From 1d4684d3bbf89e37a9e46244101e96b769fa8bac Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:57:23 -0700 Subject: [PATCH] gui: implement VideoToolbox hardware decoding on macOS and iOS --- CMakeLists.txt | 6 +- gui/CMakeLists.txt | 27 +++- gui/menu/menu_game.c | 37 ++++- gui/ui/ui_sdl.c | 38 +++++ gui/ui/ui_sdl.h | 3 + gui/ui/ui_sdl_videotoolbox.h | 15 ++ gui/ui/ui_sdl_videotoolbox.m | 288 +++++++++++++++++++++++++++++++++++ lib/gamepad/video.c | 6 +- 8 files changed, 408 insertions(+), 12 deletions(-) create mode 100644 gui/ui/ui_sdl_videotoolbox.h create mode 100644 gui/ui/ui_sdl_videotoolbox.m diff --git a/CMakeLists.txt b/CMakeLists.txt index 449be7b..e74caa7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,9 +14,9 @@ if(UNIX AND NOT APPLE AND NOT ANDROID) endif() if(APPLE) - # FFmpeg's configure step expects both compiler drivers even though Vanilla - # itself is written in C. - enable_language(CXX) + # FFmpeg's configure step expects both compiler drivers. Objective-C is + # used by the VideoToolbox-to-Metal renderer. + enable_language(CXX OBJC) endif() if(ANDROID OR MSVC OR IOS) diff --git a/gui/CMakeLists.txt b/gui/CMakeLists.txt index da4e304..78448a9 100644 --- a/gui/CMakeLists.txt +++ b/gui/CMakeLists.txt @@ -58,6 +58,13 @@ function(compile_ffmpeg _target_name _system_name _system_processor) ) endif() + if(APPLE) + list(APPEND FFMPEG_CONFIGURE_ARGS + --enable-videotoolbox + --enable-hwaccel=h264_videotoolbox + ) + endif() + if (NOT "${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "${_system_processor}" OR NOT "${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "${_system_name}") set(FFMPEG_TARGET_OS ${_system_name}) @@ -88,7 +95,6 @@ function(compile_ffmpeg _target_name _system_name _system_processor) --strip=${CMAKE_STRIP} --host-cc=clang --host-ld=clang - --enable-videotoolbox ) elseif(APPLE) set(FFMPEG_TARGET_OS "darwin") @@ -336,6 +342,15 @@ if (ANDROID) ) endif() +if (APPLE) + list(APPEND VANILLA_GUI_SRC + ui/ui_sdl_videotoolbox.m + ) + set_source_files_properties(ui/ui_sdl_videotoolbox.m PROPERTIES + COMPILE_OPTIONS "-fobjc-arc" + ) +endif() + if (LINUX) list(APPEND VANILLA_GUI_SRC ui/ui_sdl_linux.c @@ -446,10 +461,6 @@ elseif(IOS) XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "${CMAKE_OSX_DEPLOYMENT_TARGET}" ) - set_target_properties(vanilla PROPERTIES - XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO" - ) - # Compile storyboard add_custom_command( TARGET vanilla POST_BUILD @@ -530,12 +541,14 @@ if (NOT VANILLA_BUILD_VENDORED) find_package(SDL2 REQUIRED) find_package(SDL2_ttf REQUIRED) find_package(SDL2_image REQUIRED) + set(_sdl_extension_libs SDL2_ttf::SDL2_ttf SDL2_image::SDL2_image) +else() + set(_sdl_extension_libs SDL2_ttf SDL2_image) endif() target_link_libraries(vanillagui PRIVATE ${_sdl_lib} - SDL2_ttf - SDL2_image + ${_sdl_extension_libs} libvanilla m ) diff --git a/gui/menu/menu_game.c b/gui/menu/menu_game.c index d9f9312..3cff50e 100644 --- a/gui/menu/menu_game.c +++ b/gui/menu/menu_game.c @@ -21,6 +21,7 @@ #include "menu_common.h" #include "menu_main.h" #include "ui/ui_anim.h" +#include "ui/ui_sdl.h" #include "ui/ui_util.h" #ifdef ANDROID @@ -248,6 +249,20 @@ static enum AVPixelFormat drm_get_format(struct AVCodecContext *s, const enum AV return AV_PIX_FMT_NONE; } +#ifdef VANILLA_VIDEOTOOLBOX_AVAILABLE +static enum AVPixelFormat videotoolbox_get_format(struct AVCodecContext *s, const enum AVPixelFormat *fmt) +{ + while (*fmt != AV_PIX_FMT_NONE) { + if (*fmt == AV_PIX_FMT_VIDEOTOOLBOX) { + return *fmt; + } + fmt++; + } + + return AV_PIX_FMT_NONE; +} +#endif + #ifdef ANDROID static enum AVPixelFormat mediacodec_get_format(struct AVCodecContext *s, const enum AVPixelFormat *fmt) @@ -272,6 +287,9 @@ typedef struct { enum HwDecoderType { #ifdef ANDROID HWDEC_TYPE_MEDIACODEC, +#endif +#ifdef VANILLA_VIDEOTOOLBOX_AVAILABLE + HWDEC_TYPE_VIDEOTOOLBOX, #endif HWDEC_TYPE_NVDEC, HWDEC_TYPE_VAAPI, @@ -421,6 +439,12 @@ int vpi_decode_init(vpi_decode_state_t *s) decoders[HWDEC_TYPE_MEDIACODEC].get_format = mediacodec_get_format; #endif +#ifdef VANILLA_VIDEOTOOLBOX_AVAILABLE + decoders[HWDEC_TYPE_VIDEOTOOLBOX].name = "VideoToolbox"; + decoders[HWDEC_TYPE_VIDEOTOOLBOX].codec = avcodec_find_decoder(AV_CODEC_ID_H264); + decoders[HWDEC_TYPE_VIDEOTOOLBOX].get_format = videotoolbox_get_format; +#endif + decoders[HWDEC_TYPE_NVDEC].name = "NVDEC"; decoders[HWDEC_TYPE_NVDEC].codec = avcodec_find_decoder_by_name("h264_cuvid"); decoders[HWDEC_TYPE_NVDEC].get_format = nvdec_get_format; @@ -445,6 +469,17 @@ int vpi_decode_init(vpi_decode_state_t *s) int r = VANILLA_ERR_GENERIC; if (!vpi_config.force_software_decode) { +#ifdef VANILLA_VIDEOTOOLBOX_AVAILABLE + if (r != VANILLA_SUCCESS && vui_sdl_videotoolbox_available(s->vui)) { + vpi_decode_exit(s); + ffmpeg_err = av_hwdevice_ctx_create(&s->hw_device_ctx, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, NULL, NULL, 0); + if (ffmpeg_err >= 0) { + r = open_decoder(s, &decoders[HWDEC_TYPE_VIDEOTOOLBOX]); + } else { + vpilog("Failed to create VideoToolbox device: %s\n", av_err2str(ffmpeg_err)); + } + } +#endif #ifdef ANDROID // MediaCodec renders decoded buffers directly into the SurfaceTexture // owned by SDL's external OES texture. @@ -1017,9 +1052,9 @@ void *vpi_event_loop(void *arg) case VANILLA_EVENT_VIDEO: // If decoder context/thread is not set up, set up now if (!vpi_decode_alloc) { + s.vui = vui; int ret = vpi_decode_init(&s); if (ret >= 0) { - s.vui = vui; s.thread_running = 1; if (pthread_create(&decode_thread, NULL, vpi_decode_loop, &s) == 0) { diff --git a/gui/ui/ui_sdl.c b/gui/ui/ui_sdl.c index 4627237..3531c68 100644 --- a/gui/ui/ui_sdl.c +++ b/gui/ui/ui_sdl.c @@ -19,6 +19,10 @@ #include "ui_sdl_android.h" #endif +#ifdef VANILLA_VIDEOTOOLBOX_AVAILABLE +#include "ui_sdl_videotoolbox.h" +#endif + #ifdef VANILLA_HAS_EGL #include #include @@ -104,6 +108,9 @@ typedef struct { struct timeval toast_expiry; AVFrame *frame; AVFrame *held_frame; // keeps the displayed dmabuf alive until the next frame replaces it +#ifdef VANILLA_VIDEOTOOLBOX_AVAILABLE + vui_sdl_videotoolbox_context_t *videotoolbox; +#endif uint64_t present_frame_sequence; Uint64 update_tick_origin; Uint64 update_tick_frequency; @@ -842,6 +849,13 @@ int vui_init_sdl(vui_context_t *ctx, int fullscreen) vpilog("EGL unavailable, VAAPI will be disabled\n"); } +#ifdef VANILLA_VIDEOTOOLBOX_AVAILABLE + sdl_ctx->videotoolbox = vui_sdl_videotoolbox_create(sdl_ctx->renderer); + if (!sdl_ctx->videotoolbox) { + vpilog("VideoToolbox zero-copy rendering unavailable; using software decoding\n"); + } +#endif + // Open audio output device SDL_AudioSpec desired = {0}, obtained; desired.freq = 48000; @@ -1051,6 +1065,10 @@ void vui_close_sdl(vui_context_t *ctx) SDL_CloseAudioDevice(sdl_ctx->audio); +#ifdef VANILLA_VIDEOTOOLBOX_AVAILABLE + vui_sdl_videotoolbox_destroy(&sdl_ctx->videotoolbox); +#endif + SDL_DestroyRenderer(sdl_ctx->renderer); SDL_DestroyWindow(sdl_ctx->window); @@ -1060,6 +1078,17 @@ void vui_close_sdl(vui_context_t *ctx) SDL_Quit(); } +int vui_sdl_videotoolbox_available(vui_context_t *ctx) +{ +#ifdef VANILLA_VIDEOTOOLBOX_AVAILABLE + if (ctx && ctx->platform_data) { + vui_sdl_context_t *sdl_ctx = (vui_sdl_context_t *) ctx->platform_data; + return sdl_ctx->videotoolbox != NULL; + } +#endif + return 0; +} + float lerp(float a, float b, float f) { return a * (1.0 - f) + (b * f); @@ -2217,6 +2246,10 @@ int vui_update_sdl(vui_context_t *vui) case AV_PIX_FMT_YUV420P: get_texture_from_cpu_frame(sdl_ctx, sdl_ctx->frame); break; + + // AV_PIX_FMT_VIDEOTOOLBOX does not need to be handled here because + // it is handled further down by vui_sdl_videotoolbox_render + } av_frame_unref(sdl_ctx->held_frame); @@ -2245,6 +2278,11 @@ int vui_update_sdl(vui_context_t *vui) SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255); SDL_RenderClear(renderer); +#ifdef VANILLA_VIDEOTOOLBOX_AVAILABLE + if (sdl_ctx->held_frame->format == AV_PIX_FMT_VIDEOTOOLBOX) { + vui_sdl_videotoolbox_render(sdl_ctx->videotoolbox, renderer, sdl_ctx->held_frame); + } else +#endif if (sdl_ctx->game_tex) { #ifdef ANDROID if (sdl_ctx->game_tex == sdl_ctx->android_video_tex) { diff --git a/gui/ui/ui_sdl.h b/gui/ui/ui_sdl.h index cb29204..2cfe416 100644 --- a/gui/ui/ui_sdl.h +++ b/gui/ui/ui_sdl.h @@ -9,6 +9,9 @@ int vui_init_sdl(vui_context_t *ctx, int fullscreen); void vui_close_sdl(vui_context_t *ctx); +// Return whether direct VideoToolbox to display path is available (i.e. if Metal is being used) +int vui_sdl_videotoolbox_available(vui_context_t *ctx); + /** * Main loop * diff --git a/gui/ui/ui_sdl_videotoolbox.h b/gui/ui/ui_sdl_videotoolbox.h new file mode 100644 index 0000000..edd53ca --- /dev/null +++ b/gui/ui/ui_sdl_videotoolbox.h @@ -0,0 +1,15 @@ +#ifndef VANILLA_UI_SDL_VIDEOTOOLBOX_H +#define VANILLA_UI_SDL_VIDEOTOOLBOX_H + +#include +#include + +typedef struct vui_sdl_videotoolbox_context vui_sdl_videotoolbox_context_t; + +vui_sdl_videotoolbox_context_t *vui_sdl_videotoolbox_create(SDL_Renderer *renderer); +void vui_sdl_videotoolbox_destroy(vui_sdl_videotoolbox_context_t **context); +int vui_sdl_videotoolbox_render(vui_sdl_videotoolbox_context_t *context, + SDL_Renderer *renderer, + const AVFrame *frame); + +#endif // VANILLA_UI_SDL_VIDEOTOOLBOX_H diff --git a/gui/ui/ui_sdl_videotoolbox.m b/gui/ui/ui_sdl_videotoolbox.m new file mode 100644 index 0000000..51e5069 --- /dev/null +++ b/gui/ui/ui_sdl_videotoolbox.m @@ -0,0 +1,288 @@ +#include "ui_sdl_videotoolbox.h" + +#import +#import +#import +#import +#import + +#include +#include + +#include "platform.h" + +typedef struct { + vector_float4 red; + vector_float4 green; + vector_float4 blue; +} vui_yuv_conversion_t; + +@interface VUIVideoToolboxRenderer : NSObject +@property(nonatomic, strong) id device; +@property(nonatomic, strong) id rgbaPipeline; +@property(nonatomic, strong) id bgraPipeline; +@property(nonatomic, strong) id sampler; +@property(nonatomic) CVMetalTextureCacheRef textureCache; +@end + +static NSString *const vuiVideoShaderSource = + @"#include \n" + "using namespace metal;\n" + "struct RasterizerData {\n" + " float4 position [[position]];\n" + " float2 texcoord;\n" + "};\n" + "struct YUVConversion {\n" + " float4 red;\n" + " float4 green;\n" + " float4 blue;\n" + "};\n" + "vertex RasterizerData vanilla_video_vertex(uint vertexID [[vertex_id]]) {\n" + " constexpr float2 positions[] = {\n" + " float2(-1.0, 1.0), float2(1.0, 1.0),\n" + " float2(-1.0, -1.0), float2(1.0, -1.0)\n" + " };\n" + " constexpr float2 texcoords[] = {\n" + " float2(0.0, 0.0), float2(1.0, 0.0),\n" + " float2(0.0, 1.0), float2(1.0, 1.0)\n" + " };\n" + " RasterizerData output;\n" + " output.position = float4(positions[vertexID], 0.0, 1.0);\n" + " output.texcoord = texcoords[vertexID];\n" + " return output;\n" + "}\n" + "fragment float4 vanilla_video_fragment(\n" + " RasterizerData input [[stage_in]],\n" + " texture2d luma [[texture(0)]],\n" + " texture2d chroma [[texture(1)]],\n" + " sampler videoSampler [[sampler(0)]],\n" + " constant YUVConversion &conversion [[buffer(0)]]) {\n" + " const float y = luma.sample(videoSampler, input.texcoord).r;\n" + " const float2 cbcr = chroma.sample(videoSampler, input.texcoord).rg;\n" + " const float4 sample = float4(y, cbcr, 1.0);\n" + " return float4(dot(conversion.red, sample),\n" + " dot(conversion.green, sample),\n" + " dot(conversion.blue, sample), 1.0);\n" + "}\n"; + +static id vui_create_pipeline(id device, id library, MTLPixelFormat pixelFormat) +{ + MTLRenderPipelineDescriptor *descriptor = [[MTLRenderPipelineDescriptor alloc] init]; + descriptor.vertexFunction = [library newFunctionWithName:@"vanilla_video_vertex"]; + descriptor.fragmentFunction = [library newFunctionWithName:@"vanilla_video_fragment"]; + descriptor.colorAttachments[0].pixelFormat = pixelFormat; + + NSError *error = nil; + id pipeline = [device newRenderPipelineStateWithDescriptor:descriptor error:&error]; + if (!pipeline) { + vpilog("Failed to create VideoToolbox Metal pipeline: %s\n", error.localizedDescription.UTF8String); + } + return pipeline; +} + +@implementation VUIVideoToolboxRenderer + +- (instancetype)initWithSDLRenderer:(SDL_Renderer *)renderer +{ + self = [super init]; + if (!self) { + return nil; + } + + SDL_RendererInfo rendererInfo; + if (SDL_GetRendererInfo(renderer, &rendererInfo) < 0 || !rendererInfo.name || strcmp(rendererInfo.name, "metal") != 0) { + vpilog("VideoToolbox requires SDL's Metal renderer\n"); + return nil; + } + + CAMetalLayer *layer = (__bridge CAMetalLayer *) SDL_RenderGetMetalLayer(renderer); + self.device = layer.device; + if (!self.device) { + vpilog("SDL's Metal renderer did not provide a Metal device\n"); + return nil; + } + + CVReturn result = CVMetalTextureCacheCreate(kCFAllocatorDefault, NULL, self.device, NULL, &_textureCache); + if (result != kCVReturnSuccess) { + vpilog("Failed to create CoreVideo Metal texture cache: %d\n", result); + return nil; + } + + NSError *error = nil; + id library = [self.device newLibraryWithSource:vuiVideoShaderSource options:nil error:&error]; + if (!library) { + vpilog("Failed to compile VideoToolbox Metal shader: %s\n", error.localizedDescription.UTF8String); + return nil; + } + + self.rgbaPipeline = vui_create_pipeline(self.device, library, MTLPixelFormatRGBA8Unorm); + self.bgraPipeline = vui_create_pipeline(self.device, library, MTLPixelFormatBGRA8Unorm); + if (!self.rgbaPipeline && !self.bgraPipeline) { + return nil; + } + + MTLSamplerDescriptor *samplerDescriptor = [[MTLSamplerDescriptor alloc] init]; + samplerDescriptor.minFilter = MTLSamplerMinMagFilterLinear; + samplerDescriptor.magFilter = MTLSamplerMinMagFilterLinear; + samplerDescriptor.sAddressMode = MTLSamplerAddressModeClampToEdge; + samplerDescriptor.tAddressMode = MTLSamplerAddressModeClampToEdge; + self.sampler = [self.device newSamplerStateWithDescriptor:samplerDescriptor]; + if (!self.sampler) { + vpilog("Failed to create VideoToolbox Metal sampler\n"); + return nil; + } + + return self; +} + +- (void)dealloc +{ + if (_textureCache) { + CFRelease(_textureCache); + } +} + +@end + +static vui_yuv_conversion_t vui_yuv_conversion(const AVFrame *frame, OSType pixelFormat) +{ + const int fullRange = pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange || frame->color_range == AVCOL_RANGE_JPEG; + const int bt709 = frame->colorspace == AVCOL_SPC_BT709; + const float kr = bt709 ? 0.2126f : 0.2990f; + const float kb = bt709 ? 0.0722f : 0.1140f; + const float kg = 1.0f - kr - kb; + const float yOffset = fullRange ? 0.0f : 16.0f / 255.0f; + const float chromaOffset = 128.0f / 255.0f; + const float yScale = fullRange ? 1.0f : 255.0f / 219.0f; + const float chromaScale = fullRange ? 1.0f : 255.0f / 224.0f; + + const float redCr = 2.0f * (1.0f - kr) * chromaScale; + const float blueCb = 2.0f * (1.0f - kb) * chromaScale; + const float greenCb = -2.0f * kb * (1.0f - kb) / kg * chromaScale; + const float greenCr = -2.0f * kr * (1.0f - kr) / kg * chromaScale; + + vui_yuv_conversion_t conversion = { + .red = { yScale, 0.0f, redCr, -yScale * yOffset - redCr * chromaOffset }, + .green = { yScale, greenCb, greenCr, -yScale * yOffset - (greenCb + greenCr) * chromaOffset }, + .blue = { yScale, blueCb, 0.0f, -yScale * yOffset - blueCb * chromaOffset } + }; + return conversion; +} + +vui_sdl_videotoolbox_context_t *vui_sdl_videotoolbox_create(SDL_Renderer *renderer) +{ + VUIVideoToolboxRenderer *context = [[VUIVideoToolboxRenderer alloc] initWithSDLRenderer:renderer]; + return (__bridge_retained vui_sdl_videotoolbox_context_t *) context; +} + +void vui_sdl_videotoolbox_destroy(vui_sdl_videotoolbox_context_t **context) +{ + if (!context || !*context) { + return; + } + CFBridgingRelease(*context); + *context = NULL; +} + +int vui_sdl_videotoolbox_render(vui_sdl_videotoolbox_context_t *opaque, SDL_Renderer *renderer, const AVFrame *frame) +{ + if (!opaque || !renderer || !frame || frame->format != AV_PIX_FMT_VIDEOTOOLBOX || !frame->data[3]) { + return 0; + } + + VUIVideoToolboxRenderer *context = (__bridge VUIVideoToolboxRenderer *) opaque; + CVPixelBufferRef pixelBuffer = (CVPixelBufferRef) frame->data[3]; + OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); + if (CVPixelBufferGetPlaneCount(pixelBuffer) != 2 || (pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange && pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)) { + vpilog("VideoToolbox returned unsupported pixel format: 0x%08x\n", pixelFormat); + return 0; + } + + SDL_Texture *target = SDL_GetRenderTarget(renderer); + Uint32 targetFormat = 0; + int targetWidth = 0; + int targetHeight = 0; + if (!target || SDL_QueryTexture(target, &targetFormat, NULL, &targetWidth, &targetHeight) < 0) { + vpilog("VideoToolbox rendering requires an SDL texture target\n"); + return 0; + } + + id pipeline = nil; + if (targetFormat == SDL_PIXELFORMAT_RGBA32) { + pipeline = context.rgbaPipeline; + } else if (targetFormat == SDL_PIXELFORMAT_BGRA32) { + pipeline = context.bgraPipeline; + } + if (!pipeline) { + vpilog("VideoToolbox cannot render to SDL pixel format 0x%08x\n", targetFormat); + return 0; + } + + CVMetalTextureRef lumaRef = NULL; + CVMetalTextureRef chromaRef = NULL; + CVReturn result = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, + context.textureCache, + pixelBuffer, + NULL, + MTLPixelFormatR8Unorm, + CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), + CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), + 0, + &lumaRef + ); + if (result == kCVReturnSuccess) { + result = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, + context.textureCache, + pixelBuffer, + NULL, + MTLPixelFormatRG8Unorm, + CVPixelBufferGetWidthOfPlane(pixelBuffer, 1), + CVPixelBufferGetHeightOfPlane(pixelBuffer, 1), + 1, + &chromaRef + ); + } + + id luma = lumaRef ? CVMetalTextureGetTexture(lumaRef) : nil; + id chroma = chromaRef ? CVMetalTextureGetTexture(chromaRef) : nil; + if (result != kCVReturnSuccess || !luma || !chroma) { + vpilog("Failed to expose VideoToolbox frame as Metal textures: %d\n", result); + if (chromaRef) CFRelease(chromaRef); + if (lumaRef) CFRelease(lumaRef); + return 0; + } + + if (SDL_RenderFlush(renderer) < 0) { + vpilog("Failed to flush SDL before VideoToolbox render: %s\n", SDL_GetError()); + CFRelease(chromaRef); + CFRelease(lumaRef); + return 0; + } + + id encoder = (__bridge id) SDL_RenderGetMetalCommandEncoder(renderer); + if (!encoder) { + vpilog("SDL did not provide a Metal command encoder\n"); + CFRelease(chromaRef); + CFRelease(lumaRef); + return 0; + } + + MTLViewport viewport = { 0.0, 0.0, targetWidth, targetHeight, 0.0, 1.0 }; + MTLScissorRect scissor = { 0, 0, targetWidth, targetHeight }; + vui_yuv_conversion_t conversion = vui_yuv_conversion(frame, pixelFormat); + + [encoder setViewport:viewport]; + [encoder setScissorRect:scissor]; + [encoder setRenderPipelineState:pipeline]; + [encoder setFragmentTexture:luma atIndex:0]; + [encoder setFragmentTexture:chroma atIndex:1]; + [encoder setFragmentSamplerState:context.sampler atIndex:0]; + [encoder setFragmentBytes:&conversion length:sizeof(conversion) atIndex:0]; + [encoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4]; + + CFRelease(chromaRef); + CFRelease(lumaRef); + return 1; +} diff --git a/lib/gamepad/video.c b/lib/gamepad/video.c index a674242..dfe942a 100644 --- a/lib/gamepad/video.c +++ b/lib/gamepad/video.c @@ -180,7 +180,11 @@ void handle_video_packet(gamepad_context_t *ctx, VideoPacket *vp) memset(video_segments, 0, sizeof(video_segments)); - frame_decode_num++; + if (is_idr) { + frame_decode_num = 0; + } else { + frame_decode_num++; + } if (!video_complete_frame && !is_idr) { send_idr_request_to_console(ctx->socket_msg);