Implement simple retry mechanism to improve raspberry pi frame delay (#438)

* Implement simple retry mechanism to fix raspberry pi decoders

* rework

* further reduce latency

With these changes, the Pi Zero 2W is now equivalent to the gamepad/Orange Pi in latency

---------

Co-authored-by: itsmattkc <34096995+itsmattkc@users.noreply.github.com>

gui/menu/menu_game.c

@@ -1,14 +1,18 @@
 #include "menu_game.h"
 
 #include <assert.h>
+#include <limits.h>
 #include <libavformat/avformat.h>
 #include <libavcodec/avcodec.h>
 #include <libavutil/channel_layout.h>
 #include <libavutil/opt.h>
+#include <libavutil/time.h>
 #include <libswscale/swscale.h>
 #include <stdatomic.h>
 #include <stdio.h>
 #include <sys/time.h>
+#include <time.h>
+#include <unistd.h>
 #include <vanilla.h>
 
 #include "config.h"
@@ -29,8 +33,14 @@ static char vpi_toast_string[VPI_TOAST_MAX_LEN];
 static struct timeval vpi_toast_expiry;
 static int vpi_toast_number = 0;
 
+// Arbitrary limits so loops don't go on forever
+static const long VPI_DECODE_POLL_US = 500;
+static const long VPI_DECODE_MAX_POLLS = 40;
+
 AVFrame *vpi_present_frame = 0;
 pthread_mutex_t vpi_present_frame_mutex = PTHREAD_MUTEX_INITIALIZER;
+pthread_cond_t vpi_present_frame_cond = PTHREAD_COND_INITIALIZER;
+uint64_t vpi_present_frame_sequence = 0;
 
 static AVFormatContext *recording_fmt_ctx = 0;
 static AVStream *recording_vstr;
@@ -41,6 +51,8 @@ static char screenshot_buf[4096] = {0};
 static const int VIDEO_STREAM_INDEX = 0;
 static const int AUDIO_STREAM_INDEX = 1;
 
+static pthread_t vpi_event_thread;
+
 int vpi_egl_available = 0;
 
 static int status_lbl;
@@ -276,14 +288,33 @@ static int is_v4l2request_available(void)
 
 void vpi_decode_exit(vpi_decode_state_t *s)
 {
+    for (size_t i = 0; i < VPI_DECODE_QUEUE_CAPACITY; i++) {
+        av_packet_free(&s->packet_queue[i]);
+    }
+    s->packet_queue_read = 0;
+    s->packet_queue_write = 0;
+    s->packet_queue_count = 0;
+
+    if (s->cond_init) {
+        pthread_cond_destroy(&s->cond);
+        s->cond_init = 0;
+    }
+
+    if (s->mutex_init) {
+        pthread_mutex_destroy(&s->mutex);
+        s->mutex_init = 0;
+    }
+
     if (s->frame)
         av_frame_free(&s->frame);
 
-    if (s->pkt)
-	    av_packet_free(&s->pkt);
-
-    if (vpi_present_frame)
+    pthread_mutex_lock(&vpi_present_frame_mutex);
+    if (vpi_present_frame) {
 	    av_frame_free(&vpi_present_frame);
+        vpi_present_frame_sequence++;
+        pthread_cond_broadcast(&vpi_present_frame_cond);
+    }
+    pthread_mutex_unlock(&vpi_present_frame_mutex);
 
     if (s->codec_ctx)
         avcodec_free_context(&s->codec_ctx);
@@ -427,27 +458,39 @@ int vpi_decode_init(vpi_decode_state_t *s)
         return r;
     }
 
-	AVFrame *decoding_frame = av_frame_alloc();
-    if (!decoding_frame) {
-        vpilog("Failed to allocate AVFrame\n");
-        return VANILLA_ERR_GENERIC;
-    }
-
+    pthread_mutex_lock(&vpi_present_frame_mutex);
 	vpi_present_frame = av_frame_alloc();
+    pthread_mutex_unlock(&vpi_present_frame_mutex);
 	if (!vpi_present_frame) {
 		vpilog("Failed to allocate AVFrame\n");
-		return VANILLA_ERR_GENERIC;
-	}
-
-	s->pkt = av_packet_alloc();
-	if (!s->pkt) {
-		vpilog("Failed to allocate AVPacket\n");
-		return VANILLA_ERR_GENERIC;
+        goto fail;
 	}
 
 	s->frame = av_frame_alloc();
+    if (!s->frame) {
+		vpilog("Failed to allocate AVFrame\n");
+        goto fail;
+    }
+
+    if (pthread_mutex_init(&s->mutex, 0) != 0) {
+		vpilog("Failed to init decoder mutex\n");
+        goto fail;
+    }
+
+    s->mutex_init = 1;
+
+    if (pthread_cond_init(&s->cond, 0) != 0) {
+		vpilog("Failed to init decoder condition variable\n");
+        goto fail;
+    }
+
+    s->cond_init = 1;
 
     return VANILLA_SUCCESS;
+
+fail:
+    vpi_decode_exit(s);
+    return VANILLA_ERR_GENERIC;
 }
 
 int64_t get_recording_timestamp(AVRational timebase)
@@ -654,35 +697,292 @@ void vpi_game_shutdown()
     vpi_game_queued_error = VANILLA_ERR_SHUTDOWN;
 }
 
-static pthread_t vpi_event_thread;
+static void vpi_publish_decoded_frame(vpi_decode_state_t *s)
+{
+    vui_context_t *vui = s->vui;
+
+    pthread_mutex_lock(&vpi_present_frame_mutex);
+
+    // Send frame out to display
+    av_frame_unref(vpi_present_frame);
+    av_frame_move_ref(vpi_present_frame, s->frame);
+    vpi_present_frame_sequence++;
+
+    // If user has requested a screenshot, dump the screenshot to a file
+    if (screenshot_buf[0] != 0) {
+        dump_frame_to_file(vpi_present_frame, screenshot_buf);
+        screenshot_buf[0] = 0;
+    }
+
+    // Alert display that frame is ready
+    pthread_cond_broadcast(&vpi_present_frame_cond);
+    pthread_mutex_unlock(&vpi_present_frame_mutex);
+
+    // Now that we have our first frame, switch UI to game mode if not already
+    // FIXME: Not thread safe? Not a huge deal but might want to fix some day
+    if (!vui_game_mode_get(vui)) {
+        vui_game_mode_set(vui, 1);
+        vui_audio_set_enabled(vui, 1);
+    }
+}
+
+static int vpi_receive_decoded_frames(vpi_decode_state_t *s)
+{
+    int received = 0;
+
+    while (s->thread_running) {
+        // Attempt to receive frames from decoder
+        int err = avcodec_receive_frame(s->codec_ctx, s->frame);
+        if (err == AVERROR(EAGAIN) || err == AVERROR_EOF) {
+            break;
+        }
+        if (err < 0) {
+            // Return unexpected error to caller
+            vpilog("Failed to receive frame from decoder: %s (%i)\n", av_err2str(err), err);
+            return err;
+        }
+
+        // Received a frame, send it out for publishing
+        vpi_publish_decoded_frame(s);
+        received++;
+    }
+
+    return received;
+}
+
+static void vpi_decode_stop(vpi_decode_state_t *s)
+{
+    pthread_mutex_lock(&s->mutex);
+    s->thread_running = 0;
+    pthread_cond_broadcast(&s->cond);
+    pthread_mutex_unlock(&s->mutex);
+}
+
+static AVPacket *vpi_decode_get_next_packet(vpi_decode_state_t *s, int wait)
+{
+    AVPacket *pkt = NULL;
+
+    pthread_mutex_lock(&s->mutex);
+
+    // Wait for incoming frames from console/event loop.
+    while (s->thread_running && s->packet_queue_count == 0) {
+        // Wait infinitely if caller has allowed it
+        if (wait) {
+            pthread_cond_wait(&s->cond, &s->mutex);
+            continue;
+        }
+
+        // Otherwise, only wait for a short time before checking
+        struct timespec deadline;
+        timespec_get(&deadline, TIME_UTC);
+        deadline.tv_nsec += VPI_DECODE_POLL_US * 1000;
+        if (deadline.tv_nsec >= 1000000000) {
+            deadline.tv_sec++;
+            deadline.tv_nsec -= 1000000000;
+        }
+        pthread_cond_timedwait(&s->cond, &s->mutex, &deadline);
+        break;
+    }
+
+    // If we got a new packet, pull it from the queue to return to caller
+    if (s->thread_running && s->packet_queue_count > 0) {
+        pkt = s->packet_queue[s->packet_queue_read];
+        s->packet_queue[s->packet_queue_read] = NULL;
+        s->packet_queue_read = (s->packet_queue_read + 1) % VPI_DECODE_QUEUE_CAPACITY;
+        s->packet_queue_count--;
+
+        // Alert event thread in case it was waiting for queue to free up
+        pthread_cond_broadcast(&s->cond);
+    }
+    pthread_mutex_unlock(&s->mutex);
+
+    return pkt;
+}
+
+static int vpi_decode_drain(vpi_decode_state_t *s,
+                            size_t *outstanding_packets)
+{
+    int received = vpi_receive_decoded_frames(s);
+
+    // Decoder either returned nothing or an error, pass it up to the caller
+    if (received <= 0) {
+        return received;
+    }
+
+    // Received frames, substract from outstanding_packets
+    size_t frame_count = (size_t) received;
+
+    if (*outstanding_packets > frame_count) {
+        *outstanding_packets -= frame_count;
+    } else {
+        *outstanding_packets = 0;
+    }
+
+    return received;
+}
+
+static int vpi_decode_submit_packet(vpi_decode_state_t *s, AVPacket *pkt,
+                                    size_t *outstanding_packets)
+{
+    int err = AVERROR_EXIT;
+
+    for (int poll = 0; s->thread_running; poll++) {
+        err = avcodec_send_packet(s->codec_ctx, pkt);
+        if (err != AVERROR(EAGAIN) || poll >= VPI_DECODE_MAX_POLLS) {
+            break;
+        }
+
+        int received = vpi_decode_drain(s, outstanding_packets);
+        if (received < 0) {
+            return received;
+        }
+        if (received == 0) {
+            av_usleep(VPI_DECODE_POLL_US);
+        }
+    }
+
+    if (!s->thread_running) {
+        return AVERROR_EXIT;
+    }
+    if (err < 0) {
+        vpilog("Failed to send packet to decoder: %s (%i)\n",
+               av_err2str(err), err);
+        vanilla_request_idr();
+        return 0;
+    }
+
+    (*outstanding_packets)++;
+    return 1;
+}
+
+void *vpi_decode_loop(void *arg)
+{
+    vpi_decode_state_t *s = (vpi_decode_state_t *) arg;
+    size_t outstanding_packets = 0;
+    int idle_output_polls = 0;
+
+    while (s->thread_running) {
+        // Wait for a packet from the event loop/console
+        AVPacket *pkt = vpi_decode_get_next_packet(s, outstanding_packets == 0);
+
+        // If we got a packet, submit it to the decoder
+        if (pkt) {
+            int submitted = vpi_decode_submit_packet(s, pkt, &outstanding_packets);
+            av_packet_free(&pkt);
+
+            // If there was an error, break
+            if (submitted < 0) {
+                break;
+            }
+            if (submitted > 0) {
+                idle_output_polls = 0;
+            }
+        } else if (!s->thread_running) {
+            break;
+        }
+
+        int received = vpi_decode_drain(s, &outstanding_packets);
+        if (received < 0) {
+            break;
+        }
+        if (received > 0) {
+            idle_output_polls = 0;
+        } else if (outstanding_packets > 0 && ++idle_output_polls >= VPI_DECODE_MAX_POLLS) {
+            outstanding_packets = 0;
+            idle_output_polls = 0;
+        }
+    }
+
+    vpi_decode_stop(s);
+
+    return NULL;
+}
+
+static int vpi_decode_enqueue(vpi_decode_state_t *s,
+                              const uint8_t *data, size_t size)
+{
+    // Allocate new packet
+    AVPacket *pkt = av_packet_alloc();
+    if (!pkt) {
+        return AVERROR(ENOMEM);
+    }
+
+    // Allocate data buffers
+    int err = av_new_packet(pkt, (int) size);
+    if (err < 0) {
+        av_packet_free(&pkt);
+        return err;
+    }
+
+    // Copy data into packet and set timestamps (which some decoders may need)
+    memcpy(pkt->data, data, size);
+    pkt->pts = av_gettime_relative();
+    pkt->dts = pkt->pts;
+
+    // Acquire lock
+    pthread_mutex_lock(&s->mutex);
+
+    // If decoder thread is at max capacity, wait until it's processed a few more
+    while (s->thread_running && s->packet_queue_count == VPI_DECODE_QUEUE_CAPACITY) {
+        pthread_cond_wait(&s->cond, &s->mutex);
+    }
+
+    // Since we may have waited earlier, one last check before we queue this frame up
+    if (!s->thread_running) {
+        pthread_mutex_unlock(&s->mutex);
+        av_packet_free(&pkt);
+        return AVERROR_EXIT;
+    }
+
+    // Queue frame up for decoder and signal
+    s->packet_queue[s->packet_queue_write] = pkt;
+    s->packet_queue_write = (s->packet_queue_write + 1) % VPI_DECODE_QUEUE_CAPACITY;
+    s->packet_queue_count++;
+    pthread_cond_signal(&s->cond);
+    pthread_mutex_unlock(&s->mutex);
+
+    return 0;
+}
 
 void *vpi_event_loop(void *arg)
 {
     vui_context_t *vui = (vui_context_t *) arg;
-    static int vpi_decode_alloc = 0;
-
-    static vpi_decode_state_t s;
+    int vpi_decode_alloc = 0;
+    pthread_t decode_thread;
     vanilla_event_t event;
+    vpi_decode_state_t s;
+
+    memset(&s, 0, sizeof(s));
+
     while (vpi_game_queued_error != VANILLA_ERR_SHUTDOWN && vanilla_wait_event(&event)) {
         int stop = 0;
 
         switch (event.type) {
         case VANILLA_EVENT_VIDEO:
+            // If decoder context/thread is not set up, set up now
             if (!vpi_decode_alloc) {
                 int ret = vpi_decode_init(&s);
                 if (ret >= 0) {
-                    vpi_decode_alloc = 1;
+                    s.vui = vui;
+                    s.thread_running = 1;
+
+                    if (pthread_create(&decode_thread, NULL, vpi_decode_loop, &s) == 0) {
+                        vpi_decode_alloc = 1;
+                    } else {
+                        vpilog("Failed to create decode thread\n");
+                        vpi_decode_stop(&s);
+                        vpi_decode_exit(&s);
+                    }
                 }
             }
 
             if (vpi_decode_alloc) {
-                AVPacket *pkt = s.pkt;
-
-                // Send packet to decoder
-                pkt->data = event.data;
-                pkt->size = event.size;
-
+                // If we're recording, create a packet from data and send to muxer
                 if (recording_fmt_ctx) {
+                    AVPacket *pkt = av_packet_alloc();
+
+                    pkt->data = event.data;
+                    pkt->size = event.size;
                     pkt->stream_index = VIDEO_STREAM_INDEX;
 
                     int64_t ts = get_recording_timestamp(recording_vstr->time_base);
@@ -692,59 +992,15 @@ void *vpi_event_loop(void *arg)
 
                     av_interleaved_write_frame(recording_fmt_ctx, pkt);
 
-                    // av_interleaved_write_frame() eventually calls av_packet_unref(),
-                    // so we must put the references back in
-                    pkt->data = event.data;
-                    pkt->size = event.size;
+                    av_packet_free(&pkt);
                 }
 
-                int err = avcodec_send_packet(s.codec_ctx, pkt);
-
+                // Send data to decoder thread
+                int err = vpi_decode_enqueue(&s, event.data, event.size);
                 if (err < 0) {
-                    vpilog("Failed to send packet to decoder: %s (%i)\n", av_err2str(err), err);
-                    // return 0;
-
+                    vpilog("Failed to queue packet for decoder: %s (%i)\n",
+                           av_err2str(err), err);
                     vanilla_request_idr();
-                } else {
-                    int err;
-
-                    int ret = 1;
-
-                    // Retrieve frame from decoder
-                    while (1) {
-                        err = avcodec_receive_frame(s.codec_ctx, s.frame);
-                        if (err == AVERROR(EAGAIN)) {
-                            // Decoder wants another packet before it can output a frame. Silently exit.
-                            break;
-                        } else if (err < 0) {
-                            vpilog("Failed to receive frame from decoder: %i\n", err);
-                            ret = 0;
-                            break;
-                        } else {
-                            pthread_mutex_lock(&vpi_present_frame_mutex);
-
-                            // Swap refs from decoding_frame to present_frame
-                            av_frame_unref(vpi_present_frame);
-                            av_frame_move_ref(vpi_present_frame, s.frame);
-                            vpi_present_frame->pts = s.frame->pts;
-
-                            if (screenshot_buf[0] != 0) {
-                                // Dump this frame into file
-                                dump_frame_to_file(vpi_present_frame, screenshot_buf);
-                                screenshot_buf[0] = 0;
-                            }
-
-                            pthread_mutex_unlock(&vpi_present_frame_mutex);
-
-                            // Not thread safe?
-                            if (!vui_game_mode_get(vui)) {
-                                vui_game_mode_set(vui, 1);
-                                vui_audio_set_enabled(vui, 1);
-                            }
-                        }
-                    }
-
-                    // return ret;
                 }
             }
             break;
@@ -779,9 +1035,11 @@ void *vpi_event_loop(void *arg)
         }
 
 		vanilla_free_event(&event);
-	}
+    }
 
     if (vpi_decode_alloc) {
+        vpi_decode_stop(&s);
+        pthread_join(decode_thread, 0);
         vpi_decode_exit(&s);
         vpi_decode_alloc = 0;
     }

gui/menu/menu_game.h

@@ -4,21 +4,35 @@
 #include <libavcodec/avcodec.h>
 #include <libavutil/frame.h>
 #include <pthread.h>
+#include <stdint.h>
+#include <stdatomic.h>
 #include <sys/time.h>
 
 #include "ui/ui.h"
 
 #define VPI_TOAST_MAX_LEN 1024
+#define VPI_DECODE_QUEUE_CAPACITY 8
 
 typedef struct {
+    vui_context_t *vui;
     AVCodecContext *codec_ctx;
-    AVPacket *pkt;
     AVFrame *frame;
     AVBufferRef *hw_device_ctx;
+    AVPacket *packet_queue[VPI_DECODE_QUEUE_CAPACITY];
+    size_t packet_queue_read;
+    size_t packet_queue_write;
+    size_t packet_queue_count;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    int mutex_init;
+    int cond_init;
+    _Atomic int thread_running;
 } vpi_decode_state_t;
 
 extern AVFrame *vpi_present_frame;
 extern pthread_mutex_t vpi_present_frame_mutex;
+extern pthread_cond_t vpi_present_frame_cond;
+extern uint64_t vpi_present_frame_sequence;
 extern int vpi_egl_available;
 
 void vpi_menu_game(vui_context_t *vui, void *v);

gui/tests/latencytest.c

@@ -1,6 +1,7 @@
 #include <libavcodec/avcodec.h>
 #include <libavformat/avformat.h>
 #include <libavutil/avutil.h>
+#include <libavutil/time.h>
 #include <unistd.h>
 
 #include "menu/menu_game.h"
@@ -49,10 +50,11 @@ int main(int argc, const char **argv)
 
     int ret = 1;
 
+    AVPacket *pkt = av_packet_alloc();
     int64_t pts = 0;
 
     while (1) {
-        err = av_read_frame(fmt_ctx, s.pkt);
+        err = av_read_frame(fmt_ctx, pkt);
         if (err < 0) {
             if (err == AVERROR_EOF) {
                 ret = 0;
@@ -62,32 +64,47 @@ int main(int argc, const char **argv)
             break;
         }
 
-        if (s.pkt->stream_index != 0) {
+        if (pkt->stream_index != 0) {
             continue;
         }
 
         // Retrieve frame from decoder
-        s.pkt->pts = pts++;
-        err = avcodec_send_packet(s.codec_ctx, s.pkt);
+        pkt->pts = pts++;
+        err = avcodec_send_packet(s.codec_ctx, pkt);
         if (err < 0) {
             fprintf(stderr, "avcodec_send_packet: %i\n", err);
             break;
         }
-        printf("    Sent: %li\n", s.pkt->pts);
-        usleep(17000);
+        printf("    Sent: %li\n", pkt->pts);
 
-        while ((err = avcodec_receive_frame(s.codec_ctx, s.frame)) >= 0) {
-            printf("Received: %li\n", s.frame->pts);
+        const int64_t interval = 500; // Arbitrary time to wait for a new frame
+        const int64_t wait = av_gettime_relative();
+        const int64_t max_wait = 16667; // 1 / 60 FPS frame
+        while (1) {
+            err = avcodec_receive_frame(s.codec_ctx, s.frame);
+            if (err >= 0) {
+                printf("Received: %li\n", s.frame->pts);
+                continue;
+            } else if (err != AVERROR(EAGAIN)) {
+                fprintf(stderr, "avcodec_receive_frame: %i\n", err);
+                goto fail;
+            }
+
+            if (av_gettime_relative() + interval >= wait + max_wait) {
+                break;
+            }
+
+            usleep(interval);
         }
 
-
-
-        if (err < 0 && err != AVERROR(EAGAIN)) {
-            fprintf(stderr, "avcodec_receive_frame: %i\n", err);
-            break;
+        const int64_t gap = av_gettime_relative() - (wait + max_wait);
+        if (gap > 0) {
+            usleep(gap);
         }
     }
 
+fail:
+    av_packet_free(&pkt);
     vpi_decode_exit(&s);
 
     return ret;

gui/ui/ui_sdl.c

@@ -1,6 +1,7 @@
 #include "ui_sdl.h"
 
 #include <stdatomic.h>
+#include <errno.h>
 #include <math.h>
 #include <SDL2/SDL.h>
 #include <SDL2/SDL_hints.h>
@@ -9,6 +10,7 @@
 #include <SDL_power.h>
 #include <SDL_ttf.h>
 #include <pthread.h>
+#include <time.h>
 #include <vanilla.h>
 #include <libavutil/hwcontext.h>
 #include <unistd.h>
@@ -83,6 +85,7 @@ typedef struct {
     struct timeval toast_expiry;
     AVFrame *frame;
     AVFrame *held_frame; // keeps the displayed dmabuf alive until the next frame replaces it
+    uint64_t present_frame_sequence;
 	SDL_Texture *pw_tex;
 
 	uint8_t audio_buffer[AUDIO_BUFFER_COUNT][AUDIO_BUFFER_SIZE];
@@ -1917,10 +1920,12 @@ int vui_update_sdl(vui_context_t *vui)
         main_tex = sdl_ctx->layer_data[0];
     } else {
         pthread_mutex_lock(&vpi_present_frame_mutex);
-		if (vpi_present_frame && vpi_present_frame->format != -1) {
+		if (vpi_present_frame
+            && sdl_ctx->present_frame_sequence != vpi_present_frame_sequence
+            && vpi_present_frame->format != -1) {
 			av_frame_move_ref(sdl_ctx->frame, vpi_present_frame);
-            sdl_ctx->frame->pts = vpi_present_frame->pts;
 		}
+        sdl_ctx->present_frame_sequence = vpi_present_frame_sequence;
         pthread_mutex_unlock(&vpi_present_frame_mutex);
 
 		if (sdl_ctx->frame->format != -1) {
@@ -1934,10 +1939,22 @@ int vui_update_sdl(vui_context_t *vui)
                 // this is provided as an option only.
                 if (vpi_config.fast_drm) {
                     if (!drm_ctx) {
-                        vui_sdl_drm_initialize(&drm_ctx, sdl_ctx->window);
+                        if (vui_sdl_drm_initialize(&drm_ctx,
+                                                   sdl_ctx->window)) {
+                            // Blank screen so DRM doesn't appear to render over UI
+                            SDL_SetRenderTarget(renderer, NULL);
+                            SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
+                            SDL_RenderClear(renderer);
+                            SDL_RenderPresent(renderer);
+                        }
+                    }
+                    if (drm_ctx &&
+                        vui_sdl_drm_present(drm_ctx, sdl_ctx->frame)) {
+                        handle_final_blit = 0;
+                    } else {
+                        get_texture_from_drm_prime_frame(sdl_ctx,
+                                                         sdl_ctx->frame);
                     }
-                    vui_sdl_drm_present(drm_ctx, sdl_ctx->frame);
-                    handle_final_blit = 0;
                 } else {
                     get_texture_from_drm_prime_frame(sdl_ctx, sdl_ctx->frame);
                 }
@@ -2082,15 +2099,15 @@ int vui_update_sdl(vui_context_t *vui)
             dst_rect->y = 0;
             dst_rect->w = tex_w;
             dst_rect->h = tex_h;
-        } else if ((int64_t)out_w * tex_h > (int64_t)out_h * tex_w) {
+        } else if ((uint64_t)out_w * tex_h > (uint64_t)out_h * tex_w) {
             dst_rect->h = out_h;
             dst_rect->y = 0;
-            dst_rect->w = out_h * tex_w / tex_h;
+            dst_rect->w = (uint64_t)out_h * tex_w / tex_h;
             dst_rect->x = (out_w - dst_rect->w) / 2;
         } else {
             dst_rect->w = out_w;
             dst_rect->x = 0;
-            dst_rect->h = out_w * tex_h / tex_w;
+            dst_rect->h = (uint64_t)out_w * tex_h / tex_w;
             dst_rect->y = (out_h - dst_rect->h) / 2;
         }
 
@@ -2107,10 +2124,33 @@ int vui_update_sdl(vui_context_t *vui)
     const Uint32 target = 5; // No need to update faster than 200Hz (gamepad polls at 180Hz, but this is easier to calculate)
     Uint32 frame_delta = SDL_GetTicks() - last_update_time;
     if (frame_delta < target) {
-        SDL_Delay(target - frame_delta);
+        Uint32 wait_ms = target - frame_delta;
+        if (vui->game_mode) {
+            struct timespec deadline;
+            timespec_get(&deadline, TIME_UTC);
+            deadline.tv_nsec += (long) wait_ms * 1000000;
+            if (deadline.tv_nsec >= 1000000000) {
+                deadline.tv_sec++;
+                deadline.tv_nsec -= 1000000000;
+            }
+
+            pthread_mutex_lock(&vpi_present_frame_mutex);
+            while (sdl_ctx->present_frame_sequence == vpi_present_frame_sequence) {
+                int err = pthread_cond_timedwait(
+                    &vpi_present_frame_cond,
+                    &vpi_present_frame_mutex,
+                    &deadline
+                );
+                if (err == ETIMEDOUT) {
+                    break;
+                }
+            }
+            pthread_mutex_unlock(&vpi_present_frame_mutex);
+        } else {
+            SDL_Delay(wait_ms);
+        }
     }
     last_update_time = SDL_GetTicks();
 
     return !vui->quit;
 }
-

gui/ui/ui_sdl_drm.c

@@ -22,15 +22,61 @@ typedef struct vanilla_drm_ctx_t {
 	int fd;
 	uint32_t crtc;
 	int crtc_index;
+	uint32_t crtc_width;
+	uint32_t crtc_height;
 	uint32_t plane_id;
 	int got_plane;
+    uint32_t plane_fb_id_property;
+    int atomic_async;
 	uint32_t fb_id;
 	int got_fb;
+    uint32_t frame_width;
+    uint32_t frame_height;
+    uint32_t frame_format;
+    int32_t dst_x;
+    int32_t dst_y;
+    uint32_t dst_width;
+    uint32_t dst_height;
     vanilla_drm_handle_t handle_cache[MAX_HANDLE_CACHE];
     size_t handle_cache_count;
 } vanilla_drm_ctx_t;
 
-static int find_plane(const int drmfd, const int crtcidx, const uint32_t format, uint32_t *const pplane_id)
+static uint64_t get_object_property(const int drmfd, const uint32_t object_id,
+                                    const uint32_t object_type,
+                                    const char *name,
+                                    uint32_t *property_id)
+{
+    drmModeObjectPropertiesPtr props =
+        drmModeObjectGetProperties(drmfd, object_id, object_type);
+    if (!props) {
+        return 0;
+    }
+
+    uint64_t value = 0;
+    for (uint32_t i = 0; i < props->count_props; i++) {
+        drmModePropertyPtr prop = drmModeGetProperty(drmfd, props->props[i]);
+        if (!prop) {
+            continue;
+        }
+        if (strcmp(prop->name, name) == 0) {
+            if (property_id) {
+                *property_id = prop->prop_id;
+            }
+            value = props->prop_values[i];
+            drmModeFreeProperty(prop);
+            break;
+        }
+        drmModeFreeProperty(prop);
+    }
+
+    drmModeFreeObjectProperties(props);
+    return value;
+}
+
+static int find_plane(const int drmfd, const int crtcidx,
+                      const uint32_t format, const int require_atomic,
+                      uint32_t *const pplane_id,
+                      uint32_t *const pfb_id_property)
 {
     drmModePlaneResPtr planes;
     drmModePlanePtr plane;
@@ -48,7 +94,7 @@ static int find_plane(const int drmfd, const int crtcidx, const uint32_t format,
         plane = drmModeGetPlane(drmfd, planes->planes[i]);
         if (!plane) {
             vpilog("drmModeGetPlane failed: %s\n", strerror(errno));
-            break;
+            continue;
         }
 
         if (!(plane->possible_crtcs & (1 << crtcidx))) {
@@ -65,24 +111,101 @@ static int find_plane(const int drmfd, const int crtcidx, const uint32_t format,
             continue;
         }
 
+        uint32_t fb_id_property = 0;
+        uint32_t type_property = 0;
+        uint64_t plane_type = get_object_property(
+            drmfd, plane->plane_id, DRM_MODE_OBJECT_PLANE, "type",
+            &type_property);
+        if (type_property && plane_type != DRM_PLANE_TYPE_OVERLAY) {
+            drmModeFreePlane(plane);
+            continue;
+        }
+        if (require_atomic) {
+            get_object_property(drmfd, plane->plane_id,
+                                DRM_MODE_OBJECT_PLANE, "FB_ID",
+                                &fb_id_property);
+            if (!fb_id_property) {
+                drmModeFreePlane(plane);
+                continue;
+            }
+        }
+
         *pplane_id = plane->plane_id;
+        *pfb_id_property = fb_id_property;
         drmModeFreePlane(plane);
         break;
     }
 
-    if (i == planes->count_planes) ret = -1;
+    if (i == planes->count_planes) {
+        ret = -1;
+    }
 
     drmModeFreePlaneResources(planes);
     return ret;
 }
 
+static int set_plane_async(vanilla_drm_ctx_t *ctx, uint32_t fb_id)
+{
+    drmModeAtomicReqPtr req = drmModeAtomicAlloc();
+    if (!req) {
+        return -1;
+    }
+
+    int ret = drmModeAtomicAddProperty(
+        req, ctx->plane_id, ctx->plane_fb_id_property, fb_id);
+    if (ret >= 0) {
+        ret = drmModeAtomicCommit(
+            ctx->fd, req, DRM_MODE_PAGE_FLIP_ASYNC, NULL);
+    }
+
+    drmModeAtomicFree(req);
+    return ret;
+}
+
+static void fit_frame_to_crtc(const vanilla_drm_ctx_t *ctx,
+                              uint32_t frame_width, uint32_t frame_height,
+                              int32_t *dst_x, int32_t *dst_y,
+                              uint32_t *dst_width, uint32_t *dst_height)
+{
+    if (!ctx->crtc_width || !ctx->crtc_height ||
+        !frame_width || !frame_height) {
+        *dst_x = 0;
+        *dst_y = 0;
+        *dst_width = frame_width;
+        *dst_height = frame_height;
+        return;
+    }
+
+    if ((uint64_t)ctx->crtc_width * frame_height >
+        (uint64_t)ctx->crtc_height * frame_width) {
+        *dst_height = ctx->crtc_height;
+        *dst_width = (uint64_t)ctx->crtc_height * frame_width / frame_height;
+        *dst_x = (ctx->crtc_width - *dst_width) / 2;
+        *dst_y = 0;
+    } else {
+        *dst_width = ctx->crtc_width;
+        *dst_height = (uint64_t)ctx->crtc_width * frame_height / frame_width;
+        *dst_x = 0;
+        *dst_y = (ctx->crtc_height - *dst_height) / 2;
+    }
+}
+
 int vui_sdl_drm_present(vanilla_drm_ctx_t *ctx, AVFrame *frame)
 {
     const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *) frame->data[0];
     const uint32_t format = desc->layers[0].format;
 
     if (!ctx->got_plane) {
-        if (find_plane(ctx->fd, ctx->crtc_index, format, &ctx->plane_id) < 0) {
+        int found = find_plane(ctx->fd, ctx->crtc_index, format,
+                               ctx->atomic_async, &ctx->plane_id,
+                               &ctx->plane_fb_id_property);
+        if (found < 0 && ctx->atomic_async) {
+            ctx->atomic_async = 0;
+            found = find_plane(ctx->fd, ctx->crtc_index, format, 0,
+                               &ctx->plane_id,
+                               &ctx->plane_fb_id_property);
+        }
+        if (found < 0) {
             vpilog("Failed to find plane for format: %x\n", format);
             return 0;
         } else {
@@ -149,10 +272,44 @@ int vui_sdl_drm_present(vanilla_drm_ctx_t *ctx, AVFrame *frame)
         return 0;
     }
 
-    if (drmModeSetPlane(ctx->fd, ctx->plane_id, ctx->crtc, new_fb, 0,
-                    0, 0, frame->width, frame->height,
-                    0, 0, frame->width << 16, frame->height << 16) != 0) {
+    int32_t dst_x;
+    int32_t dst_y;
+    uint32_t dst_width;
+    uint32_t dst_height;
+    fit_frame_to_crtc(ctx, frame->width, frame->height,
+                      &dst_x, &dst_y, &dst_width, &dst_height);
+
+    int geometry_changed =
+        !ctx->got_fb ||
+        ctx->frame_width != (uint32_t) frame->width ||
+        ctx->frame_height != (uint32_t) frame->height ||
+        ctx->frame_format != format ||
+        ctx->dst_x != dst_x || ctx->dst_y != dst_y ||
+        ctx->dst_width != dst_width || ctx->dst_height != dst_height;
+
+    int presented = 0;
+    if (!geometry_changed && ctx->atomic_async) {
+        if (set_plane_async(ctx, new_fb) == 0) {
+            presented = 1;
+        } else {
+            /*
+             * Some drivers advertise atomic async flips but reject a
+             * particular plane configuration. Fall back permanently rather
+             * than logging and retrying the unsupported path every frame.
+             */
+            vpilog("Atomic async plane update failed, using vblank updates: %s\n",
+                   strerror(errno));
+            ctx->atomic_async = 0;
+        }
+    }
+
+    if (!presented &&
+        drmModeSetPlane(ctx->fd, ctx->plane_id, ctx->crtc, new_fb, 0,
+                        dst_x, dst_y, dst_width, dst_height,
+                        0, 0, frame->width << 16,
+                        frame->height << 16) != 0) {
         vpilog("Failed to set plane: %s\n", strerror(errno));
+        drmModeRmFB(ctx->fd, new_fb);
         return 0;
     }
 
@@ -162,57 +319,95 @@ int vui_sdl_drm_present(vanilla_drm_ctx_t *ctx, AVFrame *frame)
     }
     ctx->fb_id = new_fb;
     ctx->got_fb = 1;
+    ctx->frame_width = frame->width;
+    ctx->frame_height = frame->height;
+    ctx->frame_format = format;
+    ctx->dst_x = dst_x;
+    ctx->dst_y = dst_y;
+    ctx->dst_width = dst_width;
+    ctx->dst_height = dst_height;
 
     return 1;
 }
 
 int vui_sdl_drm_initialize(vanilla_drm_ctx_t **c, SDL_Window *window)
 {
+    *c = NULL;
     vanilla_drm_ctx_t *ctx = (vanilla_drm_ctx_t *) malloc(sizeof(vanilla_drm_ctx_t));
-    *c = ctx;
+    if (!ctx) {
+        vpilog("Failed to allocate DRM context\n");
+        return 0;
+    }
 
     memset(ctx, 0, sizeof(vanilla_drm_ctx_t));
 
     SDL_SysWMinfo wmi;
     SDL_VERSION(&wmi.version);
-    if (!SDL_GetWindowWMInfo(window, &wmi)) return 0;
-    if (wmi.subsystem != SDL_SYSWM_KMSDRM) return 0;
+    if (!SDL_GetWindowWMInfo(window, &wmi) ||
+        wmi.subsystem != SDL_SYSWM_KMSDRM) {
+        free(ctx);
+        return 0;
+    }
 
     ctx->fd = wmi.info.kmsdrm.drm_fd;
 
 	int ret = 0;
+    uint64_t async_cap = 0;
+    if (drmGetCap(ctx->fd, DRM_CAP_ATOMIC_ASYNC_PAGE_FLIP, &async_cap) == 0 &&
+        async_cap &&
+        drmSetClientCap(ctx->fd, DRM_CLIENT_CAP_ATOMIC, 1) == 0) {
+        ctx->atomic_async = 1;
+        vpilog("DRM atomic async plane updates enabled\n");
+    } else {
+        vpilog("DRM atomic async plane updates unavailable; updates will wait for vblank\n");
+    }
 
 	// Find DRM output
 	drmModeResPtr res = drmModeGetResources(ctx->fd);
+	if (!res) {
+        vpilog("Failed to get DRM resources: %s\n", strerror(errno));
+        free(ctx);
+        return 0;
+    }
 
 	for (int i = 0; i < res->count_connectors; i++) {
 		drmModeConnectorPtr c = drmModeGetConnector(ctx->fd, res->connectors[i]);
-		if (c->encoder_id) {
+		if (c && c->encoder_id) {
 			drmModeEncoderPtr enc = drmModeGetEncoder(ctx->fd, c->encoder_id);
-			if (enc->crtc_id) {
+			if (enc && enc->crtc_id) {
 				drmModeCrtcPtr crtc = drmModeGetCrtc(ctx->fd, enc->crtc_id);
 
-				// Good! We can use this connector :)
-				ctx->crtc = crtc->crtc_id;
+				if (crtc) {
+					// Good! We can use this connector :)
+					ctx->crtc = crtc->crtc_id;
+					ctx->crtc_width = crtc->mode_valid ? crtc->mode.hdisplay : crtc->width;
+					ctx->crtc_height = crtc->mode_valid ? crtc->mode.vdisplay : crtc->height;
 
-                for (int j = 0; j < res->count_crtcs; j++) {
-                    if (res->crtcs[j] == crtc->crtc_id) {
-                        ctx->crtc_index = j;
-                        break;
+                    for (int j = 0; j < res->count_crtcs; j++) {
+                        if (res->crtcs[j] == crtc->crtc_id) {
+                            ctx->crtc_index = j;
+                            break;
+                        }
                     }
-                }
 
-                ret = 1;
+                    ret = ctx->crtc_width > 0 && ctx->crtc_height > 0;
 
-				drmModeFreeCrtc(crtc);
+					drmModeFreeCrtc(crtc);
+				}
 			}
-			drmModeFreeEncoder(enc);
+			if (enc) drmModeFreeEncoder(enc);
 		}
-		drmModeFreeConnector(c);
+		if (c) drmModeFreeConnector(c);
+		if (ret) break;
 	}
 
 	// Free DRM resources
 	drmModeFreeResources(res);
+	if (!ret) {
+        vpilog("Failed to find an active DRM output\n");
+        free(ctx);
+        return 0;
+    }
 
     ctx->got_plane = 0;
     ctx->got_fb = 0;
@@ -222,6 +417,7 @@ int vui_sdl_drm_initialize(vanilla_drm_ctx_t **c, SDL_Window *window)
         h->handle = 0;
     }
 
+    *c = ctx;
     return ret;
 }