From 80b240a89d5eea1201b3c98a60ac4c05f5fd75f1 Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:00 +0200 Subject: [PATCH 01/16] recording: add video_record_allow_frame_drop and video_record_fifo_frames Wires two record_ffmpeg queue knobs through the settings_def single-source tables, record_params and the Recording settings list (shown for the ffmpeg driver only). video_record_allow_frame_drop bool, default true video_record_fifo_frames uint, default 32, 8..128 step 4 (Lakka: advanced only) Both are read when a recording starts; the driver side is the next patch. msg_hash_lbl_str.h regenerated with tools/gen_lbl_str.py, keeping only the two new rows. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- config.def.h | 9 +++++++++ configuration.h | 2 ++ menu/cbs/menu_cbs_sublabel.c | 2 ++ menu/menu_displaylist.c | 14 +++++++++----- msg_hash_lbl_str.h | 2 ++ record/record_driver.c | 2 ++ record/record_driver.h | 4 ++++ settings/settings_def_recording_video.h | 10 ++++++++++ tools/settings_table.reference | 6 ++++++ 9 files changed, 46 insertions(+), 5 deletions(-) diff --git a/config.def.h b/config.def.h index ac7dde5a5c38..7b9397a02b81 100644 --- a/config.def.h +++ b/config.def.h @@ -1196,6 +1196,15 @@ /* Record post-shaded GPU output instead of raw game footage if available. */ #define DEFAULT_GPU_RECORD false +/* Drop recorded video frames when the encoder queue is full, instead + * of stalling the frontend until it drains. */ +#define DEFAULT_VIDEO_RECORD_ALLOW_FRAME_DROP true + +/* Depth of the recording video queue, in frames. */ +#define DEFAULT_VIDEO_RECORD_FIFO_FRAMES 32 +#define MIN_VIDEO_RECORD_FIFO_FRAMES 8 +#define MAX_VIDEO_RECORD_FIFO_FRAMES 128 + /* Watch shader files for changes and auto-apply as necessary. */ #define DEFAULT_VIDEO_SHADER_WATCH_FILES false diff --git a/configuration.h b/configuration.h index 8f69a2d2e5d9..64c8928b5168 100644 --- a/configuration.h +++ b/configuration.h @@ -426,6 +426,7 @@ typedef struct settings unsigned window_auto_dims_max; unsigned video_record_threads; + unsigned video_record_fifo_frames; unsigned libnx_overclock; unsigned ai_service_mode; @@ -669,6 +670,7 @@ typedef struct settings bool video_disable_composition; bool video_post_filter_record; bool video_gpu_record; + bool video_record_allow_frame_drop; bool video_gpu_screenshot; bool video_allow_rotate; bool video_shared_context; diff --git a/menu/cbs/menu_cbs_sublabel.c b/menu/cbs/menu_cbs_sublabel.c index 3fd282305705..a3fc7c1a7605 100644 --- a/menu/cbs/menu_cbs_sublabel.c +++ b/menu/cbs/menu_cbs_sublabel.c @@ -2434,6 +2434,8 @@ int menu_cbs_init_bind_sublabel(menu_file_list_cbs_t *cbs, { MENU_ENUM_LABEL_CLOUD_SYNC_RESOLVE_KEEP_SERVER, MENU_ENUM_SUBLABEL_CLOUD_SYNC_RESOLVE_KEEP_SERVER }, { MENU_ENUM_LABEL_CORE_MANAGER_LIST, MENU_ENUM_SUBLABEL_CORE_MANAGER_LIST }, { MENU_ENUM_LABEL_VIDEO_POST_FILTER_RECORD, MENU_ENUM_SUBLABEL_VIDEO_POST_FILTER_RECORD }, + { MENU_ENUM_LABEL_VIDEO_RECORD_ALLOW_FRAME_DROP, MENU_ENUM_SUBLABEL_VIDEO_RECORD_ALLOW_FRAME_DROP }, + { MENU_ENUM_LABEL_VIDEO_RECORD_FIFO_FRAMES, MENU_ENUM_SUBLABEL_VIDEO_RECORD_FIFO_FRAMES }, { MENU_ENUM_LABEL_NETPLAY_NICKNAME, MENU_ENUM_SUBLABEL_NETPLAY_NICKNAME }, { MENU_ENUM_LABEL_CHEEVOS_USERNAME, MENU_ENUM_SUBLABEL_CHEEVOS_USERNAME }, { MENU_ENUM_LABEL_CHEEVOS_PASSWORD, MENU_ENUM_SUBLABEL_CHEEVOS_PASSWORD }, diff --git a/menu/menu_displaylist.c b/menu/menu_displaylist.c index 2544192fa5fc..cd8e5c4870b2 100644 --- a/menu/menu_displaylist.c +++ b/menu/menu_displaylist.c @@ -11494,6 +11494,8 @@ unsigned menu_displaylist_build_list( {MENU_ENUM_LABEL_VIDEO_RECORD_THREADS, PARSE_ONLY_UINT, false}, {MENU_ENUM_LABEL_VIDEO_POST_FILTER_RECORD, PARSE_ONLY_BOOL, false}, {MENU_ENUM_LABEL_VIDEO_GPU_RECORD, PARSE_ONLY_BOOL, false}, + {MENU_ENUM_LABEL_VIDEO_RECORD_ALLOW_FRAME_DROP, PARSE_ONLY_BOOL, false}, + {MENU_ENUM_LABEL_VIDEO_RECORD_FIFO_FRAMES, PARSE_ONLY_UINT, false}, {MENU_ENUM_LABEL_STREAMING_MODE, PARSE_ONLY_UINT, false}, {MENU_ENUM_LABEL_VIDEO_STREAM_QUALITY, PARSE_ONLY_UINT, false}, {MENU_ENUM_LABEL_STREAM_CONFIG, PARSE_ONLY_PATH, false}, @@ -11506,11 +11508,13 @@ unsigned menu_displaylist_build_list( build_list[3].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_RECORD_THREADS */ build_list[4].checked = has_video; /* MENU_ENUM_LABEL_VIDEO_POST_FILTER_RECORD */ build_list[5].checked = has_video; /* MENU_ENUM_LABEL_VIDEO_GPU_RECORD */ - build_list[6].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_MODE */ - build_list[7].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_STREAM_QUALITY */ - build_list[8].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAM_CONFIG */ - build_list[9].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_TITLE */ - build_list[10].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_URL */ + build_list[6].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_RECORD_ALLOW_FRAME_DROP */ + build_list[7].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_RECORD_FIFO_FRAMES */ + build_list[8].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_MODE */ + build_list[9].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_STREAM_QUALITY */ + build_list[10].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAM_CONFIG */ + build_list[11].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_TITLE */ + build_list[12].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_URL */ for (i = 0; i < ARRAY_SIZE(build_list); i++) { diff --git a/msg_hash_lbl_str.h b/msg_hash_lbl_str.h index b9e141c726e7..6ea3989676c6 100644 --- a/msg_hash_lbl_str.h +++ b/msg_hash_lbl_str.h @@ -1719,6 +1719,8 @@ #define MENU_ENUM_LABEL_VIDEO_OVERSCAN_CORRECTION_TOP_STR "video_overscan_correction_top" #define MENU_ENUM_LABEL_VIDEO_POST_FILTER_RECORD_STR "video_post_filter_record" #define MENU_ENUM_LABEL_VIDEO_PRESENT_TIMING_FROM_DISPLAY_STR "video_present_timing_from_display" +#define MENU_ENUM_LABEL_VIDEO_RECORD_ALLOW_FRAME_DROP_STR "video_record_allow_frame_drop" +#define MENU_ENUM_LABEL_VIDEO_RECORD_FIFO_FRAMES_STR "video_record_fifo_frames" #define MENU_ENUM_LABEL_VIDEO_RECORD_QUALITY_STR "video_record_quality" #define MENU_ENUM_LABEL_VIDEO_RECORD_THREADS_STR "video_record_threads" #define MENU_ENUM_LABEL_VIDEO_REFRESH_RATE_STR "video_refresh_rate" diff --git a/record/record_driver.c b/record/record_driver.c index f861e57cbb14..e923dbba026f 100644 --- a/record/record_driver.c +++ b/record/record_driver.c @@ -300,6 +300,8 @@ bool recording_init(void) params.video_stream_scale_factor = settings->uints.video_stream_scale_factor; params.video_record_threads = settings->uints.video_record_threads; params.streaming_mode = settings->uints.streaming_mode; + params.allow_frame_drop = settings->bools.video_record_allow_frame_drop; + params.video_fifo_frames = settings->uints.video_record_fifo_frames; params.out_dims = VIDEO_SCALE_PACK( av_info->geometry.base_width, av_info->geometry.base_height); diff --git a/record/record_driver.h b/record/record_driver.h index 69f3a9d93fe8..80d2fbd04d72 100644 --- a/record/record_driver.h +++ b/record/record_driver.h @@ -70,6 +70,8 @@ struct record_params unsigned video_stream_scale_factor; unsigned video_record_threads; unsigned streaming_mode; + /* Video queue depth in frames (driver clamps/defaults out-of-range). */ + unsigned video_fifo_frames; /* Aspect ratio of input video. Parameters are passed to the muxer, * the video itself is not scaled. @@ -82,6 +84,8 @@ struct record_params enum ffemu_pix_format pix_fmt; bool video_gpu_record; + /* Drop frames when the encoder queue is full instead of blocking. */ + bool allow_frame_drop; }; struct record_video_data diff --git a/settings/settings_def_recording_video.h b/settings/settings_def_recording_video.h index b62c38f85dc1..f362f3d1baa6 100644 --- a/settings/settings_def_recording_video.h +++ b/settings/settings_def_recording_video.h @@ -14,3 +14,13 @@ S_BOOL(video_gpu_record, VIDEO_GPU_RECORD, DEFAULT_GPU_RECORD, SD_FLAG_NONE, 0, 0, "Use GPU Recording", "Record output of GPU shaded material if available.") +S_BOOL(video_record_allow_frame_drop, VIDEO_RECORD_ALLOW_FRAME_DROP, + "video_record_allow_frame_drop", + DEFAULT_VIDEO_RECORD_ALLOW_FRAME_DROP, SD_FLAG_NONE, 0, 0, + "Drop Frames When Encoder Falls Behind", + "When the recording encoder can't keep up, skip frames in the recording instead of slowing down the game. Disable to keep every frame at the cost of stutter.") +S_UINT(video_record_fifo_frames, VIDEO_RECORD_FIFO_FRAMES, + "video_record_fifo_frames", + DEFAULT_VIDEO_RECORD_FIFO_FRAMES, SD_FLAG_LAKKA_ADVANCED, SDESC_RANGE_MINMAX, 0, MIN_VIDEO_RECORD_FIFO_FRAMES, MAX_VIDEO_RECORD_FIFO_FRAMES, 4, MIN_VIDEO_RECORD_FIFO_FRAMES, setting_action_ok_uint, NULL, + "Recording Video Buffer (Frames)", + "Frames queued between the game and the encoder. Larger values absorb encoder hiccups but use more memory. Applies to the next recording.") diff --git a/tools/settings_table.reference b/tools/settings_table.reference index 5470d8fbc838..2009827ca1f2 100644 --- a/tools/settings_table.reference +++ b/tools/settings_table.reference @@ -309,6 +309,7 @@ bool menu_show_reboot 1 1 bool menu_show_shutdown 1 1 bool video_post_filter_record 1 0 bool video_gpu_record 1 0 +bool video_record_allow_frame_drop 1 1 bool menu_rgui_full_width_layout 1 1 bool video_shader_enable 1 1 bool quick_menu_show_save_core_overrides 1 1 @@ -601,6 +602,7 @@ uint audio_output_layout 1 0 uint microphone_block_frames 1 0 uint menu_thumbnail_preview_threads 1 1 uint menu_file_browser_extension_display 1 MENU_FILE_BROWSER_EXTENSION_DISPLAY_ALWAYS +uint video_record_fifo_frames 1 32 uint video_scale 1 3 uint video_window_opacity 1 100 uint video_stream_port 1 56400 @@ -999,6 +1001,7 @@ bool menu_show_reboot 1 1 bool menu_show_shutdown 1 1 bool video_post_filter_record 1 0 bool video_gpu_record 1 0 +bool video_record_allow_frame_drop 1 1 bool menu_rgui_full_width_layout 1 1 bool video_shader_enable 1 1 bool quick_menu_show_save_core_overrides 1 1 @@ -1285,6 +1288,7 @@ uint audio_output_layout 1 0 uint microphone_block_frames 1 0 uint menu_thumbnail_preview_threads 1 1 uint menu_file_browser_extension_display 1 MENU_FILE_BROWSER_EXTENSION_DISPLAY_ALWAYS +uint video_record_fifo_frames 1 32 uint video_scale 1 3 uint video_window_opacity 1 100 uint video_stream_port 1 56400 @@ -1692,6 +1696,7 @@ bool menu_show_reboot 1 1 bool menu_show_shutdown 1 1 bool video_post_filter_record 1 0 bool video_gpu_record 1 0 +bool video_record_allow_frame_drop 1 1 bool menu_rgui_full_width_layout 1 1 bool video_shader_enable 1 1 bool quick_menu_show_save_core_overrides 1 1 @@ -1994,6 +1999,7 @@ uint audio_output_layout 1 0 uint microphone_block_frames 1 0 uint menu_thumbnail_preview_threads 1 1 uint menu_file_browser_extension_display 1 MENU_FILE_BROWSER_EXTENSION_DISPLAY_ALWAYS +uint video_record_fifo_frames 1 32 uint video_scale 1 3 uint video_window_opacity 1 100 uint video_stream_port 1 56400 From 04524befee6c9ffb842ada03b762263e2f1c000c Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:00 +0200 Subject: [PATCH 02/16] record_ffmpeg: drop instead of stalling when the encoder falls behind With the encoder behind, a full queue blocked the frontend in push_video/push_audio, so a slow encoder slowed the game down. With video_record_allow_frame_drop (default on): - a frame that doesn't fit is dropped; the count rides with the next queued frame and the encoder leaves a pts gap for it, so video stays aligned with audio - an audio chunk that doesn't fit is owed as silence and written ahead of the next chunk that does, and anything still owed is encoded at finalize, so the audio track keeps its length - drops are reported every 64 frames and summarised at teardown The video queue depth comes from video_record_fifo_frames (8..128). The audio queue holds 2 s at the core's rate instead of ~0.36 s at 48 kHz. With drop off, both paths block as before. ffmpeg_pipeline_test gains a drop-mode pass that checks the surviving frames come out in order, pixel-exact and at their own timestamps. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- record/drivers/record_ffmpeg.c | 236 +++++++++++++++--- .../ffmpeg_pipeline/ffmpeg_pipeline_test.c | 69 ++++- .../record/ffmpeg_pipeline/stubs_retroarch.c | 18 ++ settings/settings_def_recording_video.h | 2 +- 4 files changed, 283 insertions(+), 42 deletions(-) diff --git a/record/drivers/record_ffmpeg.c b/record/drivers/record_ffmpeg.c index 0b4b6a1190fa..db2030cb7e49 100644 --- a/record/drivers/record_ffmpeg.c +++ b/record/drivers/record_ffmpeg.c @@ -70,6 +70,7 @@ extern "C" { #endif #include "../../retroarch.h" +#include "../../runloop.h" #include "../../verbosity.h" #ifndef FFMPEG3 @@ -247,8 +248,31 @@ typedef struct ffmpeg retro_eventcount_t space; bool data_init; bool space_init; + + /* Drop-on-full accounting. Written by the producer only; the + * totals are read after the encoder thread has been joined. */ + uint64_t video_frames_in; + uint64_t video_frames_dropped; + uint64_t video_frames_logged; + /* Frames dropped since the last queued one; rides along with the + * next frame so the encoder leaves a pts gap for them. */ + unsigned video_pts_pending; + unsigned video_fifo_frames; + bool allow_frame_drop; + + /* Audio bytes dropped on a full queue and still owed to it as + * silence, so the sample-count timeline never loses time. */ + size_t audio_silence_pending; + uint64_t audio_bytes_dropped; } ffmpeg_t; +/* attr_fifo record: the frame plus the drops right before it. */ +struct ff_video_attr +{ + struct record_video_data vid; + unsigned pts_skip; +}; + /* How long a push waits for room before looking at alive again. The * encoder notifies space on every read and deinit_thread() on the way * out, so this only bounds a wake that never comes. */ @@ -1037,12 +1061,27 @@ static bool ffmpeg_init_muxer_post(ffmpeg_t *handle) return avformat_write_header(handle->muxer.ctx, NULL) >= 0; } -#define MAX_FRAMES 32 +/* Audio queue depth in seconds of input audio. */ +#define AUDIO_FIFO_SECONDS 2 +/* Video queue depth, from video_record_fifo_frames. */ +#define DEFAULT_VIDEO_FIFO_FRAMES 32 +#define MIN_VIDEO_FIFO_FRAMES 8 +#define MAX_VIDEO_FIFO_FRAMES 128 +/* One warning per this many newly dropped frames. */ +#define DROP_LOG_INTERVAL 64 static void ffmpeg_thread(void *data); static bool init_thread(ffmpeg_t *handle) { + unsigned vf = handle->video_fifo_frames; + double rate = handle->params.samplerate > 0.0 + ? handle->params.samplerate : 48000.0; + + if (vf < MIN_VIDEO_FIFO_FRAMES || vf > MAX_VIDEO_FIFO_FRAMES) + vf = DEFAULT_VIDEO_FIFO_FRAMES; + handle->video_fifo_frames = vf; + handle->data_init = retro_eventcount_init(&handle->data); handle->space_init = retro_eventcount_init(&handle->space); if (!handle->data_init || !handle->space_init) @@ -1061,14 +1100,16 @@ static bool init_thread(ffmpeg_t *handle) * larger attr ring cannot admit a frame the video ring lacks * room for.) */ handle->fifos_init = - retro_spsc_init(&handle->audio_fifo, 32000 * sizeof(int16_t) * - handle->params.channels * MAX_FRAMES / 60) /* Some arbitrary max size. */ - && retro_spsc_init(&handle->attr_fifo, sizeof(struct record_video_data) * MAX_FRAMES) + retro_spsc_init(&handle->audio_fifo, (size_t)(rate * AUDIO_FIFO_SECONDS) + * handle->params.channels * sizeof(int16_t)) + && retro_spsc_init(&handle->attr_fifo, sizeof(struct ff_video_attr) * vf) && retro_spsc_init(&handle->video_fifo, - VIDEO_SCALE_AREA(handle->params.fb_dims) * - handle->video.pix_size * MAX_FRAMES); + (size_t)VIDEO_SCALE_AREA(handle->params.fb_dims) * + handle->video.pix_size * vf); if (!handle->fifos_init) { + RARCH_ERR("[FFmpeg] Failed to allocate recording queues" + " (%u video frames).\n", vf); retro_spsc_free(&handle->audio_fifo); retro_spsc_free(&handle->attr_fifo); retro_spsc_free(&handle->video_fifo); @@ -1080,6 +1121,20 @@ static bool init_thread(ffmpeg_t *handle) retro_atomic_store_release_int(&handle->alive, 1); handle->thread = sthread_create(ffmpeg_thread, handle); + if (!handle->thread) + { + RARCH_ERR("[FFmpeg] Failed to create encoder thread.\n"); + retro_atomic_store_release_int(&handle->alive, 0); + retro_eventcount_free(&handle->data); + retro_eventcount_free(&handle->space); + handle->data_init = handle->space_init = false; + return false; + } + + RARCH_LOG("[FFmpeg] Video queue: %u frames (%.1f MiB), drop-on-full: %s.\n", + vf, ((double)VIDEO_SCALE_AREA(handle->params.fb_dims) + * handle->video.pix_size * vf) / (1024.0 * 1024.0), + handle->allow_frame_drop ? "yes" : "no"); return true; } @@ -1120,6 +1175,20 @@ static void ffmpeg_free(void *data) deinit_thread(handle); deinit_thread_buf(handle); + if (handle->video_frames_in > 0) + RARCH_LOG("[FFmpeg] Recording ended: %llu video frames in," + " %llu dropped (%.2f%%).\n", + (unsigned long long)handle->video_frames_in, + (unsigned long long)handle->video_frames_dropped, + 100.0 * (double)handle->video_frames_dropped + / (double)handle->video_frames_in); + if (handle->audio_bytes_dropped && handle->params.samplerate > 0.0) + RARCH_LOG("[FFmpeg] Recording ended: %.2f s of audio replaced" + " with silence.\n", + (double)handle->audio_bytes_dropped + / (handle->params.samplerate + * handle->params.channels * sizeof(int16_t))); + if (handle->audio.codec) { #if FFMPEG8 @@ -1206,8 +1275,10 @@ static void *ffmpeg_new(const struct record_params *params) avformat_network_init(); #endif - handle->params = *params; - handle->pkt = av_packet_alloc(); + handle->params = *params; + handle->pkt = av_packet_alloc(); + handle->allow_frame_drop = params->allow_frame_drop; + handle->video_fifo_frames = params->video_fifo_frames; switch (params->preset) { @@ -1255,12 +1326,31 @@ static void *ffmpeg_new(const struct record_params *params) return NULL; } +static bool ffmpeg_video_fifo_has_room(ffmpeg_t *handle, size_t video_bytes) +{ + return retro_spsc_write_avail(&handle->attr_fifo) + >= sizeof(struct ff_video_attr) + && retro_spsc_write_avail(&handle->video_fifo) >= video_bytes; +} + +static void ffmpeg_report_drops(uint64_t dropped) +{ + char msg[128]; + size_t _len = snprintf(msg, sizeof(msg), + "Recording: %llu frames dropped (encoder too slow)", + (unsigned long long)dropped); + RARCH_WARN("[FFmpeg] %s.\n", msg); + runloop_msg_queue_push(msg, _len, 1, 180, false, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_WARNING); +} + static bool ffmpeg_push_video(void *data, const struct record_video_data *vid) { unsigned y; unsigned rows; - struct record_video_data attr_data; + size_t video_bytes; + struct ff_video_attr attr; bool drop_frame = false; ffmpeg_t *handle = (ffmpeg_t*)data; int offset = 0; @@ -1276,61 +1366,98 @@ static bool ffmpeg_push_video(void *data, if (drop_frame) return true; + if (!retro_atomic_load_acquire_int(&handle->alive)) + return false; + /* Tightly pack our frame to conserve memory. * libretro tends to use a very large pitch. */ - attr_data = *vid; + attr.vid = *vid; + attr.pts_skip = 0; - if (attr_data.is_dupe) + if (attr.vid.is_dupe) { - attr_data.dims = 0; - attr_data.pitch = 0; + attr.vid.dims = 0; + attr.vid.pitch = 0; } else - attr_data.pitch = (int)(VIDEO_SCALE_W(attr_data.dims) + attr.vid.pitch = (int)(VIDEO_SCALE_W(attr.vid.dims) * handle->video.pix_size); - rows = VIDEO_SCALE_H(attr_data.dims); + rows = VIDEO_SCALE_H(attr.vid.dims); + video_bytes = (size_t)rows * attr.vid.pitch; + + handle->video_frames_in++; + + /* Drop mode: a full queue costs this frame, never the frontend. */ + if ( handle->allow_frame_drop + && !ffmpeg_video_fifo_has_room(handle, video_bytes)) + { + handle->video_frames_dropped++; + handle->video_pts_pending++; + if (handle->video_frames_dropped - handle->video_frames_logged + >= DROP_LOG_INTERVAL) + { + handle->video_frames_logged = handle->video_frames_dropped; + ffmpeg_report_drops(handle->video_frames_dropped); + } + return true; + } for (;;) { - /* Room for the attr and for the frame's bytes: the old check - * only asked the attr fifo, relying on the two being sized in - * step; the rings are sized independently now (power-of-two - * rounding), so ask both. */ int key; if (!retro_atomic_load_acquire_int(&handle->alive)) return false; - if ( retro_spsc_write_avail(&handle->attr_fifo) >= sizeof(attr_data) - && retro_spsc_write_avail(&handle->video_fifo) - >= (size_t)rows * attr_data.pitch) + if (ffmpeg_video_fifo_has_room(handle, video_bytes)) break; key = retro_eventcount_prepare_wait(&handle->space); if ( !retro_atomic_load_acquire_int(&handle->alive) - || ( retro_spsc_write_avail(&handle->attr_fifo) >= sizeof(attr_data) - && retro_spsc_write_avail(&handle->video_fifo) - >= (size_t)rows * attr_data.pitch)) + || ffmpeg_video_fifo_has_room(handle, video_bytes)) retro_eventcount_cancel_wait(&handle->space); else retro_eventcount_commit_wait_timeout(&handle->space, key, FFMPEG_PUSH_WAIT_US); } + attr.pts_skip = handle->video_pts_pending; + handle->video_pts_pending = 0; + /* Frame first, attr last: the encoder takes the attr as the * signal that a whole frame is behind it, and the ring's * release/acquire on each write orders the rows before it. */ for (y = 0; y < rows; y++, offset += vid->pitch) retro_spsc_write(&handle->video_fifo, - (const uint8_t*)vid->data + offset, attr_data.pitch); + (const uint8_t*)vid->data + offset, attr.vid.pitch); - retro_spsc_write(&handle->attr_fifo, &attr_data, sizeof(attr_data)); + retro_spsc_write(&handle->attr_fifo, &attr, sizeof(attr)); retro_eventcount_notify(&handle->data); return true; } +/* Writes as much owed silence as fits, in whole frames. */ +static void ffmpeg_audio_write_silence(ffmpeg_t *handle) +{ + static const int16_t zeros[1024] = {0}; + size_t avail = retro_spsc_write_avail(&handle->audio_fifo); + size_t todo = handle->audio_silence_pending; + size_t frame = handle->params.channels * sizeof(int16_t); + + if (todo > avail) + todo = avail - (avail % frame); + + handle->audio_silence_pending -= todo; + while (todo) + { + size_t chunk = todo < sizeof(zeros) ? todo : sizeof(zeros); + retro_spsc_write(&handle->audio_fifo, zeros, chunk); + todo -= chunk; + } +} + static bool ffmpeg_push_audio(void *data, const struct record_audio_data *audio_data) { @@ -1345,6 +1472,25 @@ static bool ffmpeg_push_audio(void *data, need = audio_data->frames * handle->params.channels * sizeof(int16_t); + if (handle->allow_frame_drop) + { + if (!retro_atomic_load_acquire_int(&handle->alive)) + return false; + /* A chunk that doesn't fit is owed as silence, not waited for. */ + if (handle->audio_silence_pending) + ffmpeg_audio_write_silence(handle); + if ( !handle->audio_silence_pending + && retro_spsc_write_avail(&handle->audio_fifo) >= need) + retro_spsc_write(&handle->audio_fifo, audio_data->data, need); + else + { + handle->audio_silence_pending += need; + handle->audio_bytes_dropped += need; + } + retro_eventcount_notify(&handle->data); + return true; + } + for (;;) { int key; @@ -1466,11 +1612,14 @@ static void ffmpeg_scale_input(ffmpeg_t *handle, } static bool ffmpeg_push_video_thread(ffmpeg_t *handle, - const struct record_video_data *vid) + const struct record_video_data *vid, unsigned pts_skip) { if (!vid->is_dupe) ffmpeg_scale_input(handle, vid); + /* Leave a gap for dropped frames so A/V stays aligned. */ + handle->video.frame_cnt += pts_skip; + handle->video.conv_frame->pts = handle->video.frame_cnt; if (!encode_video(handle, handle->video.conv_frame)) @@ -1875,7 +2024,7 @@ static void ffmpeg_flush_buffers(ffmpeg_t *handle) do { - struct record_video_data attr_buf; + struct ff_video_attr attr_buf; did_work = false; @@ -1906,14 +2055,29 @@ static void ffmpeg_flush_buffers(ffmpeg_t *handle) { retro_spsc_read(&handle->attr_fifo, &attr_buf, sizeof(attr_buf)); retro_spsc_read(&handle->video_fifo, video_buf, - VIDEO_SCALE_H(attr_buf.dims) * attr_buf.pitch); - attr_buf.data = video_buf; - ffmpeg_push_video_thread(handle, &attr_buf); + VIDEO_SCALE_H(attr_buf.vid.dims) * attr_buf.vid.pitch); + attr_buf.vid.data = video_buf; + ffmpeg_push_video_thread(handle, &attr_buf.vid, attr_buf.pts_skip); did_work = true; } }while (did_work); + /* Pay back silence still owed, so audio runs as long as video. */ + if (handle->config.audio_enable && audio_buf) + { + memset(audio_buf, 0, audio_buf_size); + while (handle->audio_silence_pending >= audio_buf_size) + { + struct record_audio_data aud = {0}; + aud.frames = handle->audio.codec->frame_size; + aud.data = audio_buf; + ffmpeg_push_audio_thread(handle, &aud, true); + handle->audio_silence_pending -= audio_buf_size; + } + handle->audio_silence_pending = 0; + } + /* Flush out last audio. Skip on OOM - audio_buf is the * destination for ffmpeg_flush_audio's internal fifo_read * (via ffmpeg_push_audio_thread) and NULL would NULL-deref. */ @@ -1961,7 +2125,7 @@ static void ffmpeg_thread(void *data) while (retro_atomic_load_acquire_int(&ff->alive)) { - struct record_video_data attr_buf; + struct ff_video_attr attr_buf; bool avail_video = false; bool avail_audio = false; @@ -1990,11 +2154,11 @@ static void ffmpeg_thread(void *data) { retro_spsc_read(&ff->attr_fifo, &attr_buf, sizeof(attr_buf)); retro_spsc_read(&ff->video_fifo, video_buf, - VIDEO_SCALE_H(attr_buf.dims) * attr_buf.pitch); + VIDEO_SCALE_H(attr_buf.vid.dims) * attr_buf.vid.pitch); retro_eventcount_notify(&ff->space); - attr_buf.data = video_buf; - ffmpeg_push_video_thread(ff, &attr_buf); + attr_buf.vid.data = video_buf; + ffmpeg_push_video_thread(ff, &attr_buf.vid, attr_buf.pts_skip); } if (avail_audio && audio_buf) diff --git a/samples/record/ffmpeg_pipeline/ffmpeg_pipeline_test.c b/samples/record/ffmpeg_pipeline/ffmpeg_pipeline_test.c index 449c3040dc09..fa15478b3a08 100644 --- a/samples/record/ffmpeg_pipeline/ffmpeg_pipeline_test.c +++ b/samples/record/ffmpeg_pipeline/ffmpeg_pipeline_test.c @@ -10,12 +10,18 @@ * (frames are pushed as fast as they are taken), the blocking wait * when the queues are full, and the drain after the encoder thread * is joined. A frame out of order, torn, or from the wrong slot - * fails the pixel check. */ + * fails the pixel check. + * + * A second pass runs in drop mode (allow_frame_drop, the smallest + * video queue): frames the encoder can't take in time are dropped, + * so it checks the survivors come out in order, pixel-exact, at the + * timestamp of the frame they were pushed as. */ #include #include #include #include +#include #include "../../../record/record_driver.h" #include "../../../record/drivers/record_ffmpeg.h" @@ -83,7 +89,26 @@ static int check_frame(const AVFrame *f, unsigned n) return 1; } -int main(void) +/* Which frame this is, from its top-left pixel. */ +static int frame_number(const AVFrame *f) +{ + unsigned n; + for (n = 0; n < FRAMES; n++) + { + uint32_t want = ((n * 37u) & 0xffu) << 16 + | ((n * 91u) & 0xffu) << 8 | ((n * 13u) & 0xffu); + uint32_t px; + if (f->format == AV_PIX_FMT_GBRP) + px = (f->data[2][0] << 16) | (f->data[0][0] << 8) | f->data[1][0]; + else + px = (f->data[0][2] << 16) | (f->data[0][1] << 8) | f->data[0][0]; + if (px == want) + return (int)n; + } + return -1; +} + +static int run(bool drop) { struct record_params params; struct record_video_data vid; @@ -107,6 +132,8 @@ int main(void) params.aspect_ratio = (float)W / H; params.preset = RECORD_CONFIG_TYPE_RECORDING_LOSSLESS_QUALITY; params.pix_fmt = FFEMU_PIX_ARGB8888; + params.allow_frame_drop = drop; + params.video_fifo_frames = drop ? 8 : 32; remove(OUT); rec = record_ffmpeg.init(¶ms); @@ -149,6 +176,7 @@ int main(void) AVPacket *pkt = av_packet_alloc(); AVFrame *frm = av_frame_alloc(); int vstream = -1, i; + int last = -1; unsigned got = 0; if (avformat_open_input(&fmt, OUT, NULL, NULL) < 0 @@ -188,7 +216,26 @@ int main(void) } while (avcodec_receive_frame(dec, frm) == 0) { - if (!check_frame(frm, got)) + int n = drop ? frame_number(frm) : (int)got; + if (drop) + { + /* In order, and at the pushed frame's own time. */ + double t = frm->best_effort_timestamp + * av_q2d(fmt->streams[vstream]->time_base); + if (n <= last) + { + fprintf(stderr, "frame %d after %d\n", n, last); + return 1; + } + if (t < n / FPS - 0.002 || t > n / FPS + 0.002) + { + fprintf(stderr, "frame %d at %.4f s, want %.4f s\n", + n, t, n / FPS); + return 1; + } + last = n; + } + if (n < 0 || !check_frame(frm, (unsigned)n)) return 1; got++; av_frame_unref(frm); @@ -196,7 +243,7 @@ int main(void) if (flushing) break; } - if (got != FRAMES) + if (drop ? got == 0 : got != FRAMES) { fprintf(stderr, "decoded %u frames, pushed %u\n", got, FRAMES); return 1; @@ -205,9 +252,21 @@ int main(void) avformat_close_input(&fmt); av_packet_free(&pkt); av_frame_free(&frm); + if (drop) + printf("[pass] drop mode: %u of %u frames kept, in order, " + "pixel-exact, on time\n", got, FRAMES); + else + printf("[pass] %u frames through record_ffmpeg, all in order " + "and pixel-exact\n", FRAMES); } remove(OUT); - printf("[pass] %u frames through record_ffmpeg, all in order and pixel-exact\n", FRAMES); + return 0; +} + +int main(void) +{ + if (run(false) || run(true)) + return 1; printf("ALL OK\n"); return 0; } diff --git a/samples/record/ffmpeg_pipeline/stubs_retroarch.c b/samples/record/ffmpeg_pipeline/stubs_retroarch.c index 63b07e375ebb..c44472f2042b 100644 --- a/samples/record/ffmpeg_pipeline/stubs_retroarch.c +++ b/samples/record/ffmpeg_pipeline/stubs_retroarch.c @@ -1,6 +1,9 @@ /* What record_ffmpeg.c reaches for in RetroArch, stubbed. */ #include #include +#include +#include +#include void RARCH_LOG(const char *fmt, ...) { @@ -17,3 +20,18 @@ void RARCH_ERR(const char *fmt, ...) vfprintf(stderr, fmt, ap); va_end(ap); } + +void RARCH_WARN(const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); +} + +void runloop_msg_queue_push(const char *msg, size_t len, + unsigned prio, unsigned duration, bool flush, + char *title, enum message_queue_icon icon, + enum message_queue_category category) +{ +} diff --git a/settings/settings_def_recording_video.h b/settings/settings_def_recording_video.h index f362f3d1baa6..6c0c8be9e81a 100644 --- a/settings/settings_def_recording_video.h +++ b/settings/settings_def_recording_video.h @@ -18,7 +18,7 @@ S_BOOL(video_record_allow_frame_drop, VIDEO_RECORD_ALLOW_FRAME_DROP, "video_record_allow_frame_drop", DEFAULT_VIDEO_RECORD_ALLOW_FRAME_DROP, SD_FLAG_NONE, 0, 0, "Drop Frames When Encoder Falls Behind", - "When the recording encoder can't keep up, skip frames in the recording instead of slowing down the game. Disable to keep every frame at the cost of stutter.") + "When the recording or streaming encoder can't keep up, skip video frames and fill audio gaps with silence instead of slowing down the game. Disable to keep every frame at the cost of stutter.") S_UINT(video_record_fifo_frames, VIDEO_RECORD_FIFO_FRAMES, "video_record_fifo_frames", DEFAULT_VIDEO_RECORD_FIFO_FRAMES, SD_FLAG_LAKKA_ADVANCED, SDESC_RANGE_MINMAX, 0, MIN_VIDEO_RECORD_FIFO_FRAMES, MAX_VIDEO_RECORD_FIFO_FRAMES, 4, MIN_VIDEO_RECORD_FIFO_FRAMES, setting_action_ok_uint, NULL, From ae78fd9aa46141322e865ef43248fca96bd22251 Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:00 +0200 Subject: [PATCH 03/16] record_ffmpeg: make RTMP streaming presets ingest-compliant The Twitch/YouTube/Facebook/Kick modes reused the recording presets: CRF (unbounded VBR), the encoder's default GOP (~4 s for x264 at 60 fps), audio at whatever rate was closest to the core's and no audio bitrate. Ingest servers want CBR H.264, a 2 s keyframe interval and AAC at 44.1/48 kHz. For the FLV/RTMP modes: - CBR video at 2500/4500/6000 kbps (low/med/high), 1 s VBV buffer, x264 nal-hrd=cbr, no B-frames - a fixed 2 s keyframe interval: x264 scenecut=0, and keyframes are forced on time rather than counted in frames, since pts gaps from dropped frames or cores presenting every other tick stretch a frame-counted GOP - AAC 160 kbps at 48 kHz Rate control goes through the generic AVCodecContext fields so it carries over to other encoders. Local/custom streaming and the recording presets are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- record/drivers/record_ffmpeg.c | 89 ++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/record/drivers/record_ffmpeg.c b/record/drivers/record_ffmpeg.c index db2030cb7e49..38bf23c8f58a 100644 --- a/record/drivers/record_ffmpeg.c +++ b/record/drivers/record_ffmpeg.c @@ -205,6 +205,12 @@ struct ff_config_param bool video_qscale; int video_global_quality; int video_bit_rate; + /* Rate control / GOP for bitrate-constrained outputs (RTMP). + * 0 (or <0 for max_b_frames) leaves the encoder default. */ + int video_max_rate; + int video_buf_size; + float video_gop_seconds; + int video_max_b_frames; AVDictionary *video_opts; AVDictionary *audio_opts; @@ -264,6 +270,10 @@ typedef struct ffmpeg * silence, so the sample-count timeline never loses time. */ size_t audio_silence_pending; uint64_t audio_bytes_dropped; + + /* Encoder thread: pts of the last keyframe forced for + * video_gop_seconds. */ + int64_t video_last_key_pts; } ffmpeg_t; /* attr_fifo record: the frame plus the drops right before it. */ @@ -663,6 +673,21 @@ static bool ffmpeg_init_video(ffmpeg_t *handle) else if (params->video_bit_rate) video->codec->bit_rate = params->video_bit_rate; + if (params->video_max_rate) + video->codec->rc_max_rate = params->video_max_rate; + if (params->video_buf_size) + video->codec->rc_buffer_size = params->video_buf_size; + if (params->video_max_b_frames >= 0) + video->codec->max_b_frames = params->video_max_b_frames; + if (params->video_gop_seconds > 0.0f) + { + int gop = (int)(params->video_gop_seconds * param->fps + / params->frame_drop_ratio + 0.5f); + if (gop < 1) + gop = 1; + video->codec->gop_size = gop; + } + if (handle->muxer.ctx->oformat->flags & AVFMT_GLOBALHEADER) video->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; @@ -696,6 +721,48 @@ static bool ffmpeg_init_video(ffmpeg_t *handle) return true; } +/* RTMP ingest (Twitch, YouTube, Facebook, Kick) wants H.264 + AAC-LC in + * FLV, CBR video, a fixed 2 s keyframe interval and 44.1/48 kHz audio. + * Twitch caps non-partner video at 6000 kbps and audio at 160 kbps. */ +static void ffmpeg_config_rtmp(struct ff_config_param *params, + unsigned preset) +{ + int kbps; + + switch (preset) + { + case RECORD_CONFIG_TYPE_STREAMING_LOW_QUALITY: + kbps = 2500; + break; + case RECORD_CONFIG_TYPE_STREAMING_MED_QUALITY: + kbps = 4500; + break; + default: + kbps = 6000; + break; + } + + params->video_qscale = false; + params->video_bit_rate = kbps * 1000; + params->video_max_rate = kbps * 1000; + params->video_buf_size = kbps * 1000; + params->video_gop_seconds = 2.0f; + params->video_max_b_frames = 0; + + /* CBR instead of CRF. nal-hrd=cbr makes x264 pad to a true CBR + * stream, which is what the ingest servers expect. */ + av_dict_set(¶ms->video_opts, "crf", NULL, 0); + av_dict_set(¶ms->video_opts, "nal-hrd", "cbr", 0); + /* No extra keyframes on scene cuts: ingest wants a fixed cadence. + * (keyint_min can't do this; x264 clamps it to keyint/2+1.) */ + av_dict_set(¶ms->video_opts, "x264-params", "scenecut=0", 0); + + params->audio_qscale = false; + params->audio_bit_rate = 160000; + params->sample_rate = 48000; + av_dict_set(¶ms->audio_opts, "audio_global_quality", NULL, 0); +} + static bool ffmpeg_init_config_common(struct ff_config_param *params, unsigned preset, bool video_gpu_record, @@ -704,6 +771,8 @@ static bool ffmpeg_init_config_common(struct ff_config_param *params, unsigned streaming_mode, unsigned video_record_threads) { + params->video_max_b_frames = -1; + switch (preset) { case RECORD_CONFIG_TYPE_RECORDING_LOW_QUALITY: @@ -896,7 +965,10 @@ static bool ffmpeg_init_config_common(struct ff_config_param *params, || streaming_mode == STREAMING_MODE_TWITCH || streaming_mode == STREAMING_MODE_FACEBOOK || streaming_mode == STREAMING_MODE_KICK) + { strlcpy_lit(params->format, "flv", sizeof(params->format)); + ffmpeg_config_rtmp(params, preset); + } else strlcpy_lit(params->format, "mpegts", sizeof(params->format)); } @@ -920,6 +992,7 @@ static bool ffmpeg_init_config(struct ff_config_param *params, params->threads = 1; params->frame_drop_ratio = 1; params->audio_enable = true; + params->video_max_b_frames = -1; if (!config) return true; @@ -1622,6 +1695,22 @@ static bool ffmpeg_push_video_thread(ffmpeg_t *handle, handle->video.conv_frame->pts = handle->video.frame_cnt; + /* gop_size counts frames, so pts gaps (drops, cores presenting + * every other tick) stretch it in time; force keyframes on time. */ + handle->video.conv_frame->pict_type = AV_PICTURE_TYPE_NONE; + if (handle->config.video_gop_seconds > 0.0f) + { + int64_t ticks = (int64_t)(handle->config.video_gop_seconds + * handle->params.fps / handle->video.frame_drop_ratio + 0.5); + if (ticks < 1) + ticks = 1; + if (handle->video.frame_cnt - handle->video_last_key_pts >= ticks) + { + handle->video.conv_frame->pict_type = AV_PICTURE_TYPE_I; + handle->video_last_key_pts = handle->video.frame_cnt; + } + } + if (!encode_video(handle, handle->video.conv_frame)) return false; From 91dd6ce00edf7f2f311a28fe0af769903b693152 Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:00 +0200 Subject: [PATCH 04/16] record_ffmpeg: optional hardware H.264 encoder with libx264 fallback New setting video_record_hw_encoder (default on for HAVE_LAKKA_SWITCH, off elsewhere). When set and the built-in preset would use libx264, these encoders are tried in order and the first that opens is used: h264_nvv4l2, h264_nvmpi, h264_v4l2m2m, h264_rkmpp, h264_nvenc, h264_amf, h264_qsv, h264_videotoolbox, h264_mediacodec All take system-memory frames; VAAPI is skipped because it needs a hw_frames_ctx upload path. Input is YUV420P if the encoder takes it, else NV12 (via swscale). x264 private options are not passed to them; rate control/GOP come from the generic AVCodecContext fields, plus rc=cbr for the RTMP presets. CRF presets get a bitrate derived from 0.05/0.1/0.2 bits per pixel per frame (low/med/high), floor 500 kbps. If none open, libx264 is used as before. Custom configs are untouched. ffmpeg_init_video is split so one encoder attempt is a self-contained alloc/configure/open that frees its context on failure. Also sets AVCodecContext.framerate, which some hardware rate controllers need. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- config.def.h | 8 + configuration.h | 1 + menu/cbs/menu_cbs_sublabel.c | 1 + menu/menu_displaylist.c | 14 +- msg_hash_lbl_str.h | 1 + record/drivers/record_ffmpeg.c | 312 +++++++++++++++++------- record/record_driver.c | 1 + record/record_driver.h | 2 + settings/settings_def_recording_video.h | 5 + tools/settings_table.reference | 3 + 10 files changed, 258 insertions(+), 90 deletions(-) diff --git a/config.def.h b/config.def.h index 7b9397a02b81..ca29e0a2a4a9 100644 --- a/config.def.h +++ b/config.def.h @@ -1200,6 +1200,14 @@ * of stalling the frontend until it drains. */ #define DEFAULT_VIDEO_RECORD_ALLOW_FRAME_DROP true +/* Use a hardware H.264 encoder for recording/streaming when the FFmpeg + * build has one that works, falling back to libx264. */ +#ifdef HAVE_LAKKA_SWITCH +#define DEFAULT_VIDEO_RECORD_HW_ENCODER true +#else +#define DEFAULT_VIDEO_RECORD_HW_ENCODER false +#endif + /* Depth of the recording video queue, in frames. */ #define DEFAULT_VIDEO_RECORD_FIFO_FRAMES 32 #define MIN_VIDEO_RECORD_FIFO_FRAMES 8 diff --git a/configuration.h b/configuration.h index 64c8928b5168..a582c324e02c 100644 --- a/configuration.h +++ b/configuration.h @@ -671,6 +671,7 @@ typedef struct settings bool video_post_filter_record; bool video_gpu_record; bool video_record_allow_frame_drop; + bool video_record_hw_encoder; bool video_gpu_screenshot; bool video_allow_rotate; bool video_shared_context; diff --git a/menu/cbs/menu_cbs_sublabel.c b/menu/cbs/menu_cbs_sublabel.c index a3fc7c1a7605..8628ed2b673e 100644 --- a/menu/cbs/menu_cbs_sublabel.c +++ b/menu/cbs/menu_cbs_sublabel.c @@ -2436,6 +2436,7 @@ int menu_cbs_init_bind_sublabel(menu_file_list_cbs_t *cbs, { MENU_ENUM_LABEL_VIDEO_POST_FILTER_RECORD, MENU_ENUM_SUBLABEL_VIDEO_POST_FILTER_RECORD }, { MENU_ENUM_LABEL_VIDEO_RECORD_ALLOW_FRAME_DROP, MENU_ENUM_SUBLABEL_VIDEO_RECORD_ALLOW_FRAME_DROP }, { MENU_ENUM_LABEL_VIDEO_RECORD_FIFO_FRAMES, MENU_ENUM_SUBLABEL_VIDEO_RECORD_FIFO_FRAMES }, + { MENU_ENUM_LABEL_VIDEO_RECORD_HW_ENCODER, MENU_ENUM_SUBLABEL_VIDEO_RECORD_HW_ENCODER }, { MENU_ENUM_LABEL_NETPLAY_NICKNAME, MENU_ENUM_SUBLABEL_NETPLAY_NICKNAME }, { MENU_ENUM_LABEL_CHEEVOS_USERNAME, MENU_ENUM_SUBLABEL_CHEEVOS_USERNAME }, { MENU_ENUM_LABEL_CHEEVOS_PASSWORD, MENU_ENUM_SUBLABEL_CHEEVOS_PASSWORD }, diff --git a/menu/menu_displaylist.c b/menu/menu_displaylist.c index cd8e5c4870b2..1c9c63a9b045 100644 --- a/menu/menu_displaylist.c +++ b/menu/menu_displaylist.c @@ -11495,6 +11495,7 @@ unsigned menu_displaylist_build_list( {MENU_ENUM_LABEL_VIDEO_POST_FILTER_RECORD, PARSE_ONLY_BOOL, false}, {MENU_ENUM_LABEL_VIDEO_GPU_RECORD, PARSE_ONLY_BOOL, false}, {MENU_ENUM_LABEL_VIDEO_RECORD_ALLOW_FRAME_DROP, PARSE_ONLY_BOOL, false}, + {MENU_ENUM_LABEL_VIDEO_RECORD_HW_ENCODER, PARSE_ONLY_BOOL, false}, {MENU_ENUM_LABEL_VIDEO_RECORD_FIFO_FRAMES, PARSE_ONLY_UINT, false}, {MENU_ENUM_LABEL_STREAMING_MODE, PARSE_ONLY_UINT, false}, {MENU_ENUM_LABEL_VIDEO_STREAM_QUALITY, PARSE_ONLY_UINT, false}, @@ -11509,12 +11510,13 @@ unsigned menu_displaylist_build_list( build_list[4].checked = has_video; /* MENU_ENUM_LABEL_VIDEO_POST_FILTER_RECORD */ build_list[5].checked = has_video; /* MENU_ENUM_LABEL_VIDEO_GPU_RECORD */ build_list[6].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_RECORD_ALLOW_FRAME_DROP */ - build_list[7].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_RECORD_FIFO_FRAMES */ - build_list[8].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_MODE */ - build_list[9].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_STREAM_QUALITY */ - build_list[10].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAM_CONFIG */ - build_list[11].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_TITLE */ - build_list[12].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_URL */ + build_list[7].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_RECORD_HW_ENCODER */ + build_list[8].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_RECORD_FIFO_FRAMES */ + build_list[9].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_MODE */ + build_list[10].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_STREAM_QUALITY */ + build_list[11].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAM_CONFIG */ + build_list[12].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_TITLE */ + build_list[13].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_URL */ for (i = 0; i < ARRAY_SIZE(build_list); i++) { diff --git a/msg_hash_lbl_str.h b/msg_hash_lbl_str.h index 6ea3989676c6..310bce1c520a 100644 --- a/msg_hash_lbl_str.h +++ b/msg_hash_lbl_str.h @@ -1721,6 +1721,7 @@ #define MENU_ENUM_LABEL_VIDEO_PRESENT_TIMING_FROM_DISPLAY_STR "video_present_timing_from_display" #define MENU_ENUM_LABEL_VIDEO_RECORD_ALLOW_FRAME_DROP_STR "video_record_allow_frame_drop" #define MENU_ENUM_LABEL_VIDEO_RECORD_FIFO_FRAMES_STR "video_record_fifo_frames" +#define MENU_ENUM_LABEL_VIDEO_RECORD_HW_ENCODER_STR "video_record_hw_encoder" #define MENU_ENUM_LABEL_VIDEO_RECORD_QUALITY_STR "video_record_quality" #define MENU_ENUM_LABEL_VIDEO_RECORD_THREADS_STR "video_record_threads" #define MENU_ENUM_LABEL_VIDEO_REFRESH_RATE_STR "video_refresh_rate" diff --git a/record/drivers/record_ffmpeg.c b/record/drivers/record_ffmpeg.c index 38bf23c8f58a..e5de52fdec85 100644 --- a/record/drivers/record_ffmpeg.c +++ b/record/drivers/record_ffmpeg.c @@ -211,6 +211,9 @@ struct ff_config_param int video_buf_size; float video_gop_seconds; int video_max_b_frames; + /* Bits per pixel per frame used to derive a bitrate for hardware + * encoders when the preset is CRF-based (hardware has no CRF). */ + float video_hw_bpp; AVDictionary *video_opts; AVDictionary *audio_opts; @@ -550,66 +553,196 @@ static bool ffmpeg_init_audio(ffmpeg_t *handle, const char *audio_resampler) return true; } -static bool ffmpeg_init_video(ffmpeg_t *handle) +/* Returns the encoder's supported pixel formats, terminated by + * AV_PIX_FMT_NONE, or NULL if unrestricted / unknown. */ +static const enum AVPixelFormat *ffmpeg_codec_pix_fmts(const AVCodec *codec) +{ +#if HAVE_AVCODEC_GET_SUPPORTED_CONFIG + const void *fmts = NULL; + if (avcodec_get_supported_config(NULL, codec, + AV_CODEC_CONFIG_PIX_FORMAT, 0, &fmts, NULL) < 0) + return NULL; + return (const enum AVPixelFormat*)fmts; +#else + return codec->pix_fmts; +#endif +} + +/* H.264 encoders that take frames from system memory, tried in order. + * VAAPI is left out: it needs a hw_frames_ctx upload path. */ +static const char *const ffmpeg_hw_h264_encoders[] = { + "h264_nvv4l2", /* Tegra (L4T FFmpeg) */ + "h264_nvmpi", /* Tegra/Jetson (jetson-ffmpeg) */ + "h264_v4l2m2m", /* V4L2 mem2mem: RPi, Exynos, ... */ + "h264_rkmpp", /* Rockchip */ + "h264_nvenc", /* NVIDIA desktop */ + "h264_amf", /* AMD */ + "h264_qsv", /* Intel */ + "h264_videotoolbox", /* Apple */ + "h264_mediacodec", /* Android */ + NULL +}; + +/* Pick YUV420P, else NV12, from what the encoder accepts. */ +static enum AVPixelFormat ffmpeg_hw_pick_pix_fmt(const AVCodec *codec) +{ + unsigned i; + bool has_nv12 = false; + const enum AVPixelFormat *fmts = ffmpeg_codec_pix_fmts(codec); + + if (!fmts) + return AV_PIX_FMT_YUV420P; + + for (i = 0; fmts[i] != AV_PIX_FMT_NONE; i++) + { + if (fmts[i] == AV_PIX_FMT_YUV420P) + return AV_PIX_FMT_YUV420P; + if (fmts[i] == AV_PIX_FMT_NV12) + has_nv12 = true; + } + return has_nv12 ? AV_PIX_FMT_NV12 : AV_PIX_FMT_NONE; +} + +/* Allocates and opens handle->video.codec; frees it on failure. */ +static bool ffmpeg_open_video_codec(ffmpeg_t *handle, + const AVCodec *codec, enum AVPixelFormat pix_fmt, + unsigned out_w, unsigned out_h, bool hw, AVDictionary **opts) { - size_t size; - AVFrame *frame; struct ff_config_param *params = &handle->config; struct ff_video_info *video = &handle->video; struct record_params *param = &handle->params; - const AVCodec *codec = NULL; - unsigned out_w, out_h; + AVCodecContext *c; - if (*params->vcodec) - codec = avcodec_find_encoder_by_name(params->vcodec); - else + /* Ensure even dimensions for chroma-subsampled pixel formats. + * Odd dimensions cause encoder init failure with e.g. libx264. + * Round up (pad) rather than down so no source pixels are lost. */ + if ( pix_fmt == AV_PIX_FMT_YUV420P + || pix_fmt == AV_PIX_FMT_YUV422P + || pix_fmt == AV_PIX_FMT_NV12) { - /* By default, lossless video. */ - av_dict_set(¶ms->video_opts, "qp", "0", 0); - codec = avcodec_find_encoder_by_name("libx264rgb"); + out_w = (out_w + 1) & ~1; + out_h = (out_h + 1) & ~1; } - if (!codec) + if (!(c = avcodec_alloc_context3(codec))) + return false; + + c->codec_type = AVMEDIA_TYPE_VIDEO; + c->width = out_w; + c->height = out_h; + c->time_base = av_d2q((double) + params->frame_drop_ratio /param->fps, 1000000); /* Arbitrary big number. */ + c->framerate = av_inv_q(c->time_base); + c->sample_aspect_ratio = av_d2q( + param->aspect_ratio * out_h / out_w, 255); + c->pix_fmt = pix_fmt; + c->thread_count = params->threads; + + if (params->video_qscale) + { + c->flags |= AV_CODEC_FLAG_QSCALE; + c->global_quality = params->video_global_quality; + } + else if (params->video_bit_rate) + c->bit_rate = params->video_bit_rate; + else if (hw) + { + /* No CRF in hardware; derive a bitrate from the preset. */ + float bpp = params->video_hw_bpp > 0.0f + ? params->video_hw_bpp : 0.1f; + double rate = (double)bpp * out_w * out_h + * param->fps / params->frame_drop_ratio; + if (rate < 500000.0) + rate = 500000.0; + c->bit_rate = (int64_t)rate; + } + + if (params->video_max_rate) + c->rc_max_rate = params->video_max_rate; + if (params->video_buf_size) + c->rc_buffer_size = params->video_buf_size; + if (params->video_max_b_frames >= 0) + c->max_b_frames = params->video_max_b_frames; + if (params->video_gop_seconds > 0.0f) + { + int gop = (int)(params->video_gop_seconds * param->fps + / params->frame_drop_ratio + 0.5f); + if (gop < 1) + gop = 1; + c->gop_size = gop; + } + + if (handle->muxer.ctx->oformat->flags & AVFMT_GLOBALHEADER) + c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + + if (avcodec_open2(c, codec, opts) != 0) { - RARCH_ERR("[FFmpeg] Cannot find vcodec %s.\n", - *params->vcodec ? params->vcodec : "libx264rgb"); + avcodec_free_context(&c); return false; } - video->encoder = codec; + video->codec = c; + video->encoder = codec; + param->out_dims = VIDEO_SCALE_PACK(out_w, out_h); + return true; +} - /* Don't use swscaler unless format is not something "in-house" scaler - * supports. - * - * libswscale doesn't scale RGB -> RGB correctly (goes via YUV first), - * and it's non-trivial to fix upstream as it's heavily geared towards YUV. - * If we're dealing with strange formats or YUV, just use libswscale. - */ - if (params->out_pix_fmt != AV_PIX_FMT_NONE) +/* Try the hardware H.264 encoders this FFmpeg build has, in order. */ +static bool ffmpeg_open_hw_h264(ffmpeg_t *handle, + unsigned out_w, unsigned out_h, enum AVPixelFormat *out_fmt) +{ + unsigned i; + struct ff_config_param *params = &handle->config; + + for (i = 0; ffmpeg_hw_h264_encoders[i]; i++) { - video->pix_fmt = params->out_pix_fmt; - if (video->pix_fmt != AV_PIX_FMT_BGR24 && video->pix_fmt != AV_PIX_FMT_RGB32) - video->use_sws = true; + AVDictionary *opts = NULL; + enum AVPixelFormat fmt; + const AVCodec *codec = avcodec_find_encoder_by_name( + ffmpeg_hw_h264_encoders[i]); - switch (video->pix_fmt) + if (!codec) + continue; + if ((fmt = ffmpeg_hw_pick_pix_fmt(codec)) == AV_PIX_FMT_NONE) { - case AV_PIX_FMT_BGR24: - video->scaler.out_fmt = SCALER_FMT_BGR24; - break; + RARCH_LOG("[FFmpeg] %s: no YUV420P/NV12 input, skipping.\n", + codec->name); + continue; + } - case AV_PIX_FMT_RGB32: - video->scaler.out_fmt = SCALER_FMT_ARGB8888; - break; + /* x264's private options mean nothing to these. Ask for CBR where + * there's an "rc" knob; an encoder that rejects it fails to open + * and the next one is tried. */ + if (params->video_max_rate) + av_dict_set(&opts, "rc", "cbr", 0); - default: - break; + if (ffmpeg_open_video_codec(handle, codec, fmt, out_w, out_h, + true, &opts)) + { + av_dict_free(&opts); + *out_fmt = fmt; + RARCH_LOG("[FFmpeg] Using hardware encoder %s (%s).\n", + codec->name, av_get_pix_fmt_name(fmt)); + return true; } + av_dict_free(&opts); + RARCH_WARN("[FFmpeg] Hardware encoder %s failed to open.\n", + codec->name); } - else /* Use BGR24 as default out format. */ - { - video->pix_fmt = AV_PIX_FMT_BGR24; - video->scaler.out_fmt = SCALER_FMT_BGR24; - } + return false; +} + +static bool ffmpeg_init_video(ffmpeg_t *handle) +{ + size_t size; + AVFrame *frame; + struct ff_config_param *params = &handle->config; + struct ff_video_info *video = &handle->video; + struct record_params *param = &handle->params; + const AVCodec *codec = NULL; + enum AVPixelFormat pix_fmt = AV_PIX_FMT_BGR24; + bool opened = false; + unsigned out_w, out_h; switch (param->pix_fmt) { @@ -635,65 +768,73 @@ static bool ffmpeg_init_video(ffmpeg_t *handle) return false; } - video->codec = avcodec_alloc_context3(codec); - /* Useful to set scale_factor to 2 for chroma subsampled formats to * maintain full chroma resolution. (Or just use 4:4:4 or RGB ...) */ out_w = (float)VIDEO_SCALE_W(param->out_dims) * params->scale_factor; out_h = (float)VIDEO_SCALE_H(param->out_dims) * params->scale_factor; - /* Ensure even dimensions for chroma-subsampled pixel formats. - * Odd dimensions cause encoder init failure with e.g. libx264. - * Round up (pad) rather than down so no source pixels are lost. */ - if ( video->pix_fmt == AV_PIX_FMT_YUV420P - || video->pix_fmt == AV_PIX_FMT_YUV422P) + /* Hardware H.264 replaces libx264 for the built-in presets only; + * an explicit vcodec in a custom config is left alone. */ + if ( param->hw_encoder + && !params->conf + && string_is_equal(params->vcodec, "libx264")) { - out_w = (out_w + 1) & ~1; - out_h = (out_h + 1) & ~1; + opened = ffmpeg_open_hw_h264(handle, out_w, out_h, &pix_fmt); + if (!opened) + RARCH_WARN("[FFmpeg] No usable hardware H.264 encoder," + " falling back to libx264.\n"); } - param->out_dims = VIDEO_SCALE_PACK(out_w, out_h); - video->codec->codec_type = AVMEDIA_TYPE_VIDEO; - video->codec->width = out_w; - video->codec->height = out_h; - video->codec->time_base = av_d2q((double) - params->frame_drop_ratio /param->fps, 1000000); /* Arbitrary big number. */ - video->codec->sample_aspect_ratio = av_d2q( - param->aspect_ratio * out_h / out_w, 255); - video->codec->pix_fmt = video->pix_fmt; + if (!opened) + { + if (*params->vcodec) + codec = avcodec_find_encoder_by_name(params->vcodec); + else + { + /* By default, lossless video. */ + av_dict_set(¶ms->video_opts, "qp", "0", 0); + codec = avcodec_find_encoder_by_name("libx264rgb"); + } - video->codec->thread_count = params->threads; + if (!codec) + { + RARCH_ERR("[FFmpeg] Cannot find vcodec %s.\n", + *params->vcodec ? params->vcodec : "libx264rgb"); + return false; + } - if (params->video_qscale) - { - video->codec->flags |= AV_CODEC_FLAG_QSCALE; - video->codec->global_quality = params->video_global_quality; - } - else if (params->video_bit_rate) - video->codec->bit_rate = params->video_bit_rate; + if (params->out_pix_fmt != AV_PIX_FMT_NONE) + pix_fmt = params->out_pix_fmt; - if (params->video_max_rate) - video->codec->rc_max_rate = params->video_max_rate; - if (params->video_buf_size) - video->codec->rc_buffer_size = params->video_buf_size; - if (params->video_max_b_frames >= 0) - video->codec->max_b_frames = params->video_max_b_frames; - if (params->video_gop_seconds > 0.0f) - { - int gop = (int)(params->video_gop_seconds * param->fps - / params->frame_drop_ratio + 0.5f); - if (gop < 1) - gop = 1; - video->codec->gop_size = gop; + if (!ffmpeg_open_video_codec(handle, codec, pix_fmt, out_w, out_h, + false, params->video_opts ? ¶ms->video_opts : NULL)) + return false; } - if (handle->muxer.ctx->oformat->flags & AVFMT_GLOBALHEADER) - video->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + out_w = VIDEO_SCALE_W(param->out_dims); + out_h = VIDEO_SCALE_H(param->out_dims); - if (avcodec_open2(video->codec, codec, params->video_opts ? - ¶ms->video_opts : NULL) != 0) - return false; + /* Don't use swscaler unless format is not something "in-house" scaler + * supports. + * + * libswscale doesn't scale RGB -> RGB correctly (goes via YUV first), + * and it's non-trivial to fix upstream as it's heavily geared towards YUV. + * If we're dealing with strange formats or YUV, just use libswscale. + */ + video->pix_fmt = pix_fmt; + switch (pix_fmt) + { + case AV_PIX_FMT_BGR24: + video->scaler.out_fmt = SCALER_FMT_BGR24; + break; + case AV_PIX_FMT_RGB32: + video->scaler.out_fmt = SCALER_FMT_ARGB8888; + break; + default: + video->use_sws = true; + break; + } video->frame_drop_ratio = params->frame_drop_ratio; @@ -786,6 +927,7 @@ static bool ffmpeg_init_config_common(struct ff_config_param *params, strlcpy_lit(params->vcodec, "libx264", sizeof(params->vcodec)); strlcpy_lit(params->acodec, "aac", sizeof(params->acodec)); + params->video_hw_bpp = 0.05f; av_dict_set(¶ms->video_opts, "preset", "ultrafast", 0); av_dict_set(¶ms->video_opts, "tune", "film", 0); av_dict_set(¶ms->video_opts, "crf", "35", 0); @@ -802,6 +944,7 @@ static bool ffmpeg_init_config_common(struct ff_config_param *params, strlcpy_lit(params->vcodec, "libx264", sizeof(params->vcodec)); strlcpy_lit(params->acodec, "aac", sizeof(params->acodec)); + params->video_hw_bpp = 0.1f; av_dict_set(¶ms->video_opts, "preset", "superfast", 0); av_dict_set(¶ms->video_opts, "tune", "film", 0); av_dict_set(¶ms->video_opts, "crf", "25", 0); @@ -818,6 +961,7 @@ static bool ffmpeg_init_config_common(struct ff_config_param *params, strlcpy_lit(params->vcodec, "libx264", sizeof(params->vcodec)); strlcpy_lit(params->acodec, "aac", sizeof(params->acodec)); + params->video_hw_bpp = 0.2f; av_dict_set(¶ms->video_opts, "preset", "superfast", 0); av_dict_set(¶ms->video_opts, "tune", "film", 0); av_dict_set(¶ms->video_opts, "crf", "15", 0); diff --git a/record/record_driver.c b/record/record_driver.c index e923dbba026f..60ebb137f580 100644 --- a/record/record_driver.c +++ b/record/record_driver.c @@ -302,6 +302,7 @@ bool recording_init(void) params.streaming_mode = settings->uints.streaming_mode; params.allow_frame_drop = settings->bools.video_record_allow_frame_drop; params.video_fifo_frames = settings->uints.video_record_fifo_frames; + params.hw_encoder = settings->bools.video_record_hw_encoder; params.out_dims = VIDEO_SCALE_PACK( av_info->geometry.base_width, av_info->geometry.base_height); diff --git a/record/record_driver.h b/record/record_driver.h index 80d2fbd04d72..b9d39c2d64f1 100644 --- a/record/record_driver.h +++ b/record/record_driver.h @@ -86,6 +86,8 @@ struct record_params bool video_gpu_record; /* Drop frames when the encoder queue is full instead of blocking. */ bool allow_frame_drop; + /* Prefer a hardware H.264 encoder for the built-in presets. */ + bool hw_encoder; }; struct record_video_data diff --git a/settings/settings_def_recording_video.h b/settings/settings_def_recording_video.h index 6c0c8be9e81a..a38f97dab6d5 100644 --- a/settings/settings_def_recording_video.h +++ b/settings/settings_def_recording_video.h @@ -19,6 +19,11 @@ S_BOOL(video_record_allow_frame_drop, VIDEO_RECORD_ALLOW_FRAME_DROP, DEFAULT_VIDEO_RECORD_ALLOW_FRAME_DROP, SD_FLAG_NONE, 0, 0, "Drop Frames When Encoder Falls Behind", "When the recording or streaming encoder can't keep up, skip video frames and fill audio gaps with silence instead of slowing down the game. Disable to keep every frame at the cost of stutter.") +S_BOOL(video_record_hw_encoder, VIDEO_RECORD_HW_ENCODER, + "video_record_hw_encoder", + DEFAULT_VIDEO_RECORD_HW_ENCODER, SD_FLAG_NONE, 0, 0, + "Use Hardware Video Encoder", + "Encode H.264 with a hardware encoder when one is available, falling back to software (libx264). Applies to the built-in quality presets, not to custom configs.") S_UINT(video_record_fifo_frames, VIDEO_RECORD_FIFO_FRAMES, "video_record_fifo_frames", DEFAULT_VIDEO_RECORD_FIFO_FRAMES, SD_FLAG_LAKKA_ADVANCED, SDESC_RANGE_MINMAX, 0, MIN_VIDEO_RECORD_FIFO_FRAMES, MAX_VIDEO_RECORD_FIFO_FRAMES, 4, MIN_VIDEO_RECORD_FIFO_FRAMES, setting_action_ok_uint, NULL, diff --git a/tools/settings_table.reference b/tools/settings_table.reference index 2009827ca1f2..820a9577773d 100644 --- a/tools/settings_table.reference +++ b/tools/settings_table.reference @@ -310,6 +310,7 @@ bool menu_show_shutdown 1 1 bool video_post_filter_record 1 0 bool video_gpu_record 1 0 bool video_record_allow_frame_drop 1 1 +bool video_record_hw_encoder 1 0 bool menu_rgui_full_width_layout 1 1 bool video_shader_enable 1 1 bool quick_menu_show_save_core_overrides 1 1 @@ -1002,6 +1003,7 @@ bool menu_show_shutdown 1 1 bool video_post_filter_record 1 0 bool video_gpu_record 1 0 bool video_record_allow_frame_drop 1 1 +bool video_record_hw_encoder 1 0 bool menu_rgui_full_width_layout 1 1 bool video_shader_enable 1 1 bool quick_menu_show_save_core_overrides 1 1 @@ -1697,6 +1699,7 @@ bool menu_show_shutdown 1 1 bool video_post_filter_record 1 0 bool video_gpu_record 1 0 bool video_record_allow_frame_drop 1 1 +bool video_record_hw_encoder 1 0 bool menu_rgui_full_width_layout 1 1 bool video_shader_enable 1 1 bool quick_menu_show_save_core_overrides 1 1 From e7d5e3ef74a611e860cf6b33e309b3d09d71215b Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:00 +0200 Subject: [PATCH 05/16] record_ffmpeg: run encoder threads at lower priority in drop mode With a CPU-bound software core, libx264's worker threads, the swscale/AAC work on the encoder thread and the core's main thread all compete at the same priority, so recording slows the game even when the queues never fill. In drop-on-full mode, run everything encoder-side at nice +10: - the driver's encoder thread renices itself on start - threads spawned during the video/audio avcodec_open2() (x264's pool and lookahead) are found by diffing /proc/self/task and reniced afterwards; lowering the main thread around the open instead isn't reversible without CAP_SYS_NICE The encoder then gets what the core leaves, and any shortfall shows up as dropped frames rather than game slowdown. Linux only; skipped when drop-on-full is off, since starving the encoder there means stalls. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- record/drivers/record_ffmpeg.c | 123 +++++++++++++++++++++++++++++++-- 1 file changed, 116 insertions(+), 7 deletions(-) diff --git a/record/drivers/record_ffmpeg.c b/record/drivers/record_ffmpeg.c index e5de52fdec85..9cda3d92f97b 100644 --- a/record/drivers/record_ffmpeg.c +++ b/record/drivers/record_ffmpeg.c @@ -44,6 +44,15 @@ #include #endif +#if defined(__linux__) +#include +#include +#include +#include +#include +#define FFMPEG_HAVE_THREAD_NICE 1 +#endif + #include "record_ffmpeg.h" #ifdef __cplusplus @@ -1289,6 +1298,80 @@ static bool ffmpeg_init_muxer_post(ffmpeg_t *handle) static void ffmpeg_thread(void *data); +/* Encoder threads run this much nicer than the core, in drop mode + * only: with blocking on, a starved encoder just stalls the game. */ +#define ENCODER_NICE_DELTA 10 +#define MAX_TRACKED_TIDS 512 + +#ifdef FFMPEG_HAVE_THREAD_NICE +struct ff_tid_set +{ + pid_t tids[MAX_TRACKED_TIDS]; + unsigned count; +}; + +static void ffmpeg_tids_snapshot(struct ff_tid_set *set) +{ + struct dirent *ent; + DIR *dir = opendir("/proc/self/task"); + + set->count = 0; + if (!dir) + return; + while ((ent = readdir(dir)) && set->count < MAX_TRACKED_TIDS) + { + long tid = strtol(ent->d_name, NULL, 10); + if (tid > 0) + set->tids[set->count++] = (pid_t)tid; + } + closedir(dir); +} + +static void ffmpeg_renice_tid(pid_t tid, int delta) +{ + int cur; + + errno = 0; + cur = getpriority(PRIO_PROCESS, (id_t)tid); + if (cur == -1 && errno) + return; + /* Raising nice never needs privileges. */ + setpriority(PRIO_PROCESS, (id_t)tid, cur + delta); +} + +/* Renice threads an encoder spawned during avcodec_open2(). They + * inherit the main thread's priority, and lowering that around the + * open can't be undone without CAP_SYS_NICE. */ +static unsigned ffmpeg_renice_new_threads(const struct ff_tid_set *before) +{ + unsigned i, j, n = 0; + struct ff_tid_set *after = (struct ff_tid_set*)malloc(sizeof(*after)); + + if (!after) + return 0; + ffmpeg_tids_snapshot(after); + for (i = 0; i < after->count; i++) + { + bool seen = false; + for (j = 0; j < before->count; j++) + { + if (after->tids[i] == before->tids[j]) + { + seen = true; + break; + } + } + if (!seen) + { + ffmpeg_renice_tid(after->tids[i], ENCODER_NICE_DELTA); + n++; + } + } + free(after); + return n; +} +#endif + static bool init_thread(ffmpeg_t *handle) { unsigned vf = handle->video_fifo_frames; @@ -1522,13 +1605,33 @@ static void *ffmpeg_new(const struct record_params *params) if (!ffmpeg_init_muxer_pre(handle)) goto error; - if (!ffmpeg_init_video(handle)) - goto error; - - if ( handle->config.audio_enable - && !ffmpeg_init_audio(handle, - params->audio_resampler)) - goto error; + { + bool ok; +#ifdef FFMPEG_HAVE_THREAD_NICE + struct ff_tid_set *before = NULL; + if (handle->allow_frame_drop + && (before = (struct ff_tid_set*)malloc(sizeof(*before)))) + ffmpeg_tids_snapshot(before); +#endif + ok = ffmpeg_init_video(handle) + && ( !handle->config.audio_enable + || ffmpeg_init_audio(handle, params->audio_resampler)); +#ifdef FFMPEG_HAVE_THREAD_NICE + if (before) + { + if (ok) + { + unsigned n = ffmpeg_renice_new_threads(before); + if (n) + RARCH_LOG("[FFmpeg] Lowered priority of %u encoder" + " thread(s).\n", n); + } + free(before); + } +#endif + if (!ok) + goto error; + } if (!ffmpeg_init_muxer_post(handle)) goto error; @@ -2356,6 +2459,12 @@ static void ffmpeg_thread(void *data) (ff->audio.codec->frame_size * ff->params.channels * sizeof(int16_t)) : 0; void *audio_buf = audio_buf_size ? av_malloc(audio_buf_size) : NULL; +#ifdef FFMPEG_HAVE_THREAD_NICE + /* Also covers threads an encoder spawns lazily from here. */ + if (ff->allow_frame_drop) + ffmpeg_renice_tid((pid_t)syscall(SYS_gettid), ENCODER_NICE_DELTA); +#endif + while (retro_atomic_load_acquire_int(&ff->alive)) { struct ff_video_attr attr_buf; From 6ef4a55abd6699d76ac502f85bdd69be0c9be6fa Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:01 +0200 Subject: [PATCH 06/16] record_ffmpeg: route libav* logging into RetroArch's log FFmpeg logged to stderr through its default callback, so codec messages never reached RetroArch's log or log file, which is what a user can attach after a crash. While recording, install an av_log callback that assembles lines FFmpeg builds from several calls and forwards them as RARCH_ERR/WARN/LOG/DBG by level, tagged [FFmpeg/lav]. A partial line is flushed when the next piece comes from another context, so two threads' messages are never spliced. FFmpeg's level follows the frontend log level. The default callback is restored when the recording is freed. Adds verbosity_get_log_level(). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- record/drivers/record_ffmpeg.c | 105 ++++++++++++++++++ .../record/ffmpeg_pipeline/stubs_retroarch.c | 14 +++ verbosity.c | 5 + verbosity.h | 2 + 4 files changed, 126 insertions(+) diff --git a/record/drivers/record_ffmpeg.c b/record/drivers/record_ffmpeg.c index 9cda3d92f97b..3eb4f6c333a0 100644 --- a/record/drivers/record_ffmpeg.c +++ b/record/drivers/record_ffmpeg.c @@ -1559,11 +1559,114 @@ static void ffmpeg_free(void *data) free(handle); + av_log_set_callback(av_log_default_callback); + #if FFMPEG3 avformat_network_deinit(); #endif } +/* Route libav* logging into RetroArch's log, which is flushed per line + * and so survives a crash. FFmpeg's level follows the frontend log + * level: Debug gets AV_LOG_DEBUG (codec internals, e.g. the nvv4l2 + * encoder's steps), Info gets AV_LOG_INFO, and so on. */ +/* FFmpeg often builds one line out of several av_log() calls, so + * pieces are collected until a newline. Calls come from any thread. */ +static slock_t *ffmpeg_av_log_lock; +static char ffmpeg_av_log_line[1024]; +static size_t ffmpeg_av_log_len; +static int ffmpeg_av_log_level; +static int ffmpeg_av_log_prefix = 1; +static void *ffmpeg_av_log_ctx; + +static void ffmpeg_av_log_emit(int level, const char *line) +{ + if (level <= AV_LOG_ERROR) + RARCH_ERR("[FFmpeg/lav] %s\n", line); + else if (level <= AV_LOG_WARNING) + RARCH_WARN("[FFmpeg/lav] %s\n", line); + else if (level <= AV_LOG_INFO) + RARCH_LOG("[FFmpeg/lav] %s\n", line); + else + RARCH_DBG("[FFmpeg/lav] %s\n", line); +} + +static void ffmpeg_av_log_cb(void *avcl, int level, const char *fmt, + va_list vl) +{ + char piece[1024]; + size_t _len; + + if (level > av_log_get_level()) + return; + + if (ffmpeg_av_log_lock) + slock_lock(ffmpeg_av_log_lock); + + /* A partial line from another context (another thread or codec) + * is finished on its own rather than spliced with this one. */ + if (ffmpeg_av_log_len && avcl != ffmpeg_av_log_ctx) + { + ffmpeg_av_log_emit(ffmpeg_av_log_level, ffmpeg_av_log_line); + ffmpeg_av_log_len = 0; + ffmpeg_av_log_line[0] = '\0'; + ffmpeg_av_log_prefix = 1; + } + ffmpeg_av_log_ctx = avcl; + + av_log_format_line2(avcl, level, fmt, vl, piece, sizeof(piece), + &ffmpeg_av_log_prefix); + + if (!ffmpeg_av_log_len || level < ffmpeg_av_log_level) + ffmpeg_av_log_level = level; + ffmpeg_av_log_len += strlcpy(ffmpeg_av_log_line + ffmpeg_av_log_len, + piece, sizeof(ffmpeg_av_log_line) - ffmpeg_av_log_len); + if (ffmpeg_av_log_len >= sizeof(ffmpeg_av_log_line)) + ffmpeg_av_log_len = sizeof(ffmpeg_av_log_line) - 1; + + _len = ffmpeg_av_log_len; + if ( (_len && ffmpeg_av_log_line[_len - 1] == '\n') + || _len == sizeof(ffmpeg_av_log_line) - 1) + { + while (_len && (ffmpeg_av_log_line[_len - 1] == '\n' + || ffmpeg_av_log_line[_len - 1] == '\r')) + ffmpeg_av_log_line[--_len] = '\0'; + if (_len) + ffmpeg_av_log_emit(ffmpeg_av_log_level, ffmpeg_av_log_line); + ffmpeg_av_log_len = 0; + ffmpeg_av_log_line[0] = '\0'; + } + + if (ffmpeg_av_log_lock) + slock_unlock(ffmpeg_av_log_lock); +} + +static void ffmpeg_av_log_init(void) +{ + int av_level; + + if (!verbosity_is_enabled()) + av_level = AV_LOG_ERROR; + else + { + switch (verbosity_get_log_level()) + { + case 0: av_level = AV_LOG_DEBUG; break; + case 1: av_level = AV_LOG_INFO; break; + case 2: av_level = AV_LOG_WARNING; break; + default: av_level = AV_LOG_ERROR; break; + } + } + + /* Created once and kept: the callback may still be in use by + * another libav* user when a recording ends. */ + if (!ffmpeg_av_log_lock) + ffmpeg_av_log_lock = slock_new(); + + av_log_set_level(av_level); + av_log_set_callback(ffmpeg_av_log_cb); +} + static void *ffmpeg_new(const struct record_params *params) { ffmpeg_t *handle = (ffmpeg_t*)calloc(1, sizeof(*handle)); @@ -1575,6 +1678,8 @@ static void *ffmpeg_new(const struct record_params *params) avformat_network_init(); #endif + ffmpeg_av_log_init(); + handle->params = *params; handle->pkt = av_packet_alloc(); handle->allow_frame_drop = params->allow_frame_drop; diff --git a/samples/record/ffmpeg_pipeline/stubs_retroarch.c b/samples/record/ffmpeg_pipeline/stubs_retroarch.c index c44472f2042b..25d18fa1b858 100644 --- a/samples/record/ffmpeg_pipeline/stubs_retroarch.c +++ b/samples/record/ffmpeg_pipeline/stubs_retroarch.c @@ -35,3 +35,17 @@ void runloop_msg_queue_push(const char *msg, size_t len, enum message_queue_category category) { } + +void RARCH_DBG(const char *fmt, ...) +{ +} + +bool verbosity_is_enabled(void) +{ + return false; +} + +unsigned verbosity_get_log_level(void) +{ + return 1; +} diff --git a/verbosity.c b/verbosity.c index 78d2ebc33627..ef2bc33d53db 100644 --- a/verbosity.c +++ b/verbosity.c @@ -149,6 +149,11 @@ void verbosity_set_log_level(unsigned level) verbosity_log_level = level; } +unsigned verbosity_get_log_level(void) +{ + return verbosity_log_level; +} + void verbosity_enable(void) { main_verbosity_st.verbosity = true; diff --git a/verbosity.h b/verbosity.h index 6bacb5f93bfe..ea6dbdc70fbb 100644 --- a/verbosity.h +++ b/verbosity.h @@ -47,6 +47,8 @@ void verbosity_disable(void); void verbosity_set_log_level(unsigned level); +unsigned verbosity_get_log_level(void); + bool *verbosity_get_ptr(void); void retro_main_log_file_deinit(void); From abff25b87dc93dd1e4ab2ba1210bcc4537ddd629 Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:01 +0200 Subject: [PATCH 07/16] video: BGRX readback for GPU recording (gl, vulkan) GPU (post-shaded) recording, which hardware-rendered cores are forced into, read every frame with read_viewport(), which converts the 32-bit readback to BGR24 on the main thread inside the frame budget. The encoder thread then converts BGR24 to YUV again, and the frame is queued at the viewport rounded up to powers of two. - Add an optional video_driver_t::read_viewport_bgrx() that returns the viewport as 32-bit B,G,R,X plus its row order. It is the last slot, so positional driver tables leave it NULL. - gl: desktop GL already reads back GL_BGRA, so rows are copied as-is (RGBA is swizzled on GLES), on both the PBO and synchronous paths. - vulkan: B8G8R8A8 swapchains copy rows as-is, RGBA ones swizzle. - Recording uses it when the driver has it, feeding the encoder ARGB8888: the main thread's work per frame becomes a plain copy and the encoder thread converts once. Little-endian hosts only, since FFEMU_PIX_ARGB8888 is a native-endian word. The threaded wrapper keeps its own readback on the video thread. - The encoder queue is sized to the exact viewport (a resize already ends the recording): 1280x720 rather than 2048x1024 per frame. The BGR24 path is unchanged for screenshots and other drivers. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- gfx/drivers/gl2.c | 77 ++++++++++++++++------- gfx/drivers/vulkan.c | 76 +++++++++++++++++++--- gfx/video_driver.c | 24 +++++-- gfx/video_driver.h | 5 ++ libretro-common/include/gfx/video_frame.h | 36 +++++++++++ record/record_driver.c | 29 ++++++--- record/record_driver.h | 2 + 7 files changed, 206 insertions(+), 43 deletions(-) diff --git a/gfx/drivers/gl2.c b/gfx/drivers/gl2.c index 1cf77dc39054..9bd1378e81b0 100644 --- a/gfx/drivers/gl2.c +++ b/gfx/drivers/gl2.c @@ -2554,7 +2554,8 @@ static void gl2_renderchain_bind_prev_texture( } #ifdef HAVE_GL_ASYNC_READBACK -static bool gl2_read_pbo(gl2_t *gl, uint8_t *buffer) +/* bgrx: write 32-bit BGRX (read_viewport_bgrx) instead of BGR24. */ +static bool gl2_read_pbo(gl2_t *gl, uint8_t *buffer, bool bgrx) { const uint8_t *ptr = NULL; #ifdef HAVE_OPENGLES3 @@ -2583,17 +2584,31 @@ static bool gl2_read_pbo(gl2_t *gl, uint8_t *buffer) VIDEO_SCALE_W(gl->video_dims)); unsigned rb_h = MIN(VIDEO_SCALE_H(gl->vp.dims), VIDEO_SCALE_H(gl->video_dims)); - video_frame_convert_rgba_to_bgr( - (const void*)ptr, - buffer, - rb_w * sizeof(uint32_t), - rb_w * 3, - rb_w, - rb_h); + if (bgrx) + video_frame_copy_to_bgrx(buffer, VIDEO_SCALE_W(gl->vp.dims), + VIDEO_SCALE_H(gl->vp.dims), ptr, rb_w * 4, rb_w, rb_h, true); + else + video_frame_convert_rgba_to_bgr( + (const void*)ptr, + buffer, + rb_w * sizeof(uint32_t), + rb_w * 3, + rb_w, + rb_h); } #else ptr = (const uint8_t*)glMapBuffer(GL_PIXEL_PACK_BUFFER, GL_READ_ONLY); - if (ptr) + if (ptr && bgrx) + { + /* Already BGRA (GL_BGRA readback): rows are copied as-is. */ + unsigned rb_w = MIN(VIDEO_SCALE_W(gl->vp.dims), + VIDEO_SCALE_W(gl->video_dims)); + unsigned rb_h = MIN(VIDEO_SCALE_H(gl->vp.dims), + VIDEO_SCALE_H(gl->video_dims)); + video_frame_copy_to_bgrx(buffer, VIDEO_SCALE_W(gl->vp.dims), + VIDEO_SCALE_H(gl->vp.dims), ptr, rb_w * 4, rb_w, rb_h, false); + } + else if (ptr) { struct scaler_ctx *ctx = &gl->pbo_readback_scaler; scaler_ctx_scale_direct(ctx, buffer, ptr); @@ -2619,7 +2634,7 @@ static bool gl2_read_pbo(gl2_t *gl, uint8_t *buffer) static bool gl2_renderchain_read_viewport( gl2_t *gl, - uint8_t *buffer, bool is_idle) + uint8_t *buffer, bool is_idle, bool bgrx) { size_t num_pixels = 0; @@ -2653,7 +2668,7 @@ static bool gl2_renderchain_read_viewport( #ifdef HAVE_GL_ASYNC_READBACK if (gl->flags & GL2_FLAG_PBO_READBACK_ENABLE) { - if (!gl2_read_pbo(gl, buffer)) + if (!gl2_read_pbo(gl, buffer, bgrx)) goto error; } else @@ -2686,13 +2701,19 @@ static bool gl2_renderchain_read_viewport( VIDEO_SCALE_W(gl->video_dims)); unsigned rb_h = MIN(VIDEO_SCALE_H(gl->vp.dims), VIDEO_SCALE_H(gl->video_dims)); - video_frame_convert_rgba_to_bgr( - (const void*)gl->readback_buffer_screenshot, - buffer, - rb_w * sizeof(uint32_t), - rb_w * 3, - rb_w, - rb_h); + if (bgrx) + video_frame_copy_to_bgrx(buffer, VIDEO_SCALE_W(gl->vp.dims), + VIDEO_SCALE_H(gl->vp.dims), + (const uint8_t*)gl->readback_buffer_screenshot, + rb_w * 4, rb_w, rb_h, true); + else + video_frame_convert_rgba_to_bgr( + (const void*)gl->readback_buffer_screenshot, + buffer, + rb_w * sizeof(uint32_t), + rb_w * 3, + rb_w, + rb_h); } free(gl->readback_buffer_screenshot); @@ -6616,7 +6637,20 @@ static bool gl2_read_viewport(void *data, uint8_t *buffer, bool is_idle) if (!gl) return false; - return gl2_renderchain_read_viewport(gl, buffer, is_idle); + return gl2_renderchain_read_viewport(gl, buffer, is_idle, false); +} + +static bool gl2_read_viewport_bgrx(void *data, uint8_t *buffer, + bool is_idle, bool *bottom_up) +{ + gl2_t *gl = (gl2_t*)data; + + if (!gl) + return false; + + /* glReadPixels returns the bottom row first. */ + *bottom_up = true; + return gl2_renderchain_read_viewport(gl, buffer, is_idle, true); } #if defined(HAVE_GL_ASYNC_READBACK) && !defined(HAVE_OPENGLES) && !defined(HAVE_PSGL) @@ -6639,7 +6673,7 @@ static bool gl2_record_read(void *data, uint8_t *buffer) gl->flags &= ~GL2_FLAG_PBO_READBACK_ENABLE; return false; } - return gl2_read_pbo(gl, buffer); + return gl2_read_pbo(gl, buffer, false); } #endif @@ -7575,8 +7609,9 @@ video_driver_t video_gl2 = { #ifdef HAVE_OPENXR gl2_get_vr_frame_state, gl2_set_vr_content_info, - gl2_get_video_views_status + gl2_get_video_views_status, #endif + gl2_read_viewport_bgrx }; gfx_display_ctx_driver_t gfx_display_ctx_gl = { diff --git a/gfx/drivers/vulkan.c b/gfx/drivers/vulkan.c index 4e1c83c88782..f5f17cc5cf7a 100644 --- a/gfx/drivers/vulkan.c +++ b/gfx/drivers/vulkan.c @@ -13422,11 +13422,13 @@ video_record_read_t vulkan_get_record_read(void) return vulkan_record_read; } -static bool vulkan_read_viewport(void *data, uint8_t *buffer, bool is_idle) +/* bgrx: write top-down 32-bit BGRX (read_viewport_bgrx) instead of + * bottom-up BGR24. */ +static bool vulkan_read_viewport_internal(vk_t *vk, uint8_t *buffer, + bool is_idle, bool bgrx) { VkFormat format; struct vk_texture *staging = NULL; - vk_t *vk = (vk_t*)data; if (!vk) return false; @@ -13478,7 +13480,8 @@ static bool vulkan_read_viewport(void *data, uint8_t *buffer, bool is_idle) if (staging->memory == VK_NULL_HANDLE) return false; - buffer += 3 * (VIDEO_SCALE_H(vk->vp.dims) - 1) * VIDEO_SCALE_W(vk->vp.dims); + if (!bgrx) + buffer += 3 * (VIDEO_SCALE_H(vk->vp.dims) - 1) * VIDEO_SCALE_W(vk->vp.dims); vkMapMemory(vk->context->device, staging->memory, staging->offset, staging->size, 0, (void**)&src); @@ -13494,9 +13497,27 @@ static bool vulkan_read_viewport(void *data, uint8_t *buffer, bool is_idle) vkInvalidateMappedMemoryRanges(vk->context->device, 1, &range); } - ctx->in_stride = (int)staging->stride; - ctx->out_stride = -(int)VIDEO_SCALE_W(vk->vp.dims) * 3; - scaler_ctx_scale_direct(ctx, buffer, src); + if (bgrx) + { + /* Only the region vulkan_readback() copied is valid. */ + unsigned vp_w = VIDEO_SCALE_W(vk->vp.dims); + unsigned vp_h = VIDEO_SCALE_H(vk->vp.dims); + unsigned cp_w = VIDEO_SCALE_W(vk->readback.copied_dims); + unsigned cp_h = VIDEO_SCALE_H(vk->readback.copied_dims); + if (!cp_w || cp_w > vp_w) + cp_w = vp_w; + if (!cp_h || cp_h > vp_h) + cp_h = vp_h; + video_frame_copy_to_bgrx(buffer, vp_w, vp_h, + src, staging->stride, cp_w, cp_h, + format != VK_FORMAT_B8G8R8A8_UNORM); + } + else + { + ctx->in_stride = (int)staging->stride; + ctx->out_stride = -(int)VIDEO_SCALE_W(vk->vp.dims) * 3; + scaler_ctx_scale_direct(ctx, buffer, src); + } vkUnmapMemory(vk->context->device, staging->memory); } @@ -13579,10 +13600,24 @@ static bool vulkan_read_viewport(void *data, uint8_t *buffer, bool is_idle) cp_width = vp_width; if (cp_height == 0 || cp_height > vp_height) cp_height = vp_height; - if (cp_width < vp_width || cp_height < vp_height) - memset(buffer, 0, (size_t)vp_width * vp_height * 3); - buffer += 3 * (vp_height - 1) * vp_width; + if (bgrx) + { + /* vp_width here is clamped to video_width; keep the + * recording's vp.width row pitch. */ + video_frame_copy_to_bgrx(buffer, VIDEO_SCALE_W(vk->vp.dims), + VIDEO_SCALE_H(vk->vp.dims), + src, staging->stride, cp_width, cp_height, + format != VK_FORMAT_B8G8R8A8_UNORM); + format = VK_FORMAT_UNDEFINED; /* skip the BGR24 conversion */ + } + else + { + if (cp_width < vp_width || cp_height < vp_height) + memset(buffer, 0, (size_t)vp_width * vp_height * 3); + + buffer += 3 * (vp_height - 1) * vp_width; + } switch (format) { @@ -13615,6 +13650,9 @@ static bool vulkan_read_viewport(void *data, uint8_t *buffer, bool is_idle) } break; + case VK_FORMAT_UNDEFINED: + break; + default: RARCH_ERR("[Vulkan] Unexpected swapchain format.\n"); break; @@ -13626,6 +13664,25 @@ static bool vulkan_read_viewport(void *data, uint8_t *buffer, bool is_idle) return true; } +static bool vulkan_read_viewport(void *data, uint8_t *buffer, bool is_idle) +{ + vk_t *vk = (vk_t*)data; + if (!vk) + return false; + return vulkan_read_viewport_internal(vk, buffer, is_idle, false); +} + +static bool vulkan_read_viewport_bgrx(void *data, uint8_t *buffer, + bool is_idle, bool *bottom_up) +{ + vk_t *vk = (vk_t*)data; + if (!vk) + return false; + /* The staging copy is top row first. */ + *bottom_up = false; + return vulkan_read_viewport_internal(vk, buffer, is_idle, true); +} + #ifdef VULKAN_HDR_SWAPCHAIN /* IEEE-754 half (binary16) -> float, for reading an RGBA16F scRGB backbuffer * on the CPU. Handles normals, subnormals, sign and inf/nan. */ @@ -14411,6 +14468,7 @@ video_driver_t video_vulkan = { vulkan_set_vr_content_info, vulkan_get_video_views_status, #endif + vulkan_read_viewport_bgrx }; gfx_display_ctx_driver_t gfx_display_ctx_vulkan = { diff --git a/gfx/video_driver.c b/gfx/video_driver.c index d938e1dc9fdb..20a81dec9b88 100644 --- a/gfx/video_driver.c +++ b/gfx/video_driver.c @@ -1851,6 +1851,7 @@ static void recording_dump_frame( if (video_st->record_gpu_buffer) { const uint8_t *gpu_frame = video_st->record_gpu_buffer; + bool bottom_up = true; #ifdef HAVE_THREADS int taken = -2; if (video_st->thread_wrapper_active) @@ -1901,7 +1902,14 @@ static void recording_dump_frame( /* Big bottleneck. * Since we might need to do read-backs asynchronously, * it might take 3-4 times before this returns true. */ - if (!( vid->read_viewport + if (record_st->gpu_bgrx) + { + if (!( vid->read_viewport_bgrx + && vid->read_viewport_bgrx(video_st->data, + video_st->record_gpu_buffer, is_idle, &bottom_up))) + return; + } + else if (!( vid->read_viewport && vid->read_viewport( video_st->data, video_st->record_gpu_buffer, is_idle))) return; @@ -1909,11 +1917,15 @@ static void recording_dump_frame( } ffemu_data.dims = record_st->gpu_dims; - ffemu_data.pitch = (int)(VIDEO_SCALE_W(ffemu_data.dims) * 3); - ffemu_data.data = gpu_frame - + (VIDEO_SCALE_H(ffemu_data.dims) - 1) * ffemu_data.pitch; - - ffemu_data.pitch = -ffemu_data.pitch; + ffemu_data.pitch = (int)(VIDEO_SCALE_W(ffemu_data.dims) + * (record_st->gpu_bgrx ? 4 : 3)); + ffemu_data.data = gpu_frame; + if (bottom_up) + { + ffemu_data.data = gpu_frame + + (VIDEO_SCALE_H(ffemu_data.dims) - 1) * ffemu_data.pitch; + ffemu_data.pitch = -ffemu_data.pitch; + } } else ffemu_data.is_dupe = !data; diff --git a/gfx/video_driver.h b/gfx/video_driver.h index 81ce9c1d66dd..01f0885678c3 100644 --- a/gfx/video_driver.h +++ b/gfx/video_driver.h @@ -1284,6 +1284,11 @@ typedef struct video_driver unsigned (*get_video_views_status)(void *data); #endif + /* Optional: read_viewport as 32-bit B,G,R,X, viewport width * 4 + * per row, with *bottom_up set to the row order written. Lets GPU + * recording skip the BGR24 conversion. */ + bool (*read_viewport_bgrx)(void *data, uint8_t *buffer, bool is_idle, + bool *bottom_up); } video_driver_t; #ifdef HAVE_OPENXR diff --git a/libretro-common/include/gfx/video_frame.h b/libretro-common/include/gfx/video_frame.h index 98f46b704def..e08e0b9e3cb1 100644 --- a/libretro-common/include/gfx/video_frame.h +++ b/libretro-common/include/gfx/video_frame.h @@ -24,6 +24,8 @@ #define _LIBRETRO_SDK_VIDEO_FRAME_H #include +#include +#include #include #include @@ -207,6 +209,40 @@ static INLINE void video_frame_convert_rgba_to_bgr( } } +/* Copy w x h 32-bit pixels into a dst_w x dst_h buffer of B,G,R,X + * bytes with a row pitch of dst_w * 4, keeping the source row order. + * src_rgba: source bytes are R,G,B,A and are swapped; otherwise they + * are already B,G,R,A/X and rows are copied as-is. Anything outside + * the w x h source region is cleared to black. */ +static INLINE void video_frame_copy_to_bgrx( + uint8_t *dst, unsigned dst_w, unsigned dst_h, + const uint8_t *src, size_t src_pitch, + unsigned w, unsigned h, bool src_rgba) +{ + unsigned x, y; + + if (w > dst_w) + w = dst_w; + if (h > dst_h) + h = dst_h; + if (w < dst_w || h < dst_h) + memset(dst, 0, (size_t)dst_w * dst_h * 4); + + for (y = 0; y < h; y++, dst += (size_t)dst_w * 4, src += src_pitch) + { + if (!src_rgba) + memcpy(dst, src, (size_t)w * 4); + else + for (x = 0; x < w; x++) + { + dst[4 * x + 0] = src[4 * x + 2]; + dst[4 * x + 1] = src[4 * x + 1]; + dst[4 * x + 2] = src[4 * x + 0]; + dst[4 * x + 3] = 0xff; + } + } +} + static INLINE void video_pixel_frame_scale( struct scaler_ctx *scaler, void *output, const void *data, diff --git a/record/record_driver.c b/record/record_driver.c index 60ebb137f580..7975273b8c01 100644 --- a/record/record_driver.c +++ b/record/record_driver.c @@ -377,9 +377,9 @@ bool recording_init(void) } params.out_dims = vp.dims; - params.fb_dims = VIDEO_SCALE_PACK( - next_pow2(VIDEO_SCALE_W(vp.dims)), - next_pow2(VIDEO_SCALE_H(vp.dims))); + /* Every frame is exactly the viewport (a resize ends the + * recording), so size the encoder's queue to it. */ + params.fb_dims = vp.dims; if (video_force_aspect && (VIDEO_DRIVER_ASPECT_RATIO(video_st) > 0.0f)) @@ -387,18 +387,33 @@ bool recording_init(void) else params.aspect_ratio = (float)VIDEO_SCALE_W(vp.dims) / VIDEO_SCALE_H(vp.dims); - params.pix_fmt = FFEMU_PIX_BGR24; + /* The driver's 32-bit readback skips a per-frame BGR24 + * conversion; FFEMU_PIX_ARGB8888 is a native-endian word, so B,G,R,X + * bytes only on little-endian hosts. The threaded wrapper reads back + * on its own thread and hands over BGR24. */ +#ifdef MSB_FIRST + recording_st->gpu_bgrx = false; +#else + recording_st->gpu_bgrx = + !video_st->thread_wrapper_active + && video_st->current_video->read_viewport_bgrx; +#endif + params.pix_fmt = recording_st->gpu_bgrx + ? FFEMU_PIX_ARGB8888 : FFEMU_PIX_BGR24; recording_st->gpu_dims = vp.dims; - RARCH_LOG("[Recording] %s %ux%u.\n", msg_hash_to_str(MSG_DETECTED_VIEWPORT_OF), - VIDEO_SCALE_W(vp.dims), VIDEO_SCALE_H(vp.dims)); + RARCH_LOG("[Recording] %s %ux%u (%s readback).\n", + msg_hash_to_str(MSG_DETECTED_VIEWPORT_OF), + VIDEO_SCALE_W(vp.dims), VIDEO_SCALE_H(vp.dims), + recording_st->gpu_bgrx ? "BGRX" : "BGR24"); - gpu_size = VIDEO_SCALE_AREA(vp.dims) * 3; + gpu_size = VIDEO_SCALE_AREA(vp.dims) * (recording_st->gpu_bgrx ? 4 : 3); if (!(video_st->record_gpu_buffer = (uint8_t*)malloc(gpu_size))) return false; } else { + recording_st->gpu_bgrx = false; if (recording_state.out_dims) params.out_dims = recording_state.out_dims; diff --git a/record/record_driver.h b/record/record_driver.h index b9d39c2d64f1..1935f59d9dc3 100644 --- a/record/record_driver.h +++ b/record/record_driver.h @@ -127,6 +127,8 @@ struct recording * word - VIDEO_SCALE_PACK's layout, so a resize is one comparison * against a freshly packed viewport rather than two. */ unsigned gpu_dims; + /* GPU recording reads through read_viewport_bgrx. */ + bool gpu_bgrx; /* --size's override of the recording's output size, same layout; * zero when it was not given, which is one test instead of two. */ From 278826a4c371e0c1dc3c137514bac3ed17e0c878 Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:01 +0200 Subject: [PATCH 08/16] video: BGRX readback for glcore, gl1, sdl2 and sdl3; fix sdl2 readback Extend read_viewport_bgrx() to the other Linux-capable drivers. - glcore: the async PBO readback reads GL_BGRA on desktop GL (GLES keeps RGBA), with the BGR24 scaler's input format to match, so recording gets a plain row copy as on gl. - gl1: synchronous RGBA readback, swizzled to BGRX. - sdl3: sdl3_capture_viewport() can emit BGRX, top row first. - sdl2: read_viewport() converted the whole window surface (SDL_GetWindowSurface() on a window that has a renderer), copied surface->h * pitch into a buffer sized for the viewport, top row first where callers expect bottom row first, and leaked the converted surface on every call, so GPU recording and screenshots came out as garbage. Do what sdl3 does: redraw the last core frame into the game viewport and SDL_RenderReadPixels() exactly that, as BGRX or as flipped BGR24. The drivers that pass OpenXR slots positionally get NULLs for them so the new slot lands in place. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- gfx/drivers/gl1.c | 56 +++++++++++++----- gfx/drivers/gl3.c | 86 +++++++++++++++++++++------ gfx/drivers/sdl2_gfx.c | 128 +++++++++++++++++++++++++++++++---------- gfx/drivers/sdl3_gfx.c | 42 +++++++++++--- 4 files changed, 244 insertions(+), 68 deletions(-) diff --git a/gfx/drivers/gl1.c b/gfx/drivers/gl1.c index 4458db32fa59..3b8acc918a41 100644 --- a/gfx/drivers/gl1.c +++ b/gfx/drivers/gl1.c @@ -2867,13 +2867,12 @@ static void gl1_viewport_info(void *data, struct video_viewport *vp) VIDEO_POS_PUT_Y(vp->pos, top_dist); } -static bool gl1_read_viewport(void *data, uint8_t *buffer, bool is_idle) +/* bgrx: write bottom-up 32-bit BGRX (read_viewport_bgrx) instead of + * bottom-up BGR24. */ +static bool gl1_read_viewport_internal(gl1_t *gl1, uint8_t *buffer, + bool is_idle, bool bgrx) { unsigned num_pixels = 0; - gl1_t *gl1 = (gl1_t*)data; - - if (!gl1) - return false; num_pixels = VIDEO_SCALE_AREA(gl1->vp.dims); gl1->readback_buffer_screenshot = malloc(num_pixels * sizeof(uint32_t)); @@ -2895,13 +2894,19 @@ static bool gl1_read_viewport(void *data, uint8_t *buffer, bool is_idle) unsigned vd_h = VIDEO_SCALE_H(gl1->vp.full_dims); unsigned rb_w = (VIDEO_SCALE_W(gl1->vp.dims) > vd_w) ? vd_w : VIDEO_SCALE_W(gl1->vp.dims); unsigned rb_h = (VIDEO_SCALE_H(gl1->vp.dims) > vd_h) ? vd_h : VIDEO_SCALE_H(gl1->vp.dims); - video_frame_convert_rgba_to_bgr( - (const void*)gl1->readback_buffer_screenshot, - buffer, - rb_w * sizeof(uint32_t), - rb_w * 3, - rb_w, - rb_h); + if (bgrx) + video_frame_copy_to_bgrx(buffer, VIDEO_SCALE_W(gl1->vp.dims), + VIDEO_SCALE_H(gl1->vp.dims), + (const uint8_t*)gl1->readback_buffer_screenshot, + rb_w * 4, rb_w, rb_h, true); + else + video_frame_convert_rgba_to_bgr( + (const void*)gl1->readback_buffer_screenshot, + buffer, + rb_w * sizeof(uint32_t), + rb_w * 3, + rb_w, + rb_h); } free(gl1->readback_buffer_screenshot); @@ -2910,6 +2915,25 @@ static bool gl1_read_viewport(void *data, uint8_t *buffer, bool is_idle) return true; } +static bool gl1_read_viewport(void *data, uint8_t *buffer, bool is_idle) +{ + gl1_t *gl1 = (gl1_t*)data; + if (!gl1) + return false; + return gl1_read_viewport_internal(gl1, buffer, is_idle, false); +} + +static bool gl1_read_viewport_bgrx(void *data, uint8_t *buffer, + bool is_idle, bool *bottom_up) +{ + gl1_t *gl1 = (gl1_t*)data; + if (!gl1) + return false; + /* glReadPixels returns the bottom row first. */ + *bottom_up = true; + return gl1_read_viewport_internal(gl1, buffer, is_idle, true); +} + static void gl1_set_texture_frame(void *data, const void *frame, bool rgb32, unsigned dims, float alpha) @@ -3664,7 +3688,13 @@ video_driver_t video_gl1 = { #endif NULL, /* invalidate_hw_render_cache */ gl1_read_viewport_hdr, - &gl1_raster_font + &gl1_raster_font, +#ifdef HAVE_OPENXR + NULL, /* get_vr_frame_state */ + NULL, /* set_vr_content_info */ + NULL, /* get_video_views_status */ +#endif + gl1_read_viewport_bgrx }; gfx_display_ctx_driver_t gfx_display_ctx_gl1 = { diff --git a/gfx/drivers/gl3.c b/gfx/drivers/gl3.c index 33f757a80a67..bf2fbe419907 100644 --- a/gfx/drivers/gl3.c +++ b/gfx/drivers/gl3.c @@ -1931,7 +1931,11 @@ static bool gl3_init_pbo_readback(gl3_t *gl) scaler->out_height = VIDEO_SCALE_H(gl->vp.dims); scaler->in_stride = VIDEO_SCALE_W(gl->vp.dims) * sizeof(uint32_t); scaler->out_stride = VIDEO_SCALE_W(gl->vp.dims) * 3; - scaler->in_fmt = SCALER_FMT_ABGR8888; +#ifdef HAVE_OPENGLES + scaler->in_fmt = SCALER_FMT_ABGR8888; /* RGBA readback */ +#else + scaler->in_fmt = SCALER_FMT_ARGB8888; /* BGRA readback */ +#endif scaler->out_fmt = SCALER_FMT_BGR24; scaler->scaler_type = SCALER_TYPE_POINT; @@ -1975,9 +1979,17 @@ static void gl3_pbo_async_readback(gl3_t *gl) gl->pbo_readback_index = 0; gl->pbo_readback_valid[gl->pbo_readback_index] = true; + /* Desktop GL reads BGRA, which both the recording's BGRX path and + * the BGR24 scaler want; GLES only guarantees RGBA. Keep in sync + * with the scaler in gl3_init_pbo_readback(). */ glReadPixels(VIDEO_POS_X(gl->vp.pos), VIDEO_POS_Y(gl->vp.pos), VIDEO_SCALE_W(gl->vp.dims), VIDEO_SCALE_H(gl->vp.dims), - GL_RGBA, GL_UNSIGNED_BYTE, NULL); +#ifdef HAVE_OPENGLES + GL_RGBA, GL_UNSIGNED_BYTE, +#else + GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, +#endif + NULL); if (gl->scrgb.active && gl->scrgb.fbo) glBindFramebuffer(GL_READ_FRAMEBUFFER, 0); glBindBuffer(GL_PIXEL_PACK_BUFFER, 0); @@ -4589,14 +4601,13 @@ static bool gl3_read_viewport_hdr(void *data, uint16_t *buffer, return true; } -static bool gl3_read_viewport(void *data, uint8_t *buffer, bool is_idle) +/* bgrx: write bottom-up 32-bit BGRX (read_viewport_bgrx) instead of + * bottom-up BGR24. */ +static bool gl3_read_viewport_internal(gl3_t *gl, uint8_t *buffer, + bool is_idle, bool bgrx) { - gl3_t *gl = (gl3_t*)data; size_t num_pixels = 0; - if (!gl) - return false; - if (gl->flags & GL3_FLAG_USE_SHARED_CONTEXT) gl->ctx_driver->bind_hw_render(gl->ctx_data, false); @@ -4633,7 +4644,19 @@ static bool gl3_read_viewport(void *data, uint8_t *buffer, bool is_idle) glBindBuffer(GL_PIXEL_PACK_BUFFER, gl->pbo_readback[gl->pbo_readback_index]); ptr = glMapBufferRange(GL_PIXEL_PACK_BUFFER, 0, num_pixels * sizeof(uint32_t), GL_MAP_READ_BIT); - scaler_ctx_scale_direct(ctx, buffer, ptr); + if (ptr && bgrx) + video_frame_copy_to_bgrx(buffer, VIDEO_SCALE_W(gl->vp.dims), + VIDEO_SCALE_H(gl->vp.dims), + (const uint8_t*)ptr, VIDEO_SCALE_W(gl->vp.dims) * 4, + VIDEO_SCALE_W(gl->vp.dims), VIDEO_SCALE_H(gl->vp.dims), +#ifdef HAVE_OPENGLES + true +#else + false +#endif + ); + else if (ptr) + scaler_ctx_scale_direct(ctx, buffer, ptr); glUnmapBuffer(GL_PIXEL_PACK_BUFFER); glBindBuffer(GL_PIXEL_PACK_BUFFER, 0); } @@ -4665,13 +4688,19 @@ static bool gl3_read_viewport(void *data, uint8_t *buffer, bool is_idle) VIDEO_SCALE_W(gl->video_dims)); unsigned rb_h = MIN(VIDEO_SCALE_H(gl->vp.dims), VIDEO_SCALE_H(gl->video_dims)); - video_frame_convert_rgba_to_bgr( - (const void*)gl->readback_buffer_screenshot, - buffer, - rb_w * sizeof(uint32_t), - rb_w * 3, - rb_w, - rb_h); + if (bgrx) + video_frame_copy_to_bgrx(buffer, VIDEO_SCALE_W(gl->vp.dims), + VIDEO_SCALE_H(gl->vp.dims), + (const uint8_t*)gl->readback_buffer_screenshot, + rb_w * 4, rb_w, rb_h, true); + else + video_frame_convert_rgba_to_bgr( + (const void*)gl->readback_buffer_screenshot, + buffer, + rb_w * sizeof(uint32_t), + rb_w * 3, + rb_w, + rb_h); } free(gl->readback_buffer_screenshot); @@ -4693,6 +4722,25 @@ static bool gl3_read_viewport(void *data, uint8_t *buffer, bool is_idle) return false; } +static bool gl3_read_viewport(void *data, uint8_t *buffer, bool is_idle) +{ + gl3_t *gl = (gl3_t*)data; + if (!gl) + return false; + return gl3_read_viewport_internal(gl, buffer, is_idle, false); +} + +static bool gl3_read_viewport_bgrx(void *data, uint8_t *buffer, + bool is_idle, bool *bottom_up) +{ + gl3_t *gl = (gl3_t*)data; + if (!gl) + return false; + /* glReadPixels returns the bottom row first. */ + *bottom_up = true; + return gl3_read_viewport_internal(gl, buffer, is_idle, true); +} + static void gl3_update_cpu_texture(gl3_t *gl, struct gl3_streamed_texture *streamed, const void *frame, unsigned dims, unsigned pitch) @@ -7552,7 +7600,13 @@ video_driver_t video_gl3 = { #endif NULL, /* invalidate_hw_render_cache */ gl3_read_viewport_hdr, - &gl3_raster_font + &gl3_raster_font, +#ifdef HAVE_OPENXR + NULL, /* get_vr_frame_state */ + NULL, /* set_vr_content_info */ + NULL, /* get_video_views_status */ +#endif + gl3_read_viewport_bgrx }; gfx_display_ctx_driver_t gfx_display_ctx_gl3 = { diff --git a/gfx/drivers/sdl2_gfx.c b/gfx/drivers/sdl2_gfx.c index 822c1ac7b0f8..fc125d4f83fb 100644 --- a/gfx/drivers/sdl2_gfx.c +++ b/gfx/drivers/sdl2_gfx.c @@ -587,26 +587,12 @@ static void check_window(sdl2_video_t *vid) } } -static bool sdl2_gfx_frame(void *data, const void *frame, - unsigned dims, uint64_t frame_count, - unsigned pitch, const char *msg, video_frame_info_t *video_info) +/* Draw the current core frame into the game viewport, honouring + * rotation. Also used by the viewport readback, which redraws rather + * than read the presented backbuffer (undefined after + * SDL_RenderPresent on several backends). */ +static void sdl2_blit_frame(sdl2_video_t *vid) { - char title[128]; - sdl2_video_t *vid = (sdl2_video_t*)data; -#ifdef HAVE_MENU - bool menu_is_alive = (video_info->menu_st_flags & MENU_ST_FLAG_ALIVE) ? true : false; -#endif - - if (vid->flags & SDL2_FLAG_SHOULD_RESIZE) - sdl_refresh_viewport(vid); - - if (frame) - { - SDL_RenderClear(vid->renderer); - sdl_refresh_input_size(vid, false, vid->video.rgb32, dims, pitch); - SDL_UpdateTexture(vid->frame.tex, NULL, frame, pitch); - } - { /* For 90/270 degree rotation the frame must not be clipped by the * aspect-corrected game viewport. SDL_RenderCopyEx() scales the @@ -644,6 +630,29 @@ static bool sdl2_gfx_frame(void *data, const void *frame, SDL_RenderCopyEx(vid->renderer, vid->frame.tex, NULL, NULL, vid->rotation, NULL, SDL_FLIP_NONE); } +} + +static bool sdl2_gfx_frame(void *data, const void *frame, + unsigned dims, uint64_t frame_count, + unsigned pitch, const char *msg, video_frame_info_t *video_info) +{ + char title[128]; + sdl2_video_t *vid = (sdl2_video_t*)data; +#ifdef HAVE_MENU + bool menu_is_alive = (video_info->menu_st_flags & MENU_ST_FLAG_ALIVE) ? true : false; +#endif + + if (vid->flags & SDL2_FLAG_SHOULD_RESIZE) + sdl_refresh_viewport(vid); + + if (frame) + { + SDL_RenderClear(vid->renderer); + sdl_refresh_input_size(vid, false, vid->video.rgb32, dims, pitch); + SDL_UpdateTexture(vid->frame.tex, NULL, frame, pitch); + } + + sdl2_blit_frame(vid); #ifdef HAVE_MENU { @@ -892,28 +901,79 @@ static void sdl2_gfx_viewport_info(void *data, struct video_viewport *vp) *vp = vid->vp; } -static bool sdl2_gfx_read_viewport(void *data, uint8_t *buffer, bool is_idle) +/* Redraw the last core frame into the game viewport and read it back. + * bgrx: 32-bit B,G,R,X, top row first; otherwise BGR24 bottom row + * first, as read_viewport() callers expect. Menu, widget and OSD + * passes are not replayed. Nothing is presented; the next frame + * redraws. */ +static bool sdl2_capture_viewport(sdl2_video_t *vid, uint8_t *buffer, + bool bgrx) { - SDL_Surface *surf = NULL, *bgr24 = NULL; - sdl2_video_t *vid = (sdl2_video_t*)data; + SDL_Rect saved_vp, game_vp; + int pitch; + bool ok; +#if SDL_VERSION_ATLEAST(2, 0, 5) + Uint32 fmt = bgrx ? SDL_PIXELFORMAT_BGRA32 : SDL_PIXELFORMAT_BGR24; +#else + Uint32 fmt = bgrx ? SDL_PIXELFORMAT_ARGB8888 : SDL_PIXELFORMAT_BGR24; +#endif - if (!is_idle) - video_driver_cached_frame(); + if (!vid || !vid->renderer || !vid->frame.tex + || !VIDEO_SCALE_W(vid->vp.dims) || !VIDEO_SCALE_H(vid->vp.dims)) + return false; - surf = SDL_GetWindowSurface(vid->window); - bgr24 = SDL_ConvertSurfaceFormat(surf, SDL_PIXELFORMAT_BGR24, 0); + pitch = (int)VIDEO_SCALE_W(vid->vp.dims) * (bgrx ? 4 : 3); + game_vp.x = VIDEO_POS_X(vid->vp.pos); + game_vp.y = VIDEO_POS_Y(vid->vp.pos); + game_vp.w = (int)VIDEO_SCALE_W(vid->vp.dims); + game_vp.h = (int)VIDEO_SCALE_H(vid->vp.dims); - if (!bgr24) + SDL_RenderGetViewport(vid->renderer, &saved_vp); + SDL_RenderSetViewport(vid->renderer, &game_vp); + SDL_RenderClear(vid->renderer); + sdl2_blit_frame(vid); + /* NULL rect: the current viewport, the size of the caller's buffer. */ + ok = SDL_RenderReadPixels(vid->renderer, NULL, fmt, buffer, pitch) == 0; + SDL_RenderSetViewport(vid->renderer, &saved_vp); + + if (!ok) { - RARCH_WARN("[SDL2] Failed to convert viewport data to BGR24: %s.", SDL_GetError()); + RARCH_WARN("[SDL2] Failed to read viewport: %s.\n", SDL_GetError()); return false; } - memcpy(buffer, bgr24->pixels, bgr24->h * bgr24->pitch); + if (!bgrx) + { + unsigned y, h = VIDEO_SCALE_H(vid->vp.dims); + uint8_t *tmp = (uint8_t*)malloc((size_t)pitch); + if (!tmp) + return false; + for (y = 0; y < h / 2; y++) + { + uint8_t *a = buffer + (size_t)y * pitch; + uint8_t *b = buffer + (size_t)(h - 1 - y) * pitch; + memcpy(tmp, a, pitch); + memcpy(a, b, pitch); + memcpy(b, tmp, pitch); + } + free(tmp); + } return true; } +static bool sdl2_gfx_read_viewport(void *data, uint8_t *buffer, bool is_idle) +{ + return sdl2_capture_viewport((sdl2_video_t*)data, buffer, false); +} + +static bool sdl2_gfx_read_viewport_bgrx(void *data, uint8_t *buffer, + bool is_idle, bool *bottom_up) +{ + *bottom_up = false; + return sdl2_capture_viewport((sdl2_video_t*)data, buffer, true); +} + static void sdl2_poke_set_filtering(void *data, unsigned index, bool smooth, bool ctx_scaling) { sdl2_video_t *vid = (sdl2_video_t*)data; @@ -2395,10 +2455,16 @@ video_driver_t video_sdl2 = { NULL, /* invalidate_hw_render_cache */ NULL, /* read_viewport_hdr */ #if SDL_VERSION_ATLEAST(2, 0, 18) - &sdl2_raster_font + &sdl2_raster_font, #else - NULL /* sdl2_raster_font needs SDL_RenderGeometry */ + NULL, /* sdl2_raster_font needs SDL_RenderGeometry */ +#endif +#ifdef HAVE_OPENXR + NULL, /* get_vr_frame_state */ + NULL, /* set_vr_content_info */ + NULL, /* get_video_views_status */ #endif + sdl2_gfx_read_viewport_bgrx }; #if SDL_VERSION_ATLEAST(2, 0, 18) diff --git a/gfx/drivers/sdl3_gfx.c b/gfx/drivers/sdl3_gfx.c index 99ee6c0fba3f..1c548daa5b05 100644 --- a/gfx/drivers/sdl3_gfx.c +++ b/gfx/drivers/sdl3_gfx.c @@ -361,11 +361,17 @@ static void sdl3_blit_frame(sdl3_video_t *vid) * flipped because read_viewport callers expect bottom-up data. * Must run before SDL_RenderPresent - afterwards the backbuffer * contents are undefined on several backends. */ -static bool sdl3_capture_viewport(sdl3_video_t *vid, uint8_t *buffer) +/* bgrx: write 32-bit B,G,R,X top row first (read_viewport_bgrx); + * otherwise BGR24 bottom row first, as read_viewport() callers expect. */ +static bool sdl3_capture_viewport(sdl3_video_t *vid, uint8_t *buffer, + bool bgrx) { SDL_Surface *surf; int w, h, y; bool ok = true; + size_t bpp = bgrx ? 4 : 3; + SDL_PixelFormat fmt = bgrx ? SDL_PIXELFORMAT_BGRA32 + : SDL_PIXELFORMAT_BGR24; /* NULL rect = the current render viewport, which is the * aspect-corrected game viewport at this point in the frame. */ @@ -382,18 +388,19 @@ static bool sdl3_capture_viewport(sdl3_video_t *vid, uint8_t *buffer) h = (surf->h < (int)VIDEO_SCALE_H(vid->vp.dims)) ? surf->h : (int)VIDEO_SCALE_H(vid->vp.dims); if (w < (int)VIDEO_SCALE_W(vid->vp.dims) || h < (int)VIDEO_SCALE_H(vid->vp.dims)) - memset(buffer, 0, VIDEO_SCALE_AREA(vid->vp.dims) * 3); + memset(buffer, 0, VIDEO_SCALE_AREA(vid->vp.dims) * bpp); for (y = 0; y < h; y++) { + int dst_row = bgrx ? y : (h - 1 - y); if (!SDL_ConvertPixels(w, 1, surf->format, (const uint8_t*)surf->pixels + (size_t)y * surf->pitch, surf->pitch, - SDL_PIXELFORMAT_BGR24, - buffer + (size_t)(h - 1 - y) * VIDEO_SCALE_W(vid->vp.dims) * 3, - (int)VIDEO_SCALE_W(vid->vp.dims) * 3)) + fmt, + buffer + (size_t)dst_row * VIDEO_SCALE_W(vid->vp.dims) * bpp, + (int)(VIDEO_SCALE_W(vid->vp.dims) * bpp))) { - RARCH_WARN("[SDL3] Failed to convert viewport data to BGR24: %s.\n", + RARCH_WARN("[SDL3] Failed to convert viewport data: %s.\n", SDL_GetError()); ok = false; break; @@ -624,7 +631,20 @@ static bool sdl3_gfx_read_viewport(void *data, uint8_t *buffer, bool is_idle) SDL_RenderClear(vid->renderer); sdl3_blit_frame(vid); - return sdl3_capture_viewport(vid, buffer); + return sdl3_capture_viewport(vid, buffer, false); +} + +static bool sdl3_gfx_read_viewport_bgrx(void *data, uint8_t *buffer, + bool is_idle, bool *bottom_up) +{ + sdl3_video_t *vid = (sdl3_video_t*)data; + + /* Same redraw-and-capture as sdl3_gfx_read_viewport(). */ + SDL_RenderClear(vid->renderer); + sdl3_blit_frame(vid); + + *bottom_up = false; + return sdl3_capture_viewport(vid, buffer, true); } /* Applies a new window size / fullscreen in place, without tearing @@ -2045,7 +2065,13 @@ video_driver_t video_sdl3 = { #endif NULL, /* invalidate_hw_render_cache */ NULL, /* read_viewport_hdr */ - &sdl3_raster_font + &sdl3_raster_font, +#ifdef HAVE_OPENXR + NULL, /* get_vr_frame_state */ + NULL, /* set_vr_content_info */ + NULL, /* get_video_views_status */ +#endif + sdl3_gfx_read_viewport_bgrx }; gfx_display_ctx_driver_t gfx_display_ctx_sdl3 = { From 9220c6fe3c857101d9ea878d937fac95176506ba Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:01 +0200 Subject: [PATCH 09/16] recording: performance counters for the recording path Register frontend performance counters for each stage of a recording, reported at exit with perfcnt_enable, so its cost can be placed: record_readback main thread: GPU readback record_push_video main thread: queueing a frame for the encoder record_push_audio audio path: queueing audio record_scale encoder thread: colour conversion and scaling record_encode_video encoder thread: video encode and mux record_encode_audio encoder thread: audio resample, encode and mux Counters are only touched with perfcnt_enable set, and each has a single writer. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- gfx/video_driver.c | 26 +++++--- record/drivers/record_ffmpeg.c | 66 ++++++++++++++++++- .../record/ffmpeg_pipeline/stubs_retroarch.c | 13 ++++ 3 files changed, 93 insertions(+), 12 deletions(-) diff --git a/gfx/video_driver.c b/gfx/video_driver.c index 20a81dec9b88..95b4189105b4 100644 --- a/gfx/video_driver.c +++ b/gfx/video_driver.c @@ -164,6 +164,7 @@ static void video_driver_read_vp_params(struct video_vp_param_snap *ps); #include "../driver.h" #include "../file_path_special.h" #include "../list_special.h" +#include "../performance_counters.h" #include "../retroarch.h" #include "../verbosity.h" #include "../command.h" @@ -1835,6 +1836,9 @@ void video_driver_gpu_record_deinit(void) video_st->record_gpu_buffer = NULL; } +/* Main thread GPU readback while recording, for perfcnt_enable. */ +static struct retro_perf_counter record_perf_readback = {0}; + static void recording_dump_frame( const void *data, unsigned dims, size_t pitch, bool is_idle) { @@ -1869,6 +1873,8 @@ static void recording_dump_frame( #endif { struct video_viewport vp; + bool ok; + bool perf = runloop_state_get_ptr()->perfcnt_enable; vp.pos = VIDEO_POS_PACK(0, 0); vp.dims = 0; @@ -1902,16 +1908,18 @@ static void recording_dump_frame( /* Big bottleneck. * Since we might need to do read-backs asynchronously, * it might take 3-4 times before this returns true. */ + performance_counter_init(record_perf_readback, "record_readback"); + performance_counter_start_plus(perf, record_perf_readback); if (record_st->gpu_bgrx) - { - if (!( vid->read_viewport_bgrx - && vid->read_viewport_bgrx(video_st->data, - video_st->record_gpu_buffer, is_idle, &bottom_up))) - return; - } - else if (!( vid->read_viewport - && vid->read_viewport( - video_st->data, video_st->record_gpu_buffer, is_idle))) + ok = vid->read_viewport_bgrx + && vid->read_viewport_bgrx(video_st->data, + video_st->record_gpu_buffer, is_idle, &bottom_up); + else + ok = vid->read_viewport + && vid->read_viewport( + video_st->data, video_st->record_gpu_buffer, is_idle); + performance_counter_stop_plus(perf, record_perf_readback); + if (!ok) return; } diff --git a/record/drivers/record_ffmpeg.c b/record/drivers/record_ffmpeg.c index 3eb4f6c333a0..ad35e500e73d 100644 --- a/record/drivers/record_ffmpeg.c +++ b/record/drivers/record_ffmpeg.c @@ -80,6 +80,7 @@ extern "C" { #include "../../retroarch.h" #include "../../runloop.h" +#include "../../performance_counters.h" #include "../../verbosity.h" #ifndef FFMPEG3 @@ -1296,6 +1297,28 @@ static bool ffmpeg_init_muxer_post(ffmpeg_t *handle) /* One warning per this many newly dropped frames. */ #define DROP_LOG_INTERVAL 64 +/* Reported at exit with perfcnt_enable. push_* run on the frontend's + * thread, the rest on the encoder thread; one writer each. */ +static struct retro_perf_counter ffmpeg_perf_push_video = {0}; +static struct retro_perf_counter ffmpeg_perf_push_audio = {0}; +static struct retro_perf_counter ffmpeg_perf_scale = {0}; +static struct retro_perf_counter ffmpeg_perf_encode_video = {0}; +static struct retro_perf_counter ffmpeg_perf_encode_audio = {0}; + +static bool ffmpeg_perf_enabled(void) +{ + return runloop_state_get_ptr()->perfcnt_enable; +} + +static void ffmpeg_perf_init(void) +{ + performance_counter_init(ffmpeg_perf_push_video, "record_push_video"); + performance_counter_init(ffmpeg_perf_push_audio, "record_push_audio"); + performance_counter_init(ffmpeg_perf_scale, "record_scale"); + performance_counter_init(ffmpeg_perf_encode_video, "record_encode_video"); + performance_counter_init(ffmpeg_perf_encode_audio, "record_encode_audio"); +} + static void ffmpeg_thread(void *data); /* Encoder threads run this much nicer than the core, in drop mode @@ -1680,6 +1703,8 @@ static void *ffmpeg_new(const struct record_params *params) ffmpeg_av_log_init(); + ffmpeg_perf_init(); + handle->params = *params; handle->pkt = av_packet_alloc(); handle->allow_frame_drop = params->allow_frame_drop; @@ -1769,7 +1794,7 @@ static void ffmpeg_report_drops(uint64_t dropped) MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_WARNING); } -static bool ffmpeg_push_video(void *data, +static bool ffmpeg_push_video_impl(void *data, const struct record_video_data *vid) { unsigned y; @@ -1883,7 +1908,7 @@ static void ffmpeg_audio_write_silence(ffmpeg_t *handle) } } -static bool ffmpeg_push_audio(void *data, +static bool ffmpeg_push_audio_impl(void *data, const struct record_audio_data *audio_data) { ffmpeg_t *handle = (ffmpeg_t*)data; @@ -1940,6 +1965,28 @@ static bool ffmpeg_push_audio(void *data, return true; } +static bool ffmpeg_push_video(void *data, + const struct record_video_data *vid) +{ + bool ret; + bool perf = ffmpeg_perf_enabled(); + performance_counter_start_plus(perf, ffmpeg_perf_push_video); + ret = ffmpeg_push_video_impl(data, vid); + performance_counter_stop_plus(perf, ffmpeg_perf_push_video); + return ret; +} + +static bool ffmpeg_push_audio(void *data, + const struct record_audio_data *audio_data) +{ + bool ret; + bool perf = ffmpeg_perf_enabled(); + performance_counter_start_plus(perf, ffmpeg_perf_push_audio); + ret = ffmpeg_push_audio_impl(data, audio_data); + performance_counter_stop_plus(perf, ffmpeg_perf_push_audio); + return ret; +} + static bool encode_video(ffmpeg_t *handle, AVFrame *frame) { int ret; @@ -2039,8 +2086,15 @@ static void ffmpeg_scale_input(ffmpeg_t *handle, static bool ffmpeg_push_video_thread(ffmpeg_t *handle, const struct record_video_data *vid, unsigned pts_skip) { + bool ok; + bool perf = ffmpeg_perf_enabled(); + if (!vid->is_dupe) + { + performance_counter_start_plus(perf, ffmpeg_perf_scale); ffmpeg_scale_input(handle, vid); + performance_counter_stop_plus(perf, ffmpeg_perf_scale); + } /* Leave a gap for dropped frames so A/V stays aligned. */ handle->video.frame_cnt += pts_skip; @@ -2063,7 +2117,10 @@ static bool ffmpeg_push_video_thread(ffmpeg_t *handle, } } - if (!encode_video(handle, handle->video.conv_frame)) + performance_counter_start_plus(perf, ffmpeg_perf_encode_video); + ok = encode_video(handle, handle->video.conv_frame); + performance_counter_stop_plus(perf, ffmpeg_perf_encode_video); + if (!ok) return false; handle->video.frame_cnt++; @@ -2563,6 +2620,7 @@ static void ffmpeg_thread(void *data) size_t audio_buf_size = ff->config.audio_enable ? (ff->audio.codec->frame_size * ff->params.channels * sizeof(int16_t)) : 0; void *audio_buf = audio_buf_size ? av_malloc(audio_buf_size) : NULL; + bool perf = ffmpeg_perf_enabled(); #ifdef FFMPEG_HAVE_THREAD_NICE /* Also covers threads an encoder spawns lazily from here. */ @@ -2618,7 +2676,9 @@ static void ffmpeg_thread(void *data) aud.frames = ff->audio.codec->frame_size; aud.data = audio_buf; + performance_counter_start_plus(perf, ffmpeg_perf_encode_audio); ffmpeg_push_audio_thread(ff, &aud, true); + performance_counter_stop_plus(perf, ffmpeg_perf_encode_audio); } } diff --git a/samples/record/ffmpeg_pipeline/stubs_retroarch.c b/samples/record/ffmpeg_pipeline/stubs_retroarch.c index 25d18fa1b858..cf3a7955fd23 100644 --- a/samples/record/ffmpeg_pipeline/stubs_retroarch.c +++ b/samples/record/ffmpeg_pipeline/stubs_retroarch.c @@ -4,6 +4,7 @@ #include #include #include +#include "../../../runloop.h" void RARCH_LOG(const char *fmt, ...) { @@ -49,3 +50,15 @@ unsigned verbosity_get_log_level(void) { return 1; } + +static runloop_state_t stub_runloop_state; + +runloop_state_t *runloop_state_get_ptr(void) +{ + return &stub_runloop_state; +} + +void rarch_perf_register(struct retro_perf_counter *perf) +{ + perf->registered = true; +} From 63ef4a65fdeebd5bb1967f2523558745c1795e39 Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:01 +0200 Subject: [PATCH 10/16] record_ffmpeg: timestamp video from the audio clock Video pts was one tick per frame handed to the encoder, i.e. it assumed the core delivers a frame on every retro_run(). Many don't: a 30 fps game on a 60 Hz core (Dolphin's Wind Waker, for one) only calls the video callback for new frames, and frame skipping and run-ahead do the same. Those recordings played their video at the wrong speed: a 26-minute Wind Waker session produced 13 minutes of video under 26 minutes of audio, and RetroArch's own player, which is clocked by audio, hung once the video track ran out. Count every audio frame the core produced, which is the core's real elapsed time, and place each video frame at that point on the timeline through the same pts skip dropped frames use. Frames that arrive every other tick get pts 0, 2, 4, ... Without audio, the one-tick-per-frame accounting is kept. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- record/drivers/record_ffmpeg.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/record/drivers/record_ffmpeg.c b/record/drivers/record_ffmpeg.c index ad35e500e73d..fdc7d7137142 100644 --- a/record/drivers/record_ffmpeg.c +++ b/record/drivers/record_ffmpeg.c @@ -276,6 +276,13 @@ typedef struct ffmpeg /* Frames dropped since the last queued one; rides along with the * next frame so the encoder leaves a pts gap for them. */ unsigned video_pts_pending; + /* With audio, video is placed on the audio clock: every audio frame + * the core produced is its real elapsed time, and video_next_pts is + * the pts the encoder gives the next queued frame. Cores that don't + * present every tick (30 fps games on a 60 Hz core, frame skip, + * run-ahead) then still get the right spacing. Producer only. */ + uint64_t audio_frames_in; + int64_t video_next_pts; unsigned video_fifo_frames; bool allow_frame_drop; @@ -1873,6 +1880,20 @@ static bool ffmpeg_push_video_impl(void *data, } attr.pts_skip = handle->video_pts_pending; + if ( handle->config.audio_enable + && handle->audio_frames_in + && handle->params.samplerate > 0.0) + { + /* This frame's place on the audio timeline, in codec time base + * units; never behind the previous frame. */ + double secs = (double)handle->audio_frames_in + / handle->params.samplerate; + int64_t target = (int64_t)(secs * handle->params.fps + / handle->video.frame_drop_ratio + 0.5); + attr.pts_skip = (target > handle->video_next_pts) + ? (unsigned)(target - handle->video_next_pts) : 0; + } + handle->video_next_pts += (int64_t)attr.pts_skip + 1; handle->video_pts_pending = 0; /* Frame first, attr last: the encoder takes the attr as the @@ -1921,6 +1942,7 @@ static bool ffmpeg_push_audio_impl(void *data, return true; need = audio_data->frames * handle->params.channels * sizeof(int16_t); + handle->audio_frames_in += audio_data->frames; if (handle->allow_frame_drop) { From 1412aed395971f668631b9f7f2446158c7f8b78f Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:01 +0200 Subject: [PATCH 11/16] recording: optional "Record Game Only" (no menu, OSD or menu time) GPU recording reads the backbuffer at the end of the frame, so the volume and other OSD messages, notification widgets, statistics, input overlays and, with content running behind it, the menu end up in the recording, and the menu's filler silence puts the time spent in the menu into it too. That is what's on screen, which some users expect, so this adds a switch rather than changing the default. New setting video_record_game_only ("Record Game Only", default off). It travels with the frame, like gpu_recording. When on: - gl, glcore: the async PBO readback is issued right after the core's image (shader chain, PQ->SDR conversion) is in the backbuffer, before any UI is drawn. Under scRGB output the backbuffer only gets the image at the final composite, so that path reads back at the end. - vulkan: after the final pass, end the render pass, copy the backbuffer to the readback staging buffer and resume with keep_render_pass (LOAD), so the UI draws over the unchanged image. HDR output keeps the end-of-frame readback, which needs its tonemap. - audio_driver_menu_sample() doesn't push the menu's filler silence, so menu time is left out, as its frames already are. When off, behaviour is as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- audio/audio_driver.c | 11 ++++- config.def.h | 4 ++ configuration.h | 1 + gfx/drivers/gl2.c | 55 ++++++++++++++++--------- gfx/drivers/gl3.c | 51 ++++++++++++++--------- gfx/drivers/vulkan.c | 49 +++++++++++++++++++++- gfx/video_driver.c | 2 + gfx/video_driver.h | 3 ++ menu/cbs/menu_cbs_sublabel.c | 1 + menu/menu_displaylist.c | 18 ++++---- msg_hash_lbl_str.h | 1 + settings/settings_def_recording_video.h | 5 +++ tools/settings_table.reference | 3 ++ 13 files changed, 154 insertions(+), 50 deletions(-) diff --git a/audio/audio_driver.c b/audio/audio_driver.c index 7ab3782ade80..1e0b917a25b0 100644 --- a/audio/audio_driver.c +++ b/audio/audio_driver.c @@ -9092,9 +9092,15 @@ void audio_driver_menu_sample(bool by_clock) if ((AUDIO_FLAGS_GET(audio_st) & AUDIO_FLAG_SUSPENDED)) check_flush = false; + /* Record Game Only: the menu's filler silence (and so the time spent + * in the menu) stays out of the recording, like its frames do. */ + if (settings->bools.video_record_game_only) + recording_st = NULL; + while (sample_count > 1024) { - if ( recording_st->data && + if ( recording_st && + recording_st->data && recording_st->driver && recording_st->driver->push_audio) { @@ -9117,7 +9123,8 @@ void audio_driver_menu_sample(bool by_clock) sample_count -= 1024; } - if ( recording_st->data + if ( recording_st + && recording_st->data && recording_st->driver && recording_st->driver->push_audio) { diff --git a/config.def.h b/config.def.h index ca29e0a2a4a9..a42a42049ada 100644 --- a/config.def.h +++ b/config.def.h @@ -1200,6 +1200,10 @@ * of stalling the frontend until it drains. */ #define DEFAULT_VIDEO_RECORD_ALLOW_FRAME_DROP true +/* Record only the game: GPU recordings leave out the menu, OSD, + * widgets and overlays, and menu time is left out of recordings. */ +#define DEFAULT_VIDEO_RECORD_GAME_ONLY false + /* Use a hardware H.264 encoder for recording/streaming when the FFmpeg * build has one that works, falling back to libx264. */ #ifdef HAVE_LAKKA_SWITCH diff --git a/configuration.h b/configuration.h index a582c324e02c..647cc2d6ac27 100644 --- a/configuration.h +++ b/configuration.h @@ -671,6 +671,7 @@ typedef struct settings bool video_post_filter_record; bool video_gpu_record; bool video_record_allow_frame_drop; + bool video_record_game_only; bool video_record_hw_encoder; bool video_gpu_screenshot; bool video_allow_rotate; diff --git a/gfx/drivers/gl2.c b/gfx/drivers/gl2.c index 9bd1378e81b0..408ac6208d1f 100644 --- a/gfx/drivers/gl2.c +++ b/gfx/drivers/gl2.c @@ -4039,6 +4039,30 @@ static INLINE void gl2_draw_texture(gl2_t *gl) } #endif +static void gl2_pbo_async_readback(gl2_t *gl); + +/* The recording's per-frame async readback, or its teardown once + * recording has stopped. skip_in_menu: don't read back while the menu + * is drawn. */ +static void gl2_record_readback(gl2_t *gl, bool skip_in_menu) +{ + if (!(gl->flags & GL2_FLAG_PBO_READBACK_ENABLE)) + return; + + if (!(gl->flags & GL2_FLAG_GPU_RECORDING)) + { + glDeleteBuffers(4, gl->pbo_readback); + scaler_ctx_gen_reset(&gl->pbo_readback_scaler); + gl->flags &= ~GL2_FLAG_PBO_READBACK_ENABLE; + } +#ifdef HAVE_MENU + else if (skip_in_menu && (gl->flags & GL2_FLAG_MENU_TEXTURE_ENABLE)) + ; +#endif + else + gl2_pbo_async_readback(gl); +} + static void gl2_pbo_async_readback(gl2_t *gl) { #ifdef HAVE_OPENGLES @@ -5045,8 +5069,15 @@ static bool gl2_frame(void *data, const void *frame, * displayable image. Same choke-point pattern as glcore. */ if (gl->scrgb.active) gl2_bind_fb(gl2_ui_target_fbo(gl)); - else if (gl2_needs_pq_downconvert(gl)) - gl2_encode_pq_to_sdr(gl); + else + { + if (gl2_needs_pq_downconvert(gl)) + gl2_encode_pq_to_sdr(gl); + /* Record Game Only: read back before any UI is drawn. Under + * scRGB the image only lands at the final composite. */ + if (video_info->record_game_only) + gl2_record_readback(gl, false); + } #ifdef HAVE_OVERLAY if ((gl->flags & GL2_FLAG_OVERLAY_ENABLE) && overlay_behind_menu) @@ -5203,24 +5234,8 @@ static bool gl2_frame(void *data, const void *frame, 4, GL_RGBA, GL_UNSIGNED_BYTE, gl->readback_buffer_screenshot); - else if (gl->flags & GL2_FLAG_PBO_READBACK_ENABLE) - { - /* If recording has stopped, tear down PBO readback */ - if (!(gl->flags & GL2_FLAG_GPU_RECORDING)) - { - glDeleteBuffers(4, gl->pbo_readback); - scaler_ctx_gen_reset(&gl->pbo_readback_scaler); - gl->flags &= ~GL2_FLAG_PBO_READBACK_ENABLE; - } - else - { -#ifdef HAVE_MENU - /* Don't readback if we're in menu mode. */ - if (!(gl->flags & GL2_FLAG_MENU_TEXTURE_ENABLE)) -#endif - gl2_pbo_async_readback(gl); - } - } + else if (!video_info->record_game_only || gl->scrgb.active) + gl2_record_readback(gl, !video_info->record_game_only); /* The backbuffer is what it is until the swap, so the copy is * taken now. Not from the BFI light dupes, which recurse in here diff --git a/gfx/drivers/gl3.c b/gfx/drivers/gl3.c index bf2fbe419907..a0b7b53418e5 100644 --- a/gfx/drivers/gl3.c +++ b/gfx/drivers/gl3.c @@ -1961,6 +1961,29 @@ static void gl3_deinit_pbo_readback(gl3_t *gl) scaler_ctx_gen_reset(&gl->pbo_readback_scaler); } +static void gl3_pbo_async_readback(gl3_t *gl); + +/* The recording's per-frame async readback, or its teardown once + * recording has stopped. skip_in_menu: don't read back while the menu + * is drawn. */ +static void gl3_record_readback(gl3_t *gl, bool skip_in_menu) +{ + if (!(gl->flags & GL3_FLAG_PBO_READBACK_ENABLE)) + return; + + if (!(gl->flags & GL3_FLAG_GPU_RECORDING)) + { + gl3_deinit_pbo_readback(gl); + gl->flags &= ~GL3_FLAG_PBO_READBACK_ENABLE; + } +#ifdef HAVE_MENU + else if (skip_in_menu && (gl->flags & GL3_FLAG_MENU_TEXTURE_ENABLE)) + ; +#endif + else + gl3_pbo_async_readback(gl); +} + static void gl3_pbo_async_readback(gl3_t *gl) { glBindBuffer(GL_PIXEL_PACK_BUFFER, @@ -6303,13 +6326,18 @@ static bool gl3_frame(void *data, const void *frame, if (gl->scrgb.active) glBindFramebuffer(GL_FRAMEBUFFER, gl3_ui_target_fbo(gl, video_info->dims)); - else if (gl3_needs_pq_downconvert(gl)) + else { /* Convert the PQ frame to SDR now, straight into the backbuffer, * so the UI below draws over a displayable image in its own * space - no UI layer, no linear composite, nothing else about * the SDR path changes. */ - gl3_encode_pq_to_sdr(gl, width, height); + if (gl3_needs_pq_downconvert(gl)) + gl3_encode_pq_to_sdr(gl, width, height); + /* Record Game Only: read back before any UI is drawn. Under + * scRGB the image only lands at the final composite. */ + if (video_info->record_game_only) + gl3_record_readback(gl, false); } #ifdef HAVE_SLANG @@ -6539,23 +6567,8 @@ static bool gl3_frame(void *data, const void *frame, && !gl->video_info.source_hdr10) glBindFramebuffer(GL_READ_FRAMEBUFFER, 0); } - else if (gl->flags & GL3_FLAG_PBO_READBACK_ENABLE) - { - /* If recording has stopped, tear down PBO readback */ - if (!(gl->flags & GL3_FLAG_GPU_RECORDING)) - { - gl3_deinit_pbo_readback(gl); - gl->flags &= ~GL3_FLAG_PBO_READBACK_ENABLE; - } - else - { -#ifdef HAVE_MENU - /* Don't readback if we're in menu mode. */ - if (!(gl->flags & GL3_FLAG_MENU_TEXTURE_ENABLE)) -#endif - gl3_pbo_async_readback(gl); - } - } + else if (!video_info->record_game_only || gl->scrgb.active) + gl3_record_readback(gl, !video_info->record_game_only); gl3_hw_ring_drawn(gl); gl->present_wait = 0; diff --git a/gfx/drivers/vulkan.c b/gfx/drivers/vulkan.c index f5f17cc5cf7a..72e99c154afd 100644 --- a/gfx/drivers/vulkan.c +++ b/gfx/drivers/vulkan.c @@ -10473,6 +10473,8 @@ static bool vulkan_frame(void *data, const void *frame, #ifdef VULKAN_HDR_SWAPCHAIN bool use_offscreen_buffer = false; #endif + /* Recording's readback was already taken this frame, before the UI. */ + bool record_readback_done = false; #ifdef HAVE_OPENXR bool xr_stereo = (vk->flags & VK_FLAG_OPEN_XR) != 0; @@ -11341,6 +11343,50 @@ static bool vulkan_frame(void *data, const void *frame, (vulkan_filter_chain_t*)filter_chain, vk->cmd, &vk->video_vp, vk->mvp.data); + /* Record Game Only: copy the game image out before the UI is + * drawn, then resume the pass with LOAD. HDR output keeps the + * end-of-frame readback, which needs its tonemap pass. */ + if ( video_info->record_game_only + && (vk->flags & VK_FLAG_READBACK_STREAMED) + && (vk->flags & VK_FLAG_GPU_RECORDING) +#ifdef VULKAN_HDR_SWAPCHAIN + && !(vk->context->flags & VK_CTX_FLAG_HDR_ENABLE) +#endif + && (backbuffer->image != VK_NULL_HANDLE) + && (vk->context->flags & VK_CTX_FLAG_HAS_ACQUIRED_SWAPCHAIN)) + { + vkCmdEndRenderPass(vk->cmd); + + VULKAN_IMAGE_LAYOUT_TRANSITION( + vk->cmd, + backbuffer->image, + VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + VK_ACCESS_TRANSFER_READ_BIT, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT); + + vulkan_readback(vk, backbuffer); + + VULKAN_IMAGE_LAYOUT_TRANSITION( + vk->cmd, + backbuffer->image, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, + VK_ACCESS_TRANSFER_READ_BIT, + VK_ACCESS_COLOR_ATTACHMENT_READ_BIT + | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); + + /* Same framebuffer, LOAD instead of CLEAR. */ + rp_info.renderPass = vk->keep_render_pass; + vkCmdBeginRenderPass(vk->cmd, &rp_info, VK_SUBPASS_CONTENTS_INLINE); + + record_readback_done = true; + } + #ifdef VULKAN_HDR_SWAPCHAIN end_pass = true; end_main_pass = true; @@ -11626,7 +11672,8 @@ static bool vulkan_frame(void *data, const void *frame, } if ( (vk->flags & VK_FLAG_READBACK_PENDING) - || (vk->flags & VK_FLAG_READBACK_STREAMED)) + || ( (vk->flags & VK_FLAG_READBACK_STREAMED) + && !record_readback_done)) { VkImageLayout backbuffer_layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; #ifdef VULKAN_HDR_SWAPCHAIN diff --git a/gfx/video_driver.c b/gfx/video_driver.c index 95b4189105b4..bfc9b0829437 100644 --- a/gfx/video_driver.c +++ b/gfx/video_driver.c @@ -5886,6 +5886,8 @@ void video_driver_build_info(video_frame_info_t *video_info) video_info->msg_queue_delay = runloop_st->msg_queue_delay; video_info->runloop_is_paused = (runloop_st->flags & RUNLOOP_FLAG_PAUSED) ? true : false; video_info->gpu_recording = recording_state_get_ptr()->enable; + video_info->record_game_only = video_info->gpu_recording + && settings->bools.video_record_game_only; video_info->core_running = !(runloop_st->flags & RUNLOOP_FLAG_PAUSED); video_info->menu_content_rate = false; #ifdef HAVE_MENU diff --git a/gfx/video_driver.h b/gfx/video_driver.h index 01f0885678c3..2a53f6c3fc93 100644 --- a/gfx/video_driver.h +++ b/gfx/video_driver.h @@ -699,6 +699,9 @@ typedef struct video_frame_info * draws them: it travels with the frame, so the thread that draws * never reads the recording state the main thread writes. */ bool gpu_recording; + /* Record Game Only: the driver reads the recording's frame back + * before drawing any UI over it. */ + bool record_game_only; bool threaded_present_repeat; /* The threaded presenter is holding each push to the display's * vblank: the setting is on and the wrapper is running. */ diff --git a/menu/cbs/menu_cbs_sublabel.c b/menu/cbs/menu_cbs_sublabel.c index 8628ed2b673e..5d7219d78f57 100644 --- a/menu/cbs/menu_cbs_sublabel.c +++ b/menu/cbs/menu_cbs_sublabel.c @@ -2435,6 +2435,7 @@ int menu_cbs_init_bind_sublabel(menu_file_list_cbs_t *cbs, { MENU_ENUM_LABEL_CORE_MANAGER_LIST, MENU_ENUM_SUBLABEL_CORE_MANAGER_LIST }, { MENU_ENUM_LABEL_VIDEO_POST_FILTER_RECORD, MENU_ENUM_SUBLABEL_VIDEO_POST_FILTER_RECORD }, { MENU_ENUM_LABEL_VIDEO_RECORD_ALLOW_FRAME_DROP, MENU_ENUM_SUBLABEL_VIDEO_RECORD_ALLOW_FRAME_DROP }, + { MENU_ENUM_LABEL_VIDEO_RECORD_GAME_ONLY, MENU_ENUM_SUBLABEL_VIDEO_RECORD_GAME_ONLY }, { MENU_ENUM_LABEL_VIDEO_RECORD_FIFO_FRAMES, MENU_ENUM_SUBLABEL_VIDEO_RECORD_FIFO_FRAMES }, { MENU_ENUM_LABEL_VIDEO_RECORD_HW_ENCODER, MENU_ENUM_SUBLABEL_VIDEO_RECORD_HW_ENCODER }, { MENU_ENUM_LABEL_NETPLAY_NICKNAME, MENU_ENUM_SUBLABEL_NETPLAY_NICKNAME }, diff --git a/menu/menu_displaylist.c b/menu/menu_displaylist.c index 1c9c63a9b045..24ddd1ba7ec5 100644 --- a/menu/menu_displaylist.c +++ b/menu/menu_displaylist.c @@ -11494,6 +11494,7 @@ unsigned menu_displaylist_build_list( {MENU_ENUM_LABEL_VIDEO_RECORD_THREADS, PARSE_ONLY_UINT, false}, {MENU_ENUM_LABEL_VIDEO_POST_FILTER_RECORD, PARSE_ONLY_BOOL, false}, {MENU_ENUM_LABEL_VIDEO_GPU_RECORD, PARSE_ONLY_BOOL, false}, + {MENU_ENUM_LABEL_VIDEO_RECORD_GAME_ONLY, PARSE_ONLY_BOOL, false}, {MENU_ENUM_LABEL_VIDEO_RECORD_ALLOW_FRAME_DROP, PARSE_ONLY_BOOL, false}, {MENU_ENUM_LABEL_VIDEO_RECORD_HW_ENCODER, PARSE_ONLY_BOOL, false}, {MENU_ENUM_LABEL_VIDEO_RECORD_FIFO_FRAMES, PARSE_ONLY_UINT, false}, @@ -11509,14 +11510,15 @@ unsigned menu_displaylist_build_list( build_list[3].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_RECORD_THREADS */ build_list[4].checked = has_video; /* MENU_ENUM_LABEL_VIDEO_POST_FILTER_RECORD */ build_list[5].checked = has_video; /* MENU_ENUM_LABEL_VIDEO_GPU_RECORD */ - build_list[6].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_RECORD_ALLOW_FRAME_DROP */ - build_list[7].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_RECORD_HW_ENCODER */ - build_list[8].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_RECORD_FIFO_FRAMES */ - build_list[9].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_MODE */ - build_list[10].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_STREAM_QUALITY */ - build_list[11].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAM_CONFIG */ - build_list[12].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_TITLE */ - build_list[13].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_URL */ + build_list[6].checked = has_video; /* MENU_ENUM_LABEL_VIDEO_RECORD_GAME_ONLY */ + build_list[7].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_RECORD_ALLOW_FRAME_DROP */ + build_list[8].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_RECORD_HW_ENCODER */ + build_list[9].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_RECORD_FIFO_FRAMES */ + build_list[10].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_MODE */ + build_list[11].checked = is_ffmpeg; /* MENU_ENUM_LABEL_VIDEO_STREAM_QUALITY */ + build_list[12].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAM_CONFIG */ + build_list[13].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_TITLE */ + build_list[14].checked = is_ffmpeg; /* MENU_ENUM_LABEL_STREAMING_URL */ for (i = 0; i < ARRAY_SIZE(build_list); i++) { diff --git a/msg_hash_lbl_str.h b/msg_hash_lbl_str.h index 310bce1c520a..85e08da8f2c6 100644 --- a/msg_hash_lbl_str.h +++ b/msg_hash_lbl_str.h @@ -1721,6 +1721,7 @@ #define MENU_ENUM_LABEL_VIDEO_PRESENT_TIMING_FROM_DISPLAY_STR "video_present_timing_from_display" #define MENU_ENUM_LABEL_VIDEO_RECORD_ALLOW_FRAME_DROP_STR "video_record_allow_frame_drop" #define MENU_ENUM_LABEL_VIDEO_RECORD_FIFO_FRAMES_STR "video_record_fifo_frames" +#define MENU_ENUM_LABEL_VIDEO_RECORD_GAME_ONLY_STR "video_record_game_only" #define MENU_ENUM_LABEL_VIDEO_RECORD_HW_ENCODER_STR "video_record_hw_encoder" #define MENU_ENUM_LABEL_VIDEO_RECORD_QUALITY_STR "video_record_quality" #define MENU_ENUM_LABEL_VIDEO_RECORD_THREADS_STR "video_record_threads" diff --git a/settings/settings_def_recording_video.h b/settings/settings_def_recording_video.h index a38f97dab6d5..8cec2e32c3e6 100644 --- a/settings/settings_def_recording_video.h +++ b/settings/settings_def_recording_video.h @@ -19,6 +19,11 @@ S_BOOL(video_record_allow_frame_drop, VIDEO_RECORD_ALLOW_FRAME_DROP, DEFAULT_VIDEO_RECORD_ALLOW_FRAME_DROP, SD_FLAG_NONE, 0, 0, "Drop Frames When Encoder Falls Behind", "When the recording or streaming encoder can't keep up, skip video frames and fill audio gaps with silence instead of slowing down the game. Disable to keep every frame at the cost of stutter.") +S_BOOL(video_record_game_only, VIDEO_RECORD_GAME_ONLY, + "video_record_game_only", + DEFAULT_VIDEO_RECORD_GAME_ONLY, SD_FLAG_NONE, 0, 0, + "Record Game Only", + "Leave the menu, notifications, on-screen messages and overlays out of GPU recordings, and leave out the time spent in the menu. When off, recordings capture exactly what is on screen.") S_BOOL(video_record_hw_encoder, VIDEO_RECORD_HW_ENCODER, "video_record_hw_encoder", DEFAULT_VIDEO_RECORD_HW_ENCODER, SD_FLAG_NONE, 0, 0, diff --git a/tools/settings_table.reference b/tools/settings_table.reference index 820a9577773d..86624af831c9 100644 --- a/tools/settings_table.reference +++ b/tools/settings_table.reference @@ -310,6 +310,7 @@ bool menu_show_shutdown 1 1 bool video_post_filter_record 1 0 bool video_gpu_record 1 0 bool video_record_allow_frame_drop 1 1 +bool video_record_game_only 1 0 bool video_record_hw_encoder 1 0 bool menu_rgui_full_width_layout 1 1 bool video_shader_enable 1 1 @@ -1003,6 +1004,7 @@ bool menu_show_shutdown 1 1 bool video_post_filter_record 1 0 bool video_gpu_record 1 0 bool video_record_allow_frame_drop 1 1 +bool video_record_game_only 1 0 bool video_record_hw_encoder 1 0 bool menu_rgui_full_width_layout 1 1 bool video_shader_enable 1 1 @@ -1699,6 +1701,7 @@ bool menu_show_shutdown 1 1 bool video_post_filter_record 1 0 bool video_gpu_record 1 0 bool video_record_allow_frame_drop 1 1 +bool video_record_game_only 1 0 bool video_record_hw_encoder 1 0 bool menu_rgui_full_width_layout 1 1 bool video_shader_enable 1 1 From 7069425ebc235750e1f6b7cbd34037c2848f90a4 Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:01 +0200 Subject: [PATCH 12/16] recording: apply the content rotation to raw (non-GPU) recordings Raw recordings encoded the core's frames as handed over, ignoring the rotation the display applies (core-requested or the user's video rotation), so vertical games came out sideways. GPU recordings were unaffected since they read back the already-rotated screen. Pass the rotation to the recorder, describe the output with width and height swapped for quarter turns (unless --size gave one), and rotate each frame on the encoder thread before scaling. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- record/drivers/record_ffmpeg.c | 56 ++++++++++++++++++++++++++++++++++ record/record_driver.c | 13 ++++++++ record/record_driver.h | 4 +++ 3 files changed, 73 insertions(+) diff --git a/record/drivers/record_ffmpeg.c b/record/drivers/record_ffmpeg.c index fdc7d7137142..c31d7525a02a 100644 --- a/record/drivers/record_ffmpeg.c +++ b/record/drivers/record_ffmpeg.c @@ -283,6 +283,9 @@ typedef struct ffmpeg * run-ahead) then still get the right spacing. Producer only. */ uint64_t audio_frames_in; int64_t video_next_pts; + /* Encoder thread: scratch for params.rotation, fb_dims pixels, + * allocated on first use. */ + uint8_t *video_rot_buf; unsigned video_fifo_frames; bool allow_frame_drop; @@ -1519,6 +1522,9 @@ static void ffmpeg_free(void *data) / (handle->params.samplerate * handle->params.channels * sizeof(int16_t))); + av_free(handle->video_rot_buf); + handle->video_rot_buf = NULL; + if (handle->audio.codec) { #if FFMPEG8 @@ -2105,12 +2111,62 @@ static void ffmpeg_scale_input(ffmpeg_t *handle, shrunk); } +/* Rotate a packed frame by rot quarter turns counter-clockwise + * (libretro convention) from src (w x h, src_pitch bytes per row) into + * dst, tightly packed at the rotated width. */ +static void ffmpeg_rotate_frame(uint8_t *dst, const uint8_t *src, + unsigned w, unsigned h, int src_pitch, unsigned bpp, unsigned rot) +{ + unsigned x, y; + unsigned dw = (rot & 1) ? h : w; + + for (y = 0; y < h; y++) + { + const uint8_t *row = src + (ptrdiff_t)y * src_pitch; + for (x = 0; x < w; x++) + { + unsigned dx, dy; + switch (rot) + { + case 1: dx = y; dy = w - 1 - x; break; /* 90 CCW */ + case 2: dx = w - 1 - x; dy = h - 1 - y; break; /* 180 */ + default: dx = h - 1 - y; dy = x; break; /* 270 CCW */ + } + memcpy(dst + ((size_t)dy * dw + dx) * bpp, row + (size_t)x * bpp, + bpp); + } + } +} + static bool ffmpeg_push_video_thread(ffmpeg_t *handle, const struct record_video_data *vid, unsigned pts_skip) { + struct record_video_data rotated; bool ok; bool perf = ffmpeg_perf_enabled(); + if (handle->params.rotation && !vid->is_dupe && vid->data) + { + unsigned bpp = (unsigned)handle->video.pix_size; + unsigned w = VIDEO_SCALE_W(vid->dims); + unsigned h = VIDEO_SCALE_H(vid->dims); + if (!handle->video_rot_buf) + handle->video_rot_buf = (uint8_t*)av_malloc( + VIDEO_SCALE_AREA(handle->params.fb_dims) * bpp); + if ( handle->video_rot_buf + && (size_t)w * h <= VIDEO_SCALE_AREA(handle->params.fb_dims)) + { + unsigned rot = handle->params.rotation & 3; + ffmpeg_rotate_frame(handle->video_rot_buf, + (const uint8_t*)vid->data, w, h, vid->pitch, bpp, rot); + rotated = *vid; + rotated.data = handle->video_rot_buf; + rotated.dims = (rot & 1) ? VIDEO_SCALE_PACK(h, w) : vid->dims; + rotated.pitch = (int)(VIDEO_SCALE_W(rotated.dims) * bpp); + vid = &rotated; + } + } + if (!vid->is_dupe) { performance_counter_start_plus(perf, ffmpeg_perf_scale); diff --git a/record/record_driver.c b/record/record_driver.c index 7975273b8c01..4d847764db62 100644 --- a/record/record_driver.c +++ b/record/record_driver.c @@ -424,6 +424,19 @@ bool recording_init(void) params.aspect_ratio = (float)VIDEO_SCALE_W(params.out_dims) / VIDEO_SCALE_H(params.out_dims); + /* Raw core frames are unrotated; the GPU path reads back the + * already-rotated screen. Rotate them like the display does. */ + params.rotation = retroarch_get_rotation() % 4; + if (params.rotation & 1) + { + if (!recording_state.out_dims) + params.out_dims = VIDEO_SCALE_PACK( + VIDEO_SCALE_H(params.out_dims), + VIDEO_SCALE_W(params.out_dims)); + if (params.aspect_ratio > 0.0f) + params.aspect_ratio = 1.0f / params.aspect_ratio; + } + #ifdef HAVE_VIDEO_FILTER if (settings->bools.video_post_filter_record && !!video_st->state_filter) diff --git a/record/record_driver.h b/record/record_driver.h index 1935f59d9dc3..80476ee80791 100644 --- a/record/record_driver.h +++ b/record/record_driver.h @@ -88,6 +88,10 @@ struct record_params bool allow_frame_drop; /* Prefer a hardware H.264 encoder for the built-in presets. */ bool hw_encoder; + /* Quarter turns counter-clockwise to rotate incoming frames by. + * out_dims and aspect_ratio describe the rotated output, fb_dims + * the incoming frames. */ + unsigned rotation; }; struct record_video_data From d68ba6781ea15bc1e40ad64420655b87da4dcf74 Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:01 +0200 Subject: [PATCH 13/16] record_ffmpeg: scale frames smaller than the output size to fill it ffmpeg_scale_input() clamped the scaling destination to the source size whenever a frame was smaller than the output. That was meant for the one pixel of padding added to reach even dimensions, but applied to any smaller frame: frames below the core's base geometry landed in the top-left corner of a black frame (Mupen64Plus-Next with Angrylion and GPU recording off), and video_record_scale_factor > 1 had no effect, since every frame is smaller than the scaled output. Scale every frame to the output size; only a frame exactly one pixel short of it in a dimension (the even-size padding) is copied 1:1. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- record/drivers/record_ffmpeg.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/record/drivers/record_ffmpeg.c b/record/drivers/record_ffmpeg.c index c31d7525a02a..2f12b16efd7c 100644 --- a/record/drivers/record_ffmpeg.c +++ b/record/drivers/record_ffmpeg.c @@ -2073,12 +2073,12 @@ static void ffmpeg_scale_input(ffmpeg_t *handle, { unsigned src_w = VIDEO_SCALE_W(vid->dims); unsigned src_h = VIDEO_SCALE_H(vid->dims); - /* When output was padded to even dimensions, clamp the scaling - * destination to the source size. */ + /* Scale every frame to the output size, except one exactly a pixel + * short of it (the padding to even dimensions), which is copied 1:1. */ unsigned out_w = VIDEO_SCALE_W(handle->params.out_dims); unsigned out_h = VIDEO_SCALE_H(handle->params.out_dims); - unsigned dst_w = (src_w < out_w) ? src_w : out_w; - unsigned dst_h = (src_h < out_h) ? src_h : out_h; + unsigned dst_w = (src_w + 1 == out_w) ? src_w : out_w; + unsigned dst_h = (src_h + 1 == out_h) ? src_h : out_h; /* Attempt to preserve more information if we scale down. */ bool shrunk = dst_w < src_w || dst_h < src_h; From 477213640986fff6fc393f0235a9d63b2e4ec5c2 Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:01 +0200 Subject: [PATCH 14/16] record_ffmpeg: fill audio gaps when the core stops producing audio A core that produces no audio for a while, rather than producing silence, sent nothing to the recorder, so that time was missing from the audio track and everything after it was out of sync (e.g. Mupen64Plus' 64DD loading). With video timestamps taken from the audio clock it would also stall the video timeline. Once 4 video frames in a row arrive without any audio, owe one frame period (sample_rate / fps, with the fraction carried) of silence per video frame, backfilling the 4 already seen, through the existing owed-silence mechanism. The silence goes ahead of later audio in both drop and blocking mode. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- record/drivers/record_ffmpeg.c | 65 ++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/record/drivers/record_ffmpeg.c b/record/drivers/record_ffmpeg.c index 2f12b16efd7c..57fbcdadfcd0 100644 --- a/record/drivers/record_ffmpeg.c +++ b/record/drivers/record_ffmpeg.c @@ -283,6 +283,10 @@ typedef struct ffmpeg * run-ahead) then still get the right spacing. Producer only. */ uint64_t audio_frames_in; int64_t video_next_pts; + /* For cores that stop producing audio: ffmpeg_fill_audio_gap(). */ + uint64_t audio_frames_at_last_video; + unsigned video_frames_without_audio; + double silence_frac; /* Encoder thread: scratch for params.rotation, fb_dims pixels, * allocated on first use. */ uint8_t *video_rot_buf; @@ -1807,6 +1811,54 @@ static void ffmpeg_report_drops(uint64_t dropped) MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_WARNING); } +static void ffmpeg_audio_write_silence(ffmpeg_t *handle); + +/* Video frames in a row without audio before the core is taken to have + * gone silent: short enough to be imperceptible, long enough that cores + * delivering audio in bursts aren't affected. */ +#define SILENT_VIDEO_FRAMES 4 + +/* A core that produces no audio for a while (rather than silence) left + * that time out of the audio track, desyncing everything after it, and + * with video on the audio clock it would stall video too. After + * SILENT_VIDEO_FRAMES frames without audio, owe one frame period of + * silence per video frame, backfilling the ones already seen. */ +static void ffmpeg_fill_audio_gap(ffmpeg_t *handle) +{ + unsigned frames_to_fill; + double samples; + uint64_t whole; + + if ( !handle->config.audio_enable + || handle->params.samplerate <= 0.0 + || handle->params.fps <= 0.0) + return; + + if (handle->audio_frames_in != handle->audio_frames_at_last_video) + { + handle->audio_frames_at_last_video = handle->audio_frames_in; + handle->video_frames_without_audio = 0; + return; + } + + if (++handle->video_frames_without_audio < SILENT_VIDEO_FRAMES) + return; + + frames_to_fill = (handle->video_frames_without_audio == SILENT_VIDEO_FRAMES) + ? SILENT_VIDEO_FRAMES : 1; + samples = frames_to_fill * handle->params.samplerate + / handle->params.fps + handle->silence_frac; + whole = (uint64_t)samples; + handle->silence_frac = samples - (double)whole; + + handle->audio_frames_in += whole; + handle->audio_silence_pending += (size_t)whole + * handle->params.channels * sizeof(int16_t); + handle->audio_frames_at_last_video = handle->audio_frames_in; + ffmpeg_audio_write_silence(handle); + retro_eventcount_notify(&handle->data); +} + static bool ffmpeg_push_video_impl(void *data, const struct record_video_data *vid) { @@ -1850,6 +1902,7 @@ static bool ffmpeg_push_video_impl(void *data, rows = VIDEO_SCALE_H(attr.vid.dims); video_bytes = (size_t)rows * attr.vid.pitch; + ffmpeg_fill_audio_gap(handle); handle->video_frames_in++; /* Drop mode: a full queue costs this frame, never the frontend. */ @@ -1975,12 +2028,20 @@ static bool ffmpeg_push_audio_impl(void *data, if (!retro_atomic_load_acquire_int(&handle->alive)) return false; - if (retro_spsc_write_avail(&handle->audio_fifo) >= need) + /* Silence owed from a gap goes in first, in order. */ + if (handle->audio_silence_pending) + { + ffmpeg_audio_write_silence(handle); + retro_eventcount_notify(&handle->data); + } + if ( !handle->audio_silence_pending + && retro_spsc_write_avail(&handle->audio_fifo) >= need) break; key = retro_eventcount_prepare_wait(&handle->space); if ( !retro_atomic_load_acquire_int(&handle->alive) - || retro_spsc_write_avail(&handle->audio_fifo) >= need) + || ( !handle->audio_silence_pending + && retro_spsc_write_avail(&handle->audio_fifo) >= need)) retro_eventcount_cancel_wait(&handle->space); else retro_eventcount_commit_wait_timeout(&handle->space, key, From ff98fd504bd7f0a6281e3f91a00e6f0815072b33 Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:01 +0200 Subject: [PATCH 15/16] record_ffmpeg: drop the global 'ctx', close the output on failure The muxer context also lived in a non-static global named 'ctx', an exported symbol in every binary that links this driver, used by finalize to close the output. Use handle->muxer.ctx throughout. A recording that fails after the output was opened (an encoder that won't open, say) never reached finalize, so its file stayed open. ffmpeg_free() closes it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- record/drivers/record_ffmpeg.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/record/drivers/record_ffmpeg.c b/record/drivers/record_ffmpeg.c index 57fbcdadfcd0..4e0d7bde7884 100644 --- a/record/drivers/record_ffmpeg.c +++ b/record/drivers/record_ffmpeg.c @@ -315,8 +315,6 @@ struct ff_video_attr * out, so this only bounds a wake that never comes. */ #define FFMPEG_PUSH_WAIT_US 100000 -AVFormatContext *ctx; - /* Returns the encoder's list of supported sample formats, terminated by * AV_SAMPLE_FMT_NONE, or NULL if the encoder does not restrict sample * formats (or the list could not be queried). */ @@ -1231,6 +1229,7 @@ static bool ffmpeg_init_config(struct ff_config_param *params, static bool ffmpeg_init_muxer_pre(ffmpeg_t *handle) { + AVFormatContext *ctx; #if !FFMPEG3 size_t _len; #endif @@ -1594,6 +1593,11 @@ static void ffmpeg_free(void *data) /* The muxer context owns its streams, their codec parameters, its * metadata and its url; releasing just the struct left the rest * behind on every session. */ + if ( handle->muxer.ctx + && handle->muxer.ctx->pb + && !(handle->muxer.ctx->oformat + && (handle->muxer.ctx->oformat->flags & AVFMT_NOFILE))) + avio_closep(&handle->muxer.ctx->pb); /* failed before finalize */ avformat_free_context(handle->muxer.ctx); av_packet_free(&handle->pkt); @@ -2744,7 +2748,7 @@ static bool ffmpeg_finalize(void *data) /* Write final data. */ av_write_trailer(handle->muxer.ctx); - avio_close(ctx->pb); + avio_closep(&handle->muxer.ctx->pb); return true; } From 709d2fa1005ccd731cb903d5397d2924dadcb90f Mon Sep 17 00:00:00 2001 From: LibretroAdmin Date: Thu, 8 Oct 2026 19:27:09 +0200 Subject: [PATCH 16/16] recording: rotated raw aspect, quiet drop mode, lowered encoders by inheritance, frame format across a threading switch - A raw recording of a turned picture takes the display's aspect as it is, which is already the turned one; only an aspect worked out from the output size follows the turn, and --size keeps its own shape (record_raw_geometry). - Drop mode says nothing while it records: dropped frames are counted and the total is logged when the recording ends. - In drop mode the encoders are opened on a thread that has lowered its own priority, so the threads they start run lowered and no other thread is touched. - Only the frontend-thread perf counters are registered, as the menu reads and resets them live. - A GPU recording opened for 32-bit frames gets them widened from BGR24 when the readback in charge hands over BGR24 - the threaded wrapper's, or a driver without the 32-bit one - into its own buffer. - At the end of a recording the real audio left in the queue is encoded ahead of the silence still owed; the gap fill counts every frame the core presents, frame_drop_ratio or not; the output is closed whenever it is open; an encoder that leaves the CBR request unused is named. ffmpeg_pipeline_test checks the geometry, that drop mode drops frames without a warning or an on-screen message, and that its encoder threads run lowered. threaded_video's gpu record format lane records on Vulkan unthreaded and threaded with a 32-bit recording open, and reads every row of what it is handed. --- gfx/video_driver.c | 42 +++- record/drivers/record_ffmpeg.c | 212 +++++++----------- record/record_driver.c | 20 +- record/record_driver.h | 27 ++- .../gfx/threaded_video/threaded_video_test.c | 131 +++++++++++ .../ffmpeg_pipeline/ffmpeg_pipeline_test.c | 120 +++++++++- .../record/ffmpeg_pipeline/stubs_retroarch.c | 7 + 7 files changed, 397 insertions(+), 162 deletions(-) diff --git a/gfx/video_driver.c b/gfx/video_driver.c index bfc9b0829437..37050dc6f776 100644 --- a/gfx/video_driver.c +++ b/gfx/video_driver.c @@ -1839,6 +1839,24 @@ void video_driver_gpu_record_deinit(void) /* Main thread GPU readback while recording, for perfcnt_enable. */ static struct retro_perf_counter record_perf_readback = {0}; +/* BGR24 pixels to B,G,R,X. @dst may be @src: walked from the end, each + * pixel is read before anything is written over it. */ +VIDEO_NOINLINE static void recording_bgr24_to_bgrx(uint8_t *dst, + const uint8_t *src, size_t area) +{ + size_t i = area; + while (i--) + { + uint8_t b = src[3 * i + 0]; + uint8_t g = src[3 * i + 1]; + uint8_t r = src[3 * i + 2]; + dst[4 * i + 0] = b; + dst[4 * i + 1] = g; + dst[4 * i + 2] = r; + dst[4 * i + 3] = 0xff; + } +} + static void recording_dump_frame( const void *data, unsigned dims, size_t pitch, bool is_idle) { @@ -1869,6 +1887,15 @@ static void recording_dump_frame( record_st->driver->push_video(record_st->data, &ffemu_data); return; } + /* The wrapper reads back BGR24 on its own thread; a recording + * opened for 32-bit frames (threaded video switched on since) + * has it widened into the buffer sized for those. */ + if (taken == 1 && record_st->gpu_bgrx) + { + recording_bgr24_to_bgrx(video_st->record_gpu_buffer, gpu_frame, + VIDEO_SCALE_AREA(record_st->gpu_dims)); + gpu_frame = video_st->record_gpu_buffer; + } if (taken == -2) #endif { @@ -1910,14 +1937,21 @@ static void recording_dump_frame( * it might take 3-4 times before this returns true. */ performance_counter_init(record_perf_readback, "record_readback"); performance_counter_start_plus(perf, record_perf_readback); - if (record_st->gpu_bgrx) - ok = vid->read_viewport_bgrx - && vid->read_viewport_bgrx(video_st->data, - video_st->record_gpu_buffer, is_idle, &bottom_up); + if (record_st->gpu_bgrx && vid->read_viewport_bgrx) + ok = vid->read_viewport_bgrx(video_st->data, + video_st->record_gpu_buffer, is_idle, &bottom_up); else + { + /* BGR24, as the driver now up reads it back: widened when + * the recording was opened for 32-bit frames. */ ok = vid->read_viewport && vid->read_viewport( video_st->data, video_st->record_gpu_buffer, is_idle); + if (ok && record_st->gpu_bgrx) + recording_bgr24_to_bgrx(video_st->record_gpu_buffer, + video_st->record_gpu_buffer, + VIDEO_SCALE_AREA(record_st->gpu_dims)); + } performance_counter_stop_plus(perf, record_perf_readback); if (!ok) return; diff --git a/record/drivers/record_ffmpeg.c b/record/drivers/record_ffmpeg.c index 4e0d7bde7884..8d2da4368634 100644 --- a/record/drivers/record_ffmpeg.c +++ b/record/drivers/record_ffmpeg.c @@ -45,7 +45,6 @@ #endif #if defined(__linux__) -#include #include #include #include @@ -272,7 +271,6 @@ typedef struct ffmpeg * totals are read after the encoder thread has been joined. */ uint64_t video_frames_in; uint64_t video_frames_dropped; - uint64_t video_frames_logged; /* Frames dropped since the last queued one; rides along with the * next frame so the encoder leaves a pts gap for them. */ unsigned video_pts_pending; @@ -733,14 +731,18 @@ static bool ffmpeg_open_hw_h264(ffmpeg_t *handle, } /* x264's private options mean nothing to these. Ask for CBR where - * there's an "rc" knob; an encoder that rejects it fails to open - * and the next one is tried. */ + * there's an "rc" knob; an encoder without one leaves the option + * unused, and the stream is then held only by the generic rate + * limits. */ if (params->video_max_rate) av_dict_set(&opts, "rc", "cbr", 0); if (ffmpeg_open_video_codec(handle, codec, fmt, out_w, out_h, true, &opts)) { + if (av_dict_get(opts, "rc", NULL, 0)) + RARCH_WARN("[FFmpeg] %s has no rate control option: not CBR.\n", + codec->name); av_dict_free(&opts); *out_fmt = fmt; RARCH_LOG("[FFmpeg] Using hardware encoder %s (%s).\n", @@ -1307,16 +1309,12 @@ static bool ffmpeg_init_muxer_post(ffmpeg_t *handle) #define DEFAULT_VIDEO_FIFO_FRAMES 32 #define MIN_VIDEO_FIFO_FRAMES 8 #define MAX_VIDEO_FIFO_FRAMES 128 -/* One warning per this many newly dropped frames. */ -#define DROP_LOG_INTERVAL 64 -/* Reported at exit with perfcnt_enable. push_* run on the frontend's - * thread, the rest on the encoder thread; one writer each. */ +/* Reported at exit with perfcnt_enable, and readable live from the + * menu, which reads and resets them on the frontend's thread: so only + * the push_* side, which runs there. */ static struct retro_perf_counter ffmpeg_perf_push_video = {0}; static struct retro_perf_counter ffmpeg_perf_push_audio = {0}; -static struct retro_perf_counter ffmpeg_perf_scale = {0}; -static struct retro_perf_counter ffmpeg_perf_encode_video = {0}; -static struct retro_perf_counter ffmpeg_perf_encode_audio = {0}; static bool ffmpeg_perf_enabled(void) { @@ -1327,9 +1325,6 @@ static void ffmpeg_perf_init(void) { performance_counter_init(ffmpeg_perf_push_video, "record_push_video"); performance_counter_init(ffmpeg_perf_push_audio, "record_push_audio"); - performance_counter_init(ffmpeg_perf_scale, "record_scale"); - performance_counter_init(ffmpeg_perf_encode_video, "record_encode_video"); - performance_counter_init(ffmpeg_perf_encode_audio, "record_encode_audio"); } static void ffmpeg_thread(void *data); @@ -1337,74 +1332,38 @@ static void ffmpeg_thread(void *data); /* Encoder threads run this much nicer than the core, in drop mode * only: with blocking on, a starved encoder just stalls the game. */ #define ENCODER_NICE_DELTA 10 -#define MAX_TRACKED_TIDS 512 #ifdef FFMPEG_HAVE_THREAD_NICE -struct ff_tid_set -{ - pid_t tids[MAX_TRACKED_TIDS]; - unsigned count; -}; - -static void ffmpeg_tids_snapshot(struct ff_tid_set *set) -{ - struct dirent *ent; - DIR *dir = opendir("/proc/self/task"); - - set->count = 0; - if (!dir) - return; - while ((ent = readdir(dir)) && set->count < MAX_TRACKED_TIDS) - { - long tid = strtol(ent->d_name, NULL, 10); - if (tid > 0) - set->tids[set->count++] = (pid_t)tid; - } - closedir(dir); -} - -static void ffmpeg_renice_tid(pid_t tid, int delta) +/* Linux nice is per thread and a new thread takes its creator's, so a + * thread that lowers itself before it opens the encoders has every + * thread they spawn run lowered too, and nothing else. */ +static void ffmpeg_lower_this_thread(void) { int cur; + id_t tid = (id_t)syscall(SYS_gettid); errno = 0; - cur = getpriority(PRIO_PROCESS, (id_t)tid); + cur = getpriority(PRIO_PROCESS, tid); if (cur == -1 && errno) return; /* Raising nice never needs privileges. */ - setpriority(PRIO_PROCESS, (id_t)tid, cur + delta); + setpriority(PRIO_PROCESS, tid, cur + ENCODER_NICE_DELTA); } -/* Renice threads an encoder spawned during avcodec_open2(). They - * inherit the main thread's priority, and lowering that around the - * open can't be undone without CAP_SYS_NICE. */ -static unsigned ffmpeg_renice_new_threads(const struct ff_tid_set *before) +typedef struct { - unsigned i, j, n = 0; - struct ff_tid_set *after = (struct ff_tid_set*)malloc(sizeof(*after)); + ffmpeg_t *handle; + const char *audio_resampler; + bool ok; +} ffmpeg_open_job_t; - if (!after) - return 0; - ffmpeg_tids_snapshot(after); - for (i = 0; i < after->count; i++) - { - bool seen = false; - for (j = 0; j < before->count; j++) - { - if (after->tids[i] == before->tids[j]) - { - seen = true; - break; - } - } - if (!seen) - { - ffmpeg_renice_tid(after->tids[i], ENCODER_NICE_DELTA); - n++; - } - } - free(after); - return n; +static void ffmpeg_open_codecs_lowered(void *data) +{ + ffmpeg_open_job_t *job = (ffmpeg_open_job_t*)data; + ffmpeg_lower_this_thread(); + job->ok = ffmpeg_init_video(job->handle) + && ( !job->handle->config.audio_enable + || ffmpeg_init_audio(job->handle, job->audio_resampler)); } #endif @@ -1593,11 +1552,9 @@ static void ffmpeg_free(void *data) /* The muxer context owns its streams, their codec parameters, its * metadata and its url; releasing just the struct left the rest * behind on every session. */ - if ( handle->muxer.ctx - && handle->muxer.ctx->pb - && !(handle->muxer.ctx->oformat - && (handle->muxer.ctx->oformat->flags & AVFMT_NOFILE))) - avio_closep(&handle->muxer.ctx->pb); /* failed before finalize */ + /* Failed before finalize: the open ffmpeg_init_muxer_pre made */ + if (handle->muxer.ctx && handle->muxer.ctx->pb) + avio_closep(&handle->muxer.ctx->pb); avformat_free_context(handle->muxer.ctx); av_packet_free(&handle->pkt); @@ -1758,28 +1715,29 @@ static void *ffmpeg_new(const struct record_params *params) { bool ok; + bool opened = false; #ifdef FFMPEG_HAVE_THREAD_NICE - struct ff_tid_set *before = NULL; - if (handle->allow_frame_drop - && (before = (struct ff_tid_set*)malloc(sizeof(*before)))) - ffmpeg_tids_snapshot(before); -#endif - ok = ffmpeg_init_video(handle) - && ( !handle->config.audio_enable - || ffmpeg_init_audio(handle, params->audio_resampler)); -#ifdef FFMPEG_HAVE_THREAD_NICE - if (before) + /* In drop mode the encoders are opened on a thread that has + * lowered itself, so the threads they start run lowered. */ + if (handle->allow_frame_drop) { - if (ok) + ffmpeg_open_job_t job; + sthread_t *opener; + job.handle = handle; + job.audio_resampler = params->audio_resampler; + job.ok = false; + if ((opener = sthread_create(ffmpeg_open_codecs_lowered, &job))) { - unsigned n = ffmpeg_renice_new_threads(before); - if (n) - RARCH_LOG("[FFmpeg] Lowered priority of %u encoder" - " thread(s).\n", n); + sthread_join(opener); + ok = job.ok; + opened = true; } - free(before); } #endif + if (!opened) + ok = ffmpeg_init_video(handle) + && ( !handle->config.audio_enable + || ffmpeg_init_audio(handle, params->audio_resampler)); if (!ok) goto error; } @@ -1804,17 +1762,6 @@ static bool ffmpeg_video_fifo_has_room(ffmpeg_t *handle, size_t video_bytes) && retro_spsc_write_avail(&handle->video_fifo) >= video_bytes; } -static void ffmpeg_report_drops(uint64_t dropped) -{ - char msg[128]; - size_t _len = snprintf(msg, sizeof(msg), - "Recording: %llu frames dropped (encoder too slow)", - (unsigned long long)dropped); - RARCH_WARN("[FFmpeg] %s.\n", msg); - runloop_msg_queue_push(msg, _len, 1, 180, false, NULL, - MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_WARNING); -} - static void ffmpeg_audio_write_silence(ffmpeg_t *handle); /* Video frames in a row without audio before the core is taken to have @@ -1877,6 +1824,11 @@ static bool ffmpeg_push_video_impl(void *data, if (!handle || !vid) return false; + /* Every frame the core presents is a frame period of its time, + * whether or not frame_drop_ratio keeps it. */ + if (retro_atomic_load_acquire_int(&handle->alive)) + ffmpeg_fill_audio_gap(handle); + drop_frame = handle->video.frame_drop_count++ % handle->video.frame_drop_ratio; @@ -1906,21 +1858,15 @@ static bool ffmpeg_push_video_impl(void *data, rows = VIDEO_SCALE_H(attr.vid.dims); video_bytes = (size_t)rows * attr.vid.pitch; - ffmpeg_fill_audio_gap(handle); handle->video_frames_in++; /* Drop mode: a full queue costs this frame, never the frontend. */ if ( handle->allow_frame_drop && !ffmpeg_video_fifo_has_room(handle, video_bytes)) { + /* Counted, and said once when the recording ends */ handle->video_frames_dropped++; handle->video_pts_pending++; - if (handle->video_frames_dropped - handle->video_frames_logged - >= DROP_LOG_INTERVAL) - { - handle->video_frames_logged = handle->video_frames_dropped; - ffmpeg_report_drops(handle->video_frames_dropped); - } return true; } @@ -2208,7 +2154,6 @@ static bool ffmpeg_push_video_thread(ffmpeg_t *handle, { struct record_video_data rotated; bool ok; - bool perf = ffmpeg_perf_enabled(); if (handle->params.rotation && !vid->is_dupe && vid->data) { @@ -2234,9 +2179,7 @@ static bool ffmpeg_push_video_thread(ffmpeg_t *handle, if (!vid->is_dupe) { - performance_counter_start_plus(perf, ffmpeg_perf_scale); ffmpeg_scale_input(handle, vid); - performance_counter_stop_plus(perf, ffmpeg_perf_scale); } /* Leave a gap for dropped frames so A/V stays aligned. */ @@ -2260,9 +2203,7 @@ static bool ffmpeg_push_video_thread(ffmpeg_t *handle, } } - performance_counter_start_plus(perf, ffmpeg_perf_encode_video); ok = encode_video(handle, handle->video.conv_frame); - performance_counter_stop_plus(perf, ffmpeg_perf_encode_video); if (!ok) return false; @@ -2628,21 +2569,13 @@ static bool ffmpeg_push_audio_thread(ffmpeg_t *handle, return true; } -static void ffmpeg_flush_audio(ffmpeg_t *handle, void *audio_buf, - size_t audio_buf_size) +static void ffmpeg_flush_audio(ffmpeg_t *handle) { - size_t avail = retro_spsc_read_avail(&handle->audio_fifo); - - if (avail) + if ( handle->audio.frames_in_buffer + && encode_audio(handle, false)) { - struct record_audio_data aud = {0}; - - retro_spsc_read(&handle->audio_fifo, audio_buf, avail); - - aud.frames = avail / (sizeof(int16_t) * handle->params.channels); - aud.data = audio_buf; - - ffmpeg_push_audio_thread(handle, &aud, false); + handle->audio.frame_cnt += handle->audio.frames_in_buffer; + handle->audio.frames_in_buffer = 0; } encode_audio(handle, true); @@ -2704,6 +2637,22 @@ static void ffmpeg_flush_buffers(ffmpeg_t *handle) } }while (did_work); + /* The real audio left in the queue was queued ahead of any silence + * still owed: it goes first, held in the encoder's frame until the + * silence completes it. */ + if (handle->config.audio_enable && audio_buf) + { + size_t avail = retro_spsc_read_avail(&handle->audio_fifo); + if (avail) + { + struct record_audio_data aud = {0}; + retro_spsc_read(&handle->audio_fifo, audio_buf, avail); + aud.frames = avail / (sizeof(int16_t) * handle->params.channels); + aud.data = audio_buf; + ffmpeg_push_audio_thread(handle, &aud, true); + } + } + /* Pay back silence still owed, so audio runs as long as video. */ if (handle->config.audio_enable && audio_buf) { @@ -2719,11 +2668,9 @@ static void ffmpeg_flush_buffers(ffmpeg_t *handle) handle->audio_silence_pending = 0; } - /* Flush out last audio. Skip on OOM - audio_buf is the - * destination for ffmpeg_flush_audio's internal fifo_read - * (via ffmpeg_push_audio_thread) and NULL would NULL-deref. */ + /* Flush out last audio: a short final frame, then the encoder. */ if (handle->config.audio_enable && audio_buf) - ffmpeg_flush_audio(handle, audio_buf, audio_buf_size); + ffmpeg_flush_audio(handle); /* Flush out last video. */ encode_video(handle, NULL); @@ -2763,12 +2710,11 @@ static void ffmpeg_thread(void *data) size_t audio_buf_size = ff->config.audio_enable ? (ff->audio.codec->frame_size * ff->params.channels * sizeof(int16_t)) : 0; void *audio_buf = audio_buf_size ? av_malloc(audio_buf_size) : NULL; - bool perf = ffmpeg_perf_enabled(); #ifdef FFMPEG_HAVE_THREAD_NICE /* Also covers threads an encoder spawns lazily from here. */ if (ff->allow_frame_drop) - ffmpeg_renice_tid((pid_t)syscall(SYS_gettid), ENCODER_NICE_DELTA); + ffmpeg_lower_this_thread(); #endif while (retro_atomic_load_acquire_int(&ff->alive)) @@ -2819,9 +2765,7 @@ static void ffmpeg_thread(void *data) aud.frames = ff->audio.codec->frame_size; aud.data = audio_buf; - performance_counter_start_plus(perf, ffmpeg_perf_encode_audio); ffmpeg_push_audio_thread(ff, &aud, true); - performance_counter_stop_plus(perf, ffmpeg_perf_encode_audio); } } diff --git a/record/record_driver.c b/record/record_driver.c index 4d847764db62..e06633c425c9 100644 --- a/record/record_driver.c +++ b/record/record_driver.c @@ -417,25 +417,13 @@ bool recording_init(void) if (recording_state.out_dims) params.out_dims = recording_state.out_dims; - if (video_force_aspect && - (VIDEO_DRIVER_ASPECT_RATIO(video_st) > 0.0f)) - params.aspect_ratio = VIDEO_DRIVER_ASPECT_RATIO(video_st); - else - params.aspect_ratio = (float)VIDEO_SCALE_W(params.out_dims) - / VIDEO_SCALE_H(params.out_dims); - /* Raw core frames are unrotated; the GPU path reads back the * already-rotated screen. Rotate them like the display does. */ params.rotation = retroarch_get_rotation() % 4; - if (params.rotation & 1) - { - if (!recording_state.out_dims) - params.out_dims = VIDEO_SCALE_PACK( - VIDEO_SCALE_H(params.out_dims), - VIDEO_SCALE_W(params.out_dims)); - if (params.aspect_ratio > 0.0f) - params.aspect_ratio = 1.0f / params.aspect_ratio; - } + record_raw_geometry(¶ms.out_dims, ¶ms.aspect_ratio, + params.rotation, + video_force_aspect ? VIDEO_DRIVER_ASPECT_RATIO(video_st) : 0.0f, + recording_state.out_dims != 0); #ifdef HAVE_VIDEO_FILTER if (settings->bools.video_post_filter_record diff --git a/record/record_driver.h b/record/record_driver.h index 80476ee80791..187c24713732 100644 --- a/record/record_driver.h +++ b/record/record_driver.h @@ -4,6 +4,8 @@ #include #include +#include + #include "../gfx/video_defines.h" /* VIDEO_SCALE_PACK */ enum ffemu_pix_format @@ -72,6 +74,10 @@ struct record_params unsigned streaming_mode; /* Video queue depth in frames (driver clamps/defaults out-of-range). */ unsigned video_fifo_frames; + /* Quarter turns counter-clockwise to rotate incoming frames by. + * out_dims and aspect_ratio describe the rotated output, fb_dims + * the incoming frames. */ + unsigned rotation; /* Aspect ratio of input video. Parameters are passed to the muxer, * the video itself is not scaled. @@ -88,10 +94,6 @@ struct record_params bool allow_frame_drop; /* Prefer a hardware H.264 encoder for the built-in presets. */ bool hw_encoder; - /* Quarter turns counter-clockwise to rotate incoming frames by. - * out_dims and aspect_ratio describe the rotated output, fb_dims - * the incoming frames. */ - unsigned rotation; }; struct record_video_data @@ -122,6 +124,23 @@ typedef struct record_driver } record_driver_t; +/* A raw recording's output size and display aspect when the display + * shows the core's frames turned @rotation quarter turns. @out_dims + * holds the core's size, turned here, unless it is --size's + * (@user_dims), which is kept as given. @display_aspect is the + * viewport's, which is already the turned picture's; 0 lets the + * output size decide. */ +static INLINE void record_raw_geometry(unsigned *out_dims, float *aspect, + unsigned rotation, float display_aspect, bool user_dims) +{ + if ((rotation & 1) && !user_dims) + *out_dims = VIDEO_SCALE_PACK(VIDEO_SCALE_H(*out_dims), + VIDEO_SCALE_W(*out_dims)); + *aspect = (display_aspect > 0.0f) + ? display_aspect + : (float)VIDEO_SCALE_W(*out_dims) / VIDEO_SCALE_H(*out_dims); +} + struct recording { const record_driver_t *driver; diff --git a/samples/gfx/threaded_video/threaded_video_test.c b/samples/gfx/threaded_video/threaded_video_test.c index 4ea79c8f53d6..51b48e46c374 100644 --- a/samples/gfx/threaded_video/threaded_video_test.c +++ b/samples/gfx/threaded_video/threaded_video_test.c @@ -41,6 +41,7 @@ #include "../../../gfx/video_driver.h" #include "../../../gfx/gfx_instrument.h" #include "../../../gfx/video_thread_wrapper.h" +#include "../../../record/record_driver.h" #include "../../../gfx/font_driver.h" #include "../../../menu/menu_driver.h" #include "../../../menu/menu_setting.h" @@ -6549,6 +6550,134 @@ static void readback_once(bool threaded) run_frames(4); } +/* ------------------------------------------------------------------ */ +/* Lane: a GPU recording's frames keep the format it was opened for */ +/* A GPU recording opened unthreaded on a driver with the 32-bit */ +/* readback takes B,G,R,X frames. Threaded Video switched on during */ +/* it puts the wrapper's readback in charge, which hands over BGR24 */ +/* from a buffer of its own: those frames were pushed as 32-bit, */ +/* reading a third more than the wrapper's buffer holds and showing */ +/* garbage. The recorder must get 32-bit rows of its own buffer */ +/* either way, every row of them readable. */ +/* ------------------------------------------------------------------ */ + +static unsigned reclane_pushes; +static int reclane_pitch; +static const uint8_t *reclane_data; +static unsigned reclane_rows; +static unsigned reclane_sum; + +static bool reclane_push_video(void *data, + const struct record_video_data *v) +{ + unsigned y; + size_t row; + (void)data; + if (v->is_dupe || !v->data) + return true; + reclane_pushes++; + reclane_pitch = v->pitch; + reclane_data = (const uint8_t*)v->data; + /* Each row's last byte, as the encoder's copy reads every row */ + row = (size_t)(v->pitch < 0 ? -v->pitch : v->pitch); + for (y = 0; y < reclane_rows; y++) + reclane_sum += ((const uint8_t*)v->data)[(ptrdiff_t)y * v->pitch + + (ptrdiff_t)row - 1]; + return true; +} + +static bool reclane_push_audio(void *data, + const struct record_audio_data *a) +{ + (void)data; + (void)a; + return true; +} + +static const record_driver_t reclane_driver = { + NULL, NULL, reclane_push_video, reclane_push_audio, NULL, "reclane" +}; + +static void record_format_once(bool threaded) +{ + struct video_viewport vp; + video_driver_state_t *video_st = video_state_get_ptr(); + recording_state_t *rec_st = recording_state_get_ptr(); + const char *mode = threaded ? "threaded" : "unthreaded"; + recording_state_t saved; + uint8_t *saved_buf; + uint8_t *buf; + size_t area; + unsigned i; + + set_threaded_via_setting(threaded); + run_frames(4); + expect_wrapper(threaded, "record format lane"); + /* Frames the core makes are recorded; the menu's dupes are not */ + if (menu_is_up()) + command_event(CMD_EVENT_MENU_TOGGLE, NULL); + CHECK(!menu_is_up(), "record format lane (%s): menu still up", mode); + run_frames(2); + if (threaded) + video_thread_wait_idle(); + + memset(&vp, 0, sizeof(vp)); + video_driver_get_viewport_info(&vp); + area = VIDEO_SCALE_AREA(vp.dims); + CHECK(area > 0, "record format lane (%s): no viewport", mode); + if (!area || !(buf = (uint8_t*)malloc(area * 4))) + return; + + /* A recording opened for 32-bit frames, as on this driver unthreaded */ + saved = *rec_st; + saved_buf = video_st->record_gpu_buffer; + reclane_pushes = 0; + reclane_rows = VIDEO_SCALE_H(vp.dims); + rec_st->driver = &reclane_driver; + rec_st->gpu_dims = vp.dims; + rec_st->gpu_bgrx = true; + rec_st->enable = true; + video_st->record_gpu_buffer = buf; + rec_st->data = (void*)&reclane_driver; + + /* The wrapper's readback starts a few frames behind */ + for (i = 0; i < 32 && !reclane_pushes; i++) + run_frames(1); + + rec_st->data = NULL; + if (threaded) + video_thread_wait_idle(); + *rec_st = saved; + video_st->record_gpu_buffer = saved_buf; + + CHECK(reclane_pushes > 0, "record format lane (%s): no frame recorded", + mode); + if (reclane_pushes) + { + CHECK(reclane_pitch == (int)VIDEO_SCALE_W(vp.dims) * 4 + || reclane_pitch == -(int)VIDEO_SCALE_W(vp.dims) * 4, + "record format lane (%s): rows of %d bytes for a %u wide " + "32-bit recording", mode, reclane_pitch, VIDEO_SCALE_W(vp.dims)); + CHECK(reclane_data >= buf && reclane_data < buf + area * 4, + "record format lane (%s): the frame is not in the recording's " + "own buffer", mode); + } + free(buf); +} + +static void lane_record_format(void) +{ + unsigned had = failures; + + record_format_once(false); + record_format_once(true); + set_threaded_via_setting(false); + run_frames(2); + + if (failures == had) + fprintf(stderr, "[pass] gpu record format lane\n"); +} + static void lane_gpu_readback(void) { unsigned had = failures; @@ -7241,6 +7370,8 @@ int main(int argc, char *argv[]) lane_x11_wsi_connection(); if (real_driver() && !strcmp(getenv("HARNESS_VIDEO_DRIVER"), "vulkan")) lane_gpu_readback(); + if (real_driver() && !strcmp(getenv("HARNESS_VIDEO_DRIVER"), "vulkan")) + lane_record_format(); if (real_driver()) lane_x11_grabbed_mouse(); if ( real_driver() diff --git a/samples/record/ffmpeg_pipeline/ffmpeg_pipeline_test.c b/samples/record/ffmpeg_pipeline/ffmpeg_pipeline_test.c index fa15478b3a08..82922dc26463 100644 --- a/samples/record/ffmpeg_pipeline/ffmpeg_pipeline_test.c +++ b/samples/record/ffmpeg_pipeline/ffmpeg_pipeline_test.c @@ -15,7 +15,14 @@ * A second pass runs in drop mode (allow_frame_drop, the smallest * video queue): frames the encoder can't take in time are dropped, * so it checks the survivors come out in order, pixel-exact, at the - * timestamp of the frame they were pushed as. */ + * timestamp of the frame they were pushed as, that frames were in + * fact dropped, and that the frontend was told nothing while it + * recorded: no log line and no on-screen message from the pushes. + * On Linux, drop mode also has the encoder's own threads run at a + * lower priority than the thread that opened the recording. + * + * A geometry lane checks the output size and display aspect of a raw + * recording of a frame the display turns a quarter. */ #include #include @@ -29,6 +36,17 @@ #include #include +#if defined(__linux__) +#include +#include +#include +#include +#include +#endif + +extern unsigned stub_warn_calls; +extern unsigned stub_msg_calls; + #define W 160 #define H 120 #define FRAMES 240 @@ -108,6 +126,33 @@ static int frame_number(const AVFrame *f) return -1; } +#if defined(__linux__) +/* Threads of this process running at a higher nice value than the + * calling thread. */ +static unsigned threads_lowered(void) +{ + struct dirent *ent; + unsigned n = 0; + int me = getpriority(PRIO_PROCESS, (id_t)syscall(SYS_gettid)); + DIR *dir = opendir("/proc/self/task"); + if (!dir) + return 0; + while ((ent = readdir(dir))) + { + long tid = strtol(ent->d_name, NULL, 10); + int nice; + if (tid <= 0) + continue; + errno = 0; + nice = getpriority(PRIO_PROCESS, (id_t)tid); + if (!errno && nice > me) + n++; + } + closedir(dir); + return n; +} +#endif + static int run(bool drop) { struct record_params params; @@ -128,7 +173,8 @@ static int run(bool drop) params.channels = 2; params.video_record_scale_factor = 1; params.video_stream_scale_factor = 1; - params.video_record_threads = 1; + /* Drop mode with encoder threads of its own, to see them lowered */ + params.video_record_threads = drop ? 2 : 1; params.aspect_ratio = (float)W / H; params.preset = RECORD_CONFIG_TYPE_RECORDING_LOSSLESS_QUALITY; params.pix_fmt = FFEMU_PIX_ARGB8888; @@ -143,6 +189,16 @@ static int run(bool drop) return 1; } +#if defined(__linux__) + if (drop && !threads_lowered()) + { + fprintf(stderr, "drop mode: no encoder thread runs lowered\n"); + return 1; + } +#endif + + stub_warn_calls = 0; + stub_msg_calls = 0; memset(pcm, 0, sizeof(pcm)); for (n = 0; n < FRAMES; n++) { @@ -164,6 +220,12 @@ static int run(bool drop) return 1; } } + if (drop && (stub_warn_calls || stub_msg_calls)) + { + fprintf(stderr, "drop mode: %u warnings and %u on-screen messages" + " while recording\n", stub_warn_calls, stub_msg_calls); + return 1; + } record_ffmpeg.finalize(rec); record_ffmpeg.free(rec); free(fb); @@ -243,7 +305,7 @@ static int run(bool drop) if (flushing) break; } - if (drop ? got == 0 : got != FRAMES) + if (drop ? (got == 0 || got >= FRAMES) : got != FRAMES) { fprintf(stderr, "decoded %u frames, pushed %u\n", got, FRAMES); return 1; @@ -263,9 +325,59 @@ static int run(bool drop) return 0; } +/* A raw recording of frames the display turns a quarter: the output + * is turned, and its aspect is the display's, which is already the + * turned picture's - not that inverted again. */ +static int geometry(void) +{ + unsigned dims; + float aspect; + + /* 320x240 core frame, shown turned on a 3:4 display */ + dims = VIDEO_SCALE_PACK(320, 240); + record_raw_geometry(&dims, &aspect, 1, 0.75f, false); + if (dims != VIDEO_SCALE_PACK(240, 320) || aspect != 0.75f) + { + fprintf(stderr, "geometry: turned frame recorded %ux%u at %.3f," + " want 240x320 at 0.750\n", VIDEO_SCALE_W(dims), + VIDEO_SCALE_H(dims), aspect); + return 1; + } + /* No display aspect: the turned size decides */ + dims = VIDEO_SCALE_PACK(320, 240); + record_raw_geometry(&dims, &aspect, 3, 0.0f, false); + if (dims != VIDEO_SCALE_PACK(240, 320) || aspect != 0.75f) + { + fprintf(stderr, "geometry: turned size gives %ux%u at %.3f\n", + VIDEO_SCALE_W(dims), VIDEO_SCALE_H(dims), aspect); + return 1; + } + /* --size is kept as given, and its own shape is its aspect */ + dims = VIDEO_SCALE_PACK(640, 480); + record_raw_geometry(&dims, &aspect, 1, 0.0f, true); + if (dims != VIDEO_SCALE_PACK(640, 480) || aspect < 1.333f + || aspect > 1.334f) + { + fprintf(stderr, "geometry: --size became %ux%u at %.3f\n", + VIDEO_SCALE_W(dims), VIDEO_SCALE_H(dims), aspect); + return 1; + } + /* Unturned: as it was */ + dims = VIDEO_SCALE_PACK(320, 240); + record_raw_geometry(&dims, &aspect, 2, 1.25f, false); + if (dims != VIDEO_SCALE_PACK(320, 240) || aspect != 1.25f) + { + fprintf(stderr, "geometry: half turn gives %ux%u at %.3f\n", + VIDEO_SCALE_W(dims), VIDEO_SCALE_H(dims), aspect); + return 1; + } + printf("[pass] raw recording geometry under rotation\n"); + return 0; +} + int main(void) { - if (run(false) || run(true)) + if (geometry() || run(false) || run(true)) return 1; printf("ALL OK\n"); return 0; diff --git a/samples/record/ffmpeg_pipeline/stubs_retroarch.c b/samples/record/ffmpeg_pipeline/stubs_retroarch.c index cf3a7955fd23..91c83b377ea6 100644 --- a/samples/record/ffmpeg_pipeline/stubs_retroarch.c +++ b/samples/record/ffmpeg_pipeline/stubs_retroarch.c @@ -22,9 +22,15 @@ void RARCH_ERR(const char *fmt, ...) va_end(ap); } +/* What the frontend was told, counted: drop mode must say nothing + * while it records. */ +unsigned stub_warn_calls = 0; +unsigned stub_msg_calls = 0; + void RARCH_WARN(const char *fmt, ...) { va_list ap; + stub_warn_calls++; va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); @@ -35,6 +41,7 @@ void runloop_msg_queue_push(const char *msg, size_t len, char *title, enum message_queue_icon icon, enum message_queue_category category) { + stub_msg_calls++; } void RARCH_DBG(const char *fmt, ...)