From 82ee27c20e0b429c712298bcfe056313c50d6146 Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:01 +0200 Subject: [PATCH 1/4] ffmpeg core: don't hang on gaps in the video track Playing a file whose video track has a gap of more than about two seconds (frames dropped while recording, or any variable frame rate source) froze RetroArch at that point. The main thread waits for the next frame, and the decode thread only decodes a video packet once audio has caught up to it, so across the gap it decodes audio that nobody takes until the audio queue is full, then blocks too. Wait for the next frame as before, but give up once the decoder has a second or more of audio buffered ahead, which only happens across a gap: hold the frame on screen, keep playing audio, and pick the next frame up when the decoder reaches it. The interpolation factor is clamped, since a held frame would push it outside [0, 1]. ffmpeg_sweep_test plays a clip with a 4 s hole in its video track and checks audio and video both carry on past it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- cores/libretro-ffmpeg/ffmpeg_core.c | 69 ++++++++++++++++++- .../cores/ffmpeg_sweep/ffmpeg_sweep_test.c | 60 +++++++++++++++- 2 files changed, 124 insertions(+), 5 deletions(-) diff --git a/cores/libretro-ffmpeg/ffmpeg_core.c b/cores/libretro-ffmpeg/ffmpeg_core.c index e1a3229fa962..3bc81f054813 100644 --- a/cores/libretro-ffmpeg/ffmpeg_core.c +++ b/cores/libretro-ffmpeg/ffmpeg_core.c @@ -421,6 +421,42 @@ static bool video_buffer_wait_for_finished_slot(video_buffer_t *video_buffer, return VB_STATUS(video_buffer, VB_TAIL(video_buffer)) == KB_FINISHED; } +/* Audio decoded ahead of playback this deep means the next video frame + * lies across a gap in the video track (frames dropped while + * recording, variable frame rate). The audio queue is 2 s. */ +#define VIDEO_GAP_AUDIO_BACKLOG 1.0 + +static double audio_backlog_seconds(void); + +/* Main thread: waits for slot @index to be finished, like the wait + * above, but gives up across a gap in the video track. The decode + * thread only decodes a video packet once audio has caught up to it, + * so with this thread waiting and not taking audio it fills the audio + * queue and blocks too. The caller holds the frame on screen instead + * and keeps playing audio until the decoder reaches the next one. */ +static bool video_buffer_wait_for_frame(video_buffer_t *video_buffer, + unsigned index, retro_atomic_int_t *thread_dead) +{ + for (;;) + { + int key; + if (VB_STATUS(video_buffer, index) == KB_FINISHED) + return true; + if ( retro_atomic_load_acquire_int(thread_dead) + || audio_backlog_seconds() >= VIDEO_GAP_AUDIO_BACKLOG) + return false; + key = retro_eventcount_prepare_wait(&video_buffer->ec); + if ( VB_STATUS(video_buffer, index) == KB_FINISHED + || retro_atomic_load_acquire_int(thread_dead)) + { + retro_eventcount_cancel_wait(&video_buffer->ec); + continue; + } + /* Bounded: the audio backlog grows without a notify. */ + retro_eventcount_commit_wait_timeout(&video_buffer->ec, key, 10000); + } +} + /* Wakes every wait on the buffer to re-test: the decode thread is * gone, or a waiter has something else to look at. */ static void video_buffer_wake(video_buffer_t *video_buffer) @@ -2423,6 +2459,16 @@ static int seek_adjust(int target) return 0; } +static double audio_backlog_seconds(void) +{ + if ( AUDIO_STREAMS_NUM_STR <= 0 + || !g_ctx.audio_decode_fifo_init + || MEDIA_STR.sample_rate <= 0) + return 0.0; + return (double)retro_spsc_read_avail(&AUDIO_DECODE_FIFO_STR) + / (MEDIA_STR.sample_rate * sizeof(int16_t) * 2); +} + void CORE_PREFIX(retro_run)(void) { double min_pts; @@ -2662,9 +2708,11 @@ void CORE_PREFIX(retro_run)(void) if (!VIDEO_BUFFER_STR) break; - if (!DECODE_THREAD_DEAD_STR) - video_buffer_wait_for_finished_slot(VIDEO_BUFFER_STR, - &g_ctx.decode_thread_dead); + /* Gap in the video track: hold the current frame. */ + if ( !DECODE_THREAD_DEAD_STR + && !video_buffer_wait_for_frame(VIDEO_BUFFER_STR, + VB_TAIL(VIDEO_BUFFER_STR), &g_ctx.decode_thread_dead)) + break; if (!DECODE_THREAD_DEAD_STR) { @@ -2732,6 +2780,11 @@ void CORE_PREFIX(retro_run)(void) } mix_factor = (min_pts - FRAMES_STR[0].pts) / (FRAMES_STR[1].pts - FRAMES_STR[0].pts); + /* A frame held across a gap leaves min_pts past both. */ + if (!(mix_factor >= 0.0f)) + mix_factor = 0.0f; + else if (mix_factor > 1.0f) + mix_factor = 1.0f; if (!TEMPORAL_INTERPOLATION_STR) mix_factor = 1.0f; @@ -2786,6 +2839,16 @@ void CORE_PREFIX(retro_run)(void) if (!VIDEO_BUFFER_STR) break; + /* Gap in the video track: keep holding the frame on screen. + * The next one is in the slot after the held one. */ + if ( !DECODE_THREAD_DEAD_STR + && !video_buffer_wait_for_frame(VIDEO_BUFFER_STR, + g_ctx.held_slot + ? (g_ctx.held_slot->index + 1) % VIDEO_BUFFER_STR->capacity + : (unsigned)VB_TAIL(VIDEO_BUFFER_STR), + &g_ctx.decode_thread_dead)) + break; + /* The frame on screen is about to be replaced (or, when * several are due at once, skipped), so its slot can go * back. It has to: the tail does not move until it does. */ diff --git a/samples/cores/ffmpeg_sweep/ffmpeg_sweep_test.c b/samples/cores/ffmpeg_sweep/ffmpeg_sweep_test.c index 135136ec9120..46f3896132b5 100644 --- a/samples/cores/ffmpeg_sweep/ffmpeg_sweep_test.c +++ b/samples/cores/ffmpeg_sweep/ffmpeg_sweep_test.c @@ -32,7 +32,7 @@ static const char *current = ""; /* ---- the host ----------------------------------------------------- */ -static unsigned frames_seen, batches_seen; +static unsigned frames_seen, batches_seen, fresh_frames; static uint64_t audio_frames; static bool quiet_log = true; @@ -86,8 +86,10 @@ static bool env_cb(unsigned cmd, void *data) static void video_cb(const void *d, unsigned w, unsigned h, size_t p) { - (void)d; (void)w; (void)h; (void)p; + (void)w; (void)h; (void)p; frames_seen++; + if (d) + fresh_frames++; } static size_t audio_batch_cb(const int16_t *data, size_t frames) @@ -213,6 +215,58 @@ static void clip_case(const char *name, const char *path, const char *args, clips_run++; } +/* A video track with a hole in it, as a recording that dropped frames + * leaves: the core has to play across it, not wait for a frame that + * is seconds away while the audio queue fills behind it. */ +static void gap_case(const char *path) +{ + struct retro_game_info info; + struct retro_system_av_info av; + unsigned frames, fresh; + double fps; + + current = "video gap"; + if (!make_clip(path, + "-f lavfi -i testsrc=size=160x120:rate=30:duration=8 " + "-f lavfi -i sine=f=440:r=48000:duration=8 " + "-vf \"select='not(between(t,2,6))'\" -fps_mode passthrough " + "-c:v mpeg4 -c:a pcm_s16le")) + { + printf(" (ffmpeg would not make the gap clip; skipped)\n"); + return; + } + + memset(&info, 0, sizeof(info)); + info.path = path; + audio_frames = 0; + + retro_init(); + if (!retro_load_game(&info)) + { + CHECK(false, "the core refused the gap clip"); + retro_deinit(); + return; + } + memset(&av, 0, sizeof(av)); + retro_get_system_av_info(&av); + /* Through the gap (2 s to 6 s) and out the other side. */ + fps = av.timing.fps > 0.0 ? av.timing.fps : 30.0; + frames = (unsigned)(6.5 * fps); + alarm(60); + run_frames(frames); + fresh = fresh_frames; + run_frames((unsigned)fps); + alarm(600); + printf(" %-22s %.2f s of audio, %u new frames after the gap\n", + "video gap", (double)audio_frames / 48000.0, fresh_frames - fresh); + CHECK(audio_frames >= 7 * 48000, + "audio stopped at %.2f s, in the gap", + (double)audio_frames / 48000.0); + CHECK(fresh_frames > fresh, "no video after the gap"); + retro_unload_game(); + retro_deinit(); +} + /* A file that is not what it says, or is cut short. The core has to * refuse it without reading past the end of it. */ static void reject_case(const char *name, const char *path) @@ -307,6 +361,8 @@ int main(void) "-f lavfi -i sine=f=440:r=48000:duration=2 -c:v mpeg4 -c:a pcm_s16le", true, true); + gap_case("/tmp/sw_gap.mkv"); + printf(" files that are not what they claim\n"); /* A media file cut in half: the header parses, the stream does * not reach where the index says it does. */ From 9548fb427eae0f463cfbad378b908f7b8e7f9f34 Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:01 +0200 Subject: [PATCH 2/4] ffmpeg core: detect working decoders instead of taking the first avcodec_find_decoder() returns one decoder per codec id, whichever the build registered first, so which decoder played a file depended on registration order and on what that platform's FFmpeg includes. A platform's hardware video decoder (h264_nvv4l2, h264_v4l2m2m, h264_rkmpp, h264_cuvid, h264_mediacodec, ...) was only used if the build moved it to the front (the L4T FFmpeg patches do, to make every application use nvv4l2), and a default decoder that can't work there failed the file even when another decoder for the codec was built in. For every audio, video and subtitle stream, collect the non-experimental decoders for its codec and try them in order: 1. video only: platform hardware decoders (AV_CODEC_CAP_HARDWARE), unless disabled by the new core option 'Use platform hardware decoders' (ffmpeg_hw_wrappers, default enabled); 2. the default decoder (still libdav1d first for AV1), with the existing hwaccel set-up for video; 3. any other software decoder. For audio and video, open each and decode from the start until a frame comes out, then rewind; keep the first whose frame this core can use (video: a format swscale accepts, or the hwaccel format it set up, so a wrapper that only returns DRM-PRIME surfaces is skipped; audio: a sample format swresample converts). Subtitles, inputs that can't be rewound, and codecs with a single decoder take the first decoder that opens. The hwaccel configuration now comes from the decoder actually being opened. The chosen decoder is logged, and a missing one names the codec. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- cores/libretro-ffmpeg/ffmpeg_core.c | 267 ++++++++++++++++++++++++++-- 1 file changed, 257 insertions(+), 10 deletions(-) diff --git a/cores/libretro-ffmpeg/ffmpeg_core.c b/cores/libretro-ffmpeg/ffmpeg_core.c index 3bc81f054813..d5e823c5f7ef 100644 --- a/cores/libretro-ffmpeg/ffmpeg_core.c +++ b/cores/libretro-ffmpeg/ffmpeg_core.c @@ -4,6 +4,7 @@ #include #include #include +#include #ifdef RARCH_INTERNAL #ifdef HAVE_CONFIG_H @@ -23,6 +24,7 @@ extern "C" { #include #include #include +#include #ifdef HAVE_SWRESAMPLE #include #endif @@ -780,6 +782,9 @@ typedef struct ffmpeg_core_ctx enum AVPixelFormat hw_pix_fmt; bool force_sw_decoder; #endif + /* Try platform hardware decoders (h264_nvv4l2, h264_v4l2m2m, + * h264_mediacodec, ...) in decoder detection: ffmpeg_hw_wrappers. */ + bool use_hw_wrappers; /* Stream bookkeeping */ AVCodecContext *actx[MAX_STREAMS]; @@ -916,6 +921,7 @@ static ffmpeg_core_ctx_t g_ctx; #define HW_DECODER_STR (g_ctx.hw_decoder) #define HW_DECODING_ENABLED_STR (g_ctx.hw_decoding_enabled) #define FORCE_SW_DECODER_STR (g_ctx.force_sw_decoder) +#define USE_HW_WRAPPERS_STR (g_ctx.use_hw_wrappers) #endif #define ACTX_STR (g_ctx.actx) #define SCTX_STR (g_ctx.sctx) @@ -2017,6 +2023,9 @@ void CORE_PREFIX(retro_set_environment)(retro_environment_t cb) #if ENABLE_HW_ACCEL { "ffmpeg_hw_decoder", "Use Hardware decoder (restart); off|auto|" "cuda|d3d11va|drm|dxva2|mediacodec|opencl|qsv|vaapi|vdpau|videotoolbox" }, +#endif +#if !FFMPEG3 + { "ffmpeg_hw_wrappers", "Use platform hardware decoders (restart); enabled|disabled" }, #endif { "ffmpeg_sw_decoder_threads", "Software decoder thread count (restart); auto|1|2|4|6|8|10|12|14|16" }, #if defined(HAVE_OPENGL) || defined(HAVE_OPENGLES) @@ -2145,6 +2154,17 @@ static void check_variables(bool firststart) retro_atomic_store_release_int(&g_ctx.color_space, space); } + if (firststart) + { + struct retro_variable wrap_var = {0}; + wrap_var.key = "ffmpeg_hw_wrappers"; + USE_HW_WRAPPERS_STR = true; + if ( CORE_PREFIX(environ_cb)(RETRO_ENVIRONMENT_GET_VARIABLE, &wrap_var) + && wrap_var.value + && !strcmp(wrap_var.value, "disabled")) + USE_HW_WRAPPERS_STR = false; + } + #if ENABLE_HW_ACCEL if (firststart) { @@ -2974,7 +2994,7 @@ static enum AVPixelFormat init_hw_decoder(struct AVCodecContext *ctx, #endif int ret = 0; enum AVPixelFormat decoder_pix_fmt = AV_PIX_FMT_NONE; - const AVCodec *codec = retro_find_decoder(FCTX_STR->streams[VIDEO_STREAM_INDEX_STR]->codecpar->codec_id); + const AVCodec *codec = ctx->codec; #if !FFMPEG3 for (i = 0;; i++) @@ -3128,17 +3148,169 @@ static enum AVPixelFormat get_format(AVCodecContext *ctx, } #endif -static bool open_codec(AVCodecContext **ctx, enum AVMediaType type, unsigned index) +/* Decoder detection: try every decoder for a stream's codec - for + * video, platform hardware decoders first (ffmpeg_hw_wrappers), then + * the default one (with the hwaccel path), then other software ones - + * and keep the first that opens and, for audio and video, decodes a + * frame this core can use. avcodec_find_decoder() alone gives whichever + * the build registered first, which need not work here. */ +#define MAX_DECODER_CANDIDATES 16 +#define DECODER_PROBE_PACKETS 64 + +/* Can this core use the decoded frame? Video: a software format + * swscale accepts, or the hwaccel format select_decoder() set up. + * Audio: any packed or planar sample format swresample converts. */ +static bool frame_usable(const AVFrame *frame, enum AVMediaType type) { - int ret = 0; - const AVCodec *codec = retro_find_decoder(FCTX_STR->streams[index]->codecpar->codec_id); - if (!codec) + if (type == AVMEDIA_TYPE_AUDIO) + return frame->nb_samples > 0 + && av_get_bytes_per_sample((enum AVSampleFormat)frame->format) > 0; + { - log_cb(RETRO_LOG_ERROR, "[FFMPEG] Couldn't find suitable decoder\n"); + const AVPixFmtDescriptor *desc = + av_pix_fmt_desc_get((enum AVPixelFormat)frame->format); + + if (!desc) + return false; + if (desc->flags & AV_PIX_FMT_FLAG_HWACCEL) + { +#if ENABLE_HW_ACCEL + return HW_DECODING_ENABLED_STR && frame->format == g_ctx.hw_pix_fmt; +#else + return false; +#endif + } + return sws_isSupportedInput((enum AVPixelFormat)frame->format) > 0; + } +} + +static bool input_is_seekable(void) +{ + return FCTX_STR->pb + && (FCTX_STR->pb->seekable & AVIO_SEEKABLE_NORMAL) + && !(FCTX_STR->iformat->flags & AVFMT_NOFILE); +} + +/* Rewind the demuxer to the start after a probe. */ +static bool rewind_input(unsigned index) +{ + int64_t start = FCTX_STR->streams[index]->start_time; + if (start == AV_NOPTS_VALUE) + start = 0; + if (av_seek_frame(FCTX_STR, (int)index, start, AVSEEK_FLAG_BACKWARD) < 0) return false; + avformat_flush(FCTX_STR); + return true; +} + +/* Decode from the start of the stream until one frame comes out. True + * if that frame is usable. The caller rewinds. */ +static bool probe_decoder(AVCodecContext *ctx, enum AVMediaType type, + unsigned index) +{ + int packets = 0; + bool decoded = false; + bool usable = false; + AVPacket *pkt = av_packet_alloc(); + AVFrame *frame = av_frame_alloc(); + + if (!pkt || !frame) + goto end; + + while (!decoded && packets < DECODER_PROBE_PACKETS + && av_read_frame(FCTX_STR, pkt) >= 0) + { + if (pkt->stream_index == (int)index) + { + int ret = avcodec_send_packet(ctx, pkt); + packets++; + if (ret < 0 && ret != AVERROR(EAGAIN)) + { + av_packet_unref(pkt); + break; + } + if (avcodec_receive_frame(ctx, frame) == 0) + decoded = true; + } + av_packet_unref(pkt); + } + + /* Decoders with delay may hold the first frame until drained. */ + if (!decoded && packets) + { + avcodec_send_packet(ctx, NULL); + if (avcodec_receive_frame(ctx, frame) == 0) + decoded = true; + } + + usable = decoded && frame_usable(frame, type); + if (decoded && !usable) + log_cb(RETRO_LOG_INFO, "[FFMPEG] %s decodes to %s, which this core can't use.\n", + ctx->codec->name, type == AVMEDIA_TYPE_AUDIO + ? av_get_sample_fmt_name((enum AVSampleFormat)frame->format) + : av_get_pix_fmt_name((enum AVPixelFormat)frame->format)); + +end: + av_frame_free(&frame); + av_packet_free(&pkt); + avcodec_flush_buffers(ctx); + return usable; +} + +static int decoder_candidates(enum AVCodecID id, enum AVMediaType type, + const AVCodec **list, int max) +{ + int n = 0; + const AVCodec *def = retro_find_decoder(id); +#if !FFMPEG3 + int pass; + bool allow_hw = (type == AVMEDIA_TYPE_VIDEO) && USE_HW_WRAPPERS_STR; +#endif + +#if FFMPEG3 + if (def) + list[n++] = def; +#else + /* 0: platform hardware decoders (video only), + * 1: the default decoder, 2: any other software decoder. */ + for (pass = 0; pass < 3 && n < max; pass++) + { + void *it = NULL; + const AVCodec *c = NULL; + + if (pass == 1) + { + if (def && !(def->capabilities & AV_CODEC_CAP_HARDWARE)) + list[n++] = def; + continue; + } + + while ((c = av_codec_iterate(&it)) && n < max) + { + bool hw = (c->capabilities & AV_CODEC_CAP_HARDWARE) != 0; + if (c->id != id || !av_codec_is_decoder(c)) + continue; + if (c->capabilities & AV_CODEC_CAP_EXPERIMENTAL) + continue; + if (pass == 0 ? !(hw && allow_hw) : (hw || c == def)) + continue; + list[n++] = c; + } } +#endif + return n; +} + +/* Allocate and open ctx for codec. Video gets the hwaccel set-up for + * non-wrapper decoders, exactly as before. */ +static bool open_codec_with(AVCodecContext **ctx, const AVCodec *codec, + enum AVMediaType type, unsigned index) +{ + int ret = 0; *ctx = avcodec_alloc_context3(codec); + if (!*ctx) + return false; avcodec_parameters_to_context((*ctx), FCTX_STR->streams[index]->codecpar); if (type == AVMEDIA_TYPE_VIDEO) @@ -3146,8 +3318,20 @@ static bool open_codec(AVCodecContext **ctx, enum AVMediaType type, unsigned ind VIDEO_STREAM_INDEX_STR = index; #if ENABLE_HW_ACCEL - VCTX_STR->get_format = get_format; - g_ctx.hw_pix_fmt = select_decoder((*ctx), NULL); + g_ctx.hw_pix_fmt = AV_PIX_FMT_NONE; + HW_DECODING_ENABLED_STR = false; + if (codec->capabilities & AV_CODEC_CAP_HARDWARE) + { + /* A hardware wrapper decodes by itself into frames it hands + * back; no hwaccel device or get_format() hook. */ + log_cb(RETRO_LOG_INFO, "[FFMPEG] Trying hardware decoder %s.\n", + codec->name); + } + else + { + (*ctx)->get_format = get_format; + g_ctx.hw_pix_fmt = select_decoder((*ctx), NULL); + } #else select_decoder((*ctx), NULL); #endif @@ -3156,13 +3340,76 @@ static bool open_codec(AVCodecContext **ctx, enum AVMediaType type, unsigned ind if ((ret = avcodec_open2(*ctx, codec, NULL)) < 0) { #ifdef __cplusplus - log_cb(RETRO_LOG_ERROR, "[FFMPEG] Could not open codec: %d\n", ret); + log_cb(RETRO_LOG_ERROR, "[FFMPEG] Could not open codec %s: %d\n", + codec->name, ret); #else - log_cb(RETRO_LOG_ERROR, "[FFMPEG] Could not open codec: %s\n", av_err2str(ret)); + log_cb(RETRO_LOG_ERROR, "[FFMPEG] Could not open codec %s: %s\n", + codec->name, av_err2str(ret)); #endif + avcodec_free_context(ctx); + return false; + } + + return true; +} + +static bool open_codec(AVCodecContext **ctx, enum AVMediaType type, unsigned index) +{ + const AVCodec *list[MAX_DECODER_CANDIDATES]; + enum AVCodecID id = FCTX_STR->streams[index]->codecpar->codec_id; + int i, n; + bool probe; + + *ctx = NULL; + n = decoder_candidates(id, type, list, MAX_DECODER_CANDIDATES); + if (!n) + { + log_cb(RETRO_LOG_ERROR, "[FFMPEG] This FFmpeg build has no decoder for %s (%s).\n", + avcodec_get_name(id), av_get_media_type_string(type)); + return false; + } + + /* Probing reads packets, so the input must be rewindable; otherwise + * take the first decoder that opens, as before. With one candidate, + * or for subtitles, there is nothing worth probing. */ + probe = n > 1 + && (type == AVMEDIA_TYPE_VIDEO || type == AVMEDIA_TYPE_AUDIO) + && input_is_seekable(); + + for (i = 0; i < n; i++) + { + if (!open_codec_with(ctx, list[i], type, index)) + continue; + + if (!probe) + break; + + if (probe_decoder(*ctx, type, index)) + { + if (!rewind_input(index)) + log_cb(RETRO_LOG_WARN, "[FFMPEG] Could not rewind after probing the decoder.\n"); + break; + } + + log_cb(RETRO_LOG_INFO, "[FFMPEG] Decoder %s failed its probe, trying the next.\n", + list[i]->name); + avcodec_free_context(ctx); + if (!rewind_input(index)) + { + log_cb(RETRO_LOG_ERROR, "[FFMPEG] Could not rewind after probing the decoder.\n"); + return false; + } + } + + if (!*ctx) + { + log_cb(RETRO_LOG_ERROR, "[FFMPEG] No working decoder for %s (%s).\n", + avcodec_get_name(id), av_get_media_type_string(type)); return false; } + log_cb(RETRO_LOG_INFO, "[FFMPEG] Using %s decoder %s.\n", + av_get_media_type_string(type), (*ctx)->codec->name); return true; } From bc201fb349e44b3154177b822e476fcea6f4ee5d Mon Sep 17 00:00:00 2001 From: gavin_darkglider Date: Thu, 8 Oct 2026 17:28:01 +0200 Subject: [PATCH 3/4] ffmpeg core: claim more media formats, but only those the build can open The player claimed a fixed list of 28 extensions, and the frontend's media type table only routed those to it, so common files (.ogv, .mka, .opus, .aac, .ac3, .aiff, .wv, .ape, .mts, ...) didn't open with the built-in player at all. - The core keeps its list and adds 31 more extensions, each only if this build has the demuxer and, where the extension implies a codec (.ogv Theora, .ape APE, .wv WavPack, .ac3 AC-3, ...), the decoder. RetroArch's platforms ship very different FFmpeg builds, so valid_extensions is built once, at runtime. - libretro_ffmpeg_ffmpeg_core_supports_extension() exposes the same check; msg_hash.c lists the extra extensions as MOVIE_OTHER / MUSIC_OTHER (after everything else, so existing claims win) and path_is_media_type() only routes them to the player when it passes. MPV builds route them unconditionally. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tpe3orhELpViSQpZcsTTjr --- cores/internal_cores.h | 4 ++ cores/libretro-ffmpeg/ffmpeg_core.c | 101 +++++++++++++++++++++++++++- msg_hash.c | 34 ++++++++++ msg_hash.h | 4 ++ retroarch.c | 12 ++++ 5 files changed, 154 insertions(+), 1 deletion(-) diff --git a/cores/internal_cores.h b/cores/internal_cores.h index 7a886462e9b0..ce615719c811 100644 --- a/cores/internal_cores.h +++ b/cores/internal_cores.h @@ -91,6 +91,10 @@ unsigned libretro_ffmpeg_retro_api_version(void); void libretro_ffmpeg_retro_get_system_info(struct retro_system_info *info); +/* True if this build's FFmpeg can open files with this (lowercase) + * extension; extensions the core always claimed are always true. */ +bool libretro_ffmpeg_ffmpeg_core_supports_extension(const char *ext); + void libretro_ffmpeg_retro_get_system_av_info(struct retro_system_av_info *info); void libretro_ffmpeg_retro_set_environment(retro_environment_t cb); diff --git a/cores/libretro-ffmpeg/ffmpeg_core.c b/cores/libretro-ffmpeg/ffmpeg_core.c index d5e823c5f7ef..efa1179a7846 100644 --- a/cores/libretro-ffmpeg/ffmpeg_core.c +++ b/cores/libretro-ffmpeg/ffmpeg_core.c @@ -1983,13 +1983,112 @@ void CORE_PREFIX(retro_set_controller_port_device)(unsigned port, unsigned devic (void)device; } +/* Formats this core has always claimed. */ +#define FFMPEG_CORE_BASE_EXTENSIONS \ + "mkv|avi|f4v|f4f|3gp|ogm|flv|mp4|mp3|flac|ogg|m4a|webm|3g2|mov|wmv|mpg|mpeg|vob|asf|divx|m2p|m2ts|ps|ts|mxf|wma|wav" + +/* Further formats, claimed only when this FFmpeg build can open them: + * RetroArch runs on platforms whose FFmpeg builds differ widely, so + * each needs its demuxer and, where the format implies one, its + * decoder. Keep in sync with the media types in msg_hash.c. */ +static const struct +{ + const char *ext; + const char *demuxer; + enum AVCodecID codec; /* AV_CODEC_ID_NONE: container only */ +} ffmpeg_core_extra_formats[] = { + { "ogv", "ogg", AV_CODEC_ID_THEORA }, + { "oga", "ogg", AV_CODEC_ID_VORBIS }, + { "opus", "ogg", AV_CODEC_ID_OPUS }, + { "spx", "ogg", AV_CODEC_ID_SPEEX }, + { "mka", "matroska", AV_CODEC_ID_NONE }, + { "mk3d", "matroska", AV_CODEC_ID_NONE }, + { "weba", "matroska", AV_CODEC_ID_NONE }, + { "m4v", "mov", AV_CODEC_ID_NONE }, + { "m4b", "mov", AV_CODEC_ID_AAC }, + { "aac", "aac", AV_CODEC_ID_AAC }, + { "ac3", "ac3", AV_CODEC_ID_AC3 }, + { "eac3", "eac3", AV_CODEC_ID_EAC3 }, + { "dts", "dts", AV_CODEC_ID_DTS }, + { "mts", "mpegts", AV_CODEC_ID_NONE }, + { "m2t", "mpegts", AV_CODEC_ID_NONE }, + { "m2v", "mpegvideo", AV_CODEC_ID_MPEG2VIDEO }, + { "mp2", "mp3", AV_CODEC_ID_MP2 }, + { "aif", "aiff", AV_CODEC_ID_NONE }, + { "aiff", "aiff", AV_CODEC_ID_NONE }, + { "wv", "wv", AV_CODEC_ID_WAVPACK }, + { "ape", "ape", AV_CODEC_ID_APE }, + { "tta", "tta", AV_CODEC_ID_TTA }, + { "mpc", "mpc", AV_CODEC_ID_MUSEPACK7 }, + { "amr", "amr", AV_CODEC_ID_AMR_NB }, + { "au", "au", AV_CODEC_ID_NONE }, + { "caf", "caf", AV_CODEC_ID_NONE }, + { "rm", "rm", AV_CODEC_ID_NONE }, + { "rmvb", "rm", AV_CODEC_ID_NONE }, + { "nut", "nut", AV_CODEC_ID_NONE }, + { "dv", "dv", AV_CODEC_ID_DVVIDEO }, + { "ivf", "ivf", AV_CODEC_ID_NONE }, +}; + +static bool ffmpeg_core_extra_format_ok(unsigned i) +{ + if (!av_find_input_format(ffmpeg_core_extra_formats[i].demuxer)) + return false; + if ( ffmpeg_core_extra_formats[i].codec != AV_CODEC_ID_NONE + && !avcodec_find_decoder(ffmpeg_core_extra_formats[i].codec)) + return false; + return true; +} + +/* Exported for the frontend's media type detection (msg_hash.c lists + * these extensions as movie/music): true if this build can open ext. + * Base formats are always claimed. */ +/* Appends s to the extension list in buf, if it fits. */ +static size_t ffmpeg_core_append(char *buf, size_t len, size_t size, + const char *s) +{ + size_t n = strlen(s); + if (len + n + 1 > size) + return len; + memcpy(buf + len, s, n + 1); + return len + n; +} + +bool CORE_PREFIX(ffmpeg_core_supports_extension)(const char *ext) +{ + unsigned i; + for (i = 0; i < sizeof(ffmpeg_core_extra_formats) + / sizeof(ffmpeg_core_extra_formats[0]); i++) + if (!strcmp(ext, ffmpeg_core_extra_formats[i].ext)) + return ffmpeg_core_extra_format_ok(i); + return true; +} + void CORE_PREFIX(retro_get_system_info)(struct retro_system_info *info) { + static char extensions[512]; + + if (!*extensions) + { + unsigned i; + size_t _len = ffmpeg_core_append(extensions, 0, sizeof(extensions), + FFMPEG_CORE_BASE_EXTENSIONS); + for (i = 0; i < sizeof(ffmpeg_core_extra_formats) + / sizeof(ffmpeg_core_extra_formats[0]); i++) + { + if (!ffmpeg_core_extra_format_ok(i)) + continue; + _len = ffmpeg_core_append(extensions, _len, sizeof(extensions), "|"); + _len = ffmpeg_core_append(extensions, _len, sizeof(extensions), + ffmpeg_core_extra_formats[i].ext); + } + } + memset(info, 0, sizeof(*info)); info->library_name = "FFmpeg"; info->library_version = "v1"; info->need_fullpath = true; - info->valid_extensions = "mkv|avi|f4v|f4f|3gp|ogm|flv|mp4|mp3|flac|ogg|m4a|webm|3g2|mov|wmv|mpg|mpeg|vob|asf|divx|m2p|m2ts|ps|ts|mxf|wma|wav"; + info->valid_extensions = extensions; } void CORE_PREFIX(retro_get_system_av_info)(struct retro_system_av_info *info) diff --git a/msg_hash.c b/msg_hash.c index 85ec33e34002..0fbf256d0e84 100644 --- a/msg_hash.c +++ b/msg_hash.c @@ -692,6 +692,40 @@ enum msg_file_type msg_hash_to_file_type(const char *value) { "weba", FILE_TYPE_WEBA }, #endif #endif +#if defined(HAVE_FFMPEG) || defined(HAVE_MPV) + /* Further media formats; with FFmpeg, only routed to the player + * if its build can open them. Extensions claimed above win. */ + { "ogv", FILE_TYPE_MOVIE_OTHER }, + { "mk3d", FILE_TYPE_MOVIE_OTHER }, + { "mts", FILE_TYPE_MOVIE_OTHER }, + { "m2t", FILE_TYPE_MOVIE_OTHER }, + { "m2v", FILE_TYPE_MOVIE_OTHER }, + { "rm", FILE_TYPE_MOVIE_OTHER }, + { "rmvb", FILE_TYPE_MOVIE_OTHER }, + { "nut", FILE_TYPE_MOVIE_OTHER }, + { "dv", FILE_TYPE_MOVIE_OTHER }, + { "ivf", FILE_TYPE_MOVIE_OTHER }, + { "oga", FILE_TYPE_MUSIC_OTHER }, + { "opus", FILE_TYPE_MUSIC_OTHER }, + { "spx", FILE_TYPE_MUSIC_OTHER }, + { "mka", FILE_TYPE_MUSIC_OTHER }, + { "weba", FILE_TYPE_MUSIC_OTHER }, + { "m4b", FILE_TYPE_MUSIC_OTHER }, + { "aac", FILE_TYPE_MUSIC_OTHER }, + { "ac3", FILE_TYPE_MUSIC_OTHER }, + { "eac3", FILE_TYPE_MUSIC_OTHER }, + { "dts", FILE_TYPE_MUSIC_OTHER }, + { "mp2", FILE_TYPE_MUSIC_OTHER }, + { "aif", FILE_TYPE_MUSIC_OTHER }, + { "aiff", FILE_TYPE_MUSIC_OTHER }, + { "wv", FILE_TYPE_MUSIC_OTHER }, + { "ape", FILE_TYPE_MUSIC_OTHER }, + { "tta", FILE_TYPE_MUSIC_OTHER }, + { "mpc", FILE_TYPE_MUSIC_OTHER }, + { "amr", FILE_TYPE_MUSIC_OTHER }, + { "au", FILE_TYPE_MUSIC_OTHER }, + { "caf", FILE_TYPE_MUSIC_OTHER }, +#endif #ifdef HAVE_IMAGEVIEWER { "jpg", FILE_TYPE_JPEG }, { "JPG", FILE_TYPE_JPEG }, diff --git a/msg_hash.h b/msg_hash.h index 8ddf15d362b0..a6d218f06a7a 100644 --- a/msg_hash.h +++ b/msg_hash.h @@ -218,6 +218,10 @@ enum msg_file_type FILE_TYPE_PS, FILE_TYPE_TS, FILE_TYPE_MXF, + /* Further formats the built-in media player claims when its build + * can open them (see ffmpeg_core_supports_extension()). */ + FILE_TYPE_MOVIE_OTHER, + FILE_TYPE_MUSIC_OTHER, FILE_TYPE_JPEG, FILE_TYPE_PNG, diff --git a/retroarch.c b/retroarch.c index 93d375b3349a..b17b200e6594 100644 --- a/retroarch.c +++ b/retroarch.c @@ -237,6 +237,9 @@ #include "version_git.h" #include "retroarch.h" +#ifdef HAVE_FFMPEG +#include "cores/internal_cores.h" +#endif #include "accessibility.h" @@ -3118,6 +3121,15 @@ enum rarch_content_type path_is_media_type(const char *path) case FILE_TYPE_TS: case FILE_TYPE_MXF: return RARCH_CONTENT_MOVIE; + case FILE_TYPE_MOVIE_OTHER: + case FILE_TYPE_MUSIC_OTHER: +#ifdef HAVE_FFMPEG + /* Only if this build's FFmpeg can open it. */ + if (!libretro_ffmpeg_ffmpeg_core_supports_extension(ext_lower)) + return RARCH_CONTENT_NONE; +#endif + return (msg_hash_to_file_type(ext_lower) == FILE_TYPE_MOVIE_OTHER) + ? RARCH_CONTENT_MOVIE : RARCH_CONTENT_MUSIC; case FILE_TYPE_WMA: case FILE_TYPE_M4A: #endif From 15c7f15c1741c4b9d57f085d5d969513c87baf54 Mon Sep 17 00:00:00 2001 From: LibretroAdmin Date: Thu, 8 Oct 2026 18:15:02 +0200 Subject: [PATCH 4/4] ffmpeg core: an unload frees its codec contexts and a hardware device has one owner The codec contexts are freed at unload whatever the FFmpeg version, not only closed, and a hardware device is handed to the codec context that uses it rather than referenced from it, so freeing that context closes the device - every decoder the probing opens and drops included. ffmpeg_sweep_test runs its ASan pass with leak detection: every load and unload in it must give back what it took. --- .github/workflows/Linux-samples-cores.yml | 11 ++++++----- cores/libretro-ffmpeg/ffmpeg_core.c | 23 +++-------------------- 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/.github/workflows/Linux-samples-cores.yml b/.github/workflows/Linux-samples-cores.yml index 4be4aebe7a1c..03e9fdd28acc 100644 --- a/.github/workflows/Linux-samples-cores.yml +++ b/.github/workflows/Linux-samples-cores.yml @@ -165,15 +165,16 @@ jobs: # empty or absent, then teardown while the decode thread is # still working. # - # Leak detection off: libavutil's HW-device probing keeps a - # few objects, as the audio-pipe lane already has it. The - # libav* TSan suppressions are the same ones and for the - # same reason. + # With leak detection: every load and unload in it, the + # decoder probing included, must give back its codec + # contexts and hardware devices. The libav* TSan + # suppressions are the audio-pipe lane's, for the same + # reason. make clean all ./ffmpeg_sweep_test echo "[pass] ffmpeg_sweep_test" make clean all SANITIZER=address,undefined - ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=1 \ + ASAN_OPTIONS=detect_leaks=1:detect_stack_use_after_return=1 \ UBSAN_OPTIONS=print_stacktrace=1 timeout 900 ./ffmpeg_sweep_test echo "[pass] ffmpeg_sweep_test (ASan)" make clean all SANITIZER=thread diff --git a/cores/libretro-ffmpeg/ffmpeg_core.c b/cores/libretro-ffmpeg/ffmpeg_core.c index efa1179a7846..66df446d887e 100644 --- a/cores/libretro-ffmpeg/ffmpeg_core.c +++ b/cores/libretro-ffmpeg/ffmpeg_core.c @@ -625,9 +625,6 @@ static retro_input_state_t CORE_PREFIX(input_state_cb); #define FFMPEG3 ((LIBAVUTIL_VERSION_INT < AV_VERSION_INT(56, 6, 100)) || \ (LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 10, 100))) #endif -#ifndef FFMPEG8 -#define FFMPEG8 (LIBAVCODEC_VERSION_MAJOR >= 62) -#endif #define MAX_STREAMS 8 /* Sentinel return value indicating no frame available (EAGAIN/EOF) */ @@ -3156,7 +3153,7 @@ static enum AVPixelFormat init_hw_decoder(struct AVCodecContext *ctx, decoder_pix_fmt = AV_PIX_FMT_NONE; } else - ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); + ctx->hw_device_ctx = hw_device_ctx; /* the context owns it */ } return decoder_pix_fmt; @@ -4532,32 +4529,18 @@ void CORE_PREFIX(retro_unload_game)(void) g_ctx.decoded_frame_cnt = 0; AUDIO_FRAMES_STR = 0; + /* Freed, not only closed: avcodec_close() leaves the context + * allocated. */ for (i = 0; i < MAX_STREAMS; i++) { -#if FFMPEG8 if (SCTX_STR[i]) avcodec_free_context(&SCTX_STR[i]); if (ACTX_STR[i]) avcodec_free_context(&ACTX_STR[i]); -#else - if (SCTX_STR[i]) - avcodec_close(SCTX_STR[i]); - if (ACTX_STR[i]) - avcodec_close(ACTX_STR[i]); -#endif - SCTX_STR[i] = NULL; - ACTX_STR[i] = NULL; } if (VCTX_STR) - { -#if FFMPEG8 avcodec_free_context(&VCTX_STR); -#else - avcodec_close(VCTX_STR); -#endif - VCTX_STR = NULL; - } if (FCTX_STR) {