diff --git a/.github/workflows/Linux-samples-cores.yml b/.github/workflows/Linux-samples-cores.yml index 4be4aebe7a1c..03e9fdd28acc 100644 --- a/.github/workflows/Linux-samples-cores.yml +++ b/.github/workflows/Linux-samples-cores.yml @@ -165,15 +165,16 @@ jobs: # empty or absent, then teardown while the decode thread is # still working. # - # Leak detection off: libavutil's HW-device probing keeps a - # few objects, as the audio-pipe lane already has it. The - # libav* TSan suppressions are the same ones and for the - # same reason. + # With leak detection: every load and unload in it, the + # decoder probing included, must give back its codec + # contexts and hardware devices. The libav* TSan + # suppressions are the audio-pipe lane's, for the same + # reason. make clean all ./ffmpeg_sweep_test echo "[pass] ffmpeg_sweep_test" make clean all SANITIZER=address,undefined - ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=1 \ + ASAN_OPTIONS=detect_leaks=1:detect_stack_use_after_return=1 \ UBSAN_OPTIONS=print_stacktrace=1 timeout 900 ./ffmpeg_sweep_test echo "[pass] ffmpeg_sweep_test (ASan)" make clean all SANITIZER=thread diff --git a/cores/internal_cores.h b/cores/internal_cores.h index 7a886462e9b0..ce615719c811 100644 --- a/cores/internal_cores.h +++ b/cores/internal_cores.h @@ -91,6 +91,10 @@ unsigned libretro_ffmpeg_retro_api_version(void); void libretro_ffmpeg_retro_get_system_info(struct retro_system_info *info); +/* True if this build's FFmpeg can open files with this (lowercase) + * extension; extensions the core always claimed are always true. */ +bool libretro_ffmpeg_ffmpeg_core_supports_extension(const char *ext); + void libretro_ffmpeg_retro_get_system_av_info(struct retro_system_av_info *info); void libretro_ffmpeg_retro_set_environment(retro_environment_t cb); diff --git a/cores/libretro-ffmpeg/ffmpeg_core.c b/cores/libretro-ffmpeg/ffmpeg_core.c index e1a3229fa962..66df446d887e 100644 --- a/cores/libretro-ffmpeg/ffmpeg_core.c +++ b/cores/libretro-ffmpeg/ffmpeg_core.c @@ -4,6 +4,7 @@ #include #include #include +#include #ifdef RARCH_INTERNAL #ifdef HAVE_CONFIG_H @@ -23,6 +24,7 @@ extern "C" { #include #include #include +#include #ifdef HAVE_SWRESAMPLE #include #endif @@ -421,6 +423,42 @@ static bool video_buffer_wait_for_finished_slot(video_buffer_t *video_buffer, return VB_STATUS(video_buffer, VB_TAIL(video_buffer)) == KB_FINISHED; } +/* Audio decoded ahead of playback this deep means the next video frame + * lies across a gap in the video track (frames dropped while + * recording, variable frame rate). The audio queue is 2 s. */ +#define VIDEO_GAP_AUDIO_BACKLOG 1.0 + +static double audio_backlog_seconds(void); + +/* Main thread: waits for slot @index to be finished, like the wait + * above, but gives up across a gap in the video track. The decode + * thread only decodes a video packet once audio has caught up to it, + * so with this thread waiting and not taking audio it fills the audio + * queue and blocks too. The caller holds the frame on screen instead + * and keeps playing audio until the decoder reaches the next one. */ +static bool video_buffer_wait_for_frame(video_buffer_t *video_buffer, + unsigned index, retro_atomic_int_t *thread_dead) +{ + for (;;) + { + int key; + if (VB_STATUS(video_buffer, index) == KB_FINISHED) + return true; + if ( retro_atomic_load_acquire_int(thread_dead) + || audio_backlog_seconds() >= VIDEO_GAP_AUDIO_BACKLOG) + return false; + key = retro_eventcount_prepare_wait(&video_buffer->ec); + if ( VB_STATUS(video_buffer, index) == KB_FINISHED + || retro_atomic_load_acquire_int(thread_dead)) + { + retro_eventcount_cancel_wait(&video_buffer->ec); + continue; + } + /* Bounded: the audio backlog grows without a notify. */ + retro_eventcount_commit_wait_timeout(&video_buffer->ec, key, 10000); + } +} + /* Wakes every wait on the buffer to re-test: the decode thread is * gone, or a waiter has something else to look at. */ static void video_buffer_wake(video_buffer_t *video_buffer) @@ -587,9 +625,6 @@ static retro_input_state_t CORE_PREFIX(input_state_cb); #define FFMPEG3 ((LIBAVUTIL_VERSION_INT < AV_VERSION_INT(56, 6, 100)) || \ (LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 10, 100))) #endif -#ifndef FFMPEG8 -#define FFMPEG8 (LIBAVCODEC_VERSION_MAJOR >= 62) -#endif #define MAX_STREAMS 8 /* Sentinel return value indicating no frame available (EAGAIN/EOF) */ @@ -744,6 +779,9 @@ typedef struct ffmpeg_core_ctx enum AVPixelFormat hw_pix_fmt; bool force_sw_decoder; #endif + /* Try platform hardware decoders (h264_nvv4l2, h264_v4l2m2m, + * h264_mediacodec, ...) in decoder detection: ffmpeg_hw_wrappers. */ + bool use_hw_wrappers; /* Stream bookkeeping */ AVCodecContext *actx[MAX_STREAMS]; @@ -880,6 +918,7 @@ static ffmpeg_core_ctx_t g_ctx; #define HW_DECODER_STR (g_ctx.hw_decoder) #define HW_DECODING_ENABLED_STR (g_ctx.hw_decoding_enabled) #define FORCE_SW_DECODER_STR (g_ctx.force_sw_decoder) +#define USE_HW_WRAPPERS_STR (g_ctx.use_hw_wrappers) #endif #define ACTX_STR (g_ctx.actx) #define SCTX_STR (g_ctx.sctx) @@ -1941,13 +1980,112 @@ void CORE_PREFIX(retro_set_controller_port_device)(unsigned port, unsigned devic (void)device; } +/* Formats this core has always claimed. */ +#define FFMPEG_CORE_BASE_EXTENSIONS \ + "mkv|avi|f4v|f4f|3gp|ogm|flv|mp4|mp3|flac|ogg|m4a|webm|3g2|mov|wmv|mpg|mpeg|vob|asf|divx|m2p|m2ts|ps|ts|mxf|wma|wav" + +/* Further formats, claimed only when this FFmpeg build can open them: + * RetroArch runs on platforms whose FFmpeg builds differ widely, so + * each needs its demuxer and, where the format implies one, its + * decoder. Keep in sync with the media types in msg_hash.c. */ +static const struct +{ + const char *ext; + const char *demuxer; + enum AVCodecID codec; /* AV_CODEC_ID_NONE: container only */ +} ffmpeg_core_extra_formats[] = { + { "ogv", "ogg", AV_CODEC_ID_THEORA }, + { "oga", "ogg", AV_CODEC_ID_VORBIS }, + { "opus", "ogg", AV_CODEC_ID_OPUS }, + { "spx", "ogg", AV_CODEC_ID_SPEEX }, + { "mka", "matroska", AV_CODEC_ID_NONE }, + { "mk3d", "matroska", AV_CODEC_ID_NONE }, + { "weba", "matroska", AV_CODEC_ID_NONE }, + { "m4v", "mov", AV_CODEC_ID_NONE }, + { "m4b", "mov", AV_CODEC_ID_AAC }, + { "aac", "aac", AV_CODEC_ID_AAC }, + { "ac3", "ac3", AV_CODEC_ID_AC3 }, + { "eac3", "eac3", AV_CODEC_ID_EAC3 }, + { "dts", "dts", AV_CODEC_ID_DTS }, + { "mts", "mpegts", AV_CODEC_ID_NONE }, + { "m2t", "mpegts", AV_CODEC_ID_NONE }, + { "m2v", "mpegvideo", AV_CODEC_ID_MPEG2VIDEO }, + { "mp2", "mp3", AV_CODEC_ID_MP2 }, + { "aif", "aiff", AV_CODEC_ID_NONE }, + { "aiff", "aiff", AV_CODEC_ID_NONE }, + { "wv", "wv", AV_CODEC_ID_WAVPACK }, + { "ape", "ape", AV_CODEC_ID_APE }, + { "tta", "tta", AV_CODEC_ID_TTA }, + { "mpc", "mpc", AV_CODEC_ID_MUSEPACK7 }, + { "amr", "amr", AV_CODEC_ID_AMR_NB }, + { "au", "au", AV_CODEC_ID_NONE }, + { "caf", "caf", AV_CODEC_ID_NONE }, + { "rm", "rm", AV_CODEC_ID_NONE }, + { "rmvb", "rm", AV_CODEC_ID_NONE }, + { "nut", "nut", AV_CODEC_ID_NONE }, + { "dv", "dv", AV_CODEC_ID_DVVIDEO }, + { "ivf", "ivf", AV_CODEC_ID_NONE }, +}; + +static bool ffmpeg_core_extra_format_ok(unsigned i) +{ + if (!av_find_input_format(ffmpeg_core_extra_formats[i].demuxer)) + return false; + if ( ffmpeg_core_extra_formats[i].codec != AV_CODEC_ID_NONE + && !avcodec_find_decoder(ffmpeg_core_extra_formats[i].codec)) + return false; + return true; +} + +/* Exported for the frontend's media type detection (msg_hash.c lists + * these extensions as movie/music): true if this build can open ext. + * Base formats are always claimed. */ +/* Appends s to the extension list in buf, if it fits. */ +static size_t ffmpeg_core_append(char *buf, size_t len, size_t size, + const char *s) +{ + size_t n = strlen(s); + if (len + n + 1 > size) + return len; + memcpy(buf + len, s, n + 1); + return len + n; +} + +bool CORE_PREFIX(ffmpeg_core_supports_extension)(const char *ext) +{ + unsigned i; + for (i = 0; i < sizeof(ffmpeg_core_extra_formats) + / sizeof(ffmpeg_core_extra_formats[0]); i++) + if (!strcmp(ext, ffmpeg_core_extra_formats[i].ext)) + return ffmpeg_core_extra_format_ok(i); + return true; +} + void CORE_PREFIX(retro_get_system_info)(struct retro_system_info *info) { + static char extensions[512]; + + if (!*extensions) + { + unsigned i; + size_t _len = ffmpeg_core_append(extensions, 0, sizeof(extensions), + FFMPEG_CORE_BASE_EXTENSIONS); + for (i = 0; i < sizeof(ffmpeg_core_extra_formats) + / sizeof(ffmpeg_core_extra_formats[0]); i++) + { + if (!ffmpeg_core_extra_format_ok(i)) + continue; + _len = ffmpeg_core_append(extensions, _len, sizeof(extensions), "|"); + _len = ffmpeg_core_append(extensions, _len, sizeof(extensions), + ffmpeg_core_extra_formats[i].ext); + } + } + memset(info, 0, sizeof(*info)); info->library_name = "FFmpeg"; info->library_version = "v1"; info->need_fullpath = true; - info->valid_extensions = "mkv|avi|f4v|f4f|3gp|ogm|flv|mp4|mp3|flac|ogg|m4a|webm|3g2|mov|wmv|mpg|mpeg|vob|asf|divx|m2p|m2ts|ps|ts|mxf|wma|wav"; + info->valid_extensions = extensions; } void CORE_PREFIX(retro_get_system_av_info)(struct retro_system_av_info *info) @@ -1981,6 +2119,9 @@ void CORE_PREFIX(retro_set_environment)(retro_environment_t cb) #if ENABLE_HW_ACCEL { "ffmpeg_hw_decoder", "Use Hardware decoder (restart); off|auto|" "cuda|d3d11va|drm|dxva2|mediacodec|opencl|qsv|vaapi|vdpau|videotoolbox" }, +#endif +#if !FFMPEG3 + { "ffmpeg_hw_wrappers", "Use platform hardware decoders (restart); enabled|disabled" }, #endif { "ffmpeg_sw_decoder_threads", "Software decoder thread count (restart); auto|1|2|4|6|8|10|12|14|16" }, #if defined(HAVE_OPENGL) || defined(HAVE_OPENGLES) @@ -2109,6 +2250,17 @@ static void check_variables(bool firststart) retro_atomic_store_release_int(&g_ctx.color_space, space); } + if (firststart) + { + struct retro_variable wrap_var = {0}; + wrap_var.key = "ffmpeg_hw_wrappers"; + USE_HW_WRAPPERS_STR = true; + if ( CORE_PREFIX(environ_cb)(RETRO_ENVIRONMENT_GET_VARIABLE, &wrap_var) + && wrap_var.value + && !strcmp(wrap_var.value, "disabled")) + USE_HW_WRAPPERS_STR = false; + } + #if ENABLE_HW_ACCEL if (firststart) { @@ -2423,6 +2575,16 @@ static int seek_adjust(int target) return 0; } +static double audio_backlog_seconds(void) +{ + if ( AUDIO_STREAMS_NUM_STR <= 0 + || !g_ctx.audio_decode_fifo_init + || MEDIA_STR.sample_rate <= 0) + return 0.0; + return (double)retro_spsc_read_avail(&AUDIO_DECODE_FIFO_STR) + / (MEDIA_STR.sample_rate * sizeof(int16_t) * 2); +} + void CORE_PREFIX(retro_run)(void) { double min_pts; @@ -2662,9 +2824,11 @@ void CORE_PREFIX(retro_run)(void) if (!VIDEO_BUFFER_STR) break; - if (!DECODE_THREAD_DEAD_STR) - video_buffer_wait_for_finished_slot(VIDEO_BUFFER_STR, - &g_ctx.decode_thread_dead); + /* Gap in the video track: hold the current frame. */ + if ( !DECODE_THREAD_DEAD_STR + && !video_buffer_wait_for_frame(VIDEO_BUFFER_STR, + VB_TAIL(VIDEO_BUFFER_STR), &g_ctx.decode_thread_dead)) + break; if (!DECODE_THREAD_DEAD_STR) { @@ -2732,6 +2896,11 @@ void CORE_PREFIX(retro_run)(void) } mix_factor = (min_pts - FRAMES_STR[0].pts) / (FRAMES_STR[1].pts - FRAMES_STR[0].pts); + /* A frame held across a gap leaves min_pts past both. */ + if (!(mix_factor >= 0.0f)) + mix_factor = 0.0f; + else if (mix_factor > 1.0f) + mix_factor = 1.0f; if (!TEMPORAL_INTERPOLATION_STR) mix_factor = 1.0f; @@ -2786,6 +2955,16 @@ void CORE_PREFIX(retro_run)(void) if (!VIDEO_BUFFER_STR) break; + /* Gap in the video track: keep holding the frame on screen. + * The next one is in the slot after the held one. */ + if ( !DECODE_THREAD_DEAD_STR + && !video_buffer_wait_for_frame(VIDEO_BUFFER_STR, + g_ctx.held_slot + ? (g_ctx.held_slot->index + 1) % VIDEO_BUFFER_STR->capacity + : (unsigned)VB_TAIL(VIDEO_BUFFER_STR), + &g_ctx.decode_thread_dead)) + break; + /* The frame on screen is about to be replaced (or, when * several are due at once, skipped), so its slot can go * back. It has to: the tail does not move until it does. */ @@ -2911,7 +3090,7 @@ static enum AVPixelFormat init_hw_decoder(struct AVCodecContext *ctx, #endif int ret = 0; enum AVPixelFormat decoder_pix_fmt = AV_PIX_FMT_NONE; - const AVCodec *codec = retro_find_decoder(FCTX_STR->streams[VIDEO_STREAM_INDEX_STR]->codecpar->codec_id); + const AVCodec *codec = ctx->codec; #if !FFMPEG3 for (i = 0;; i++) @@ -2974,7 +3153,7 @@ static enum AVPixelFormat init_hw_decoder(struct AVCodecContext *ctx, decoder_pix_fmt = AV_PIX_FMT_NONE; } else - ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); + ctx->hw_device_ctx = hw_device_ctx; /* the context owns it */ } return decoder_pix_fmt; @@ -3065,17 +3244,169 @@ static enum AVPixelFormat get_format(AVCodecContext *ctx, } #endif -static bool open_codec(AVCodecContext **ctx, enum AVMediaType type, unsigned index) +/* Decoder detection: try every decoder for a stream's codec - for + * video, platform hardware decoders first (ffmpeg_hw_wrappers), then + * the default one (with the hwaccel path), then other software ones - + * and keep the first that opens and, for audio and video, decodes a + * frame this core can use. avcodec_find_decoder() alone gives whichever + * the build registered first, which need not work here. */ +#define MAX_DECODER_CANDIDATES 16 +#define DECODER_PROBE_PACKETS 64 + +/* Can this core use the decoded frame? Video: a software format + * swscale accepts, or the hwaccel format select_decoder() set up. + * Audio: any packed or planar sample format swresample converts. */ +static bool frame_usable(const AVFrame *frame, enum AVMediaType type) { - int ret = 0; - const AVCodec *codec = retro_find_decoder(FCTX_STR->streams[index]->codecpar->codec_id); - if (!codec) + if (type == AVMEDIA_TYPE_AUDIO) + return frame->nb_samples > 0 + && av_get_bytes_per_sample((enum AVSampleFormat)frame->format) > 0; + { - log_cb(RETRO_LOG_ERROR, "[FFMPEG] Couldn't find suitable decoder\n"); + const AVPixFmtDescriptor *desc = + av_pix_fmt_desc_get((enum AVPixelFormat)frame->format); + + if (!desc) + return false; + if (desc->flags & AV_PIX_FMT_FLAG_HWACCEL) + { +#if ENABLE_HW_ACCEL + return HW_DECODING_ENABLED_STR && frame->format == g_ctx.hw_pix_fmt; +#else + return false; +#endif + } + return sws_isSupportedInput((enum AVPixelFormat)frame->format) > 0; + } +} + +static bool input_is_seekable(void) +{ + return FCTX_STR->pb + && (FCTX_STR->pb->seekable & AVIO_SEEKABLE_NORMAL) + && !(FCTX_STR->iformat->flags & AVFMT_NOFILE); +} + +/* Rewind the demuxer to the start after a probe. */ +static bool rewind_input(unsigned index) +{ + int64_t start = FCTX_STR->streams[index]->start_time; + if (start == AV_NOPTS_VALUE) + start = 0; + if (av_seek_frame(FCTX_STR, (int)index, start, AVSEEK_FLAG_BACKWARD) < 0) return false; + avformat_flush(FCTX_STR); + return true; +} + +/* Decode from the start of the stream until one frame comes out. True + * if that frame is usable. The caller rewinds. */ +static bool probe_decoder(AVCodecContext *ctx, enum AVMediaType type, + unsigned index) +{ + int packets = 0; + bool decoded = false; + bool usable = false; + AVPacket *pkt = av_packet_alloc(); + AVFrame *frame = av_frame_alloc(); + + if (!pkt || !frame) + goto end; + + while (!decoded && packets < DECODER_PROBE_PACKETS + && av_read_frame(FCTX_STR, pkt) >= 0) + { + if (pkt->stream_index == (int)index) + { + int ret = avcodec_send_packet(ctx, pkt); + packets++; + if (ret < 0 && ret != AVERROR(EAGAIN)) + { + av_packet_unref(pkt); + break; + } + if (avcodec_receive_frame(ctx, frame) == 0) + decoded = true; + } + av_packet_unref(pkt); + } + + /* Decoders with delay may hold the first frame until drained. */ + if (!decoded && packets) + { + avcodec_send_packet(ctx, NULL); + if (avcodec_receive_frame(ctx, frame) == 0) + decoded = true; + } + + usable = decoded && frame_usable(frame, type); + if (decoded && !usable) + log_cb(RETRO_LOG_INFO, "[FFMPEG] %s decodes to %s, which this core can't use.\n", + ctx->codec->name, type == AVMEDIA_TYPE_AUDIO + ? av_get_sample_fmt_name((enum AVSampleFormat)frame->format) + : av_get_pix_fmt_name((enum AVPixelFormat)frame->format)); + +end: + av_frame_free(&frame); + av_packet_free(&pkt); + avcodec_flush_buffers(ctx); + return usable; +} + +static int decoder_candidates(enum AVCodecID id, enum AVMediaType type, + const AVCodec **list, int max) +{ + int n = 0; + const AVCodec *def = retro_find_decoder(id); +#if !FFMPEG3 + int pass; + bool allow_hw = (type == AVMEDIA_TYPE_VIDEO) && USE_HW_WRAPPERS_STR; +#endif + +#if FFMPEG3 + if (def) + list[n++] = def; +#else + /* 0: platform hardware decoders (video only), + * 1: the default decoder, 2: any other software decoder. */ + for (pass = 0; pass < 3 && n < max; pass++) + { + void *it = NULL; + const AVCodec *c = NULL; + + if (pass == 1) + { + if (def && !(def->capabilities & AV_CODEC_CAP_HARDWARE)) + list[n++] = def; + continue; + } + + while ((c = av_codec_iterate(&it)) && n < max) + { + bool hw = (c->capabilities & AV_CODEC_CAP_HARDWARE) != 0; + if (c->id != id || !av_codec_is_decoder(c)) + continue; + if (c->capabilities & AV_CODEC_CAP_EXPERIMENTAL) + continue; + if (pass == 0 ? !(hw && allow_hw) : (hw || c == def)) + continue; + list[n++] = c; + } } +#endif + return n; +} + +/* Allocate and open ctx for codec. Video gets the hwaccel set-up for + * non-wrapper decoders, exactly as before. */ +static bool open_codec_with(AVCodecContext **ctx, const AVCodec *codec, + enum AVMediaType type, unsigned index) +{ + int ret = 0; *ctx = avcodec_alloc_context3(codec); + if (!*ctx) + return false; avcodec_parameters_to_context((*ctx), FCTX_STR->streams[index]->codecpar); if (type == AVMEDIA_TYPE_VIDEO) @@ -3083,8 +3414,20 @@ static bool open_codec(AVCodecContext **ctx, enum AVMediaType type, unsigned ind VIDEO_STREAM_INDEX_STR = index; #if ENABLE_HW_ACCEL - VCTX_STR->get_format = get_format; - g_ctx.hw_pix_fmt = select_decoder((*ctx), NULL); + g_ctx.hw_pix_fmt = AV_PIX_FMT_NONE; + HW_DECODING_ENABLED_STR = false; + if (codec->capabilities & AV_CODEC_CAP_HARDWARE) + { + /* A hardware wrapper decodes by itself into frames it hands + * back; no hwaccel device or get_format() hook. */ + log_cb(RETRO_LOG_INFO, "[FFMPEG] Trying hardware decoder %s.\n", + codec->name); + } + else + { + (*ctx)->get_format = get_format; + g_ctx.hw_pix_fmt = select_decoder((*ctx), NULL); + } #else select_decoder((*ctx), NULL); #endif @@ -3093,16 +3436,79 @@ static bool open_codec(AVCodecContext **ctx, enum AVMediaType type, unsigned ind if ((ret = avcodec_open2(*ctx, codec, NULL)) < 0) { #ifdef __cplusplus - log_cb(RETRO_LOG_ERROR, "[FFMPEG] Could not open codec: %d\n", ret); + log_cb(RETRO_LOG_ERROR, "[FFMPEG] Could not open codec %s: %d\n", + codec->name, ret); #else - log_cb(RETRO_LOG_ERROR, "[FFMPEG] Could not open codec: %s\n", av_err2str(ret)); + log_cb(RETRO_LOG_ERROR, "[FFMPEG] Could not open codec %s: %s\n", + codec->name, av_err2str(ret)); #endif + avcodec_free_context(ctx); return false; } return true; } +static bool open_codec(AVCodecContext **ctx, enum AVMediaType type, unsigned index) +{ + const AVCodec *list[MAX_DECODER_CANDIDATES]; + enum AVCodecID id = FCTX_STR->streams[index]->codecpar->codec_id; + int i, n; + bool probe; + + *ctx = NULL; + n = decoder_candidates(id, type, list, MAX_DECODER_CANDIDATES); + if (!n) + { + log_cb(RETRO_LOG_ERROR, "[FFMPEG] This FFmpeg build has no decoder for %s (%s).\n", + avcodec_get_name(id), av_get_media_type_string(type)); + return false; + } + + /* Probing reads packets, so the input must be rewindable; otherwise + * take the first decoder that opens, as before. With one candidate, + * or for subtitles, there is nothing worth probing. */ + probe = n > 1 + && (type == AVMEDIA_TYPE_VIDEO || type == AVMEDIA_TYPE_AUDIO) + && input_is_seekable(); + + for (i = 0; i < n; i++) + { + if (!open_codec_with(ctx, list[i], type, index)) + continue; + + if (!probe) + break; + + if (probe_decoder(*ctx, type, index)) + { + if (!rewind_input(index)) + log_cb(RETRO_LOG_WARN, "[FFMPEG] Could not rewind after probing the decoder.\n"); + break; + } + + log_cb(RETRO_LOG_INFO, "[FFMPEG] Decoder %s failed its probe, trying the next.\n", + list[i]->name); + avcodec_free_context(ctx); + if (!rewind_input(index)) + { + log_cb(RETRO_LOG_ERROR, "[FFMPEG] Could not rewind after probing the decoder.\n"); + return false; + } + } + + if (!*ctx) + { + log_cb(RETRO_LOG_ERROR, "[FFMPEG] No working decoder for %s (%s).\n", + avcodec_get_name(id), av_get_media_type_string(type)); + return false; + } + + log_cb(RETRO_LOG_INFO, "[FFMPEG] Using %s decoder %s.\n", + av_get_media_type_string(type), (*ctx)->codec->name); + return true; +} + static bool codec_is_image(enum AVCodecID id) { switch (id) @@ -4123,32 +4529,18 @@ void CORE_PREFIX(retro_unload_game)(void) g_ctx.decoded_frame_cnt = 0; AUDIO_FRAMES_STR = 0; + /* Freed, not only closed: avcodec_close() leaves the context + * allocated. */ for (i = 0; i < MAX_STREAMS; i++) { -#if FFMPEG8 if (SCTX_STR[i]) avcodec_free_context(&SCTX_STR[i]); if (ACTX_STR[i]) avcodec_free_context(&ACTX_STR[i]); -#else - if (SCTX_STR[i]) - avcodec_close(SCTX_STR[i]); - if (ACTX_STR[i]) - avcodec_close(ACTX_STR[i]); -#endif - SCTX_STR[i] = NULL; - ACTX_STR[i] = NULL; } if (VCTX_STR) - { -#if FFMPEG8 avcodec_free_context(&VCTX_STR); -#else - avcodec_close(VCTX_STR); -#endif - VCTX_STR = NULL; - } if (FCTX_STR) { diff --git a/msg_hash.c b/msg_hash.c index 85ec33e34002..0fbf256d0e84 100644 --- a/msg_hash.c +++ b/msg_hash.c @@ -692,6 +692,40 @@ enum msg_file_type msg_hash_to_file_type(const char *value) { "weba", FILE_TYPE_WEBA }, #endif #endif +#if defined(HAVE_FFMPEG) || defined(HAVE_MPV) + /* Further media formats; with FFmpeg, only routed to the player + * if its build can open them. Extensions claimed above win. */ + { "ogv", FILE_TYPE_MOVIE_OTHER }, + { "mk3d", FILE_TYPE_MOVIE_OTHER }, + { "mts", FILE_TYPE_MOVIE_OTHER }, + { "m2t", FILE_TYPE_MOVIE_OTHER }, + { "m2v", FILE_TYPE_MOVIE_OTHER }, + { "rm", FILE_TYPE_MOVIE_OTHER }, + { "rmvb", FILE_TYPE_MOVIE_OTHER }, + { "nut", FILE_TYPE_MOVIE_OTHER }, + { "dv", FILE_TYPE_MOVIE_OTHER }, + { "ivf", FILE_TYPE_MOVIE_OTHER }, + { "oga", FILE_TYPE_MUSIC_OTHER }, + { "opus", FILE_TYPE_MUSIC_OTHER }, + { "spx", FILE_TYPE_MUSIC_OTHER }, + { "mka", FILE_TYPE_MUSIC_OTHER }, + { "weba", FILE_TYPE_MUSIC_OTHER }, + { "m4b", FILE_TYPE_MUSIC_OTHER }, + { "aac", FILE_TYPE_MUSIC_OTHER }, + { "ac3", FILE_TYPE_MUSIC_OTHER }, + { "eac3", FILE_TYPE_MUSIC_OTHER }, + { "dts", FILE_TYPE_MUSIC_OTHER }, + { "mp2", FILE_TYPE_MUSIC_OTHER }, + { "aif", FILE_TYPE_MUSIC_OTHER }, + { "aiff", FILE_TYPE_MUSIC_OTHER }, + { "wv", FILE_TYPE_MUSIC_OTHER }, + { "ape", FILE_TYPE_MUSIC_OTHER }, + { "tta", FILE_TYPE_MUSIC_OTHER }, + { "mpc", FILE_TYPE_MUSIC_OTHER }, + { "amr", FILE_TYPE_MUSIC_OTHER }, + { "au", FILE_TYPE_MUSIC_OTHER }, + { "caf", FILE_TYPE_MUSIC_OTHER }, +#endif #ifdef HAVE_IMAGEVIEWER { "jpg", FILE_TYPE_JPEG }, { "JPG", FILE_TYPE_JPEG }, diff --git a/msg_hash.h b/msg_hash.h index 8ddf15d362b0..a6d218f06a7a 100644 --- a/msg_hash.h +++ b/msg_hash.h @@ -218,6 +218,10 @@ enum msg_file_type FILE_TYPE_PS, FILE_TYPE_TS, FILE_TYPE_MXF, + /* Further formats the built-in media player claims when its build + * can open them (see ffmpeg_core_supports_extension()). */ + FILE_TYPE_MOVIE_OTHER, + FILE_TYPE_MUSIC_OTHER, FILE_TYPE_JPEG, FILE_TYPE_PNG, diff --git a/retroarch.c b/retroarch.c index 93d375b3349a..b17b200e6594 100644 --- a/retroarch.c +++ b/retroarch.c @@ -237,6 +237,9 @@ #include "version_git.h" #include "retroarch.h" +#ifdef HAVE_FFMPEG +#include "cores/internal_cores.h" +#endif #include "accessibility.h" @@ -3118,6 +3121,15 @@ enum rarch_content_type path_is_media_type(const char *path) case FILE_TYPE_TS: case FILE_TYPE_MXF: return RARCH_CONTENT_MOVIE; + case FILE_TYPE_MOVIE_OTHER: + case FILE_TYPE_MUSIC_OTHER: +#ifdef HAVE_FFMPEG + /* Only if this build's FFmpeg can open it. */ + if (!libretro_ffmpeg_ffmpeg_core_supports_extension(ext_lower)) + return RARCH_CONTENT_NONE; +#endif + return (msg_hash_to_file_type(ext_lower) == FILE_TYPE_MOVIE_OTHER) + ? RARCH_CONTENT_MOVIE : RARCH_CONTENT_MUSIC; case FILE_TYPE_WMA: case FILE_TYPE_M4A: #endif diff --git a/samples/cores/ffmpeg_sweep/ffmpeg_sweep_test.c b/samples/cores/ffmpeg_sweep/ffmpeg_sweep_test.c index 135136ec9120..46f3896132b5 100644 --- a/samples/cores/ffmpeg_sweep/ffmpeg_sweep_test.c +++ b/samples/cores/ffmpeg_sweep/ffmpeg_sweep_test.c @@ -32,7 +32,7 @@ static const char *current = ""; /* ---- the host ----------------------------------------------------- */ -static unsigned frames_seen, batches_seen; +static unsigned frames_seen, batches_seen, fresh_frames; static uint64_t audio_frames; static bool quiet_log = true; @@ -86,8 +86,10 @@ static bool env_cb(unsigned cmd, void *data) static void video_cb(const void *d, unsigned w, unsigned h, size_t p) { - (void)d; (void)w; (void)h; (void)p; + (void)w; (void)h; (void)p; frames_seen++; + if (d) + fresh_frames++; } static size_t audio_batch_cb(const int16_t *data, size_t frames) @@ -213,6 +215,58 @@ static void clip_case(const char *name, const char *path, const char *args, clips_run++; } +/* A video track with a hole in it, as a recording that dropped frames + * leaves: the core has to play across it, not wait for a frame that + * is seconds away while the audio queue fills behind it. */ +static void gap_case(const char *path) +{ + struct retro_game_info info; + struct retro_system_av_info av; + unsigned frames, fresh; + double fps; + + current = "video gap"; + if (!make_clip(path, + "-f lavfi -i testsrc=size=160x120:rate=30:duration=8 " + "-f lavfi -i sine=f=440:r=48000:duration=8 " + "-vf \"select='not(between(t,2,6))'\" -fps_mode passthrough " + "-c:v mpeg4 -c:a pcm_s16le")) + { + printf(" (ffmpeg would not make the gap clip; skipped)\n"); + return; + } + + memset(&info, 0, sizeof(info)); + info.path = path; + audio_frames = 0; + + retro_init(); + if (!retro_load_game(&info)) + { + CHECK(false, "the core refused the gap clip"); + retro_deinit(); + return; + } + memset(&av, 0, sizeof(av)); + retro_get_system_av_info(&av); + /* Through the gap (2 s to 6 s) and out the other side. */ + fps = av.timing.fps > 0.0 ? av.timing.fps : 30.0; + frames = (unsigned)(6.5 * fps); + alarm(60); + run_frames(frames); + fresh = fresh_frames; + run_frames((unsigned)fps); + alarm(600); + printf(" %-22s %.2f s of audio, %u new frames after the gap\n", + "video gap", (double)audio_frames / 48000.0, fresh_frames - fresh); + CHECK(audio_frames >= 7 * 48000, + "audio stopped at %.2f s, in the gap", + (double)audio_frames / 48000.0); + CHECK(fresh_frames > fresh, "no video after the gap"); + retro_unload_game(); + retro_deinit(); +} + /* A file that is not what it says, or is cut short. The core has to * refuse it without reading past the end of it. */ static void reject_case(const char *name, const char *path) @@ -307,6 +361,8 @@ int main(void) "-f lavfi -i sine=f=440:r=48000:duration=2 -c:v mpeg4 -c:a pcm_s16le", true, true); + gap_case("/tmp/sw_gap.mkv"); + printf(" files that are not what they claim\n"); /* A media file cut in half: the header parses, the stream does * not reach where the index says it does. */