vf_libplacebo vulkan changes

configure

@@ -327,7 +327,6 @@ External library support:
   --disable-securetransport disable Secure Transport, needed for TLS support
                            on OSX if openssl and gnutls are not used [autodetect]
   --enable-vapoursynth     enable VapourSynth demuxer [no]
-  --disable-vulkan         disable Vulkan code [autodetect]
   --disable-xlib           disable xlib [autodetect]
   --disable-zlib           disable zlib [autodetect]
 
@@ -354,6 +353,8 @@ External library support:
   --disable-vaapi          disable Video Acceleration API (mainly Unix/Intel) code [autodetect]
   --disable-vdpau          disable Nvidia Video Decode and Presentation API for Unix code [autodetect]
   --disable-videotoolbox   disable VideoToolbox code [autodetect]
+  --disable-vulkan         disable Vulkan code [autodetect]
+  --enable-vulkan-encode   enable Vulkan encoding code [no]
 
 Toolchain options:
   --arch=ARCH              select architecture [$arch]
@@ -1915,6 +1916,7 @@ HWACCEL_LIBRARY_LIST="
     mmal
     omx
     opencl
+    vulkan_encode
 "
 
 DOCUMENT_LIST="
@@ -2514,6 +2516,7 @@ CONFIG_EXTRA="
     vp56dsp
     vp8dsp
     wma_freqs
+    vulkan_encode
     wmv2dsp
 "
 
@@ -3020,6 +3023,8 @@ av1_vaapi_hwaccel_deps="vaapi VADecPictureParameterBufferAV1_bit_depth_idx"
 av1_vaapi_hwaccel_select="av1_decoder"
 av1_vdpau_hwaccel_deps="vdpau VdpPictureInfoAV1"
 av1_vdpau_hwaccel_select="av1_decoder"
+av1_vulkan_hwaccel_deps="vulkan"
+av1_vulkan_hwaccel_select="av1_decoder"
 h263_vaapi_hwaccel_deps="vaapi"
 h263_vaapi_hwaccel_select="h263_decoder"
 h263_videotoolbox_hwaccel_deps="videotoolbox"
@@ -3038,6 +3043,8 @@ h264_vdpau_hwaccel_deps="vdpau"
 h264_vdpau_hwaccel_select="h264_decoder"
 h264_videotoolbox_hwaccel_deps="videotoolbox"
 h264_videotoolbox_hwaccel_select="h264_decoder"
+h264_vulkan_hwaccel_deps="vulkan"
+h264_vulkan_hwaccel_select="h264_decoder"
 hevc_d3d11va_hwaccel_deps="d3d11va DXVA_PicParams_HEVC"
 hevc_d3d11va_hwaccel_select="hevc_decoder"
 hevc_d3d11va2_hwaccel_deps="d3d11va DXVA_PicParams_HEVC"
@@ -3052,6 +3059,8 @@ hevc_vdpau_hwaccel_deps="vdpau VdpPictureInfoHEVC"
 hevc_vdpau_hwaccel_select="hevc_decoder"
 hevc_videotoolbox_hwaccel_deps="videotoolbox"
 hevc_videotoolbox_hwaccel_select="hevc_decoder"
+hevc_vulkan_hwaccel_deps="vulkan"
+hevc_vulkan_hwaccel_select="hevc_decoder"
 mjpeg_nvdec_hwaccel_deps="nvdec"
 mjpeg_nvdec_hwaccel_select="mjpeg_decoder"
 mjpeg_vaapi_hwaccel_deps="vaapi"
@@ -3133,6 +3142,7 @@ qsvenc_select="qsv"
 qsvvpp_select="qsv"
 vaapi_encode_deps="vaapi"
 v4l2_m2m_deps="linux_videodev2_h sem_timedwait"
+vulkan_encode_deps="vulkan"
 
 bilateral_cuda_filter_deps="ffnvcodec"
 bilateral_cuda_filter_deps_any="cuda_nvcc cuda_llvm"
@@ -3185,6 +3195,7 @@ h264_qsv_encoder_select="atsc_a53 qsvenc"
 h264_rkmpp_decoder_deps="rkmpp"
 h264_rkmpp_decoder_select="h264_mp4toannexb_bsf"
 h264_vaapi_encoder_select="atsc_a53 cbs_h264 vaapi_encode"
+h264_vulkan_encoder_select="cbs_h264 vulkan_encode"
 h264_v4l2m2m_decoder_deps="v4l2_m2m h264_v4l2_m2m"
 h264_v4l2m2m_decoder_select="h264_mp4toannexb_bsf"
 h264_v4l2m2m_encoder_deps="v4l2_m2m h264_v4l2_m2m"
@@ -3640,7 +3651,9 @@ blend_vulkan_filter_deps="vulkan spirv_compiler"
 boxblur_filter_deps="gpl"
 boxblur_opencl_filter_deps="opencl gpl"
 bs2b_filter_deps="libbs2b"
+bwdif_vulkan_filter_deps="vulkan spirv_compiler"
 chromaber_vulkan_filter_deps="vulkan spirv_compiler"
+color_vulkan_filter_deps="vulkan spirv_compiler"
 colorkey_opencl_filter_deps="opencl"
 colormatrix_filter_deps="gpl"
 convolution_opencl_filter_deps="opencl"
@@ -3697,6 +3710,7 @@ minterpolate_filter_select="scene_sad"
 mptestsrc_filter_deps="gpl"
 negate_filter_deps="lut_filter"
 nlmeans_opencl_filter_deps="opencl"
+nlmeans_vulkan_filter_deps="vulkan spirv_compiler"
 nnedi_filter_deps="gpl"
 ocr_filter_deps="libtesseract"
 ocv_filter_deps="libopencv"
@@ -7040,8 +7054,8 @@ enabled crystalhd && check_lib crystalhd "stdint.h libcrystalhd/libcrystalhd_if.
          "in maintaining it."
 
 if enabled vulkan; then
-    check_pkg_config_header_only vulkan "vulkan >= 1.2.189" "vulkan/vulkan.h" "defined VK_VERSION_1_2" ||
-        check_cpp_condition vulkan "vulkan/vulkan.h" "defined(VK_VERSION_1_3) || (defined(VK_VERSION_1_2) && VK_HEADER_VERSION >= 189)"
+    check_pkg_config_header_only vulkan "vulkan >= 1.3.238" "vulkan/vulkan.h" "defined VK_VERSION_1_3" ||
+        check_cpp_condition vulkan "vulkan/vulkan.h" "defined(VK_VERSION_1_4) || (defined(VK_VERSION_1_3) && VK_HEADER_VERSION >= 238)"
 fi
 
 if enabled x86; then

libavcodec/Makefile

@@ -170,6 +170,7 @@ OBJS-$(CONFIG_VP3DSP)                  += vp3dsp.o
 OBJS-$(CONFIG_VP56DSP)                 += vp56dsp.o
 OBJS-$(CONFIG_VP8DSP)                  += vp8dsp.o
 OBJS-$(CONFIG_V4L2_M2M)                += v4l2_m2m.o v4l2_context.o v4l2_buffers.o v4l2_fmt.o
+OBJS-$(CONFIG_VULKAN_ENCODE)           += vulkan_encode.o
 OBJS-$(CONFIG_WMA_FREQS)               += wma_freqs.o
 OBJS-$(CONFIG_WMV2DSP)                 += wmv2dsp.o
 
@@ -419,6 +420,8 @@ OBJS-$(CONFIG_H264_VAAPI_ENCODER)      += vaapi_encode_h264.o h264_levels.o \
 OBJS-$(CONFIG_H264_VIDEOTOOLBOX_ENCODER) += videotoolboxenc.o
 OBJS-$(CONFIG_H264_V4L2M2M_DECODER)    += v4l2_m2m_dec.o
 OBJS-$(CONFIG_H264_V4L2M2M_ENCODER)    += v4l2_m2m_enc.o
+OBJS-$(CONFIG_H264_VULKAN_ENCODER)     += vulkan_encode_h264.o h264_levels.o \
+                                          h2645data.o
 OBJS-$(CONFIG_HAP_DECODER)             += hapdec.o hap.o
 OBJS-$(CONFIG_HAP_ENCODER)             += hapenc.o hap.o
 OBJS-$(CONFIG_HCA_DECODER)             += hcadec.o
@@ -982,12 +985,14 @@ OBJS-$(CONFIG_NVDEC)                      += nvdec.o
 OBJS-$(CONFIG_VAAPI)                      += vaapi_decode.o
 OBJS-$(CONFIG_VIDEOTOOLBOX)               += videotoolbox.o
 OBJS-$(CONFIG_VDPAU)                      += vdpau.o
+OBJS-$(CONFIG_VULKAN)                     += vulkan.o vulkan_video.o
 
 OBJS-$(CONFIG_AV1_D3D11VA_HWACCEL)        += dxva2_av1.o
 OBJS-$(CONFIG_AV1_DXVA2_HWACCEL)          += dxva2_av1.o
 OBJS-$(CONFIG_AV1_NVDEC_HWACCEL)          += nvdec_av1.o
 OBJS-$(CONFIG_AV1_VAAPI_HWACCEL)          += vaapi_av1.o
 OBJS-$(CONFIG_AV1_VDPAU_HWACCEL)          += vdpau_av1.o
+OBJS-$(CONFIG_AV1_VULKAN_HWACCEL)         += vulkan_av1.o
 OBJS-$(CONFIG_H263_VAAPI_HWACCEL)         += vaapi_mpeg4.o
 OBJS-$(CONFIG_H263_VIDEOTOOLBOX_HWACCEL)  += videotoolbox.o
 OBJS-$(CONFIG_H264_D3D11VA_HWACCEL)       += dxva2_h264.o
@@ -997,12 +1002,14 @@ OBJS-$(CONFIG_H264_QSV_HWACCEL)           += qsvdec.o
 OBJS-$(CONFIG_H264_VAAPI_HWACCEL)         += vaapi_h264.o
 OBJS-$(CONFIG_H264_VDPAU_HWACCEL)         += vdpau_h264.o
 OBJS-$(CONFIG_H264_VIDEOTOOLBOX_HWACCEL)  += videotoolbox.o
+OBJS-$(CONFIG_H264_VULKAN_HWACCEL)        += vulkan_decode.o vulkan_h264.o
 OBJS-$(CONFIG_HEVC_D3D11VA_HWACCEL)       += dxva2_hevc.o
 OBJS-$(CONFIG_HEVC_DXVA2_HWACCEL)         += dxva2_hevc.o
 OBJS-$(CONFIG_HEVC_NVDEC_HWACCEL)         += nvdec_hevc.o
 OBJS-$(CONFIG_HEVC_QSV_HWACCEL)           += qsvdec.o
 OBJS-$(CONFIG_HEVC_VAAPI_HWACCEL)         += vaapi_hevc.o h265_profile_level.o
 OBJS-$(CONFIG_HEVC_VDPAU_HWACCEL)         += vdpau_hevc.o h265_profile_level.o
+OBJS-$(CONFIG_HEVC_VULKAN_HWACCEL)        += vulkan_decode.o vulkan_hevc.o
 OBJS-$(CONFIG_MJPEG_NVDEC_HWACCEL)        += nvdec_mjpeg.o
 OBJS-$(CONFIG_MJPEG_VAAPI_HWACCEL)        += vaapi_mjpeg.o
 OBJS-$(CONFIG_MPEG1_NVDEC_HWACCEL)        += nvdec_mpeg12.o
@@ -1290,6 +1297,7 @@ SKIPHEADERS-$(CONFIG_XVMC)             += xvmc.h
 SKIPHEADERS-$(CONFIG_VAAPI)            += vaapi_decode.h vaapi_hevc.h vaapi_encode.h
 SKIPHEADERS-$(CONFIG_VDPAU)            += vdpau.h vdpau_internal.h
 SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX)     += videotoolbox.h vt_internal.h
+SKIPHEADERS-$(CONFIG_VULKAN)           += vulkan.h vulkan_video.h vulkan_encode.h vulkan_decode.h vulkan_video_codec_av1std.h vulkan_video_codec_av1std_decode.h
 SKIPHEADERS-$(CONFIG_V4L2_M2M)         += v4l2_buffers.h v4l2_context.h v4l2_m2m.h
 SKIPHEADERS-$(CONFIG_ZLIB)             += zlib_wrapper.h
 

libavcodec/allcodecs.c

@@ -852,6 +852,7 @@ extern const FFCodec ff_h264_qsv_encoder;
 extern const FFCodec ff_h264_v4l2m2m_encoder;
 extern const FFCodec ff_h264_vaapi_encoder;
 extern const FFCodec ff_h264_videotoolbox_encoder;
+extern const FFCodec ff_h264_vulkan_encoder;
 extern const FFCodec ff_hevc_amf_encoder;
 extern const FFCodec ff_hevc_cuvid_decoder;
 extern const FFCodec ff_hevc_mediacodec_decoder;

libavcodec/av1dec.c

@@ -27,6 +27,7 @@
 #include "libavutil/opt.h"
 #include "avcodec.h"
 #include "av1_parse.h"
+#include "decode.h"
 #include "av1dec.h"
 #include "atsc_a53.h"
 #include "bytestream.h"
@@ -268,8 +269,10 @@ static void skip_mode_params(AV1DecContext *s)
     int second_forward_idx, second_forward_hint;
     int ref_hint, dist, i;
 
-    if (!header->skip_mode_present)
-        return;
+    if (header->frame_type == AV1_FRAME_KEY ||
+	header->frame_type == AV1_FRAME_INTRA_ONLY ||
+	!header->reference_select || !seq->enable_order_hint)
+      return;
 
     forward_idx  = -1;
     backward_idx = -1;
@@ -446,7 +449,8 @@ static int get_pixel_format(AVCodecContext *avctx)
                      CONFIG_AV1_D3D11VA_HWACCEL * 2 + \
                      CONFIG_AV1_NVDEC_HWACCEL + \
                      CONFIG_AV1_VAAPI_HWACCEL + \
-                     CONFIG_AV1_VDPAU_HWACCEL)
+                     CONFIG_AV1_VDPAU_HWACCEL + \
+                     CONFIG_AV1_VULKAN_HWACCEL)
     enum AVPixelFormat pix_fmts[HWACCEL_MAX + 2], *fmtp = pix_fmts;
 
     if (seq->seq_profile == 2 && seq->color_config.high_bitdepth)
@@ -526,6 +530,9 @@ static int get_pixel_format(AVCodecContext *avctx)
 #endif
 #if CONFIG_AV1_VDPAU_HWACCEL
         *fmtp++ = AV_PIX_FMT_VDPAU;
+#endif
+#if CONFIG_AV1_VULKAN_HWACCEL
+        *fmtp++ = AV_PIX_FMT_VULKAN;
 #endif
         break;
     case AV_PIX_FMT_YUV420P10:
@@ -544,6 +551,44 @@ static int get_pixel_format(AVCodecContext *avctx)
 #endif
 #if CONFIG_AV1_VDPAU_HWACCEL
         *fmtp++ = AV_PIX_FMT_VDPAU;
+#endif
+#if CONFIG_AV1_VULKAN_HWACCEL
+        *fmtp++ = AV_PIX_FMT_VULKAN;
+#endif
+        break;
+    case AV_PIX_FMT_YUV420P12:
+#if CONFIG_AV1_VULKAN_HWACCEL
+        *fmtp++ = AV_PIX_FMT_VULKAN;
+#endif
+        break;
+    case AV_PIX_FMT_YUV422P:
+#if CONFIG_AV1_VULKAN_HWACCEL
+        *fmtp++ = AV_PIX_FMT_VULKAN;
+#endif
+        break;
+    case AV_PIX_FMT_YUV422P10:
+#if CONFIG_AV1_VULKAN_HWACCEL
+        *fmtp++ = AV_PIX_FMT_VULKAN;
+#endif
+        break;
+    case AV_PIX_FMT_YUV422P12:
+#if CONFIG_AV1_VULKAN_HWACCEL
+        *fmtp++ = AV_PIX_FMT_VULKAN;
+#endif
+        break;
+    case AV_PIX_FMT_YUV444P:
+#if CONFIG_AV1_VULKAN_HWACCEL
+        *fmtp++ = AV_PIX_FMT_VULKAN;
+#endif
+        break;
+    case AV_PIX_FMT_YUV444P10:
+#if CONFIG_AV1_VULKAN_HWACCEL
+        *fmtp++ = AV_PIX_FMT_VULKAN;
+#endif
+        break;
+    case AV_PIX_FMT_YUV444P12:
+#if CONFIG_AV1_VULKAN_HWACCEL
+        *fmtp++ = AV_PIX_FMT_VULKAN;
 #endif
         break;
     case AV_PIX_FMT_GRAY8:
@@ -863,8 +908,7 @@ static int av1_frame_alloc(AVCodecContext *avctx, AV1Frame *f)
     if (avctx->hwaccel) {
         const AVHWAccel *hwaccel = avctx->hwaccel;
         if (hwaccel->frame_priv_data_size) {
-            f->hwaccel_priv_buf =
-                av_buffer_allocz(hwaccel->frame_priv_data_size);
+            f->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(avctx, hwaccel);
             if (!f->hwaccel_priv_buf) {
                 ret = AVERROR(ENOMEM);
                 goto fail;
@@ -1412,6 +1456,9 @@ static void av1_decode_flush(AVCodecContext *avctx)
         av_buffer_unref(&itut_t35.payload_ref);
 
     ff_cbs_flush(s->cbc);
+
+    if (avctx->hwaccel && avctx->hwaccel->flush)
+        avctx->hwaccel->flush(avctx);
 }
 
 #define OFFSET(x) offsetof(AV1DecContext, x)
@@ -1464,6 +1511,9 @@ const FFCodec ff_av1_decoder = {
 #if CONFIG_AV1_VDPAU_HWACCEL
         HWACCEL_VDPAU(av1),
 #endif
+#if CONFIG_AV1_VULKAN_HWACCEL
+        HWACCEL_VULKAN(av1),
+#endif
 
         NULL
     },

libavcodec/avcodec.c

@@ -39,6 +39,7 @@
 #include "decode.h"
 #include "encode.h"
 #include "frame_thread_encoder.h"
+#include "hwconfig.h"
 #include "internal.h"
 #include "thread.h"
 
@@ -459,9 +460,7 @@ av_cold int avcodec_close(AVCodecContext *avctx)
 
         av_buffer_unref(&avci->pool);
 
-        if (avctx->hwaccel && avctx->hwaccel->uninit)
-            avctx->hwaccel->uninit(avctx);
-        av_freep(&avci->hwaccel_priv_data);
+        ff_hwaccel_uninit(avctx);
 
         av_bsf_free(&avci->bsf);
 

libavcodec/avcodec.h

@@ -2253,6 +2253,25 @@ typedef struct AVHWAccel {
      * that avctx->hwaccel_priv_data is invalid.
      */
     int (*frame_params)(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx);
+
+    /**
+     * Copy necessary context variables from a previous thread context to the current one.
+     * For thread-safe hwaccels only.
+     */
+    int (*update_thread_context)(AVCodecContext *dst, const AVCodecContext *src);
+
+    /**
+     * Callback to free the hwaccel-specific frame data.
+     *
+     * @param hwctx a pointer to an AVHWDeviceContext.
+     * @param data the per-frame hardware accelerator private data to be freed.
+     */
+    void (*free_frame_priv)(void *hwctx, uint8_t *data);
+
+    /**
+     * Callback to flush the hwaccel state.
+     */
+    void (*flush)(AVCodecContext *avctx);
 } AVHWAccel;
 
 /**

libavcodec/cbs_av1.h

@@ -215,6 +215,8 @@ typedef struct AV1RawFrameHeader {
     uint8_t uniform_tile_spacing_flag;
     uint8_t tile_cols_log2;
     uint8_t tile_rows_log2;
+    uint8_t tile_start_col_sb[AV1_MAX_TILE_COLS];
+    uint8_t tile_start_row_sb[AV1_MAX_TILE_COLS];
     uint8_t width_in_sbs_minus_1[AV1_MAX_TILE_COLS];
     uint8_t height_in_sbs_minus_1[AV1_MAX_TILE_ROWS];
     uint16_t context_update_tile_id;

libavcodec/cbs_av1_syntax_template.c

@@ -626,6 +626,10 @@ static int FUNC(tile_info)(CodedBitstreamContext *ctx, RWContext *rw,
 
         tile_width_sb = (sb_cols + (1 << current->tile_cols_log2) - 1) >>
             current->tile_cols_log2;
+
+        for (int off = 0, i = 0; off < sb_cols; off += tile_width_sb)
+            current->tile_start_col_sb[i++] = off;
+
         current->tile_cols = (sb_cols + tile_width_sb - 1) / tile_width_sb;
 
         min_log2_tile_rows = FFMAX(min_log2_tiles - current->tile_cols_log2, 0);
@@ -634,6 +638,10 @@ static int FUNC(tile_info)(CodedBitstreamContext *ctx, RWContext *rw,
 
         tile_height_sb = (sb_rows + (1 << current->tile_rows_log2) - 1) >>
             current->tile_rows_log2;
+
+        for (int off = 0, i = 0; off < sb_rows; off += tile_height_sb)
+            current->tile_start_row_sb[i++] = off;
+
         current->tile_rows = (sb_rows + tile_height_sb - 1) / tile_height_sb;
 
         for (i = 0; i < current->tile_cols - 1; i++)
@@ -652,6 +660,7 @@ static int FUNC(tile_info)(CodedBitstreamContext *ctx, RWContext *rw,
 
         start_sb = 0;
         for (i = 0; start_sb < sb_cols && i < AV1_MAX_TILE_COLS; i++) {
+            current->tile_start_col_sb[i] = start_sb;
             max_width = FFMIN(sb_cols - start_sb, max_tile_width_sb);
             ns(max_width, width_in_sbs_minus_1[i], 1, i);
             size_sb = current->width_in_sbs_minus_1[i] + 1;
@@ -669,6 +678,7 @@ static int FUNC(tile_info)(CodedBitstreamContext *ctx, RWContext *rw,
 
         start_sb = 0;
         for (i = 0; start_sb < sb_rows && i < AV1_MAX_TILE_ROWS; i++) {
+            current->tile_start_row_sb[i] = start_sb;
             max_height = FFMIN(sb_rows - start_sb, max_tile_height_sb);
             ns(max_height, height_in_sbs_minus_1[i], 1, i);
             size_sb = current->height_in_sbs_minus_1[i] + 1;