Fixes for HW accelerated encoders to keep pix_fmt

Author: Josh5

lib/encoders/nvenc.py

@@ -143,21 +143,53 @@ def generate_default_args(self):
                 generic_kwargs["-hwaccel_output_format"] = "cuda"
         return generic_kwargs, advanced_kwargs
 
-    def generate_filtergraphs(self, software_filters, hw_smart_filters):
+    def generate_filtergraphs(self, settings, has_sw_filters, hw_smart_filters, target_fmt="nv12"):
         """
-        Generate the required filter for enabling NVENC HW acceleration
+        Generate the required filter for enabling NVENC/CUDA HW acceleration.
 
         :return:
         """
-        filters = []
-        if software_filters:
-            # Software filters mean the decoder was outputting nv12 surfaces. We need to upload back to cuda first
-            filters.append('hwupload_cuda')
+        generic_kwargs = {}
+        advanced_kwargs = {}
+        hw_filter_args = []
+        sw_filter_prefix_args = []
+        sw_filter_suffix_args = []
+
+        # Check for HW accelerated decode mode
+        # All decode methods ('cuda', 'nvdec', 'cuvid') are handled by the same
+        # filtergraph logic and output CUDA fames. The main FFmpeg command handles the specific decoder.
+        hw_decode = (settings.get_setting('nvenc_decoding_method') or '').lower() in ('cuda', 'nvdec', 'cuvid')
+
+        # Check software format to use
+        sw_fmt = "p010le" if target_fmt == "p010" else "nv12"
+
+        # If we have SW filters:
+        if has_sw_filters:
+            # If we have SW filters and HW decode (CUDA/NVDEC) is enabled, make decoder produce SW frames
+            if hw_decode:
+                generic_kwargs['-hwaccel_output_format'] = sw_fmt
+            # Add filter to upload software frames to CUDA for CUDA filters
+            # Note, format conversion (if any - eg yuv422p10le -> p010le) happens after the software filters.
+            # If a user applies a custom software filter that does not support the pix_fmt, then will need to prefix it with 'format=p010le'
+            sw_filter_suffix_args.append(f'format={sw_fmt}|cuda,hwupload_cuda')
+        # If we have no software filters:
+        else:
+            # Add hwupload filter that can handle when the frame was decoded in software or hardware
+            hw_filter_args.append(f'format={sw_fmt}|cuda,hwupload_cuda')
+
+        # Loop over any HW smart filters to be applied and add them as required.
         for smart_filter in hw_smart_filters:
             if smart_filter.get('scale'):
                 scale_values = smart_filter.get('scale')
-                filters.append('scale_cuda={}:-1'.format(scale_values[0]))
-        return filters
+                hw_filter_args.append('scale_cuda={}:-1'.format(scale_values["width"]))
+
+        return {
+            "generic_kwargs":        generic_kwargs,
+            "advanced_kwargs":       advanced_kwargs,
+            "hw_filter_args":        hw_filter_args,
+            "sw_filter_prefix_args": sw_filter_prefix_args,
+            "sw_filter_suffix_args": sw_filter_suffix_args,
+        }
 
     def encoder_details(self, encoder):
         hardware_devices = list_available_cuda_devices()

lib/encoders/qsv.py

@@ -88,32 +88,52 @@ def generate_default_args(self):
             }
         return generic_kwargs, advanced_kwargs
 
-    def generate_filtergraphs(self, settings, software_filters, hw_smart_filters):
+    def generate_filtergraphs(self, settings, has_sw_filters, hw_smart_filters, target_fmt="nv12"):
         """
         Generate the required filter for enabling QSV HW acceleration
 
         :return:
         """
         generic_kwargs = {}
         advanced_kwargs = {}
-        filter_args = []
-        # If we are using software filters, then disable qsv surfaces.
-        # Instead, output software frames
-        if software_filters:
-            # If we are decoding with QSV, then output to software frames
-            if settings.get_setting('qsv_decoding_method') in ['qsv']:
-                generic_kwargs['-hwaccel_output_format'] = 'nv12'
+        hw_filter_args = []
+        sw_filter_prefix_args = []
+        sw_filter_suffix_args = []
+
+        # Check if we are decoding with QSV
+        hw_decode = settings.get_setting('qsv_decoding_method') in ['qsv']
+        # Check software format to use
+        sw_fmt = "p010le" if target_fmt == "p010" else "nv12"
+
+        # If we have SW filters:
+        if has_sw_filters:
+            # If we have SW filters and HW decode is enabled, make decoder produce SW frames
+            if hw_decode:
+                generic_kwargs['-hwaccel_output_format'] = sw_fmt
             # Add filter to upload software frames to QSV for QSV filters
-            filter_args.append('hwupload=extra_hw_frames=64,format=qsv')
-        # If we have no software filters, but we are decoding in software, then we still need to upload surfaces to QSV
-        elif settings.get_setting('qsv_decoding_method') not in ['qsv']:
-            filter_args.append('hwupload=extra_hw_frames=64,format=qsv')
+            # Note, format conversion (if any - eg yuv422p10le -> p010le) happens after the software filters.
+            # If a user applies a custom software filter that does not support the pix_fmt, then will need to prefix it with 'format=p010le'
+            sw_filter_suffix_args.append(
+                f'format={sw_fmt}|qsv,hwupload=extra_hw_frames=64,format=qsv,vpp_qsv=format={target_fmt}')
+        # If we have no software filters:
+        else:
+            # Add hwupload filter that can handle when the frame was decoded in software or hardware
+            hw_filter_args.append(f'format={sw_fmt}|qsv,hwupload=extra_hw_frames=64,format=qsv,vpp_qsv=format={target_fmt}')
+
         # Loop over any HW smart filters to be applied and add them as required.
         for smart_filter in hw_smart_filters:
             if smart_filter.get('scale'):
                 scale_values = smart_filter.get('scale')
-                filter_args.append('scale_qsv=w={}:h={}'.format(scale_values[0], scale_values[1]))
-        return generic_kwargs, advanced_kwargs, filter_args
+                hw_filter_args.append('scale_qsv=w={}:h=-1'.format(scale_values["width"]))
+
+        # Return built args
+        return {
+            "generic_kwargs":        generic_kwargs,
+            "advanced_kwargs":       advanced_kwargs,
+            "hw_filter_args":        hw_filter_args,
+            "sw_filter_prefix_args": sw_filter_prefix_args,
+            "sw_filter_suffix_args": sw_filter_suffix_args,
+        }
 
     def encoder_details(self, encoder):
         provides = self.provides()
@@ -196,7 +216,8 @@ def get_qsv_decoding_method_form_settings(self):
             "label":          "Enable HW Accelerated Decoding",
             "description":    "Warning: Ensure your device supports decoding the source video codec or it will fail.\n"
                               "This enables full hardware transcode with QSV, using only GPU memory for the entire video transcode.\n"
-                              "If filters are configured in the plugin, decoder will output NV12 software surfaces which are slightly slower.",
+                              "If filters are configured in the plugin, decoder will output NV12 or P010LE software surfaces to\n"
+                              "those filters which will be slightly slower.",
             "sub_setting":    True,
             "input_type":     "select",
             "select_options": [

lib/encoders/vaapi.py

@@ -122,13 +122,51 @@ def generate_default_args(self):
 
         return generic_kwargs, advanced_kwargs
 
-    def generate_filtergraphs(self):
+    def generate_filtergraphs(self, settings, has_sw_filters, hw_smart_filters, target_fmt="nv12"):
         """
         Generate the required filter for enabling VAAPI HW acceleration
 
         :return:
         """
-        return ["format=nv12|vaapi,hwupload"]
+        generic_kwargs = {}
+        advanced_kwargs = {}
+        hw_filter_args = []
+        sw_filter_prefix_args = []
+        sw_filter_suffix_args = []
+
+        # Check if we are decoding with VAAPI
+        hw_decode = settings.get_setting('vaapi_enabled_hw_decoding')
+        # Check software format to use
+        sw_fmt = "p010le" if target_fmt == "p010" else "nv12"
+
+        # If we have SW filters:
+        if has_sw_filters:
+            # If we have SW filters and HW decode is enabled, make decoder produce SW frames
+            if hw_decode:
+                generic_kwargs['-hwaccel_output_format'] = sw_fmt
+            # Add filter to upload software frames to VAAPI for VAAPI filters
+            # Note, format conversion (if any - eg yuv422p10le -> p010le) happens after the software filters.
+            # If a user applies a custom software filter that does not support the pix_fmt, then will need to prefix it with 'format=p010le'
+            sw_filter_suffix_args.append(f'format={sw_fmt}|vaapi,hwupload')
+        # If we have no software filters:
+        else:
+            # Add hwupload filter that can handle when the frame was decoded in software or hardware
+            hw_filter_args.append(f'format={sw_fmt}|vaapi,hwupload')
+
+        # Loop over any HW smart filters to be applied and add them as required.
+        for smart_filter in hw_smart_filters:
+            if smart_filter.get('scale'):
+                scale_values = smart_filter.get('scale')
+                hw_filter_args.append('scale_vaapi=w={}:h=-1'.format(scale_values["width"]))
+
+        # Return built args
+        return {
+            "generic_kwargs":        generic_kwargs,
+            "advanced_kwargs":       advanced_kwargs,
+            "hw_filter_args":        hw_filter_args,
+            "sw_filter_prefix_args": sw_filter_prefix_args,
+            "sw_filter_suffix_args": sw_filter_suffix_args,
+        }
 
     def encoder_details(self, encoder):
         provides = self.provides()
@@ -215,10 +253,14 @@ def get_vaapi_device_form_settings(self):
         return values
 
     def get_vaapi_enabled_hw_decoding_form_settings(self):
+        # TODO: Change this to a select dropdown like the others
         values = {
             "label":       "Enable HW Decoding",
+            "description": "Warning: Ensure your device supports decoding the source video codec or it will fail.\n"
+                           "This enables full hardware transcode with VAAPI, using only GPU memory for the entire video transcode.\n"
+                           "If filters are configured in the plugin, decoder will output NV12 or P010LE software surfaces to\n"
+                           "those filters which will be slightly slower.",
             "sub_setting": True,
-            "description": "Will fallback to software decoding and hardware encoding when the input codec is not supported.",
         }
         if self.settings.get_setting('mode') not in ['standard']:
             values["display"] = "hidden"