From 1b532926ab276969a9eefc8e15bbe1747bbc5767 Mon Sep 17 00:00:00 2001 From: alireza787b Date: Mon, 27 Jul 2026 07:58:48 +0000 Subject: [PATCH] fix(VideoManager): preserve Windows software frames --- .../GStreamer/gstqgc/gstqgcvideosinkbin.cc | 35 +----- test/VideoManager/GStreamer/GStreamerTest.cc | 2 + test/VideoManager/GStreamer/GStreamerTest.h | 2 + .../GStreamer/gstqgc/GStreamerGstQgcTest.cc | 105 +++++++++++++++++- 4 files changed, 107 insertions(+), 37 deletions(-) diff --git a/src/VideoManager/VideoReceiver/GStreamer/gstqgc/gstqgcvideosinkbin.cc b/src/VideoManager/VideoReceiver/GStreamer/gstqgc/gstqgcvideosinkbin.cc index 5b5691142bd2..302757d4b81c 100644 --- a/src/VideoManager/VideoReceiver/GStreamer/gstqgc/gstqgcvideosinkbin.cc +++ b/src/VideoManager/VideoReceiver/GStreamer/gstqgc/gstqgcvideosinkbin.cc @@ -296,43 +296,12 @@ static gboolean wireGpuPath(GstQgcVideoSinkBin* self, GstElement* videosink, Gst } #else GstElement* glupload = nullptr; -#if defined(QGC_HAS_GST_D3D11_GPU_PATH) - // Memory-aware D3D11 conversion: d3d11upload lifts a software decoder's system-memory frame into - // D3D11Memory and d3d11convert normalizes any non-Qt-renderable format (e.g. 4:4:4/packed) to a sink - // format. Both passthrough (no copy) when the HW decoder already delivers an advertised D3D11Memory - // format, so the zero-copy fast path is preserved. Missing factories fall back to the direct link. - // D3D11-only is safe: gpuZeroCopyAllowedForCurrentGraphicsApi() routes a D3D12 RHI to the CPU path, so - // wireGpuPath only runs under D3D11 RHI where the decoder family is aligned to D3D11Memory output. - GstElement* d3d11upload = gst_element_factory_make("d3d11upload", nullptr); - GstElement* d3d11convert = gst_element_factory_make("d3d11convert", nullptr); - if (d3d11upload && d3d11convert) { - if (!chain.adopt(d3d11upload)) { - gst_clear_object(&d3d11convert); - GST_ERROR_OBJECT(self, "Failed to add d3d11upload to GPU path"); - return FALSE; - } - if (!chain.adopt(d3d11convert)) { - GST_ERROR_OBJECT(self, "Failed to add d3d11convert to GPU path"); - return FALSE; - } - if (!chain.linkChain({d3d11upload, d3d11convert, capsf, videosink}) || !chain.ghostSink(d3d11upload)) { - GST_ERROR_OBJECT(self, "Failed to link/ghost d3d11upload→d3d11convert→capsfilter→qgcqvideosink"); - return FALSE; - } - } else { - gst_clear_object(&d3d11upload); - gst_clear_object(&d3d11convert); - if (!chain.linkChain({capsf, videosink}) || !chain.ghostSink(capsf)) { - GST_ERROR_OBJECT(self, "Failed to link/ghost qgcqvideosink (GPU path, d3d11 convert unavailable)"); - return FALSE; - } - } -#else + // Keep native GPU memory direct. buildGpuCapsString() also offers system memory, which + // qgcqvideosink maps through its CPU fallback instead of uploading software-decoded frames. if (!chain.linkChain({capsf, videosink}) || !chain.ghostSink(capsf)) { GST_ERROR_OBJECT(self, "Failed to link/ghost qgcqvideosink (GPU path)"); return FALSE; } -#endif #endif chain.commit(); diff --git a/test/VideoManager/GStreamer/GStreamerTest.cc b/test/VideoManager/GStreamer/GStreamerTest.cc index 9f539c615f21..20a570c4215e 100644 --- a/test/VideoManager/GStreamer/GStreamerTest.cc +++ b/test/VideoManager/GStreamer/GStreamerTest.cc @@ -534,6 +534,8 @@ QGC_GST_SKIP_TEST(_testDmaBufTiledImportAvoidsTexStorage) QGC_GST_SKIP_TEST(_testVulkanDispatchDemotesToCpu) QGC_GST_SKIP_TEST(_testCapsCacheInvalidation) QGC_GST_SKIP_TEST(_testGpuZeroCopyFallback) +QGC_GST_SKIP_TEST(_testWindowsGpuSinkPreservesSoftwarePixels) +QGC_GST_SKIP_TEST(_testWindowsGpuSinkPreservesDirectD3D11Memory) QGC_GST_SKIP_TEST(_testAppsinkTeardownUnderLoad) QGC_GST_SKIP_TEST(_testBridgeDispatcherFanout) QGC_GST_SKIP_TEST(_testHwBufferMapTexturesGuard) diff --git a/test/VideoManager/GStreamer/GStreamerTest.h b/test/VideoManager/GStreamer/GStreamerTest.h index f1658fe73461..f45f2c6cc76f 100644 --- a/test/VideoManager/GStreamer/GStreamerTest.h +++ b/test/VideoManager/GStreamer/GStreamerTest.h @@ -45,6 +45,8 @@ private slots: void _testVulkanDispatchDemotesToCpu(); void _testCapsCacheInvalidation(); void _testGpuZeroCopyFallback(); + void _testWindowsGpuSinkPreservesSoftwarePixels(); + void _testWindowsGpuSinkPreservesDirectD3D11Memory(); void _testAppsinkTeardownUnderLoad(); void _testBridgeDispatcherFanout(); void _testHwBufferMapTexturesGuard(); diff --git a/test/VideoManager/GStreamer/gstqgc/GStreamerGstQgcTest.cc b/test/VideoManager/GStreamer/gstqgc/GStreamerGstQgcTest.cc index a919f3339cba..fd511b37bab5 100644 --- a/test/VideoManager/GStreamer/gstqgc/GStreamerGstQgcTest.cc +++ b/test/VideoManager/GStreamer/gstqgc/GStreamerGstQgcTest.cc @@ -47,15 +47,17 @@ #include #include #include +#include +#include #include #include #include #include #include -#include -#include #include #include +#include +#include #include #include #include @@ -407,6 +409,10 @@ void GStreamerTest::_testQgcVideoSinkBinGpuZeroCopyProperty() #if defined(QGC_GST_BIN_USE_GLUPLOAD) || (defined(Q_OS_LINUX) && defined(QGC_GST_BIN_USE_DMABUF)) bool sawGlupload = false; bool sawGlColorConvert = false; +#endif +#if defined(Q_OS_WIN) && (defined(QGC_HAS_GST_D3D11_GPU_PATH) || defined(QGC_HAS_GST_D3D12_GPU_PATH)) + bool sawD3D11Upload = false; + bool sawD3D11Convert = false; #endif gboolean done = FALSE; GValue val = G_VALUE_INIT; @@ -423,6 +429,14 @@ void GStreamerTest::_testQgcVideoSinkBinGpuZeroCopyProperty() if (name && QString::fromUtf8(name).startsWith(QStringLiteral("glcolorconvert"))) { sawGlColorConvert = true; } +#endif +#if defined(Q_OS_WIN) && (defined(QGC_HAS_GST_D3D11_GPU_PATH) || defined(QGC_HAS_GST_D3D12_GPU_PATH)) + if (name && QString::fromUtf8(name).startsWith(QStringLiteral("d3d11upload"))) { + sawD3D11Upload = true; + } + if (name && QString::fromUtf8(name).startsWith(QStringLiteral("d3d11convert"))) { + sawD3D11Convert = true; + } #endif g_free(name); g_value_reset(&val); @@ -445,6 +459,16 @@ void GStreamerTest::_testQgcVideoSinkBinGpuZeroCopyProperty() GstCaps* caps = nullptr; g_object_get(formatFilter, "caps", &caps, NULL); QVERIFY2(caps, "GPU bin format capsfilter has null caps"); +#if defined(Q_OS_WIN) && (defined(QGC_HAS_GST_D3D11_GPU_PATH) || defined(QGC_HAS_GST_D3D12_GPU_PATH)) + bool hasSystemMemoryCaps = false; + for (guint i = 0; i < gst_caps_get_size(caps); ++i) { + const GstCapsFeatures* features = gst_caps_get_features(caps, i); + if (!features || gst_caps_features_is_equal(features, GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY)) { + hasSystemMemoryCaps = true; + break; + } + } +#endif gchar* capsStr = gst_caps_to_string(caps); const QString s = QString::fromUtf8(capsStr ? capsStr : ""); g_free(capsStr); @@ -475,6 +499,9 @@ void GStreamerTest::_testQgcVideoSinkBinGpuZeroCopyProperty() qUtf8Printable(QStringLiteral("Direct DMABuf bin caps must not advertise GLMemory first: ") + s)); #elif defined(Q_OS_WIN) && (defined(QGC_HAS_GST_D3D11_GPU_PATH) || defined(QGC_HAS_GST_D3D12_GPU_PATH)) QCOMPARE(elementCount, 2); + QVERIFY2(!sawD3D11Upload, "Windows GPU sink must not upload software-decoded frames to D3D11"); + QVERIFY2(!sawD3D11Convert, "Windows GPU sink must not force software-decoded frames through D3D11 conversion"); + QVERIFY2(hasSystemMemoryCaps, "Windows GPU sink must retain a system-memory fallback for software decoders"); #if defined(QGC_HAS_GST_D3D11_GPU_PATH) QVERIFY2(s.contains(QStringLiteral("memory:D3D11Memory")), qUtf8Printable(QStringLiteral("GPU bin caps missing memory:D3D11Memory: ") + s)); @@ -539,16 +566,20 @@ struct PipelineRunResult int frameCount = 0; QSize lastFrameSize; QVideoFrameFormat::PixelFormat lastPixelFormat = QVideoFrameFormat::Format_Invalid; + QColor centerPixel; + bool centerPixelValid = false; bool eos = false; QString errorMessage; }; PipelineRunResult runPipelineThroughQVideoSink(QVideoSink& videoSink, QGCQVideoSinkController*& outController, - const char* capsLine, int numBuffers = 5, bool gpuZerocopy = false) + const char* capsLine, int numBuffers = 5, bool gpuZerocopy = false, + const char* sourceProperties = "", bool captureCenterPixel = false) { PipelineRunResult r; const QString launch = - QStringLiteral("videotestsrc num-buffers=%1 ! %2 ! qgcvideosinkbin name=sink gpu-zerocopy=%3") + QStringLiteral("videotestsrc %1 num-buffers=%2 ! %3 ! qgcvideosinkbin name=sink gpu-zerocopy=%4") + .arg(QString::fromUtf8(sourceProperties)) .arg(numBuffers) .arg(QString::fromUtf8(capsLine)) .arg(gpuZerocopy ? "true" : "false"); @@ -583,6 +614,13 @@ PipelineRunResult runPipelineThroughQVideoSink(QVideoSink& videoSink, QGCQVideoS ++r.frameCount; r.lastFrameSize = f.size(); r.lastPixelFormat = f.pixelFormat(); + if (captureCenterPixel) { + const QImage image = f.toImage(); + if (!image.isNull()) { + r.centerPixel = image.pixelColor(image.width() / 2, image.height() / 2); + r.centerPixelValid = true; + } + } }); gst_object_unref(sinkBin); @@ -680,6 +718,65 @@ void GStreamerTest::_testGpuZeroCopyFallback() #endif } +void GStreamerTest::_testWindowsGpuSinkPreservesSoftwarePixels() +{ +#if defined(Q_OS_WIN) && defined(QGC_HAS_GST_D3D11_GPU_PATH) + GStreamer::redirectGLibLogging(); + QVERIFY2(GStreamer::completeInit(), "completeInit failed"); + + QVideoSink videoSink; + QGCQVideoSinkController* controller = nullptr; + const auto result = runPipelineThroughQVideoSink(videoSink, controller, + "video/x-raw,format=BGRA,width=320,height=240,framerate=30/1", + /*numBuffers*/ 5, /*gpuZerocopy*/ true, "pattern=red", + /*captureCenterPixel*/ true); + + QVERIFY2(result.eos, qUtf8Printable(QStringLiteral("Software-frame pipeline: %1").arg(result.errorMessage))); + QTRY_VERIFY_WITH_TIMEOUT(result.frameCount > 0, 2000); + QCOMPARE(result.lastPixelFormat, QVideoFrameFormat::Format_BGRA8888); + QVERIFY2(result.centerPixelValid, "System-memory fallback frame could not be mapped for pixel verification"); + QVERIFY2((result.centerPixel.red() > 200) && (result.centerPixel.green() < 50) && (result.centerPixel.blue() < 50), + qUtf8Printable(QStringLiteral("Expected a red software frame, received RGB(%1,%2,%3)") + .arg(result.centerPixel.red()) + .arg(result.centerPixel.green()) + .arg(result.centerPixel.blue()))); +#else + QSKIP("Windows D3D11 GPU sink path is unavailable in this build"); +#endif +} + +void GStreamerTest::_testWindowsGpuSinkPreservesDirectD3D11Memory() +{ +#if defined(Q_OS_WIN) && defined(QGC_HAS_GST_D3D11_GPU_PATH) + GStreamer::redirectGLibLogging(); + QVERIFY2(GStreamer::completeInit(), "completeInit failed"); + + GstElementFactory* uploadFactory = gst_element_factory_find("d3d11upload"); + if (!uploadFactory) { + QSKIP("d3d11upload is unavailable"); + } + gst_object_unref(uploadFactory); + + QVideoSink videoSink; + QGCQVideoSinkController* controller = nullptr; + auto result = + runPipelineThroughQVideoSink(videoSink, controller, + "video/x-raw,format=NV12,width=320,height=240,framerate=30/1 ! d3d11upload ! " + "video/x-raw(memory:D3D11Memory),format=NV12", + /*numBuffers*/ 5, /*gpuZerocopy*/ true, "pattern=red"); + if (!result.eos && (result.errorMessage.contains(QStringLiteral("D3D"), Qt::CaseInsensitive) || + result.errorMessage.contains(QStringLiteral("device"), Qt::CaseInsensitive))) { + QSKIP(qPrintable(QStringLiteral("D3D11 device unavailable in this runner: %1").arg(result.errorMessage))); + } + + QVERIFY2(result.eos, qUtf8Printable(QStringLiteral("D3D11 pipeline: %1").arg(result.errorMessage))); + QTRY_VERIFY_WITH_TIMEOUT(result.frameCount > 0, 2000); + QCOMPARE(result.lastPixelFormat, QVideoFrameFormat::Format_NV12); +#else + QSKIP("D3D11 GPU sink path is only available in supported Windows builds"); +#endif +} + void GStreamerTest::_testAppsinkTeardownUnderLoad() { GStreamer::redirectGLibLogging();