test: Add CUDA graph tests for non-batching TensorRT models

qa/L0_cuda_graph/test.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2020-2026, NVIDIA CORPORATION. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -321,8 +321,75 @@ fi
 set -e
 
 set +e
-if [ `grep -c "Context with profile default \[0\] is launching CUDA graph " $SERVER_LOG` != "0" ]; then
-    echo -e "\n***\n*** Failed. Expected 0 execution with CUDA graph\n***"
+# A non-batching model must launch the graph it captured.
+if [ `grep -c "Context with profile default \[0\] is launching CUDA graph " $SERVER_LOG` != "1" ]; then
+    echo -e "\n***\n*** Failed. Expected 1 execution with CUDA graph\n***"
+    RET=1
+fi
+
+if [ `grep -c "Context with profile default \[0\] is being executed for " $SERVER_LOG` != "0" ]; then
+    echo -e "\n***\n*** Failed. Expected 0 execution without CUDA graph\n***"
+    RET=1
+fi
+
+if [ `grep -c "captured CUDA graph for" $SERVER_LOG` != "1" ]; then
+    echo -e "\n***\n*** Failed. Expected 1 CUDA graph to be captured\n***"
+    RET=1
+fi
+set -e
+
+kill $SERVER_PID
+wait $SERVER_PID
+
+# TrtCudaGraphTest.test_nobatch_dynamic_shape
+# Non-batching model with a dynamic shape and an explicit graph_spec
+# (batch_size: 0). The captured graph must be launched when the request shape
+# matches the graph_spec.
+rm -rf ${DATADIR} && mkdir -p ${DATADIR}
+cp -r /data/inferenceserver/${REPO_VERSION}/qa_variable_model_repository/plan_nobatch_float32_float32_float32 ${DATADIR}/
+
+CLIENT_LOG="./nobatch_dynamic_shape.client.log"
+SERVER_LOG="./nobatch_dynamic_shape.inference_server.log"
+echo "optimization { \
+    cuda { \
+        graphs: true \
+        graph_spec [ { \
+            batch_size: 0 \
+            input { key: \"INPUT0\" value: {dim : [16]} } \
+            input { key: \"INPUT1\" value: {dim : [16]} } \
+} ] } }" >> ${DATADIR}/plan_nobatch_float32_float32_float32/config.pbtxt
+
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    exit 1
+fi
+
+set +e
+python $TRT_CUDA_GRAPH_TEST TrtCudaGraphTest.test_nobatch_dynamic_shape plan_nobatch>>$CLIENT_LOG 2>&1
+if [ $? -ne 0 ]; then
+    echo -e "\n***\n*** Test Failed\n***"
+    cat $CLIENT_LOG
+    RET=1
+else
+    check_test_results $TEST_RESULT_FILE 1
+    if [ $? -ne 0 ]; then
+        cat $CLIENT_LOG
+        echo -e "\n***\n*** Test Result Verification Failed\n***"
+        RET=1
+    fi
+fi
+set -e
+
+set +e
+if [ `grep -c "Context with profile 6 \[6\] is launching CUDA graph " $SERVER_LOG` != "1" ]; then
+    echo -e "\n***\n*** Failed. Expected 1 execution with CUDA graph\n***"
+    RET=1
+fi
+
+if [ `grep -c "Context with profile 6 \[6\] is being executed for " $SERVER_LOG` != "1" ]; then
+    echo -e "\n***\n*** Failed. Expected 1 execution without CUDA graph\n***"
     RET=1
 fi
 

qa/L0_cuda_graph/trt_cuda_graph_test.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -27,15 +27,15 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import sys
-
-sys.path.append("../common")
-
 import unittest
 
-import infer_util as iu
 import numpy as np
-import test_util as tu
-from tritonclient.utils import *
+from tritonclient.utils import InferenceServerException
+
+sys.path.append("../common")
+
+import infer_util as iu  # noqa: E402
+import test_util as tu  # noqa: E402
 
 
 class TrtCudaGraphTest(tu.TestResultCollector):
@@ -155,6 +155,13 @@ def test_range_dynamic_shape(self):
     def test_nobatch_fixed_shape(self):
         self._check_infer((16,), 0)
 
+    def test_nobatch_dynamic_shape(self):
+        # Non-batching model with a dynamic shape. The captured graph must be
+        # launched when the request shape matches the graph_spec.
+        self._check_infer((16,), 0)
+        # A different request shape must use regular execution.
+        self._check_infer((20,), 0)
+
 
 if __name__ == "__main__":
     if len(sys.argv) > 2: