From 90ac4723202af2fa5729a46f8426dc495f90c0ab Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 23 Jul 2026 20:46:52 +0530 Subject: [PATCH 1/6] Update --- qa/L0_cuda_graph/test.sh | 73 ++++++++++++++++++++++++- qa/L0_cuda_graph/trt_cuda_graph_test.py | 7 ++- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/qa/L0_cuda_graph/test.sh b/qa/L0_cuda_graph/test.sh index e1bfe2057f..09a7c8c0d3 100755 --- a/qa/L0_cuda_graph/test.sh +++ b/qa/L0_cuda_graph/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2026, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -321,8 +321,75 @@ fi set -e set +e -if [ `grep -c "Context with profile default \[0\] is launching CUDA graph " $SERVER_LOG` != "0" ]; then - echo -e "\n***\n*** Failed. Expected 0 execution with CUDA graph\n***" +# A non-batching model must launch the graph it captured. +if [ `grep -c "Context with profile default \[0\] is launching CUDA graph " $SERVER_LOG` != "1" ]; then + echo -e "\n***\n*** Failed. Expected 1 execution with CUDA graph\n***" + RET=1 +fi + +if [ `grep -c "Context with profile default \[0\] is being executed for " $SERVER_LOG` != "0" ]; then + echo -e "\n***\n*** Failed. Expected 0 execution without CUDA graph\n***" + RET=1 +fi + +if [ `grep -c "captured CUDA graph for" $SERVER_LOG` != "1" ]; then + echo -e "\n***\n*** Failed. Expected 1 CUDA graph to be captured\n***" + RET=1 +fi +set -e + +kill $SERVER_PID +wait $SERVER_PID + +# TrtCudaGraphTest.test_nobatch_dynamic_shape +# Non-batching model with a dynamic shape and an explicit graph_spec +# (batch_size: 0). The captured graph must be launched when the request shape +# matches the graph_spec. +rm -rf ${DATADIR} && mkdir -p ${DATADIR} +cp -r /data/inferenceserver/${REPO_VERSION}/qa_variable_model_repository/plan_nobatch_float32_float32_float32 ${DATADIR}/ + +CLIENT_LOG="./nobatch_dynamic_shape.client.log" +SERVER_LOG="./nobatch_dynamic_shape.inference_server.log" +echo "optimization { \ + cuda { \ + graphs: true \ + graph_spec [ { \ + batch_size: 0 \ + input { key: \"INPUT0\" value: {dim : [16]} } \ + input { key: \"INPUT1\" value: {dim : [16]} } \ +} ] } }" >> ${DATADIR}/plan_nobatch_float32_float32_float32/config.pbtxt + +run_server +if [ "$SERVER_PID" == "0" ]; then + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 +fi + +set +e +python $TRT_CUDA_GRAPH_TEST TrtCudaGraphTest.test_nobatch_dynamic_shape plan_nobatch>>$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Failed\n***" + cat $CLIENT_LOG + RET=1 +else + check_test_results $TEST_RESULT_FILE 1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Result Verification Failed\n***" + RET=1 + fi +fi +set -e + +set +e +if [ `grep -c "is launching CUDA graph " $SERVER_LOG` != "1" ]; then + echo -e "\n***\n*** Failed. Expected 1 execution with CUDA graph\n***" + RET=1 +fi + +if [ `grep -c "is being executed for " $SERVER_LOG` != "0" ]; then + echo -e "\n***\n*** Failed. Expected 0 execution without CUDA graph\n***" RET=1 fi diff --git a/qa/L0_cuda_graph/trt_cuda_graph_test.py b/qa/L0_cuda_graph/trt_cuda_graph_test.py index c77ee5e5f4..740b08a4a4 100755 --- a/qa/L0_cuda_graph/trt_cuda_graph_test.py +++ b/qa/L0_cuda_graph/trt_cuda_graph_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -155,6 +155,11 @@ def test_range_dynamic_shape(self): def test_nobatch_fixed_shape(self): self._check_infer((16,), 0) + def test_nobatch_dynamic_shape(self): + # Non-batching model with a dynamic shape. The captured graph must be + # launched when the request shape matches the graph_spec. + self._check_infer((16,), 0) + if __name__ == "__main__": if len(sys.argv) > 2: From 5ab4a57905c15ad3794f1778a9d35e3f7e1e1762 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 23 Jul 2026 20:55:41 +0530 Subject: [PATCH 2/6] Update --- qa/L0_cuda_graph/trt_cuda_graph_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qa/L0_cuda_graph/trt_cuda_graph_test.py b/qa/L0_cuda_graph/trt_cuda_graph_test.py index 740b08a4a4..c0be5683ee 100755 --- a/qa/L0_cuda_graph/trt_cuda_graph_test.py +++ b/qa/L0_cuda_graph/trt_cuda_graph_test.py @@ -27,15 +27,15 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys +import unittest -sys.path.append("../common") +import numpy as np +from tritonclient.utils import InferenceServerException -import unittest +sys.path.append("../common") import infer_util as iu -import numpy as np import test_util as tu -from tritonclient.utils import * class TrtCudaGraphTest(tu.TestResultCollector): From b87d050e18ecd29c67d22931968797fd8b5606ff Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Fri, 24 Jul 2026 17:47:29 +0530 Subject: [PATCH 3/6] Update --- qa/L0_cuda_graph/trt_cuda_graph_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qa/L0_cuda_graph/trt_cuda_graph_test.py b/qa/L0_cuda_graph/trt_cuda_graph_test.py index c0be5683ee..2280bb7f0d 100755 --- a/qa/L0_cuda_graph/trt_cuda_graph_test.py +++ b/qa/L0_cuda_graph/trt_cuda_graph_test.py @@ -26,16 +26,16 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import os import sys import unittest +import infer_util as iu +import test_util as tu import numpy as np from tritonclient.utils import InferenceServerException -sys.path.append("../common") - -import infer_util as iu -import test_util as tu +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../common")) class TrtCudaGraphTest(tu.TestResultCollector): From b32eb4ce0660ec62c724c7f3c6e1b9db20a4ef7f Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Fri, 24 Jul 2026 17:48:56 +0530 Subject: [PATCH 4/6] Update --- qa/L0_cuda_graph/trt_cuda_graph_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/L0_cuda_graph/trt_cuda_graph_test.py b/qa/L0_cuda_graph/trt_cuda_graph_test.py index 2280bb7f0d..67931cf1ad 100755 --- a/qa/L0_cuda_graph/trt_cuda_graph_test.py +++ b/qa/L0_cuda_graph/trt_cuda_graph_test.py @@ -31,8 +31,8 @@ import unittest import infer_util as iu -import test_util as tu import numpy as np +import test_util as tu from tritonclient.utils import InferenceServerException sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../common")) From 4b8854cd87a1a94ec590a2f7c2f7a8fcc7db593c Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Fri, 24 Jul 2026 17:50:54 +0530 Subject: [PATCH 5/6] Undo change --- qa/L0_cuda_graph/trt_cuda_graph_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qa/L0_cuda_graph/trt_cuda_graph_test.py b/qa/L0_cuda_graph/trt_cuda_graph_test.py index 67931cf1ad..221a690612 100755 --- a/qa/L0_cuda_graph/trt_cuda_graph_test.py +++ b/qa/L0_cuda_graph/trt_cuda_graph_test.py @@ -26,7 +26,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os import sys import unittest @@ -35,7 +34,7 @@ import test_util as tu from tritonclient.utils import InferenceServerException -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../common")) +sys.path.append("../common") class TrtCudaGraphTest(tu.TestResultCollector): From e8b6f00b93cf5e288ea5b4add0f809518300f4b4 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Fri, 24 Jul 2026 19:07:50 +0530 Subject: [PATCH 6/6] Update --- qa/L0_cuda_graph/test.sh | 6 +++--- qa/L0_cuda_graph/trt_cuda_graph_test.py | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/qa/L0_cuda_graph/test.sh b/qa/L0_cuda_graph/test.sh index 09a7c8c0d3..6cfadafde6 100755 --- a/qa/L0_cuda_graph/test.sh +++ b/qa/L0_cuda_graph/test.sh @@ -383,13 +383,13 @@ fi set -e set +e -if [ `grep -c "is launching CUDA graph " $SERVER_LOG` != "1" ]; then +if [ `grep -c "Context with profile 6 \[6\] is launching CUDA graph " $SERVER_LOG` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 execution with CUDA graph\n***" RET=1 fi -if [ `grep -c "is being executed for " $SERVER_LOG` != "0" ]; then - echo -e "\n***\n*** Failed. Expected 0 execution without CUDA graph\n***" +if [ `grep -c "Context with profile 6 \[6\] is being executed for " $SERVER_LOG` != "1" ]; then + echo -e "\n***\n*** Failed. Expected 1 execution without CUDA graph\n***" RET=1 fi diff --git a/qa/L0_cuda_graph/trt_cuda_graph_test.py b/qa/L0_cuda_graph/trt_cuda_graph_test.py index 221a690612..d308613777 100755 --- a/qa/L0_cuda_graph/trt_cuda_graph_test.py +++ b/qa/L0_cuda_graph/trt_cuda_graph_test.py @@ -29,13 +29,14 @@ import sys import unittest -import infer_util as iu import numpy as np -import test_util as tu from tritonclient.utils import InferenceServerException sys.path.append("../common") +import infer_util as iu # noqa: E402 +import test_util as tu # noqa: E402 + class TrtCudaGraphTest(tu.TestResultCollector): MODELNAME = "plan" @@ -158,6 +159,8 @@ def test_nobatch_dynamic_shape(self): # Non-batching model with a dynamic shape. The captured graph must be # launched when the request shape matches the graph_spec. self._check_infer((16,), 0) + # A different request shape must use regular execution. + self._check_infer((20,), 0) if __name__ == "__main__":