diff --git a/qa/L0_cuda_graph/test.sh b/qa/L0_cuda_graph/test.sh index e1bfe2057f..6cfadafde6 100755 --- a/qa/L0_cuda_graph/test.sh +++ b/qa/L0_cuda_graph/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2026, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -321,8 +321,75 @@ fi set -e set +e -if [ `grep -c "Context with profile default \[0\] is launching CUDA graph " $SERVER_LOG` != "0" ]; then - echo -e "\n***\n*** Failed. Expected 0 execution with CUDA graph\n***" +# A non-batching model must launch the graph it captured. +if [ `grep -c "Context with profile default \[0\] is launching CUDA graph " $SERVER_LOG` != "1" ]; then + echo -e "\n***\n*** Failed. Expected 1 execution with CUDA graph\n***" + RET=1 +fi + +if [ `grep -c "Context with profile default \[0\] is being executed for " $SERVER_LOG` != "0" ]; then + echo -e "\n***\n*** Failed. Expected 0 execution without CUDA graph\n***" + RET=1 +fi + +if [ `grep -c "captured CUDA graph for" $SERVER_LOG` != "1" ]; then + echo -e "\n***\n*** Failed. Expected 1 CUDA graph to be captured\n***" + RET=1 +fi +set -e + +kill $SERVER_PID +wait $SERVER_PID + +# TrtCudaGraphTest.test_nobatch_dynamic_shape +# Non-batching model with a dynamic shape and an explicit graph_spec +# (batch_size: 0). The captured graph must be launched when the request shape +# matches the graph_spec. +rm -rf ${DATADIR} && mkdir -p ${DATADIR} +cp -r /data/inferenceserver/${REPO_VERSION}/qa_variable_model_repository/plan_nobatch_float32_float32_float32 ${DATADIR}/ + +CLIENT_LOG="./nobatch_dynamic_shape.client.log" +SERVER_LOG="./nobatch_dynamic_shape.inference_server.log" +echo "optimization { \ + cuda { \ + graphs: true \ + graph_spec [ { \ + batch_size: 0 \ + input { key: \"INPUT0\" value: {dim : [16]} } \ + input { key: \"INPUT1\" value: {dim : [16]} } \ +} ] } }" >> ${DATADIR}/plan_nobatch_float32_float32_float32/config.pbtxt + +run_server +if [ "$SERVER_PID" == "0" ]; then + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 +fi + +set +e +python $TRT_CUDA_GRAPH_TEST TrtCudaGraphTest.test_nobatch_dynamic_shape plan_nobatch>>$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Failed\n***" + cat $CLIENT_LOG + RET=1 +else + check_test_results $TEST_RESULT_FILE 1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Result Verification Failed\n***" + RET=1 + fi +fi +set -e + +set +e +if [ `grep -c "Context with profile 6 \[6\] is launching CUDA graph " $SERVER_LOG` != "1" ]; then + echo -e "\n***\n*** Failed. Expected 1 execution with CUDA graph\n***" + RET=1 +fi + +if [ `grep -c "Context with profile 6 \[6\] is being executed for " $SERVER_LOG` != "1" ]; then + echo -e "\n***\n*** Failed. Expected 1 execution without CUDA graph\n***" RET=1 fi diff --git a/qa/L0_cuda_graph/trt_cuda_graph_test.py b/qa/L0_cuda_graph/trt_cuda_graph_test.py index c77ee5e5f4..d308613777 100755 --- a/qa/L0_cuda_graph/trt_cuda_graph_test.py +++ b/qa/L0_cuda_graph/trt_cuda_graph_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,15 +27,15 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys - -sys.path.append("../common") - import unittest -import infer_util as iu import numpy as np -import test_util as tu -from tritonclient.utils import * +from tritonclient.utils import InferenceServerException + +sys.path.append("../common") + +import infer_util as iu # noqa: E402 +import test_util as tu # noqa: E402 class TrtCudaGraphTest(tu.TestResultCollector): @@ -155,6 +155,13 @@ def test_range_dynamic_shape(self): def test_nobatch_fixed_shape(self): self._check_infer((16,), 0) + def test_nobatch_dynamic_shape(self): + # Non-batching model with a dynamic shape. The captured graph must be + # launched when the request shape matches the graph_spec. + self._check_infer((16,), 0) + # A different request shape must use regular execution. + self._check_infer((20,), 0) + if __name__ == "__main__": if len(sys.argv) > 2: