From 1d63485809f78b0ee2bdcb5117d1dd57627040e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Mon, 20 Jul 2026 14:53:29 -0500 Subject: [PATCH] test: add custom metrics reload regression --- .../custom_metrics_reload_test.py | 59 +++++++++++++++++++ qa/L0_backend_python/custom_metrics/test.sh | 13 +++- .../custom_metrics_reload/config.pbtxt | 43 ++++++++++++++ .../custom_metrics_reload/model.py | 54 +++++++++++++++++ 4 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 qa/L0_backend_python/custom_metrics/custom_metrics_reload_test.py create mode 100644 qa/python_models/custom_metrics_reload/config.pbtxt create mode 100644 qa/python_models/custom_metrics_reload/model.py diff --git a/qa/L0_backend_python/custom_metrics/custom_metrics_reload_test.py b/qa/L0_backend_python/custom_metrics/custom_metrics_reload_test.py new file mode 100644 index 0000000000..3a15f093f8 --- /dev/null +++ b/qa/L0_backend_python/custom_metrics/custom_metrics_reload_test.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import pathlib + +import numpy as np +import tritonclient.http as httpclient + +_tritonserver_ipaddr = os.environ.get("TRITONSERVER_IPADDR", "localhost") + + +class TestCustomMetricsReload: + def _infer(self, client, model_name): + result = client.infer(model_name, [], client_timeout=240) + output0 = result.as_numpy("OUTPUT0") + assert np.array_equal(output0, np.array([1.0], dtype=np.float32)) + + def test_histogram_metric_survives_model_reload(self): + model_name = "custom_metrics_reload" + model_path = pathlib.Path("models") / model_name / "1" / "model.py" + + with httpclient.InferenceServerClient(f"{_tritonserver_ipaddr}:8000") as client: + assert client.is_model_ready(model_name) + + for _ in range(6): + self._infer(client, model_name) + + os.utime(model_path) + client.load_model(model_name) + assert client.is_model_ready(model_name) + + for _ in range(6): + self._infer(client, model_name) diff --git a/qa/L0_backend_python/custom_metrics/test.sh b/qa/L0_backend_python/custom_metrics/test.sh index 9020c7ebfd..35281237f9 100755 --- a/qa/L0_backend_python/custom_metrics/test.sh +++ b/qa/L0_backend_python/custom_metrics/test.sh @@ -30,7 +30,7 @@ CLIENT_LOG="./custom_metrics_client.log" TEST_RESULT_FILE='test_results.txt' source ../../common/util.sh -SERVER_ARGS="--model-repository=${MODELDIR}/custom_metrics/models --backend-directory=${BACKEND_DIR} --log-verbose=1" +SERVER_ARGS="--model-repository=${MODELDIR}/custom_metrics/models --backend-directory=${BACKEND_DIR} --model-control-mode=explicit --load-model=* --log-verbose=1" SERVER_LOG="./custom_metrics_server.log" RET=0 @@ -40,6 +40,10 @@ mkdir -p models/custom_metrics/1/ cp ../../python_models/custom_metrics/model.py models/custom_metrics/1/ cp ../../python_models/custom_metrics/config.pbtxt models/custom_metrics +mkdir -p models/custom_metrics_reload/1/ +cp ../../python_models/custom_metrics_reload/model.py models/custom_metrics_reload/1/ +cp ../../python_models/custom_metrics_reload/config.pbtxt models/custom_metrics_reload + run_server if [ "$SERVER_PID" == "0" ]; then echo -e "\n***\n*** Failed to start $SERVER\n***" @@ -57,6 +61,13 @@ if [ $? -ne 0 ]; then RET=1 fi +python3 -m pytest --junitxml="custom_metrics_reload.report.xml" custom_metrics_reload_test.py >> $CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + echo -e "\n***\n*** 'Custom Metrics Reload' test FAILED. \n***" + cat $CLIENT_LOG + RET=1 +fi + set -e kill_server diff --git a/qa/python_models/custom_metrics_reload/config.pbtxt b/qa/python_models/custom_metrics_reload/config.pbtxt new file mode 100644 index 0000000000..1086ee7d92 --- /dev/null +++ b/qa/python_models/custom_metrics_reload/config.pbtxt @@ -0,0 +1,43 @@ +# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +name: "custom_metrics_reload" +backend: "python" + +output [ + { + name: "OUTPUT0" + data_type: TYPE_FP32 + dims: [ 1 ] + } +] + +instance_group [ + { + count: 3 + kind: KIND_CPU + } +] diff --git a/qa/python_models/custom_metrics_reload/model.py b/qa/python_models/custom_metrics_reload/model.py new file mode 100644 index 0000000000..484f81a947 --- /dev/null +++ b/qa/python_models/custom_metrics_reload/model.py @@ -0,0 +1,54 @@ +# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import numpy as np +import triton_python_backend_utils as pb_utils + + +class TritonPythonModel: + def initialize(self, args): + metric_family = pb_utils.MetricFamily( + name="test_custom_metrics_reload_histogram", + description="test histogram survives python model reload", + kind=pb_utils.MetricFamily.HISTOGRAM, + ) + + self._metric = metric_family.Metric( + labels={"model": "custom_metrics_reload"}, + buckets=[0.1, 1.0, 2.5, 5.0, 10.0], + ) + self._metric_family = metric_family + + def execute(self, requests): + responses = [] + for _ in requests: + self._metric.observe(0.5) + responses.append( + pb_utils.InferenceResponse( + [pb_utils.Tensor("OUTPUT0", np.array([1.0], dtype=np.float32))] + ) + ) + return responses