diff --git a/CMakeLists.txt b/CMakeLists.txt index bd91a52109..da4902ac5f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,6 +167,7 @@ if (OPENSSL_ROOT_DIR) endif() set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/${LIB_DIR}/cmake/protobuf") +set(_FINDPACKAGE_UTF8_RANGE_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/${LIB_DIR}/cmake/utf8_range") set(_FINDPACKAGE_OPENTELEMETRY_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/opentelemetry-cpp/${LIB_DIR}/cmake/opentelemetry-cpp") @@ -199,6 +200,7 @@ ExternalProject_Add(triton-server BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/triton-server" CMAKE_CACHE_ARGS -DProtobuf_DIR:PATH=${_FINDPACKAGE_PROTOBUF_CONFIG_DIR} + -Dutf8_range_DIR:PATH=${_FINDPACKAGE_UTF8_RANGE_CONFIG_DIR} ${_CMAKE_ARGS_OPENSSL_ROOT_DIR} ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} diff --git a/Dockerfile.QA b/Dockerfile.QA index 739afd63ab..106738d8fb 100644 --- a/Dockerfile.QA +++ b/Dockerfile.QA @@ -352,8 +352,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN rm -f /usr/bin/python && \ ln -s /usr/bin/python3 /usr/bin/python -RUN pip3 install --upgrade "numpy<2" pillow attrdict future "grpcio>=1.81.1" requests gsutil \ - "awscli<=1.36.40" six grpcio-channelz prettytable virtualenv \ +RUN pip3 install --upgrade "numpy<2" pillow attrdict future "grpcio==1.81.1" requests gsutil \ + "awscli<=1.36.40" six "grpcio-channelz==1.81.1" prettytable virtualenv \ check-jsonschema # go needed for example go client test. diff --git a/Dockerfile.sdk b/Dockerfile.sdk index cdb07a9868..1fa015f494 100644 --- a/Dockerfile.sdk +++ b/Dockerfile.sdk @@ -83,7 +83,7 @@ RUN apt-get update && \ software-properties-common \ vim \ wget && \ - pip3 install --upgrade grpcio-tools cmake==4.0.3 auditwheel + pip3 install --upgrade "grpcio-tools==1.81.1" cmake==4.0.3 auditwheel ENV CMAKE_POLICY_MINIMUM_REQUIRED=3.5 @@ -182,7 +182,7 @@ RUN apt-get update && \ python3-wheel \ vim \ wget && \ - pip3 install "grpcio>=1.81.1" grpcio-tools && \ + pip3 install "grpcio==1.81.1" "grpcio-tools==1.81.1" && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*; diff --git a/qa/L0_lifecycle/lifecycle_test.py b/qa/L0_lifecycle/lifecycle_test.py index 2dba1c25d5..142fb0494a 100755 --- a/qa/L0_lifecycle/lifecycle_test.py +++ b/qa/L0_lifecycle/lifecycle_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,6 +26,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# Local test modules are imported after adjusting sys.path for ../common. +# flake8: noqa: E402 import sys sys.path.append("../common") @@ -308,7 +310,7 @@ def test_parse_error_no_model_config(self): self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - md = triton_client.get_model_metadata(model_name, "1") + triton_client.get_model_metadata(model_name, "1") self.assertTrue( False, "expected model '" @@ -2256,7 +2258,6 @@ def test_multiple_model_repository_control_startup_models(self): def test_model_repository_index(self): # use model control EXPLICIT and --load-model to load a subset of models # in model repository - tensor_shape = (1, 16) model_bases = ["plan", "libtorch", "simple_libtorch"] # Sanity check on loaded models @@ -2578,8 +2579,8 @@ def test_file_override_security(self): self.assertTrue(os.path.exists(os.path.join(model_basepath, existing_file_rel))) # Symlinks - ## No easy way to inject symlink into generated temp model dir, so for - ## testing sake, make a fixed symlink path in /tmp. + # No easy way to inject symlink into generated temp model dir, so for + # testing sake, make a fixed symlink path in /tmp. escape_dir_symlink_rel = os.path.join("..", "escape_symlink") escape_dir_symlink_full = "/tmp/escape_symlink" self.assertEqual( @@ -2670,7 +2671,7 @@ def callback(user_data, result, error): except InferenceServerException as ex: self.assertIn( "failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: " - + "Failed to connect to remote host: connect: Connection refused (111)", + + "Failed to connect to remote host: Connection refused", ex.message(), ) @@ -2682,7 +2683,7 @@ def callback(user_data, result, error): # Previous requests should succeed for result in async_results: - if type(result) == InferenceServerException: + if isinstance(result, InferenceServerException): raise result output_data = result.as_numpy("OUTPUT0") np.testing.assert_allclose( @@ -2732,9 +2733,16 @@ def callback(user_data, result, error): ) self.assertTrue(False, "expected error for new inference during shutdown") except InferenceServerException as ex: - # The first request received by the gRPC endpoint while shutting down returns CANCELLED - # each subsequent request returns Connection refused - self.assertIn("CANCELLED", ex.message()) + # The first request received by the gRPC endpoint while shutting down + # historically returned CANCELLED; gRPC >= 1.81 drops the connection + # immediately, so it now returns "Connection refused" like the + # subsequent requests. Accept either. + self.assertTrue( + "CANCELLED" in ex.message() + or "Failed to connect to remote host: Connection refused" + in ex.message(), + f"unexpected shutdown error: {ex.message()}", + ) # 2: New sequence with existing sequence ID try: triton_client.infer(model_name, inputs, sequence_id=1, sequence_start=True) @@ -2742,7 +2750,7 @@ def callback(user_data, result, error): except InferenceServerException as ex: self.assertIn( "failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: " - + "Failed to connect to remote host: connect: Connection refused (111)", + + "Failed to connect to remote host: Connection refused", ex.message(), ) # 3: Continuing sequence after shutdown @@ -2752,7 +2760,7 @@ def callback(user_data, result, error): except InferenceServerException as ex: self.assertIn( "failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: " - + "Failed to connect to remote host: connect: Connection refused (111)", + + "Failed to connect to remote host: Connection refused", ex.message(), ) @@ -2764,7 +2772,7 @@ def callback(user_data, result, error): # Previous requests should succeed for result in async_results: - if type(result) == InferenceServerException: + if isinstance(result, InferenceServerException): raise result output_data = result.as_numpy("OUTPUT") np.testing.assert_allclose( @@ -2815,7 +2823,7 @@ def callback(user_data, result, error): except InferenceServerException as ex: self.assertIn( "failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: " - + "Failed to connect to remote host: connect: Connection refused (111)", + + "Failed to connect to remote host: Connection refused", ex.message(), ) @@ -2827,7 +2835,7 @@ def callback(user_data, result, error): # Previous requests should succeed for result in async_results: - if type(result) == InferenceServerException: + if isinstance(result, InferenceServerException): raise result output_data = result.as_numpy("OUTPUT0") np.testing.assert_allclose( @@ -3073,7 +3081,7 @@ def _load_unload(): # This test can replicate a load while async unloading on machines with # sufficient concurrency. Regardless on whether it is replicated or not, # the server must not crash. - if load_before_unload_finish[0] == False: + if load_before_unload_finish[0] is False: # Track non-replication on test printout via statistics. warning_msg = "Cannot replicate a load while async unloading. CPU count: {}. num_threads: {}.".format( multiprocessing.cpu_count(), num_threads @@ -3299,7 +3307,9 @@ def test_model_config_overwite(self): """ # Ensure the model has been loaded w/ the expected (different from override) config. - self.assertTrue(original_config != None and original_config != override_config) + self.assertTrue( + original_config is not None and original_config != override_config + ) # Reload the model with the overriding configuration value. triton_client.load_model(model_name, config=override_config) @@ -3362,7 +3372,6 @@ def test_shutdown_while_loading(self): def test_shutdown_with_live_connection(self): model_name = "add_sub" model_shape = (16,) - from geventhttpclient.response import HTTPConnectionClosed input_data = np.ones(shape=model_shape, dtype=np.float32) inputs = [ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index de46e970e4..fe5934506b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -90,6 +90,17 @@ endif() # find_package(re2 REQUIRED) +# protobuf +# +# protobuf v33's libprotobuf.a calls into abseil internally, and the +# tritonserver executable statically links libprotobuf.a (via the gRPC +# endpoint). Linking the protobuf package target directly ensures its own +# abseil dependencies are pulled onto the executable's link line, so builds +# that do not enable metrics/GPU (which otherwise pull abseil incidentally) +# still resolve those symbols. protobuf carries its own abseil deps, so no +# abseil target is referenced here. +find_package(Protobuf CONFIG REQUIRED) + # # tritonserver executable # @@ -150,6 +161,7 @@ target_link_libraries( triton-common-logging # from repo-common triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core + protobuf::libprotobuf # brings its own abseil deps ) if(${TRITON_ENABLE_ASAN}) diff --git a/src/grpc/grpc_server.cc b/src/grpc/grpc_server.cc index 10da64776e..46f34be03f 100644 --- a/src/grpc/grpc_server.cc +++ b/src/grpc/grpc_server.cc @@ -866,10 +866,15 @@ CommonHandler::RegisterModelConfig() err = TRITONSERVER_MessageSerializeToJson( model_config_message, &buffer, &byte_size); if (err == nullptr) { - ::google::protobuf::util::JsonStringToMessage( - ::google::protobuf::stringpiece_internal::StringPiece( - buffer, (int)byte_size), - response->mutable_config()); + const auto parse_status = + ::google::protobuf::util::JsonStringToMessage( + absl::string_view(buffer, byte_size), + response->mutable_config()); + if (!parse_status.ok()) { + err = TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INTERNAL, + std::string(parse_status.message()).c_str()); + } } TRITONSERVER_MessageDelete(model_config_message); }