From d898a0fe64788f5ff0d20a222394b51e8ef2ada1 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Thu, 11 Jun 2026 08:33:40 -0700 Subject: [PATCH 1/7] ci(TRI-1406): prepare server for 26.06 (versions, TRT 11 QA compat, enroot env fix) (#8828) --- TRITON_VERSION | 2 +- build.py | 4 +- qa/common/gen_common.py | 22 ++++++ .../gen_qa_dyna_sequence_implicit_models.py | 51 ++++++++---- qa/common/gen_qa_dyna_sequence_models.py | 67 ++++++++++------ qa/common/gen_qa_identity_models.py | 49 ++++++++---- qa/common/gen_qa_implicit_models.py | 45 +++++++---- qa/common/gen_qa_model_repository | 8 +- qa/common/gen_qa_models.py | 77 ++++++++++++------- qa/common/gen_qa_sequence_models.py | 65 +++++++++++----- qa/common/gen_qa_trt_format_models.py | 37 +++++---- qa/common/gen_qa_trt_plugin_models.py | 27 ++++--- qa/common/test_util.py | 17 ++++ 13 files changed, 326 insertions(+), 145 deletions(-) diff --git a/TRITON_VERSION b/TRITON_VERSION index 6a166a54c5..38a7743781 100644 --- a/TRITON_VERSION +++ b/TRITON_VERSION @@ -1 +1 @@ -2.70.0dev +2.70.0 diff --git a/build.py b/build.py index 9111c06c3a..e12a074aa3 100755 --- a/build.py +++ b/build.py @@ -75,8 +75,8 @@ "triton_container_version": "26.06dev", "upstream_container_version": "26.05", "ort_version": "1.24.4", - "ort_openvino_version": "2026.1.0", - "standalone_openvino_version": "2026.1.0", + "ort_openvino_version": "2026.2.0", + "standalone_openvino_version": "2026.2.0", "dcgm_version": "4.5.3-1", "rhel_py_version": "3.12.3", } diff --git a/qa/common/gen_common.py b/qa/common/gen_common.py index db0869ef38..00d23e1e54 100644 --- a/qa/common/gen_common.py +++ b/qa/common/gen_common.py @@ -146,6 +146,28 @@ def np_to_torch_dtype(np_dtype): return None +def trt_set_dynamic_range(tensor, lo, hi): + """Set ITensor.dynamic_range on TRT versions that support it. + + Removed in TensorRT 11+ (strongly-typed networks). Silently skip on + versions where the attribute is gone so the QA model-gen scripts stay + compatible with both old and new TRT.""" + try: + tensor.dynamic_range = (lo, hi) + except AttributeError: + pass # ITensor.dynamic_range removed in TensorRT 11+ (strongly-typed) + + +def trt_cast_tensor(network, tensor, target_dtype): + """Insert an explicit dtype cast that works on both TRT 8.5+ (add_cast) + and older TRT (add_identity + set_output_type). Returns the cast layer.""" + if hasattr(network, "add_cast"): + return network.add_cast(tensor, target_dtype) + layer = network.add_identity(tensor) + layer.set_output_type(0, target_dtype) + return layer + + def openvino_save_model(model_version_dir, model): import openvino as ov diff --git a/qa/common/gen_qa_dyna_sequence_implicit_models.py b/qa/common/gen_qa_dyna_sequence_implicit_models.py index c69ca28eab..784e63ffb5 100755 --- a/qa/common/gen_qa_dyna_sequence_implicit_models.py +++ b/qa/common/gen_qa_dyna_sequence_implicit_models.py @@ -30,7 +30,12 @@ import os import numpy as np -from gen_common import np_to_model_dtype, np_to_onnx_dtype, np_to_trt_dtype +from gen_common import ( + np_to_model_dtype, + np_to_onnx_dtype, + np_to_trt_dtype, + trt_set_dynamic_range, +) FLAGS = None np_dtype_string = np.dtype(object) @@ -388,7 +393,10 @@ def create_plan_modelfile(models_dir, model_version, max_batch, dtype, shape): not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype, so this was a no-op. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) input_state_cond_temp = network.add_elementwise( ready0, not_start.get_output(0), trt.ElementWiseOperation.SUM @@ -527,7 +535,10 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype, so this was a no-op. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) input_state_cond_temp = network.add_elementwise( ready0, not_start.get_output(0), trt.ElementWiseOperation.SUM @@ -552,11 +563,19 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).name = "OUTPUT" network.mark_output(out0.get_output(0)) - out0.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output already has + # trt_dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ out0_state.get_output(0).name = "OUTPUT_STATE" network.mark_output(out0_state.get_output(0)) - out0_state.get_output(0).dtype = trt_dtype + try: + out0_state.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) in_state0.allowed_formats = 1 << int(trt_memory_format) @@ -566,21 +585,23 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0_state.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in_state0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - out0_state.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in_state0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(out0_state, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() diff --git a/qa/common/gen_qa_dyna_sequence_models.py b/qa/common/gen_qa_dyna_sequence_models.py index 1a26890f32..c4cb5bf26a 100755 --- a/qa/common/gen_qa_dyna_sequence_models.py +++ b/qa/common/gen_qa_dyna_sequence_models.py @@ -36,6 +36,7 @@ np_to_torch_dtype, np_to_trt_dtype, openvino_save_model, + trt_set_dynamic_range, ) FLAGS = None @@ -101,15 +102,26 @@ def create_plan_shape_tensor_modelfile( resized_out0 = resize_layer.get_output(0) shape_out0.get_output(0).name = "SHAPE_OUTPUT" - shape_out0.get_output(0).dtype = trt.int64 + # ITensor.dtype setter removed in TRT 11; shape/elementwise/resize + # outputs already have the correct dtype. + try: + shape_out0.get_output(0).dtype = trt.int64 + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output_for_shapes(shape_out0.get_output(0)) out0.name = "OUTPUT" - out0.dtype = trt.int32 + try: + out0.dtype = trt.int32 + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(out0) resized_out0.name = "RESIZED_OUTPUT" - resized_out0.dtype = trt_dtype + try: + resized_out0.dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(resized_out0) shape_in0.allowed_formats = 1 << int(trt_memory_format) @@ -121,20 +133,22 @@ def create_plan_shape_tensor_modelfile( resized_out0.allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - dummy_in0.dynamic_range = (-128.0, 127.0) - resized_out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - end0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(dummy_in0, -128.0, 127.0) + trt_set_dynamic_range(resized_out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(end0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_prefix = [] @@ -353,7 +367,12 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).name = "OUTPUT" network.mark_output(out0.get_output(0)) - out0.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output already has + # trt_dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format) @@ -361,21 +380,23 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - end0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - corrid0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(end0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) + trt_set_dynamic_range(corrid0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_shape = [] diff --git a/qa/common/gen_qa_identity_models.py b/qa/common/gen_qa_identity_models.py index 426d939d9e..bc87333205 100755 --- a/qa/common/gen_qa_identity_models.py +++ b/qa/common/gen_qa_identity_models.py @@ -37,6 +37,7 @@ np_to_onnx_dtype, np_to_trt_dtype, openvino_save_model, + trt_set_dynamic_range, ) FLAGS = None @@ -552,14 +553,18 @@ def create_plan_dynamic_rf_modelfile( out_node = network.add_identity(in_node) out_node.get_output(0).name = "OUTPUT{}".format(io_num) - out_node.get_output(0).dtype = trt_dtype + # Identity preserves input dtype; the ITensor.dtype setter was + # removed in TensorRT 11. Older TRT versions still accept it. + try: + out_node.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(out_node.get_output(0)) out_node.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in_node.dynamic_range = (-128.0, 127.0) - out_node.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in_node, -128.0, 127.0) + trt_set_dynamic_range(out_node.get_output(0), -128.0, 127.0) min_shape = [] opt_shape = [] max_shape = [] @@ -583,14 +588,17 @@ def create_plan_dynamic_rf_modelfile( profile.set_shape("INPUT{}".format(io_num), min_shape, opt_shape, max_shape) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() config.flags = flags @@ -668,18 +676,26 @@ def create_plan_shape_tensor_modelfile( dummy_out_node.name = "DUMMY_OUTPUT{}".format(io_num) - dummy_out_node.dtype = trt_dtype + # The ITensor.dtype setter was removed in TensorRT 11; resize and + # shape layers already produce the correct dtype, so suppress the + # AttributeError instead of changing the older-TRT behavior. + try: + dummy_out_node.dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(dummy_out_node) dummy_out_node.allowed_formats = 1 << int(trt_memory_format) - out_node.get_output(0).dtype = trt.int64 + try: + out_node.get_output(0).dtype = trt.int64 + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output_for_shapes(out_node.get_output(0)) out_node.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in_node.dynamic_range = (-128.0, 127.0) - out_node.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in_node, -128.0, 127.0) + trt_set_dynamic_range(out_node.get_output(0), -128.0, 127.0) config = builder.create_builder_config() min_prefix = [] @@ -707,14 +723,17 @@ def create_plan_shape_tensor_modelfile( config.add_optimization_profile(profile) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config.flags = flags diff --git a/qa/common/gen_qa_implicit_models.py b/qa/common/gen_qa_implicit_models.py index c0800098ec..4b91ca24ea 100755 --- a/qa/common/gen_qa_implicit_models.py +++ b/qa/common/gen_qa_implicit_models.py @@ -37,6 +37,7 @@ np_to_onnx_dtype, np_to_torch_dtype, np_to_trt_dtype, + trt_set_dynamic_range, ) FLAGS = None @@ -923,7 +924,11 @@ def create_plan_modelfile(models_dir, model_version, max_batch, dtype, shape): not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype (both inputs do), so this + # call was a no-op on modern TRT. Guard for older versions. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) internal_state = network.add_elementwise( in_state0, not_start.get_output(0), trt.ElementWiseOperation.PROD ) @@ -1033,7 +1038,11 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype (both inputs do), so this + # call was a no-op on modern TRT. Guard for older versions. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) internal_state = network.add_elementwise( in_state0, not_start.get_output(0), trt.ElementWiseOperation.PROD ) @@ -1049,8 +1058,16 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0_state.get_output(0).name = "OUTPUT_STATE" network.mark_output(out0_state.get_output(0)) - out0.get_output(0).dtype = trt_dtype - out0_state.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output dtype + # already matches trt_dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ + try: + out0_state.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format) @@ -1058,20 +1075,22 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in_state0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in_state0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_shape = [] diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 1038897618..ca873fe8f1 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -626,7 +626,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: openvino.ubuntu.$CI_JOB_ID" enroot create --name openvino.ubuntu.$CI_JOB_ID /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: openvino.ubuntu.$CI_JOB_ID" - enroot start --root --rw -m /tmp:/tmp openvino.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT + enroot start --root --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR openvino.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${OPENVINOSCRIPT} failed" exit 1 @@ -636,7 +636,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: onnxruntime.ubuntu.$CI_JOB_ID" enroot create --name onnxruntime.ubuntu.$CI_JOB_ID /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: onnxruntime.ubuntu.$CI_JOB_ID" - enroot start --root --rw -m /tmp:/tmp onnxruntime.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT + enroot start --root --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR onnxruntime.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${ONNXSCRIPT} failed" exit 1 @@ -653,7 +653,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: pytorch.$CI_JOB_ID" enroot create --name pytorch.$CI_JOB_ID /tmp/pytorch.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: pytorch.$CI_JOB_ID" - enroot start --rw -m /tmp:/tmp pytorch.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT + enroot start --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR pytorch.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${TORCHSCRIPT} failed" exit 1 @@ -664,7 +664,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: tensorrt.$CI_JOB_ID" enroot create --name tensorrt.$CI_JOB_ID /tmp/tensorrt.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: tensorrt.$CI_JOB_ID" - enroot start --rw -m /tmp:/tmp tensorrt.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TRTSCRIPT + enroot start --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR tensorrt.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TRTSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${TRTSCRIPT} failed" exit 1 diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py index d305f07ce3..2008cd824a 100755 --- a/qa/common/gen_qa_models.py +++ b/qa/common/gen_qa_models.py @@ -40,6 +40,8 @@ np_to_torch_dtype, np_to_trt_dtype, openvino_save_model, + trt_cast_tensor, + trt_set_dynamic_range, ) FLAGS = None @@ -95,11 +97,11 @@ def create_plan_dynamic_rf_modelfile( # FIXME: Remove support check when jetson supports TRT 8.5 (DLIS-4256) if tu.support_trt_uint8(): if trt_input_dtype == trt.uint8: - in0_cast = network.add_identity(in0) - in0_cast.set_output_type(0, trt.float32) + # TensorRT 11 removed set_output_type on identity layers; the + # equivalent on TRT 8.5+ is add_cast. + in0_cast = trt_cast_tensor(network, in0, trt.float32) in0 = in0_cast.get_output(0) - in1_cast = network.add_identity(in1) - in1_cast.set_output_type(0, trt.float32) + in1_cast = trt_cast_tensor(network, in1, trt.float32) in1 = in1_cast.get_output(0) add = network.add_elementwise(in0, in1, trt.ElementWiseOperation.SUM) @@ -111,19 +113,25 @@ def create_plan_dynamic_rf_modelfile( # FIXME: Remove support check when jetson supports TRT 8.5 (DLIS-4256) if tu.support_trt_uint8(): if trt_output0_dtype == trt.uint8: - out0 = network.add_identity(out0.get_output(0)) - out0.set_output_type(0, trt.uint8) + out0 = trt_cast_tensor(network, out0.get_output(0), trt.uint8) if trt_output1_dtype == trt.uint8: - out1 = network.add_identity(out1.get_output(0)) - out1.set_output_type(0, trt.uint8) + out1 = trt_cast_tensor(network, out1.get_output(0), trt.uint8) out0.get_output(0).name = "OUTPUT0" out1.get_output(0).name = "OUTPUT1" network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - out0.get_output(0).dtype = trt_output0_dtype - out1.get_output(0).dtype = trt_output1_dtype + # ITensor.dtype setter removed in TRT 11; cast above already produced + # the desired output dtype on modern TRT. + try: + out0.get_output(0).dtype = trt_output0_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ + try: + out1.get_output(0).dtype = trt_output1_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) in1.allowed_formats = 1 << int(trt_memory_format) @@ -131,13 +139,12 @@ def create_plan_dynamic_rf_modelfile( out1.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_input_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in1.dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in1, -128.0, 127.0) if trt_output0_dtype == trt.int8: - out0.get_output(0).dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(out0.get_output(0), -128.0, 127.0) if trt_output1_dtype == trt.int8: - out1.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(out1.get_output(0), -128.0, 127.0) min_shape = [] opt_shape = [] max_shape = [] @@ -159,15 +166,19 @@ def create_plan_dynamic_rf_modelfile( profile.set_shape("INPUT0", min_shape, opt_shape, max_shape) profile.set_shape("INPUT1", min_shape, opt_shape, max_shape) - flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + flags = 0 + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() config.flags = flags @@ -416,8 +427,15 @@ def create_plan_fixed_rf_modelfile( network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - out0.get_output(0).dtype = trt_output0_dtype - out1.get_output(0).dtype = trt_output1_dtype + # ITensor.dtype setter removed in TRT 11; output dtype already matches. + try: + out0.get_output(0).dtype = trt_output0_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ + try: + out1.get_output(0).dtype = trt_output1_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) in1.allowed_formats = 1 << int(trt_memory_format) @@ -425,13 +443,12 @@ def create_plan_fixed_rf_modelfile( out1.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_input_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in1.dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in1, -128.0, 127.0) if trt_output0_dtype == trt.int8: - out0.get_output(0).dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(out0.get_output(0), -128.0, 127.0) if trt_output1_dtype == trt.int8: - out1.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(out1.get_output(0), -128.0, 127.0) config = builder.create_builder_config() min_shape = [] @@ -450,15 +467,19 @@ def create_plan_fixed_rf_modelfile( profile.set_shape("INPUT0", min_shape, opt_shape, max_shape) profile.set_shape("INPUT1", min_shape, opt_shape, max_shape) - flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + flags = 0 + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() diff --git a/qa/common/gen_qa_sequence_models.py b/qa/common/gen_qa_sequence_models.py index f8d89a5f9e..6de55c3ea5 100755 --- a/qa/common/gen_qa_sequence_models.py +++ b/qa/common/gen_qa_sequence_models.py @@ -37,6 +37,7 @@ np_to_torch_dtype, np_to_trt_dtype, openvino_save_model, + trt_set_dynamic_range, ) FLAGS = None @@ -90,15 +91,28 @@ def create_plan_shape_tensor_modelfile( shape_out0 = network.add_shape(resized_out0) shape_out0.get_output(0).name = "SHAPE_OUTPUT" - shape_out0.get_output(0).dtype = trt.int64 + # The ITensor.dtype setter was removed in TensorRT 11. The shape, resize + # and elementwise outputs already have the correct dtype, so on TRT 11+ + # this assignment is unnecessary; keep the explicit form for older TRT + # under a guard. + try: + shape_out0.get_output(0).dtype = trt.int64 + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output_for_shapes(shape_out0.get_output(0)) out0.name = "OUTPUT" - out0.dtype = trt_dtype + try: + out0.dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(out0) resized_out0.name = "RESIZED_OUTPUT" - resized_out0.dtype = trt_dtype + try: + resized_out0.dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(resized_out0) in0.allowed_formats = 1 << int(trt_memory_format) @@ -110,20 +124,22 @@ def create_plan_shape_tensor_modelfile( resized_out0.allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - resized_out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(resized_out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_prefix = [] @@ -306,7 +322,12 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).name = "OUTPUT" network.mark_output(out0.get_output(0)) - out0.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output already has + # the correct dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format) @@ -314,19 +335,21 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_shape = [] diff --git a/qa/common/gen_qa_trt_format_models.py b/qa/common/gen_qa_trt_format_models.py index 5f2cadd69e..fc59d7c9e1 100755 --- a/qa/common/gen_qa_trt_format_models.py +++ b/qa/common/gen_qa_trt_format_models.py @@ -32,7 +32,7 @@ import numpy as np import tensorrt as trt import test_util as tu -from gen_common import np_to_model_dtype, np_to_trt_dtype +from gen_common import np_to_model_dtype, np_to_trt_dtype, trt_set_dynamic_range np_dtype_string = np.dtype(object) @@ -98,30 +98,35 @@ def create_plan_modelfile( add = network.add_elementwise(in0, in1, trt.ElementWiseOperation.SUM) sub = network.add_elementwise(in0, in1, trt.ElementWiseOperation.SUB) - out0 = network.add_identity(add.get_output(0)) - out1 = network.add_identity(sub.get_output(0)) + # TRT 11 removed Layer.set_output_type; on modern TRT use add_cast to + # produce the desired output dtype. On older TRT, fall back to the + # original identity + set_output_type pattern. + if hasattr(network, "add_cast"): + out0 = network.add_cast(add.get_output(0), trt_output0_dtype) + out1 = network.add_cast(sub.get_output(0), trt_output1_dtype) + else: + out0 = network.add_identity(add.get_output(0)) + out1 = network.add_identity(sub.get_output(0)) + out0.set_output_type(0, trt_output0_dtype) + out1.set_output_type(0, trt_output1_dtype) out0.get_output(0).name = "OUTPUT0" out1.get_output(0).name = "OUTPUT1" network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - out0.set_output_type(0, trt_output0_dtype) - out1.set_output_type(0, trt_output1_dtype) - in0.allowed_formats = 1 << int(trt_input_memory_format) in1.allowed_formats = 1 << int(trt_input_memory_format) out0.get_output(0).allowed_formats = 1 << int(trt_output_memory_format) out1.get_output(0).allowed_formats = 1 << int(trt_output_memory_format) if trt_input_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in1.dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in1, -128.0, 127.0) if trt_output0_dtype == trt.int8: - out0.get_output(0).dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(out0.get_output(0), -128.0, 127.0) if trt_output1_dtype == trt.int8: - out1.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(out1.get_output(0), -128.0, 127.0) min_shape = [] opt_shape = [] max_shape = [] @@ -146,14 +151,18 @@ def create_plan_modelfile( # Commenting this because from I/O Formats from TensorRT Developer Guide: # The build will fail if TensorRT cannot build an engine without introducing such reformatting. The failure may happen only for some target platforms, because of what formats are supported by kernels for those platforms. # flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + flags = 0 + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() config.flags = flags diff --git a/qa/common/gen_qa_trt_plugin_models.py b/qa/common/gen_qa_trt_plugin_models.py index 9fd23d92a8..9c01de6d6f 100755 --- a/qa/common/gen_qa_trt_plugin_models.py +++ b/qa/common/gen_qa_trt_plugin_models.py @@ -40,6 +40,13 @@ trt.init_libnvinfer_plugins(TRT_LOGGER, "") +# TRT 11 removed the IPluginV2 registry surface (plugin_creator_list). +# Decide V2 vs V3 once at import time and use the same flag for both +# plugin creation and network.add_plugin_v* dispatch. add_plugin_v2 is +# still bound on INetworkDefinition in TRT 11, so hasattr() on the +# network is not a safe gate -- only this registry probe is. +TRT_USES_V3_PLUGINS = not hasattr(trt.get_plugin_registry(), "plugin_creator_list") + def get_trt_plugin(plugin_name): plugin = None @@ -48,8 +55,9 @@ def get_trt_plugin(plugin_name): # branches and V3 on rel-11.0 (and TRT 11 removed the V2 plugin # registry surface). Pick the matching API at runtime. registry = trt.get_plugin_registry() - use_v3 = not hasattr(registry, "plugin_creator_list") - plugin_creators = registry.all_creators if use_v3 else registry.plugin_creator_list + plugin_creators = ( + registry.all_creators if TRT_USES_V3_PLUGINS else registry.plugin_creator_list + ) for plugin_creator in plugin_creators: if (plugin_creator.name == "CustomHardmax") and ( plugin_name == "CustomHardmax" @@ -62,7 +70,7 @@ def get_trt_plugin(plugin_name): if field_collection is None: raise RuntimeError("Plugin not found: " + plugin_name) - if use_v3: + if TRT_USES_V3_PLUGINS: plugin = plugin_creator.create_plugin( name=plugin_name, field_collection=field_collection, @@ -116,16 +124,17 @@ def create_plan_modelfile( input_layer = network.add_input( name="INPUT0", dtype=trt_input_dtype, shape=input_with_batchsize ) - # add_plugin_v2 was removed in TRT 11; add_plugin_v3 has existed since - # TRT 10.0. Pick the API that exists on this TRT install; the plugin - # object returned by get_trt_plugin() is matched to the same version. + # add_plugin_v2 is still bound on INetworkDefinition in TRT 11 but + # rejects IPluginV3 objects; dispatch on the same TRT_USES_V3_PLUGINS + # flag that get_trt_plugin() used to pick the plugin object kind so + # both halves agree on V2 vs V3. plugin_obj = get_trt_plugin(plugin_name) - if hasattr(network, "add_plugin_v2"): - plugin_layer = network.add_plugin_v2(inputs=[input_layer], plugin=plugin_obj) - else: + if TRT_USES_V3_PLUGINS: plugin_layer = network.add_plugin_v3( inputs=[input_layer], shape_inputs=[], plugin=plugin_obj ) + else: + plugin_layer = network.add_plugin_v2(inputs=[input_layer], plugin=plugin_obj) plugin_layer.get_output(0).name = "OUTPUT0" network.mark_output(plugin_layer.get_output(0)) diff --git a/qa/common/test_util.py b/qa/common/test_util.py index 46a42668bf..6f918ded6c 100755 --- a/qa/common/test_util.py +++ b/qa/common/test_util.py @@ -137,6 +137,12 @@ def validate_for_trt_model( # FIXME: Remove this check when jetson supports TRT 8.5 (DLIS-4256) if not support_trt_uint8(): supported_datatypes.remove(np.uint8) + # TRT 11+ removed the implicit-precision INT8 path (BuilderFlag.INT8 + # + dynamic_range); strongly-typed networks require explicit QDQ which + # the QA generators don't emit. Exclude int8 plan models on TRT 11+. + if not support_trt_int8_implicit_precision(): + if np.int8 in supported_datatypes: + supported_datatypes.remove(np.int8) if not input_dtype in supported_datatypes: return False if not output0_dtype in supported_datatypes: @@ -355,6 +361,17 @@ def support_trt_uint8(): return hasattr(trt, "uint8") +def support_trt_int8_implicit_precision(): + """Return True if the installed TensorRT supports the implicit-precision + INT8 path (BuilderFlag.INT8 + per-tensor dynamic_range). Removed in + TensorRT 11+ where strongly-typed networks are mandatory.""" + try: + import tensorrt as trt + except ImportError: + return False + return hasattr(trt.BuilderFlag, "INT8") + + def check_gpus_compute_capability(min_capability): """ Check if all GPUs have a compute capability greater than or equal to the given value. From e5f5faf751e597b4eecd6667b23cb889e44b69ad Mon Sep 17 00:00:00 2001 From: mattwittwer Date: Thu, 11 Jun 2026 09:40:44 -0700 Subject: [PATCH 2/7] feat: Enable PyTorch2 Batching Tests (#8814) (#8831) --- qa/L0_torch_aoti/test.sh | 57 +++- qa/L0_torch_aoti/torch_aoti_infer_test.py | 351 ++++++++++++++++++--- qa/common/gen_qa_implicit_models.py | 366 +++++++++++++++++++++- qa/common/gen_qa_model_repository | 2 + qa/common/gen_qa_models.py | 165 +++++++++- 5 files changed, 875 insertions(+), 66 deletions(-) diff --git a/qa/L0_torch_aoti/test.sh b/qa/L0_torch_aoti/test.sh index f37751c55e..9a09921e10 100755 --- a/qa/L0_torch_aoti/test.sh +++ b/qa/L0_torch_aoti/test.sh @@ -62,6 +62,7 @@ DATADIR=${DATADIR:="/data/inferenceserver/${REPO_VERSION}"} TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends +SERVER_TIMEOUT=${SERVER_TIMEOUT:=120} # PyTorch on SBSA requires libgomp to be loaded first. See the following # GitHub issue for more information: @@ -79,7 +80,9 @@ export BACKENDS # Copy the models into the model repository echo -e "${COLOR_DARK}Setting up model repository in ${MODELDIR}${COLOR_RESET}" -rm -rf ${MODELDIR} && mkdir -p ${MODELDIR} +BAD_MODELDIR=`pwd`/bad_models +rm -rf ${MODELDIR} ${BAD_MODELDIR} +mkdir -p ${MODELDIR} models=( "torch_aoti_complex_index" "torch_aoti_complex_named" @@ -89,6 +92,8 @@ models=( "torch_aoti_int64_int64" "torch_aoti_float16_float16" "torch_aoti_float32_float32" + "torch_aoti_variable_float32" + "torch_aoti_multi_instance_float32" "torchvision_aoti" ) for model in "${models[@]}"; do @@ -96,6 +101,19 @@ for model in "${models[@]}"; do echo -e "${COLOR_DARK}ls ${MODELDIR}/${model}${COLOR_RESET}" ls -lha ${MODELDIR}/${model} done + +# Sequence-batching AOTI models live in the implicit-state sequence repository. +sequence_models=( + "torch_aoti_sequence_float32" + "torch_aoti_sequence_initstate_float32" + "torch_aoti_sequence_forward_float32" +) +for model in "${sequence_models[@]}"; do + cp -r ${DATADIR}/qa_sequence_implicit_model_repository/${model} ${MODELDIR}/${model} + echo -e "${COLOR_DARK}ls ${MODELDIR}/${model}${COLOR_RESET}" + ls -lha ${MODELDIR}/${model} +done + echo -e "${COLOR_DARK}ls ${MODELDIR}${COLOR_RESET}" ls -lha ${MODELDIR} @@ -133,11 +151,42 @@ fi echo -e "${COLOR_DARK}Killing server (pid: ${SERVER_PID})${COLOR_RESET}" kill -s SIGINT ${SERVER_PID} wait ${SERVER_PID} || true -echo -e "${COLOR_DARK}Removing model repository${COLOR_RESET}" -for model in "${models[@]}"; do - rm -rf ${MODELDIR}/${model} + +# Negative tests: these models declare unsupported types (TYPE_STRING CORRID / +# state) and must fail to load. Start a separate server (exit-on-error=false) so +# it stays up despite the load failures, then assert the models are not ready. +echo -e "${COLOR_DARK}Negative (load-failure) tests${COLOR_RESET}" +mkdir -p ${BAD_MODELDIR} +bad_models=( + "torch_aoti_sequence_bad_corrid" + "torch_aoti_sequence_bad_state" +) +for model in "${bad_models[@]}"; do + cp -r ${DATADIR}/qa_sequence_implicit_model_repository/${model} ${BAD_MODELDIR}/${model} done +SERVER_ARGS="--model-repository=${BAD_MODELDIR} --exit-on-error=false --log-verbose=1" +SERVER_LOG="./torch_aoti_negative-server.log" +run_server_tolive +if [[ "${SERVER_PID}" -eq 0 ]]; then + echo -e "${COLOR_ERROR}\n***\n*** Failed to start ${SERVER} (negative phase)\n***${COLOR_RESET}" 1>&2 + cat ${SERVER_LOG} 1>&2 + RET=1 +else + wait_for_model_stable ${SERVER_TIMEOUT} + for model in "${bad_models[@]}"; do + code=$(curl -s -o /dev/null -w "%{http_code}" localhost:8000/v2/models/${model}/ready) + if [[ "${code}" == "200" ]]; then + echo -e "${COLOR_ERROR}*** Negative model '${model}' unexpectedly loaded (ready)${COLOR_RESET}" 1>&2 + RET=1 + else + echo -e "${COLOR_INFO}*** Negative model '${model}' correctly failed to load${COLOR_RESET}" + fi + done + kill -s SIGINT ${SERVER_PID} + wait ${SERVER_PID} || true +fi + # Report results and exit. if [[ ${RET} -ne 0 ]]; then echo -e "${COLOR_ERROR}\n***\n*** Test Suite FAILED\n***${COLOR_RESET}" &1>2 diff --git a/qa/L0_torch_aoti/torch_aoti_infer_test.py b/qa/L0_torch_aoti/torch_aoti_infer_test.py index 2b93f31a48..60cae41360 100755 --- a/qa/L0_torch_aoti/torch_aoti_infer_test.py +++ b/qa/L0_torch_aoti/torch_aoti_infer_test.py @@ -30,10 +30,13 @@ sys.path.append("../common") import unittest +from concurrent.futures import ThreadPoolExecutor +import numpy as np import test_util as tu import torch import tritonclient.http as http +from tritonclient.utils import InferenceServerException class TorchAotiTest(tu.TestResultCollector): @@ -203,81 +206,325 @@ def test_simple_model(self): torch.float16, torch.float32, ] + # The simple AOTI add/sub model is compiled with a dynamic batch + # dimension and configured with max_batch_size: 8. Exercise a range of + # batch sizes (including 1) so we validate that batched inputs are + # assembled and batched outputs are scattered back per-row correctly. + batch_sizes = [1, 4, 8] for io_type in io_types: MODEL_NAME = self._get_simple_model_name(io_type) - INPUT_SHAPE = (16,) - OUTPUT_SHAPE = (16,) TRITON_IO_TYPE = self._dtype_to_triton_dtype(io_type) - input_data = ( - self._get_simple_input_data(INPUT_SHAPE, io_type), - self._get_simple_input_data(INPUT_SHAPE, io_type), - ) + for batch_size in batch_sizes: + INPUT_SHAPE = (batch_size, 16) + OUTPUT_SHAPE = (batch_size, 16) - with http.InferenceServerClient("localhost:8000") as client: - inputs = [ - http.InferInput("ARGS[0]", input_data[0].shape, TRITON_IO_TYPE), - http.InferInput("ARGS[1]", input_data[1].shape, TRITON_IO_TYPE), - ] + input_data = ( + self._get_simple_input_data(INPUT_SHAPE, io_type), + self._get_simple_input_data(INPUT_SHAPE, io_type), + ) - inputs[0].set_data_from_numpy(input_data[0], binary_data=True) - inputs[1].set_data_from_numpy(input_data[1], binary_data=True) + with http.InferenceServerClient("localhost:8000") as client: + inputs = [ + http.InferInput("ARGS[0]", input_data[0].shape, TRITON_IO_TYPE), + http.InferInput("ARGS[1]", input_data[1].shape, TRITON_IO_TYPE), + ] - output_names = [ - "RESULT", - ] + inputs[0].set_data_from_numpy(input_data[0], binary_data=True) + inputs[1].set_data_from_numpy(input_data[1], binary_data=True) - outputs = [] - for output_name in output_names: - outputs.append( - http.InferRequestedOutput(output_name, binary_data=True) - ) + output_names = [ + "RESULT", + ] - output_data = [] - results = client.infer(MODEL_NAME, inputs, outputs=outputs) + outputs = [] + for output_name in output_names: + outputs.append( + http.InferRequestedOutput(output_name, binary_data=True) + ) + + output_data = [] + results = client.infer(MODEL_NAME, inputs, outputs=outputs) - for output_name in output_names: - output_data.append(results.as_numpy(output_name)) + for output_name in output_names: + output_data.append(results.as_numpy(output_name)) - self.assertEqual(len(outputs), len(output_data)) - for data in output_data: - self.assertEqual(data.shape, OUTPUT_SHAPE) - self.assertTrue((data == input_data[0] + input_data[1]).all()) + self.assertEqual(len(outputs), len(output_data)) + for data in output_data: + self.assertEqual(data.shape, OUTPUT_SHAPE) + self.assertTrue((data == input_data[0] + input_data[1]).all()) def test_torchvision(self): + # torchvision_aoti is exported with a dynamic batch dim (max_batch_size + # 8), so exercise batching of a real, higher-rank [N,3,224,224] model. MODEL_NAME = "torchvision_aoti" - INPUT_SHAPE = (1, 3, 224, 224) - OUTPUT_SHAPE = (1, 1000) - - input_data = self._get_torchvision_input_data(INPUT_SHAPE) - input_data[0][0] = 1.0 + with http.InferenceServerClient("localhost:8000") as client: + for batch_size in (1, 4, 8): + input_data = self._get_torchvision_input_data((batch_size, 3, 224, 224)) + inputs = [http.InferInput("ARGS[0]", input_data.shape, "FP32")] + inputs[0].set_data_from_numpy(input_data, binary_data=True) + outputs = [http.InferRequestedOutput("RESULT", binary_data=True)] + results = client.infer(MODEL_NAME, inputs, outputs=outputs) + data = results.as_numpy("RESULT") + self.assertEqual(data.shape, (batch_size, 1000)) + output_tensor = torch.from_numpy(data) + self.assertTrue(torch.isfinite(output_tensor).all().item()) + def test_batch_size_limit(self): + # A request whose batch exceeds max_batch_size (8) must be rejected; + # exactly max_batch_size must succeed. + MODEL_NAME = "torch_aoti_float32_float32" with http.InferenceServerClient("localhost:8000") as client: + ok = self._get_simple_input_data((8, 16), torch.float32) inputs = [ - http.InferInput("ARGS[0]", input_data.shape, "FP32"), + http.InferInput("ARGS[0]", ok.shape, "FP32"), + http.InferInput("ARGS[1]", ok.shape, "FP32"), ] - - inputs[0].set_data_from_numpy(input_data, binary_data=True) - - output_names = [ - "RESULT", + inputs[0].set_data_from_numpy(ok, binary_data=True) + inputs[1].set_data_from_numpy(ok, binary_data=True) + outputs = [http.InferRequestedOutput("RESULT", binary_data=True)] + client.infer(MODEL_NAME, inputs, outputs=outputs) # batch == max OK + + too_big = self._get_simple_input_data((16, 16), torch.float32) + big_inputs = [ + http.InferInput("ARGS[0]", too_big.shape, "FP32"), + http.InferInput("ARGS[1]", too_big.shape, "FP32"), ] + big_inputs[0].set_data_from_numpy(too_big, binary_data=True) + big_inputs[1].set_data_from_numpy(too_big, binary_data=True) + with self.assertRaises(InferenceServerException): + client.infer(MODEL_NAME, big_inputs, outputs=outputs) - outputs = [] - for output_name in output_names: - outputs.append(http.InferRequestedOutput(output_name, binary_data=True)) + def _infer_add(self, model_name, a, b, triton_type="FP32"): + # Run the two-input add model and return its RESULT output. + with http.InferenceServerClient("localhost:8000") as client: + inputs = [ + http.InferInput("ARGS[0]", a.shape, triton_type), + http.InferInput("ARGS[1]", b.shape, triton_type), + ] + inputs[0].set_data_from_numpy(a, binary_data=True) + inputs[1].set_data_from_numpy(b, binary_data=True) + outputs = [http.InferRequestedOutput("RESULT", binary_data=True)] + return client.infer(model_name, inputs, outputs=outputs).as_numpy("RESULT") - output_data = [] - results = client.infer(MODEL_NAME, inputs, outputs=outputs) + def _execution_count(self, model_name): + with http.InferenceServerClient("localhost:8000") as client: + stats = client.get_inference_statistics(model_name=model_name) + return int(stats["model_stats"][0]["execution_count"]) + + def _infer_one_row(self, model_name): + a = self._get_simple_input_data((1, 16), torch.float32) + b = self._get_simple_input_data((1, 16), torch.float32) + out = self._infer_add(model_name, a, b) + self.assertTrue((out == a + b).all()) + + def test_dynamic_batching_coalescing(self): + # Fire many concurrent single-row requests and confirm the dynamic + # batcher coalesced them into far fewer backend executions than requests. + MODEL_NAME = "torch_aoti_float32_float32" + num_requests = 200 + before = self._execution_count(MODEL_NAME) + with ThreadPoolExecutor(max_workers=32) as pool: + futures = [ + pool.submit(self._infer_one_row, MODEL_NAME) + for _ in range(num_requests) + ] + for future in futures: + future.result() + executions = self._execution_count(MODEL_NAME) - before + self.assertGreater(executions, 0) + self.assertLess(executions, num_requests) + + def test_multi_instance(self): + # Concurrent requests against a 2-instance model must all be correct. + MODEL_NAME = "torch_aoti_multi_instance_float32" + with ThreadPoolExecutor(max_workers=16) as pool: + futures = [pool.submit(self._infer_one_row, MODEL_NAME) for _ in range(64)] + for future in futures: + future.result() + + def test_variable_shape_batching(self): + # The variable model has dims [-1]; exercise different feature lengths + # across batch sizes. + MODEL_NAME = "torch_aoti_variable_float32" + for batch_size in (1, 4, 8): + for feature in (1, 16, 64): + a = np.random.randn(batch_size, feature).astype(np.float32) + b = np.random.randn(batch_size, feature).astype(np.float32) + out = self._infer_add(MODEL_NAME, a, b) + self.assertEqual(out.shape, (batch_size, feature)) + self.assertTrue((out == a + b).all()) + + +class TorchAotiSequenceTest(tu.TestResultCollector): + # The AOTI sequence model (see gen_qa_implicit_models.py) is a running + # accumulator that resets on the sequence start and adds the correlation id + # to each emitted output: + # new_state = INPUT0 + INPUT_STATE * (1 - START) + # OUTPUT0 = (new_state + CORRID) * READY + # Triton's sequence scheduler synthesizes the START / READY / CORRID control + # tensors and manages the implicit state, so the client only sends INPUT__0 + # (the correlation id is supplied via sequence_id). + MODEL_NAME = "torch_aoti_sequence_float32" + + def _infer_step( + self, + client, + seq_id, + value, + start, + end, + model=None, + in_name="INPUT__0", + out_name="OUTPUT__0", + ): + data = np.full((1, 1), value, dtype=np.float32) + inputs = [http.InferInput(in_name, data.shape, "FP32")] + inputs[0].set_data_from_numpy(data, binary_data=True) + outputs = [http.InferRequestedOutput(out_name, binary_data=True)] + result = client.infer( + model or self.MODEL_NAME, + inputs, + outputs=outputs, + sequence_id=seq_id, + sequence_start=start, + sequence_end=end, + ) + return result.as_numpy(out_name) + + def test_single_sequence(self): + seq_id = 100 + steps = [2.0, 3.0, 4.0, 5.0] + # Output is the running sum plus the correlation id. + expected = np.cumsum(steps) + seq_id + with http.InferenceServerClient("localhost:8000") as client: + for i, value in enumerate(steps): + out = self._infer_step( + client, + seq_id=seq_id, + value=value, + start=(i == 0), + end=(i == len(steps) - 1), + ) + self.assertEqual(out.shape, (1, 1)) + self.assertAlmostEqual(float(out[0, 0]), float(expected[i]), places=3) + + def test_interleaved_sequences(self): + # Two concurrent sequences must keep independent state. Each output is + # the per-sequence running sum plus that sequence's correlation id. + seqs = { + 201: {"steps": [1.0, 1.0, 1.0, 1.0], "sum": 0.0}, + 202: {"steps": [10.0, 20.0, 30.0, 40.0], "sum": 0.0}, + } + with http.InferenceServerClient("localhost:8000") as client: + num_steps = len(next(iter(seqs.values()))["steps"]) + for i in range(num_steps): + for seq_id, st in seqs.items(): + value = st["steps"][i] + st["sum"] += value + out = self._infer_step( + client, + seq_id=seq_id, + value=value, + start=(i == 0), + end=(i == num_steps - 1), + ) + self.assertAlmostEqual( + float(out[0, 0]), st["sum"] + seq_id, places=3 + ) - for output_name in output_names: - output_data.append(results.as_numpy(output_name)) + def test_many_concurrent_sequences(self): + # Fill every batch slot with a distinct live sequence (max_batch_size is + # 8); each must keep independent state (+ its own correlation id) as they + # are stepped in lockstep. + seq_ids = list(range(300, 308)) # 8 concurrent sequences == slots + sums = {s: 0.0 for s in seq_ids} + num_steps = 5 + with http.InferenceServerClient("localhost:8000") as client: + for i in range(num_steps): + for s in seq_ids: + value = float(s % 7 + 1) + sums[s] += value + out = self._infer_step( + client, + seq_id=s, + value=value, + start=(i == 0), + end=(i == num_steps - 1), + ) + self.assertAlmostEqual(float(out[0, 0]), sums[s] + s, places=3) + + def test_staggered_sequences(self): + # Sequences of different lengths begin and end at different ticks on a + # shared timeline, so they overlap: some finish (freeing their batch + # slot) while others stay live and new ones start into the freed slots. + # Each sequence keeps independent state that resets on its own START. + # plan: seq_id -> (first_tick, values) + plans = { + 400: (0, [1.0, 2.0]), # ticks 0-1 + 401: (0, [3.0, 4.0, 5.0, 6.0]), # ticks 0-3 + 402: (2, [10.0, 10.0, 10.0]), # ticks 2-4 (starts after 400 ends) + 403: (3, [7.0, 8.0]), # ticks 3-4 + } + last_tick = max(start + len(values) - 1 for start, values in plans.values()) + running = {seq_id: 0.0 for seq_id in plans} + with http.InferenceServerClient("localhost:8000") as client: + for tick in range(last_tick + 1): + for seq_id, (first_tick, values) in plans.items(): + idx = tick - first_tick + if idx < 0 or idx >= len(values): + continue # sequence not live at this tick + running[seq_id] += values[idx] + out = self._infer_step( + client, + seq_id=seq_id, + value=values[idx], + start=(idx == 0), + end=(idx == len(values) - 1), + ) + self.assertAlmostEqual( + float(out[0, 0]), running[seq_id] + seq_id, places=3 + ) - self.assertEqual(len(outputs), len(output_data)) - for data in output_data: - self.assertEqual(data.shape, OUTPUT_SHAPE) - output_tensor = torch.from_numpy(data) - self.assertTrue(torch.isfinite(output_tensor).all().item()) + def test_initial_state_sequence(self): + # Model relies on a declared zero initial_state (no START reset). Output + # is the running sum (no correlation id for this variant). + model = "torch_aoti_sequence_initstate_float32" + steps = [2.0, 3.0, 4.0, 5.0] + expected = np.cumsum(steps) + with http.InferenceServerClient("localhost:8000") as client: + for i, value in enumerate(steps): + out = self._infer_step( + client, + seq_id=500, + value=value, + start=(i == 0), + end=(i == len(steps) - 1), + model=model, + ) + self.assertAlmostEqual(float(out[0, 0]), float(expected[i]), places=3) + + def test_forward_interface_sequence(self): + # Same sequence artifact, but control/state addressed via the forward + # interface (ARGS[...] / RESULT[...]). Behaviour must match the ordinal + # model: running sum + correlation id. + model = "torch_aoti_sequence_forward_float32" + seq_id = 600 + steps = [2.0, 3.0, 4.0] + expected = np.cumsum(steps) + seq_id + with http.InferenceServerClient("localhost:8000") as client: + for i, value in enumerate(steps): + out = self._infer_step( + client, + seq_id=seq_id, + value=value, + start=(i == 0), + end=(i == len(steps) - 1), + model=model, + in_name="ARGS[0]", + out_name="RESULT[0]", + ) + self.assertAlmostEqual(float(out[0, 0]), float(expected[i]), places=3) if __name__ == "__main__": diff --git a/qa/common/gen_qa_implicit_models.py b/qa/common/gen_qa_implicit_models.py index 4b91ca24ea..d3b28da96a 100755 --- a/qa/common/gen_qa_implicit_models.py +++ b/qa/common/gen_qa_implicit_models.py @@ -1246,6 +1246,352 @@ def create_plan_modelconfig(models_dir, max_batch, dtype, shape): cfile.write(config) +def create_torch_aoti_modelfile(models_dir, model_version, max_batch, dtype, shape): + # AOT Inductor (PT2) sequence model. The forward arguments map positionally + # to the model's ordinal inputs/outputs, which the config addresses directly: + # INPUT__0 = INPUT0 (data), INPUT__1 = INPUT_STATE (implicit state), + # INPUT__2 = START (control), INPUT__3 = READY (control), + # INPUT__4 = CORRID (control) + # OUTPUT__0 = out (data), OUTPUT__1 = new_state (implicit state) + if dtype not in (np.float32, np.int32): + return + + torch_dtype = np_to_torch_dtype(dtype) + model_name = tu.get_sequence_model_name("torch_aoti", dtype) + shape = [abs(ips) for ips in shape] + + class SequenceNet(nn.Module): + def __init__(self): + super(SequenceNet, self).__init__() + + def forward(self, INPUT0, INPUT_STATE, START, READY, CORRID): + # On sequence START, reset the running state to INPUT0; otherwise + # accumulate onto the carried state. The emitted output adds the + # correlation id so tests can confirm CORRID delivery, and READY + # gates it (active batch slots have READY == 1). + keep = (1 - START).to(INPUT_STATE.dtype) + new_state = INPUT0 + INPUT_STATE * keep + out = (new_state + CORRID.to(new_state.dtype)) * READY.to(new_state.dtype) + return out, new_state + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = SequenceNet().to(device).eval() + + # Export with a dynamic first (batch) dimension so the AOTI artifact accepts + # any batch size in [1, max_batch]. The correlation id is delivered as an + # INT32 tensor (see config), independent of the model's data type. + export_batch = 2 if max_batch > 0 else 1 + data_shape = [export_batch] + list(shape) + ctrl_shape = [export_batch, 1] + sample_inputs = ( + torch.zeros(data_shape, dtype=torch_dtype, device=device), + torch.zeros(data_shape, dtype=torch_dtype, device=device), + torch.zeros(ctrl_shape, dtype=torch_dtype, device=device), + torch.zeros(ctrl_shape, dtype=torch_dtype, device=device), + torch.zeros(ctrl_shape, dtype=torch.int32, device=device), + ) + + dynamic_shapes = None + if max_batch > 0: + batch = torch.export.Dim("batch", min=1, max=max_batch) + dynamic_shapes = ( + {0: batch}, + {0: batch}, + {0: batch}, + {0: batch}, + {0: batch}, + ) + + model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) + try: + os.makedirs(model_version_dir) + except OSError: + pass # ignore existing dir + + exported_model = torch.export.export( + model, sample_inputs, dynamic_shapes=dynamic_shapes + ) + torch._inductor.aoti_compile_and_package( + exported_model, package_path=model_version_dir + "/model.pt2" + ) + + +def create_torch_aoti_modelconfig(models_dir, max_batch, dtype, shape): + if dtype not in (np.float32, np.int32): + return + + model_name = tu.get_sequence_model_name("torch_aoti", dtype) + config_dir = models_dir + "/" + model_name + control_type = "int32" if dtype == np.int32 else "fp32" + + config = f""" +name: "{model_name}" +backend: "pytorch" +platform: "torch_aoti" +max_batch_size: {max_batch} +sequence_batching {{ + max_sequence_idle_microseconds: 5000000 + control_input [ + {{ + name: "INPUT__2" + control [ + {{ + kind: CONTROL_SEQUENCE_START + {control_type}_false_true: [ 0, 1 ] + }} + ] + }}, + {{ + name: "INPUT__3" + control [ + {{ + kind: CONTROL_SEQUENCE_READY + {control_type}_false_true: [ 0, 1 ] + }} + ] + }}, + {{ + name: "INPUT__4" + control [ + {{ + kind: CONTROL_SEQUENCE_CORRID + data_type: TYPE_INT32 + }} + ] + }} + ] + state [ + {{ + input_name: "INPUT__1" + output_name: "OUTPUT__1" + data_type: {np_to_model_dtype(dtype)} + dims: [ {tu.shape_to_dims_str(shape)} ] + }} + ] +}} +input [ + {{ + name: "INPUT__0" + data_type: {np_to_model_dtype(dtype)} + dims: [ {tu.shape_to_dims_str(shape)} ] + }} +] +output [ + {{ + name: "OUTPUT__0" + data_type: {np_to_model_dtype(dtype)} + dims: [ {tu.shape_to_dims_str(shape)} ] + }} +] +instance_group [ + {{ + kind: {"KIND_GPU" if torch.cuda.is_available() else "KIND_CPU"} + }} +] +""" + + try: + os.makedirs(config_dir) + except OSError: + pass # ignore existing dir + + with open(config_dir + "/config.pbtxt", "w") as cfile: + cfile.write(config) + + +def create_torch_aoti_forward_modelconfig(models_dir, model_version): + # Config-only variant of the float32 sequence model that addresses the + # control/state tensors via the forward-argument interface (ARGS[...] / + # RESULT[...]) instead of the ordinal INPUT__N / OUTPUT__N names. It reuses + # the float32 sequence artifact (5 positional inputs -> ARGS[0..4], two + # outputs -> RESULT[0], RESULT[1]). + import shutil + + src = models_dir + "/" + tu.get_sequence_model_name("torch_aoti", np.float32) + src_pt2 = src + "/" + str(model_version) + "/model.pt2" + if not os.path.exists(src_pt2): + print(f"warning: {src_pt2} not found; skipping forward-interface model") + return + + model_name = "torch_aoti_sequence_forward_float32" + dst_dir = models_dir + "/" + model_name + "/" + str(model_version) + try: + os.makedirs(dst_dir) + except OSError: + pass # ignore existing dir + shutil.copy(src_pt2, dst_dir + "/model.pt2") + + config = f""" +name: "{model_name}" +backend: "pytorch" +platform: "torch_aoti" +max_batch_size: 8 +sequence_batching {{ + max_sequence_idle_microseconds: 5000000 + control_input [ + {{ name: "ARGS[2]" control [{{ kind: CONTROL_SEQUENCE_START fp32_false_true: [ 0, 1 ] }}] }}, + {{ name: "ARGS[3]" control [{{ kind: CONTROL_SEQUENCE_READY fp32_false_true: [ 0, 1 ] }}] }}, + {{ name: "ARGS[4]" control [{{ kind: CONTROL_SEQUENCE_CORRID data_type: TYPE_INT32 }}] }} + ] + state [ + {{ + input_name: "ARGS[1]" + output_name: "RESULT[1]" + data_type: {np_to_model_dtype(np.float32)} + dims: [ 1 ] + }} + ] +}} +input [ + {{ name: "ARGS[0]" data_type: {np_to_model_dtype(np.float32)} dims: [ 1 ] }} +] +output [ + {{ name: "RESULT[0]" data_type: {np_to_model_dtype(np.float32)} dims: [ 1 ] }} +] +instance_group [{{ kind: {"KIND_GPU" if torch.cuda.is_available() else "KIND_CPU"} }}] +""" + with open(models_dir + "/" + model_name + "/config.pbtxt", "w") as f: + f.write(config) + print(f"Created forward-interface sequence model {model_name}") + + +def create_torch_aoti_initstate_model(models_dir, model_version, max_batch=8): + # Sequence model that relies on a declared zero initial_state rather than a + # START-driven reset: new_state = INPUT0 + INPUT_STATE (accumulate). On the + # first step the state input is the zero initial_state, so it behaves as a + # running sum that resets when the sequence (and its state) is recycled. + model_name = "torch_aoti_sequence_initstate_float32" + dst_dir = models_dir + "/" + model_name + "/" + str(model_version) + try: + os.makedirs(dst_dir) + except OSError: + pass # ignore existing dir + + class SequenceNet(nn.Module): + def forward(self, INPUT0, INPUT_STATE, START, READY): + new_state = INPUT0 + INPUT_STATE + out = new_state * READY.to(new_state.dtype) + return out, new_state + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = SequenceNet().to(device).eval() + batch = torch.export.Dim("batch", min=1, max=max_batch) + sample = ( + torch.zeros(2, 1, dtype=torch.float32, device=device), + torch.zeros(2, 1, dtype=torch.float32, device=device), + torch.zeros(2, 1, dtype=torch.float32, device=device), + torch.zeros(2, 1, dtype=torch.float32, device=device), + ) + ds = ({0: batch}, {0: batch}, {0: batch}, {0: batch}) + ep = torch.export.export(model, sample, dynamic_shapes=ds) + torch._inductor.aoti_compile_and_package(ep, package_path=dst_dir + "/model.pt2") + + config = f""" +name: "{model_name}" +backend: "pytorch" +platform: "torch_aoti" +max_batch_size: {max_batch} +sequence_batching {{ + max_sequence_idle_microseconds: 5000000 + control_input [ + {{ name: "INPUT__2" control [{{ kind: CONTROL_SEQUENCE_START fp32_false_true: [ 0, 1 ] }}] }}, + {{ name: "INPUT__3" control [{{ kind: CONTROL_SEQUENCE_READY fp32_false_true: [ 0, 1 ] }}] }} + ] + state [ + {{ + input_name: "INPUT__1" + output_name: "OUTPUT__1" + data_type: {np_to_model_dtype(np.float32)} + dims: [ 1 ] + initial_state: {{ + name: "zero state" + data_type: {np_to_model_dtype(np.float32)} + dims: [ 1 ] + zero_data: true + }} + }} + ] +}} +input [ + {{ name: "INPUT__0" data_type: {np_to_model_dtype(np.float32)} dims: [ 1 ] }} +] +output [ + {{ name: "OUTPUT__0" data_type: {np_to_model_dtype(np.float32)} dims: [ 1 ] }} +] +instance_group [{{ kind: {"KIND_GPU" if torch.cuda.is_available() else "KIND_CPU"} }}] +""" + with open(models_dir + "/" + model_name + "/config.pbtxt", "w") as f: + f.write(config) + print(f"Created initial-state sequence model {model_name}") + + +def create_torch_aoti_negative_configs(models_dir, model_version): + # Config-only negative models that reuse the float32 sequence artifact but + # declare an unsupported TYPE_STRING correlation id / state. These must fail + # to load; the L0 test starts a dedicated server and asserts the failure. + import shutil + + src = models_dir + "/" + tu.get_sequence_model_name("torch_aoti", np.float32) + src_pt2 = src + "/" + str(model_version) + "/model.pt2" + if not os.path.exists(src_pt2): + print(f"warning: {src_pt2} not found; skipping negative models") + return + + fp32 = np_to_model_dtype(np.float32) + gpu = "KIND_GPU" if torch.cuda.is_available() else "KIND_CPU" + variants = { + "torch_aoti_sequence_bad_corrid": f""" +name: "torch_aoti_sequence_bad_corrid" +backend: "pytorch" +platform: "torch_aoti" +max_batch_size: 8 +sequence_batching {{ + max_sequence_idle_microseconds: 5000000 + control_input [ + {{ name: "INPUT__2" control [{{ kind: CONTROL_SEQUENCE_START fp32_false_true: [ 0, 1 ] }}] }}, + {{ name: "INPUT__3" control [{{ kind: CONTROL_SEQUENCE_READY fp32_false_true: [ 0, 1 ] }}] }}, + {{ name: "INPUT__4" control [{{ kind: CONTROL_SEQUENCE_CORRID data_type: TYPE_STRING }}] }} + ] + state [ + {{ input_name: "INPUT__1" output_name: "OUTPUT__1" data_type: {fp32} dims: [ 1 ] }} + ] +}} +input [ {{ name: "INPUT__0" data_type: {fp32} dims: [ 1 ] }} ] +output [ {{ name: "OUTPUT__0" data_type: {fp32} dims: [ 1 ] }} ] +instance_group [{{ kind: {gpu} }}] +""", + "torch_aoti_sequence_bad_state": f""" +name: "torch_aoti_sequence_bad_state" +backend: "pytorch" +platform: "torch_aoti" +max_batch_size: 8 +sequence_batching {{ + max_sequence_idle_microseconds: 5000000 + control_input [ + {{ name: "INPUT__2" control [{{ kind: CONTROL_SEQUENCE_START fp32_false_true: [ 0, 1 ] }}] }}, + {{ name: "INPUT__3" control [{{ kind: CONTROL_SEQUENCE_READY fp32_false_true: [ 0, 1 ] }}] }} + ] + state [ + {{ input_name: "INPUT__1" output_name: "OUTPUT__1" data_type: TYPE_STRING dims: [ 1 ] }} + ] +}} +input [ {{ name: "INPUT__0" data_type: {fp32} dims: [ 1 ] }} ] +output [ {{ name: "OUTPUT__0" data_type: {fp32} dims: [ 1 ] }} ] +instance_group [{{ kind: {gpu} }}] +""", + } + for model_name, config in variants.items(): + dst_dir = models_dir + "/" + model_name + "/" + str(model_version) + try: + os.makedirs(dst_dir) + except OSError: + pass # ignore existing dir + shutil.copy(src_pt2, dst_dir + "/model.pt2") + with open(models_dir + "/" + model_name + "/config.pbtxt", "w") as f: + f.write(config) + print(f"Created negative sequence model {model_name}") + + def create_models(models_dir, dtype, shape, initial_state, no_batch=True): model_version = 1 @@ -1288,6 +1634,18 @@ def create_models(models_dir, dtype, shape, initial_state, no_batch=True): models_dir, model_version, 0, dtype, shape + suffix, initial_state ) + if FLAGS.torch_aoti: + # AOTI sequence models are generated with first-dim batching enabled. + create_torch_aoti_modelconfig(models_dir, 8, dtype, shape) + create_torch_aoti_modelfile(models_dir, model_version, 8, dtype, shape) + # Generate the float32-only variants once (they reuse / extend the + # float32 sequence artifact): forward-interface naming, declared zero + # initial_state, and the negative (unsupported-type) load-failure models. + if dtype == np.float32 and no_batch: + create_torch_aoti_forward_modelconfig(models_dir, model_version) + create_torch_aoti_initstate_model(models_dir, model_version, 8) + create_torch_aoti_negative_configs(models_dir, model_version) + if __name__ == "__main__": parser = argparse.ArgumentParser() @@ -1328,6 +1686,12 @@ def create_models(models_dir, dtype, shape, initial_state, no_batch=True): action="store_true", help="Generate Pytorch LibTorch models", ) + parser.add_argument( + "--torch-aoti", + required=False, + action="store_true", + help="Generate PyTorch AOT Inductor (PT2) sequence models", + ) parser.add_argument( "--openvino", required=False, @@ -1356,7 +1720,7 @@ def create_models(models_dir, dtype, shape, initial_state, no_batch=True): if FLAGS.tensorrt: import tensorrt as trt - if FLAGS.libtorch: + if FLAGS.libtorch or FLAGS.torch_aoti: import torch from torch import nn diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index ca873fe8f1..7952ef5cb8 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -277,6 +277,8 @@ python3 $TRITON_MDLS_SRC_DIR/gen_qa_sequence_models.py --libtorch --variable --m chmod -R 777 $TRITON_MDLS_QA_VARIABLE_SEQUENCE_MODEL python3 $TRITON_MDLS_SRC_DIR/gen_qa_implicit_models.py --libtorch --models_dir=$TRITON_MDLS_QA_SEQUENCE_IMPLICIT_MODEL chmod -R 777 $TRITON_MDLS_QA_SEQUENCE_IMPLICIT_MODEL +python3 $TRITON_MDLS_SRC_DIR/gen_qa_implicit_models.py --torch-aoti --models_dir=$TRITON_MDLS_QA_SEQUENCE_IMPLICIT_MODEL +chmod -R 777 $TRITON_MDLS_QA_SEQUENCE_IMPLICIT_MODEL python3 $TRITON_MDLS_SRC_DIR/gen_qa_implicit_models.py --libtorch --variable --models_dir=$TRITON_MDLS_QA_VARIABLE_SEQUENCE_IMPLICIT_MODEL chmod -R 777 $TRITON_MDLS_QA_VARIABLE_SEQUENCE_IMPLICIT_MODEL python3 $TRITON_MDLS_SRC_DIR/gen_qa_dyna_sequence_models.py --libtorch --models_dir=$TRITON_MDLS_QA_DYNA_SEQUENCE_MODEL diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py index 2008cd824a..bf34cb4eb9 100755 --- a/qa/common/gen_qa_models.py +++ b/qa/common/gen_qa_models.py @@ -1383,6 +1383,7 @@ def np_to_dtype(np_dtype): def create_torch_aoti_model_file( models_dir, + max_batch, model_version, input_shape, input_dtype, @@ -1451,11 +1452,32 @@ def forward(self, INPUT0: torch.Tensor, INPUT1: torch.Tensor) -> torch.Tensor: model.to(device) model = model.eval() - sample_inputs = generate_torch_aoti_sample_inputs(input_shape, input_dtype, device) + # When batching is enabled, the AOTI artifact must be compiled with a + # dynamic first (batch) dimension. Otherwise the compiled model specializes + # to a static shape and rejects any batch size other than the one used at + # export time. We export with a representative batch > 1 and declare dim 0 + # of each input as dynamic over [1, max_batch]. + if max_batch > 0: + export_input_shape = [2] + list(input_shape) + else: + export_input_shape = list(input_shape) + + sample_inputs = generate_torch_aoti_sample_inputs( + export_input_shape, input_dtype, device + ) + + dynamic_shapes = None + if max_batch > 0: + batch_dim = torch.export.Dim("batch", min=1, max=max_batch) + # One spec per positional argument of AddSubNet.forward(INPUT0, INPUT1). + dynamic_shapes = ({0: batch_dim}, {0: batch_dim}) + package_path = os.path.join(model_version_dir, "model.pt2") try: - exported_model = torch.export.export(model, sample_inputs) + exported_model = torch.export.export( + model, sample_inputs, dynamic_shapes=dynamic_shapes + ) torch._inductor.aoti_compile_and_package( exported_model, package_path=package_path, @@ -1640,15 +1662,22 @@ def create_torchvision_aoti_model_file( model = model.to(device) model = model.eval() - SHAPE = (max_batch, 3, 224, 224) + # When batching is enabled, export with a dynamic first (batch) dimension so + # the AOTI artifact accepts any batch size in [1, max_batch]. A batch>=2 + # sample is used to avoid 0/1 specialization. + if max_batch > 0: + SHAPE = (2, 3, 224, 224) + dynamic_shapes = ({0: torch.export.Dim("batch", min=1, max=max_batch)},) + else: + SHAPE = (1, 3, 224, 224) + dynamic_shapes = None - # Example input tensor with batch size 1 and 3 color channels (RGB), height and width of 224 sample_inputs = (torch.zeros(SHAPE, dtype=torch.float32, device=device),) package_path = os.path.join(model_version_dir, "model.pt2") try: - ep = torch.export.export(model, sample_inputs) + ep = torch.export.export(model, sample_inputs, dynamic_shapes=dynamic_shapes) torch._inductor.aoti_compile_and_package(ep, package_path=package_path) except Exception as e: print( @@ -1759,6 +1788,7 @@ def create_libtorch_modelconfig( def create_torch_aoti_model_config( models_dir, + max_batch, input_shape, output_shape, input_dtype, @@ -1777,7 +1807,6 @@ def create_torch_aoti_model_config( else: version_policy_str = "{ all { }}" - # Use a different model name for the non-batching variant model_name = tu.get_model_name( "torch_aoti", input_dtype, @@ -1793,7 +1822,12 @@ def create_torch_aoti_model_config( backend: "pytorch" name: "{model_name}" platform: "torch_aoti" +max_batch_size: {max_batch} version_policy: {version_policy_str} +dynamic_batching {{ + preferred_batch_size: [ 4, 8 ] + max_queue_delay_microseconds: 1000 +}} input [ {{ name: "ARGS[0]" @@ -1836,6 +1870,112 @@ def create_torch_aoti_model_config( print(f"Created {label_path}") +def create_torch_aoti_variable_model(models_dir, max_batch=8): + # AOTI add model exported with two dynamic dimensions (batch + feature) so + # batching can be exercised on a model whose non-batch shape is variable + # (dims: [-1]). Float32 only. + model_name = "torch_aoti_variable_float32" + model_version_dir = os.path.join(models_dir, model_name, "1") + try: + os.makedirs(model_version_dir) + except OSError: + pass # ignore existing dir + + print(f"{_color_green}Creating model {model_name}{_color_reset}") + + class AddNet(nn.Module): + def forward(self, INPUT0, INPUT1): + return INPUT0 + INPUT1 + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = AddNet().to(device).eval() + + # Sample uses batch>=2 and feature>=2 to avoid 0/1 specialization. + sample = ( + torch.zeros(2, 4, dtype=torch.float32, device=device), + torch.zeros(2, 4, dtype=torch.float32, device=device), + ) + batch = torch.export.Dim("batch", min=1, max=max_batch) + feature = torch.export.Dim("feature", min=1, max=512) + dynamic_shapes = ( + {0: batch, 1: feature}, + {0: batch, 1: feature}, + ) + try: + ep = torch.export.export(model, sample, dynamic_shapes=dynamic_shapes) + torch._inductor.aoti_compile_and_package( + ep, package_path=os.path.join(model_version_dir, "model.pt2") + ) + except Exception as e: + print( + f"{_color_red}error: Failed to create model {model_name}: {e}{_color_reset}", + file=sys.stderr, + ) + return + + config = f""" +backend: "pytorch" +name: "{model_name}" +platform: "torch_aoti" +max_batch_size: {max_batch} +dynamic_batching {{ max_queue_delay_microseconds: 1000 }} +input [ + {{ name: "ARGS[0]" data_type: TYPE_FP32 dims: [ -1 ] }}, + {{ name: "ARGS[1]" data_type: TYPE_FP32 dims: [ -1 ] }} +] +output [ + {{ name: "RESULT" data_type: TYPE_FP32 dims: [ -1 ] }} +] +instance_group [{{ kind: {"KIND_GPU" if torch.cuda.is_available() else "KIND_CPU"} }}] +""" + with open(os.path.join(models_dir, model_name, "config.pbtxt"), "w") as f: + f.write(config) + print(f"Created config for {model_name}") + + +def create_torch_aoti_multi_instance_model(models_dir, max_batch=8): + # Config-only variant that reuses the float32 add model artifact but runs + # with two instances, to exercise batching across multiple model instances. + import shutil + + src_name = tu.get_model_name("torch_aoti", np.float32, np.float32, None) + src_pt2 = os.path.join(models_dir, src_name, "1", "model.pt2") + if not os.path.exists(src_pt2): + print( + f"{_color_yellow}warning: {src_pt2} not found; skipping multi-instance " + f"model{_color_reset}" + ) + return + + model_name = "torch_aoti_multi_instance_float32" + model_version_dir = os.path.join(models_dir, model_name, "1") + try: + os.makedirs(model_version_dir) + except OSError: + pass # ignore existing dir + shutil.copy(src_pt2, os.path.join(model_version_dir, "model.pt2")) + + print(f"{_color_green}Creating model {model_name}{_color_reset}") + config = f""" +backend: "pytorch" +name: "{model_name}" +platform: "torch_aoti" +max_batch_size: {max_batch} +dynamic_batching {{ max_queue_delay_microseconds: 1000 }} +input [ + {{ name: "ARGS[0]" data_type: TYPE_FP32 dims: [ 16 ] }}, + {{ name: "ARGS[1]" data_type: TYPE_FP32 dims: [ 16 ] }} +] +output [ + {{ name: "RESULT" data_type: TYPE_FP32 dims: [ 16 ] }} +] +instance_group [{{ count: 2 kind: {"KIND_GPU" if torch.cuda.is_available() else "KIND_CPU"} }}] +""" + with open(os.path.join(models_dir, model_name, "config.pbtxt"), "w") as f: + f.write(config) + print(f"Created config for {model_name}") + + def create_torch_aoti_complex_model_config( models_dir, ): @@ -2405,6 +2545,7 @@ def create_models( # max-batch 8 if create_torch_aoti_model_file( models_dir, + 8, model_version, input_shape, input_dtype, @@ -2412,6 +2553,7 @@ def create_models( ): create_torch_aoti_model_config( models_dir, + 8, input_shape, output0_shape, input_dtype, @@ -3061,9 +3203,14 @@ def create_fixed_models( ) if create_torch_aoti_complex_model_file(FLAGS.models_dir): create_torch_aoti_complex_model_config(FLAGS.models_dir) + # Batching coverage models: variable non-batch dim, and a multi-instance + # variant that reuses the float32 add model artifact. + create_torch_aoti_variable_model(FLAGS.models_dir) + create_torch_aoti_multi_instance_model(FLAGS.models_dir) if FLAGS.torchvision_aoti: - # TODO: Add support for variable batch size and version policy for torchvision AOTI models. print(f"{_color_blue}TorchVision AOTI model generation requested{_color_reset}") - if create_torchvision_aoti_model_file(FLAGS.models_dir, 1): - create_torchvision_aoti_model_config(FLAGS.models_dir, 1) + # Export with a dynamic batch dimension (max_batch_size 8) to exercise + # batching of a real, higher-rank ([N,3,224,224]) model. + if create_torchvision_aoti_model_file(FLAGS.models_dir, 8): + create_torchvision_aoti_model_config(FLAGS.models_dir, 8) From fe1f873f54e25691f0141f8dfd3f25e11f356ee6 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 11 Jun 2026 22:28:17 +0530 Subject: [PATCH 3/7] test: Fix L0_perf_tensorrt_llm (#8823) (#8832) --- qa/L0_perf_tensorrt_llm/test.sh | 9 ++++++--- qa/common/trtllm_util.sh | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/qa/L0_perf_tensorrt_llm/test.sh b/qa/L0_perf_tensorrt_llm/test.sh index 5bf418c5c5..29057c0feb 100755 --- a/qa/L0_perf_tensorrt_llm/test.sh +++ b/qa/L0_perf_tensorrt_llm/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -29,6 +29,7 @@ RET=0 BASE_DIR=$(pwd) NUM_GPUS=${NUM_GPUS:=1} TENSORRTLLM_BACKEND_REPO_TAG=${TENSORRTLLM_BACKEND_REPO_TAG:="main"} +TRITON_REPO_ORG=${TRITON_REPO_ORG:="https://github.com/triton-inference-server"} TRT_ROOT="/usr/local/tensorrt" MODEL_NAME="gpt2_tensorrt_llm" @@ -76,6 +77,9 @@ function upgrade_openmpi { } # Update environment variables + export PATH=/opt/hpcx/ompi/bin:$PATH + export LD_LIBRARY_PATH=/opt/hpcx/ompi/lib:$LD_LIBRARY_PATH + if ! grep -q '/opt/hpcx/ompi/bin' ~/.bashrc; then echo 'export PATH=/opt/hpcx/ompi/bin:$PATH' >>~/.bashrc fi @@ -84,7 +88,6 @@ function upgrade_openmpi { echo 'export LD_LIBRARY_PATH=/opt/hpcx/ompi/lib:$LD_LIBRARY_PATH' >>~/.bashrc fi ldconfig - source ~/.bashrc cd "$BASE_DIR" mpirun --version } @@ -96,7 +99,7 @@ build_gpt2_tensorrt_engine prepare_model_repository # Install perf_analyzer -pip3 install tritonclient +pip3 install perf_analyzer ARCH="amd64" STATIC_BATCH=1 diff --git a/qa/common/trtllm_util.sh b/qa/common/trtllm_util.sh index 81ecb2d770..e36bcffc35 100755 --- a/qa/common/trtllm_util.sh +++ b/qa/common/trtllm_util.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -53,7 +53,7 @@ function build_gpt2_tensorrt_engine { trtllm-build --checkpoint_dir "./c-model/gpt2/${NUM_GPUS}-gpu/" \ --gpt_attention_plugin float16 \ --remove_input_padding enable \ - --paged_kv_cache enable \ + --kv_cache_type paged \ --gemm_plugin float16 \ --workers "${NUM_GPUS}" \ --output_dir "${ENGINES_DIR}" From d4d7275d517a5faf6fb18d154e6b54df8ee9c725 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Fri, 12 Jun 2026 12:53:13 -0700 Subject: [PATCH 4/7] fix(qa): TRT 11 strongly-typed cast for plan_float* output dtype (#8836) --- qa/common/gen_qa_models.py | 44 ++++++++++++++------------------------ 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py index bf34cb4eb9..935140f539 100755 --- a/qa/common/gen_qa_models.py +++ b/qa/common/gen_qa_models.py @@ -109,30 +109,20 @@ def create_plan_dynamic_rf_modelfile( out0 = add if not swap else sub out1 = sub if not swap else add - # uint8 conversion after operations - # FIXME: Remove support check when jetson supports TRT 8.5 (DLIS-4256) - if tu.support_trt_uint8(): - if trt_output0_dtype == trt.uint8: - out0 = trt_cast_tensor(network, out0.get_output(0), trt.uint8) - if trt_output1_dtype == trt.uint8: - out1 = trt_cast_tensor(network, out1.get_output(0), trt.uint8) + # TRT 11 strongly-typed networks: ITensor.dtype setter no longer coerces + # output dtype, so insert an explicit cast whenever the elementwise op's + # natural output dtype differs from the declared output dtype. Covers + # both float<->float and uint8 cases. + if out0.get_output(0).dtype != trt_output0_dtype: + out0 = trt_cast_tensor(network, out0.get_output(0), trt_output0_dtype) + if out1.get_output(0).dtype != trt_output1_dtype: + out1 = trt_cast_tensor(network, out1.get_output(0), trt_output1_dtype) out0.get_output(0).name = "OUTPUT0" out1.get_output(0).name = "OUTPUT1" network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - # ITensor.dtype setter removed in TRT 11; cast above already produced - # the desired output dtype on modern TRT. - try: - out0.get_output(0).dtype = trt_output0_dtype - except AttributeError: - pass # ITensor.dtype setter removed in TensorRT 11+ - try: - out1.get_output(0).dtype = trt_output1_dtype - except AttributeError: - pass # ITensor.dtype setter removed in TensorRT 11+ - in0.allowed_formats = 1 << int(trt_memory_format) in1.allowed_formats = 1 << int(trt_memory_format) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) @@ -422,21 +412,19 @@ def create_plan_fixed_rf_modelfile( out0 = add if not swap else sub out1 = sub if not swap else add + # TRT 11 strongly-typed networks: ITensor.dtype setter no longer coerces + # output dtype, so insert an explicit cast whenever the elementwise op's + # natural output dtype differs from the declared output dtype. + if out0.get_output(0).dtype != trt_output0_dtype: + out0 = trt_cast_tensor(network, out0.get_output(0), trt_output0_dtype) + if out1.get_output(0).dtype != trt_output1_dtype: + out1 = trt_cast_tensor(network, out1.get_output(0), trt_output1_dtype) + out0.get_output(0).name = "OUTPUT0" out1.get_output(0).name = "OUTPUT1" network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - # ITensor.dtype setter removed in TRT 11; output dtype already matches. - try: - out0.get_output(0).dtype = trt_output0_dtype - except AttributeError: - pass # ITensor.dtype setter removed in TensorRT 11+ - try: - out1.get_output(0).dtype = trt_output1_dtype - except AttributeError: - pass # ITensor.dtype setter removed in TensorRT 11+ - in0.allowed_formats = 1 << int(trt_memory_format) in1.allowed_formats = 1 << int(trt_memory_format) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) From a4aab2b7e5e0499d96be77304e406f86ba4dee7f Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Tue, 16 Jun 2026 11:24:17 -0700 Subject: [PATCH 5/7] docs(server): update README and versions for r26.06 (#8844) --- Dockerfile.sdk | 2 +- README.md | 11 +++-------- build.py | 4 ++-- deploy/aws/values.yaml | 2 +- deploy/fleetcommand/Chart.yaml | 2 +- deploy/fleetcommand/values.yaml | 6 +++--- deploy/gcp/values.yaml | 2 +- .../perf-analyzer-script/triton_client.yaml | 2 +- .../server-deployer/build_and_push.sh | 4 ++-- .../server-deployer/chart/triton/Chart.yaml | 2 +- .../server-deployer/chart/triton/values.yaml | 6 +++--- .../server-deployer/data-test/schema.yaml | 2 +- .../server-deployer/schema.yaml | 4 ++-- .../gke-marketplace-app/trt-engine/README.md | 6 +++--- deploy/k8s-onprem/values.yaml | 2 +- deploy/oci/values.yaml | 2 +- docs/customization_guide/compose.md | 18 +++++++++--------- docs/getting_started/llm.md | 4 ++-- docs/introduction/release_notes.md | 4 ++-- docs/user_guide/performance_tuning.md | 4 ++-- python/openai/README.md | 6 +++--- qa/common/gen_jetson_trt_models | 2 +- qa/common/gen_qa_model_repository | 2 +- 23 files changed, 47 insertions(+), 52 deletions(-) diff --git a/Dockerfile.sdk b/Dockerfile.sdk index b2181abe6e..d6eed1e363 100644 --- a/Dockerfile.sdk +++ b/Dockerfile.sdk @@ -29,7 +29,7 @@ # # Base image on the minimum Triton container -ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:26.05-py3-min +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:26.06-py3-min ARG TRITON_CLIENT_REPO_SUBDIR=clientrepo ARG TRITON_REPO_ORGANIZATION=http://github.com/triton-inference-server diff --git a/README.md b/README.md index fdb6b2a5bf..46c47848db 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,6 @@ --> [![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) ->[!WARNING] ->You are currently on the `main` branch which tracks under-development progress ->towards the next release. The current release is version [2.69.0](https://github.com/triton-inference-server/server/releases/latest) ->and corresponds to the 26.05 container release on NVIDIA GPU Cloud (NGC). - # Triton Inference Server Triton Inference Server is an open source inference serving software that @@ -90,16 +85,16 @@ Inference Server with the ```bash # Step 1: Create the example model repository -git clone -b r26.05 https://github.com/triton-inference-server/server.git +git clone -b r26.06 https://github.com/triton-inference-server/server.git cd server/docs/examples ./fetch_models.sh # Step 2: Launch triton from the NGC Triton container -docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:26.05-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx +docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:26.06-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx # Step 3: Sending an Inference Request # In a separate console, launch the image_client example from the NGC Triton SDK container -docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:26.05-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg +docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:26.06-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg # Inference should return the following Image '/workspace/images/mug.jpg': diff --git a/build.py b/build.py index e12a074aa3..fa23ee1742 100755 --- a/build.py +++ b/build.py @@ -71,8 +71,8 @@ # DEFAULT_TRITON_VERSION_MAP = { - "release_version": "2.70.0dev", - "triton_container_version": "26.06dev", + "release_version": "2.70.0", + "triton_container_version": "26.06", "upstream_container_version": "26.05", "ort_version": "1.24.4", "ort_openvino_version": "2026.2.0", diff --git a/deploy/aws/values.yaml b/deploy/aws/values.yaml index c94f832aa8..81700622b4 100644 --- a/deploy/aws/values.yaml +++ b/deploy/aws/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:26.05-py3 + imageName: nvcr.io/nvidia/tritonserver:26.06-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/fleetcommand/Chart.yaml b/deploy/fleetcommand/Chart.yaml index bd360e7955..8a9cc7a56d 100644 --- a/deploy/fleetcommand/Chart.yaml +++ b/deploy/fleetcommand/Chart.yaml @@ -26,7 +26,7 @@ apiVersion: v1 # appVersion is the Triton version; update when changing release -appVersion: 2.69.0 +appVersion: 2.70.0 description: Triton Inference Server (Fleet Command) name: triton-inference-server # version is the Chart version; update when changing anything in the chart diff --git a/deploy/fleetcommand/values.yaml b/deploy/fleetcommand/values.yaml index b911db4afd..33e525bceb 100644 --- a/deploy/fleetcommand/values.yaml +++ b/deploy/fleetcommand/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:26.05-py3 + imageName: nvcr.io/nvidia/tritonserver:26.06-py3 pullPolicy: IfNotPresent numGpus: 1 serverCommand: tritonserver @@ -47,13 +47,13 @@ image: # # To set model control mode, uncomment and configure below # TODO: Fix the following url, it is invalid - # See https://github.com/triton-inference-server/server/blob/r26.05/docs/user_guide/model_management.md + # See https://github.com/triton-inference-server/server/blob/r26.06/docs/user_guide/model_management.md # for more details #- --model-control-mode=explicit|poll|none # # Additional server args # - # see https://github.com/triton-inference-server/server/blob/r26.05/README.md + # see https://github.com/triton-inference-server/server/blob/r26.06/README.md # for more details service: diff --git a/deploy/gcp/values.yaml b/deploy/gcp/values.yaml index 9784c9d252..6418ab82c0 100644 --- a/deploy/gcp/values.yaml +++ b/deploy/gcp/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:26.05-py3 + imageName: nvcr.io/nvidia/tritonserver:26.06-py3 pullPolicy: IfNotPresent modelRepositoryPath: gs://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml index 0e1347f4fd..3c1b93dfe6 100644 --- a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml +++ b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml @@ -33,7 +33,7 @@ metadata: namespace: default spec: containers: - - image: nvcr.io/nvidia/tritonserver:26.05-py3-sdk + - image: nvcr.io/nvidia/tritonserver:26.06-py3-sdk imagePullPolicy: Always name: nv-triton-client securityContext: diff --git a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh index 4b4468d89d..6e166361ae 100755 --- a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh +++ b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh @@ -28,8 +28,8 @@ export REGISTRY=gcr.io/$(gcloud config get-value project | tr ':' '/') export APP_NAME=tritonserver export MAJOR_VERSION=2.67 -export MINOR_VERSION=2.69.0 -export NGC_VERSION=26.05-py3 +export MINOR_VERSION=2.70.0 +export NGC_VERSION=26.06-py3 docker pull nvcr.io/nvidia/$APP_NAME:$NGC_VERSION diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml index d150f0e8d7..d1323108e3 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml @@ -28,4 +28,4 @@ apiVersion: v1 appVersion: "2.68" description: Triton Inference Server name: triton-inference-server -version: 2.69.0 +version: 2.70.0 diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml index 362107e71a..17232cdd2b 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml @@ -31,14 +31,14 @@ maxReplicaCount: 3 tritonProtocol: HTTP # HPA GPU utilization autoscaling target HPATargetAverageValue: 85 -modelRepositoryPath: gs://triton_sample_models/26.05 -publishedVersion: '2.69.0' +modelRepositoryPath: gs://triton_sample_models/26.06 +publishedVersion: '2.70.0' gcpMarketplace: true image: registry: gcr.io repository: nvidia-ngc-public/tritonserver - tag: 26.05-py3 + tag: 26.06-py3 pullPolicy: IfNotPresent # modify the model repository here to match your GCP storage bucket numGpus: 1 diff --git a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml index 4c312c9880..6a68da95d6 100644 --- a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.69.0' + publishedVersion: '2.70.0' publishedVersionMetadata: releaseNote: >- Initial release. diff --git a/deploy/gke-marketplace-app/server-deployer/schema.yaml b/deploy/gke-marketplace-app/server-deployer/schema.yaml index ccf3b157c4..2e95769c0c 100644 --- a/deploy/gke-marketplace-app/server-deployer/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/schema.yaml @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.69.0' + publishedVersion: '2.70.0' publishedVersionMetadata: releaseNote: >- Initial release. @@ -89,7 +89,7 @@ properties: modelRepositoryPath: type: string title: Bucket where models are stored. Please make sure the user/service account to create the GKE app has permission to this GCS bucket. Read Triton documentation on configs and formatting details, supporting TensorRT, TensorFlow, Pytorch, Onnx ... etc. - default: gs://triton_sample_models/26.05 + default: gs://triton_sample_models/26.06 image.ldPreloadPath: type: string title: Leave this empty by default. Triton allows users to create custom layers for backend such as TensorRT plugin, the compiled shared library must be provided via LD_PRELOAD environment variable. diff --git a/deploy/gke-marketplace-app/trt-engine/README.md b/deploy/gke-marketplace-app/trt-engine/README.md index fff7466da4..6c6f26acbd 100644 --- a/deploy/gke-marketplace-app/trt-engine/README.md +++ b/deploy/gke-marketplace-app/trt-engine/README.md @@ -33,7 +33,7 @@ ``` docker run --gpus all -it --network host \ --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \ - -v ~:/scripts nvcr.io/nvidia/tensorrt:26.05-py3 + -v ~:/scripts nvcr.io/nvidia/tensorrt:26.06-py3 pip install onnx six torch tf2onnx tensorflow @@ -57,7 +57,7 @@ mkdir -p engines python3 builder.py -m models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/model.ckpt -o engines/bert_large_int8_bs1_s128.engine -b 1 -s 128 -c models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/ -v models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/vocab.txt --int8 --fp16 --strict --calib-num 1 -iln -imh -gsutil cp bert_large_int8_bs1_s128.engine gs://triton_sample_models/26.05/bert/1/model.plan +gsutil cp bert_large_int8_bs1_s128.engine gs://triton_sample_models/26.06/bert/1/model.plan ``` -For each Triton upgrade, container version used to generate the model, and the model path in GCS `gs://triton_sample_models/26.05/` should be updated accordingly with the correct version. +For each Triton upgrade, container version used to generate the model, and the model path in GCS `gs://triton_sample_models/26.06/` should be updated accordingly with the correct version. diff --git a/deploy/k8s-onprem/values.yaml b/deploy/k8s-onprem/values.yaml index 3d788f3f17..8b83d1cf98 100644 --- a/deploy/k8s-onprem/values.yaml +++ b/deploy/k8s-onprem/values.yaml @@ -30,7 +30,7 @@ tags: openshift: false image: - imageName: nvcr.io/nvidia/tritonserver:26.05-py3 + imageName: nvcr.io/nvidia/tritonserver:26.06-py3 pullPolicy: IfNotPresent modelRepositoryServer: < Replace with the IP Address of your file server > modelRepositoryPath: /srv/models diff --git a/deploy/oci/values.yaml b/deploy/oci/values.yaml index df5d60066d..3ef505aa92 100644 --- a/deploy/oci/values.yaml +++ b/deploy/oci/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:26.05-py3 + imageName: nvcr.io/nvidia/tritonserver:26.06-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://https://.compat.objectstorage..oraclecloud.com:443/triton-inference-server-repository numGpus: 1 diff --git a/docs/customization_guide/compose.md b/docs/customization_guide/compose.md index e922d27fbe..45eb95eda0 100644 --- a/docs/customization_guide/compose.md +++ b/docs/customization_guide/compose.md @@ -46,8 +46,8 @@ The `compose.py` script can be found in the Simply clone the repository and run `compose.py` to create a custom container. Note: Created container version will depend on the branch that was cloned. For example branch - [r26.05](https://github.com/triton-inference-server/server/tree/r26.05) -should be used to create a image based on the NGC 26.05 Triton release. + [r26.06](https://github.com/triton-inference-server/server/tree/r26.06) +should be used to create a image based on the NGC 26.06 Triton release. `compose.py` provides `--backend`, `--repoagent` options that allow you to specify which backends and repository agents to include in the custom image. @@ -78,20 +78,20 @@ For example, running ``` python3 compose.py --backend pytorch --repoagent checksum ``` -on branch [r26.05](https://github.com/triton-inference-server/server/tree/r26.05) pulls: -- `min` container `nvcr.io/nvidia/tritonserver:26.05-py3-min` -- `full` container `nvcr.io/nvidia/tritonserver:26.05-py3` +on branch [r26.06](https://github.com/triton-inference-server/server/tree/r26.06) pulls: +- `min` container `nvcr.io/nvidia/tritonserver:26.06-py3-min` +- `full` container `nvcr.io/nvidia/tritonserver:26.06-py3` Alternatively, users can specify the version of Triton container to pull from any branch by either: 1. Adding flag `--container-version ` to branch ``` -python3 compose.py --backend pytorch --repoagent checksum --container-version 26.05 +python3 compose.py --backend pytorch --repoagent checksum --container-version 26.06 ``` 2. Specifying `--image min, --image full,`. The user is responsible for specifying compatible `min` and `full` containers. ``` -python3 compose.py --backend pytorch --repoagent checksum --image min,nvcr.io/nvidia/tritonserver:26.05-py3-min --image full,nvcr.io/nvidia/tritonserver:26.05-py3 +python3 compose.py --backend pytorch --repoagent checksum --image min,nvcr.io/nvidia/tritonserver:26.06-py3-min --image full,nvcr.io/nvidia/tritonserver:26.06-py3 ``` Method 1 and 2 will result in the same composed container. Furthermore, `--image` flag overrides the `--container-version` flag when both are specified. @@ -102,8 +102,8 @@ Note: 2. vLLM and TensorRT-LLM backends are currently not supported backends for `compose.py`. If you want to build additional backends on top of these backends, it would be better to [build it yourself](#build-it-yourself) by using -`nvcr.io/nvidia/tritonserver:26.05-vllm-python-py3` or -`nvcr.io/nvidia/tritonserver:26.05-trtllm-python-py3` as a `min` container. +`nvcr.io/nvidia/tritonserver:26.06-vllm-python-py3` or +`nvcr.io/nvidia/tritonserver:26.06-trtllm-python-py3` as a `min` container. ### CPU-only container composition diff --git a/docs/getting_started/llm.md b/docs/getting_started/llm.md index 6ea9a5aa33..2ae5f82290 100644 --- a/docs/getting_started/llm.md +++ b/docs/getting_started/llm.md @@ -282,7 +282,7 @@ The above needs to be done manually with your favorite editor. Once finished, pl -v $(pwd)/all_models:/opt/all_models \ -v $(pwd)/scripts:/opt/scripts \ -v $(pwd)/Phi-3-mini-4k-instruct:/opt/Phi-3-mini-4k-instruct \ - nvcr.io/nvidia/tritonserver:26.05-trtllm-python-py3 + nvcr.io/nvidia/tritonserver:26.06-trtllm-python-py3 # Launch Server python3 ../scripts/launch_triton_server.py --model_repo ../all_models/inflight_batcher_llm --world_size 1 @@ -308,7 +308,7 @@ The above needs to be done manually with your favorite editor. Once finished, pl - export RELEASE="26.05" + export RELEASE="26.06" docker run -it --net=host --gpus '"device=0"' nvcr.io/nvidia/tritonserver:${RELEASE}-py3-sdk 17. ## Download the Phi-3 tokenizer diff --git a/docs/introduction/release_notes.md b/docs/introduction/release_notes.md index 19fc0f22f3..6c15ceb35a 100644 --- a/docs/introduction/release_notes.md +++ b/docs/introduction/release_notes.md @@ -25,9 +25,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> -# [Triton Inference Server Release 26.05](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-26-05.html#rel-26-05) +# [Triton Inference Server Release 26.06](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-26-06.html#rel-26-06) -The Triton Inference Server container image, release 26.05, is available +The Triton Inference Server container image, release 26.06, is available on [NGC](https://ngc.nvidia.com/catalog/containers/nvidia:tritonserver) and is open source on [GitHub](https://github.com/triton-inference-server/server). Release notes can diff --git a/docs/user_guide/performance_tuning.md b/docs/user_guide/performance_tuning.md index ff837a4629..4d5eee16a8 100644 --- a/docs/user_guide/performance_tuning.md +++ b/docs/user_guide/performance_tuning.md @@ -235,7 +235,7 @@ with a `tritonserver` binary. ```bash # Start server container -docker run -ti --rm --gpus=all --network=host -v $PWD:/mnt --name triton-server nvcr.io/nvidia/tritonserver:26.05-py3 +docker run -ti --rm --gpus=all --network=host -v $PWD:/mnt --name triton-server nvcr.io/nvidia/tritonserver:26.06-py3 # Start serving your models tritonserver --model-repository=/mnt/models @@ -284,7 +284,7 @@ by setting the `-u` flag, such as `perf_analyzer -m densenet_onnx -u ```bash # Start the SDK container interactively -docker run -ti --rm --gpus=all --network=host -v $PWD:/mnt --name triton-client nvcr.io/nvidia/tritonserver:26.05-py3-sdk +docker run -ti --rm --gpus=all --network=host -v $PWD:/mnt --name triton-client nvcr.io/nvidia/tritonserver:26.06-py3-sdk # Benchmark model being served from step 3 perf_analyzer -m densenet_onnx --concurrency-range 1:4 diff --git a/python/openai/README.md b/python/openai/README.md index 92d91d4db7..f8a1e868d1 100644 --- a/python/openai/README.md +++ b/python/openai/README.md @@ -46,7 +46,7 @@ docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ - nvcr.io/nvidia/tritonserver:26.05-vllm-python-py3 + nvcr.io/nvidia/tritonserver:26.06-vllm-python-py3 ``` 2. Launch the OpenAI-compatible Triton Inference Server: @@ -355,7 +355,7 @@ Currently, OpenAI-Compatible Frontend supports loading embedding models and embe docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ - nvcr.io/nvidia/tritonserver:26.05-vllm-python-py3 + nvcr.io/nvidia/tritonserver:26.06-vllm-python-py3 ``` 2. Launch the OpenAI-compatible Triton Inference Server: @@ -451,7 +451,7 @@ docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ -e TRTLLM_ORCHESTRATOR=1 \ - nvcr.io/nvidia/tritonserver:26.05-trtllm-python-py3 + nvcr.io/nvidia/tritonserver:26.06-trtllm-python-py3 ``` 2. Install dependencies inside the container: diff --git a/qa/common/gen_jetson_trt_models b/qa/common/gen_jetson_trt_models index 4d491fa2a1..658f272f86 100755 --- a/qa/common/gen_jetson_trt_models +++ b/qa/common/gen_jetson_trt_models @@ -34,7 +34,7 @@ # Make all generated files accessible outside of container umask 0000 # Set the version of the models -TRITON_VERSION=${TRITON_VERSION:=26.05} +TRITON_VERSION=${TRITON_VERSION:=26.06} # Set the CUDA device to use NVIDIA_VISIBLE_DEVICES=${NVIDIA_VISIBLE_DEVICES:=0} # Set TensorRT image diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 7952ef5cb8..7fae1a0b23 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -66,7 +66,7 @@ log_message.status "Changing working directory to the script directory to: " "${ cd ${TRITON_MDLS_BASE_SCRIPT_DIR} log_message.status "define: default values" -TRITON_VERSION=${TRITON_VERSION:=26.05} +TRITON_VERSION=${TRITON_VERSION:=26.06} ONNX_VERSION=1.20.1 ONNX_OPSET=0 OPENVINO_VERSION=2024.5.0 From 43efd1099f2e58f7a746a849b67b97d93b28773a Mon Sep 17 00:00:00 2001 From: Matthew Wittwer Date: Mon, 22 Jun 2026 18:23:45 +0000 Subject: [PATCH 6/7] update the expected number of models --- qa/L0_server_status/server_status_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/qa/L0_server_status/server_status_test.py b/qa/L0_server_status/server_status_test.py index 9a0f26e7b9..a39cd0b908 100755 --- a/qa/L0_server_status/server_status_test.py +++ b/qa/L0_server_status/server_status_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -428,6 +428,9 @@ def test_model_specific_infer(self): self.assertTrue(False, "unexpected error {}".format(ex)) +EXPECTED_MODEL_STATS: int = 144 + + class ModelMetadataTest(tu.TestResultCollector): """ These tests must be run after the ServerMetadataTest. See test.sh @@ -723,8 +726,8 @@ def test_infer_stats_no_model(self): stats = infer_stats.model_stats self.assertEqual( len(stats), - 125, - "expected 125 infer stats for all ready versions of all model", + EXPECTED_MODEL_STATS, + f"expected {EXPECTED_MODEL_STATS} infer stats for all ready versions of all model", ) except InferenceServerException as ex: From 7ae4bb3ab063211cb7cf98108bd26f4d79a7631a Mon Sep 17 00:00:00 2001 From: Matthew Wittwer Date: Mon, 22 Jun 2026 22:55:25 +0000 Subject: [PATCH 7/7] dynamically check for expected models ready --- qa/L0_server_status/server_status_test.py | 26 +++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/qa/L0_server_status/server_status_test.py b/qa/L0_server_status/server_status_test.py index a39cd0b908..7903c34799 100755 --- a/qa/L0_server_status/server_status_test.py +++ b/qa/L0_server_status/server_status_test.py @@ -428,9 +428,6 @@ def test_model_specific_infer(self): self.assertTrue(False, "unexpected error {}".format(ex)) -EXPECTED_MODEL_STATS: int = 144 - - class ModelMetadataTest(tu.TestResultCollector): """ These tests must be run after the ServerMetadataTest. See test.sh @@ -718,16 +715,33 @@ def test_infer_stats_no_model(self): self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - # Returns infer stats for ALL models + ready versions + # get_inference_statistics with no model returns one entry per + # ready model version. Derive the expected count from the model + # repository index (the set of currently-ready versions) rather + # than hard-coding it, so adding/removing QA models can not + # silently break this test. + index = triton_client.get_model_repository_index() infer_stats = triton_client.get_inference_statistics() if pair[1] == "http": stats = infer_stats["model_stats"] + expected_model_stats = sum( + 1 for model in index if model.get("state") == "READY" + ) else: stats = infer_stats.model_stats + expected_model_stats = sum( + 1 for model in index.models if model.state == "READY" + ) + self.assertGreater( + expected_model_stats, + 0, + "expected at least one ready model version in the repository index", + ) self.assertEqual( len(stats), - EXPECTED_MODEL_STATS, - f"expected {EXPECTED_MODEL_STATS} infer stats for all ready versions of all model", + expected_model_stats, + f"expected {expected_model_stats} infer stats (one per ready " + f"model version per the repository index), got {len(stats)}", ) except InferenceServerException as ex: