diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fcd86c8884da..ef261b4272e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1325,55 +1325,22 @@ common-files: &common_files | tests/unittest/utils/test_util.py | tests/unittest/utils/torch_ref.py | tests/unittest/utils/util.py | - triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/1/model.py | - triton_backend/all_models/gpt/postprocessing/1/model.py | - triton_backend/all_models/gpt/preprocessing/1/model.py | - triton_backend/all_models/gpt/tensorrt_llm/1/model.py | - triton_backend/all_models/inflight_batcher_llm/postprocessing/1/model.py | - triton_backend/all_models/inflight_batcher_llm/preprocessing/1/model.py | - triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py | - triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/decode.py | - triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/triton_decoder.py | - triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/model.py | triton_backend/all_models/llmapi/tensorrt_llm/1/helpers.py | triton_backend/all_models/llmapi/tensorrt_llm/1/model.py | - triton_backend/all_models/multimodal/multimodal_encoders/1/model.py | - triton_backend/all_models/multimodal/multimodal_encoders/1/multimodal_utils.py | - triton_backend/all_models/tests/test_decode.py | triton_backend/all_models/tests/test_llmapi_python_backend.py | - triton_backend/all_models/tests/test_multi_image_preprocess.py | - triton_backend/all_models/tests/test_multimodal_encoders.py | - triton_backend/all_models/tests/test_python_backend.py | - triton_backend/all_models/tests/test_triton_decoder.py | - triton_backend/all_models/whisper/whisper_bls/1/fbank.py | - triton_backend/all_models/whisper/whisper_bls/1/model.py | - triton_backend/all_models/whisper/whisper_bls/1/tokenizer.py | - triton_backend/ci/L0_backend_trtllm/base_metrics_verification_tests.py | - triton_backend/ci/L0_backend_trtllm/custom_metrics_verification_tests.py | - triton_backend/inflight_batcher_llm/client/__init__.py | - triton_backend/inflight_batcher_llm/client/e2e_grpc_speculative_decoding_client.py | - triton_backend/inflight_batcher_llm/client/end_to_end_grpc_client.py | - triton_backend/inflight_batcher_llm/client/inflight_batcher_llm_client.py | triton_backend/scripts/launch_triton_server.py | triton_backend/tools/__init__.py | triton_backend/tools/fill_template.py | - triton_backend/tools/gpt/benchmark_core_model.py | - triton_backend/tools/gpt/client.py | - triton_backend/tools/gpt/client_async.py | - triton_backend/tools/gpt/end_to_end_test.py | - triton_backend/tools/gpt/gen_input_data.py | triton_backend/tools/inflight_batcher_llm/benchmark_core_model.py | triton_backend/tools/inflight_batcher_llm/end_to_end_test.py | triton_backend/tools/inflight_batcher_llm/speculative_decoding_test.py | triton_backend/tools/inflight_batcher_llm/test_max_queue_size.py | triton_backend/tools/llmapi_client.py | - triton_backend/tools/multimodal/client.py | triton_backend/tools/tests/__init__.py | triton_backend/tools/tests/test_fill_template.py | triton_backend/tools/tests/test_llmapi_cancel.py | triton_backend/tools/utils/__init__.py | - triton_backend/tools/utils/utils.py | - triton_backend/tools/whisper/client.py + triton_backend/tools/utils/utils.py )$ # Used by ruff hooks: main ruff (exclude: *legacy_files) and @@ -2699,55 +2666,22 @@ legacy-files: &legacy_files | tests/unittest/utils/test_util.py | tests/unittest/utils/torch_ref.py | tests/unittest/utils/util.py | - triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/1/model.py | - triton_backend/all_models/gpt/postprocessing/1/model.py | - triton_backend/all_models/gpt/preprocessing/1/model.py | - triton_backend/all_models/gpt/tensorrt_llm/1/model.py | - triton_backend/all_models/inflight_batcher_llm/postprocessing/1/model.py | - triton_backend/all_models/inflight_batcher_llm/preprocessing/1/model.py | - triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py | - triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/decode.py | - triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/triton_decoder.py | - triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/model.py | triton_backend/all_models/llmapi/tensorrt_llm/1/helpers.py | triton_backend/all_models/llmapi/tensorrt_llm/1/model.py | - triton_backend/all_models/multimodal/multimodal_encoders/1/model.py | - triton_backend/all_models/multimodal/multimodal_encoders/1/multimodal_utils.py | - triton_backend/all_models/tests/test_decode.py | triton_backend/all_models/tests/test_llmapi_python_backend.py | - triton_backend/all_models/tests/test_multi_image_preprocess.py | - triton_backend/all_models/tests/test_multimodal_encoders.py | - triton_backend/all_models/tests/test_python_backend.py | - triton_backend/all_models/tests/test_triton_decoder.py | - triton_backend/all_models/whisper/whisper_bls/1/fbank.py | - triton_backend/all_models/whisper/whisper_bls/1/model.py | - triton_backend/all_models/whisper/whisper_bls/1/tokenizer.py | - triton_backend/ci/L0_backend_trtllm/base_metrics_verification_tests.py | - triton_backend/ci/L0_backend_trtllm/custom_metrics_verification_tests.py | - triton_backend/inflight_batcher_llm/client/__init__.py | - triton_backend/inflight_batcher_llm/client/e2e_grpc_speculative_decoding_client.py | - triton_backend/inflight_batcher_llm/client/end_to_end_grpc_client.py | - triton_backend/inflight_batcher_llm/client/inflight_batcher_llm_client.py | triton_backend/scripts/launch_triton_server.py | triton_backend/tools/__init__.py | triton_backend/tools/fill_template.py | - triton_backend/tools/gpt/benchmark_core_model.py | - triton_backend/tools/gpt/client.py | - triton_backend/tools/gpt/client_async.py | - triton_backend/tools/gpt/end_to_end_test.py | - triton_backend/tools/gpt/gen_input_data.py | triton_backend/tools/inflight_batcher_llm/benchmark_core_model.py | triton_backend/tools/inflight_batcher_llm/end_to_end_test.py | triton_backend/tools/inflight_batcher_llm/speculative_decoding_test.py | triton_backend/tools/inflight_batcher_llm/test_max_queue_size.py | triton_backend/tools/llmapi_client.py | - triton_backend/tools/multimodal/client.py | triton_backend/tools/tests/__init__.py | triton_backend/tools/tests/test_fill_template.py | triton_backend/tools/tests/test_llmapi_cancel.py | triton_backend/tools/utils/__init__.py | - triton_backend/tools/utils/utils.py | - triton_backend/tools/whisper/client.py + triton_backend/tools/utils/utils.py )$ # <<<< END AUTO-GENERATED >>>> diff --git a/docker/Dockerfile.multi b/docker/Dockerfile.multi index c4c91d9fad71..e724eb4cc629 100644 --- a/docker/Dockerfile.multi +++ b/docker/Dockerfile.multi @@ -181,21 +181,9 @@ ENV TRT_LLM_GIT_COMMIT=${GIT_COMMIT} \ RUN --mount=type=bind,source=scripts/generate_container_oss_attribution.sh,target=/mnt/gen_attribution.sh \ bash /mnt/gen_attribution.sh "release" "${TRT_LLM_VER}" "${TARGETARCH}" -FROM wheel AS tritonbuild - -WORKDIR /src/tensorrt_llm -RUN pip install /src/tensorrt_llm/build/tensorrt_llm*.whl -COPY ./triton_backend/ ./triton_backend/ -ARG TRITON_BASE_TAG -RUN bash ./triton_backend/inflight_batcher_llm/scripts/build.sh -s "r${TRITON_BASE_TAG%-py3}" - - FROM release AS tritonrelease WORKDIR /app/tensorrt_llm COPY ./triton_backend/all_models ./triton_backend/all_models COPY ./triton_backend/scripts ./triton_backend/scripts COPY ./triton_backend/tools ./triton_backend/tools -COPY ./triton_backend/inflight_batcher_llm/scripts ./triton_backend/inflight_batcher_llm/scripts -COPY ./triton_backend/inflight_batcher_llm/client ./triton_backend/inflight_batcher_llm/client -COPY --from=tritonbuild /opt/tritonserver/backends/tensorrtllm /opt/tritonserver/backends/tensorrtllm diff --git a/docs/source/developer-guide/ci-overview.md b/docs/source/developer-guide/ci-overview.md index 30cc613a2e38..908315c3e3c1 100644 --- a/docs/source/developer-guide/ci-overview.md +++ b/docs/source/developer-guide/ci-overview.md @@ -24,14 +24,14 @@ Integration tests are listed under `tests/integration/test_lists/test-db/`. Most - `stage`: either `pre_merge` or `post_merge`. - `backend`: `pytorch`, `tensorrt` or `triton`. -Example from `l0_a100.yml`: +Example from `l0_a30.yml`: ```yaml terms: - stage: post_merge - backend: triton + stage: pre_merge + backend: pytorch tests: - - triton_server/test_triton.py::test_gpt_ib_ptuning[gpt-ib-ptuning] + - unittest/_torch/sampler/test_beam_search.py ``` ## Unit tests @@ -61,8 +61,8 @@ Manually searching YAML and Groovy files can be tedious. The helper script `scripts/test_to_stage_mapping.py` automates the lookup: ```bash -python scripts/test_to_stage_mapping.py --tests "triton_server/test_triton.py::test_gpt_ib_ptuning[gpt-ib-ptuning]" -python scripts/test_to_stage_mapping.py --tests gpt_ib_ptuning +python scripts/test_to_stage_mapping.py --tests "unittest/_torch/sampler/test_beam_search.py" +python scripts/test_to_stage_mapping.py --tests test_beam_search python scripts/test_to_stage_mapping.py --stages A100X-Triton-Post-Merge-1 python scripts/test_to_stage_mapping.py --test-list my_tests.txt python scripts/test_to_stage_mapping.py --test-list my_tests.yml diff --git a/jenkins/Build.groovy b/jenkins/Build.groovy index 2d0d9730c3cb..7297eb14e88e 100644 --- a/jenkins/Build.groovy +++ b/jenkins/Build.groovy @@ -427,22 +427,10 @@ def runLLMBuild(pipeline, buildFlags, tarName, is_linux_x86_64) sh "cd ${LLM_ROOT} && python3 scripts/build_cpp_examples.py" } - // Build tritonserver artifacts - def llmPath = sh (script: "realpath ${LLM_ROOT}",returnStdout: true).trim() - // TODO: Remove after the cmake version is upgraded to 3.31.8 - // Get triton tag from docker/dockerfile.multi - def tritonShortTag = "r26.04" - sh "cd ${LLM_ROOT}/triton_backend/inflight_batcher_llm && mkdir build && cd build && cmake .. -DTRTLLM_DIR=${llmPath} -DTRITON_COMMON_REPO_TAG=${tritonShortTag} -DTRITON_CORE_REPO_TAG=${tritonShortTag} -DTRITON_THIRD_PARTY_REPO_TAG=${tritonShortTag} -DTRITON_BACKEND_REPO_TAG=${tritonShortTag} -DUSE_CXX11_ABI=ON && make -j${buildJobs} install" - // Step 3: packaging wheels into tarfile sh "cp ${LLM_ROOT}/build/tensorrt_llm-*.whl TensorRT-LLM/" - // Step 4: packaging tritonserver artifacts into tarfile - sh "mkdir -p TensorRT-LLM/triton_backend/inflight_batcher_llm/" - sh "cp ${LLM_ROOT}/triton_backend/inflight_batcher_llm/build/libtriton_tensorrtllm.so TensorRT-LLM/triton_backend/inflight_batcher_llm/" - sh "cp ${LLM_ROOT}/triton_backend/inflight_batcher_llm/build/trtllmExecutorWorker TensorRT-LLM/triton_backend/inflight_batcher_llm/" - - // Step 5: packaging benchmark and required cpp dependencies into tarfile + // Step 4: packaging benchmark and required cpp dependencies into tarfile sh "mkdir -p TensorRT-LLM/benchmarks/cpp" sh "cp ${LLM_ROOT}/cpp/build/benchmarks/bertBenchmark TensorRT-LLM/benchmarks/cpp" sh "cp ${LLM_ROOT}/cpp/build/benchmarks/gptManagerBenchmark TensorRT-LLM/benchmarks/cpp" @@ -450,7 +438,7 @@ def runLLMBuild(pipeline, buildFlags, tarName, is_linux_x86_64) sh "cp ${LLM_ROOT}/cpp/build/tensorrt_llm/libtensorrt_llm.so TensorRT-LLM/benchmarks/cpp" sh "cp ${LLM_ROOT}/cpp/build/tensorrt_llm/plugins/libnvinfer_plugin_tensorrt_llm.so TensorRT-LLM/benchmarks/cpp" - // Step 6: packaging attribution files into tarfile when they exist + // Step 5: packaging attribution files into tarfile when they exist sh "mkdir -p TensorRT-LLM/attribution" sh "cp ${LLM_ROOT}/cpp/build/attribution/missing_files.json TensorRT-LLM/attribution/ || true" sh "cp ${LLM_ROOT}/cpp/build/attribution/import_payload.json TensorRT-LLM/attribution/ || true" diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index c2c0af6233c8..5d5737a305bc 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -3570,12 +3570,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO } } - trtllm_utils.llmExecStepWithRetry(pipeline, script: "mkdir -p /opt/tritonserver/backends/tensorrtllm") - def isAarch64 = config.contains("aarch64") - if (!isAarch64) { - trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmPath} && cp TensorRT-LLM/triton_backend/inflight_batcher_llm/libtriton_tensorrtllm.so /opt/tritonserver/backends/tensorrtllm/") - trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmPath} && cp TensorRT-LLM/triton_backend/inflight_batcher_llm/trtllmExecutorWorker /opt/tritonserver/backends/tensorrtllm/") - } trtllm_utils.llmExecStepWithRetry(pipeline, script: "git config --global --add safe.directory \"*\"") } // timeout 45 min } @@ -4484,8 +4478,8 @@ def launchTestJobs(pipeline, testFilter) // "A30-TensorRT-Post-Merge-6": ["a30", "l0_a30", 6, 6], "A30-CPP-Post-Merge-1": ["a30", "l0_a30", 1, 2], "A30-CPP-Post-Merge-2": ["a30", "l0_a30", 2, 2], - "A30-Triton-Post-Merge-1": ["a30", "l0_a30", 1, 2], - "A30-Triton-Post-Merge-2": ["a30", "l0_a30", 2, 2], + // "A30-Triton-Post-Merge-1": ["a30", "l0_a30", 1, 2], + // "A30-Triton-Post-Merge-2": ["a30", "l0_a30", 2, 2], // "A100X-TensorRT-Post-Merge-1": ["a100x", "l0_a100", 1, 6], // "A100X-TensorRT-Post-Merge-2": ["a100x", "l0_a100", 2, 6], // "A100X-TensorRT-Post-Merge-3": ["a100x", "l0_a100", 3, 6], diff --git a/jenkins/current_image_tags.properties b/jenkins/current_image_tags.properties index 08de3db7e1f3..b4648ceb7aed 100644 --- a/jenkins/current_image_tags.properties +++ b/jenkins/current_image_tags.properties @@ -13,8 +13,8 @@ # images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead. IMAGE_NAME=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm -LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.04-py3-x86_64-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202606292100-15694 -LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.04-py3-aarch64-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202606292100-15694 -LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-trt10.16.1.11-skip-tritondevel-202606292100-15694 -LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-trt10.16.1.11-skip-tritondevel-202606292100-15694 -LLM_SBSA_WHEEL_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-trt10.16.1.11-skip-tritondevel-202606292100-15694 +LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.04-py3-x86_64-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202607072201-15907 +LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.04-py3-aarch64-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202607072201-15907 +LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-trt10.16.1.11-skip-tritondevel-202607072201-15907 +LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-trt10.16.1.11-skip-tritondevel-202607072201-15907 +LLM_SBSA_WHEEL_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-trt10.16.1.11-skip-tritondevel-202607072201-15907 diff --git a/legacy-files.txt b/legacy-files.txt index 45575618ba58..d6c8b23308aa 100644 --- a/legacy-files.txt +++ b/legacy-files.txt @@ -1317,52 +1317,19 @@ tests/unittest/utils/test_prebuilt_whl_cpp_extensions.py tests/unittest/utils/test_util.py tests/unittest/utils/torch_ref.py tests/unittest/utils/util.py -triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/1/model.py -triton_backend/all_models/gpt/postprocessing/1/model.py -triton_backend/all_models/gpt/preprocessing/1/model.py -triton_backend/all_models/gpt/tensorrt_llm/1/model.py -triton_backend/all_models/inflight_batcher_llm/postprocessing/1/model.py -triton_backend/all_models/inflight_batcher_llm/preprocessing/1/model.py -triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py -triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/decode.py -triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/triton_decoder.py -triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/model.py triton_backend/all_models/llmapi/tensorrt_llm/1/helpers.py triton_backend/all_models/llmapi/tensorrt_llm/1/model.py -triton_backend/all_models/multimodal/multimodal_encoders/1/model.py -triton_backend/all_models/multimodal/multimodal_encoders/1/multimodal_utils.py -triton_backend/all_models/tests/test_decode.py triton_backend/all_models/tests/test_llmapi_python_backend.py -triton_backend/all_models/tests/test_multi_image_preprocess.py -triton_backend/all_models/tests/test_multimodal_encoders.py -triton_backend/all_models/tests/test_python_backend.py -triton_backend/all_models/tests/test_triton_decoder.py -triton_backend/all_models/whisper/whisper_bls/1/fbank.py -triton_backend/all_models/whisper/whisper_bls/1/model.py -triton_backend/all_models/whisper/whisper_bls/1/tokenizer.py -triton_backend/ci/L0_backend_trtllm/base_metrics_verification_tests.py -triton_backend/ci/L0_backend_trtllm/custom_metrics_verification_tests.py -triton_backend/inflight_batcher_llm/client/__init__.py -triton_backend/inflight_batcher_llm/client/e2e_grpc_speculative_decoding_client.py -triton_backend/inflight_batcher_llm/client/end_to_end_grpc_client.py -triton_backend/inflight_batcher_llm/client/inflight_batcher_llm_client.py triton_backend/scripts/launch_triton_server.py triton_backend/tools/__init__.py triton_backend/tools/fill_template.py -triton_backend/tools/gpt/benchmark_core_model.py -triton_backend/tools/gpt/client.py -triton_backend/tools/gpt/client_async.py -triton_backend/tools/gpt/end_to_end_test.py -triton_backend/tools/gpt/gen_input_data.py triton_backend/tools/inflight_batcher_llm/benchmark_core_model.py triton_backend/tools/inflight_batcher_llm/end_to_end_test.py triton_backend/tools/inflight_batcher_llm/speculative_decoding_test.py triton_backend/tools/inflight_batcher_llm/test_max_queue_size.py triton_backend/tools/llmapi_client.py -triton_backend/tools/multimodal/client.py triton_backend/tools/tests/__init__.py triton_backend/tools/tests/test_fill_template.py triton_backend/tools/tests/test_llmapi_cancel.py triton_backend/tools/utils/__init__.py triton_backend/tools/utils/utils.py -triton_backend/tools/whisper/client.py diff --git a/pyproject.toml b/pyproject.toml index 51c939b9a4d6..39feca1560f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1374,55 +1374,22 @@ exclude = [ "tests/unittest/utils/test_util.py", "tests/unittest/utils/torch_ref.py", "tests/unittest/utils/util.py", - "triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/1/model.py", - "triton_backend/all_models/gpt/postprocessing/1/model.py", - "triton_backend/all_models/gpt/preprocessing/1/model.py", - "triton_backend/all_models/gpt/tensorrt_llm/1/model.py", - "triton_backend/all_models/inflight_batcher_llm/postprocessing/1/model.py", - "triton_backend/all_models/inflight_batcher_llm/preprocessing/1/model.py", - "triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py", - "triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/decode.py", - "triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/triton_decoder.py", - "triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/model.py", "triton_backend/all_models/llmapi/tensorrt_llm/1/helpers.py", "triton_backend/all_models/llmapi/tensorrt_llm/1/model.py", - "triton_backend/all_models/multimodal/multimodal_encoders/1/model.py", - "triton_backend/all_models/multimodal/multimodal_encoders/1/multimodal_utils.py", - "triton_backend/all_models/tests/test_decode.py", "triton_backend/all_models/tests/test_llmapi_python_backend.py", - "triton_backend/all_models/tests/test_multi_image_preprocess.py", - "triton_backend/all_models/tests/test_multimodal_encoders.py", - "triton_backend/all_models/tests/test_python_backend.py", - "triton_backend/all_models/tests/test_triton_decoder.py", - "triton_backend/all_models/whisper/whisper_bls/1/fbank.py", - "triton_backend/all_models/whisper/whisper_bls/1/model.py", - "triton_backend/all_models/whisper/whisper_bls/1/tokenizer.py", - "triton_backend/ci/L0_backend_trtllm/base_metrics_verification_tests.py", - "triton_backend/ci/L0_backend_trtllm/custom_metrics_verification_tests.py", - "triton_backend/inflight_batcher_llm/client/__init__.py", - "triton_backend/inflight_batcher_llm/client/e2e_grpc_speculative_decoding_client.py", - "triton_backend/inflight_batcher_llm/client/end_to_end_grpc_client.py", - "triton_backend/inflight_batcher_llm/client/inflight_batcher_llm_client.py", "triton_backend/scripts/launch_triton_server.py", "triton_backend/tools/__init__.py", "triton_backend/tools/fill_template.py", - "triton_backend/tools/gpt/benchmark_core_model.py", - "triton_backend/tools/gpt/client.py", - "triton_backend/tools/gpt/client_async.py", - "triton_backend/tools/gpt/end_to_end_test.py", - "triton_backend/tools/gpt/gen_input_data.py", "triton_backend/tools/inflight_batcher_llm/benchmark_core_model.py", "triton_backend/tools/inflight_batcher_llm/end_to_end_test.py", "triton_backend/tools/inflight_batcher_llm/speculative_decoding_test.py", "triton_backend/tools/inflight_batcher_llm/test_max_queue_size.py", "triton_backend/tools/llmapi_client.py", - "triton_backend/tools/multimodal/client.py", "triton_backend/tools/tests/__init__.py", "triton_backend/tools/tests/test_fill_template.py", "triton_backend/tools/tests/test_llmapi_cancel.py", "triton_backend/tools/utils/__init__.py", "triton_backend/tools/utils/utils.py", - "triton_backend/tools/whisper/client.py", ] # <<<< END AUTO-GENERATED >>>> diff --git a/ruff-legacy.toml b/ruff-legacy.toml index 4b8efc9a80ea..2f9d1bd00745 100644 --- a/ruff-legacy.toml +++ b/ruff-legacy.toml @@ -1334,55 +1334,22 @@ include = [ "tests/unittest/utils/test_util.py", "tests/unittest/utils/torch_ref.py", "tests/unittest/utils/util.py", - "triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/1/model.py", - "triton_backend/all_models/gpt/postprocessing/1/model.py", - "triton_backend/all_models/gpt/preprocessing/1/model.py", - "triton_backend/all_models/gpt/tensorrt_llm/1/model.py", - "triton_backend/all_models/inflight_batcher_llm/postprocessing/1/model.py", - "triton_backend/all_models/inflight_batcher_llm/preprocessing/1/model.py", - "triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py", - "triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/decode.py", - "triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/triton_decoder.py", - "triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/model.py", "triton_backend/all_models/llmapi/tensorrt_llm/1/helpers.py", "triton_backend/all_models/llmapi/tensorrt_llm/1/model.py", - "triton_backend/all_models/multimodal/multimodal_encoders/1/model.py", - "triton_backend/all_models/multimodal/multimodal_encoders/1/multimodal_utils.py", - "triton_backend/all_models/tests/test_decode.py", "triton_backend/all_models/tests/test_llmapi_python_backend.py", - "triton_backend/all_models/tests/test_multi_image_preprocess.py", - "triton_backend/all_models/tests/test_multimodal_encoders.py", - "triton_backend/all_models/tests/test_python_backend.py", - "triton_backend/all_models/tests/test_triton_decoder.py", - "triton_backend/all_models/whisper/whisper_bls/1/fbank.py", - "triton_backend/all_models/whisper/whisper_bls/1/model.py", - "triton_backend/all_models/whisper/whisper_bls/1/tokenizer.py", - "triton_backend/ci/L0_backend_trtllm/base_metrics_verification_tests.py", - "triton_backend/ci/L0_backend_trtllm/custom_metrics_verification_tests.py", - "triton_backend/inflight_batcher_llm/client/__init__.py", - "triton_backend/inflight_batcher_llm/client/e2e_grpc_speculative_decoding_client.py", - "triton_backend/inflight_batcher_llm/client/end_to_end_grpc_client.py", - "triton_backend/inflight_batcher_llm/client/inflight_batcher_llm_client.py", "triton_backend/scripts/launch_triton_server.py", "triton_backend/tools/__init__.py", "triton_backend/tools/fill_template.py", - "triton_backend/tools/gpt/benchmark_core_model.py", - "triton_backend/tools/gpt/client.py", - "triton_backend/tools/gpt/client_async.py", - "triton_backend/tools/gpt/end_to_end_test.py", - "triton_backend/tools/gpt/gen_input_data.py", "triton_backend/tools/inflight_batcher_llm/benchmark_core_model.py", "triton_backend/tools/inflight_batcher_llm/end_to_end_test.py", "triton_backend/tools/inflight_batcher_llm/speculative_decoding_test.py", "triton_backend/tools/inflight_batcher_llm/test_max_queue_size.py", "triton_backend/tools/llmapi_client.py", - "triton_backend/tools/multimodal/client.py", "triton_backend/tools/tests/__init__.py", "triton_backend/tools/tests/test_fill_template.py", "triton_backend/tools/tests/test_llmapi_cancel.py", "triton_backend/tools/utils/__init__.py", "triton_backend/tools/utils/utils.py", - "triton_backend/tools/whisper/client.py", ] [lint] diff --git a/scripts/test_to_stage_mapping.py b/scripts/test_to_stage_mapping.py index d51623a80c9d..04626131b658 100644 --- a/scripts/test_to_stage_mapping.py +++ b/scripts/test_to_stage_mapping.py @@ -10,8 +10,8 @@ Example usage:: python scripts/test_to_stage_mapping.py --tests \\ - "triton_server/test_triton.py::test_gpt_ib_ptuning[gpt-ib-ptuning]" - python scripts/test_to_stage_mapping.py --tests gpt_ib_ptuning + "unittest/_torch/sampler/test_beam_search.py" + python scripts/test_to_stage_mapping.py --tests test_beam_search python scripts/test_to_stage_mapping.py --stages \\ A100X-Triton-Post-Merge-1 diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index 11c08a284c74..3769f51ec40d 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -27,7 +27,7 @@ from defs.trt_test_alternative import (check_call, check_call_negative_test, check_output, print_info, print_warning) -from .common import convert_weights, get_mmlu_accuracy, venv_check_call +from .common import get_mmlu_accuracy, venv_check_call from .conftest import (get_device_count, get_sm_version, llm_models_root, skip_no_sm120, skip_nvlink_inactive, skip_post_blackwell, skip_pre_ada, skip_pre_blackwell, skip_pre_hopper, @@ -72,62 +72,29 @@ def test_gpt3_175b_1layers_build_only(llm_root, llm_venv, engine_dir): @pytest.mark.parametrize("model_name,model_path", [ ("DeepSeek-R1-Distill-Qwen-1.5B", "DeepSeek-R1-Distill-Qwen-1.5B"), ]) -def test_qwen_e2e_cpprunner_large_new_tokens(model_name, model_path, llm_venv, - qwen_example_root, cmodel_dir, - engine_dir): - """RCCA: https://nvbugs/5238105""" - model_dir = convert_weights( - llm_venv=llm_venv, - example_root=qwen_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=f"{llm_models_root()}/{model_path}", - ) - - build_cmd = [ - "trtllm-build", f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", "--gemm_plugin=float16", - "--max_num_tokens=32768" - ] +def test_qwen_e2e_cpprunner_large_new_tokens(model_name, model_path, llm_venv): + """RCCA: https://nvbugs/5238105 - none of n>1 sequences may be empty.""" + from tensorrt_llm import LLM, SamplingParams - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) + prompt = r"<｜begin▁of▁sentence｜><｜User｜>The operation $\otimes$ is defined for all nonzero numbers by $a \otimes b = \frac{a^{2}}{b}$. Determine $[(1 \otimes 2) \otimes 3] - [1 \otimes (2 \otimes 3)]$. Let's think step by step and output the final answer within \boxed{}.<｜Assistant｜>" - from transformers import AutoTokenizer + sampling_params = SamplingParams( + max_tokens=1024, + n=4, + temperature=0.6, + top_p=1.0, + top_k=1024, + ) - from tensorrt_llm.runtime import PYTHON_BINDINGS + with LLM(model=f"{llm_models_root()}/{model_path}", + max_batch_size=8, + max_seq_len=4224) as llm: + outputs = llm.generate([prompt], sampling_params=sampling_params) - if PYTHON_BINDINGS: - from tensorrt_llm.runtime import ModelRunnerCpp - tokenizer = AutoTokenizer.from_pretrained( - f"{llm_models_root()}/{model_path}", - trust_remote_code=True, - use_fast=False) - - message = r"<｜begin▁of▁sentence｜><｜User｜>The operation $\otimes$ is defined for all nonzero numbers by $a \otimes b = \frac{a^{2}}{b}$. Determine $[(1 \otimes 2) \otimes 3] - [1 \otimes (2 \otimes 3)]$. Let's think step by step and output the final answer within \boxed{}.<｜Assistant｜>" - - inputs = tokenizer(message, return_tensors='pt', - add_special_tokens=False)['input_ids'] - - runner = ModelRunnerCpp.from_dir(engine_dir=f"{engine_dir}", - max_input_len=128, - max_output_len=4096, - max_batch_size=8) - - outputs = runner.generate(inputs, - end_id=tokenizer.eos_token_id, - pad_id=tokenizer.pad_token_id, - temperature=0.6, - top_p=1.0, - top_k=1024, - max_new_tokens=1024, - return_dict=True, - min_length=1, - num_return_sequences=4, - output_sequence_lengths=True) - - seq_lengths = outputs['sequence_lengths'] - assert not (seq_lengths == 0).any( - ), f"Found zero length in sequence_lengths tensor: {seq_lengths}" + completions = outputs[0].outputs + seq_lengths = [len(c.token_ids) for c in completions] + assert all(length > 0 for length in seq_lengths), \ + f"Found zero-length completion: {seq_lengths}" # TODO replace the trtllm_bench_prolog diff --git a/tests/integration/defs/thirdparty/test_cmake_third_party.py b/tests/integration/defs/thirdparty/test_cmake_third_party.py index 6ba7389fb4fe..15349577ea0a 100644 --- a/tests/integration/defs/thirdparty/test_cmake_third_party.py +++ b/tests/integration/defs/thirdparty/test_cmake_third_party.py @@ -21,9 +21,6 @@ "3rdparty/CMakeLists.txt", # Historical use of ExternalProject_Add that is not yet migrated to 3rdparty "cpp/tensorrt_llm/deep_ep/CMakeLists.txt", - # Historical build that is not included in the wheel build and thus exempt - # from the third-party process. - "triton_backend/inflight_batcher_llm/*", "build", # Default build directory "cpp/build", # Default extension module build directory ] diff --git a/tests/integration/defs/triton_server/build_model.sh b/tests/integration/defs/triton_server/build_model.sh deleted file mode 100755 index 1b96b46e354a..000000000000 --- a/tests/integration/defs/triton_server/build_model.sh +++ /dev/null @@ -1,680 +0,0 @@ -#!/usr/bin/bash - -install_requirements() { - pip install -r requirements.txt - #This is to WAR an issue with latest transformers package not being compatible with sentencepiece 0.1.99 apparently - pip install sentencepiece --upgrade -} - -MODEL=$1 - -GPT2=$LLM_MODELS_ROOT/gpt2 -GPT2_MEDIUM=$LLM_MODELS_ROOT/gpt2-medium -GPT2_NEXT_PTUNING=$LLM_MODELS_ROOT/email_composition -OPT_125M=$LLM_MODELS_ROOT/opt-125m -GPTJ=$LLM_MODELS_ROOT/gpt-j-6b -MISTRAL=$LLM_MODELS_ROOT/mistral-7b-v0.1 -GPT_2B=$LLM_MODELS_ROOT/GPT-2B-001_bf16_tp1.nemo -GPT_2B_LORA=$LLM_MODELS_ROOT/lora/gpt-next-2b -VICUNA=$LLM_MODELS_ROOT/vicuna-7b-v1.3 -MEDUSA_VICUNA=$LLM_MODELS_ROOT/medusa-vicuna-7b-v1.3/ -EAGLE_VICUNA=$LLM_MODELS_ROOT/EAGLE-Vicuna-7B-v1.3/ -BART=$LLM_MODELS_ROOT/bart-large-cnn/ -T5=$LLM_MODELS_ROOT/t5-small/ -BLIP2_OPT_2_7B=$LLM_MODELS_ROOT/blip2-opt-2.7b -LLAVA_7B=$LLM_MODELS_ROOT/llava-1.5-7b-hf -VILA1_5_3B=$LLM_MODELS_ROOT/vila/VILA1.5-3b -VILA_PATH=$LLM_MODELS_ROOT/vila/VILA -LLAMA_3_2_11B_VISION=$LLM_MODELS_ROOT/llama-3.2-models/Llama-3.2-11B-Vision-Instruct -WHISPER_LAREGE_V3=$LLM_MODELS_ROOT/whisper-models/large-v3 -LLAVA_ONEVISION_7B=$LLM_MODELS_ROOT/llava-onevision-qwen2-7b-ov-hf -QWEN2_VL_7B=$LLM_MODELS_ROOT/Qwen2-VL-7B-Instruct -set -e - -pkill -9 -f tritonserver || true - -pushd $LLM_ROOT/ - -# install deps -pip3 install -r requirements-dev.txt - -if [ "$MODEL" = "gpt" ] || [ "$MODEL" = "gpt-disaggregated-serving-bls" ]; then - - # GPT2 - pushd examples/models/core/gpt - - install_requirements - - echo "Convert GPT from HF" - python3 convert_checkpoint.py --model_dir ${GPT2} --dtype float16 --output_dir ./c-model/gpt2/fp16 - - echo "Build GPT: float16 | remove_input_padding" - trtllm-build --checkpoint_dir ./c-model/gpt2/fp16 \ - --gpt_attention_plugin float16 \ - --gemm_plugin float16 \ - --context_fmha enable \ - --remove_input_padding enable \ - --max_batch_size 8 \ - --max_seq_len 1024 \ - --max_num_tokens 7392 \ - --output_dir trt_engine/gpt2/fp16/1-gpu/ - - popd # examples/models/core/gpt - -fi - -if [ "$MODEL" = "opt" ]; then - - pushd examples/models/contrib/opt - - install_requirements - - echo "Convert OPT from HF" - python3 convert_checkpoint.py --model_dir ${OPT_125M} --dtype float16 --output_dir ./c-model/opt-125m/fp16 - - echo "OPT builder" - trtllm-build --checkpoint_dir ./c-model/opt-125m/fp16 \ - --gemm_plugin float16 \ - --gpt_attention_plugin float16 \ - --context_fmha=enable \ - --max_batch_size 8 \ - --max_seq_len 1024 \ - --max_num_tokens 7392 \ - --output_dir trt_engine/opt-125m/fp16/1-gpu/ - - - popd # examples/models/contrib/opt - -fi - -if [ "$MODEL" = "mistral" ]; then - - pushd examples/models/core/llama - - install_requirements - - echo "Convert Mistral from HF" - python3 convert_checkpoint.py --dtype float16 \ - --n_layer 2 --n_positions 32768 \ - --output_dir ./c-model/mistral-7b/fp16 - - echo "Build Mistral" - trtllm-build --model_config ./c-model/mistral-7b/fp16/config.json \ - --context_fmha=enable \ - --gpt_attention_plugin float16 \ - --gemm_plugin float16 \ - --max_input_len 8192 \ - --max_batch_size 8 \ - --max_num_tokens 8192 \ - --output_dir mistral_7b_outputs - - popd # examples/models/core/llama - -fi - -if [ "$MODEL" = "mistral-ib" ]; then - - pushd examples/models/core/llama - - install_requirements - - echo "Convert Mistral from HF" - python3 convert_checkpoint.py --dtype float16 \ - --model_dir ${MISTRAL} \ - --output_dir ./c-model/mistral-7b/fp16 - - echo "Build Mistral with inflight batching" - trtllm-build --checkpoint_dir ./c-model/mistral-7b/fp16/ \ - --context_fmha=enable \ - --remove_input_padding=enable \ - --kv_cache_type=paged \ - --gpt_attention_plugin float16 \ - --gemm_plugin float16 \ - --max_batch_size 1 \ - --max_seq_len 9216 \ - --use_paged_context_fmha disable \ - --max_beam_width 2 \ - --output_dir ib_mistral_7b_outputs - - popd # examples/models/core/llama - -fi - -if [ "$MODEL" = "gptj" ]; then - - pushd examples/models/contrib/gptj - - install_requirements - - echo "Convert GPT-J from HF" - python3 convert_checkpoint.py --dtype float16 --n_layer 2 --output_dir ./c-model/gpt-j-6b/fp16 - - echo "Build GPT-J" - trtllm-build --model_config ./c-model/gpt-j-6b/fp16/config.json \ - --context_fmha=enable \ - --gpt_attention_plugin float16 \ - --gemm_plugin float16 \ - --max_batch_size 8 \ - --output_dir gptj_outputs - - popd # examples/models/contrib/gptj - -fi - -if [ "$MODEL" = "gpt-ib" ]; then - - # GPT2 - pushd examples/models/core/gpt - - install_requirements - - echo "Convert GPT from HF" - python3 convert_checkpoint.py --model_dir ${GPT2} --dtype float16 --output_dir ./c-model/gpt2/fp16 - - echo "Build GPT: float16" - trtllm-build --checkpoint_dir ./c-model/gpt2/fp16 \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable \ - --kv_cache_type paged \ - --use_paged_context_fmha enable \ - --gemm_plugin float16 \ - --max_batch_size 8 \ - --max_num_tokens 7392 \ - --gather_generation_logits \ - --output_dir trt_engine/gpt2-ib/fp16/1-gpu/ - - popd # examples/models/core/gpt - -fi - -if [ "$MODEL" = "gpt-ib-lad" ]; then - - # GPT2 - pushd examples/models/core/gpt - - # install_requirements - - echo "Convert GPT from HF" - python3 convert_checkpoint.py --model_dir ${GPT2} --dtype float16 --output_dir ./c-model/gpt2/fp16 - - echo "Build GPT: float16" - trtllm-build --checkpoint_dir ./c-model/gpt2/fp16 \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable \ - --kv_cache_type paged \ - --use_paged_context_fmha enable \ - --gemm_plugin float16 \ - --max_batch_size 8 \ - --max_num_tokens 7392 \ - --gather_generation_logits \ - --max_draft_len=83 \ - --speculative_decoding_mode=lookahead_decoding \ - --output_dir trt_engine/gpt2-ib-lad/fp16/1-gpu/ - - popd # examples/models/core/gpt - -fi - -if [ "$MODEL" = "bart-ib" ] || [ "$MODEL" = "t5-ib" ]; then - - pushd examples/models/core/enc_dec - - if [ "$MODEL" = "bart-ib" ]; then - MODEL_DIR=${BART} - MODEL_TYPE="bart" - elif [ "$MODEL" = "t5-ib" ]; then - MODEL_DIR=${T5} - MODEL_TYPE="t5" - fi - echo "Convert ${MODEL_TYPE} from HF" - python3 convert_checkpoint.py --model_type ${MODEL_TYPE} --model_dir ${MODEL_DIR} --dtype float16 --output_dir ./c-model/${MODEL}/fp16 - - echo "Build Encoder: " - trtllm-build --checkpoint_dir ./c-model/${MODEL}/fp16/encoder \ - --output_dir trt_engine/${MODEL}/fp16/1-gpu/encoder \ - --kv_cache_type disabled --moe_plugin disable \ - --max_beam_width 1 \ - --max_batch_size 8 --max_input_len 512 --max_seq_len 512 \ - --gemm_plugin float16 \ - --bert_attention_plugin float16 \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable --context_fmha disable - - - echo "Build Decoder:" - trtllm-build --checkpoint_dir ./c-model/${MODEL}/fp16/decoder \ - --output_dir trt_engine/${MODEL}/fp16/1-gpu/decoder \ - --moe_plugin disable \ - --max_beam_width 1 \ - --max_batch_size 8 --max_input_len 1 --max_seq_len 512 --max_encoder_input_len 512 \ - --gemm_plugin float16 \ - --bert_attention_plugin float16 \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable --context_fmha disable - - popd # examples/models/core/enc_dec - -fi - - -if [ "$MODEL" = "gpt-medium-ib" ]; then - - # GPT2 - pushd examples/models/core/gpt - - install_requirements - - echo "Convert GPT2 medium from HF" - python3 convert_checkpoint.py --model_dir ${GPT2_MEDIUM} --dtype float16 --output_dir ./c-model/gpt2-medium/fp16 - - echo "Build GPT2 medium control: float16" - trtllm-build --checkpoint_dir ./c-model/gpt2-medium/fp16 \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable \ - --kv_cache_type paged \ - --gemm_plugin float16 \ - --context_fmha enable \ - --use_paged_context_fmha enable \ - --max_batch_size 8 \ - --max_num_tokens 7392 \ - --gather_generation_logits \ - --output_dir trt_engine/gpt2-medium-ib/fp16/1-gpu/ - - echo "Build GPT2 medium target: float16" - trtllm-build --checkpoint_dir ./c-model/gpt2-medium/fp16 \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable \ - --kv_cache_type paged \ - --gemm_plugin float16 \ - --context_fmha enable \ - --use_paged_context_fmha enable \ - --max_draft_len 5 \ - --speculative_decoding_mode draft_tokens_external \ - --max_batch_size 8 \ - --max_num_tokens 7392 \ - --gather_generation_logits \ - --output_dir trt_engine/gpt2-medium-ib-target/fp16/1-gpu/ - - popd # examples/models/core/gpt - -fi - -if [ "$MODEL" = "gpt-ib-ptuning" ]; then - - # GPT2 - pushd examples/models/core/gpt - - install_requirements - - echo "Convert GPT from NeMo" - python3 convert_checkpoint.py --nemo_ckpt_path ${GPT2_NEXT_PTUNING}/megatron_converted_8b_tp4_pp1.nemo --dtype float16 --output_dir ./c-model/email_composition/fp16 - - echo "Convert ptuning table" - python3 nemo_prompt_convert.py -i ${GPT2_NEXT_PTUNING}/email_composition.nemo -o email_composition.npy - - cp ${GPT2_NEXT_PTUNING}/input.csv ./ - - echo "Build GPT: float16" - trtllm-build --checkpoint_dir ./c-model/email_composition/fp16 \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable \ - --kv_cache_type paged \ - --gemm_plugin float16 \ - --context_fmha enable \ - --max_batch_size 4 --max_seq_len 256 --max_beam_width 1 \ - --max_num_tokens 512 \ - --output_dir trt_engine/email_composition/fp16/1-gpu/ \ - --max_prompt_embedding_table_size 300 - - popd # examples/models/core/gpt - -fi - -if [ "$MODEL" = "gpt-2b-ib-lora" ]; then - - # GPT-2B - pushd examples/models/core/gpt - - install_requirements - - echo "Convert GPT from NeMo" - python3 convert_checkpoint.py --nemo_ckpt_path ${GPT_2B} --dtype float16 --output_dir ./c-model/gpt-2b-lora/fp16 - - echo "Build GPT: float16" - trtllm-build --checkpoint_dir ./c-model/gpt-2b-lora/fp16 \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable \ - --kv_cache_type paged \ - --gemm_plugin float16 \ - --lora_plugin float16 \ - --lora_dir ${GPT_2B_LORA}/gpt2b_lora-900.nemo \ - --lora_ckpt_source nemo \ - --lora_target_modules attn_qkv \ - --max_batch_size 8 \ - --max_num_tokens 7392 \ - --output_dir trt_engine/gpt-2b-lora-ib/fp16/1-gpu/ - - python3 nemo_lora_convert.py -i ${GPT_2B_LORA}/gpt2b_lora-900.nemo \ - -o gpt-2b-lora-train-900 --write-cpp-runtime-tensors --storage-type float16 - python3 nemo_lora_convert.py -i ${GPT_2B_LORA}/gpt2b_lora-900.nemo \ - -o gpt-2b-lora-train-900-tllm --storage-type float16 - cp ${GPT_2B_LORA}/gpt2b_lora-900.nemo . - - cp ${GPT_2B_LORA}/input.csv . - - popd # examples/models/core/gpt -fi - -if [ "$MODEL" = "gpt-gather-logits" ]; then - - # GPT2 - pushd examples/models/core/gpt - - install_requirements - - echo "Convert GPT from HF" - python3 convert_checkpoint.py --model_dir ${GPT2} --dtype float16 --output_dir ./c-model/gpt2/fp16 - - echo "Build GPT: float16 | gather_all_token_logits" - trtllm-build --checkpoint_dir ./c-model/gpt2/fp16 \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable \ - --kv_cache_type paged \ - --gemm_plugin float16 \ - --context_fmha enable \ - --max_batch_size 128 --max_seq_len 600 \ - --gather_all_token_logits \ - --output_dir trt_engine/gpt2-gather-logits/fp16/1-gpu/ \ - --max_num_tokens 38400 - - popd # examples/models/core/gpt - -fi - -if [ "$MODEL" = "medusa" ]; then - - # Medusa - pushd examples/medusa - - install_requirements - - echo "Convert Medusa from HF" - python convert_checkpoint.py --model_dir ${VICUNA} \ - --medusa_model_dir ${MEDUSA_VICUNA} \ - --output_dir ./tllm_checkpoint_1gpu_medusa \ - --dtype float16 \ - --num_medusa_heads 4 - - echo "Build Medusa: float16" - trtllm-build --checkpoint_dir ./tllm_checkpoint_1gpu_medusa \ - --output_dir ./tmp/medusa/7B/trt_engines/fp16/1-gpu/ \ - --gemm_plugin float16 \ - --speculative_decoding_mode medusa \ - --max_batch_size 8 --max_seq_len 600 \ - --max_num_tokens 2400 - - popd # examples/medusa - -fi - -if [ "$MODEL" = "eagle" ]; then - - # Eagle - pushd examples/eagle - - install_requirements - - echo "Convert Eagle from HF" - python convert_checkpoint.py --model_dir ${VICUNA} \ - --eagle_model_dir ${EAGLE_VICUNA} \ - --output_dir ./tllm_checkpoint_1gpu_eagle \ - --dtype float16 \ - --num_eagle_layers 4 - - echo "Build Eagle: float16" - trtllm-build --checkpoint_dir ./tllm_checkpoint_1gpu_eagle \ - --output_dir ./tmp/eagle/7B/trt_engines/fp16/1-gpu/ \ - --gemm_plugin float16 \ - --speculative_decoding_mode eagle \ - --max_batch_size 8 --max_seq_len 600 \ - --max_num_tokens 2400 - - popd # examples/eagle - -fi - -if [ "$MODEL" = "gpt-gather-generation-logits" ]; then - - # GPT2 - pushd examples/models/core/gpt - - install_requirements - - echo "Convert GPT from HF" - python3 convert_checkpoint.py --model_dir ${GPT2} --dtype float16 --output_dir ./c-model/gpt2/fp16 - - # draft model, only gather_generation_logits - echo "Build GPT: float16 | gather_all_token_logits" - trtllm-build --checkpoint_dir ./c-model/gpt2/fp16 \ - --max_batch_size 4 \ - --max_seq_len 640 \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable \ - --kv_cache_type paged \ - --context_fmha enable \ - --max_num_tokens 38400 \ - --use_paged_context_fmha enable \ - --gather_generation_logits \ - --output_dir trt_engine/gpt2-draft-gather-generation-logits/fp16/1-gpu/ - - popd # examples/models/core/gpt - -fi - -if [ "$MODEL" = "blip2-opt" ]; then - - pushd examples/models/core/multimodal - - echo "Convert OPT from HF" - python3 ../../contrib/opt/convert_checkpoint.py --model_type blip2 --model_dir ${BLIP2_OPT_2_7B} --dtype float16 --output_dir ./c-model/opt-2.7b/fp16 - - echo "OPT builder" - trtllm-build --checkpoint_dir ./c-model/opt-2.7b/fp16 \ - --gemm_plugin float16 \ - --max_beam_width 1 \ - --max_batch_size 8 \ - --max_multimodal_len 256 \ - --max_input_len 924 \ - --max_seq_len 1024 \ - --use_paged_context_fmha enable \ - --output_dir trt_engines/opt-2.7b/fp16/1-gpu - - python build_multimodal_engine.py --model_type blip2 --model_path ${BLIP2_OPT_2_7B} --max_batch_size 8 - - popd # examples/models/core/multimodal - -fi - -if [ "$MODEL" = "llava" ]; then - - pushd examples/models/core/multimodal - - echo "Convert LLaMA from HF" - python3 ../llama/convert_checkpoint.py --model_dir ${LLAVA_7B} --dtype float16 --output_dir ./c-model/llava-1.5-7b-hf/fp16 - - echo "LLAVA builder" - trtllm-build --checkpoint_dir ./c-model/llava-1.5-7b-hf/fp16 \ - --gemm_plugin float16 \ - --max_batch_size 8 \ - --max_input_len 2048 \ - --max_seq_len 2560 \ - --max_multimodal_len 4608 \ - --output_dir trt_engines/llava-1.5-7b-hf/fp16/1-gpu - - python build_multimodal_engine.py --model_path ${LLAVA_7B} --model_type llava --max_batch_size 8 - - popd # examples/models/core/multimodal - -fi - -if [ "$MODEL" = "llava_fp8" ]; then - - pushd examples/models/core/multimodal - - echo "Convert LLaVA from HF to FP8" - python3 ../../../quantization/quantize.py \ - --model_dir ${LLAVA_7B} \ - --dtype float16 \ - --qformat fp8 \ - --kv_cache_dtype fp8 \ - --output_dir ./c-model/llava-1.5-7b-hf/fp8 \ - --calib_size 512 - - echo "LLAVA builder for FP8" - trtllm-build --checkpoint_dir ./c-model/llava-1.5-7b-hf/fp8 \ - --gemm_plugin auto \ - --max_batch_size 8 \ - --max_input_len 2048 \ - --max_seq_len 2560 \ - --max_multimodal_len 4608 \ - --output_dir trt_engines/llava-1.5-7b-hf/fp8/1-gpu - - python build_multimodal_engine.py --model_path ${LLAVA_7B} --model_type llava --max_batch_size 8 - - popd # examples/models/core/multimodal - -fi - -if [ "$MODEL" = "vila" ]; then - - echo "Install vila requirements" - pip install -r $LLM_BACKEND_ROOT/all_models/multimodal/requirements-vila.txt - - pushd examples/models/core/multimodal - - echo "Convert LLaMA from HF" - python3 ../llama/convert_checkpoint.py --model_dir ${VILA1_5_3B} --dtype float16 --output_dir ./c-model/vila1.5-3b/fp16 - - echo "VILA builder" - trtllm-build --checkpoint_dir ./c-model/vila1.5-3b/fp16 \ - --gemm_plugin float16 \ - --max_batch_size 8 \ - --max_input_len 2048 \ - --max_seq_len 2560 \ - --max_multimodal_len 6272 \ - --output_dir trt_engines/vila1.5-3b/fp16/1-gpu - - python build_multimodal_engine.py --model_path ${VILA1_5_3B} --model_type vila --max_batch_size 32 --vila_path ${VILA_PATH} - - popd # examples/models/core/multimodal -fi - -if [ "$MODEL" = "mllama" ]; then - - echo "Install mllama requirements" - pip install -r $LLM_BACKEND_ROOT/all_models/multimodal/requirements-mllama.txt - - pushd examples/models/core/multimodal - - echo "Convert mllama from HF" - python3 ../mllama/convert_checkpoint.py --model_dir ${LLAMA_3_2_11B_VISION} --dtype bfloat16 --output_dir ./c-model/Llama-3.2-11B-Vision-Instruct/bf16 - - echo "mllama builder" - trtllm-build --checkpoint_dir ./c-model/Llama-3.2-11B-Vision-Instruct/bf16 \ - --gemm_plugin auto \ - --max_batch_size 8 \ - --max_input_len 2048 \ - --max_seq_len 2560 \ - --max_encoder_input_len 8200 \ - --output_dir trt_engines/Llama-3.2-11B-Vision-Instruct/bf16/1-gpu - - python build_multimodal_engine.py --model_path ${LLAMA_3_2_11B_VISION} \ - --model_type mllama --max_batch_size 8 \ - --output_dir tmp/trt_engines/Llama-3.2-11B-Vision-Instruct/multimodal_encoder - - popd # examples/models/core/multimodal - -fi - -if [ "$MODEL" = "whisper" ]; then - - pushd examples/models/core/whisper - - echo "Convert OpenAI Whisper Checkpoint" - python3 convert_checkpoint.py --model_dir ${WHISPER_LAREGE_V3} --output_dir ./c-model/${MODEL}/tllm_checkpoint - - echo "Build Whisper Encoder: " - trtllm-build --checkpoint_dir ./c-model/${MODEL}/tllm_checkpoint/encoder \ - --output_dir trt_engine/${MODEL}/encoder \ - --moe_plugin disable \ - --max_batch_size 8 \ - --gemm_plugin disable \ - --bert_attention_plugin float16 \ - --max_input_len 3000 --max_seq_len=3000 - - echo "Build Whisper Decoder: " - trtllm-build --checkpoint_dir ./c-model/${MODEL}/tllm_checkpoint/decoder \ - --output_dir trt_engine/${MODEL}/decoder \ - --moe_plugin disable \ - --max_beam_width 4 \ - --max_batch_size 8 \ - --max_seq_len 114 \ - --max_input_len 14 \ - --max_encoder_input_len 3000 \ - --gemm_plugin float16 \ - --bert_attention_plugin float16 \ - --gpt_attention_plugin float16 - popd # examples/models/core/whisper - -fi - -if [ "$MODEL" = "llava_onevision" ]; then - - echo "Install llava_onevision requirements" - pip install -r $LLM_BACKEND_ROOT/all_models/multimodal/requirements-llava-onevision.txt - - pushd examples/models/core/multimodal - - echo "Convert Qwen from HF" - python3 ../qwen/convert_checkpoint.py --model_dir ${LLAVA_ONEVISION_7B} --dtype float16 --output_dir ./c-model/llava-7b/fp16 - - echo "Qwen builder" - trtllm-build --checkpoint_dir ./c-model/llava-7b/fp16 \ - --gemm_plugin float16 \ - --max_batch_size 1 \ - --max_input_len 7500 \ - --max_seq_len 7600 \ - --max_multimodal_len 7300 \ - --output_dir trt_engines/llava-onevision-7b/fp16/1-gpu - - python build_multimodal_engine.py --model_path ${LLAVA_ONEVISION_7B} --model_type llava_onevision --max_batch_size 16 --output_dir tmp/trt_engines/llava-onevision-qwen2-7b-ov-hf/multimodal_encoder - - popd # examples/models/core/multimodal - -fi - -if [ "$MODEL" = "qwen2_vl" ]; then - echo "Install llava_onevision requirements" - pip install -r $LLM_BACKEND_ROOT/all_models/multimodal/requirements-qwen2vl.txt - - pushd examples/models/core/multimodal - - echo "Convert Qwen from HF" - python3 ../qwen/convert_checkpoint.py --model_dir ${QWEN2_VL_7B} --dtype float16 --output_dir ./c-model/qwen2-vl-7b/fp16 - - echo "Qwen builder" - trtllm-build --checkpoint_dir ./c-model/qwen2-vl-7b/fp16 \ - --gemm_plugin float16 \ - --max_batch_size 4 \ - --max_input_len 2048 \ - --max_seq_len 76307200 \ - --max_multimodal_len 1296 \ - --output_dir trt_engines/qwen2-vl-7b/fp16/1-gpu - - python build_multimodal_engine.py --model_path ${QWEN2_VL_7B} --model_type qwen2_vl --output_dir tmp/trt_engines/Qwen2-VL-7B-Instruct/multimodal_encoder - - popd # examples/models/core/multimodal - -fi - -popd # $LLM_ROOT diff --git a/tests/integration/defs/triton_server/common.py b/tests/integration/defs/triton_server/common.py index a9416cf838c7..bad12b9e10c3 100644 --- a/tests/integration/defs/triton_server/common.py +++ b/tests/integration/defs/triton_server/common.py @@ -1,92 +1,7 @@ -import json import os -import re -import subprocess -import sys -import tempfile import time -from difflib import SequenceMatcher -import pytest - -from .conftest import venv_check_call, venv_check_output -from .trt_test_alternative import (check_call, check_output, print_error, - print_info) - -try: - import psutil -except ModuleNotFoundError: - check_call(f"pip3 install psutil", shell=True) - - -def install_venv_custom_package(package_name): - pip_command = [sys.executable, "-m", "pip", "install", package_name] - - try: - subprocess.check_call(pip_command) - print(f"Successfully installed {package_name}") - except subprocess.CalledProcessError as e: - print(f"Failed to install {package_name}. Error: {e}") - - -# Install cuda-python in venv -install_venv_custom_package("cuda-python") - -from cuda.bindings import driver as cuda_driver - - -def getSMVersion(): - # Init - err_tuple = cuda_driver.cuInit(0) - err = err_tuple[0] if isinstance(err_tuple, tuple) else err_tuple - if err != cuda_driver.CUresult.CUDA_SUCCESS: - raise RuntimeError(f"CUDA initialization failed with error code {err}") - - # Device - err, cuDevice = cuda_driver.cuDeviceGet(0) - if err != cuda_driver.CUresult.CUDA_SUCCESS: - raise RuntimeError(f"Failed to get CUDA device with error code {err}") - - # Get target architecture - err, sm_major = cuda_driver.cuDeviceGetAttribute( - cuda_driver.CUdevice_attribute. - CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, cuDevice) - if err != cuda_driver.CUresult.CUDA_SUCCESS: - raise RuntimeError( - f"Failed to get compute capability major with error code {err}") - - err, sm_minor = cuda_driver.cuDeviceGetAttribute( - cuda_driver.CUdevice_attribute. - CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, cuDevice) - if err != cuda_driver.CUresult.CUDA_SUCCESS: - raise RuntimeError( - f"Failed to get compute capability minor with error code {err}") - - return sm_major * 10 + sm_minor - - -skip_pre_ada = pytest.mark.skipif( - getSMVersion() < 89, - reason="This test is not supported in pre-Ada architecture") - - -def query_gpu_name(): - cmd = r"nvidia-smi --query-gpu=name --format=csv,noheader | head -n1" - gpu_name = check_output(f"{cmd}", shell=True).strip() - - return gpu_name - - -def get_gpu_full_name(): - cmd = r"nvidia-smi -L | head -n1" - gpu_info = check_output(f"{cmd}", shell=True).strip() - - # Extract GPU name using regex pattern - pattern = r"GPU \d+: (.*?) \(UUID:" - match = re.search(pattern, gpu_info) - assert match is not None, f"Failed to extract GPU name from: {gpu_info}" - - return match.group(1).strip() +from .trt_test_alternative import check_call, check_output, print_info def check_server_ready(http_port="8000", timeout_timer=None, sleep_interval=5): @@ -120,81 +35,6 @@ def check_server_ready(http_port="8000", timeout_timer=None, sleep_interval=5): ) -def assert_pattern_match_target(pattern, content, target_value): - match = re.search(pattern, content) - assert match is not None, f"'{pattern}' has no matches." - num_match = int(match.group(1)) - assert num_match == target_value, f"'{pattern}' check failed, {num_match} does not equal to target {target_value}" - - -def check_server_metrics(metrics_port="8002", - batching_strategy="", - kv_cache_reuse=""): - metrics = check_output(f"curl 0.0.0.0:{metrics_port}/metrics 2>&1", - shell=True).strip() - print_info(metrics) - - pattern_request_success = r'nv_inference_request_success\{model="tensorrt_llm",version="1"\} (\d)' - assert_pattern_match_target(pattern_request_success, metrics, 1) - pattern_inference_count = r'nv_inference_count\{model="tensorrt_llm",version="1"\} (\d)' - assert_pattern_match_target(pattern_inference_count, metrics, 1) - pattern_exec_count = r'nv_inference_exec_count\{model="tensorrt_llm",version="1"\} (\d)' - assert_pattern_match_target(pattern_exec_count, metrics, 1) - if kv_cache_reuse == "False": - pattern_kv_cache_block_used = r'nv_trt_llm_kv_cache_block_metrics\{kv_cache_block_type="used",model="tensorrt_llm",version="1"\} (\d)' - assert_pattern_match_target(pattern_kv_cache_block_used, metrics, 0) - if batching_strategy == "inflight_fused_batching": - pattern_generation_requests = ( - r'nv_trt_llm_inflight_batcher_metrics' - r'\{inflight_batcher_specific_metric="generation_requests",model="tensorrt_llm",version="1"\} (\d)' - ) - assert_pattern_match_target(pattern_generation_requests, metrics, 0) - - -def search_and_replace(file_path, search_words, replace_words): - with open(file_path, 'r') as file: - original_contents = file.read() - updated_contents = re.sub(search_words, replace_words, - original_contents) - with open(file_path, 'w') as file: - file.write(updated_contents) - - -def prepare_ib_model_repo(llm_backend_repo_root, new_model_repo, model_name=""): - origin_model_repo = os.path.join(llm_backend_repo_root, "all_models", - "inflight_batcher_llm") - check_call(f"rm -rf {new_model_repo}", shell=True) - check_call(f"cp -R {origin_model_repo} {new_model_repo}", shell=True) - - if model_name == "whisper": - whisper_model_repo = os.path.join(llm_backend_repo_root, "all_models", - "whisper", "whisper_bls") - check_call(f"cp -R {whisper_model_repo} {new_model_repo}", shell=True) - - -def prepare_custom_config(llm_backend_repo_root, new_model_repo, - new_config_name): - tensorrt_llm_config = os.path.join(llm_backend_repo_root, "all_models", - "inflight_batcher_llm", "tensorrt_llm") - new_config = os.path.join(new_model_repo, new_config_name) - check_call(f"cp -R {tensorrt_llm_config} {new_config}", shell=True) - - -def prepare_multimodal_model_repo(llm_backend_repo_root, new_model_repo, - dir_name): - origin_model_repo = os.path.join(llm_backend_repo_root, "all_models", - "multimodal", dir_name) - check_call(f"cp -R {origin_model_repo} {new_model_repo}", shell=True) - - -def prepare_disaggregated_serving_model_repo(llm_backend_repo_root, - new_model_repo): - origin_model_repo = os.path.join(llm_backend_repo_root, "all_models", - "disaggregated_serving", - "disaggregated_serving_bls") - check_call(f"cp -R {origin_model_repo} {new_model_repo}", shell=True) - - def prepare_llmapi_model_repo(llm_backend_repo_root, new_model_repo): origin_model_repo = os.path.join(llm_backend_repo_root, "all_models", "llmapi") @@ -217,517 +57,3 @@ def set_llmapi_decoupled_mode(new_model_repo, decoupled): f.write("\nmodel_transaction_policy {\n" f" decoupled: {str(bool(decoupled)).lower()}\n" "}\n") - - -def modify_ib_config_pbtxt(REPO_PATH, - DECODER_ENGINE_PATH, - TOKENIZER_PATH, - llm_backend_repo_root, - DECOUPLED_MODE, - MAX_TOKENS_IN_KV_CACHE, - MAX_ATTENTION_WINDOW_SIZE, - BATCH_SCHEDULER_POLICY, - BATCHING_STRATEGY, - KV_CACHE_FREE_GPU_MEM_FRACTION, - EXCLUDE_INPUT_IN_OUTPUT, - ENABLE_TRT_OVERLAP, - TRITON_MAX_BATCH_SIZE, - MAX_QUEUE_DELAY_MICROSECONDS, - MAX_BEAM_WIDTH, - ENABLE_KV_CACHE_REUSE, - NORMALIZE_LOG_PROBS, - ENABLE_CHUNKED_CONTEXT, - GPU_DEVICE_IDS, - DECODING_MODE, - PREPROCESSING_INSTANCE_COUNT, - POSTPROCESSING_INSTANCE_COUNT, - ACCUMULATE_TOKEN, - BLS_INSTANCE_COUNT, - TENSORRT_LLM_TARGET_MODEL_NAME="tensorrt_llm_target", - TENSORRT_LLM_DRAFT_MODEL_NAME="tensorrt_llm_draft", - BACKEND="tensorrtllm", - GPU_WEIGHTS_PERCENT="1.0", - ENCODER_ENGINE_PATH="", - MULTIMODAL_ENGINE_PATH="", - DRAFT_ENGINE_PATH="", - TARGET_ENGINE_PATH="", - MAX_QUEUE_SIZE="0", - ENABLE_CONTEXT_FMHA_FP32_ACC="False", - PARTICIPANT_IDS="", - PARTICIPANT_IDS_DRAFT="", - PARTICIPANT_IDS_TARGET="", - SPEC_DEC_FAST_LOGITS="0", - EXECUTOR_LOOKAHEAD_WINDOW="", - EXECUTOR_LOOKAHEAD_NGRAM="", - EXECUTOR_LOOKAHEAD_VERIFICATION_SET="", - MAX_NUM_IMAGES="1", - CROSS_KV_CACHE_FRACTION="", - ENCODER_INPUT_FEATURES_DTYPE="TYPE_FP16", - GUIDED_DECODING_BACKEND="", - XGRAMMAR_TOKENIZER_INFO_PATH="", - PROMPT_EMBEDDING_TABLE_DTYPE="TYPE_FP16"): - fill_template_py = os.path.join(llm_backend_repo_root, "tools", - "fill_template.py") - tensorrt_llm_config = os.path.join(llm_backend_repo_root, REPO_PATH, - "tensorrt_llm", "config.pbtxt") - preprocessing_config = os.path.join(llm_backend_repo_root, REPO_PATH, - "preprocessing", "config.pbtxt") - postprocessing_config = os.path.join(llm_backend_repo_root, REPO_PATH, - "postprocessing", "config.pbtxt") - ensemble_config = os.path.join(llm_backend_repo_root, REPO_PATH, "ensemble", - "config.pbtxt") - tensorrt_llm_bls_config = os.path.join(llm_backend_repo_root, REPO_PATH, - "tensorrt_llm_bls", "config.pbtxt") - whisper_bls_config = os.path.join(llm_backend_repo_root, REPO_PATH, - "whisper_bls", "config.pbtxt") - disaggregated_serving_bls_config = os.path.join( - llm_backend_repo_root, REPO_PATH, "disaggregated_serving_bls", - "config.pbtxt") - - if MULTIMODAL_ENGINE_PATH != "": - multimodal_enc_config = os.path.join(llm_backend_repo_root, REPO_PATH, - "multimodal_encoders", - "config.pbtxt") - - check_call( - f"python3 {fill_template_py} -i {multimodal_enc_config} triton_max_batch_size:{TRITON_MAX_BATCH_SIZE}," \ - f"multimodal_model_path:{MULTIMODAL_ENGINE_PATH},encoder_input_features_data_type:{ENCODER_INPUT_FEATURES_DTYPE}," \ - f"prompt_embedding_table_data_type:{PROMPT_EMBEDDING_TABLE_DTYPE}," \ - f"hf_model_path:{TOKENIZER_PATH}", - shell=True) - check_call( - f"python3 {fill_template_py} -i {tensorrt_llm_bls_config} tensorrt_llm_model_name:tensorrt_llm," \ - f"multimodal_encoders_name:multimodal_encoders", - shell=True) - check_call( - f"python3 {fill_template_py} -i {preprocessing_config} max_num_images:{MAX_NUM_IMAGES}", - shell=True) - - if DRAFT_ENGINE_PATH != "": - llm_draft_config = os.path.join(llm_backend_repo_root, REPO_PATH, - "tensorrt_llm_draft", "config.pbtxt") - search_words = 'name: "tensorrt_llm"' - replace_words = 'name: "tensorrt_llm_draft"' - search_and_replace(llm_draft_config, search_words, replace_words) - check_call( - f"python3 {fill_template_py} -i {llm_draft_config} 'triton_backend:{BACKEND},engine_dir:{DRAFT_ENGINE_PATH},decoupled_mode:{DECOUPLED_MODE}," \ - f"max_tokens_in_paged_kv_cache:{MAX_TOKENS_IN_KV_CACHE},max_attention_window_size:{MAX_ATTENTION_WINDOW_SIZE},batch_scheduler_policy:{BATCH_SCHEDULER_POLICY}," \ - f"batching_strategy:{BATCHING_STRATEGY}," \ - f"kv_cache_free_gpu_mem_fraction:{KV_CACHE_FREE_GPU_MEM_FRACTION},enable_trt_overlap:{ENABLE_TRT_OVERLAP}," \ - f"exclude_input_in_output:{EXCLUDE_INPUT_IN_OUTPUT},triton_max_batch_size:{TRITON_MAX_BATCH_SIZE}," \ - f"max_queue_delay_microseconds:{MAX_QUEUE_DELAY_MICROSECONDS},max_beam_width:{MAX_BEAM_WIDTH}," \ - f"enable_kv_cache_reuse:{ENABLE_KV_CACHE_REUSE},normalize_log_probs:{NORMALIZE_LOG_PROBS}," \ - f"enable_chunked_context:{ENABLE_CHUNKED_CONTEXT},gpu_device_ids:{GPU_DEVICE_IDS},decoding_mode:{DECODING_MODE}," \ - f"gpu_weights_percent:{GPU_WEIGHTS_PERCENT},encoder_engine_dir:{ENCODER_ENGINE_PATH},max_queue_size:{MAX_QUEUE_SIZE}," \ - f"speculative_decoding_fast_logits:{SPEC_DEC_FAST_LOGITS}," \ - f"lookahead_window_size:{EXECUTOR_LOOKAHEAD_WINDOW}," \ - f"lookahead_ngram_size:{EXECUTOR_LOOKAHEAD_NGRAM}," \ - f"lookahead_verification_set_size:{EXECUTOR_LOOKAHEAD_VERIFICATION_SET}," \ - f"encoder_input_features_data_type:{ENCODER_INPUT_FEATURES_DTYPE}," \ - f"prompt_embedding_table_data_type:{PROMPT_EMBEDDING_TABLE_DTYPE}," \ - f"participant_ids:{PARTICIPANT_IDS_DRAFT}," \ - f"logits_datatype:TYPE_FP32'", - shell=True) - if TARGET_ENGINE_PATH != "": - llm_target_config = os.path.join(llm_backend_repo_root, REPO_PATH, - "tensorrt_llm_target", "config.pbtxt") - search_words = 'name: "tensorrt_llm"' - replace_words = 'name: "tensorrt_llm_target"' - search_and_replace(llm_target_config, search_words, replace_words) - check_call( - f"python3 {fill_template_py} -i {llm_target_config} 'triton_backend:{BACKEND},engine_dir:{TARGET_ENGINE_PATH},decoupled_mode:{DECOUPLED_MODE}," \ - f"max_tokens_in_paged_kv_cache:{MAX_TOKENS_IN_KV_CACHE},max_attention_window_size:{MAX_ATTENTION_WINDOW_SIZE},batch_scheduler_policy:{BATCH_SCHEDULER_POLICY}," \ - f"batching_strategy:{BATCHING_STRATEGY}," \ - f"kv_cache_free_gpu_mem_fraction:{KV_CACHE_FREE_GPU_MEM_FRACTION},enable_trt_overlap:{ENABLE_TRT_OVERLAP}," \ - f"exclude_input_in_output:{EXCLUDE_INPUT_IN_OUTPUT},triton_max_batch_size:{TRITON_MAX_BATCH_SIZE}," \ - f"max_queue_delay_microseconds:{MAX_QUEUE_DELAY_MICROSECONDS},max_beam_width:{MAX_BEAM_WIDTH}," \ - f"enable_kv_cache_reuse:true,normalize_log_probs:{NORMALIZE_LOG_PROBS}," \ - f"enable_chunked_context:{ENABLE_CHUNKED_CONTEXT},gpu_device_ids:{GPU_DEVICE_IDS},decoding_mode:{DECODING_MODE}," \ - f"gpu_weights_percent:{GPU_WEIGHTS_PERCENT},encoder_engine_dir:{ENCODER_ENGINE_PATH},max_queue_size:{MAX_QUEUE_SIZE}," \ - f"speculative_decoding_fast_logits:{SPEC_DEC_FAST_LOGITS}," \ - f"lookahead_window_size:{EXECUTOR_LOOKAHEAD_WINDOW}," \ - f"lookahead_ngram_size:{EXECUTOR_LOOKAHEAD_NGRAM}," \ - f"lookahead_verification_set_size:{EXECUTOR_LOOKAHEAD_VERIFICATION_SET}," \ - f"encoder_input_features_data_type:{ENCODER_INPUT_FEATURES_DTYPE}," \ - f"prompt_embedding_table_data_type:{PROMPT_EMBEDDING_TABLE_DTYPE}," \ - f"participant_ids:{PARTICIPANT_IDS_TARGET}," \ - f"logits_datatype:TYPE_FP32'", - shell=True) - - check_call( - f"python3 {fill_template_py} -i {preprocessing_config} tokenizer_dir:{TOKENIZER_PATH}," \ - f"triton_max_batch_size:{TRITON_MAX_BATCH_SIZE},preprocessing_instance_count:{PREPROCESSING_INSTANCE_COUNT}," \ - f"multimodal_model_path:{MULTIMODAL_ENGINE_PATH},engine_dir:{DECODER_ENGINE_PATH}", - shell=True) - check_call( - f"python3 {fill_template_py} -i {postprocessing_config} tokenizer_dir:{TOKENIZER_PATH}," \ - f"triton_max_batch_size:{TRITON_MAX_BATCH_SIZE},postprocessing_instance_count:{POSTPROCESSING_INSTANCE_COUNT}", - shell=True) - check_call( - f"python3 {fill_template_py} -i {ensemble_config} triton_max_batch_size:{TRITON_MAX_BATCH_SIZE},logits_datatype:TYPE_FP32", - shell=True) - check_call( - f"python3 {fill_template_py} -i {tensorrt_llm_bls_config} triton_max_batch_size:{TRITON_MAX_BATCH_SIZE}," \ - f"decoupled_mode:{DECOUPLED_MODE},accumulate_tokens:{ACCUMULATE_TOKEN},bls_instance_count:{BLS_INSTANCE_COUNT}," \ - f"tensorrt_llm_model_name:{TENSORRT_LLM_TARGET_MODEL_NAME},tensorrt_llm_draft_model_name:{TENSORRT_LLM_DRAFT_MODEL_NAME},logits_datatype:TYPE_FP32," \ - f"prompt_embedding_table_data_type:{PROMPT_EMBEDDING_TABLE_DTYPE}", - shell=True) - - check_call( - f"python3 {fill_template_py} -i {tensorrt_llm_config} 'triton_backend:{BACKEND},engine_dir:{DECODER_ENGINE_PATH},decoupled_mode:{DECOUPLED_MODE}," \ - f"max_tokens_in_paged_kv_cache:{MAX_TOKENS_IN_KV_CACHE},max_attention_window_size:{MAX_ATTENTION_WINDOW_SIZE},batch_scheduler_policy:{BATCH_SCHEDULER_POLICY}," \ - f"batching_strategy:{BATCHING_STRATEGY}," \ - f"kv_cache_free_gpu_mem_fraction:{KV_CACHE_FREE_GPU_MEM_FRACTION},cross_kv_cache_fraction:{CROSS_KV_CACHE_FRACTION},enable_trt_overlap:{ENABLE_TRT_OVERLAP}," \ - f"exclude_input_in_output:{EXCLUDE_INPUT_IN_OUTPUT},triton_max_batch_size:{TRITON_MAX_BATCH_SIZE}," \ - f"max_queue_delay_microseconds:{MAX_QUEUE_DELAY_MICROSECONDS},max_beam_width:{MAX_BEAM_WIDTH}," \ - f"enable_kv_cache_reuse:{ENABLE_KV_CACHE_REUSE},normalize_log_probs:{NORMALIZE_LOG_PROBS}," \ - f"enable_chunked_context:{ENABLE_CHUNKED_CONTEXT},gpu_device_ids:{GPU_DEVICE_IDS},decoding_mode:{DECODING_MODE}," \ - f"gpu_weights_percent:{GPU_WEIGHTS_PERCENT},encoder_engine_dir:{ENCODER_ENGINE_PATH},max_queue_size:{MAX_QUEUE_SIZE}," \ - f"enable_context_fmha_fp32_acc:{ENABLE_CONTEXT_FMHA_FP32_ACC}," \ - f"encoder_input_features_data_type:{ENCODER_INPUT_FEATURES_DTYPE}," \ - f"prompt_embedding_table_data_type:{PROMPT_EMBEDDING_TABLE_DTYPE}," \ - f"participant_ids:{PARTICIPANT_IDS}," \ - f"logits_datatype:TYPE_FP32,guided_decoding_backend:{GUIDED_DECODING_BACKEND},tokenizer_dir:{TOKENIZER_PATH},xgrammar_tokenizer_info_path:{XGRAMMAR_TOKENIZER_INFO_PATH}'", - shell=True) - - if os.path.exists(whisper_bls_config): - check_call( - f"python3 {fill_template_py} -i {whisper_bls_config} engine_dir:{ENCODER_ENGINE_PATH}," \ - f"n_mels:128,zero_pad:false,triton_max_batch_size:{TRITON_MAX_BATCH_SIZE},decoupled_mode:{DECOUPLED_MODE}", - shell=True) - if os.path.exists(disaggregated_serving_bls_config): - check_call( - f"python3 {fill_template_py} -i {disaggregated_serving_bls_config} 'triton_max_batch_size:{TRITON_MAX_BATCH_SIZE}," \ - f"decoupled_mode:{DECOUPLED_MODE},disaggregated_serving_bls_count:{BLS_INSTANCE_COUNT}," \ - "context_model_name:context,generation_model_name:generation,logits_datatype:TYPE_FP32'", - shell=True) - - -def modify_disaggregated_serving_config_pbtxt(llm_backend_repo_root, REPO_PATH): - check_call(f"cp -R {REPO_PATH}/tensorrt_llm {REPO_PATH}/generation", - shell=True) - check_call(f"mv {REPO_PATH}/tensorrt_llm {REPO_PATH}/context", shell=True) - check_call( - f"mv {REPO_PATH}/disaggregated_serving_bls {REPO_PATH}/tensorrt_llm", - shell=True) - - tensorrt_llm_config = os.path.join(llm_backend_repo_root, REPO_PATH, - "tensorrt_llm", "config.pbtxt") - search_and_replace(tensorrt_llm_config, 'name: "disaggregated_serving_bls"', - 'name: "tensorrt_llm"') - context_config = os.path.join(llm_backend_repo_root, REPO_PATH, "context", - "config.pbtxt") - search_and_replace(context_config, 'name: "tensorrt_llm"', - 'name: "context"') - generation_config = os.path.join(llm_backend_repo_root, REPO_PATH, - "generation", "config.pbtxt") - search_and_replace(generation_config, 'name: "tensorrt_llm"', - 'name: "generation"') - - -def validate_by_sequence_matcher(output_result, golden_results, threshold): - rankings = {} - for golden_result in golden_results: - output_result = output_result.strip() - golden_result = golden_result.strip() - matcher = SequenceMatcher(None, output_result, golden_result) - # Get the similarity ratio and populate rankings dict - similarity_ratio = matcher.ratio() - rankings[str(similarity_ratio)] = golden_result - - # Find out the highest_similarity_ratio - highest_similarity_ratio, golden_result = max(rankings.items(), - key=lambda x: float(x[0])) - print_info(f"output_result: {output_result}") - print_info( - f"rankings(similarity_ratio:golden_result):\n{json.dumps(rankings, indent=4)}" - ) - - if float(highest_similarity_ratio) < threshold: - pytest.fail( - f"highest_similarity_ratio {highest_similarity_ratio} is less than {threshold}" - ) - - -def validate_by_keyword(output_result, keyword): - if keyword not in output_result: - pytest.fail(f"FAIL! \"{keyword}\" not in output:\n{output_result}") - else: - print_info(f"PASS! \"{keyword}\" in output:\n{output_result}") - - -def run_cpp_backend_tests(feature_name, llm_backend_venv, - inflight_batcher_llm_client_root, tokenizer_dir): - # Chooses script - script_name = "" - if feature_name in [ - "test_basic", "test_log_probs", "test_request_id", "test_n_returns" - ]: - script_name = f"{inflight_batcher_llm_client_root}/inflight_batcher_llm_client.py" - elif feature_name in ["test_stop_words", "test_embedding_bias"]: - script_name = f"{inflight_batcher_llm_client_root}/end_to_end_grpc_client.py" - - # Run command - if "inflight_batcher_llm_client.py" in script_name: - run_cmd = [ - f"{script_name}", - f"--tokenizer-dir={tokenizer_dir}", - ] - - if feature_name == "test_log_probs": - run_cmd += [ - "--request-output-len=10", - "--return-log-probs", - "--top-k=2", - ] - elif feature_name == "test_request_id": - run_cmd += [ - "--request-id=my_request", - ] - elif feature_name == "test_n_returns": - run_cmd += [ - "--num-return-sequences=3", - ] - - venv_check_call(llm_backend_venv, run_cmd) - elif "end_to_end_grpc_client.py" in script_name: - if feature_name == "test_stop_words": - run_cmd = [ - f"{script_name}", - f"-o=10", - "-p=\"The only thing we have to fear is\"", - "--stop-words=\" government\"", - ] - output = venv_check_output(llm_backend_venv, run_cmd) - print_info(f"The test output is:\n{output}") - with tempfile.NamedTemporaryFile( - dir=llm_backend_venv.get_working_directory(), - mode='w', - delete=False) as temp_file: - temp_file.write(output) - temp_file.close() - check_call( - f"grep -v \"that the government will\" {temp_file.name}", - shell=True) - if feature_name == "test_embedding_bias": - run_cmd = [ - f"{script_name}", - f"-o=10", - "-p=\"The only thing we have to fear is\"", - "--embedding-bias-words=\" government\"", - "--embedding-bias-weights=-20", - ] - output = venv_check_output(llm_backend_venv, run_cmd) - print_info(f"The test output is:\n{output}") - with tempfile.NamedTemporaryFile( - dir=llm_backend_venv.get_working_directory(), - mode='w', - delete=False) as temp_file: - temp_file.write(output) - temp_file.close() - check_call( - f"grep -v \"that the government will\" {temp_file.name}", - shell=True) - - -def run_cpp_streaming_backend_tests(feature_name, - llm_backend_venv, - inflight_batcher_llm_client_root, - tokenizer_dir, - model_name="", - e2e_model=""): - # Chooses script - script_name = "" - if feature_name in ["test_basic"]: - script_name = f"{inflight_batcher_llm_client_root}/inflight_batcher_llm_client.py" - elif feature_name in ["batched_inputs"] and e2e_model == "tensorrt_llm_bls": - script_name = f"{inflight_batcher_llm_client_root}/end_to_end_grpc_client.py" - - # Run command - if "inflight_batcher_llm_client.py" in script_name: - run_cmd = [ - f"{script_name}", - "--streaming", - f"--tokenizer-dir={tokenizer_dir}", - ] - - if feature_name == "test_basic": - venv_check_call(llm_backend_venv, run_cmd) - elif "end_to_end_grpc_client.py" in script_name: - raw_input = """["This is a test","I want you to","The cat is"]""" - raw_output = "" - gpu_name = query_gpu_name() - run_cmd = [ - f"{script_name}", - "--streaming", - "-o=5", - f"--model-name={e2e_model}", - f"-p={raw_input}", - "--batch-inputs", - "--overwrite-output-text", - ] - if "H100" in gpu_name: - if "gpt" in model_name.lower(): - raw_output = """[" of the power of the"," know that I am not"," a very good cat."]""" - elif "llama" in model_name.lower(): - raw_output = """["of the emergency alert","know that I am not", "out of the bag."]""" - if raw_output != "": - run_cmd += [ - f"--expected-outputs={raw_output}", - "--check-outputs", - ] - if feature_name == "batched_inputs": - venv_check_call(llm_backend_venv, run_cmd) - - -def retrieve_latency_value(log): - m = re.search(r"Latency: (\d+\.\d+) ms", log) - latency_value = None - if m is not None: - latency_value = m.group(1).strip() - - assert latency_value is not None, f"Did not find latency value in log: {log}." - return float(latency_value) - - -def get_pid_by_name(process_name): - proc_pid = None - for proc in psutil.process_iter(['pid', 'name']): - # Skip zombie process. - if proc.info['name'] == process_name and proc.status( - ) != psutil.STATUS_ZOMBIE: - proc_pid = proc.info['pid'] - break - - assert proc_pid, f"Fail to get a valid process pid of {process_name}." - return proc_pid - - -def get_rss_usage_bytes_by_pid(pid): - rss = None - try: - process = psutil.Process(pid) - rss = process.memory_info().rss - except psutil.NoSuchProcess: - print_error(f"Process with PID {pid} no longer exists.") - except psutil.AccessDenied: - print_error(f"Access denied to process with PID {pid}.") - except Exception as e: - print_error(f"An error occurred: {e}") - - assert rss is not None, f"Fail to get RSS usage of pid {pid}." - return rss - - -def check_avg_rss_increasement(llm_backend_venv, - process_name, - inference_cmd, - rss_increase_bytes_threshold=64, - warm_up_times=10, - total_run_times=20): - pid = get_pid_by_name(process_name) - rss_usage_before_inference = get_rss_usage_bytes_by_pid(pid) - - # Warm-up. - time = 1 - for _ in range(warm_up_times): - venv_check_call(llm_backend_venv, inference_cmd) - current_rss_usage = get_rss_usage_bytes_by_pid(pid) - print_info( - f"The RSS usage after {time} inference request is: {current_rss_usage} bytes." - ) - time += 1 - - rss_usage_after_warmup = get_rss_usage_bytes_by_pid(pid) - - # Calculate average RSS increasement. - if total_run_times <= warm_up_times: - raise ValueError(f"total_run_times must larger than {warm_up_times}.") - for _ in range(total_run_times - warm_up_times): - venv_check_call(llm_backend_venv, inference_cmd) - current_rss_usage = get_rss_usage_bytes_by_pid(pid) - print_info( - f"The RSS usage after {time} inference request is: {current_rss_usage} bytes." - ) - time += 1 - - rss_usage_final_run = get_rss_usage_bytes_by_pid(pid) - avg_rss_increasement = (rss_usage_final_run - rss_usage_after_warmup) // ( - total_run_times - warm_up_times) - - print_info(f"Checking RSS usage of process: {process_name}.") - print_info( - f"The RSS usage before inference is: {rss_usage_before_inference} bytes." - ) - print_info( - f"The RSS usage after {warm_up_times} times warm-up run is: {rss_usage_after_warmup} bytes." - ) - print_info( - f"The RSS usage after {total_run_times} times run is: {rss_usage_final_run} bytes." - ) - print_info( - f"The average RSS increasement after warm-up is: {avg_rss_increasement} bytes." - ) - - if avg_rss_increasement > rss_increase_bytes_threshold: - pytest.fail( - f"The average RSS increasement: {avg_rss_increasement} bytes > threshold: {rss_increase_bytes_threshold} bytes." - ) - else: - print_info( - f"The average RSS increasement: {avg_rss_increasement} bytes <= threshold: {rss_increase_bytes_threshold} bytes." - ) - - -def parse_endpoint_generated_outputs(output_text, - max_tokens, - stream, - count_tokens=False, - check_repetition=False): - print_info("Analyzing the outputs...") - pattern = r'"text_output"\s*:\s*"(.*)"' - matches = re.findall(pattern, output_text) - assert matches is not None, "No matching outputs." - print_info(f"The matched output tokens are:\n{matches}") - - if count_tokens: - num_tokens = max_tokens if stream is True else 1 - num_matches = len(matches) - assert num_tokens == num_matches, f"The output token amount: {num_matches} is not matching expected: {num_tokens}." - - if check_repetition: - from collections import Counter - match_counts = Counter(matches) - total_matches = len(matches) - for match, count in match_counts.items(): - repetition_rate = (count / total_matches) * 100 - assert repetition_rate <= 50, f"Repetition rate of '{match}' is {repetition_rate}%, which is beyond the allowed threshold." - - -def parse_endpoint_generated_json_outputs(output_text, check_repetition=False): - print_info("Analyzing the outputs...") - try: - # Parse the JSON string - output_json = json.loads(output_text) - - # Extract the text_output field - text_output = output_json.get("text_output", []) - - # Print the output tokens - print_info(f"The matched output tokens are:\n{text_output}") - - # Check for repetition in text_output - if check_repetition: - if isinstance(text_output, list) and len(text_output) > 0: - from collections import Counter - item_counts = Counter(text_output) - duplicates = { - item: count - for item, count in item_counts.items() if count > 1 - } - - assert not duplicates, f"Repetition found in text_output: {duplicates}" - else: - print_error("text_output is not a list or is empty.") - except json.JSONDecodeError as e: - print_error(f"Error parsing JSON: {e}") diff --git a/tests/integration/defs/triton_server/local_venv.py b/tests/integration/defs/triton_server/local_venv.py deleted file mode 100644 index 77dfb810de34..000000000000 --- a/tests/integration/defs/triton_server/local_venv.py +++ /dev/null @@ -1,113 +0,0 @@ -""" -Local implementation of our Python venv testing. -All the bootstrapping is done with our original core work. -""" -import copy -import os -import tempfile -import textwrap as tw - -from defs.runner_interface import PythonRunnerInterface -from defs.trt_test_alternative import check_call, check_output - - -class PythonVenvRunnerImpl(PythonRunnerInterface): - """ - Object that understands how to run Python scripts in a virtual environment. - Local implementation of our runner. - - Args: - pip_opts (list): Options to pass to pip when installing packages - venv_dir (str): Path to the virtualenv root directory, or None if this is - an externally-built virtualenv - venv_bin (str): Path to the Python executable to use when running tests - workspace (str): Path to the test workspace - """ - - def __init__(self, pip_opts, venv_dir, venv_bin, workspace): - self._venv_bin = venv_bin - self.venv_dir = venv_dir - self.pip_opts = pip_opts - self.venv_exe_name = os.path.basename(venv_bin) - self.venv_exe_dir = os.path.dirname(venv_bin) - self.workspace = workspace - self._new_env = os.environ.copy() - - def get_working_directory(self, translate_wsl_path=True): - """ - Common interface required by RunnerInterface. - Both TRTExecutableRunner and VirtualenvRunner both have a workspace attribute but they are - of different types (function vs variable), as a result, this function is introduced instead. - """ - return self.workspace - - def _get_repro_envvars(self): - return sorted(os.environ) - - def set_working_directory(self, value): - """ - Common interface required by RunnerInterface. - Both TRTExecutableRunner and VirtualenvRunner both have a workspace attribute but they are - of different types (function vs variable), as a result, this function is introduced instead. - """ - self.workspace = value - - def _run_internal(self, script, caller, print_script: bool): - - if isinstance(script, bytes): - script = script.decode() - if print_script: - print("Run with Python: {}".format(self._venv_bin)) - print("=== BEGIN SCRIPT ===") - print(script) - print("=== END SCRIPT =====") - - if not os.path.exists(self.workspace): - os.makedirs(self.workspace, exist_ok=True) - f = tempfile.NamedTemporaryFile(dir=self.workspace, - mode="w", - delete=False) - f.write(script) - # On Windows, we have to first close the file before we can read it - f.close() - try: - path = f.name - out = caller([self._venv_bin, path]) - finally: - os.remove(f.name) - - return out - - def run(self, body, caller=check_call, print_script=True): - """Run Python code in a script with a predefined prolog.""" - script = "" - script += tw.dedent(body) - return self._run_internal(script, caller, print_script) - - def run_output(self, body): - """Runs Python code and captures the output in a string.""" - return self.run(body, caller=check_output, print_script=False) - - def run_raw(self, script, caller=check_call, print_script=True): - """Run Python code without any pre-processing.""" - return self._run_internal(script, caller, print_script) - - def run_cmd(self, args, caller=check_call, env=None, print_script=True): - """Call on the command-line (can be used to run Python script files).""" - if not os.path.exists(self.workspace): - os.makedirs(self.workspace, exist_ok=True) - - call_args = [self._venv_bin] + args - if env: - new_env = copy.deepcopy(os.environ) - new_env.update(env) - else: - new_env = os.environ - - return caller(call_args, env=new_env) - - def install_packages(self, packages): - """Install Python packages by name.""" - - def install_from_requirements(self, requirements_file): - pass diff --git a/tests/integration/defs/triton_server/runner_interface.py b/tests/integration/defs/triton_server/runner_interface.py deleted file mode 100644 index d6ad0956f5ca..000000000000 --- a/tests/integration/defs/triton_server/runner_interface.py +++ /dev/null @@ -1,57 +0,0 @@ -""" - File that holds interfaces for runners. - Needs to be in this file to prevent cyclic loops. -""" - -from abc import ABC, abstractmethod - - -class RunnerInterface(ABC): - """Extend this class to use with runner related functions such as misc.temp_wd""" - - @abstractmethod - def get_working_directory(self): - """ - Since workspace is a function in TRTExectuableRunner and workspace is a variable in VritualenvRunner we introduce this particular - function as a common interface between the two runners. - """ - - @abstractmethod - def set_working_directory(self, value): - pass - - @abstractmethod - def run(self): - pass - - -class PythonRunnerInterface(RunnerInterface, ABC): - """ - A public interface for runners working with Python based tests. - Implementation is dependent on local and remote modes and so implementations - can be found under misc/venv_runner_remote.py and misc/venv_runner_local.py - """ - - @abstractmethod - def install_from_requirements(self, requirements_file): - """Install Python packages listed in the given requirements file.""" - - @abstractmethod - def install_packages(self, packages): - """Install Python packages by name.""" - - @abstractmethod - def run_cmd(self, args, caller, env, print_script): - """Call on the command-line (can be used to run Python script files).""" - - @abstractmethod - def run_raw(self, script, caller, print_script): - """Run Python code without any pre-processing.""" - - @abstractmethod - def run_output(self, body): - """Runs Python code and captures the output in a string.""" - - @abstractmethod - def run(self, body, caller, print_script): - """Run Python code in a script with a predefined prolog.""" diff --git a/tests/integration/defs/triton_server/test.sh b/tests/integration/defs/triton_server/test.sh deleted file mode 100755 index eb3c904d87ab..000000000000 --- a/tests/integration/defs/triton_server/test.sh +++ /dev/null @@ -1,1692 +0,0 @@ -#!/usr/bin/bash - -MODEL=$1 -DECODER_ENGINE_PATH=$2 -TOKENIZER_PATH=$3 -TOKENIZER_TYPE=$4 -DRAFT_ENGINE_PATH=$5 -TARGET_ENGINE_PATH=$6 -ENCODER_ENGINE_PATH=$7 -MULTIMODAL_ENGINE_PATH=$8 - -set -ex -set -o pipefail -nvidia-smi -pushd $LLM_BACKEND_ROOT -source tools/utils.sh -TRITON_REPO="triton_repo" - -kill_triton_server () { - pkill -9 -f trtllmExecutorWorker || true - pkill -9 -f tritonserver -} - -# Kill titonserver if it is still pending from previous test -kill_triton_server || true - -if [ "$MODEL" = "mistral" ] || [ "$MODEL" = "mistral-ib" ] || [ "$MODEL" = "mistral-ib-mm" ]; then - MAX_ATTENTION_WINDOW_SIZE="2048" - MAX_SEQUENCE_LEN="8704" # max_input_len + max_output_len -elif [ "$MODEL" = "t5-ib" ] || [ "$MODEL" = "bart-ib" ]; then - MAX_ATTENTION_WINDOW_SIZE="" - MAX_SEQUENCE_LEN="4096" # for enc-dec, choose a sufficient size of max token in kv cache to avoid no free block error -elif [ "$MODEL" = "whisper" ]; then - MAX_ATTENTION_WINDOW_SIZE="" - MAX_SEQUENCE_LEN="24000" # WAR to avoid no free block errors -else - MAX_ATTENTION_WINDOW_SIZE="" - MAX_SEQUENCE_LEN="2048" -fi - -if [ "$MODEL" = "mllama" ]; then - ENCODER_INPUT_FEATURES_DTYPE="TYPE_BF16" -else - ENCODER_INPUT_FEATURES_DTYPE="TYPE_FP16" -fi - -if [ "$MODEL" = "gpt" ] || [ "$MODEL" = "opt" ] || [ "$MODEL" = "gptj" ] || [ "$MODEL" = "mistral" ]; then - rm -rf ${TRITON_REPO} - cp -R all_models/gpt ${TRITON_REPO} - - # Modify config.pbtxt - python3 tools/fill_template.py -i ${TRITON_REPO}/tensorrt_llm/config.pbtxt engine_dir:${DECODER_ENGINE_PATH} - python3 tools/fill_template.py -i ${TRITON_REPO}/preprocessing/config.pbtxt tokenizer_dir:${TOKENIZER_PATH} - python3 tools/fill_template.py -i ${TRITON_REPO}/postprocessing/config.pbtxt tokenizer_dir:${TOKENIZER_PATH} - - # Launch Triton Server - mpirun --allow-run-as-root \ - -n 1 /opt/tritonserver/bin/tritonserver \ - --model-repository=${TRITON_REPO} \ - --disable-auto-complete-config \ - --backend-config=python,shm-region-prefix-name=prefix0_ : & - export SERVER_PID=$! - wait_for_server_ready ${SERVER_PID} 1200 ${TRITON_HTTP_PORT} - - pushd tools/gpt/ - - # Client - python3 client.py \ - --text="Born in north-east France, Soyer trained as a" \ - --output_len=10 \ - --protocol=http \ - --tokenizer_dir ${TOKENIZER_PATH} - - python3 client.py \ - --text="Born in north-east France, Soyer trained as a" \ - --output_len=10 \ - --protocol=grpc \ - --tokenizer_dir ${TOKENIZER_PATH} - - # Async Client - python3 client_async.py \ - --text="Born in north-east France, Soyer trained as a" \ - --output_len=10 \ - --protocol=http \ - --tokenizer_dir ${TOKENIZER_PATH} - - python3 client_async.py \ - --text="Born in north-east France, Soyer trained as a" \ - --output_len=10 \ - --protocol=grpc \ - --tokenizer_dir ${TOKENIZER_PATH} - - # End to end test - python3 end_to_end_test.py \ - --tokenizer_dir ${TOKENIZER_PATH} - - # Benchmark Core Model - python3 benchmark_core_model.py \ - --batch_size=8 --start_len=128 --output_len=20 \ - --protocol=http --mode=sync - - python3 benchmark_core_model.py \ - --batch_size=8 --start_len=128 --output_len=20 \ - --protocol=grpc --mode=sync - - python3 benchmark_core_model.py \ - --batch_size=8 --start_len=128 --output_len=20 \ - --protocol=http --mode=async - - python3 benchmark_core_model.py \ - --batch_size=8 --start_len=128 --output_len=20 \ - --protocol=grpc --mode=async - - # Benchmark using Perf Analyzer - python3 gen_input_data.py - # FIXME(kaiyu): Uncomment this when perf_analyzer is available. - # perf_analyzer -m tensorrt_llm -v \ - # -b 8 --input-data input_data.json \ - # --concurrency-range 2 \ - # -i http \ - # -u 'localhost:8000' - - # perf_analyzer -m tensorrt_llm -v \ - # -b 8 --input-data input_data.json \ - # --concurrency-range 2 \ - # -i grpc \ - # -u 'localhost:8001' - - kill ${SERVER_PID} - - popd # tools/gpt - -fi - -print_test_params () { - - echo "----------------------------------" - echo " Test parameters:" - echo "----------------------------------" - echo "BACKEND: ${BACKEND}" - echo "BATCHING_STRATEGY: ${BATCHING_STRATEGY}" - echo "MAX_TOKENS_IN_KV_CACHE: ${MAX_TOKENS_IN_KV_CACHE}" - echo "MAX_ATTENTION_WINDOW_SIZE: ${MAX_ATTENTION_WINDOW_SIZE}" - echo "BATCH_SCHEDULER_POLICY: ${BATCH_SCHEDULER_POLICY}" - echo "KV_CACHE_FREE_GPU_MEM_FRACTION: ${KV_CACHE_FREE_GPU_MEM_FRACTION}" - echo "CROSS_KV_CACHE_FRACTION: ${CROSS_KV_CACHE_FRACTION}" - echo "EXCLUDE_INPUT_IN_OUTPUT: ${EXCLUDE_INPUT_IN_OUTPUT}" - echo "TRITON_MAX_BATCH_SIZE: ${TRITON_MAX_BATCH_SIZE}" - echo "MAX_QUEUE_DELAY_MICROSECONDS: ${MAX_QUEUE_DELAY_MICROSECONDS}" - echo "MAX_BEAM_WIDTH: ${MAX_BEAM_WIDTH}" - echo "ENABLE_KV_CACHE_REUSE: ${ENABLE_KV_CACHE_REUSE}" - echo "E2E_MODEL_NAME: ${E2E_MODEL_NAME}" - echo "TENSORRT_LLM_MODEL_NAME: ${TENSORRT_LLM_MODEL_NAME}" - echo "TENSORRT_LLM_TARGET_MODEL_NAME: ${TENSORRT_LLM_TARGET_MODEL_NAME}" - echo "TENSORRT_LLM_DRAFT_MODEL_NAME: ${TENSORRT_LLM_DRAFT_MODEL_NAME}" - echo "ACCUMULATE_TOKEN: ${ACCUMULATE_TOKEN}" - echo "BLS_INSTANCE_COUNT: ${BLS_INSTANCE_COUNT}" - echo "PREPROCESSING_INSTANCE_COUNT: ${PREPROCESSING_INSTANCE_COUNT}" - echo "POSTPROCESSING_INSTANCE_COUNT: ${POSTPROCESSING_INSTANCE_COUNT}" - echo "NORMALIZE_LOG_PROBS: ${NORMALIZE_LOG_PROBS}" - echo "ENABLE_CHUNKED_CONTEXT: ${ENABLE_CHUNKED_CONTEXT}" - echo "GPU_DEVICE_IDS: ${GPU_DEVICE_IDS}" - echo "DECODING_MODE: ${DECODING_MODE}" - echo "MAX_QUEUE_SIZE: ${MAX_QUEUE_SIZE}" - echo "ENABLE_CONTEXT_FMHA_FP32_ACC: ${ENABLE_CONTEXT_FMHA_FP32_ACC}" - echo "PROMPT_EMBEDDING_TABLE_DTYPE: ${PROMPT_EMBEDDING_TABLE_DTYPE}" - echo "run_all_tests: ${run_all_tests}" - echo "----------------------------------" -} - -fill_triton_repo () { - - if [ "${DRAFT_ENGINE_PATH}" != "" ] && [ "${DRAFT_ENGINE_PATH}" != "skip" ]; then - cp -R ${TRITON_REPO}/tensorrt_llm ${TRITON_REPO}/tensorrt_llm_draft - sed -i 's/name: "tensorrt_llm"/name: "tensorrt_llm_draft"/g' ${TRITON_REPO}/tensorrt_llm_draft/config.pbtxt - fi - - if [ "${TARGET_ENGINE_PATH}" != "" ] && [ "${TARGET_ENGINE_PATH}" != "skip" ]; then - cp -R ${TRITON_REPO}/tensorrt_llm ${TRITON_REPO}/tensorrt_llm_target - sed -i 's/name: "tensorrt_llm"/name: "tensorrt_llm_target"/g' ${TRITON_REPO}/tensorrt_llm_target/config.pbtxt - fi - - echo "Filling triton repository at ${TRITON_REPO}/tensorrt_llm with engine ${DECODER_ENGINE_PATH}" - python3 tools/fill_template.py -i ${TRITON_REPO}/tensorrt_llm/config.pbtxt triton_backend:${BACKEND},engine_dir:${DECODER_ENGINE_PATH},decoupled_mode:${DECOUPLED_MODE},max_tokens_in_paged_kv_cache:${MAX_TOKENS_IN_KV_CACHE},max_attention_window_size:${MAX_ATTENTION_WINDOW_SIZE},batch_scheduler_policy:${BATCH_SCHEDULER_POLICY},batching_strategy:${BATCHING_STRATEGY},kv_cache_free_gpu_mem_fraction:${KV_CACHE_FREE_GPU_MEM_FRACTION},exclude_input_in_output:${EXCLUDE_INPUT_IN_OUTPUT},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS},max_beam_width:${MAX_BEAM_WIDTH},enable_kv_cache_reuse:${ENABLE_KV_CACHE_REUSE},normalize_log_probs:${NORMALIZE_LOG_PROBS},enable_chunked_context:${ENABLE_CHUNKED_CONTEXT},gpu_device_ids:${GPU_DEVICE_IDS},decoding_mode:${DECODING_MODE},max_queue_size:${MAX_QUEUE_SIZE},enable_context_fmha_fp32_acc:${ENABLE_CONTEXT_FMHA_FP32_ACC},encoder_input_features_data_type:${ENCODER_INPUT_FEATURES_DTYPE},prompt_embedding_table_data_type:${PROMPT_EMBEDDING_TABLE_DTYPE},logits_datatype:TYPE_FP32,lookahead_window_size:${LOOKAHEAD_WINDOW_SIZE},lookahead_ngram_size:${LOOKAHEAD_NGRAM_SIZE},lookahead_verification_set_size:${LOOKAHEAD_VERIFICATION_SET_SIZE} - python3 tools/fill_template.py -i ${TRITON_REPO}/preprocessing/config.pbtxt tokenizer_dir:${TOKENIZER_PATH},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},preprocessing_instance_count:${PREPROCESSING_INSTANCE_COUNT} - python3 tools/fill_template.py -i ${TRITON_REPO}/postprocessing/config.pbtxt tokenizer_dir:${TOKENIZER_PATH},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},postprocessing_instance_count:${POSTPROCESSING_INSTANCE_COUNT} - python3 tools/fill_template.py -i ${TRITON_REPO}/ensemble/config.pbtxt triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},logits_datatype:TYPE_FP32 - - if [ "${DRAFT_ENGINE_PATH}" != "" ] && [ "${DRAFT_ENGINE_PATH}" != "skip" ] && [ "${TARGET_ENGINE_PATH}" != "" ] && [ "${TARGET_ENGINE_PATH}" != "skip" ]; then - python3 tools/fill_template.py -i ${TRITON_REPO}/tensorrt_llm_bls/config.pbtxt triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},decoupled_mode:${DECOUPLED_MODE},accumulate_tokens:${ACCUMULATE_TOKEN},bls_instance_count:${BLS_INSTANCE_COUNT},tensorrt_llm_model_name:${TENSORRT_LLM_TARGET_MODEL_NAME},logits_datatype:TYPE_FP32,tensorrt_llm_draft_model_name:${TENSORRT_LLM_DRAFT_MODEL_NAME},prompt_embedding_table_data_type:${PROMPT_EMBEDDING_TABLE_DTYPE} - else - python3 tools/fill_template.py -i ${TRITON_REPO}/tensorrt_llm_bls/config.pbtxt triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},decoupled_mode:${DECOUPLED_MODE},accumulate_tokens:${ACCUMULATE_TOKEN},bls_instance_count:${BLS_INSTANCE_COUNT},tensorrt_llm_model_name:${TENSORRT_LLM_MODEL_NAME},logits_datatype:TYPE_FP32,tensorrt_llm_draft_model_name:"",prompt_embedding_table_data_type:${PROMPT_EMBEDDING_TABLE_DTYPE} - fi - - if [ "${DRAFT_ENGINE_PATH}" != "" ] && [ "${DRAFT_ENGINE_PATH}" != "skip" ]; then - echo "Filling triton repository at ${TRITON_REPO}/tensorrt_llm_draft with engine ${DRAFT_ENGINE_PATH}" - python3 tools/fill_template.py -i ${TRITON_REPO}/tensorrt_llm_draft/config.pbtxt triton_backend:${BACKEND},engine_dir:${DRAFT_ENGINE_PATH},decoupled_mode:${DECOUPLED_MODE},max_tokens_in_paged_kv_cache:${MAX_TOKENS_IN_KV_CACHE},max_attention_window_size:${MAX_ATTENTION_WINDOW_SIZE},batch_scheduler_policy:${BATCH_SCHEDULER_POLICY},batching_strategy:${BATCHING_STRATEGY},kv_cache_free_gpu_mem_fraction:${KV_CACHE_FREE_GPU_MEM_FRACTION},exclude_input_in_output:${EXCLUDE_INPUT_IN_OUTPUT},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS},max_beam_width:${MAX_BEAM_WIDTH},enable_kv_cache_reuse:${ENABLE_KV_CACHE_REUSE},normalize_log_probs:${NORMALIZE_LOG_PROBS},enable_chunked_context:${ENABLE_CHUNKED_CONTEXT},gpu_device_ids:${GPU_DEVICE_IDS},decoding_mode:${DECODING_MODE},max_queue_size:${MAX_QUEUE_SIZE},encoder_input_features_data_type:${ENCODER_INPUT_FEATURES_DTYPE},prompt_embedding_table_data_type:${PROMPT_EMBEDDING_TABLE_DTYPE},logits_datatype:TYPE_FP32 - - fi - - if [ "${TARGET_ENGINE_PATH}" != "" ] && [ "${TARGET_ENGINE_PATH}" != "skip" ]; then - echo "Filling triton repository at ${TRITON_REPO}/tensorrt_llm_target with engine ${TARGET_ENGINE_PATH}" - python3 tools/fill_template.py -i ${TRITON_REPO}/tensorrt_llm_target/config.pbtxt triton_backend:${BACKEND},engine_dir:${TARGET_ENGINE_PATH},decoupled_mode:${DECOUPLED_MODE},max_tokens_in_paged_kv_cache:${MAX_TOKENS_IN_KV_CACHE},max_attention_window_size:${MAX_ATTENTION_WINDOW_SIZE},batch_scheduler_policy:${BATCH_SCHEDULER_POLICY},batching_strategy:${BATCHING_STRATEGY},kv_cache_free_gpu_mem_fraction:${KV_CACHE_FREE_GPU_MEM_FRACTION},exclude_input_in_output:${EXCLUDE_INPUT_IN_OUTPUT},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS},max_beam_width:${MAX_BEAM_WIDTH},enable_kv_cache_reuse:true,normalize_log_probs:${NORMALIZE_LOG_PROBS},enable_chunked_context:${ENABLE_CHUNKED_CONTEXT},gpu_device_ids:${GPU_DEVICE_IDS},decoding_mode:${DECODING_MODE},max_queue_size:${MAX_QUEUE_SIZE},encoder_input_features_data_type:${ENCODER_INPUT_FEATURES_DTYPE},prompt_embedding_table_data_type:${PROMPT_EMBEDDING_TABLE_DTYPE},logits_datatype:TYPE_FP32 - - fi - - # encoder-decoder model only - if [ "${CROSS_KV_CACHE_FRACTION}" != "" ]; then - python3 tools/fill_template.py -i ${TRITON_REPO}/tensorrt_llm/config.pbtxt cross_kv_cache_fraction:${CROSS_KV_CACHE_FRACTION} - fi - - if [ "${ENCODER_ENGINE_PATH}" != "" ] && [ "${ENCODER_ENGINE_PATH}" != "skip" ]; then - python3 tools/fill_template.py -i ${TRITON_REPO}/tensorrt_llm/config.pbtxt encoder_engine_dir:${ENCODER_ENGINE_PATH} - fi - - if [ "${MULTIMODAL_ENGINE_PATH}" != "" ] && [ "${MULTIMODAL_ENGINE_PATH}" != "skip" ]; then - cp all_models/multimodal/ensemble ${TRITON_REPO} -r - cp all_models/multimodal/multimodal_encoders ${TRITON_REPO} -r - python3 tools/fill_template.py -i ${TRITON_REPO}/ensemble/config.pbtxt triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},logits_datatype:TYPE_FP32 - python3 tools/fill_template.py -i ${TRITON_REPO}/preprocessing/config.pbtxt multimodal_model_path:${MULTIMODAL_ENGINE_PATH},engine_dir:${DECODER_ENGINE_PATH} - python3 tools/fill_template.py -i ${TRITON_REPO}/multimodal_encoders/config.pbtxt triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},multimodal_model_path:${MULTIMODAL_ENGINE_PATH},encoder_input_features_data_type:${ENCODER_INPUT_FEATURES_DTYPE},prompt_embedding_table_data_type:${PROMPT_EMBEDDING_TABLE_DTYPE},hf_model_path:${TOKENIZER_PATH} - python3 tools/fill_template.py -i ${TRITON_REPO}/tensorrt_llm_bls/config.pbtxt multimodal_encoders_name:multimodal_encoders - - fi - if [ "$MODEL" = "whisper" ]; then - cp all_models/whisper/whisper_bls ${TRITON_REPO} -r - rm -r ${TRITON_REPO}/preprocessing ${TRITON_REPO}/postprocessing ${TRITON_REPO}/ensemble ${TRITON_REPO}/tensorrt_llm_bls - python3 tools/fill_template.py -i ${TRITON_REPO}/whisper_bls/config.pbtxt engine_dir:${ENCODER_ENGINE_PATH},n_mels:128,zero_pad:false,triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},decoupled_mode:${DECOUPLED_MODE} - wget -nc --directory-prefix=${TRITON_REPO}/whisper_bls/1 https://raw.githubusercontent.com/openai/whisper/main/whisper/assets/multilingual.tiktoken - wget -nc --directory-prefix=${TRITON_REPO}/whisper_bls/1 https://raw.githubusercontent.com/openai/whisper/main/whisper/assets/mel_filters.npz - fi - - if [ "$MODEL" = "gpt-disaggregated-serving-bls" ]; then - cp -r ${TRITON_REPO}/tensorrt_llm ${TRITON_REPO}/generation - mv ${TRITON_REPO}/tensorrt_llm ${TRITON_REPO}/context - cp -r all_models/disaggregated_serving/disaggregated_serving_bls/ ${TRITON_REPO} - python3 tools/fill_template.py -i ${TRITON_REPO}/disaggregated_serving_bls/config.pbtxt triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},decoupled_mode:${DECOUPLED_MODE},disaggregated_serving_bls_count:${BLS_INSTANCE_COUNT},context_model_name:context,generation_model_name:generation,logits_datatype:TYPE_FP32 - - mv ${TRITON_REPO}/disaggregated_serving_bls ${TRITON_REPO}/tensorrt_llm - sed 's/name: "disaggregated_serving_bls"/name: "tensorrt_llm"/' -i ${TRITON_REPO}/tensorrt_llm/config.pbtxt - sed 's/name: "tensorrt_llm"/name: "context"/' -i ${TRITON_REPO}/context/config.pbtxt - sed 's/name: "tensorrt_llm"/name: "generation"/' -i ${TRITON_REPO}/generation/config.pbtxt - fi -} - -launch_triton_server () { - - print_test_params - - rm -rf ${TRITON_REPO} - cp -R all_models/inflight_batcher_llm ${TRITON_REPO} - - fill_triton_repo - - # Launch Triton Server - /opt/tritonserver/bin/tritonserver \ - --disable-auto-complete-config --model-repository=${TRITON_REPO} --http-port ${TRITON_HTTP_PORT} --grpc-port ${TRITON_GRPC_PORT} --metrics-port ${TRITON_METRICS_PORT} > log.txt 2>&1 & - export SERVER_PID=$! - - wait_for_server_ready ${SERVER_PID} 1200 ${TRITON_HTTP_PORT} -} - -test_stop_words() { - # test to run for all combinations of flags - EXCL_INPUT_IN_OUTPUT_FLAG="" - [ "${EXCLUDE_INPUT_IN_OUTPUT}" = "true" ] && EXCL_INPUT_IN_OUTPUT_FLAG="--exclude-input-in-output" - - STREAMING_FLAG="" - [ "${STREAMING}" = "true" ] && STREAMING_FLAG="--streaming" - - BEAM_FLAG="--beam-width 1" - [ "${DECODING_MODE}" = "beam_search" ] && BEAM_FLAG="--beam-width 2" - - # Test client - pushd inflight_batcher_llm/client - if [[ $MODEL = "gpt-ib" ]]; then - PROMPT="The only thing we have to fear is" - OUTLEN=10 - - ORIGINAL_OUTPUT=$(python3 end_to_end_grpc_client.py ${STREAMING_FLAG} -o ${OUTLEN} -p "${PROMPT}" 2>&1 | tail -n 1) - echo "original output" - echo $ORIGINAL_OUTPUT - # should be something like "[...] that the government will [...]" - - # examples of stop words that won't affect generation - # "government" isn't tokenized like " government" - # " that the public" doesn't match entirely the generated string - TEST_OUTPUT=$(python3 end_to_end_grpc_client.py ${STREAMING_FLAG} -o ${OUTLEN} -p "${PROMPT}" --stop-words "government" " that the public" 2>&1 | tail -n 1) - [[ "${ORIGINAL_OUTPUT}" == "${TEST_OUTPUT}" ]] - - # check that output finishes at "government" - TEST_OUTPUT=$(python3 end_to_end_grpc_client.py ${STREAMING_FLAG} -o ${OUTLEN} -p "${PROMPT}" --stop-words " lorem" " government" 2>&1 | tail -n 1) - [[ "${TEST_OUTPUT}" == *"government" ]] - TEST_OUTPUT=$(python3 end_to_end_grpc_client.py ${STREAMING_FLAG} -o ${OUTLEN} -p "${PROMPT}" --stop-words " that the government" 2>&1 | tail -n 1) - [[ "${TEST_OUTPUT}" == *"government" ]] - else - PROMPT="What does Jonathan mean?" - OUTLEN=10 - - # Only the BLS backend supports stop word detection on word level. - # The word Jonathan has multiple tokenizations which is not detected in TRT-LLM. - TEST_OUTPUT=$(python3 end_to_end_grpc_client.py ${STREAMING_FLAG} -o ${OUTLEN} -p "${PROMPT}" --stop-words "Jonathan" --model-name "tensorrt_llm_bls" $BEAM_FLAG 2>&1 | tail -n 1) - fi - - popd -} - -run_cpp_trtllm_backend_tests () { - - # test to run for all combinations of flags - EXCL_INPUT_IN_OUTPUT_FLAG="" - [ "${EXCLUDE_INPUT_IN_OUTPUT}" = "true" ] && EXCL_INPUT_IN_OUTPUT_FLAG="--exclude-input-in-output" - - STREAMING_FLAG="" - [ "${STREAMING}" = "true" ] && STREAMING_FLAG="--streaming" - - # Test client - pushd inflight_batcher_llm/client - - if [ $MAX_ATTENTION_WINDOW_SIZE ]; then - # test using a longer input - # TODO: Once we switch to using real weights, add `--check-output` arg - python3 inflight_batcher_llm_client.py \ - ${STREAMING_FLAG} \ - --tokenizer-dir ${TOKENIZER_PATH} \ - --input-tokens-csv='../../tools/dataset/long_input.csv' \ - --output-tokens-csv='../../tools/dataset/long_output.csv' \ - ${EXCL_INPUT_IN_OUTPUT_FLAG} \ - 2>&1 | tee output_long_input - - # If no prompt in output, check that output sequence isn't an empty list of tokens - if $EXCL_INPUT_IN_OUTPUT_FLAG; then - grep -o "Output sequence starts with: \[1, 3189, 28809, 28707, 7234, 574, 3441, 1236, 28723, 28705" output_long_input - else - grep -o "Output sequence\( starts with\)\?:\s*\[\([0-9]*\,\?\s\?\)*\]" output_long_input - fi - fi - - # testing output accuracy for real weights only - CHECK_OUTPUT_FLAG="" - if [ $MODEL = "gpt-ib" ]; then - CHECK_OUTPUT_FLAG="--check-output" - fi - - python3 inflight_batcher_llm_client.py \ - ${STREAMING_FLAG} \ - ${CHECK_OUTPUT_FLAG} \ - ${EXCL_INPUT_IN_OUTPUT_FLAG} \ - --tokenizer-dir ${TOKENIZER_PATH} - - #Check that metrics work as expected by looking at number of successful requests for tensorrt_llm - num_success=$(curl localhost:${TRITON_METRICS_PORT}/metrics 2>&1 | grep nv_inference_request_success\{model=\"tensorrt_llm\" | cut -d " " -f 2) - if (( num_success <= 0 )); then - exit 1 - else - echo "Number of successful requests: $num_success" - fi - - if [[ "$run_all_tests" == "true" && "$BATCHING_STRATEGY" == "inflight_fused_batching" ]]; then - - # testing output accuracy for real weights only - if [[ $MODEL = "gpt-ib" ]] || [[ $MODEL = "mistral-ib-streaming" ]]; then - popd - - test_stop_words - - pushd inflight_batcher_llm/client - fi - - # Stop request - python3 inflight_batcher_llm_client.py \ - ${STREAMING_FLAG} \ - --request-output-len=128 \ - --stop-after-ms 100 \ - --tokenizer-dir ${TOKENIZER_PATH} \ - --request-id 1 \ - 2>&1 | tee output_w_stop - grep "Got cancellation response" output_w_stop - - if [[ "${STREAMING}" == "true" ]]; then - # Request cancellation - python3 inflight_batcher_llm_client.py \ - ${EXCL_INPUT_IN_OUTPUT_FLAG} \ - --streaming \ - --request-output-len=128 \ - --stop-after-ms 100 \ - --request-id 1 \ - --stop-via-request-cancel \ - --tokenizer-dir ${TOKENIZER_PATH} 2>&1 | tee output_w_stop - - grep "Request is cancelled" output_w_stop - fi - - if [[ -n "${1}" && -n "${2}" && -n "${3}" ]]; then - python3 inflight_batcher_llm_client.py \ - ${EXCL_INPUT_IN_OUTPUT_FLAG} \ - ${STREAMING_FLAG} \ - --request-output-len=128 \ - --end-id $3 \ - --request-id 1 \ - --tokenizer-dir ${TOKENIZER_PATH} \ - --input-tokens-csv=$1 \ - --output-tokens-csv=$2 \ - --check-output - fi - - #test with return log probs - python3 inflight_batcher_llm_client.py \ - ${STREAMING_FLAG} \ - --request-output-len=10 \ - --tokenizer-dir ${TOKENIZER_PATH} \ - --return-log-probs --top-k 2 \ - 2>&1 | tee output_log_probs - - #test with string request id - python3 inflight_batcher_llm_client.py \ - ${STREAMING_FLAG} \ - ${CHECK_OUTPUT_FLAG} \ - --tokenizer-dir ${TOKENIZER_PATH} \ - --request-id my_request 2>&1 | tee output_str_request - - # n-return requires the decoupled mode. - if [[ "${DECOUPLED_MODE}" == "True" ]]; then - #test with n returns - python3 inflight_batcher_llm_client.py \ - ${STREAMING_FLAG} \ - ${CHECK_OUTPUT_FLAG} \ - --tokenizer-dir ${TOKENIZER_PATH} \ - --num-return-sequences 2 2>&1 | tee output_n_return - fi - - - # Test triton metrics are present and have non-zero values (when applicable). - TRITON_METRICS_LOG="triton_metrics.out" - curl localhost:${TRITON_METRICS_PORT}/metrics -o ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_request_metrics\{model="tensorrt_llm",request_type="context",version="1"\} [0-9]+$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_request_metrics\{model="tensorrt_llm",request_type="scheduled",version="1"\} [0-9]+$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_request_metrics\{model="tensorrt_llm",request_type="max",version="1"\} [1-9][0-9]*$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_request_metrics\{model="tensorrt_llm",request_type="active",version="1"\} [0-9]+$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_request_metrics\{model="tensorrt_llm",request_type="waiting",version="1"\} [0-9]+$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_runtime_memory_metrics\{memory_type="pinned",model="tensorrt_llm",version="1"\} [1-9][0-9]*$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_runtime_memory_metrics\{memory_type="gpu",model="tensorrt_llm",version="1"\} [1-9][0-9]*$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_runtime_memory_metrics\{memory_type="cpu",model="tensorrt_llm",version="1"\} [1-9][0-9]*$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_kv_cache_block_metrics\{kv_cache_block_type="tokens_per",model="tensorrt_llm",version="1"\} [1-9][0-9]*$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_kv_cache_block_metrics\{kv_cache_block_type="used",model="tensorrt_llm",version="1"\} [0-9]*$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_kv_cache_block_metrics\{kv_cache_block_type="free",model="tensorrt_llm",version="1"\} [1-9][0-9]*$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_kv_cache_block_metrics\{kv_cache_block_type="max",model="tensorrt_llm",version="1"\} [1-9][0-9]*$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_kv_cache_block_metrics\{kv_cache_block_type="fraction",model="tensorrt_llm",version="1"\} [0-9]*\.?[0-9]+$' ${TRITON_METRICS_LOG} - if [[ "${BATCHING_STRATEGY}" == "v1" ]]; then - grep -E 'nv_trt_llm_inflight_batcher_metrics\{model="tensorrt_llm",v1_specific_metric="num_ctx_tokens",version="1"\} [0-9]+$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_inflight_batcher_metrics\{model="tensorrt_llm",v1_specific_metric="num_gen_tokens",version="1"\} [0-9]+$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_inflight_batcher_metrics\{model="tensorrt_llm",v1_specific_metric="empty_gen_slots",version="1"\} [0-9]+$' ${TRITON_METRICS_LOG} - else - grep -E 'nv_trt_llm_inflight_batcher_metrics\{inflight_batcher_specific_metric="paused_requests",model="tensorrt_llm",version="1"\} [0-9]+$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_inflight_batcher_metrics\{inflight_batcher_specific_metric="micro_batch_id",model="tensorrt_llm",version="1"\} [0-9]+$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_inflight_batcher_metrics\{inflight_batcher_specific_metric="generation_requests",model="tensorrt_llm",version="1"\} [0-9]+$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_inflight_batcher_metrics\{inflight_batcher_specific_metric="total_context_tokens",model="tensorrt_llm",version="1"\} [0-9]+$' ${TRITON_METRICS_LOG} - fi - grep -E 'nv_trt_llm_general_metrics\{general_type="iteration_counter",model="tensorrt_llm",version="1"\} [1-9][0-9]*$' ${TRITON_METRICS_LOG} - grep -E 'nv_trt_llm_general_metrics\{general_type="timestamp",model="tensorrt_llm",version="1"\} [1-9][0-9]*$' ${TRITON_METRICS_LOG} - rm ${TRITON_METRICS_LOG} - fi - - popd # inflight_batcher_llm/client - - # End to end test - pushd tools/inflight_batcher_llm - - # HTTP client cannot be used with decoupled mode. - if [[ "${DECOUPLED_MODE}" == "False" ]]; then - python3 benchmark_core_model.py \ - ${EXCL_INPUT_IN_OUTPUT_FLAG} \ - --concurrency 8 \ - -i http \ - --max-input-len 300 \ - dataset \ - --dataset ../dataset/mini_cnn_eval.json \ - --tokenizer-dir ${TOKENIZER_PATH} - fi - - if [[ "$run_all_tests" == "true" ]]; then - # Note: streaming flag is not set to 1 for these benchmarks regardless - # of the value of $STREAMING. - DECOUPLED_FLAG="" - [ "${DECOUPLED_MODE}" = "True" ] && DECOUPLED_FLAG="--decoupled" - - python3 benchmark_core_model.py \ - ${DECOUPLED_FLAG} \ - ${EXCL_INPUT_IN_OUTPUT_FLAG} \ - --concurrency 8 \ - -i grpc \ - --max-input-len 300 \ - --num-requests 80 \ - dataset \ - --dataset ../dataset/mini_cnn_eval.json \ - --tokenizer-dir ${TOKENIZER_PATH} - - # Performance check. - python3 benchmark_core_model.py \ - ${DECOUPLED_FLAG} \ - ${CHECK_PERF_JSON_ARGS} \ - --check-perf-key ${MODEL}-${BACKEND} \ - --check-perf-rtol 0.05 \ - --check-perf-atol 50 \ - --concurrency 8 \ - -i grpc \ - --max-input-len 300 \ - --request-rate -1 \ - --num-requests 1000 \ - token-norm-dist \ - --input-mean 128 --input-stdev 0 \ - --output-mean 20 --output-stdev 0 - - python3 benchmark_core_model.py \ - ${DECOUPLED_FLAG} \ - -i grpc --max-input-len 1000 \ - --request-rate -1 \ - token-from-histogram --histogram-key example - - fi - - - popd # tools/inflight_batcher_llm -} - -run_cpp_e2e_backend_tests () { - - STREAMING_FLAG="" - [ "${STREAMING}" = "true" ] && STREAMING_FLAG="--streaming" - - OVERWRITE_OUTPUT_TEXT_FLAG="" - [ "${ACCUMULATE_TOKEN}" = "true" ] && OVERWRITE_OUTPUT_TEXT_FLAG="--overwrite-output-text" - - EXCL_INPUT_IN_OUTPUT_FLAG="" - [ "${EXCLUDE_INPUT_IN_OUTPUT}" = "true" ] && EXCL_INPUT_IN_OUTPUT_FLAG="--exclude-input-in-output" - - pushd inflight_batcher_llm/client - - # testing output accuracy for real weights only - if [[ $MODEL = "gpt-ib" || $MODEL = "gpt-ib-streaming" ]]; then - - python3 end_to_end_grpc_client.py \ - ${STREAMING_FLAG} \ - --output-len 10 --prompt "The only thing we have to fear is" \ - ${OVERWRITE_OUTPUT_TEXT_FLAG} \ - ${EXCL_INPUT_IN_OUTPUT_FLAG} \ - --model-name "$E2E_MODEL_NAME" | tee output_e2e - grep "that the government will" output_e2e - if [[ "$EXCL_INPUT_IN_OUTPUT_FLAG" != "" ]]; then - grep -v "The only thing we have to fear is" output_e2e - fi - - if [[ "$run_all_tests" == "true" && "$BATCHING_STRATEGY" == "inflight_fused_batching" ]]; then - # test with embedding bias - python3 end_to_end_grpc_client.py \ - ${STREAMING_FLAG} \ - ${OVERWRITE_OUTPUT_TEXT_FLAG} \ - -o 10 \ - -p "The only thing we have to fear is" \ - --embedding-bias-words " government" \ - --embedding-bias-weights -20 \ - --model-name "$E2E_MODEL_NAME" \ - ${EXCL_INPUT_IN_OUTPUT_FLAG} \ - 2>&1 | tee output_w_bias - grep -v "that the government will" output_w_bias - if [[ "$EXCL_INPUT_IN_OUTPUT_FLAG" != "" ]]; then - grep -v "The only thing we have to fear is" output_e2e - fi - - #Only run batched test in streaming for now since it requires decoupled mode - if [[ "$DECOUPLED_MODE" == "true" ]]; then - # test with batched requests - python3 end_to_end_grpc_client.py \ - ${STREAMING_FLAG} \ - ${OVERWRITE_OUTPUT_TEXT_FLAG} \ - ${EXCL_INPUT_IN_OUTPUT_FLAG} \ - -o 5 \ - --model-name "$E2E_MODEL_NAME" \ - -p '["This is a test","I want you to","The cat is"]' \ - --batch-inputs --check-outputs --expected-outputs '[" of the power of the"," know that I am not"," a very good cat."]' - fi - fi - fi - - popd # inflight_batcher_llm/client - - # End to end test - pushd tools/inflight_batcher_llm - # end_to_end_test.py doesn't support streaming - if [[ "${STREAMING}" == "false" ]]; then - python3 end_to_end_test.py \ - --concurrency 8 \ - -i http \ - --max-input-len 200 \ - --test-bls \ - --dataset ../dataset/mini_cnn_eval.json - - if [[ "$run_all_tests" == "true" ]]; then - python3 end_to_end_test.py \ - --concurrency 8 \ - -i grpc \ - --max-input-len 200 \ - --test-bls \ - --dataset ../dataset/mini_cnn_eval.json - fi - fi - - popd # tools/inflight_batcher_llm -} - -run_cpp_trtllm_queue_size_tests () { - # Test client - echo "25229,291,7379,251522,39854,5754,251514,315,32906,14297,398,261" > input.csv - pushd tools/inflight_batcher_llm - EXTRA_FLAGS="" - if [[ "${DECOUPLED_MODE}" == "True" ]]; then - EXTRA_FLAGS="-p grpc -u localhost:8001" - fi - python3 test_max_queue_size.py --input-tokens-csv ../../input.csv --request-output-len 256 --num-requests 100 ${EXTRA_FLAGS} - - popd # tools/inflight_batcher_llm -} - -BACKENDS=( "tensorrtllm" "python" ) -BATCHING_STRATEGIES=( "inflight_fused_batching" ) -MAX_TOKENS_IN_KV_CACHES=( "" $MAX_SEQUENCE_LEN ) -BATCH_SCHEDULER_POLICIES=( "guaranteed_no_evict" "max_utilization" ) -KV_CACHE_FREE_GPU_MEM_FRACTIONS=( "0.2" "" ) -CROSS_KV_CACHE_FRACTION="" -ENABLE_CHUNKED_CONTEXTS=( "false" "true" ) - -BACKEND="tensorrtllm" -TRITON_MAX_BATCH_SIZE="128" -MAX_QUEUE_DELAY_MICROSECONDS="0" -MAX_BEAM_WIDTH="1" -ENABLE_KV_CACHE_REUSE="false" -E2E_MODEL_NAME="ensemble" -TENSORRT_LLM_MODEL_NAME="tensorrt_llm" -TENSORRT_LLM_DRAFT_MODEL_NAME="tensorrt_llm_draft" -TENSORRT_LLM_TARGET_MODEL_NAME="tensorrt_llm_target" -ACCUMULATE_TOKEN="false" -EXCLUDE_INPUT_IN_OUTPUT="false" -BLS_INSTANCE_COUNT="1" -PREPROCESSING_INSTANCE_COUNT="1" -POSTPROCESSING_INSTANCE_COUNT="1" -NORMALIZE_LOG_PROBS="true" -TRITON_HTTP_PORT="8000" -TRITON_GRPC_PORT="8001" -TRITON_METRICS_PORT="8002" -GPU_DEVICE_IDS="" -DECODING_MODE="top_k_top_p" -MAX_QUEUE_SIZE="0" -PROMPT_EMBEDDING_TABLE_DTYPE="TYPE_FP16" - -if [ "$MODEL" = "gpt-ib" ] || [ "$MODEL" = "mistral-ib" ] || [ "$MODEL" = "mistral-ib-mm" ]; then - - # Non-streaming tests, decoupled is false - DECOUPLED_MODE="False" - STREAMING="false" - - # ------------------------------- - # Param sweep test - # ------------------------------- - run_all_tests="true" - for BACKEND in "${BACKENDS[@]}"; do - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - for MAX_TOKENS_IN_KV_CACHE in "${MAX_TOKENS_IN_KV_CACHES[@]}"; do - for BATCH_SCHEDULER_POLICY in "${BATCH_SCHEDULER_POLICIES[@]}"; do - for KV_CACHE_FREE_GPU_MEM_FRACTION in "${KV_CACHE_FREE_GPU_MEM_FRACTIONS[@]}"; do - for ENABLE_CHUNKED_CONTEXT in "${ENABLE_CHUNKED_CONTEXTS[@]}"; do - - # Because the runners are shared, the default value of 0.9 doesn't work, so skip - # if max_tokens_in_kv_cache is also empty - if [[ "${KV_CACHE_FREE_GPU_MEM_FRACTION}" == "" && "${MAX_TOKENS_IN_KV_CACHE}" == "" ]]; then - continue - fi - if [[ "${BATCHING_STRATEGY}" == "v1" && "${BATCH_SCHEDULER_POLICY}" == "max_utilization" ]]; then - continue - fi - # For V1, batchScheduler currently cannot properly estimate kvCache usage - if [[ "${BATCHING_STRATEGY}" == "v1" && "${MAX_TOKENS_IN_KV_CACHE}" != "" ]]; then - continue - fi - # mistral is built without chunked context support - if [[ "$MODEL" = "mistral-ib" && "${ENABLE_CHUNKED_CONTEXT}" == "true" ]]; then - continue - fi - if [[ "$MODEL" = "mistral-ib-mm" && "${ENABLE_CHUNKED_CONTEXT}" == "true" ]]; then - continue - fi - if [[ "$MODEL" = "mistral-ib-mm" ]]; then - export TRTLLM_ORCHESTRATOR=1 - fi - - launch_triton_server - run_cpp_trtllm_backend_tests - run_cpp_e2e_backend_tests - kill_triton_server - run_all_tests="false" - done - done - done - done - done - done - BACKEND="${BACKENDS[0]}" - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - ENABLE_CHUNKED_CONTEXT="${ENABLE_CHUNKED_CONTEXTS[0]}" - - # ------------------------------- - # Exclude input in output test - # ------------------------------- - EXCLUDE_INPUT_IN_OUTPUT="true" - run_all_tests="false" - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - launch_triton_server - run_cpp_trtllm_backend_tests - run_cpp_e2e_backend_tests - kill_triton_server - done - EXCLUDE_INPUT_IN_OUTPUT="false" - - # ------------------------------- - # Max queue delay microseconds - # ------------------------------- - run_all_tests="false" - MAX_QUEUE_DELAY_MICROSECONDS="1000000" - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - launch_triton_server - run_cpp_trtllm_backend_tests - run_cpp_e2e_backend_tests - kill_triton_server - done - MAX_QUEUE_DELAY_MICROSECONDS="0" - - # ------------------------------- - # Max queue size - # ------------------------------- - run_all_tests="false" - MAX_QUEUE_SIZE="6" - TRITON_MAX_BATCH_SIZE="1" - BATCHING_STRATEGY="inflight_fused_batching" - - for BACKEND in "${BACKENDS[@]}"; do - launch_triton_server - run_cpp_trtllm_queue_size_tests - kill_triton_server - done - - MAX_QUEUE_SIZE="0" - TRITON_MAX_BATCH_SIZE="128" - BACKEND="${BACKENDS[0]}" - - # ------------------------------- - # Python BLS - # ------------------------------- - ACCUMULATE_TOKENS=( "false" "true" ) - E2E_MODEL_NAMES=( "ensemble" "tensorrt_llm_bls" ) - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - for E2E_MODEL_NAME in "${E2E_MODEL_NAMES[@]}"; do - for ACCUMULATE_TOKEN in "${ACCUMULATE_TOKENS[@]}"; do - - if [[ "${E2E_MODEL_NAME}" == "ensemble" && "${ACCUMULATE_TOKEN}" == "true" ]]; then - continue - fi - launch_triton_server - run_cpp_e2e_backend_tests - kill_triton_server - done - done - done - E2E_MODEL_NAME="ensemble" - ACCUMULATE_TOKEN="false" -fi - -if [ "$MODEL" = "gpt-ib-streaming" ]; then - - DECOUPLED_MODE="True" - STREAMING="true" - run_all_tests="true" - - for BACKEND in "${BACKENDS[@]}"; do - run_all_tests="true" - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - for MAX_TOKENS_IN_KV_CACHE in "${MAX_TOKENS_IN_KV_CACHES[@]}"; do - for BATCH_SCHEDULER_POLICY in "${BATCH_SCHEDULER_POLICIES[@]}"; do - for KV_CACHE_FREE_GPU_MEM_FRACTION in "${KV_CACHE_FREE_GPU_MEM_FRACTIONS[@]}"; do - for ENABLE_CHUNKED_CONTEXT in "${ENABLE_CHUNKED_CONTEXTS[@]}"; do - - # Because the runners are shared, the default value of 0.9 doesn't work, so skip - # if max_tokens_in_kv_cache is also empty - if [[ "${KV_CACHE_FREE_GPU_MEM_FRACTION}" == "" && "${MAX_TOKENS_IN_KV_CACHE}" == "" ]]; then - continue - fi - if [[ "${BATCHING_STRATEGY}" == "v1" && "${BATCH_SCHEDULER_POLICY}" == "max_utilization" ]]; then - continue - fi - # For V1, batchScheduler currently cannot properly estimate kvCache usage - if [[ "${BATCHING_STRATEGY}" == "v1" && "${MAX_TOKENS_IN_KV_CACHE}" != "" ]]; then - continue - fi - - launch_triton_server - run_cpp_trtllm_backend_tests '../../tools/dataset/short_input_end_id.csv' '../../tools/dataset/short_output_end_id.csv' 268 - run_cpp_e2e_backend_tests - kill_triton_server - - run_all_tests="false" - done - done - done - done - done - done - BACKEND="${BACKENDS[0]}" - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - ENABLE_CHUNKED_CONTEXT="${ENABLE_CHUNKED_CONTEXTS[0]}" - - # -------------------- - # Python BLS test - # -------------------- - ACCUMULATE_TOKENS=( "false" "true" ) - E2E_MODEL_NAMES=( "ensemble" "tensorrt_llm_bls" ) - run_all_tests="true" - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - for E2E_MODEL_NAME in "${E2E_MODEL_NAMES[@]}"; do - for ACCUMULATE_TOKEN in "${ACCUMULATE_TOKENS[@]}"; do - - if [[ "${E2E_MODEL_NAME}" == "ensemble" && "${ACCUMULATE_TOKEN}" == "true" ]]; then - continue - fi - launch_triton_server - run_cpp_e2e_backend_tests - kill_triton_server - done - done - done - E2E_MODEL_NAME="ensemble" - ACCUMULATE_TOKEN="false" - run_all_tests="false" -fi - -if [ "$MODEL" = "mistral-ib-streaming" ]; then - - DECOUPLED_MODE="True" - STREAM=("true" "false") - EXCLUDE_INPUT_IN_OUTPUT_OPTS=("true" "false") - run_all_tests="true" - MAX_BEAM_WIDTH="2" - DECODING_MODES=("top_k_top_p" "beam_search") - - # -------------------- - # Python BLS test - # -------------------- - ACCUMULATE_TOKENS=( "false" ) - E2E_MODEL_NAMES=( "tensorrt_llm_bls" ) - run_all_tests="true" - for BATCHING_STRATEGY in "inflight_fused_batching"; do - for E2E_MODEL_NAME in "${E2E_MODEL_NAMES[@]}"; do - for ACCUMULATE_TOKEN in "${ACCUMULATE_TOKENS[@]}"; do - for STREAMING in "${STREAM[@]}"; do - for EXCLUDE_INPUT_IN_OUTPUT in "${EXCLUDE_INPUT_IN_OUTPUT_OPTS[@]}"; do - for DECODING_MODE in ${DECODING_MODES[@]}; do - if [[ "${E2E_MODEL_NAME}" == "ensemble" && "${ACCUMULATE_TOKEN}" == "true" ]]; then - continue - fi - launch_triton_server - test_stop_words - kill_triton_server - done - done - done - done - done - done - MAX_BEAM_WIDTH="1" - E2E_MODEL_NAME="ensemble" - ACCUMULATE_TOKEN="false" - run_all_tests="false" - DECODING_MODE="top_k_top_p" -fi - -if [ "$MODEL" = "gpt-ib-speculative-decoding-bls" ]; then - # -------------------- - # Python BLS test - # -------------------- - DECOUPLED_MODE="False" - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - USE_DRAFT_LOGITS_VALUES=( "true" "false" ) - - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - for USE_DRAFT_LOGITS in "${USE_DRAFT_LOGITS_VALUES[@]}"; do - - - if [[ "${BATCHING_STRATEGY}" == "v1" ]]; then - continue - fi - draft_args="--num-draft-tokens=5" - if [[ "${USE_DRAFT_LOGITS}" == "true" ]]; then - # with draft logit compare the outputs are not deterministic so we just - draft_args="--num-draft-tokens=5 --return-generation-logits --use-draft-logits --disable-output-comparison" - fi - ENABLE_KV_CACHE_REUSE="true" - launch_triton_server - - # Test client - pushd tools/inflight_batcher_llm - - python3 speculative_decoding_test.py \ - --max-input-len 200 \ - --dataset ../dataset/mini_cnn_eval_spec_decoding.json \ - --url-target=localhost:8001 \ - --url-draft=localhost:8001 \ - --url-control=localhost:8001 \ - --draft-tensorrt-llm-model-name="${TENSORRT_LLM_DRAFT_MODEL_NAME}" \ - --target-tensorrt-llm-model-name="${TENSORRT_LLM_TARGET_MODEL_NAME}" \ - --bls-speculative-tensorrt-llm-model-name="tensorrt_llm_bls" \ - --execute-bls-speculative-decoding \ - ${draft_args} \ - --verbose - - popd # inflight_batcher_llm/client - - kill_triton_server - done - done -fi - -if [ "$MODEL" = "gpt-ib-ptuning" ]; then - - #Generate reference output - pushd $LLM_ROOT/examples/models/core/gpt - - # Input with virtual tokens: - python3 $LLM_ROOT/examples/run.py \ - --max_output_len=8 \ - --vocab_file=c-model/email_composition/fp16/tokenizer.model \ - --prompt_table_path=email_composition.npy \ - --input_file=input.csv \ - --engine_dir ${DECODER_ENGINE_PATH} \ - --output_csv output_w_prompt.csv \ - --enable_context_fmha_fp32_acc \ - --no-kv_cache_enable_block_reuse - - #Input w/o virtual tokens: - echo "25229,291,7379,251522,39854,5754,251514,315,32906,14297,398,261" > input_wo_prompt.csv - python3 $LLM_ROOT/examples/run.py \ - --max_output_len=8 \ - --vocab_file=c-model/email_composition/fp16/tokenizer.model \ - --input_file=input_wo_prompt.csv \ - --engine_dir ${DECODER_ENGINE_PATH} \ - --output_csv output_wo_prompt.csv \ - --enable_context_fmha_fp32_acc \ - --no-kv_cache_enable_block_reuse - - popd - - DECOUPLED_MODE="False" - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - ENABLE_CONTEXT_FMHA_FP32_ACC="True" - - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - if [[ "${BATCHING_STRATEGY}" == "v1" ]]; then - continue - fi - - launch_triton_server - - # Test client - pushd inflight_batcher_llm/client - - python3 inflight_batcher_llm_client.py \ - --prompt-embedding-table $LLM_ROOT/examples/models/core/gpt/email_composition.npy \ - --prompt-task-id 0 \ - --input-tokens-csv $LLM_ROOT/examples/models/core/gpt/input.csv \ - --output-tokens-csv $LLM_ROOT/examples/models/core/gpt/output_w_prompt.csv \ - --check-output \ - --request-output-len 8 - - python3 inflight_batcher_llm_client.py --input-tokens-csv $LLM_ROOT/examples/models/core/gpt/input_wo_prompt.csv --output-tokens-csv $LLM_ROOT/examples/models/core/gpt/output_wo_prompt.csv --check-output --request-output-len 8 - - popd # inflight_batcher_llm/client - - kill_triton_server - done -fi - -if [ "$MODEL" = "gpt-2b-ib-lora" ]; then - - #Generate reference output - pushd $LLM_ROOT/examples/models/core/gpt - - # Input with virtual tokens: - python3 $LLM_ROOT/examples/run.py \ - --max_output_len=8 \ - --lora_dir=gpt2b_lora-900.nemo \ - --lora_ckpt_source nemo \ - --lora_task_uids 0 \ - --engine_dir ${DECODER_ENGINE_PATH} \ - --input_file input.csv \ - --output_csv output.csv \ - --use_py_session \ - --tokenizer_dir ${TOKENIZER_PATH} - - popd - - DECOUPLED_MODE="False" - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - - # LoRA is not supported in V1 - if [[ "${BATCHING_STRATEGY}" == "v1" ]]; then - continue - fi - - launch_triton_server - - # Test client - pushd inflight_batcher_llm/client - - python3 inflight_batcher_llm_client.py \ - --input-tokens-csv $LLM_ROOT/examples/models/core/gpt/input.csv \ - --output-tokens-csv $LLM_ROOT/examples/models/core/gpt/output.csv \ - --check-output --request-output-len 8 \ - --lora-path $LLM_ROOT/examples/models/core/gpt/gpt-2b-lora-train-900 \ - --lora-task-id 12345 - - python3 inflight_batcher_llm_client.py \ - --input-tokens-csv $LLM_ROOT/examples/models/core/gpt/input.csv \ - --output-tokens-csv $LLM_ROOT/examples/models/core/gpt/output.csv \ - --check-output --request-output-len 8 \ - --lora-task-id 12345 - - ACCUMULATE_TOKENS=( "false" "true" ) - E2E_MODEL_NAMES=( "ensemble" "tensorrt_llm_bls" ) - for E2E_MODEL_NAME in "${E2E_MODEL_NAMES[@]}"; do - for ACCUMULATE_TOKEN in "${ACCUMULATE_TOKENS[@]}"; do - - if [[ "${E2E_MODEL_NAME}" == "ensemble" && "${ACCUMULATE_TOKEN}" == "true" ]]; then - continue - fi - python3 end_to_end_grpc_client.py \ - ${STREAMING_FLAG} \ - --output-len 100 --prompt "After Washington had returned to Williamsburg, Dinwiddie ordered him to lead a larger force to assist Trent in his work. While en route, Washington learned of Trent's retreat. Since Tanaghrisson had promised support to the British, Washington continued toward Fort Duquesne and met with the Mingo leader. Learning of a French scouting party in the area, Washington, with Tanaghrisson and his party, surprised the Canadians on May 28 in what became known as the Battle of Jumonville Glen. They killed many of the Canadians, including their commanding officer, Joseph Coulon de Jumonville, whose head was reportedly split open by Tanaghrisson with a tomahawk. The historian Fred Anderson suggests that Tanaghrisson was acting to gain the support of the British and regain authority over his own people. They had been inclined to support the French, with whom they had long trading relationships. One of Tanaghrisson's men told Contrecoeur that Jumonville had been killed by British musket fire. Question: Upon learning of a French scounting party in the area, what did Washington do? Answer:" \ - ${OVERWRITE_OUTPUT_TEXT_FLAG} \ - --lora-path $LLM_ROOT/examples/models/core/gpt/gpt-2b-lora-train-900 \ - --lora-task-id 12345 \ - --model-name "$E2E_MODEL_NAME" | tee "output_e2e_${E2E_MODEL_NAME}_${ACCUMULATE_TOKENS}" - - grep "Answer: He surprised the Canadians on May 28 in what became known as the Battle of Jumonville" "output_e2e_${E2E_MODEL_NAME}_${ACCUMULATE_TOKENS}" - rt=$? - if [ ${rt} -ne 0 ]; then - echo "FAIL" - exit 1 - else - echo "PASS" - fi - done - done - popd # inflight_batcher_llm/client - - #run_cpp_e2e_backend_tests - kill_triton_server - done -fi - -if [ "$MODEL" = "gpt-speculative-decoding" ]; then - - DECOUPLED_MODE="False" - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - - # Speculative decoding is not supported in V1 - if [[ "${BATCHING_STRATEGY}" == "v1" ]]; then - continue - fi - - TRITON_HTTP_PORT="8000" - TRITON_GRPC_PORT="8001" - TRITON_METRICS_PORT="8002" - ENABLE_KV_CACHE_REUSE="true" - launch_triton_server - - TRITON_HTTP_PORT="8003" - TRITON_GRPC_PORT="8004" - TRITON_METRICS_PORT="8005" - # TODO(nkorobov): Draft model can benefit from enable KV cache. - # Add --enable_context_fmha --use_paged_context_fmha to its build command - ENABLE_KV_CACHE_REUSE="false" - launch_triton_server - - # Test client - pushd tools/inflight_batcher_llm - - python3 speculative_decoding_test.py \ - --max-input-len 200 \ - --dataset ../dataset/mini_cnn_eval_spec_decoding.json \ - --url-draft localhost:8004 \ - --url-target localhost:8001 \ - --url-control localhost:8001 \ - --draft-tensorrt-llm-model-name="${TENSORRT_LLM_DRAFT_MODEL_NAME}" \ - --target-tensorrt-llm-model-name="${TENSORRT_LLM_TARGET_MODEL_NAME}" \ - --verbose - - popd # inflight_batcher_llm/client - - kill_triton_server - done -fi - -if [ "$MODEL" = "gpt-disaggregated-serving-bls" ]; then - - DECOUPLED_MODE="False" - STREAMING="false" - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="0.2" - export TRTLLM_USE_MPI_KVCACHE="1" - - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - - # Disaggregated Serving is not supported in v1 batching - if [[ "${BATCHING_STRATEGY}" == "v1" ]]; then - continue - fi - - launch_triton_server - run_cpp_e2e_backend_tests - - kill_triton_server - done - - export TRTLLM_USE_MPI_KVCACHE="0" -fi - -if [ "$MODEL" = "gpt-gather-logits" ]; then - - if [ "${DRAFT_ENGINE_PATH}" == "" ]; then - # normal gather logits test - DECOUPLED_MODE="False" - MAX_NUM_SEQUENCE="${MAX_NUM_SEQUENCES[0]}" - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - ENABLE_TRT_OVERLAP="${ENABLE_TRT_OVERLAPS[0]}" - - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - - launch_triton_server - - # Test client - pushd inflight_batcher_llm/client - - # Kaiyu: nvbugs 4796041 - # python3 inflight_batcher_llm_client.py \ - # --tokenizer-dir ${TOKENIZER_PATH} \ - # --return-context-logits \ - # --return-generation-logits - - python3 inflight_batcher_llm_client.py \ - --tokenizer-dir ${TOKENIZER_PATH} - popd # inflight_batcher_llm/client - - pushd tools/inflight_batcher_llm - # Kaiyu: nvbugs 4796041 - # python3 end_to_end_test.py \ - # -i http \ - # --max-input-len 192 \ - # --return-context-logits \ - # --return-generation-logits \ - # --dataset ../dataset/mini_cnn_eval.json - - python3 end_to_end_test.py \ - -i http \ - --max-input-len 192 \ - --dataset ../dataset/mini_cnn_eval.json - - popd # tools/inflight_batcher_llm - - kill_triton_server - done - - else - # test with speculative decoding - # speculative decoding return draft model draft token logits - # and target model accepted token logits - - DECOUPLED_MODE="False" - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - ENABLE_TRT_OVERLAP="${ENABLE_TRT_OVERLAPS[0]}" - - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - - # Speculative decoding is not supported in V1 - if [[ "${BATCHING_STRATEGY}" == "v1" ]]; then - continue - fi - - TRITON_HTTP_PORT="8000" - TRITON_GRPC_PORT="8001" - TRITON_METRICS_PORT="8002" - ENABLE_KV_CACHE_REUSE="true" - launch_triton_server - - TRITON_HTTP_PORT="8003" - TRITON_GRPC_PORT="8004" - TRITON_METRICS_PORT="8005" - # TODO(nkorobov): Draft model can benefit from enable KV cache. - # Add --enable_context_fmha --use_paged_context_fmha to its build command - ENABLE_KV_CACHE_REUSE="false" - launch_triton_server - - # Test client - pushd tools/inflight_batcher_llm - - python3 speculative_decoding_test.py \ - --max-input-len 128 \ - --dataset ../dataset/mini_cnn_eval_spec_decoding.json \ - --url-draft localhost:8004 \ - --url-target localhost:8001 \ - --url-control localhost:8001 \ - --draft-tensorrt-llm-model-name="${TENSORRT_LLM_DRAFT_MODEL_NAME}" \ - --target-tensorrt-llm-model-name="${TENSORRT_LLM_TARGET_MODEL_NAME}" \ - --num-draft-tokens=5 \ - --return-target-model-accepted-token-logits \ - --return-draft-model-draft-logits \ - --verbose - - popd # inflight_batcher_llm/client - - kill_triton_server - done - fi - -fi - -if [ "$MODEL" = "medusa" ]; then - # To make sure that torch is not a dependency for C++ backend - # pip3 uninstall -y torch - - # Test streaming - DECOUPLED_MODE="True" - STREAMING="true" - run_all_tests="true" - - MAX_NUM_SEQUENCE="${MAX_NUM_SEQUENCES[0]}" - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - BATCHING_STRATEGY="${BATCHING_STRATEGIES[0]}" - DECODING_MODE="medusa" - - END_ID_MEDUSA=1284 - MEDUSA_INPUT_IDS_PATH='../../tools/dataset/short_input_end_id_medusa.csv' - MEDUSA_OUTPUT_IDS_PATH='../../tools/dataset/short_output_end_id_medusa.csv' - - launch_triton_server - run_cpp_trtllm_backend_tests ${MEDUSA_INPUT_IDS_PATH} ${MEDUSA_OUTPUT_IDS_PATH} ${END_ID_MEDUSA} - kill_triton_server - # FIXME: grpc e2e test returns different result (because it is Medusa and not GPT) and has some problems with spaces - - # Test non-streaming - DECOUPLED_MODE="False" - launch_triton_server - # Test client - pushd inflight_batcher_llm/client - python3 inflight_batcher_llm_client.py \ - --request-output-len=128 \ - --end-id ${END_ID_MEDUSA} \ - --request-id 1 \ - --tokenizer-dir ${TOKENIZER_PATH} \ - --input-tokens-csv ${MEDUSA_INPUT_IDS_PATH} \ - --output-tokens-csv ${MEDUSA_OUTPUT_IDS_PATH} \ - --check-output - popd # inflight_batcher_llm/client - kill_triton_server -fi - -if [ "$MODEL" = "eagle" ]; then - # To make sure that torch is not a dependency for C++ backend - # pip3 uninstall -y torch - - # Test streaming - DECOUPLED_MODE="True" - STREAMING="true" - run_all_tests="true" - - MAX_NUM_SEQUENCE="${MAX_NUM_SEQUENCES[0]}" - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - BATCHING_STRATEGY="${BATCHING_STRATEGIES[0]}" - # chunked context is not supported yet. - ENABLE_CHUNKED_CONTEXT="false" - DECODING_MODE="eagle" - - END_ID_EAGLE=1284 - # Use the same I/O files as eagle is based on the same vicuna-v1.3-7b as medusa. - EAGLE_INPUT_IDS_PATH='../../tools/dataset/short_input_end_id_medusa.csv' - EAGLE_OUTPUT_IDS_PATH='../../tools/dataset/short_output_end_id_eagle.csv' - - for BACKEND in "${BACKENDS[@]}"; do - launch_triton_server - run_cpp_trtllm_backend_tests ${EAGLE_INPUT_IDS_PATH} ${EAGLE_OUTPUT_IDS_PATH} ${END_ID_EAGLE} - kill_triton_server - done - # FIXME: grpc e2e test returns different result (because it is Eagle and not GPT) and has some problems with spaces - - # Test non-streaming - DECOUPLED_MODE="False" - launch_triton_server - # Test client - pushd inflight_batcher_llm/client - python3 inflight_batcher_llm_client.py \ - --request-output-len=128 \ - --end-id ${END_ID_EAGLE} \ - --request-id 1 \ - --tokenizer-dir ${TOKENIZER_PATH} \ - --input-tokens-csv ${EAGLE_INPUT_IDS_PATH} \ - --output-tokens-csv ${EAGLE_OUTPUT_IDS_PATH} \ - --check-output - popd # inflight_batcher_llm/client - kill_triton_server -fi - -if [ "$MODEL" = "bart-ib" ] || [ "$MODEL" = "t5-ib" ]; then - - # Non-streaming tests, decoupled is false - DECOUPLED_MODE="False" - STREAMING="false" - - # enc-dec models only support inflight_fused_batching, with chunked context disabled - CHECK_PERF_JSON_ARGS="" - BATCHING_STRATEGY="inflight_fused_batching" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - ENABLE_CHUNKED_CONTEXT="false" - CROSS_KV_CACHE_FRACTION="0.5" - - # ------------------------------- - # Param sweep test - # ------------------------------- - run_all_tests="true" - for BACKEND in "${BACKENDS[@]}"; do - for MAX_TOKENS_IN_KV_CACHE in "${MAX_TOKENS_IN_KV_CACHES[@]}"; do - for KV_CACHE_FREE_GPU_MEM_FRACTION in "${KV_CACHE_FREE_GPU_MEM_FRACTIONS[@]}"; do - # Because the runners are shared, the default value of 0.9 doesn't work, so skip - # if max_tokens_in_kv_cache is also empty - if [[ "${KV_CACHE_FREE_GPU_MEM_FRACTION}" == "" && "${MAX_TOKENS_IN_KV_CACHE}" == "" ]]; then - continue - fi - - #Encoder-decoder models are not yet supported in python backend - if [[ "${BACKEND}" == "python" ]]; then - continue - fi - - launch_triton_server - run_cpp_trtllm_backend_tests - run_cpp_e2e_backend_tests - kill_triton_server - run_all_tests="false" - done - done - done - BACKEND="${BACKENDS[0]}" - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - - # ------------------------------- - # Exclude input in output test - # ------------------------------- - EXCLUDE_INPUT_IN_OUTPUT="true" - run_all_tests="false" - launch_triton_server - run_cpp_trtllm_backend_tests - run_cpp_e2e_backend_tests - kill_triton_server - EXCLUDE_INPUT_IN_OUTPUT="false" - - # ------------------------------- - # Max queue delay microseconds - # ------------------------------- - run_all_tests="false" - MAX_QUEUE_DELAY_MICROSECONDS="1000000" - launch_triton_server - run_cpp_trtllm_backend_tests - run_cpp_e2e_backend_tests - kill_triton_server - MAX_QUEUE_DELAY_MICROSECONDS="0" - - # ------------------------------- - # Python BLS - # ------------------------------- - - ACCUMULATE_TOKENS=( "false" "true" ) - E2E_MODEL_NAMES=( "ensemble" "tensorrt_llm_bls" ) - for E2E_MODEL_NAME in "${E2E_MODEL_NAMES[@]}"; do - for ACCUMULATE_TOKEN in "${ACCUMULATE_TOKENS[@]}"; do - - if [[ "${E2E_MODEL_NAME}" == "ensemble" && "${ACCUMULATE_TOKEN}" == "true" ]]; then - continue - fi - launch_triton_server - run_cpp_e2e_backend_tests - kill_triton_server - done - done - E2E_MODEL_NAME="ensemble" - ACCUMULATE_TOKEN="false" - - # Reset - CROSS_KV_CACHE_FRACTION="" -fi - -if [ "$MODEL" = "blip2-opt" ]; then - - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - - # Test none-streaming - DECOUPLED_MODE="False" - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - launch_triton_server - python3 tools/multimodal/client.py --model_type blip2 | tee multimodal_output - grep -oi "singapore" multimodal_output - kill_triton_server - done - - # Test streaming - DECOUPLED_MODE="True" - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - launch_triton_server - python3 tools/multimodal/client.py --model_type blip2 --streaming | tee multimodal_output - grep -oi "sing" multimodal_output - kill_triton_server - done - DECOUPLED_MODE="False" - - # Python BLS - DECOUPLED_MODE="True" - ACCUMULATE_TOKENS=( "false" "true" ) - E2E_MODEL_NAMES=( "ensemble" "tensorrt_llm_bls" ) - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - for E2E_MODEL_NAME in "${E2E_MODEL_NAMES[@]}"; do - for ACCUMULATE_TOKEN in "${ACCUMULATE_TOKENS[@]}"; do - - if [[ "${E2E_MODEL_NAME}" == "ensemble" && "${ACCUMULATE_TOKEN}" == "true" ]]; then - continue - fi - launch_triton_server - python3 tools/multimodal/client.py --model_type blip2 --use_bls --streaming | tee multimodal_output - grep -oi "sing" multimodal_output - kill_triton_server - done - done - done - E2E_MODEL_NAME="ensemble" - ACCUMULATE_TOKEN="false" - DECOUPLED_MODE="False" - - # Test kv cache reuse - ENABLE_KV_CACHE_REUSE="True" - for BATCHING_STRATEGY in "${BATCHING_STRATEGIES[@]}"; do - launch_triton_server - python3 tools/multimodal/client.py --text "Question: Can you identify which city is depicted in this image based on the landmarks, architecture, and overall scenery? Please provide the name of the city along with any notable features that led you to your conclusion. Answer:" --model_type blip2 --prompt_table_extra_id 1 - python3 tools/multimodal/client.py --text "Question: Can you identify which city is depicted in this image based on the landmarks, architecture, and overall scenery? Please provide the name of the city along with any notable features that led you to your conclusion. Answer:" --model_type blip2 --prompt_table_extra_id 1 | tee multimodal_output - grep -oi "singapore" multimodal_output - kill_triton_server - done - ENABLE_KV_CACHE_REUSE="False" -fi - -if [ "$MODEL" = "mllama" ]; then - - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - CROSS_KV_CACHE_FRACTION="0.5" - BATCHING_STRATEGY="inflight_fused_batching" - - # Test none-streaming - DECOUPLED_MODE="False" - for BACKEND in "${BACKENDS[@]}"; do - if [[ "${BACKEND}" == "python" ]]; then - continue - fi - launch_triton_server - python3 tools/multimodal/client.py --model_type mllama | tee multimodal_output - grep -oi "singapore" multimodal_output - kill_triton_server - done - - # Test streaming - DECOUPLED_MODE="True" - for BACKEND in "${BACKENDS[@]}"; do - launch_triton_server - python3 tools/multimodal/client.py --model_type mllama --streaming | tee multimodal_output - grep -oi "singapore" multimodal_output - kill_triton_server - done - DECOUPLED_MODE="False" - CROSS_KV_CACHE_FRACTION="" -fi - - -if [ "$MODEL" = "whisper" ]; then - - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[1]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - # enc-dec models only support inflight_fused_batching, with chunked context disabled - BATCHING_STRATEGY="inflight_fused_batching" - ENABLE_CHUNKED_CONTEXT="false" - EXCLUDE_INPUT_IN_OUTPUT="true" - CROSS_KV_CACHE_FRACTION="0.5" - wget -nc https://raw.githubusercontent.com/yuekaizhang/Triton-ASR-Client/main/datasets/mini_en/wav/1221-135766-0002.wav - # Test none-streaming - DECOUPLED_MODE="False" - pip install tiktoken soundfile - launch_triton_server - python3 tools/whisper/client.py --audio-path 1221-135766-0002.wav - kill_triton_server - - # Test streaming - DECOUPLED_MODE="True" - launch_triton_server - python3 tools/whisper/client.py --audio-path 1221-135766-0002.wav --streaming - kill_triton_server - - EXCLUDE_INPUT_IN_OUTPUT="false" - DECOUPLED_MODE="False" - CROSS_KV_CACHE_FRACTION="" -fi - -if [ "$MODEL" = "llava_onevision" ]; then - - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - - # Test none-streaming - DECOUPLED_MODE="False" - BATCHING_STRATEGY="inflight_fused_batching" - launch_triton_server - python3 tools/multimodal/client.py --model_type llava_onevision --end-id 151645 --pad-id 151643 | tee multimodal_output - grep -oi "singapore" multimodal_output - kill_triton_server - - # Test streaming - DECOUPLED_MODE="True" - BATCHING_STRATEGY="inflight_fused_batching" - launch_triton_server - python3 tools/multimodal/client.py --model_type llava_onevision --streaming --end-id 151645 --pad-id 151643 | tee multimodal_output - grep -oi "sing" multimodal_output - kill_triton_server - - # Test with video input - DECOUPLED_MODE="False" - BATCHING_STRATEGY="inflight_fused_batching" - launch_triton_server - VIDEO_PATH=$TOKENIZER_PATH'/../video-neva/test_video/video_test.mp4' - python3 tools/multimodal/client.py --model_type llava_onevision --end-id 151645 --pad-id 151643 --text "What is in this video?" --video $VIDEO_PATH --video_num_frames 8 | tee multimodal_output - grep -oi "robotic hand" multimodal_output - kill_triton_server - -fi - -if [ "$MODEL" = "qwen2_vl" ]; then - - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - - # Test none-streaming - DECOUPLED_MODE="False" - BATCHING_STRATEGY="inflight_fused_batching" - launch_triton_server - python3 tools/multimodal/client.py --model_type qwen2_vl --end-id 151645 --pad-id 151643 | tee multimodal_output - grep -oi "Singapore" multimodal_output - kill_triton_server - - # Test streaming - DECOUPLED_MODE="True" - BATCHING_STRATEGY="inflight_fused_batching" - launch_triton_server - python3 tools/multimodal/client.py --model_type qwen2_vl --streaming --end-id 151645 --pad-id 151643 | tee multimodal_output - grep -oi "Singapore" multimodal_output - kill_triton_server - -fi - -if [ "$MODEL" = "llava" ]; then - - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[1]}" - ENABLE_CHUNKED_CONTEXT="${ENABLE_CHUNKED_CONTEXTS[1]}" - echo "ENABLE_CHUNKED_CONTEXT: ${ENABLE_CHUNKED_CONTEXT}" - - # Test none-streaming - DECOUPLED_MODE="False" - BATCHING_STRATEGY="inflight_fused_batching" - launch_triton_server - python3 tools/multimodal/client.py --model_type llava --end-id 2 --pad-id 32001 | tee multimodal_output - grep -oi "Singapore" multimodal_output - kill_triton_server - - # Test streaming - DECOUPLED_MODE="True" - BATCHING_STRATEGY="inflight_fused_batching" - launch_triton_server - python3 tools/multimodal/client.py --model_type llava --streaming --end-id 2 --pad-id 32001 | tee multimodal_output - grep -oi "Singapore" multimodal_output - kill_triton_server - -fi - -if [ "$MODEL" = "llava_fp8" ]; then - - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[1]}" - ENABLE_CHUNKED_CONTEXT="${ENABLE_CHUNKED_CONTEXTS[1]}" - echo "ENABLE_CHUNKED_CONTEXT: ${ENABLE_CHUNKED_CONTEXT}" - - # Test none-streaming - DECOUPLED_MODE="False" - BATCHING_STRATEGY="inflight_fused_batching" - launch_triton_server - python3 tools/multimodal/client.py --model_type llava --end-id 2 --pad-id 32001 | tee multimodal_output - grep -oi "Singapore" multimodal_output - kill_triton_server - - # Test streaming - DECOUPLED_MODE="True" - BATCHING_STRATEGY="inflight_fused_batching" - launch_triton_server - python3 tools/multimodal/client.py --model_type llava --streaming --end-id 2 --pad-id 32001 | tee multimodal_output - grep -oi "Singapore" multimodal_output - kill_triton_server - -fi - -if [ "$MODEL" = "gpt-ib-lad" ]; then - # Test streaming - DECOUPLED_MODE="False" - STREAMING="true" - run_all_tests="true" - - MAX_NUM_SEQUENCE="${MAX_NUM_SEQUENCES[0]}" - MAX_TOKENS_IN_KV_CACHE="${MAX_TOKENS_IN_KV_CACHES[0]}" - BATCH_SCHEDULER_POLICY="${BATCH_SCHEDULER_POLICIES[0]}" - KV_CACHE_FREE_GPU_MEM_FRACTION="${KV_CACHE_FREE_GPU_MEM_FRACTIONS[0]}" - BATCHING_STRATEGY="${BATCHING_STRATEGIES[0]}" - DECODING_MODE="lookahead" - - # Lookahead parameters - LOOKAHEAD_WINDOW_SIZE=7 - LOOKAHEAD_NGRAM_SIZE=7 - LOOKAHEAD_VERIFICATION_SET_SIZE=7 - - LOOKAHEAD_CONFIG="--lookahead_config=[${LOOKAHEAD_WINDOW_SIZE},${LOOKAHEAD_NGRAM_SIZE},${LOOKAHEAD_VERIFICATION_SET_SIZE}]" - - launch_triton_server - # Test client - pushd inflight_batcher_llm/client - python3 inflight_batcher_llm_client.py \ - ${LOOKAHEAD_CONFIG} \ - --tokenizer-dir ${TOKENIZER_PATH} - popd # inflight_batcher_llm/client - kill_triton_server - -fi - -popd # $LLM_BACKEND_ROOT diff --git a/tests/integration/defs/triton_server/test_triton.py b/tests/integration/defs/triton_server/test_triton.py deleted file mode 100644 index 6d710ad5ba1d..000000000000 --- a/tests/integration/defs/triton_server/test_triton.py +++ /dev/null @@ -1,501 +0,0 @@ -import os -import subprocess - -import pytest - - -def run_shell_command(command, llm_root): - env = os.environ.copy() - env["LLM_ROOT"] = llm_root - env["LLM_BACKEND_ROOT"] = os.path.join(llm_root, "triton_backend") - env["NVIDIA_TRITON_SERVER_VERSION"] = os.environ.get( - "NVIDIA_TRITON_SERVER_VERSION", "25.03") - subprocess.run(command, env=env, check=True, shell=True) - - -def build_model(model_name, llm_root, tritonserver_test_root): - """Build the model required for the test.""" - env = os.environ.copy() - env["LLM_ROOT"] = llm_root - env["LLM_BACKEND_ROOT"] = os.path.join(llm_root, "triton_backend") - env["LLM_MODELS_ROOT"] = os.environ.get("LLM_MODELS_ROOT", - "/scratch.trt_llm_data/llm-models") - subprocess.run(f"bash {tritonserver_test_root}/build_model.sh {model_name}", - env=env, - check=True, - shell=True) - - -@pytest.fixture -def test_name(request): - return request.param - - -@pytest.fixture -def model_path(test_name): - """Returns the appropriate model path based on the test name.""" - model_mapping = { - "gpt": "gpt2", - "opt": "opt-125m", - "mistral": "mistral-7b-v0.1", - "mistral-ib": "mistral-7b-v0.1", - "mistral-ib-streaming": "mistral-7b-v0.1", - "mistral-ib-mm": "mistral-7b-v0.1", - "gptj": "gpt-j-6b", - "gpt-ib": "gpt2", - "gpt-ib-streaming": "gpt2", - "gpt-ib-ptuning": "gpt2", - "gpt-ib-lad": "gpt2", - "gpt-speculative-decoding": "gpt2", - "gpt-ib-speculative-decoding-bls": "gpt2", - "gpt-2b-ib-lora": "gpt-next/gpt-next-tokenizer-hf-v2", - "gpt-gather-logits": "gpt2", - "medusa": "vicuna-7b-v1.3", - "eagle": "vicuna-7b-v1.3", - "bart-ib": "bart-large-cnn", - "t5-ib": "t5-small", - "blip2-opt": "blip2-opt-2.7b", - "mllama": "llama-3.2-models/Llama-3.2-11B-Vision-Instruct", - "whisper": "whisper-large-v3", - "gpt-disaggregated-serving-bls": "gpt2", - "llava_onevision": "llava-onevision-qwen2-7b-ov-hf", - "qwen2_vl": "Qwen2-VL-7B-Instruct", - "llava": "llava-1.5-7b-hf", - "llava_fp8": "llava-1.5-7b-hf" - } - model_cache_root = os.environ.get("LLM_MODELS_ROOT", - "/scratch.trt_llm_data/llm-models") - return os.path.join(model_cache_root, model_mapping.get(test_name, "")) - - -@pytest.fixture -def engine_dir(test_name, llm_root): - """Returns the appropriate engine directory based on the test name.""" - engine_mapping = { - "gpt": "models/core/gpt/trt_engine/gpt2/fp16/1-gpu/", - "opt": "models/contrib/opt/trt_engine/opt-125m/fp16/1-gpu/", - "mistral": "models/core/llama/mistral_7b_outputs", - "mistral-ib": "models/core/llama/ib_mistral_7b_outputs", - "mistral-ib-streaming": "models/core/llama/ib_mistral_7b_outputs", - "mistral-ib-mm": "models/core/llama/ib_mistral_7b_outputs", - "gptj": "models/contrib/gptj/gptj_outputs", - "gpt-ib": "models/core/gpt/trt_engine/gpt2-ib/fp16/1-gpu/", - "gpt-ib-streaming": "models/core/gpt/trt_engine/gpt2-ib/fp16/1-gpu/", - "gpt-ib-ptuning": - "models/core/gpt/trt_engine/email_composition/fp16/1-gpu/", - "gpt-ib-lad": "models/core/gpt/trt_engine/gpt2-ib-lad/fp16/1-gpu/", - "gpt-2b-ib-lora": - "models/core/gpt/trt_engine/gpt-2b-lora-ib/fp16/1-gpu/", - "medusa": "medusa/tmp/medusa/7B/trt_engines/fp16/1-gpu/", - "eagle": "eagle/tmp/eagle/7B/trt_engines/fp16/1-gpu/", - "bart-ib": "models/core/enc_dec/trt_engine/bart-ib/fp16/1-gpu/", - "t5-ib": "models/core/enc_dec/trt_engine/t5-ib/fp16/1-gpu/", - "blip2-opt": "models/core/multimodal/trt_engines/opt-2.7b/fp16/1-gpu", - "mllama": - "models/core/multimodal/trt_engines/Llama-3.2-11B-Vision-Instruct/bf16/1-gpu", - "whisper": "models/core/whisper/trt_engine/whisper", - "gpt-disaggregated-serving-bls": - "models/core/gpt/trt_engine/gpt2/fp16/1-gpu/", - "llava_onevision": - "models/core/multimodal/trt_engines/llava-onevision-7b/fp16/1-gpu", - "qwen2_vl": "models/core/multimodal/trt_engines/qwen2-vl-7b/fp16/1-gpu", - "llava": - "models/core/multimodal/trt_engines/llava-1.5-7b-hf/fp16/1-gpu", - "llava_fp8": - "models/core/multimodal/trt_engines/llava-1.5-7b-hf/fp8/1-gpu" - } - return os.path.join(llm_root, "examples/", - engine_mapping.get(test_name, "")) - - -@pytest.mark.parametrize("test_name", ["gpt"], indirect=True) -def test_gpt(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - # Build the model - build_model(test_name, llm_root, tritonserver_test_root) - - # Run the test - tokenizer_type = "auto" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["mistral"], indirect=True) -def test_mistral(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "llama" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["gptj"], indirect=True) -def test_gptj(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "auto" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["mistral-ib"], indirect=True) -def test_mistral_ib(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "llama" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["mistral-ib-streaming"], indirect=True) -def test_mistral_ib_streaming(tritonserver_test_root, test_name, llm_root, - model_path, engine_dir): - build_model("mistral-ib", llm_root, tritonserver_test_root) - tokenizer_type = "llama" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["mistral-ib-mm"], indirect=True) -def test_mistral_ib_mm(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model("mistral-ib", llm_root, tritonserver_test_root) - tokenizer_type = "llama" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["gpt-ib"], indirect=True) -def test_gpt_ib(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "auto" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["gpt-ib-streaming"], indirect=True) -def test_gpt_ib_streaming(tritonserver_test_root, test_name, llm_root, - model_path, engine_dir): - build_model("gpt-ib", llm_root, tritonserver_test_root) - tokenizer_type = "auto" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["gpt-ib-ptuning"], indirect=True) -def test_gpt_ib_ptuning(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model("gpt-ib-ptuning", llm_root, tritonserver_test_root) - tokenizer_type = "auto" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["gpt-ib-lad"], indirect=True) -def test_gpt_ib_lad(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "auto" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["gpt-2b-ib-lora"], indirect=True) -def test_gpt_2b_ib_lora(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "auto" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["gpt-ib-speculative-decoding-bls"], - indirect=True) -def test_gpt_ib_speculative_decoding_bls(tritonserver_test_root, test_name, - llm_root, model_path): - # Build the draft model - build_model("gpt-ib", llm_root, tritonserver_test_root) - # Build the control & target model - build_model("gpt-medium-ib", llm_root, tritonserver_test_root) - - tokenizer_type = "auto" - - draft_engine_path = f"{llm_root}/examples/models/core/gpt/trt_engine/gpt2-ib/fp16/1-gpu/" - control_engine_path = f"{llm_root}/examples/models/core/gpt/trt_engine/gpt2-medium-ib/fp16/1-gpu/" - target_engine_path = f"{llm_root}/examples/models/core/gpt/trt_engine/gpt2-medium-ib-target/fp16/1-gpu/" - - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh gpt-ib-speculative-decoding-bls {control_engine_path} " - f"{model_path} {tokenizer_type} {draft_engine_path} {target_engine_path}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["gpt-speculative-decoding"], - indirect=True) -def test_gpt_speculative_decoding(tritonserver_test_root, test_name, llm_root, - model_path): - # Build the draft model - build_model("gpt-ib", llm_root, tritonserver_test_root) - # Build the control & target model - build_model("gpt-medium-ib", llm_root, tritonserver_test_root) - - tokenizer_type = "auto" - - draft_engine_path = f"{llm_root}/examples/models/core/gpt/trt_engine/gpt2-ib/fp16/1-gpu/" - control_engine_path = f"{llm_root}/examples/models/core/gpt/trt_engine/gpt2-medium-ib/fp16/1-gpu/" - target_engine_path = f"{llm_root}/examples/models/core/gpt/trt_engine/gpt2-medium-ib-target/fp16/1-gpu/" - - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh gpt-speculative-decoding {control_engine_path} " - f"{model_path} {tokenizer_type} {draft_engine_path} {target_engine_path}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["gpt-gather-logits"], indirect=True) -def test_gpt_gather_logits(tritonserver_test_root, test_name, llm_root, - model_path): - # Standard gather logits test - build_model("gpt-gather-logits", llm_root, tritonserver_test_root) - - tokenizer_type = "auto" - engine_path = f"{llm_root}/examples/models/core/gpt/trt_engine/gpt2-gather-logits/fp16/1-gpu/" - - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh gpt-gather-logits {engine_path} {model_path} {tokenizer_type}", - llm_root) - - # Speculative decoding return draft model draft token logits test - build_model("gpt-gather-generation-logits", llm_root, - tritonserver_test_root) - build_model("gpt-medium-ib", llm_root, tritonserver_test_root) - - draft_engine_path = f"{llm_root}/examples/models/core/gpt/trt_engine/gpt2-draft-gather-generation-logits/fp16/1-gpu/" - control_engine_path = f"{llm_root}/examples/models/core/gpt/trt_engine/gpt2-medium-ib/fp16/1-gpu/" - target_engine_path = f"{llm_root}/examples/models/core/gpt/trt_engine/gpt2-medium-ib-target/fp16/1-gpu/" - - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh gpt-gather-logits {control_engine_path} " - f"{model_path} {tokenizer_type} {draft_engine_path} {target_engine_path}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["medusa"], indirect=True) -def test_medusa(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "llama" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["eagle"], indirect=True) -def test_eagle(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "llama" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["whisper"], indirect=True) -def test_whisper(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "auto" - decoder_path = f"{engine_dir}/decoder" - encoder_path = f"{engine_dir}/encoder" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {decoder_path} {model_path} {tokenizer_type} skip skip {encoder_path}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["t5-ib"], indirect=True) -def test_t5_ib(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "auto" - decoder_path = f"{engine_dir}/decoder" - encoder_path = f"{engine_dir}/encoder" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {decoder_path} {model_path} {tokenizer_type} skip skip {encoder_path}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["bart-ib"], indirect=True) -def test_bart_ib(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "auto" - decoder_path = f"{engine_dir}/decoder" - encoder_path = f"{engine_dir}/encoder" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {decoder_path} {model_path} {tokenizer_type} skip skip {encoder_path}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["blip2-opt"], indirect=True) -def test_blip2_opt(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "auto" - visual_encoder_path = f"{llm_root}/examples/models/core/multimodal/tmp/trt_engines/blip2-opt-2.7b/multimodal_encoder/" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type} skip skip skip {visual_encoder_path}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["mllama"], indirect=True) -def test_mllama(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "auto" - visual_encoder_path = f"{llm_root}/examples/models/core/multimodal/tmp/trt_engines/Llama-3.2-11B-Vision-Instruct/multimodal_encoder/" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type} skip skip skip {visual_encoder_path}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["gpt-disaggregated-serving-bls"], - indirect=True) -def test_gpt_disaggregated_serving_bls(tritonserver_test_root, test_name, - llm_root, model_path, engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "auto" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["llava"], indirect=True) -def test_llava(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "auto" - visual_encoder_path = f"{llm_root}/examples/models/core/multimodal/tmp/trt_engines/llava-1.5-7b-hf/multimodal_encoder/" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type} skip skip skip {visual_encoder_path}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["llava_fp8"], indirect=True) -def test_llava_fp8(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "auto" - visual_encoder_path = f"{llm_root}/examples/models/core/multimodal/tmp/trt_engines/llava-1.5-7b-hf/multimodal_encoder/" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type} skip skip skip {visual_encoder_path}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["llava_onevision"], indirect=True) -def test_llava_onevision(tritonserver_test_root, test_name, llm_root, - model_path, engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "auto" - visual_encoder_path = f"{llm_root}/examples/models/core/multimodal/tmp/trt_engines/llava-onevision-qwen2-7b-ov-hf/multimodal_encoder/" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type} skip skip skip {visual_encoder_path}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["qwen2_vl"], indirect=True) -def test_qwen2_vl(tritonserver_test_root, test_name, llm_root, model_path, - engine_dir): - build_model(test_name, llm_root, tritonserver_test_root) - tokenizer_type = "auto" - visual_encoder_path = f"{llm_root}/examples/models/core/multimodal/tmp/trt_engines/Qwen2-VL-7B-Instruct/multimodal_encoder/" - run_shell_command( - f"cd {tritonserver_test_root} && ./test.sh {test_name} {engine_dir} {model_path} {tokenizer_type} skip skip skip {visual_encoder_path}", - llm_root) - - -@pytest.mark.parametrize("test_name", ["python-bls-unit-tests"], indirect=True) -def test_python_bls_unit_tests(tritonserver_test_root, test_name, llm_root): - run_shell_command( - f"cd {llm_root}/triton_backend && PYTHONPATH=all_models/inflight_batcher_llm/tensorrt_llm_bls/1 " - "python3 -m pytest all_models/tests/test_*decode*.py", llm_root) - run_shell_command( - f"cd {llm_root}/triton_backend && PYTHONPATH=all_models/inflight_batcher_llm/tensorrt_llm/1 " - "python3 -m pytest all_models/tests/test_python_backend.py", llm_root) - - -@pytest.mark.parametrize("test_name", ["python-preproc-unit-tests"], - indirect=True) -def test_python_preproc_unit_tests(tritonserver_test_root, test_name, llm_root): - run_shell_command( - f"cd {llm_root}/triton_backend && PYTHONPATH=all_models/inflight_batcher_llm/preprocessing/1 " - "python3 -m pytest all_models/tests/test_multi_image_preprocess.py", - llm_root) - - -@pytest.mark.parametrize("test_name", ["python-multimodal-encoders-unit-tests"], - indirect=True) -def test_python_multimodal_encoders_unit_tests(tritonserver_test_root, - test_name, llm_root): - run_shell_command( - f"cd {llm_root}/triton_backend && PYTHONPATH=all_models/multimodal/multimodal_encoders/1 " - "python3 -m pytest all_models/tests/test_multimodal_encoders.py", - llm_root) - - -@pytest.mark.parametrize("test_name", ["fill-template"], indirect=True) -def test_fill_template(tritonserver_test_root, test_name, llm_root): - run_shell_command( - f"cd {llm_root}/triton_backend && PYTHONPATH=tools/ python3 -m pytest tools/tests/test_fill_template.py", - llm_root) - - -@pytest.mark.parametrize("test_name", ["llmapi-unit-tests"], indirect=True) -def test_llmapi_unit_tests(tritonserver_test_root, test_name, llm_root): - run_shell_command( - f"cd {llm_root}/triton_backend && PYTHONPATH=all_models/llmapi/tensorrt_llm/1 " - "python3 -m pytest all_models/tests/test_llmapi_python_backend.py", - llm_root) - - -@pytest.mark.parametrize("test_name", ["cpp-unit-tests"], indirect=True) -def test_cpp_unit_tests(tritonserver_test_root, test_name, llm_root): - # Build the inflight_batcher_llm - run_shell_command( - f"cd {llm_root}/triton_backend/inflight_batcher_llm && " - "rm -rf build && " - "mkdir -p build", llm_root) - - # Get the value of TRITON_SHORT_TAG from docker/Dockerfile.multi - import subprocess - triton_short_tag = subprocess.check_output( - [f"{llm_root}/jenkins/scripts/get_triton_tag.sh", llm_root], - text=True).strip() - print(f"using triton tag from docker/Dockerfile.multi: {triton_short_tag}") - run_shell_command( - f"cd {llm_root}/triton_backend/inflight_batcher_llm/build && " - f"cmake .. -DTRTLLM_DIR={llm_root} -DCMAKE_INSTALL_PREFIX=install/ " - f"-DBUILD_TESTS=ON -DUSE_CXX11_ABI=ON " - f"-DTRITON_COMMON_REPO_TAG={triton_short_tag} " - f"-DTRITON_CORE_REPO_TAG={triton_short_tag} " - f"-DTRITON_THIRD_PARTY_REPO_TAG={triton_short_tag} " - f"-DTRITON_BACKEND_REPO_TAG={triton_short_tag} " - "&& make -j8 install", llm_root) - - # Run the cpp unit tests - run_shell_command( - f"cd {llm_root}/triton_backend/inflight_batcher_llm/build/tests && " - "./utilsTest", llm_root) - run_shell_command( - f"cd {llm_root}/triton_backend/inflight_batcher_llm/build/tests && " - "./modelStateTest", llm_root) diff --git a/tests/integration/test_lists/test-db/l0_a30.yml b/tests/integration/test_lists/test-db/l0_a30.yml index c1a6ec959f87..726b8792e5b6 100644 --- a/tests/integration/test_lists/test-db/l0_a30.yml +++ b/tests/integration/test_lists/test-db/l0_a30.yml @@ -87,25 +87,6 @@ l0_a30: - accuracy/test_llm_api.py::TestQwen2_7BInstruct::test_weight_only - examples/test_granite.py::test_llm_granite[granite-3.0-2b-instruct-bfloat16] # 5 mins - examples/test_ngram.py::test_llm_ngram_1gpu[no_streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs2] # 1 min -- condition: - ranges: - system_gpu_count: - gte: 1 - lte: 1 - wildcards: - gpu: - - '*a30*' - linux_distribution_name: ubuntu* - terms: - stage: post_merge - backend: triton - tests: - - triton_server/test_triton.py::test_cpp_unit_tests[cpp-unit-tests] - - triton_server/test_triton.py::test_python_bls_unit_tests[python-bls-unit-tests] - - triton_server/test_triton.py::test_python_preproc_unit_tests[python-preproc-unit-tests] - - triton_server/test_triton.py::test_fill_template[fill-template] - - triton_server/test_triton.py::test_python_multimodal_encoders_unit_tests[python-multimodal-encoders-unit-tests] - - triton_server/test_triton.py::test_llmapi_unit_tests[llmapi-unit-tests] # ------------- AutoDeploy Backend Stages --------------- - condition: ranges: diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 69ffda67e910..bc1837cafd74 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -373,8 +373,6 @@ test_e2e.py::test_openai_chat_example[trt] SKIP (https://nvbugs/5477444) test_e2e.py::test_openai_completions_example[trt] SKIP (https://nvbugs/5701450) test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus[DeepSeek-R1-W4AFP8-DeepSeek-R1/DeepSeek-R1-W4AFP8] SKIP (https://nvbugs/5836830) test_e2e.py::test_trtllm_bench_iteration_log[TRT-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B] SKIP (https://nvbugs/5448523) -triton_server/test_triton.py::test_cpp_unit_tests[cpp-unit-tests] SKIP (https://nvbugs/5619359) -triton_server/test_triton.py::test_python_bls_unit_tests[python-bls-unit-tests] SKIP (https://nvbugs/5477392) unittest/_torch/attention/sparse/deepseek_v4/test_compressor_kernel.py::test_prefill_varlen[varlen_hd512_overlap] SKIP (https://nvbugs/6426860) unittest/_torch/misc/test_share_tensor.py::TestShareTensor::test_share_tensor_different_dtypes SKIP (https://nvbugs/6418021) unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend[act=Relu2-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize] SKIP (https://nvbugs/5989912) diff --git a/triton_backend/all_models/disaggregated_serving/README.md b/triton_backend/all_models/disaggregated_serving/README.md deleted file mode 100644 index 9ebb917c752c..000000000000 --- a/triton_backend/all_models/disaggregated_serving/README.md +++ /dev/null @@ -1,123 +0,0 @@ - - -# Running Disaggregated Serving with Triton TensorRT LLM Backend - -## Overview - -Disaggregated serving refers to a technique that uses separate GPUs for -running the context and generation phases of LLM inference. - -For Triton integration, a BLS model named -[_disaggregated\_serving\_bls_](./disaggregated_serving_bls/1/model.py) -has been created that orchestrates the disaggregated serving pipeline. This -BLS model requires the TRT-LLM model names that are going to be used for -context and generation phases. - -This example assumes access to a two GPU device systems with CUDA_VISIBLE_DEVICES -set to `0,1`. - -## Model Repository Setup and Start Server - -1. Setup the model repository as instructed in the [LLaMa](../docs/llama.md) -guide. - -2. Create context and generation models with the desired tensor-parallel -configuration. We will be using `context` and `generation` model names for -context and generation models respectively. The context and generation models -should be copying the config -[tensorrt_llm](../inflight_batcher_llm/tensorrt_llm/) model. - -3. Set the `participant_ids` for context and generation models to `1` and `2` respectively. - -4. Set the `gpu_device_ids` for context and generation models to `0` and `1` respectively. - -5. Set the `context_model_name` and `generation_model_name` to `context` and `generation` in the -[disaggregated_serving_bls](./disaggregated_serving_bls/config.pbtxt) model configuration. - -Your model repository should look like below: - -``` -disaggreagted_serving/ -|-- context -| |-- 1 -| `-- config.pbtxt -|-- disaggregated_serving_bls -| |-- 1 -| | `-- model.py -| `-- config.pbtxt -|-- ensemble -| |-- 1 -| `-- config.pbtxt -|-- generation -| |-- 1 -| `-- config.pbtxt -|-- postprocessing -| |-- 1 -| | `-- model.py -| `-- config.pbtxt -`-- preprocessing - |-- 1 - | `-- model.py - `-- config.pbtxt -``` - -6. Rename the `tensorrt_llm` model in the `ensemble` config.pbtxt file to `disaggregated_serving_bls`. - -7. Launch the Triton Server: - -``` -python3 scripts/launch_triton_server.py --world_size 3 --tensorrt_llm_model_name context,generation --multi-model --disable-spawn-processes -``` - -> ![NOTE] -> -> The world size should be equal to `tp*pp` of context model + `tp*pp` of generation model + 1. -> The additional process is required for the orchestrator. - -6. Send a request to the server. - -``` -python3 inflight_batcher_llm/client/end_to_end_grpc_client.py -S -p "Machine learning is" -``` - -## Creating Multiple Copies of the Context and Generation Models (Data Parallelism) - -You can also create multiple copies of the context and generation models. This can be -achieved by setting the `participant_ids` and `gpu_device_ids` for each instance. - -For example, if you have a context model with `tp=2` and you want to create 2 -copies of it, you can set the `participant_ids` to `1,2;3,4`, -`gpu_device_ids` to `0,1;2,3` (assuming a 4-GPU system), and set the `count` -in `instance_groups` section of the model configuration to 2. This will create 2 -copies of the context model where the first copy will be on GPU 0 and 1, and the -second copy will be on GPU 2 and 3. - -## Known Issues - -1. Only C++ version of the backend is supported right now. diff --git a/triton_backend/all_models/disaggregated_serving/disaggregated_serving.md b/triton_backend/all_models/disaggregated_serving/disaggregated_serving.md deleted file mode 100644 index 9ebb917c752c..000000000000 --- a/triton_backend/all_models/disaggregated_serving/disaggregated_serving.md +++ /dev/null @@ -1,123 +0,0 @@ - - -# Running Disaggregated Serving with Triton TensorRT LLM Backend - -## Overview - -Disaggregated serving refers to a technique that uses separate GPUs for -running the context and generation phases of LLM inference. - -For Triton integration, a BLS model named -[_disaggregated\_serving\_bls_](./disaggregated_serving_bls/1/model.py) -has been created that orchestrates the disaggregated serving pipeline. This -BLS model requires the TRT-LLM model names that are going to be used for -context and generation phases. - -This example assumes access to a two GPU device systems with CUDA_VISIBLE_DEVICES -set to `0,1`. - -## Model Repository Setup and Start Server - -1. Setup the model repository as instructed in the [LLaMa](../docs/llama.md) -guide. - -2. Create context and generation models with the desired tensor-parallel -configuration. We will be using `context` and `generation` model names for -context and generation models respectively. The context and generation models -should be copying the config -[tensorrt_llm](../inflight_batcher_llm/tensorrt_llm/) model. - -3. Set the `participant_ids` for context and generation models to `1` and `2` respectively. - -4. Set the `gpu_device_ids` for context and generation models to `0` and `1` respectively. - -5. Set the `context_model_name` and `generation_model_name` to `context` and `generation` in the -[disaggregated_serving_bls](./disaggregated_serving_bls/config.pbtxt) model configuration. - -Your model repository should look like below: - -``` -disaggreagted_serving/ -|-- context -| |-- 1 -| `-- config.pbtxt -|-- disaggregated_serving_bls -| |-- 1 -| | `-- model.py -| `-- config.pbtxt -|-- ensemble -| |-- 1 -| `-- config.pbtxt -|-- generation -| |-- 1 -| `-- config.pbtxt -|-- postprocessing -| |-- 1 -| | `-- model.py -| `-- config.pbtxt -`-- preprocessing - |-- 1 - | `-- model.py - `-- config.pbtxt -``` - -6. Rename the `tensorrt_llm` model in the `ensemble` config.pbtxt file to `disaggregated_serving_bls`. - -7. Launch the Triton Server: - -``` -python3 scripts/launch_triton_server.py --world_size 3 --tensorrt_llm_model_name context,generation --multi-model --disable-spawn-processes -``` - -> ![NOTE] -> -> The world size should be equal to `tp*pp` of context model + `tp*pp` of generation model + 1. -> The additional process is required for the orchestrator. - -6. Send a request to the server. - -``` -python3 inflight_batcher_llm/client/end_to_end_grpc_client.py -S -p "Machine learning is" -``` - -## Creating Multiple Copies of the Context and Generation Models (Data Parallelism) - -You can also create multiple copies of the context and generation models. This can be -achieved by setting the `participant_ids` and `gpu_device_ids` for each instance. - -For example, if you have a context model with `tp=2` and you want to create 2 -copies of it, you can set the `participant_ids` to `1,2;3,4`, -`gpu_device_ids` to `0,1;2,3` (assuming a 4-GPU system), and set the `count` -in `instance_groups` section of the model configuration to 2. This will create 2 -copies of the context model where the first copy will be on GPU 0 and 1, and the -second copy will be on GPU 2 and 3. - -## Known Issues - -1. Only C++ version of the backend is supported right now. diff --git a/triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/1/model.py b/triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/1/model.py deleted file mode 100644 index 14e8de75a876..000000000000 --- a/triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/1/model.py +++ /dev/null @@ -1,138 +0,0 @@ -import json - -import triton_python_backend_utils as pb_utils - - -def read_parameter_as_type(value, name, pytype=str): - if value == "": - return None - if value.startswith("${") and value.endswith("}"): - return None - if pytype is bool: - return value.lower() in ["1", "true"] - try: - result = pytype(value) - return result - except: - pb_utils.Logger.log_warning( - f"Could not read parameter '{name}' with value '{value}', will use default." - ) - return None - - -def get_parameter(model_config, name, pytype=str): - if name not in model_config['parameters']: - return None - return read_parameter_as_type( - model_config['parameters'][name]['string_value'], name, pytype) - - -class TritonPythonModel: - """Your Python model must use the same class name. Every Python model - that is created must have "TritonPythonModel" as the class name. - """ - - def initialize(self, args): - """`initialize` is called only once when the model is being loaded. - Implementing `initialize` function is optional. This function allows - the model to initialize any state associated with this model. - - Parameters - ---------- - args : dict - Both keys and values are strings. The dictionary keys and values are: - * model_config: A JSON string containing the model configuration - * model_instance_kind: A string containing model instance kind - * model_instance_device_id: A string containing model instance device ID - * model_repository: Model repository path - * model_version: Model version - * model_name: Model name - """ - model_config = json.loads(args['model_config']) - self.context_model_name = get_parameter(model_config, - "context_model_name") - self.generation_model_name = get_parameter(model_config, - "generation_model_name") - self.decoupled = pb_utils.using_decoupled_model_transaction_policy( - model_config) - - def create_context_request(self, request): - inputs = request.inputs() - triton_request = pb_utils.InferenceRequest( - model_name=self.context_model_name, - inputs=inputs, - parameters={"request_type": "context_only"}, - requested_output_names=[]) - return triton_request - - def create_generation_request(self, request, context_response): - inputs = request.inputs() - context_phase_params = pb_utils.get_output_tensor_by_name( - context_response, "context_phase_params") - if context_phase_params is None: - raise pb_utils.TritonModelException( - "Context response must have an output named context phase params" - ) - inputs.append(context_phase_params) - triton_request = pb_utils.InferenceRequest( - model_name=self.generation_model_name, - inputs=inputs, - parameters={"request_type": "generation_only"}, - requested_output_names=[]) - return triton_request - - def execute(self, requests): - """`execute` must be implemented in every Python model. `execute` - function receives a list of pb_utils.InferenceRequest as the only - argument. This function is called when an inference is requested - for this model. - - Parameters - ---------- - requests : list - A list of pb_utils.InferenceRequest - - Returns - ------- - list - A list of pb_utils.InferenceResponse. The length of this list must - be the same as `requests` - """ - for request in requests: - context_request = self.create_context_request(request) - context_responses = context_request.exec(decoupled=self.decoupled) - if self.decoupled: - context_responses = list(context_responses) - assert len( - context_responses) == 1, "Expected 1 context response" - - if self.decoupled: - context_response = context_responses[0] - else: - context_response = context_responses - if context_response.has_error(): - raise pb_utils.TritonModelException( - f"Context model {self.context_model_name} failed with error: {context_response.error().message()}" - ) - generation_request = self.create_generation_request( - request, context_response) - - # TODO(itabrizian): Send the context response to reduce TTFT in decoupled case. - # It requires adding the generated token to the generation request - # to avoid sending the first token multiple times. - responses = generation_request.exec(decoupled=self.decoupled) - - if self.decoupled: - for response in responses: - if response.has_error(): - raise pb_utils.TritonModelException( - f"Generation model {self.generation_model_name} failed with error: {response.error().message()}" - ) - request.get_response_sender().send(response) - - request.get_response_sender().send( - flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) - else: - request.get_response_sender().send( - responses, - flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) diff --git a/triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/config.pbtxt b/triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/config.pbtxt deleted file mode 100644 index 267bfa2efc49..000000000000 --- a/triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/config.pbtxt +++ /dev/null @@ -1,471 +0,0 @@ -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "disaggregated_serving_bls" -backend: "python" -max_batch_size: ${triton_max_batch_size} - -model_transaction_policy { - decoupled: ${decoupled_mode} -} - -input [ - { - name: "input_ids" - data_type: TYPE_INT32 - dims: [ -1 ] - allow_ragged_batch: true - }, - { - name: "input_lengths" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "request_output_len" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "num_return_sequences" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "draft_input_ids" - data_type: TYPE_INT32 - dims: [ -1 ] - optional: true - allow_ragged_batch: true - }, - { - name: "decoder_input_ids" - data_type: TYPE_INT32 - dims: [ -1 ] - optional: true - allow_ragged_batch: true - }, - { - name: "decoder_input_lengths" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "draft_logits" - data_type: TYPE_FP32 - dims: [ -1, -1 ] - optional: true - allow_ragged_batch: true - }, - { - name: "draft_acceptance_threshold" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "end_id" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "pad_id" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "stop_words_list" - data_type: TYPE_INT32 - dims: [ 2, -1 ] - optional: true - allow_ragged_batch: true - }, - { - name: "bad_words_list" - data_type: TYPE_INT32 - dims: [ 2, -1 ] - optional: true - allow_ragged_batch: true - }, - { - name: "embedding_bias" - data_type: TYPE_FP32 - dims: [ -1 ] - optional: true - allow_ragged_batch: true - }, - { - name: "beam_width" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "temperature" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "runtime_top_k" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "runtime_top_p" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "runtime_top_p_min" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "runtime_top_p_decay" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "runtime_top_p_reset_ids" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "len_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "early_stopping" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "prompt_ignore_length" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "repetition_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "min_tokens" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "beam_search_diversity_rate" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "presence_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "frequency_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "seed" - data_type: TYPE_UINT64 - dims: [ 1 ] - optional: true - }, - { - name: "return_log_probs" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "return_context_logits" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "return_generation_logits" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "return_perf_metrics" - data_type: TYPE_BOOL - dims: [ 1 ] - reshape: { shape: [ ] } - optional: true - }, - { - name: "exclude_input_in_output" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "stop" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "streaming" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "prompt_embedding_table" - data_type: TYPE_FP16 - dims: [ -1, -1 ] - optional: true - allow_ragged_batch: true - }, - { - name: "prompt_table_extra_ids" - data_type: TYPE_UINT64 - dims: [ -1 ] - optional: true - allow_ragged_batch: true - }, - { - name: "prompt_vocab_size" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - # the unique task ID for the given LoRA. - # To perform inference with a specific LoRA for the first time `lora_task_id` `lora_weights` and `lora_config` must all be given. - # The LoRA will be cached, so that subsequent requests for the same task only require `lora_task_id`. - # If the cache is full the oldest LoRA will be evicted to make space for new ones. An error is returned if `lora_task_id` is not cached. - { - name: "lora_task_id" - data_type: TYPE_UINT64 - dims: [ 1 ] - optional: true - }, - # weights for a lora adapter shape [ num_lora_modules_layers, D x Hi + Ho x D ] - # where the last dimension holds the in / out adapter weights for the associated module (e.g. attn_qkv) and model layer - # each of the in / out tensors are first flattened and then concatenated together in the format above. - # D=adapter_size (R value), Hi=hidden_size_in, Ho=hidden_size_out. - { - name: "lora_weights" - data_type: TYPE_FP16 - dims: [ -1, -1 ] - optional: true - allow_ragged_batch: true - }, - # module identifier (same size a first dimension of lora_weights) - # See LoraModule::ModuleType for model id mapping - # - # "attn_qkv": 0 # compbined qkv adapter - # "attn_q": 1 # q adapter - # "attn_k": 2 # k adapter - # "attn_v": 3 # v adapter - # "attn_dense": 4 # adapter for the dense layer in attention - # "mlp_h_to_4h": 5 # for llama2 adapter for gated mlp layer after attention / RMSNorm: up projection - # "mlp_4h_to_h": 6 # for llama2 adapter for gated mlp layer after attention / RMSNorm: down projection - # "mlp_gate": 7 # for llama2 adapter for gated mlp later after attention / RMSNorm: gate - # - # last dim holds [ module_id, layer_idx, adapter_size (D aka R value) ] - { - name: "lora_config" - data_type: TYPE_INT32 - dims: [ -1, 3 ] - optional: true - allow_ragged_batch: true - }, - { - name: "guided_decoding_guide_type" - data_type: TYPE_STRING - dims: [ 1 ] - optional: true - allow_ragged_batch: true - }, - { - name: "guided_decoding_guide" - data_type: TYPE_STRING - dims: [ 1 ] - optional: true - allow_ragged_batch: true - }, - { - name: "return_num_output_tokens" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "return_num_input_tokens" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - } -] -output [ - { - name: "output_ids" - data_type: TYPE_INT32 - dims: [ -1, -1 ] - }, - { - name: "sequence_length" - data_type: TYPE_INT32 - dims: [ -1 ] - }, - { - name: "cum_log_probs" - data_type: TYPE_FP32 - dims: [ -1 ] - }, - { - name: "output_log_probs" - data_type: TYPE_FP32 - dims: [ -1, -1 ] - }, - { - name: "context_logits" - data_type: ${logits_datatype} - dims: [ -1, -1 ] - }, - { - name: "generation_logits" - data_type: ${logits_datatype} - dims: [ -1, -1, -1 ] - }, - { - name: "batch_index" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "sequence_index" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "kv_cache_alloc_new_blocks" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "kv_cache_reused_blocks" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "kv_cache_alloc_total_blocks" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "arrival_time_ns" - data_type: TYPE_INT64 - dims: [ 1 ] - }, - { - name: "first_scheduled_time_ns" - data_type: TYPE_INT64 - dims: [ 1 ] - }, - { - name: "first_token_time_ns" - data_type: TYPE_INT64 - dims: [ 1 ] - }, - { - name: "last_token_time_ns" - data_type: TYPE_INT64 - dims: [ 1 ] - }, - { - name: "acceptance_rate" - data_type: TYPE_FP32 - dims: [ 1 ] - }, - { - name: "total_accepted_draft_tokens" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "total_draft_tokens" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "num_output_tokens" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "num_input_tokens" - data_type: TYPE_INT32 - dims: [ 1 ] - } -] -instance_group [ - { - count: ${disaggregated_serving_bls_count} - kind : KIND_CPU - } -] - -parameters: { - key: "context_model_name" - value: { - string_value: "${context_model_name}" - } -} -parameters: { - key: "generation_model_name" - value: { - string_value: "${generation_model_name}" - } -} diff --git a/triton_backend/all_models/gpt/ensemble/1/.tmp b/triton_backend/all_models/gpt/ensemble/1/.tmp deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/triton_backend/all_models/gpt/ensemble/config.pbtxt b/triton_backend/all_models/gpt/ensemble/config.pbtxt deleted file mode 100755 index 978579f2005b..000000000000 --- a/triton_backend/all_models/gpt/ensemble/config.pbtxt +++ /dev/null @@ -1,250 +0,0 @@ -name: "ensemble" -platform: "ensemble" -max_batch_size: 1024 -input [ - { - name: "text_input" - data_type: TYPE_STRING - dims: [ -1 ] - }, - { - name: "max_tokens" - data_type: TYPE_INT32 - dims: [ -1 ] - }, - { - name: "bad_words" - data_type: TYPE_STRING - dims: [ -1 ] - }, - { - name: "stop_words" - data_type: TYPE_STRING - dims: [ -1 ] - }, - { - name: "end_id" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "pad_id" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "top_k" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "top_p" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "temperature" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "length_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "early_stopping" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "prompt_ignore_length" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "repetition_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "min_tokens" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "presence_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "frequency_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "seed" - data_type: TYPE_UINT64 - dims: [ 1 ] - optional: true - }, - { - name: "beam_width" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "output_log_probs" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - } -] -output [ - { - name: "text_output" - data_type: TYPE_STRING - dims: [ -1, -1 ] - } -] -ensemble_scheduling { - step [ - { - model_name: "preprocessing" - model_version: -1 - input_map { - key: "QUERY" - value: "text_input" - } - input_map { - key: "REQUEST_OUTPUT_LEN" - value: "max_tokens" - } - input_map { - key: "BAD_WORDS_DICT" - value: "bad_words" - } - input_map { - key: "STOP_WORDS_DICT" - value: "stop_words" - } - output_map { - key: "REQUEST_INPUT_LEN" - value: "_REQUEST_INPUT_LEN" - } - output_map { - key: "INPUT_ID" - value: "_INPUT_ID" - } - output_map { - key: "REQUEST_OUTPUT_LEN" - value: "_REQUEST_OUTPUT_LEN" - } - }, - { - model_name: "tensorrt_llm" - model_version: -1 - input_map { - key: "input_ids" - value: "_INPUT_ID" - } - input_map { - key: "input_lengths" - value: "_REQUEST_INPUT_LEN" - } - input_map { - key: "request_output_len" - value: "_REQUEST_OUTPUT_LEN" - } - input_map { - key: "end_id" - value: "end_id" - } - input_map { - key: "pad_id" - value: "pad_id" - } - input_map { - key: "runtime_top_k" - value: "top_k" - } - input_map { - key: "runtime_top_p" - value: "top_p" - } - input_map { - key: "temperature" - value: "temperature" - } - input_map { - key: "len_penalty" - value: "length_penalty" - } - input_map { - key: "early_stopping" - value: "early_stopping" - } - input_map { - key: "prompt_ignore_length" - value: "prompt_ignore_length" - } - input_map { - key: "repetition_penalty" - value: "repetition_penalty" - } - input_map { - key: "min_tokens" - value: "min_tokens" - } - input_map { - key: "presence_penalty" - value: "presence_penalty" - } - input_map { - key: "frequency_penalty" - value: "frequency_penalty" - } - input_map { - key: "seed" - value: "seed" - } - input_map { - key: "beam_width" - value: "beam_width" - } - input_map { - key: "output_log_probs" - value: "output_log_probs" - } - output_map { - key: "output_ids" - value: "_TOKENS_BATCH" - } - }, - { - model_name: "postprocessing" - model_version: -1 - input_map { - key: "TOKENS_BATCH" - value: "_TOKENS_BATCH" - } - output_map { - key: "OUTPUT" - value: "text_output" - } - } - ] -} diff --git a/triton_backend/all_models/gpt/postprocessing/1/model.py b/triton_backend/all_models/gpt/postprocessing/1/model.py deleted file mode 100644 index 006a98cf399d..000000000000 --- a/triton_backend/all_models/gpt/postprocessing/1/model.py +++ /dev/null @@ -1,119 +0,0 @@ -# -*- coding: utf-8 -*- -import json - -import numpy as np -import triton_python_backend_utils as pb_utils -from transformers import AutoTokenizer - - -class TritonPythonModel: - """Your Python model must use the same class name. Every Python model - that is created must have "TritonPythonModel" as the class name. - """ - - def initialize(self, args): - """`initialize` is called only once when the model is being loaded. - Implementing `initialize` function is optional. This function allows - the model to initialize any state associated with this model. - Parameters - ---------- - args : dict - Both keys and values are strings. The dictionary keys and values are: - * model_config: A JSON string containing the model configuration - * model_instance_kind: A string containing model instance kind - * model_instance_device_id: A string containing model instance device ID - * model_repository: Model repository path - * model_version: Model version - * model_name: Model name - """ - # Parse model configs - model_config = json.loads(args['model_config']) - tokenizer_dir = model_config['parameters']['tokenizer_dir'][ - 'string_value'] - - self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir, - legacy=False, - padding_side="left", - trust_remote_code=True) - if not self.tokenizer.pad_token: - self.tokenizer.pad_token = self.tokenizer.eos_token - - # Parse model output configs - output_config = pb_utils.get_output_config_by_name( - model_config, "OUTPUT") - - # Convert Triton types to numpy types - self.output_dtype = pb_utils.triton_string_to_numpy( - output_config['data_type']) - - def execute(self, requests): - """`execute` must be implemented in every Python model. `execute` - function receives a list of pb_utils.InferenceRequest as the only - argument. This function is called when an inference is requested - for this model. Depending on the batching configuration (e.g. Dynamic - Batching) used, `requests` may contain multiple requests. Every - Python model, must create one pb_utils.InferenceResponse for every - pb_utils.InferenceRequest in `requests`. If there is an error, you can - set the error argument when creating a pb_utils.InferenceResponse. - Parameters - ---------- - requests : list - A list of pb_utils.InferenceRequest - Returns - ------- - list - A list of pb_utils.InferenceResponse. The length of this list must - be the same as `requests` - """ - - responses = [] - - # Every Python backend must iterate over everyone of the requests - # and create a pb_utils.InferenceResponse for each of them. - for idx, request in enumerate(requests): - # Get input tensors - tokens_batch = pb_utils.get_input_tensor_by_name( - request, 'TOKENS_BATCH').as_numpy() - - # Reshape Input - # tokens_batch = tokens_batch.reshape([-1, tokens_batch.shape[0]]) - # tokens_batch = tokens_batch.T - - # Postprocessing output data. - outputs = self._postprocessing(tokens_batch) - - # Create output tensors. You need pb_utils.Tensor - # objects to create pb_utils.InferenceResponse. - output_tensor = pb_utils.Tensor( - 'OUTPUT', - np.array(outputs).astype(self.output_dtype)) - - # Create InferenceResponse. You can set an error here in case - # there was a problem with handling this inference request. - # Below is an example of how you can set errors in inference - # response: - # - # pb_utils.InferenceResponse( - # output_tensors=..., TritonError("An error occurred")) - inference_response = pb_utils.InferenceResponse( - output_tensors=[output_tensor]) - responses.append(inference_response) - - # You should return a list of pb_utils.InferenceResponse. Length - # of this list must match the length of `requests` list. - return responses - - def finalize(self): - """`finalize` is called only once when the model is being unloaded. - Implementing `finalize` function is optional. This function allows - the model to perform any necessary clean ups before exit. - """ - print('Cleaning up...') - - def _postprocessing(self, tokens_batch): - outputs = [] - for beam_tokens in tokens_batch: - for tokens in beam_tokens: - output = self.tokenizer.decode(tokens) - outputs.append(output.encode('utf8')) - return outputs diff --git a/triton_backend/all_models/gpt/postprocessing/config.pbtxt b/triton_backend/all_models/gpt/postprocessing/config.pbtxt deleted file mode 100755 index 04b56cab226b..000000000000 --- a/triton_backend/all_models/gpt/postprocessing/config.pbtxt +++ /dev/null @@ -1,32 +0,0 @@ -name: "postprocessing" -backend: "python" -max_batch_size: 1024 -dynamic_batching {} -input [ - { - name: "TOKENS_BATCH" - data_type: TYPE_INT32 - dims: [ -1, -1 ] - } -] -output [ - { - name: "OUTPUT" - data_type: TYPE_STRING - dims: [ -1, -1 ] - } -] - -parameters { - key: "tokenizer_dir" - value: { - string_value: "${tokenizer_dir}" - } -} - -instance_group [ - { - count: 1 - kind: KIND_CPU - } -] diff --git a/triton_backend/all_models/gpt/preprocessing/1/model.py b/triton_backend/all_models/gpt/preprocessing/1/model.py deleted file mode 100644 index a301e16b747d..000000000000 --- a/triton_backend/all_models/gpt/preprocessing/1/model.py +++ /dev/null @@ -1,196 +0,0 @@ -# -*- coding: utf-8 -*- -import json -from typing import List - -import numpy as np -import torch -import triton_python_backend_utils as pb_utils -from torch.nn.utils.rnn import pad_sequence -from transformers import AutoTokenizer - - -class TritonPythonModel: - """Your Python model must use the same class name. Every Python model - that is created must have "TritonPythonModel" as the class name. - """ - - def initialize(self, args): - """`initialize` is called only once when the model is being loaded. - Implementing `initialize` function is optional. This function allows - the model to initialize any state associated with this model. - Parameters - ---------- - args : dict - Both keys and values are strings. The dictionary keys and values are: - * model_config: A JSON string containing the model configuration - * model_instance_kind: A string containing model instance kind - * model_instance_device_id: A string containing model instance device ID - * model_repository: Model repository path - * model_version: Model version - * model_name: Model name - """ - # Parse model configs - model_config = json.loads(args['model_config']) - tokenizer_dir = model_config['parameters']['tokenizer_dir'][ - 'string_value'] - - self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir, - padding_side='left', - legacy=False, - trust_remote_code=True) - if not self.tokenizer.pad_token: - self.tokenizer.pad_token = self.tokenizer.eos_token - - self.pad_id = self.tokenizer.encode(self.tokenizer.pad_token, - add_special_tokens=False)[0] - - # Parse model output configs and convert Triton types to numpy types - input_names = [ - "INPUT_ID", "REQUEST_INPUT_LEN", "BAD_WORDS_IDS", "STOP_WORDS_IDS" - ] - for input_name in input_names: - setattr( - self, - input_name.lower() + "_dtype", - pb_utils.triton_string_to_numpy( - pb_utils.get_output_config_by_name( - model_config, input_name)['data_type'])) - - def execute(self, requests): - """`execute` must be implemented in every Python model. `execute` - function receives a list of pb_utils.InferenceRequest as the only - argument. This function is called when an inference is requested - for this model. Depending on the batching configuration (e.g. Dynamic - Batching) used, `requests` may contain multiple requests. Every - Python model, must create one pb_utils.InferenceResponse for every - pb_utils.InferenceRequest in `requests`. If there is an error, you can - set the error argument when creating a pb_utils.InferenceResponse. - Parameters - ---------- - requests : list - A list of pb_utils.InferenceRequest - Returns - ------- - list - A list of pb_utils.InferenceResponse. The length of this list must - be the same as `requests` - """ - - responses = [] - - # Every Python backend must iterate over everyone of the requests - # and create a pb_utils.InferenceResponse for each of them. - for idx, request in enumerate(requests): - # Get input tensors - query = pb_utils.get_input_tensor_by_name(request, - 'QUERY').as_numpy() - request_output_len = pb_utils.get_input_tensor_by_name( - request, 'REQUEST_OUTPUT_LEN').as_numpy() - - bad_words_dict = pb_utils.get_input_tensor_by_name( - request, 'BAD_WORDS_DICT').as_numpy() - stop_words_dict = pb_utils.get_input_tensor_by_name( - request, 'STOP_WORDS_DICT').as_numpy() - - # Preprocessing input data. - input_id, request_input_len = self._create_request(query) - bad_words = self._to_word_list_format(bad_words_dict) - stop_words = self._to_word_list_format(stop_words_dict) - - # Create output tensors. You need pb_utils.Tensor - # objects to create pb_utils.InferenceResponse. - input_id_tensor = pb_utils.Tensor( - 'INPUT_ID', - np.array(input_id).astype(self.input_id_dtype)) - request_input_len_tensor = pb_utils.Tensor( - 'REQUEST_INPUT_LEN', - np.array(request_input_len).astype( - self.request_input_len_dtype)) - request_output_len_tensor = pb_utils.Tensor('REQUEST_OUTPUT_LEN', - request_output_len) - bad_words_ids_tensor = pb_utils.Tensor('BAD_WORDS_IDS', bad_words) - stop_words_ids_tensor = pb_utils.Tensor('STOP_WORDS_IDS', - stop_words) - - # Create InferenceResponse. You can set an error here in case - # there was a problem with handling this inference request. - # Below is an example of how you can set errors in inference - # response: - # - # pb_utils.InferenceResponse( - # output_tensors=..., TritonError("An error occurred")) - inference_response = pb_utils.InferenceResponse(output_tensors=[ - input_id_tensor, bad_words_ids_tensor, stop_words_ids_tensor, - request_input_len_tensor, request_output_len_tensor - ]) - responses.append(inference_response) - - # You should return a list of pb_utils.InferenceResponse. Length - # of this list must match the length of `requests` list. - return responses - - def finalize(self): - """`finalize` is called only once when the model is being unloaded. - Implementing `finalize` function is optional. This function allows - the model to perform any necessary clean ups before exit. - """ - print('Cleaning up...') - - def _create_request(self, query): - """ - query : batch string (2D numpy array) - """ - start_ids = [ - torch.IntTensor(self.tokenizer.encode(s[0].decode())) for s in query - ] - start_lengths = torch.IntTensor([[len(ids)] for ids in start_ids]) - - start_ids = pad_sequence(start_ids, - batch_first=True, - padding_value=self.pad_id) - # input_len = min(start_lengths) - #attn_mask = torch.ones((batch_size, input_len, input_len)).tril() - - return start_ids, start_lengths - - def _to_word_list_format(self, word_lists: List[List[str | bytes]]): - ''' - word_lists format: - len(word_lists) == batch_size - word_lists[i] means the words associated to batch item i. A "word" may actually be any string. Like "lorem" or "lorem ipsum". - ''' - assert self.tokenizer != None, "need to set tokenizer" - - if word_lists is None: - # Return an empty array of shape (1,2,0) - return np.empty([1, 2, 0], dtype="int32") - - flat_ids = [] - offsets = [] - for word_list in word_lists: - item_flat_ids = [] - item_offsets = [] - - for word in word_list: - if isinstance(word, bytes): - word = word.decode() - - ids = self.tokenizer.encode(word, add_special_tokens=False) - - if len(ids) == 0: - continue - - item_flat_ids += ids - item_offsets.append(len(ids)) - - flat_ids.append(np.array(item_flat_ids)) - offsets.append(np.cumsum(np.array(item_offsets))) - - pad_to = max(1, max(len(ids) for ids in flat_ids)) - - for i, (ids, offs) in enumerate(zip(flat_ids, offsets)): - flat_ids[i] = np.pad(ids, (0, pad_to - len(ids)), constant_values=0) - offsets[i] = np.pad(offs, (0, pad_to - len(offs)), - constant_values=-1) - - return np.array([flat_ids, offsets], dtype="int32").transpose((1, 0, 2)) diff --git a/triton_backend/all_models/gpt/preprocessing/config.pbtxt b/triton_backend/all_models/gpt/preprocessing/config.pbtxt deleted file mode 100644 index c36b2b81618e..000000000000 --- a/triton_backend/all_models/gpt/preprocessing/config.pbtxt +++ /dev/null @@ -1,71 +0,0 @@ -name: "preprocessing" -backend: "python" -max_batch_size: 1024 -input [ - { - name: "QUERY" - data_type: TYPE_STRING - dims: [ -1 ] - }, - { - name: "BAD_WORDS_DICT" - data_type: TYPE_STRING - dims: [ -1 ] - }, - { - name: "STOP_WORDS_DICT" - data_type: TYPE_STRING - dims: [ -1 ] - }, - { - name: "REQUEST_OUTPUT_LEN" - data_type: TYPE_INT32 - dims: [ -1 ] - } -] -output [ - { - name: "INPUT_ID" - data_type: TYPE_INT32 - dims: [ -1 ] - }, - { - name: "REQUEST_INPUT_LEN" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "BAD_WORDS_IDS" - data_type: TYPE_INT32 - dims: [ 2, -1 ] - }, - { - name: "STOP_WORDS_IDS" - data_type: TYPE_INT32 - dims: [ 2, -1 ] - }, - { - name: "REQUEST_OUTPUT_LEN" - data_type: TYPE_INT32 - dims: [ -1 ] - }, - { - name: "PROMPT_LEARNING_TASK_NAME_IDS" - data_type: TYPE_INT32 - dims: [ 1 ] - } -] - -parameters { - key: "tokenizer_dir" - value: { - string_value: "${tokenizer_dir}" - } -} - -instance_group [ - { - count: 1 - kind: KIND_CPU - } -] diff --git a/triton_backend/all_models/gpt/tensorrt_llm/1/model.py b/triton_backend/all_models/gpt/tensorrt_llm/1/model.py deleted file mode 100644 index cb8cad377fee..000000000000 --- a/triton_backend/all_models/gpt/tensorrt_llm/1/model.py +++ /dev/null @@ -1,203 +0,0 @@ -import json - -import torch -import triton_python_backend_utils as pb_utils -from torch import from_numpy - -from tensorrt_llm.runtime import ModelRunner, SamplingConfig - - -def mpi_comm(): - from mpi4py import MPI - return MPI.COMM_WORLD - - -def mpi_rank(): - return mpi_comm().Get_rank() - - -def get_engine_name(model, dtype, tp_size, rank): - return '{}_{}_tp{}_rank{}.engine'.format(model, dtype, tp_size, rank) - - -def get_input_tensor_by_name(request, name): - tensor = pb_utils.get_input_tensor_by_name(request, name) - if tensor is not None: - # Triton tensor -> numpy tensor -> PyTorch tensor - return from_numpy(tensor.as_numpy()) - else: - return tensor - - -def get_input_scalar_by_name(request, name): - tensor = pb_utils.get_input_tensor_by_name(request, name) - if tensor is not None: - # Triton tensor -> numpy tensor -> first scalar - tensor = tensor.as_numpy() - return tensor.reshape((tensor.size, ))[0] - else: - return tensor - - -class TritonPythonModel: - """Your Python model must use the same class name. Every Python model - that is created must have "TritonPythonModel" as the class name. - """ - - def initialize(self, args): - """`initialize` is called only once when the model is being loaded. - Implementing `initialize` function is optional. This function allows - the model to initialize any state associated with this model. - - Parameters - ---------- - args : dict - Both keys and values are strings. The dictionary keys and values are: - * model_config: A JSON string containing the model configuration - * model_instance_kind: A string containing model instance kind - * model_instance_device_id: A string containing model instance device ID - * model_repository: Model repository path - * model_version: Model version - * model_name: Model name - """ - model_config = json.loads(args['model_config']) - engine_dir = model_config['parameters']['engine_dir']['string_value'] - self.comm = mpi_comm() - self.rank = mpi_rank() - self.runner = ModelRunner.from_dir(engine_dir=engine_dir, - rank=self.rank) - if self.rank != 0: - while (True): - self.execute([None]) - - def execute(self, requests): - """`execute` must be implemented in every Python model. `execute` - function receives a list of pb_utils.InferenceRequest as the only - argument. This function is called when an inference is requested - for this model. - - Parameters - ---------- - requests : list - A list of pb_utils.InferenceRequest - - Returns - ------- - list - A list of pb_utils.InferenceResponse. The length of this list must - be the same as `requests` - """ - responses = [] - - # Every Python backend must iterate through list of requests and create - # an instance of pb_utils.InferenceResponse class for each of them. You - # should avoid storing any of the input Tensors in the class attributes - # as they will be overridden in subsequent inference requests. You can - # make a copy of the underlying NumPy array and store it if it is - # required. - for request in requests: - # Perform inference on the request and append it to responses list... - inputs = {} - if self.rank == 0: - inputs['input_ids'] = get_input_tensor_by_name( - request, 'input_ids') - inputs['input_lengths'] = get_input_tensor_by_name( - request, 'input_lengths') - inputs['request_output_len'] = get_input_scalar_by_name( - request, 'request_output_len') - inputs['end_id'] = get_input_scalar_by_name(request, 'end_id') - inputs['pad_id'] = get_input_scalar_by_name(request, 'pad_id') - inputs['beam_width'] = get_input_scalar_by_name( - request, 'beam_width') - inputs['temperature'] = get_input_scalar_by_name( - request, 'temperature') - inputs['runtime_top_k'] = get_input_scalar_by_name( - request, 'runtime_top_k') - inputs['runtime_top_p'] = get_input_scalar_by_name( - request, 'runtime_top_p') - inputs['len_penalty'] = get_input_scalar_by_name( - request, 'len_penalty') - inputs['repetition_penalty'] = get_input_scalar_by_name( - request, 'repetition_penalty') - inputs['min_tokens'] = get_input_scalar_by_name( - request, 'min_tokens') - inputs['presence_penalty'] = get_input_scalar_by_name( - request, 'presence_penalty') - inputs['frequency_penalty'] = get_input_scalar_by_name( - request, 'frequency_penalty') - inputs['seed'] = get_input_scalar_by_name(request, 'seed') - inputs['output_log_probs'] = get_input_scalar_by_name( - request, 'output_log_probs') - - # Broadcast requests to other clients - inputs = self.comm.bcast(inputs, root=0) - input_ids = inputs['input_ids'].cuda() - end_id = inputs['end_id'] - pad_id = inputs['pad_id'] - - sampling_config = SamplingConfig(end_id=end_id, pad_id=pad_id) - if inputs['beam_width'] is not None: - sampling_config.num_beams = inputs['beam_width'] - if inputs['temperature'] is not None: - sampling_config.temperature = inputs['temperature'] - if inputs['runtime_top_k'] is not None: - sampling_config.top_k = inputs['runtime_top_k'] - if inputs['runtime_top_p'] is not None: - sampling_config.top_p = inputs['runtime_top_p'] - if inputs['len_penalty'] is not None: - sampling_config.length_penalty = inputs['len_penalty'] - if inputs['repetition_penalty'] is not None: - sampling_config.repetition_penalty = inputs[ - 'repetition_penalty'] - if inputs['min_tokens'] is not None: - sampling_config.min_tokens = inputs['min_tokens'] - if inputs['presence_penalty'] is not None: - sampling_config.presence_penalty = inputs['presence_penalty'] - if inputs['frequency_penalty'] is not None: - sampling_config.frequency_penalty = inputs['frequency_penalty'] - sampling_config.seed = inputs['seed'] - sampling_config.output_log_probs = inputs['output_log_probs'] - sampling_config.return_dict = True - - outputs = self.runner.generate(input_ids, None, sampling_config) - output_ids = outputs["output_ids"] - - if self.rank == 0: - # Create output tensors. You need pb_utils.Tensor - # objects to create pb_utils.InferenceResponse. - torch.cuda.synchronize() - output_tensors = [ - pb_utils.Tensor("output_ids", - output_ids.cpu().numpy()) - ] - - if sampling_config.output_log_probs: - # [max_seq_len, batch_size, num_beams] -> [batch_size, max_seq_len, num_beams] - log_probs = self.runner.session.log_probs_tiled.transpose( - 0, 1).cpu().numpy() - output_tensors.append( - pb_utils.Tensor("log_probs", log_probs)) - - # Create InferenceResponse. You can set an error here in case - # there was a problem with handling this inference request. - # Below is an example of how you can set errors in inference - # response: - # - # pb_utils.InferenceResponse( - # output_tensors=..., TritonError("An error occurred")) - - inference_response = pb_utils.InferenceResponse(output_tensors) - else: - inference_response = pb_utils.InferenceResponse([]) - responses.append(inference_response) - - # You must return a list of pb_utils.InferenceResponse. Length - # of this list must match the length of `requests` list. - return responses - - def finalize(self): - """`finalize` is called only once when the model is being unloaded. - Implementing `finalize` function is optional. This function allows - the model to perform any necessary clean ups before exit. - """ - return diff --git a/triton_backend/all_models/gpt/tensorrt_llm/config.pbtxt b/triton_backend/all_models/gpt/tensorrt_llm/config.pbtxt deleted file mode 100644 index a7abcf997eab..000000000000 --- a/triton_backend/all_models/gpt/tensorrt_llm/config.pbtxt +++ /dev/null @@ -1,160 +0,0 @@ -name: "tensorrt_llm" -backend: "python" -max_batch_size: 1024 - -# # Uncomment this for dynamic_batching -# dynamic_batching { -# max_queue_delay_microseconds: 50000 -# } - -input [ - { - name: "input_ids" - data_type: TYPE_INT32 - dims: [ -1 ] - }, - { - name: "input_lengths" - data_type: TYPE_INT32 - dims: [ 1 ] - reshape: { shape: [ ] } - }, - { - name: "request_output_len" - data_type: TYPE_INT32 - dims: [ -1 ] - }, - { - name: "end_id" - data_type: TYPE_INT32 - dims: [ 1 ] - reshape: { shape: [ ] } - }, - { - name: "pad_id" - data_type: TYPE_INT32 - dims: [ 1 ] - reshape: { shape: [ ] } - }, - { - name: "beam_width" - data_type: TYPE_INT32 - dims: [ 1 ] - reshape: { shape: [ ] } - optional: true - }, - { - name: "temperature" - data_type: TYPE_FP32 - dims: [ 1 ] - reshape: { shape: [ ] } - optional: true - }, - { - name: "runtime_top_k" - data_type: TYPE_INT32 - dims: [ 1 ] - reshape: { shape: [ ] } - optional: true - }, - { - name: "runtime_top_p" - data_type: TYPE_FP32 - dims: [ 1 ] - reshape: { shape: [ ] } - optional: true - }, - { - name: "len_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - reshape: { shape: [ ] } - optional: true - }, - { - name: "early_stopping" - data_type: TYPE_INT32 - dims: [ 1 ] - reshape: { shape: [ ] } - optional: true - }, - { - name: "prompt_ignore_length" - data_type: TYPE_INT32 - dims: [ 1 ] - reshape: { shape: [ ] } - optional: true - }, - { - name: "repetition_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - reshape: { shape: [ ] } - optional: true - }, - { - name: "min_tokens" - data_type: TYPE_INT32 - dims: [ 1 ] - reshape: { shape: [ ] } - optional: true - }, - { - name: "presence_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - reshape: { shape: [ ] } - optional: true - }, - { - name: "frequency_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - reshape: { shape: [ ] } - optional: true - }, - { - name: "seed" - data_type: TYPE_UINT64 - dims: [ 1 ] - reshape: { shape: [ ] } - optional: true - }, - { - name: "output_log_probs" - data_type: TYPE_BOOL - dims: [ 1 ] - reshape: { shape: [ ] } - optional: true - } -] -output [ - { - name: "output_ids" - data_type: TYPE_INT32 - dims: [ -1, -1 ] - }, - { - name: "log_probs" - data_type: TYPE_FP32 - dims: [ -1, -1 ] - } -] -instance_group [ - { - count: 1 - kind : KIND_CPU - } -] -parameters { - key: "engine_dir" - value: { - string_value: "${engine_dir}" - } -} -parameters: { - key: "FORCE_CPU_ONLY_INPUT_TENSORS" - value: { - string_value: "no" - } -} diff --git a/triton_backend/all_models/inflight_batcher_llm/ensemble/1/.tmp b/triton_backend/all_models/inflight_batcher_llm/ensemble/1/.tmp deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/triton_backend/all_models/inflight_batcher_llm/ensemble/config.pbtxt b/triton_backend/all_models/inflight_batcher_llm/ensemble/config.pbtxt deleted file mode 100644 index 0bfa7ab90a44..000000000000 --- a/triton_backend/all_models/inflight_batcher_llm/ensemble/config.pbtxt +++ /dev/null @@ -1,732 +0,0 @@ -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "ensemble" -platform: "ensemble" -max_batch_size: ${triton_max_batch_size} -input [ - { - name: "text_input" - data_type: TYPE_STRING - dims: [ 1 ] - }, - { - name: "decoder_text_input" - data_type: TYPE_STRING - dims: [ 1 ] - optional: true - }, - { - name: "max_tokens" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "num_return_sequences" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "bad_words" - data_type: TYPE_STRING - dims: [ -1 ] - optional: true - }, - { - name: "stop_words" - data_type: TYPE_STRING - dims: [ -1 ] - optional: true - }, - { - name: "exclude_input_in_output" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "end_id" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "pad_id" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "top_k" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "top_p" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "temperature" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "length_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "early_stopping" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "prompt_ignore_length" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "repetition_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "min_tokens" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "presence_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "frequency_penalty" - data_type: TYPE_FP32 - dims: [ 1 ] - optional: true - }, - { - name: "seed" - data_type: TYPE_UINT64 - dims: [ 1 ] - optional: true - }, - { - name: "return_log_probs" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "return_context_logits" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "return_generation_logits" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "return_perf_metrics" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "return_num_input_tokens" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "return_num_output_tokens" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "beam_width" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "stream" - data_type: TYPE_BOOL - dims: [ 1 ] - optional: true - }, - { - name: "prompt_embedding_table" - data_type: TYPE_FP16 - dims: [ -1, -1 ] - optional: true - }, - { - name: "prompt_table_extra_id" - data_type: TYPE_UINT64 - dims: [ 1 ] - optional: true - }, - { - name: "prompt_vocab_size" - data_type: TYPE_INT32 - dims: [ 1 ] - optional: true - }, - { - name: "embedding_bias_words" - data_type: TYPE_STRING - dims: [ -1 ] - optional: true - }, - { - name: "embedding_bias_weights" - data_type: TYPE_FP32 - dims: [ -1 ] - optional: true - }, - # the unique task ID for the given LoRA. - # To perform inference with a specific LoRA for the first time `lora_task_id` `lora_weights` and `lora_config` must all be given. - # The LoRA will be cached, so that subsequent requests for the same task only require `lora_task_id`. - # If the cache is full the oldest LoRA will be evicted to make space for new ones. An error is returned if `lora_task_id` is not cached. - { - name: "lora_task_id" - data_type: TYPE_UINT64 - dims: [ 1 ] - optional: true - }, - # weights for a lora adapter shape [ num_lora_modules_layers, D x Hi + Ho x D ] - # where the last dimension holds the in / out adapter weights for the associated module (e.g. attn_qkv) and model layer - # each of the in / out tensors are first flattened and then concatenated together in the format above. - # D=adapter_size (R value), Hi=hidden_size_in, Ho=hidden_size_out. - { - name: "lora_weights" - data_type: TYPE_FP16 - dims: [ -1, -1 ] - optional: true - allow_ragged_batch: true - }, - # module identifier (same size a first dimension of lora_weights) - # See LoraModule::ModuleType for model id mapping - # - # "attn_qkv": 0 # compbined qkv adapter - # "attn_q": 1 # q adapter - # "attn_k": 2 # k adapter - # "attn_v": 3 # v adapter - # "attn_dense": 4 # adapter for the dense layer in attention - # "mlp_h_to_4h": 5 # for llama2 adapter for gated mlp layer after attention / RMSNorm: up projection - # "mlp_4h_to_h": 6 # for llama2 adapter for gated mlp layer after attention / RMSNorm: down projection - # "mlp_gate": 7 # for llama2 adapter for gated mlp later after attention / RMSNorm: gate - # - # last dim holds [ module_id, layer_idx, adapter_size (D aka R value) ] - { - name: "lora_config" - data_type: TYPE_INT32 - dims: [ -1, 3 ] - optional: true - allow_ragged_batch: true - }, - { - name: "guided_decoding_guide_type" - data_type: TYPE_STRING - dims: [ 1 ] - optional: true - allow_ragged_batch: true - }, - { - name: "guided_decoding_guide" - data_type: TYPE_STRING - dims: [ 1 ] - optional: true - allow_ragged_batch: true - } -] -output [ - { - name: "text_output" - data_type: TYPE_STRING - dims: [ -1 ] - }, - { - name: "cum_log_probs" - data_type: TYPE_FP32 - dims: [ -1 ] - }, - { - name: "output_log_probs" - data_type: TYPE_FP32 - dims: [ -1, -1 ] - }, - { - name: "context_logits" - data_type: ${logits_datatype} - dims: [ -1, -1 ] - }, - { - name: "generation_logits" - data_type: ${logits_datatype} - dims: [ -1, -1, -1 ] - }, - { - name: "batch_index" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "sequence_index" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "kv_cache_alloc_new_blocks" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "kv_cache_reused_blocks" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "kv_cache_alloc_total_blocks" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "kv_cache_alloc_total_blocks" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "arrival_time_ns" - data_type: TYPE_INT64 - dims: [ 1 ] - }, - { - name: "first_scheduled_time_ns" - data_type: TYPE_INT64 - dims: [ 1 ] - }, - { - name: "first_token_time_ns" - data_type: TYPE_INT64 - dims: [ 1 ] - }, - { - name: "last_token_time_ns" - data_type: TYPE_INT64 - dims: [ 1 ] - }, - { - name: "acceptance_rate" - data_type: TYPE_FP32 - dims: [ 1 ] - }, - { - name: "total_accepted_draft_tokens" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "total_draft_tokens" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "num_input_tokens" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "num_output_tokens" - data_type: TYPE_INT32 - dims: [ 1 ] - } -] -ensemble_scheduling { - step [ - { - model_name: "preprocessing" - model_version: -1 - input_map { - key: "QUERY" - value: "text_input" - } - input_map { - key: "DECODER_QUERY" - value: "decoder_text_input" - } - input_map { - key: "REQUEST_OUTPUT_LEN" - value: "max_tokens" - } - input_map { - key: "BAD_WORDS_DICT" - value: "bad_words" - } - input_map { - key: "STOP_WORDS_DICT" - value: "stop_words" - } - input_map { - key: "EMBEDDING_BIAS_WORDS" - value: "embedding_bias_words" - } - input_map { - key: "EMBEDDING_BIAS_WEIGHTS" - value: "embedding_bias_weights" - } - input_map { - key: "END_ID" - value: "end_id" - } - input_map { - key: "PAD_ID" - value: "pad_id" - } - input_map { - key: "PROMPT_TABLE_EXTRA_ID" - value: "prompt_table_extra_id" - } - output_map { - key: "REQUEST_INPUT_LEN" - value: "_REQUEST_INPUT_LEN" - } - output_map { - key: "INPUT_ID" - value: "_INPUT_ID" - } - output_map { - key: "REQUEST_DECODER_INPUT_LEN" - value: "_REQUEST_DECODER_INPUT_LEN" - } - output_map { - key: "DECODER_INPUT_ID" - value: "_DECODER_INPUT_ID" - } - output_map { - key: "REQUEST_OUTPUT_LEN" - value: "_REQUEST_OUTPUT_LEN" - } - output_map { - key: "STOP_WORDS_IDS" - value: "_STOP_WORDS_IDS" - } - output_map { - key: "BAD_WORDS_IDS" - value: "_BAD_WORDS_IDS" - } - output_map { - key: "EMBEDDING_BIAS" - value: "_EMBEDDING_BIAS" - } - output_map { - key: "OUT_END_ID" - value: "_PREPROCESSOR_END_ID" - } - output_map { - key: "OUT_PAD_ID" - value: "_PREPROCESSOR_PAD_ID" - } - output_map { - key: "OUT_PROMPT_TABLE_EXTRA_IDS" - value: "_OUT_PROMPT_TABLE_EXTRA_IDS" - } - }, - { - model_name: "tensorrt_llm" - model_version: -1 - input_map { - key: "input_ids" - value: "_INPUT_ID" - } - input_map { - key: "decoder_input_ids" - value: "_DECODER_INPUT_ID" - } - input_map { - key: "input_lengths" - value: "_REQUEST_INPUT_LEN" - } - input_map { - key: "decoder_input_lengths" - value: "_REQUEST_DECODER_INPUT_LEN" - } - input_map { - key: "exclude_input_in_output" - value: "exclude_input_in_output" - } - input_map { - key: "request_output_len" - value: "_REQUEST_OUTPUT_LEN" - } - input_map { - key: "end_id" - value: "_PREPROCESSOR_END_ID" - } - input_map { - key: "pad_id" - value: "_PREPROCESSOR_PAD_ID" - } - input_map { - key: "embedding_bias" - value: "_EMBEDDING_BIAS" - } - input_map { - key: "runtime_top_k" - value: "top_k" - } - input_map { - key: "runtime_top_p" - value: "top_p" - } - input_map { - key: "temperature" - value: "temperature" - } - input_map { - key: "len_penalty" - value: "length_penalty" - } - input_map { - key: "early_stopping" - value: "early_stopping" - } - input_map { - key: "prompt_ignore_length" - value: "prompt_ignore_length" - } - input_map { - key: "repetition_penalty" - value: "repetition_penalty" - } - input_map { - key: "min_tokens" - value: "min_tokens" - } - input_map { - key: "presence_penalty" - value: "presence_penalty" - } - input_map { - key: "frequency_penalty" - value: "frequency_penalty" - } - input_map { - key: "seed" - value: "seed" - } - input_map { - key: "return_log_probs" - value: "return_log_probs" - } - input_map { - key: "return_context_logits" - value: "return_context_logits" - } - input_map { - key: "return_generation_logits" - value: "return_generation_logits" - } - input_map { - key: "return_perf_metrics" - value: "return_perf_metrics" - } - input_map { - key: "return_num_input_tokens" - value: "return_num_input_tokens" - } - input_map { - key: "return_num_output_tokens" - value: "return_num_output_tokens" - } - input_map { - key: "num_return_sequences" - value: "num_return_sequences" - } - input_map { - key: "beam_width" - value: "beam_width" - } - input_map { - key: "streaming" - value: "stream" - } - input_map { - key: "prompt_embedding_table" - value: "prompt_embedding_table" - } - input_map { - key: "prompt_vocab_size" - value: "prompt_vocab_size" - } - input_map { - key: "stop_words_list" - value: "_STOP_WORDS_IDS" - } - input_map { - key: "bad_words_list" - value: "_BAD_WORDS_IDS" - } - input_map { - key: "prompt_table_extra_ids" - value: "_OUT_PROMPT_TABLE_EXTRA_IDS" - }, - input_map { - key: "lora_task_id", - value: "lora_task_id" - }, - input_map { - key: "lora_weights", - value: "lora_weights" - }, - input_map { - key: "lora_config", - value: "lora_config" - }, - input_map { - key: "guided_decoding_guide_type", - value: "guided_decoding_guide_type" - }, - input_map { - key: "guided_decoding_guide", - value: "guided_decoding_guide" - } - output_map { - key: "output_ids" - value: "_TOKENS_BATCH" - } - output_map { - key: "sequence_length" - value: "_SEQUENCE_LENGTH" - }, - output_map { - key: "cum_log_probs" - value: "cum_log_probs" - } - output_map { - key: "output_log_probs" - value: "output_log_probs" - }, - output_map { - key: "context_logits" - value: "context_logits" - }, - output_map { - key: "generation_logits" - value: "generation_logits" - }, - output_map { - key: "batch_index" - value: "batch_index" - }, - output_map { - key: "sequence_index" - value: "sequence_index" - }, - output_map { - key: "kv_cache_alloc_new_blocks" - value: "kv_cache_alloc_new_blocks" - }, - output_map { - key: "kv_cache_reused_blocks" - value: "kv_cache_reused_blocks" - }, - output_map { - key: "kv_cache_alloc_total_blocks" - value: "kv_cache_alloc_total_blocks" - }, - output_map { - key: "arrival_time_ns" - value: "arrival_time_ns" - }, - output_map { - key: "first_scheduled_time_ns" - value: "first_scheduled_time_ns" - }, - output_map { - key: "first_token_time_ns" - value: "first_token_time_ns" - }, - output_map { - key: "last_token_time_ns" - value: "last_token_time_ns" - }, - output_map { - key: "acceptance_rate" - value: "acceptance_rate" - }, - output_map { - key: "total_accepted_draft_tokens" - value: "total_accepted_draft_tokens" - }, - output_map { - key: "total_draft_tokens" - value: "total_draft_tokens" - }, - output_map { - key: "num_input_tokens" - value: "num_input_tokens" - }, - output_map { - key: "num_output_tokens" - value: "num_output_tokens" - } - }, - { - model_name: "postprocessing" - model_version: -1 - input_map { - key: "TOKENS_BATCH" - value: "_TOKENS_BATCH" - } - input_map { - key: "SEQUENCE_LENGTH" - value: "_SEQUENCE_LENGTH" - } - output_map { - key: "OUTPUT" - value: "text_output" - } - } - ] -} diff --git a/triton_backend/all_models/inflight_batcher_llm/postprocessing/1/model.py b/triton_backend/all_models/inflight_batcher_llm/postprocessing/1/model.py deleted file mode 100644 index 70d88ae785c1..000000000000 --- a/triton_backend/all_models/inflight_batcher_llm/postprocessing/1/model.py +++ /dev/null @@ -1,177 +0,0 @@ -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import json - -import numpy as np -import triton_python_backend_utils as pb_utils -from transformers import AutoTokenizer - - -class TritonPythonModel: - """Your Python model must use the same class name. Every Python model - that is created must have "TritonPythonModel" as the class name. - """ - - def initialize(self, args): - """`initialize` is called only once when the model is being loaded. - Implementing `initialize` function is optional. This function allows - the model to initialize any state associated with this model. - Parameters - ---------- - args : dict - Both keys and values are strings. The dictionary keys and values are: - * model_config: A JSON string containing the model configuration - * model_instance_kind: A string containing model instance kind - * model_instance_device_id: A string containing model instance device ID - * model_repository: Model repository path - * model_version: Model version - * model_name: Model name - """ - # Parse model configs - model_config = json.loads(args['model_config']) - tokenizer_dir = model_config['parameters']['tokenizer_dir'][ - 'string_value'] - - skip_special_tokens = model_config['parameters'].get( - 'skip_special_tokens') - if skip_special_tokens is not None: - skip_special_tokens_str = skip_special_tokens['string_value'].lower( - ) - if skip_special_tokens_str in [ - 'true', 'false', '1', '0', 't', 'f', 'y', 'n', 'yes', 'no' - ]: - self.skip_special_tokens = skip_special_tokens_str in [ - 'true', '1', 't', 'y', 'yes' - ] - else: - print( - f"[TensorRT-LLM][WARNING] Don't setup 'skip_special_tokens' correctly (set value is {skip_special_tokens['string_value']}). Set it as True by default." - ) - self.skip_special_tokens = True - else: - print( - f"[TensorRT-LLM][WARNING] Don't setup 'skip_special_tokens'. Set it as True by default." - ) - self.skip_special_tokens = True - - self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir, - legacy=False, - padding_side='left', - trust_remote_code=True) - if not self.tokenizer.pad_token: - self.tokenizer.pad_token = self.tokenizer.eos_token - - # Parse model output configs - output_config = pb_utils.get_output_config_by_name( - model_config, "OUTPUT") - - # Convert Triton types to numpy types - self.output_dtype = pb_utils.triton_string_to_numpy( - output_config['data_type']) - - def execute(self, requests): - """`execute` must be implemented in every Python model. `execute` - function receives a list of pb_utils.InferenceRequest as the only - argument. This function is called when an inference is requested - for this model. Depending on the batching configuration (e.g. Dynamic - Batching) used, `requests` may contain multiple requests. Every - Python model, must create one pb_utils.InferenceResponse for every - pb_utils.InferenceRequest in `requests`. If there is an error, you can - set the error argument when creating a pb_utils.InferenceResponse. - Parameters - ---------- - requests : list - A list of pb_utils.InferenceRequest - Returns - ------- - list - A list of pb_utils.InferenceResponse. The length of this list must - be the same as `requests` - """ - - tokens_batch = [] - sequence_lengths = [] - for idx, request in enumerate(requests): - for input_tensor in request.inputs(): - if input_tensor.name() == "TOKENS_BATCH": - tokens_batch.append(input_tensor.as_numpy()) - elif input_tensor.name() == "SEQUENCE_LENGTH": - sequence_lengths.append(input_tensor.as_numpy()) - else: - raise ValueError(f"unknown input {input_tensor.name}") - - # batch decode - list_of_tokens = [] - req_idx_offset = 0 - req_idx_offsets = [req_idx_offset] - for idx, token_batch in enumerate(tokens_batch): - for batch_idx, beam_tokens in enumerate(token_batch): - for beam_idx, tokens in enumerate(beam_tokens): - seq_len = sequence_lengths[idx][batch_idx][beam_idx] - list_of_tokens.append(tokens[:seq_len]) - req_idx_offset += 1 - - req_idx_offsets.append(req_idx_offset) - - all_outputs = self.tokenizer.batch_decode( - list_of_tokens, skip_special_tokens=self.skip_special_tokens) - - # construct responses - responses = [] - for idx, request in enumerate(requests): - req_outputs = [ - x.encode('utf8') - for x in all_outputs[req_idx_offsets[idx]:req_idx_offsets[idx + - 1]] - ] - - output_tensor = pb_utils.Tensor( - 'OUTPUT', - np.array(req_outputs).astype(self.output_dtype)) - - outputs = [output_tensor] - - # Create InferenceResponse. You can set an error here in case - # there was a problem with handling this inference request. - # Below is an example of how you can set errors in inference - # response: - # - # pb_utils.InferenceResponse( - # output_tensors=..., TritonError("An error occurred")) - inference_response = pb_utils.InferenceResponse( - output_tensors=outputs) - responses.append(inference_response) - # You should return a list of pb_utils.InferenceResponse. Length - # of this list must match the length of `requests` list. - return responses - - def finalize(self): - """`finalize` is called only once when the model is being unloaded. - Implementing `finalize` function is optional. This function allows - the model to perform any necessary clean ups before exit. - """ - print('Cleaning up...') diff --git a/triton_backend/all_models/inflight_batcher_llm/postprocessing/config.pbtxt b/triton_backend/all_models/inflight_batcher_llm/postprocessing/config.pbtxt deleted file mode 100644 index 9f3655ef4d78..000000000000 --- a/triton_backend/all_models/inflight_batcher_llm/postprocessing/config.pbtxt +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "postprocessing" -backend: "python" -max_batch_size: ${triton_max_batch_size} -dynamic_batching {} -input [ - { - name: "TOKENS_BATCH" - data_type: TYPE_INT32 - dims: [ -1, -1 ] - }, - { - name: "SEQUENCE_LENGTH" - data_type: TYPE_INT32 - dims: [ -1 ] - } -] -output [ - { - name: "OUTPUT" - data_type: TYPE_STRING - dims: [ -1 ] - } -] - -parameters { - key: "tokenizer_dir" - value: { - string_value: "${tokenizer_dir}" - } -} - -parameters { - key: "skip_special_tokens" - value: { - string_value: "${skip_special_tokens}" - } -} - -instance_group [ - { - count: ${postprocessing_instance_count} - kind: KIND_CPU - } -] diff --git a/triton_backend/all_models/inflight_batcher_llm/preprocessing/1/model.py b/triton_backend/all_models/inflight_batcher_llm/preprocessing/1/model.py deleted file mode 100755 index 8e605f8ce5ba..000000000000 --- a/triton_backend/all_models/inflight_batcher_llm/preprocessing/1/model.py +++ /dev/null @@ -1,1175 +0,0 @@ -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import base64 -import io -import json -import os -from collections import defaultdict -from typing import Dict, List, Tuple - -import numpy as np -import requests -import triton_python_backend_utils as pb_utils -from PIL import Image -from transformers import AutoConfig, AutoProcessor, AutoTokenizer, T5Tokenizer - - -class TritonPythonModel: - """Your Python model must use the same class name. Every Python model - that is created must have "TritonPythonModel" as the class name. - """ - - def initialize(self, args): - """`initialize` is called only once when the model is being loaded. - Implementing `initialize` function is optional. This function allows - the model to initialize any state associated with this model. - Parameters - ---------- - args : dict - Both keys and values are strings. The dictionary keys and values are: - * model_config: A JSON string containing the model configuration - * model_instance_kind: A string containing model instance kind - * model_instance_device_id: A string containing model instance device ID - * model_repository: Model repository path - * model_version: Model version - * model_name: Model name - """ - # Parse model configs - model_config = json.loads(args['model_config']) - tokenizer_dir = model_config['parameters']['tokenizer_dir'][ - 'string_value'] - - add_special_tokens = model_config['parameters'].get( - 'add_special_tokens') - multimodal_model_path = model_config['parameters'][ - 'multimodal_model_path']['string_value'] - max_num_images = model_config['parameters'].get('max_num_images') - - if max_num_images is not None: - max_num_images_str = max_num_images['string_value'] - if max_num_images_str.isdigit(): - self.max_num_images = int(max_num_images_str) - else: - print( - f"[TensorRT-LLM][WARNING] 'max_num_images' parameter is not set correctly (value is {max_num_images_str}). Will be set to None" - ) - self.max_num_images = None - else: - print( - f"[TensorRT-LLM][WARNING] Don't setup 'max_num_images'. Set it as None by default." - ) - self.max_num_images = None - if multimodal_model_path == "${multimodal_model_path}" or multimodal_model_path == "": - multimodal_model_path = None - - if add_special_tokens is not None: - add_special_tokens_str = add_special_tokens['string_value'].lower() - if add_special_tokens_str in [ - 'true', 'false', '1', '0', 't', 'f', 'y', 'n', 'yes', 'no' - ]: - self.add_special_tokens = add_special_tokens_str in [ - 'true', '1', 't', 'y', 'yes' - ] - else: - print( - f"[TensorRT-LLM][WARNING] Don't setup 'add_special_tokens' correctly (set value is {add_special_tokens['string_value']}). Set it as True by default." - ) - self.add_special_tokens = True - else: - print( - f"[TensorRT-LLM][WARNING] Don't setup 'add_special_tokens'. Set it as True by default." - ) - self.add_special_tokens = True - - self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir, - legacy=False, - padding_side='left', - trust_remote_code=True) - - if isinstance(self.tokenizer, T5Tokenizer): - bos_id = self.tokenizer.bos_token_id - self.tokenizer_bos_id = bos_id if bos_id is not None else -1 - - if not self.tokenizer.pad_token: - self.tokenizer.pad_token = self.tokenizer.eos_token - - self.tokenizer_end_id = self.tokenizer.encode( - self.tokenizer.eos_token, add_special_tokens=False)[0] - self.tokenizer_pad_id = self.tokenizer.encode( - self.tokenizer.pad_token, add_special_tokens=False)[0] - self.vocab_size = self.tokenizer.vocab_size - - self.is_multimodal = False - self.model_type = None - self.vision_preprocessor = None - - if multimodal_model_path is not None: - self.is_multimodal = True - multimodal_model_path = os.path.join(multimodal_model_path, - 'config.json') - with open(multimodal_model_path, 'r') as f: - visual_model_config = json.load(f) - self.model_type = visual_model_config['builder_config'][ - 'model_type'] - - assert self.model_type in [ - 'llava', 'blip2-opt', 'pixtral', 'vila', 'mllama', - 'llava_onevision', 'qwen2_vl' - ], f"[TensorRT-LLM][ERROR] Currently supported multi-modal models are llava, blip2-opt, pixtral, vila, mllama, llava_onevision and qwen2_vl. Got {self.model_type}." - - assert self.model_type != 'llava_onevison' or self.max_num_images is None or self.max_num_images <= 1, f"LLaVA-OneVsion is not support multi image inference currently." - - llm_model_path = model_config['parameters']['gpt_model_path'][ - 'string_value'] - llm_model_path = os.path.join(llm_model_path, 'config.json') - with open(llm_model_path, 'r') as f: - llm_model_config = json.load(f) - self.vocab_size = int( - llm_model_config["pretrained_config"]["vocab_size"]) - self._setup_ptable_shape(llm_model_config) - - if self.model_type in [ - 'mllama', 'llava_onevision', 'qwen2_vl', 'pixtral' - ]: - full_processor = AutoProcessor.from_pretrained( - tokenizer_dir, trust_remote_code=True) - self.hf_config = AutoConfig.from_pretrained(tokenizer_dir) - self.vision_preprocessor = VisionPreProcessor( - self.model_type, - full_processor, - model_config, - self.hf_config, - ) - - # Parse model output configs and convert Triton types to numpy types - output_names = [ - "INPUT_ID", "DECODER_INPUT_ID", "REQUEST_INPUT_LEN", - "REQUEST_DECODER_INPUT_LEN", "BAD_WORDS_IDS", "STOP_WORDS_IDS", - "OUT_END_ID", "OUT_PAD_ID", "OUT_PROMPT_TABLE_EXTRA_IDS", - "PIXEL_VALUES", "IMAGE_SIZES" - ] - input_names = ["EMBEDDING_BIAS_WORDS", "EMBEDDING_BIAS_WEIGHTS"] - for input_name in input_names: - setattr( - self, - input_name.lower() + "_dtype", - pb_utils.triton_string_to_numpy( - pb_utils.get_input_config_by_name(model_config, - input_name)['data_type'])) - - for output_name in output_names: - setattr( - self, - output_name.lower() + "_dtype", - pb_utils.triton_string_to_numpy( - pb_utils.get_output_config_by_name( - model_config, output_name)['data_type'])) - - def _setup_ptable_shape(self, llm_model_config): - max_prompt_embedding_table_size = llm_model_config['build_config'][ - 'max_prompt_embedding_table_size'] - max_batch_size = llm_model_config['build_config']['max_batch_size'] - - num_multimodal_features = max_prompt_embedding_table_size // max_batch_size - hidden_size = llm_model_config['pretrained_config']['hidden_size'] - if self.max_num_images is not None: - num_multimodal_features = num_multimodal_features // self.max_num_images - - self.ptable_shape = (-1, num_multimodal_features, hidden_size) - - def execute(self, requests): - """`execute` must be implemented in every Python model. `execute` - function receives a list of pb_utils.InferenceRequest as the only - argument. This function is called when an inference is requested - for this model. Depending on the batching configuration (e.g. Dynamic - Batching) used, `requests` may contain multiple requests. Every - Python model, must create one pb_utils.InferenceResponse for every - pb_utils.InferenceRequest in `requests`. If there is an error, you can - set the error argument when creating a pb_utils.InferenceResponse. - Parameters - ---------- - requests : list - A list of pb_utils.InferenceRequest - Returns - ------- - list - A list of pb_utils.InferenceResponse. The length of this list must - be the same as `requests` - """ - - responses = [] - - # Every Python backend must iterate over everyone of the requests - # and create a pb_utils.InferenceResponse for each of them. - for idx, request in enumerate(requests): - # Get input tensors - query = pb_utils.get_input_tensor_by_name(request, - 'QUERY').as_numpy() - batch_size = query.shape[0] - - decoder_query = pb_utils.get_input_tensor_by_name( - request, 'DECODER_QUERY') - if decoder_query is not None: - decoder_query = decoder_query.as_numpy() - - request_output_len = pb_utils.get_input_tensor_by_name( - request, 'REQUEST_OUTPUT_LEN').as_numpy() - - bad_words_dict = pb_utils.get_input_tensor_by_name( - request, 'BAD_WORDS_DICT') - if bad_words_dict is not None: - bad_words_dict = bad_words_dict.as_numpy() - - stop_words_dict = pb_utils.get_input_tensor_by_name( - request, 'STOP_WORDS_DICT') - if stop_words_dict is not None: - stop_words_dict = stop_words_dict.as_numpy() - - embedding_bias_words = pb_utils.get_input_tensor_by_name( - request, 'EMBEDDING_BIAS_WORDS') - if embedding_bias_words is not None: - embedding_bias_words = embedding_bias_words.as_numpy() - - embedding_bias_weights = pb_utils.get_input_tensor_by_name( - request, 'EMBEDDING_BIAS_WEIGHTS') - if embedding_bias_weights is not None: - embedding_bias_weights = embedding_bias_weights.as_numpy() - - # Take the end_id from the input tensors - # If not specified, use tokenizer to get end_id - end_id = pb_utils.get_input_tensor_by_name(request, 'END_ID') - if end_id is not None: - end_id = end_id.as_numpy() - else: - end_id = [[self.tokenizer_end_id]] * batch_size - - # Take the pad_id from the input tensors - # If not specified, use tokenizer to get pad_id - pad_id = pb_utils.get_input_tensor_by_name(request, 'PAD_ID') - if pad_id is not None: - pad_id = pad_id.as_numpy() - else: - pad_id = [[self.tokenizer_pad_id]] * batch_size - - # Take the extra_id from the input tensors - # Extra id is used in kv cache reuse for p-tuning - prompt_table_extra_id = pb_utils.get_input_tensor_by_name( - request, 'PROMPT_TABLE_EXTRA_ID') - if prompt_table_extra_id is not None: - prompt_table_extra_id = prompt_table_extra_id.as_numpy() - assert prompt_table_extra_id.shape[ - 0] == batch_size, "Prompt table extra id must have the same batch size as Query" - assert prompt_table_extra_id.shape[ - 1] == 1, "Multiple IDs cannot be provided for a single image" - - # Preprocessing vision input passed as a url or bytes tensor - img_urls = pb_utils.get_input_tensor_by_name(request, 'IMAGE_URL') - image_bytes = pb_utils.get_input_tensor_by_name( - request, 'IMAGE_BYTES') - video_bytes = pb_utils.get_input_tensor_by_name( - request, 'VIDEO_BYTES') - vision_processed_tensors = [] - visual_tokens = [] - # Pixtral supports text-only input - if self.is_multimodal and (img_urls or image_bytes or video_bytes - or self.model_type == 'pixtral'): - assert self.vision_preprocessor != None, "Vision preprocessor for preparing images before encoding is None" - processed_tensors = {} - if self.model_type == 'mllama': - processed_tensors = self.vision_preprocessor.mllama_process( - queries=query.astype(str).tolist(), - img_urls=img_urls, - image_bytes=image_bytes, - ) - elif self.model_type == 'llava_onevision': - if video_bytes is None: - processed_tensors, visual_tokens = self.vision_preprocessor.llava_onevision_process_image( - queries=query.astype(str).tolist(), - img_urls=img_urls, - image_bytes=image_bytes, - ) - else: - processed_tensors, visual_tokens = self.vision_preprocessor.llava_onevision_process_video( - queries=query.astype(str).tolist(), - video_bytes=video_bytes, - ) - elif self.model_type == 'qwen2_vl': - processed_tensors = self.vision_preprocessor.qwen2_vl_process_image( - queries=query.astype(str).tolist(), - img_urls=img_urls, - image_bytes=image_bytes, - ) - qwen2vl_input_id_tensor = processed_tensors.get("INPUT_IDS") - processed_tensors.pop("INPUT_IDS") - qwen2vl_input_length_tensor = processed_tensors.get( - "REQUEST_INPUT_LEN") - processed_tensors.pop("REQUEST_INPUT_LEN") - elif self.model_type == 'pixtral': - image_sizes = pb_utils.get_input_tensor_by_name( - request, 'IMAGE_SIZES') - processed_tensors, visual_tokens = self.vision_preprocessor.pixtral_process( - queries=query.astype(str).tolist(), - img_urls=img_urls, - image_bytes=image_bytes, - image_sizes=image_sizes, - ) - pixtral_input_id_tensor = processed_tensors.pop("INPUT_IDS") - request_input_len = np.array( - [[len(input_ids_for_batch)] - for input_ids_for_batch in pixtral_input_id_tensor]) - else: - raise ValueError( - "Unsupported model type for IMAGE_BYTES or IMAGE_URL inputs" - ) - vision_processed_tensors = [ - pb_utils.Tensor.from_dlpack(k, v) - for k, v in processed_tensors.items() - ] - else: - assert self.model_type != "llava_onevision", "Image processing requires IMAGE_BYTES or IMAGE_URL to be provided" - - # Preprocessing input data. - # For the LLaVA_OneVision model, num_multimodal_features is not a fixed value - if self.model_type != 'pixtral': - input_id, request_input_len = self._create_request( - query, visual_tokens) - if decoder_query is not None: - decoder_input_id, request_decoder_input_len = self._create_request( - decoder_query) - else: - decoder_input_id = pad_id * np.ones((batch_size, 1), np.int32) - request_decoder_input_len = 1 * np.ones( - (batch_size, 1), np.int32) - - bad_words = self._to_word_list_format(bad_words_dict, batch_size) - stop_words = self._to_word_list_format(stop_words_dict, batch_size) - - embedding_bias = self._get_embedding_bias( - embedding_bias_words, embedding_bias_weights, - self.embedding_bias_weights_dtype, batch_size) - - if prompt_table_extra_id is not None and self.model_type != 'qwen2_vl': - prompt_table_extra_ids = np.zeros_like(input_id) - for i in range(batch_size): - prompt_table_extra_ids[i] = np.where( - input_id[i] >= self.vocab_size, - prompt_table_extra_id[i], 0) - - # Create output tensors. You need pb_utils.Tensor - # objects to create pb_utils.InferenceResponse. - # Qwen2-VL model has special logic to process input ids - if self.model_type == 'qwen2_vl': - input_id_tensor = pb_utils.Tensor.from_dlpack( - 'INPUT_ID', qwen2vl_input_id_tensor) - request_input_len_tensor = pb_utils.Tensor.from_dlpack( - 'REQUEST_INPUT_LEN', qwen2vl_input_length_tensor) - elif self.model_type == 'pixtral': - input_id_tensor = pb_utils.Tensor( - 'INPUT_ID', - pixtral_input_id_tensor.numpy().astype(self.input_id_dtype)) - request_input_len_tensor = pb_utils.Tensor( - 'REQUEST_INPUT_LEN', - request_input_len.astype(self.request_input_len_dtype)) - else: - input_id_tensor = pb_utils.Tensor( - 'INPUT_ID', input_id.astype(self.input_id_dtype)) - request_input_len_tensor = pb_utils.Tensor( - 'REQUEST_INPUT_LEN', - request_input_len.astype(self.request_input_len_dtype)) - decoder_input_id_tensor = pb_utils.Tensor( - 'DECODER_INPUT_ID', - decoder_input_id.astype(self.decoder_input_id_dtype)) - request_decoder_input_len_tensor = pb_utils.Tensor( - 'REQUEST_DECODER_INPUT_LEN', - request_decoder_input_len.astype( - self.request_decoder_input_len_dtype)) - request_output_len_tensor = pb_utils.Tensor('REQUEST_OUTPUT_LEN', - request_output_len) - bad_words_ids_tensor = pb_utils.Tensor('BAD_WORDS_IDS', bad_words) - stop_words_ids_tensor = pb_utils.Tensor('STOP_WORDS_IDS', - stop_words) - embedding_bias_tensor = pb_utils.Tensor('EMBEDDING_BIAS', - embedding_bias) - end_id_tensor = pb_utils.Tensor('OUT_END_ID', - np.array(end_id, dtype=np.int32)) - pad_id_tensor = pb_utils.Tensor('OUT_PAD_ID', - np.array(pad_id, dtype=np.int32)) - if prompt_table_extra_id is not None: - prompt_table_extra_ids_tensor = pb_utils.Tensor( - 'OUT_PROMPT_TABLE_EXTRA_IDS', - np.array(prompt_table_extra_ids, - dtype=self.out_prompt_table_extra_ids_dtype)) - inference_response = pb_utils.InferenceResponse(output_tensors=[ - input_id_tensor, decoder_input_id_tensor, - bad_words_ids_tensor, stop_words_ids_tensor, - request_input_len_tensor, request_decoder_input_len_tensor, - request_output_len_tensor, embedding_bias_tensor, - end_id_tensor, pad_id_tensor, prompt_table_extra_ids_tensor - ] + vision_processed_tensors) - else: - inference_response = pb_utils.InferenceResponse(output_tensors=[ - input_id_tensor, decoder_input_id_tensor, - bad_words_ids_tensor, stop_words_ids_tensor, - request_input_len_tensor, request_decoder_input_len_tensor, - request_output_len_tensor, embedding_bias_tensor, - end_id_tensor, pad_id_tensor - ] + vision_processed_tensors) - responses.append(inference_response) - # You should return a list of pb_utils.InferenceResponse. Length - # of this list must match the length of `requests` list. - return responses - - def finalize(self): - """`finalize` is called only once when the model is being unloaded. - Implementing `finalize` function is optional. This function allows - the model to perform any necessary clean ups before exit. - """ - print('Cleaning up...') - - def _split_prompt_by_images(self, concatenated_ids, image_token_index=-200): - """ - Splits tokenized prompts by image placeholders for each sample in the batch. - - Args: - concatenated_ids (np.ndarray): A batch of concatenated token IDs, where image placeholders are indicated by `image_token_index`. - - Returns: - List[List[np.ndarray]]: A list containing lists of token ID arrays for each prompt segment, per batch sample. - """ - batch_splits = [] - for batch in concatenated_ids: - zero_indices = np.where(batch == image_token_index)[0] - start_idx = 0 - splits = [] - for idx in zero_indices: - if start_idx != idx: - splits.append(batch[start_idx:idx].reshape(1, -1)) - start_idx = idx + 1 - if start_idx < len(batch): - splits.append(batch[start_idx:].reshape(1, -1)) - - splits = [split for split in splits if split.size > 0] - batch_splits.append(splits) - - return batch_splits - - def _setup_fake_prompts(self, batch_size, batch_split_prompts): - """ - Replaces image placeholders with unique fake prompt IDs for multi-image inputs. - - Args: - batch_size (int): The number of samples in the batch. - batch_split_prompts (List[List[np.ndarray]]): Tokenized prompt segments for each batch sample. - - Returns: - np.ndarray: An array of input IDs with image placeholders replaced by fake prompt IDs. - """ - - num_multimodal_features = self.ptable_shape[1] - input_ids_list = [] - - for batch_idx in range(batch_size): - splits = batch_split_prompts[batch_idx] - sample_input_ids = [splits[0]] - sample_fake_prompt_counter = self.vocab_size - - for split_idx in range(len(splits) - 1): - fake_prompt_id = np.arange( - sample_fake_prompt_counter, - sample_fake_prompt_counter + num_multimodal_features) - sample_fake_prompt_counter += num_multimodal_features - fake_prompt_id = np.expand_dims(fake_prompt_id, axis=0) - sample_input_ids.append(fake_prompt_id) - sample_input_ids.append(splits[split_idx + 1]) - - sample_input_ids = np.concatenate(sample_input_ids, axis=1) - input_ids_list.append(sample_input_ids) - - # Pad the input_ids to the same length for bs > 1 - max_seq_len = max( - [sample_input_ids.shape[1] for sample_input_ids in input_ids_list]) - input_ids_padded = [] - for sample_input_ids in input_ids_list: - seq_len = sample_input_ids.shape[1] - pad_width = max_seq_len - seq_len - if pad_width > 0: - sample_input_ids_padded = np.pad( - sample_input_ids, ((0, 0), (0, pad_width)), - 'constant', - constant_values=self.tokenizer_pad_id) - else: - sample_input_ids_padded = sample_input_ids - input_ids_padded.append(sample_input_ids_padded) - - input_ids = np.stack(input_ids_padded) - input_ids = input_ids.reshape(batch_size, -1).astype(np.int32) - - return input_ids - - def _process_multi_image_inputs(self, query, image_token_index=-200): - """ - Processes input queries that contain multiple images by tokenizing the input strings and inserting image_token_index between the parts. - - Args: - query (np.ndarray): Batch of input strings. - - Returns: - List[np.ndarray]: List of tokenized input IDs for each sample. - """ - start_ids = [] - for s in query: - parts = s[0].decode().split('') - num_images = len(parts) - 1 - if num_images > self.max_num_images: - raise ValueError( - f"The number of images in the request ({num_images}) exceeds the maximum allowed ({self.max_num_images})." - ) - tokenized_parts = [ - self.tokenizer.encode(part, add_special_tokens=False) - for part in parts - ] - - # Insert `image_token_index` between the parts to represent - final_ids = [] - for i, part in enumerate(tokenized_parts): - final_ids.extend(part) - if i < len(tokenized_parts) - 1: - final_ids.append(image_token_index) - - start_ids.append(np.array(final_ids).astype(int)) - - return start_ids - - def _create_request(self, query, visual_tokens=None): - """ - query : batch string (2D numpy array) - """ - if isinstance(self.tokenizer, T5Tokenizer): - start_ids = [ - np.array([self.tokenizer_bos_id] + self.tokenizer.encode( - s[0].decode(), add_special_tokens=self.add_special_tokens)). - astype(int) for s in query - ] - else: - # Qwen2-VL input id is calculated when processing image - if 'qwen2_vl' == self.model_type: - return None, None - if self.is_multimodal and self.max_num_images and self.max_num_images > 1: - start_ids = self._process_multi_image_inputs(query) - - else: - start_ids = [ - np.array( - self.tokenizer.encode( - s[0].decode(), - add_special_tokens=self.add_special_tokens)).astype( - int) for s in query - ] - - if self.is_multimodal: - if 'blip2' in self.model_type or 'mllama' == self.model_type: - pre_prompt = None - post_prompt = None - elif 'llava' == self.model_type: - pre_prompt = "USER:\n" - post_prompt = " ASSISTANT:" - elif 'vila' == self.model_type: - pre_prompt = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: " - post_prompt = " ASSISTANT:" - elif 'llava_onevision' == self.model_type: - pre_prompt = "<|im_start|>user " - post_prompt = "<|im_end|><|im_start|>assistant\n" - pre_prompt_id = np.array( - self.tokenizer.encode( - pre_prompt, - add_special_tokens=self.add_special_tokens, - padding=True)) if pre_prompt is not None else np.array( - [], dtype=int) - - post_prompt_id = np.array( - self.tokenizer.encode( - post_prompt, - add_special_tokens=self.add_special_tokens, - padding=True)) if post_prompt is not None else np.array( - [], dtype=int) - - if self.max_num_images and self.max_num_images > 1: - concatenated_ids = [ - np.concatenate((pre_prompt_id, ids, post_prompt_id), axis=0) - for ids in start_ids - ] - batch_split_prompts = self._split_prompt_by_images( - concatenated_ids) - start_ids = self._setup_fake_prompts(query.shape[0], - batch_split_prompts) - elif self.model_type == 'llava_onevision': - fake_prompt_ids = [] - extra_id = np.array( - self.tokenizer.encode( - '\n', - add_special_tokens=self.add_special_tokens, - padding=True)) - for tokens in visual_tokens: - prompt_id = np.arange(self.vocab_size, - self.vocab_size + tokens) - fake_prompt_ids.append(prompt_id) - start_ids = [ - np.concatenate((pre_prompt_id, prompt_id, extra_id, ids, - post_prompt_id), - axis=0) - for prompt_id, ids in zip(fake_prompt_ids, start_ids) - ] - else: - fake_prompt_id = np.arange( - self.vocab_size, self.vocab_size + self.ptable_shape[1]) - start_ids = [ - np.concatenate( - (pre_prompt_id, fake_prompt_id, ids, post_prompt_id), - axis=0) for ids in start_ids - ] - - start_lengths = np.array([[len(ids)] for ids in start_ids]).astype(int) - - max_len = 0 - for seq in start_ids: - max_len = max(max_len, seq.shape[0]) - start_ids = np.stack([ - np.pad(seq, (0, max_len - seq.shape[0]), - 'constant', - constant_values=(0, self.tokenizer_pad_id)) - for seq in start_ids - ]) - - return start_ids, start_lengths - - def _to_word_list_format(self, word_lists: List[List[str | bytes]], - batch_size): - ''' - word_lists format: - len(word_lists) == batch_size - word_lists[i] means the words associated to batch item i. A "word" may actually be any string. Like "lorem" or "lorem ipsum". - ''' - assert self.tokenizer != None, "need to set tokenizer" - - if word_lists is None: - # Return an empty array of shape (1,2,0) - return np.empty([batch_size, 2, 0], dtype="int32") - - flat_ids = [] - offsets = [] - for word_list in word_lists: - item_flat_ids = [] - item_offsets = [] - - for word in word_list: - if isinstance(word, bytes): - word = word.decode() - - ids = self.tokenizer.encode(word, add_special_tokens=False) - if len(ids) == 0: - continue - - item_flat_ids += ids - item_offsets.append(len(ids)) - - flat_ids.append(np.array(item_flat_ids)) - offsets.append(np.cumsum(np.array(item_offsets))) - - pad_to = max(1, max(len(ids) for ids in flat_ids)) - - for i, (ids, offs) in enumerate(zip(flat_ids, offsets)): - flat_ids[i] = np.pad(ids, (0, pad_to - len(ids)), constant_values=0) - offsets[i] = np.pad(offs, (0, pad_to - len(offs)), - constant_values=-1) - - return np.array([flat_ids, offsets], dtype="int32").transpose((1, 0, 2)) - - def _get_embedding_bias(self, embedding_bias_words, embedding_bias_weights, - bias_dtype, batch_size): - - assert self.tokenizer != None, "need to set tokenizer" - - if embedding_bias_words is None or embedding_bias_weights is None: - return np.empty([batch_size, 0], - dtype=self.embedding_bias_weights_dtype) - - batch_embedding_bias = [] - for words, weights in zip(embedding_bias_words, embedding_bias_weights): - - vocab_size = len(self.tokenizer.vocab) - embedding_bias = [0.] * vocab_size - - assert len(words) == len( - weights - ), "Embedding bias words must have same dimension as embedding bias weights" - - for word, weight in zip(words, weights): - if isinstance(word, bytes): - word = word.decode() - ids = self.tokenizer.encode(word) - - if len(ids) == 0: - continue - - for id in ids: - embedding_bias[id] += weight - - batch_embedding_bias.append(np.array(embedding_bias)) - - return np.array(batch_embedding_bias, dtype=bias_dtype) - - -class VisionPreProcessor: - """ A class that can load images from url requests, and process them via a vision model processor, - in preparation for the vision encoder. - """ - - def __init__(self, - vision_model_type, - vision_model_processor, - preprocessor_model_config=None, - hf_config=None): - preprocessor_model_config = preprocessor_model_config or {} - - # import libraries that are only relevant for multimodal models - import torch - from torch.utils.dlpack import from_dlpack - - # NOTE: Due to the behavior of MPI initialization, it is recommended to avoid using import tensorrt_llm - # except for the specific modules tensorrt_llm and multimodal_encoders. - # As a result, the function str_dtype_to_torch has been copied directly from tensorrt_llm._utils. - _str_to_torch_dtype_dict = dict( - bfloat16=torch.bfloat16, - float16=torch.float16, - float32=torch.float32, - int64=torch.int64, - int32=torch.int32, - int8=torch.int8, - bool=torch.bool, - fp8=torch.float8_e4m3fn, - ) - - def str_dtype_to_torch(dtype): - ret = _str_to_torch_dtype_dict.get(dtype) - assert ret is not None, f'Unsupported dtype: {dtype}' - return ret - - self.load_images_tensor = lambda tensor: tensor if not hasattr( - tensor, 'to_dlpack') else from_dlpack(tensor.to_dlpack()) - - # extract expected output tensor dtype - self.output_str_dtypes = {} - for properties in preprocessor_model_config.get('output', []): - dtype = properties['data_type'] - self.output_str_dtypes[properties['name']] = np.dtype( - pb_utils.triton_string_to_numpy(dtype)).name - - # create method for converting output tensors batch to the expected type - self.convert_tensor_list_to_tensor = lambda tensor_list: torch.concat( - [ - torch.from_numpy(x) if isinstance(x, np.ndarray) else x - for x in tensor_list - ], - dim=0) - self.convert_tensor_to_str_dtype = lambda tensor, dtype: tensor.to( - str_dtype_to_torch(dtype)) - - # create model-specific processor - self.vision_model_processor = vision_model_processor - self.vision_model_type = vision_model_type - - if vision_model_type == 'pixtral': - assert hf_config is not None, "Pixtral model requires hf_config to be set" - self.vocab_size = hf_config.text_config.vocab_size - self.image_size = hf_config.vision_config.image_size - self.image_token_index = hf_config.image_token_index - - def load_images_from_urls(self, img_urls): - images = [] - for img_url in img_urls: - img_url = img_url.decode() - if img_url.startswith("data:image/jpeg;base64,"): - image_base64 = img_url.split(",")[1] - # Decode the base64 string - image_data = base64.b64decode(image_base64) - # Create a BytesIO object from the decoded data - image_buffer = io.BytesIO(image_data) - images.append(Image.open(image_buffer).convert("RGB")) - else: - images.append( - Image.open(requests.get(img_url, - stream=True).raw).convert("RGB")) - return images - - def mllama_process(self, queries, img_urls=None, image_bytes=None): - vision_processed_tensors = {} - if img_urls is not None or image_bytes is not None: - if img_urls is not None: - # download and read images - images = [ - self.load_images_from_urls(urls) - for urls in img_urls.as_numpy() - ] - else: - images = [ - img for img_list in self.load_images_tensor(image_bytes) - for img in img_list - ] - - batch_size = len(images) - - preprocessor_outputs = {} - possible_output_names = [ - 'PIXEL_VALUES', 'ASPECT_RATIO_IDS', 'ASPECT_RATIO_MASK', - 'CROSS_ATTENTION_MASK' - ] - for batch_id in range(batch_size): - # Preprocess images and query - processed_vision_data = self.vision_model_processor( - images=images[batch_id], - text=queries[batch_id], - return_tensors="pt") - # Reshape pixel_values to [num_images, *HWC/CHW] - val = processed_vision_data["pixel_values"] - val = val.reshape(1, -1, *(val.shape[-3:])) - processed_vision_data["pixel_values"] = val - - # Create vision output tensors - for key in possible_output_names: - val = processed_vision_data.get(key.lower()) - if val is not None: - if key not in preprocessor_outputs: - preprocessor_outputs[key] = [] - preprocessor_outputs[key].append(val) - - for key, tensor_list in preprocessor_outputs.items(): - val = self.convert_tensor_list_to_tensor(tensor_list) - if key in self.output_str_dtypes: - val = self.convert_tensor_to_str_dtype( - val, self.output_str_dtypes[key]) - vision_processed_tensors[key] = val - return vision_processed_tensors - - def llava_onevision_process_image(self, - queries, - img_urls=None, - image_bytes=None): - - import torch - vision_processed_tensors = {} - if img_urls is not None: - # download and read images - images = [ - self.load_images_from_urls(urls) - for urls in img_urls.as_numpy() - ] - else: - images = [ - img for img_list in self.load_images_tensor(image_bytes) - for img in img_list - ] - - batch_size = len(images) - assert len( - queries - ) == batch_size, f"Image must have the same batch size as Query." - preprocessor_outputs = {} - possible_output_names = ['PIXEL_VALUES', 'IMAGE_SIZES'] - visual_tokens = [] - for batch_id in range(batch_size): - # Preprocess images and query - processed_vision_data = self.vision_model_processor( - images=images[batch_id], text='', return_tensors="pt") - visual_tokens.append(processed_vision_data['input_ids'].shape[1]) - # Create vision output tensors - for key in possible_output_names: - val = processed_vision_data.get(key.lower()) - if val is not None: - if key not in preprocessor_outputs: - preprocessor_outputs[key] = [] - preprocessor_outputs[key].append(val) - - max_patch = max(x.shape[1] - for x in preprocessor_outputs['PIXEL_VALUES']) - preprocessor_outputs['PIXEL_VALUES'] = [ - torch.nn.functional.pad( - image, (0, 0, 0, 0, 0, 0, 0, max_patch - image.shape[1], 0, 0), - mode='constant') - for image in preprocessor_outputs['PIXEL_VALUES'] - ] - # Add a dimension image_sizes to match the dimensions defined in config.pbtxt - for elem in preprocessor_outputs['IMAGE_SIZES']: - elem.unsqueeze_(1) - for key, tensor_list in preprocessor_outputs.items(): - val = self.convert_tensor_list_to_tensor(tensor_list) - if key in self.output_str_dtypes: - val = self.convert_tensor_to_str_dtype( - val, self.output_str_dtypes[key]) - vision_processed_tensors[key] = val - return vision_processed_tensors, visual_tokens - - def llava_onevision_process_video(self, queries, video_bytes=None): - import torch - vision_processed_tensors = {} - videos = [video for video in self.load_images_tensor(video_bytes)] - - batch_size = len(videos) - assert len( - queries - ) == batch_size, f"Video must have the same batch size as Query." - preprocessor_outputs = {} - preprocessor_outputs['PIXEL_VALUES'] = [] - preprocessor_outputs['IS_VIDEO_INPUT'] = [] - visual_tokens = [] - for batch_id in range(len(queries)): - processed_vision_data = self.vision_model_processor( - videos=list(videos[batch_id]), - text='