Skip to content

Remove tensorrt backend #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
QiJune wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Remove tensorrt backend #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6174b2c
[None][feat] Remove legacy TensorRT Python backend
Jun 22, 2026
59eb547
[None][feat] Decouple C++ runtime from the TensorRT library (nvinfer1)
Jun 22, 2026
331ca87
[None][test] Clean up TensorRT-removal stragglers and fix test collec…
Jun 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
7 changes: 3 additions & 4 deletions cpp/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,8 +249,8 @@ if(ENABLE_MULTI_DEVICE)
endif()

# TRT dependencies
find_package(TensorRT 10 REQUIRED COMPONENTS OnnxParser)
set(TRT_LIB TensorRT::NvInfer)
# TensorRT dependency removed: the C++ tree no longer links the TensorRT library.
set(TRT_LIB "")

get_filename_component(TRT_LLM_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR} PATH)

Expand Down Expand Up @@ -291,7 +291,6 @@ include_directories(
${CUDAToolkit_INCLUDE_DIRS}
${CUDAToolkit_INCLUDE_DIRS}/cccl
${CUDNN_ROOT_DIR}/include
$<TARGET_PROPERTY:TensorRT::NvInfer,INTERFACE_INCLUDE_DIRECTORIES>
${maybe_nvtx_includedir}
${CMAKE_BINARY_DIR}/_deps/cutlass-src/include
${CMAKE_BINARY_DIR}/_deps/cutlass-src/tools/util/include
Expand Down Expand Up @@ -683,6 +682,6 @@ if(MEASURE_BUILD_TIME)
endif()

set(BUILD_WHEEL_TARGETS
tensorrt_llm;nvinfer_plugin_tensorrt_llm
tensorrt_llm
CACHE STRING "Targets used to build wheel")
add_custom_target(build_wheel_targets DEPENDS ${BUILD_WHEEL_TARGETS})
4 changes: 2 additions & 2 deletions cpp/include/tensorrt_llm/batch_manager/cacheTransceiver.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ class CacheTransceiver : public BaseCacheTransceiver
public:
CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheManager,
executor::kv_cache::CacheState::ModelConfig const& cacheStateModelCfg, runtime::WorldConfig const& worldConfig,
std::vector<SizeType32> const& attentionLayerNumPerPP, nvinfer1::DataType dataType,
std::vector<SizeType32> const& attentionLayerNumPerPP, tensorrt_llm::DataType dataType,
executor::kv_cache::CacheState::AttentionType attentionType
= executor::kv_cache::CacheState::AttentionType::kDEFAULT,
std::optional<executor::CacheTransceiverConfig> cacheTransceiverConfig = std::nullopt,
Expand All @@ -242,7 +242,7 @@ class CacheTransceiver : public BaseCacheTransceiver

CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheManager, std::vector<SizeType32> numKvHeadsPerLayer,
SizeType32 sizePerHead, SizeType32 tokensPerBlock, runtime::WorldConfig const& worldConfig,
std::vector<SizeType32> const& attentionLayerNumPerPP, nvinfer1::DataType dataType,
std::vector<SizeType32> const& attentionLayerNumPerPP, tensorrt_llm::DataType dataType,
executor::kv_cache::CacheState::AttentionType attentionType
= executor::kv_cache::CacheState::AttentionType::kDEFAULT,
std::optional<executor::CacheTransceiverConfig> cacheTransceiverConfig = std::nullopt,
Expand Down
4 changes: 2 additions & 2 deletions cpp/include/tensorrt_llm/batch_manager/createNewDecoderRequests.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,14 @@ class CreateNewDecoderRequests : Algorithm
std::vector<executor::LookaheadDecodingConfig>>
operator()(runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig,
executor::DecodingConfig const& decodingConfig, RequestVector const& contextRequests,
nvinfer1::DataType logitsType, DecoderInputBuffers& inputBuffers, runtime::decoder::DecoderState& decoderState,
tensorrt_llm::DataType logitsType, DecoderInputBuffers& inputBuffers, runtime::decoder::DecoderState& decoderState,
CudaStream const& runtimeStream, CudaStream const& decoderStream, SizeType32 maxSequenceLength,
SizeType32 beamWidth, OptionalRef<MedusaBuffers const> medusaBuffers) const;

[[nodiscard]] std::tuple<std::vector<SharedConstPtr>, std::vector<executor::LookaheadDecodingConfig>>
createDecoderRequests(RequestVector const& finishedContextRequests, TensorPtr const& inputIds,
executor::DecodingConfig const& decodingConfig, runtime::decoder::DecoderState& decoderState,
nvinfer1::DataType logitsType, runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig,
tensorrt_llm::DataType logitsType, runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig,
runtime::CudaStream const& runtimeStream, runtime::CudaStream const& decoderStream,
SizeType32 maxSequenceLength, OptionalRef<MedusaBuffers const> medusaBuffers) const;

Expand Down
4 changes: 2 additions & 2 deletions cpp/include/tensorrt_llm/batch_manager/guidedDecoder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ class GuidedDecoder
using BitmaskT = uint32_t;

GuidedDecoder(executor::GuidedDecodingConfig const& guidedDecodingConfig, SizeType32 maxNumSequences,
SizeType32 vocabSizePadded, nvinfer1::DataType logitsDtype, runtime::BufferManager const& runtimeBufferManager);
SizeType32 vocabSizePadded, tensorrt_llm::DataType logitsDtype, runtime::BufferManager const& runtimeBufferManager);
void build(ScheduledRequests const& scheduledRequests);
void execute(DecoderInputBuffers const& decoderInputBuffers, runtime::BufferManager const& runtimeBufferManager);

Expand All @@ -51,7 +51,7 @@ class GuidedDecoder
SizeType32 mMaxNumSequences;
SizeType32 mVocabSizePadded;
SizeType32 mBitmaskSize; // CeilDiv(vocabSizePadded, 32)
nvinfer1::DataType mLogitsDtype;
tensorrt_llm::DataType mLogitsDtype;

TensorPtr mLogitsBitmask; // [mMaxNumRequests, mBitmaskSize]
TensorPtr mLogitsBitmaskHost; // [mMaxNumRequests, mBitmaskSize]
Expand Down
53 changes: 0 additions & 53 deletions cpp/include/tensorrt_llm/batch_manager/handleContextLogits.h
View file
Open in desktop

This file was deleted.

53 changes: 0 additions & 53 deletions cpp/include/tensorrt_llm/batch_manager/handleGenerationLogits.h
View file
Open in desktop

This file was deleted.

36 changes: 18 additions & 18 deletions cpp/include/tensorrt_llm/batch_manager/kvCacheManager.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
#include "tensorrt_llm/runtime/iBuffer.h"
#include "tensorrt_llm/runtime/iTensor.h"
#include "tensorrt_llm/runtime/worldConfig.h"
#include <NvInferRuntime.h>
#include "tensorrt_llm/common/tllmDataType.h"

#include <algorithm>
#include <array>
Expand Down Expand Up @@ -128,7 +128,7 @@ struct PoolConfiguration
{
SizeType32 windowSize;
SizeType32 sizePerHead;
nvinfer1::DataType dtype;
tensorrt_llm::DataType dtype;
};

struct LinearAttentionMetadata
Expand Down Expand Up @@ -838,7 +838,7 @@ class WindowBlockManager
using BlockMap = std::unordered_multimap<size_t, BlockPtr>;
using BlockMapIterRange = std::pair<BlockMap::const_iterator, BlockMap::const_iterator>;

explicit WindowBlockManager(nvinfer1::DataType dtype, SizeType32 windowSize,
explicit WindowBlockManager(tensorrt_llm::DataType dtype, SizeType32 windowSize,
std::vector<SizeType32> const& managedLayers, std::vector<SizeType32> const& numKvHeadsPerLayer,
SizeType32 sizePerHead, SizeType32 tokensPerBlock, bool isSWA, SizeType32 blocksInPrimaryPool,
SizeType32 blocksInSecondaryPool, SizeType32 maxNumSequences, std::shared_ptr<runtime::CudaStream> stream,
Expand Down Expand Up @@ -1003,7 +1003,7 @@ class WindowBlockManager
//! host pools with mixed precisions when constructed with a per-window
//! dtype map. Empty pools or NVFP4-scale pools are routed through the
//! per-pool tensor metadata instead.
[[nodiscard]] nvinfer1::DataType getDataType() const noexcept
[[nodiscard]] tensorrt_llm::DataType getDataType() const noexcept
{
return mDataType;
}
Expand Down Expand Up @@ -1093,7 +1093,7 @@ class WindowBlockManager
[[nodiscard]] SizeType32 getNumEltsPerContainer() const
{
#ifdef ENABLE_FP4
return mDataType == nvinfer1::DataType::kFP4 ? 2 : 1;
return mDataType == tensorrt_llm::DataType::kFP4 ? 2 : 1;
#else
return 1;
#endif
Expand Down Expand Up @@ -1158,7 +1158,7 @@ class WindowBlockManager
return mLayerToIndexWithinPool.at(layerIdx);
}

void setOffsets(kernels::KVCacheIndex* offsetsPtr, nvinfer1::Dims const& offsetsShape, SizeType32 beamIdx,
void setOffsets(kernels::KVCacheIndex* offsetsPtr, tensorrt_llm::Dims const& offsetsShape, SizeType32 beamIdx,
SizeType32 blockIdx, KVCacheBlock::IdType blockId) const;

//! \brief Bring offloaded block from secondary to primary memory.
Expand Down Expand Up @@ -1319,7 +1319,7 @@ class WindowBlockManager
}

private:
nvinfer1::DataType mDataType;
tensorrt_llm::DataType mDataType;
SizeType32 mWindowSize;

// Number of blocks in pools
Expand Down Expand Up @@ -1445,7 +1445,7 @@ class BlockManager
explicit BlockManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead,
SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences,
CudaStreamPtr stream, SizeType32 maxSequenceLength, SizeType32 maxBeamWidth,
std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkBubbleLength,
std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkBubbleLength,
SizeType32 chunkSize, CacheType cacheType = CacheType::kSELF,
std::optional<executor::RetentionPriority> secondaryOffloadMinPriority = std::nullopt,
std::shared_ptr<KVCacheEventManager> eventManager = nullptr, bool enablePartialReuse = true,
Expand Down Expand Up @@ -1527,7 +1527,7 @@ class BlockManager

void releaseLastBlock(GenerationRequest& sequence, SizeType32 windowSize);

void setOffsets(kernels::KVCacheIndex* offsetsPtr, nvinfer1::Dims const& offsetsShape, SizeType32 beamIdx,
void setOffsets(kernels::KVCacheIndex* offsetsPtr, tensorrt_llm::Dims const& offsetsShape, SizeType32 beamIdx,
SizeType32 blockIdx, KVCacheBlock::IdType blockId, SizeType32 windowSize) const;

//! \brief Combined prefix reuse analysis — single radix tree walk.
Expand Down Expand Up @@ -1585,9 +1585,9 @@ class BlockManager

//! \brief Convenience: window_size -> dataType, derived from getPoolConfigurations().
//! For one-pool-per-window managers only; multi-pool-per-window will collide.
[[nodiscard]] std::map<SizeType32, nvinfer1::DataType> getDataTypePerWindow() const
[[nodiscard]] std::map<SizeType32, tensorrt_llm::DataType> getDataTypePerWindow() const
{
std::map<SizeType32, nvinfer1::DataType> result;
std::map<SizeType32, tensorrt_llm::DataType> result;
for (auto const& [windowSize, manager] : mWindowBlockManagers)
{
result[windowSize] = manager.getDataType();
Expand All @@ -1600,7 +1600,7 @@ class BlockManager
return mWindowBlockManagers.at(windowSize).getSizePerHead();
}

[[nodiscard]] nvinfer1::DataType getDataTypeForWindow(SizeType32 windowSize) const
[[nodiscard]] tensorrt_llm::DataType getDataTypeForWindow(SizeType32 windowSize) const
{
return mWindowBlockManagers.at(windowSize).getDataType();
}
Expand Down Expand Up @@ -2153,7 +2153,7 @@ class BaseKVCacheManager
/// head_dim=512). Empty vector = uniform @p sizePerHead / @p dtype across all windows.
/// @return Map from window size to tuple of (primary blocks, secondary blocks)
[[nodiscard]] static BlocksPerWindow calculateMaxNumBlocks(executor::KvCacheConfig const& config,
nvinfer1::DataType dtype, std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead,
tensorrt_llm::DataType dtype, std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead,
SizeType32 tokensPerBlock, tensorrt_llm::runtime::WorldConfig const& worldConfig,
std::map<SizeType32, std::vector<SizeType32>> const& windowSizeToLayers, uint64_t allottedPrimaryMemBytes,
uint64_t allottedSecondaryMemBytes, size_t extraCostMemory, SizeType32 kvFactor, SizeType32 maxBatchSize,
Expand Down Expand Up @@ -2240,7 +2240,7 @@ class KVCacheManager : public BaseKVCacheManager
//! and disagg transfer machinery applies natively. Empty vector = uniform.
KVCacheManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock,
BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth,
std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength,
std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength,
CudaStreamPtr stream, SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse = false,
CacheType cacheType = CacheType::kSELF,
std::optional<executor::RetentionPriority> secondaryOffloadMinPriority = std::nullopt,
Expand All @@ -2254,7 +2254,7 @@ class KVCacheManager : public BaseKVCacheManager

KVCacheManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock,
BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth,
std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength,
std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength,
int64_t stream, SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse = false,
CacheType cacheType = CacheType::kSELF,
std::optional<executor::RetentionPriority> secondaryOffloadMinPriority = std::nullopt,
Expand All @@ -2268,7 +2268,7 @@ class KVCacheManager : public BaseKVCacheManager

KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, SizeType32 sizePerHead, SizeType32 tokensPerBlock,
BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth,
std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength,
std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength,
CudaStreamPtr stream, SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse = true,
CacheType cacheType = CacheType::kSELF,
std::optional<executor::RetentionPriority> secondaryOffloadMinPriority = std::nullopt,
Expand All @@ -2282,7 +2282,7 @@ class KVCacheManager : public BaseKVCacheManager

KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, SizeType32 sizePerHead, SizeType32 tokensPerBlock,
BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth,
std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength,
std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength,
int64_t stream, SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse = false,
CacheType cacheType = CacheType::kSELF, bool enablePartialReuse = true, bool copyOnpartialReuse = true,
bool enableIndexerKCache = false, SizeType32 indexerKCacheQuantBlockSize = 128,
Expand Down Expand Up @@ -2628,7 +2628,7 @@ class KVCacheManager : public BaseKVCacheManager
SizeType32 mMaxNumSequences;
// Maximum beam width
SizeType32 mMaxBeamWidth;
nvinfer1::DataType mDataType;
tensorrt_llm::DataType mDataType;
// Maximum kv cache length per sequence
SizeType32 mMaxAttentionWindow;
// Number of tokens per block
Expand Down
Loading
Loading