From 86ddf5d7774199fa1ac38da6893a3342cc72dd67 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 5 May 2026 02:43:31 +0000 Subject: [PATCH 01/16] docs: sync LMSYS SGLang blog cards --- docs_new/index.mdx | 60 +++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/docs_new/index.mdx b/docs_new/index.mdx index 6a5b1ed19ddb..908d517386da 100644 --- a/docs_new/index.mdx +++ b/docs_new/index.mdx @@ -83,7 +83,7 @@ It is designed to deliver low-latency and high-throughput inference across a wid }} > DeepSeek-V4 on Day 0: From Fast Inference to Verified RL with SGLang and Miles - {"DeepSeek-V4 on Day 0: From Fast Inference to Verified RL with SGLang and Miles"} + {"Updating 1T parameters in seconds \u2014 P2P weight transfer in Large Scale Distributed RL"}

- {"April 25, 2026"} + {"April 29, 2026"}

HiSparse: Turbocharging Sparse Attention with Hierarchical Memory - {"HiSparse: Turbocharging Sparse Attention with Hierarchical Memory"} + {"DeepSeek-V4 on Day 0: From Fast Inference to Verified RL with SGLang and Miles"}

- {"April 10, 2026"} + {"April 25, 2026"}

Highlights of SGLang at NVIDIA GTC 2026 - {"Highlights of SGLang at NVIDIA GTC 2026"} + {"HiSparse: Turbocharging Sparse Attention with Hierarchical Memory"}

- {"March 31, 2026"} + {"April 10, 2026"}

Elastic EP in SGLang: Achieving Partial Failure Tolerance for DeepSeek MoE Deployments - {"Elastic EP in SGLang: Achieving Partial Failure Tolerance for DeepSeek MoE Deployments"} + {"Highlights of SGLang at NVIDIA GTC 2026"}

- {"March 25, 2026"} + {"March 31, 2026"}

ROCm Support for Miles: Large-Scale RL Post-Training on AMD Instinct\u2122 GPUs - {"ROCm Support for Miles: Large-Scale RL Post-Training on AMD Instinct\u2122 GPUs"} + {"Elastic EP in SGLang: Achieving Partial Failure Tolerance for DeepSeek MoE Deployments"}

- {"March 17, 2026"} + {"March 25, 2026"}

SGLang Adds Day-0 Support for NVIDIA Nemotron 3 Super for building High-Efficiency Multi-Agent Systems - {"SGLang Adds Day-0 Support for NVIDIA Nemotron 3 Super for building High-Efficiency Multi-Agent Systems"} + {"ROCm Support for Miles: Large-Scale RL Post-Training on AMD Instinct\u2122 GPUs"}

- {"March 11, 2026"} + {"March 17, 2026"}

From fad2580716968192ec9f6585326299ba931fc156 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 29 May 2026 03:36:00 +0000 Subject: [PATCH 02/16] docs: sync LMSYS SGLang blog cards --- docs_new/index.mdx | 60 +++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/docs_new/index.mdx b/docs_new/index.mdx index 908d517386da..a68238e5bfbf 100644 --- a/docs_new/index.mdx +++ b/docs_new/index.mdx @@ -83,7 +83,7 @@ It is designed to deliver low-latency and high-throughput inference across a wid }} > Updating 1T parameters in seconds \u2014 P2P weight transfer in Large Scale Distributed RL - {"Updating 1T parameters in seconds \u2014 P2P weight transfer in Large Scale Distributed RL"} + {"Win on TCO: How AMD Instinct\u2122 MI355X Achieves Cost-Competitive Distributed Inference Through SGLang with MoRI"}

- {"April 29, 2026"} + {"May 28, 2026"}

DeepSeek-V4 on Day 0: From Fast Inference to Verified RL with SGLang and Miles - {"DeepSeek-V4 on Day 0: From Fast Inference to Verified RL with SGLang and Miles"} + {"Updating 1T parameters in seconds \u2014 P2P weight transfer in Large Scale Distributed RL"}

- {"April 25, 2026"} + {"April 29, 2026"}

HiSparse: Turbocharging Sparse Attention with Hierarchical Memory - {"HiSparse: Turbocharging Sparse Attention with Hierarchical Memory"} + {"DeepSeek-V4 on Day 0: From Fast Inference to Verified RL with SGLang and Miles"}

- {"April 10, 2026"} + {"April 25, 2026"}

Highlights of SGLang at NVIDIA GTC 2026 - {"Highlights of SGLang at NVIDIA GTC 2026"} + {"HiSparse: Turbocharging Sparse Attention with Hierarchical Memory"}

- {"March 31, 2026"} + {"April 10, 2026"}

Elastic EP in SGLang: Achieving Partial Failure Tolerance for DeepSeek MoE Deployments - {"Elastic EP in SGLang: Achieving Partial Failure Tolerance for DeepSeek MoE Deployments"} + {"Highlights of SGLang at NVIDIA GTC 2026"}

- {"March 25, 2026"} + {"March 31, 2026"}

ROCm Support for Miles: Large-Scale RL Post-Training on AMD Instinct\u2122 GPUs - {"ROCm Support for Miles: Large-Scale RL Post-Training on AMD Instinct\u2122 GPUs"} + {"Elastic EP in SGLang: Achieving Partial Failure Tolerance for DeepSeek MoE Deployments"}

- {"March 17, 2026"} + {"March 25, 2026"}

From 09b7c559ef1ddfbc9ae508a2dbbcc8397d4da4b0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 1 Jun 2026 17:38:37 +0000 Subject: [PATCH 03/16] docs: sync LMSYS SGLang blog cards --- docs_new/index.mdx | 60 +++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/docs_new/index.mdx b/docs_new/index.mdx index a68238e5bfbf..7eeeb55b1dfc 100644 --- a/docs_new/index.mdx +++ b/docs_new/index.mdx @@ -83,7 +83,7 @@ It is designed to deliver low-latency and high-throughput inference across a wid }} > Win on TCO: How AMD Instinct\u2122 MI355X Achieves Cost-Competitive Distributed Inference Through SGLang with MoRI - {"Win on TCO: How AMD Instinct\u2122 MI355X Achieves Cost-Competitive Distributed Inference Through SGLang with MoRI"} + {"Heterogeneous CPU + GPU EPD Disaggregation to Boost VLM Serving"}

- {"May 28, 2026"} + {"May 29, 2026"}

Updating 1T parameters in seconds \u2014 P2P weight transfer in Large Scale Distributed RL - {"Updating 1T parameters in seconds \u2014 P2P weight transfer in Large Scale Distributed RL"} + {"Win on TCO: How AMD Instinct\u2122 MI355X Achieves Cost-Competitive Distributed Inference Through SGLang with MoRI"}

- {"April 29, 2026"} + {"May 28, 2026"}

DeepSeek-V4 on Day 0: From Fast Inference to Verified RL with SGLang and Miles - {"DeepSeek-V4 on Day 0: From Fast Inference to Verified RL with SGLang and Miles"} + {"Updating 1T parameters in seconds \u2014 P2P weight transfer in Large Scale Distributed RL"}

- {"April 25, 2026"} + {"April 29, 2026"}

HiSparse: Turbocharging Sparse Attention with Hierarchical Memory - {"HiSparse: Turbocharging Sparse Attention with Hierarchical Memory"} + {"DeepSeek-V4 on Day 0: From Fast Inference to Verified RL with SGLang and Miles"}

- {"April 10, 2026"} + {"April 25, 2026"}

Highlights of SGLang at NVIDIA GTC 2026 - {"Highlights of SGLang at NVIDIA GTC 2026"} + {"HiSparse: Turbocharging Sparse Attention with Hierarchical Memory"}

- {"March 31, 2026"} + {"April 10, 2026"}

Elastic EP in SGLang: Achieving Partial Failure Tolerance for DeepSeek MoE Deployments - {"Elastic EP in SGLang: Achieving Partial Failure Tolerance for DeepSeek MoE Deployments"} + {"Highlights of SGLang at NVIDIA GTC 2026"}

- {"March 25, 2026"} + {"March 31, 2026"}

From ff3f90ce2ed3d09de70d3da2ec1495e3e0d05d30 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Tue, 5 May 2026 11:33:37 +0300 Subject: [PATCH 04/16] add fine-grain L1-L2 data xfer statistics --- .../sglang/srt/managers/cache_controller.py | 236 +++++++++++++++++- python/sglang/srt/mem_cache/hiradix_cache.py | 46 +++- 2 files changed, 269 insertions(+), 13 deletions(-) diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 56057ade66f6..a4991930ef3f 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -28,6 +28,9 @@ PoolName, PoolTransfer, ) +from sglang.srt.observability.metrics_collector import ( + HiCacheL1L2TransferMetricsCollector, +) if TYPE_CHECKING: from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator @@ -146,6 +149,18 @@ class HiCacheAck(NamedTuple): finish_event: device_module.Event node_ids: List[int] + # Number of KV token slots moved by this merged operation. + token_count: int + + # Number of KV blocks moved. For HiCache this should be token_count // page_size. + block_count: int + + # Estimated total bytes moved for this operation. + byte_count: int + + # Host-side fallback timer start. Used only if device event elapsed_time is unavailable. + start_time_ns: int + class TransferBuffer: """ @@ -263,6 +278,8 @@ def __init__( pp_rank: int = 0, pp_size: int = 1, enable_storage_metrics: bool = False, + enable_metrics: bool = False, + extra_metric_labels: Optional[dict[str, str]] = None, ): self.tp_group = tp_group self.attn_cp_group = attn_cp_group @@ -286,6 +303,28 @@ def __init__( self.pp_size = pp_size self.enable_storage_metrics = enable_storage_metrics + # init L1/L2 transfer metrics collection (device-host transfers triggered by write/load). + self.enable_l1_l2_transfer_metrics = enable_metrics + self.hicache_l1_l2_transfer_metrics_collector = None + + self.hicache_l1_l2_transfer_totals = { + "offload": { + "events": 0, + "blocks": 0, + "bytes": 0, + "xfer_us": 0, + }, + "onboard": { + "events": 0, + "blocks": 0, + "bytes": 0, + "xfer_us": 0, + }, + } + + if self.enable_l1_l2_transfer_metrics: + self._init_l1_l2_transfer_metrics(extra_metric_labels) + # Draft KV pool support (best-effort piggyback on target L2/L3 ops). self.has_draft = False self.mem_pool_device_draft = None @@ -720,6 +759,9 @@ def start_writing(self) -> None: start_event = device_module.Event() finish_event = device_module.Event() + token_count = int(host_indices.numel()) + start_time_ns = time.perf_counter_ns() + start_event.record() with device_module.stream(self.write_stream): start_event.wait(self.write_stream) @@ -742,7 +784,15 @@ def start_writing(self) -> None: if device_indices.is_cuda: device_indices.record_stream(self.write_stream) - self.ack_write_queue.append(HiCacheAck(start_event, finish_event, op.node_ids)) + self.ack_write_queue.append( + self._make_hicache_ack( + start_event=start_event, + finish_event=finish_event, + node_ids=op.node_ids, + token_count=token_count, + start_time_ns=start_time_ns, + ) + ) def load( self, @@ -794,6 +844,10 @@ def start_loading(self) -> int: ) self.load_queue.clear() producer_event = self.layer_done_counter.events[producer_id] + + token_count = int(host_indices.numel()) + start_time_ns = time.perf_counter_ns() + producer_event.start_event.record() with device_module.stream(self.load_stream): @@ -824,10 +878,12 @@ def start_loading(self) -> int: device_indices.record_stream(self.load_stream) self.ack_load_queue.append( - HiCacheAck( + self._make_hicache_ack( start_event=producer_event.start_event, finish_event=producer_event.finish_event, node_ids=op.node_ids, + token_count=token_count, + start_time_ns=start_time_ns, ) ) return producer_id @@ -1234,3 +1290,179 @@ def backup_thread_func(self): except Empty: continue + + def _init_l1_l2_transfer_metrics( + self, + extra_metric_labels: Optional[dict[str, str]] = None, + ) -> None: + """Initialize Prometheus metrics for L1<->L2 transfer accounting.""" + + from sglang.srt.distributed import ( + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + ) + from sglang.srt.layers.dp_attention import ( + get_attention_dp_rank, + get_attention_tp_rank, + get_attention_tp_size, + is_dp_attention_enabled, + ) + + if is_dp_attention_enabled(): + tp_rank = get_attention_tp_rank() + tp_size = get_attention_tp_size() + dp_rank = get_attention_dp_rank() + else: + tp_rank = get_tensor_model_parallel_rank() + tp_size = get_tensor_model_parallel_world_size() + dp_rank = 0 + + attn_cp_rank, attn_cp_size = self.get_attn_cp_rank_and_size() + + labels = { + "tp_rank": str(tp_rank), + "tp_size": str(tp_size), + "dp_rank": str(dp_rank), + "pp_rank": str(self.pp_rank), + "pp_size": str(self.pp_size), + "attn_cp_rank": str(attn_cp_rank), + "attn_cp_size": str(attn_cp_size), + "io_backend": str(self.io_backend), + } + + if extra_metric_labels: + labels.update({k: str(v) for k, v in extra_metric_labels.items()}) + + self.hicache_l1_l2_transfer_metrics_collector = ( + HiCacheL1L2TransferMetricsCollector(labels) + ) + + def _host_pool_bytes_per_token(self) -> int: + """Return bytes per token slot for the host-side KV pools. + + Handles both a normal HostKVCache and HostPoolGroup-style wrappers. + """ + if hasattr(self.mem_pool_host, "entries"): + return sum( + int(entry.host_pool.size_per_token) + for entry in self.mem_pool_host.entries + ) + + return int(getattr(self.mem_pool_host, "size_per_token", 0)) + + def _estimate_l1_l2_transfer_bytes(self, token_count: int) -> int: + """Estimate total bytes moved for one L1<->L2 transfer operation.""" + bytes_per_token = self._host_pool_bytes_per_token() + total = int(token_count) * bytes_per_token + + if self.has_draft and self.mem_pool_host_draft is not None: + total += int(token_count) * int( + getattr(self.mem_pool_host_draft, "size_per_token", 0) + ) + + return total + + def _make_hicache_ack( + self, + *, + start_event: device_module.Event, + finish_event: device_module.Event, + node_ids: List[int], + token_count: int, + start_time_ns: int, + ) -> HiCacheAck: + block_count = int(token_count) // int(self.page_size) + byte_count = self._estimate_l1_l2_transfer_bytes(token_count) + + return HiCacheAck( + start_event=start_event, + finish_event=finish_event, + node_ids=node_ids, + token_count=int(token_count), + block_count=block_count, + byte_count=byte_count, + start_time_ns=start_time_ns, + ) + + def _transfer_elapsed_us(self, ack: HiCacheAck) -> int: + """Return transfer duration in microseconds. + + Prefer device event timing. Fall back to host elapsed time for devices/backends + that do not expose elapsed_time(). + """ + try: + return max(0, int(ack.start_event.elapsed_time(ack.finish_event) * 1000)) + except Exception: + return max(0, int((time.perf_counter_ns() - ack.start_time_ns) // 1000)) + + def record_l1_l2_transfer_complete( + self, + *, + direction: str, + ack: HiCacheAck, + ) -> None: + """Record logs and Prometheus metrics after a transfer ack completes. + + direction: + - "offload": L1 -> L2 + - "onboard": L2 -> L1 + """ + if direction == "offload": + action = "Offload" + src = "sglang_hicache::L1" + dst = "sglang_hicache::L2" + elif direction == "onboard": + action = "Onboard" + src = "sglang_hicache::L2" + dst = "sglang_hicache::L1" + else: + raise ValueError(f"Unknown HiCache L1/L2 transfer direction: {direction}") + + xfer_us = self._transfer_elapsed_us(ack) + ts_us = time.time_ns() // 1000 + + logger.info( + "%s transfer complete ts_us=%d blocks=%d bytes=%d xfer_us=%d " + "bandwidth=%.2fGB/s " + 'src="%s" dst="%s"', + action, + ts_us, + ack.block_count, + ack.byte_count, + xfer_us, + ack.byte_count * 0.001 / xfer_us if xfer_us > 0 else 0, + src, + dst, + ) + + if self.hicache_l1_l2_transfer_metrics_collector is not None: + self.hicache_l1_l2_transfer_metrics_collector.record_transfer( + direction=direction, + src=src, + dst=dst, + blocks=ack.block_count, + bytes_=ack.byte_count, + xfer_us=xfer_us, + ) + + totals = self.hicache_l1_l2_transfer_totals[direction] + totals["events"] += 1 + totals["blocks"] += ack.block_count + totals["bytes"] += ack.byte_count + totals["xfer_us"] += xfer_us + + logger.info( + '%s transfer cumulative direction="%s" total_events=%d ' + "total_blocks=%d total_bytes=%d total_xfer_us=%d " + "bandwidth=%.2fGB/s cumulative " + 'src="%s" dst="%s"', + action, + direction, + totals["events"], + totals["blocks"], + totals["bytes"], + totals["xfer_us"], + totals["bytes"] * 0.001 / totals["xfer_us"] if totals["xfer_us"] > 0 else 0, + src, + dst, + ) diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py index aef02c2ca5d7..f3faf53b9570 100644 --- a/python/sglang/srt/mem_cache/hiradix_cache.py +++ b/python/sglang/srt/mem_cache/hiradix_cache.py @@ -154,6 +154,8 @@ def __init__(self, params: CacheInitParams, server_args: ServerArgs): pp_rank=self.pp_rank, pp_size=self.pp_size, enable_storage_metrics=self.enable_storage_metrics, + enable_metrics=params.enable_metrics, + extra_metric_labels=self.extra_metric_labels, ) self._apply_storage_runtime_config( storage_backend=server_args.hicache_storage_backend, @@ -799,9 +801,16 @@ def writing_check(self, write_back=False): if write_back: # blocking till all write back complete while len(self.ongoing_write_through) > 0: - for _, finish_event, ack_list in self.cache_controller.ack_write_queue: - finish_event.synchronize() - for ack_id in ack_list: + for ack in self.cache_controller.ack_write_queue: + ack.finish_event.synchronize() + + # Record the L1->L2 transfer completion for this write-back operation. + self.cache_controller.record_l1_l2_transfer_complete( + direction="offload", + ack=ack, + ) + + for ack_id in ack.node_ids: node, backup_len = self.ongoing_write_through.pop(ack_id) # DMA confirmed -- block is now on host. self._record_store_event(node, medium=StorageMedium.CPU) @@ -816,8 +825,8 @@ def writing_check(self, write_back=False): return finish_count = 0 - for _, finish_event, ack_list in self.cache_controller.ack_write_queue: - if not finish_event.query(): + for ack in self.cache_controller.ack_write_queue: + if not ack.finish_event.query(): break finish_count += 1 queue_size = torch.tensor(finish_count, dtype=torch.int, device="cpu") @@ -826,9 +835,16 @@ def writing_check(self, write_back=False): finish_count = int(queue_size.item()) while finish_count > 0: - _, finish_event, ack_list = self.cache_controller.ack_write_queue.pop(0) - finish_event.synchronize() - for ack_id in ack_list: + ack = self.cache_controller.ack_write_queue.pop(0) + ack.finish_event.synchronize() + + # Record the L1->L2 transfer completion for this write-through operation. + self.cache_controller.record_l1_l2_transfer_complete( + direction="offload", + ack=ack, + ) + + for ack_id in ack.node_ids: node, backup_len = self.ongoing_write_through.pop(ack_id) # DMA confirmed -- block is now on host. self._record_store_event(node, medium=StorageMedium.CPU) @@ -839,13 +855,21 @@ def writing_check(self, write_back=False): def loading_check(self): finish_count = 0 - for _, finish_event, ack_list in self.cache_controller.ack_load_queue: - if not finish_event.query(): + for ack in self.cache_controller.ack_load_queue: + if not ack.finish_event.query(): # the KV cache loading is still ongoing break + + # ensure completion of loading, before recording transfer completion and updating cache state + ack.finish_event.synchronize() + self.cache_controller.record_l1_l2_transfer_complete( + direction="onboard", + ack=ack, + ) + finish_count += 1 # no need to sync across TP workers as batch forwarding is synced - for ack_id in ack_list: + for ack_id in ack.node_ids: end_node = self.ongoing_load_back.pop(ack_id) self.dec_lock_ref(end_node) From 5a773fa0a374a8a2a8c3b9d4ea65b80ec9a9c19c Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Tue, 5 May 2026 11:34:33 +0300 Subject: [PATCH 05/16] add fine-grain L1-L2 data xfer metrics to /metrics --- .../srt/observability/metrics_collector.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 942ad8f5e8df..0f7ad27c9732 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -1802,6 +1802,90 @@ def log_storage_metrics(self, storage_metrics: Optional[StorageMetrics] = None): self._log_histogram(self.histogram_backup_bandwidth, v) +class HiCacheL1L2TransferMetricsCollector: + """Prometheus metrics for HiCache L1<->L2 KV block transfers. + + L1: device/GPU KV cache. + L2: host/CPU KV cache. + """ + + def __init__(self, labels: Optional[dict[str, str]] = None): + from prometheus_client import Counter, Histogram + + self.labels = labels or {} + labelnames = list(self.labels.keys()) + ["direction", "src", "dst"] + + self.transfer_events_total = Counter( + "sglang:hicache_l1_l2_transfer_events_total", + "Total number of completed HiCache L1<->L2 KV block transfer events.", + labelnames=labelnames, + ) + + self.transfer_blocks_total = Counter( + "sglang:hicache_l1_l2_transfer_blocks_total", + "Total number of KV cache blocks transferred between HiCache L1 and L2.", + labelnames=labelnames, + ) + + self.transfer_bytes_total = Counter( + "sglang:hicache_l1_l2_transfer_bytes_total", + "Total number of KV cache bytes transferred between HiCache L1 and L2.", + labelnames=labelnames, + ) + + self.transfer_time_us_total = Counter( + "sglang:hicache_l1_l2_transfer_time_us_total", + "Total measured transfer time in microseconds for HiCache L1<->L2 KV block transfers.", + labelnames=labelnames, + ) + + self.transfer_duration_us = Histogram( + "sglang:hicache_l1_l2_transfer_duration_us", + "Observed duration in microseconds for one completed HiCache L1<->L2 KV block transfer.", + labelnames=labelnames, + buckets=( + 100, + 250, + 500, + 1_000, + 2_500, + 5_000, + 10_000, + 25_000, + 50_000, + 100_000, + 250_000, + 500_000, + 1_000_000, + 2_500_000, + 5_000_000, + ), + ) + + def record_transfer( + self, + *, + direction: str, + src: str, + dst: str, + blocks: int, + bytes_: int, + xfer_us: int, + ) -> None: + metric_labels = { + **self.labels, + "direction": direction, + "src": src, + "dst": dst, + } + + self.transfer_events_total.labels(**metric_labels).inc() + self.transfer_blocks_total.labels(**metric_labels).inc(blocks) + self.transfer_bytes_total.labels(**metric_labels).inc(bytes_) + self.transfer_time_us_total.labels(**metric_labels).inc(xfer_us) + self.transfer_duration_us.labels(**metric_labels).observe(xfer_us) + + class ExpertDispatchCollector(_StatLoggerDIMixin): def __init__(self, ep_size: int) -> None: from prometheus_client import Histogram as _PromHistogram From 093a2e0b5d4d2c5cd4ab7d01bebc3b73af9117b0 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Tue, 5 May 2026 11:35:17 +0300 Subject: [PATCH 06/16] add fine-grain L1-L2 data xfer statistics for hi_mamba_radix_cache --- .../srt/mem_cache/hi_mamba_radix_cache.py | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py index c3e4c7a80405..667b2e8a226c 100644 --- a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py +++ b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py @@ -383,9 +383,9 @@ def writing_check(self, write_back=False): if write_back: # blocking till all write back complete while len(self.ongoing_write_through) > 0: - for _, finish_event, ack_list in self.cache_controller.ack_write_queue: - finish_event.synchronize() - for ack_id in ack_list: + for ack in self.cache_controller.ack_write_queue: + ack.finish_event.synchronize() + for ack_id in ack.node_ids: backuped_node = self.ongoing_write_through.pop(ack_id) self._record_store_event( backuped_node, medium=StorageMedium.CPU @@ -400,8 +400,8 @@ def writing_check(self, write_back=False): return finish_count = 0 - for _, finish_event, ack_list in self.cache_controller.ack_write_queue: - if not finish_event.query(): + for ack in self.cache_controller.ack_write_queue: + if not ack.finish_event.query(): break finish_count += 1 @@ -415,9 +415,16 @@ def writing_check(self, write_back=False): finish_count = int(queue_size.item()) while finish_count > 0: - _, finish_event, ack_list = self.cache_controller.ack_write_queue.pop(0) - finish_event.synchronize() - for ack_id in ack_list: + ack = self.cache_controller.ack_write_queue.pop(0) + ack.finish_event.synchronize() + + # Record the L1->L2 transfer completion for this write-back operation. + self.cache_controller.record_l1_l2_transfer_complete( + direction="offload", + ack=ack, + ) + + for ack_id in ack.node_ids: backuped_node = self.ongoing_write_through.pop(ack_id) self._record_store_event(backuped_node, medium=StorageMedium.CPU) self.dec_lock_ref(backuped_node) @@ -427,12 +434,20 @@ def writing_check(self, write_back=False): def loading_check(self): finish_count = 0 - for _, finish_event, ack_list in self.cache_controller.ack_load_queue: - if not finish_event.query(): + for ack in self.cache_controller.ack_load_queue: + if not ack.finish_event.query(): # the KV cache loading is still ongoing break + + # ensure completion of loading, before recording transfer completion and updating cache state + ack.finish_event.synchronize() + self.cache_controller.record_l1_l2_transfer_complete( + direction="onboard", + ack=ack, + ) + finish_count += 1 - for ack_id in ack_list: + for ack_id in ack.node_ids: end_node = self.ongoing_load_back.pop(ack_id) self.dec_lock_ref(end_node) From 3a7eab590b4826b4e7ff54e2fae93077a62726b6 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Tue, 5 May 2026 11:36:01 +0300 Subject: [PATCH 07/16] add fine-grain L1-L2 data xfer statistics for hybrid_cache --- .../hybrid_cache/hybrid_cache_controller.py | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index cf14fb6c3283..0411ab3711ee 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -11,9 +11,6 @@ import torch from sglang.srt.managers.cache_controller import CacheOperation as BaseCacheOperation -from sglang.srt.managers.cache_controller import ( - HiCacheAck, -) from sglang.srt.managers.cache_controller import ( HiCacheController as BaseHiCacheController, ) @@ -172,6 +169,8 @@ def __init__( pp_size: int = 1, transfer_layer_num: Optional[int] = None, enable_storage_metrics: bool = False, + enable_metrics: bool = False, + extra_metric_labels: Optional[dict[str, str]] = None, ): startup_storage_backend = storage_backend self.extra_host_mem_release_queues: dict[PoolName, Queue[torch.Tensor]] = {} @@ -192,6 +191,8 @@ def __init__( pp_rank=pp_rank, pp_size=pp_size, enable_storage_metrics=enable_storage_metrics, + enable_metrics=enable_metrics, + extra_metric_labels=extra_metric_labels, ) # Override layer_num: hybrid models transfer all layers (For example, Linear Model (KV + Mamba)), # not just the full attention layers reported by full_kv_pool. @@ -401,6 +402,10 @@ def start_writing(self) -> None: self.write_queue.clear() start_event = device_module.Event() finish_event = device_module.Event() + + token_count = int(host_indices.numel()) + start_time_ns = time.perf_counter_ns() + start_event.record() with device_module.stream(self.write_stream): start_event.wait(self.write_stream) @@ -418,7 +423,15 @@ def start_writing(self) -> None: device_indices, resolved_pool_transfers, ) - self.ack_write_queue.append(HiCacheAck(start_event, finish_event, op.node_ids)) + self.ack_write_queue.append( + self._make_hicache_ack( + start_event=start_event, + finish_event=finish_event, + node_ids=op.node_ids, + token_count=token_count, + start_time_ns=start_time_ns, + ) + ) def load( self, @@ -473,6 +486,10 @@ def start_loading(self) -> int: ) self.load_queue.clear() producer_event = self.layer_done_counter.events[producer_id] + + token_count = int(host_indices.numel()) + start_time_ns = time.perf_counter_ns() + producer_event.start_event.record() with device_module.stream(self.load_stream): producer_event.start_event.wait(self.load_stream) @@ -493,10 +510,12 @@ def start_loading(self) -> int: resolved_pool_transfers, ) self.ack_load_queue.append( - HiCacheAck( - producer_event.start_event, - producer_event.finish_event, - op.node_ids, + self._make_hicache_ack( + start_event=producer_event.start_event, + finish_event=producer_event.finish_event, + node_ids=op.node_ids, + token_count=token_count, + start_time_ns=start_time_ns, ) ) return producer_id From d617dc888fa37d6fe3bc33bcbbb7486611f16f4d Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Tue, 5 May 2026 11:36:30 +0300 Subject: [PATCH 08/16] add fine-grain L1-L2 data xfer statistics for unified_radix_cache --- .../srt/mem_cache/unified_radix_cache.py | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index c9ba6ec86637..2fa02bbd38cb 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -2138,9 +2138,9 @@ def writing_check(self, write_back: bool = False) -> None: if write_back: # Blocking: wait for all pending write-backs while self.ongoing_write_through: - for _, finish_event, ack_list in cc.ack_write_queue: - finish_event.synchronize() - for ack_id in ack_list: + for ack in cc.ack_write_queue: + ack.finish_event.synchronize() + for ack_id in ack.node_ids: entry = self.ongoing_write_through.pop(ack_id, None) if entry is not None: node, params = entry @@ -2157,8 +2157,8 @@ def writing_check(self, write_back: bool = False) -> None: return finish_count = 0 - for _, finish_event, ack_list in cc.ack_write_queue: - if not finish_event.query(): + for ack in cc.ack_write_queue: + if not ack.finish_event.query(): break finish_count += 1 @@ -2169,9 +2169,16 @@ def writing_check(self, write_back: bool = False) -> None: # Process completed acks while finish_count > 0: - _, finish_event, ack_list = cc.ack_write_queue.pop(0) - finish_event.synchronize() - for ack_id in ack_list: + ack = cc.ack_write_queue.pop(0) + ack.finish_event.synchronize() + + # Post-ack callback for cache controller (e.g. to trigger next write-backs) + self.cache_controller.record_l1_l2_transfer_complete( + direction="onboard", + ack=ack, + ) + + for ack_id in ack.node_ids: node, params = self.ongoing_write_through.pop(ack_id) self._record_store_event(node, medium=StorageMedium.CPU) self.dec_lock_ref(node, params) @@ -2185,11 +2192,19 @@ def loading_check(self) -> None: if cc is None or not self.ongoing_load_back: return finish_count = 0 - for _, finish_event, ack_list in cc.ack_load_queue: - if not finish_event.query(): + for ack in cc.ack_load_queue: + if not ack.finish_event.query(): break + + # ensure completion of loading, before recording transfer completion and updating cache state + ack.finish_event.synchronize() + self.cache_controller.record_l1_l2_transfer_complete( + direction="onboard", + ack=ack, + ) + finish_count += 1 - for ack_id in ack_list: + for ack_id in ack.node_ids: node, lock_params = self.ongoing_load_back.pop(ack_id) self.dec_lock_ref(node, lock_params) del cc.ack_load_queue[:finish_count] From 27e9395fb8191fb27e6b47e959a00c2171efa01f Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Tue, 2 Jun 2026 01:21:21 +0300 Subject: [PATCH 09/16] fix mislabeled direction --- python/sglang/srt/mem_cache/unified_radix_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 2fa02bbd38cb..c5cad50a457a 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -2174,7 +2174,7 @@ def writing_check(self, write_back: bool = False) -> None: # Post-ack callback for cache controller (e.g. to trigger next write-backs) self.cache_controller.record_l1_l2_transfer_complete( - direction="onboard", + direction="offload", ack=ack, ) From c3661198359554dfcbb49df89141e65a8b167ed3 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Tue, 2 Jun 2026 01:40:22 +0300 Subject: [PATCH 10/16] show fine-grain transfer statistics in debug mode --- python/sglang/srt/managers/cache_controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index a4991930ef3f..f77fef192968 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -1421,7 +1421,7 @@ def record_l1_l2_transfer_complete( xfer_us = self._transfer_elapsed_us(ack) ts_us = time.time_ns() // 1000 - logger.info( + logger.debug( "%s transfer complete ts_us=%d blocks=%d bytes=%d xfer_us=%d " "bandwidth=%.2fGB/s " 'src="%s" dst="%s"', @@ -1451,7 +1451,7 @@ def record_l1_l2_transfer_complete( totals["bytes"] += ack.byte_count totals["xfer_us"] += xfer_us - logger.info( + logger.debug( '%s transfer cumulative direction="%s" total_events=%d ' "total_blocks=%d total_bytes=%d total_xfer_us=%d " "bandwidth=%.2fGB/s cumulative " From 351255935f3b28acd57f7f38838ca3bb66c57c1c Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Wed, 3 Jun 2026 10:19:03 -0700 Subject: [PATCH 11/16] add fine-grain xfer statistics to disagg --- .../disaggregation/decode_kvcache_offload_manager.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py b/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py index bccdd809b5c4..c6230c860b74 100644 --- a/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py +++ b/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py @@ -219,9 +219,13 @@ def check_offload_progress(self): def _check_offload_progress(self, finish_count): """Check the progress of offload from device to host.""" while finish_count > 0: - _, finish_event, ack_list = self.cache_controller.ack_write_queue.pop(0) - finish_event.synchronize() - for ack_id in ack_list: + ack = self.cache_controller.ack_write_queue.pop(0) + ack.finish_event.synchronize() + self.cache_controller.record_l1_l2_transfer_complete( + direction="offload", + ack=ack, + ) + for ack_id in ack.node_ids: ( req, host_indices, From fca6b1696f5d0cca5caca8ffd5dac42f2795c567 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Wed, 3 Jun 2026 10:21:03 -0700 Subject: [PATCH 12/16] Fix HiCacheAck test consumers --- .../test_specv2_kvcache_offloading.py | 13 +++++++++---- .../mem_cache/test_unified_radix_cache_unittest.py | 12 ++++++------ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/test/registered/disaggregation/test_specv2_kvcache_offloading.py b/test/registered/disaggregation/test_specv2_kvcache_offloading.py index 0cd5c77bdc64..7351aee00550 100644 --- a/test/registered/disaggregation/test_specv2_kvcache_offloading.py +++ b/test/registered/disaggregation/test_specv2_kvcache_offloading.py @@ -8,6 +8,7 @@ """ import unittest +from types import SimpleNamespace from unittest.mock import MagicMock import torch @@ -90,6 +91,10 @@ def synchronize(self): pass +def _ack(node_id: int): + return SimpleNamespace(finish_event=_FinishedEvent(), node_ids=[node_id]) + + class TestReleaseFinishedReq(unittest.TestCase): """Tests for _release_finished_req overallocation cleanup.""" @@ -292,7 +297,7 @@ def test_unfinished_offload_ack_does_not_free_incremental_slots(self): 8, ) manager.cache_controller = MagicMock() - manager.cache_controller.ack_write_queue = [(None, _FinishedEvent(), [7])] + manager.cache_controller.ack_write_queue = [_ack(7)] manager._trigger_backup = MagicMock(return_value="last_hash") manager._check_offload_progress(1) @@ -326,7 +331,7 @@ def test_offload_kv_cache_tracks_inflight_write_until_ack(self): self.assertEqual(manager.offloaded_state[req.rid].inc_len, 4) manager.cache_controller.write.assert_called_once() - manager.cache_controller.ack_write_queue = [(None, _FinishedEvent(), [1])] + manager.cache_controller.ack_write_queue = [_ack(1)] manager._trigger_backup = MagicMock(return_value="last_hash") manager._check_offload_progress(1) @@ -369,7 +374,7 @@ def test_finished_offload_ack_waits_for_other_inflight_writes(self): 8, ) manager.cache_controller = MagicMock() - manager.cache_controller.ack_write_queue = [(None, _FinishedEvent(), [8])] + manager.cache_controller.ack_write_queue = [_ack(8)] manager._trigger_backup = MagicMock(return_value="last_hash") manager._check_offload_progress(1) @@ -399,7 +404,7 @@ def test_finished_request_releases_all_committed_slots_after_last_offload_ack( 12, ) manager.cache_controller = MagicMock() - manager.cache_controller.ack_write_queue = [(None, _FinishedEvent(), [9])] + manager.cache_controller.ack_write_queue = [_ack(9)] manager._trigger_backup = MagicMock(return_value="last_hash") manager._check_offload_progress(1) diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 4cb5319658e7..4d2de0f99f87 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -390,8 +390,8 @@ def _load_back_node(self, tree, node): self.assertTrue(loaded) producer_id = tree.ready_to_load_host_cache() self.assertNotEqual(producer_id, -1) - for _, finish_event, _ in list(tree.cache_controller.ack_load_queue): - finish_event.synchronize() + for ack in list(tree.cache_controller.ack_load_queue): + ack.finish_event.synchronize() tree.loading_check() def test_kv_events_store_and_remove_full_blocks(self): @@ -1914,8 +1914,8 @@ def _load_back_node(self, tree, node): self.assertTrue(loaded) producer_id = tree.ready_to_load_host_cache() self.assertNotEqual(producer_id, -1) - for _, finish_event, _ in list(tree.cache_controller.ack_load_queue): - finish_event.synchronize() + for ack in list(tree.cache_controller.ack_load_queue): + ack.finish_event.synchronize() tree.loading_check() return node.component_data[ComponentType.FULL].value @@ -2383,8 +2383,8 @@ def _release_ongoing_load_back_locks(self, tree): def _finish_pending_loads(self, tree): producer_id = tree.ready_to_load_host_cache() self.assertNotEqual(producer_id, -1) - for _, finish_event, _ in list(tree.cache_controller.ack_load_queue): - finish_event.synchronize() + for ack in list(tree.cache_controller.ack_load_queue): + ack.finish_event.synchronize() tree.loading_check() def _match_tokens_for_chain(self, chain): From 5dfd7ed65122cf3854387c74c00ca1f0844db4e8 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Wed, 3 Jun 2026 10:21:11 -0700 Subject: [PATCH 13/16] Skip HiCache transfer timing when disabled --- .../sglang/srt/managers/cache_controller.py | 83 ++++++++++--------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index f77fef192968..5dc502aa175c 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -1407,6 +1407,13 @@ def record_l1_l2_transfer_complete( - "offload": L1 -> L2 - "onboard": L2 -> L1 """ + should_log = logger.isEnabledFor(logging.DEBUG) + should_record_metrics = ( + self.hicache_l1_l2_transfer_metrics_collector is not None + ) + if not should_log and not should_record_metrics: + return + if direction == "offload": action = "Offload" src = "sglang_hicache::L1" @@ -1419,23 +1426,24 @@ def record_l1_l2_transfer_complete( raise ValueError(f"Unknown HiCache L1/L2 transfer direction: {direction}") xfer_us = self._transfer_elapsed_us(ack) - ts_us = time.time_ns() // 1000 - - logger.debug( - "%s transfer complete ts_us=%d blocks=%d bytes=%d xfer_us=%d " - "bandwidth=%.2fGB/s " - 'src="%s" dst="%s"', - action, - ts_us, - ack.block_count, - ack.byte_count, - xfer_us, - ack.byte_count * 0.001 / xfer_us if xfer_us > 0 else 0, - src, - dst, - ) - if self.hicache_l1_l2_transfer_metrics_collector is not None: + if should_log: + ts_us = time.time_ns() // 1000 + logger.debug( + "%s transfer complete ts_us=%d blocks=%d bytes=%d xfer_us=%d " + "bandwidth=%.2fGB/s " + 'src="%s" dst="%s"', + action, + ts_us, + ack.block_count, + ack.byte_count, + xfer_us, + ack.byte_count * 0.001 / xfer_us if xfer_us > 0 else 0, + src, + dst, + ) + + if should_record_metrics: self.hicache_l1_l2_transfer_metrics_collector.record_transfer( direction=direction, src=src, @@ -1445,24 +1453,25 @@ def record_l1_l2_transfer_complete( xfer_us=xfer_us, ) - totals = self.hicache_l1_l2_transfer_totals[direction] - totals["events"] += 1 - totals["blocks"] += ack.block_count - totals["bytes"] += ack.byte_count - totals["xfer_us"] += xfer_us - - logger.debug( - '%s transfer cumulative direction="%s" total_events=%d ' - "total_blocks=%d total_bytes=%d total_xfer_us=%d " - "bandwidth=%.2fGB/s cumulative " - 'src="%s" dst="%s"', - action, - direction, - totals["events"], - totals["blocks"], - totals["bytes"], - totals["xfer_us"], - totals["bytes"] * 0.001 / totals["xfer_us"] if totals["xfer_us"] > 0 else 0, - src, - dst, - ) + if should_log: + totals = self.hicache_l1_l2_transfer_totals[direction] + totals["events"] += 1 + totals["blocks"] += ack.block_count + totals["bytes"] += ack.byte_count + totals["xfer_us"] += xfer_us + + logger.debug( + '%s transfer cumulative direction="%s" total_events=%d ' + "total_blocks=%d total_bytes=%d total_xfer_us=%d " + "bandwidth=%.2fGB/s cumulative " + 'src="%s" dst="%s"', + action, + direction, + totals["events"], + totals["blocks"], + totals["bytes"], + totals["xfer_us"], + totals["bytes"] * 0.001 / totals["xfer_us"] if totals["xfer_us"] > 0 else 0, + src, + dst, + ) From 9420afc3aa1df4f393372e524ca91439ce868baf Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Wed, 3 Jun 2026 10:21:21 -0700 Subject: [PATCH 14/16] Record blocking HiCache write-back transfers --- python/sglang/srt/mem_cache/hi_mamba_radix_cache.py | 4 ++++ python/sglang/srt/mem_cache/unified_radix_cache.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py index 667b2e8a226c..5abea09304cc 100644 --- a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py +++ b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py @@ -385,6 +385,10 @@ def writing_check(self, write_back=False): while len(self.ongoing_write_through) > 0: for ack in self.cache_controller.ack_write_queue: ack.finish_event.synchronize() + self.cache_controller.record_l1_l2_transfer_complete( + direction="offload", + ack=ack, + ) for ack_id in ack.node_ids: backuped_node = self.ongoing_write_through.pop(ack_id) self._record_store_event( diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index c5cad50a457a..b98016471a47 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -2140,6 +2140,10 @@ def writing_check(self, write_back: bool = False) -> None: while self.ongoing_write_through: for ack in cc.ack_write_queue: ack.finish_event.synchronize() + cc.record_l1_l2_transfer_complete( + direction="offload", + ack=ack, + ) for ack_id in ack.node_ids: entry = self.ongoing_write_through.pop(ack_id, None) if entry is not None: From 23c4982f590f016b1f11af18c4b6697138b22898 Mon Sep 17 00:00:00 2001 From: Harry Xie Date: Wed, 3 Jun 2026 10:21:30 -0700 Subject: [PATCH 15/16] Remove redundant HiCache transfer counters --- .../sglang/srt/observability/metrics_collector.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 0f7ad27c9732..82cfb97ffe7e 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -1815,12 +1815,6 @@ def __init__(self, labels: Optional[dict[str, str]] = None): self.labels = labels or {} labelnames = list(self.labels.keys()) + ["direction", "src", "dst"] - self.transfer_events_total = Counter( - "sglang:hicache_l1_l2_transfer_events_total", - "Total number of completed HiCache L1<->L2 KV block transfer events.", - labelnames=labelnames, - ) - self.transfer_blocks_total = Counter( "sglang:hicache_l1_l2_transfer_blocks_total", "Total number of KV cache blocks transferred between HiCache L1 and L2.", @@ -1833,12 +1827,6 @@ def __init__(self, labels: Optional[dict[str, str]] = None): labelnames=labelnames, ) - self.transfer_time_us_total = Counter( - "sglang:hicache_l1_l2_transfer_time_us_total", - "Total measured transfer time in microseconds for HiCache L1<->L2 KV block transfers.", - labelnames=labelnames, - ) - self.transfer_duration_us = Histogram( "sglang:hicache_l1_l2_transfer_duration_us", "Observed duration in microseconds for one completed HiCache L1<->L2 KV block transfer.", @@ -1879,10 +1867,8 @@ def record_transfer( "dst": dst, } - self.transfer_events_total.labels(**metric_labels).inc() self.transfer_blocks_total.labels(**metric_labels).inc(blocks) self.transfer_bytes_total.labels(**metric_labels).inc(bytes_) - self.transfer_time_us_total.labels(**metric_labels).inc(xfer_us) self.transfer_duration_us.labels(**metric_labels).observe(xfer_us) From b570cabbae9ffc9dee8b86e9dd9c86e68a899357 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Thu, 4 Jun 2026 16:28:32 +0000 Subject: [PATCH 16/16] Fix HiCache transfer timing and enable_metrics threading 1. Transfer durations were always host wall-clock, never device time. Every event pair fed to elapsed_time() was created without enable_timing=True, so on CUDA elapsed_time() raises 'Both events must be created with enable_timing=True' and _transfer_elapsed_us silently falls into its except-fallback. The fallback measures enqueue -> ack-poll latency, not DMA time: an idle onboard logged xfer_us=5751205 (5.75 s, 0.03 GB/s) for a 161 MB transfer that takes ~25 ms, because the ack sat until the next request triggered loading_check. Create the pairs used for timing (write start/finish, LayerLoadingEvent start + last layer event) with enable_timing=True so the device path actually runs. 2. enable_metrics was only threaded into HiRadixCache's controller. The decode-side offload manager and all five HybridCacheController construction sites in hybrid_pool_assembler.py left it defaulted to False, so disagg-decode / hybrid / unified stacks recorded DEBUG logs but never Prometheus, even with --enable-metrics. Pass enable_metrics and extra_metric_labels through. 3. Drop the unrelated docs_new/index.mdx blog-card rotation (same change was already removed from the sibling PR #26976). --- docs_new/index.mdx | 60 +++++++++---------- .../decode_kvcache_offload_manager.py | 2 + .../sglang/srt/managers/cache_controller.py | 18 ++++-- .../hybrid_cache/hybrid_cache_controller.py | 6 +- .../hybrid_cache/hybrid_pool_assembler.py | 10 ++++ 5 files changed, 60 insertions(+), 36 deletions(-) diff --git a/docs_new/index.mdx b/docs_new/index.mdx index 7eeeb55b1dfc..908d517386da 100644 --- a/docs_new/index.mdx +++ b/docs_new/index.mdx @@ -83,7 +83,7 @@ It is designed to deliver low-latency and high-throughput inference across a wid }} > Heterogeneous CPU + GPU EPD Disaggregation to Boost VLM Serving - {"Heterogeneous CPU + GPU EPD Disaggregation to Boost VLM Serving"} + {"Updating 1T parameters in seconds \u2014 P2P weight transfer in Large Scale Distributed RL"}

- {"May 29, 2026"} + {"April 29, 2026"}

Win on TCO: How AMD Instinct\u2122 MI355X Achieves Cost-Competitive Distributed Inference Through SGLang with MoRI - {"Win on TCO: How AMD Instinct\u2122 MI355X Achieves Cost-Competitive Distributed Inference Through SGLang with MoRI"} + {"DeepSeek-V4 on Day 0: From Fast Inference to Verified RL with SGLang and Miles"}

- {"May 28, 2026"} + {"April 25, 2026"}

Updating 1T parameters in seconds \u2014 P2P weight transfer in Large Scale Distributed RL - {"Updating 1T parameters in seconds \u2014 P2P weight transfer in Large Scale Distributed RL"} + {"HiSparse: Turbocharging Sparse Attention with Hierarchical Memory"}

- {"April 29, 2026"} + {"April 10, 2026"}

DeepSeek-V4 on Day 0: From Fast Inference to Verified RL with SGLang and Miles - {"DeepSeek-V4 on Day 0: From Fast Inference to Verified RL with SGLang and Miles"} + {"Highlights of SGLang at NVIDIA GTC 2026"}

- {"April 25, 2026"} + {"March 31, 2026"}

HiSparse: Turbocharging Sparse Attention with Hierarchical Memory - {"HiSparse: Turbocharging Sparse Attention with Hierarchical Memory"} + {"Elastic EP in SGLang: Achieving Partial Failure Tolerance for DeepSeek MoE Deployments"}

- {"April 10, 2026"} + {"March 25, 2026"}

Highlights of SGLang at NVIDIA GTC 2026 - {"Highlights of SGLang at NVIDIA GTC 2026"} + {"ROCm Support for Miles: Large-Scale RL Post-Training on AMD Instinct\u2122 GPUs"}

- {"March 31, 2026"} + {"March 17, 2026"}

diff --git a/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py b/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py index c6230c860b74..a2385411c882 100644 --- a/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py +++ b/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py @@ -99,6 +99,8 @@ def __init__( storage_backend=server_args.hicache_storage_backend, model_name=server_args.served_model_name, storage_backend_extra_config=hicache_storage_backend_extra_config, + enable_metrics=server_args.enable_metrics, + extra_metric_labels=server_args.extra_metric_labels, ) self.ongoing_offload = {} diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 5dc502aa175c..871b300b3d5e 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -57,8 +57,16 @@ class LayerLoadingEvent: def __init__(self, num_layers: int): self._num_layers = num_layers - self.load_events = [device_module.Event() for _ in range(num_layers)] - self.start_event = device_module.Event() # start event on controller stream + # The last layer's event doubles as finish_event; together with + # start_event it is used for elapsed_time() in transfer metrics, which + # requires both events to be created with enable_timing=True. + self.load_events = [ + device_module.Event(enable_timing=(i == num_layers - 1)) + for i in range(num_layers) + ] + self.start_event = device_module.Event( + enable_timing=True + ) # start event on controller stream def complete(self, layer_index: int): assert 0 <= layer_index < self._num_layers @@ -756,8 +764,10 @@ def start_writing(self) -> None: ) self.write_queue.clear() - start_event = device_module.Event() - finish_event = device_module.Event() + # enable_timing so record_l1_l2_transfer_complete can use + # elapsed_time() for the actual transfer duration. + start_event = device_module.Event(enable_timing=True) + finish_event = device_module.Event(enable_timing=True) token_count = int(host_indices.numel()) start_time_ns = time.perf_counter_ns() diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index 0411ab3711ee..e533140b6acd 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -400,8 +400,10 @@ def start_writing(self) -> None: self.move_hybrid_indices(op) ) self.write_queue.clear() - start_event = device_module.Event() - finish_event = device_module.Event() + # enable_timing so record_l1_l2_transfer_complete can use + # elapsed_time() for the actual transfer duration. + start_event = device_module.Event(enable_timing=True) + finish_event = device_module.Event(enable_timing=True) token_count = int(host_indices.numel()) start_time_ns = time.perf_counter_ns() diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py index e12c9d350372..007a785fa3bb 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py @@ -152,6 +152,8 @@ def build_kv_only_stack( pp_size=pp_size, transfer_layer_num=transfer_layer_num, enable_storage_metrics=enable_storage_metrics, + enable_metrics=params.enable_metrics, + extra_metric_labels=server_args.extra_metric_labels, ) return host_pool_group, cache_controller @@ -236,6 +238,8 @@ def build_hybrid_swa_stack( pp_size=pp_size, transfer_layer_num=transfer_layer_num, enable_storage_metrics=enable_storage_metrics, + enable_metrics=params.enable_metrics, + extra_metric_labels=server_args.extra_metric_labels, ) return host_pool_group, cache_controller @@ -489,6 +493,8 @@ def build_deepseek_v4_hicache_stack( pp_size=pp_size, transfer_layer_num=transfer_layer_num, enable_storage_metrics=enable_storage_metrics, + enable_metrics=params.enable_metrics, + extra_metric_labels=server_args.extra_metric_labels, ) return host_pool_group, cache_controller @@ -569,6 +575,8 @@ def build_hybrid_mamba_stack( pp_size=pp_size, transfer_layer_num=transfer_layer_num, enable_storage_metrics=enable_storage_metrics, + enable_metrics=params.enable_metrics, + extra_metric_labels=server_args.extra_metric_labels, ) return host_pool_group, cache_controller @@ -641,6 +649,8 @@ def build_anchor_sidecar_stack( pp_size=pp_size, transfer_layer_num=transfer_layer_num, enable_storage_metrics=enable_storage_metrics, + enable_metrics=params.enable_metrics, + extra_metric_labels=server_args.extra_metric_labels, ) return host_pool_group, cache_controller