From 93ba338e6972c2cd54026ac91da59b0fe32b191a Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Tue, 5 May 2026 11:33:37 +0300
Subject: [PATCH 01/28] add fine-grain L1-L2 data xfer statistics

---
 .../sglang/srt/managers/cache_controller.py   | 236 +++++++++++++++++-
 python/sglang/srt/mem_cache/hiradix_cache.py  |  81 +++---
 2 files changed, 285 insertions(+), 32 deletions(-)

diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py
index ca276bc11479..0c675cdaa645 100644
--- a/python/sglang/srt/managers/cache_controller.py
+++ b/python/sglang/srt/managers/cache_controller.py
@@ -28,6 +28,9 @@
     PoolName,
     PoolTransfer,
 )
+from sglang.srt.observability.metrics_collector import (
+    HiCacheL1L2TransferMetricsCollector,
+)
 
 if TYPE_CHECKING:
     from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
@@ -148,6 +151,18 @@ class HiCacheAck(NamedTuple):
     finish_event: device_module.Event
     node_ids: List[int]
 
+    # Number of KV token slots moved by this merged operation.
+    token_count: int
+
+    # Number of KV blocks moved. For HiCache this should be token_count // page_size.
+    block_count: int
+
+    # Estimated total bytes moved for this operation.
+    byte_count: int
+
+    # Host-side fallback timer start. Used only if device event elapsed_time is unavailable.
+    start_time_ns: int
+
 
 class StorageOperation:
     counter = 0
@@ -225,6 +240,8 @@ def __init__(
         model_name: Optional[str] = None,
         storage_backend_extra_config: Optional[dict] = None,
         enable_storage_metrics: bool = False,
+        enable_metrics: bool = False,
+        extra_metric_labels: Optional[dict[str, str]] = None,
     ):
         self.tp_group = tp_group
         self.attn_cp_group = attn_cp_group
@@ -247,6 +264,28 @@ def __init__(
         self.storage_backend_type = None
         self.enable_storage_metrics = enable_storage_metrics
 
+        # init L1/L2 transfer metrics collection (device-host transfers triggered by write/load).
+        self.enable_l1_l2_transfer_metrics = enable_metrics
+        self.hicache_l1_l2_transfer_metrics_collector = None
+
+        self.hicache_l1_l2_transfer_totals = {
+            "offload": {
+                "events": 0,
+                "blocks": 0,
+                "bytes": 0,
+                "xfer_us": 0,
+            },
+            "onboard": {
+                "events": 0,
+                "blocks": 0,
+                "bytes": 0,
+                "xfer_us": 0,
+            },
+        }
+
+        if self.enable_l1_l2_transfer_metrics:
+            self._init_l1_l2_transfer_metrics(extra_metric_labels)
+
         # Draft KV pool support (best-effort piggyback on target L2/L3 ops).
         self.has_draft = False
         self.mem_pool_device_draft = None
@@ -692,6 +731,9 @@ def start_writing(self) -> None:
         start_event = device_module.Event()
         finish_event = device_module.Event()
 
+        token_count = int(host_indices.numel())
+        start_time_ns = time.perf_counter_ns()
+
         start_event.record()
         with device_module.stream(self.write_stream):
             start_event.wait(self.write_stream)
@@ -714,7 +756,15 @@ def start_writing(self) -> None:
             if device_indices.is_cuda:
                 device_indices.record_stream(self.write_stream)
 
-        self.ack_write_queue.append(HiCacheAck(start_event, finish_event, op.node_ids))
+        self.ack_write_queue.append(
+            self._make_hicache_ack(
+                start_event=start_event,
+                finish_event=finish_event,
+                node_ids=op.node_ids,
+                token_count=token_count,
+                start_time_ns=start_time_ns,
+            )
+        )
 
     def load(
         self,
@@ -766,6 +816,10 @@ def start_loading(self) -> int:
         )
         self.load_queue.clear()
         producer_event = self.layer_done_counter.events[producer_id]
+
+        token_count = int(host_indices.numel())
+        start_time_ns = time.perf_counter_ns()
+
         producer_event.start_event.record()
 
         with device_module.stream(self.load_stream):
@@ -796,10 +850,12 @@ def start_loading(self) -> int:
                 device_indices.record_stream(self.load_stream)
 
         self.ack_load_queue.append(
-            HiCacheAck(
+            self._make_hicache_ack(
                 start_event=producer_event.start_event,
                 finish_event=producer_event.finish_event,
                 node_ids=op.node_ids,
+                token_count=token_count,
+                start_time_ns=start_time_ns,
             )
         )
         return producer_id
@@ -1206,3 +1262,179 @@ def backup_thread_func(self):
 
             except Empty:
                 continue
+
+    def _init_l1_l2_transfer_metrics(
+        self,
+        extra_metric_labels: Optional[dict[str, str]] = None,
+    ) -> None:
+        """Initialize Prometheus metrics for L1<->L2 transfer accounting."""
+
+        from sglang.srt.distributed import (
+            get_tensor_model_parallel_rank,
+            get_tensor_model_parallel_world_size,
+        )
+        from sglang.srt.layers.dp_attention import (
+            get_attention_dp_rank,
+            get_attention_tp_rank,
+            get_attention_tp_size,
+            is_dp_attention_enabled,
+        )
+
+        if is_dp_attention_enabled():
+            tp_rank = get_attention_tp_rank()
+            tp_size = get_attention_tp_size()
+            dp_rank = get_attention_dp_rank()
+        else:
+            tp_rank = get_tensor_model_parallel_rank()
+            tp_size = get_tensor_model_parallel_world_size()
+            dp_rank = 0
+
+        attn_cp_rank, attn_cp_size = self.get_attn_cp_rank_and_size()
+
+        labels = {
+            "tp_rank": str(tp_rank),
+            "tp_size": str(tp_size),
+            "dp_rank": str(dp_rank),
+            "pp_rank": str(self.pp_rank),
+            "pp_size": str(self.pp_size),
+            "attn_cp_rank": str(attn_cp_rank),
+            "attn_cp_size": str(attn_cp_size),
+            "io_backend": str(self.io_backend),
+        }
+
+        if extra_metric_labels:
+            labels.update({k: str(v) for k, v in extra_metric_labels.items()})
+
+        self.hicache_l1_l2_transfer_metrics_collector = (
+            HiCacheL1L2TransferMetricsCollector(labels)
+        )
+
+    def _host_pool_bytes_per_token(self) -> int:
+        """Return bytes per token slot for the host-side KV pools.
+
+        Handles both a normal HostKVCache and HostPoolGroup-style wrappers.
+        """
+        if hasattr(self.mem_pool_host, "entries"):
+            return sum(
+                int(entry.host_pool.size_per_token)
+                for entry in self.mem_pool_host.entries
+            )
+
+        return int(getattr(self.mem_pool_host, "size_per_token", 0))
+
+    def _estimate_l1_l2_transfer_bytes(self, token_count: int) -> int:
+        """Estimate total bytes moved for one L1<->L2 transfer operation."""
+        bytes_per_token = self._host_pool_bytes_per_token()
+        total = int(token_count) * bytes_per_token
+
+        if self.has_draft and self.mem_pool_host_draft is not None:
+            total += int(token_count) * int(
+                getattr(self.mem_pool_host_draft, "size_per_token", 0)
+            )
+
+        return total
+
+    def _make_hicache_ack(
+        self,
+        *,
+        start_event: device_module.Event,
+        finish_event: device_module.Event,
+        node_ids: List[int],
+        token_count: int,
+        start_time_ns: int,
+    ) -> HiCacheAck:
+        block_count = int(token_count) // int(self.page_size)
+        byte_count = self._estimate_l1_l2_transfer_bytes(token_count)
+
+        return HiCacheAck(
+            start_event=start_event,
+            finish_event=finish_event,
+            node_ids=node_ids,
+            token_count=int(token_count),
+            block_count=block_count,
+            byte_count=byte_count,
+            start_time_ns=start_time_ns,
+        )
+
+    def _transfer_elapsed_us(self, ack: HiCacheAck) -> int:
+        """Return transfer duration in microseconds.
+
+        Prefer device event timing. Fall back to host elapsed time for devices/backends
+        that do not expose elapsed_time().
+        """
+        try:
+            return max(0, int(ack.start_event.elapsed_time(ack.finish_event) * 1000))
+        except Exception:
+            return max(0, int((time.perf_counter_ns() - ack.start_time_ns) // 1000))
+
+    def record_l1_l2_transfer_complete(
+        self,
+        *,
+        direction: str,
+        ack: HiCacheAck,
+    ) -> None:
+        """Record logs and Prometheus metrics after a transfer ack completes.
+
+        direction:
+        - "offload": L1 -> L2
+        - "onboard": L2 -> L1
+        """
+        if direction == "offload":
+            action = "Offload"
+            src = "sglang_hicache::L1"
+            dst = "sglang_hicache::L2"
+        elif direction == "onboard":
+            action = "Onboard"
+            src = "sglang_hicache::L2"
+            dst = "sglang_hicache::L1"
+        else:
+            raise ValueError(f"Unknown HiCache L1/L2 transfer direction: {direction}")
+
+        xfer_us = self._transfer_elapsed_us(ack)
+        ts_us = time.time_ns() // 1000
+
+        logger.info(
+            "%s transfer complete ts_us=%d blocks=%d bytes=%d xfer_us=%d "
+            "bandwidth=%.2fGB/s "
+            'src="%s" dst="%s"',
+            action,
+            ts_us,
+            ack.block_count,
+            ack.byte_count,
+            xfer_us,
+            ack.byte_count * 0.001 / xfer_us if xfer_us > 0 else 0,
+            src,
+            dst,
+        )
+
+        if self.hicache_l1_l2_transfer_metrics_collector is not None:
+            self.hicache_l1_l2_transfer_metrics_collector.record_transfer(
+                direction=direction,
+                src=src,
+                dst=dst,
+                blocks=ack.block_count,
+                bytes_=ack.byte_count,
+                xfer_us=xfer_us,
+            )
+
+        totals = self.hicache_l1_l2_transfer_totals[direction]
+        totals["events"] += 1
+        totals["blocks"] += ack.block_count
+        totals["bytes"] += ack.byte_count
+        totals["xfer_us"] += xfer_us
+
+        logger.info(
+            '%s transfer cumulative direction="%s" total_events=%d '
+            "total_blocks=%d total_bytes=%d total_xfer_us=%d "
+            "bandwidth=%.2fGB/s cumulative "
+            'src="%s" dst="%s"',
+            action,
+            direction,
+            totals["events"],
+            totals["blocks"],
+            totals["bytes"],
+            totals["xfer_us"],
+            totals["bytes"] * 0.001 / totals["xfer_us"] if totals["xfer_us"] > 0 else 0,
+            src,
+            dst,
+        )
diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py
index 92e842ad51cb..48cc407cc916 100644
--- a/python/sglang/srt/mem_cache/hiradix_cache.py
+++ b/python/sglang/srt/mem_cache/hiradix_cache.py
@@ -157,6 +157,8 @@ def __init__(self, params: CacheInitParams, server_args: ServerArgs):
                 model_name=server_args.served_model_name,
                 storage_backend_extra_config=extra_config,
                 enable_storage_metrics=self.enable_storage_metrics,
+                enable_metrics=params.enable_metrics,
+                extra_metric_labels=self.extra_metric_labels,
             )
         self._apply_storage_runtime_config(
             storage_backend=server_args.hicache_storage_backend,
@@ -910,10 +912,20 @@ def writing_check(self, write_back=False):
         if write_back:
             # blocking till all write back complete
             while len(self.ongoing_write_through) > 0:
-                for _, finish_event, ack_list in self.cache_controller.ack_write_queue:
-                    finish_event.synchronize()
-                    for ack_id in ack_list:
-                        self._finish_write_through_ack(ack_id, release_lock=False)
+                for ack in self.cache_controller.ack_write_queue:
+                    ack.finish_event.synchronize()
+                    matched = False
+                    for ack_id in ack.node_ids:
+                        if ack_id in self.ongoing_write_through:
+                            self._finish_write_through_ack(
+                                ack_id, release_lock=False
+                            )
+                            matched = True
+                    if matched:
+                        self.cache_controller.record_l1_l2_transfer_complete(
+                            direction="offload",
+                            ack=ack,
+                        )
                 self.cache_controller.ack_write_queue.clear()
                 assert len(self.ongoing_write_through) == 0
             return
@@ -923,44 +935,53 @@ def writing_check(self, write_back=False):
         # host memory pressure), so a conditional skip desyncs the NCCL op
         # sequence and deadlocks under TP > 1. (Matches UnifiedRadixCache.)
         finish_count = 0
-        if self.pp_rank == 0:
-            for _, finish_event, ack_list in self.cache_controller.ack_write_queue:
-                if not finish_event.query():
-                    break
-                finish_count += 1
+        for ack in self.cache_controller.ack_write_queue:
+            if not ack.finish_event.query():
+                break
+            finish_count += 1
         finish_count_tensor = torch.tensor(finish_count, dtype=torch.int, device="cpu")
-        self._all_reduce(finish_count_tensor, torch.distributed.ReduceOp.MIN)
-        finish_count = finish_count_tensor.item()
+        # Keep cache state transitions identical across CPxTP participants.
+        self._all_reduce_attn_groups(finish_count_tensor, torch.distributed.ReduceOp.MIN)
+        finish_count = int(finish_count_tensor.item())
 
         if finish_count > 0:
             logger.debug(f"Process {finish_count} write back operations")
         while finish_count > 0:
-            _, finish_event, ack_list = self.cache_controller.ack_write_queue.pop(0)
-            finish_event.synchronize()
-            for ack_id in ack_list:
-                self._finish_write_through_ack(ack_id, release_lock=True)
+            ack = self.cache_controller.ack_write_queue.pop(0)
+            ack.finish_event.synchronize()
+            matched = False
+            for ack_id in ack.node_ids:
+                if ack_id in self.ongoing_write_through:
+                    self._finish_write_through_ack(ack_id, release_lock=True)
+                    matched = True
+            if matched:
+                self.cache_controller.record_l1_l2_transfer_complete(
+                    direction="offload",
+                    ack=ack,
+                )
             finish_count -= 1
 
     def loading_check(self):
         finish_count = 0
-        if self.pp_rank == 0:
-            for _, finish_event, ack_list in self.cache_controller.ack_load_queue:
-                if not finish_event.query():
-                    break
-                finish_count += 1
-        finish_count_tensor = torch.tensor(finish_count, dtype=torch.int, device="cpu")
-        self._all_reduce(finish_count_tensor, torch.distributed.ReduceOp.MIN)
-        finish_count = finish_count_tensor.item()
+        for ack in self.cache_controller.ack_load_queue:
+            if not ack.finish_event.query():
+                # the KV cache loading is still ongoing
+                break
 
-        if finish_count > 0:
-            logger.debug(f"Process {finish_count} load operations")
-        while finish_count > 0:
-            _, finish_event, ack_list = self.cache_controller.ack_load_queue.pop(0)
-            finish_event.synchronize()
-            for ack_id in ack_list:
+            # ensure completion of loading, before recording transfer completion and updating cache state
+            ack.finish_event.synchronize()
+            self.cache_controller.record_l1_l2_transfer_complete(
+                direction="onboard",
+                ack=ack,
+            )
+
+            finish_count += 1
+            # no need to sync across TP workers as batch forwarding is synced
+            for ack_id in ack.node_ids:
                 end_node = self.ongoing_load_back.pop(ack_id)
                 self.dec_lock_ref(end_node)
-            finish_count -= 1
+
+        del self.cache_controller.ack_load_queue[:finish_count]
 
     def is_load_back_event_done(self, consumer_index: int) -> bool:
         """Return True after the local load-back event is complete."""

From 47cacf068f8cbb2d3311b38d9ac03f632f2af908 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Tue, 5 May 2026 11:34:33 +0300
Subject: [PATCH 02/28] add fine-grain L1-L2 data xfer metrics to /metrics

---
 .../srt/observability/metrics_collector.py    | 84 +++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py
index d1a9cca04e28..64a6a51a5f7f 100644
--- a/python/sglang/srt/observability/metrics_collector.py
+++ b/python/sglang/srt/observability/metrics_collector.py
@@ -1826,6 +1826,90 @@ def log_storage_metrics(self, storage_metrics: Optional[StorageMetrics] = None):
             self._log_histogram(self.histogram_backup_bandwidth, v)
 
 
+class HiCacheL1L2TransferMetricsCollector:
+    """Prometheus metrics for HiCache L1<->L2 KV block transfers.
+
+    L1: device/GPU KV cache.
+    L2: host/CPU KV cache.
+    """
+
+    def __init__(self, labels: Optional[dict[str, str]] = None):
+        from prometheus_client import Counter, Histogram
+
+        self.labels = labels or {}
+        labelnames = list(self.labels.keys()) + ["direction", "src", "dst"]
+
+        self.transfer_events_total = Counter(
+            "sglang:hicache_l1_l2_transfer_events_total",
+            "Total number of completed HiCache L1<->L2 KV block transfer events.",
+            labelnames=labelnames,
+        )
+
+        self.transfer_blocks_total = Counter(
+            "sglang:hicache_l1_l2_transfer_blocks_total",
+            "Total number of KV cache blocks transferred between HiCache L1 and L2.",
+            labelnames=labelnames,
+        )
+
+        self.transfer_bytes_total = Counter(
+            "sglang:hicache_l1_l2_transfer_bytes_total",
+            "Total number of KV cache bytes transferred between HiCache L1 and L2.",
+            labelnames=labelnames,
+        )
+
+        self.transfer_time_us_total = Counter(
+            "sglang:hicache_l1_l2_transfer_time_us_total",
+            "Total measured transfer time in microseconds for HiCache L1<->L2 KV block transfers.",
+            labelnames=labelnames,
+        )
+
+        self.transfer_duration_us = Histogram(
+            "sglang:hicache_l1_l2_transfer_duration_us",
+            "Observed duration in microseconds for one completed HiCache L1<->L2 KV block transfer.",
+            labelnames=labelnames,
+            buckets=(
+                100,
+                250,
+                500,
+                1_000,
+                2_500,
+                5_000,
+                10_000,
+                25_000,
+                50_000,
+                100_000,
+                250_000,
+                500_000,
+                1_000_000,
+                2_500_000,
+                5_000_000,
+            ),
+        )
+
+    def record_transfer(
+        self,
+        *,
+        direction: str,
+        src: str,
+        dst: str,
+        blocks: int,
+        bytes_: int,
+        xfer_us: int,
+    ) -> None:
+        metric_labels = {
+            **self.labels,
+            "direction": direction,
+            "src": src,
+            "dst": dst,
+        }
+
+        self.transfer_events_total.labels(**metric_labels).inc()
+        self.transfer_blocks_total.labels(**metric_labels).inc(blocks)
+        self.transfer_bytes_total.labels(**metric_labels).inc(bytes_)
+        self.transfer_time_us_total.labels(**metric_labels).inc(xfer_us)
+        self.transfer_duration_us.labels(**metric_labels).observe(xfer_us)
+
+
 class ExpertDispatchCollector(_StatLoggerDIMixin):
     def __init__(self, ep_size: int) -> None:
         from prometheus_client import Histogram as _PromHistogram

From 6836c8941afddddf727ccf0a71b1896692698d14 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Tue, 5 May 2026 11:35:17 +0300
Subject: [PATCH 03/28] add fine-grain L1-L2 data xfer statistics for
 hi_mamba_radix_cache

---
 .../srt/mem_cache/hi_mamba_radix_cache.py     | 36 +++++++++++++------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
index 381af58be2a7..68b20d9cbcb2 100644
--- a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
+++ b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
@@ -383,9 +383,9 @@ def writing_check(self, write_back=False):
         if write_back:
             # blocking till all write back complete
             while len(self.ongoing_write_through) > 0:
-                for _, finish_event, ack_list in self.cache_controller.ack_write_queue:
-                    finish_event.synchronize()
-                    for ack_id in ack_list:
+                for ack in self.cache_controller.ack_write_queue:
+                    ack.finish_event.synchronize()
+                    for ack_id in ack.node_ids:
                         backuped_node = self.ongoing_write_through.pop(ack_id)
                         self._record_store_event(
                             backuped_node, medium=StorageMedium.CPU
@@ -401,8 +401,8 @@ def writing_check(self, write_back=False):
         # independently (no cross-rank sync).
         finish_count = 0
         if len(self.ongoing_write_through) > 0:
-            for _, finish_event, ack_list in self.cache_controller.ack_write_queue:
-                if not finish_event.query():
+            for ack in self.cache_controller.ack_write_queue:
+                if not ack.finish_event.query():
                     break
                 finish_count += 1
 
@@ -416,9 +416,16 @@ def writing_check(self, write_back=False):
         finish_count = int(queue_size.item())
 
         while finish_count > 0:
-            _, finish_event, ack_list = self.cache_controller.ack_write_queue.pop(0)
-            finish_event.synchronize()
-            for ack_id in ack_list:
+            ack = self.cache_controller.ack_write_queue.pop(0)
+            ack.finish_event.synchronize()
+
+            # Record the L1->L2 transfer completion for this write-back operation.
+            self.cache_controller.record_l1_l2_transfer_complete(
+                direction="offload",
+                ack=ack,
+            )
+
+            for ack_id in ack.node_ids:
                 backuped_node = self.ongoing_write_through.pop(ack_id)
                 self._record_store_event(backuped_node, medium=StorageMedium.CPU)
                 self.dec_lock_ref(backuped_node)
@@ -430,9 +437,18 @@ def loading_check(self):
         # Every rank must enter the all_reduce below; ongoing_load_back can
         # diverge across ranks.
         finish_count = 0
-        for _, finish_event, ack_list in self.cache_controller.ack_load_queue:
-            if not finish_event.query():
+        for ack in self.cache_controller.ack_load_queue:
+            if not ack.finish_event.query():
+                # the KV cache loading is still ongoing
                 break
+
+            # ensure completion of loading, before recording transfer completion and updating cache state
+            ack.finish_event.synchronize()
+            self.cache_controller.record_l1_l2_transfer_complete(
+                direction="onboard",
+                ack=ack,
+            )
+
             finish_count += 1
 
         queue_size = torch.tensor(finish_count, dtype=torch.int, device="cpu")

From 8daf3698f4b898452a0d90c23fa699d5211ddcfb Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Tue, 5 May 2026 11:36:01 +0300
Subject: [PATCH 04/28] add fine-grain L1-L2 data xfer statistics for
 hybrid_cache

---
 .../hybrid_cache/hybrid_cache_controller.py   | 35 ++++++++++++++-----
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py
index 011a877e79ff..76e937de2f74 100644
--- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py
+++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py
@@ -11,9 +11,6 @@
 import torch
 
 from sglang.srt.managers.cache_controller import CacheOperation as BaseCacheOperation
-from sglang.srt.managers.cache_controller import (
-    HiCacheAck,
-)
 from sglang.srt.managers.cache_controller import (
     HiCacheController as BaseHiCacheController,
 )
@@ -172,6 +169,8 @@ def __init__(
         storage_backend_extra_config: Optional[dict] = None,
         transfer_layer_num: Optional[int] = None,
         enable_storage_metrics: bool = False,
+        enable_metrics: bool = False,
+        extra_metric_labels: Optional[dict[str, str]] = None,
     ):
         startup_storage_backend = storage_backend
         self.extra_host_mem_release_queues: dict[PoolName, Queue[torch.Tensor]] = {}
@@ -191,6 +190,8 @@ def __init__(
             model_name=model_name,
             storage_backend_extra_config=storage_backend_extra_config,
             enable_storage_metrics=enable_storage_metrics,
+            enable_metrics=enable_metrics,
+            extra_metric_labels=extra_metric_labels,
         )
         # Override layer_num: hybrid models transfer all layers (For example, Linear Model (KV + Mamba)),
         # not just the full attention layers reported by full_kv_pool.
@@ -410,6 +411,10 @@ def start_writing(self) -> None:
         self.write_queue.clear()
         start_event = device_module.Event()
         finish_event = device_module.Event()
+
+        token_count = int(host_indices.numel())
+        start_time_ns = time.perf_counter_ns()
+
         start_event.record()
         with device_module.stream(self.write_stream):
             start_event.wait(self.write_stream)
@@ -434,7 +439,15 @@ def start_writing(self) -> None:
                 device_indices,
                 resolved_pool_transfers,
             )
-        self.ack_write_queue.append(HiCacheAck(start_event, finish_event, op.node_ids))
+        self.ack_write_queue.append(
+            self._make_hicache_ack(
+                start_event=start_event,
+                finish_event=finish_event,
+                node_ids=op.node_ids,
+                token_count=token_count,
+                start_time_ns=start_time_ns,
+            )
+        )
 
     def load(
         self,
@@ -489,6 +502,10 @@ def start_loading(self) -> int:
         )
         self.load_queue.clear()
         producer_event = self.layer_done_counter.events[producer_id]
+
+        token_count = int(host_indices.numel())
+        start_time_ns = time.perf_counter_ns()
+
         producer_event.start_event.record()
         with device_module.stream(self.load_stream):
             producer_event.start_event.wait(self.load_stream)
@@ -521,10 +538,12 @@ def start_loading(self) -> int:
                 resolved_pool_transfers,
             )
         self.ack_load_queue.append(
-            HiCacheAck(
-                producer_event.start_event,
-                producer_event.finish_event,
-                op.node_ids,
+            self._make_hicache_ack(
+                start_event=producer_event.start_event,
+                finish_event=producer_event.finish_event,
+                node_ids=op.node_ids,
+                token_count=token_count,
+                start_time_ns=start_time_ns,
             )
         )
         return producer_id

From da90db6363468ccb7305fcbb52f01532016ad137 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Tue, 5 May 2026 11:36:30 +0300
Subject: [PATCH 05/28] add fine-grain L1-L2 data xfer statistics for
 unified_radix_cache

---
 python/sglang/srt/mem_cache/hiradix_cache.py  | 36 +++++++------
 .../srt/mem_cache/unified_radix_cache.py      | 53 +++++++++++++------
 2 files changed, 56 insertions(+), 33 deletions(-)

diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py
index 48cc407cc916..ce5cbf117857 100644
--- a/python/sglang/srt/mem_cache/hiradix_cache.py
+++ b/python/sglang/srt/mem_cache/hiradix_cache.py
@@ -935,13 +935,13 @@ def writing_check(self, write_back=False):
         # host memory pressure), so a conditional skip desyncs the NCCL op
         # sequence and deadlocks under TP > 1. (Matches UnifiedRadixCache.)
         finish_count = 0
-        for ack in self.cache_controller.ack_write_queue:
-            if not ack.finish_event.query():
-                break
-            finish_count += 1
+        if self.pp_rank == 0:
+            for ack in self.cache_controller.ack_write_queue:
+                if not ack.finish_event.query():
+                    break
+                finish_count += 1
         finish_count_tensor = torch.tensor(finish_count, dtype=torch.int, device="cpu")
-        # Keep cache state transitions identical across CPxTP participants.
-        self._all_reduce_attn_groups(finish_count_tensor, torch.distributed.ReduceOp.MIN)
+        self._all_reduce(finish_count_tensor, torch.distributed.ReduceOp.MIN)
         finish_count = int(finish_count_tensor.item())
 
         if finish_count > 0:
@@ -963,25 +963,29 @@ def writing_check(self, write_back=False):
 
     def loading_check(self):
         finish_count = 0
-        for ack in self.cache_controller.ack_load_queue:
-            if not ack.finish_event.query():
-                # the KV cache loading is still ongoing
-                break
+        if self.pp_rank == 0:
+            for ack in self.cache_controller.ack_load_queue:
+                if not ack.finish_event.query():
+                    break
+                finish_count += 1
+        finish_count_tensor = torch.tensor(finish_count, dtype=torch.int, device="cpu")
+        self._all_reduce(finish_count_tensor, torch.distributed.ReduceOp.MIN)
+        finish_count = int(finish_count_tensor.item())
 
-            # ensure completion of loading, before recording transfer completion and updating cache state
+        if finish_count > 0:
+            logger.debug(f"Process {finish_count} load operations")
+        while finish_count > 0:
+            ack = self.cache_controller.ack_load_queue.pop(0)
             ack.finish_event.synchronize()
+            # ensure completion of loading, before recording transfer completion and updating cache state
             self.cache_controller.record_l1_l2_transfer_complete(
                 direction="onboard",
                 ack=ack,
             )
-
-            finish_count += 1
-            # no need to sync across TP workers as batch forwarding is synced
             for ack_id in ack.node_ids:
                 end_node = self.ongoing_load_back.pop(ack_id)
                 self.dec_lock_ref(end_node)
-
-        del self.cache_controller.ack_load_queue[:finish_count]
+            finish_count -= 1
 
     def is_load_back_event_done(self, consumer_index: int) -> bool:
         """Return True after the local load-back event is complete."""
diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py
index fa26bbee505b..6debad03a287 100644
--- a/python/sglang/srt/mem_cache/unified_radix_cache.py
+++ b/python/sglang/srt/mem_cache/unified_radix_cache.py
@@ -2349,11 +2349,18 @@ def writing_check(self, write_back: bool = False) -> None:
         if write_back:
             # Blocking: wait for all pending write-backs
             while self.ongoing_write_through:
-                for _, finish_event, ack_list in cc.ack_write_queue:
-                    finish_event.synchronize()
-                    for ack_id in ack_list:
+                for ack in cc.ack_write_queue:
+                    ack.finish_event.synchronize()
+                    matched = False
+                    for ack_id in ack.node_ids:
                         if ack_id in self.ongoing_write_through:
                             self._finish_write_through_ack(ack_id)
+                            matched = True
+                    if matched:
+                        cc.record_l1_l2_transfer_complete(
+                            direction="offload",
+                            ack=ack,
+                        )
                 cc.ack_write_queue.clear()
                 assert len(self.ongoing_write_through) == 0
             return
@@ -2362,21 +2369,28 @@ def writing_check(self, write_back: bool = False) -> None:
         # diverge across ranks (e.g. write_backup returning 0 on a subset).
         finish_count = 0
         if self.pp_rank == 0:
-            for _, finish_event, ack_list in cc.ack_write_queue:
-                if not finish_event.query():
+            for ack in cc.ack_write_queue:
+                if not ack.finish_event.query():
                     break
                 finish_count += 1
-
         finish_count_tensor = torch.tensor(finish_count, dtype=torch.int, device="cpu")
         self._all_reduce(finish_count_tensor, torch.distributed.ReduceOp.MIN)
-        finish_count = finish_count_tensor.item()
+        finish_count = int(finish_count_tensor.item())
 
         # Process completed acks
         while finish_count > 0:
-            _, finish_event, ack_list = cc.ack_write_queue.pop(0)
-            finish_event.synchronize()
-            for ack_id in ack_list:
-                self._finish_write_through_ack(ack_id)
+            ack = cc.ack_write_queue.pop(0)
+            ack.finish_event.synchronize()
+            matched = False
+            for ack_id in ack.node_ids:
+                if ack_id in self.ongoing_write_through:
+                    self._finish_write_through_ack(ack_id)
+                    matched = True
+            if matched:
+                self.cache_controller.record_l1_l2_transfer_complete(
+                    direction="offload",
+                    ack=ack,
+                )
             finish_count -= 1
 
     def loading_check(self) -> None:
@@ -2388,18 +2402,23 @@ def loading_check(self) -> None:
         # diverge across ranks.
         finish_count = 0
         if self.pp_rank == 0:
-            for _, finish_event, ack_list in cc.ack_load_queue:
-                if not finish_event.query():
+            for ack in cc.ack_load_queue:
+                if not ack.finish_event.query():
                     break
                 finish_count += 1
         finish_count_tensor = torch.tensor(finish_count, dtype=torch.int, device="cpu")
         self._all_reduce(finish_count_tensor, torch.distributed.ReduceOp.MIN)
-        finish_count = finish_count_tensor.item()
+        finish_count = int(finish_count_tensor.item())
 
         while finish_count > 0:
-            _, finish_event, ack_list = cc.ack_load_queue.pop(0)
-            finish_event.synchronize()
-            for ack_id in ack_list:
+            ack = cc.ack_load_queue.pop(0)
+            ack.finish_event.synchronize()
+            # ensure completion of loading, before recording transfer completion and updating cache state
+            self.cache_controller.record_l1_l2_transfer_complete(
+                direction="onboard",
+                ack=ack,
+            )
+            for ack_id in ack.node_ids:
                 node, lock_params, host_lock_params = self.ongoing_load_back.pop(ack_id)
                 self.dec_lock_ref(node, lock_params)
                 self.dec_host_lock_ref(node, host_lock_params)

From dc852407eab35ec18a0c05bea1d44c12d5d43784 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Tue, 2 Jun 2026 01:40:22 +0300
Subject: [PATCH 06/28] show fine-grain transfer statistics in debug mode

---
 python/sglang/srt/managers/cache_controller.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py
index 0c675cdaa645..6b8b78bfc1a4 100644
--- a/python/sglang/srt/managers/cache_controller.py
+++ b/python/sglang/srt/managers/cache_controller.py
@@ -1393,7 +1393,7 @@ def record_l1_l2_transfer_complete(
         xfer_us = self._transfer_elapsed_us(ack)
         ts_us = time.time_ns() // 1000
 
-        logger.info(
+        logger.debug(
             "%s transfer complete ts_us=%d blocks=%d bytes=%d xfer_us=%d "
             "bandwidth=%.2fGB/s "
             'src="%s" dst="%s"',
@@ -1423,7 +1423,7 @@ def record_l1_l2_transfer_complete(
         totals["bytes"] += ack.byte_count
         totals["xfer_us"] += xfer_us
 
-        logger.info(
+        logger.debug(
             '%s transfer cumulative direction="%s" total_events=%d '
             "total_blocks=%d total_bytes=%d total_xfer_us=%d "
             "bandwidth=%.2fGB/s cumulative "

From bb3729068c0ab41f576f64fc3e4c1c9220b78a68 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Wed, 3 Jun 2026 10:19:03 -0700
Subject: [PATCH 07/28] add fine-grain xfer statistics to disagg

---
 .../disaggregation/decode_kvcache_offload_manager.py   | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py b/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py
index 6321efb2e94d..c49dee7b175b 100644
--- a/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py
+++ b/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py
@@ -219,9 +219,13 @@ def check_offload_progress(self):
     def _check_offload_progress(self, finish_count):
         """Check the progress of offload from device to host."""
         while finish_count > 0:
-            _, finish_event, ack_list = self.cache_controller.ack_write_queue.pop(0)
-            finish_event.synchronize()
-            for ack_id in ack_list:
+            ack = self.cache_controller.ack_write_queue.pop(0)
+            ack.finish_event.synchronize()
+            self.cache_controller.record_l1_l2_transfer_complete(
+                direction="offload",
+                ack=ack,
+            )
+            for ack_id in ack.node_ids:
                 (
                     req,
                     host_indices,

From 0fa635c6c07209a6817808d88467bbcff47ef751 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Wed, 3 Jun 2026 10:21:03 -0700
Subject: [PATCH 08/28] Fix HiCacheAck test consumers

---
 .../test_specv2_kvcache_offloading.py               | 13 +++++++++----
 .../mem_cache/test_unified_radix_cache_unittest.py  | 12 ++++++------
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/test/registered/disaggregation/test_specv2_kvcache_offloading.py b/test/registered/disaggregation/test_specv2_kvcache_offloading.py
index 296138eaa673..91a4ba6fd8b7 100644
--- a/test/registered/disaggregation/test_specv2_kvcache_offloading.py
+++ b/test/registered/disaggregation/test_specv2_kvcache_offloading.py
@@ -8,6 +8,7 @@
 """
 
 import unittest
+from types import SimpleNamespace
 from unittest.mock import MagicMock
 
 import torch
@@ -91,6 +92,10 @@ def synchronize(self):
         pass
 
 
+def _ack(node_id: int):
+    return SimpleNamespace(finish_event=_FinishedEvent(), node_ids=[node_id])
+
+
 class TestReleaseFinishedReq(unittest.TestCase):
     """Tests for _release_finished_req overallocation cleanup."""
 
@@ -293,7 +298,7 @@ def test_unfinished_offload_ack_does_not_free_incremental_slots(self):
             8,
         )
         manager.cache_controller = MagicMock()
-        manager.cache_controller.ack_write_queue = [(None, _FinishedEvent(), [7])]
+        manager.cache_controller.ack_write_queue = [_ack(7)]
         manager._trigger_backup = MagicMock(return_value="last_hash")
 
         manager._check_offload_progress(1)
@@ -327,7 +332,7 @@ def test_offload_kv_cache_tracks_inflight_write_until_ack(self):
         self.assertEqual(manager.offloaded_state[req.rid].inc_len, 4)
         manager.cache_controller.write.assert_called_once()
 
-        manager.cache_controller.ack_write_queue = [(None, _FinishedEvent(), [1])]
+        manager.cache_controller.ack_write_queue = [_ack(1)]
         manager._trigger_backup = MagicMock(return_value="last_hash")
 
         manager._check_offload_progress(1)
@@ -370,7 +375,7 @@ def test_finished_offload_ack_waits_for_other_inflight_writes(self):
             8,
         )
         manager.cache_controller = MagicMock()
-        manager.cache_controller.ack_write_queue = [(None, _FinishedEvent(), [8])]
+        manager.cache_controller.ack_write_queue = [_ack(8)]
         manager._trigger_backup = MagicMock(return_value="last_hash")
 
         manager._check_offload_progress(1)
@@ -400,7 +405,7 @@ def test_finished_request_releases_all_committed_slots_after_last_offload_ack(
             12,
         )
         manager.cache_controller = MagicMock()
-        manager.cache_controller.ack_write_queue = [(None, _FinishedEvent(), [9])]
+        manager.cache_controller.ack_write_queue = [_ack(9)]
         manager._trigger_backup = MagicMock(return_value="last_hash")
 
         manager._check_offload_progress(1)
diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py
index 67bcf88f504b..17d6fbc360ab 100644
--- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py
+++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py
@@ -497,8 +497,8 @@ def _load_back_node(self, tree, node):
         self.assertTrue(loaded)
         producer_id = tree.ready_to_load_host_cache()
         self.assertNotEqual(producer_id, -1)
-        for _, finish_event, _ in list(tree.cache_controller.ack_load_queue):
-            finish_event.synchronize()
+        for ack in list(tree.cache_controller.ack_load_queue):
+            ack.finish_event.synchronize()
         tree.loading_check()
 
     def test_kv_events_store_and_remove_full_blocks(self):
@@ -2792,8 +2792,8 @@ def _load_back_node(self, tree, node):
         self.assertTrue(loaded)
         producer_id = tree.ready_to_load_host_cache()
         self.assertNotEqual(producer_id, -1)
-        for _, finish_event, _ in list(tree.cache_controller.ack_load_queue):
-            finish_event.synchronize()
+        for ack in list(tree.cache_controller.ack_load_queue):
+            ack.finish_event.synchronize()
         tree.loading_check()
         return node.component_data[ComponentType.FULL].value
 
@@ -3294,8 +3294,8 @@ def _release_ongoing_load_back_locks(self, tree):
     def _finish_pending_loads(self, tree):
         producer_id = tree.ready_to_load_host_cache()
         self.assertNotEqual(producer_id, -1)
-        for _, finish_event, _ in list(tree.cache_controller.ack_load_queue):
-            finish_event.synchronize()
+        for ack in list(tree.cache_controller.ack_load_queue):
+            ack.finish_event.synchronize()
         tree.loading_check()
 
     def _match_tokens_for_chain(self, chain):

From 47563ca43fc409bf7ba7e72bee3d079e35ecdf9d Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Wed, 3 Jun 2026 10:21:11 -0700
Subject: [PATCH 09/28] Skip HiCache transfer timing when disabled

---
 .../sglang/srt/managers/cache_controller.py   | 83 ++++++++++---------
 1 file changed, 46 insertions(+), 37 deletions(-)

diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py
index 6b8b78bfc1a4..b4025294392b 100644
--- a/python/sglang/srt/managers/cache_controller.py
+++ b/python/sglang/srt/managers/cache_controller.py
@@ -1379,6 +1379,13 @@ def record_l1_l2_transfer_complete(
         - "offload": L1 -> L2
         - "onboard": L2 -> L1
         """
+        should_log = logger.isEnabledFor(logging.DEBUG)
+        should_record_metrics = (
+            self.hicache_l1_l2_transfer_metrics_collector is not None
+        )
+        if not should_log and not should_record_metrics:
+            return
+
         if direction == "offload":
             action = "Offload"
             src = "sglang_hicache::L1"
@@ -1391,23 +1398,24 @@ def record_l1_l2_transfer_complete(
             raise ValueError(f"Unknown HiCache L1/L2 transfer direction: {direction}")
 
         xfer_us = self._transfer_elapsed_us(ack)
-        ts_us = time.time_ns() // 1000
-
-        logger.debug(
-            "%s transfer complete ts_us=%d blocks=%d bytes=%d xfer_us=%d "
-            "bandwidth=%.2fGB/s "
-            'src="%s" dst="%s"',
-            action,
-            ts_us,
-            ack.block_count,
-            ack.byte_count,
-            xfer_us,
-            ack.byte_count * 0.001 / xfer_us if xfer_us > 0 else 0,
-            src,
-            dst,
-        )
 
-        if self.hicache_l1_l2_transfer_metrics_collector is not None:
+        if should_log:
+            ts_us = time.time_ns() // 1000
+            logger.debug(
+                "%s transfer complete ts_us=%d blocks=%d bytes=%d xfer_us=%d "
+                "bandwidth=%.2fGB/s "
+                'src="%s" dst="%s"',
+                action,
+                ts_us,
+                ack.block_count,
+                ack.byte_count,
+                xfer_us,
+                ack.byte_count * 0.001 / xfer_us if xfer_us > 0 else 0,
+                src,
+                dst,
+            )
+
+        if should_record_metrics:
             self.hicache_l1_l2_transfer_metrics_collector.record_transfer(
                 direction=direction,
                 src=src,
@@ -1417,24 +1425,25 @@ def record_l1_l2_transfer_complete(
                 xfer_us=xfer_us,
             )
 
-        totals = self.hicache_l1_l2_transfer_totals[direction]
-        totals["events"] += 1
-        totals["blocks"] += ack.block_count
-        totals["bytes"] += ack.byte_count
-        totals["xfer_us"] += xfer_us
-
-        logger.debug(
-            '%s transfer cumulative direction="%s" total_events=%d '
-            "total_blocks=%d total_bytes=%d total_xfer_us=%d "
-            "bandwidth=%.2fGB/s cumulative "
-            'src="%s" dst="%s"',
-            action,
-            direction,
-            totals["events"],
-            totals["blocks"],
-            totals["bytes"],
-            totals["xfer_us"],
-            totals["bytes"] * 0.001 / totals["xfer_us"] if totals["xfer_us"] > 0 else 0,
-            src,
-            dst,
-        )
+        if should_log:
+            totals = self.hicache_l1_l2_transfer_totals[direction]
+            totals["events"] += 1
+            totals["blocks"] += ack.block_count
+            totals["bytes"] += ack.byte_count
+            totals["xfer_us"] += xfer_us
+
+            logger.debug(
+                '%s transfer cumulative direction="%s" total_events=%d '
+                "total_blocks=%d total_bytes=%d total_xfer_us=%d "
+                "bandwidth=%.2fGB/s cumulative "
+                'src="%s" dst="%s"',
+                action,
+                direction,
+                totals["events"],
+                totals["blocks"],
+                totals["bytes"],
+                totals["xfer_us"],
+                totals["bytes"] * 0.001 / totals["xfer_us"] if totals["xfer_us"] > 0 else 0,
+                src,
+                dst,
+            )

From 2bbd28e98a60471e15216a369b2bf8b51a6b127b Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Wed, 3 Jun 2026 10:21:21 -0700
Subject: [PATCH 10/28] Record blocking HiCache write-back transfers

---
 python/sglang/srt/mem_cache/hi_mamba_radix_cache.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
index 68b20d9cbcb2..7f278e2172bb 100644
--- a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
+++ b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
@@ -385,6 +385,10 @@ def writing_check(self, write_back=False):
             while len(self.ongoing_write_through) > 0:
                 for ack in self.cache_controller.ack_write_queue:
                     ack.finish_event.synchronize()
+                    self.cache_controller.record_l1_l2_transfer_complete(
+                        direction="offload",
+                        ack=ack,
+                    )
                     for ack_id in ack.node_ids:
                         backuped_node = self.ongoing_write_through.pop(ack_id)
                         self._record_store_event(

From 70df64bcbc70684e7b5676330965e594a37d0f35 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Wed, 3 Jun 2026 10:21:30 -0700
Subject: [PATCH 11/28] Remove redundant HiCache transfer counters

---
 .../sglang/srt/observability/metrics_collector.py  | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py
index 64a6a51a5f7f..4fbdc11fc5fc 100644
--- a/python/sglang/srt/observability/metrics_collector.py
+++ b/python/sglang/srt/observability/metrics_collector.py
@@ -1839,12 +1839,6 @@ def __init__(self, labels: Optional[dict[str, str]] = None):
         self.labels = labels or {}
         labelnames = list(self.labels.keys()) + ["direction", "src", "dst"]
 
-        self.transfer_events_total = Counter(
-            "sglang:hicache_l1_l2_transfer_events_total",
-            "Total number of completed HiCache L1<->L2 KV block transfer events.",
-            labelnames=labelnames,
-        )
-
         self.transfer_blocks_total = Counter(
             "sglang:hicache_l1_l2_transfer_blocks_total",
             "Total number of KV cache blocks transferred between HiCache L1 and L2.",
@@ -1857,12 +1851,6 @@ def __init__(self, labels: Optional[dict[str, str]] = None):
             labelnames=labelnames,
         )
 
-        self.transfer_time_us_total = Counter(
-            "sglang:hicache_l1_l2_transfer_time_us_total",
-            "Total measured transfer time in microseconds for HiCache L1<->L2 KV block transfers.",
-            labelnames=labelnames,
-        )
-
         self.transfer_duration_us = Histogram(
             "sglang:hicache_l1_l2_transfer_duration_us",
             "Observed duration in microseconds for one completed HiCache L1<->L2 KV block transfer.",
@@ -1903,10 +1891,8 @@ def record_transfer(
             "dst": dst,
         }
 
-        self.transfer_events_total.labels(**metric_labels).inc()
         self.transfer_blocks_total.labels(**metric_labels).inc(blocks)
         self.transfer_bytes_total.labels(**metric_labels).inc(bytes_)
-        self.transfer_time_us_total.labels(**metric_labels).inc(xfer_us)
         self.transfer_duration_us.labels(**metric_labels).observe(xfer_us)
 
 

From aa68fd5896c85e1643340f473170a0e475ccafca Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Thu, 4 Jun 2026 12:55:26 -0700
Subject: [PATCH 12/28] Fix HiCache transfer metric registration

---
 .../srt/observability/metrics_collector.py    | 96 +++++++++++--------
 1 file changed, 58 insertions(+), 38 deletions(-)

diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py
index 4fbdc11fc5fc..2e7d170d7b8c 100644
--- a/python/sglang/srt/observability/metrics_collector.py
+++ b/python/sglang/srt/observability/metrics_collector.py
@@ -1826,53 +1826,72 @@ def log_storage_metrics(self, storage_metrics: Optional[StorageMetrics] = None):
             self._log_histogram(self.histogram_backup_bandwidth, v)
 
 
-class HiCacheL1L2TransferMetricsCollector:
+class HiCacheL1L2TransferMetricsCollector(_StatLoggerDIMixin):
     """Prometheus metrics for HiCache L1<->L2 KV block transfers.
 
     L1: device/GPU KV cache.
     L2: host/CPU KV cache.
     """
 
+    _metric_cache: dict[
+        tuple[tuple[str, ...], type, type],
+        tuple[Any, Any, Any],
+    ] = {}
+
     def __init__(self, labels: Optional[dict[str, str]] = None):
-        from prometheus_client import Counter, Histogram
+        from prometheus_client import Counter as _PromCounter
+        from prometheus_client import Histogram as _PromHistogram
+
+        Counter = self._counter_cls or _PromCounter
+        Histogram = self._histogram_cls or _PromHistogram
 
         self.labels = labels or {}
         labelnames = list(self.labels.keys()) + ["direction", "src", "dst"]
+        cache_key = (tuple(labelnames), Counter, Histogram)
+
+        cached_metrics = self._metric_cache.get(cache_key)
+        if cached_metrics is None:
+            cached_metrics = (
+                Counter(
+                    "sglang:hicache_l1_l2_transfer_blocks_total",
+                    "Total number of KV cache blocks transferred between HiCache L1 and L2.",
+                    labelnames=labelnames,
+                ),
+                Counter(
+                    "sglang:hicache_l1_l2_transfer_bytes_total",
+                    "Total number of KV cache bytes transferred between HiCache L1 and L2.",
+                    labelnames=labelnames,
+                ),
+                Histogram(
+                    "sglang:hicache_l1_l2_transfer_duration_us",
+                    "Observed device-event duration in microseconds for one completed HiCache L1<->L2 KV block transfer.",
+                    labelnames=labelnames,
+                    buckets=(
+                        100,
+                        250,
+                        500,
+                        1_000,
+                        2_500,
+                        5_000,
+                        10_000,
+                        25_000,
+                        50_000,
+                        100_000,
+                        250_000,
+                        500_000,
+                        1_000_000,
+                        2_500_000,
+                        5_000_000,
+                    ),
+                ),
+            )
+            self._metric_cache[cache_key] = cached_metrics
 
-        self.transfer_blocks_total = Counter(
-            "sglang:hicache_l1_l2_transfer_blocks_total",
-            "Total number of KV cache blocks transferred between HiCache L1 and L2.",
-            labelnames=labelnames,
-        )
-
-        self.transfer_bytes_total = Counter(
-            "sglang:hicache_l1_l2_transfer_bytes_total",
-            "Total number of KV cache bytes transferred between HiCache L1 and L2.",
-            labelnames=labelnames,
-        )
-
-        self.transfer_duration_us = Histogram(
-            "sglang:hicache_l1_l2_transfer_duration_us",
-            "Observed duration in microseconds for one completed HiCache L1<->L2 KV block transfer.",
-            labelnames=labelnames,
-            buckets=(
-                100,
-                250,
-                500,
-                1_000,
-                2_500,
-                5_000,
-                10_000,
-                25_000,
-                50_000,
-                100_000,
-                250_000,
-                500_000,
-                1_000_000,
-                2_500_000,
-                5_000_000,
-            ),
-        )
+        (
+            self.transfer_blocks_total,
+            self.transfer_bytes_total,
+            self.transfer_duration_us,
+        ) = cached_metrics
 
     def record_transfer(
         self,
@@ -1882,7 +1901,7 @@ def record_transfer(
         dst: str,
         blocks: int,
         bytes_: int,
-        xfer_us: int,
+        xfer_us: Optional[int],
     ) -> None:
         metric_labels = {
             **self.labels,
@@ -1893,7 +1912,8 @@ def record_transfer(
 
         self.transfer_blocks_total.labels(**metric_labels).inc(blocks)
         self.transfer_bytes_total.labels(**metric_labels).inc(bytes_)
-        self.transfer_duration_us.labels(**metric_labels).observe(xfer_us)
+        if xfer_us is not None:
+            self.transfer_duration_us.labels(**metric_labels).observe(xfer_us)
 
 
 class ExpertDispatchCollector(_StatLoggerDIMixin):

From 9f2542788cf6600ee2557c58389fe89030885d62 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Thu, 4 Jun 2026 12:55:37 -0700
Subject: [PATCH 13/28] Tighten HiCache transfer metric accounting

---
 .../sglang/srt/managers/cache_controller.py   | 78 +++++++++++--------
 .../hybrid_cache/hybrid_cache_controller.py   |  6 --
 2 files changed, 47 insertions(+), 37 deletions(-)

diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py
index b4025294392b..037020256409 100644
--- a/python/sglang/srt/managers/cache_controller.py
+++ b/python/sglang/srt/managers/cache_controller.py
@@ -160,9 +160,6 @@ class HiCacheAck(NamedTuple):
     # Estimated total bytes moved for this operation.
     byte_count: int
 
-    # Host-side fallback timer start. Used only if device event elapsed_time is unavailable.
-    start_time_ns: int
-
 
 class StorageOperation:
     counter = 0
@@ -263,6 +260,7 @@ def __init__(
         self.storage_backend = None
         self.storage_backend_type = None
         self.enable_storage_metrics = enable_storage_metrics
+        self._warned_unknown_host_pool_bytes = False
 
         # init L1/L2 transfer metrics collection (device-host transfers triggered by write/load).
         self.enable_l1_l2_transfer_metrics = enable_metrics
@@ -732,8 +730,6 @@ def start_writing(self) -> None:
         finish_event = device_module.Event()
 
         token_count = int(host_indices.numel())
-        start_time_ns = time.perf_counter_ns()
-
         start_event.record()
         with device_module.stream(self.write_stream):
             start_event.wait(self.write_stream)
@@ -762,7 +758,6 @@ def start_writing(self) -> None:
                 finish_event=finish_event,
                 node_ids=op.node_ids,
                 token_count=token_count,
-                start_time_ns=start_time_ns,
             )
         )
 
@@ -818,8 +813,6 @@ def start_loading(self) -> int:
         producer_event = self.layer_done_counter.events[producer_id]
 
         token_count = int(host_indices.numel())
-        start_time_ns = time.perf_counter_ns()
-
         producer_event.start_event.record()
 
         with device_module.stream(self.load_stream):
@@ -855,7 +848,6 @@ def start_loading(self) -> int:
                 finish_event=producer_event.finish_event,
                 node_ids=op.node_ids,
                 token_count=token_count,
-                start_time_ns=start_time_ns,
             )
         )
         return producer_id
@@ -1315,12 +1307,31 @@ def _host_pool_bytes_per_token(self) -> int:
         Handles both a normal HostKVCache and HostPoolGroup-style wrappers.
         """
         if hasattr(self.mem_pool_host, "entries"):
-            return sum(
-                int(entry.host_pool.size_per_token)
-                for entry in self.mem_pool_host.entries
-            )
+            bytes_per_token = 0
+            for entry in self.mem_pool_host.entries:
+                size_per_token = getattr(entry.host_pool, "size_per_token", None)
+                if size_per_token is None:
+                    self._warn_unknown_host_pool_bytes(entry.host_pool)
+                    continue
+                bytes_per_token += int(size_per_token)
+            return bytes_per_token
+
+        size_per_token = getattr(self.mem_pool_host, "size_per_token", None)
+        if size_per_token is None:
+            self._warn_unknown_host_pool_bytes(self.mem_pool_host)
+            return 0
+
+        return int(size_per_token)
 
-        return int(getattr(self.mem_pool_host, "size_per_token", 0))
+    def _warn_unknown_host_pool_bytes(self, host_pool) -> None:
+        if self._warned_unknown_host_pool_bytes:
+            return
+        self._warned_unknown_host_pool_bytes = True
+        logger.warning(
+            "Unable to estimate HiCache L1/L2 transfer bytes for host pool type %s: "
+            "missing size_per_token.",
+            type(host_pool).__name__,
+        )
 
     def _estimate_l1_l2_transfer_bytes(self, token_count: int) -> int:
         """Estimate total bytes moved for one L1<->L2 transfer operation."""
@@ -1328,9 +1339,13 @@ def _estimate_l1_l2_transfer_bytes(self, token_count: int) -> int:
         total = int(token_count) * bytes_per_token
 
         if self.has_draft and self.mem_pool_host_draft is not None:
-            total += int(token_count) * int(
-                getattr(self.mem_pool_host_draft, "size_per_token", 0)
+            draft_size_per_token = getattr(
+                self.mem_pool_host_draft, "size_per_token", None
             )
+            if draft_size_per_token is None:
+                self._warn_unknown_host_pool_bytes(self.mem_pool_host_draft)
+            else:
+                total += int(token_count) * int(draft_size_per_token)
 
         return total
 
@@ -1341,9 +1356,10 @@ def _make_hicache_ack(
         finish_event: device_module.Event,
         node_ids: List[int],
         token_count: int,
-        start_time_ns: int,
     ) -> HiCacheAck:
-        block_count = int(token_count) // int(self.page_size)
+        block_count = (int(token_count) + int(self.page_size) - 1) // int(
+            self.page_size
+        )
         byte_count = self._estimate_l1_l2_transfer_bytes(token_count)
 
         return HiCacheAck(
@@ -1353,19 +1369,14 @@ def _make_hicache_ack(
             token_count=int(token_count),
             block_count=block_count,
             byte_count=byte_count,
-            start_time_ns=start_time_ns,
         )
 
-    def _transfer_elapsed_us(self, ack: HiCacheAck) -> int:
-        """Return transfer duration in microseconds.
-
-        Prefer device event timing. Fall back to host elapsed time for devices/backends
-        that do not expose elapsed_time().
-        """
+    def _transfer_elapsed_us(self, ack: HiCacheAck) -> Optional[int]:
+        """Return device-event transfer duration in microseconds, when available."""
         try:
             return max(0, int(ack.start_event.elapsed_time(ack.finish_event) * 1000))
         except Exception:
-            return max(0, int((time.perf_counter_ns() - ack.start_time_ns) // 1000))
+            return None
 
     def record_l1_l2_transfer_complete(
         self,
@@ -1402,15 +1413,15 @@ def record_l1_l2_transfer_complete(
         if should_log:
             ts_us = time.time_ns() // 1000
             logger.debug(
-                "%s transfer complete ts_us=%d blocks=%d bytes=%d xfer_us=%d "
+                "%s transfer complete ts_us=%d blocks=%d bytes=%d xfer_us=%s "
                 "bandwidth=%.2fGB/s "
                 'src="%s" dst="%s"',
                 action,
                 ts_us,
                 ack.block_count,
                 ack.byte_count,
-                xfer_us,
-                ack.byte_count * 0.001 / xfer_us if xfer_us > 0 else 0,
+                xfer_us if xfer_us is not None else "unavailable",
+                ack.byte_count * 0.001 / xfer_us if xfer_us else 0,
                 src,
                 dst,
             )
@@ -1430,7 +1441,8 @@ def record_l1_l2_transfer_complete(
             totals["events"] += 1
             totals["blocks"] += ack.block_count
             totals["bytes"] += ack.byte_count
-            totals["xfer_us"] += xfer_us
+            if xfer_us is not None:
+                totals["xfer_us"] += xfer_us
 
             logger.debug(
                 '%s transfer cumulative direction="%s" total_events=%d '
@@ -1443,7 +1455,11 @@ def record_l1_l2_transfer_complete(
                 totals["blocks"],
                 totals["bytes"],
                 totals["xfer_us"],
-                totals["bytes"] * 0.001 / totals["xfer_us"] if totals["xfer_us"] > 0 else 0,
+                (
+                    totals["bytes"] * 0.001 / totals["xfer_us"]
+                    if totals["xfer_us"] > 0
+                    else 0
+                ),
                 src,
                 dst,
             )
diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py
index 76e937de2f74..ef4bf1241bd9 100644
--- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py
+++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py
@@ -413,8 +413,6 @@ def start_writing(self) -> None:
         finish_event = device_module.Event()
 
         token_count = int(host_indices.numel())
-        start_time_ns = time.perf_counter_ns()
-
         start_event.record()
         with device_module.stream(self.write_stream):
             start_event.wait(self.write_stream)
@@ -445,7 +443,6 @@ def start_writing(self) -> None:
                 finish_event=finish_event,
                 node_ids=op.node_ids,
                 token_count=token_count,
-                start_time_ns=start_time_ns,
             )
         )
 
@@ -504,8 +501,6 @@ def start_loading(self) -> int:
         producer_event = self.layer_done_counter.events[producer_id]
 
         token_count = int(host_indices.numel())
-        start_time_ns = time.perf_counter_ns()
-
         producer_event.start_event.record()
         with device_module.stream(self.load_stream):
             producer_event.start_event.wait(self.load_stream)
@@ -543,7 +538,6 @@ def start_loading(self) -> int:
                 finish_event=producer_event.finish_event,
                 node_ids=op.node_ids,
                 token_count=token_count,
-                start_time_ns=start_time_ns,
             )
         )
         return producer_id

From c14fcb7b7f34975cc2c7216ab49bb5895af47bb0 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Thu, 4 Jun 2026 12:55:57 -0700
Subject: [PATCH 14/28] Fix HiCache write-through ack state handling

---
 .../decode_kvcache_offload_manager.py         | 23 ++++-----
 .../srt/mem_cache/hi_mamba_radix_cache.py     | 51 +++++++++++--------
 python/sglang/srt/mem_cache/hiradix_cache.py  | 30 ++++++-----
 .../srt/mem_cache/unified_radix_cache.py      | 20 ++++----
 4 files changed, 67 insertions(+), 57 deletions(-)

diff --git a/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py b/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py
index c49dee7b175b..552f05ba0fba 100644
--- a/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py
+++ b/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py
@@ -221,19 +221,13 @@ def _check_offload_progress(self, finish_count):
         while finish_count > 0:
             ack = self.cache_controller.ack_write_queue.pop(0)
             ack.finish_event.synchronize()
-            self.cache_controller.record_l1_l2_transfer_complete(
-                direction="offload",
-                ack=ack,
-            )
+            matched = False
             for ack_id in ack.node_ids:
-                (
-                    req,
-                    host_indices,
-                    incremental_tokens,
-                    start_time,
-                    start,
-                    end,
-                ) = self.ongoing_offload.pop(ack_id)
+                entry = self.ongoing_offload.pop(ack_id, None)
+                if entry is None:
+                    continue
+                matched = True
+                req, host_indices, incremental_tokens, start_time, start, end = entry
 
                 self._mark_offload_finished(req.rid)
                 prior_hash = (
@@ -251,6 +245,11 @@ def _check_offload_progress(self, finish_count):
                     state = self.offloaded_state.get(req.rid)
                     start_offset = state.prefill_len if state is not None else start
                     self._release_finished_req(req, start_offset)
+            if matched:
+                self.cache_controller.record_l1_l2_transfer_complete(
+                    direction="offload",
+                    ack=ack,
+                )
             finish_count -= 1
 
     def _release_finished_req(self, req: Req, start_offset: int):
diff --git a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
index 7f278e2172bb..81ad82212079 100644
--- a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
+++ b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
@@ -379,23 +379,34 @@ def _inc_hit_count(self, node: TreeNode, chunked=False):
             # write to host if the node is not backuped
             self.write_backup(node)
 
+    def _finish_write_through_ack(self, ack_id: int, *, release_lock: bool) -> bool:
+        backuped_node = self.ongoing_write_through.pop(ack_id, None)
+        if backuped_node is None:
+            return False
+
+        self._record_store_event(backuped_node, medium=StorageMedium.CPU)
+        if release_lock:
+            self.dec_lock_ref(backuped_node)
+        if self.enable_storage:
+            self.write_backup_storage(backuped_node)
+        return True
+
     def writing_check(self, write_back=False):
         if write_back:
             # blocking till all write back complete
             while len(self.ongoing_write_through) > 0:
                 for ack in self.cache_controller.ack_write_queue:
                     ack.finish_event.synchronize()
-                    self.cache_controller.record_l1_l2_transfer_complete(
-                        direction="offload",
-                        ack=ack,
-                    )
+                    matched = False
                     for ack_id in ack.node_ids:
-                        backuped_node = self.ongoing_write_through.pop(ack_id)
-                        self._record_store_event(
-                            backuped_node, medium=StorageMedium.CPU
+                        matched |= self._finish_write_through_ack(
+                            ack_id, release_lock=False
+                        )
+                    if matched:
+                        self.cache_controller.record_l1_l2_transfer_complete(
+                            direction="offload",
+                            ack=ack,
                         )
-                        if self.enable_storage:
-                            self.write_backup_storage(backuped_node)
                 self.cache_controller.ack_write_queue.clear()
                 assert len(self.ongoing_write_through) == 0
             return
@@ -423,18 +434,16 @@ def writing_check(self, write_back=False):
             ack = self.cache_controller.ack_write_queue.pop(0)
             ack.finish_event.synchronize()
 
-            # Record the L1->L2 transfer completion for this write-back operation.
-            self.cache_controller.record_l1_l2_transfer_complete(
-                direction="offload",
-                ack=ack,
-            )
-
+            matched = False
             for ack_id in ack.node_ids:
-                backuped_node = self.ongoing_write_through.pop(ack_id)
-                self._record_store_event(backuped_node, medium=StorageMedium.CPU)
-                self.dec_lock_ref(backuped_node)
-                if self.enable_storage:
-                    self.write_backup_storage(backuped_node)
+                matched |= self._finish_write_through_ack(
+                    ack_id, release_lock=True
+                )
+            if matched:
+                self.cache_controller.record_l1_l2_transfer_complete(
+                    direction="offload",
+                    ack=ack,
+                )
             finish_count -= 1
 
     def loading_check(self):
@@ -446,8 +455,6 @@ def loading_check(self):
                 # the KV cache loading is still ongoing
                 break
 
-            # ensure completion of loading, before recording transfer completion and updating cache state
-            ack.finish_event.synchronize()
             self.cache_controller.record_l1_l2_transfer_complete(
                 direction="onboard",
                 ack=ack,
diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py
index ce5cbf117857..c28301077852 100644
--- a/python/sglang/srt/mem_cache/hiradix_cache.py
+++ b/python/sglang/srt/mem_cache/hiradix_cache.py
@@ -799,7 +799,7 @@ def _track_write_through_node(self, node: TreeNode, backup_len: int) -> None:
     def _replace_pending_write_through_node(
         self, old_node: TreeNode, new_nodes: List[TreeNode]
     ) -> None:
-        ack_id = old_node.write_through_pending_id
+        ack_id = getattr(old_node, "write_through_pending_id", None)
         if ack_id is None:
             return
 
@@ -824,17 +824,21 @@ def _replace_pending_write_through_node(
             node.write_through_pending_id = ack_id
         self.ongoing_write_through[ack_id] = (lock_node, backup_len, updated_nodes)
 
-    def _finish_write_through_ack(self, ack_id: int, *, release_lock: bool) -> None:
-        lock_node, backup_len, publish_nodes = self.ongoing_write_through.pop(ack_id)
+    def _finish_write_through_ack(self, ack_id: int, *, release_lock: bool) -> bool:
+        pending = self.ongoing_write_through.pop(ack_id, None)
+        if pending is None:
+            return False
+
+        lock_node, backup_len, publish_nodes = pending
         for node in publish_nodes:
-            if node.write_through_pending_id == ack_id:
+            if getattr(node, "write_through_pending_id", None) == ack_id:
                 node.write_through_pending_id = None
-            # DMA confirmed -- block is now on host.
             self._record_store_event(node, medium=StorageMedium.CPU)
         if self.enable_storage:
             self.write_backup_storage(lock_node, backup_len)
         if release_lock:
             self.dec_lock_ref(lock_node)
+        return True
 
     def write_backup_storage(self, node: TreeNode, backup_len: Optional[int] = None):
         # Recover pre-split data via walk-and-concat if node was split.
@@ -916,11 +920,9 @@ def writing_check(self, write_back=False):
                     ack.finish_event.synchronize()
                     matched = False
                     for ack_id in ack.node_ids:
-                        if ack_id in self.ongoing_write_through:
-                            self._finish_write_through_ack(
-                                ack_id, release_lock=False
-                            )
-                            matched = True
+                        matched |= self._finish_write_through_ack(
+                            ack_id, release_lock=False
+                        )
                     if matched:
                         self.cache_controller.record_l1_l2_transfer_complete(
                             direction="offload",
@@ -949,11 +951,12 @@ def writing_check(self, write_back=False):
         while finish_count > 0:
             ack = self.cache_controller.ack_write_queue.pop(0)
             ack.finish_event.synchronize()
+
             matched = False
             for ack_id in ack.node_ids:
-                if ack_id in self.ongoing_write_through:
-                    self._finish_write_through_ack(ack_id, release_lock=True)
-                    matched = True
+                matched |= self._finish_write_through_ack(
+                    ack_id, release_lock=True
+                )
             if matched:
                 self.cache_controller.record_l1_l2_transfer_complete(
                     direction="offload",
@@ -977,7 +980,6 @@ def loading_check(self):
         while finish_count > 0:
             ack = self.cache_controller.ack_load_queue.pop(0)
             ack.finish_event.synchronize()
-            # ensure completion of loading, before recording transfer completion and updating cache state
             self.cache_controller.record_l1_l2_transfer_complete(
                 direction="onboard",
                 ack=ack,
diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py
index 6debad03a287..4eaabc110c15 100644
--- a/python/sglang/srt/mem_cache/unified_radix_cache.py
+++ b/python/sglang/srt/mem_cache/unified_radix_cache.py
@@ -92,6 +92,7 @@ def __init__(self, tree_components: tuple[ComponentType, ...], priority: int = 0
         self.hash_value = None
         self.hit_count = 0
         self.priority = priority
+        self.write_through_pending_id: Optional[int] = None
         self.lru_prev: list[UnifiedTreeNode | None] = [None] * (
             _NUM_COMPONENT_TYPES * 2
         )
@@ -1643,8 +1644,12 @@ def _replace_pending_write_through_node(
             updated_nodes,
         )
 
-    def _finish_write_through_ack(self, ack_id: int) -> None:
-        lock_node, lock_params, publish_nodes = self.ongoing_write_through.pop(ack_id)
+    def _finish_write_through_ack(self, ack_id: int) -> bool:
+        pending = self.ongoing_write_through.pop(ack_id, None)
+        if pending is None:
+            return False
+
+        lock_node, lock_params, publish_nodes = pending
         for node in publish_nodes:
             if node.write_through_pending_id == ack_id:
                 node.write_through_pending_id = None
@@ -1656,6 +1661,7 @@ def _finish_write_through_ack(self, ack_id: int) -> None:
             # suffix; the prefix fragment must be persisted as well.
             for node in publish_nodes:
                 self.write_backup_storage(node)
+        return True
 
     def load_back(
         self,
@@ -2353,9 +2359,7 @@ def writing_check(self, write_back: bool = False) -> None:
                     ack.finish_event.synchronize()
                     matched = False
                     for ack_id in ack.node_ids:
-                        if ack_id in self.ongoing_write_through:
-                            self._finish_write_through_ack(ack_id)
-                            matched = True
+                        matched |= self._finish_write_through_ack(ack_id)
                     if matched:
                         cc.record_l1_l2_transfer_complete(
                             direction="offload",
@@ -2381,11 +2385,10 @@ def writing_check(self, write_back: bool = False) -> None:
         while finish_count > 0:
             ack = cc.ack_write_queue.pop(0)
             ack.finish_event.synchronize()
+
             matched = False
             for ack_id in ack.node_ids:
-                if ack_id in self.ongoing_write_through:
-                    self._finish_write_through_ack(ack_id)
-                    matched = True
+                matched |= self._finish_write_through_ack(ack_id)
             if matched:
                 self.cache_controller.record_l1_l2_transfer_complete(
                     direction="offload",
@@ -2413,7 +2416,6 @@ def loading_check(self) -> None:
         while finish_count > 0:
             ack = cc.ack_load_queue.pop(0)
             ack.finish_event.synchronize()
-            # ensure completion of loading, before recording transfer completion and updating cache state
             self.cache_controller.record_l1_l2_transfer_complete(
                 direction="onboard",
                 ack=ack,

From 2848801bd30f830580a692e92973f67d973a7898 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Thu, 4 Jun 2026 12:58:08 -0700
Subject: [PATCH 15/28] Keep HiCache transfer totals with metrics

---
 python/sglang/srt/managers/cache_controller.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py
index 037020256409..78b13a1e8e76 100644
--- a/python/sglang/srt/managers/cache_controller.py
+++ b/python/sglang/srt/managers/cache_controller.py
@@ -1436,14 +1436,18 @@ def record_l1_l2_transfer_complete(
                 xfer_us=xfer_us,
             )
 
-        if should_log:
-            totals = self.hicache_l1_l2_transfer_totals[direction]
-            totals["events"] += 1
-            totals["blocks"] += ack.block_count
-            totals["bytes"] += ack.byte_count
-            if xfer_us is not None:
-                totals["xfer_us"] += xfer_us
+        # Accumulate totals whenever transfer observation is active. This keeps
+        # cumulative debug logs consistent if DEBUG logging is enabled after metrics
+        # collection has already been running, while preserving the fully-disabled
+        # early-exit path above.
+        totals = self.hicache_l1_l2_transfer_totals[direction]
+        totals["events"] += 1
+        totals["blocks"] += ack.block_count
+        totals["bytes"] += ack.byte_count
+        if xfer_us is not None:
+            totals["xfer_us"] += xfer_us
 
+        if should_log:
             logger.debug(
                 '%s transfer cumulative direction="%s" total_events=%d '
                 "total_blocks=%d total_bytes=%d total_xfer_us=%d "

From de5c53920ba80372911012e5eeb58dff9149383e Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Thu, 4 Jun 2026 20:18:17 +0000
Subject: [PATCH 16/28] fix(hicache): remove duplicate write_through_pending_id
 init in UnifiedTreeNode

The fix commit added write_through_pending_id earlier in __init__ (after
priority, grouped with other node state), but main already had the same
assignment at the end of __init__. Remove the trailing duplicate.
---
 python/sglang/srt/mem_cache/unified_radix_cache.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py
index 4eaabc110c15..19b38b8e18dc 100644
--- a/python/sglang/srt/mem_cache/unified_radix_cache.py
+++ b/python/sglang/srt/mem_cache/unified_radix_cache.py
@@ -101,7 +101,6 @@ def __init__(self, tree_components: tuple[ComponentType, ...], priority: int = 0
         )
         self.id = UnifiedTreeNode.counter
         UnifiedTreeNode.counter += 1
-        self.write_through_pending_id: Optional[int] = None
 
     def component(self, component_type: ComponentType) -> ComponentData:
         return self.component_data[component_type]

From 7c93e075c444afe91f84f019f2c365b167791fa7 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Thu, 4 Jun 2026 21:19:57 +0000
Subject: [PATCH 17/28] fix(hicache/metrics): wire enable_metrics to
 HybridCacheController and disagg offload path

HybridCacheController and the disagg offload HiCacheController were created
without enable_metrics / extra_metric_labels, so HiCacheL1L2TransferMetrics
Collector was never initialised on those paths even when --enable-metrics is
set.

Thread enable_metrics and extra_metric_labels through:
- hybrid_pool_assembler: all 5 build_*_stack functions, all 6 StackStrategy
  build() methods (including the base), and the top-level strategy.build()
  dispatch call -- following the exact same pattern as enable_storage_metrics.
- decode_kvcache_offload_manager: pass server_args.enable_metrics and
  server_args.extra_metric_labels when constructing HiCacheController.
---
 .../decode_kvcache_offload_manager.py         |  2 +
 .../hybrid_cache/hybrid_pool_assembler.py     | 50 +++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py b/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py
index 552f05ba0fba..f750c558f416 100644
--- a/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py
+++ b/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py
@@ -99,6 +99,8 @@ def __init__(
             storage_backend=server_args.hicache_storage_backend,
             model_name=server_args.served_model_name,
             storage_backend_extra_config=hicache_storage_backend_extra_config,
+            enable_metrics=server_args.enable_metrics,
+            extra_metric_labels=server_args.extra_metric_labels,
         )
 
         self.ongoing_offload = {}
diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py
index daaf05e63c12..64e3dc4e4111 100644
--- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py
+++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py
@@ -119,6 +119,8 @@ def build_kv_only_stack(
     model_name: Optional[str] = None,
     storage_backend_extra_config: Optional[dict] = None,
     enable_storage_metrics: bool = False,
+    enable_metrics: bool = False,
+    extra_metric_labels: Optional[dict[str, str]] = None,
 ) -> tuple[HostPoolGroup, HybridCacheController]:
     transfer_layer_num = len(full_layer_mapping)
     kv_host_pool = build_kv_host_pool(
@@ -156,6 +158,8 @@ def build_kv_only_stack(
         storage_backend_extra_config=storage_backend_extra_config,
         transfer_layer_num=transfer_layer_num,
         enable_storage_metrics=enable_storage_metrics,
+        enable_metrics=enable_metrics,
+        extra_metric_labels=extra_metric_labels,
     )
     return host_pool_group, cache_controller
 
@@ -182,6 +186,8 @@ def build_hybrid_swa_stack(
     model_name: Optional[str] = None,
     storage_backend_extra_config: Optional[dict] = None,
     enable_storage_metrics: bool = False,
+    enable_metrics: bool = False,
+    extra_metric_labels: Optional[dict[str, str]] = None,
 ) -> tuple[HostPoolGroup, HybridCacheController]:
     transfer_layer_num = len(full_layer_mapping | swa_layer_mapping)
     kv_host_pool = build_kv_host_pool(
@@ -238,6 +244,8 @@ def build_hybrid_swa_stack(
         storage_backend_extra_config=storage_backend_extra_config,
         transfer_layer_num=transfer_layer_num,
         enable_storage_metrics=enable_storage_metrics,
+        enable_metrics=enable_metrics,
+        extra_metric_labels=extra_metric_labels,
     )
     return host_pool_group, cache_controller
 
@@ -285,6 +293,8 @@ def build_deepseek_v4_hicache_stack(
     model_name: Optional[str] = None,
     storage_backend_extra_config: Optional[dict] = None,
     enable_storage_metrics: bool = False,
+    enable_metrics: bool = False,
+    extra_metric_labels: Optional[dict[str, str]] = None,
 ) -> tuple[HostPoolGroup, HybridCacheController]:
     # TODO(hzh0425): Support PP for deepseek v4 with hicache
     transfer_layer_num = kvcache.end_layer - kvcache.start_layer
@@ -475,6 +485,8 @@ def build_deepseek_v4_hicache_stack(
         storage_backend_extra_config=storage_backend_extra_config,
         transfer_layer_num=transfer_layer_num,
         enable_storage_metrics=enable_storage_metrics,
+        enable_metrics=enable_metrics,
+        extra_metric_labels=extra_metric_labels,
     )
     return host_pool_group, cache_controller
 
@@ -501,6 +513,8 @@ def build_hybrid_mamba_stack(
     model_name: Optional[str] = None,
     storage_backend_extra_config: Optional[dict] = None,
     enable_storage_metrics: bool = False,
+    enable_metrics: bool = False,
+    extra_metric_labels: Optional[dict[str, str]] = None,
 ) -> tuple[HostPoolGroup, HybridCacheController]:
     transfer_layer_num = len(full_layer_mapping | mamba_layer_mapping)
     mamba_allocator = params.req_to_token_pool.mamba_allocator
@@ -556,6 +570,8 @@ def build_hybrid_mamba_stack(
         storage_backend_extra_config=storage_backend_extra_config,
         transfer_layer_num=transfer_layer_num,
         enable_storage_metrics=enable_storage_metrics,
+        enable_metrics=enable_metrics,
+        extra_metric_labels=extra_metric_labels,
     )
     return host_pool_group, cache_controller
 
@@ -581,6 +597,8 @@ def build_anchor_sidecar_stack(
     model_name: Optional[str] = None,
     storage_backend_extra_config: Optional[dict] = None,
     enable_storage_metrics: bool = False,
+    enable_metrics: bool = False,
+    extra_metric_labels: Optional[dict[str, str]] = None,
 ) -> tuple[HostPoolGroup, HybridCacheController]:
     transfer_layer_num = len(full_layer_mapping)
     kv_host_pool = build_kv_host_pool(
@@ -626,6 +644,8 @@ def build_anchor_sidecar_stack(
         storage_backend_extra_config=storage_backend_extra_config,
         transfer_layer_num=transfer_layer_num,
         enable_storage_metrics=enable_storage_metrics,
+        enable_metrics=enable_metrics,
+        extra_metric_labels=extra_metric_labels,
     )
     return host_pool_group, cache_controller
 
@@ -669,6 +689,8 @@ def build(
         prefetch_threshold: int = 256,
         model_name: Optional[str] = None,
         enable_storage_metrics: bool = False,
+        enable_metrics: bool = False,
+        extra_metric_labels: Optional[dict[str, str]] = None,
     ) -> StackBuildResult:
         raise NotImplementedError
 
@@ -699,6 +721,8 @@ def build(
         prefetch_threshold=256,
         model_name=None,
         enable_storage_metrics=False,
+        enable_metrics=False,
+        extra_metric_labels=None,
     ):
         from sglang.srt.mem_cache.base_prefix_cache import EvictParams
 
@@ -719,6 +743,8 @@ def build(
             model_name=model_name,
             storage_backend_extra_config=storage_backend_extra_config,
             enable_storage_metrics=enable_storage_metrics,
+            enable_metrics=enable_metrics,
+            extra_metric_labels=extra_metric_labels,
         )
         sidecars = [
             SidecarPoolSpec(
@@ -777,6 +803,8 @@ def build(
         prefetch_threshold=256,
         model_name=None,
         enable_storage_metrics=False,
+        enable_metrics=False,
+        extra_metric_labels=None,
     ):
         from sglang.srt.mem_cache.base_prefix_cache import EvictParams
 
@@ -803,6 +831,8 @@ def build(
             model_name=model_name,
             storage_backend_extra_config=storage_backend_extra_config,
             enable_storage_metrics=enable_storage_metrics,
+            enable_metrics=enable_metrics,
+            extra_metric_labels=extra_metric_labels,
         )
         return StackBuildResult(
             host_pool_group=host_pool_group,
@@ -853,6 +883,8 @@ def build(
         prefetch_threshold=256,
         model_name=None,
         enable_storage_metrics=False,
+        enable_metrics=False,
+        extra_metric_labels=None,
     ):
         from sglang.srt.mem_cache.base_prefix_cache import EvictParams
 
@@ -878,6 +910,8 @@ def build(
             model_name=model_name,
             storage_backend_extra_config=storage_backend_extra_config,
             enable_storage_metrics=enable_storage_metrics,
+            enable_metrics=enable_metrics,
+            extra_metric_labels=extra_metric_labels,
         )
         return StackBuildResult(
             host_pool_group=host_pool_group,
@@ -914,6 +948,8 @@ def build(
         prefetch_threshold=256,
         model_name=None,
         enable_storage_metrics=False,
+        enable_metrics=False,
+        extra_metric_labels=None,
     ):
         from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
 
@@ -944,6 +980,8 @@ def build(
             model_name=model_name,
             storage_backend_extra_config=storage_backend_extra_config,
             enable_storage_metrics=enable_storage_metrics,
+            enable_metrics=enable_metrics,
+            extra_metric_labels=extra_metric_labels,
         )
         return StackBuildResult(
             host_pool_group=host_pool_group,
@@ -995,6 +1033,8 @@ def build(
         prefetch_threshold=256,
         model_name=None,
         enable_storage_metrics=False,
+        enable_metrics=False,
+        extra_metric_labels=None,
     ):
         from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
 
@@ -1018,6 +1058,8 @@ def build(
             model_name=model_name,
             storage_backend_extra_config=storage_backend_extra_config,
             enable_storage_metrics=enable_storage_metrics,
+            enable_metrics=enable_metrics,
+            extra_metric_labels=extra_metric_labels,
         )
         return StackBuildResult(
             host_pool_group=host_pool_group,
@@ -1116,6 +1158,8 @@ def attach_hybrid_pool_to_unified_cache(
             prefetch_threshold=storage_prefetch_threshold,
             model_name=server_args.served_model_name,
             enable_storage_metrics=cache._enable_metrics_flag,
+            enable_metrics=cache._enable_metrics_flag,
+            extra_metric_labels=cache.extra_metric_labels,
         )
         _apply_stack_result(cache, kvcache, params, result)
     except Exception:
@@ -1131,6 +1175,8 @@ def attach_hybrid_dsa_pool_to_hiradix_cache(
     extra_config: dict,
     prefetch_threshold: int,
     enable_storage_metrics: bool,
+    enable_metrics: bool = False,
+    extra_metric_labels: Optional[dict[str, str]] = None,
     load_cache_event,
     attn_cp_group: Optional[torch.distributed.ProcessGroup] = None,
     attn_tp_group: Optional[torch.distributed.ProcessGroup] = None,
@@ -1167,6 +1213,8 @@ def attach_hybrid_dsa_pool_to_hiradix_cache(
             model_name=server_args.served_model_name,
             storage_backend_extra_config=extra_config,
             enable_storage_metrics=enable_storage_metrics,
+            enable_metrics=enable_metrics,
+            extra_metric_labels=extra_metric_labels,
         )
         radix_cache.full_kv_pool_host = host_pool_group.get_pool(PoolName.KV)
         radix_cache.token_to_kv_pool_host = host_pool_group
@@ -1223,6 +1271,8 @@ def attach_hybrid_pool_to_mamba_cache(
             model_name=server_args.served_model_name,
             storage_backend_extra_config=extra_config,
             enable_storage_metrics=enable_storage_metrics,
+            enable_metrics=enable_metrics,
+            extra_metric_labels=extra_metric_labels,
         )
         mamba_cache.full_kv_pool_host = host_pool_group.get_pool(PoolName.KV)
         mamba_cache.mamba_pool_host = host_pool_group.get_pool(PoolName.MAMBA)

From 20f97cb8517f7af392d8a4dfff34faa4fc2e83c0 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Thu, 4 Jun 2026 21:01:52 -0700
Subject: [PATCH 18/28] fix(hicache/metrics): fix byte estimation for hybrid
 sidecar pool transfers

The old _host_pool_bytes_per_token summed size_per_token across every
HostPoolGroup entry and multiplied by the anchor KV token count. This
over-counts sidecar pools (Mamba, SWA, DSA indexer) that use a different
index cardinality: MambaPoolHost uses page_size=1 (one slot per node) while
the KV pool uses page_size=N, so Mamba bytes were inflated by N.

Replace with _host_pool_size_per_token(pool_name) for named per-pool lookup,
_transfer_index_count(transfer) for actual sidecar index counts, and update
_estimate_l1_l2_transfer_bytes / _make_hicache_ack to accept pool_transfers
so each sidecar contributes its own (actual_index_count * size_per_token).
---
 .../sglang/srt/managers/cache_controller.py   | 56 ++++++++++++-------
 1 file changed, 35 insertions(+), 21 deletions(-)

diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py
index 78b13a1e8e76..78d5927250a5 100644
--- a/python/sglang/srt/managers/cache_controller.py
+++ b/python/sglang/srt/managers/cache_controller.py
@@ -1301,28 +1301,27 @@ def _init_l1_l2_transfer_metrics(
             HiCacheL1L2TransferMetricsCollector(labels)
         )
 
-    def _host_pool_bytes_per_token(self) -> int:
-        """Return bytes per token slot for the host-side KV pools.
-
-        Handles both a normal HostKVCache and HostPoolGroup-style wrappers.
-        """
-        if hasattr(self.mem_pool_host, "entries"):
-            bytes_per_token = 0
-            for entry in self.mem_pool_host.entries:
-                size_per_token = getattr(entry.host_pool, "size_per_token", None)
-                if size_per_token is None:
-                    self._warn_unknown_host_pool_bytes(entry.host_pool)
-                    continue
-                bytes_per_token += int(size_per_token)
-            return bytes_per_token
-
+    def _host_pool_size_per_token(self, pool_name=None) -> int:
+        """Return size_per_token for a specific named pool, or the whole pool for plain caches."""
+        if hasattr(self.mem_pool_host, "entry_map") and pool_name is not None:
+            entry = self.mem_pool_host.entry_map.get(pool_name)
+            if entry is None:
+                return 0
+            return int(getattr(entry.host_pool, "size_per_token", 0) or 0)
         size_per_token = getattr(self.mem_pool_host, "size_per_token", None)
         if size_per_token is None:
             self._warn_unknown_host_pool_bytes(self.mem_pool_host)
             return 0
-
         return int(size_per_token)
 
+    def _transfer_index_count(self, transfer) -> int:
+        """Return the number of index slots in a PoolTransfer (host preferred, device fallback)."""
+        if transfer.host_indices is not None:
+            return int(transfer.host_indices.numel())
+        if transfer.device_indices is not None:
+            return int(transfer.device_indices.numel())
+        return 0
+
     def _warn_unknown_host_pool_bytes(self, host_pool) -> None:
         if self._warned_unknown_host_pool_bytes:
             return
@@ -1333,10 +1332,24 @@ def _warn_unknown_host_pool_bytes(self, host_pool) -> None:
             type(host_pool).__name__,
         )
 
-    def _estimate_l1_l2_transfer_bytes(self, token_count: int) -> int:
-        """Estimate total bytes moved for one L1<->L2 transfer operation."""
-        bytes_per_token = self._host_pool_bytes_per_token()
-        total = int(token_count) * bytes_per_token
+    def _estimate_l1_l2_transfer_bytes(
+        self,
+        token_count: int,
+        pool_transfers: Optional[List[PoolTransfer]] = None,
+    ) -> int:
+        """Estimate total bytes moved for one L1<->L2 transfer operation.
+
+        For hybrid caches (Mamba, SWA, DSA sidecars) each sidecar pool may use
+        a different index count than the primary KV token count, so per-transfer
+        index counts are used rather than assuming every pool moves token_count slots.
+        """
+        total = int(token_count) * self._host_pool_size_per_token(PoolName.KV)
+
+        for transfer in pool_transfers or []:
+            total += (
+                self._transfer_index_count(transfer)
+                * self._host_pool_size_per_token(transfer.name)
+            )
 
         if self.has_draft and self.mem_pool_host_draft is not None:
             draft_size_per_token = getattr(
@@ -1356,11 +1369,12 @@ def _make_hicache_ack(
         finish_event: device_module.Event,
         node_ids: List[int],
         token_count: int,
+        pool_transfers: Optional[List[PoolTransfer]] = None,
     ) -> HiCacheAck:
         block_count = (int(token_count) + int(self.page_size) - 1) // int(
             self.page_size
         )
-        byte_count = self._estimate_l1_l2_transfer_bytes(token_count)
+        byte_count = self._estimate_l1_l2_transfer_bytes(token_count, pool_transfers)
 
         return HiCacheAck(
             start_event=start_event,

From f50d1caed56247e8e184fe5cae26c1e6a53e7193 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Thu, 4 Jun 2026 21:02:15 -0700
Subject: [PATCH 19/28] fix(hicache/mamba): add enable_metrics and
 extra_metric_labels params to attach_hybrid_pool_to_mamba_cache

The function body already forwarded these two names to build_hybrid_mamba_stack,
but they were never declared in the signature, causing a NameError at startup for
any Mamba model with HiCache enabled.
---
 python/sglang/srt/mem_cache/hi_mamba_radix_cache.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
index 81ad82212079..a438d3e8fca7 100644
--- a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
+++ b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
@@ -151,6 +151,8 @@ def __init__(self, params: CacheInitParams, server_args: ServerArgs):
             prefetch_threshold=prefetch_threshold,
             load_cache_event=self.load_cache_event,
             enable_storage_metrics=self.enable_storage_metrics,
+            enable_metrics=params.enable_metrics,
+            extra_metric_labels=self.extra_metric_labels,
             attn_cp_group=params.attn_cp_cache_group,
             attn_tp_group=params.attn_tp_cache_group,
         )

From 50c8c7186e23549d275e56791e5b47909c7ff559 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Thu, 4 Jun 2026 21:03:22 -0700
Subject: [PATCH 20/28] fix(hicache/dsa): propagate enable_metrics and
 extra_metric_labels to DSA attach path

The DSA (MLATokenToKVPool / DeepSeek V4) branch called
attach_hybrid_dsa_pool_to_hiradix_cache without these args, so the
HybridCacheController was always constructed with enable_metrics=False. The
non-DSA else-branch correctly passed both params; this brings the DSA branch
into parity.
---
 python/sglang/srt/mem_cache/hiradix_cache.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py
index c28301077852..ef724a401c4e 100644
--- a/python/sglang/srt/mem_cache/hiradix_cache.py
+++ b/python/sglang/srt/mem_cache/hiradix_cache.py
@@ -136,6 +136,8 @@ def __init__(self, params: CacheInitParams, server_args: ServerArgs):
                 extra_config=extra_config,
                 prefetch_threshold=prefetch_threshold,
                 enable_storage_metrics=self.enable_storage_metrics,
+                enable_metrics=self._enable_metrics_flag,
+                extra_metric_labels=self.extra_metric_labels,
                 load_cache_event=self.load_cache_event,
                 attn_cp_group=self.attn_cp_group,
                 attn_tp_group=self.attn_tp_group,

From 6afe77ad2b6374948b218cc6b216d8243f37241a Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Thu, 4 Jun 2026 21:04:05 -0700
Subject: [PATCH 21/28] fix(hicache/metrics): pass resolved_pool_transfers to
 _make_hicache_ack in hybrid write/load paths

Completes the byte-estimation fix in cache_controller.py: without forwarding
resolved_pool_transfers here, sidecar pools (Mamba, SWA, DSA indexer) would
still contribute zero bytes to the ack, leaving only KV bytes accounted for.
---
 .../srt/mem_cache/hybrid_cache/hybrid_cache_controller.py       | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py
index ef4bf1241bd9..6ebca96c3592 100644
--- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py
+++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py
@@ -443,6 +443,7 @@ def start_writing(self) -> None:
                 finish_event=finish_event,
                 node_ids=op.node_ids,
                 token_count=token_count,
+                pool_transfers=resolved_pool_transfers,
             )
         )
 
@@ -538,6 +539,7 @@ def start_loading(self) -> int:
                 finish_event=producer_event.finish_event,
                 node_ids=op.node_ids,
                 token_count=token_count,
+                pool_transfers=resolved_pool_transfers,
             )
         )
         return producer_id

From ee6b9184ff3b95dfc606f5fb72f1ae9358c3faef Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Thu, 4 Jun 2026 21:04:29 -0700
Subject: [PATCH 22/28] fix(hicache/mamba): add enable_metrics and
 extra_metric_labels params to attach_hybrid_pool_to_mamba_cache

The function body already forwarded these two names to build_hybrid_mamba_stack,
but they were never declared in the signature, causing a NameError at startup for
any Mamba model with HiCache enabled.
---
 .../sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py
index 64e3dc4e4111..86b9c82da8fd 100644
--- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py
+++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py
@@ -1238,6 +1238,8 @@ def attach_hybrid_pool_to_mamba_cache(
     prefetch_threshold: int,
     load_cache_event,
     enable_storage_metrics: bool = False,
+    enable_metrics: bool = False,
+    extra_metric_labels: Optional[dict[str, str]] = None,
     attn_cp_group: Optional[torch.distributed.ProcessGroup] = None,
     attn_tp_group: Optional[torch.distributed.ProcessGroup] = None,
 ) -> None:

From ce12ca8ffb9aad8ac05a78de5d73308344aec875 Mon Sep 17 00:00:00 2001
From: Ishan Dhanani <ishandhanani@gmail.com>
Date: Mon, 8 Jun 2026 13:57:07 +0000
Subject: [PATCH 23/28] fix(hicache/metrics): make L1/L2 transfer metrics emit
 real data on CUDA

Two issues left the transfer metrics non-functional on GPU after merging main:

1. Durations were never recorded. record_l1_l2_transfer_complete feeds
   ack.start_event.elapsed_time(finish_event) into the duration histogram, but
   the events were created without enable_timing=True, so elapsed_time() always
   raises on CUDA and _transfer_elapsed_us returns None -> the duration_us
   histogram and bandwidth got no data (xfer_us=unavailable, bandwidth=0).
   Create the event pairs used for timing with enable_timing=True
   (LayerLoadingEvent.start_event + last layer event; write-path start/finish in
   cache_controller and hybrid_cache_controller). The Optional[int] host
   fallback is preserved for backends without elapsed_time().

2. The Prometheus collector was never created without L3 storage. main moved
   self.pp_rank/pp_size assignment into _generate_storage_config, which only runs
   when a storage backend is attached; _init_l1_l2_transfer_metrics (needed for
   the collector) reads self.pp_rank and crashed (AttributeError) or was skipped.
   Derive pp_rank/pp_size in __init__ before calling _init so the collector is
   built for plain HiCache too.

Verified on 1xL40S: offload/onboard duration histogram now populates
(count == debug cumulative events), ~26 GB/s device timing, gating intact
(no --enable-metrics -> 0 series).
---
 .../sglang/srt/managers/cache_controller.py   | 24 +++++++++++++++----
 .../hybrid_cache/hybrid_cache_controller.py   |  6 +++--
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py
index 78d5927250a5..5061d4f0929e 100644
--- a/python/sglang/srt/managers/cache_controller.py
+++ b/python/sglang/srt/managers/cache_controller.py
@@ -59,8 +59,17 @@
 class LayerLoadingEvent:
     def __init__(self, num_layers: int):
         self._num_layers = num_layers
-        self.load_events = [device_module.Event() for _ in range(num_layers)]
-        self.start_event = device_module.Event()  # start event on controller stream
+        # The last layer's event doubles as finish_event; together with
+        # start_event it feeds elapsed_time() in record_l1_l2_transfer_complete,
+        # which requires both events to be created with enable_timing=True.
+        # Other per-layer events stay timing-free (only used for stream waits).
+        self.load_events = [
+            device_module.Event(enable_timing=(i == num_layers - 1))
+            for i in range(num_layers)
+        ]
+        self.start_event = device_module.Event(
+            enable_timing=True
+        )  # start event on controller stream
 
     def complete(self, layer_index: int):
         assert 0 <= layer_index < self._num_layers
@@ -282,6 +291,11 @@ def __init__(
         }
 
         if self.enable_l1_l2_transfer_metrics:
+            # pp_rank/pp_size are otherwise only set in _generate_storage_config,
+            # which runs only when a storage backend is attached. Derive them here
+            # so the transfer-metrics labels are available without L3 storage.
+            self.pp_rank = get_pipeline_model_parallel_rank()
+            self.pp_size = get_pipeline_model_parallel_world_size()
             self._init_l1_l2_transfer_metrics(extra_metric_labels)
 
         # Draft KV pool support (best-effort piggyback on target L2/L3 ops).
@@ -726,8 +740,10 @@ def start_writing(self) -> None:
             )
         self.write_queue.clear()
 
-        start_event = device_module.Event()
-        finish_event = device_module.Event()
+        # enable_timing so record_l1_l2_transfer_complete can use elapsed_time()
+        # for the real device transfer duration (host fallback returns None).
+        start_event = device_module.Event(enable_timing=True)
+        finish_event = device_module.Event(enable_timing=True)
 
         token_count = int(host_indices.numel())
         start_event.record()
diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py
index 6ebca96c3592..40df95c8730e 100644
--- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py
+++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py
@@ -409,8 +409,10 @@ def start_writing(self) -> None:
                 self.move_hybrid_indices(op)
             )
         self.write_queue.clear()
-        start_event = device_module.Event()
-        finish_event = device_module.Event()
+        # enable_timing so record_l1_l2_transfer_complete can use elapsed_time()
+        # for the real device transfer duration (host fallback returns None).
+        start_event = device_module.Event(enable_timing=True)
+        finish_event = device_module.Event(enable_timing=True)
 
         token_count = int(host_indices.numel())
         start_event.record()

From cc7a3957717b09a9e9d00e439767de734534c59b Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Mon, 8 Jun 2026 11:40:35 -0700
Subject: [PATCH 24/28] test(hicache/metrics): add CPU unit tests for L1/L2
 transfer metrics

Add test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py.
CPU-only via _ControllerStub; covers collector call-chain and
record_l1_l2_transfer_complete with DI recording doubles.
---
 .../mem_cache/test_hicache_l1l2_metrics.py    | 639 ++++++++++++++++++
 1 file changed, 639 insertions(+)
 create mode 100644 test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py

diff --git a/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py b/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py
new file mode 100644
index 000000000000..1cdb61ad350e
--- /dev/null
+++ b/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py
@@ -0,0 +1,639 @@
+"""Unit tests for HiCache L1/L2 transfer metrics.
+
+Covers:
+- HiCacheL1L2TransferMetricsCollector.record_transfer
+- HiCacheController helper methods: _transfer_index_count, _host_pool_size_per_token,
+  _estimate_l1_l2_transfer_bytes, _make_hicache_ack, _transfer_elapsed_us
+- HiCacheController.record_l1_l2_transfer_complete (totals, early-exit, direction labels)
+
+All tests run on CPU; no GPU device or distributed process group is required.
+"""
+
+from sglang.test.ci.ci_register import register_cpu_ci
+
+register_cpu_ci(est_time=3, suite="base-a-test-cpu")
+
+import logging
+import types
+import unittest
+from unittest.mock import MagicMock, patch
+
+import torch
+
+from sglang.srt.managers.cache_controller import HiCacheAck, HiCacheController
+from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
+from sglang.srt.observability.metrics_collector import HiCacheL1L2TransferMetricsCollector
+
+
+# ── Test doubles ──────────────────────────────────────────────────────────────
+
+
+class _RecordingCounter:
+    """prometheus_client.Counter stand-in.
+
+    .labels() returns self so the inc() call chains directly, and every call
+    is recorded for assertion.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.inc_calls: list = []
+        self.label_calls: list = []
+
+    def labels(self, **kwargs):
+        self.label_calls.append(dict(kwargs))
+        return self
+
+    def inc(self, amount=1):
+        self.inc_calls.append(amount)
+
+
+class _RecordingHistogram(_RecordingCounter):
+    """prometheus_client.Histogram stand-in."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.observe_calls: list = []
+
+    def observe(self, value):
+        self.observe_calls.append(value)
+
+
+# ── Host-pool helpers ─────────────────────────────────────────────────────────
+
+
+class _PlainHostPool:
+    """Pool with a flat size_per_token attribute — the common HiCache case."""
+
+    def __init__(self, size_per_token: int):
+        self.size_per_token = size_per_token
+
+
+class _UnknownPool:
+    """Pool without size_per_token and without entry_map — triggers the warning path."""
+
+
+class _HybridPool:
+    """Pool with entry_map — models the hybrid/sidecar cache host pool."""
+
+    def __init__(self, pool_map: dict):
+        # pool_map: {PoolName: object with .host_pool.size_per_token}  or  None value
+        self.entry_map = pool_map
+
+
+def _hybrid_entry(size_per_token: int):
+    """Build a minimal entry_map value (entry.host_pool.size_per_token)."""
+    return types.SimpleNamespace(host_pool=types.SimpleNamespace(size_per_token=size_per_token))
+
+
+# ── Controller stub ───────────────────────────────────────────────────────────
+
+
+class _ControllerStub:
+    """Minimal stand-in that lets HiCacheController unbound methods execute on CPU.
+
+    Real methods are bound at class level; tests set instance attributes to
+    configure the method's observable state.
+    """
+
+    def __init__(self):
+        self.page_size = 16
+        self.has_draft = False
+        self.mem_pool_host = _PlainHostPool(2048)
+        self.mem_pool_host_draft = None
+        self._warned_unknown_host_pool_bytes = False
+        self.hicache_l1_l2_transfer_metrics_collector = None
+        self.hicache_l1_l2_transfer_totals = {
+            "offload": {"events": 0, "blocks": 0, "bytes": 0, "xfer_us": 0},
+            "onboard": {"events": 0, "blocks": 0, "bytes": 0, "xfer_us": 0},
+        }
+
+    # Bind real HiCacheController methods
+    _warn_unknown_host_pool_bytes = HiCacheController._warn_unknown_host_pool_bytes
+    _host_pool_size_per_token = HiCacheController._host_pool_size_per_token
+    _transfer_index_count = HiCacheController._transfer_index_count
+    _estimate_l1_l2_transfer_bytes = HiCacheController._estimate_l1_l2_transfer_bytes
+    _make_hicache_ack = HiCacheController._make_hicache_ack
+    _transfer_elapsed_us = HiCacheController._transfer_elapsed_us
+    record_l1_l2_transfer_complete = HiCacheController.record_l1_l2_transfer_complete
+
+
+# ── Ack factory ───────────────────────────────────────────────────────────────
+
+
+def _make_ack(
+    *,
+    node_ids=None,
+    token_count=64,
+    block_count=4,
+    byte_count=65536,
+    start_event=None,
+    finish_event=None,
+) -> HiCacheAck:
+    return HiCacheAck(
+        start_event=start_event or MagicMock(),
+        finish_event=finish_event or MagicMock(),
+        node_ids=node_ids if node_ids is not None else [1, 2],
+        token_count=token_count,
+        block_count=block_count,
+        byte_count=byte_count,
+    )
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# HiCacheL1L2TransferMetricsCollector
+# ══════════════════════════════════════════════════════════════════════════════
+
+
+class TestHiCacheL1L2TransferCollectorDI(unittest.TestCase):
+    """DI hook attrs must default to None so prometheus_client is used unchanged."""
+
+    def test_di_attrs_default_none(self):
+        self.assertIsNone(HiCacheL1L2TransferMetricsCollector._counter_cls)
+        self.assertIsNone(HiCacheL1L2TransferMetricsCollector._histogram_cls)
+
+
+class TestHiCacheL1L2TransferCollectorRecordTransfer(unittest.TestCase):
+    """Verify record_transfer updates counters and histograms correctly."""
+
+    class _Collector(HiCacheL1L2TransferMetricsCollector):
+        _counter_cls = _RecordingCounter
+        _histogram_cls = _RecordingHistogram
+
+    def setUp(self):
+        # Clear the metric cache so every test gets fresh Recording instances.
+        HiCacheL1L2TransferMetricsCollector._metric_cache.clear()
+
+    def tearDown(self):
+        HiCacheL1L2TransferMetricsCollector._metric_cache.clear()
+
+    def _make_collector(self, extra_labels=None):
+        labels = {"tp_rank": "0", "io_backend": "test"}
+        if extra_labels:
+            labels.update(extra_labels)
+        return self._Collector(labels=labels)
+
+    def test_record_offload_increments_blocks_counter(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="offload",
+            src="sglang_hicache::L1",
+            dst="sglang_hicache::L2",
+            blocks=8,
+            bytes_=131072,
+            xfer_us=None,
+        )
+        self.assertEqual(c.transfer_blocks_total.inc_calls, [8])
+
+    def test_record_offload_increments_bytes_counter(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="offload",
+            src="sglang_hicache::L1",
+            dst="sglang_hicache::L2",
+            blocks=8,
+            bytes_=131072,
+            xfer_us=None,
+        )
+        self.assertEqual(c.transfer_bytes_total.inc_calls, [131072])
+
+    def test_record_onboard_increments_blocks_and_bytes(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="onboard",
+            src="sglang_hicache::L2",
+            dst="sglang_hicache::L1",
+            blocks=4,
+            bytes_=65536,
+            xfer_us=None,
+        )
+        self.assertEqual(c.transfer_blocks_total.inc_calls, [4])
+        self.assertEqual(c.transfer_bytes_total.inc_calls, [65536])
+
+    def test_record_with_xfer_us_observes_histogram(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="offload",
+            src="sglang_hicache::L1",
+            dst="sglang_hicache::L2",
+            blocks=2,
+            bytes_=32768,
+            xfer_us=5000,
+        )
+        self.assertEqual(c.transfer_duration_us.observe_calls, [5000])
+
+    def test_record_without_xfer_us_skips_histogram(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="offload",
+            src="sglang_hicache::L1",
+            dst="sglang_hicache::L2",
+            blocks=2,
+            bytes_=32768,
+            xfer_us=None,
+        )
+        self.assertEqual(c.transfer_duration_us.observe_calls, [])
+
+    def test_metric_labels_contain_direction_src_dst(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="offload",
+            src="sglang_hicache::L1",
+            dst="sglang_hicache::L2",
+            blocks=1,
+            bytes_=1024,
+            xfer_us=None,
+        )
+        labels_used = c.transfer_blocks_total.label_calls[0]
+        self.assertEqual(labels_used["direction"], "offload")
+        self.assertEqual(labels_used["src"], "sglang_hicache::L1")
+        self.assertEqual(labels_used["dst"], "sglang_hicache::L2")
+        # Extra labels from constructor are also present
+        self.assertEqual(labels_used["tp_rank"], "0")
+
+    def test_multiple_transfers_accumulate(self):
+        c = self._make_collector()
+        for _ in range(3):
+            c.record_transfer(
+                direction="onboard",
+                src="sglang_hicache::L2",
+                dst="sglang_hicache::L1",
+                blocks=2,
+                bytes_=4096,
+                xfer_us=100,
+            )
+        self.assertEqual(c.transfer_blocks_total.inc_calls, [2, 2, 2])
+        self.assertEqual(c.transfer_duration_us.observe_calls, [100, 100, 100])
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# HiCacheController._transfer_index_count
+# ══════════════════════════════════════════════════════════════════════════════
+
+
+class TestHiCacheControllerTransferIndexCount(unittest.TestCase):
+    def setUp(self):
+        self.ctrl = _ControllerStub()
+
+    def test_host_indices_preferred(self):
+        xfer = PoolTransfer(name=PoolName.KV, host_indices=torch.arange(32))
+        self.assertEqual(self.ctrl._transfer_index_count(xfer), 32)
+
+    def test_device_indices_fallback_when_host_none(self):
+        xfer = PoolTransfer(
+            name=PoolName.MAMBA,
+            host_indices=None,
+            device_indices=torch.arange(16),
+        )
+        self.assertEqual(self.ctrl._transfer_index_count(xfer), 16)
+
+    def test_both_none_returns_zero(self):
+        xfer = PoolTransfer(name=PoolName.SWA, host_indices=None, device_indices=None)
+        self.assertEqual(self.ctrl._transfer_index_count(xfer), 0)
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# HiCacheController._host_pool_size_per_token
+# ══════════════════════════════════════════════════════════════════════════════
+
+
+class TestHiCacheControllerHostPoolSizePerToken(unittest.TestCase):
+    def setUp(self):
+        self.ctrl = _ControllerStub()
+
+    def test_plain_pool_returns_size_per_token(self):
+        self.ctrl.mem_pool_host = _PlainHostPool(4096)
+        result = self.ctrl._host_pool_size_per_token(PoolName.KV)
+        self.assertEqual(result, 4096)
+
+    def test_plain_pool_called_without_pool_name(self):
+        self.ctrl.mem_pool_host = _PlainHostPool(1024)
+        result = self.ctrl._host_pool_size_per_token()
+        self.assertEqual(result, 1024)
+
+    def test_entry_map_pool_found(self):
+        entry = _hybrid_entry(size_per_token=512)
+        self.ctrl.mem_pool_host = _HybridPool({PoolName.MAMBA: entry})
+        result = self.ctrl._host_pool_size_per_token(PoolName.MAMBA)
+        self.assertEqual(result, 512)
+
+    def test_entry_map_pool_not_found_returns_zero(self):
+        entry = _hybrid_entry(size_per_token=512)
+        self.ctrl.mem_pool_host = _HybridPool({PoolName.MAMBA: entry})
+        result = self.ctrl._host_pool_size_per_token(PoolName.SWA)
+        self.assertEqual(result, 0)
+
+    def test_unknown_pool_warns_and_returns_zero(self):
+        self.ctrl.mem_pool_host = _UnknownPool()
+        with self.assertLogs("sglang.srt.managers.cache_controller", level="WARNING"):
+            result = self.ctrl._host_pool_size_per_token(PoolName.KV)
+        self.assertEqual(result, 0)
+
+    def test_unknown_pool_warns_only_once(self):
+        self.ctrl.mem_pool_host = _UnknownPool()
+        with self.assertLogs("sglang.srt.managers.cache_controller", level="WARNING") as cm:
+            self.ctrl._host_pool_size_per_token(PoolName.KV)
+            self.ctrl._host_pool_size_per_token(PoolName.KV)
+        # The flag should have been set after the first call, suppressing the second.
+        self.assertTrue(self.ctrl._warned_unknown_host_pool_bytes)
+        self.assertEqual(len(cm.output), 1)
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# HiCacheController._estimate_l1_l2_transfer_bytes
+# ══════════════════════════════════════════════════════════════════════════════
+
+
+class TestHiCacheControllerEstimateBytes(unittest.TestCase):
+    def setUp(self):
+        self.ctrl = _ControllerStub()
+        self.ctrl.mem_pool_host = _PlainHostPool(2048)  # 2 KiB per KV token
+
+    def test_plain_kv_only(self):
+        # 32 tokens × 2048 bytes = 65536
+        result = self.ctrl._estimate_l1_l2_transfer_bytes(token_count=32)
+        self.assertEqual(result, 32 * 2048)
+
+    def test_zero_tokens_returns_zero(self):
+        result = self.ctrl._estimate_l1_l2_transfer_bytes(token_count=0)
+        self.assertEqual(result, 0)
+
+    def test_with_sidecar_pool_transfer(self):
+        mamba_size = 256
+        entry = _hybrid_entry(size_per_token=mamba_size)
+        self.ctrl.mem_pool_host = _HybridPool(
+            {PoolName.KV: _hybrid_entry(2048), PoolName.MAMBA: entry}
+        )
+        xfer = PoolTransfer(name=PoolName.MAMBA, host_indices=torch.arange(8))
+        # KV: 16 tokens × 2048 + Mamba sidecar: 8 indices × 256
+        expected = 16 * 2048 + 8 * mamba_size
+        result = self.ctrl._estimate_l1_l2_transfer_bytes(
+            token_count=16, pool_transfers=[xfer]
+        )
+        self.assertEqual(result, expected)
+
+    def test_with_draft_pool(self):
+        self.ctrl.has_draft = True
+        self.ctrl.mem_pool_host_draft = _PlainHostPool(512)
+        # KV: 10 tokens × 2048 + draft: 10 tokens × 512
+        expected = 10 * 2048 + 10 * 512
+        result = self.ctrl._estimate_l1_l2_transfer_bytes(token_count=10)
+        self.assertEqual(result, expected)
+
+    def test_draft_pool_missing_size_per_token_warns_and_omits(self):
+        self.ctrl.has_draft = True
+        self.ctrl.mem_pool_host_draft = _UnknownPool()
+        with self.assertLogs("sglang.srt.managers.cache_controller", level="WARNING"):
+            result = self.ctrl._estimate_l1_l2_transfer_bytes(token_count=4)
+        # Draft bytes omitted; only KV contribution.
+        self.assertEqual(result, 4 * 2048)
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# HiCacheController._make_hicache_ack — block_count ceiling division
+# ══════════════════════════════════════════════════════════════════════════════
+
+
+class TestHiCacheControllerMakeHicacheAck(unittest.TestCase):
+    def _make_ack_via_ctrl(self, token_count, page_size=16):
+        ctrl = _ControllerStub()
+        ctrl.page_size = page_size
+        ctrl.mem_pool_host = _PlainHostPool(1024)
+        return ctrl._make_hicache_ack(
+            start_event=MagicMock(),
+            finish_event=MagicMock(),
+            node_ids=[1],
+            token_count=token_count,
+        )
+
+    def test_block_count_exact_page(self):
+        ack = self._make_ack_via_ctrl(token_count=32, page_size=16)
+        self.assertEqual(ack.block_count, 2)
+
+    def test_block_count_partial_page(self):
+        ack = self._make_ack_via_ctrl(token_count=33, page_size=16)
+        self.assertEqual(ack.block_count, 3)
+
+    def test_block_count_single_token(self):
+        ack = self._make_ack_via_ctrl(token_count=1, page_size=16)
+        self.assertEqual(ack.block_count, 1)
+
+    def test_block_count_full_single_page(self):
+        ack = self._make_ack_via_ctrl(token_count=16, page_size=16)
+        self.assertEqual(ack.block_count, 1)
+
+    def test_byte_count_matches_estimate(self):
+        ctrl = _ControllerStub()
+        ctrl.page_size = 16
+        ctrl.mem_pool_host = _PlainHostPool(2048)
+        ack = ctrl._make_hicache_ack(
+            start_event=MagicMock(),
+            finish_event=MagicMock(),
+            node_ids=[1],
+            token_count=10,
+        )
+        # _estimate_l1_l2_transfer_bytes(10) = 10 * 2048 = 20480
+        self.assertEqual(ack.byte_count, 10 * 2048)
+
+    def test_token_count_stored_correctly(self):
+        ack = self._make_ack_via_ctrl(token_count=48, page_size=16)
+        self.assertEqual(ack.token_count, 48)
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# HiCacheController._transfer_elapsed_us
+# ══════════════════════════════════════════════════════════════════════════════
+
+
+class TestHiCacheControllerTransferElapsedUs(unittest.TestCase):
+    def setUp(self):
+        self.ctrl = _ControllerStub()
+
+    def _ack_with_elapsed(self, elapsed_ms: float) -> HiCacheAck:
+        start = MagicMock()
+        finish = MagicMock()
+        start.elapsed_time.return_value = elapsed_ms
+        return _make_ack(start_event=start, finish_event=finish)
+
+    def test_converts_ms_to_us(self):
+        ack = self._ack_with_elapsed(1.5)
+        result = self.ctrl._transfer_elapsed_us(ack)
+        self.assertEqual(result, 1500)
+
+    def test_exact_milliseconds(self):
+        ack = self._ack_with_elapsed(10.0)
+        result = self.ctrl._transfer_elapsed_us(ack)
+        self.assertEqual(result, 10000)
+
+    def test_clamps_negative_to_zero(self):
+        ack = self._ack_with_elapsed(-5.0)
+        result = self.ctrl._transfer_elapsed_us(ack)
+        self.assertEqual(result, 0)
+
+    def test_exception_returns_none(self):
+        # Non-CUDA backends raise on elapsed_time() (timing not supported).
+        start = MagicMock()
+        start.elapsed_time.side_effect = RuntimeError("CUDA not available")
+        ack = _make_ack(start_event=start)
+        result = self.ctrl._transfer_elapsed_us(ack)
+        self.assertIsNone(result)
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# HiCacheController.record_l1_l2_transfer_complete
+# ══════════════════════════════════════════════════════════════════════════════
+
+
+class TestRecordL1L2TransferComplete(unittest.TestCase):
+    """Test totals accumulation, early-exit, direction labels, and collector call."""
+
+    def _make_ctrl(self, *, with_collector: bool = True) -> _ControllerStub:
+        ctrl = _ControllerStub()
+        if with_collector:
+            ctrl.hicache_l1_l2_transfer_metrics_collector = MagicMock()
+        # Return a fixed xfer_us so totals tests are deterministic.
+        ctrl._transfer_elapsed_us = MagicMock(return_value=2000)
+        return ctrl
+
+    # ── early exit ────────────────────────────────────────────────────────────
+
+    def test_early_exit_leaves_totals_unchanged(self):
+        ctrl = _ControllerStub()
+        ctrl.hicache_l1_l2_transfer_metrics_collector = None
+        ack = _make_ack(block_count=4, byte_count=65536)
+
+        with patch("sglang.srt.managers.cache_controller.logger") as mock_log:
+            mock_log.isEnabledFor.return_value = False
+            ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
+
+        totals = ctrl.hicache_l1_l2_transfer_totals["offload"]
+        self.assertEqual(totals["events"], 0)
+        self.assertEqual(totals["blocks"], 0)
+        self.assertEqual(totals["bytes"], 0)
+
+    # ── totals accumulation ───────────────────────────────────────────────────
+
+    def test_offload_totals_accumulate_over_two_calls(self):
+        ctrl = self._make_ctrl()
+        ack = _make_ack(block_count=4, byte_count=65536)
+        ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
+        ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
+
+        totals = ctrl.hicache_l1_l2_transfer_totals["offload"]
+        self.assertEqual(totals["events"], 2)
+        self.assertEqual(totals["blocks"], 8)
+        self.assertEqual(totals["bytes"], 131072)
+
+    def test_onboard_totals_accumulate_independently(self):
+        ctrl = self._make_ctrl()
+        ack = _make_ack(block_count=2, byte_count=32768)
+        ctrl.record_l1_l2_transfer_complete(direction="onboard", ack=ack)
+
+        on = ctrl.hicache_l1_l2_transfer_totals["onboard"]
+        off = ctrl.hicache_l1_l2_transfer_totals["offload"]
+        self.assertEqual(on["events"], 1)
+        self.assertEqual(off["events"], 0)
+
+    def test_xfer_us_accumulates_in_totals(self):
+        ctrl = self._make_ctrl()
+        ctrl._transfer_elapsed_us = MagicMock(return_value=1000)
+        ack = _make_ack()
+        ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
+        ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
+
+        self.assertEqual(
+            ctrl.hicache_l1_l2_transfer_totals["offload"]["xfer_us"], 2000
+        )
+
+    def test_xfer_us_none_leaves_total_zero(self):
+        ctrl = self._make_ctrl()
+        ctrl._transfer_elapsed_us = MagicMock(return_value=None)
+        ack = _make_ack()
+        ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
+
+        self.assertEqual(
+            ctrl.hicache_l1_l2_transfer_totals["offload"]["xfer_us"], 0
+        )
+
+    # ── invalid direction ─────────────────────────────────────────────────────
+
+    def test_invalid_direction_raises_value_error(self):
+        ctrl = self._make_ctrl()
+        ack = _make_ack()
+        with self.assertRaises(ValueError):
+            ctrl.record_l1_l2_transfer_complete(direction="sideways", ack=ack)
+
+    # ── collector call-through ────────────────────────────────────────────────
+
+    def test_collector_called_with_correct_args_offload(self):
+        ctrl = self._make_ctrl()
+        ctrl._transfer_elapsed_us = MagicMock(return_value=5000)
+        ack = _make_ack(block_count=3, byte_count=49152)
+
+        ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
+
+        ctrl.hicache_l1_l2_transfer_metrics_collector.record_transfer.assert_called_once_with(
+            direction="offload",
+            src="sglang_hicache::L1",
+            dst="sglang_hicache::L2",
+            blocks=3,
+            bytes_=49152,
+            xfer_us=5000,
+        )
+
+    def test_collector_called_with_correct_args_onboard(self):
+        ctrl = self._make_ctrl()
+        ctrl._transfer_elapsed_us = MagicMock(return_value=3000)
+        ack = _make_ack(block_count=6, byte_count=98304)
+
+        ctrl.record_l1_l2_transfer_complete(direction="onboard", ack=ack)
+
+        ctrl.hicache_l1_l2_transfer_metrics_collector.record_transfer.assert_called_once_with(
+            direction="onboard",
+            src="sglang_hicache::L2",
+            dst="sglang_hicache::L1",
+            blocks=6,
+            bytes_=98304,
+            xfer_us=3000,
+        )
+
+    def test_offload_src_dst_labels(self):
+        ctrl = self._make_ctrl()
+        ack = _make_ack()
+        ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
+        call_kwargs = (
+            ctrl.hicache_l1_l2_transfer_metrics_collector.record_transfer.call_args.kwargs
+        )
+        self.assertEqual(call_kwargs["src"], "sglang_hicache::L1")
+        self.assertEqual(call_kwargs["dst"], "sglang_hicache::L2")
+
+    def test_onboard_src_dst_labels(self):
+        ctrl = self._make_ctrl()
+        ack = _make_ack()
+        ctrl.record_l1_l2_transfer_complete(direction="onboard", ack=ack)
+        call_kwargs = (
+            ctrl.hicache_l1_l2_transfer_metrics_collector.record_transfer.call_args.kwargs
+        )
+        self.assertEqual(call_kwargs["src"], "sglang_hicache::L2")
+        self.assertEqual(call_kwargs["dst"], "sglang_hicache::L1")
+
+    def test_no_collector_does_not_call_record_transfer(self):
+        ctrl = _ControllerStub()
+        ctrl.hicache_l1_l2_transfer_metrics_collector = None
+        ctrl._transfer_elapsed_us = MagicMock(return_value=1000)
+        ack = _make_ack()
+
+        # Set DEBUG logging so we don't early-exit (should_log=True keeps it running).
+        cache_ctrl_logger = logging.getLogger("sglang.srt.managers.cache_controller")
+        old_level = cache_ctrl_logger.level
+        try:
+            cache_ctrl_logger.setLevel(logging.DEBUG)
+            ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
+        finally:
+            cache_ctrl_logger.setLevel(old_level)
+
+        # Totals should still be updated; no collector to assert on.
+        self.assertEqual(ctrl.hicache_l1_l2_transfer_totals["offload"]["events"], 1)
+
+
+if __name__ == "__main__":
+    unittest.main()

From 98fd202898c13b931ec867b6ab06c83e36b58ca3 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Mon, 8 Jun 2026 12:01:33 -0700
Subject: [PATCH 25/28] test(hicache/metrics): add Prometheus exposition tests

Add isolated-registry tests verifying metric names, label values,
and counter/histogram sample values in generate_latest() output.
---
 .../mem_cache/test_hicache_l1l2_metrics.py    | 245 ++++++++++++++++++
 1 file changed, 245 insertions(+)

diff --git a/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py b/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py
index 1cdb61ad350e..77c7cc5a2ccd 100644
--- a/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py
+++ b/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py
@@ -635,5 +635,250 @@ def test_no_collector_does_not_call_record_transfer(self):
         self.assertEqual(ctrl.hicache_l1_l2_transfer_totals["offload"]["events"], 1)
 
 
+# ══════════════════════════════════════════════════════════════════════════════
+# Prometheus text exposition — real prometheus_client, isolated registry
+# ══════════════════════════════════════════════════════════════════════════════
+
+
+class TestHiCacheL1L2TransferCollectorPrometheusOutput(unittest.TestCase):
+    """Verify that record_transfer produces correct Prometheus text exposition.
+
+    Uses an isolated CollectorRegistry per test so metrics don't leak into
+    the global registry and tests remain independent of each other.
+    """
+
+    def setUp(self):
+        import prometheus_client
+        from prometheus_client import CollectorRegistry
+
+        self.registry = CollectorRegistry()
+        _reg = self.registry
+        _PC = prometheus_client.Counter
+        _PH = prometheus_client.Histogram
+
+        # Wrapper classes that forward construction to the real prometheus_client
+        # types but bind them to our isolated registry.  Each setUp() call
+        # produces a new class object, so the _metric_cache key is unique and
+        # a fresh Counter/Histogram pair is created every test.
+        class _BoundCounter:
+            def __init__(self, *args, **kwargs):
+                kwargs["registry"] = _reg
+                self._inner = _PC(*args, **kwargs)
+
+            def labels(self, **kwargs):
+                return self._inner.labels(**kwargs)
+
+        class _BoundHistogram:
+            def __init__(self, *args, **kwargs):
+                kwargs["registry"] = _reg
+                self._inner = _PH(*args, **kwargs)
+
+            def labels(self, **kwargs):
+                return self._inner.labels(**kwargs)
+
+        class _TestCollector(HiCacheL1L2TransferMetricsCollector):
+            _counter_cls = _BoundCounter
+            _histogram_cls = _BoundHistogram
+
+        self._TestCollector = _TestCollector
+        HiCacheL1L2TransferMetricsCollector._metric_cache.clear()
+
+    def tearDown(self):
+        HiCacheL1L2TransferMetricsCollector._metric_cache.clear()
+
+    def _make_collector(self):
+        return self._TestCollector(labels={"tp_rank": "0", "io_backend": "nixl"})
+
+    def _exposition(self) -> str:
+        from prometheus_client import generate_latest
+
+        return generate_latest(self.registry).decode("utf-8")
+
+    # ── metric names ──────────────────────────────────────────────────────────
+
+    def test_blocks_counter_name_in_exposition(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="offload",
+            src="sglang_hicache::L1",
+            dst="sglang_hicache::L2",
+            blocks=5,
+            bytes_=81920,
+            xfer_us=None,
+        )
+        self.assertIn("sglang:hicache_l1_l2_transfer_blocks_total", self._exposition())
+
+    def test_bytes_counter_name_in_exposition(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="offload",
+            src="sglang_hicache::L1",
+            dst="sglang_hicache::L2",
+            blocks=1,
+            bytes_=16384,
+            xfer_us=None,
+        )
+        self.assertIn("sglang:hicache_l1_l2_transfer_bytes_total", self._exposition())
+
+    def test_duration_histogram_name_in_exposition(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="onboard",
+            src="sglang_hicache::L2",
+            dst="sglang_hicache::L1",
+            blocks=2,
+            bytes_=32768,
+            xfer_us=3000,
+        )
+        self.assertIn("sglang:hicache_l1_l2_transfer_duration_us", self._exposition())
+
+    # ── label values ──────────────────────────────────────────────────────────
+
+    def test_direction_label_in_exposition(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="onboard",
+            src="sglang_hicache::L2",
+            dst="sglang_hicache::L1",
+            blocks=3,
+            bytes_=49152,
+            xfer_us=None,
+        )
+        self.assertIn('direction="onboard"', self._exposition())
+
+    def test_src_and_dst_labels_in_exposition(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="offload",
+            src="sglang_hicache::L1",
+            dst="sglang_hicache::L2",
+            blocks=1,
+            bytes_=1024,
+            xfer_us=None,
+        )
+        output = self._exposition()
+        self.assertIn('src="sglang_hicache::L1"', output)
+        self.assertIn('dst="sglang_hicache::L2"', output)
+
+    def test_constructor_labels_in_exposition(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="offload",
+            src="sglang_hicache::L1",
+            dst="sglang_hicache::L2",
+            blocks=1,
+            bytes_=1024,
+            xfer_us=None,
+        )
+        output = self._exposition()
+        self.assertIn('tp_rank="0"', output)
+        self.assertIn('io_backend="nixl"', output)
+
+    # ── counter values ────────────────────────────────────────────────────────
+
+    def _find_metric_line(self, exposition: str, name_fragment: str, **label_filters) -> str:
+        """Return the first non-comment exposition line matching name_fragment
+        and all label_filters, or raise AssertionError."""
+        for line in exposition.splitlines():
+            if line.startswith("#"):
+                continue
+            if name_fragment not in line:
+                continue
+            if all(f'{k}="{v}"' in line for k, v in label_filters.items()):
+                return line
+        self.fail(
+            f"No metric line found for fragment={name_fragment!r} "
+            f"labels={label_filters} in:\n{exposition}"
+        )
+
+    def test_blocks_counter_value(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="offload",
+            src="sglang_hicache::L1",
+            dst="sglang_hicache::L2",
+            blocks=7,
+            bytes_=114688,
+            xfer_us=None,
+        )
+        line = self._find_metric_line(
+            self._exposition(),
+            "hicache_l1_l2_transfer_blocks_total",
+            direction="offload",
+        )
+        self.assertTrue(line.endswith(" 7.0"), f"Expected value 7.0 in: {line!r}")
+
+    def test_bytes_counter_value(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="onboard",
+            src="sglang_hicache::L2",
+            dst="sglang_hicache::L1",
+            blocks=1,
+            bytes_=32768,
+            xfer_us=None,
+        )
+        line = self._find_metric_line(
+            self._exposition(),
+            "hicache_l1_l2_transfer_bytes_total",
+            direction="onboard",
+        )
+        self.assertTrue(line.endswith(" 32768.0"), f"Expected value 32768.0 in: {line!r}")
+
+    # ── histogram ─────────────────────────────────────────────────────────────
+
+    def test_duration_histogram_sum_when_xfer_us_provided(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="offload",
+            src="sglang_hicache::L1",
+            dst="sglang_hicache::L2",
+            blocks=1,
+            bytes_=1024,
+            xfer_us=2500,
+        )
+        line = self._find_metric_line(
+            self._exposition(),
+            "hicache_l1_l2_transfer_duration_us_sum",
+            direction="offload",
+        )
+        self.assertTrue(line.endswith(" 2500.0"), f"Expected sum 2500.0 in: {line!r}")
+
+    def test_duration_histogram_count_when_xfer_us_provided(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="offload",
+            src="sglang_hicache::L1",
+            dst="sglang_hicache::L2",
+            blocks=1,
+            bytes_=1024,
+            xfer_us=1000,
+        )
+        line = self._find_metric_line(
+            self._exposition(),
+            "hicache_l1_l2_transfer_duration_us_count",
+            direction="offload",
+        )
+        self.assertTrue(line.endswith(" 1.0"), f"Expected count 1.0 in: {line!r}")
+
+    def test_duration_histogram_not_observed_when_xfer_us_none(self):
+        c = self._make_collector()
+        c.record_transfer(
+            direction="offload",
+            src="sglang_hicache::L1",
+            dst="sglang_hicache::L2",
+            blocks=1,
+            bytes_=1024,
+            xfer_us=None,
+        )
+        # Count must be 0 — no observation recorded.
+        line = self._find_metric_line(
+            self._exposition(),
+            "hicache_l1_l2_transfer_duration_us_count",
+            direction="offload",
+        )
+        self.assertTrue(line.endswith(" 0.0"), f"Expected count 0.0 in: {line!r}")
+
+
 if __name__ == "__main__":
     unittest.main()

From f47933ae593adc2c009ad63781ddc246230947c6 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Mon, 8 Jun 2026 12:19:43 -0700
Subject: [PATCH 26/28] test(hicache/metrics): fix registration and minor test
 adjustments

---
 .../mem_cache/test_hicache_l1l2_metrics.py    | 21 ++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py b/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py
index 77c7cc5a2ccd..da9659f5af53 100644
--- a/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py
+++ b/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py
@@ -871,13 +871,20 @@ def test_duration_histogram_not_observed_when_xfer_us_none(self):
             bytes_=1024,
             xfer_us=None,
         )
-        # Count must be 0 — no observation recorded.
-        line = self._find_metric_line(
-            self._exposition(),
-            "hicache_l1_l2_transfer_duration_us_count",
-            direction="offload",
-        )
-        self.assertTrue(line.endswith(" 0.0"), f"Expected count 0.0 in: {line!r}")
+        # No observation was recorded, so prometheus_client never instantiates a
+        # labeled child for the histogram — no _count series is emitted at all.
+        exposition = self._exposition()
+        for line in exposition.splitlines():
+            if line.startswith("#"):
+                continue
+            if (
+                "hicache_l1_l2_transfer_duration_us_count" in line
+                and 'direction="offload"' in line
+            ):
+                self.fail(
+                    "Expected no duration _count series when xfer_us is None, "
+                    f"but found: {line!r}\nin:\n{exposition}"
+                )
 
 
 if __name__ == "__main__":

From 1823d820e4b1737f7df712ff2077b703d9e856f0 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Mon, 8 Jun 2026 14:05:38 -0700
Subject: [PATCH 27/28] test(hicache/metrics): trim to 9 essential tests

Drop helper/edge-case tests; keep 2 DI-double, 3 controller-stub,
and 4 Prometheus exposition tests.
---
 .../mem_cache/test_hicache_l1l2_metrics.py    | 619 +-----------------
 1 file changed, 20 insertions(+), 599 deletions(-)

diff --git a/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py b/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py
index da9659f5af53..1feb15401804 100644
--- a/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py
+++ b/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py
@@ -1,11 +1,5 @@
 """Unit tests for HiCache L1/L2 transfer metrics.
 
-Covers:
-- HiCacheL1L2TransferMetricsCollector.record_transfer
-- HiCacheController helper methods: _transfer_index_count, _host_pool_size_per_token,
-  _estimate_l1_l2_transfer_bytes, _make_hicache_ack, _transfer_elapsed_us
-- HiCacheController.record_l1_l2_transfer_complete (totals, early-exit, direction labels)
-
 All tests run on CPU; no GPU device or distributed process group is required.
 """
 
@@ -13,15 +7,10 @@
 
 register_cpu_ci(est_time=3, suite="base-a-test-cpu")
 
-import logging
-import types
 import unittest
-from unittest.mock import MagicMock, patch
-
-import torch
+from unittest.mock import MagicMock
 
 from sglang.srt.managers.cache_controller import HiCacheAck, HiCacheController
-from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
 from sglang.srt.observability.metrics_collector import HiCacheL1L2TransferMetricsCollector
 
 
@@ -29,18 +18,12 @@
 
 
 class _RecordingCounter:
-    """prometheus_client.Counter stand-in.
-
-    .labels() returns self so the inc() call chains directly, and every call
-    is recorded for assertion.
-    """
+    """prometheus_client.Counter stand-in that records every call."""
 
     def __init__(self, *args, **kwargs):
         self.inc_calls: list = []
-        self.label_calls: list = []
 
     def labels(self, **kwargs):
-        self.label_calls.append(dict(kwargs))
         return self
 
     def inc(self, amount=1):
@@ -58,61 +41,19 @@ def observe(self, value):
         self.observe_calls.append(value)
 
 
-# ── Host-pool helpers ─────────────────────────────────────────────────────────
-
-
-class _PlainHostPool:
-    """Pool with a flat size_per_token attribute — the common HiCache case."""
-
-    def __init__(self, size_per_token: int):
-        self.size_per_token = size_per_token
-
-
-class _UnknownPool:
-    """Pool without size_per_token and without entry_map — triggers the warning path."""
-
-
-class _HybridPool:
-    """Pool with entry_map — models the hybrid/sidecar cache host pool."""
-
-    def __init__(self, pool_map: dict):
-        # pool_map: {PoolName: object with .host_pool.size_per_token}  or  None value
-        self.entry_map = pool_map
-
-
-def _hybrid_entry(size_per_token: int):
-    """Build a minimal entry_map value (entry.host_pool.size_per_token)."""
-    return types.SimpleNamespace(host_pool=types.SimpleNamespace(size_per_token=size_per_token))
-
-
 # ── Controller stub ───────────────────────────────────────────────────────────
 
 
 class _ControllerStub:
-    """Minimal stand-in that lets HiCacheController unbound methods execute on CPU.
-
-    Real methods are bound at class level; tests set instance attributes to
-    configure the method's observable state.
-    """
+    """Minimal stand-in that lets HiCacheController unbound methods run on CPU."""
 
     def __init__(self):
-        self.page_size = 16
-        self.has_draft = False
-        self.mem_pool_host = _PlainHostPool(2048)
-        self.mem_pool_host_draft = None
-        self._warned_unknown_host_pool_bytes = False
         self.hicache_l1_l2_transfer_metrics_collector = None
         self.hicache_l1_l2_transfer_totals = {
             "offload": {"events": 0, "blocks": 0, "bytes": 0, "xfer_us": 0},
             "onboard": {"events": 0, "blocks": 0, "bytes": 0, "xfer_us": 0},
         }
 
-    # Bind real HiCacheController methods
-    _warn_unknown_host_pool_bytes = HiCacheController._warn_unknown_host_pool_bytes
-    _host_pool_size_per_token = HiCacheController._host_pool_size_per_token
-    _transfer_index_count = HiCacheController._transfer_index_count
-    _estimate_l1_l2_transfer_bytes = HiCacheController._estimate_l1_l2_transfer_bytes
-    _make_hicache_ack = HiCacheController._make_hicache_ack
     _transfer_elapsed_us = HiCacheController._transfer_elapsed_us
     record_l1_l2_transfer_complete = HiCacheController.record_l1_l2_transfer_complete
 
@@ -120,57 +61,35 @@ def __init__(self):
 # ── Ack factory ───────────────────────────────────────────────────────────────
 
 
-def _make_ack(
-    *,
-    node_ids=None,
-    token_count=64,
-    block_count=4,
-    byte_count=65536,
-    start_event=None,
-    finish_event=None,
-) -> HiCacheAck:
+def _make_ack(*, block_count=4, byte_count=65536) -> HiCacheAck:
     return HiCacheAck(
-        start_event=start_event or MagicMock(),
-        finish_event=finish_event or MagicMock(),
-        node_ids=node_ids if node_ids is not None else [1, 2],
-        token_count=token_count,
+        start_event=MagicMock(),
+        finish_event=MagicMock(),
+        node_ids=[1, 2],
+        token_count=64,
         block_count=block_count,
         byte_count=byte_count,
     )
 
 
 # ══════════════════════════════════════════════════════════════════════════════
-# HiCacheL1L2TransferMetricsCollector
+# HiCacheL1L2TransferMetricsCollector — call-chain (DI doubles)
 # ══════════════════════════════════════════════════════════════════════════════
 
 
-class TestHiCacheL1L2TransferCollectorDI(unittest.TestCase):
-    """DI hook attrs must default to None so prometheus_client is used unchanged."""
-
-    def test_di_attrs_default_none(self):
-        self.assertIsNone(HiCacheL1L2TransferMetricsCollector._counter_cls)
-        self.assertIsNone(HiCacheL1L2TransferMetricsCollector._histogram_cls)
-
-
 class TestHiCacheL1L2TransferCollectorRecordTransfer(unittest.TestCase):
-    """Verify record_transfer updates counters and histograms correctly."""
-
     class _Collector(HiCacheL1L2TransferMetricsCollector):
         _counter_cls = _RecordingCounter
         _histogram_cls = _RecordingHistogram
 
     def setUp(self):
-        # Clear the metric cache so every test gets fresh Recording instances.
         HiCacheL1L2TransferMetricsCollector._metric_cache.clear()
 
     def tearDown(self):
         HiCacheL1L2TransferMetricsCollector._metric_cache.clear()
 
-    def _make_collector(self, extra_labels=None):
-        labels = {"tp_rank": "0", "io_backend": "test"}
-        if extra_labels:
-            labels.update(extra_labels)
-        return self._Collector(labels=labels)
+    def _make_collector(self):
+        return self._Collector(labels={"tp_rank": "0", "io_backend": "test"})
 
     def test_record_offload_increments_blocks_counter(self):
         c = self._make_collector()
@@ -184,31 +103,6 @@ def test_record_offload_increments_blocks_counter(self):
         )
         self.assertEqual(c.transfer_blocks_total.inc_calls, [8])
 
-    def test_record_offload_increments_bytes_counter(self):
-        c = self._make_collector()
-        c.record_transfer(
-            direction="offload",
-            src="sglang_hicache::L1",
-            dst="sglang_hicache::L2",
-            blocks=8,
-            bytes_=131072,
-            xfer_us=None,
-        )
-        self.assertEqual(c.transfer_bytes_total.inc_calls, [131072])
-
-    def test_record_onboard_increments_blocks_and_bytes(self):
-        c = self._make_collector()
-        c.record_transfer(
-            direction="onboard",
-            src="sglang_hicache::L2",
-            dst="sglang_hicache::L1",
-            blocks=4,
-            bytes_=65536,
-            xfer_us=None,
-        )
-        self.assertEqual(c.transfer_blocks_total.inc_calls, [4])
-        self.assertEqual(c.transfer_bytes_total.inc_calls, [65536])
-
     def test_record_with_xfer_us_observes_histogram(self):
         c = self._make_collector()
         c.record_transfer(
@@ -221,262 +115,6 @@ def test_record_with_xfer_us_observes_histogram(self):
         )
         self.assertEqual(c.transfer_duration_us.observe_calls, [5000])
 
-    def test_record_without_xfer_us_skips_histogram(self):
-        c = self._make_collector()
-        c.record_transfer(
-            direction="offload",
-            src="sglang_hicache::L1",
-            dst="sglang_hicache::L2",
-            blocks=2,
-            bytes_=32768,
-            xfer_us=None,
-        )
-        self.assertEqual(c.transfer_duration_us.observe_calls, [])
-
-    def test_metric_labels_contain_direction_src_dst(self):
-        c = self._make_collector()
-        c.record_transfer(
-            direction="offload",
-            src="sglang_hicache::L1",
-            dst="sglang_hicache::L2",
-            blocks=1,
-            bytes_=1024,
-            xfer_us=None,
-        )
-        labels_used = c.transfer_blocks_total.label_calls[0]
-        self.assertEqual(labels_used["direction"], "offload")
-        self.assertEqual(labels_used["src"], "sglang_hicache::L1")
-        self.assertEqual(labels_used["dst"], "sglang_hicache::L2")
-        # Extra labels from constructor are also present
-        self.assertEqual(labels_used["tp_rank"], "0")
-
-    def test_multiple_transfers_accumulate(self):
-        c = self._make_collector()
-        for _ in range(3):
-            c.record_transfer(
-                direction="onboard",
-                src="sglang_hicache::L2",
-                dst="sglang_hicache::L1",
-                blocks=2,
-                bytes_=4096,
-                xfer_us=100,
-            )
-        self.assertEqual(c.transfer_blocks_total.inc_calls, [2, 2, 2])
-        self.assertEqual(c.transfer_duration_us.observe_calls, [100, 100, 100])
-
-
-# ══════════════════════════════════════════════════════════════════════════════
-# HiCacheController._transfer_index_count
-# ══════════════════════════════════════════════════════════════════════════════
-
-
-class TestHiCacheControllerTransferIndexCount(unittest.TestCase):
-    def setUp(self):
-        self.ctrl = _ControllerStub()
-
-    def test_host_indices_preferred(self):
-        xfer = PoolTransfer(name=PoolName.KV, host_indices=torch.arange(32))
-        self.assertEqual(self.ctrl._transfer_index_count(xfer), 32)
-
-    def test_device_indices_fallback_when_host_none(self):
-        xfer = PoolTransfer(
-            name=PoolName.MAMBA,
-            host_indices=None,
-            device_indices=torch.arange(16),
-        )
-        self.assertEqual(self.ctrl._transfer_index_count(xfer), 16)
-
-    def test_both_none_returns_zero(self):
-        xfer = PoolTransfer(name=PoolName.SWA, host_indices=None, device_indices=None)
-        self.assertEqual(self.ctrl._transfer_index_count(xfer), 0)
-
-
-# ══════════════════════════════════════════════════════════════════════════════
-# HiCacheController._host_pool_size_per_token
-# ══════════════════════════════════════════════════════════════════════════════
-
-
-class TestHiCacheControllerHostPoolSizePerToken(unittest.TestCase):
-    def setUp(self):
-        self.ctrl = _ControllerStub()
-
-    def test_plain_pool_returns_size_per_token(self):
-        self.ctrl.mem_pool_host = _PlainHostPool(4096)
-        result = self.ctrl._host_pool_size_per_token(PoolName.KV)
-        self.assertEqual(result, 4096)
-
-    def test_plain_pool_called_without_pool_name(self):
-        self.ctrl.mem_pool_host = _PlainHostPool(1024)
-        result = self.ctrl._host_pool_size_per_token()
-        self.assertEqual(result, 1024)
-
-    def test_entry_map_pool_found(self):
-        entry = _hybrid_entry(size_per_token=512)
-        self.ctrl.mem_pool_host = _HybridPool({PoolName.MAMBA: entry})
-        result = self.ctrl._host_pool_size_per_token(PoolName.MAMBA)
-        self.assertEqual(result, 512)
-
-    def test_entry_map_pool_not_found_returns_zero(self):
-        entry = _hybrid_entry(size_per_token=512)
-        self.ctrl.mem_pool_host = _HybridPool({PoolName.MAMBA: entry})
-        result = self.ctrl._host_pool_size_per_token(PoolName.SWA)
-        self.assertEqual(result, 0)
-
-    def test_unknown_pool_warns_and_returns_zero(self):
-        self.ctrl.mem_pool_host = _UnknownPool()
-        with self.assertLogs("sglang.srt.managers.cache_controller", level="WARNING"):
-            result = self.ctrl._host_pool_size_per_token(PoolName.KV)
-        self.assertEqual(result, 0)
-
-    def test_unknown_pool_warns_only_once(self):
-        self.ctrl.mem_pool_host = _UnknownPool()
-        with self.assertLogs("sglang.srt.managers.cache_controller", level="WARNING") as cm:
-            self.ctrl._host_pool_size_per_token(PoolName.KV)
-            self.ctrl._host_pool_size_per_token(PoolName.KV)
-        # The flag should have been set after the first call, suppressing the second.
-        self.assertTrue(self.ctrl._warned_unknown_host_pool_bytes)
-        self.assertEqual(len(cm.output), 1)
-
-
-# ══════════════════════════════════════════════════════════════════════════════
-# HiCacheController._estimate_l1_l2_transfer_bytes
-# ══════════════════════════════════════════════════════════════════════════════
-
-
-class TestHiCacheControllerEstimateBytes(unittest.TestCase):
-    def setUp(self):
-        self.ctrl = _ControllerStub()
-        self.ctrl.mem_pool_host = _PlainHostPool(2048)  # 2 KiB per KV token
-
-    def test_plain_kv_only(self):
-        # 32 tokens × 2048 bytes = 65536
-        result = self.ctrl._estimate_l1_l2_transfer_bytes(token_count=32)
-        self.assertEqual(result, 32 * 2048)
-
-    def test_zero_tokens_returns_zero(self):
-        result = self.ctrl._estimate_l1_l2_transfer_bytes(token_count=0)
-        self.assertEqual(result, 0)
-
-    def test_with_sidecar_pool_transfer(self):
-        mamba_size = 256
-        entry = _hybrid_entry(size_per_token=mamba_size)
-        self.ctrl.mem_pool_host = _HybridPool(
-            {PoolName.KV: _hybrid_entry(2048), PoolName.MAMBA: entry}
-        )
-        xfer = PoolTransfer(name=PoolName.MAMBA, host_indices=torch.arange(8))
-        # KV: 16 tokens × 2048 + Mamba sidecar: 8 indices × 256
-        expected = 16 * 2048 + 8 * mamba_size
-        result = self.ctrl._estimate_l1_l2_transfer_bytes(
-            token_count=16, pool_transfers=[xfer]
-        )
-        self.assertEqual(result, expected)
-
-    def test_with_draft_pool(self):
-        self.ctrl.has_draft = True
-        self.ctrl.mem_pool_host_draft = _PlainHostPool(512)
-        # KV: 10 tokens × 2048 + draft: 10 tokens × 512
-        expected = 10 * 2048 + 10 * 512
-        result = self.ctrl._estimate_l1_l2_transfer_bytes(token_count=10)
-        self.assertEqual(result, expected)
-
-    def test_draft_pool_missing_size_per_token_warns_and_omits(self):
-        self.ctrl.has_draft = True
-        self.ctrl.mem_pool_host_draft = _UnknownPool()
-        with self.assertLogs("sglang.srt.managers.cache_controller", level="WARNING"):
-            result = self.ctrl._estimate_l1_l2_transfer_bytes(token_count=4)
-        # Draft bytes omitted; only KV contribution.
-        self.assertEqual(result, 4 * 2048)
-
-
-# ══════════════════════════════════════════════════════════════════════════════
-# HiCacheController._make_hicache_ack — block_count ceiling division
-# ══════════════════════════════════════════════════════════════════════════════
-
-
-class TestHiCacheControllerMakeHicacheAck(unittest.TestCase):
-    def _make_ack_via_ctrl(self, token_count, page_size=16):
-        ctrl = _ControllerStub()
-        ctrl.page_size = page_size
-        ctrl.mem_pool_host = _PlainHostPool(1024)
-        return ctrl._make_hicache_ack(
-            start_event=MagicMock(),
-            finish_event=MagicMock(),
-            node_ids=[1],
-            token_count=token_count,
-        )
-
-    def test_block_count_exact_page(self):
-        ack = self._make_ack_via_ctrl(token_count=32, page_size=16)
-        self.assertEqual(ack.block_count, 2)
-
-    def test_block_count_partial_page(self):
-        ack = self._make_ack_via_ctrl(token_count=33, page_size=16)
-        self.assertEqual(ack.block_count, 3)
-
-    def test_block_count_single_token(self):
-        ack = self._make_ack_via_ctrl(token_count=1, page_size=16)
-        self.assertEqual(ack.block_count, 1)
-
-    def test_block_count_full_single_page(self):
-        ack = self._make_ack_via_ctrl(token_count=16, page_size=16)
-        self.assertEqual(ack.block_count, 1)
-
-    def test_byte_count_matches_estimate(self):
-        ctrl = _ControllerStub()
-        ctrl.page_size = 16
-        ctrl.mem_pool_host = _PlainHostPool(2048)
-        ack = ctrl._make_hicache_ack(
-            start_event=MagicMock(),
-            finish_event=MagicMock(),
-            node_ids=[1],
-            token_count=10,
-        )
-        # _estimate_l1_l2_transfer_bytes(10) = 10 * 2048 = 20480
-        self.assertEqual(ack.byte_count, 10 * 2048)
-
-    def test_token_count_stored_correctly(self):
-        ack = self._make_ack_via_ctrl(token_count=48, page_size=16)
-        self.assertEqual(ack.token_count, 48)
-
-
-# ══════════════════════════════════════════════════════════════════════════════
-# HiCacheController._transfer_elapsed_us
-# ══════════════════════════════════════════════════════════════════════════════
-
-
-class TestHiCacheControllerTransferElapsedUs(unittest.TestCase):
-    def setUp(self):
-        self.ctrl = _ControllerStub()
-
-    def _ack_with_elapsed(self, elapsed_ms: float) -> HiCacheAck:
-        start = MagicMock()
-        finish = MagicMock()
-        start.elapsed_time.return_value = elapsed_ms
-        return _make_ack(start_event=start, finish_event=finish)
-
-    def test_converts_ms_to_us(self):
-        ack = self._ack_with_elapsed(1.5)
-        result = self.ctrl._transfer_elapsed_us(ack)
-        self.assertEqual(result, 1500)
-
-    def test_exact_milliseconds(self):
-        ack = self._ack_with_elapsed(10.0)
-        result = self.ctrl._transfer_elapsed_us(ack)
-        self.assertEqual(result, 10000)
-
-    def test_clamps_negative_to_zero(self):
-        ack = self._ack_with_elapsed(-5.0)
-        result = self.ctrl._transfer_elapsed_us(ack)
-        self.assertEqual(result, 0)
-
-    def test_exception_returns_none(self):
-        # Non-CUDA backends raise on elapsed_time() (timing not supported).
-        start = MagicMock()
-        start.elapsed_time.side_effect = RuntimeError("CUDA not available")
-        ack = _make_ack(start_event=start)
-        result = self.ctrl._transfer_elapsed_us(ack)
-        self.assertIsNone(result)
-
 
 # ══════════════════════════════════════════════════════════════════════════════
 # HiCacheController.record_l1_l2_transfer_complete
@@ -484,34 +122,12 @@ def test_exception_returns_none(self):
 
 
 class TestRecordL1L2TransferComplete(unittest.TestCase):
-    """Test totals accumulation, early-exit, direction labels, and collector call."""
-
-    def _make_ctrl(self, *, with_collector: bool = True) -> _ControllerStub:
+    def _make_ctrl(self) -> _ControllerStub:
         ctrl = _ControllerStub()
-        if with_collector:
-            ctrl.hicache_l1_l2_transfer_metrics_collector = MagicMock()
-        # Return a fixed xfer_us so totals tests are deterministic.
+        ctrl.hicache_l1_l2_transfer_metrics_collector = MagicMock()
         ctrl._transfer_elapsed_us = MagicMock(return_value=2000)
         return ctrl
 
-    # ── early exit ────────────────────────────────────────────────────────────
-
-    def test_early_exit_leaves_totals_unchanged(self):
-        ctrl = _ControllerStub()
-        ctrl.hicache_l1_l2_transfer_metrics_collector = None
-        ack = _make_ack(block_count=4, byte_count=65536)
-
-        with patch("sglang.srt.managers.cache_controller.logger") as mock_log:
-            mock_log.isEnabledFor.return_value = False
-            ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
-
-        totals = ctrl.hicache_l1_l2_transfer_totals["offload"]
-        self.assertEqual(totals["events"], 0)
-        self.assertEqual(totals["blocks"], 0)
-        self.assertEqual(totals["bytes"], 0)
-
-    # ── totals accumulation ───────────────────────────────────────────────────
-
     def test_offload_totals_accumulate_over_two_calls(self):
         ctrl = self._make_ctrl()
         ack = _make_ack(block_count=4, byte_count=65536)
@@ -523,52 +139,10 @@ def test_offload_totals_accumulate_over_two_calls(self):
         self.assertEqual(totals["blocks"], 8)
         self.assertEqual(totals["bytes"], 131072)
 
-    def test_onboard_totals_accumulate_independently(self):
-        ctrl = self._make_ctrl()
-        ack = _make_ack(block_count=2, byte_count=32768)
-        ctrl.record_l1_l2_transfer_complete(direction="onboard", ack=ack)
-
-        on = ctrl.hicache_l1_l2_transfer_totals["onboard"]
-        off = ctrl.hicache_l1_l2_transfer_totals["offload"]
-        self.assertEqual(on["events"], 1)
-        self.assertEqual(off["events"], 0)
-
-    def test_xfer_us_accumulates_in_totals(self):
-        ctrl = self._make_ctrl()
-        ctrl._transfer_elapsed_us = MagicMock(return_value=1000)
-        ack = _make_ack()
-        ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
-        ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
-
-        self.assertEqual(
-            ctrl.hicache_l1_l2_transfer_totals["offload"]["xfer_us"], 2000
-        )
-
-    def test_xfer_us_none_leaves_total_zero(self):
-        ctrl = self._make_ctrl()
-        ctrl._transfer_elapsed_us = MagicMock(return_value=None)
-        ack = _make_ack()
-        ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
-
-        self.assertEqual(
-            ctrl.hicache_l1_l2_transfer_totals["offload"]["xfer_us"], 0
-        )
-
-    # ── invalid direction ─────────────────────────────────────────────────────
-
-    def test_invalid_direction_raises_value_error(self):
-        ctrl = self._make_ctrl()
-        ack = _make_ack()
-        with self.assertRaises(ValueError):
-            ctrl.record_l1_l2_transfer_complete(direction="sideways", ack=ack)
-
-    # ── collector call-through ────────────────────────────────────────────────
-
     def test_collector_called_with_correct_args_offload(self):
         ctrl = self._make_ctrl()
         ctrl._transfer_elapsed_us = MagicMock(return_value=5000)
         ack = _make_ack(block_count=3, byte_count=49152)
-
         ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
 
         ctrl.hicache_l1_l2_transfer_metrics_collector.record_transfer.assert_called_once_with(
@@ -584,7 +158,6 @@ def test_collector_called_with_correct_args_onboard(self):
         ctrl = self._make_ctrl()
         ctrl._transfer_elapsed_us = MagicMock(return_value=3000)
         ack = _make_ack(block_count=6, byte_count=98304)
-
         ctrl.record_l1_l2_transfer_complete(direction="onboard", ack=ack)
 
         ctrl.hicache_l1_l2_transfer_metrics_collector.record_transfer.assert_called_once_with(
@@ -596,44 +169,6 @@ def test_collector_called_with_correct_args_onboard(self):
             xfer_us=3000,
         )
 
-    def test_offload_src_dst_labels(self):
-        ctrl = self._make_ctrl()
-        ack = _make_ack()
-        ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
-        call_kwargs = (
-            ctrl.hicache_l1_l2_transfer_metrics_collector.record_transfer.call_args.kwargs
-        )
-        self.assertEqual(call_kwargs["src"], "sglang_hicache::L1")
-        self.assertEqual(call_kwargs["dst"], "sglang_hicache::L2")
-
-    def test_onboard_src_dst_labels(self):
-        ctrl = self._make_ctrl()
-        ack = _make_ack()
-        ctrl.record_l1_l2_transfer_complete(direction="onboard", ack=ack)
-        call_kwargs = (
-            ctrl.hicache_l1_l2_transfer_metrics_collector.record_transfer.call_args.kwargs
-        )
-        self.assertEqual(call_kwargs["src"], "sglang_hicache::L2")
-        self.assertEqual(call_kwargs["dst"], "sglang_hicache::L1")
-
-    def test_no_collector_does_not_call_record_transfer(self):
-        ctrl = _ControllerStub()
-        ctrl.hicache_l1_l2_transfer_metrics_collector = None
-        ctrl._transfer_elapsed_us = MagicMock(return_value=1000)
-        ack = _make_ack()
-
-        # Set DEBUG logging so we don't early-exit (should_log=True keeps it running).
-        cache_ctrl_logger = logging.getLogger("sglang.srt.managers.cache_controller")
-        old_level = cache_ctrl_logger.level
-        try:
-            cache_ctrl_logger.setLevel(logging.DEBUG)
-            ctrl.record_l1_l2_transfer_complete(direction="offload", ack=ack)
-        finally:
-            cache_ctrl_logger.setLevel(old_level)
-
-        # Totals should still be updated; no collector to assert on.
-        self.assertEqual(ctrl.hicache_l1_l2_transfer_totals["offload"]["events"], 1)
-
 
 # ══════════════════════════════════════════════════════════════════════════════
 # Prometheus text exposition — real prometheus_client, isolated registry
@@ -641,11 +176,8 @@ def test_no_collector_does_not_call_record_transfer(self):
 
 
 class TestHiCacheL1L2TransferCollectorPrometheusOutput(unittest.TestCase):
-    """Verify that record_transfer produces correct Prometheus text exposition.
-
-    Uses an isolated CollectorRegistry per test so metrics don't leak into
-    the global registry and tests remain independent of each other.
-    """
+    """Verifies metric names, label values, and sample values in real Prometheus
+    text output using an isolated CollectorRegistry per test."""
 
     def setUp(self):
         import prometheus_client
@@ -656,10 +188,8 @@ def setUp(self):
         _PC = prometheus_client.Counter
         _PH = prometheus_client.Histogram
 
-        # Wrapper classes that forward construction to the real prometheus_client
-        # types but bind them to our isolated registry.  Each setUp() call
-        # produces a new class object, so the _metric_cache key is unique and
-        # a fresh Counter/Histogram pair is created every test.
+        # Each setUp() defines new class objects, giving a unique _metric_cache
+        # key so every test gets fresh Counter/Histogram instances.
         class _BoundCounter:
             def __init__(self, *args, **kwargs):
                 kwargs["registry"] = _reg
@@ -694,91 +224,9 @@ def _exposition(self) -> str:
 
         return generate_latest(self.registry).decode("utf-8")
 
-    # ── metric names ──────────────────────────────────────────────────────────
-
-    def test_blocks_counter_name_in_exposition(self):
-        c = self._make_collector()
-        c.record_transfer(
-            direction="offload",
-            src="sglang_hicache::L1",
-            dst="sglang_hicache::L2",
-            blocks=5,
-            bytes_=81920,
-            xfer_us=None,
-        )
-        self.assertIn("sglang:hicache_l1_l2_transfer_blocks_total", self._exposition())
-
-    def test_bytes_counter_name_in_exposition(self):
-        c = self._make_collector()
-        c.record_transfer(
-            direction="offload",
-            src="sglang_hicache::L1",
-            dst="sglang_hicache::L2",
-            blocks=1,
-            bytes_=16384,
-            xfer_us=None,
-        )
-        self.assertIn("sglang:hicache_l1_l2_transfer_bytes_total", self._exposition())
-
-    def test_duration_histogram_name_in_exposition(self):
-        c = self._make_collector()
-        c.record_transfer(
-            direction="onboard",
-            src="sglang_hicache::L2",
-            dst="sglang_hicache::L1",
-            blocks=2,
-            bytes_=32768,
-            xfer_us=3000,
-        )
-        self.assertIn("sglang:hicache_l1_l2_transfer_duration_us", self._exposition())
-
-    # ── label values ──────────────────────────────────────────────────────────
-
-    def test_direction_label_in_exposition(self):
-        c = self._make_collector()
-        c.record_transfer(
-            direction="onboard",
-            src="sglang_hicache::L2",
-            dst="sglang_hicache::L1",
-            blocks=3,
-            bytes_=49152,
-            xfer_us=None,
-        )
-        self.assertIn('direction="onboard"', self._exposition())
-
-    def test_src_and_dst_labels_in_exposition(self):
-        c = self._make_collector()
-        c.record_transfer(
-            direction="offload",
-            src="sglang_hicache::L1",
-            dst="sglang_hicache::L2",
-            blocks=1,
-            bytes_=1024,
-            xfer_us=None,
-        )
-        output = self._exposition()
-        self.assertIn('src="sglang_hicache::L1"', output)
-        self.assertIn('dst="sglang_hicache::L2"', output)
-
-    def test_constructor_labels_in_exposition(self):
-        c = self._make_collector()
-        c.record_transfer(
-            direction="offload",
-            src="sglang_hicache::L1",
-            dst="sglang_hicache::L2",
-            blocks=1,
-            bytes_=1024,
-            xfer_us=None,
-        )
-        output = self._exposition()
-        self.assertIn('tp_rank="0"', output)
-        self.assertIn('io_backend="nixl"', output)
-
-    # ── counter values ────────────────────────────────────────────────────────
-
     def _find_metric_line(self, exposition: str, name_fragment: str, **label_filters) -> str:
-        """Return the first non-comment exposition line matching name_fragment
-        and all label_filters, or raise AssertionError."""
+        """Return the first non-comment line matching name_fragment and all
+        label_filters, or fail the test."""
         for line in exposition.splitlines():
             if line.startswith("#"):
                 continue
@@ -787,7 +235,7 @@ def _find_metric_line(self, exposition: str, name_fragment: str, **label_filters
             if all(f'{k}="{v}"' in line for k, v in label_filters.items()):
                 return line
         self.fail(
-            f"No metric line found for fragment={name_fragment!r} "
+            f"No metric line for fragment={name_fragment!r} "
             f"labels={label_filters} in:\n{exposition}"
         )
 
@@ -825,8 +273,6 @@ def test_bytes_counter_value(self):
         )
         self.assertTrue(line.endswith(" 32768.0"), f"Expected value 32768.0 in: {line!r}")
 
-    # ── histogram ─────────────────────────────────────────────────────────────
-
     def test_duration_histogram_sum_when_xfer_us_provided(self):
         c = self._make_collector()
         c.record_transfer(
@@ -861,31 +307,6 @@ def test_duration_histogram_count_when_xfer_us_provided(self):
         )
         self.assertTrue(line.endswith(" 1.0"), f"Expected count 1.0 in: {line!r}")
 
-    def test_duration_histogram_not_observed_when_xfer_us_none(self):
-        c = self._make_collector()
-        c.record_transfer(
-            direction="offload",
-            src="sglang_hicache::L1",
-            dst="sglang_hicache::L2",
-            blocks=1,
-            bytes_=1024,
-            xfer_us=None,
-        )
-        # No observation was recorded, so prometheus_client never instantiates a
-        # labeled child for the histogram — no _count series is emitted at all.
-        exposition = self._exposition()
-        for line in exposition.splitlines():
-            if line.startswith("#"):
-                continue
-            if (
-                "hicache_l1_l2_transfer_duration_us_count" in line
-                and 'direction="offload"' in line
-            ):
-                self.fail(
-                    "Expected no duration _count series when xfer_us is None, "
-                    f"but found: {line!r}\nin:\n{exposition}"
-                )
-
 
 if __name__ == "__main__":
     unittest.main()

From 169c7adbd6bfb72d20910759981084021dfd5255 Mon Sep 17 00:00:00 2001
From: Harry Xie <harryx@nvidia.com>
Date: Thu, 11 Jun 2026 00:48:55 +0300
Subject: [PATCH 28/28] fix: apply black formatting fixes

---
 python/sglang/srt/managers/cache_controller.py      |  7 +++----
 python/sglang/srt/mem_cache/hi_mamba_radix_cache.py |  4 +---
 python/sglang/srt/mem_cache/hiradix_cache.py        |  4 +---
 .../unit/mem_cache/test_hicache_l1l2_metrics.py     | 13 +++++++++----
 4 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py
index 5061d4f0929e..5dd24300aeac 100644
--- a/python/sglang/srt/managers/cache_controller.py
+++ b/python/sglang/srt/managers/cache_controller.py
@@ -1362,10 +1362,9 @@ def _estimate_l1_l2_transfer_bytes(
         total = int(token_count) * self._host_pool_size_per_token(PoolName.KV)
 
         for transfer in pool_transfers or []:
-            total += (
-                self._transfer_index_count(transfer)
-                * self._host_pool_size_per_token(transfer.name)
-            )
+            total += self._transfer_index_count(
+                transfer
+            ) * self._host_pool_size_per_token(transfer.name)
 
         if self.has_draft and self.mem_pool_host_draft is not None:
             draft_size_per_token = getattr(
diff --git a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
index a438d3e8fca7..3a4d12d3dce9 100644
--- a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
+++ b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py
@@ -438,9 +438,7 @@ def writing_check(self, write_back=False):
 
             matched = False
             for ack_id in ack.node_ids:
-                matched |= self._finish_write_through_ack(
-                    ack_id, release_lock=True
-                )
+                matched |= self._finish_write_through_ack(ack_id, release_lock=True)
             if matched:
                 self.cache_controller.record_l1_l2_transfer_complete(
                     direction="offload",
diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py
index ef724a401c4e..2d2650835b5f 100644
--- a/python/sglang/srt/mem_cache/hiradix_cache.py
+++ b/python/sglang/srt/mem_cache/hiradix_cache.py
@@ -956,9 +956,7 @@ def writing_check(self, write_back=False):
 
             matched = False
             for ack_id in ack.node_ids:
-                matched |= self._finish_write_through_ack(
-                    ack_id, release_lock=True
-                )
+                matched |= self._finish_write_through_ack(ack_id, release_lock=True)
             if matched:
                 self.cache_controller.record_l1_l2_transfer_complete(
                     direction="offload",
diff --git a/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py b/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py
index 1feb15401804..e06e7af9c13d 100644
--- a/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py
+++ b/test/registered/unit/mem_cache/test_hicache_l1l2_metrics.py
@@ -11,8 +11,9 @@
 from unittest.mock import MagicMock
 
 from sglang.srt.managers.cache_controller import HiCacheAck, HiCacheController
-from sglang.srt.observability.metrics_collector import HiCacheL1L2TransferMetricsCollector
-
+from sglang.srt.observability.metrics_collector import (
+    HiCacheL1L2TransferMetricsCollector,
+)
 
 # ── Test doubles ──────────────────────────────────────────────────────────────
 
@@ -224,7 +225,9 @@ def _exposition(self) -> str:
 
         return generate_latest(self.registry).decode("utf-8")
 
-    def _find_metric_line(self, exposition: str, name_fragment: str, **label_filters) -> str:
+    def _find_metric_line(
+        self, exposition: str, name_fragment: str, **label_filters
+    ) -> str:
         """Return the first non-comment line matching name_fragment and all
         label_filters, or fail the test."""
         for line in exposition.splitlines():
@@ -271,7 +274,9 @@ def test_bytes_counter_value(self):
             "hicache_l1_l2_transfer_bytes_total",
             direction="onboard",
         )
-        self.assertTrue(line.endswith(" 32768.0"), f"Expected value 32768.0 in: {line!r}")
+        self.assertTrue(
+            line.endswith(" 32768.0"), f"Expected value 32768.0 in: {line!r}"
+        )
 
     def test_duration_histogram_sum_when_xfer_us_provided(self):
         c = self._make_collector()