chore(bench): support block-size and vllm bench load overrides

benchmarks/README.md

@@ -74,10 +74,60 @@ python benchmarks/run_bench.py \
 `--backend all` runs the full comparison matrix sequentially: `baseline`,
 `lmcache`, `daser-chunk`, and `daser-prefix`. Use `--backend baseline`,
 `--backend lmcache`, `--backend daser-chunk`, or `--backend daser-prefix` for a
-single backend. `--backend daser` still starts DaseR with the mode selected by
-`--cache-reuse-mode`, and `--backend vllm` is accepted as a compatibility alias
-for `baseline`. `--cache-reuse-mode` is DaseR-only; baseline and LMCache use the
-same prompts but record `reuse_mode` as `none` in their manifests.
+single backend. Use comma-separated rows such as
+`--backend baseline,lmcache,daser-prefix` to run a subset. Baseline and LMCache
+use the same prompts but record `reuse_mode` as `none` in their manifests.
+
+Use synthetic vLLM benchmark traffic when you need to control prompt length
+directly instead of loading IMDB or LongBench records. This mode starts the same
+services but sends load with `vllm bench serve` against the OpenAI-compatible
+`/v1/completions` endpoint:
+
+```bash
+python benchmarks/run_bench.py \
+  --backend baseline,lmcache,daser-prefix \
+  --load-generator vllm-bench \
+  --model /data/<user>/model/models/Qwen/Qwen3-8B \
+  --store-dir /data/<user>/daser_bench/vllmbench_len8192 \
+  --gpu-id 2 \
+  --gpu-util 0.85 \
+  --max-num-seqs 32 \
+  --block-size 128 \
+  --bench-num-prompts 1000 \
+  --bench-input-len 8192 \
+  --bench-output-len 1 \
+  --bench-request-rate inf \
+  --bench-max-concurrency 16 \
+  --bench-seed 42
+```
+
+`--backend baseline,lmcache,daser-prefix` runs the OpenAI-compatible rows. It
+omits `daser-chunk` because DaseR chunk mode uses DaseR-specific `/documents`
+and `/infer` endpoints instead of the OpenAI completions endpoint. If
+`--load-generator vllm-bench` is combined with a selection that includes
+`daser-chunk`, such as `--backend all` or `--backend daser-chunk`, the runner
+fails fast and asks you to select only `baseline,lmcache,daser-prefix` or use
+`--load-generator internal`.
+
+The vLLM-bench-specific knobs are:
+
+| Option | Meaning |
+|--------|---------|
+| `--bench-num-prompts` | Number of random prompts to send |
+| `--bench-input-len` | Random dataset input length passed to `vllm bench serve` |
+| `--bench-output-len` | Random dataset output length; defaults to `--gen-max-tokens` |
+| `--bench-request-rate` | Requests per second; `inf` sends all requests immediately |
+| `--bench-max-concurrency` | Maximum concurrent requests; defaults to `--max-inflight` |
+| `--bench-random-prefix-len` | Fixed prefix tokens before random context tokens |
+| `--bench-random-range-ratio` | Symmetric random length range ratio |
+| `--bench-seed` | Seed reused for cold and warm phases |
+| `--bench-burstiness` | Request arrival burstiness passed to `vllm bench serve` |
+
+For LMCache and DaseR prefix, cold and warm phases run `vllm bench serve` twice
+with the same seed and length parameters so the second pass can reuse the first
+pass. Raw vLLM bench JSON is saved beside each backend's `results.json` as
+`vllm_bench_baseline.json`, `vllm_bench_cold.json`, and
+`vllm_bench_warm.json`.
 
 Generation defaults are deterministic across backends: `--gen-temperature`
 defaults to `0.0`, `--gen-top-p` defaults to `1.0`, and `--gen-seed` defaults
@@ -255,16 +305,20 @@ cache hit rates from multiple sources:
 - `vllm_external_prefix_cache_hit_rate`: vLLM Prometheus external prefix cache
   hit ratio from `vllm:external_prefix_cache_*` counter deltas.
 - `backend_server_cache_hit_rate`: summary hit ratio used for backend
-  comparison. DaseR reports its internal
-  `daser_external_prefix_cache_*` counters, which the connector records with
-  the same queried-token / accepted-token semantics as vLLM's
-  `vllm:external_prefix_cache_*` counters. LMCache reports MP server token hit
-  counters when available and falls back to request counters only when token
-  counters are absent. DaseR control-plane lookup counters are still kept in raw
-  metrics as `daser_prometheus_tokens` and `daser_prometheus_requests`.
+  comparison. DaseR reports token-level
+  `daser_cache_matched_tokens_total / daser_cache_requested_tokens_total`
+  counter deltas. LMCache reports MP server token hit counters when available
+  and falls back to status prefetch token counters. Request-level counters and
+  external-prefix counters are kept in raw metrics for diagnostics but are not
+  used as the summary backend hit rate.
 - `metrics`: raw vLLM Prometheus, backend Prometheus, backend status counter
   deltas, and all named hit-ratio candidates.
 
+`--load-generator vllm-bench` writes raw vLLM bench JSON files and wraps each
+phase with the same backend metric snapshots as the internal load generator, so
+its cold/warm summaries include token-level `backend_server_cache_hit_rate`
+when the backend exposes the required counters.
+
 For datasets with answer labels, each summary also includes
 `answer_contains_accuracy`; datasets without labels report `null`.
 

benchmarks/bench_load.py

@@ -17,7 +17,7 @@
 from benchmarks.utils.constants import (
     BLOCK_TOKENS,
     COMPARISON_IOURING_MEM,
-    SLOT_SIZE,
+    slot_size_for_block_tokens,
 )
 from benchmarks.utils.datasets import (
     BenchmarkSample,
@@ -29,6 +29,7 @@
 from benchmarks.utils.loadgen import (
     PhaseResult,
     RequestResult,
+    backend_server_hit_rate,
     run_daser_chunk,
     run_daser_prefix,
     run_lmcache,
@@ -69,6 +70,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser.add_argument("--longbench-dir")
     parser.add_argument("--datasets", default=None)
     parser.add_argument("--max-samples", type=int, default=20)
+    parser.add_argument("--block-size", type=int, default=BLOCK_TOKENS)
     parser.add_argument("--max-context-tokens", type=int, default=0)
     parser.add_argument("--no-dedup-context", action="store_true")
     parser.add_argument("--max-inflight", type=int, default=32)
@@ -91,6 +93,11 @@ async def main_async(args: argparse.Namespace) -> None:
         if args.prepare_only
         else BenchmarkManifest.read(_required(args.manifest, "--manifest"))
     )
+    if manifest is not None and args.block_size != manifest.block_size:
+        raise ValueError(
+            f"--block-size={args.block_size} does not match manifest "
+            f"block_size={manifest.block_size}"
+        )
     model = args.model if args.prepare_only else manifest.model
     store_dir = args.store_dir if args.prepare_only else manifest.store_dir
     if model is None:
@@ -105,7 +112,10 @@ async def main_async(args: argparse.Namespace) -> None:
     model_config = AutoConfig.from_pretrained(model, trust_remote_code=True)
     chunk_aligned_prompts = True
     prompts = build_prompt_payloads(
-        tokenizer, samples, chunk_aligned=chunk_aligned_prompts
+        tokenizer,
+        samples,
+        chunk_aligned=chunk_aligned_prompts,
+        block_tokens=args.block_size,
     )
     token_counts = count_prompt_payload_tokens(tokenizer, prompts)
     effective_max_context_tokens = _effective_max_context_tokens(
@@ -120,22 +130,29 @@ async def main_async(args: argparse.Namespace) -> None:
     if _should_dedup_context(args):
         samples = dedup_by_context(samples)
         prompts = build_prompt_payloads(
-            tokenizer, samples, chunk_aligned=chunk_aligned_prompts
+            tokenizer,
+            samples,
+            chunk_aligned=chunk_aligned_prompts,
+            block_tokens=args.block_size,
         )
         token_counts = count_prompt_payload_tokens(tokenizer, prompts)
     samples = interleave_samples(samples)
     prompts = build_prompt_payloads(
-        tokenizer, samples, chunk_aligned=chunk_aligned_prompts
+        tokenizer,
+        samples,
+        chunk_aligned=chunk_aligned_prompts,
+        block_tokens=args.block_size,
     )
     token_counts = count_prompt_payload_tokens(tokenizer, prompts)
-    total_blocks, max_prompt_blocks = workload_blocks(token_counts, BLOCK_TOKENS)
+    total_blocks, max_prompt_blocks = workload_blocks(token_counts, args.block_size)
+    slot_size = slot_size_for_block_tokens(args.block_size)
     sizing = None
     if args.prepare_only:
         capacity_limits = _capacity_limits(args, store_dir)
         sizing = derive_benchmark_sizing(
             total_blocks=total_blocks,
             max_prompt_blocks=max_prompt_blocks,
-            slot_size=SLOT_SIZE,
+            slot_size=slot_size,
             mode=COMPARISON_IOURING_MEM,
             evict=args.evict,
             capacity_limits=capacity_limits,
@@ -161,6 +178,7 @@ async def main_async(args: argparse.Namespace) -> None:
         total_prompt_tokens=sum(token_counts),
         total_blocks=total_blocks,
         max_prompt_blocks=max_prompt_blocks,
+        block_size=args.block_size,
         evict=args.evict,
         sizing=sizing,
     )
@@ -391,6 +409,7 @@ def _common_config_for_run(
     total_blocks: int,
     max_prompt_blocks: int,
     evict: bool,
+    block_size: int,
     sizing: BenchmarkSizing | None,
 ) -> dict[str, Any]:
     """Build benchmark config without re-inferring capacities during load.
@@ -406,6 +425,7 @@ def _common_config_for_run(
         total_prompt_tokens: Total selected prompt tokens.
         total_blocks: Total selected KV blocks.
         max_prompt_blocks: Largest selected prompt in KV blocks.
+        block_size: vLLM KV block size in tokens.
         evict: Whether eviction sizing was requested.
         sizing: Prepare-time sizing, required for prepare-only invocations.
 
@@ -429,6 +449,7 @@ def _common_config_for_run(
             "total_prompt_tokens": total_prompt_tokens,
             "total_blocks": total_blocks,
             "max_prompt_blocks": max_prompt_blocks,
+            "block_size": block_size,
         }
 
     if prepare_only:
@@ -442,6 +463,7 @@ def _common_config_for_run(
             "total_prompt_tokens": total_prompt_tokens,
             "total_blocks": total_blocks,
             "max_prompt_blocks": max_prompt_blocks,
+            "block_size": block_size,
             "derived_l1_size_bytes": sizing.daser_l1_bytes,
             "derived_l1_size": format_capacity(sizing.daser_l1_bytes),
             "derived_l2_size_bytes": sizing.daser_l2_bytes,
@@ -463,6 +485,7 @@ def _common_config_for_run(
         "total_prompt_tokens": total_prompt_tokens,
         "total_blocks": total_blocks,
         "max_prompt_blocks": max_prompt_blocks,
+        "block_size": block_size,
         "derived_l1_size_bytes": manifest.l1_size_bytes,
         "derived_l1_size": format_capacity(manifest.l1_size_bytes),
         "derived_l2_size_bytes": manifest.l2_size_bytes,
@@ -549,22 +572,7 @@ def _serialise_phase(
 
 
 def _backend_server_hit_rate(hit_ratios: dict[str, Any]) -> float | None:
-    if (
-        hit_ratios.get("daser_external_prefix") is not None
-        or hit_ratios.get("daser_prometheus_tokens") is not None
-        or hit_ratios.get("daser_prometheus_requests") is not None
-    ):
-        ratio = hit_ratios.get("daser_external_prefix")
-        return float(ratio) if ratio is not None else None
-    for key in (
-        "lmcache_prometheus_lookup",
-        "lmcache_prometheus_retrieve",
-        "lmcache_status_prefetch",
-    ):
-        ratio = hit_ratios.get(key)
-        if ratio is not None:
-            return float(ratio)
-    return None
+    return backend_server_hit_rate(hit_ratios)
 
 
 def main(argv: list[str] | None = None) -> None:

benchmarks/bench_start_servers.py

@@ -37,6 +37,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser.add_argument("--max-num-seqs", type=int, default=32)
     parser.add_argument("--max-num-batched-tokens", type=int, default=0)
     parser.add_argument("--max-model-len", type=int, default=0)
+    parser.add_argument("--block-size", type=int, default=16)
     parser.add_argument("--l1-size", type=parse_size_bytes, default="256gib")
     parser.add_argument("--l2-size", type=parse_size_bytes, default="300gib")
     parser.add_argument(
@@ -73,6 +74,7 @@ async def main_async(args: argparse.Namespace) -> None:
         max_num_batched_tokens=(
             args.max_num_batched_tokens if args.max_num_batched_tokens > 0 else None
         ),
+        block_size=args.block_size,
         reuse_mode=args.cache_reuse_mode,
         transfer_mode=args.transfer_mode,
         vllm_port=args.vllm_port,