diff --git a/benchmarks/tracelab/README.md b/benchmarks/tracelab/README.md new file mode 100644 index 0000000..273634d --- /dev/null +++ b/benchmarks/tracelab/README.md @@ -0,0 +1,51 @@ +# TraceLab coding-agent benchmark + +This adapter converts TraceLab coding-agent sessions into the CSV schema used +by TraceLab's closed-loop runner. It preserves source round order, input and +output lengths, reusable-prefix lengths, and wall-clock tool gaps. DaseR does +not vendor the TraceLab dataset, DuckDB environment, or runner. + +## Prepare a replay + +Run the exporter with an environment that provides DuckDB: + +```bash +PYTHONPATH="$PWD" /path/to/tracelab/python \ + -m benchmarks.tracelab.prepare_trace \ + --db /path/to/coding_trace.duckdb \ + --model /path/to/model \ + --tensor-parallel-size 2 \ + --max-sessions 12 \ + --pause-window-rounds 8 \ + --out /path/to/scratch/trace.csv +``` + +Use `--dense-window-rounds` with `--max-dense-gap-seconds` to select contiguous +long-prefix windows. Dense selection rejects overlapping source rounds instead +of turning them into artificial zero-time dependencies. + +The exporter prints selected session/round counts and model-derived peak KV +capacity. Model, database, and output paths are local inputs and must not be +committed. + +## Build a model-native token pool + +Models with custom tokenizers must use token IDs produced by that model's own +tokenizer: + +```bash +/path/to/python -m benchmarks.tracelab.prepare_token_pool \ + --model /path/to/model \ + --text-file /path/to/corpus.txt \ + --output /path/to/scratch/model_tokens.u32 \ + --limit 1000000 \ + --trust-remote-code +``` + +The output is a headerless little-endian sequence of unsigned 32-bit token +IDs. Configure the closed-loop runner to submit those IDs directly. For later +rounds, append the exact generated token IDs returned by vLLM; decoding and +re-tokenizing output can change the prefix. + +Large inputs and outputs belong in project-approved scratch storage, never +`/tmp`. diff --git a/benchmarks/tracelab/__init__.py b/benchmarks/tracelab/__init__.py new file mode 100644 index 0000000..e15e71a --- /dev/null +++ b/benchmarks/tracelab/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +"""TraceLab coding-agent benchmark preparation utilities.""" diff --git a/benchmarks/tracelab/prepare_token_pool.py b/benchmarks/tracelab/prepare_token_pool.py new file mode 100644 index 0000000..5a42d4a --- /dev/null +++ b/benchmarks/tracelab/prepare_token_pool.py @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Build a GLM-native binary token pool for the TraceLab replay runner.""" + +from __future__ import annotations + +import argparse +from array import array +from pathlib import Path +import sys +from typing import Any + + +def build_token_pool(corpus: Path, tokenizer: Any, limit: int) -> list[int]: + """Tokenize a text corpus into ordinary model-native token IDs. + + Args: + corpus: UTF-8 text corpus with one sample per line. + tokenizer: Hugging Face-compatible tokenizer with ``encode`` and + ``vocab_size`` attributes. + limit: Maximum number of token IDs to return. + + Returns: + Up to ``limit`` token IDs, excluding tokenizer-added special tokens. + + Thread-safety: + Performs synchronous file IO and tokenizer calls. Do not call it from + an asyncio hot path or share a mutable tokenizer concurrently. + """ + + if limit <= 0: + raise ValueError("token limit must be positive") + vocab_size = int(tokenizer.vocab_size) + if vocab_size <= 0: + raise ValueError("tokenizer vocab_size must be positive") + special_token_ids = { + int(token_id) for token_id in getattr(tokenizer, "all_special_ids", ()) + } + + pool: list[int] = [] + with corpus.open(encoding="utf-8") as file: + for line in file: + if not line.strip(): + continue + token_ids = tokenizer.encode(line, add_special_tokens=False) + for token_id in token_ids: + value = int(token_id) + if value in special_token_ids: + continue + if value < 0 or value >= vocab_size: + raise ValueError( + f"ordinary token ID {value} is outside [0, {vocab_size})" + ) + pool.append(value) + if len(pool) == limit: + return pool + + if not pool: + raise ValueError("text corpus produced an empty token pool") + return pool + + +def write_token_pool(token_ids: list[int], output: Path) -> None: + """Write token IDs as headerless little-endian unsigned 32-bit integers. + + Args: + token_ids: Non-negative IDs that fit in ``u32``. + output: Destination consumed by TraceLab ``--token-pool-file``. + + Returns: + ``None``. + + Thread-safety: + Performs synchronous file IO and replaces ``output``. The destination + must not be read or written concurrently. + """ + + if not token_ids: + raise ValueError("token pool must not be empty") + values = array("I", token_ids) + if values.itemsize != 4: + raise RuntimeError("this platform does not provide 32-bit unsigned arrays") + if sys.byteorder != "little": + values.byteswap() + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("wb") as file: + values.tofile(file) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Build a model-native u32 token pool for TraceLab" + ) + parser.add_argument("--model", required=True) + parser.add_argument("--text-file", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--limit", type=int, default=1_000_000) + parser.add_argument("--trust-remote-code", action="store_true") + return parser.parse_args() + + +def main() -> None: + """Generate the token pool described by command-line arguments. + + Args: + None. + + Returns: + ``None``. + + Thread-safety: + Runs synchronously as a standalone preparation command. + """ + + from transformers import AutoTokenizer + + args = _parse_args() + tokenizer = AutoTokenizer.from_pretrained( + args.model, + trust_remote_code=args.trust_remote_code, + ) + token_ids = build_token_pool(args.text_file, tokenizer, args.limit) + write_token_pool(token_ids, args.output) + print( + f"wrote {len(token_ids)} model-native tokens to {args.output} " + f"(min={min(token_ids)}, max={max(token_ids)}, " + f"vocab_size={tokenizer.vocab_size})" + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/tracelab/prepare_trace.py b/benchmarks/tracelab/prepare_trace.py new file mode 100644 index 0000000..48c8c62 --- /dev/null +++ b/benchmarks/tracelab/prepare_trace.py @@ -0,0 +1,503 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Export real TraceLab session gaps for the official closed-loop runner.""" + +from __future__ import annotations + +import argparse +import csv +from dataclasses import dataclass +import json +import math +from pathlib import Path +import random +from typing import Any, Iterable + +from benchmarks.utils.constants import slot_size_for_block_tokens + +_FIELDS = ( + "id", + "input_len", + "output_len", + "arrival_time", + "round_idx", + "tool_wait_after_ms", + "prefix_len", +) +_BLOCK_TOKENS = 16 + + +@dataclass(frozen=True) +class Round: + """One replayable TraceLab round. + + Args: + input_len: Newly appended prompt tokens. + output_len: Trace-observed decode tokens. + prefix_len: Reusable historical prompt tokens. + first_event_ms: Wall-clock timestamp of the first round event. + last_event_ms: Wall-clock timestamp of the last round event. + + Thread-safety: + Immutable and safe to share between preparation threads. + """ + + input_len: int + output_len: int + prefix_len: int + first_event_ms: int + last_event_ms: int + + +@dataclass(frozen=True) +class PreparedSession: + """A validated closed-loop session and its source gap diagnostics. + + Args: + source_id: Stable composite TraceLab session identity. + rounds: Source-ordered replay rounds. + negative_gaps: Adjacent event boundaries that overlap in the trace. + + Thread-safety: + Immutable and safe to share between preparation threads. + """ + + source_id: str + rounds: tuple[Round, ...] + negative_gaps: int + + +def prepare_sessions(rows: Iterable[tuple[Any, ...]]) -> list[PreparedSession]: + """Group ordered DuckDB rows into validated TraceLab sessions. + + Args: + rows: Query rows containing composite identity, token counts, and event + bounds in the order returned by :func:`load_rows`. + + Returns: + Sessions with at least two rounds and valid event timestamps. + + Thread-safety: + Pure with respect to caller-owned rows. + """ + + grouped: dict[str, list[Round]] = {} + for source_id, input_len, output_len, prefix_len, first_ms, last_ms in rows: + if first_ms is None or last_ms is None: + continue + grouped.setdefault(str(source_id), []).append( + Round( + input_len=max(1, int(input_len or 0)), + output_len=max(1, int(output_len or 0)), + prefix_len=max(0, int(prefix_len or 0)), + first_event_ms=int(first_ms), + last_event_ms=int(last_ms), + ) + ) + + sessions: list[PreparedSession] = [] + for source_id, rounds in grouped.items(): + if len(rounds) < 2: + continue + negative_gaps = sum( + right.first_event_ms < left.last_event_ms + for left, right in zip(rounds, rounds[1:], strict=False) + ) + sessions.append(PreparedSession(source_id, tuple(rounds), negative_gaps)) + return sessions + + +def select_sessions( + sessions: list[PreparedSession], + *, + max_sessions: int, + max_model_len: int, + seed: int, +) -> list[PreparedSession]: + """Select whole sessions without truncating or rescaling trace rounds. + + Args: + sessions: Candidate sessions. + max_sessions: Maximum number of sessions to return. + max_model_len: Model context limit for prompt plus output tokens. + seed: Deterministic shuffle seed. + + Returns: + Up to ``max_sessions`` sessions whose every round fits the model. + + Thread-safety: + Does not mutate ``sessions`` or global random state. + """ + + if max_sessions <= 0 or max_model_len <= 0: + raise ValueError("session and model limits must be positive") + eligible = [ + session + for session in sessions + if all( + round_.prefix_len + round_.input_len + round_.output_len <= max_model_len + for round_ in session.rounds + ) + ] + random.Random(seed).shuffle(eligible) + return eligible[:max_sessions] + + +def select_pause_windows( + sessions: list[PreparedSession], + *, + max_sessions: int, + max_model_len: int, + window_rounds: int, + min_pause_s: float, + max_pause_s: float, + max_prefix_tokens: int, +) -> list[PreparedSession]: + """Select long-prefix windows around real, bounded TraceLab pauses. + + Args: + sessions: Candidate complete sessions. + max_sessions: Maximum number of windows to return. + max_model_len: Model context limit for prompt plus output tokens. + window_rounds: Maximum contiguous rounds retained per session. + min_pause_s: Minimum source pause represented by each window. + max_pause_s: Maximum source pause represented by each window. + max_prefix_tokens: Largest pause-frontier prefix to admit. + + Returns: + Deterministic windows ordered by decreasing pause-frontier prefix. + + Thread-safety: + Does not mutate caller-owned sessions. + """ + + if ( + max_sessions <= 0 + or max_model_len <= 0 + or window_rounds < 2 + or min_pause_s < 0 + or max_pause_s < min_pause_s + or max_prefix_tokens <= 0 + ): + raise ValueError("invalid pause-window limits") + candidates: list[tuple[int, PreparedSession]] = [] + for session in sessions: + best: tuple[int, int] | None = None + for index, (left, right) in enumerate( + zip(session.rounds, session.rounds[1:], strict=False) + ): + gap_s = max(0, right.first_event_ms - left.last_event_ms) / 1000.0 + context_tokens = right.prefix_len + right.input_len + right.output_len + if ( + min_pause_s <= gap_s <= max_pause_s + and left.prefix_len <= max_prefix_tokens + and context_tokens <= max_model_len + and (best is None or left.prefix_len > best[0]) + ): + best = (left.prefix_len, index) + if best is None: + continue + prefix_tokens, pause_index = best + before = window_rounds // 2 + start = max(0, pause_index + 1 - before) + end = min(len(session.rounds), start + window_rounds) + start = max(0, end - window_rounds) + window = session.rounds[start:end] + window_gaps_s = [ + max(0, right.first_event_ms - left.last_event_ms) / 1000.0 + for left, right in zip(window, window[1:], strict=False) + ] + if max(window_gaps_s, default=0.0) <= max_pause_s and all( + round_.prefix_len + round_.input_len + round_.output_len <= max_model_len + for round_ in window + ): + negative_gaps = sum( + right.first_event_ms < left.last_event_ms + for left, right in zip(window, window[1:], strict=False) + ) + candidates.append( + ( + prefix_tokens, + PreparedSession(session.source_id, window, negative_gaps), + ) + ) + candidates.sort(key=lambda item: (-item[0], item[1].source_id)) + return [session for _, session in candidates[:max_sessions]] + + +def select_dense_windows( + sessions: list[PreparedSession], + *, + max_sessions: int, + max_model_len: int, + window_rounds: int, + max_gap_s: float, + min_prefix_tokens: int, + max_prefix_tokens: int, +) -> list[PreparedSession]: + """Select long-prefix windows with only short, non-overlapping gaps. + + Args: + sessions: Candidate complete sessions. + max_sessions: Maximum number of windows to return. + max_model_len: Model context limit for prompt plus output tokens. + window_rounds: Exact number of contiguous rounds per window. + max_gap_s: Largest source gap allowed inside a window. + min_prefix_tokens: Minimum peak prefix required in a window. + max_prefix_tokens: Largest prefix admitted in a window. + + Returns: + One deterministic best window per selected session, ordered by total + reusable prefix tokens. + + Thread-safety: + Does not mutate caller-owned sessions. + """ + + if ( + max_sessions <= 0 + or max_model_len <= 0 + or window_rounds < 2 + or max_gap_s < 0 + or min_prefix_tokens < 0 + or max_prefix_tokens < min_prefix_tokens + ): + raise ValueError("invalid dense-window limits") + max_gap_ms = max_gap_s * 1000.0 + candidates: list[tuple[int, PreparedSession]] = [] + for session in sessions: + best: tuple[int, tuple[Round, ...]] | None = None + for start in range(len(session.rounds) - window_rounds + 1): + window = session.rounds[start : start + window_rounds] + prefixes = [round_.prefix_len for round_ in window] + gaps_ms = [ + right.first_event_ms - left.last_event_ms + for left, right in zip(window, window[1:], strict=False) + ] + if ( + max(prefixes) < min_prefix_tokens + or max(prefixes) > max_prefix_tokens + or any(gap_ms < 0 or gap_ms > max_gap_ms for gap_ms in gaps_ms) + or any( + round_.prefix_len + round_.input_len + round_.output_len + > max_model_len + for round_ in window + ) + ): + continue + prefix_total = sum(prefixes) + if best is None or prefix_total > best[0]: + best = (prefix_total, window) + if best is not None: + prefix_total, window = best + candidates.append( + (prefix_total, PreparedSession(session.source_id, window, 0)) + ) + candidates.sort(key=lambda item: (-item[0], item[1].source_id)) + return [session for _, session in candidates[:max_sessions]] + + +def write_trace( + sessions: list[PreparedSession], + output: Path, + *, + arrival_rate: float, + seed: int, +) -> None: + """Write official TraceLab closed-loop CSV using real inter-round gaps. + + Args: + sessions: Selected sessions. + output: Destination CSV path. + arrival_rate: Poisson session arrival rate in sessions per second. + seed: Deterministic arrival seed. + + Returns: + ``None``. + + Thread-safety: + Performs synchronous planning-time file IO. The destination must not be + written concurrently. + """ + + if not sessions: + raise ValueError("at least one session is required") + if not math.isfinite(arrival_rate) or arrival_rate <= 0: + raise ValueError("arrival_rate must be finite and positive") + rng = random.Random(seed) + arrival_ms = 0.0 + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("w", newline="", encoding="utf-8") as file: + writer = csv.DictWriter(file, fieldnames=_FIELDS) + writer.writeheader() + for session_id, session in enumerate(sessions): + if session_id: + arrival_ms += rng.expovariate(arrival_rate) * 1000.0 + for round_idx, round_ in enumerate(session.rounds): + gap_ms = 0 + if round_idx + 1 < len(session.rounds): + gap_ms = max( + 0, + session.rounds[round_idx + 1].first_event_ms + - round_.last_event_ms, + ) + writer.writerow( + { + "id": session_id, + "input_len": round_.input_len, + "output_len": round_.output_len, + "arrival_time": f"{arrival_ms:.6f}", + "round_idx": round_idx, + "tool_wait_after_ms": gap_ms, + "prefix_len": round_.prefix_len, + } + ) + + +def load_rows(database: Path, provider: str) -> list[tuple[Any, ...]]: + """Load source-ordered token and real event-bound rows from TraceLab. + + Args: + database: TraceLab DuckDB path. + provider: ``all``, ``claude``, or ``codex``. + + Returns: + Query rows accepted by :func:`prepare_sessions`. + + Thread-safety: + Opens a read-only database connection for this call. + """ + + try: + import duckdb + except ImportError as exc: + raise RuntimeError( + "duckdb is required only for trace preparation; run this script " + "from TraceLab's uv environment" + ) from exc + where = "" if provider == "all" else "WHERE r.provider = ?" + params = [] if provider == "all" else [provider] + query = f""" + SELECT + concat(r.project, '|', r.session_id, '|', r.session_file) AS source_id, + r.newly_append_tokens, + r.output_tokens, + r.prefix_tokens, + min(epoch_ms(t.timestamp)) AS first_event_ms, + max(epoch_ms(t.timestamp)) AS last_event_ms + FROM rounds r + LEFT JOIN timing_events t USING (round_pk) + {where} + GROUP BY r.project, r.session_id, r.session_file, r.round_index, + r.round_pk, r.ingest_seq, r.newly_append_tokens, + r.output_tokens, r.prefix_tokens + ORDER BY min(r.ingest_seq), r.round_index, r.round_pk + """ + connection = duckdb.connect(str(database), read_only=True) + try: + return connection.execute(query, params).fetchall() + finally: + connection.close() + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse TraceLab preparation arguments. + + Args: + argv: Optional argument vector without the executable name. + + Returns: + Validated command-line arguments. + + Thread-safety: + Pure except for ``argparse`` error handling. + """ + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--db", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--tensor-parallel-size", type=int, default=2) + parser.add_argument("--max-model-len", type=int, default=1_048_576) + parser.add_argument("--max-sessions", type=int, default=12) + parser.add_argument("--arrival-rate", type=float, default=0.05) + parser.add_argument("--seed", type=int, default=11) + parser.add_argument("--pause-window-rounds", type=int, default=0) + parser.add_argument("--dense-window-rounds", type=int, default=0) + parser.add_argument("--min-pause-seconds", type=float, default=30.0) + parser.add_argument("--max-pause-seconds", type=float, default=300.0) + parser.add_argument("--max-dense-gap-seconds", type=float, default=1.0) + parser.add_argument("--min-dense-prefix-tokens", type=int, default=65_536) + parser.add_argument("--max-prefix-tokens", type=int, default=260_000) + parser.add_argument("--provider", choices=("all", "claude", "codex"), default="all") + args = parser.parse_args(argv) + if args.tensor_parallel_size <= 0: + parser.error("--tensor-parallel-size must be positive") + if args.pause_window_rounds and args.dense_window_rounds: + parser.error("pause and dense window modes are mutually exclusive") + return args + + +def main(argv: list[str] | None = None) -> None: + """Prepare one replay CSV and print its capacity summary. + + Args: + argv: Optional argument vector without the executable name. + + Returns: + ``None``. + + Thread-safety: + Performs synchronous DuckDB and output-file IO as a standalone command. + """ + + args = parse_args(argv) + sessions = prepare_sessions(load_rows(args.db, args.provider)) + if args.pause_window_rounds: + selected = select_pause_windows( + sessions, + max_sessions=args.max_sessions, + max_model_len=args.max_model_len, + window_rounds=args.pause_window_rounds, + min_pause_s=args.min_pause_seconds, + max_pause_s=args.max_pause_seconds, + max_prefix_tokens=args.max_prefix_tokens, + ) + elif args.dense_window_rounds: + selected = select_dense_windows( + sessions, + max_sessions=args.max_sessions, + max_model_len=args.max_model_len, + window_rounds=args.dense_window_rounds, + max_gap_s=args.max_dense_gap_seconds, + min_prefix_tokens=args.min_dense_prefix_tokens, + max_prefix_tokens=args.max_prefix_tokens, + ) + else: + selected = select_sessions( + sessions, + max_sessions=args.max_sessions, + max_model_len=args.max_model_len, + seed=args.seed, + ) + write_trace(selected, args.out, arrival_rate=args.arrival_rate, seed=args.seed + 1) + bytes_per_token = ( + slot_size_for_block_tokens(args.model, _BLOCK_TOKENS, args.tensor_parallel_size) + // _BLOCK_TOKENS + ) + peak_tokens = sum( + max(round_.prefix_len for round_ in session.rounds) for session in selected + ) + summary = { + "sessions": len(selected), + "rounds": sum(len(session.rounds) for session in selected), + "negative_gaps_clamped": sum(session.negative_gaps for session in selected), + "sum_session_peak_tokens": peak_tokens, + "sum_session_peak_kv_gib": peak_tokens * bytes_per_token / 1024**3, + "bytes_per_token": bytes_per_token, + "output": str(args.out), + } + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/test_tracelab_prepare.py b/tests/unit/test_tracelab_prepare.py new file mode 100644 index 0000000..fcdc88f --- /dev/null +++ b/tests/unit/test_tracelab_prepare.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tests for TraceLab wall-clock-gap preparation.""" + +import csv + +from benchmarks.tracelab.prepare_trace import ( + prepare_sessions, + select_dense_windows, + select_pause_windows, + select_sessions, + write_trace, +) + + +def test_prepare_trace_preserves_lengths_and_clamps_overlapping_gap(tmp_path) -> None: + rows = [ + ("a", 100, 10, 0, 1_000, 2_000), + ("a", 20, 5, 110, 1_900, 2_500), + ("b", 200, 10, 0, 3_000, 3_500), + ("b", 30, 5, 210, 5_000, 5_500), + ] + sessions = prepare_sessions(rows) + selected = select_sessions(sessions, max_sessions=2, max_model_len=1_000, seed=1) + output = tmp_path / "trace.csv" + write_trace(selected, output, arrival_rate=1.0, seed=2) + + with output.open(newline="", encoding="utf-8") as file: + emitted = list(csv.DictReader(file)) + + by_input = {int(row["input_len"]): row for row in emitted} + assert by_input[100]["tool_wait_after_ms"] == "0" + assert by_input[200]["tool_wait_after_ms"] == "1500" + assert by_input[20]["prefix_len"] == "110" + assert sum(session.negative_gaps for session in selected) == 1 + + windows = select_pause_windows( + sessions, + max_sessions=1, + max_model_len=1_000, + window_rounds=2, + min_pause_s=1.0, + max_pause_s=2.0, + max_prefix_tokens=1_000, + ) + assert [round_.input_len for round_ in windows[0].rounds] == [200, 30] + + +def test_select_dense_windows_rejects_overlapping_and_long_gaps() -> None: + sessions = prepare_sessions( + [ + ("dense", 10, 1, 70_000, 1_000, 1_100), + ("dense", 10, 1, 71_000, 1_200, 1_300), + ("dense", 10, 1, 72_000, 1_400, 1_500), + ("dense", 10, 1, 90_000, 5_000, 5_100), + ("overlap", 10, 1, 100_000, 1_000, 1_200), + ("overlap", 10, 1, 110_000, 1_100, 1_300), + ("overlap", 10, 1, 120_000, 1_400, 1_500), + ] + ) + + windows = select_dense_windows( + sessions, + max_sessions=2, + max_model_len=300_000, + window_rounds=3, + max_gap_s=0.5, + min_prefix_tokens=65_536, + max_prefix_tokens=200_000, + ) + + assert len(windows) == 1 + assert windows[0].source_id == "dense" + assert [round_.prefix_len for round_ in windows[0].rounds] == [ + 70_000, + 71_000, + 72_000, + ] + assert windows[0].negative_gaps == 0 diff --git a/tests/unit/test_tracelab_token_pool.py b/tests/unit/test_tracelab_token_pool.py new file mode 100644 index 0000000..0dfa582 --- /dev/null +++ b/tests/unit/test_tracelab_token_pool.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tests for TraceLab model-native token-pool preparation.""" + +from array import array + +import pytest + +from benchmarks.tracelab.prepare_token_pool import ( + build_token_pool, + write_token_pool, +) + + +class _Tokenizer: + vocab_size = 16 + all_special_ids = [16] + + def encode(self, text: str, *, add_special_tokens: bool) -> list[int]: + assert add_special_tokens is False + return [int(value) for value in text.split()] + + +def test_build_and_write_token_pool(tmp_path) -> None: + corpus = tmp_path / "corpus.txt" + corpus.write_text("1 2 3\n\n4 5\n", encoding="utf-8") + + token_ids = build_token_pool(corpus, _Tokenizer(), limit=4) + output = tmp_path / "tokens.u32" + write_token_pool(token_ids, output) + + values = array("I") + with output.open("rb") as file: + values.fromfile(file, 4) + assert token_ids == [1, 2, 3, 4] + assert values.tolist() == token_ids + + +def test_build_token_pool_drops_declared_special_token(tmp_path) -> None: + corpus = tmp_path / "corpus.txt" + corpus.write_text("15 16\n", encoding="utf-8") + + assert build_token_pool(corpus, _Tokenizer(), limit=10) == [15] + + +def test_build_token_pool_rejects_unknown_out_of_range_token(tmp_path) -> None: + corpus = tmp_path / "corpus.txt" + corpus.write_text("15 17\n", encoding="utf-8") + + with pytest.raises(ValueError, match="outside"): + build_token_pool(corpus, _Tokenizer(), limit=10)