diff --git a/mbo/hash/measurements/hash_benchmark_report.py b/mbo/hash/measurements/hash_benchmark_report.py index 9c80185..961b044 100644 --- a/mbo/hash/measurements/hash_benchmark_report.py +++ b/mbo/hash/measurements/hash_benchmark_report.py @@ -55,9 +55,10 @@ import tarfile import tempfile -# README tables show this (dense) set; the stored FULL dataset holds ~3x more -# (a slow exponential) for the curve. Mirror kReadmeSizes in hash_benchmark.cc. -_README_SIZES = [1, 3, 7, 8, 11, 15, 16, 19, 22, 27, 32, 47, 48, 63, 64, 256, 1024, 4096] +# README tables read their sizes from the data itself (see _throughput_table), +# so there is no size list here to drift. The curated README set = kReadmeSizes +# in hash_benchmark.cc (the FAST mode); the tables are rendered from a fast-mode +# dataset, the ns-vs-length chart from the dense FULL dataset. # Ordering uses the benchmark's algorithm keys (data keys); _LABEL_128 only # renames "mumbo" to "jumbo" for display in the 128-bit table. @@ -394,12 +395,13 @@ def line(cells): return "\n".join([line(headers), "| " + " | ".join(sep) + " |"] + [line(r) for r in rows]) -def _throughput_table(data, preferred, relabel, sizes): - # Length-per-row, algorithm-per-column: with ~18 README sizes this reads far - # better than 18 columns, and each row's bold marks the fastest algorithm at - # that length. +def _throughput_table(data, preferred, relabel): + # Length-per-row, algorithm-per-column; each row's bold marks the fastest + # algorithm at that length. The sizes are read from the data's own buckets + # (the single source of truth), so there is no second size list to drift from + # the C++ kReadmeSizes - feed this the README (fast-mode) dataset. algos = _order(list(data), preferred) - sizes = [s for s in sizes if any(str(s) in data[a] for a in algos)] + sizes = sorted({int(s) for a in data.values() for s in a}) headers = ["Length"] + [relabel.get(a, a) for a in algos] rows = [] for size in sizes: @@ -447,11 +449,11 @@ def render_tables(results): "", f"### 64-bit one-shot throughput (ns/op, {agg}; lower is better)", "", - _throughput_table(results["throughput64"], _ORDER_64, {}, _README_SIZES), + _throughput_table(results["throughput64"], _ORDER_64, {}), "", f"### 128-bit one-shot throughput (ns/op, {agg}; native-128 algorithms only)", "", - _throughput_table(results["throughput128"], _ORDER_128, _LABEL_128, _README_SIZES), + _throughput_table(results["throughput128"], _ORDER_128, _LABEL_128), "", f"### Mixed-length latency (ns/hash, {agg}; lower is better)", "", diff --git a/mbo/hash/measurements/run_measurements.py b/mbo/hash/measurements/run_measurements.py index a7bd80e..664a16c 100755 --- a/mbo/hash/measurements/run_measurements.py +++ b/mbo/hash/measurements/run_measurements.py @@ -98,17 +98,26 @@ def main(argv): extras = [] # extra files packed alongside the canonical in the per-machine bundle if not args.skip_perf: - print(">>> [perf] full performance sweep (runs solo for clean numbers)", file=sys.stderr) + cfg = ["--config", args.config] if args.config else [] + print(">>> [perf] full sweep (curve) + fast sweep (README tables), solo for clean numbers", file=sys.stderr) + # FULL sweep: the dense ns-vs-length curve + the authoritative raw (compare.py). + subprocess.run( + [*report, "run", "--mode", "full", "--reps", str(args.reps), + "--raw", os.path.join(data, "raw.json.gz"), "--out", os.path.join(data, "results.json")] + cfg, + check=True, + ) + canonical = newest(os.path.join(data, "*_results.json")) + extras.append(newest(os.path.join(data, "*_raw.json.gz"))) + # FAST sweep: the README tables. Its buckets ARE kReadmeSizes, and the tool + # reads the sizes from THIS data (no hardcoded list to drift from the C++). with open(os.path.join(data, "README_tables.md"), "w") as tables: subprocess.run( - [*report, "run", "--mode", "full", "--reps", str(args.reps), - "--raw", os.path.join(data, "raw.json.gz"), "--out", os.path.join(data, "results.json"), "--tables"] - + (["--config", args.config] if args.config else []), + [*report, "run", "--mode", "fast", "--reps", str(args.reps), + "--out", os.path.join(data, "results_readme.json"), "--tables"] + cfg, stdout=tables, check=True, ) - canonical = newest(os.path.join(data, "*_results.json")) - extras.append(newest(os.path.join(data, "*_raw.json.gz"))) - print(">>> [perf] rendering ns-vs-length charts (64-bit + 128-bit, log-log)", file=sys.stderr) + extras.append(newest(os.path.join(data, "*_results_readme.json"))) + print(">>> [perf] rendering ns-vs-length charts from the full sweep (64/128, log-log)", file=sys.stderr) subprocess.run([*report, "plot", "--results", canonical, "--out", os.path.join(data, "hash_throughput.svg")], check=True) for width in ("64", "128"): shutil.copy(