Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions mbo/hash/measurements/hash_benchmark_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,10 @@
import tarfile
import tempfile

# README tables show this (dense) set; the stored FULL dataset holds ~3x more
# (a slow exponential) for the curve. Mirror kReadmeSizes in hash_benchmark.cc.
_README_SIZES = [1, 3, 7, 8, 11, 15, 16, 19, 22, 27, 32, 47, 48, 63, 64, 256, 1024, 4096]
# README tables read their sizes from the data itself (see _throughput_table),
# so there is no size list here to drift. The curated README set = kReadmeSizes
# in hash_benchmark.cc (the FAST mode); the tables are rendered from a fast-mode
# dataset, the ns-vs-length chart from the dense FULL dataset.

# Ordering uses the benchmark's algorithm keys (data keys); _LABEL_128 only
# renames "mumbo" to "jumbo" for display in the 128-bit table.
Expand Down Expand Up @@ -394,12 +395,13 @@ def line(cells):
return "\n".join([line(headers), "| " + " | ".join(sep) + " |"] + [line(r) for r in rows])


def _throughput_table(data, preferred, relabel, sizes):
# Length-per-row, algorithm-per-column: with ~18 README sizes this reads far
# better than 18 columns, and each row's bold marks the fastest algorithm at
# that length.
def _throughput_table(data, preferred, relabel):
# Length-per-row, algorithm-per-column; each row's bold marks the fastest
# algorithm at that length. The sizes are read from the data's own buckets
# (the single source of truth), so there is no second size list to drift from
# the C++ kReadmeSizes - feed this the README (fast-mode) dataset.
algos = _order(list(data), preferred)
sizes = [s for s in sizes if any(str(s) in data[a] for a in algos)]
sizes = sorted({int(s) for a in data.values() for s in a})
headers = ["Length"] + [relabel.get(a, a) for a in algos]
rows = []
for size in sizes:
Expand Down Expand Up @@ -447,11 +449,11 @@ def render_tables(results):
"",
f"### 64-bit one-shot throughput (ns/op, {agg}; lower is better)",
"",
_throughput_table(results["throughput64"], _ORDER_64, {}, _README_SIZES),
_throughput_table(results["throughput64"], _ORDER_64, {}),
"",
f"### 128-bit one-shot throughput (ns/op, {agg}; native-128 algorithms only)",
"",
_throughput_table(results["throughput128"], _ORDER_128, _LABEL_128, _README_SIZES),
_throughput_table(results["throughput128"], _ORDER_128, _LABEL_128),
"",
f"### Mixed-length latency (ns/hash, {agg}; lower is better)",
"",
Expand Down
23 changes: 16 additions & 7 deletions mbo/hash/measurements/run_measurements.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,17 +98,26 @@ def main(argv):
extras = [] # extra files packed alongside the canonical in the per-machine bundle

if not args.skip_perf:
print(">>> [perf] full performance sweep (runs solo for clean numbers)", file=sys.stderr)
cfg = ["--config", args.config] if args.config else []
print(">>> [perf] full sweep (curve) + fast sweep (README tables), solo for clean numbers", file=sys.stderr)
# FULL sweep: the dense ns-vs-length curve + the authoritative raw (compare.py).
subprocess.run(
[*report, "run", "--mode", "full", "--reps", str(args.reps),
"--raw", os.path.join(data, "raw.json.gz"), "--out", os.path.join(data, "results.json")] + cfg,
check=True,
)
canonical = newest(os.path.join(data, "*_results.json"))
extras.append(newest(os.path.join(data, "*_raw.json.gz")))
# FAST sweep: the README tables. Its buckets ARE kReadmeSizes, and the tool
# reads the sizes from THIS data (no hardcoded list to drift from the C++).
with open(os.path.join(data, "README_tables.md"), "w") as tables:
subprocess.run(
[*report, "run", "--mode", "full", "--reps", str(args.reps),
"--raw", os.path.join(data, "raw.json.gz"), "--out", os.path.join(data, "results.json"), "--tables"]
+ (["--config", args.config] if args.config else []),
[*report, "run", "--mode", "fast", "--reps", str(args.reps),
"--out", os.path.join(data, "results_readme.json"), "--tables"] + cfg,
stdout=tables, check=True,
)
canonical = newest(os.path.join(data, "*_results.json"))
extras.append(newest(os.path.join(data, "*_raw.json.gz")))
print(">>> [perf] rendering ns-vs-length charts (64-bit + 128-bit, log-log)", file=sys.stderr)
extras.append(newest(os.path.join(data, "*_results_readme.json")))
print(">>> [perf] rendering ns-vs-length charts from the full sweep (64/128, log-log)", file=sys.stderr)
subprocess.run([*report, "plot", "--results", canonical, "--out", os.path.join(data, "hash_throughput.svg")], check=True)
for width in ("64", "128"):
shutil.copy(
Expand Down
Loading