From e4f9b5160e22be789fb4afc121c08c6d5f030c56 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 12 Jul 2026 23:29:55 +0100 Subject: [PATCH 1/4] chore(hash): integrate measurements as a dev package + gate the Results table - Drop the nested mbo/hash/measurements module and its .bazelignore entry so it is an ordinary dev package (built/tested in dev + CI). It is still stripped from release archives by release_prep.sh, so nothing shipped depends on it. - Add //mbo/hash/measurements:quality_sh_test, a bashtest that runs `hash_benchmark_report.py quality --check` (README Results table in sync with the curated data), plus drift-detection and idempotence checks. - Add a no-deps-on-measurements pre-commit guard: no .bazel/.bzl file outside the dir may reference //mbo/hash/measurements, so the release strip cannot leave a dangling reference. - Export //mbo/hash:README.md (public) for the test; refresh measurements/README and the hash README Methodology (how to measure on another machine). --- .bazelignore | 1 - .pre-commit-config.yaml | 11 ++++ CHANGELOG.md | 2 + mbo/hash/BUILD.bazel | 8 +++ mbo/hash/README.md | 7 +++ mbo/hash/measurements/BUILD.bazel | 35 ++++++++++++ mbo/hash/measurements/MODULE.bazel | 29 ---------- mbo/hash/measurements/README.md | 23 ++++---- mbo/hash/measurements/quality_sh_test.sh | 70 ++++++++++++++++++++++++ 9 files changed, 146 insertions(+), 40 deletions(-) create mode 100644 mbo/hash/measurements/BUILD.bazel delete mode 100644 mbo/hash/measurements/MODULE.bazel create mode 100755 mbo/hash/measurements/quality_sh_test.sh diff --git a/.bazelignore b/.bazelignore index 1a91016a..de18a698 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1,2 +1 @@ bazel-mbo -mbo/hash/measurements diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8333dd69..fc690bc6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -55,6 +55,17 @@ repos: language: pygrep entry: \s*#\s*include files: MODULE.bazel + - id: no-deps-on-measurements + name: no deps on mbo/hash/measurements + description: | + mbo/hash/measurements is dev-only tooling, stripped from release archives + (release_prep.sh EXCLUDES). Nothing outside it may load from or depend on + it (label `//mbo/hash/measurements`), or a release build would carry a + dangling reference. Dependency edges stay one-way: measurements -> library. + language: pygrep + entry: //mbo/hash/measurements + files: \.(bazel|bzl)$ + exclude: ^mbo/hash/measurements/ - id: github-bazelmod-patch-applies name: github-bazelmod-patch-applies description: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ad2ae6a..49a04f3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ - Added the `quality` command to `hash_benchmark_report.py`: it generates the SMHasher3 "Results" table in `mbo/hash/README.md` from a curated source (verify with `--check`, source measured verdict/score from a fresh run with `--smhasher `). - Fixed SMHasher3 result parsing so an invalid hash name or crash reads as `ERROR` instead of a false `PASS`, and the pass/fail score and failing families now parse from the run Summary; legacy/short SMHasher3 names are aliased to their registered form when a measured dataset is loaded. +- Integrated `mbo/hash/measurements` as a normal dev package (dropped its nested module and `.bazelignore` entry; still stripped from release archives) and added a `quality_sh_test` bazel test gating the generated Results table. +- Added a `no-deps-on-measurements` pre-commit guard so nothing outside the release-stripped `mbo/hash/measurements` may depend on it (which would dangle in releases). # 0.13.1 diff --git a/mbo/hash/BUILD.bazel b/mbo/hash/BUILD.bazel index 0b3dcfa7..a695c478 100644 --- a/mbo/hash/BUILD.bazel +++ b/mbo/hash/BUILD.bazel @@ -23,6 +23,14 @@ load(":internal/hash_mangle_seed.bzl", "mangle_seed_gen") package(default_visibility = ["//visibility:private"]) +# Exported so the dev-only measurements quality test (under mbo/hash/measurements, +# stripped from release archives) can verify this README's generated SMHasher3 +# table. Public visibility, so this line references no release-stripped package. +exports_files( + ["README.md"], + visibility = ["//visibility:public"], +) + # Create custom bazel flag `--//mbo/hash:mangle_seed`: any printable-ASCII # string (user name, release tag, date, ...) selecting the mangle constant. # Folded to a bucket inside `mangle_seed_gen`, so caches converge (see diff --git a/mbo/hash/README.md b/mbo/hash/README.md index b5104454..2643f481 100644 --- a/mbo/hash/README.md +++ b/mbo/hash/README.md @@ -568,3 +568,10 @@ reads as the compact MUM hash rather than a second tuned one: Full logs are not committed; regenerate as above. Last run (2026-07): all three in-house hashes clean - `mumbo-64`/`jumbo-128` and `dumbo-64` PASS 188 / 188. +- To produce measurements on another machine (or refresh a machine's numbers), + run the tooling from the repo root: `mbo/hash/measurements/run_measurements.py` + writes one Git-LFS bundle per machine (perf sweep + SMHasher3), then + `hash_benchmark_report.py publish --bundles <...>` re-renders the performance + tables and charts above, and `hash_benchmark_report.py quality --smhasher +` refreshes the Results table. Full per-machine and publish steps are + in `mbo/hash/measurements/README.md`. diff --git a/mbo/hash/measurements/BUILD.bazel b/mbo/hash/measurements/BUILD.bazel new file mode 100644 index 00000000..cf7f450b --- /dev/null +++ b/mbo/hash/measurements/BUILD.bazel @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) The helly25 authors (helly25.com) +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dev-only measurement tooling for mbo/hash (benchmark + SMHasher3 quality). The +# whole directory is stripped from release archives (release_prep.sh EXCLUDES), so +# nothing shipped depends on it; these targets exist only for the dev build/CI. + +load("@helly25_bashtest//bashtest:bashtest.bzl", "bashtest") + +package(default_visibility = ["//visibility:private"]) + +# Guards that the generated SMHasher3 "Results" table in //mbo/hash/README.md +# stays in sync with the curated source of truth in hash_benchmark_report.py +# (`quality --check`). Uses the system python3 - the tool has no third-party deps. +bashtest( + name = "quality_sh_test", + size = "small", + srcs = ["quality_sh_test.sh"], + data = [ + "hash_benchmark_report.py", + "//mbo/hash:README.md", + ], +) diff --git a/mbo/hash/measurements/MODULE.bazel b/mbo/hash/measurements/MODULE.bazel deleted file mode 100644 index a2eb9698..00000000 --- a/mbo/hash/measurements/MODULE.bazel +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) The helly25 authors (helly25.com) -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Separate DEV-ONLY module for the mbo/hash measurement tooling. It is listed in -# the repository-root .bazelignore, so it is never part of the `helly25_mbo` -# module, its //... target universe, or any published/BCR offering. This keeps -# measurement/plotting dev dependencies out of the library's dependency closure. -# -# The report script (hash_benchmark_report.py) is pure Python standard library -# and needs no bazel to run; it shells out to `bazel run //mbo/hash:hash_benchmark` -# from the repository root. This module exists to host any future measurement -# dependencies in isolation. See README.md. - -module( - name = "helly25_mbo_hash_measurements", - version = "0.0.0", -) diff --git a/mbo/hash/measurements/README.md b/mbo/hash/measurements/README.md index 769c41f3..12a3e500 100644 --- a/mbo/hash/measurements/README.md +++ b/mbo/hash/measurements/README.md @@ -2,11 +2,13 @@ Dev-only tooling to run `//mbo/hash:hash_benchmark`, store the results with full provenance, render the README performance tables, and plot ns-vs-length curves. -This is a **pure development dependency**: it is its own Bazel module -(`MODULE.bazel` here) and is listed in the repository-root `.bazelignore`, so it -is never part of the `helly25_mbo` module, its `//...` target universe, or any -published/BCR offering. Consumers of the library never pull any of this in -(including heavier plotting dependencies). +This is a **pure development dependency**: the whole `mbo/hash/measurements` +directory is stripped from release archives (`release_prep.sh` EXCLUDES), so it is +never part of any published/BCR offering and consumers of the library never pull +any of it in (including any heavier plotting dependencies). It is an ordinary dev +package of the `helly25_mbo` module - built and tested in dev/CI - and a +pre-commit guard forbids anything outside it from depending on it, so the release +strip can never leave a dangling reference. This is a living design doc - update it when the layout or policy changes. @@ -31,9 +33,10 @@ even invert the true ordering. So we: ```text mbo/hash/measurements/ README.md # this design doc - MODULE.bazel # separate dev module (isolates plotting deps) + BUILD.bazel # dev-only targets: the quality-table verify test (stripped from releases) run_measurements.py # per-machine runner: one full sweep + smhasher -> a data bundle (nothing else) - hash_benchmark_report.py # run / store / tables / plot / bundle / publish / verify (stdlib only) + hash_benchmark_report.py # run / store / tables / plot / smhasher / bundle / publish / verify / quality (stdlib only) + quality_sh_test.sh # bazel test: the README SMHasher3 Results table matches the curated data build_smhasher3.sh # reproducible SMHasher3 build (clone + fixes + install plugin + container gcc) smhasher3/mbohash.cpp # in-house mumbo/jumbo and dumbo SMHasher3 registration (includes the real headers) charts/__*.svg # published per-machine charts (committed; rendered by `publish`, verifiable) @@ -118,9 +121,9 @@ logs. Across several machines that accumulates, so: the ONLY thing that writes to the tree; the measurement run writes nothing but the bundle, so it stays authoritative and several runs can go back-to-back. - The loose run outputs in `data/` are staging only and git-ignored; `bundle` - packs them into the `.tgz`. This directory is dev-only and `.bazelignore`'d, so - it never enters the module or BCR (the release archive carries LFS pointer - files here, which is harmless). + packs them into the `.tgz`. This directory is dev-only and stripped from release + archives, so it never enters a published/BCR offering (the release archive + carries LFS pointer files here, which is harmless). ## Regenerate / contribute a machine diff --git a/mbo/hash/measurements/quality_sh_test.sh b/mbo/hash/measurements/quality_sh_test.sh new file mode 100755 index 00000000..c1bc25b9 --- /dev/null +++ b/mbo/hash/measurements/quality_sh_test.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) The helly25 authors (helly25.com) +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Guards the generated SMHasher3 "Results" table in mbo/hash/README.md: it must +# stay in sync with the curated source of truth in hash_benchmark_report.py +# (`quality --check`), the drift guard must actually fail on a changed table (so +# it is not a no-op), and regenerating must be idempotent (the committed table is +# already in canonical form). + +# shellcheck disable=SC2317 # Functions are called by the bashtest runner. + +set -euo pipefail + +# shellcheck disable=SC1090,SC1091,SC2154 +source "${helly25_bashtest}" + +REPORT="${TEST_SRCDIR}/${TEST_WORKSPACE}/mbo/hash/measurements/hash_benchmark_report.py" +README="${TEST_SRCDIR}/${TEST_WORKSPACE}/mbo/hash/README.md" +declare -r REPORT README + +[[ -f ${REPORT} ]] || die "hash_benchmark_report.py not found." +[[ -f ${README} ]] || die "README.md not found." + +PYTHON="${PYTHON:-python3}" +command -v "${PYTHON}" >/dev/null || die "python3 not found on PATH." +declare -r PYTHON + +function quality() { + "${PYTHON}" "${REPORT}" quality "$@" +} + +# The committed table must match the curated data - this is the drift guard. +function test::committed_table_is_in_sync() { + quality --check --readme "${README}" \ + || die "README SMHasher3 Results table is stale; run: hash_benchmark_report.py quality" +} + +# The guard must actually catch drift: a table with a flipped score must fail. +function test::check_detects_drift() { + local drift="${TEST_TMPDIR}/drift_README.md" + sed 's#| 7/186 #| 9/186 #' "${README}" >"${drift}" + cmp -s "${README}" "${drift}" && die "drift fixture is identical - the score edit did not match." + if quality --check --readme "${drift}" 2>/dev/null; then + die "quality --check passed on a changed table; the drift guard is a no-op." + fi +} + +# Regenerating into a copy of the committed README must change nothing. +function test::regenerate_is_idempotent() { + local copy="${TEST_TMPDIR}/regen_README.md" + cp "${README}" "${copy}" + quality --readme "${copy}" >/dev/null || die "quality failed to write the table." + cmp -s "${README}" "${copy}" || die "regenerating changed the README - it was not in canonical form." +} + +test_runner From d176b3eb16f17f5244d4bb6776d32a8c44df9a64 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 12 Jul 2026 23:34:33 +0100 Subject: [PATCH 2/4] docs(hash): show a runnable sh example for producing measurements on another machine --- mbo/hash/README.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/mbo/hash/README.md b/mbo/hash/README.md index 2643f481..8583bfe1 100644 --- a/mbo/hash/README.md +++ b/mbo/hash/README.md @@ -569,9 +569,19 @@ reads as the compact MUM hash rather than a second tuned one: three in-house hashes clean - `mumbo-64`/`jumbo-128` and `dumbo-64` PASS 188 / 188. - To produce measurements on another machine (or refresh a machine's numbers), - run the tooling from the repo root: `mbo/hash/measurements/run_measurements.py` - writes one Git-LFS bundle per machine (perf sweep + SMHasher3), then - `hash_benchmark_report.py publish --bundles <...>` re-renders the performance - tables and charts above, and `hash_benchmark_report.py quality --smhasher -` refreshes the Results table. Full per-machine and publish steps are - in `mbo/hash/measurements/README.md`. + run the tooling from the repo root (full per-machine and publish steps in + [`measurements/README.md`](measurements/README.md)): + + ```sh + # On the new machine: one perf sweep + the SMHasher3 battery, packed into a + # per-machine Git-LFS bundle whose path the script prints. + mbo/hash/measurements/run_measurements.py --config clang --jobs 4 + git add mbo/hash/measurements/data/.tgz + git lfs push origin HEAD && git push + + # Re-render this README from the chosen bundles, then refresh the SMHasher3 + # Results table from the same bundle's measured data. + mbo/hash/measurements/hash_benchmark_report.py publish --bundles data/.tgz + mbo/hash/measurements/hash_benchmark_report.py quality --smhasher data/.tgz + git add mbo/hash/README.md mbo/hash/measurements/charts + ``` From 7ffed2999dede49b96f096df483b3201e6e37ca6 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 12 Jul 2026 23:53:11 +0100 Subject: [PATCH 3/4] docs(hash): intro - make clear dumbo also passes SMHasher3 (188/188) The intro credited only mumbo/jumbo with passing and appended "dumbo is the compact companion", which read as if dumbo did not pass. All three in-house hashes pass 188/188; dumbo is a compact single-lane companion with a different profile, not a lower-quality one. --- mbo/hash/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mbo/hash/README.md b/mbo/hash/README.md index 8583bfe1..28d6b512 100644 --- a/mbo/hash/README.md +++ b/mbo/hash/README.md @@ -2,10 +2,12 @@ Fast, constexpr-safe, non-cryptographic hashing, built around the in-house **mumbo/jumbo and dumbo** family: notice-free, pure Apache-2.0, and MUM-based -(widening multiply). `mumbo` (64-bit) and its native 128-bit sibling `jumbo` -both pass [SMHasher3](https://gitlab.com/fwojcik/smhasher3) clean (188/188 - -`jumbo` is the only clean native 128 we measured) and post the best mixed-length -latency in our benchmarks; `dumbo` is the compact companion. +(widening multiply). All three pass [SMHasher3](https://gitlab.com/fwojcik/smhasher3) +clean (188/188): `mumbo` (64-bit) and its native 128-bit sibling `jumbo` (the +only clean native 128 we measured) post the best mixed-length latency in our +benchmarks, and `dumbo` is a compact single-lane companion with a very different +profile - fastest here on tiny keys, slower on bulk - that trades reach for size, +not quality. It also ships a **build-seed mangle** (`hash_mangle.h`): restricted/limited, constexpr-safe compile-time hash mangling with release-time rotation enforcement. From 611440bdbc7ab5424643fed6638af942e3b237d1 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Mon, 13 Jul 2026 00:20:24 +0100 Subject: [PATCH 4/4] feat(hash): generate BOTH quality tables from JSON (manual) + a measured bundle Replace the curated _SMH_RESULTS_ROWS with a clean separation of concerns: - hash_algorithms.json holds ONLY manual/editorial per-instance columns (role, notice, seeded, streaming, starlark, available_via, smhasher name), one entry per (algorithm, width) so both tables separate 64/128 consistently (murmur3 is 128-only - its GetHash64 folds the 128; xxh3 is native at both). - Measured verdict/score/failures are re-parsed from a data bundle's logs (_measured_from_bundle; the stored smhasher.json predates parser fixes) and joined by smhasher name; a _MISSING_MEASURED stopgap covers FNV-1a-64/ MurmurHash3-128 (invalid-name logs) until a bundle carries them, warned loudly. - now renders BOTH the Algorithm-overview (SMHasher3 PASS/FAIL derived from the measurement) and the Results table (measured score + families) between their own marker regions; gates both. - New command: all bundles at the same source SHA must report identical measurements (verdicts are machine-independent). Verified: the two committed bundles agree. - quality_sh_test reads the LFS bundles (data dep); the test CI job fetches them via git lfs pull (actions/checkout leaves pointers). --- .github/workflows/test.yml | 5 + CHANGELOG.md | 4 +- mbo/hash/README.md | 51 +-- mbo/hash/measurements/BUILD.bazel | 11 +- mbo/hash/measurements/README.md | 24 +- mbo/hash/measurements/hash_algorithms.json | 136 ++++++++ .../measurements/hash_benchmark_report.py | 329 ++++++++++++------ mbo/hash/measurements/quality_sh_test.sh | 42 ++- 8 files changed, 447 insertions(+), 155 deletions(-) create mode 100644 mbo/hash/measurements/hash_algorithms.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 09395638..32b8fdb6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,6 +58,11 @@ jobs: continue-on-error: ${{inputs.continue-on-error}} steps: - uses: actions/checkout@v6 + - name: Fetch measurement bundles (Git LFS) + # The `//mbo/hash/measurements:quality_sh_test` bazel test reads the + # committed data bundles for the SMHasher3 quality tables; actions/checkout + # leaves them as LFS pointers, so materialize just those (each is small). + run: git lfs pull --include="mbo/hash/measurements/data/*.tgz" - uses: bazelbuild/setup-bazelisk@v3 - name: Install GCC if: ${{inputs.compiler == 'gcc' && inputs.gcc_version != ''}} diff --git a/CHANGELOG.md b/CHANGELOG.md index 49a04f3d..0ce81a09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # 0.13.2 -- Added the `quality` command to `hash_benchmark_report.py`: it generates the SMHasher3 "Results" table in `mbo/hash/README.md` from a curated source (verify with `--check`, source measured verdict/score from a fresh run with `--smhasher `). +- Added the `quality` command to `hash_benchmark_report.py`: it generates both the "Algorithm overview" and SMHasher3 "Results" tables in `mbo/hash/README.md` from `hash_algorithms.json` (manual/editorial columns) merged with the measured verdict/score/failures re-parsed from a data bundle (`--bundle`); `--check` gates them, and a `consistency` command verifies all bundles from the same source SHA agree (SMHasher3 verdicts are machine-independent). - Fixed SMHasher3 result parsing so an invalid hash name or crash reads as `ERROR` instead of a false `PASS`, and the pass/fail score and failing families now parse from the run Summary; legacy/short SMHasher3 names are aliased to their registered form when a measured dataset is loaded. -- Integrated `mbo/hash/measurements` as a normal dev package (dropped its nested module and `.bazelignore` entry; still stripped from release archives) and added a `quality_sh_test` bazel test gating the generated Results table. +- Integrated `mbo/hash/measurements` as a normal dev package (dropped its nested module and `.bazelignore` entry; still stripped from release archives) and added a `quality_sh_test` bazel test gating the generated tables; the test CI job now fetches the measurement bundles (Git LFS) so it reads real data. - Added a `no-deps-on-measurements` pre-commit guard so nothing outside the release-stripped `mbo/hash/measurements` may depend on it (which would dangle in releases). # 0.13.1 diff --git a/mbo/hash/README.md b/mbo/hash/README.md index 28d6b512..35dbc255 100644 --- a/mbo/hash/README.md +++ b/mbo/hash/README.md @@ -72,17 +72,22 @@ Three entry points, split by contract: This is the at-a-glance map; the `SMHasher3` column is a PASS/FAIL summary only. For the exact score and the failing families see [Quality: SMHasher3](#quality-smhasher3). -| Algorithm | Widths | Available via | Starlark | NOTICE | Seeded | Streaming | SMHasher3 | -| ----------- | ------ | --------------------------------- | -------- | ----------------------- | ------ | --------- | --------- | -| `mumbo` | 64 | `hash.h` (default 64/32) | no | none (in-house) | yes | yes | PASS | -| `jumbo` | 128 | `hash.h` (default 128) | no | none (in-house) | yes | yes (64) | PASS | -| `murmur3` | 64/128 | `hash.h` | no | none (public domain) | yes | no | FAIL | -| `siphash` | 64 | `hash.h` | no | none (CC0) | keyed | yes | PASS | -| `fnv1a` | 64 | `hash.h` | yes | none (public domain) | yes | no | FAIL | -| `dumbo` | 64 | `hash.h` | yes | none (in-house) | yes | no | PASS | -| `rapidhash` | 64 | `hash_extra.h` + `:hash_extra_cc` | no | **MIT - ship NOTICE** | yes | no | PASS | -| `xxh64` | 64 | `hash_extra.h` + `:hash_extra_cc` | no | **BSD-2 - ship NOTICE** | yes | yes | FAIL | -| `xxh3` | 64/128 | `hash_extra.h` + `:hash_extra_cc` | no | **BSD-2 - ship NOTICE** | yes | no | FAIL | + + +| Algorithm | Bits | Available via | Starlark | NOTICE | Seeded | Streaming | SMHasher3 | +| ----------- | ---: | --------------------------------- | -------- | ----------------------- | ------ | --------- | --------- | +| `mumbo` | 64 | `hash.h` (default 64/32) | no | none (in-house) | yes | yes | PASS | +| `jumbo` | 128 | `hash.h` (default 128) | no | none (in-house) | yes | yes (64) | PASS | +| `murmur3` | 128 | `hash.h` | no | none (public domain) | yes | no | FAIL | +| `siphash` | 64 | `hash.h` | no | none (CC0) | keyed | yes | PASS | +| `fnv1a` | 64 | `hash.h` | yes | none (public domain) | yes | no | FAIL | +| `dumbo` | 64 | `hash.h` | yes | none (in-house) | yes | no | PASS | +| `rapidhash` | 64 | `hash_extra.h` + `:hash_extra_cc` | no | **MIT - ship NOTICE** | yes | no | PASS | +| `xxh64` | 64 | `hash_extra.h` + `:hash_extra_cc` | no | **BSD-2 - ship NOTICE** | yes | yes | FAIL | +| `xxh3` | 64 | `hash_extra.h` + `:hash_extra_cc` | no | **BSD-2 - ship NOTICE** | yes | no | FAIL | +| `xxh3` | 128 | `hash_extra.h` + `:hash_extra_cc` | no | **BSD-2 - ship NOTICE** | yes | no | FAIL | + + Notes: the **Starlark** column marks the hashes also implemented at build time in [`hash.bzl`](hash.bzl) (`hash.dumbo` and `hash.fnv1a`), kept byte-for-byte @@ -432,18 +437,18 @@ numbers are directly comparable. -| Algorithm | Bits | Role in mbo/hash | SMHasher3 result | Failures | -| ----------- | ---: | ------------------------- | ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `dumbo` | 64 | `hash.h` (compact MUM) | PASS | none | -| `fnv1a` | 64 | `hash.h` | 7/186 | nearly every family: Avalanche, BIC, Sparse, Cyclic, Permutation, Text, TwoBytes, Bitflip, PerlinNoise, and the complete Seed* cluster | -| `mumbo` | 64 | default (64/32/streaming) | PASS | none | -| `rapidhash` | 64 | extra (`hash_extra_cc`) | PASS | none | -| `siphash` | 64 | `hash.h` (keyed PRF) | PASS | none | -| `xxh3` | 64 | extra (`hash_extra_cc`) | 166/188 | BIC [3, 8, 11], Sparse [20/3], PerlinNoise [2], Bitflip [8], SeedZeroes [1280, 8448], SeedSparse [2, 3] | -| `xxh64` | 64 | extra (`hash_extra_cc`) | 181/188 | SeedBlockLen [15, 19, 21, 26, 29, 30], SeedBIC [8] | -| `jumbo` | 128 | default (128) | PASS | none | -| `murmur3` | 128 | `hash.h` | 123/188 | BIC, Zeroes, Permutation, and the complete Seed* cluster (11 families) | -| `xxh3` | 128 | extra (`hash_extra_cc`) | 162/188 | BIC [3, 8, 15], Sparse [20/3], PerlinNoise [2], Bitflip [3, 4, 8], SeedZeroes [1280, 8448], SeedSparse [2, 3], SeedBlockLen [8, 12-16], SeedBlockOffset [0-5], SeedBIC [3, 8] | +| Algorithm | Bits | Role in mbo/hash | SMHasher3 result | Failures | +| ----------- | ---: | ------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `dumbo` | 64 | `hash.h` (compact MUM) | PASS | none | +| `fnv1a` | 64 | `hash.h` | 7/186 | nearly every family: Avalanche, BIC, Sparse, Cyclic, Permutation, Text, TwoBytes, Bitflip, PerlinNoise, and the complete Seed* cluster | +| `mumbo` | 64 | default (64/32/streaming) | PASS | none | +| `rapidhash` | 64 | extra (`hash_extra_cc`) | PASS | none | +| `siphash` | 64 | `hash.h` (keyed PRF) | PASS | none | +| `xxh3` | 64 | extra (`hash_extra_cc`) | 166/188 | BIC [3, 8, 11], Sparse [20/3], PerlinNoise [2], Bitflip [8], SeedZeroes [1280, 8448], SeedSparse [2, 3], SeedBlockLen [8, 13, 14, 15, 16], SeedBlockOffset [0, 1, 2, 3, 4], SeedBIC [3, 8] | +| `xxh64` | 64 | extra (`hash_extra_cc`) | 181/188 | SeedBlockLen [15, 19, 21, 26, 29, 30], SeedBIC [8] | +| `jumbo` | 128 | default (128) | PASS | none | +| `murmur3` | 128 | `hash.h` | 123/188 | BIC, Zeroes, Permutation, and the complete Seed* cluster (11 families) | +| `xxh3` | 128 | extra (`hash_extra_cc`) | 162/188 | BIC [3, 8, 15], Sparse [20/3], PerlinNoise [2], Bitflip [3, 4, 8], SeedZeroes [1280, 8448], SeedSparse [2, 3], SeedBlockLen [8, 12, 13, 14, 15, 16], SeedBlockOffset [0, 1, 2, 3, 4, 5], SeedBIC [3, 8] | diff --git a/mbo/hash/measurements/BUILD.bazel b/mbo/hash/measurements/BUILD.bazel index cf7f450b..2eda359a 100644 --- a/mbo/hash/measurements/BUILD.bazel +++ b/mbo/hash/measurements/BUILD.bazel @@ -21,15 +21,18 @@ load("@helly25_bashtest//bashtest:bashtest.bzl", "bashtest") package(default_visibility = ["//visibility:private"]) -# Guards that the generated SMHasher3 "Results" table in //mbo/hash/README.md -# stays in sync with the curated source of truth in hash_benchmark_report.py -# (`quality --check`). Uses the system python3 - the tool has no third-party deps. +# Guards the generated SMHasher3 quality tables in //mbo/hash/README.md (overview +# + Results): `quality --check` (they match hash_algorithms.json x a measured +# bundle) and `consistency` (all bundles agree). Uses the system python3 (the tool +# has no third-party deps) and the Git-LFS bundles (materialized by test.yml / a +# local `git lfs pull`). bashtest( name = "quality_sh_test", size = "small", srcs = ["quality_sh_test.sh"], data = [ + "hash_algorithms.json", "hash_benchmark_report.py", "//mbo/hash:README.md", - ], + ] + glob(["data/*.tgz"]), ) diff --git a/mbo/hash/measurements/README.md b/mbo/hash/measurements/README.md index 12a3e500..ac8e564a 100644 --- a/mbo/hash/measurements/README.md +++ b/mbo/hash/measurements/README.md @@ -35,8 +35,9 @@ mbo/hash/measurements/ README.md # this design doc BUILD.bazel # dev-only targets: the quality-table verify test (stripped from releases) run_measurements.py # per-machine runner: one full sweep + smhasher -> a data bundle (nothing else) - hash_benchmark_report.py # run / store / tables / plot / smhasher / bundle / publish / verify / quality (stdlib only) - quality_sh_test.sh # bazel test: the README SMHasher3 Results table matches the curated data + hash_benchmark_report.py # run / store / tables / plot / smhasher / bundle / publish / verify / quality / consistency (stdlib only) + hash_algorithms.json # manual/editorial SOT for the quality tables (role, notice, seeded, starlark, ...; NOT measured data) + quality_sh_test.sh # bazel test: the README quality tables match hash_algorithms.json x a bundle, and bundles agree build_smhasher3.sh # reproducible SMHasher3 build (clone + fixes + install plugin + container gcc) smhasher3/mbohash.cpp # in-house mumbo/jumbo and dumbo SMHasher3 registration (includes the real headers) charts/__*.svg # published per-machine charts (committed; rendered by `publish`, verifiable) @@ -193,6 +194,25 @@ in `_SMHASHER_NAMES` and match `SMHasher3 --list`. Last verified run (2026-07): **mumbo-64/jumbo-128** and **dumbo-64** all PASS 188/188 (dumbo redesigned into a compact single-lane MUM hash). +### Generated quality tables (`quality`, `consistency`) + +The two SMHasher3 tables in `../README.md` (the "Algorithm overview" and the +"Results" table) are GENERATED, not hand-edited. `hash_benchmark_report.py quality +--bundle ` renders both between their `` markers, +merging `hash_algorithms.json` (the manual/editorial columns) with the measured +verdict/score/failures re-parsed from the bundle's logs (the stored `smhasher.json` +predates parser fixes, so the LOGS are the truth). `quality --check` verifies the +committed tables without writing - the `quality_sh_test` bazel test / CI gate. + +Two names the committed bundles have no real data for (`FNV-1a-64` / +`MurmurHash3-128`, run under invalid short names) are supplied by a hardcoded +`_MISSING_MEASURED` stopgap, warned about on every render; delete each once a +bundle carries the real measurement. + +`hash_benchmark_report.py consistency` cross-checks that all bundles from the same +source git SHA report identical measurements (a SMHasher3 verdict is a property of +the algorithm, not the machine), so a divergent or broken run is caught. + ## Output filenames Loose staging files the tool writes are prefixed `YYYYMMDD_HHMMSS_` (local wall diff --git a/mbo/hash/measurements/hash_algorithms.json b/mbo/hash/measurements/hash_algorithms.json new file mode 100644 index 00000000..1bd8d3d3 --- /dev/null +++ b/mbo/hash/measurements/hash_algorithms.json @@ -0,0 +1,136 @@ +{ + "_comment": [ + "Manual/editorial source of truth for the mbo/hash README tables, rendered by", + "`hash_benchmark_report.py quality`. This holds ONLY human-authored per-instance", + "data; the measured columns (SMHasher3 verdict/score/failing families) are NOT", + "here - they come from a data bundle (`quality --bundle`), joined by `smhasher` name.", + "One entry per (algorithm, width) instance, so both README tables separate 64/128", + "consistently (murmur3 is 128-only: its GetHash64 is the folded 128; xxh3 is native", + "at both). `starlark` marks a hash also ported in //mbo/hash:hash.bzl. Edit here;", + "run `quality` to re-render." + ], + "algorithms": [ + { + "algo": "dumbo", + "bits": 64, + "smhasher": "dumbo-64", + "available_via": "`hash.h`", + "role": "`hash.h` (compact MUM)", + "notice": "none (in-house)", + "seeded": "yes", + "streaming": "no", + "starlark": "yes" + }, + { + "algo": "fnv1a", + "bits": 64, + "smhasher": "FNV-1a-64", + "available_via": "`hash.h`", + "role": "`hash.h`", + "notice": "none (public domain)", + "seeded": "yes", + "streaming": "no", + "starlark": "yes" + }, + { + "algo": "mumbo", + "bits": 64, + "smhasher": "mumbo-64", + "available_via": "`hash.h` (default 64/32)", + "role": "default (64/32/streaming)", + "notice": "none (in-house)", + "seeded": "yes", + "streaming": "yes", + "starlark": "no" + }, + { + "algo": "rapidhash", + "bits": 64, + "smhasher": "rapidhash", + "available_via": "`hash_extra.h` + `:hash_extra_cc`", + "role": "extra (`hash_extra_cc`)", + "notice": "**MIT - ship NOTICE**", + "seeded": "yes", + "streaming": "no", + "starlark": "no" + }, + { + "algo": "siphash", + "bits": 64, + "smhasher": "SipHash-2-4", + "available_via": "`hash.h`", + "role": "`hash.h` (keyed PRF)", + "notice": "none (CC0)", + "seeded": "keyed", + "streaming": "yes", + "starlark": "no" + }, + { + "algo": "xxh3", + "bits": 64, + "smhasher": "XXH3-64", + "available_via": "`hash_extra.h` + `:hash_extra_cc`", + "role": "extra (`hash_extra_cc`)", + "notice": "**BSD-2 - ship NOTICE**", + "seeded": "yes", + "streaming": "no", + "starlark": "no" + }, + { + "algo": "xxh64", + "bits": 64, + "smhasher": "XXH-64", + "available_via": "`hash_extra.h` + `:hash_extra_cc`", + "role": "extra (`hash_extra_cc`)", + "notice": "**BSD-2 - ship NOTICE**", + "seeded": "yes", + "streaming": "yes", + "starlark": "no" + }, + { + "algo": "jumbo", + "bits": 128, + "smhasher": "jumbo-128", + "available_via": "`hash.h` (default 128)", + "role": "default (128)", + "notice": "none (in-house)", + "seeded": "yes", + "streaming": "yes (64)", + "starlark": "no" + }, + { + "algo": "murmur3", + "bits": 128, + "smhasher": "MurmurHash3-128", + "available_via": "`hash.h`", + "role": "`hash.h`", + "notice": "none (public domain)", + "seeded": "yes", + "streaming": "no", + "starlark": "no" + }, + { + "algo": "xxh3", + "bits": 128, + "smhasher": "XXH3-128", + "available_via": "`hash_extra.h` + `:hash_extra_cc`", + "role": "extra (`hash_extra_cc`)", + "notice": "**BSD-2 - ship NOTICE**", + "seeded": "yes", + "streaming": "no", + "starlark": "no" + } + ], + "overview_order": [ + "mumbo-64", + "jumbo-128", + "MurmurHash3-128", + "SipHash-2-4", + "FNV-1a-64", + "dumbo-64", + "rapidhash", + "XXH-64", + "XXH3-64", + "XXH3-128" + ] +} diff --git a/mbo/hash/measurements/hash_benchmark_report.py b/mbo/hash/measurements/hash_benchmark_report.py index 069a36cf..3c7a5bdb 100644 --- a/mbo/hash/measurements/hash_benchmark_report.py +++ b/mbo/hash/measurements/hash_benchmark_report.py @@ -88,43 +88,44 @@ _SMHASHER_ALL = ["mumbo", "jumbo", "dumbo", "fnv1a", "xxh64", "xxh3", "rapidhash", "siphash", "murmur3"] # Legacy / short SMHasher3 names -> the current registered name, applied when a -# measured map is loaded so an older dataset (recorded before a name was pinned) -# still joins onto _SMH_RESULTS_ROWS. The false-PASS bug came from the bare short -# name `FNV-1a`, which the binary does not register (the real name is `FNV-1a-64`; -# likewise `MurmurHash3` -> `MurmurHash3-128`). Confirm names with `SMHasher3 --list`. +# bundle's logs are re-parsed so an older dataset (recorded before a name was +# pinned) still joins onto the algorithm instances by `smhasher` name. The +# false-PASS bug came from the bare short name `FNV-1a`, which the binary does not +# register (the real name is `FNV-1a-64`; likewise `MurmurHash3` -> +# `MurmurHash3-128`). Confirm names with `SMHasher3 --list`. _SMH_NAME_ALIASES = { "FNV-1a": "FNV-1a-64", "MurmurHash3": "MurmurHash3-128", } -# Curated source of truth for the "## Quality: SMHasher3" -> "### Results" table, -# rendered into the README between the _SMH_BEGIN/_SMH_END markers by the -# `quality` command (verify with `quality --check`). Each row is -# (algo, bits, role, smhasher, verdict, score, failures): -# - smhasher is the SMHasher3 registration name; `quality --smhasher ` -# joins a fresh measurement onto the row by this name, so the verdict/score -# below act as the checked-in fallback (used when no bundle is passed, e.g. -# `--check` in CI) and measured data overrides them when supplied. -# - verdict/score are our measurements on the pinned SMHasher3 build; score is -# (passed, total) for a FAIL and None for a PASS. Totals differ per hash -# (e.g. siphash 186, fnv1a 186, most 188), so they are NOT derived here. -# - role and failures are HUMAN-authored. The Failures text is editorial: a few -# failing families are listed as "Family [indices]" (as SMHasher3's Summary -# "Failures:" block reports them); a hash that fails most of the battery is -# summarized in prose. Re-derive verdict/score/families from a fresh `smhasher` -# run (its JSON carries the parsed Summary), then edit the wording here. -_SMH_RESULTS_ROWS = [ - ("dumbo", 64, "`hash.h` (compact MUM)", "dumbo-64", "PASS", None, "none"), - ("fnv1a", 64, "`hash.h`", "FNV-1a-64", "FAIL", (7, 186), "nearly every family: Avalanche, BIC, Sparse, Cyclic, Permutation, Text, TwoBytes, Bitflip, PerlinNoise, and the complete Seed* cluster"), - ("mumbo", 64, "default (64/32/streaming)", "mumbo-64", "PASS", None, "none"), - ("rapidhash", 64, "extra (`hash_extra_cc`)", "rapidhash", "PASS", None, "none"), - ("siphash", 64, "`hash.h` (keyed PRF)", "SipHash-2-4", "PASS", None, "none"), - ("xxh3", 64, "extra (`hash_extra_cc`)", "XXH3-64", "FAIL", (166, 188), "BIC [3, 8, 11], Sparse [20/3], PerlinNoise [2], Bitflip [8], SeedZeroes [1280, 8448], SeedSparse [2, 3]"), - ("xxh64", 64, "extra (`hash_extra_cc`)", "XXH-64", "FAIL", (181, 188), "SeedBlockLen [15, 19, 21, 26, 29, 30], SeedBIC [8]"), - ("jumbo", 128, "default (128)", "jumbo-128", "PASS", None, "none"), - ("murmur3", 128, "`hash.h`", "MurmurHash3-128", "FAIL", (123, 188), "BIC, Zeroes, Permutation, and the complete Seed* cluster (11 families)"), - ("xxh3", 128, "extra (`hash_extra_cc`)", "XXH3-128", "FAIL", (162, 188), "BIC [3, 8, 15], Sparse [20/3], PerlinNoise [2], Bitflip [3, 4, 8], SeedZeroes [1280, 8448], SeedSparse [2, 3], SeedBlockLen [8, 12-16], SeedBlockOffset [0-5], SeedBIC [3, 8]"), -] +# The manual/editorial SOT for the README quality tables lives in this JSON (per +# instance: role, notice, seeded, streaming, starlark, ... - NOT measured data). +# The measured columns (verdict/score/failures) come from a data bundle. +_ALGORITHMS_JSON = os.path.join(os.path.dirname(os.path.abspath(__file__)), "hash_algorithms.json") + +# TEMPORARY stopgap: the committed bundles were recorded under the invalid short +# names `FNV-1a`/`MurmurHash3` (rejected by the pinned SMHasher3 build), so their +# logs are the "Invalid hash" stub and carry no real data for these two. Hardcode +# the known values until a bundle carries a real `FNV-1a-64`/`MurmurHash3-128` +# measurement, then DELETE the matching entry. `failures` is the editorial summary +# (both fail most of the battery, so listing every family would be noise). +_MISSING_MEASURED = { + "FNV-1a-64": { + "verdict": "FAIL", + "passed": 7, + "total": 186, + "failures": [ + "nearly every family: Avalanche, BIC, Sparse, Cyclic, Permutation, " + "Text, TwoBytes, Bitflip, PerlinNoise, and the complete Seed* cluster" + ], + }, + "MurmurHash3-128": { + "verdict": "FAIL", + "passed": 123, + "total": 188, + "failures": ["BIC, Zeroes, Permutation, and the complete Seed* cluster (11 families)"], + }, +} def _timestamp(): @@ -314,11 +315,11 @@ def _machine_label(ctx): _SMH_FAILBLOCK_RE = re.compile(r"^\s+(?P[A-Za-z][\w-]*)\s*:\s*\[(?P[^\]]*)\]\s*$") -def _smhasher_one(cmd_prefix, name, raw_dir, stamp): - """Run one SMHasher3 battery and parse its verdict + failing families.""" - print(f"$ {' '.join(cmd_prefix)} {name}", file=sys.stderr) - proc = subprocess.run([*cmd_prefix, name], capture_output=True, text=True, check=False) - text = proc.stdout + proc.stderr +def _parse_smhasher(text, returncode=0): + """Parse one SMHasher3 battery's captured output into a result entry (pure, no + I/O). Used both for a live run and for re-parsing a bundle's saved log with the + CURRENT parser - the committed bundles' stored `smhasher.json` predates parser + fixes (score-less), so the LOGS, re-parsed here, are the measured truth.""" verdict_match = _SMH_VERDICT_RE.search(text) invalid = re.search(r"Invalid hash '([^']*)' specified", text) # A completed battery ALWAYS ends with "Overall result: PASS|FAIL (p / n)". @@ -339,8 +340,8 @@ def _smhasher_one(cmd_prefix, name, raw_dir, stamp): verdict, passed, total = "ERROR", None, None if invalid: error = f"invalid SMHasher3 name {invalid.group(1)!r} - check `SMHasher3 --list`" - elif proc.returncode != 0: - error = f"SMHasher3 exited {proc.returncode} with no 'Overall result' line" + elif returncode != 0: + error = f"SMHasher3 exited {returncode} with no 'Overall result' line" else: error = "no 'Overall result' line - crashed or truncated output" # Failing families as "Family [indices]" (e.g. "BIC [3, 8, 11]"), read from @@ -362,15 +363,23 @@ def _smhasher_one(cmd_prefix, name, raw_dir, stamp): failures.append(f"{block_match.group('family')} [{block_match.group('idx').strip()}]") elif line.strip(): break # a non-blank, non-matching line ends the block (e.g. the "---" rule) - entry = { + return { "verdict": verdict, "score": (f"{passed} / {total}" if passed is not None and total is not None else None), "passed": passed, "total": total, "failures": failures, - "returncode": proc.returncode, + "returncode": returncode, "error": error, } + + +def _smhasher_one(cmd_prefix, name, raw_dir, stamp): + """Run one SMHasher3 battery, parse it, and save the raw log.""" + print(f"$ {' '.join(cmd_prefix)} {name}", file=sys.stderr) + proc = subprocess.run([*cmd_prefix, name], capture_output=True, text=True, check=False) + text = proc.stdout + proc.stderr + entry = _parse_smhasher(text, proc.returncode) if raw_dir: os.makedirs(raw_dir, exist_ok=True) log_path = os.path.join(raw_dir, f"{stamp}_smhasher_{name}.log") @@ -650,55 +659,73 @@ def _dump_canonical(results, path): _PERF_BEGIN = "" _PERF_END = "" -# --- quality: the SMHasher3 "Results" table, rendered from _SMH_RESULTS_ROWS --- +# --- quality: the README tables, from hash_algorithms.json (manual) x a bundle +# (measured). Two generated regions: the "Algorithm overview" and the SMHasher3 +# "Results" table. Manual columns come from the JSON; verdict/score/failures from +# the measurement (see render_overview_table / render_results_table). +_OVERVIEW_BEGIN = "" +_OVERVIEW_END = "" _SMH_BEGIN = "" _SMH_END = "" -def render_results_table(measured=None): - """Render the SMHasher3 "### Results" table from the curated _SMH_RESULTS_ROWS. +def _load_algorithms(): + """Load the manual/editorial per-instance data + the overview row order.""" + data = _load_json(_ALGORITHMS_JSON) + return data["algorithms"], data["overview_order"] + + +def _measured_entry(name, measured): + """The measured entry for `name`, aborting loudly if it is missing or did not + complete (ERROR) - a broken/absent measurement must never render a plausible + row. Callers cover the known gaps via `_fill_missing_measured` first.""" + entry = measured.get(name) + if entry is None: + raise SystemExit(f"{name}: no measured data - pass a bundle that measured it (or add a stopgap)") + if entry["verdict"] not in ("PASS", "FAIL"): + raise SystemExit( + f"{name}: measured verdict {entry['verdict']!r} is not PASS/FAIL - the battery did not " + f"complete ({entry.get('error') or 'bad name/crash'}); fix the measurement before regenerating" + ) + return entry + - When `measured` (a {smhasher-name: entry} map from a fresh `smhasher` run) is - given, each row's verdict/score is taken from it by name; the checked-in - fallback is used only for names the measurement does not cover (so a partial - run - e.g. just the mbo hashes - still renders a complete table). A measured - verdict that disagrees with the curated Failures wording is warned about, and - a battery that did not complete (ERROR) aborts loudly rather than shipping a - misleading row. Role/Bits/Failures are always the human-authored columns.""" +def render_results_table(algorithms, measured): + """Render the SMHasher3 "### Results" table: manual role/bits from the JSON, + verdict/score/failures from the measurement (joined by smhasher name).""" headers = ["Algorithm", "Bits", "Role in mbo/hash", "SMHasher3 result", "Failures"] aligns = ["l", "r", "l", "l", "l"] rows = [] - for algo, bits, role, name, verdict, score, failures in _SMH_RESULTS_ROWS: - meas = (measured or {}).get(name) - if meas: - verdict = meas.get("verdict", verdict) - if meas.get("passed") is not None and meas.get("total") is not None: - score = (meas["passed"], meas["total"]) - if verdict == "PASS" and failures != "none": - print(f"WARNING: {name}: measured PASS but curated Failures is {failures!r}", file=sys.stderr) - elif verdict == "FAIL" and failures == "none": - print(f"WARNING: {name}: measured FAIL but curated Failures says 'none'", file=sys.stderr) - if verdict == "PASS": - # A PASS renders as "PASS" (no number); a measured PASS legitimately - # carries passed==total, so only the editorial Failures must be "none". - if failures != "none": - raise SystemExit(f"{name}: a PASS row must have Failures 'none' (got {failures!r})") - if score and score[0] < score[1]: - print(f"WARNING: {name}: verdict PASS but score {score[0]}/{score[1]} shows failures", file=sys.stderr) - result = "PASS" - elif verdict == "FAIL": - if not score or score[0] >= score[1]: - raise SystemExit(f"{name}: a FAIL row needs a passed= total: + raise SystemExit(f"{algo['smhasher']}: FAIL without a valid passed/total score") + result = f"{passed}/{total}" + failures = ", ".join(entry.get("failures") or []) or "none" + rows.append([f"`{algo['algo']}`", str(algo["bits"]), algo["role"], result, failures]) + return _md_table(headers, rows, aligns) + + +def render_overview_table(algorithms, measured, overview_order): + """Render the "## Algorithm overview" table: manual columns from the JSON, the + `SMHasher3` PASS/FAIL derived from the measured verdict (so it cannot disagree + with the Results table). Rows follow the overview's own curated order.""" + headers = ["Algorithm", "Bits", "Available via", "Starlark", "NOTICE", "Seeded", "Streaming", "SMHasher3"] + aligns = ["l", "r", "l", "l", "l", "l", "l", "l"] + by_name = {algo["smhasher"]: algo for algo in algorithms} + rows = [] + for name in overview_order: + algo = by_name[name] + entry = _measured_entry(name, measured) + rows.append([ + f"`{algo['algo']}`", str(algo["bits"]), algo["available_via"], algo["starlark"], + algo["notice"], algo["seeded"], algo["streaming"], entry["verdict"], + ]) return _md_table(headers, rows, aligns) @@ -711,25 +738,49 @@ def _extract_bundle(path, dest): tar.extractall(dest) # noqa: S202 - our own bundle, older Python -def _load_smhasher(path): - """Load a measured SMHasher3 map {name: {verdict, passed, total, ...}} from - either a `smhasher` run's JSON or a data bundle .tgz that packs one. Legacy / - short names are normalized to their registered form (see _SMH_NAME_ALIASES) so - an older dataset still joins onto the Results rows.""" - if path.endswith((".tgz", ".tar.gz")): - with tempfile.TemporaryDirectory() as tmp: - _extract_bundle(path, tmp) - members = [f for f in os.listdir(tmp) if "smhasher" in f and f.endswith(".json")] - if not members: - raise SystemExit(f"no *smhasher*.json inside {path}") - data = _load_json(os.path.join(tmp, sorted(members)[-1])) - else: - data = _load_json(path) - smh = dict(data.get("smhasher", data)) - for alias, canonical in _SMH_NAME_ALIASES.items(): - if alias in smh and canonical not in smh: # do not clobber a real entry - smh[canonical] = smh.pop(alias) - return smh +_SMH_LOG_RE = re.compile(r"^(\d{8}_\d{6})_smhasher_(.+)\.log(\.gz)?$") + + +def _measured_from_bundle(path): + """Re-parse a bundle's per-algorithm SMHasher3 logs into a measured map + {registered-name: entry} with the CURRENT parser. The bundle's stored + smhasher.json is ignored (it predates parser fixes, so it is score-less); the + LOGS are the measured truth. The newest log per name wins, and legacy / short + log names are normalized to their registered form (see _SMH_NAME_ALIASES).""" + latest = {} # name -> (stamp, text) + with tempfile.TemporaryDirectory() as tmp: + _extract_bundle(path, tmp) + for fname in sorted(os.listdir(tmp)): + match = _SMH_LOG_RE.match(fname) + if not match: + continue + stamp, name, gz = match.group(1), match.group(2), match.group(3) + opener = gzip.open if gz else open + with opener(os.path.join(tmp, fname), "rt") as handle: + text = handle.read() + if name not in latest or stamp >= latest[name][0]: + latest[name] = (stamp, text) + measured = {} + for name, (_stamp, text) in latest.items(): + measured[_SMH_NAME_ALIASES.get(name, name)] = _parse_smhasher(text) + return measured + + +def _fill_missing_measured(measured): + """Overlay the hardcoded stopgap (`_MISSING_MEASURED`) for any name the bundle + lacks valid data for (absent, or an ERROR from a bad-name/crashed run). Warns + on every substitution so the stopgap is never silent. Returns a new map.""" + out = dict(measured) + for name, stub in _MISSING_MEASURED.items(): + current = out.get(name) + if current is None or current.get("verdict") == "ERROR": + print( + f"WARNING: {name}: no valid measurement in the bundle; using the hardcoded stopgap " + f"(delete _MISSING_MEASURED[{name!r}] once a bundle carries real data).", + file=sys.stderr, + ) + out[name] = {**stub, "score": f"{stub['passed']} / {stub['total']}", "returncode": 0, "error": None} + return out def _render_charts(full, stem, charts_dir, subtitle): @@ -818,10 +869,14 @@ def main(argv): p_verify.add_argument("--readme", default="mbo/hash/README.md") p_verify.add_argument("--charts-dir", default="mbo/hash/measurements/charts") - p_quality = sub.add_parser("quality", help="render the SMHasher3 Results table into the README from the curated rows (+ optional measured bundle)") + p_quality = sub.add_parser("quality", help="render the Algorithm-overview + SMHasher3 Results tables into the README from hash_algorithms.json (manual) and a measured bundle") p_quality.add_argument("--readme", default="mbo/hash/README.md") - p_quality.add_argument("--smhasher", help="measured smhasher JSON or data bundle .tgz to source verdict/score from (default: the checked-in curated values)") - p_quality.add_argument("--check", action="store_true", help="verify the README table matches instead of writing (exit 1 on drift)") + p_quality.add_argument("--bundle", required=True, help="data bundle .tgz whose per-algorithm SMHasher3 logs are re-parsed for verdict/score/failures") + p_quality.add_argument("--check", action="store_true", help="verify the README tables match instead of writing (exit 1 on drift)") + + p_consistency = sub.add_parser("consistency", help="verify all data bundles from the same source SHA report identical SMHasher3 measurements (quality is machine-independent)") + p_consistency.add_argument("--bundles", nargs="+", help="bundles to compare (default: all data/*.tgz)") + p_consistency.add_argument("--data-dir", default="mbo/hash/measurements/data") args = parser.parse_args(argv) # One stamp per invocation, so all files a run writes share it. Every @@ -975,24 +1030,76 @@ def main(argv): return 0 if args.command == "quality": - measured = _load_smhasher(args.smhasher) if args.smhasher else None - region = "\n".join([_SMH_BEGIN, "", render_results_table(measured), "", _SMH_END]) + algorithms, overview_order = _load_algorithms() + measured = _fill_missing_measured(_measured_from_bundle(args.bundle)) text = open(args.readme).read() - if _SMH_BEGIN not in text or _SMH_END not in text: - raise SystemExit(f"markers not found in {args.readme}; add a {_SMH_BEGIN} ... {_SMH_END} region") - new = text[: text.index(_SMH_BEGIN)] + region + text[text.index(_SMH_END) + len(_SMH_END) :] + regions = [ + (_OVERVIEW_BEGIN, _OVERVIEW_END, render_overview_table(algorithms, measured, overview_order)), + (_SMH_BEGIN, _SMH_END, render_results_table(algorithms, measured)), + ] + new = text + for begin, end, body in regions: + if begin not in new or end not in new: + raise SystemExit(f"markers not found in {args.readme}; add a {begin} ... {end} region") + block = "\n".join([begin, "", body, "", end]) + new = new[: new.index(begin)] + block + new[new.index(end) + len(end) :] if args.check: if new != text: - print(f"VERIFY FAILED: the SMHasher3 Results table in {args.readme} is stale; run `quality`", file=sys.stderr) + print(f"VERIFY FAILED: the generated tables in {args.readme} are stale; run `quality`", file=sys.stderr) return 1 - print("VERIFY OK: the SMHasher3 Results table matches the curated data", file=sys.stderr) + print("VERIFY OK: the generated overview + Results tables match hash_algorithms.json + the bundle", file=sys.stderr) return 0 if new != text: with open(args.readme, "w") as handle: handle.write(new) - print(f"wrote the SMHasher3 Results table into {args.readme}", file=sys.stderr) + print(f"wrote the overview + Results tables into {args.readme}", file=sys.stderr) else: - print(f"the SMHasher3 Results table is already current in {args.readme}", file=sys.stderr) + print(f"the generated tables are already current in {args.readme}", file=sys.stderr) + return 0 + + if args.command == "consistency": + bundles = args.bundles or sorted( + os.path.join(args.data_dir, f) for f in os.listdir(args.data_dir) if f.endswith(".tgz") + ) + if not bundles: + raise SystemExit(f"no bundles found (looked in {args.data_dir})") + # Group by the source git SHA embedded in the bundle filename + # (_c___.tgz). SMHasher3 verdicts are a + # property of the algorithm, not the machine, so bundles at the same SHA + # must report identical measurements - a difference means a broken run. + by_sha = {} + for bundle in bundles: + match = re.search(r"_([0-9a-fA-F]{8})_\d{8}_\d{6}\.tgz$", os.path.basename(bundle)) + sha = match.group(1) if match else os.path.basename(bundle) + by_sha.setdefault(sha, []).append((bundle, _measured_from_bundle(bundle))) + + def _key(entry): + if entry is None: + return None + return (entry["verdict"], entry.get("passed"), entry.get("total"), tuple(entry.get("failures") or [])) + + problems = [] + for sha, group in sorted(by_sha.items()): + if len(group) < 2: + print(f"consistency: SHA {sha}: only 1 bundle, nothing to cross-check", file=sys.stderr) + continue + ref_bundle, ref = group[0] + names = sorted(set().union(*(set(meas) for _bundle, meas in group))) + for bundle, meas in group[1:]: + for name in names: + if _key(ref.get(name)) != _key(meas.get(name)): + problems.append( + f"SHA {sha}: {name} differs between {os.path.basename(ref_bundle)} " + f"and {os.path.basename(bundle)}: {_key(ref.get(name))} vs {_key(meas.get(name))}" + ) + for problem in problems: + print(f"VERIFY FAILED: {problem}", file=sys.stderr) + if problems: + return 1 + print( + f"VERIFY OK: {len(bundles)} bundle(s) in {len(by_sha)} source-SHA group(s) agree on all SMHasher3 measurements", + file=sys.stderr, + ) return 0 return 1 diff --git a/mbo/hash/measurements/quality_sh_test.sh b/mbo/hash/measurements/quality_sh_test.sh index c1bc25b9..14eeb642 100755 --- a/mbo/hash/measurements/quality_sh_test.sh +++ b/mbo/hash/measurements/quality_sh_test.sh @@ -15,11 +15,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Guards the generated SMHasher3 "Results" table in mbo/hash/README.md: it must -# stay in sync with the curated source of truth in hash_benchmark_report.py -# (`quality --check`), the drift guard must actually fail on a changed table (so -# it is not a no-op), and regenerating must be idempotent (the committed table is -# already in canonical form). +# Guards the generated SMHasher3 quality tables in mbo/hash/README.md (the +# Algorithm-overview and the Results table), rendered by +# `hash_benchmark_report.py quality` from hash_algorithms.json (manual data) x a +# measured data bundle. Asserts: the committed tables are in sync (`quality +# --check`); the drift guard actually fails on a changed table; regenerating is +# idempotent; and all committed bundles agree on their measurements +# (`consistency`, since SMHasher3 verdicts are machine-independent). # shellcheck disable=SC2317 # Functions are called by the bashtest runner. @@ -28,28 +30,36 @@ set -euo pipefail # shellcheck disable=SC1090,SC1091,SC2154 source "${helly25_bashtest}" -REPORT="${TEST_SRCDIR}/${TEST_WORKSPACE}/mbo/hash/measurements/hash_benchmark_report.py" +MEAS="${TEST_SRCDIR}/${TEST_WORKSPACE}/mbo/hash/measurements" +REPORT="${MEAS}/hash_benchmark_report.py" README="${TEST_SRCDIR}/${TEST_WORKSPACE}/mbo/hash/README.md" -declare -r REPORT README +declare -r MEAS REPORT README [[ -f ${REPORT} ]] || die "hash_benchmark_report.py not found." [[ -f ${README} ]] || die "README.md not found." +# The data bundles are Git-LFS: CI materializes them (see test.yml) and a local +# run needs `git lfs pull`. An unmaterialized pointer is tiny, so guard on size. +BUNDLES=("${MEAS}"/data/*.tgz) +[[ -f ${BUNDLES[0]} ]] || die "no data bundles found (run: git lfs pull)." +[[ $(wc -c <"${BUNDLES[0]}") -gt 1024 ]] || die "bundle is a Git-LFS pointer; run: git lfs pull." +declare -r BUNDLE="${BUNDLES[0]}" + PYTHON="${PYTHON:-python3}" command -v "${PYTHON}" >/dev/null || die "python3 not found on PATH." declare -r PYTHON function quality() { - "${PYTHON}" "${REPORT}" quality "$@" + "${PYTHON}" "${REPORT}" quality --bundle "${BUNDLE}" "$@" } -# The committed table must match the curated data - this is the drift guard. -function test::committed_table_is_in_sync() { +# The committed tables must match hash_algorithms.json x the bundle measurements. +function test::committed_tables_are_in_sync() { quality --check --readme "${README}" \ - || die "README SMHasher3 Results table is stale; run: hash_benchmark_report.py quality" + || die "README quality tables are stale; run: hash_benchmark_report.py quality --bundle " } -# The guard must actually catch drift: a table with a flipped score must fail. +# The guard must actually catch drift: a flipped score must fail --check. function test::check_detects_drift() { local drift="${TEST_TMPDIR}/drift_README.md" sed 's#| 7/186 #| 9/186 #' "${README}" >"${drift}" @@ -63,8 +73,14 @@ function test::check_detects_drift() { function test::regenerate_is_idempotent() { local copy="${TEST_TMPDIR}/regen_README.md" cp "${README}" "${copy}" - quality --readme "${copy}" >/dev/null || die "quality failed to write the table." + quality --readme "${copy}" >/dev/null 2>&1 || die "quality failed to write the tables." cmp -s "${README}" "${copy}" || die "regenerating changed the README - it was not in canonical form." } +# All committed bundles at the same source SHA must report identical measurements. +function test::bundles_agree_per_source_sha() { + "${PYTHON}" "${REPORT}" consistency --bundles "${BUNDLES[@]}" \ + || die "committed bundles disagree on SMHasher3 measurements." +} + test_runner