From 9099d2fbe1c204023a17ea5dde4f3b745df2d886 Mon Sep 17 00:00:00 2001 From: ghangz <152254226+ghangz@users.noreply.github.com> Date: Wed, 1 Jul 2026 11:41:46 +0800 Subject: [PATCH 1/2] Add FlashMLA benchmark performance gate --- tools/benchmark_perf_gate.py | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 tools/benchmark_perf_gate.py diff --git a/tools/benchmark_perf_gate.py b/tools/benchmark_perf_gate.py new file mode 100644 index 00000000..431946f4 --- /dev/null +++ b/tools/benchmark_perf_gate.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Compare baseline and current performance JSON and fail on regressions.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +METRIC = 'throughput' +TOLERANCE = 0.05 + + +def load(path: Path) -> dict[str, float]: + data = json.loads(path.read_text(encoding="utf-8")) + if isinstance(data, list): + return {str(item["name"]): float(item[METRIC]) for item in data} + return {str(k): float(v[METRIC] if isinstance(v, dict) else v) for k, v in data.items()} + + +def compare(baseline: dict[str, float], current: dict[str, float]) -> dict[str, object]: + rows: list[dict[str, object]] = [] + failed = False + for name, old in sorted(baseline.items()): + if name not in current: + rows.append({"name": name, "status": "missing-current"}) + failed = True + continue + new = current[name] + ratio = (new - old) / old if old else 0.0 + status = "regression" if ratio < -TOLERANCE else "ok" + failed = failed or status != "ok" + rows.append({"name": name, "baseline": old, "current": new, "delta_ratio": ratio, "status": status}) + return {"ok": not failed, "metric": METRIC, "rows": rows} + + +def self_test() -> None: + data = compare({"case": 100.0}, {"case": 99.0}) + assert data["ok"] + print(json.dumps({"ok": True, "rows": len(data["rows"])}, ensure_ascii=False)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("baseline") + parser.add_argument("current") + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + if args.self_test: + self_test() + return 0 + result = compare(load(Path(args.baseline)), load(Path(args.current))) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From d4bec1fa8064e28a09ae891b59651a797638d7d9 Mon Sep 17 00:00:00 2001 From: ghangz <152254226+ghangz@users.noreply.github.com> Date: Wed, 1 Jul 2026 15:36:45 +0800 Subject: [PATCH 2/2] Address PR review feedback - Add FlashMLA benchmark performance gate - Strengthen benchmark perf gate self test --- tools/benchmark_perf_gate.py | 38 ++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/tools/benchmark_perf_gate.py b/tools/benchmark_perf_gate.py index 431946f4..ee5c0526 100644 --- a/tools/benchmark_perf_gate.py +++ b/tools/benchmark_perf_gate.py @@ -7,7 +7,7 @@ import json from pathlib import Path -METRIC = 'throughput' +METRIC = "throughput" TOLERANCE = 0.05 @@ -15,7 +15,10 @@ def load(path: Path) -> dict[str, float]: data = json.loads(path.read_text(encoding="utf-8")) if isinstance(data, list): return {str(item["name"]): float(item[METRIC]) for item in data} - return {str(k): float(v[METRIC] if isinstance(v, dict) else v) for k, v in data.items()} + return { + str(k): float(v[METRIC] if isinstance(v, dict) else v) + for k, v in data.items() + } def compare(baseline: dict[str, float], current: dict[str, float]) -> dict[str, object]: @@ -30,25 +33,44 @@ def compare(baseline: dict[str, float], current: dict[str, float]) -> dict[str, ratio = (new - old) / old if old else 0.0 status = "regression" if ratio < -TOLERANCE else "ok" failed = failed or status != "ok" - rows.append({"name": name, "baseline": old, "current": new, "delta_ratio": ratio, "status": status}) + rows.append( + { + "name": name, + "baseline": old, + "current": new, + "delta_ratio": ratio, + "status": status, + } + ) return {"ok": not failed, "metric": METRIC, "rows": rows} def self_test() -> None: - data = compare({"case": 100.0}, {"case": 99.0}) - assert data["ok"] - print(json.dumps({"ok": True, "rows": len(data["rows"])}, ensure_ascii=False)) + data_ok = compare({"case": 100.0}, {"case": 99.0}) + data_regression = compare({"case": 100.0}, {"case": 90.0}) + data_missing = compare({"case": 100.0}, {}) + if not data_ok["ok"] or data_regression["ok"] or data_missing["ok"]: + raise RuntimeError( + { + "ok_case": data_ok, + "regression_case": data_regression, + "missing_case": data_missing, + } + ) + print(json.dumps({"ok": True, "rows": len(data_ok["rows"])}, ensure_ascii=False)) def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("baseline") - parser.add_argument("current") + parser.add_argument("baseline", nargs="?") + parser.add_argument("current", nargs="?") parser.add_argument("--self-test", action="store_true") args = parser.parse_args() if args.self_test: self_test() return 0 + if not args.baseline or not args.current: + parser.error("baseline and current are required unless --self-test is used") result = compare(load(Path(args.baseline)), load(Path(args.current))) print(json.dumps(result, ensure_ascii=False, indent=2)) return 0 if result["ok"] else 1