From 377bfa569fe2e50a56da4844340c151fd6ec6de3 Mon Sep 17 00:00:00 2001 From: mlik-sudo Date: Mon, 15 Dec 2025 10:47:31 -0500 Subject: [PATCH] chore: bootstrap proof pack scripts and safer dispatch --- benchmarks/compare.sh | 80 +++++++++++++++++++++++++++++++++++++ benchmarks/results/.gitkeep | 0 hub/tools/dispatch.py | 56 ++++++++++++++------------ scripts/proof.sh | 7 ++++ scripts/run-tests.sh | 12 ++++++ 5 files changed, 130 insertions(+), 25 deletions(-) create mode 100644 benchmarks/compare.sh create mode 100644 benchmarks/results/.gitkeep create mode 100644 scripts/proof.sh create mode 100644 scripts/run-tests.sh diff --git a/benchmarks/compare.sh b/benchmarks/compare.sh new file mode 100644 index 0000000..1d22a40 --- /dev/null +++ b/benchmarks/compare.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + ./benchmarks/compare.sh + +Both files must be JSONL produced by the harness, with numeric `duration_ms` fields. +Example: + ./benchmarks/compare.sh project-space/benchmarks/results/smoke_results.jsonl project-space/benchmarks/results/smoke_results.jsonl +EOF +} + +if [[ ${1:-} == "-h" || ${1:-} == "--help" ]]; then + usage + exit 0 +fi + +baseline=${1:-} +candidate=${2:-} + +if [[ -z ${baseline} || -z ${candidate} ]]; then + usage + exit 2 +fi + +python3 - <<'PY' +import json +import statistics +import sys +from pathlib import Path + +baseline = Path(sys.argv[1]) +candidate = Path(sys.argv[2]) + +for p in (baseline, candidate): + if not p.exists(): + raise SystemExit(f"missing file: {p}") + +def read_durations(path: Path) -> list[int]: + out: list[int] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + val = row.get("duration_ms") + if isinstance(val, (int, float)): + out.append(int(val)) + return out + +def p95(values: list[int]) -> int: + if not values: + return 0 + values = sorted(values) + k = max(0, int(0.95 * (len(values) - 1))) + return values[k] + +b = read_durations(baseline) +c = read_durations(candidate) + +if not b or not c: + raise SystemExit("no durations found in one or both files") + +b_avg = int(statistics.mean(b)) +c_avg = int(statistics.mean(c)) + +b_p95 = p95(b) +c_p95 = p95(c) + +def pct(a: int, b: int) -> float: + if a == 0: + return 0.0 + return (b - a) * 100.0 / a + +print(f"baseline: n={len(b)} avg_ms={b_avg} p95_ms={b_p95}") +print(f"candidate: n={len(c)} avg_ms={c_avg} p95_ms={c_p95}") +print(f"delta: avg={pct(b_avg, c_avg):+.2f}% p95={pct(b_p95, c_p95):+.2f}%") +PY +"${baseline}" "${candidate}" diff --git a/benchmarks/results/.gitkeep b/benchmarks/results/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/hub/tools/dispatch.py b/hub/tools/dispatch.py index bf5d31b..dd9e9c0 100644 --- a/hub/tools/dispatch.py +++ b/hub/tools/dispatch.py @@ -1,15 +1,17 @@ import argparse import json -import subprocess import pathlib +import subprocess import os # --- Argument Parsing --- -parser = argparse.ArgumentParser(description="Dispatch tasks to CLI agents based on a task.json file.") +parser = argparse.ArgumentParser( + description="Dispatch tasks to CLI agents based on a task.json file." +) parser.add_argument( "--request", required=True, - help="Path to the request task.json file, relative to the repository root." + help="Path to the request task.json file, relative to the repository root.", ) args = parser.parse_args() @@ -23,63 +25,67 @@ task = json.loads(req_path.read_text(encoding="utf-8")) except (FileNotFoundError, json.JSONDecodeError) as e: print(f"Error: Could not read or parse the task file at '{req_path}'. Details: {e}") - exit(1) + raise SystemExit(1) # --- Output Directory Setup --- out_dir = pathlib.Path("responses") / task.get("id", "unknown-task") out_dir.mkdir(parents=True, exist_ok=True) print(f"Output directory created at: {out_dir}") -# --- CLI Runner Function --- -def run_cli(cmd, out_file): + +def run_cli(cmd: list[str], out_file: pathlib.Path) -> None: """Executes a command and writes its stdout and stderr to files.""" + print(f"Running command: {' '.join(cmd)}") + try: - # Using shell=True for simplicity, but ensure commands are constructed safely. - res = subprocess.run(" ".join(cmd), capture_output=True, text=True, shell=True, check=True) - out_file.write_text(res.stdout) + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + out_file.write_text(res.stdout, encoding="utf-8") if res.stderr: - (out_file.parent / (out_file.name + ".stderr")).write_text(res.stderr) + (out_file.parent / (out_file.name + ".stderr")).write_text( + res.stderr, encoding="utf-8" + ) print(f"Successfully wrote output to {out_file}") except subprocess.CalledProcessError as e: error_message = ( - f"Command failed with exit code {e.returncode}\\n" - f"--- STDOUT ---\\n{e.stdout}\\n" - f"--- STDERR ---\\n{e.stderr}" + f"Command failed with exit code {e.returncode}\n" + f"--- STDOUT ---\n{e.stdout}\n" + f"--- STDERR ---\n{e.stderr}" ) - out_file.write_text(error_message) + out_file.write_text(error_message, encoding="utf-8") print(error_message) except Exception as e: error_message = f"An unexpected error occurred: {e}" - out_file.write_text(error_message) + out_file.write_text(error_message, encoding="utf-8") print(error_message) + # --- Placeholder CLI Calls --- # In a real implementation, you would build these commands dynamically based on the task drivers. task_id = task.get("id", "unknown-task") -print("\\n--- Dispatching to Gemini ---") +print("\n--- Dispatching to Gemini ---") gemini_out = out_dir / "gemini-cli" gemini_out.mkdir(exist_ok=True) run_cli( - ["echo", f"'Gemini CLI output for task {task_id}'"], - gemini_out / "output.txt" + ["echo", f"Gemini CLI output for task {task_id}"], + gemini_out / "output.txt", ) -print("\\n--- Dispatching to Codex ---") +print("\n--- Dispatching to Codex ---") codex_out = out_dir / "codex-cli" codex_out.mkdir(exist_ok=True) run_cli( - ["echo", f"'Codex CLI output for task {task_id}'"], - codex_out / "output.txt" + ["echo", f"Codex CLI output for task {task_id}"], + codex_out / "output.txt", ) -print("\\n--- Dispatching to Claude ---") +print("\n--- Dispatching to Claude ---") claude_out = out_dir / "claude-code" claude_out.mkdir(exist_ok=True) run_cli( - ["echo", f"'Claude Code CLI output for task {task_id}'"], - claude_out / "output.txt" + ["echo", f"Claude Code CLI output for task {task_id}"], + claude_out / "output.txt", ) -print("\\nDispatch script finished.") +print("\nDispatch script finished.") diff --git a/scripts/proof.sh b/scripts/proof.sh new file mode 100644 index 0000000..a819119 --- /dev/null +++ b/scripts/proof.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Convenience wrapper for the Proof Pack. +# Keeps README instructions stable while routing to Makefile. + +make proof diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh new file mode 100644 index 0000000..61dcc2f --- /dev/null +++ b/scripts/run-tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Runs repository tests in a predictable way. +# Intended to be called by `make test-all`. + +if ! command -v pytest >/dev/null 2>&1; then + echo "pytest not found. Install dev deps: pip install -r requirements-dev.txt" >&2 + exit 1 +fi + +pytest -q tests/