From 232b3307fb6e695ca22b35206a9a85e3ac94c993 Mon Sep 17 00:00:00 2001 From: kelkalot Date: Sat, 4 Jul 2026 20:38:00 +0200 Subject: [PATCH] Make --tools optional when using traces/mined Allow probe/derive runs to omit --tools when traces or mined inputs provide probes. Add a reusable _TOOLS_OPTION and _require_probe_source check, handle trace-only runs by using an empty tools fingerprint, and error when no probe sources or when the derived battery is empty. Bump package version to v0.3.1, update README and VALIDATION notes to reflect the behavior change, and add tests covering trace-only, mined-only, missing-source, and empty-battery cases. --- README.md | 4 ++++ VALIDATION.md | 5 +++-- probelock/__init__.py | 2 +- probelock/cli.py | 40 ++++++++++++++++++++++++++++----- pyproject.toml | 2 +- tests/test_cli.py | 52 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 95 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f4d990d..2290163 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,10 @@ uv run probelock probe --tools tools.json --traces traces.json \ --endpoint http://localhost:11434/v1 --model llama3.1:8b -o candidate.lock ``` +`--tools` is optional here: traced probes replay their own embedded tool definitions, +so a trace-only run (`probe --traces traces.json ...`) needs no schema file. The same +holds for `--mined` below. Provide `--tools` when you also want the synthetic battery. + A traces file is a small, stable JSON schema probelock defines itself — **not** raw OpenTelemetry — because OTel's own span attribute layout is not stable across libraries or versions (litellm has already changed where it puts request/response attributes once, and diff --git a/VALIDATION.md b/VALIDATION.md index 32ca5ac..b4ba4e3 100644 --- a/VALIDATION.md +++ b/VALIDATION.md @@ -264,8 +264,9 @@ force the code path this test targets. - `llama-server -fa` requires an explicit value (`--flash-attn auto`) on recent builds; the bare flag misparses a following `-hf` argument as its value. -- `probelock probe --traces` still requires `--tools`; the traced probes carry - their own embedded tool definitions regardless of what's in the schema file. +- `probelock probe --traces` required `--tools` at the time of this validation, + even though traced probes carry their own embedded tool definitions. Fixed in + v0.3.1: `--tools` is now optional whenever `--traces` or `--mined` supply probes. - Running two `llama-server` instances concurrently on one GPU crashed one of them (Metal contention) mid-probe during testing. Don't run concurrent GPU inference workloads on a single-GPU host. diff --git a/probelock/__init__.py b/probelock/__init__.py index a34bb6c..bfc6dab 100644 --- a/probelock/__init__.py +++ b/probelock/__init__.py @@ -5,4 +5,4 @@ model version, quantization, or runtime. """ -__version__ = "0.3.0" +__version__ = "0.3.1" diff --git a/probelock/cli.py b/probelock/cli.py index de1f38b..095527e 100644 --- a/probelock/cli.py +++ b/probelock/cli.py @@ -160,14 +160,27 @@ def _mined_battery(path: Path): return accepted, [to_probe(p) for p in accepted] +_TOOLS_OPTION = typer.Option( + None, "--tools", "-t", help="OpenAI-style tools JSON file (optional when --traces " + "or --mined supply probes — those carry their own embedded tool definitions)." +) + + +def _require_probe_source(tools, traces, mined) -> None: + if tools is None and traces is None and mined is None: + _err("Provide at least one probe source: --tools, --traces, or --mined.") + raise typer.Exit(2) + + @app.command() def derive( - tools: Path = typer.Option(..., "--tools", "-t", help="OpenAI-style tools JSON file."), + tools: Optional[Path] = _TOOLS_OPTION, traces: Optional[Path] = _TRACES_OPTION, mined: Optional[Path] = _MINED_OPTION, ) -> None: """Show the probe battery that would be generated from a toolset (transparency).""" - probes = derive_probes(_load_tools_or_exit(tools)) + _require_probe_source(tools, traces, mined) + probes = derive_probes(_load_tools_or_exit(tools)) if tools is not None else [] if traces is not None: probes = probes + derive_traced_probes(_load_traces_or_exit(traces)) if mined is not None: @@ -183,7 +196,7 @@ def derive( @app.command() def probe( - tools: Path = typer.Option(..., "--tools", "-t", help="OpenAI-style tools JSON file."), + tools: Optional[Path] = _TOOLS_OPTION, simulate: Optional[Path] = typer.Option( None, "--simulate", "-s", help="Run against a deterministic profile (no model)." ), @@ -215,9 +228,18 @@ def probe( ), ) -> None: """Run the probe battery and produce a capability lockfile.""" - tool_list = _load_tools_or_exit(tools) - probes = derive_probes(tool_list) - fingerprint = tools_fingerprint(tool_list) + _require_probe_source(tools, traces, mined) + if tools is not None: + tool_list = _load_tools_or_exit(tools) + probes = derive_probes(tool_list) + fingerprint = tools_fingerprint(tool_list) + else: + # Trace-only run: traced/mined probes replay their own embedded tool + # definitions, so there is no schema battery and no toolset to fingerprint. + # The empty fingerprint is stable, so two trace-only lockfiles diff cleanly, + # while a trace-only vs schema-derived pair still flags tools_changed. + probes = [] + fingerprint = "" traces_fp = None if traces is not None: @@ -264,6 +286,12 @@ def probe( elif fps: traces_fp = fps[0] + if not probes: + # Possible without --tools: a traces file that derives nothing, or a mined + # file with no accepted probes. A zero-probe lockfile can gate nothing. + _err("The probe battery is empty — nothing derived from the given sources.") + raise typer.Exit(2) + try: if simulate is not None: client = SimulatedClient(_load_json_or_exit(simulate, "simulate profile")) diff --git a/pyproject.toml b/pyproject.toml index b583a7a..2156b81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "probelock" -version = "0.3.0" +version = "0.3.1" description = "A capability lockfile for local models. Catch silent regressions when you swap a model, quant, or runtime." readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_cli.py b/tests/test_cli.py index 41d626b..4a2711c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -588,3 +588,55 @@ def test_ingest_accepts_multiple_logs_as_one_corpus(tmp_path): assert cats.count("tool_selection") == 2 # incl. the continuation across files assert cats.count("schema_validity") == 5 assert cats.count("no_tool") == 1 + + +# --- --tools optional when traces/mined supply probes (validation rough edge) ------- + + +def test_probe_traces_only_needs_no_tools_file(tmp_path): + lock = tmp_path / "traced.lock" + result = runner.invoke(app, [ + "probe", "--traces", str(_SAMPLE_TRACES), "--simulate", str(_SIM_Q8), + "-o", str(lock)]) + assert result.exit_code == 0, result.output + data = json.loads(lock.read_text()) + assert data["tools_fingerprint"] == "" # no schema battery ran + assert data["traces_fingerprint"] + assert "tool_selection" in data["capabilities"] # traced probes replayed + assert data["n_probes"] > 0 + + +def test_derive_traces_only_needs_no_tools_file(): + result = runner.invoke(app, ["derive", "--traces", str(_SAMPLE_TRACES)]) + assert result.exit_code == 0, result.output + assert "::traced::" in result.output + + +def test_probe_mined_only_needs_no_tools_file(tmp_path): + mined = _ingest(tmp_path) + runner.invoke(app, ["traces", "review", str(mined), "--auto-accept", "schema_validity"]) + lock = tmp_path / "mined-only.lock" + result = runner.invoke(app, [ + "probe", "--mined", str(mined), "--simulate", str(_SIM_Q8), + "--allow-sensitive", "-o", str(lock)]) + assert result.exit_code == 0, result.output + caps = json.loads(lock.read_text())["capabilities"] + assert list(caps) == ["traced_schema_validity"] + + +def test_probe_without_any_source_exits_2(): + result = runner.invoke(app, ["probe", "--simulate", str(_SIM_Q8)]) + assert result.exit_code == 2, result.output + assert "--tools, --traces, or --mined" in result.output + result = runner.invoke(app, ["derive"]) + assert result.exit_code == 2, result.output + + +def test_probe_with_empty_battery_exits_2(tmp_path): + # a mined file with nothing accepted and no --tools: the battery is empty and a + # zero-probe lockfile can gate nothing + mined = _ingest(tmp_path) # everything still pending + result = runner.invoke(app, [ + "probe", "--mined", str(mined), "--simulate", str(_SIM_Q8)]) + assert result.exit_code == 2, result.output + assert "empty" in result.output