diff --git a/README.md b/README.md index f4d990d..2290163 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,10 @@ uv run probelock probe --tools tools.json --traces traces.json \ --endpoint http://localhost:11434/v1 --model llama3.1:8b -o candidate.lock ``` +`--tools` is optional here: traced probes replay their own embedded tool definitions, +so a trace-only run (`probe --traces traces.json ...`) needs no schema file. The same +holds for `--mined` below. Provide `--tools` when you also want the synthetic battery. + A traces file is a small, stable JSON schema probelock defines itself — **not** raw OpenTelemetry — because OTel's own span attribute layout is not stable across libraries or versions (litellm has already changed where it puts request/response attributes once, and diff --git a/VALIDATION.md b/VALIDATION.md index 32ca5ac..b4ba4e3 100644 --- a/VALIDATION.md +++ b/VALIDATION.md @@ -264,8 +264,9 @@ force the code path this test targets. - `llama-server -fa` requires an explicit value (`--flash-attn auto`) on recent builds; the bare flag misparses a following `-hf` argument as its value. -- `probelock probe --traces` still requires `--tools`; the traced probes carry - their own embedded tool definitions regardless of what's in the schema file. +- `probelock probe --traces` required `--tools` at the time of this validation, + even though traced probes carry their own embedded tool definitions. Fixed in + v0.3.1: `--tools` is now optional whenever `--traces` or `--mined` supply probes. - Running two `llama-server` instances concurrently on one GPU crashed one of them (Metal contention) mid-probe during testing. Don't run concurrent GPU inference workloads on a single-GPU host. diff --git a/probelock/__init__.py b/probelock/__init__.py index a34bb6c..bfc6dab 100644 --- a/probelock/__init__.py +++ b/probelock/__init__.py @@ -5,4 +5,4 @@ model version, quantization, or runtime. """ -__version__ = "0.3.0" +__version__ = "0.3.1" diff --git a/probelock/cli.py b/probelock/cli.py index de1f38b..095527e 100644 --- a/probelock/cli.py +++ b/probelock/cli.py @@ -160,14 +160,27 @@ def _mined_battery(path: Path): return accepted, [to_probe(p) for p in accepted] +_TOOLS_OPTION = typer.Option( + None, "--tools", "-t", help="OpenAI-style tools JSON file (optional when --traces " + "or --mined supply probes — those carry their own embedded tool definitions)." +) + + +def _require_probe_source(tools, traces, mined) -> None: + if tools is None and traces is None and mined is None: + _err("Provide at least one probe source: --tools, --traces, or --mined.") + raise typer.Exit(2) + + @app.command() def derive( - tools: Path = typer.Option(..., "--tools", "-t", help="OpenAI-style tools JSON file."), + tools: Optional[Path] = _TOOLS_OPTION, traces: Optional[Path] = _TRACES_OPTION, mined: Optional[Path] = _MINED_OPTION, ) -> None: """Show the probe battery that would be generated from a toolset (transparency).""" - probes = derive_probes(_load_tools_or_exit(tools)) + _require_probe_source(tools, traces, mined) + probes = derive_probes(_load_tools_or_exit(tools)) if tools is not None else [] if traces is not None: probes = probes + derive_traced_probes(_load_traces_or_exit(traces)) if mined is not None: @@ -183,7 +196,7 @@ def derive( @app.command() def probe( - tools: Path = typer.Option(..., "--tools", "-t", help="OpenAI-style tools JSON file."), + tools: Optional[Path] = _TOOLS_OPTION, simulate: Optional[Path] = typer.Option( None, "--simulate", "-s", help="Run against a deterministic profile (no model)." ), @@ -215,9 +228,18 @@ def probe( ), ) -> None: """Run the probe battery and produce a capability lockfile.""" - tool_list = _load_tools_or_exit(tools) - probes = derive_probes(tool_list) - fingerprint = tools_fingerprint(tool_list) + _require_probe_source(tools, traces, mined) + if tools is not None: + tool_list = _load_tools_or_exit(tools) + probes = derive_probes(tool_list) + fingerprint = tools_fingerprint(tool_list) + else: + # Trace-only run: traced/mined probes replay their own embedded tool + # definitions, so there is no schema battery and no toolset to fingerprint. + # The empty fingerprint is stable, so two trace-only lockfiles diff cleanly, + # while a trace-only vs schema-derived pair still flags tools_changed. + probes = [] + fingerprint = "" traces_fp = None if traces is not None: @@ -264,6 +286,12 @@ def probe( elif fps: traces_fp = fps[0] + if not probes: + # Possible without --tools: a traces file that derives nothing, or a mined + # file with no accepted probes. A zero-probe lockfile can gate nothing. + _err("The probe battery is empty — nothing derived from the given sources.") + raise typer.Exit(2) + try: if simulate is not None: client = SimulatedClient(_load_json_or_exit(simulate, "simulate profile")) diff --git a/pyproject.toml b/pyproject.toml index b583a7a..2156b81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "probelock" -version = "0.3.0" +version = "0.3.1" description = "A capability lockfile for local models. Catch silent regressions when you swap a model, quant, or runtime." readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_cli.py b/tests/test_cli.py index 41d626b..4a2711c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -588,3 +588,55 @@ def test_ingest_accepts_multiple_logs_as_one_corpus(tmp_path): assert cats.count("tool_selection") == 2 # incl. the continuation across files assert cats.count("schema_validity") == 5 assert cats.count("no_tool") == 1 + + +# --- --tools optional when traces/mined supply probes (validation rough edge) ------- + + +def test_probe_traces_only_needs_no_tools_file(tmp_path): + lock = tmp_path / "traced.lock" + result = runner.invoke(app, [ + "probe", "--traces", str(_SAMPLE_TRACES), "--simulate", str(_SIM_Q8), + "-o", str(lock)]) + assert result.exit_code == 0, result.output + data = json.loads(lock.read_text()) + assert data["tools_fingerprint"] == "" # no schema battery ran + assert data["traces_fingerprint"] + assert "tool_selection" in data["capabilities"] # traced probes replayed + assert data["n_probes"] > 0 + + +def test_derive_traces_only_needs_no_tools_file(): + result = runner.invoke(app, ["derive", "--traces", str(_SAMPLE_TRACES)]) + assert result.exit_code == 0, result.output + assert "::traced::" in result.output + + +def test_probe_mined_only_needs_no_tools_file(tmp_path): + mined = _ingest(tmp_path) + runner.invoke(app, ["traces", "review", str(mined), "--auto-accept", "schema_validity"]) + lock = tmp_path / "mined-only.lock" + result = runner.invoke(app, [ + "probe", "--mined", str(mined), "--simulate", str(_SIM_Q8), + "--allow-sensitive", "-o", str(lock)]) + assert result.exit_code == 0, result.output + caps = json.loads(lock.read_text())["capabilities"] + assert list(caps) == ["traced_schema_validity"] + + +def test_probe_without_any_source_exits_2(): + result = runner.invoke(app, ["probe", "--simulate", str(_SIM_Q8)]) + assert result.exit_code == 2, result.output + assert "--tools, --traces, or --mined" in result.output + result = runner.invoke(app, ["derive"]) + assert result.exit_code == 2, result.output + + +def test_probe_with_empty_battery_exits_2(tmp_path): + # a mined file with nothing accepted and no --tools: the battery is empty and a + # zero-probe lockfile can gate nothing + mined = _ingest(tmp_path) # everything still pending + result = runner.invoke(app, [ + "probe", "--mined", str(mined), "--simulate", str(_SIM_Q8)]) + assert result.exit_code == 2, result.output + assert "empty" in result.output