Skip to content

Commit ef32a5f

Browse files
committed
test: authored fixtures, parametrized parity tests, hypothesis properties
Four authored fixtures in tests/fixtures/: - manifest_minimal.json — single READ tool - manifest_basic.json — six tools across the taxonomy with postures - manifest_full.json — all 6 effects, all 4 postures, metadata - decisions_basic.json — six expected (call, decision) pairs Parametrized tests confirm every fixture loads and round-trips JSON byte-stably (parse → dump → parse → dump → equal). Decision parity test walks each case in decisions_basic.json against manifest_basic.json and asserts effects + most_restrictive match. Hypothesis property tests at 1,000 examples each cover: - determinism: classify(call, manifest) yields the same Decision every call - dominance: most_restrictive is in the effects tuple - manifest fidelity: Decision.effects == ToolDefinition.effects - byte-stable rationale - round-trip stability: classify(call, M) == classify(call, parse(M.dump())) - argument independence: arguments don't influence Phase 2 classification mypy override added for tests.* to disable disallow_untyped_decorators — hypothesis's @given/@settings don't satisfy strict-decorator typing. Runtime modules stay strict. 99 tests passing, 100% coverage on every runtime module.
1 parent 67470ff commit ef32a5f

6 files changed

Lines changed: 355 additions & 4 deletions

File tree

pyproject.toml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,13 @@ pretty = true
132132
module = ["typer.*"]
133133
ignore_missing_imports = false
134134

135+
# Hypothesis's @given / @settings decorators don't propagate types in a way
136+
# mypy --strict accepts as "typed" — disabling disallow_untyped_decorators
137+
# only inside the test suite keeps strict typing on the runtime modules.
138+
[[tool.mypy.overrides]]
139+
module = ["tests.*"]
140+
disallow_untyped_decorators = false
141+
135142
[tool.pytest.ini_options]
136143
minversion = "8.0"
137144
addopts = [
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
{
2+
"manifest": "manifest_basic.json",
3+
"cases": [
4+
{
5+
"name": "read-only file read",
6+
"tool": "read_file",
7+
"expected": {
8+
"tool": "read_file",
9+
"effects": ["read"],
10+
"most_restrictive": "read"
11+
}
12+
},
13+
{
14+
"name": "scoped write",
15+
"tool": "write_file",
16+
"expected": {
17+
"tool": "write_file",
18+
"effects": ["write"],
19+
"most_restrictive": "write"
20+
}
21+
},
22+
{
23+
"name": "network read collapses to network",
24+
"tool": "fetch_url",
25+
"expected": {
26+
"tool": "fetch_url",
27+
"effects": ["network", "read"],
28+
"most_restrictive": "network"
29+
}
30+
},
31+
{
32+
"name": "subprocess execute",
33+
"tool": "shell_run",
34+
"expected": {
35+
"tool": "shell_run",
36+
"effects": ["execute"],
37+
"most_restrictive": "execute"
38+
}
39+
},
40+
{
41+
"name": "fork-and-detach spawn",
42+
"tool": "shell_daemon",
43+
"expected": {
44+
"tool": "shell_daemon",
45+
"effects": ["spawn"],
46+
"most_restrictive": "spawn"
47+
}
48+
},
49+
{
50+
"name": "destructive dominates network",
51+
"tool": "git_force_push",
52+
"expected": {
53+
"tool": "git_force_push",
54+
"effects": ["destructive", "network"],
55+
"most_restrictive": "destructive"
56+
}
57+
}
58+
]
59+
}

tests/fixtures/manifest_basic.json

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{
2+
"tools": {
3+
"read_file": {
4+
"name": "read_file",
5+
"description": "Read a file from disk.",
6+
"effects": ["read"],
7+
"permitted_postures": null,
8+
"require_confirmation": false,
9+
"metadata": {}
10+
},
11+
"write_file": {
12+
"name": "write_file",
13+
"description": "Write content to a file.",
14+
"effects": ["write"],
15+
"permitted_postures": null,
16+
"require_confirmation": false,
17+
"metadata": {}
18+
},
19+
"fetch_url": {
20+
"name": "fetch_url",
21+
"description": "Fetch a URL.",
22+
"effects": ["network", "read"],
23+
"permitted_postures": null,
24+
"require_confirmation": false,
25+
"metadata": {}
26+
},
27+
"shell_run": {
28+
"name": "shell_run",
29+
"description": "Run a shell command and wait for completion.",
30+
"effects": ["execute"],
31+
"permitted_postures": ["interactive", "autonomous"],
32+
"require_confirmation": false,
33+
"metadata": {}
34+
},
35+
"shell_daemon": {
36+
"name": "shell_daemon",
37+
"description": "Spawn a long-running daemon process.",
38+
"effects": ["spawn"],
39+
"permitted_postures": ["interactive"],
40+
"require_confirmation": true,
41+
"metadata": {}
42+
},
43+
"git_force_push": {
44+
"name": "git_force_push",
45+
"description": "Force push, irreversible if remote was rewritten.",
46+
"effects": ["destructive", "network"],
47+
"permitted_postures": ["interactive"],
48+
"require_confirmation": true,
49+
"metadata": {}
50+
}
51+
}
52+
}

tests/fixtures/manifest_full.json

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"tools": {
3+
"all_classes": {
4+
"name": "all_classes",
5+
"description": "Synthetic tool exercising every effect class and posture.",
6+
"effects": [
7+
"destructive",
8+
"spawn",
9+
"execute",
10+
"network",
11+
"write",
12+
"read"
13+
],
14+
"permitted_postures": [
15+
"interactive",
16+
"autonomous",
17+
"dry_run",
18+
"locked"
19+
],
20+
"require_confirmation": true,
21+
"metadata": {
22+
"category": "synthetic",
23+
"tags": ["coverage", "reference"],
24+
"owner": "spine-lite",
25+
"audit_priority": "high"
26+
}
27+
}
28+
}
29+
}
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"tools": {
3+
"read_file": {
4+
"name": "read_file",
5+
"effects": ["read"]
6+
}
7+
}
8+
}

tests/unit/test_classifier.py

Lines changed: 200 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,24 @@
1-
"""Tests for the classifier (basic / unit).
1+
"""Tests for the classifier.
22
3-
Property-based tests with hypothesis live in commit 5 alongside the
4-
authored fixtures. This file covers the core behaviour and the error
5-
paths.
3+
Three layers:
4+
5+
1. Unit tests covering happy paths, error paths, frozen dataclass
6+
immutability, and the public-API surface.
7+
2. Parametrized parity tests against the authored fixtures in
8+
``tests/fixtures/``: round-trip JSON byte-stability per manifest,
9+
and case-by-case decision parity for ``manifest_basic.json``.
10+
3. Hypothesis property tests for determinism, dominance, and round-trip
11+
stability — 1,000 examples each.
612
"""
713

814
from __future__ import annotations
915

16+
import json
17+
from pathlib import Path
18+
1019
import pytest
20+
from hypothesis import HealthCheck, given, settings
21+
from hypothesis import strategies as st
1122

1223
from spine_lite import (
1324
Decision,
@@ -18,6 +29,15 @@
1829
ToolCall,
1930
ToolDefinition,
2031
classify,
32+
parse_manifest,
33+
)
34+
35+
FIXTURES_DIR = Path(__file__).parent.parent / "fixtures"
36+
37+
_HYPOTHESIS_THOROUGH = settings(
38+
max_examples=1000,
39+
deadline=None,
40+
suppress_health_check=[HealthCheck.too_slow],
2141
)
2242

2343

@@ -178,3 +198,179 @@ def test_decision_classify_toolcall_in_public_api() -> None:
178198

179199
for name in ("Decision", "ToolCall", "classify"):
180200
assert name in spine_lite.__all__
201+
202+
203+
# ---------- parity tests against authored fixtures ----------
204+
205+
206+
_MANIFEST_FIXTURES = (
207+
"manifest_minimal.json",
208+
"manifest_basic.json",
209+
"manifest_full.json",
210+
)
211+
212+
213+
@pytest.mark.parametrize("fixture", _MANIFEST_FIXTURES)
214+
def test_manifest_fixture_loads_cleanly(fixture: str) -> None:
215+
raw = (FIXTURES_DIR / fixture).read_text()
216+
manifest = parse_manifest(raw)
217+
assert isinstance(manifest, Manifest)
218+
219+
220+
@pytest.mark.parametrize("fixture", _MANIFEST_FIXTURES)
221+
def test_manifest_fixture_round_trip_byte_stable(fixture: str) -> None:
222+
"""parse → dump → parse → dump produces identical bytes the second time."""
223+
raw = (FIXTURES_DIR / fixture).read_text()
224+
parsed = parse_manifest(raw)
225+
dumped_once = parsed.model_dump_json()
226+
re_parsed = parse_manifest(dumped_once)
227+
dumped_twice = re_parsed.model_dump_json()
228+
assert dumped_once == dumped_twice
229+
assert parsed == re_parsed
230+
231+
232+
def _load_decision_cases() -> list[dict[str, object]]:
233+
payload = json.loads((FIXTURES_DIR / "decisions_basic.json").read_text())
234+
cases: list[dict[str, object]] = payload["cases"]
235+
return cases
236+
237+
238+
@pytest.fixture(scope="module")
239+
def basic_manifest() -> Manifest:
240+
return parse_manifest((FIXTURES_DIR / "manifest_basic.json").read_text())
241+
242+
243+
@pytest.mark.parametrize(
244+
"case",
245+
_load_decision_cases(),
246+
ids=lambda c: str(c["name"]),
247+
)
248+
def test_decision_parity_against_fixture(
249+
case: dict[str, object],
250+
basic_manifest: Manifest,
251+
) -> None:
252+
expected = case["expected"]
253+
assert isinstance(expected, dict)
254+
255+
decision = classify(ToolCall(tool=str(case["tool"])), basic_manifest)
256+
257+
assert decision.tool == expected["tool"]
258+
expected_effects = tuple(Effect(e) for e in expected["effects"])
259+
assert decision.effects == expected_effects
260+
assert decision.most_restrictive == Effect(str(expected["most_restrictive"]))
261+
262+
263+
# ---------- hypothesis property tests ----------
264+
265+
266+
_NAME_STRATEGY = st.text(
267+
alphabet=st.characters(min_codepoint=ord("a"), max_codepoint=ord("z")),
268+
min_size=1,
269+
max_size=15,
270+
)
271+
272+
_EFFECTS_STRATEGY = st.lists(
273+
st.sampled_from(list(Effect)),
274+
min_size=1,
275+
max_size=6,
276+
).map(tuple)
277+
278+
_POSTURES_STRATEGY = st.one_of(
279+
st.none(),
280+
st.lists(
281+
st.sampled_from(list(Posture)),
282+
min_size=1,
283+
max_size=4,
284+
).map(tuple),
285+
)
286+
287+
288+
@st.composite
289+
def _tool_definition_strategy(draw: st.DrawFn, name: str) -> ToolDefinition:
290+
return ToolDefinition(
291+
name=name,
292+
description=draw(st.one_of(st.none(), st.text(max_size=30))),
293+
effects=draw(_EFFECTS_STRATEGY),
294+
permitted_postures=draw(_POSTURES_STRATEGY),
295+
require_confirmation=draw(st.booleans()),
296+
)
297+
298+
299+
@st.composite
300+
def _manifest_strategy(draw: st.DrawFn) -> Manifest:
301+
names = draw(st.lists(_NAME_STRATEGY, min_size=1, max_size=5, unique=True))
302+
tools = {name: draw(_tool_definition_strategy(name=name)) for name in names}
303+
return Manifest(tools=tools)
304+
305+
306+
@_HYPOTHESIS_THOROUGH
307+
@given(manifest=_manifest_strategy())
308+
def test_classify_is_deterministic_property(manifest: Manifest) -> None:
309+
"""classify(call, manifest) returns the same Decision on every call."""
310+
for tool_name in manifest.tools:
311+
call = ToolCall(tool=tool_name)
312+
first = classify(call, manifest)
313+
second = classify(call, manifest)
314+
assert first == second
315+
316+
317+
@_HYPOTHESIS_THOROUGH
318+
@given(manifest=_manifest_strategy())
319+
def test_classify_dominant_is_in_effects_property(manifest: Manifest) -> None:
320+
"""The Decision's most_restrictive is always a member of its effects."""
321+
for tool_name in manifest.tools:
322+
decision = classify(ToolCall(tool=tool_name), manifest)
323+
assert decision.most_restrictive in decision.effects
324+
325+
326+
@_HYPOTHESIS_THOROUGH
327+
@given(manifest=_manifest_strategy())
328+
def test_classify_effects_match_manifest_definition(manifest: Manifest) -> None:
329+
"""The Decision's effects are exactly the manifest's declared effects."""
330+
for tool_name, definition in manifest.tools.items():
331+
decision = classify(ToolCall(tool=tool_name), manifest)
332+
assert decision.effects == definition.effects
333+
334+
335+
@_HYPOTHESIS_THOROUGH
336+
@given(manifest=_manifest_strategy())
337+
def test_classify_rationale_is_byte_stable_property(manifest: Manifest) -> None:
338+
"""Identical inputs produce byte-identical rationale strings."""
339+
for tool_name in manifest.tools:
340+
call = ToolCall(tool=tool_name)
341+
a = classify(call, manifest).rationale
342+
b = classify(call, manifest).rationale
343+
assert a == b
344+
345+
346+
@_HYPOTHESIS_THOROUGH
347+
@given(manifest=_manifest_strategy())
348+
def test_classify_stable_across_manifest_round_trip(manifest: Manifest) -> None:
349+
"""Manifest → JSON → Manifest produces identical decisions for every tool."""
350+
re_parsed = parse_manifest(manifest.model_dump_json())
351+
for tool_name in manifest.tools:
352+
call = ToolCall(tool=tool_name)
353+
original = classify(call, manifest)
354+
replayed = classify(call, re_parsed)
355+
assert original == replayed
356+
357+
358+
@_HYPOTHESIS_THOROUGH
359+
@given(
360+
manifest=_manifest_strategy(),
361+
arg_payload=st.dictionaries(
362+
st.text(min_size=1, max_size=10),
363+
st.text(max_size=20),
364+
max_size=5,
365+
),
366+
)
367+
def test_classify_ignores_arguments_property(
368+
manifest: Manifest,
369+
arg_payload: dict[str, str],
370+
) -> None:
371+
"""Phase 2: arguments are stored on ToolCall but don't influence classification."""
372+
for tool_name in manifest.tools:
373+
no_args = classify(ToolCall(tool=tool_name), manifest)
374+
with_args = classify(ToolCall(tool=tool_name, arguments=arg_payload), manifest)
375+
assert no_args.effects == with_args.effects
376+
assert no_args.most_restrictive == with_args.most_restrictive

0 commit comments

Comments
 (0)