|
1 | | -"""Tests for the classifier (basic / unit). |
| 1 | +"""Tests for the classifier. |
2 | 2 |
|
3 | | -Property-based tests with hypothesis live in commit 5 alongside the |
4 | | -authored fixtures. This file covers the core behaviour and the error |
5 | | -paths. |
| 3 | +Three layers: |
| 4 | +
|
| 5 | +1. Unit tests covering happy paths, error paths, frozen dataclass |
| 6 | + immutability, and the public-API surface. |
| 7 | +2. Parametrized parity tests against the authored fixtures in |
| 8 | + ``tests/fixtures/``: round-trip JSON byte-stability per manifest, |
| 9 | + and case-by-case decision parity for ``manifest_basic.json``. |
| 10 | +3. Hypothesis property tests for determinism, dominance, and round-trip |
| 11 | + stability — 1,000 examples each. |
6 | 12 | """ |
7 | 13 |
|
8 | 14 | from __future__ import annotations |
9 | 15 |
|
| 16 | +import json |
| 17 | +from pathlib import Path |
| 18 | + |
10 | 19 | import pytest |
| 20 | +from hypothesis import HealthCheck, given, settings |
| 21 | +from hypothesis import strategies as st |
11 | 22 |
|
12 | 23 | from spine_lite import ( |
13 | 24 | Decision, |
|
18 | 29 | ToolCall, |
19 | 30 | ToolDefinition, |
20 | 31 | classify, |
| 32 | + parse_manifest, |
| 33 | +) |
| 34 | + |
| 35 | +FIXTURES_DIR = Path(__file__).parent.parent / "fixtures" |
| 36 | + |
| 37 | +_HYPOTHESIS_THOROUGH = settings( |
| 38 | + max_examples=1000, |
| 39 | + deadline=None, |
| 40 | + suppress_health_check=[HealthCheck.too_slow], |
21 | 41 | ) |
22 | 42 |
|
23 | 43 |
|
@@ -178,3 +198,179 @@ def test_decision_classify_toolcall_in_public_api() -> None: |
178 | 198 |
|
179 | 199 | for name in ("Decision", "ToolCall", "classify"): |
180 | 200 | assert name in spine_lite.__all__ |
| 201 | + |
| 202 | + |
| 203 | +# ---------- parity tests against authored fixtures ---------- |
| 204 | + |
| 205 | + |
| 206 | +_MANIFEST_FIXTURES = ( |
| 207 | + "manifest_minimal.json", |
| 208 | + "manifest_basic.json", |
| 209 | + "manifest_full.json", |
| 210 | +) |
| 211 | + |
| 212 | + |
| 213 | +@pytest.mark.parametrize("fixture", _MANIFEST_FIXTURES) |
| 214 | +def test_manifest_fixture_loads_cleanly(fixture: str) -> None: |
| 215 | + raw = (FIXTURES_DIR / fixture).read_text() |
| 216 | + manifest = parse_manifest(raw) |
| 217 | + assert isinstance(manifest, Manifest) |
| 218 | + |
| 219 | + |
| 220 | +@pytest.mark.parametrize("fixture", _MANIFEST_FIXTURES) |
| 221 | +def test_manifest_fixture_round_trip_byte_stable(fixture: str) -> None: |
| 222 | + """parse → dump → parse → dump produces identical bytes the second time.""" |
| 223 | + raw = (FIXTURES_DIR / fixture).read_text() |
| 224 | + parsed = parse_manifest(raw) |
| 225 | + dumped_once = parsed.model_dump_json() |
| 226 | + re_parsed = parse_manifest(dumped_once) |
| 227 | + dumped_twice = re_parsed.model_dump_json() |
| 228 | + assert dumped_once == dumped_twice |
| 229 | + assert parsed == re_parsed |
| 230 | + |
| 231 | + |
| 232 | +def _load_decision_cases() -> list[dict[str, object]]: |
| 233 | + payload = json.loads((FIXTURES_DIR / "decisions_basic.json").read_text()) |
| 234 | + cases: list[dict[str, object]] = payload["cases"] |
| 235 | + return cases |
| 236 | + |
| 237 | + |
| 238 | +@pytest.fixture(scope="module") |
| 239 | +def basic_manifest() -> Manifest: |
| 240 | + return parse_manifest((FIXTURES_DIR / "manifest_basic.json").read_text()) |
| 241 | + |
| 242 | + |
| 243 | +@pytest.mark.parametrize( |
| 244 | + "case", |
| 245 | + _load_decision_cases(), |
| 246 | + ids=lambda c: str(c["name"]), |
| 247 | +) |
| 248 | +def test_decision_parity_against_fixture( |
| 249 | + case: dict[str, object], |
| 250 | + basic_manifest: Manifest, |
| 251 | +) -> None: |
| 252 | + expected = case["expected"] |
| 253 | + assert isinstance(expected, dict) |
| 254 | + |
| 255 | + decision = classify(ToolCall(tool=str(case["tool"])), basic_manifest) |
| 256 | + |
| 257 | + assert decision.tool == expected["tool"] |
| 258 | + expected_effects = tuple(Effect(e) for e in expected["effects"]) |
| 259 | + assert decision.effects == expected_effects |
| 260 | + assert decision.most_restrictive == Effect(str(expected["most_restrictive"])) |
| 261 | + |
| 262 | + |
| 263 | +# ---------- hypothesis property tests ---------- |
| 264 | + |
| 265 | + |
| 266 | +_NAME_STRATEGY = st.text( |
| 267 | + alphabet=st.characters(min_codepoint=ord("a"), max_codepoint=ord("z")), |
| 268 | + min_size=1, |
| 269 | + max_size=15, |
| 270 | +) |
| 271 | + |
| 272 | +_EFFECTS_STRATEGY = st.lists( |
| 273 | + st.sampled_from(list(Effect)), |
| 274 | + min_size=1, |
| 275 | + max_size=6, |
| 276 | +).map(tuple) |
| 277 | + |
| 278 | +_POSTURES_STRATEGY = st.one_of( |
| 279 | + st.none(), |
| 280 | + st.lists( |
| 281 | + st.sampled_from(list(Posture)), |
| 282 | + min_size=1, |
| 283 | + max_size=4, |
| 284 | + ).map(tuple), |
| 285 | +) |
| 286 | + |
| 287 | + |
| 288 | +@st.composite |
| 289 | +def _tool_definition_strategy(draw: st.DrawFn, name: str) -> ToolDefinition: |
| 290 | + return ToolDefinition( |
| 291 | + name=name, |
| 292 | + description=draw(st.one_of(st.none(), st.text(max_size=30))), |
| 293 | + effects=draw(_EFFECTS_STRATEGY), |
| 294 | + permitted_postures=draw(_POSTURES_STRATEGY), |
| 295 | + require_confirmation=draw(st.booleans()), |
| 296 | + ) |
| 297 | + |
| 298 | + |
| 299 | +@st.composite |
| 300 | +def _manifest_strategy(draw: st.DrawFn) -> Manifest: |
| 301 | + names = draw(st.lists(_NAME_STRATEGY, min_size=1, max_size=5, unique=True)) |
| 302 | + tools = {name: draw(_tool_definition_strategy(name=name)) for name in names} |
| 303 | + return Manifest(tools=tools) |
| 304 | + |
| 305 | + |
| 306 | +@_HYPOTHESIS_THOROUGH |
| 307 | +@given(manifest=_manifest_strategy()) |
| 308 | +def test_classify_is_deterministic_property(manifest: Manifest) -> None: |
| 309 | + """classify(call, manifest) returns the same Decision on every call.""" |
| 310 | + for tool_name in manifest.tools: |
| 311 | + call = ToolCall(tool=tool_name) |
| 312 | + first = classify(call, manifest) |
| 313 | + second = classify(call, manifest) |
| 314 | + assert first == second |
| 315 | + |
| 316 | + |
| 317 | +@_HYPOTHESIS_THOROUGH |
| 318 | +@given(manifest=_manifest_strategy()) |
| 319 | +def test_classify_dominant_is_in_effects_property(manifest: Manifest) -> None: |
| 320 | + """The Decision's most_restrictive is always a member of its effects.""" |
| 321 | + for tool_name in manifest.tools: |
| 322 | + decision = classify(ToolCall(tool=tool_name), manifest) |
| 323 | + assert decision.most_restrictive in decision.effects |
| 324 | + |
| 325 | + |
| 326 | +@_HYPOTHESIS_THOROUGH |
| 327 | +@given(manifest=_manifest_strategy()) |
| 328 | +def test_classify_effects_match_manifest_definition(manifest: Manifest) -> None: |
| 329 | + """The Decision's effects are exactly the manifest's declared effects.""" |
| 330 | + for tool_name, definition in manifest.tools.items(): |
| 331 | + decision = classify(ToolCall(tool=tool_name), manifest) |
| 332 | + assert decision.effects == definition.effects |
| 333 | + |
| 334 | + |
| 335 | +@_HYPOTHESIS_THOROUGH |
| 336 | +@given(manifest=_manifest_strategy()) |
| 337 | +def test_classify_rationale_is_byte_stable_property(manifest: Manifest) -> None: |
| 338 | + """Identical inputs produce byte-identical rationale strings.""" |
| 339 | + for tool_name in manifest.tools: |
| 340 | + call = ToolCall(tool=tool_name) |
| 341 | + a = classify(call, manifest).rationale |
| 342 | + b = classify(call, manifest).rationale |
| 343 | + assert a == b |
| 344 | + |
| 345 | + |
| 346 | +@_HYPOTHESIS_THOROUGH |
| 347 | +@given(manifest=_manifest_strategy()) |
| 348 | +def test_classify_stable_across_manifest_round_trip(manifest: Manifest) -> None: |
| 349 | + """Manifest → JSON → Manifest produces identical decisions for every tool.""" |
| 350 | + re_parsed = parse_manifest(manifest.model_dump_json()) |
| 351 | + for tool_name in manifest.tools: |
| 352 | + call = ToolCall(tool=tool_name) |
| 353 | + original = classify(call, manifest) |
| 354 | + replayed = classify(call, re_parsed) |
| 355 | + assert original == replayed |
| 356 | + |
| 357 | + |
| 358 | +@_HYPOTHESIS_THOROUGH |
| 359 | +@given( |
| 360 | + manifest=_manifest_strategy(), |
| 361 | + arg_payload=st.dictionaries( |
| 362 | + st.text(min_size=1, max_size=10), |
| 363 | + st.text(max_size=20), |
| 364 | + max_size=5, |
| 365 | + ), |
| 366 | +) |
| 367 | +def test_classify_ignores_arguments_property( |
| 368 | + manifest: Manifest, |
| 369 | + arg_payload: dict[str, str], |
| 370 | +) -> None: |
| 371 | + """Phase 2: arguments are stored on ToolCall but don't influence classification.""" |
| 372 | + for tool_name in manifest.tools: |
| 373 | + no_args = classify(ToolCall(tool=tool_name), manifest) |
| 374 | + with_args = classify(ToolCall(tool=tool_name, arguments=arg_payload), manifest) |
| 375 | + assert no_args.effects == with_args.effects |
| 376 | + assert no_args.most_restrictive == with_args.most_restrictive |
0 commit comments