diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e98aa44..2f814ac0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,37 @@ jobs: - name: Import-linter contracts run: lint-imports + # CR-CI-001 (issue #350) — suppression ceiling. + # + # CI green is maintained in part by pytest suppression flags that hide + # whole test files. Those are tracked for removal; this guard stops the + # count growing in the meantime. + # + # MAX is a CEILING, not a floor, and it RATCHETS DOWN: whenever a phase + # removes flags, MAX drops to the new count in the same commit. Leaving + # MAX high would open a silent gap for the suppressions to regrow into. + # Any PR that raises the count fails here and needs explicit Research + # Team sign-off. When CR-CI-001 closes, MAX becomes 0 and this guard + # stays. + # + # History on this repo: 37 until this change, which brings public level + # with the private line in one step — Phase 2 removed 14 stale flags, + # Phase 3 converted 11 files to named skips (10 patent stubs + 1 Ollama), + # and Phase 4 fixed 7 genuinely-failing files. + # + # Counts only indented flag lines, never prose: this step's own comments + # would otherwise be counted and the guard would fail itself. + - name: Suppression ceiling (CR-CI-001) + run: | + MAX=5 + COUNT=$(grep -cE '^[[:space:]]+--ignore=' .github/workflows/ci.yml) + echo "pytest --ignore suppressions: $COUNT (ceiling: $MAX)" + if [ "$COUNT" -gt "$MAX" ]; then + echo "::error::Suppression count $COUNT exceeds ceiling $MAX. See issue #350 (CR-CI-001)." + echo "Adding --ignore flags requires explicit Research Team sign-off." + exit 1 + fi + - name: Run tests with coverage run: | # 26 test files reference classes from stub files emptied in a prior @@ -47,43 +78,11 @@ jobs: # test_shacl.py requires rdflib which is not in the CI dev extras. # Tracked as tech debt — fix target: v0.36.0. pytest --cov=graqle --cov-report=xml --cov-report=term -q \ - --ignore=tests/test_activation/test_pcst.py \ - --ignore=tests/test_activation/test_content_aware_pcst.py \ - --ignore=tests/test_activation/test_relevance_chunks.py \ - --ignore=tests/test_activation/test_adaptive.py \ - --ignore=tests/test_activation/test_chunk_scorer.py \ - --ignore=tests/test_activation/test_cypher_activation.py \ - --ignore=tests/test_benchmarks/test_metrics.py \ - --ignore=tests/test_connectors/test_neo4j_enhanced.py \ - --ignore=tests/test_connectors/test_tamr.py \ --ignore=tests/test_integration/test_phase3.py \ - --ignore=tests/test_integration/test_ollama_e2e.py \ - --ignore=tests/test_intelligence/test_governance_middleware.py \ - --ignore=tests/test_intelligence/test_models.py \ - --ignore=tests/test_learning/test_activation_memory.py \ - --ignore=tests/test_learning/test_gds_intelligence.py \ --ignore=tests/test_licensing/test_manager.py \ - --ignore=tests/test_ontology/test_constraint_graph.py \ - --ignore=tests/test_ontology/test_governance_v3.py \ - --ignore=tests/test_ontology/test_ontology_generator.py \ - --ignore=tests/test_ontology/test_semantic_shacl_gate.py \ - --ignore=tests/test_ontology/test_shacl_gate.py \ - --ignore=tests/test_orchestration/test_confidence_calibration.py \ - --ignore=tests/test_orchestration/test_debate.py \ --ignore=tests/test_plugins/test_mcp_server.py \ - --ignore=tests/test_server/test_middleware.py \ - --ignore=tests/test_server/test_models.py \ - --ignore=tests/test_governance/test_shacl.py \ - --ignore=tests/test_cli/test_g5_vscode_gate_install.py \ --ignore=tests/test_distribution/test_no_internal_strings.py \ - --ignore=tests/test_plugins/test_cg_14_config_drift.py \ --ignore=tests/test_plugins/test_chat_02_intent_hint.py \ - --ignore=tests/test_generation/test_phase4_tools.py \ - --ignore=tests/test_generation/test_phase10_layer1.py \ - --ignore=tests/test_plugins/test_hfci018_tool_hints.py \ - --ignore=tests/test_plugins/test_lesson_hit_count.py \ - --ignore=tests/test_plugins/test_mcp_dev_server_v015.py \ - --ignore=tests/test_plugins/test_phantom.py - name: Upload coverage report if: matrix.python-version == '3.12' diff --git a/graqle/release_gate/engine.py b/graqle/release_gate/engine.py index 1f4cf845..544826b6 100644 --- a/graqle/release_gate/engine.py +++ b/graqle/release_gate/engine.py @@ -6,8 +6,9 @@ Error handling contract (non-leaky): - Provider exceptions, timeouts, None, or malformed payloads never crash. - - They resolve to a fallback WARN verdict with a caller-safe reason string - that NEVER reveals internal thresholds, weights, or compose logic. + - They resolve to a fallback BLOCK verdict (CR-B: the gate fails CLOSED) + with a caller-safe reason string that NEVER reveals internal thresholds, + weights, or compose logic. - NaN / inf / non-string / None fields are normalized to neutral defaults. Public-surface IP redaction: @@ -99,9 +100,10 @@ async def gate( Returns ------- - ReleaseGateVerdict — never raises; failure paths return WARN. + ReleaseGateVerdict — never raises; failure paths return BLOCK (CR-B: fail closed). """ - # ── Input validation (returns WARN fallback on bad input — never raises). + # ── Input validation (returns BLOCK fallback on bad input — never raises). +# CR-B: a gate that cannot parse its own input must not pass a release. # Normalize target FIRST so every subsequent fallback uses the same value. # If target is malformed, fall back to "pypi" and surface invalid_target once. effective_target = target if target in SUPPORTED_TARGETS else "pypi" @@ -287,15 +289,34 @@ def _fallback_verdict( reason: str, review_summary: str = "internal governance error", ) -> ReleaseGateVerdict: - """Return a safe WARN verdict on provider failure or bad input. - - The `reason` is a short machine-readable tag (e.g. "review_timeout") - that callers can surface to operators. It never contains internal - threshold values, weights, or compose logic. + """Return a BLOCK verdict on provider failure or bad input. + + CR-B (Research Team ruling): the gate FAILS CLOSED. A provider timeout + and an internal error both block. A gate that returns WARN when it + could not evaluate is not a gate -- it reports low confidence and lets + the release through, which is the failure mode this exists to prevent. + + Both halves of the ruling are satisfied without a context flag here, + because BLOCK is mapped differently by target at the consumer: + + * publish targets run `graq release-gate` and exit 1 on BLOCK, so + the publish is stopped uniformly across every target; + * pull requests get a red report -- the same BLOCK verdict rendered + as a failing check, which informs without gating a merge. + + The engine keeps its never-crash and no-leak contract unchanged: it + still never raises, and `reason` is a short machine-readable tag + (e.g. "review_timeout") that never contains internal threshold values, + weights, or compose logic. + + An override exists at the workflow level: role-gated, requiring a + written justification, and written to the audit trail BEFORE anything + publishes. It is deliberately not a parameter of this function -- an + engine that can be asked to not-block is not fail-closed. """ safe_target = target if target in SUPPORTED_TARGETS else "pypi" return ReleaseGateVerdict( - verdict=Verdict.WARN, + verdict=Verdict.BLOCK, target=safe_target, blockers=(), majors=(), diff --git a/tests/test_activation/test_content_aware_pcst.py b/tests/test_activation/test_content_aware_pcst.py index 42a2d8c6..bd730564 100644 --- a/tests/test_activation/test_content_aware_pcst.py +++ b/tests/test_activation/test_content_aware_pcst.py @@ -7,6 +7,7 @@ Layer 2: Post-PCST content filter in PCSTActivation._content_filter() Layer 3: Direct file lookup bypass in GraQle._direct_file_lookup() """ +from __future__ import annotations # ── graqle:intelligence ── # module: tests.test_activation.test_content_aware_pcst @@ -29,15 +30,15 @@ raise ImportError("stub only") except (ImportError, AttributeError): pytest.skip( - "IP-protected module not yet implemented in this build - skipping.", + "patent stub: graqle.activation.pcst ships as a patent notice only " + "(EP26162901.8 / EP26166054.2). Not missing infrastructure - " + "these tests run when the IP-protected implementation ships.", allow_module_level=True, ) -from __future__ import annotations - import math import pytest diff --git a/tests/test_activation/test_pcst.py b/tests/test_activation/test_pcst.py index 9f80cc8f..db504d2a 100644 --- a/tests/test_activation/test_pcst.py +++ b/tests/test_activation/test_pcst.py @@ -18,7 +18,9 @@ from graqle.activation.pcst import PCSTActivation except ImportError: pytest.skip( - "IP-protected module not yet implemented in this build - skipping.", + "patent stub: graqle.activation.pcst ships as a patent notice only " + "(EP26162901.8 / EP26166054.2). Not missing infrastructure - " + "these tests run when the IP-protected implementation ships.", allow_module_level=True, ) diff --git a/tests/test_activation/test_relevance_chunks.py b/tests/test_activation/test_relevance_chunks.py index f26b81f7..74a09324 100644 --- a/tests/test_activation/test_relevance_chunks.py +++ b/tests/test_activation/test_relevance_chunks.py @@ -22,7 +22,9 @@ from graqle.activation.relevance import RelevanceScorer except ImportError: pytest.skip( - "IP-protected module not yet implemented in this build - skipping.", + "patent stub: graqle.activation.relevance ships as a patent notice only " + "(EP26162901.8 / EP26166054.2). Not missing infrastructure - " + "these tests run when the IP-protected implementation ships.", allow_module_level=True, ) diff --git a/tests/test_alpha_validation/test_02_g2_release_gate.py b/tests/test_alpha_validation/test_02_g2_release_gate.py index ce3dcd5c..7f8659dc 100644 --- a/tests/test_alpha_validation/test_02_g2_release_gate.py +++ b/tests/test_alpha_validation/test_02_g2_release_gate.py @@ -56,7 +56,9 @@ async def test_g2_release_gate(server_with_plan, record): bad = json.loads(bad_raw) if isinstance(bad_raw, str) else bad_raw # Either a structured error or an engine response that flags the empty diff bad_signals_problem = (bad.get("ok") is False) or ("error" in bad) or ( - bad.get("verdict") in ("FLAG", "WARN", "INSUFFICIENT_GRAPH") + # CR-B: an unevaluatable input now BLOCKs. WARN is kept for older + # engines; BLOCK is what a fail-closed gate actually returns. + bad.get("verdict") in ("BLOCK", "FLAG", "WARN", "INSUFFICIENT_GRAPH") ) assert bad_signals_problem, bad assertions += 1 diff --git a/tests/test_cli/test_g5_vscode_gate_install.py b/tests/test_cli/test_g5_vscode_gate_install.py index 68cb5dc2..7e08c3b2 100644 --- a/tests/test_cli/test_g5_vscode_gate_install.py +++ b/tests/test_cli/test_g5_vscode_gate_install.py @@ -35,59 +35,64 @@ def pkg_data() -> Path: # ── CLI --target flag (5) ──────────────────────────────────────────── -def test_cli_target_default_is_claude(): +# `CliRunner.isolated_filesystem()` was removed in newer click/typer +# (present in click 8.3, gone by 8.5). `typer[all]>=0.9` is unpinned, so CI +# resolves the newest and these tests broke there while passing locally. +# pytest's `tmp_path` gives the same isolation and does not depend on the +# runner's API surface. +def test_cli_target_default_is_claude(tmp_path): """Default --target (no flag) should NOT create .vscode/ dir.""" - with runner.isolated_filesystem() as fs: - result = runner.invoke( - app, - ["gate-install", fs, "--dry-run"], - ) - # Command may fail due to interpreter probe, but we only care - # that no vscode-extension scaffolding appears in output - assert "G5 vscode-extension" not in (result.output or "") + fs = str(tmp_path) + result = runner.invoke( + app, + ["gate-install", fs, "--dry-run"], + ) + # Command may fail due to interpreter probe, but we only care + # that no vscode-extension scaffolding appears in output + assert "G5 vscode-extension" not in (result.output or "") -def test_cli_target_vscode_extension_creates_vscode_dir(): - with runner.isolated_filesystem() as fs: - root = Path(fs) - result = runner.invoke( - app, - ["gate-install", str(root), "--target=vscode-extension"], - ) - assert (root / ".vscode").exists() - assert (root / ".vscode" / "settings.json").exists() - assert (root / ".vscode" / "tasks.json").exists() - assert (root / ".vscode" / "extensions.json").exists() - assert (root / ".mcp.json").exists() +def test_cli_target_vscode_extension_creates_vscode_dir(tmp_path): + fs = str(tmp_path) + root = Path(fs) + result = runner.invoke( + app, + ["gate-install", str(root), "--target=vscode-extension"], + ) + assert (root / ".vscode").exists() + assert (root / ".vscode" / "settings.json").exists() + assert (root / ".vscode" / "tasks.json").exists() + assert (root / ".vscode" / "extensions.json").exists() + assert (root / ".mcp.json").exists() -def test_cli_target_invalid_exits_1(): - with runner.isolated_filesystem() as fs: - result = runner.invoke( - app, ["gate-install", fs, "--target=invalid_target"], - ) - assert result.exit_code == 1 - assert "Invalid --target" in (result.output or "") +def test_cli_target_invalid_exits_1(tmp_path): + fs = str(tmp_path) + result = runner.invoke( + app, ["gate-install", fs, "--target=invalid_target"], + ) + assert result.exit_code == 1 + assert "Invalid --target" in (result.output or "") -def test_cli_target_vscode_dry_run_does_not_write(pkg_data): - with runner.isolated_filesystem() as fs: - root = Path(fs) - result = runner.invoke( - app, - ["gate-install", str(root), "--target=vscode-extension", "--dry-run"], - ) - assert not (root / ".vscode").exists() - assert not (root / ".mcp.json").exists() +def test_cli_target_vscode_dry_run_does_not_write(pkg_data, tmp_path): + fs = str(tmp_path) + root = Path(fs) + result = runner.invoke( + app, + ["gate-install", str(root), "--target=vscode-extension", "--dry-run"], + ) + assert not (root / ".vscode").exists() + assert not (root / ".mcp.json").exists() -def test_cli_target_vscode_shows_would_write_in_dry_run(): - with runner.isolated_filesystem() as fs: - result = runner.invoke( - app, - ["gate-install", fs, "--target=vscode-extension", "--dry-run"], - ) - assert "would-write" in (result.output or "") +def test_cli_target_vscode_shows_would_write_in_dry_run(tmp_path): + fs = str(tmp_path) + result = runner.invoke( + app, + ["gate-install", fs, "--target=vscode-extension", "--dry-run"], + ) + assert "would-write" in (result.output or "") # ── VS Code file creation (6) ──────────────────────────────────────── @@ -241,6 +246,13 @@ async def test_mcp_handler_vscode_target_routes_to_helper(tmp_path: Path, pkg_da class _Srv: _graph_file = None + # `_handle_gate_install` resolves the project root through this method. + # Bind the real implementation rather than faking it, so the test + # exercises production resolution instead of a stub that can drift. + _project_root_from_graph_file = ( + mcp.KogniDevServer._project_root_from_graph_file + ) + result = json.loads(await mcp.KogniDevServer._handle_gate_install( _Srv(), {"target": "vscode-extension", "dry_run": False}, )) diff --git a/tests/test_generation/test_phase10_layer1.py b/tests/test_generation/test_phase10_layer1.py index cdd1abc1..e58d3b72 100644 --- a/tests/test_generation/test_phase10_layer1.py +++ b/tests/test_generation/test_phase10_layer1.py @@ -17,6 +17,26 @@ import pytest +@pytest.fixture(autouse=True) +def _disable_r22_shacl_gate(monkeypatch): + """Disable the R22 SHACL output gate for these single-tool unit tests. + + The gate requires a governance trace conforming to the ADR-209 chain -- + at least one graq_inspect, graq_context and graq_impact step. These tests + invoke ONE tool in isolation, so the trace legitimately has a single step + and the gate correctly blocks the output. That is the gate working as + designed, not a defect in the tool under test. + + The module reads GRAQLE_SHACL_GATE once at import time into a constant, so + setting the env var here would have no effect; the constant is patched + directly. Scoped to this file -- the gate stays enabled everywhere else. + """ + monkeypatch.setattr( + "graqle.plugins.mcp_dev_server._SHACL_GATE_ENABLED", False, raising=False + ) + + + # ────────────────────────────────────────────────────────────────────────────── # 1. GovernanceMiddleware — cumulative radius anti-gaming # ────────────────────────────────────────────────────────────────────────────── diff --git a/tests/test_generation/test_phase4_tools.py b/tests/test_generation/test_phase4_tools.py index 06950dfb..3c3a19e2 100644 --- a/tests/test_generation/test_phase4_tools.py +++ b/tests/test_generation/test_phase4_tools.py @@ -35,6 +35,26 @@ from graqle.routing import MCP_TOOL_TO_TASK, TASK_RECOMMENDATIONS +@pytest.fixture(autouse=True) +def _disable_r22_shacl_gate(monkeypatch): + """Disable the R22 SHACL output gate for these single-tool unit tests. + + The gate requires a governance trace conforming to the ADR-209 chain -- + at least one graq_inspect, graq_context and graq_impact step. These tests + invoke ONE tool in isolation, so the trace legitimately has a single step + and the gate correctly blocks the output. That is the gate working as + designed, not a defect in the tool under test. + + The module reads GRAQLE_SHACL_GATE once at import time into a constant, so + setting the env var here would have no effect; the constant is patched + directly. Scoped to this file -- the gate stays enabled everywhere else. + """ + monkeypatch.setattr( + "graqle.plugins.mcp_dev_server._SHACL_GATE_ENABLED", False, raising=False + ) + + + # --------------------------------------------------------------------------- # Mock helpers (reuse pattern from test_phase35_tools.py) # --------------------------------------------------------------------------- diff --git a/tests/test_learning/test_activation_memory.py b/tests/test_learning/test_activation_memory.py index 2c114125..d42611de 100644 --- a/tests/test_learning/test_activation_memory.py +++ b/tests/test_learning/test_activation_memory.py @@ -27,7 +27,9 @@ raise ImportError("stub only (ActivationMemoryConfig missing)") except (ImportError, AttributeError): pytest.skip( - "IP-protected module not yet implemented in this build - skipping.", + "patent stub: graqle.learning.activation_memory ships as a patent notice only " + "(EP26162901.8 / EP26166054.2). Not missing infrastructure - " + "these tests run when the IP-protected implementation ships.", allow_module_level=True, ) diff --git a/tests/test_learning/test_gds_intelligence.py b/tests/test_learning/test_gds_intelligence.py index 7df2da1c..29e5fa74 100644 --- a/tests/test_learning/test_gds_intelligence.py +++ b/tests/test_learning/test_gds_intelligence.py @@ -3,6 +3,7 @@ Tests both the NetworkX fallback mode (no Neo4j required) and the data structures. Neo4j GDS tests are skipped unless a live Neo4j instance with GDS plugin is available. """ +from __future__ import annotations # ── graqle:intelligence ── # module: tests.test_learning.test_gds_intelligence @@ -25,15 +26,15 @@ raise ImportError("stub only") except (ImportError, AttributeError): pytest.skip( - "IP-protected module not yet implemented in this build - skipping.", + "patent stub: graqle.learning.gds_intelligence ships as a patent notice only " + "(EP26162901.8 / EP26166054.2). Not missing infrastructure - " + "these tests run when the IP-protected implementation ships.", allow_module_level=True, ) -from __future__ import annotations - from graqle.core.graph import Graqle from graqle.learning.gds_intelligence import ( Community, diff --git a/tests/test_ontology/test_constraint_graph.py b/tests/test_ontology/test_constraint_graph.py index 355adb10..251796ce 100644 --- a/tests/test_ontology/test_constraint_graph.py +++ b/tests/test_ontology/test_constraint_graph.py @@ -18,7 +18,9 @@ from graqle.ontology.constraint_graph import ConstraintGraph except ImportError: pytest.skip( - "IP-protected module not yet implemented in this build - skipping.", + "patent stub: graqle.ontology.constraint_graph ships as a patent notice only " + "(EP26162901.8 / EP26166054.2). Not missing infrastructure - " + "these tests run when the IP-protected implementation ships.", allow_module_level=True, ) diff --git a/tests/test_ontology/test_governance_v3.py b/tests/test_ontology/test_governance_v3.py index 7286981e..9255f5d1 100644 --- a/tests/test_ontology/test_governance_v3.py +++ b/tests/test_ontology/test_governance_v3.py @@ -15,13 +15,22 @@ try: import importlib as _importlib _importlib.import_module("graqle.ontology.domains.governance_v3") + # This module also needs semantic_shacl_gate, a separate patent stub: + # guard BOTH or the import below still raises at collection. # Verify the key class exists in the stub import graqle.ontology.domains.governance_v3 as _stub_mod + from graqle.ontology.semantic_shacl_gate import ( # noqa: F401 + SemanticConstraint, + SemanticSHACLGate, + ) if not any(hasattr(_stub_mod, a) for a in dir(_stub_mod) if not a.startswith("_")): raise ImportError("stub only") except (ImportError, AttributeError): pytest.skip( - "IP-protected module not yet implemented in this build - skipping.", + "patent stub: graqle.ontology.domains.governance_v3 + " + "graqle.ontology.semantic_shacl_gate ship as patent notices only " + "(EP26162901.8 / EP26166054.2). Not missing infrastructure - " + "these tests run when the IP-protected implementation ships.", allow_module_level=True, ) diff --git a/tests/test_ontology/test_ontology_generator.py b/tests/test_ontology/test_ontology_generator.py index 1d22a1a7..b5fa6d0d 100644 --- a/tests/test_ontology/test_ontology_generator.py +++ b/tests/test_ontology/test_ontology_generator.py @@ -15,13 +15,19 @@ try: import importlib as _importlib _importlib.import_module("graqle.ontology.ontology_generator") + # This module also needs semantic_shacl_gate, a separate patent stub: + # guard BOTH or the import below still raises at collection. # Verify the key class exists in the stub import graqle.ontology.ontology_generator as _stub_mod + from graqle.ontology.semantic_shacl_gate import SemanticConstraint # noqa: F401 if not any(hasattr(_stub_mod, a) for a in dir(_stub_mod) if not a.startswith("_")): raise ImportError("stub only") except (ImportError, AttributeError): pytest.skip( - "IP-protected module not yet implemented in this build - skipping.", + "patent stub: graqle.ontology.ontology_generator + " + "graqle.ontology.semantic_shacl_gate ship as patent notices only " + "(EP26162901.8 / EP26166054.2). Not missing infrastructure - " + "these tests run when the IP-protected implementation ships.", allow_module_level=True, ) diff --git a/tests/test_ontology/test_semantic_shacl_gate.py b/tests/test_ontology/test_semantic_shacl_gate.py index 36c39b50..52614ae7 100644 --- a/tests/test_ontology/test_semantic_shacl_gate.py +++ b/tests/test_ontology/test_semantic_shacl_gate.py @@ -21,7 +21,9 @@ raise ImportError("stub only") except (ImportError, AttributeError): pytest.skip( - "IP-protected module not yet implemented in this build - skipping.", + "patent stub: graqle.ontology.semantic_shacl_gate ships as a patent notice only " + "(EP26162901.8 / EP26166054.2). Not missing infrastructure - " + "these tests run when the IP-protected implementation ships.", allow_module_level=True, ) diff --git a/tests/test_orchestration/test_debate.py b/tests/test_orchestration/test_debate.py index d36d0655..6a458e63 100644 --- a/tests/test_orchestration/test_debate.py +++ b/tests/test_orchestration/test_debate.py @@ -18,7 +18,9 @@ from graqle.orchestration.debate import DebateProtocol except ImportError: pytest.skip( - "IP-protected module not yet implemented in this build - skipping.", + "patent stub: graqle.orchestration.debate ships as a patent notice only " + "(EP26162901.8 / EP26166054.2). Not missing infrastructure - " + "these tests run when the IP-protected implementation ships.", allow_module_level=True, ) diff --git a/tests/test_plugins/test_cg_14_config_drift.py b/tests/test_plugins/test_cg_14_config_drift.py index 8fe4d27d..4c1e60a8 100644 --- a/tests/test_plugins/test_cg_14_config_drift.py +++ b/tests/test_plugins/test_cg_14_config_drift.py @@ -469,6 +469,14 @@ async def test_handler_invalid_action_returns_error_envelope(): # Minimal fake server — we only need the _handle_config_audit method bound class _FakeServer: _graph_file = None + # CR-012 PR-012c added `_project_root_from_graph_file` and routed the + # handler through it (mcp_dev_server.py:5058). The double must expose + # the same surface or the handler raises AttributeError before it ever + # reaches the behaviour under test. Delegating to the real helper keeps + # the double honest if that helper's semantics change. + _project_root_from_graph_file = ( + mcp.KogniDevServer._project_root_from_graph_file + ) server = _FakeServer() result_json = await mcp.KogniDevServer._handle_config_audit( server, {"action": "delete"}, @@ -483,6 +491,14 @@ async def test_handler_accept_missing_file_returns_validation_error(): class _FakeServer: _graph_file = None + # CR-012 PR-012c added `_project_root_from_graph_file` and routed the + # handler through it (mcp_dev_server.py:5058). The double must expose + # the same surface or the handler raises AttributeError before it ever + # reaches the behaviour under test. Delegating to the real helper keeps + # the double honest if that helper's semantics change. + _project_root_from_graph_file = ( + mcp.KogniDevServer._project_root_from_graph_file + ) server = _FakeServer() result = json.loads(await mcp.KogniDevServer._handle_config_audit( server, {"action": "accept", "approver": "alice"}, @@ -497,6 +513,14 @@ async def test_handler_accept_missing_approver_returns_validation_error(): class _FakeServer: _graph_file = None + # CR-012 PR-012c added `_project_root_from_graph_file` and routed the + # handler through it (mcp_dev_server.py:5058). The double must expose + # the same surface or the handler raises AttributeError before it ever + # reaches the behaviour under test. Delegating to the real helper keeps + # the double honest if that helper's semantics change. + _project_root_from_graph_file = ( + mcp.KogniDevServer._project_root_from_graph_file + ) server = _FakeServer() result = json.loads(await mcp.KogniDevServer._handle_config_audit( server, {"action": "accept", "file": "graqle.yaml"}, @@ -514,6 +538,14 @@ async def test_handler_audit_returns_drift_response(tmp_repo, monkeypatch): class _FakeServer: _graph_file = None + # CR-012 PR-012c added `_project_root_from_graph_file` and routed the + # handler through it (mcp_dev_server.py:5058). The double must expose + # the same surface or the handler raises AttributeError before it ever + # reaches the behaviour under test. Delegating to the real helper keeps + # the double honest if that helper's semantics change. + _project_root_from_graph_file = ( + mcp.KogniDevServer._project_root_from_graph_file + ) server = _FakeServer() result = json.loads(await mcp.KogniDevServer._handle_config_audit( server, {"action": "audit"}, @@ -532,6 +564,14 @@ async def test_handler_accept_rejects_path_traversal(): class _FakeServer: _graph_file = None + # CR-012 PR-012c added `_project_root_from_graph_file` and routed the + # handler through it (mcp_dev_server.py:5058). The double must expose + # the same surface or the handler raises AttributeError before it ever + # reaches the behaviour under test. Delegating to the real helper keeps + # the double honest if that helper's semantics change. + _project_root_from_graph_file = ( + mcp.KogniDevServer._project_root_from_graph_file + ) server = _FakeServer() for bad in ("../../../etc/passwd", "foo/../../bar", "/etc/passwd", "C:\\Windows\\hosts"): @@ -549,6 +589,14 @@ async def test_handler_accept_unknown_file_returns_unknown_file_envelope(tmp_rep class _FakeServer: _graph_file = None + # CR-012 PR-012c added `_project_root_from_graph_file` and routed the + # handler through it (mcp_dev_server.py:5058). The double must expose + # the same surface or the handler raises AttributeError before it ever + # reaches the behaviour under test. Delegating to the real helper keeps + # the double honest if that helper's semantics change. + _project_root_from_graph_file = ( + mcp.KogniDevServer._project_root_from_graph_file + ) server = _FakeServer() result = json.loads(await mcp.KogniDevServer._handle_config_audit( server, {"action": "accept", "file": "random.txt", "approver": "alice"}, diff --git a/tests/test_plugins/test_hfci018_tool_hints.py b/tests/test_plugins/test_hfci018_tool_hints.py index bfa69167..8507c3ea 100644 --- a/tests/test_plugins/test_hfci018_tool_hints.py +++ b/tests/test_plugins/test_hfci018_tool_hints.py @@ -15,6 +15,27 @@ from graqle.plugins.mcp_dev_server import KogniDevServer +@pytest.fixture(autouse=True) +def _disable_r22_shacl_gate(monkeypatch): + """Disable the R22 SHACL output gate for these single-tool unit tests. + + The gate requires a governance trace conforming to the ADR-209 chain -- + at least one graq_inspect, graq_context and graq_impact step. These tests + invoke ONE tool in isolation, so the trace legitimately has a single step + and the gate blocks the output, replacing the tool's response with an + error dict. The KeyError these tests raised was that substitution, not a + missing field. + + The module reads GRAQLE_SHACL_GATE once at import time into a constant, so + setting the env var here would have no effect; the constant is patched + directly. Scoped to this file -- the gate stays enabled everywhere else. + """ + monkeypatch.setattr( + "graqle.plugins.mcp_dev_server._SHACL_GATE_ENABLED", False, raising=False + ) + + + # --------------------------------------------------------------------------- # Mock objects # --------------------------------------------------------------------------- diff --git a/tests/test_plugins/test_lesson_hit_count.py b/tests/test_plugins/test_lesson_hit_count.py index 817b0d62..a03f950c 100644 --- a/tests/test_plugins/test_lesson_hit_count.py +++ b/tests/test_plugins/test_lesson_hit_count.py @@ -21,6 +21,27 @@ from graqle.plugins.mcp_dev_server import KogniDevServer + +@pytest.fixture(autouse=True) +def _disable_r22_shacl_gate(monkeypatch): + """Disable the R22 SHACL output gate for these single-tool unit tests. + + The gate requires a governance trace conforming to the ADR-209 chain -- + at least one graq_inspect, graq_context and graq_impact step. These tests + invoke ONE tool in isolation, so the trace legitimately has a single step + and the gate blocks the output, replacing the tool's response with an + error dict. The KeyError these tests raised was that substitution, not a + missing field. + + The module reads GRAQLE_SHACL_GATE once at import time into a constant, so + setting the env var here would have no effect; the constant is patched + directly. Scoped to this file -- the gate stays enabled everywhere else. + """ + monkeypatch.setattr( + "graqle.plugins.mcp_dev_server._SHACL_GATE_ENABLED", False, raising=False + ) + + # --------------------------------------------------------------------------- # Mock graph objects (same pattern as test_mcp_dev_server.py) # --------------------------------------------------------------------------- diff --git a/tests/test_plugins/test_mcp_dev_server_v015.py b/tests/test_plugins/test_mcp_dev_server_v015.py index fd826515..60486829 100644 --- a/tests/test_plugins/test_mcp_dev_server_v015.py +++ b/tests/test_plugins/test_mcp_dev_server_v015.py @@ -36,6 +36,27 @@ def _allow_shrink_for_mock_tests(monkeypatch: pytest.MonkeyPatch) -> None: KogniDevServer, ) + +@pytest.fixture(autouse=True) +def _disable_r22_shacl_gate(monkeypatch): + """Disable the R22 SHACL output gate for these single-tool unit tests. + + The gate requires a governance trace conforming to the ADR-209 chain -- + at least one graq_inspect, graq_context and graq_impact step. These tests + invoke ONE tool in isolation, so the trace legitimately has a single step + and the gate blocks the output, replacing the tool's response with an + error dict. The KeyError these tests raised was that substitution, not a + missing field. + + The module reads GRAQLE_SHACL_GATE once at import time into a constant, so + setting the env var here would have no effect; the constant is patched + directly. Scoped to this file -- the gate stays enabled everywhere else. + """ + monkeypatch.setattr( + "graqle.plugins.mcp_dev_server._SHACL_GATE_ENABLED", False, raising=False + ) + + # --------------------------------------------------------------------------- # Mock graph objects # --------------------------------------------------------------------------- diff --git a/tests/test_plugins/test_phantom.py b/tests/test_plugins/test_phantom.py index 3b75ce47..3191bfb9 100644 --- a/tests/test_plugins/test_phantom.py +++ b/tests/test_plugins/test_phantom.py @@ -16,6 +16,27 @@ import pytest + +@pytest.fixture(autouse=True) +def _disable_r22_shacl_gate(monkeypatch): + """Disable the R22 SHACL output gate for these single-tool unit tests. + + The gate requires a governance trace conforming to the ADR-209 chain -- + at least one graq_inspect, graq_context and graq_impact step. These tests + invoke ONE tool in isolation, so the trace legitimately has a single step + and the gate blocks the output, replacing the tool's response with an + error dict. The KeyError these tests raised was that substitution, not a + missing field. + + The module reads GRAQLE_SHACL_GATE once at import time into a constant, so + setting the env var here would have no effect; the constant is patched + directly. Scoped to this file -- the gate stays enabled everywhere else. + """ + monkeypatch.setattr( + "graqle.plugins.mcp_dev_server._SHACL_GATE_ENABLED", False, raising=False + ) + + # =========================================================================== # 1. SMOKE TESTS — Plugin loads without errors # =========================================================================== diff --git a/tests/test_release_gate/test_engine.py b/tests/test_release_gate/test_engine.py index 943f04ea..491c6f30 100644 --- a/tests/test_release_gate/test_engine.py +++ b/tests/test_release_gate/test_engine.py @@ -143,41 +143,41 @@ def test_gate_clear_with_single_major(): # ── 7. WARN on empty diff input ─────────────────────────────────────────── -def test_gate_warn_on_empty_diff(): +def test_gate_blocks_on_empty_diff(): eng = _engine() result = asyncio.run(eng.gate("", "pypi")) - assert result.verdict is Verdict.WARN + assert result.verdict is Verdict.BLOCK assert "invalid_diff" in result.prediction_reasons # ── 8. WARN on non-string diff ──────────────────────────────────────────── -def test_gate_warn_on_non_string_diff(): +def test_gate_blocks_on_non_string_diff(): eng = _engine() result = asyncio.run(eng.gate(None, "pypi")) # type: ignore[arg-type] - assert result.verdict is Verdict.WARN + assert result.verdict is Verdict.BLOCK assert "invalid_diff" in result.prediction_reasons # ── 9. WARN on invalid target ───────────────────────────────────────────── -def test_gate_warn_on_invalid_target(): +def test_gate_blocks_on_invalid_target(): eng = _engine() result = asyncio.run(eng.gate(SAMPLE_DIFF, "npm")) - assert result.verdict is Verdict.WARN + assert result.verdict is Verdict.BLOCK assert "invalid_target" in result.prediction_reasons # ── 10. WARN on invalid min_confidence ──────────────────────────────────── -def test_gate_warn_on_invalid_min_confidence(): +def test_gate_blocks_on_invalid_min_confidence(): eng = _engine() result = asyncio.run(eng.gate(SAMPLE_DIFF, "pypi", min_confidence=2.0)) - assert result.verdict is Verdict.WARN + assert result.verdict is Verdict.BLOCK assert "invalid_min_confidence" in result.prediction_reasons result2 = asyncio.run(eng.gate(SAMPLE_DIFF, "pypi", min_confidence=float("nan"))) - assert result2.verdict is Verdict.WARN + assert result2.verdict is Verdict.BLOCK # ── 11. Verdict dataclass is frozen ─────────────────────────────────────── @@ -217,14 +217,14 @@ def test_gate_target_vscode_marketplace(): assert result.target == "vscode-marketplace" -# ── 15. Review provider exception → WARN ───────────────────────────────── +# ── 15. Review provider exception → BLOCK (CR-B: fail closed) ───────────────────────────────── def test_gate_handles_review_provider_exception(): eng = _engine( review=FakeReviewProvider(raise_exc=RuntimeError("provider down")), ) result = asyncio.run(eng.gate(SAMPLE_DIFF, "pypi")) - assert result.verdict is Verdict.WARN + assert result.verdict is Verdict.BLOCK assert "review_error" in result.prediction_reasons @@ -262,14 +262,81 @@ def test_provider_exception_text_reaches_the_operator_log(caplog): assert "RuntimeError" not in blob -# ── 16. Prediction provider exception → WARN ───────────────────────────── +# ── CR-B: the fail-closed contract itself ──────────────────────────────── + +def test_every_unevaluatable_path_blocks_never_warns(): + """CR-B (Research ruling): a gate that could not evaluate must BLOCK. + + The defect this pins: the fallback returned WARN at confidence 0.0 with + zero findings, so an internal crash read as "low confidence, proceed" and + the release went out ungated. `Release Gate (PyPI)` was red on every PR + for exactly this reason and blocked nothing. + + A timeout and an internal error are treated identically -- neither is + evidence the release is safe, only that the gate could not say. + + Asserted over EVERY fallback entry point at once, so adding a new failure + path that returns WARN fails here rather than silently reopening the gate. + """ + # Exceptions. + for reason, eng in { + "review_error": _engine(review=FakeReviewProvider(raise_exc=RuntimeError("x"))), + "prediction_error": _engine(predict=FakePredictionProvider(raise_exc=RuntimeError("x"))), + }.items(): + result = asyncio.run(eng.gate(SAMPLE_DIFF, "pypi")) + assert result.verdict is Verdict.BLOCK, ( + f"{reason} returned {result.verdict}; the gate must fail CLOSED" + ) + assert reason in result.prediction_reasons + + # Timeouts. The real timeout is 60s, so it is shortened here rather than + # slept through -- the assertion is about the VERDICT, not the duration. + import graqle.release_gate.engine as _eng_mod + + original = _eng_mod._PROVIDER_TIMEOUT_SECONDS + try: + _eng_mod._PROVIDER_TIMEOUT_SECONDS = 0.01 + for reason, eng in { + "review_timeout": _engine(review=FakeReviewProvider(sleep=0.2)), + "prediction_timeout": _engine(predict=FakePredictionProvider(sleep=0.2)), + }.items(): + result = asyncio.run(eng.gate(SAMPLE_DIFF, "pypi")) + assert result.verdict is Verdict.BLOCK, ( + f"{reason} returned {result.verdict}; the gate must fail CLOSED" + ) + assert reason in result.prediction_reasons + finally: + _eng_mod._PROVIDER_TIMEOUT_SECONDS = original + + # Bad input is equally unevaluatable: a gate that cannot parse its own + # input has no basis to pass a release. + eng = _engine() + for bad in ( + lambda: eng.gate(SAMPLE_DIFF, "not-a-target"), + lambda: eng.gate("", "pypi"), + lambda: eng.gate(SAMPLE_DIFF, "pypi", min_confidence=2.0), + ): + assert asyncio.run(bad()).verdict is Verdict.BLOCK + + +def test_block_still_leaks_nothing(caplog): + """Failing closed must not become an excuse to leak internals.""" + eng = _engine(review=FakeReviewProvider(raise_exc=RuntimeError("secret-internal-detail"))) + result = asyncio.run(eng.gate(SAMPLE_DIFF, "pypi")) + assert result.verdict is Verdict.BLOCK + blob = json.dumps(result.to_dict()) + assert "secret-internal-detail" not in blob + assert "RuntimeError" not in blob + + +# ── 16. Prediction provider exception → BLOCK (CR-B: fail closed) ───────────────────────────── def test_gate_handles_prediction_provider_exception(): eng = _engine( predict=FakePredictionProvider(raise_exc=ValueError("predict fail")), ) result = asyncio.run(eng.gate(SAMPLE_DIFF, "pypi")) - assert result.verdict is Verdict.WARN + assert result.verdict is Verdict.BLOCK assert "prediction_error" in result.prediction_reasons @@ -319,7 +386,7 @@ class NonePred: def test_gate_fallback_verdict_serializes_safely(): eng = _engine(review=FakeReviewProvider(raise_exc=OSError("io"))) result = asyncio.run(eng.gate(SAMPLE_DIFF, "pypi")) - assert result.verdict is Verdict.WARN + assert result.verdict is Verdict.BLOCK d = result.to_dict() j = json.dumps(d) assert "review_error" in j diff --git a/tests/test_release_gate/test_mcp_tool.py b/tests/test_release_gate/test_mcp_tool.py index d26800f6..fcaf6d75 100644 --- a/tests/test_release_gate/test_mcp_tool.py +++ b/tests/test_release_gate/test_mcp_tool.py @@ -61,8 +61,9 @@ async def _stub_predict(args): "target": "pypi", })) data = json.loads(result) - # Missing diff triggers engine fallback → WARN verdict with invalid_diff - assert data["verdict"] == "WARN" + # Missing diff triggers the engine fallback. CR-B: the gate fails CLOSED, + # so an unevaluatable input BLOCKS rather than advising. + assert data["verdict"] == "BLOCK" assert "invalid_diff" in data["prediction_reasons"] @@ -81,7 +82,7 @@ async def _stub_predict(args): "target": "npm", })) data = json.loads(result) - assert data["verdict"] == "WARN" + assert data["verdict"] == "BLOCK" assert "invalid_target" in data["prediction_reasons"] diff --git a/tests/test_tamper_evidence/test_canonicalize.py b/tests/test_tamper_evidence/test_canonicalize.py index e61633e0..42103bff 100644 --- a/tests/test_tamper_evidence/test_canonicalize.py +++ b/tests/test_tamper_evidence/test_canonicalize.py @@ -20,7 +20,7 @@ ) try: - from hypothesis import given + from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st _HAS_HYPOTHESIS = True @@ -74,6 +74,12 @@ def test_canon_returns_bytes(): max_leaves=12, ) + # `too_slow` is a timing health check on INPUT GENERATION, not a + # correctness signal. Under a loaded parallel run the generator can + # miss the threshold and fail a test that is not broken. Suppressed + # here for the same reason, and in the same way, as the sibling + # property tests in test_merkle.py. + @settings(suppress_health_check=[HealthCheck.too_slow]) @given(st.dictionaries(st.text(min_size=1, max_size=12), _json_values, max_size=5)) def test_canon_property_deepcopy_invariant(record): """Property: canonicalization is stable under deep copy (no float NaN/Inf