Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 15 additions & 8 deletions src/skillspector/nodes/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -723,7 +723,18 @@ def report(state: SkillspectorState) -> dict[str, object]:
# operationally) regardless of output format; also embedded in each format's
# body / metadata below.
_attempted, _succeeded, degraded = _llm_runtime_status(use_llm, llm_call_log)
provider_available, provider_error = is_llm_available()
has_recorded_failure = any(not r.get("ok") for r in llm_call_log)
provider_unavailable = bool(
use_llm and not provider_available and has_recorded_failure
)
degraded = degraded or provider_unavailable
degraded_notice = _llm_degradation_notice(use_llm, llm_call_log)
if provider_unavailable and degraded_notice is None:
degraded_notice = (
"LLM analysis was requested but the configured provider was unavailable"
f" ({provider_error or 'unknown reason'}); results may reflect static analysis only."
)
if degraded:
logger.warning(
"LLM stage degraded: %d/%d LLM call(s) failed; report reflects static "
Expand All @@ -749,14 +760,10 @@ def report(state: SkillspectorState) -> dict[str, object]:
components, file_cache, use_llm, raw_findings, filtered_findings
)

# Fail closed on a degraded deep scan: when the LLM stage was requested but
# every call failed, the semantic analyzers were effectively skipped, so a
# SAFE verdict would rest on static analysis alone. An attacker can trigger
# this on purpose (e.g. content that breaks the LLM call) to dodge semantic
# scrutiny. Floor the recommendation at CAUTION so an install-gate ASKS
# rather than auto-allows; risk_score / severity are left untouched (they
# honestly reflect what static analysis found), and llm_degraded / llm_error
# explain why the verdict was raised.
# Fail closed on a degraded deep scan: when the provider is unavailable or
# every recorded LLM call failed, a SAFE verdict would rest on static
# analysis alone. Floor the recommendation at CAUTION so an install-gate
# ASKS rather than auto-allows; risk_score / severity are left untouched.
if degraded and risk_recommendation == "SAFE":
risk_recommendation = "CAUTION"

Expand Down
29 changes: 29 additions & 0 deletions tests/nodes/test_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -887,6 +887,35 @@ def test_degraded_scan_floors_recommendation_at_caution() -> None:
assert result["risk_recommendation"] == "CAUTION" # but never SAFE when degraded


def test_unavailable_provider_floors_recommendation_even_with_success_records(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Provider truth wins when swallowed batch failures produced false success records."""
monkeypatch.setattr(
"skillspector.nodes.report.is_llm_available",
lambda: (False, "codex binary not found"),
)
state: SkillspectorState = {
"filtered_findings": [],
"component_metadata": [],
"has_executable_scripts": False,
"manifest": {},
"output_format": "json",
"use_llm": True,
"llm_call_log": [
llm_call_record("semantic_developer_intent", ok=True),
llm_call_record("semantic_quality_policy", ok=True),
llm_call_record("semantic_security_discovery", ok=False, error="binary missing"),
],
}

result = report(state)
assert result["risk_recommendation"] == "CAUTION"
payload = json.loads(result["report_body"])
assert payload["risk_assessment"]["recommendation"] == "CAUTION"
assert payload["metadata"]["llm_available"] is False


def test_non_degraded_clean_scan_stays_safe() -> None:
"""Without degradation, a clean scan still reports SAFE (no over-flooring)."""
state: SkillspectorState = {
Expand Down