Propagate errors instead of swallowing them

README.md

@@ -413,7 +413,8 @@ The returned dict has the structure:
   "tool_version": "1.5.0",
   "ruleset": "ethereum",
   "findings": [{"id", "severity", "title", "message", "file", "line", "hint"}],
-  "summary": {"high": 0, "med": 0, "low": 0, "info": 5, "files_scanned": 12}
+  "summary": {"high": 0, "med": 0, "low": 0, "info": 5, "files_scanned": 12, "files_skipped": 1},
+  "errors": [{"file": "vendor/unreadable.go", "error": "Permission denied"}]
 }
 ```
 

run_demo.py

@@ -131,8 +131,12 @@ def print_banner():
     print(BANNER)
 
 
-def run_demo(eip_number: int = 1559, client: str = "go-ethereum"):
-    """Run a demonstration of PRSpec capabilities for any supported EIP."""
+def run_demo(eip_number: int = 1559, client: str = "go-ethereum") -> bool:
+    """Run a demonstration of PRSpec capabilities for any supported EIP.
+
+    Returns ``True`` when the demo completed and ``False`` when it bailed out,
+    so the process exit status reflects what actually happened.
+    """
     print_banner()
 
     # Import PRSpec components
@@ -146,7 +150,7 @@ def run_demo(eip_number: int = 1559, client: str = "go-ethereum"):
     except ImportError as e:
         print(f"Import error: {e}")
         print("Make sure you've installed requirements: pip install -r requirements.txt")
-        return
+        return False
 
     # Check for Rich library
     try:
@@ -166,7 +170,7 @@ def run_demo(eip_number: int = 1559, client: str = "go-ethereum"):
         print(f"   ✓ Config loaded from: {config.config_path}")
     except Exception as e:
         print(f"   Error loading config: {e}")
-        return
+        return False
 
     # Check API key
     print("\nChecking API credentials...")
@@ -176,7 +180,7 @@ def run_demo(eip_number: int = 1559, client: str = "go-ethereum"):
     except ValueError as e:
         print(f"   {e}")
         print("   Please set GEMINI_API_KEY in your .env file")
-        return
+        return False
 
     # Validate EIP support
     spec_fetcher = SpecFetcher(github_token=config.github_token)
@@ -186,7 +190,7 @@ def run_demo(eip_number: int = 1559, client: str = "go-ethereum"):
     if eip_number not in spec_fetcher.supported_eips():
         print(f"   EIP-{eip_number} is not in the registry. "
               f"Supported: {spec_fetcher.supported_eips()}")
-        return
+        return False
 
     eip_title = spec_fetcher.get_eip_title(eip_number)
     print(f"\nTarget: EIP-{eip_number} ({eip_title}) -- {client}")
@@ -214,7 +218,7 @@ def run_demo(eip_number: int = 1559, client: str = "go-ethereum"):
             print(f"   ✓ Consensus spec: {len(spec_data['consensus_spec'])} characters")
     except Exception as e:
         print(f"   Error fetching spec: {e}")
-        return
+        return False
 
     # Fetch client implementation
     print(f"\nFetching {client} implementation...")
@@ -235,7 +239,7 @@ def run_demo(eip_number: int = 1559, client: str = "go-ethereum"):
             code_files = SAMPLE_CODE[eip_number]
         else:
             print("   No sample code available for this EIP. Exiting.")
-            return
+            return False
 
     # Parse the code
     language = code_fetcher.client_language(client)
@@ -342,6 +346,7 @@ def _analyze_file(file_path, code_content):
     print("=" * 60)
     print("\nReports are in the 'output' directory.")
     print()
+    return True
 
 
 def quick_test():
@@ -387,6 +392,8 @@ def quick_test():
     args = arg_parser.parse_args()
 
     if args.test:
-        quick_test()
+        ok = quick_test()
     else:
-        run_demo(eip_number=args.eip, client=args.client)
+        ok = run_demo(eip_number=args.eip, client=args.client)
+
+    sys.exit(0 if ok else 1)

src/analyzer.py

@@ -1,6 +1,7 @@
 """LLM-based spec compliance analysis (Gemini, OpenAI, and Azure AI backends)."""
 
 import json
+import logging
 import re
 import time
 from abc import ABC, abstractmethod
@@ -9,6 +10,10 @@
 
 import requests
 
+from .errors import AnalysisError
+
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class AnalysisResult:
@@ -18,14 +23,24 @@ class AnalysisResult:
     issues: List[Dict[str, Any]]
     summary: str
     raw_response: Optional[str] = None
+    # Set when the analysis failed; carries the underlying error so callers
+    # (reports, CLI exit codes) can distinguish a failure from a verdict.
+    error: Optional[str] = None
 
     def to_dict(self) -> Dict[str, Any]:
-        return {
+        data: Dict[str, Any] = {
             "status": self.status,
             "confidence": self.confidence,
             "issues": self.issues,
             "summary": self.summary,
         }
+        if self.error:
+            data["error"] = self.error
+        return data
+
+    @property
+    def failed(self) -> bool:
+        return self.status == "ERROR"
 
     @property
     def has_issues(self) -> bool:
@@ -161,6 +176,16 @@ def _parse_json_response(self, response_text: str) -> Dict[str, Any]:
         # Strip surrounding prose — some models wrap JSON in explanation text
         text = text.strip()
 
+        if not text:
+            logger.error("LLM returned an empty response")
+            return {
+                "status": "ERROR",
+                "confidence": 0,
+                "issues": [],
+                "summary": "The model returned an empty response.",
+                "error": "empty response",
+            }
+
         try:
             return json.loads(text)
         except json.JSONDecodeError:
@@ -206,14 +231,32 @@ def _parse_json_response(self, response_text: str) -> Dict[str, Any]:
                     except json.JSONDecodeError:
                         continue
 
+        logger.error(
+            "Could not parse a JSON object out of a %d-char model response; "
+            "first 200 chars: %s", len(text), text[:200],
+        )
         return {
             "status": "ERROR",
             "confidence": 0,
             "issues": [],
             "summary": f"Failed to parse response ({len(text)} chars). "
-                       f"The model output may have been truncated."
+                       f"The model output may have been truncated.",
+            "error": "unparseable response",
         }
 
+    @staticmethod
+    def _result_from_payload(result: Dict[str, Any],
+                             raw_response: Optional[str]) -> "AnalysisResult":
+        """Build an :class:`AnalysisResult` from a parsed model payload."""
+        return AnalysisResult(
+            status=result.get("status", "UNCERTAIN"),
+            confidence=result.get("confidence", 0),
+            issues=result.get("issues", []),
+            summary=result.get("summary", ""),
+            raw_response=raw_response,
+            error=result.get("error"),
+        )
+
 
 class GeminiAnalyzer(BaseAnalyzer):
     """Gemini-backed analyzer. Uses the large context window to compare
@@ -225,8 +268,10 @@ def __init__(self, api_key: str, model: str = "gemini-2.5-pro",
         try:
             from google import genai  # type: ignore[import-untyped]
             from google.genai import types as genai_types  # type: ignore[import-untyped]
-        except ImportError:
-            raise ImportError("google-genai not installed. Run: pip install google-genai")
+        except ImportError as e:
+            raise ImportError(
+                "google-genai not installed. Run: pip install google-genai"
+            ) from e
 
         self._genai_types = genai_types
         self.client = genai.Client(api_key=api_key)
@@ -249,22 +294,25 @@ def analyze_compliance(self, spec_text: str, code_text: str,
                 ),
             )
 
-            result = self._parse_json_response(response.text)
+            if response.text is None:
+                raise AnalysisError(
+                    "Gemini returned no text (the response may have been "
+                    "blocked or truncated)"
+                )
 
-            return AnalysisResult(
-                status=result.get("status", "UNCERTAIN"),
-                confidence=result.get("confidence", 0),
-                issues=result.get("issues", []),
-                summary=result.get("summary", ""),
-                raw_response=response.text
-            )
+            result = self._parse_json_response(response.text)
+            return self._result_from_payload(result, response.text)
 
         except Exception as e:
+            logger.exception(
+                "Gemini analysis of %s failed", context.get("file_name", "<unknown>")
+            )
             return AnalysisResult(
                 status="ERROR",
                 confidence=0,
                 issues=[],
-                summary=f"Gemini analysis failed: {str(e)}"
+                summary=f"Gemini analysis failed: {str(e)}",
+                error=f"{type(e).__name__}: {e}",
             )
 
     def get_model_info(self) -> Dict[str, Any]:
@@ -286,8 +334,8 @@ def __init__(self, api_key: str, model: str = "gpt-4-turbo-preview",
         try:
             from openai import OpenAI
             self.client = OpenAI(api_key=api_key)
-        except ImportError:
-            raise ImportError("openai not installed. Run: pip install openai")
+        except ImportError as e:
+            raise ImportError("openai not installed. Run: pip install openai") from e
 
         self.model = model
         self.max_tokens = max_tokens
@@ -316,22 +364,19 @@ def analyze_compliance(self, spec_text: str, code_text: str,
             )
 
             response_text = response.choices[0].message.content
-            result = self._parse_json_response(response_text)
-
-            return AnalysisResult(
-                status=result.get("status", "UNCERTAIN"),
-                confidence=result.get("confidence", 0),
-                issues=result.get("issues", []),
-                summary=result.get("summary", ""),
-                raw_response=response_text
-            )
+            result = self._parse_json_response(response_text or "")
+            return self._result_from_payload(result, response_text)
 
         except Exception as e:
+            logger.exception(
+                "OpenAI analysis of %s failed", context.get("file_name", "<unknown>")
+            )
             return AnalysisResult(
                 status="ERROR",
                 confidence=0,
                 issues=[],
-                summary=f"OpenAI analysis failed: {str(e)}"
+                summary=f"OpenAI analysis failed: {str(e)}",
+                error=f"{type(e).__name__}: {e}",
             )
 
     def get_model_info(self) -> Dict[str, Any]:
@@ -398,7 +443,7 @@ def _post_with_retry(self, body: dict, headers: dict) -> requests.Response:
         ``HTTPError`` if every attempt is exhausted.
         """
         last_exc: Optional[Exception] = None
-        for attempt in range(self.max_retries + 1):
+        for attempt in range(max(self.max_retries, 0) + 1):
             response = self.session.post(
                 self.endpoint, json=body, headers=headers, timeout=120
             )
@@ -415,10 +460,17 @@ def _post_with_retry(self, body: dict, headers: dict) -> requests.Response:
             retry_after = response.headers.get("Retry-After")
             try:
                 delay = float(retry_after) if retry_after is not None else 2.0 ** attempt
-            except ValueError:
+            except (TypeError, ValueError):
+                logger.warning("Ignoring malformed Retry-After header: %r", retry_after)
                 delay = 2.0 ** attempt
+            logger.warning(
+                "Azure AI returned %s; retrying in %.1fs (attempt %d/%d)",
+                response.status_code, delay, attempt + 1, self.max_retries,
+            )
             time.sleep(delay)
 
+        if last_exc is None:  # pragma: no cover - defensive
+            raise AnalysisError("Azure AI request loop exited without a response")
         raise last_exc
 
     def analyze_compliance(self, spec_text: str, code_text: str,
@@ -452,21 +504,18 @@ def analyze_compliance(self, spec_text: str, code_text: str,
                 if block.get("type") == "text"
             )
             result = self._parse_json_response(response_text)
-
-            return AnalysisResult(
-                status=result.get("status", "UNCERTAIN"),
-                confidence=result.get("confidence", 0),
-                issues=result.get("issues", []),
-                summary=result.get("summary", ""),
-                raw_response=response_text
-            )
+            return self._result_from_payload(result, response_text)
 
         except Exception as e:
+            logger.exception(
+                "Azure AI analysis of %s failed", context.get("file_name", "<unknown>")
+            )
             return AnalysisResult(
                 status="ERROR",
                 confidence=0,
                 issues=[],
-                summary=f"Azure AI analysis failed: {str(e)}"
+                summary=f"Azure AI analysis failed: {str(e)}",
+                error=f"{type(e).__name__}: {e}",
             )
 
     def get_model_info(self) -> Dict[str, Any]: