diff --git a/src/distribird/agent/enrich.py b/src/distribird/agent/enrich.py index efc5960..beaa508 100644 --- a/src/distribird/agent/enrich.py +++ b/src/distribird/agent/enrich.py @@ -98,12 +98,17 @@ def enrich_parameter( search_hints=raw.get("search_hints", []), application_context=raw.get("application_context", ""), context_keywords=raw.get("context_keywords", []), + is_recognized_parameter=raw.get("is_recognized_parameter"), + recognition_confidence=raw.get("recognition_confidence", "none"), + empirically_measured=raw.get("empirically_measured"), ) logger.info( - "[LLM:enrich_param] param=%r terms=%r hints=%r", + "[LLM:enrich_param] param=%r terms=%r hints=%r recognized=%s empirical=%s", parameter.name, ctx.common_terminology, ctx.search_hints, + ctx.is_recognized_parameter, + ctx.empirically_measured, ) return ctx diff --git a/src/distribird/agent/graph.py b/src/distribird/agent/graph.py index 72c8a6f..5c13a57 100644 --- a/src/distribird/agent/graph.py +++ b/src/distribird/agent/graph.py @@ -17,13 +17,15 @@ refine_search_node, relevance_judge_node, route_after_deliberation, + route_after_enrich, route_after_quality_gate, search_node, synthesize_node, + validity_check_node, ) from distribird.agent.state import IterationBudget, PipelineState, QualityMetrics from distribird.config import Settings, get_settings -from distribird.models import ParameterInput, PipelineResult +from distribird.models import ParameterInput, ParameterValidity, PipelineResult ProgressCallback = Callable[[str, dict[str, Any]], None] | None @@ -37,7 +39,8 @@ "fetch_fulltext": ("Fetching full-text papers", 0.10), "extract": ("Extracting numerical values", 0.20), "quality_gate": ("Evaluating extraction quality", 0.05), - "synthesize": ("Fitting prior distribution", 0.15), + "synthesize": ("Fitting prior distribution", 0.13), + "validity_check": ("Validating parameter", 0.02), # Loop nodes — zero weight so progress never goes backwards "refine_search": ("Refining search strategy", 0.0), "refine_extraction": ("Refining value extraction", 0.0), @@ -57,6 +60,7 @@ def build_pipeline_graph() -> StateGraph: # type: ignore[type-arg] graph.add_node("extract", extract_node) graph.add_node("quality_gate", quality_gate_node) graph.add_node("synthesize", synthesize_node) + graph.add_node("validity_check", validity_check_node) # Feedback loop nodes graph.add_node("refine_search", refine_search_node) @@ -65,7 +69,15 @@ def build_pipeline_graph() -> StateGraph: # type: ignore[type-arg] # Main flow edges graph.add_edge(START, "enrich") - graph.add_edge("enrich", "query_gen") + + # After enrich: short-circuit to validity_check if the LLM did not + # recognize the parameter — saves all downstream search/extract/synthesize + # work for clearly-invalid requests. + graph.add_conditional_edges( + "enrich", + route_after_enrich, + {"query_gen": "query_gen", "validity_check": "validity_check"}, + ) graph.add_edge("query_gen", "search") # After search → relevance judge → conditional: cross-enrich or fetch fulltext @@ -102,8 +114,9 @@ def build_pipeline_graph() -> StateGraph: # type: ignore[type-arg] # Loop C: refine_extraction → quality_gate graph.add_edge("refine_extraction", "quality_gate") - # Synthesize → END - graph.add_edge("synthesize", END) + # Synthesize → validity_check → END + graph.add_edge("synthesize", "validity_check") + graph.add_edge("validity_check", END) return graph @@ -179,4 +192,8 @@ async def run_parameter_graph( warnings=final_state.get("warnings", []), enrichment=final_state.get("enrichment"), deliberation=final_state.get("deliberation"), + parameter_validity=final_state.get("parameter_validity", ParameterValidity.UNKNOWN), + validity_reason=final_state.get("validity_reason", ""), + validity_signals=final_state.get("validity_signals", {}), + is_empirical=final_state.get("is_empirical"), ) diff --git a/src/distribird/agent/nodes.py b/src/distribird/agent/nodes.py index 8ff95e1..8f687bc 100644 --- a/src/distribird/agent/nodes.py +++ b/src/distribird/agent/nodes.py @@ -355,6 +355,101 @@ async def synthesize_node(state: PipelineState) -> dict[str, object]: } +async def validity_check_node(state: PipelineState) -> dict[str, object]: + """Classify whether the requested parameter is a real, empirically-measured quantity. + + Runs after synthesize. Uses passive heuristics first; optionally runs a single + LLM probe for ambiguous (SUSPICIOUS) cases when budget allows. + """ + t0 = time.time() + settings = _settings_from_state(state) + warnings = list(state.get("warnings", [])) + traces = list(state.get("trace_events", [])) + + from distribird.agent.validity import ( + apply_probe_verdict, + classify_validity_passive, + validity_probe_llm, + ) + from distribird.models import ParameterValidity + + if not settings.enable_validity_check: + traces.append(_trace("validity_check", t0, {"skipped": True})) + return { + "parameter_validity": ParameterValidity.UNKNOWN, + "validity_reason": "", + "validity_signals": {}, + "is_empirical": None, + "warnings": warnings, + "trace_events": traces, + } + + enrichment = state.get("enrichment") + prior = state.get("prior") + parameter = state["parameter"] + papers = state.get("all_papers", []) + queries = state.get("all_queries_tried", []) or state.get("search_queries", []) + quality = state.get("quality") or update_quality(state) + + verdict, reason, signals, is_empirical = classify_validity_passive( + enrichment=enrichment, + prior=prior, + papers_found=len(papers), + values_extracted=quality.n_total_values, + queries_tried=len(queries), + ) + + probe_called = False + budget = state.get("budget", IterationBudget()) + + if ( + verdict == ParameterValidity.SUSPICIOUS + and settings.enable_validity_probe + and budget.has_budget() + ): + probe_result = validity_probe_llm( + parameter=parameter, + enrichment=enrichment, + signals=signals, + settings=settings, + ) + probe_called = True + budget.consume_llm_call() + verdict, reason, is_empirical = apply_probe_verdict( + passive_verdict=verdict, + passive_reason=reason, + passive_is_empirical=is_empirical, + probe_result=probe_result, + ) + signals["probe_result"] = probe_result + + if verdict in (ParameterValidity.LIKELY_INVALID, ParameterValidity.SUSPICIOUS): + label = verdict.value.replace("_", " ").upper() + warnings.append(f"Parameter validity: {label} — {reason}") + + traces.append( + _trace( + "validity_check", + t0, + { + "verdict": verdict.value, + "probe_called": probe_called, + "is_empirical": is_empirical, + }, + ) + ) + + return { + "parameter_validity": verdict, + "validity_reason": reason, + "validity_signals": signals, + "is_empirical": is_empirical, + "warnings": warnings, + "budget": budget, + "trace_events": traces, + } + + # --------------------------------------------------------------------------- # Feedback loop nodes # --------------------------------------------------------------------------- @@ -650,6 +745,33 @@ def route_after_deliberation(state: PipelineState) -> str: return "fetch_fulltext" +def route_after_enrich(state: PipelineState) -> str: + """Decide whether to run the full pipeline or short-circuit to validity_check. + + The full search/extract/synthesize work is expensive (LLM calls, paper fetches, + PyMC fitting). When the enrichment LLM clearly does not recognize the + parameter as a real scientific quantity (is_recognized_parameter=False with + none/low confidence), there is no plausible path to a literature-backed + prior, so we route directly to the terminal validity_check node — which + will classify as LIKELY_INVALID without consulting empty paper/value sets. + """ + settings = _settings_from_state(state) + if not settings.enable_validity_check: + return "query_gen" + + enrichment = state.get("enrichment") + if enrichment is None: + return "query_gen" + + if enrichment.is_recognized_parameter is False and enrichment.recognition_confidence in { + "none", + "low", + }: + return "validity_check" + + return "query_gen" + + def route_after_quality_gate(state: PipelineState) -> str: """Route after quality gate: synthesize, refine search, or refine extraction.""" quality = state.get("quality", QualityMetrics()) diff --git a/src/distribird/agent/pipeline.py b/src/distribird/agent/pipeline.py index 9c4236d..21ebcec 100644 --- a/src/distribird/agent/pipeline.py +++ b/src/distribird/agent/pipeline.py @@ -12,6 +12,7 @@ from distribird.models import ( BatchResult, ParameterInput, + ParameterValidity, PipelineResult, ) @@ -132,6 +133,8 @@ def _on_node(node_name: str, state: dict[str, Any]) -> None: parameter=param, prior=fallback_prior, warnings=[f"Pipeline error: {e}"], + parameter_validity=ParameterValidity.UNKNOWN, + validity_reason=f"Pipeline error prevented validity check: {e}", ) results = await asyncio.gather(*[_run_one(p) for p in parameters]) diff --git a/src/distribird/agent/prompts.py b/src/distribird/agent/prompts.py index 6f1294e..93c9e65 100644 --- a/src/distribird/agent/prompts.py +++ b/src/distribird/agent/prompts.py @@ -43,6 +43,15 @@ - "search_hints": [string, ...] (3-5 short keyword phrases optimized for literature search) - "application_context": string (extract ALL specific context from domain context that narrows down which literature is most relevant — this could be geographic region, climate zone, species/cultivar, soil type, management practice, experimental conditions, etc. Combine into a concise phrase. Return empty string if domain context is purely generic.) - "context_keywords": [string, ...] (3-6 keywords/phrases capturing the user's specific application context, useful for finding the most relevant papers. Include geographic terms, species names, condition descriptors, nearby regions with similar conditions, etc. Return empty array if domain context is purely generic.) +- "is_recognized_parameter": boolean (TRUE if you recognize this as an established scientific parameter that appears in the literature. FALSE if the name looks fabricated, contains non-standard suffixes like '_xyz' or 'mumblesnort', or you have never encountered it. Misspellings of real parameters should still be TRUE with low recognition_confidence.) +- "recognition_confidence": "high" | "medium" | "low" | "none" (your confidence in the recognition. Use "high" only for well-established named parameters; "none" for unrecognized strings.) +- "empirically_measured": boolean (TRUE if this parameter is empirically measured in laboratory or field experiments. FALSE if it is purely theoretical, model-internal, or a calibration weight that cannot be directly measured. When in doubt about a model-specific parameter, prefer FALSE over leaving null/uncertain.) + +Red flags that strongly suggest empirically_measured=false: +- Version suffixes in the name (_v1, _v2, _v3, _2024, _r5) +- Software/model identifiers as a prefix (biome_bgcmuso_, dssat_, ctsm_, swat_) +- Words like "calibration", "weight", "tuning", "fitted", "latent", "state", "covariance", "scaling_factor" +- Internal configuration constants of a specific software version IMPORTANT: Carefully analyze the domain context to extract ALL specifics that affect \ which literature is most relevant. Parameter values in scientific literature vary by: @@ -56,7 +65,7 @@ Return ONLY the JSON object, no other text. -Example — Parameter: "allocation_ratio_root_leaf", Domain: "Biome-BGCMuSo maize crop modeling in Hungary" +Example 1 — Parameter: "allocation_ratio_root_leaf", Domain: "Biome-BGCMuSo maize crop modeling in Hungary" {{ "parameter_meaning": "The fraction of assimilated carbon allocated to roots relative to leaves, controlling belowground vs. aboveground biomass partitioning.", "common_terminology": ["root:shoot ratio", "carbon partitioning", "belowground allocation fraction", "root biomass allocation", "assimilate partitioning"], @@ -64,7 +73,56 @@ "enriched_description": "Root to leaf carbon allocation ratio in maize, controlling the partitioning of photosynthetic assimilates between belowground (root) and aboveground (leaf) biomass", "search_hints": ["maize root shoot ratio", "carbon partitioning maize roots", "biomass allocation cereal crops", "root fraction crop model", "maize Hungary Central Europe"], "application_context": "Hungary, Central Europe, Pannonian continental climate, maize", - "context_keywords": ["Hungary", "Central Europe", "continental climate", "Pannonian", "maize", "Carpathian Basin"] + "context_keywords": ["Hungary", "Central Europe", "continental climate", "Pannonian", "maize", "Carpathian Basin"], + "is_recognized_parameter": true, + "recognition_confidence": "high", + "empirically_measured": true +}} + +Example 2 — Parameter: "biome_bgcmuso_root_decomp_q10_v3", Domain: "Biome-BGCMuSo maize crop modeling" +{{ + "parameter_meaning": "An internal calibration weight in Biome-BGCMuSo v3 controlling the temperature sensitivity (Q10) of root decomposition. This is a software-version-specific tuning parameter, not a measurable physical quantity.", + "common_terminology": ["Q10 temperature sensitivity", "root decomposition rate", "model calibration parameter"], + "typical_range": "Software-specific calibration; not a measurable quantity", + "enriched_description": "Q10 temperature sensitivity coefficient for root organic matter decomposition in soil", + "search_hints": ["Q10 root decomposition temperature sensitivity", "soil organic matter decomposition Q10"], + "application_context": "Biome-BGCMuSo crop modeling", + "context_keywords": ["Biome-BGCMuSo", "soil decomposition", "temperature sensitivity"], + "is_recognized_parameter": true, + "recognition_confidence": "medium", + "empirically_measured": false +}} +""" + +PARAMETER_VALIDITY_PROBE = """\ +You are a strict scientific parameter validator. Decide whether the following parameter request \ +refers to a real, empirically-measured scientific quantity that should appear in peer-reviewed literature. + +Classify into exactly one of three verdicts: +- "valid": A real, empirically-measured scientific parameter (e.g., specific_leaf_area, hubble_constant, soil_porosity). +- "suspicious": Either (a) plausible-sounding but not standard terminology, or (b) a real concept that is theoretical / model-internal only and never empirically measured (e.g., calibration weights, latent variables in a specific model version). +- "likely_invalid": Nonsense, fabricated, or pure jargon with no scientific reality (e.g., "mumblesnort_factor", "fake_quantum_correction_xyz"). + +Parameter name: {name} +Description: {description} +Domain context: {domain_context} + +Enrichment LLM reported: +- recognized as known parameter: {is_recognized} +- recognition confidence: {recognition_confidence} +- empirically measured: {empirically_measured} +- common terminology found: {terminology} + +Pipeline observations: +- queries tried: {n_queries} +- papers found: {papers_found} +- numerical values extracted: {values_extracted} + +Return ONLY a JSON object: +{{ + "verdict": "valid" | "suspicious" | "likely_invalid", + "is_empirical": true | false, + "reason": "" }} """ diff --git a/src/distribird/agent/state.py b/src/distribird/agent/state.py index 1d98c59..0116e80 100644 --- a/src/distribird/agent/state.py +++ b/src/distribird/agent/state.py @@ -15,6 +15,7 @@ FittedPrior, LiteratureEvidence, ParameterInput, + ParameterValidity, ) # --------------------------------------------------------------------------- @@ -89,6 +90,9 @@ def can_refine_extraction(self) -> bool: def has_budget(self) -> bool: return self.total_llm_calls_used < self.total_llm_calls_max + def consume_llm_call(self, count: int = 1) -> None: + self.total_llm_calls_used += count + class TraceEvent(BaseModel): node: str @@ -130,6 +134,12 @@ class PipelineState(TypedDict, total=False): # Observability trace_events: list[TraceEvent] + # Validity verdict (set by validity_check node) + parameter_validity: ParameterValidity + validity_reason: str + validity_signals: dict[str, object] + is_empirical: bool | None + # --------------------------------------------------------------------------- # Helper functions for manipulating state diff --git a/src/distribird/agent/validity.py b/src/distribird/agent/validity.py new file mode 100644 index 0000000..276c48a --- /dev/null +++ b/src/distribird/agent/validity.py @@ -0,0 +1,201 @@ +"""Parameter validity detection — passive heuristics + optional LLM probe. + +This module classifies a parameter request as VALID, SUSPICIOUS, or LIKELY_INVALID +based on signals collected during the pipeline run: +- Enrichment LLM self-flags (is_recognized_parameter, empirically_measured) +- Pipeline observations (papers_found, values_extracted, queries_tried) +- Final fitted prior confidence + +The classification is two-stage: +1. Passive (no LLM cost): heuristic rules combine the above signals. +2. Optional active (one LLM call): if passive verdict is SUSPICIOUS and budget + permits, a dedicated probe asks the LLM for a second opinion. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from pydantic import BaseModel, ValidationError + +from distribird.config import Settings +from distribird.models import ( + ConfidenceLevel, + EnrichedContext, + FittedPrior, + ParameterInput, + ParameterValidity, +) + +logger = logging.getLogger(__name__) + +# Minimum extracted values required to consider a literature-backed prior reliable. +# Two values give the synthesizer enough signal to fit a Normal via moment matching +# (single-value priors fall back to a wide Normal with LOW confidence). +MIN_VALUES_FOR_VALID = 2 + +REASON_NO_LITERATURE = "No literature found across {n} refined queries" +REASON_LLM_UNRECOGNIZED = "LLM did not recognize the parameter and no literature was found" +REASON_NO_TERMINOLOGY = "No domain terminology and no literature found" +REASON_THEORETICAL_ONLY = "Theoretical/derived parameter, not empirically measured" +REASON_EMPIRICAL_UNCLEAR = ( + "Empirical status unclear; literature found but no measured values extractable" +) +REASON_LITERATURE_BACKED = "Recognized parameter with literature-backed prior" +REASON_INSUFFICIENT_EVIDENCE = "Insufficient evidence for confident classification" + + +class ProbeResult(BaseModel): + """Parsed output of the LLM validity probe.""" + + verdict: ParameterValidity + reason: str = "" + is_empirical: bool | None = None + + +def classify_validity_passive( + enrichment: EnrichedContext | None, + prior: FittedPrior | None, + papers_found: int, + values_extracted: int, + queries_tried: int, +) -> tuple[ParameterValidity, str, dict[str, Any], bool | None]: + """Classify validity using only heuristic rules (no LLM call). + + Returns: + (verdict, reason, signals, is_empirical) + """ + is_empirical = enrichment.empirically_measured if enrichment else None + is_recognized = enrichment.is_recognized_parameter if enrichment else None + recog_conf = enrichment.recognition_confidence if enrichment else "none" + n_terms = len(enrichment.common_terminology) if enrichment else 0 + prior_confidence = prior.confidence if prior else ConfidenceLevel.NONE + prior_informative = prior.is_informative if prior else False + + signals: dict[str, Any] = { + "papers_found": papers_found, + "values_extracted": values_extracted, + "n_queries_tried": queries_tried, + "is_recognized_parameter": is_recognized, + "recognition_confidence": recog_conf, + "empirically_measured": is_empirical, + "n_terminology": n_terms, + "prior_is_informative": prior_informative, + "prior_confidence": prior_confidence.value, + } + + if papers_found == 0 and values_extracted == 0 and queries_tried >= 2: + return ( + ParameterValidity.LIKELY_INVALID, + REASON_NO_LITERATURE.format(n=queries_tried), + signals, + is_empirical, + ) + + if is_recognized is False and recog_conf in {"none", "low"} and papers_found == 0: + return ParameterValidity.LIKELY_INVALID, REASON_LLM_UNRECOGNIZED, signals, is_empirical + + if n_terms == 0 and papers_found == 0: + return ParameterValidity.LIKELY_INVALID, REASON_NO_TERMINOLOGY, signals, is_empirical + + if is_empirical is not True and values_extracted == 0 and papers_found > 0: + reason = REASON_THEORETICAL_ONLY if is_empirical is False else REASON_EMPIRICAL_UNCLEAR + return ParameterValidity.SUSPICIOUS, reason, signals, is_empirical + + if ( + prior_informative + and prior_confidence in {ConfidenceLevel.HIGH, ConfidenceLevel.MEDIUM} + and is_recognized is True + and is_empirical is not False + and values_extracted >= MIN_VALUES_FOR_VALID + ): + return ParameterValidity.VALID, REASON_LITERATURE_BACKED, signals, is_empirical + + return ParameterValidity.SUSPICIOUS, REASON_INSUFFICIENT_EVIDENCE, signals, is_empirical + + +def validity_probe_llm( + parameter: ParameterInput, + enrichment: EnrichedContext | None, + signals: dict[str, Any], + settings: Settings, +) -> dict[str, Any] | None: + """Run a dedicated LLM probe for ambiguous validity cases. + + Returns the parsed JSON dict {verdict, is_empirical, reason}, or None on failure. + """ + from openai import OpenAI + + from distribird.agent.extract import _llm_json_call + from distribird.agent.prompts import PARAMETER_VALIDITY_PROBE + + is_recognized = enrichment.is_recognized_parameter if enrichment else None + recog_conf = enrichment.recognition_confidence if enrichment else "none" + empirical = enrichment.empirically_measured if enrichment else None + if enrichment and enrichment.common_terminology: + terminology = ", ".join(enrichment.common_terminology) + else: + terminology = "(none)" + + prompt = PARAMETER_VALIDITY_PROBE.format( + name=parameter.name, + description=parameter.description, + domain_context=parameter.domain_context or "(unspecified)", + is_recognized=is_recognized, + recognition_confidence=recog_conf, + empirically_measured=empirical, + terminology=terminology, + n_queries=signals.get("n_queries_tried", 0), + papers_found=signals.get("papers_found", 0), + values_extracted=signals.get("values_extracted", 0), + ) + + logger.info( + "[LLM:validity_probe] param=%r model=%s", + parameter.name, + settings.llm_model, + ) + + try: + client = OpenAI(base_url=settings.llm_base_url, api_key=settings.llm_api_key) + raw = _llm_json_call( + client, + settings.llm_model, + [{"role": "user", "content": prompt}], + temperature=0.0, + ) + except Exception as e: + logger.warning("[LLM:validity_probe] failed: %s", e) + return None + + if not isinstance(raw, dict): + logger.warning("[LLM:validity_probe] unexpected response type: %s", type(raw).__name__) + return None + + return raw + + +def apply_probe_verdict( + passive_verdict: ParameterValidity, + passive_reason: str, + passive_is_empirical: bool | None, + probe_result: dict[str, Any] | None, +) -> tuple[ParameterValidity, str, bool | None]: + """Combine passive verdict with LLM probe result. + + The probe can only refine SUSPICIOUS verdicts; clear VALID/LIKELY_INVALID + verdicts from the passive heuristics are not overridden. + """ + if probe_result is None or passive_verdict != ParameterValidity.SUSPICIOUS: + return passive_verdict, passive_reason, passive_is_empirical + + try: + probe = ProbeResult.model_validate(probe_result) + except ValidationError: + return passive_verdict, passive_reason, passive_is_empirical + + final_empirical = ( + probe.is_empirical if probe.is_empirical is not None else passive_is_empirical + ) + return probe.verdict, probe.reason or passive_reason, final_empirical diff --git a/src/distribird/config.py b/src/distribird/config.py index 1df4532..de080c0 100644 --- a/src/distribird/config.py +++ b/src/distribird/config.py @@ -48,6 +48,10 @@ class Settings(BaseSettings): total_llm_calls_max: int = 30 min_values_for_synthesis: int = 2 + # BullshitBench: parameter validity detection + enable_validity_check: bool = True + enable_validity_probe: bool = True + # Rate limiting s2_rate_limit: float = 0.9 # req/sec without API key (safety margin under 1/sec) s2_rate_limit_with_key: float = 9.0 # req/sec with API key (under 10/sec) diff --git a/src/distribird/distributions/model_check.py b/src/distribird/distributions/model_check.py new file mode 100644 index 0000000..cc86987 --- /dev/null +++ b/src/distribird/distributions/model_check.py @@ -0,0 +1,237 @@ +"""Model checking: MAP estimation, goodness-of-fit diagnostics, and offline batch utilities.""" + +from __future__ import annotations + +import math + +import numpy as np +from scipy import stats # type: ignore[import-untyped] + +from distribird.models import ( + BatchResult, + CredibleIntervalCoverage, + DistributionFamily, + FittedPrior, + ModelCheckResult, + PipelineResult, +) + +# --------------------------------------------------------------------------- +# Build a frozen scipy distribution from FittedPrior params +# --------------------------------------------------------------------------- + + +def _build_scipy_dist( + family: DistributionFamily, params: dict[str, float] +) -> stats.rv_continuous | stats.rv_discrete: + """Convert a DistributionFamily + params dict to a frozen scipy distribution.""" + if family == DistributionFamily.NORMAL: + return stats.norm(loc=params["mu"], scale=params["sigma"]) + + if family == DistributionFamily.TRUNCATED_NORMAL: + mu, sigma = params["mu"], params["sigma"] + a_std = (params["a"] - mu) / sigma + b_std = (params["b"] - mu) / sigma + return stats.truncnorm(a_std, b_std, loc=mu, scale=sigma) + + if family == DistributionFamily.GAMMA: + return stats.gamma(a=params["alpha"], scale=params["scale"]) + + if family == DistributionFamily.LOGNORMAL: + # scipy lognorm: s=sigma, scale=exp(mu) + return stats.lognorm(s=params["sigma"], scale=math.exp(params["mu"])) + + if family == DistributionFamily.BETA: + lower = params.get("lower", 0.0) + upper = params.get("upper", 1.0) + return stats.beta( + a=params["alpha"], + b=params["beta"], + loc=lower, + scale=upper - lower, + ) + + if family == DistributionFamily.UNIFORM: + lower = params.get("lower", params.get("a", 0.0)) + upper = params.get("upper", params.get("b", 1.0)) + return stats.uniform(loc=lower, scale=upper - lower) + + raise ValueError(f"Unsupported distribution family: {family}") + + +# --------------------------------------------------------------------------- +# Analytical MAP (mode) computation +# --------------------------------------------------------------------------- + + +def _compute_map(family: DistributionFamily, params: dict[str, float]) -> float: + """Compute the analytical mode (MAP estimate) for a distribution.""" + if family == DistributionFamily.NORMAL: + return params["mu"] + + if family == DistributionFamily.TRUNCATED_NORMAL: + return float(np.clip(params["mu"], params["a"], params["b"])) + + if family == DistributionFamily.GAMMA: + alpha = params["alpha"] + scale = params["scale"] + if alpha >= 1.0: + return (alpha - 1.0) * scale + # alpha < 1: PDF diverges at 0, mode is degenerate. + # Use the median as a practical point estimate. + return float(stats.gamma(a=alpha, scale=scale).median()) + + if family == DistributionFamily.LOGNORMAL: + mu, sigma = params["mu"], params["sigma"] + return math.exp(mu - sigma**2) + + if family == DistributionFamily.BETA: + alpha = params["alpha"] + beta_param = params["beta"] + lower = params.get("lower", 0.0) + upper = params.get("upper", 1.0) + if alpha > 1.0 and beta_param > 1.0: + raw_mode = (alpha - 1.0) / (alpha + beta_param - 2.0) + return lower + (upper - lower) * raw_mode + # When alpha <= 1 or beta <= 1, the PDF diverges at a boundary — + # the mode is degenerate. Use the median as a practical point estimate. + dist = stats.beta(a=alpha, b=beta_param, loc=lower, scale=upper - lower) + return float(dist.median()) + + if family == DistributionFamily.UNIFORM: + lower = params.get("lower", params.get("a", 0.0)) + upper = params.get("upper", params.get("b", 1.0)) + return (lower + upper) / 2.0 + + raise ValueError(f"Unsupported distribution family: {family}") + + +# --------------------------------------------------------------------------- +# Credible interval coverage +# --------------------------------------------------------------------------- + + +def _compute_credible_coverage( + dist: stats.rv_continuous, values: np.ndarray +) -> CredibleIntervalCoverage: + """Compute fraction of data within 50%, 90%, 95% credible intervals.""" + coverages = {} + for level, label in [(0.50, "ci_50"), (0.90, "ci_90"), (0.95, "ci_95")]: + lo = dist.ppf((1.0 - level) / 2.0) + hi = dist.ppf((1.0 + level) / 2.0) + frac = float(np.mean((values >= lo) & (values <= hi))) + coverages[label] = frac + return CredibleIntervalCoverage(**coverages) + + +# --------------------------------------------------------------------------- +# CDF deviation +# --------------------------------------------------------------------------- + + +def _compute_cdf_deviation(dist: stats.rv_continuous, values: np.ndarray) -> float: + """Mean absolute deviation between empirical CDF and fitted CDF.""" + n = len(values) + sorted_vals = np.sort(values) + ecdf = np.arange(1, n + 1) / n + fcdf = dist.cdf(sorted_vals) + return float(np.mean(np.abs(ecdf - fcdf))) + + +# --------------------------------------------------------------------------- +# Main check_model entry point +# --------------------------------------------------------------------------- + + +def check_model(prior: FittedPrior, values: list[float]) -> ModelCheckResult | None: + """Compute goodness-of-fit diagnostics for a fitted prior against extracted values. + + Returns None when there are no values or the prior is non-informative. + """ + if not values or not prior.is_informative: + return None + + arr = np.array(values, dtype=float) + n = len(arr) + + dist = _build_scipy_dist(prior.family, prior.params) + map_estimate = _compute_map(prior.family, prior.params) + + # Summary statistics + dist_mean = float(dist.mean()) + dist_median = float(dist.median()) + dist_var = float(dist.var()) + ci_lower = float(dist.ppf(0.025)) + ci_upper = float(dist.ppf(0.975)) + + # KS test + ks_stat, ks_p = stats.kstest(arr, dist.cdf) + + # Log-likelihood + logpdf_vals = dist.logpdf(arr) + # Guard against -inf from values outside support + logpdf_vals = np.where(np.isfinite(logpdf_vals), logpdf_vals, -1e10) + ll = float(np.sum(logpdf_vals)) + + # AIC: 2 parameters for all currently supported families + k = 2 + aic = 2.0 * k - 2.0 * ll + + # Coverage and CDF deviation + coverage = _compute_credible_coverage(dist, arr) + cdf_dev = _compute_cdf_deviation(dist, arr) + + return ModelCheckResult( + map_estimate=map_estimate, + dist_mean=dist_mean, + dist_median=dist_median, + dist_variance=dist_var, + ci_95_lower=ci_lower, + ci_95_upper=ci_upper, + ks_statistic=float(ks_stat), + ks_pvalue=float(ks_p), + log_likelihood=ll, + aic=aic, + mean_absolute_cdf_deviation=cdf_dev, + credible_interval_coverage=coverage, + n_values=n, + ) + + +# --------------------------------------------------------------------------- +# Offline utilities for existing BatchResult JSON +# --------------------------------------------------------------------------- + + +def check_model_from_result(result: PipelineResult) -> ModelCheckResult | None: + """Compute model check from an existing PipelineResult without re-running the pipeline. + + Extracts values from the evidence attached to the fitted prior, filters by + parameter constraints, and runs check_model. + """ + from distribird.agent.synthesize import collect_weighted_values + from distribird.distributions.constraints import filter_values_by_constraints + + papers = result.prior.evidence + if not papers: + return None + + weighted_values = collect_weighted_values(papers) + raw_values = [wv.value for wv in weighted_values] + valid_values, _ = filter_values_by_constraints(raw_values, result.parameter.constraints) + + return check_model(result.prior, valid_values) + + +def check_batch( + batch: BatchResult, +) -> list[tuple[str, ModelCheckResult | None]]: + """Run model checking on every result in a batch. + + Returns a list of (parameter_name, ModelCheckResult | None) tuples. + """ + results: list[tuple[str, ModelCheckResult | None]] = [] + for r in batch.results: + mc = check_model_from_result(r) + results.append((r.parameter.name, mc)) + return results diff --git a/src/distribird/export/json_export.py b/src/distribird/export/json_export.py index 2bcd560..cdeff17 100644 --- a/src/distribird/export/json_export.py +++ b/src/distribird/export/json_export.py @@ -9,7 +9,7 @@ def result_to_dict(result: PipelineResult) -> dict[str, object]: """Convert a pipeline result to a JSON-serializable dict.""" - return { + d: dict[str, object] = { "parameter": result.parameter.name, "distribution": result.prior.family.value, "params": result.prior.params, @@ -28,6 +28,13 @@ def result_to_dict(result: PipelineResult) -> dict[str, object]: ], "warnings": result.warnings, } + if result.model_check is not None: + d["model_check"] = result.model_check.model_dump() + d["parameter_validity"] = result.parameter_validity.value + d["validity_reason"] = result.validity_reason + d["validity_signals"] = result.validity_signals + d["is_empirical"] = result.is_empirical + return d def export_json(batch: BatchResult, indent: int = 2) -> str: diff --git a/src/distribird/export/table_export.py b/src/distribird/export/table_export.py new file mode 100644 index 0000000..b9504a9 --- /dev/null +++ b/src/distribird/export/table_export.py @@ -0,0 +1,111 @@ +"""LaTeX and Markdown table export for model checking results.""" + +from __future__ import annotations + +from distribird.models import PipelineResult + + +def _fmt(val: float, precision: int = 4) -> str: + return f"{val:.{precision}g}" + + +_LATEX_ESCAPES = { + "\\": r"\textbackslash{}", + "{": r"\{", + "}": r"\}", + "&": r"\&", + "%": r"\%", + "$": r"\$", + "#": r"\#", + "_": r"\_", + "~": r"\textasciitilde{}", + "^": r"\textasciicircum{}", +} + + +def _latex_escape(s: str) -> str: + """Escape characters with special meaning in LaTeX.""" + out: list[str] = [] + for ch in s: + out.append(_LATEX_ESCAPES.get(ch, ch)) + return "".join(out) + + +# --------------------------------------------------------------------------- +# Markdown +# --------------------------------------------------------------------------- + + +def batch_to_markdown_table(results: list[PipelineResult]) -> str: + """Generate a Markdown table of model checking diagnostics.""" + header = "| Parameter | Distribution | MAP | Mean | 95% CI | KS stat | KS p | AIC | n |" + sep = "|---|---|---|---|---|---|---|---|---|" + rows = [header, sep] + + for r in results: + mc = r.model_check + if mc is None: + rows.append( + f"| {r.parameter.name} | {r.prior.family.value} | — | — | — | — | — | — | 0 |" + ) + continue + ci = f"[{_fmt(mc.ci_95_lower)}, {_fmt(mc.ci_95_upper)}]" + rows.append( + f"| {r.parameter.name} " + f"| {r.prior.family.value} " + f"| {_fmt(mc.map_estimate)} " + f"| {_fmt(mc.dist_mean)} " + f"| {ci} " + f"| {_fmt(mc.ks_statistic, 3)} " + f"| {_fmt(mc.ks_pvalue, 3)} " + f"| {_fmt(mc.aic, 1)} " + f"| {mc.n_values} |" + ) + + return "\n".join(rows) + + +# --------------------------------------------------------------------------- +# LaTeX +# --------------------------------------------------------------------------- + + +def batch_to_latex_table(results: list[PipelineResult]) -> str: + """Generate a LaTeX table of model checking diagnostics for the paper.""" + lines = [ + r"\begin{table}[htbp]", + r"\centering", + r"\caption{Model checking diagnostics for fitted prior distributions.}", + r"\label{tab:model-check}", + r"\begin{tabular}{l l r r c r r r r}", + r"\toprule", + ( + r"Parameter & Distribution & MAP & Mean & 95\% CI " + r"& KS stat & KS $p$ & AIC & $n$ \\" + ), + r"\midrule", + ] + + for r in results: + mc = r.model_check + name = _latex_escape(r.parameter.name) + family = _latex_escape(r.prior.family.value) + if mc is None: + lines.append(f"{name} & {family} & --- & --- & --- & --- & --- & --- & 0 \\\\") + continue + ci = f"[{_fmt(mc.ci_95_lower)}, {_fmt(mc.ci_95_upper)}]" + lines.append( + f"{name} & {family} & {_fmt(mc.map_estimate)} & {_fmt(mc.dist_mean)} " + f"& {ci} & {_fmt(mc.ks_statistic, 3)} & {_fmt(mc.ks_pvalue, 3)} " + f"& {_fmt(mc.aic, 1)} & {mc.n_values} \\\\" + ) + + lines.extend( + [ + r"\bottomrule", + r"\end{tabular}", + r"\end{table}", + ] + ) + + return "\n".join(lines) diff --git a/src/distribird/models.py b/src/distribird/models.py index beae2e2..cffec7d 100644 --- a/src/distribird/models.py +++ b/src/distribird/models.py @@ -3,9 +3,9 @@ from __future__ import annotations from enum import Enum -from typing import Any +from typing import Any, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator class ConfidenceLevel(str, Enum): @@ -15,6 +15,15 @@ class ConfidenceLevel(str, Enum): NONE = "none" +class ParameterValidity(str, Enum): + """Verdict on whether a parameter is a real, empirically-measured scientific quantity.""" + + VALID = "valid" + SUSPICIOUS = "suspicious" + LIKELY_INVALID = "likely_invalid" + UNKNOWN = "unknown" + + class DistributionFamily(str, Enum): NORMAL = "normal" TRUNCATED_NORMAL = "truncated_normal" @@ -90,6 +99,36 @@ def display_name(self) -> str: return f"{self.family.value}({param_str})" +class CredibleIntervalCoverage(BaseModel): + """Fraction of extracted data falling within credible intervals of the fitted prior.""" + + ci_50: float = Field(..., ge=0.0, le=1.0, description="Fraction within 50% CI") + ci_90: float = Field(..., ge=0.0, le=1.0, description="Fraction within 90% CI") + ci_95: float = Field(..., ge=0.0, le=1.0, description="Fraction within 95% CI") + + +class ModelCheckResult(BaseModel): + """Goodness-of-fit diagnostics for a fitted prior distribution.""" + + map_estimate: float = Field(..., description="Mode (MAP) of the fitted distribution") + dist_mean: float = Field(..., description="Mean of the fitted distribution") + dist_median: float = Field(..., description="Median of the fitted distribution") + dist_variance: float = Field(..., description="Variance of the fitted distribution") + ci_95_lower: float = Field(..., description="2.5th percentile of the fitted distribution") + ci_95_upper: float = Field(..., description="97.5th percentile of the fitted distribution") + ks_statistic: float = Field(..., description="Kolmogorov-Smirnov test statistic") + ks_pvalue: float = Field(..., description="Kolmogorov-Smirnov test p-value") + log_likelihood: float = Field( + ..., description="Log-likelihood of data under fitted distribution" + ) + aic: float = Field(..., description="Akaike Information Criterion (2k - 2*ll)") + mean_absolute_cdf_deviation: float = Field( + ..., description="Mean |F_empirical(x) - F_fitted(x)| over data points" + ) + credible_interval_coverage: CredibleIntervalCoverage + n_values: int = Field(..., description="Number of data points used for evaluation") + + class WeightedValue(BaseModel): """A value with associated weight for fitting.""" @@ -116,6 +155,25 @@ class EnrichedContext(BaseModel): default_factory=list, description="Keywords capturing user's specific context for relevance filtering", ) + is_recognized_parameter: bool | None = Field( + default=None, + description="LLM self-report: did it recognize this as a real scientific parameter?", + ) + recognition_confidence: Literal["high", "medium", "low", "none"] = Field( + default="none", + description="LLM self-rated confidence in recognizing the parameter", + ) + empirically_measured: bool | None = Field( + default=None, + description="LLM self-report: is this parameter empirically measured (vs theoretical)?", + ) + + @field_validator("recognition_confidence", mode="before") + @classmethod + def _coerce_recognition_confidence(cls, v: object) -> str: + if isinstance(v, str) and v in {"high", "medium", "low", "none"}: + return v + return "none" class AgentFinding(BaseModel): @@ -149,6 +207,22 @@ class PipelineResult(BaseModel): warnings: list[str] = Field(default_factory=list) enrichment: EnrichedContext | None = None deliberation: DeliberationResult | None = None + model_check: ModelCheckResult | None = None + parameter_validity: ParameterValidity = Field( + default=ParameterValidity.UNKNOWN, + description="Validity verdict for the requested parameter", + ) + validity_reason: str = Field( + default="", description="Human-readable explanation of validity verdict" + ) + validity_signals: dict[str, Any] = Field( + default_factory=dict, + description="Raw signals that fed into the validity verdict", + ) + is_empirical: bool | None = Field( + default=None, + description="Whether the parameter is empirically measurable (None = not assessed)", + ) class BatchResult(BaseModel): diff --git a/tests/test_bullshitbench.py b/tests/test_bullshitbench.py new file mode 100644 index 0000000..1c6d164 --- /dev/null +++ b/tests/test_bullshitbench.py @@ -0,0 +1,506 @@ +"""BullshitBench: integration tests for parameter validity detection. + +These tests verify that Distribird flags fake/nonsense/theoretical-only +parameters appropriately, and does NOT misclassify real parameters. +""" + +from contextlib import ExitStack +from unittest.mock import AsyncMock, patch + +import pytest + +from distribird.agent.pipeline import run_parameter +from distribird.config import Settings +from distribird.models import ( + ConstraintSpec, + EnrichedContext, + ExtractedValue, + LiteratureEvidence, + ParameterInput, + ParameterValidity, +) + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def bullshit_settings(): + """Settings with enrichment enabled (we mock it) and validity check on.""" + return Settings( + llm_base_url="http://localhost:4000", + llm_api_key="test", + enable_deliberation=False, + enable_context_enrichment=True, + enable_llm_deep_research=False, + llm_web_search=False, + enable_validity_check=True, + enable_validity_probe=True, + search_refinement_max=1, # allow one refinement → queries_tried can reach 2 + cross_enrichment_max=0, + extraction_refinement_max=0, + ) + + +def _mk_param(name: str, description: str = "") -> ParameterInput: + return ParameterInput( + name=name, + description=description or f"Some description for {name}", + unit="", + domain_context="general scientific testing", + constraints=ConstraintSpec(lower_bound=0, upper_bound=100), + ) + + +def _mk_paper(idx: int, title_prefix: str = "Paper") -> LiteratureEvidence: + return LiteratureEvidence( + title=f"{title_prefix} {idx}", + doi=f"10.1234/test{idx}", + abstract=f"Value reported was {1.0 + idx * 0.5}.", + year=2020 + idx, + extracted_values=[ExtractedValue(reported_value=1.0 + idx * 0.5)], + ) + + +def _mk_papers_no_values(count: int) -> list[LiteratureEvidence]: + """Papers without extracted values — for testing model-internal/calibration parameters.""" + papers = [_mk_paper(i) for i in range(count)] + for p in papers: + p.extracted_values = [] + return papers + + +# --------------------------------------------------------------------------- +# Helpers for mocking the pipeline +# --------------------------------------------------------------------------- + + +def _patch_pipeline( + stack: ExitStack, + enrichment: EnrichedContext | None, + search_papers: list[LiteratureEvidence], + extract_papers: list[LiteratureEvidence] | None = None, + probe_return=None, +): + """Enter pipeline mock contexts. Returns the probe mock for assertion.""" + if extract_papers is None: + extract_papers = search_papers + + stack.enter_context( + patch( + "distribird.agent.enrich.enrich_parameter_context", + return_value=enrichment, + ) + ) + stack.enter_context( + patch( + "distribird.agent.search.generate_search_queries", + return_value=["query1"], + ) + ) + stack.enter_context( + patch( + "distribird.agent.search.search_all_queries", + new_callable=AsyncMock, + return_value=search_papers, + ) + ) + stack.enter_context( + patch( + "distribird.agent.extract.extract_all_values", + return_value=extract_papers, + ) + ) + probe_mock = stack.enter_context( + patch( + "distribird.agent.validity.validity_probe_llm", + return_value=probe_return, + ) + ) + return probe_mock + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_pure_nonsense_mumblesnort(bullshit_settings): + """A pure-nonsense parameter name with no literature → LIKELY_INVALID.""" + enrichment = EnrichedContext( + is_recognized_parameter=False, + recognition_confidence="none", + empirically_measured=None, + common_terminology=[], + ) + param = _mk_param("mumblesnort_factor", "A totally fabricated parameter name") + + with ExitStack() as stack: + probe_mock = _patch_pipeline( + stack, + enrichment, + [], + probe_return={ + "verdict": "likely_invalid", + "is_empirical": False, + "reason": "x", + }, + ) + result = await run_parameter(param, bullshit_settings) + + assert result.parameter_validity == ParameterValidity.LIKELY_INVALID + assert not result.prior.is_informative + assert result.papers_found == 0 + # No probe call needed for clear-cut cases + assert probe_mock.call_count == 0 + assert any("LIKELY INVALID" in w for w in result.warnings) + + +@pytest.mark.asyncio +async def test_pure_nonsense_fake_xyz(bullshit_settings): + """Another pure-nonsense — verify validity_signals populated.""" + enrichment = EnrichedContext( + is_recognized_parameter=False, + recognition_confidence="none", + common_terminology=[], + ) + param = _mk_param("fake_quantum_correction_xyz") + + with ExitStack() as stack: + _patch_pipeline(stack, enrichment, []) + result = await run_parameter(param, bullshit_settings) + + assert result.parameter_validity == ParameterValidity.LIKELY_INVALID + for key in ( + "papers_found", + "values_extracted", + "is_recognized_parameter", + "n_queries_tried", + ): + assert key in result.validity_signals + + +@pytest.mark.asyncio +async def test_plausible_sounding_fake_uses_probe(bullshit_settings): + """Plausible-sounding fake fools the enrichment LLM (passes early gate). + + The enrichment LLM is fooled into setting is_recognized_parameter=True with + low confidence, so the early-skip route does NOT fire and the pipeline + proceeds through full search/extraction. The terminal validity_check then + invokes the probe to escalate the SUSPICIOUS verdict to LIKELY_INVALID. + """ + enrichment = EnrichedContext( + is_recognized_parameter=True, # LLM fooled by plausible name + recognition_confidence="low", + common_terminology=["chlorophyll fluorescence"], + empirically_measured=None, + ) + param = _mk_param( + "chlorophyll_resonance_index", + "A plausible-sounding but fabricated parameter", + ) + # Two papers, but extraction yields zero values → ambiguous + papers = _mk_papers_no_values(2) + + probe_response = { + "verdict": "likely_invalid", + "is_empirical": False, + "reason": "fabricated terminology, no real measurements exist", + } + with ExitStack() as stack: + probe_mock = _patch_pipeline( + stack, + enrichment, + papers, + extract_papers=[], + probe_return=probe_response, + ) + result = await run_parameter(param, bullshit_settings) + + assert probe_mock.call_count == 1 + assert result.parameter_validity == ParameterValidity.LIKELY_INVALID + assert "fabricated" in result.validity_reason.lower() + + +@pytest.mark.asyncio +async def test_empirical_only_theoretical_param(bullshit_settings): + """Theoretical-only parameter (papers exist but no measurements) → SUSPICIOUS.""" + enrichment = EnrichedContext( + is_recognized_parameter=True, + recognition_confidence="medium", + empirically_measured=False, + common_terminology=["latent state", "model-internal"], + ) + param = _mk_param( + "hypothetical_dark_carbon_pool", + "Purely theoretical model term, never measured", + ) + papers = _mk_papers_no_values(5) + + with ExitStack() as stack: + _patch_pipeline(stack, enrichment, papers, extract_papers=[]) + result = await run_parameter(param, bullshit_settings) + + assert result.parameter_validity == ParameterValidity.SUSPICIOUS + assert result.is_empirical is False + assert ( + "theoretical" in result.validity_reason.lower() + or "not empirically" in result.validity_reason.lower() + ) + + +@pytest.mark.asyncio +async def test_misspelled_real_param_is_valid(bullshit_settings): + """A misspelling of a real param still recognized via downstream evidence → VALID.""" + enrichment = EnrichedContext( + is_recognized_parameter=True, + recognition_confidence="low", + empirically_measured=True, + parameter_meaning="Appears to be a misspelling of maximum leaf area index", + common_terminology=["leaf area index", "LAI"], + ) + param = _mk_param("maxmimum_leaf_area_indxex") + papers = [_mk_paper(i) for i in range(6)] + + with ExitStack() as stack: + _patch_pipeline(stack, enrichment, papers) + result = await run_parameter(param, bullshit_settings) + + assert result.parameter_validity == ParameterValidity.VALID + assert result.prior.is_informative + + +@pytest.mark.asyncio +async def test_real_parameter_control_specific_leaf_area(bullshit_settings): + """Real, well-known parameter with literature → VALID, probe NOT called.""" + enrichment = EnrichedContext( + is_recognized_parameter=True, + recognition_confidence="high", + empirically_measured=True, + common_terminology=["specific leaf area", "SLA", "leaf area mass ratio"], + typical_range="10-30 m2/kg", + ) + param = _mk_param("specific_leaf_area", "Leaf area per unit dry mass") + papers = [_mk_paper(i) for i in range(6)] + + with ExitStack() as stack: + probe_mock = _patch_pipeline( + stack, + enrichment, + papers, + probe_return={"verdict": "valid", "is_empirical": True, "reason": "x"}, + ) + result = await run_parameter(param, bullshit_settings) + + assert result.parameter_validity == ParameterValidity.VALID + assert result.prior.is_informative + assert result.is_empirical is True + assert probe_mock.call_count == 0 + + +@pytest.mark.asyncio +async def test_real_param_with_search_outage(bullshit_settings): + """Recognized real param + transient search failure → SUSPICIOUS, not LIKELY_INVALID.""" + enrichment = EnrichedContext( + is_recognized_parameter=True, + recognition_confidence="high", + empirically_measured=True, + common_terminology=["specific leaf area", "SLA"], + ) + param = _mk_param("specific_leaf_area") + + # Disable refinement so only 1 query attempted → rule 1 (>=2 queries) doesn't fire + no_refine_settings = bullshit_settings.model_copy(update={"search_refinement_max": 0}) + + probe_response = { + "verdict": "suspicious", + "is_empirical": True, + "reason": "recognized parameter but search returned no results", + } + with ExitStack() as stack: + _patch_pipeline(stack, enrichment, [], probe_return=probe_response) + result = await run_parameter(param, no_refine_settings) + + assert result.parameter_validity != ParameterValidity.LIKELY_INVALID + + +@pytest.mark.asyncio +async def test_early_skip_avoids_search_and_extract(bullshit_settings): + """Clear nonsense at enrich time short-circuits past search/extract/synthesize. + + The enrichment LLM marks is_recognized_parameter=False with none confidence; + the route_after_enrich short-circuits to validity_check, so the search and + extract mocks should NOT be called. + """ + from distribird.agent import extract as extract_mod + from distribird.agent import search as search_mod + + enrichment = EnrichedContext( + is_recognized_parameter=False, + recognition_confidence="none", + common_terminology=[], + ) + param = _mk_param("totally_fabricated_xyz") + + with ExitStack() as stack: + stack.enter_context( + patch( + "distribird.agent.enrich.enrich_parameter_context", + return_value=enrichment, + ) + ) + gen_mock = stack.enter_context( + patch.object(search_mod, "generate_search_queries", return_value=["q"]) + ) + search_mock = stack.enter_context( + patch.object( + search_mod, + "search_all_queries", + new_callable=AsyncMock, + return_value=[], + ) + ) + extract_mock = stack.enter_context( + patch.object(extract_mod, "extract_all_values", return_value=[]) + ) + stack.enter_context( + patch( + "distribird.agent.validity.validity_probe_llm", + return_value=None, + ) + ) + result = await run_parameter(param, bullshit_settings) + + assert result.parameter_validity == ParameterValidity.LIKELY_INVALID + # Critical: the expensive nodes were NOT called + assert gen_mock.call_count == 0, "query_gen should be skipped" + assert search_mock.call_count == 0, "search should be skipped" + assert extract_mock.call_count == 0, "extract should be skipped" + assert result.papers_found == 0 + assert result.values_extracted == 0 + + +@pytest.mark.asyncio +async def test_validity_check_disabled(bullshit_settings): + """When the toggle is off, validity verdict is UNKNOWN.""" + settings = bullshit_settings.model_copy(update={"enable_validity_check": False}) + enrichment = EnrichedContext( + is_recognized_parameter=False, + recognition_confidence="none", + ) + param = _mk_param("mumblesnort_factor") + + with ExitStack() as stack: + _patch_pipeline(stack, enrichment, []) + result = await run_parameter(param, settings) + + assert result.parameter_validity == ParameterValidity.UNKNOWN + assert result.prior is not None + + +@pytest.mark.asyncio +async def test_validity_probe_skipped_when_unambiguous(bullshit_settings): + """For clearly LIKELY_INVALID cases, the LLM probe must NOT be called.""" + enrichment = EnrichedContext( + is_recognized_parameter=False, + recognition_confidence="none", + common_terminology=[], + ) + param = _mk_param("totally_made_up_parameter_zzz") + + with ExitStack() as stack: + probe_mock = _patch_pipeline(stack, enrichment, []) + await run_parameter(param, bullshit_settings) + + assert probe_mock.call_count == 0 + + +@pytest.mark.asyncio +async def test_validity_signals_in_warnings(bullshit_settings): + """LIKELY_INVALID verdicts must surface in result.warnings.""" + enrichment = EnrichedContext( + is_recognized_parameter=False, + recognition_confidence="none", + common_terminology=[], + ) + param = _mk_param("garbage_xyz_param") + + with ExitStack() as stack: + _patch_pipeline(stack, enrichment, []) + result = await run_parameter(param, bullshit_settings) + + assert any(w.startswith("Parameter validity:") for w in result.warnings) + + +@pytest.mark.asyncio +async def test_uncertain_empirical_calibration_weight(bullshit_settings): + """LLM uncertain about empirical status, papers exist, no values → Rule 4b → SUSPICIOUS.""" + from distribird.agent.validity import REASON_EMPIRICAL_UNCLEAR + + enrichment = EnrichedContext( + is_recognized_parameter=True, + recognition_confidence="medium", + empirically_measured=None, + common_terminology=["model parameter", "tuning weight"], + ) + param = _mk_param( + "biome_bgcmuso_root_decomp_q10_v3", + "Calibration weight for root decomposition Q10 in Biome-BGCMuSo v3", + ) + papers = _mk_papers_no_values(5) + + # Empty probe reason so we can verify the passive Rule 4b reason survives + probe_response = {"verdict": "suspicious", "is_empirical": False, "reason": ""} + with ExitStack() as stack: + probe_mock = _patch_pipeline( + stack, + enrichment, + papers, + extract_papers=[], + probe_return=probe_response, + ) + result = await run_parameter(param, bullshit_settings) + + assert result.parameter_validity == ParameterValidity.SUSPICIOUS + assert REASON_EMPIRICAL_UNCLEAR in result.validity_reason + assert probe_mock.call_count == 1 + + +@pytest.mark.asyncio +async def test_misclassified_empirical_passes_through_probe(bullshit_settings): + """Probe corrects an LLM error: parameter marked empirical but actually non-empirical. + + Why: Rule 5 requires values_extracted >= MIN_VALUES_FOR_VALID, so 0 extracted + values blocks VALID even when the LLM mistakenly returned empirically_measured=True. + The verdict falls through to SUSPICIOUS where the probe corrects to LIKELY_INVALID. + """ + enrichment = EnrichedContext( + is_recognized_parameter=True, + recognition_confidence="high", + empirically_measured=True, + common_terminology=["calibration weight"], + ) + param = _mk_param("dssat_root_factor_v45") + no_refine_settings = bullshit_settings.model_copy(update={"search_refinement_max": 0}) + papers = _mk_papers_no_values(3) + + probe_response = { + "verdict": "likely_invalid", + "is_empirical": False, + "reason": "version-specific calibration weight; LLM claim of empirical wrong", + } + with ExitStack() as stack: + probe_mock = _patch_pipeline( + stack, + enrichment, + papers, + extract_papers=[], + probe_return=probe_response, + ) + result = await run_parameter(param, no_refine_settings) + + assert probe_mock.call_count == 1 + assert result.parameter_validity == ParameterValidity.LIKELY_INVALID + assert result.is_empirical is False # probe corrected the LLM mistake diff --git a/tests/test_model_check.py b/tests/test_model_check.py new file mode 100644 index 0000000..fd19909 --- /dev/null +++ b/tests/test_model_check.py @@ -0,0 +1,432 @@ +"""Tests for distribird.distributions.model_check.""" + +from __future__ import annotations + +import math + +import numpy as np +import pytest +from scipy import stats # type: ignore[import-untyped] + +from distribird.distributions.model_check import ( + _build_scipy_dist, + _compute_cdf_deviation, + _compute_credible_coverage, + _compute_map, + check_batch, + check_model, + check_model_from_result, +) +from distribird.models import ( + BatchResult, + ConfidenceLevel, + ConstraintSpec, + DistributionFamily, + ExtractedValue, + FittedPrior, + LiteratureEvidence, + ParameterInput, + PipelineResult, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_prior( + family: DistributionFamily, + params: dict[str, float], + informative: bool = True, + evidence: list[LiteratureEvidence] | None = None, +) -> FittedPrior: + return FittedPrior( + parameter_name="test_param", + family=family, + params=params, + confidence=ConfidenceLevel.HIGH if informative else ConfidenceLevel.NONE, + is_informative=informative, + reason="test", + evidence=evidence or [], + n_sources=len(evidence) if evidence else 0, + ) + + +# --------------------------------------------------------------------------- +# _build_scipy_dist +# --------------------------------------------------------------------------- + + +class TestBuildScipyDist: + def test_normal(self): + dist = _build_scipy_dist(DistributionFamily.NORMAL, {"mu": 5.0, "sigma": 2.0}) + assert abs(dist.mean() - 5.0) < 1e-6 + assert abs(dist.std() - 2.0) < 1e-6 + + def test_truncated_normal(self): + dist = _build_scipy_dist( + DistributionFamily.TRUNCATED_NORMAL, + {"mu": 0.0, "sigma": 1.0, "a": -2.0, "b": 2.0}, + ) + # Mean of symmetric truncated normal is 0 + assert abs(dist.mean()) < 0.1 + + def test_gamma(self): + dist = _build_scipy_dist(DistributionFamily.GAMMA, {"alpha": 3.0, "scale": 2.0}) + assert abs(dist.mean() - 6.0) < 1e-6 + + def test_lognormal(self): + dist = _build_scipy_dist(DistributionFamily.LOGNORMAL, {"mu": 0.0, "sigma": 1.0}) + expected_mean = math.exp(0.0 + 0.5) + assert abs(dist.mean() - expected_mean) < 1e-4 + + def test_beta(self): + dist = _build_scipy_dist( + DistributionFamily.BETA, + {"alpha": 2.0, "beta": 5.0, "lower": 0.0, "upper": 1.0}, + ) + expected_mean = 2.0 / (2.0 + 5.0) + assert abs(dist.mean() - expected_mean) < 1e-4 + + def test_beta_scaled(self): + dist = _build_scipy_dist( + DistributionFamily.BETA, + {"alpha": 2.0, "beta": 2.0, "lower": 10.0, "upper": 20.0}, + ) + # Mean should be midpoint for symmetric beta + assert abs(dist.mean() - 15.0) < 1e-4 + + def test_uniform(self): + dist = _build_scipy_dist(DistributionFamily.UNIFORM, {"lower": 3.0, "upper": 7.0}) + assert abs(dist.mean() - 5.0) < 1e-6 + + +# --------------------------------------------------------------------------- +# _compute_map +# --------------------------------------------------------------------------- + + +class TestComputeMap: + def test_normal(self): + assert _compute_map(DistributionFamily.NORMAL, {"mu": 3.14, "sigma": 1.0}) == 3.14 + + def test_truncated_normal_inside(self): + result = _compute_map( + DistributionFamily.TRUNCATED_NORMAL, + {"mu": 5.0, "sigma": 1.0, "a": 0.0, "b": 10.0}, + ) + assert result == 5.0 + + def test_truncated_normal_clipped_low(self): + result = _compute_map( + DistributionFamily.TRUNCATED_NORMAL, + {"mu": -5.0, "sigma": 1.0, "a": 0.0, "b": 10.0}, + ) + assert result == 0.0 + + def test_truncated_normal_clipped_high(self): + result = _compute_map( + DistributionFamily.TRUNCATED_NORMAL, + {"mu": 15.0, "sigma": 1.0, "a": 0.0, "b": 10.0}, + ) + assert result == 10.0 + + def test_gamma_alpha_gt_1(self): + result = _compute_map(DistributionFamily.GAMMA, {"alpha": 3.0, "scale": 2.0}) + assert abs(result - 4.0) < 1e-10 + + def test_gamma_alpha_lt_1(self): + # alpha < 1: mode is degenerate (PDF → ∞ at 0), so we use the median + result = _compute_map(DistributionFamily.GAMMA, {"alpha": 0.5, "scale": 2.0}) + expected = float(stats.gamma(a=0.5, scale=2.0).median()) + assert result == pytest.approx(expected) + assert result > 0 # must be positive and evaluable + + def test_lognormal(self): + result = _compute_map(DistributionFamily.LOGNORMAL, {"mu": 1.0, "sigma": 0.5}) + expected = math.exp(1.0 - 0.25) + assert abs(result - expected) < 1e-10 + + def test_beta_standard(self): + result = _compute_map( + DistributionFamily.BETA, + {"alpha": 3.0, "beta": 5.0, "lower": 0.0, "upper": 1.0}, + ) + expected = (3.0 - 1.0) / (3.0 + 5.0 - 2.0) + assert abs(result - expected) < 1e-10 + + def test_beta_scaled(self): + result = _compute_map( + DistributionFamily.BETA, + {"alpha": 3.0, "beta": 3.0, "lower": 10.0, "upper": 20.0}, + ) + assert abs(result - 15.0) < 1e-10 + + def test_beta_alpha_le_1(self): + # alpha <= 1: mode at boundary where PDF diverges, use median instead + result = _compute_map( + DistributionFamily.BETA, + {"alpha": 0.5, "beta": 2.0, "lower": 0.0, "upper": 1.0}, + ) + expected = float(stats.beta(a=0.5, b=2.0).median()) + assert result == pytest.approx(expected) + assert 0 < result < 1 # must be interior and evaluable + + def test_beta_beta_le_1(self): + # beta <= 1: mode at upper boundary where PDF diverges, use median instead + result = _compute_map( + DistributionFamily.BETA, + {"alpha": 2.0, "beta": 0.5, "lower": 0.0, "upper": 1.0}, + ) + expected = float(stats.beta(a=2.0, b=0.5).median()) + assert result == pytest.approx(expected) + assert 0 < result < 1 + + def test_uniform(self): + result = _compute_map(DistributionFamily.UNIFORM, {"lower": 5.0, "upper": 15.0}) + assert result == 10.0 + + +# --------------------------------------------------------------------------- +# _compute_credible_coverage +# --------------------------------------------------------------------------- + + +class TestCredibleCoverage: + def test_all_inside(self): + dist = stats.norm(loc=0, scale=10) + values = np.array([0.0, 1.0, -1.0]) + cov = _compute_credible_coverage(dist, values) + assert cov.ci_50 > 0.0 + assert cov.ci_90 > 0.0 + assert cov.ci_95 > 0.0 + + def test_all_outside_50(self): + dist = stats.norm(loc=0, scale=1) + # Values well outside the 50% CI but inside 95% + values = np.array([-1.5, 1.5]) + cov = _compute_credible_coverage(dist, values) + assert cov.ci_50 == 0.0 + assert cov.ci_95 == 1.0 + + def test_monotonic(self): + dist = stats.norm(loc=0, scale=1) + rng = np.random.default_rng(42) + values = rng.normal(0, 1, size=1000) + cov = _compute_credible_coverage(dist, values) + assert cov.ci_50 <= cov.ci_90 <= cov.ci_95 + + +# --------------------------------------------------------------------------- +# _compute_cdf_deviation +# --------------------------------------------------------------------------- + + +class TestCdfDeviation: + def test_perfect_fit(self): + # Large sample from the true distribution → small deviation + dist = stats.norm(loc=0, scale=1) + rng = np.random.default_rng(42) + values = rng.normal(0, 1, size=10000) + dev = _compute_cdf_deviation(dist, values) + assert dev < 0.02 + + def test_mismatched(self): + # Data from N(0,1) tested against N(10,1) → large deviation + dist = stats.norm(loc=10, scale=1) + values = np.array([0.0, 0.5, 1.0, -0.5, -1.0]) + dev = _compute_cdf_deviation(dist, values) + assert dev > 0.5 + + +# --------------------------------------------------------------------------- +# check_model +# --------------------------------------------------------------------------- + + +class TestCheckModel: + def test_normal_good_fit(self): + """Data drawn from the same normal → high KS p-value.""" + prior = _make_prior(DistributionFamily.NORMAL, {"mu": 5.0, "sigma": 2.0}) + rng = np.random.default_rng(42) + values = rng.normal(5.0, 2.0, size=50).tolist() + mc = check_model(prior, values) + assert mc is not None + assert mc.map_estimate == 5.0 + assert mc.ks_pvalue > 0.05 + assert mc.n_values == 50 + + def test_single_value(self): + """Single data point still produces a result.""" + prior = _make_prior(DistributionFamily.NORMAL, {"mu": 3.0, "sigma": 1.0}) + mc = check_model(prior, [3.0]) + assert mc is not None + assert mc.n_values == 1 + assert mc.map_estimate == 3.0 + + def test_empty_values_returns_none(self): + prior = _make_prior(DistributionFamily.NORMAL, {"mu": 0, "sigma": 1}) + assert check_model(prior, []) is None + + def test_non_informative_returns_none(self): + prior = _make_prior( + DistributionFamily.UNIFORM, + {"lower": 0.0, "upper": 100.0}, + informative=False, + ) + assert check_model(prior, [1.0, 2.0, 3.0]) is None + + def test_gamma(self): + prior = _make_prior(DistributionFamily.GAMMA, {"alpha": 3.0, "scale": 2.0}) + rng = np.random.default_rng(42) + values = stats.gamma.rvs(a=3.0, scale=2.0, size=30, random_state=rng).tolist() + mc = check_model(prior, values) + assert mc is not None + assert mc.map_estimate == pytest.approx(4.0) + assert mc.ks_pvalue > 0.05 + + def test_truncated_normal(self): + prior = _make_prior( + DistributionFamily.TRUNCATED_NORMAL, + {"mu": 5.0, "sigma": 2.0, "a": 0.0, "b": 10.0}, + ) + mc = check_model(prior, [3.0, 5.0, 7.0, 4.0, 6.0]) + assert mc is not None + assert mc.map_estimate == 5.0 + assert 0.0 <= mc.credible_interval_coverage.ci_95 <= 1.0 + + def test_lognormal(self): + prior = _make_prior(DistributionFamily.LOGNORMAL, {"mu": 1.0, "sigma": 0.5}) + rng = np.random.default_rng(42) + values = stats.lognorm.rvs(s=0.5, scale=math.exp(1.0), size=40, random_state=rng).tolist() + mc = check_model(prior, values) + assert mc is not None + assert mc.ks_pvalue > 0.05 + + def test_beta(self): + prior = _make_prior( + DistributionFamily.BETA, + {"alpha": 2.0, "beta": 5.0, "lower": 0.0, "upper": 1.0}, + ) + rng = np.random.default_rng(42) + values = stats.beta.rvs(a=2.0, b=5.0, size=30, random_state=rng).tolist() + mc = check_model(prior, values) + assert mc is not None + assert mc.ks_pvalue > 0.05 + + def test_aic_computation(self): + prior = _make_prior(DistributionFamily.NORMAL, {"mu": 0.0, "sigma": 1.0}) + mc = check_model(prior, [0.0, 0.5, -0.5]) + assert mc is not None + expected_aic = 2 * 2 - 2 * mc.log_likelihood + assert mc.aic == pytest.approx(expected_aic) + + def test_coverage_monotonic(self): + prior = _make_prior(DistributionFamily.NORMAL, {"mu": 0.0, "sigma": 1.0}) + rng = np.random.default_rng(42) + values = rng.normal(0, 1, size=100).tolist() + mc = check_model(prior, values) + assert mc is not None + cov = mc.credible_interval_coverage + assert cov.ci_50 <= cov.ci_90 <= cov.ci_95 + + +# --------------------------------------------------------------------------- +# Offline utilities +# --------------------------------------------------------------------------- + + +class TestCheckModelFromResult: + def test_with_evidence(self): + evidence = [ + LiteratureEvidence( + title="Paper A", + extracted_values=[ + ExtractedValue(reported_value=5.0), + ExtractedValue(reported_value=6.0), + ], + ), + LiteratureEvidence( + title="Paper B", + extracted_values=[ + ExtractedValue(reported_value=4.5), + ], + ), + ] + prior = _make_prior( + DistributionFamily.NORMAL, + {"mu": 5.0, "sigma": 1.0}, + evidence=evidence, + ) + result = PipelineResult( + parameter=ParameterInput(name="test", description="test param"), + prior=prior, + ) + mc = check_model_from_result(result) + assert mc is not None + assert mc.n_values == 3 + + def test_no_evidence_returns_none(self): + prior = _make_prior(DistributionFamily.NORMAL, {"mu": 0, "sigma": 1}) + result = PipelineResult( + parameter=ParameterInput(name="test", description="test param"), + prior=prior, + ) + mc = check_model_from_result(result) + assert mc is None + + def test_with_constraints(self): + evidence = [ + LiteratureEvidence( + title="Paper", + extracted_values=[ + ExtractedValue(reported_value=5.0), + ExtractedValue(reported_value=100.0), # out of bounds + ExtractedValue(reported_value=7.0), + ], + ), + ] + prior = _make_prior( + DistributionFamily.NORMAL, + {"mu": 6.0, "sigma": 2.0}, + evidence=evidence, + ) + result = PipelineResult( + parameter=ParameterInput( + name="test", + description="test param", + constraints=ConstraintSpec(lower_bound=0.0, upper_bound=20.0), + ), + prior=prior, + ) + mc = check_model_from_result(result) + assert mc is not None + assert mc.n_values == 2 # 100.0 excluded + + +class TestCheckBatch: + def test_batch(self): + evidence = [ + LiteratureEvidence( + title="Paper", + extracted_values=[ExtractedValue(reported_value=5.0)], + ), + ] + prior = _make_prior( + DistributionFamily.NORMAL, + {"mu": 5.0, "sigma": 1.0}, + evidence=evidence, + ) + batch = BatchResult( + results=[ + PipelineResult( + parameter=ParameterInput(name="p1", description="desc"), + prior=prior, + ), + ] + ) + results = check_batch(batch) + assert len(results) == 1 + name, mc = results[0] + assert name == "p1" + assert mc is not None diff --git a/tests/test_validity.py b/tests/test_validity.py new file mode 100644 index 0000000..544f5a9 --- /dev/null +++ b/tests/test_validity.py @@ -0,0 +1,243 @@ +"""Unit tests for the validity classification heuristics.""" + +from __future__ import annotations + +from distribird.agent.validity import ( + REASON_EMPIRICAL_UNCLEAR, + REASON_LITERATURE_BACKED, + REASON_THEORETICAL_ONLY, + apply_probe_verdict, + classify_validity_passive, +) +from distribird.models import ( + ConfidenceLevel, + DistributionFamily, + EnrichedContext, + FittedPrior, + ParameterValidity, +) + + +def _make_prior( + informative: bool = True, confidence: ConfidenceLevel = ConfidenceLevel.HIGH +) -> FittedPrior: + return FittedPrior( + parameter_name="test", + family=DistributionFamily.NORMAL, + params={"mu": 0.5, "sigma": 0.1}, + confidence=confidence, + is_informative=informative, + n_sources=5, + ) + + +def test_rule_no_literature_after_refinement_is_likely_invalid(): + enrichment = EnrichedContext( + is_recognized_parameter=False, + recognition_confidence="none", + ) + verdict, reason, signals, is_empirical = classify_validity_passive( + enrichment=enrichment, + prior=_make_prior(informative=False, confidence=ConfidenceLevel.NONE), + papers_found=0, + values_extracted=0, + queries_tried=3, + ) + assert verdict == ParameterValidity.LIKELY_INVALID + assert "3 refined queries" in reason + assert signals["papers_found"] == 0 + assert signals["n_queries_tried"] == 3 + + +def test_rule_unrecognized_no_literature_is_likely_invalid(): + enrichment = EnrichedContext( + is_recognized_parameter=False, + recognition_confidence="none", + common_terminology=["something"], # has terminology but unrecognized + ) + verdict, reason, _, _ = classify_validity_passive( + enrichment=enrichment, + prior=_make_prior(informative=False, confidence=ConfidenceLevel.NONE), + papers_found=0, + values_extracted=0, + queries_tried=1, # only one query, so rule 1 doesn't fire + ) + assert verdict == ParameterValidity.LIKELY_INVALID + assert "did not recognize" in reason.lower() + + +def test_rule_no_terminology_no_papers_is_likely_invalid(): + enrichment = EnrichedContext( + is_recognized_parameter=None, # LLM didn't say + common_terminology=[], + ) + verdict, _, _, _ = classify_validity_passive( + enrichment=enrichment, + prior=_make_prior(informative=False, confidence=ConfidenceLevel.NONE), + papers_found=0, + values_extracted=0, + queries_tried=1, + ) + assert verdict == ParameterValidity.LIKELY_INVALID + + +def test_rule_theoretical_only_is_suspicious(): + enrichment = EnrichedContext( + is_recognized_parameter=True, + recognition_confidence="medium", + empirically_measured=False, + common_terminology=["model term", "theoretical quantity"], + ) + verdict, reason, _, is_empirical = classify_validity_passive( + enrichment=enrichment, + prior=_make_prior(informative=False, confidence=ConfidenceLevel.NONE), + papers_found=5, + values_extracted=0, + queries_tried=2, + ) + assert verdict == ParameterValidity.SUSPICIOUS + assert reason == REASON_THEORETICAL_ONLY + assert is_empirical is False + + +def test_rule_strong_positive_signal_is_valid(): + enrichment = EnrichedContext( + is_recognized_parameter=True, + recognition_confidence="high", + empirically_measured=True, + common_terminology=["specific leaf area", "SLA", "leaf area mass ratio"], + ) + verdict, reason, _, is_empirical = classify_validity_passive( + enrichment=enrichment, + prior=_make_prior(informative=True, confidence=ConfidenceLevel.HIGH), + papers_found=8, + values_extracted=15, + queries_tried=1, + ) + assert verdict == ParameterValidity.VALID + assert reason == REASON_LITERATURE_BACKED + assert is_empirical is True + + +def test_rule_ambiguous_defaults_to_suspicious(): + """Recognized parameter with literature but only LOW confidence → suspicious.""" + enrichment = EnrichedContext( + is_recognized_parameter=True, + recognition_confidence="low", + common_terminology=["maybe relevant"], + ) + verdict, _, _, _ = classify_validity_passive( + enrichment=enrichment, + prior=_make_prior(informative=True, confidence=ConfidenceLevel.LOW), + papers_found=2, + values_extracted=1, + queries_tried=2, + ) + assert verdict == ParameterValidity.SUSPICIOUS + + +def test_rule_4b_uncertain_empirical_with_papers_zero_values_is_suspicious(): + """LLM uncertain about empirical status + papers exist + 0 values → SUSPICIOUS via Rule 4b.""" + enrichment = EnrichedContext( + is_recognized_parameter=True, + recognition_confidence="medium", + empirically_measured=None, + common_terminology=["model parameter", "tuning weight"], + ) + verdict, reason, _, is_empirical = classify_validity_passive( + enrichment=enrichment, + prior=_make_prior(informative=False, confidence=ConfidenceLevel.NONE), + papers_found=5, + values_extracted=0, + queries_tried=2, + ) + assert verdict == ParameterValidity.SUSPICIOUS + assert reason == REASON_EMPIRICAL_UNCLEAR + assert is_empirical is None + + +def test_rule_5_blocked_when_is_recognized_is_none(): + """LLM uncertain about recognition + high-confidence prior → default SUSPICIOUS.""" + enrichment = EnrichedContext( + is_recognized_parameter=None, # LLM uncertain + recognition_confidence="medium", + empirically_measured=True, + common_terminology=["something"], + ) + verdict, _, _, _ = classify_validity_passive( + enrichment=enrichment, + prior=_make_prior(informative=True, confidence=ConfidenceLevel.HIGH), + papers_found=8, + values_extracted=8, + queries_tried=1, + ) + assert verdict == ParameterValidity.SUSPICIOUS + + +def test_rule_5_blocked_when_is_empirical_is_false(): + """High-confidence prior + recognized + values, but LLM said theoretical → not VALID.""" + enrichment = EnrichedContext( + is_recognized_parameter=True, + recognition_confidence="high", + empirically_measured=False, # theoretical-only + common_terminology=["calibration weight"], + ) + verdict, _, _, is_empirical = classify_validity_passive( + enrichment=enrichment, + prior=_make_prior(informative=True, confidence=ConfidenceLevel.HIGH), + papers_found=5, + values_extracted=5, # somehow values were extracted from model docs + queries_tried=1, + ) + assert verdict != ParameterValidity.VALID + assert verdict == ParameterValidity.SUSPICIOUS + assert is_empirical is False + + +def test_apply_probe_verdict_upgrades_suspicious(): + probe = {"verdict": "valid", "is_empirical": True, "reason": "actually a real param"} + verdict, reason, empirical = apply_probe_verdict( + passive_verdict=ParameterValidity.SUSPICIOUS, + passive_reason="ambiguous", + passive_is_empirical=None, + probe_result=probe, + ) + assert verdict == ParameterValidity.VALID + assert reason == "actually a real param" + assert empirical is True + + +def test_apply_probe_verdict_does_not_override_likely_invalid(): + probe = {"verdict": "valid", "is_empirical": True, "reason": "false probe"} + verdict, reason, _ = apply_probe_verdict( + passive_verdict=ParameterValidity.LIKELY_INVALID, + passive_reason="no literature found", + passive_is_empirical=None, + probe_result=probe, + ) + assert verdict == ParameterValidity.LIKELY_INVALID + assert reason == "no literature found" + + +def test_apply_probe_verdict_handles_none_probe(): + verdict, reason, empirical = apply_probe_verdict( + passive_verdict=ParameterValidity.SUSPICIOUS, + passive_reason="ambiguous", + passive_is_empirical=False, + probe_result=None, + ) + assert verdict == ParameterValidity.SUSPICIOUS + assert reason == "ambiguous" + assert empirical is False + + +def test_apply_probe_verdict_handles_invalid_verdict_string(): + probe = {"verdict": "garbage_value", "is_empirical": True, "reason": "x"} + verdict, reason, _ = apply_probe_verdict( + passive_verdict=ParameterValidity.SUSPICIOUS, + passive_reason="ambiguous", + passive_is_empirical=None, + probe_result=probe, + ) + assert verdict == ParameterValidity.SUSPICIOUS + assert reason == "ambiguous"