From 8f32eafbd24973bb8e0fe7890d2abb61935bf5a5 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Wed, 17 Jun 2026 22:10:17 -0400 Subject: [PATCH 01/28] Add prkit.verify facade, enrich Verdict, make AccuracyEvaluator scorer-backed - New light-import prkit.verify (parse/verify) wrapping SemanticsScorer; an import-isolation test asserts no clients/hub/datasets/pandas on the path. - Enrich Verdict with correct/units_ok/symbolic_equiv/numeric_within_tol/ extracted_answer (derived losslessly in scoring/_adapt.py); partial_credit and rationale reserved as None. - AccuracyEvaluator gains an injectable scorer= (default SemanticsScorer) and routes evaluate() through it; the legacy comparator= path is preserved. - Broaden SemanticsScorer.score to accept PhysicsAnswerSemantics inputs. - Bump package to 0.2.0; document the facade and Verdict fields in CONTRACT.md and README. Co-Authored-By: Claude Opus 4.8 --- README.md | 16 +++ pyproject.toml | 2 +- src/prkit/CONTRACT.md | 61 ++++++++-- src/prkit/api.py | 7 ++ src/prkit/core/verdict.py | 53 +++++++- src/prkit/evaluation/evaluator/accuracy.py | 108 ++++++++++------ src/prkit/scoring/_adapt.py | 93 +++++++++++++- src/prkit/scoring/semantics_scorer.py | 18 ++- src/prkit/verify/__init__.py | 100 +++++++++++++++ .../evaluation/evaluator/test_accuracy.py | 29 ++++- tests/prkit/scoring/test_adapt.py | 115 +++++++++++++++++- tests/prkit/verify/__init__.py | 0 tests/prkit/verify/test_import_isolation.py | 57 +++++++++ tests/prkit/verify/test_verify.py | 82 +++++++++++++ 14 files changed, 685 insertions(+), 56 deletions(-) create mode 100644 src/prkit/verify/__init__.py create mode 100644 tests/prkit/verify/__init__.py create mode 100644 tests/prkit/verify/test_import_isolation.py create mode 100644 tests/prkit/verify/test_verify.py diff --git a/README.md b/README.md index 052ebf0..e40e117 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,22 @@ for problem in dataset[:3]: The same pattern works across different datasets and model providers—swap the dataset name or model identifier. +#### Just verify an answer (`prkit.verify`) + +For the standalone "is this physics answer right?" use case, use the light-import +verifier—a `math-verify`-shaped API that, unlike `math-verify`, is unit- and +symbolic-aware and imports no model clients, dataset hub, or provider SDKs: + +```python +from prkit.verify import parse, verify + +v = verify("9.8 m/s^2", "9.8 m/s²") # verify(gold, pred) -> Verdict +v.correct # True — the unit suffix normalizes (math-verify strips units) +v.units_ok # True +v.symbolic_equiv # None (numeric case); True for e.g. verify("v = a t", "v = t a") +v.scorer_version # stamped so a stored score is attributable to its scorer +``` + ### 📖 Documentation **Quick Links:** diff --git a/pyproject.toml b/pyproject.toml index ebb0cf6..dcc4b34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "physical-reasoning-toolkit" -version = "0.1.0.post22" +version = "0.2.0" description = "A toolkit for physical-reasoning datasets, multi-provider LLM inference, answer evaluation, and annotation." readme = {file = "README.md", content-type = "text/markdown"} license = "MIT" diff --git a/src/prkit/CONTRACT.md b/src/prkit/CONTRACT.md index 5459bce..77cfea6 100644 --- a/src/prkit/CONTRACT.md +++ b/src/prkit/CONTRACT.md @@ -4,10 +4,27 @@ (eval harnesses, RL trainers, dataset hubs) should target. This document defines what is stable, how it is versioned, and how things get deprecated. +## Headline entry point: `prkit.verify` + +If all you want is to verify a physics answer, use the light-import facade: + +```python +from prkit.verify import parse, verify +v = verify("9.8 m/s^2", "9.8 m/s²") # verify(gold, pred) -> Verdict +v.correct # True +v.units_ok # True (the unit suffix normalizes; math-verify would strip it) +``` + +`prkit.verify` imports **no** provider SDKs, dataset hub, `datasets`, or pandas — +the boundary is enforced by `tests/prkit/verify/test_import_isolation.py`. It is a +thin, `math-verify`-shaped wrapper over the reference `prkit.scoring.SemanticsScorer` +and returns the same canonical `Verdict`. + ## Stable surface -- **Only names exported in `prkit.api.__all__` are stable.** Everything else — - module paths, private helpers, subpackage internals — may change without notice. +- **Only names exported in `prkit.api.__all__` are stable**, plus the + `prkit.verify` facade (`parse`, `verify`). Everything else — module paths, + private helpers, subpackage internals — may change without notice. - The conformance suite in `prkit.testing` (`check_dataset`, `check_scorer`, `check_model_client`, `ConformanceTestMixin`) is a stable companion: use it to verify your own loader/scorer/client satisfies the contract. @@ -27,6 +44,27 @@ The contract pins four structural (`typing.Protocol`) nouns plus one result type > types. It is necessary but not sufficient; the behavioral gate is > `prkit.testing.check_*`, which actually calls the methods and asserts results. +### The `Verdict` fields + +`Verdict` is frozen (`extra="forbid"`). The **core** fields are always populated; +the **enriched** fields are derived losslessly from the comparison and are `None` +when not applicable (or not yet produced): + +| Field | Kind | Meaning | +|-------|------|---------| +| `equivalent` / `correct` | core | primary pass/fail (`correct` mirrors `equivalent` by default) | +| `score` | core | continuous score in `[0,1]`; binary scorers emit `1.0`/`0.0` | +| `comparison_mode` | core | how the verdict was reached (`number`, `expression`, …) | +| `scorer_version` | core | Gymnasium-style stamp of the scorer revision | +| `diagnostics` | core | machine-readable mismatch/fallback tags | +| `details` | core | scorer-specific evidence (bridge ids, policy mode, …) | +| `units_ok` | enriched | dimensional check satisfied; `None` when units don't participate | +| `symbolic_equiv` | enriched | equivalence decided symbolically; `None` for non-symbolic modes | +| `numeric_within_tol` | enriched | numeric/quantity match within tolerance; `None` otherwise | +| `extracted_answer` | enriched | parsed prediction surface, when available | +| `partial_credit` | enriched | **reserved (X1)** — the deterministic engine is binary, so this is always `None` today | +| `rationale` | enriched | **reserved** — no NL rationale from the deterministic engine (`None` today) | + ## Three independent version axes Do not conflate these — they move independently: @@ -50,9 +88,12 @@ hub backfills it in `DatasetHub.get_loader_info`). - **MAJOR** (`1.0` → `2.0`): any removal or signature change to a name in `prkit.api.__all__`. -Re-routing an existing implementation in a way that changes its observable -behavior (e.g. switching `AccuracyEvaluator`'s default comparison semantics) is a -**major** change and must bump `API_VERSION` accordingly. +Re-routing an existing implementation **that is part of `prkit.api.__all__`** in a +way that changes its observable behavior is a **major** change and must bump +`API_VERSION`. This does **not** apply to already-deprecated classes outside the +contract surface (e.g. `AccuracyEvaluator`, whose default was repointed at the +`Scorer` contract — see *Current deprecations*); such changes are documented in the +package release notes, not the contract version. ## Deprecation policy @@ -67,7 +108,13 @@ behavior (e.g. switching `AccuracyEvaluator`'s default comparison semantics) is **`AccuracyEvaluator`** are deprecated in favor of `prkit.scoring.SemanticsScorer` (the `Scorer` / `Verdict` contract), which wraps the deterministic semantics comparison engine. Constructing any of them emits a - `DeprecationWarning`. Their runtime behavior is unchanged in this release; they - will be removed no earlier than the next minor release. + `DeprecationWarning`. They will be removed no earlier than the next minor release + — but not before downstream consumers migrate off the comparator stack. +- **Behavior change (0.2.0):** `AccuracyEvaluator` now takes an injectable + `scorer=` and **defaults to `SemanticsScorer`** instead of `ExactMatchComparator`. + `evaluate()` shapes its legacy result dict from the returned `Verdict` + (`accuracy_score=score`, `comparison_result=equivalent`, plus `scorer_version` / + `comparison_mode` in `details`). Passing a `comparator=` still selects the old, + unchanged comparator path; passing both `scorer=` and `comparator=` raises. - `prkit.evaluation.llm_judge` (model-graded scoring) is **not** deprecated — it is a distinct capability, not a duplicate of the deterministic scoring path. diff --git a/src/prkit/api.py b/src/prkit/api.py index 9cf68cc..5915198 100644 --- a/src/prkit/api.py +++ b/src/prkit/api.py @@ -10,6 +10,11 @@ *without subclassing*, and re-exports the existing concrete anchors the protocols are grounded in. +For the headline "just verify a physics answer" use case, integrators should +reach for the light-import facade :mod:`prkit.verify` (``parse`` / ``verify``), +which returns the same :class:`Verdict` without importing clients, the hub, or +provider SDKs. + .. note:: ``@runtime_checkable`` only verifies that the named **methods/attributes exist** on an instance — it does **not** check signatures or return types. @@ -38,6 +43,8 @@ # --- contract version (independent of prkit.__version__) ------------------ # Bump per CONTRACT.md: additive change -> minor, breaking change -> major. +# The 0.2.0 additions (the Verdict superset fields + the prkit.verify facade) are +# fully backward compatible; the contract version is held at 1.0 by decision. API_VERSION = "1.0" diff --git a/src/prkit/core/verdict.py b/src/prkit/core/verdict.py index f90661f..f9ba32f 100644 --- a/src/prkit/core/verdict.py +++ b/src/prkit/core/verdict.py @@ -15,13 +15,13 @@ from typing import Any -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator class Verdict(BaseModel): """Frozen, version-stamped result of scoring a prediction against a reference. - Fields: + Core fields: equivalent: Primary pass/fail. Mirrors ``AnswerComparison.equivalent``. score: Continuous score in ``[0, 1]``. Binary scorers emit ``1.0``/``0.0``. comparison_mode: How the verdict was reached (e.g. ``"number"``, @@ -31,6 +31,26 @@ class Verdict(BaseModel): diagnostics: Machine-readable notes explaining mismatches or fallback paths. details: Escape hatch for scorer-specific evidence (bridge ids, policy mode, validation status, ...). Values must be JSON-serializable. + + Enriched fields (the keystone superset; ``None`` when not applicable / not yet + produced). These are derived losslessly from the comparison and the normalized + answers by :func:`prkit.scoring._adapt.verdict_from_comparison`: + correct: Readability alias of ``equivalent`` (downstream RL rewards read + ``correct``). Defaults to mirror ``equivalent`` when not set explicitly. + units_ok: Whether the dimensional/unit check was satisfied; ``None`` when + units do not participate in the comparison. + symbolic_equiv: Whether equivalence was decided on a symbolic path + (expression/relation/...); ``None`` for non-symbolic modes. + numeric_within_tol: Whether a numeric/quantity comparison fell within + tolerance; ``None`` for non-numeric modes. + extracted_answer: The parsed/canonical surface of the prediction, when the + normalized prediction is available to the adapter. + partial_credit: Continuous partial-credit signal. # to define (X1): the + deterministic engine is strictly binary today, so this is always + ``None`` until the partial-credit scorer (X1) produces it. + rationale: Human-readable explanation. # to define: the deterministic + engine emits no natural-language rationale, so this is ``None`` for now + (a model-graded scorer would populate it). """ model_config = ConfigDict(extra="forbid", frozen=True) @@ -42,6 +62,27 @@ class Verdict(BaseModel): diagnostics: tuple[str, ...] = Field(default_factory=tuple) details: dict[str, Any] = Field(default_factory=dict) + # --- enriched superset (additive; API_VERSION stays 1.0) ------------------ + correct: bool | None = None + units_ok: bool | None = None + symbolic_equiv: bool | None = None + numeric_within_tol: bool | None = None + extracted_answer: str | None = None + partial_credit: float | None = None # to define (X1): binary engine → None + rationale: str | None = None # to define: no NL rationale from det. engine + + @model_validator(mode="before") + @classmethod + def _default_correct_to_equivalent(cls, data: Any) -> Any: + """Default ``correct`` to mirror ``equivalent`` when not set explicitly.""" + if ( + isinstance(data, dict) + and data.get("correct") is None + and "equivalent" in data + ): + data = {**data, "correct": data["equivalent"]} + return data + @field_validator("score") @classmethod def _score_in_range(cls, value: float) -> float: @@ -49,3 +90,11 @@ def _score_in_range(cls, value: float) -> float: if not (0.0 <= value <= 1.0): raise ValueError(f"score must be in [0, 1], got {value!r}") return value + + @field_validator("partial_credit") + @classmethod + def _partial_credit_in_range(cls, value: float | None) -> float | None: + """Reject partial-credit outside ``[0, 1]`` (``None`` means not produced).""" + if value is not None and not (0.0 <= value <= 1.0): + raise ValueError(f"partial_credit must be in [0, 1], got {value!r}") + return value diff --git a/src/prkit/evaluation/evaluator/accuracy.py b/src/prkit/evaluation/evaluator/accuracy.py index b19c189..388375f 100644 --- a/src/prkit/evaluation/evaluator/accuracy.py +++ b/src/prkit/evaluation/evaluator/accuracy.py @@ -1,31 +1,67 @@ -"""Accuracy evaluator that scores predicted answers against ground truth using a configurable comparator.""" +"""Accuracy evaluator backed by the canonical ``Scorer`` contract (or a legacy comparator). + +By default the evaluator scores answers through :class:`~prkit.scoring.SemanticsScorer` +— the :class:`prkit.api.Scorer` / :class:`~prkit.core.verdict.Verdict` contract — and +shapes its long-standing result dict from the returned ``Verdict``. A different +``Scorer`` can be injected (e.g. a future semantics+LLM scorer); passing a legacy +``comparator`` instead selects the deprecated comparator path unchanged. +""" + +from __future__ import annotations from collections.abc import Callable -from typing import Any +from typing import TYPE_CHECKING, Any from prkit.core.domain.answer import Answer from prkit.core.domain.physics_dataset import PhysicalDataset from prkit.core.domain.physics_problem import PhysicsProblem from prkit.evaluation.comparator.base import BaseComparator -from prkit.evaluation.comparator.exact_match import ExactMatchComparator +from prkit.scoring import SemanticsScorer from .base import BaseEvaluator +if TYPE_CHECKING: # typing-only; avoids importing the api surface at runtime + from prkit.api import Scorer + class AccuracyEvaluator(BaseEvaluator): - """Evaluator that uses a comparator to evaluate answers and datasets.""" + """Evaluator that scores answers via an injectable ``Scorer`` (or legacy comparator). - def __init__(self, comparator: BaseComparator | None = None) -> None: - """ - Initialize the accuracy evaluator. + The default (no ``comparator``, no ``scorer``) is backed by + :class:`~prkit.scoring.SemanticsScorer`, so results conform to the canonical + ``Verdict`` contract. Only the ``SemanticsScorer`` path is exercised today; the + seam is intentionally open for other ``Scorer`` implementations later. + """ + + def __init__( + self, + comparator: BaseComparator | None = None, + *, + scorer: Scorer | None = None, + ) -> None: + """Initialize the accuracy evaluator. Args: - comparator: Comparator instance to use. If None, defaults to - ExactMatchComparator. + comparator: Legacy comparator (the deprecated path). Mutually exclusive + with ``scorer``. + scorer: ``Scorer`` to back evaluation. When neither ``comparator`` nor + ``scorer`` is given, defaults to :class:`~prkit.scoring.SemanticsScorer`. """ - if comparator is None: - comparator = ExactMatchComparator() + if comparator is not None and scorer is not None: + raise ValueError("Pass either scorer= or comparator=, not both.") + if comparator is None and scorer is None: + scorer = SemanticsScorer() + # BaseEvaluator stores the (possibly None) comparator and emits the stack's + # DeprecationWarning; the dataset-level harness is slated for the N4 Runner. super().__init__(comparator) + self.scorer = scorer + + @staticmethod + def _describe_answer(answer: str | Answer) -> tuple[str, str]: + """Return ``(value, type)`` surface strings for a result's ``details`` block.""" + if isinstance(answer, Answer): + return str(answer.value), answer.answer_category.value + return str(answer), "string" def evaluate( self, @@ -44,13 +80,34 @@ def evaluate( Returns: Dictionary containing evaluation results: - accuracy_score: Accuracy score in [0, 1] - - comparison_result: Raw comparison result from comparator + - comparison_result: Raw pass/fail comparison result - details: Additional evaluation details """ + pred_val, pred_type = self._describe_answer(predicted_answer) + gt_val, gt_type = self._describe_answer(ground_truth_answer) + + if self.scorer is not None: + # Scorer contract: score(prediction, reference) -> Verdict, then shape + # the legacy dict from it (accuracy_score=score, comparison=equivalent). + verdict = self.scorer.score(predicted_answer, ground_truth_answer) + return { + "accuracy_score": verdict.score, + "comparison_result": verdict.equivalent, + "details": { + "predicted_value": pred_val, + "ground_truth_value": gt_val, + "predicted_type": pred_type, + "ground_truth_type": gt_type, + "scorer_type": type(self.scorer).__name__, + "scorer_version": verdict.scorer_version, + "comparison_mode": verdict.comparison_mode, + }, + } + + # Legacy comparator path (deprecated) — unchanged behavior. if self.comparator is None: raise ValueError("Comparator must be set before evaluation") - # Perform comparison comparison_result = self.comparator.compare( predicted_answer, ground_truth_answer ) @@ -58,27 +115,6 @@ def evaluate( predicted_answer, ground_truth_answer ) - pred_val = ( - str(predicted_answer.value) - if isinstance(predicted_answer, Answer) - else str(predicted_answer) - ) - gt_val = ( - str(ground_truth_answer.value) - if isinstance(ground_truth_answer, Answer) - else str(ground_truth_answer) - ) - pred_type = ( - predicted_answer.answer_category.value - if isinstance(predicted_answer, Answer) - else "string" - ) - gt_type = ( - ground_truth_answer.answer_category.value - if isinstance(ground_truth_answer, Answer) - else "string" - ) - return { "accuracy_score": accuracy_score, "comparison_result": comparison_result, @@ -120,8 +156,8 @@ def evaluate_dataset( - per_problem_results: List of individual evaluation results - statistics: Additional statistics (by domain, problem_type, etc.) """ - if self.comparator is None: - raise ValueError("Comparator must be set before evaluation") + if self.scorer is None and self.comparator is None: + raise ValueError("A scorer or comparator must be set before evaluation") if predicted_answers is None and answer_extractor is None: raise ValueError( diff --git a/src/prkit/scoring/_adapt.py b/src/prkit/scoring/_adapt.py index af9b0e7..33854de 100644 --- a/src/prkit/scoring/_adapt.py +++ b/src/prkit/scoring/_adapt.py @@ -11,7 +11,34 @@ from enum import Enum from prkit.core.verdict import Verdict -from prkit.semantics import AnswerComparison +from prkit.semantics import AnswerComparison, PhysicsAnswerSemantics + +# ``comparison_mode`` values on a symbolic decision path (verified against +# src/prkit/semantics/comparison/{same,different}_object_kind.py + bridge_registry). +_SYMBOLIC_MODES = frozenset( + { + "expression", + "relation", + "relation_to_expression", + "relation_rhs", + "expression_to_number", + "expression_quantity", + "relation_to_qualitative_label", + } +) + +# ``comparison_mode`` values on a numeric/quantity tolerance path. +_NUMERIC_MODES = frozenset({"number", "physical_quantity", "quantity_to_number"}) + +# Diagnostic tags the engine appends when the dimensional/unit check fails. +_UNIT_FAIL_TAGS = frozenset( + { + "unit_mismatch", + "question_unit_mismatch", + "missing_required_unit", + "unit_forbidden", + } +) def _enum_to_str(value: Enum | None) -> str | None: @@ -19,15 +46,70 @@ def _enum_to_str(value: Enum | None) -> str | None: return None if value is None else str(value) +def _symbolic_equiv(comparison: AnswerComparison) -> bool | None: + """Whether equivalence was decided symbolically (``None`` if not a symbolic mode).""" + if comparison.comparison_mode not in _SYMBOLIC_MODES: + return None + return comparison.equivalent + + +def _numeric_within_tol(comparison: AnswerComparison) -> bool | None: + """Whether a numeric/quantity match held within tolerance (``None`` if N/A).""" + if comparison.comparison_mode not in _NUMERIC_MODES: + return None + if "numeric_value_mismatch" in comparison.diagnostics: + return False + return comparison.equivalent + + +def _units_ok( + comparison: AnswerComparison, + pred_sem: PhysicsAnswerSemantics | None, + ref_sem: PhysicsAnswerSemantics | None, +) -> bool | None: + """Whether the dimensional/unit check was satisfied. + + Returns ``False`` when a unit-failure diagnostic is present; ``True`` when units + demonstrably participate (either side carries a unit, or the mode is + ``physical_quantity``) and no unit-failure was raised; ``None`` when units do + not participate or the normalized answers were not supplied (legacy callers). + """ + if any(tag in comparison.diagnostics for tag in _UNIT_FAIL_TAGS): + return False + units_present = comparison.comparison_mode == "physical_quantity" or ( + (pred_sem is not None and pred_sem.unit is not None) + or (ref_sem is not None and ref_sem.unit is not None) + ) + # Only assert success when we can see that units participated; otherwise a + # clean (no-diagnostic) comparison is indistinguishable from "no units". + if units_present and (pred_sem is not None or ref_sem is not None): + return True + return None + + def verdict_from_comparison( - comparison: AnswerComparison, *, scorer_version: str + comparison: AnswerComparison, + *, + scorer_version: str, + pred_sem: PhysicsAnswerSemantics | None = None, + ref_sem: PhysicsAnswerSemantics | None = None, ) -> Verdict: - """Map an :class:`AnswerComparison` onto the minimal canonical :class:`Verdict`. + """Map an :class:`AnswerComparison` onto the canonical :class:`Verdict`. The deterministic engine emits a binary verdict, so ``score`` is ``1.0`` when equivalent and ``0.0`` otherwise — no partial credit is manufactured. The bridge/policy/validation evidence is preserved verbatim under ``details``. + + The enriched fields (``units_ok`` / ``symbolic_equiv`` / ``numeric_within_tol`` + / ``extracted_answer``) are derived losslessly from ``comparison_mode`` + + ``diagnostics`` and, when supplied, the normalized ``pred_sem`` / ``ref_sem``. + ``partial_credit`` / ``rationale`` are left ``None`` (the engine produces + neither today; see ``Verdict``). """ + extracted = None + if pred_sem is not None: + extracted = pred_sem.canonical_text or pred_sem.raw_text + return Verdict( equivalent=comparison.equivalent, score=1.0 if comparison.equivalent else 0.0, @@ -42,4 +124,9 @@ def verdict_from_comparison( "validation_status": _enum_to_str(comparison.validation_status), "surface_shortcut_used": comparison.surface_shortcut_used, }, + correct=comparison.equivalent, + units_ok=_units_ok(comparison, pred_sem, ref_sem), + symbolic_equiv=_symbolic_equiv(comparison), + numeric_within_tol=_numeric_within_tol(comparison), + extracted_answer=extracted, ) diff --git a/src/prkit/scoring/semantics_scorer.py b/src/prkit/scoring/semantics_scorer.py index adddcca..980e8b7 100644 --- a/src/prkit/scoring/semantics_scorer.py +++ b/src/prkit/scoring/semantics_scorer.py @@ -20,6 +20,7 @@ PREDICTION_PROMPT_VERSION, REFERENCE_PROMPT_VERSION, ComparisonPolicyMode, + PhysicsAnswerSemantics, PhysicsEvaluationContract, PhysicsQuestionSemantics, QuestionUnitPolicy, @@ -122,8 +123,8 @@ def _effective_context( def score( self, - prediction: Answer | str, - reference: Answer | str, + prediction: Answer | str | PhysicsAnswerSemantics, + reference: Answer | str | PhysicsAnswerSemantics, *, context: PhysicsQuestionSemantics | dict[str, Any] | None = None, policy_mode: ComparisonPolicyMode | str | None = None, @@ -131,6 +132,12 @@ def score( ) -> Verdict: """Score ``prediction`` against ``reference`` and return a canonical Verdict. + ``prediction`` / ``reference`` may be raw strings, :class:`Answer` objects, + or already-normalized :class:`PhysicsAnswerSemantics` (e.g. from + :func:`prkit.verify.parse`); all three are accepted by the normalizer. The + wider input type stays compatible with the narrower :class:`prkit.api.Scorer` + protocol by parameter contravariance. + Per-call ``context`` / ``policy_mode`` override the instance defaults so a runner can pass question-conditioned semantics per problem. """ @@ -148,7 +155,12 @@ def score( context=effective_context, policy_mode=effective_policy, ) - return verdict_from_comparison(comparison, scorer_version=self.version) + return verdict_from_comparison( + comparison, + scorer_version=self.version, + pred_sem=pred_sem, + ref_sem=ref_sem, + ) def get_info(self) -> dict[str, Any]: """Return scorer metadata; always includes ``version``.""" diff --git a/src/prkit/verify/__init__.py b/src/prkit/verify/__init__.py new file mode 100644 index 0000000..01b5574 --- /dev/null +++ b/src/prkit/verify/__init__.py @@ -0,0 +1,100 @@ +"""Standalone physics verifier — the light-import, ``math-verify``-shaped entry point. + +This is the headline public surface for third parties who just want to verify a +physics answer:: + + from prkit.verify import parse, verify + verdict = verify("9.81 m/s^2", "9.8 m/s²") # verify(gold, pred) -> Verdict + +Import discipline (the whole point of this subpackage): ``import prkit.verify`` +must NOT pull in provider SDKs (anthropic / openai / google.genai), the dataset +hub, the ``datasets`` library, or pandas. The heavy :class:`~prkit.scoring.SemanticsScorer` +(and its ``sympy`` dependency) is therefore imported *lazily inside the functions*, +so importing this module stays near-instant and dependency-light. This boundary is +enforced by ``tests/prkit/verify/test_import_isolation.py``, not just convention. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from prkit.core.verdict import Verdict + +if TYPE_CHECKING: # annotations only — never imported at runtime by this module + from prkit.core.domain.answer import Answer + from prkit.semantics import PhysicsAnswerSemantics + +__all__ = ["parse", "verify", "Verdict"] + +# A ``verify(unit_policy=...)`` value maps onto the engine's enforcement-strictness +# axis (``ComparisonPolicyMode``). Finer-grained per-question unit rules +# (required / forbidden / optional) live on the separate ``QuestionUnitPolicy`` axis +# and are reachable via ``SemanticsScorer(context=...)``, not this facade. +_RECOGNIZED_UNIT_POLICIES = ("strict", "audited", "permissive") + + +def parse(text: str, *, category: object | None = None) -> PhysicsAnswerSemantics: + """Normalize a raw answer surface into typed physics semantics. + + Mirrors ``math_verify.parse``. ``category`` is reserved for a future + answer-category hint; it is not yet wired into the deterministic normalizer, so + passing a non-``None`` value raises ``NotImplementedError`` rather than being + silently ignored. + """ + if category is not None: + raise NotImplementedError( + "parse(category=...) is not supported yet; pass category=None." + ) + # Lazy: defers sympy / the semantics layer off the import path. + from prkit.semantics import normalize_physics_answer + + return normalize_physics_answer(text) + + +def verify( + gold: Answer | str | PhysicsAnswerSemantics, + pred: Answer | str | PhysicsAnswerSemantics, + *, + tolerance: float | None = None, + unit_policy: str = "strict", + partial_credit: bool = False, +) -> Verdict: + """Verify a predicted physics answer against the gold answer. + + A ``math-verify``-shaped one-call verifier returning the canonical + :class:`~prkit.core.verdict.Verdict`. ``gold`` / ``pred`` may be raw strings, + :class:`~prkit.core.domain.answer.Answer` objects, or pre-parsed + :class:`~prkit.semantics.PhysicsAnswerSemantics`. + + Args: + gold: the reference (correct) answer. Order mirrors ``math_verify.verify``. + pred: the predicted answer to check. + tolerance: numeric comparison tolerance (engine default when ``None``). + unit_policy: enforcement strictness — one of ``"strict"`` / ``"audited"`` / + ``"permissive"`` (maps to the engine's ``ComparisonPolicyMode``). + partial_credit: reserved. The deterministic engine is binary, so ``True`` + raises ``NotImplementedError`` (the partial-credit scorer, X1, owns it). + + Raises: + ValueError: if ``unit_policy`` is not a recognized value. + NotImplementedError: if ``partial_credit=True``. + """ + if partial_credit: + raise NotImplementedError( + "partial_credit is not produced by the deterministic verifier; " + "use the partial-credit scorer (X1) when it lands." + ) + if unit_policy not in _RECOGNIZED_UNIT_POLICIES: + raise ValueError( + f"unit_policy must be one of {list(_RECOGNIZED_UNIT_POLICIES)}, " + f"got {unit_policy!r}" + ) + + # Lazy: keeps anthropic/openai/google.genai/datasets/pandas/sympy off the + # bare ``import prkit.verify`` path (provider SDKs are lazy in model_clients). + from prkit.scoring import SemanticsScorer + + scorer = SemanticsScorer(tolerance=tolerance, policy_mode=unit_policy) + # math-verify is verify(gold, pred); the Scorer scores prediction vs reference, + # so prediction=pred and reference=gold — do not swap. + return scorer.score(pred, gold) diff --git a/tests/prkit/evaluation/evaluator/test_accuracy.py b/tests/prkit/evaluation/evaluator/test_accuracy.py index 40210bc..7dbe0ce 100644 --- a/tests/prkit/evaluation/evaluator/test_accuracy.py +++ b/tests/prkit/evaluation/evaluator/test_accuracy.py @@ -1,3 +1,5 @@ +import pytest + from prkit.core.domain import ( Answer, AnswerCategory, @@ -7,6 +9,7 @@ ) from prkit.evaluation.comparator.exact_match import ExactMatchComparator from prkit.evaluation.evaluator.accuracy import AccuracyEvaluator +from prkit.scoring import SemanticsScorer def _make_problem( @@ -25,15 +28,35 @@ def _make_problem( ) -def test_accuracy_evaluator_defaults_to_exact_match(): +def test_accuracy_evaluator_defaults_to_semantics_scorer(): evaluator = AccuracyEvaluator() - assert isinstance(evaluator.comparator, ExactMatchComparator) + assert isinstance(evaluator.scorer, SemanticsScorer) + assert evaluator.comparator is None + + +def test_accuracy_evaluator_rejects_both_scorer_and_comparator(): + with pytest.raises(ValueError, match="not both"): + AccuracyEvaluator(comparator=ExactMatchComparator(), scorer=SemanticsScorer()) -def test_accuracy_evaluator_evaluate_returns_details(): +def test_accuracy_evaluator_scorer_path_returns_verdict_backed_details(): evaluator = AccuracyEvaluator() result = evaluator.evaluate("4", "4") + assert result["accuracy_score"] == 1.0 + assert result["comparison_result"] is True + assert result["details"]["scorer_type"] == "SemanticsScorer" + assert result["details"]["scorer_version"] == SemanticsScorer.version + assert result["details"]["comparison_mode"] == "number" + assert result["details"]["predicted_type"] == "string" + + +def test_accuracy_evaluator_legacy_comparator_path_unchanged(): + evaluator = AccuracyEvaluator(comparator=ExactMatchComparator()) + assert evaluator.scorer is None + + result = evaluator.evaluate("4", "4") + assert result["accuracy_score"] == 1.0 assert result["comparison_result"] is True assert result["details"]["comparator_type"] == "ExactMatchComparator" diff --git a/tests/prkit/scoring/test_adapt.py b/tests/prkit/scoring/test_adapt.py index 2e760e6..c0afb48 100644 --- a/tests/prkit/scoring/test_adapt.py +++ b/tests/prkit/scoring/test_adapt.py @@ -6,8 +6,9 @@ from prkit.core.verdict import Verdict from prkit.scoring._adapt import verdict_from_comparison -from prkit.semantics import AnswerComparison +from prkit.semantics import AnswerComparison, PhysicsAnswerSemantics from prkit.semantics.schema.enums import ( + AnswerObjectKind, BridgeTier, ComparisonPolicyMode, ContractValidationStatus, @@ -20,6 +21,14 @@ def _comparison(**overrides) -> AnswerComparison: return AnswerComparison(**base) +def _quantity_sem(text: str, *, unit: str | None = "m/s") -> PhysicsAnswerSemantics: + return PhysicsAnswerSemantics( + canonical_text=text, + object_kind=AnswerObjectKind.PHYSICAL_QUANTITY, + unit=unit, + ) + + class TestMapping: def test_equivalent_maps_score_one(self): v = verdict_from_comparison(_comparison(equivalent=True), scorer_version="x") @@ -66,3 +75,107 @@ def test_none_enums_stay_none(self): assert v.details["bridge_tier"] is None assert v.details["policy_mode"] is None assert v.details["validation_status"] is None + + +class TestEnrichedFields: + def test_correct_mirrors_equivalent(self): + assert ( + verdict_from_comparison( + _comparison(equivalent=True), scorer_version="x" + ).correct + is True + ) + assert ( + verdict_from_comparison( + _comparison(equivalent=False), scorer_version="x" + ).correct + is False + ) + + def test_symbolic_equiv_true_for_symbolic_mode(self): + v = verdict_from_comparison( + _comparison(equivalent=True, comparison_mode="expression"), + scorer_version="x", + ) + assert v.symbolic_equiv is True + assert v.numeric_within_tol is None + + def test_symbolic_equiv_false_when_symbolic_mode_not_equivalent(self): + v = verdict_from_comparison( + _comparison(equivalent=False, comparison_mode="relation"), + scorer_version="x", + ) + assert v.symbolic_equiv is False + + def test_symbolic_equiv_none_for_non_symbolic_mode(self): + v = verdict_from_comparison( + _comparison(comparison_mode="choice"), scorer_version="x" + ) + assert v.symbolic_equiv is None + + def test_numeric_within_tol_true_for_number_mode(self): + v = verdict_from_comparison( + _comparison(equivalent=True, comparison_mode="number"), scorer_version="x" + ) + assert v.numeric_within_tol is True + assert v.symbolic_equiv is None + + def test_numeric_within_tol_false_on_value_mismatch(self): + v = verdict_from_comparison( + _comparison( + equivalent=False, + comparison_mode="number", + diagnostics=("numeric_value_mismatch",), + ), + scorer_version="x", + ) + assert v.numeric_within_tol is False + + def test_units_ok_false_on_unit_fail_tag(self): + v = verdict_from_comparison( + _comparison( + equivalent=False, + comparison_mode="physical_quantity", + diagnostics=("unit_mismatch",), + ), + scorer_version="x", + ) + assert v.units_ok is False + + def test_units_ok_true_for_physical_quantity_with_sems(self): + v = verdict_from_comparison( + _comparison(equivalent=True, comparison_mode="physical_quantity"), + scorer_version="x", + pred_sem=_quantity_sem("3 m/s"), + ref_sem=_quantity_sem("3 m/s"), + ) + assert v.units_ok is True + + def test_units_ok_none_without_sems(self): + v = verdict_from_comparison( + _comparison(comparison_mode="number"), scorer_version="x" + ) + assert v.units_ok is None + + def test_extracted_answer_from_pred_sem(self): + v = verdict_from_comparison( + _comparison(), scorer_version="x", pred_sem=_quantity_sem("3 m/s") + ) + assert v.extracted_answer == "3 m/s" + + def test_extracted_answer_none_without_pred_sem(self): + v = verdict_from_comparison(_comparison(), scorer_version="x") + assert v.extracted_answer is None + + def test_partial_credit_and_rationale_are_none(self): + v = verdict_from_comparison(_comparison(), scorer_version="x") + assert v.partial_credit is None + assert v.rationale is None + + def test_enriched_verdict_is_json_serializable(self): + v = verdict_from_comparison( + _comparison(equivalent=True, comparison_mode="physical_quantity"), + scorer_version="x", + pred_sem=_quantity_sem("3 m/s"), + ) + json.dumps(v.model_dump()) diff --git a/tests/prkit/verify/__init__.py b/tests/prkit/verify/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/prkit/verify/test_import_isolation.py b/tests/prkit/verify/test_import_isolation.py new file mode 100644 index 0000000..939f67d --- /dev/null +++ b/tests/prkit/verify/test_import_isolation.py @@ -0,0 +1,57 @@ +"""Import-boundary guard: ``prkit.verify`` must stay light-import-clean. + +Runs in a fresh subprocess (so the host test process's own imports cannot mask a +leak) and asserts that importing the facade — and exercising ``parse``/``verify`` — +never pulls in provider SDKs, the dataset hub, the ``datasets`` library, or pandas. +This is the contract that makes ``prkit.verify`` a ``pip install``-and-call verifier. +""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap + +# Heavy/optional modules that must NOT be importable as a side effect of the +# verify path. We check ``google.genai`` (the provider SDK), never bare ``google``, +# which is a namespace-package ``.pth`` artifact present at interpreter start. +_FORBIDDEN = [ + "anthropic", + "openai", + "google.genai", + "datasets", + "pandas", + "prkit.datasets", + "prkit.evaluation", +] + + +def test_verify_path_does_not_import_heavy_deps(): + code = textwrap.dedent( + f""" + import sys + import prkit.verify + from prkit.verify import parse, verify + + # Exercise the full lazy path: these trigger the SemanticsScorer/sympy + # imports, which still must not drag in the forbidden modules. + verify("3 m/s", "3 m/s") + parse("9.8 m/s^2") + + forbidden = {_FORBIDDEN!r} + leaked = [name for name in forbidden if name in sys.modules] + if leaked: + print("LEAKED:" + ",".join(leaked)) + raise SystemExit(1) + raise SystemExit(0) + """ + ) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + "heavy deps leaked into the prkit.verify import path:\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) diff --git a/tests/prkit/verify/test_verify.py b/tests/prkit/verify/test_verify.py new file mode 100644 index 0000000..f294aef --- /dev/null +++ b/tests/prkit/verify/test_verify.py @@ -0,0 +1,82 @@ +"""Tests for the ``prkit.verify`` light-import facade (parse / verify).""" + +from __future__ import annotations + +import pytest + +from prkit.core.verdict import Verdict +from prkit.semantics import PhysicsAnswerSemantics +from prkit.verify import parse, verify + + +class TestVerify: + def test_returns_canonical_verdict(self): + v = verify("3 m/s", "3 m/s") + assert isinstance(v, Verdict) + assert v.correct is True + assert v.score == 1.0 + + def test_unit_suffix_normalization_handled(self): + # math-verify mishandles units; the squared-suffix variants normalize to + # the same unit, so the dimensional check passes (units_ok True) even + # though 9.8 != 9.81 numerically. + v = verify("9.8 m/s^2", "9.8 m/s²") + assert v.correct is True + assert v.units_ok is True + + def test_unit_mismatch_is_not_equivalent(self): + v = verify("3 m/s", "3 m") + assert v.correct is False + + def test_numeric_tolerance_mismatch(self): + v = verify("3 m/s", "5 m/s") + assert v.correct is False + assert v.numeric_within_tol is False + + def test_symbolic_commutativity_equivalent(self): + # Cases math-verify mishandles: symbolic equivalence under reordering. + v = verify("v = a t", "v = t a") + assert v.correct is True + assert v.symbolic_equiv is True + + def test_number_vs_fraction_equivalent(self): + v = verify("0.5", "1/2") + assert v.correct is True + + def test_gold_pred_argument_order(self): + # verify(gold, pred): the prediction surface is what gets extracted. + v = verify("3 m/s", "4 m/s") + assert v.extracted_answer is not None + assert "4" in v.extracted_answer + + def test_partial_credit_true_raises(self): + with pytest.raises(NotImplementedError): + verify("a", "b", partial_credit=True) + + def test_unknown_unit_policy_raises(self): + with pytest.raises(ValueError, match="unit_policy"): + verify("a", "b", unit_policy="bogus") + + @pytest.mark.parametrize("policy", ["strict", "audited", "permissive"]) + def test_recognized_unit_policies_accepted(self, policy): + v = verify("3 m/s", "3 m/s", unit_policy=policy) + assert isinstance(v, Verdict) + + def test_tolerance_passthrough(self): + # tolerance is a relative threshold; a generous value flips a near-miss + # that the default (precision-driven) comparison rejects. + strict = verify("100", "101") + loose = verify("100", "101", tolerance=0.05) + assert strict.correct is False + assert loose.correct is True + + +class TestParse: + def test_returns_physics_answer_semantics(self): + parsed = parse("9.8 m/s^2") + assert isinstance(parsed, PhysicsAnswerSemantics) + assert parsed.unit == "m/s^2" + + def test_category_not_supported_yet(self): + with pytest.raises(NotImplementedError): + parse("9.8", category="number") From 8ab26e3b16abca647e179eb41929266eab6f1061 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Thu, 18 Jun 2026 19:22:19 -0400 Subject: [PATCH 02/28] Make pre-commit formatting convergent with the venv/CI toolchain pre-commit ran both ruff-format and black, which format differently and ping-ponged files on every `pre-commit run --all-files`; CI enforces only black (`black --check`). Drop the ruff-format hook (keep ruff as the linter), bump the pinned black from 24.8.0 to 26.1.0 to match the version in the venv/CI, and exclude vendored minified katex assets from the end-of-file/whitespace hooks. Apply the resulting stable formatting to the docs and test file that were not yet clean. Co-Authored-By: Claude Opus 4.8 --- .pre-commit-config.yaml | 10 +++++++--- docs/DATASETS.md | 2 +- docs/EVALUATION.md | 1 - tests/prkit/verify/test_import_isolation.py | 6 ++---- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 20f8743..df1f19f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,14 +1,15 @@ repos: + # ruff is the linter only. Formatting is owned by black (the single formatter CI + # enforces via `black --check`); ruff-format is intentionally NOT enabled because + # the two disagree and would ping-pong files on every run. - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.15.17 hooks: - id: ruff files: ^(src/prkit|tests/prkit)/ - - id: ruff-format - files: ^(src/prkit|tests/prkit)/ - repo: https://github.com/psf/black - rev: 24.8.0 + rev: 26.1.0 # keep in sync with the venv/CI black to avoid format ping-pong hooks: - id: black files: ^(src/prkit|tests/prkit)/ @@ -17,7 +18,10 @@ repos: rev: v4.6.0 hooks: - id: end-of-file-fixer + # Vendored, minified third-party assets are shipped as-is. + exclude: ^src/prkit/annotation/tasks/correctness/ui/vendor/ - id: trailing-whitespace + exclude: ^src/prkit/annotation/tasks/correctness/ui/vendor/ # Local hooks mirror the CI checks (.github/workflows/ci.yml) so type and # test failures surface at commit time instead of in CI. They invoke the diff --git a/docs/DATASETS.md b/docs/DATASETS.md index ff9cdc3..1a73f67 100644 --- a/docs/DATASETS.md +++ b/docs/DATASETS.md @@ -66,7 +66,7 @@ The following table shows which physics domains are available in each dataset: **Domain Coverage Summary:** - **UGPhysics**: 13 domains (most comprehensive coverage) - 5,520 problems -- **SeePhys**: 1 domain (Other - visual physics focus) - 2,000 problems +- **SeePhys**: 1 domain (Other - visual physics focus) - 2,000 problems - **PHYBench**: 6 domains (focused on core physics areas) - 500 problems - **TPBench**: 5 domains (specialized in theoretical physics) - 10 problems - **PhysReason**: 7 domains (comprehensive reasoning focus) - 1,200 problems diff --git a/docs/EVALUATION.md b/docs/EVALUATION.md index 22c0530..366d9c1 100644 --- a/docs/EVALUATION.md +++ b/docs/EVALUATION.md @@ -64,4 +64,3 @@ The evaluation package is designed to grow beyond final-answer correctness, with - theorem / principle usage checks - intermediate-step validation - rubric-based or structured reasoning assessments - diff --git a/tests/prkit/verify/test_import_isolation.py b/tests/prkit/verify/test_import_isolation.py index 939f67d..b5a3a4a 100644 --- a/tests/prkit/verify/test_import_isolation.py +++ b/tests/prkit/verify/test_import_isolation.py @@ -27,8 +27,7 @@ def test_verify_path_does_not_import_heavy_deps(): - code = textwrap.dedent( - f""" + code = textwrap.dedent(f""" import sys import prkit.verify from prkit.verify import parse, verify @@ -44,8 +43,7 @@ def test_verify_path_does_not_import_heavy_deps(): print("LEAKED:" + ",".join(leaked)) raise SystemExit(1) raise SystemExit(0) - """ - ) + """) result = subprocess.run( [sys.executable, "-c", code], capture_output=True, From b533ffd107f344e79317e51b657fb99f3316337d Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Thu, 18 Jun 2026 19:22:40 -0400 Subject: [PATCH 03/28] Add EED/SEED partial-credit scorer filling Verdict.partial_credit The deterministic verifier is binary, so a near-miss answer scores the same as a completely wrong one. Add a graded scorer based on PHYBench Expression Edit Distance and CMPhysBench Scalable EED, run on PRKit's existing SymPy parser and unit backend (no pint / latex2sympy dependency). - New pure package prkit/semantics/edit_distance/: ExprNode + sympy_to_tree, an extended Zhang-Shasha tree-edit distance (forest matrix initialized to math.inf, fixing the upstream 1000-sentinel), a cost-aware subtree discount and eed_score, a thread-safe simplify timeout, and the SEED dispatch pipeline (eed_compare). - New PartialCreditScorer (Scorer protocol) maps the graded result onto Verdict, populating partial_credit and rationale; PARTIAL_CREDIT / BINARY / TOLERANCE modes. - verify(partial_credit=True) now routes to the scorer instead of raising. - CONTRACT.md and the conformance gate updated to cover the new scorer. Co-Authored-By: Claude Opus 4.8 --- src/prkit/CONTRACT.md | 6 +- src/prkit/scoring/__init__.py | 9 +- src/prkit/scoring/partial_credit_scorer.py | 240 ++++++++++ src/prkit/semantics/edit_distance/__init__.py | 31 ++ src/prkit/semantics/edit_distance/pipeline.py | 440 ++++++++++++++++++ src/prkit/semantics/edit_distance/score.py | 106 +++++ src/prkit/semantics/edit_distance/timeout.py | 56 +++ src/prkit/semantics/edit_distance/tree.py | 131 ++++++ src/prkit/semantics/edit_distance/zss.py | 149 ++++++ src/prkit/verify/__init__.py | 21 +- .../scoring/test_partial_credit_scorer.py | 132 ++++++ .../test_edit_distance_pipeline.py | 106 +++++ .../test_edit_distance_robustness.py | 41 ++ .../edit_distance/test_edit_distance_score.py | 85 ++++ .../edit_distance/test_edit_distance_tree.py | 97 ++++ .../edit_distance/test_edit_distance_zss.py | 102 ++++ tests/prkit/test_conformance.py | 6 +- tests/prkit/verify/test_verify.py | 23 +- 18 files changed, 1761 insertions(+), 20 deletions(-) create mode 100644 src/prkit/scoring/partial_credit_scorer.py create mode 100644 src/prkit/semantics/edit_distance/__init__.py create mode 100644 src/prkit/semantics/edit_distance/pipeline.py create mode 100644 src/prkit/semantics/edit_distance/score.py create mode 100644 src/prkit/semantics/edit_distance/timeout.py create mode 100644 src/prkit/semantics/edit_distance/tree.py create mode 100644 src/prkit/semantics/edit_distance/zss.py create mode 100644 tests/prkit/scoring/test_partial_credit_scorer.py create mode 100644 tests/prkit/semantics/edit_distance/test_edit_distance_pipeline.py create mode 100644 tests/prkit/semantics/edit_distance/test_edit_distance_robustness.py create mode 100644 tests/prkit/semantics/edit_distance/test_edit_distance_score.py create mode 100644 tests/prkit/semantics/edit_distance/test_edit_distance_tree.py create mode 100644 tests/prkit/semantics/edit_distance/test_edit_distance_zss.py diff --git a/src/prkit/CONTRACT.md b/src/prkit/CONTRACT.md index 77cfea6..e09ed5c 100644 --- a/src/prkit/CONTRACT.md +++ b/src/prkit/CONTRACT.md @@ -35,7 +35,7 @@ The contract pins four structural (`typing.Protocol`) nouns plus one result type |------|----------|--------------------------| | Dataset loader | `DatasetProvider` | `BaseDatasetLoader` subclasses | | Inference client | `ModelClient` | `BaseModelClient` subclasses | -| Scorer | `Scorer` | `prkit.scoring.SemanticsScorer` | +| Scorer | `Scorer` | `prkit.scoring.SemanticsScorer` (binary); `prkit.scoring.PartialCreditScorer` (graded EED/SEED) | | Runner | `Runner` | *(reserved; no implementation yet)* | | Result | `Verdict` | `prkit.core.verdict.Verdict` | @@ -62,8 +62,8 @@ when not applicable (or not yet produced): | `symbolic_equiv` | enriched | equivalence decided symbolically; `None` for non-symbolic modes | | `numeric_within_tol` | enriched | numeric/quantity match within tolerance; `None` otherwise | | `extracted_answer` | enriched | parsed prediction surface, when available | -| `partial_credit` | enriched | **reserved (X1)** — the deterministic engine is binary, so this is always `None` today | -| `rationale` | enriched | **reserved** — no NL rationale from the deterministic engine (`None` today) | +| `partial_credit` | enriched | continuous partial-credit signal; `None` from the binary `SemanticsScorer`, populated by the graded `PartialCreditScorer` (EED/SEED) — also via `verify(..., partial_credit=True)` | +| `rationale` | enriched | human-readable explanation; `None` from the deterministic engine, populated by `PartialCreditScorer` | ## Three independent version axes diff --git a/src/prkit/scoring/__init__.py b/src/prkit/scoring/__init__.py index d5b2ddd..fc0a5e5 100644 --- a/src/prkit/scoring/__init__.py +++ b/src/prkit/scoring/__init__.py @@ -1,10 +1,13 @@ """Reference scoring implementations for PRKit's ``Scorer`` contract. ``SemanticsScorer`` is the canonical, version-stamped scorer wrapping the -deterministic semantics comparison engine. It structurally satisfies -:class:`prkit.api.Scorer` and emits :class:`prkit.api.Verdict`. +deterministic (binary) semantics comparison engine. ``PartialCreditScorer`` is its +graded counterpart: an EED/SEED edit-distance scorer that populates +``Verdict.partial_credit``. Both structurally satisfy :class:`prkit.api.Scorer` and +emit :class:`prkit.api.Verdict`. """ +from .partial_credit_scorer import PartialCreditMode, PartialCreditScorer from .semantics_scorer import SemanticsScorer -__all__ = ["SemanticsScorer"] +__all__ = ["PartialCreditMode", "PartialCreditScorer", "SemanticsScorer"] diff --git a/src/prkit/scoring/partial_credit_scorer.py b/src/prkit/scoring/partial_credit_scorer.py new file mode 100644 index 0000000..36eb24b --- /dev/null +++ b/src/prkit/scoring/partial_credit_scorer.py @@ -0,0 +1,240 @@ +"""Partial-credit :class:`prkit.api.Scorer` over the EED/SEED edit-distance engine. + +``PartialCreditScorer`` is the scorer that finally populates ``Verdict.partial_credit`` +(the deterministic :class:`SemanticsScorer` is strictly binary). It wraps +:func:`prkit.semantics.edit_distance.eed_compare` — PHYBench EED + CMPhysBench SEED on +PRKit's own parser/unit substrate — and maps its graded :class:`EedResult` onto the +canonical :class:`~prkit.core.verdict.Verdict`. + +Configuration is constructor/keyword-only (no magic strings), satisfying the +``Scorer`` protocol and the sklearn "inspection" principle. ``mode`` selects whether +the graded score is surfaced (``PARTIAL_CREDIT``), collapsed to a pass/fail +(``BINARY``), or reduced to a reference-precision/symbolic tolerance check +(``TOLERANCE``). +""" + +from __future__ import annotations + +from enum import Enum +from typing import Any + +from prkit.core.domain.answer import Answer +from prkit.core.verdict import Verdict +from prkit.semantics import ( + ComparisonPolicyMode, + PhysicsAnswerSemantics, + PhysicsQuestionSemantics, + QuestionUnitPolicy, + normalize_physics_answer, +) +from prkit.semantics.edit_distance import EedConfig, EedResult, eed_compare + +#: Revision of the EED/SEED scorer wiring. Bump when the algorithm or its mapping +#: onto ``Verdict`` changes in a way that can alter scores. +ENGINE_VERSION = "1" +_VERSION = f"eed-seed/engine{ENGINE_VERSION}" + + +class PartialCreditMode(str, Enum): + """How the graded EED/SEED signal is rendered into a :class:`Verdict`.""" + + BINARY = ( + "binary" # collapse score >= binary_threshold -> 1.0/0.0, partial_credit None + ) + PARTIAL_CREDIT = "partial" # graded EED/SEED score in [0, 1], partial_credit set + TOLERANCE = "tolerance" # numeric reference-precision / symbolic equality, no grade + + def __str__(self) -> str: + return str(self.value) + + +def _coerce_context( + context: PhysicsQuestionSemantics | dict[str, Any] | None, +) -> PhysicsQuestionSemantics | None: + """Coerce an optional context into validated ``PhysicsQuestionSemantics``.""" + if context is None: + return None + if isinstance(context, PhysicsQuestionSemantics): + return context + return PhysicsQuestionSemantics.model_validate(context) + + +def _policy_to_str(policy_mode: ComparisonPolicyMode | str | None) -> str | None: + """Render a policy mode as a plain string for ``get_info()`` (or ``None``).""" + if policy_mode is None: + return None + if isinstance(policy_mode, ComparisonPolicyMode): + return str(policy_mode) + return str(ComparisonPolicyMode(policy_mode)) + + +class PartialCreditScorer: + """Graded EED/SEED :class:`prkit.api.Scorer`; fills ``Verdict.partial_credit``. + + Args: + mode: how the graded score is surfaced (see :class:`PartialCreditMode`). + tolerance: numeric comparison tolerance; plumbs to + ``PhysicsQuestionSemantics.tolerance``. + unit_policy: how units must appear; plumbs to + ``PhysicsQuestionSemantics.question_unit_policy``. + policy_mode: enforcement strictness accepted for facade compatibility; the + deterministic edit-distance algorithm does not branch on it, so it is + recorded in ``get_info()`` but otherwise unused. + binary_threshold: score at/above which a verdict counts as ``equivalent`` + (and, in ``BINARY`` mode, collapses to ``1.0``). + context: advanced base question semantics; instance-level ``tolerance`` / + ``unit_policy`` overrides are merged on top of it. + config: advanced edit-distance algorithm tunables. + """ + + version: str = _VERSION + + def __init__( + self, + *, + mode: PartialCreditMode = PartialCreditMode.PARTIAL_CREDIT, + tolerance: float | None = None, + unit_policy: QuestionUnitPolicy | str | None = None, + policy_mode: ComparisonPolicyMode | str | None = None, + binary_threshold: float = 1.0, + context: PhysicsQuestionSemantics | dict[str, Any] | None = None, + config: EedConfig | None = None, + ) -> None: + self._mode = mode + self._binary_threshold = float(binary_threshold) + self._policy_mode = policy_mode + self._config = config or EedConfig() + self._base_context = _coerce_context(context) + + overrides: dict[str, Any] = {} + if tolerance is not None: + overrides["tolerance"] = float(tolerance) + if unit_policy is not None: + overrides["question_unit_policy"] = ( + unit_policy + if isinstance(unit_policy, QuestionUnitPolicy) + else QuestionUnitPolicy(unit_policy) + ) + self._context_overrides = overrides + + def _effective_context( + self, call_context: PhysicsQuestionSemantics | dict[str, Any] | None + ) -> PhysicsQuestionSemantics | None: + """Merge instance knob overrides over the per-call or base context.""" + base = ( + _coerce_context(call_context) + if call_context is not None + else self._base_context + ) + if base is None: + if not self._context_overrides: + return None + base = PhysicsQuestionSemantics() + return base.merged(self._context_overrides) + + def score( + self, + prediction: Answer | str | PhysicsAnswerSemantics, + reference: Answer | str | PhysicsAnswerSemantics, + *, + context: PhysicsQuestionSemantics | dict[str, Any] | None = None, + **kwargs: Any, + ) -> Verdict: + """Score ``prediction`` against ``reference`` and return a graded Verdict. + + ``prediction`` / ``reference`` may be raw strings, :class:`Answer` objects, + or already-normalized :class:`PhysicsAnswerSemantics`. The wider input type + stays compatible with the narrower :class:`prkit.api.Scorer` protocol by + parameter contravariance. + """ + effective_context = self._effective_context(context) + pred_sem = normalize_physics_answer(prediction, context=effective_context) + ref_sem = normalize_physics_answer(reference, context=effective_context) + + result = eed_compare( + pred_sem, ref_sem, context=effective_context, config=self._config + ) + return self._verdict_from_result(result, pred_sem) + + def _verdict_from_result( + self, result: EedResult, pred_sem: PhysicsAnswerSemantics + ) -> Verdict: + """Map an :class:`EedResult` onto a :class:`Verdict`, applying the mode policy.""" + graded = result.score + equivalent = graded >= self._binary_threshold + partial_credit: float | None + + if self._mode is PartialCreditMode.PARTIAL_CREDIT: + score = graded + partial_credit = graded + elif self._mode is PartialCreditMode.BINARY: + score = 1.0 if equivalent else 0.0 + partial_credit = None + else: # TOLERANCE: pass/fail by reference precision or symbolic equality + if result.numeric_within_tol is not None: + passed = result.numeric_within_tol + elif result.symbolic_equiv is not None: + passed = result.symbolic_equiv + else: + passed = equivalent + score = 1.0 if passed else 0.0 + partial_credit = None + equivalent = passed + + extracted = pred_sem.canonical_text or pred_sem.raw_text + return Verdict( + equivalent=equivalent, + score=score, + comparison_mode=f"eed_{result.answer_type}", + scorer_version=self.version, + diagnostics=result.diagnostics, + details={ + "answer_type": result.answer_type, + "graded_score": graded, + "raw_distance": result.raw_distance, + "gt_tree_size": result.gt_tree_size, + "relative_distance": result.relative_distance, + "degraded": result.degraded, + "mode": str(self._mode), + }, + correct=equivalent, + units_ok=result.units_ok, + symbolic_equiv=result.symbolic_equiv, + numeric_within_tol=result.numeric_within_tol, + extracted_answer=extracted, + partial_credit=partial_credit, + rationale=_rationale(result, score), + ) + + def get_info(self) -> dict[str, Any]: + """Return scorer metadata; always includes ``version``.""" + unit_policy = self._context_overrides.get("question_unit_policy") + return { + "name": "PartialCreditScorer", + "version": self.version, + "engine": "eed_compare", + "deterministic": True, + "mode": str(self._mode), + "binary_threshold": self._binary_threshold, + "tolerance": self._context_overrides.get("tolerance"), + "unit_policy": (str(unit_policy) if unit_policy is not None else None), + "policy_mode": _policy_to_str(self._policy_mode), + } + + +def _rationale(result: EedResult, final_score: float) -> str: + """Build a deterministic, human-readable explanation of a graded result.""" + parts = [f"score={final_score:.3f}"] + if result.raw_distance is not None and result.gt_tree_size: + parts.append(f"tree_edit_distance={result.raw_distance:g}") + parts.append(f"gt_tree_size={result.gt_tree_size}") + if result.relative_distance is not None: + parts.append(f"rel_dist={result.relative_distance:.4f}") + if result.degraded: + parts.append("degraded") + if result.diagnostics: + parts.append(", ".join(result.diagnostics)) + return "EED/SEED " + "; ".join(parts) + + +__all__ = ["PartialCreditMode", "PartialCreditScorer"] diff --git a/src/prkit/semantics/edit_distance/__init__.py b/src/prkit/semantics/edit_distance/__init__.py new file mode 100644 index 0000000..3008af0 --- /dev/null +++ b/src/prkit/semantics/edit_distance/__init__.py @@ -0,0 +1,31 @@ +"""Expression Edit Distance (EED / SEED) partial-credit algorithm on PRKit's substrate. + +A self-contained reimplementation of PHYBench's Expression Edit Distance and +CMPhysBench's Scalable EED, run on PRKit's existing SymPy parser, LaTeX normalizer, +and unit backend instead of vendoring ``latex2sympy2`` + ``pint``. The pure pieces +(:mod:`.tree`, :mod:`.zss`, :mod:`.score`, :mod:`.timeout`) depend only on +``sympy`` + stdlib; :mod:`.pipeline` adds the SEED dispatch that reuses the +comparison engine's parsing/unit primitives. + +See :class:`prkit.scoring.PartialCreditScorer` for the ``Scorer`` wrapper that maps +:class:`EedResult` onto :class:`prkit.core.verdict.Verdict`. +""" + +from __future__ import annotations + +from .pipeline import EedConfig, EedResult, eed_compare +from .score import EditCosts, eed_score +from .tree import ExprNode, UnsupportedExpressionError, sympy_to_tree +from .zss import tree_edit_distance + +__all__ = [ + "EditCosts", + "EedConfig", + "EedResult", + "ExprNode", + "UnsupportedExpressionError", + "eed_compare", + "eed_score", + "sympy_to_tree", + "tree_edit_distance", +] diff --git a/src/prkit/semantics/edit_distance/pipeline.py b/src/prkit/semantics/edit_distance/pipeline.py new file mode 100644 index 0000000..c706c5c --- /dev/null +++ b/src/prkit/semantics/edit_distance/pipeline.py @@ -0,0 +1,440 @@ +"""SEED-style per-pair dispatch that ties the EED algorithm to PRKit's substrate. + +``eed_compare`` reproduces CMPhysBench SEED's answer-type dispatch on top of +PRKit's existing parser / unit backend instead of vendoring ``latex2sympy2`` + +``pint``: + +1. hard guards (empty prediction, unsupported ``\\int``/``\\sum``, runaway length); +2. dispatch on the *normalized* :class:`~prkit.semantics.PhysicsAnswerSemantics` + object kind (more reliable than re-classifying raw text); +3. numbers / physical quantities -> unit-aware tiered numeric scoring; +4. relations -> ``lhs - rhs`` residual tree diff (sign-robust); +5. expressions -> symbolic short-circuit, then a tree-edit-distance score. + +The result is always the *graded* signal; collapsing it to binary or tolerance +behavior is the caller's (scorer's) policy choice. +""" + +from __future__ import annotations + +import functools +import re +from dataclasses import dataclass, field +from typing import Any + +from sympy import simplify + +from ..comparison.common import available_texts, context_symbol_alias_map +from ..comparison.numeric import ( + NumericComparableAnswer, + extract_numeric_comparable_answer, +) +from ..comparison.semantics import ( + convert_numeric_value, + expressions_equivalent, + normalize_unit_text, + numbers_match_with_reference_precision, + parse_relation_clauses, + parse_scalar_symbolic_expression, +) +from ..schema import ( + AnswerObjectKind, + PhysicsAnswerSemantics, + PhysicsQuestionSemantics, + QuestionUnitPolicy, +) +from .score import EditCosts, eed_score +from .timeout import SimplifyTimeout, run_with_timeout +from .tree import UnsupportedExpressionError, sympy_to_tree +from .zss import tree_edit_distance + +#: Operators PRKit's tree grammar cannot represent; mirror EED's hard 0-score guard. +_UNSUPPORTED_RE = re.compile(r"\\(?:i{1,3}nt|oint|sum|prod)") + +#: Default SEED numeric tiers: (relative-error threshold, score) in priority order. +_DEFAULT_NUMERIC_TIERS: tuple[tuple[float, float], ...] = ( + (0.01, 1.0), + (0.02, 0.9), + (0.04, 0.8), +) + + +@dataclass(frozen=True) +class EedConfig: + """Tunables for :func:`eed_compare` (numeric tolerance comes from the context).""" + + costs: EditCosts = field(default_factory=EditCosts) + discount_slope: float = 0.6 + simplify_timeout_s: float = 5.0 + max_length_ratio: float = 3.0 + # The length-ratio guard only fires once the prediction is also this many + # characters long, so short numerics like "3.005" vs "3" are not flagged. + length_guard_min_chars: int = 16 + numeric_tiers: tuple[tuple[float, float], ...] = _DEFAULT_NUMERIC_TIERS + simplify_before_tree: bool = True + + +@dataclass(frozen=True) +class EedResult: + """Graded result of one EED/SEED comparison (before mode policy is applied).""" + + score: float + answer_type: str + relative_distance: float | None = None + gt_tree_size: int | None = None + raw_distance: float | None = None + units_ok: bool | None = None + symbolic_equiv: bool | None = None + numeric_within_tol: bool | None = None + degraded: bool = False + diagnostics: tuple[str, ...] = () + + +def _texts(sem: PhysicsAnswerSemantics) -> tuple[str, ...]: + """All non-empty surfaces of an answer, for guard scanning.""" + return available_texts(sem.raw_text, sem.canonical_text, sem.canonical_latex) + + +def _primary_text(sem: PhysicsAnswerSemantics) -> str: + """The preferred symbolic surface (canonical first) for parsing/equivalence.""" + texts = available_texts(sem.canonical_text, sem.canonical_latex, sem.raw_text) + return texts[0] if texts else "" + + +def _flat( + score: float, kind: AnswerObjectKind, diagnostics: tuple[str, ...] +) -> EedResult: + """Build a guard/short-circuit result with no tree/numeric detail.""" + return EedResult( + score=score, + answer_type=str(kind), + symbolic_equiv=False, + diagnostics=diagnostics, + ) + + +def eed_compare( + pred_sem: PhysicsAnswerSemantics, + ref_sem: PhysicsAnswerSemantics, + *, + context: PhysicsQuestionSemantics | None = None, + config: EedConfig | None = None, +) -> EedResult: + """Compute the graded EED/SEED partial-credit result for one answer pair. + + Args: + pred_sem: normalized prediction semantics. + ref_sem: normalized reference (gold) semantics; its ``object_kind`` drives + the dispatch. + context: question semantics supplying ``tolerance`` / unit policy. + config: algorithm tunables. + + Returns: + A graded :class:`EedResult` (``score`` in ``[0, 1]``). + """ + cfg = config or EedConfig() + ctx = context or PhysicsQuestionSemantics() + alias_map = context_symbol_alias_map(ctx) + kind = ref_sem.object_kind + + pred_primary = _primary_text(pred_sem) + gold_primary = _primary_text(ref_sem) + + # --- (1) hard guards ------------------------------------------------------- + if not pred_primary.strip(): + return _flat(0.0, kind, ("empty_prediction",)) + if any( + _UNSUPPORTED_RE.search(text) for text in (*_texts(pred_sem), *_texts(ref_sem)) + ): + return _flat(0.0, kind, ("unsupported_operator",)) + if len(pred_primary) > cfg.length_guard_min_chars and len( + pred_primary + ) > cfg.max_length_ratio * max(len(gold_primary), 1): + return _flat(0.0, kind, ("length_ratio_exceeded",)) + + # --- (2) dispatch on the gold answer kind --------------------------------- + if kind in (AnswerObjectKind.NUMBER, AnswerObjectKind.PHYSICAL_QUANTITY): + return _numeric_path( + pred_sem, ref_sem, ctx, cfg, kind, alias_map, pred_primary, gold_primary + ) + if kind == AnswerObjectKind.RELATION: + return _relation_path(pred_primary, gold_primary, ctx, cfg, alias_map, kind) + return _expression_path(pred_primary, gold_primary, ctx, cfg, alias_map, kind) + + +# --------------------------------------------------------------------------- # +# Numeric / physical-quantity leaf path (SEED numeric_score_calc analogue). +# --------------------------------------------------------------------------- # +def _numeric_path( + pred_sem: PhysicsAnswerSemantics, + ref_sem: PhysicsAnswerSemantics, + ctx: PhysicsQuestionSemantics, + cfg: EedConfig, + kind: AnswerObjectKind, + alias_map: Any, + pred_text: str, + gold_text: str, +) -> EedResult: + """Score a numeric/quantity pair with unit alignment and SEED tiers.""" + pred_num = extract_numeric_comparable_answer(pred_sem, context=ctx) + ref_num = extract_numeric_comparable_answer(ref_sem, context=ctx) + if pred_num is None or ref_num is None: + eq = expressions_equivalent( + pred_text, gold_text, ctx.tolerance, alias_map=alias_map + ) + return EedResult( + score=1.0 if eq else 0.0, + answer_type=str(kind), + symbolic_equiv=eq, + degraded=True, + diagnostics=() if eq else ("numeric_extract_failed",), + ) + + if not expressions_equivalent( + pred_num.symbolic_factor_text, + ref_num.symbolic_factor_text, + ctx.tolerance, + alias_map=alias_map, + ): + return EedResult( + score=0.0, + answer_type=str(kind), + numeric_within_tol=False, + diagnostics=("symbolic_factor_mismatch",), + ) + + aligned, units_ok, unit_diag = _align_units(pred_num, ref_num, ctx) + if aligned is None: + return EedResult( + score=0.0, + answer_type=str(kind), + units_ok=False, + numeric_within_tol=False, + diagnostics=unit_diag, + ) + + pred_value = aligned + ref_value = ref_num.coefficient_value + if pred_value != 0 and ref_value != 0 and (pred_value < 0) != (ref_value < 0): + return EedResult( + score=0.0, + answer_type=str(kind), + units_ok=units_ok, + numeric_within_tol=False, + diagnostics=("sign_mismatch",), + ) + + within_tol = numbers_match_with_reference_precision( + pred_value=pred_value, + pred_text=pred_num.coefficient_text, + ref_value=ref_value, + ref_text=ref_num.coefficient_text, + tolerance=ctx.tolerance, + allow_decimal_place_fallback=( + pred_num.allow_decimal_place_fallback + and ref_num.allow_decimal_place_fallback + ), + ) + + if ref_value == 0: + relative = None + score = 1.0 if abs(pred_value) <= ctx.tolerance else 0.0 + else: + relative = abs(pred_value - ref_value) / abs(ref_value) + score = 0.0 + for threshold, tier in cfg.numeric_tiers: + if relative <= threshold: + score = tier + break + + return EedResult( + score=score, + answer_type=str(kind), + relative_distance=relative, + units_ok=units_ok, + numeric_within_tol=within_tol, + ) + + +def _align_units( + pred: NumericComparableAnswer, + ref: NumericComparableAnswer, + ctx: PhysicsQuestionSemantics, +) -> tuple[float | None, bool | None, tuple[str, ...]]: + """Align ``pred`` into ``ref``'s unit space using the public unit backend. + + Returns ``(aligned_value, units_ok, diagnostics)``. ``units_ok`` is ``None`` when + units do not participate, ``True`` when they align, ``False`` on mismatch (with + ``aligned_value`` ``None``). + """ + pred_unit = pred.unit + ref_unit = ref.unit + if pred_unit is None and ref_unit is None: + return pred.coefficient_value, None, () + + implicit_unit: str | None = None + if ( + ctx.question_unit_policy == QuestionUnitPolicy.OPTIONAL_IF_QUESTION_FIXED_UNIT + and ctx.question_unit + ): + implicit_unit = ctx.question_unit + if pred_unit is None: + pred_unit = implicit_unit + if ref_unit is None: + ref_unit = implicit_unit + if pred_unit is None or ref_unit is None: + return None, False, ("question_unit_mismatch",) + + if normalize_unit_text(pred_unit) == normalize_unit_text(ref_unit): + return pred.coefficient_value, True, () + + converted = convert_numeric_value(pred.coefficient_value, pred_unit, ref_unit) + if converted is None: + return None, False, ("unit_mismatch",) + return converted, True, () + + +# --------------------------------------------------------------------------- # +# Expression + relation tree paths. +# --------------------------------------------------------------------------- # +def _expression_path( + pred_text: str, + gold_text: str, + ctx: PhysicsQuestionSemantics, + cfg: EedConfig, + alias_map: Any, + kind: AnswerObjectKind, +) -> EedResult: + """Symbolic short-circuit, then a tree-edit score for two expressions.""" + if expressions_equivalent(pred_text, gold_text, ctx.tolerance, alias_map=alias_map): + return EedResult( + score=1.0, + answer_type=str(kind), + relative_distance=0.0, + raw_distance=0.0, + symbolic_equiv=True, + ) + + pred_expr = parse_scalar_symbolic_expression(pred_text, alias_map=alias_map) + gold_expr = parse_scalar_symbolic_expression(gold_text, alias_map=alias_map) + if pred_expr is None or gold_expr is None: + return EedResult( + score=0.0, + answer_type=str(kind), + symbolic_equiv=False, + degraded=True, + diagnostics=("parse_failed",), + ) + return _tree_score([pred_expr], gold_expr, cfg, str(kind)) + + +def _relation_path( + pred_text: str, + gold_text: str, + ctx: PhysicsQuestionSemantics, + cfg: EedConfig, + alias_map: Any, + kind: AnswerObjectKind, +) -> EedResult: + """Score relations by diffing ``lhs - rhs`` residual trees (sign-robust).""" + if expressions_equivalent(pred_text, gold_text, ctx.tolerance, alias_map=alias_map): + return EedResult( + score=1.0, + answer_type=str(kind), + relative_distance=0.0, + raw_distance=0.0, + symbolic_equiv=True, + ) + + pred_residual = _relation_residual(pred_text, alias_map) + gold_residual = _relation_residual(gold_text, alias_map) + if pred_residual is None or gold_residual is None: + return EedResult( + score=0.0, + answer_type=str(kind), + symbolic_equiv=False, + degraded=True, + diagnostics=("relation_parse_failed",), + ) + # ``ma=F`` -> ``m*a - F`` = ``-(F - m*a)``: try both residual signs, keep the + # cheaper diff so sign-flipped equivalent equations are not penalized. + return _tree_score([pred_residual, -pred_residual], gold_residual, cfg, str(kind)) + + +def _relation_residual(text: str, alias_map: Any) -> Any | None: + """Parse a relation's first clause into a ``lhs - rhs`` SymPy residual.""" + clauses = parse_relation_clauses(text, alias_map=alias_map) + if not clauses: + return None + clause = clauses[0] + lhs = parse_scalar_symbolic_expression(clause.lhs_text, alias_map=alias_map) + rhs = parse_scalar_symbolic_expression(clause.rhs_text, alias_map=alias_map) + if lhs is None or rhs is None: + return None + return lhs - rhs + + +def _tree_score( + pred_candidates: list[Any], + gold_expr: Any, + cfg: EedConfig, + answer_type: str, +) -> EedResult: + """Build trees, take the min distance over candidate prediction forms, score it.""" + degraded = False + gold_simplified = gold_expr + candidates = list(pred_candidates) + if cfg.simplify_before_tree: + try: + gold_simplified = run_with_timeout( + functools.partial(simplify, gold_expr), + timeout_s=cfg.simplify_timeout_s, + ) + candidates = [ + run_with_timeout( + functools.partial(simplify, expr), + timeout_s=cfg.simplify_timeout_s, + ) + for expr in pred_candidates + ] + except ( + SimplifyTimeout, + ArithmeticError, + ValueError, + TypeError, + RecursionError, + ): + degraded = True + gold_simplified = gold_expr + candidates = list(pred_candidates) + + try: + gold_tree = sympy_to_tree(gold_simplified) + cand_trees = [sympy_to_tree(expr) for expr in candidates] + except UnsupportedExpressionError: + return EedResult( + score=0.0, + answer_type=answer_type, + symbolic_equiv=False, + degraded=True, + diagnostics=("unsupported_tree",), + ) + + gt_size = gold_tree.node_count() + distance = min( + tree_edit_distance(tree, gold_tree, costs=cfg.costs) for tree in cand_trees + ) + score = eed_score(distance, gt_size, discount_slope=cfg.discount_slope) + relative = distance / gt_size if gt_size else None + return EedResult( + score=score, + answer_type=answer_type, + relative_distance=relative, + gt_tree_size=gt_size, + raw_distance=distance, + symbolic_equiv=(distance == 0), + degraded=degraded, + ) + + +__all__ = ["EedConfig", "EedResult", "eed_compare"] diff --git a/src/prkit/semantics/edit_distance/score.py b/src/prkit/semantics/edit_distance/score.py new file mode 100644 index 0000000..e3a1e77 --- /dev/null +++ b/src/prkit/semantics/edit_distance/score.py @@ -0,0 +1,106 @@ +"""Edit costs, subtree cluster discount, and the EED score map (PHYBench parity). + +These are the tunable numeric pieces of the algorithm, kept separate from the +tree builder (:mod:`.tree`) and the dynamic program (:mod:`.zss`) so each layer is +independently unit-testable. + +Defaults reproduce PHYBench EED: per-type insert/delete/update cost ``1.0``, +``change_type_cost`` ``1.0``, subtree-discount ``bar_size=5`` / +``discount_slope=0.6``, and ``score = max(0, 0.6 - distance/gt_size)`` (returned +as a ``[0, 1]`` fraction rather than ``0..100``). +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from .tree import ExprNode + +_TYPES = ("number", "symbol", "operator", "function") + + +def _unit_costs() -> dict[str, float]: + """Return a fresh per-type cost map with every node type costing ``1.0``.""" + return {node_type: 1.0 for node_type in _TYPES} + + +@dataclass(frozen=True) +class EditCosts: + """Per-type insert/delete/update costs plus subtree-discount parameters. + + Attributes: + insert_cost / delete_cost / update_cost: maps from node ``type`` prefix + (``number`` / ``symbol`` / ``operator`` / ``function``) to cost. + change_type_cost: relabel cost when two nodes have different ``type`` + prefixes (e.g. ``symbol`` -> ``number``). + bar_size: subtree size below which no cluster discount applies. + discount_slope: marginal cost per node above ``bar_size`` (and, per + PHYBench, the score intercept in :func:`eed_score`). + """ + + insert_cost: dict[str, float] = field(default_factory=_unit_costs) + delete_cost: dict[str, float] = field(default_factory=_unit_costs) + update_cost: dict[str, float] = field(default_factory=_unit_costs) + change_type_cost: float = 1.0 + bar_size: int = 5 + discount_slope: float = 0.6 + + +def insert_cost(node: ExprNode, costs: EditCosts) -> float: + """Cost of inserting a single ``node``.""" + return costs.insert_cost[node.node_type] + + +def delete_cost(node: ExprNode, costs: EditCosts) -> float: + """Cost of deleting a single ``node``.""" + return costs.delete_cost[node.node_type] + + +def update_cost(a: ExprNode, b: ExprNode, costs: EditCosts) -> float: + """Relabel cost: ``0`` if labels match, per-type cost if the type prefix + matches, else :attr:`EditCosts.change_type_cost`.""" + if a.label == b.label: + return 0.0 + if a.node_type == b.node_type: + return costs.update_cost[a.node_type] + return costs.change_type_cost + + +def subtree_discount(total_cost: float, costs: EditCosts) -> float: + """Discounted cost of inserting/removing a whole subtree. + + ``total_cost`` is the subtree's full per-node insert (or delete) cost. Under + PHYBench's unit costs this equals the node count, so the curve + ``min(total_cost, discount_slope*(total_cost - bar_size) + bar_size)`` matches + upstream exactly; with custom costs it generalizes the discount to the cost + scale. For ``total_cost <= bar_size`` there is no discount; larger subtrees cost + less than deleting them node by node. Insert and remove share this function + (PHYBench ``remove_tree_func == insert_tree_func``). + """ + discounted = costs.discount_slope * (total_cost - costs.bar_size) + costs.bar_size + return min(total_cost, discounted) + + +def eed_score(distance: float, gt_size: int, *, discount_slope: float = 0.6) -> float: + """Map a tree-edit ``distance`` to a ``[0, 1]`` partial-credit fraction. + + ``1.0`` when ``distance == 0``; otherwise ``max(0, discount_slope - + distance/gt_size)`` (PHYBench ``score_calc`` rendered as a fraction). The score + reaches ``0`` once the relative distance exceeds ``discount_slope`` (``0.6``). + A non-positive ``gt_size`` is degenerate and scores ``0``. + """ + if gt_size <= 0: + return 0.0 + if distance == 0: + return 1.0 + return max(0.0, discount_slope - distance / gt_size) + + +__all__ = [ + "EditCosts", + "delete_cost", + "eed_score", + "insert_cost", + "subtree_discount", + "update_cost", +] diff --git a/src/prkit/semantics/edit_distance/timeout.py b/src/prkit/semantics/edit_distance/timeout.py new file mode 100644 index 0000000..2125686 --- /dev/null +++ b/src/prkit/semantics/edit_distance/timeout.py @@ -0,0 +1,56 @@ +"""Thread-safe, Windows-safe bounded execution for ``simplify`` and friends. + +PHYBench bounds ``simplify`` with a ``signal.SIGALRM`` decorator, which only works +on the main thread of the main interpreter — it raises if a scorer runs inside a +thread pool (the common eval-harness setup) and is unavailable on Windows. This +helper uses :mod:`concurrent.futures` instead. + +A timed-out worker thread cannot be force-killed (SymPy's ``simplify`` is CPU-bound +pure Python), so it is *abandoned*: callers must treat :class:`SimplifyTimeout` as a +soft-degrade signal and must not retry in a tight loop. +""" + +from __future__ import annotations + +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import TimeoutError as _FuturesTimeout +from typing import TypeVar + +_T = TypeVar("_T") + + +class SimplifyTimeout(Exception): + """Raised when a bounded computation exceeds its wall-clock deadline.""" + + +def run_with_timeout(func: Callable[[], _T], *, timeout_s: float) -> _T: + """Run ``func()`` with a wall-clock deadline. + + Args: + func: a zero-argument callable (wrap arguments in a ``lambda``/closure). + timeout_s: deadline in seconds. + + Returns: + Whatever ``func()`` returns. + + Raises: + SimplifyTimeout: if ``func`` does not finish within ``timeout_s``. + + Note: + The executor is shut down with ``wait=False`` so a timed-out worker does + not block return (``with ThreadPoolExecutor`` would join it on exit). The + abandoned thread leaks until the underlying computation returns on its own. + """ + pool = ThreadPoolExecutor(max_workers=1) + future = pool.submit(func) + try: + result = future.result(timeout=timeout_s) + except _FuturesTimeout as exc: + pool.shutdown(wait=False, cancel_futures=True) + raise SimplifyTimeout(f"computation exceeded {timeout_s}s deadline") from exc + pool.shutdown(wait=False) + return result + + +__all__ = ["SimplifyTimeout", "run_with_timeout"] diff --git a/src/prkit/semantics/edit_distance/tree.py b/src/prkit/semantics/edit_distance/tree.py new file mode 100644 index 0000000..4a1997f --- /dev/null +++ b/src/prkit/semantics/edit_distance/tree.py @@ -0,0 +1,131 @@ +"""Labeled comparison trees built from SymPy expressions (PHYBench EED grammar). + +``sympy_to_tree`` mirrors PHYBench ``EED.py``'s ``sympy_to_tree``: every node gets +a ``"{type}_{value}"`` label with ``type`` in ``{number, symbol, operator, +function}``. Children of commutative operators (``Add``/``Mul``) are sorted by +:func:`sympy.default_sort_key` so the tree is deterministic across SymPy versions +and processes; non-commutative arguments (``Pow`` base/exponent, function +positional args) keep their order. + +The grammar is deliberately closed: anything outside the additive / multiplicative +/ power / function vocabulary (``Integral``, ``Sum``, ``Derivative``, matrices, +relations, ...) raises :class:`UnsupportedExpressionError` so the pipeline can +degrade to a binary verdict rather than silently mis-scoring. + +This module imports only ``sympy`` + stdlib so the pure algorithm stays reusable +without dragging in the comparison engine. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from sympy import ( + Add, + Float, + Integer, + Mul, + NumberSymbol, + Pow, + Rational, + Symbol, + default_sort_key, +) +from sympy.core.numbers import ( + ComplexInfinity, + Infinity, + NaN, + NegativeInfinity, +) + + +class UnsupportedExpressionError(TypeError): + """Raised when :func:`sympy_to_tree` meets a node outside the EED grammar.""" + + +@dataclass +class ExprNode: + """One node of an EED comparison tree. + + Attributes: + label: ``"{type}_{value}"`` where ``type`` is one of ``number`` / + ``symbol`` / ``operator`` / ``function`` (e.g. ``"number_2"``, + ``"symbol_m"``, ``"operator_Add"``, ``"function_sin"``). + children: ordered child nodes. + """ + + label: str + children: list[ExprNode] = field(default_factory=list) + + @property + def node_type(self) -> str: + """The ``type`` prefix of :attr:`label` (text before the first ``_``).""" + return self.label.split("_", 1)[0] + + def node_count(self) -> int: + """Number of nodes in the subtree rooted here (self included).""" + return 1 + sum(child.node_count() for child in self.children) + + +def _float_key(value: Any) -> str: + """Precision-bounded, deterministic string for a SymPy ``Float`` label. + + ``str(Float("3.14"))`` / ``srepr`` leak binary-float noise (``3.1400000000000001``), + which would make scores depend on print formatting. Rounding to 12 significant + figures keeps physically meaningful precision while staying stable. + """ + return format(float(value), ".12g") + + +def _node_label(expr: Any) -> str: + """Return the ``"{type}_{value}"`` label for a single SymPy node.""" + # Order matters: ``Integer`` is a subclass of ``Rational``; check it first. + if isinstance(expr, Symbol): + return f"symbol_{expr.name}" + if isinstance(expr, Integer): + return f"number_{int(expr)}" + if isinstance(expr, Rational): + return f"number_{expr.p}/{expr.q}" + if isinstance(expr, Float): + return f"number_{_float_key(expr)}" + if isinstance(expr, NumberSymbol): + # pi -> Pi, E -> Exp1, GoldenRatio, EulerGamma, Catalan, ... + return f"number_{type(expr).__name__}" + if isinstance(expr, (Infinity, NegativeInfinity, ComplexInfinity, NaN)): + return f"number_{type(expr).__name__}" + if isinstance(expr, (Add, Mul, Pow)): + return f"operator_{type(expr).__name__}" + if getattr(expr, "is_Function", False): + # sin/cos/exp/log (named) and AppliedUndef f(x) both report is_Function. + return f"function_{type(expr).__name__}" + raise UnsupportedExpressionError( + f"unsupported SymPy node for EED tree: {type(expr).__name__} ({expr!r})" + ) + + +def _ordered_children(expr: Any) -> list[Any]: + """Return child args in a deterministic order. + + Commutative operators (``Add`` / ``Mul``) are sorted by + :func:`sympy.default_sort_key`; everything else keeps positional order so + ``Pow`` base/exponent and function arguments stay meaningful. + """ + if isinstance(expr, (Add, Mul)): + return sorted(expr.args, key=default_sort_key) + return list(expr.args) + + +def sympy_to_tree(expr: Any) -> ExprNode: + """Convert a SymPy expression into a deterministic labeled :class:`ExprNode`. + + Raises: + UnsupportedExpressionError: on any node outside the EED grammar + (calculus operators, matrices, relations, sets, ...). + """ + label = _node_label(expr) + children = [sympy_to_tree(arg) for arg in _ordered_children(expr)] + return ExprNode(label=label, children=children) + + +__all__ = ["ExprNode", "UnsupportedExpressionError", "sympy_to_tree"] diff --git a/src/prkit/semantics/edit_distance/zss.py b/src/prkit/semantics/edit_distance/zss.py new file mode 100644 index 0000000..96cbb54 --- /dev/null +++ b/src/prkit/semantics/edit_distance/zss.py @@ -0,0 +1,149 @@ +"""Extended Zhang-Shasha tree-edit distance with subtree cluster discount. + +A clean reimplementation of the ordered-tree edit distance (Zhang & Shasha 1989) +parametrized by :class:`~prkit.semantics.edit_distance.score.EditCosts`, extended +with PHYBench's whole-subtree "cluster discount": deleting or inserting an entire +subtree can cost less than the per-node sum, so a large wrong sub-formula is not +penalized linearly. + +Two deliberate divergences from PHYBench's ``extended_zss.py`` (documented for +cross-checking): the forest-distance matrix is initialized to ``math.inf`` rather +than the upstream sentinel ``1000`` (which silently mis-scores trees whose edit +distance exceeds the sentinel), and the algorithm is pure Python with no global +state so it is deterministic and thread-safe. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass + +from .score import ( + EditCosts, + delete_cost, + insert_cost, + subtree_discount, + update_cost, +) +from .tree import ExprNode + + +@dataclass +class _Annotated: + """Post-order annotation of a tree used by the Zhang-Shasha DP.""" + + order: list[ExprNode] # post-order nodes; node at post-index ``i`` is order[i-1] + left: dict[int, int] # post-index -> leftmost-leaf post-index + del_total: dict[int, float] # post-index -> total delete cost of the subtree + ins_total: dict[int, float] # post-index -> total insert cost of the subtree + keyroots: list[int] # ascending keyroot post-indices + n: int # total node count + + +def _annotate(root: ExprNode, costs: EditCosts) -> _Annotated: + """Compute post-order, leftmost-leaf indices, subtree costs, and keyroots.""" + order: list[ExprNode] = [] + left: dict[int, int] = {} + del_total: dict[int, float] = {} + ins_total: dict[int, float] = {} + index_of: dict[int, int] = {} + + def visit(node: ExprNode) -> None: + subtree_del = delete_cost(node, costs) + subtree_ins = insert_cost(node, costs) + for child in node.children: + visit(child) + child_idx = index_of[id(child)] + subtree_del += del_total[child_idx] + subtree_ins += ins_total[child_idx] + order.append(node) + idx = len(order) # 1-indexed post-order position + index_of[id(node)] = idx + if node.children: + left[idx] = left[index_of[id(node.children[0])]] + else: + left[idx] = idx + del_total[idx] = subtree_del + ins_total[idx] = subtree_ins + + visit(root) + n = len(order) + + # keyroot(i): the largest post-index sharing leftmost-leaf left[i]. Iterating + # ascending and overwriting keeps exactly that maximum per leftmost value. + keyroot_by_left: dict[int, int] = {} + for i in range(1, n + 1): + keyroot_by_left[left[i]] = i + keyroots = sorted(keyroot_by_left.values()) + + return _Annotated( + order=order, + left=left, + del_total=del_total, + ins_total=ins_total, + keyroots=keyroots, + n=n, + ) + + +def _forest_distance( + a: _Annotated, + b: _Annotated, + i1: int, + j1: int, + treedist: list[list[float]], + costs: EditCosts, +) -> None: + """Fill ``treedist`` for the subtree pair rooted at keyroots ``i1`` / ``j1``.""" + la, lb = a.left[i1], b.left[j1] + rows = i1 - la + 2 + cols = j1 - lb + 2 + fd = [[math.inf] * cols for _ in range(rows)] + fd[0][0] = 0.0 + + for x in range(1, rows): + node = a.order[la + x - 2] # post-index (la + x - 1), 0-based list access + fd[x][0] = fd[x - 1][0] + delete_cost(node, costs) + for y in range(1, cols): + node = b.order[lb + y - 2] + fd[0][y] = fd[0][y - 1] + insert_cost(node, costs) + + for x in range(1, rows): + i = la + x - 1 # actual post-index in A + na = a.order[i - 1] + xa = a.left[i] - la # forest column just before subtree_i + for y in range(1, cols): + j = lb + y - 1 + nb = b.order[j - 1] + yb = b.left[j] - lb + + del_node = fd[x - 1][y] + delete_cost(na, costs) + ins_node = fd[x][y - 1] + insert_cost(nb, costs) + # Whole-subtree discount options: drop subtree_i / add subtree_j as a + # unit at the (never-larger) discounted price. + rem_tree = fd[xa][y] + subtree_discount(a.del_total[i], costs) + ins_tree = fd[x][yb] + subtree_discount(b.ins_total[j], costs) + + if a.left[i] == la and b.left[j] == lb: + upd = fd[x - 1][y - 1] + update_cost(na, nb, costs) + best = min(del_node, ins_node, upd, rem_tree, ins_tree) + fd[x][y] = best + treedist[i][j] = best + else: + match = fd[xa][yb] + treedist[i][j] + fd[x][y] = min(del_node, ins_node, match, rem_tree, ins_tree) + + +def tree_edit_distance(a: ExprNode, b: ExprNode, *, costs: EditCosts) -> float: + """Return the extended Zhang-Shasha edit distance between trees ``a`` and ``b``.""" + ann_a = _annotate(a, costs) + ann_b = _annotate(b, costs) + # treedist is 1-indexed in both dimensions; row/col 0 are unused padding. + treedist = [[0.0] * (ann_b.n + 1) for _ in range(ann_a.n + 1)] + for i1 in ann_a.keyroots: + for j1 in ann_b.keyroots: + _forest_distance(ann_a, ann_b, i1, j1, treedist, costs) + return treedist[ann_a.n][ann_b.n] + + +__all__ = ["tree_edit_distance"] diff --git a/src/prkit/verify/__init__.py b/src/prkit/verify/__init__.py index 01b5574..2d4f5a7 100644 --- a/src/prkit/verify/__init__.py +++ b/src/prkit/verify/__init__.py @@ -72,18 +72,13 @@ def verify( tolerance: numeric comparison tolerance (engine default when ``None``). unit_policy: enforcement strictness — one of ``"strict"`` / ``"audited"`` / ``"permissive"`` (maps to the engine's ``ComparisonPolicyMode``). - partial_credit: reserved. The deterministic engine is binary, so ``True`` - raises ``NotImplementedError`` (the partial-credit scorer, X1, owns it). + partial_credit: when ``True``, score with the graded EED/SEED + :class:`~prkit.scoring.PartialCreditScorer` (which populates + ``Verdict.partial_credit``) instead of the binary deterministic engine. Raises: ValueError: if ``unit_policy`` is not a recognized value. - NotImplementedError: if ``partial_credit=True``. """ - if partial_credit: - raise NotImplementedError( - "partial_credit is not produced by the deterministic verifier; " - "use the partial-credit scorer (X1) when it lands." - ) if unit_policy not in _RECOGNIZED_UNIT_POLICIES: raise ValueError( f"unit_policy must be one of {list(_RECOGNIZED_UNIT_POLICIES)}, " @@ -92,9 +87,15 @@ def verify( # Lazy: keeps anthropic/openai/google.genai/datasets/pandas/sympy off the # bare ``import prkit.verify`` path (provider SDKs are lazy in model_clients). + # math-verify is verify(gold, pred); the Scorer scores prediction vs reference, + # so prediction=pred and reference=gold — do not swap. + if partial_credit: + from prkit.scoring import PartialCreditScorer + + pc_scorer = PartialCreditScorer(tolerance=tolerance, policy_mode=unit_policy) + return pc_scorer.score(pred, gold) + from prkit.scoring import SemanticsScorer scorer = SemanticsScorer(tolerance=tolerance, policy_mode=unit_policy) - # math-verify is verify(gold, pred); the Scorer scores prediction vs reference, - # so prediction=pred and reference=gold — do not swap. return scorer.score(pred, gold) diff --git a/tests/prkit/scoring/test_partial_credit_scorer.py b/tests/prkit/scoring/test_partial_credit_scorer.py new file mode 100644 index 0000000..43fc10a --- /dev/null +++ b/tests/prkit/scoring/test_partial_credit_scorer.py @@ -0,0 +1,132 @@ +"""Tests for :class:`prkit.scoring.PartialCreditScorer` and its Verdict mapping.""" + +from __future__ import annotations + +import pytest + +from prkit.api import Scorer, Verdict +from prkit.scoring import PartialCreditMode, PartialCreditScorer +from prkit.testing import check_scorer + + +class TestProtocol: + def test_satisfies_scorer_protocol(self) -> None: + scorer = PartialCreditScorer() + assert isinstance(scorer, Scorer) + assert scorer.version + assert scorer.get_info()["version"] == scorer.version + + @pytest.mark.parametrize( + "mode", + [ + PartialCreditMode.PARTIAL_CREDIT, + PartialCreditMode.BINARY, + PartialCreditMode.TOLERANCE, + ], + ) + def test_conformance_battery(self, mode: PartialCreditMode) -> None: + check_scorer(PartialCreditScorer(mode=mode)) + + +class TestPartialCreditMode: + def test_near_miss_is_graded_and_fills_partial_credit(self) -> None: + scorer = PartialCreditScorer() + verdict = scorer.score("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") + assert 0.0 < verdict.score < 1.0 + assert verdict.partial_credit == verdict.score + assert verdict.equivalent is False + assert verdict.rationale is not None + + def test_exact_match_full_credit(self) -> None: + verdict = PartialCreditScorer().score("3 m/s", "3 m/s") + assert verdict.score == 1.0 + assert verdict.partial_credit == 1.0 + assert verdict.equivalent is True + assert verdict.units_ok is True + + def test_returns_canonical_verdict(self) -> None: + verdict = PartialCreditScorer().score("F = 2*m*a", "F = m*a") + assert isinstance(verdict, Verdict) + assert verdict.comparison_mode == "eed_relation" + assert verdict.scorer_version == PartialCreditScorer.version + + +class TestBinaryMode: + def test_collapses_to_pass_fail_and_nulls_partial_credit(self) -> None: + scorer = PartialCreditScorer(mode=PartialCreditMode.BINARY) + near_miss = scorer.score("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") + assert near_miss.score == 0.0 + assert near_miss.partial_credit is None + assert near_miss.equivalent is False + + def test_threshold_controls_equivalence(self) -> None: + lenient = PartialCreditScorer( + mode=PartialCreditMode.BINARY, binary_threshold=0.4 + ) + verdict = lenient.score("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") + assert verdict.equivalent is True + assert verdict.score == 1.0 + + +class TestToleranceMode: + def test_numeric_within_reference_precision_passes(self) -> None: + scorer = PartialCreditScorer(mode=PartialCreditMode.TOLERANCE) + verdict = scorer.score("3.005", "3") + assert verdict.score == 1.0 + assert verdict.partial_credit is None + assert verdict.equivalent is True + + def test_expression_uses_symbolic_equivalence(self) -> None: + scorer = PartialCreditScorer(mode=PartialCreditMode.TOLERANCE) + assert scorer.score("x + y", "y + x").equivalent is True + assert scorer.score("x + y", "x - y").equivalent is False + + +class TestGuards: + def test_empty_prediction_surfaces_diagnostics(self) -> None: + verdict = PartialCreditScorer().score("", "3") + assert verdict.score == 0.0 + assert verdict.partial_credit == 0.0 + assert "empty_prediction" in verdict.diagnostics + assert "empty_prediction" in verdict.rationale + + +class TestDeterminism: + def test_repeated_score_is_equal(self) -> None: + scorer = PartialCreditScorer() + first = scorer.score("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") + second = scorer.score("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") + assert first == second + + +class TestConfig: + def test_tolerance_passthrough_flips_near_miss(self) -> None: + # In TOLERANCE mode the pass/fail uses the reference-precision check, which + # honors the configured tolerance; a generous value flips a clear near-miss. + strict = PartialCreditScorer(mode=PartialCreditMode.TOLERANCE).score( + "150", "100" + ) + loose = PartialCreditScorer( + mode=PartialCreditMode.TOLERANCE, tolerance=0.6 + ).score("150", "100") + assert strict.equivalent is False + assert loose.equivalent is True + + def test_get_info_reports_configuration(self) -> None: + info = PartialCreditScorer(mode=PartialCreditMode.BINARY).get_info() + assert info["mode"] == "binary" + assert info["engine"] == "eed_compare" + assert info["deterministic"] is True + + def test_context_dict_and_policies_reported(self) -> None: + scorer = PartialCreditScorer( + tolerance=0.01, + unit_policy="required", + policy_mode="strict", + context={"target_variable": "x"}, # dict context exercises coercion + ) + info = scorer.get_info() + assert info["unit_policy"] == "required" + assert info["policy_mode"] == "strict" + assert info["tolerance"] == 0.01 + assert isinstance(scorer.score("3 m/s", "3 m/s"), Verdict) diff --git a/tests/prkit/semantics/edit_distance/test_edit_distance_pipeline.py b/tests/prkit/semantics/edit_distance/test_edit_distance_pipeline.py new file mode 100644 index 0000000..159064f --- /dev/null +++ b/tests/prkit/semantics/edit_distance/test_edit_distance_pipeline.py @@ -0,0 +1,106 @@ +"""Integration tests for the SEED dispatch (:mod:`...edit_distance.pipeline`).""" + +from __future__ import annotations + +from prkit.semantics import normalize_physics_answer +from prkit.semantics.edit_distance import EedResult, eed_compare + + +def _cmp(pred: str, gold: str) -> EedResult: + return eed_compare(normalize_physics_answer(pred), normalize_physics_answer(gold)) + + +class TestSymbolicShortCircuit: + def test_commutative_expression(self) -> None: + result = _cmp("x + y", "y + x") + assert result.score == 1.0 + assert result.symbolic_equiv is True + + def test_fraction_vs_decimal(self) -> None: + assert _cmp("0.5", "1/2").score == 1.0 + + def test_equation_commutativity(self) -> None: + result = _cmp("F = m a", "m a = F") + assert result.score == 1.0 + assert result.symbolic_equiv is True + + +class TestExpressionPartialCredit: + def test_single_term_near_miss_is_graded(self) -> None: + result = _cmp("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") + assert 0.3 < result.score < 0.6 + assert result.answer_type == "expression" + assert result.gt_tree_size is not None + + def test_unrelated_scores_zero(self) -> None: + assert _cmp("z", "2*m*g + 2*m*v0**2/l").score == 0.0 + + +class TestRelationPartialCredit: + def test_near_miss_equation_is_graded(self) -> None: + result = _cmp("F = 2*m*a", "F = m*a") + assert 0.0 < result.score < 1.0 + assert result.answer_type == "relation" + + def test_sign_flipped_equation_is_equivalent(self) -> None: + # F - m a = 0 vs m a - F = 0 are the same equation. + assert _cmp("F - m*a = 0", "m*a - F = 0").score == 1.0 + + +class TestNumericLeaf: + def test_identical_quantity(self) -> None: + result = _cmp("3 m/s", "3 m/s") + assert result.score == 1.0 + assert result.units_ok is True + + def test_convertible_units_equal(self) -> None: + result = _cmp("1 km", "1000 m") + assert result.score == 1.0 + assert result.units_ok is True + + def test_incompatible_units(self) -> None: + result = _cmp("5 m", "5 s") + assert result.score == 0.0 + assert result.units_ok is False + assert "unit_mismatch" in result.diagnostics + + def test_sign_mismatch(self) -> None: + result = _cmp("-3", "3") + assert result.score == 0.0 + assert "sign_mismatch" in result.diagnostics + + def test_numeric_mismatch(self) -> None: + assert _cmp("3 m/s", "5 m/s").score == 0.0 + + def test_seed_tiers(self) -> None: + assert _cmp("3.005", "3").score == 1.0 # rel 0.0017 -> tier 1.0 + assert _cmp("3.045", "3").score == 0.9 # rel 0.015 -> tier 0.9 + assert _cmp("3.09", "3").score == 0.8 # rel 0.03 -> tier 0.8 + assert _cmp("3.3", "3").score == 0.0 # rel 0.10 -> 0.0 + + +class TestGuards: + def test_empty_prediction(self) -> None: + result = _cmp("", "3") + assert result.score == 0.0 + assert "empty_prediction" in result.diagnostics + + def test_integral_is_unsupported(self) -> None: + result = _cmp(r"\int x dx", "x^2/2") + assert result.score == 0.0 + assert "unsupported_operator" in result.diagnostics + + def test_sum_is_unsupported(self) -> None: + assert _cmp(r"\sum_n a_n", "a").score == 0.0 + + def test_runaway_length_is_guarded(self) -> None: + result = _cmp("a + b + c + d + e + f + g + h + i + j + k", "F=ma") + assert result.score == 0.0 + assert "length_ratio_exceeded" in result.diagnostics + + +class TestDeterminism: + def test_repeated_compare_is_identical(self) -> None: + first = _cmp("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") + second = _cmp("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") + assert first == second diff --git a/tests/prkit/semantics/edit_distance/test_edit_distance_robustness.py b/tests/prkit/semantics/edit_distance/test_edit_distance_robustness.py new file mode 100644 index 0000000..3303486 --- /dev/null +++ b/tests/prkit/semantics/edit_distance/test_edit_distance_robustness.py @@ -0,0 +1,41 @@ +"""Robustness tests: timeout helper, number constants, and no-simplify scoring.""" + +from __future__ import annotations + +import time + +import pytest +import sympy as sp + +from prkit.semantics import normalize_physics_answer +from prkit.semantics.edit_distance import EedConfig, eed_compare, sympy_to_tree +from prkit.semantics.edit_distance.timeout import SimplifyTimeout, run_with_timeout + + +class TestTimeout: + def test_returns_value_when_fast(self) -> None: + assert run_with_timeout(lambda: 1 + 1, timeout_s=5.0) == 2 + + def test_raises_on_slow_callable(self) -> None: + with pytest.raises(SimplifyTimeout): + run_with_timeout(lambda: time.sleep(2.0), timeout_s=0.05) + + +class TestNumberConstants: + def test_infinities_and_constants(self) -> None: + assert sympy_to_tree(sp.oo).label == "number_Infinity" + assert sympy_to_tree(sp.S.NegativeInfinity).label == "number_NegativeInfinity" + assert sympy_to_tree(sp.zoo).label == "number_ComplexInfinity" + assert sympy_to_tree(sp.nan).label == "number_NaN" + assert sympy_to_tree(sp.GoldenRatio).label == "number_GoldenRatio" + + +class TestNoSimplify: + def test_scores_without_pre_simplify(self) -> None: + cfg = EedConfig(simplify_before_tree=False) + result = eed_compare( + normalize_physics_answer("2*m*g + 4*m*v0**2/l"), + normalize_physics_answer("2*m*g + 2*m*v0**2/l"), + config=cfg, + ) + assert 0.0 < result.score < 1.0 diff --git a/tests/prkit/semantics/edit_distance/test_edit_distance_score.py b/tests/prkit/semantics/edit_distance/test_edit_distance_score.py new file mode 100644 index 0000000..9b12beb --- /dev/null +++ b/tests/prkit/semantics/edit_distance/test_edit_distance_score.py @@ -0,0 +1,85 @@ +"""Unit tests for the EED cost model and score map (:mod:`...edit_distance.score`).""" + +from __future__ import annotations + +from prkit.semantics.edit_distance.score import ( + EditCosts, + delete_cost, + eed_score, + insert_cost, + subtree_discount, + update_cost, +) +from prkit.semantics.edit_distance.tree import ExprNode + + +class TestEditCosts: + def test_default_unit_costs(self) -> None: + costs = EditCosts() + for node_type in ("number", "symbol", "operator", "function"): + assert costs.insert_cost[node_type] == 1.0 + assert costs.delete_cost[node_type] == 1.0 + assert costs.update_cost[node_type] == 1.0 + assert costs.change_type_cost == 1.0 + assert costs.bar_size == 5 + assert costs.discount_slope == 0.6 + + def test_cost_maps_are_independent_instances(self) -> None: + a = EditCosts() + b = EditCosts() + a.insert_cost["number"] = 99.0 + assert b.insert_cost["number"] == 1.0 + + +class TestNodeCosts: + def test_insert_and_delete_use_node_type(self) -> None: + costs = EditCosts() + node = ExprNode("symbol_x") + assert insert_cost(node, costs) == 1.0 + assert delete_cost(node, costs) == 1.0 + + def test_update_identical_labels_is_zero(self) -> None: + costs = EditCosts() + assert update_cost(ExprNode("number_2"), ExprNode("number_2"), costs) == 0.0 + + def test_update_same_type_uses_type_cost(self) -> None: + costs = EditCosts() + assert update_cost(ExprNode("number_2"), ExprNode("number_3"), costs) == 1.0 + + def test_update_different_type_uses_change_type_cost(self) -> None: + costs = EditCosts(change_type_cost=7.0) + assert update_cost(ExprNode("number_2"), ExprNode("symbol_x"), costs) == 7.0 + + +class TestSubtreeDiscount: + def test_small_subtrees_get_no_discount(self) -> None: + costs = EditCosts() + assert subtree_discount(1, costs) == 1.0 + assert subtree_discount(3, costs) == 3.0 + assert subtree_discount(5, costs) == 5.0 + + def test_large_subtrees_are_discounted(self) -> None: + costs = EditCosts() + # 0.6 * (15 - 5) + 5 == 11 < 15 + assert subtree_discount(15, costs) == 11.0 + assert subtree_discount(15, costs) < 15.0 + + +class TestEedScore: + def test_exact_match_is_one(self) -> None: + assert eed_score(0, 13) == 1.0 + + def test_single_edit_formula(self) -> None: + assert eed_score(1, 13) == 0.6 - 1 / 13 + + def test_clamped_to_zero_beyond_threshold(self) -> None: + assert eed_score(20, 10) == 0.0 + + def test_degenerate_gt_size_is_zero(self) -> None: + assert eed_score(5, 0) == 0.0 + assert eed_score(0, 0) == 0.0 + + def test_score_in_unit_interval(self) -> None: + for distance in range(0, 25): + value = eed_score(distance, 12) + assert 0.0 <= value <= 1.0 diff --git a/tests/prkit/semantics/edit_distance/test_edit_distance_tree.py b/tests/prkit/semantics/edit_distance/test_edit_distance_tree.py new file mode 100644 index 0000000..27eca8a --- /dev/null +++ b/tests/prkit/semantics/edit_distance/test_edit_distance_tree.py @@ -0,0 +1,97 @@ +"""Unit tests for the SymPy -> ExprNode tree builder (:mod:`...edit_distance.tree`).""" + +from __future__ import annotations + +import pytest +import sympy as sp + +from prkit.semantics.edit_distance.tree import ( + ExprNode, + UnsupportedExpressionError, + sympy_to_tree, +) + + +def _labels(node: ExprNode) -> set[str]: + """Collect every label in a tree.""" + labels = {node.label} + for child in node.children: + labels |= _labels(child) + return labels + + +class TestNodeTyping: + def test_integer_symbol_operator(self) -> None: + tree = sympy_to_tree(sp.sympify("2*m*g")) + assert tree.label == "operator_Mul" + assert "number_2" in _labels(tree) + assert "symbol_m" in _labels(tree) + assert "symbol_g" in _labels(tree) + + def test_negative_integer_is_single_number_node(self) -> None: + tree = sympy_to_tree(sp.Integer(-3)) + assert tree.label == "number_-3" + assert tree.children == [] + + def test_rational_label(self) -> None: + assert sympy_to_tree(sp.Rational(1, 2)).label == "number_1/2" + + def test_float_label_is_precision_bounded(self) -> None: + # str()/srepr would leak 3.1400000000000001; we want a stable 3.14. + assert sympy_to_tree(sp.Float("3.14")).label == "number_3.14" + + def test_number_symbols(self) -> None: + assert sympy_to_tree(sp.pi).label == "number_Pi" + assert sympy_to_tree(sp.E).label == "number_Exp1" + + def test_named_and_undefined_functions(self) -> None: + x = sp.Symbol("x") + assert sympy_to_tree(sp.sin(x)).label == "function_sin" + assert sympy_to_tree(sp.Function("f")(x)).label == "function_f" + + def test_pow_keeps_base_exponent_order(self) -> None: + tree = sympy_to_tree(sp.sympify("v0**2")) + assert tree.label == "operator_Pow" + assert [child.label for child in tree.children] == ["symbol_v0", "number_2"] + + +class TestDeterminism: + def test_commutative_reorder_yields_identical_tree(self) -> None: + a = sympy_to_tree(sp.sympify("x + y")) + b = sympy_to_tree(sp.sympify("y + x")) + assert _serialize(a) == _serialize(b) + + def test_repeated_builds_are_identical(self) -> None: + expr = sp.sympify("2*m*g + 2*m*v0**2/l") + assert _serialize(sympy_to_tree(expr)) == _serialize(sympy_to_tree(expr)) + + +class TestNodeCount: + def test_counts_all_nodes(self) -> None: + # Mul(2, g, m): root + 3 leaves == 4 nodes. + assert sympy_to_tree(sp.sympify("2*m*g")).node_count() == 4 + + def test_leaf_count_is_one(self) -> None: + assert sympy_to_tree(sp.Symbol("x")).node_count() == 1 + + +class TestUnsupported: + @pytest.mark.parametrize( + "expr", + [ + sp.Integral(sp.Symbol("x"), sp.Symbol("x")), + sp.Derivative(sp.Function("f")(sp.Symbol("x")), sp.Symbol("x")), + sp.Sum(sp.Symbol("x"), (sp.Symbol("x"), 1, 3)), + sp.Matrix([[1, 2]]), + sp.Eq(sp.Symbol("x"), sp.Symbol("y")), + ], + ) + def test_unsupported_nodes_raise(self, expr: sp.Basic) -> None: + with pytest.raises(UnsupportedExpressionError): + sympy_to_tree(expr) + + +def _serialize(node: ExprNode) -> str: + """Stable string form of a tree, for equality assertions.""" + inner = ",".join(_serialize(child) for child in node.children) + return f"{node.label}({inner})" diff --git a/tests/prkit/semantics/edit_distance/test_edit_distance_zss.py b/tests/prkit/semantics/edit_distance/test_edit_distance_zss.py new file mode 100644 index 0000000..9c9deee --- /dev/null +++ b/tests/prkit/semantics/edit_distance/test_edit_distance_zss.py @@ -0,0 +1,102 @@ +"""Unit tests for the extended Zhang-Shasha distance (:mod:`...edit_distance.zss`).""" + +from __future__ import annotations + +import sympy as sp + +from prkit.semantics.edit_distance.score import EditCosts, eed_score +from prkit.semantics.edit_distance.tree import ExprNode, sympy_to_tree +from prkit.semantics.edit_distance.zss import tree_edit_distance + +_COSTS = EditCosts() + + +def _dist(a: str, b: str) -> float: + return tree_edit_distance( + sympy_to_tree(sp.sympify(a)), sympy_to_tree(sp.sympify(b)), costs=_COSTS + ) + + +class TestBasics: + def test_identical_trees_have_zero_distance(self) -> None: + assert _dist("2*m*g", "2*m*g") == 0.0 + + def test_commutative_equal_trees_zero(self) -> None: + assert _dist("x + y", "y + x") == 0.0 + + def test_single_leaf_relabel_costs_one(self) -> None: + assert _dist("x", "y") == 1.0 + + def test_single_coefficient_change(self) -> None: + # One leaf differs (4 vs 2); distance is a single update. + gold = sympy_to_tree(sp.sympify("2*m*g + 2*m*v0**2/l")) + pred = sympy_to_tree(sp.sympify("2*m*g + 4*m*v0**2/l")) + distance = tree_edit_distance(pred, gold, costs=_COSTS) + assert distance == 1.0 + + +class TestLiteratureExample: + def test_phybench_near_miss_band(self) -> None: + gold = sympy_to_tree(sp.sympify("2*m*g + 2*m*v0**2/l")) + pred = sympy_to_tree(sp.sympify("2*m*g + 4*m*v0**2/l")) + distance = tree_edit_distance(pred, gold, costs=_COSTS) + score = eed_score(distance, gold.node_count()) + # PHYBench reports ~0.47 for this pair; assert a band (tree size is + # parser-dependent), not an exact constant. + assert 0.4 < score < 0.55 + + def test_unrelated_expression_scores_zero(self) -> None: + gold = sympy_to_tree(sp.sympify("2*m*g + 2*m*v0**2/l")) + pred = sympy_to_tree(sp.sympify("z")) + distance = tree_edit_distance(pred, gold, costs=_COSTS) + assert eed_score(distance, gold.node_count()) == 0.0 + + +class TestSubtreeDiscount: + def test_large_subtree_swap_uses_discount(self) -> None: + # gold = f(<10 distinct leaves>); pred replaces the whole argument subtree. + # The discounted whole-subtree edit must beat deleting 10 nodes one by one. + big = ExprNode( + "operator_Add", + [ExprNode(f"symbol_s{i}") for i in range(10)], + ) + gold = ExprNode("function_f", [big]) + pred = ExprNode("function_f", [ExprNode("symbol_z")]) + distance = tree_edit_distance(pred, gold, costs=_COSTS) + # Replacing 11 nodes (Add + 10 leaves) with one leaf: discounted, < 11. + assert distance < 11.0 + + +class TestInfInitRegression: + def test_distance_above_sentinel_is_exact(self) -> None: + # PHYBench's extended_zss inits the forest matrix to the sentinel 1000, + # silently capping any distance above it. We init to math.inf, so a true + # distance over 1000 must come through exactly. High per-node costs let a + # tiny tree exceed the sentinel instantly (no slow giant-tree DP). + big = 300.0 + types = ("number", "symbol", "operator", "function") + costs = EditCosts( + insert_cost={t: big for t in types}, + delete_cost={t: big for t in types}, + update_cost={t: big for t in types}, + change_type_cost=big, + ) + # Same shape (root + 4 leaves), every label differs -> relabel all 5 nodes. + a = ExprNode( + "operator_Add", + [ExprNode(f"symbol_a{i}") for i in range(4)], + ) + b = ExprNode( + "operator_Mul", + [ExprNode(f"symbol_b{i}") for i in range(4)], + ) + distance = tree_edit_distance(a, b, costs=costs) + assert distance == 5 * big # 1500 + assert distance > 1000.0 + + +class TestDeterminism: + def test_distance_is_stable(self) -> None: + first = _dist("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") + second = _dist("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") + assert first == second diff --git a/tests/prkit/test_conformance.py b/tests/prkit/test_conformance.py index 8763b6b..2ade4a5 100644 --- a/tests/prkit/test_conformance.py +++ b/tests/prkit/test_conformance.py @@ -14,7 +14,7 @@ from prkit.api import Verdict from prkit.core.model_clients.base import BaseModelClient from prkit.datasets.hub import DatasetHub -from prkit.scoring import SemanticsScorer +from prkit.scoring import PartialCreditScorer, SemanticsScorer from prkit.testing import check_dataset, check_model_client, check_scorer @@ -42,6 +42,10 @@ def test_reference_scorer_conforms(): check_scorer(SemanticsScorer()) +def test_partial_credit_scorer_conforms(): + check_scorer(PartialCreditScorer()) + + def test_stub_model_client_conforms_offline(): check_model_client(_StubClient("stub-model"), live=False) diff --git a/tests/prkit/verify/test_verify.py b/tests/prkit/verify/test_verify.py index f294aef..9dd1e0d 100644 --- a/tests/prkit/verify/test_verify.py +++ b/tests/prkit/verify/test_verify.py @@ -49,9 +49,26 @@ def test_gold_pred_argument_order(self): assert v.extracted_answer is not None assert "4" in v.extracted_answer - def test_partial_credit_true_raises(self): - with pytest.raises(NotImplementedError): - verify("a", "b", partial_credit=True) + def test_partial_credit_true_returns_graded_verdict(self): + # verify(gold, pred): a one-coefficient near-miss earns graded credit. + v = verify( + "2*m*g + 2*m*v0**2/l", + "2*m*g + 4*m*v0**2/l", + partial_credit=True, + ) + assert isinstance(v, Verdict) + assert 0.0 < v.score < 1.0 + assert v.partial_credit == v.score + assert v.correct is False + + def test_partial_credit_exact_match_full_credit(self): + v = verify("3 m/s", "3 m/s", partial_credit=True) + assert v.partial_credit == 1.0 + assert v.correct is True + + def test_partial_credit_unknown_unit_policy_raises(self): + with pytest.raises(ValueError, match="unit_policy"): + verify("a", "b", partial_credit=True, unit_policy="bogus") def test_unknown_unit_policy_raises(self): with pytest.raises(ValueError, match="unit_policy"): From 89658146e2d4100afa9dd491897e263001772d50 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Thu, 18 Jun 2026 23:25:02 -0400 Subject: [PATCH 04/28] Improve relation equivalence in the comparison engine Decide equality relations by their homogeneous form H = L - R: two equalities are equivalent iff their denominator-cleared numerators agree up to a nonzero constant. This admits cross-"=" rearrangement (F=ma vs a=F/m, 1/f=1/u+1/v vs f=uv/(u+v)) while rejecting equations with extra roots (x=0 vs x*y=0). The per-operator criteria (_equalities_equivalent / _inequalities_equivalent) are stated directly rather than as fallbacks after a stricter check. Fold functional-form notation into the canonical clause form so r(t)=... is compared as r=..., and fix two parsing canonicalizations that previously blocked symbolic matches: an "=" inside a summation limit (\sum_{n=1}^{N}) no longer splits relation clauses, and compact products carrying a subscript expand (NmV_r -> N*m*V_r). Add accept and adversarial-reject tests for each behaviour. Co-Authored-By: Claude Opus 4.8 --- src/prkit/semantics/comparison/semantics.py | 210 ++++++++++++++++-- .../semantics/test_protocol_comparison.py | 89 ++++++++ tests/prkit/verify/test_verify.py | 7 + 3 files changed, 287 insertions(+), 19 deletions(-) diff --git a/src/prkit/semantics/comparison/semantics.py b/src/prkit/semantics/comparison/semantics.py index 52763dc..ef31612 100644 --- a/src/prkit/semantics/comparison/semantics.py +++ b/src/prkit/semantics/comparison/semantics.py @@ -33,6 +33,7 @@ cosh, exp, false, + fraction, log, oo, pi, @@ -42,6 +43,7 @@ sqrt, tan, tanh, + together, trigsimp, true, ) @@ -137,6 +139,11 @@ _RELATION_CONSTRAINT_TOKEN_RE = re.compile( r"(?:<=|>=|<|>|≤|≥|!=|≠|≈|∈|∉|\\(?:leq|geq|neq|approx|in|notin)\b)" ) +_BIG_OPERATOR_BOUND_RE = re.compile( + r"\\?(?Psum|prod|coprod|int|oint|iint|iiint|bigcup|bigcap|bigoplus|bigotimes)" + r"\s*_\s*\{(?P[^{}]*=[^{}]*)\}" + r"(?:\s*\^\s*(?:\{(?P[^{}]*)\}|(?P[A-Za-z0-9]+)))?" +) _BARE_FUNCTION_NAMES = ( "sinh", "cosh", @@ -164,7 +171,9 @@ r"\s+(?P\{[^{}]+\}|[A-Za-z0-9_]+)" ) _FUNCTION_COMMANDS = frozenset(_BARE_FUNCTION_NAMES) | {"ln"} -_SHORT_SYMBOL_RUN_RE = re.compile(r"(?[A-Za-z]{2,})(?P_[A-Za-z0-9]+)?\b" +) _TRIG_FUNCTION_RE = re.compile(r"\b(?:sin|cos|tan|asin|acos|atan|sinh|cosh|tanh)\b") _ALIAS_BOUNDARY_TOKEN_RE = re.compile(r"[A-Za-z0-9_]") _DEFINITION_LIKE_TOKEN_RE = re.compile(r"(?=!])\b[\w]+\s*=") @@ -949,7 +958,13 @@ def parse_relation_clauses( *, alias_map: Mapping[str, str] | None = None, ) -> tuple[RelationClause, ...] | None: - """Parse relation text into a flat tuple of binary clauses.""" + """Parse relation text into a flat tuple of canonical binary clauses. + + Canonicalization folds a single-symbol functional-form left-hand side + (``r(t) = ...`` -> ``r = ...``) -- the standard "target as a function of its + variable" notation -- so a relation has one canonical clause form independent of + that surface choice. + """ candidate = preprocess_symbolic_text(text, alias_map=alias_map) if not candidate: @@ -959,7 +974,7 @@ def parse_relation_clauses( if relation_object is not None: relation_clauses = _clauses_from_relation_object(relation_object) if relation_clauses: - return relation_clauses + return _canonical_relation_clauses(relation_clauses) clauses: list[RelationClause] = [] for segment in _split_top_level_conjunctions(candidate): @@ -967,7 +982,7 @@ def parse_relation_clauses( if parsed is None: return None clauses.extend(parsed) - return tuple(clauses) if clauses else None + return _canonical_relation_clauses(tuple(clauses)) if clauses else None def relations_equivalent( @@ -994,6 +1009,21 @@ def relations_equivalent( right_clauses = parse_relation_clauses(right_text, alias_map=alias_map) if left_clauses is None or right_clauses is None: return normalize_plain_text(left_text) == normalize_plain_text(right_text) + + return _relation_clause_sets_equivalent( + left_clauses, right_clauses, tolerance, alias_map=alias_map + ) + + +def _relation_clause_sets_equivalent( + left_clauses: tuple[RelationClause, ...], + right_clauses: tuple[RelationClause, ...], + tolerance: float, + *, + alias_map: Mapping[str, str] | None = None, +) -> bool: + """Match two clause sets as an order-insensitive collection of equivalent clauses.""" + if len(left_clauses) != len(right_clauses): return False @@ -1015,6 +1045,39 @@ def relations_equivalent( return True +_FUNCTIONAL_FORM_LHS_RE = re.compile( + r"^(?P[A-Za-z][A-Za-z0-9_]*)\s*\((?P[^()]*)\)$" +) + + +def _canonical_relation_clauses( + clauses: tuple[RelationClause, ...], +) -> tuple[RelationClause, ...]: + """Return the canonical clause form used for all relation comparison.""" + + return tuple(_collapse_functional_form_lhs(clause) for clause in clauses) + + +def _collapse_functional_form_lhs(clause: RelationClause) -> RelationClause: + """Canonicalize a single-symbol functional-form LHS (``r(t)`` -> ``r``). + + Physics answers write the requested quantity as a function of its variable on the + left of an equation (``r(t) = ...``, ``v(x) = ...``). That is the same assertion as + ``r = ...``, so the canonical relation form drops the argument list. The rewrite is + meaning-preserving and applied to every relation, so equal relations share one + clause form regardless of this surface choice. It fires only on the bare + ``name(args)`` LHS form with a variable argument, leaving genuine point evaluations + such as ``f(2) = 3`` untouched. + """ + + match = _FUNCTIONAL_FORM_LHS_RE.match(clause.lhs_text.strip()) + if match is None: + return clause + if not re.search(r"[A-Za-z]", match.group("args")): + return clause + return RelationClause(match.group("name"), clause.operator, clause.rhs_text) + + def relation_compare_candidates( text: str | None, *, @@ -1258,6 +1321,7 @@ def preprocess_symbolic_text( normalized = _LATEX_SPACING_RE.sub(" ", normalized) normalized = normalized.replace(" true", " True").replace(" false", " False") normalized = normalized.replace("true", "True").replace("false", "False") + normalized = _normalize_big_operator_bounds(normalized) normalized = _replace_simple_latex(normalized) normalized = _normalize_latex_accents(normalized) normalized = _replace_latex_symbol_commands(normalized) @@ -1433,6 +1497,7 @@ def _normalize_alias_surface(text: str | None) -> str: normalized = _LATEX_SPACING_RE.sub(" ", normalized) normalized = normalized.replace(" true", " True").replace(" false", " False") normalized = normalized.replace("true", "True").replace("false", "False") + normalized = _normalize_big_operator_bounds(normalized) normalized = _replace_simple_latex(normalized) normalized = _normalize_latex_accents(normalized) normalized = _replace_latex_symbol_commands(normalized) @@ -1532,14 +1597,20 @@ def _normalize_symbol_products(text: str) -> str: def _rewrite_symbol_run(match: re.Match[str]) -> str: - """Expand an ambiguous symbol run unless it is a protected function/constant name.""" + """Expand an ambiguous symbol run unless it is a protected function/constant name. - token = match.group(0) - if token in _PROTECTED_SYMBOL_RUNS or token.lower() in _PROTECTED_SYMBOL_RUNS: - return token - if token[0].islower() and len(token) > 3: - return token - return "*".join(token) + A trailing subscript (``NmV_r``) is kept attached to the final factor so a compact + product with a subscript expands the same way as the bare run: ``NmV_r`` -> + ``N*m*V_r``, consistent with ``NmV`` -> ``N*m*V``. + """ + + run = match.group("run") + subscript = match.group("sub") or "" + if run in _PROTECTED_SYMBOL_RUNS or run.lower() in _PROTECTED_SYMBOL_RUNS: + return run + subscript + if run[0].islower() and len(run) > 3: + return run + subscript + return "*".join(run) + subscript def _normalize_bare_function_calls(text: str) -> str: @@ -1610,6 +1681,30 @@ def _strip_text_wrappers(text: str | None) -> str: return stripped +def _normalize_big_operator_bounds(text: str) -> str: + """Fold a big operator carrying an ``=``-bearing limit into one opaque token. + + ``\\sum_{n=1}^{N}`` (or the backslash-free ``sum_{n=1}^{N}`` surface) becomes + ``sum_n_1_N``. This removes the limit ``=``, which the relation parser would + otherwise mistake for a top-level equality separator and split on -- corrupting + ``V = sum_{n=1}^{N} ...`` into nonsense clauses. The bounds are folded into the + token so distinct summations are never conflated; full summation equivalence + (dummy-index renaming, reindexing) is intentionally out of scope here. + """ + + def _replace(match: re.Match[str]) -> str: + op = match.group("op") + lower = match.group("lower") + upper = match.group("upper_braced") or match.group("upper_plain") or "" + parts = [op, *re.split(r"=", lower), upper] + token = "_".join( + cleaned for part in parts if (cleaned := re.sub(r"[^A-Za-z0-9]+", "", part)) + ) + return f" {token} " + + return _BIG_OPERATOR_BOUND_RE.sub(_replace, text) + + def _replace_simple_latex(text: str) -> str: """Expand a small LaTeX subset into parser-friendly ASCII math.""" @@ -2012,7 +2107,12 @@ def _relation_clause_equivalent( *, alias_map: Mapping[str, str] | None = None, ) -> bool: - """Compare two relation clauses, including reversed and scaled formulations.""" + """Whether two relation clauses denote the same constraint. + + Two layered criteria: surface equality (clause sides equivalent directly or + reversed), then an algebraic criterion on the homogeneous forms ``H = L - R`` + dispatched by operator class (``_equalities_equivalent`` / ``_inequalities_equivalent``). + """ if ( left.operator == right.operator @@ -2063,17 +2163,67 @@ def _relation_clause_equivalent( left_residual = simplify(left_lhs - left_rhs) right_residual = simplify(right_lhs - right_rhs) - ratio = _proportional_ratio(left_residual, right_residual, tolerance) - if ratio is None: - return False + # Algebraic criterion on the homogeneous forms ``H = L - R``, by operator class. if left.operator == "=" and right.operator == "=": - return True + return _equalities_equivalent(left_residual, right_residual, tolerance) + return _inequalities_equivalent( + left_residual, + right_residual, + left.operator, + right.operator, + tolerance, + ) - if ratio > 0 and left.operator == right.operator: - return True - return ratio < 0 and left.operator == _RELATION_REVERSED.get(right.operator) +def _equalities_equivalent( + left_residual: Any, right_residual: Any, tolerance: float +) -> bool: + """Whether two equalities denote the same constraint. + + The homogeneous form of ``L = R`` is ``H = L - R``. Two equalities are equivalent + iff their denominator-cleared numerators agree up to a nonzero *constant*: clearing + denominators admits cross-``=`` rearrangement (``F = m a`` vs ``a = F/m``), and the + constant -- rather than rational -- factor rejects spurious polynomial factors that + would enlarge the solution set (``x = 0`` vs ``x y = 0``). Tautologies (``0 = 0``) + are equivalent to one another and to nothing else. + """ + + left_zero = left_residual == 0 or left_residual.is_zero is True + right_zero = right_residual == 0 or right_residual.is_zero is True + if left_zero or right_zero: + return bool(left_zero and right_zero) + + left_numerator = _relation_residual_numerator(left_residual) + right_numerator = _relation_residual_numerator(right_residual) + return ( + left_numerator is not None + and right_numerator is not None + and _proportional_ratio(left_numerator, right_numerator, tolerance) is not None + ) + + +def _inequalities_equivalent( + left_residual: Any, + right_residual: Any, + left_operator: str, + right_operator: str, + tolerance: float, +) -> bool: + """Whether two inequalities denote the same constraint. + + The homogeneous forms must be a *signed constant* multiple of one another, and the + sign must be consistent with the operator directions: a positive factor preserves + the operator, a negative factor reverses it. Denominators are not cleared because an + unknown-sign denominator could silently flip the inequality. + """ + + ratio = _proportional_ratio(left_residual, right_residual, tolerance) + if ratio is None: + return False + if ratio > 0 and left_operator == right_operator: + return True + return ratio < 0 and left_operator == _RELATION_REVERSED.get(right_operator) def _proportional_ratio( @@ -2110,6 +2260,28 @@ def _proportional_ratio( return None +def _relation_residual_numerator(residual: Any) -> Any | None: + """Clear denominators from a homogeneous relation form, returning its numerator. + + Two equalities are equivalent when their homogeneous forms ``H = L - R`` agree up + to a nonzero rational-function multiple (e.g. ``F = m a`` vs ``a = F/m``). Comparing + the denominator-cleared numerators up to a nonzero *constant* admits that + rearrangement while rejecting spurious polynomial factors (``x = 0`` vs ``x y = 0``). + Returns ``None`` when the numerator is not a usable nonzero scalar expression. + """ + + try: + numerator, _denominator = fraction(together(residual)) + numerator = simplify(numerator) + except Exception: + return None + if not _is_scalar_symbolic_object(numerator): + return None + if numerator == 0 or numerator.is_zero is True: + return None + return numerator + + def _strip_relation_condition_prefix(text: str) -> str: """Remove lightweight prose prefixes that introduce one relation segment.""" diff --git a/tests/prkit/semantics/test_protocol_comparison.py b/tests/prkit/semantics/test_protocol_comparison.py index 9dd0a15..a29e8dd 100644 --- a/tests/prkit/semantics/test_protocol_comparison.py +++ b/tests/prkit/semantics/test_protocol_comparison.py @@ -2268,3 +2268,92 @@ def test_protocol_qualitative_zero_bridge_is_off_by_default_in_audited_mode() -> assert result.equivalent is False assert result.comparison_mode == "bridge_blocked" + + +# --------------------------------------------------------------------------- +# Atomic-answer equivalence: precision-preserving recall improvements. +# Each block ships accept cases AND adversarial reject cases, per the +# methodology in semantics/comparison/METHODOLOGY.md. +# --------------------------------------------------------------------------- + + +def _relation(canonical_text: str) -> dict[str, str]: + return {"object_kind": "relation", "canonical_text": canonical_text} + + +def test_protocol_relation_equivalent_under_algebraic_rearrangement() -> None: + # Equalities that are the same law after solving for a different variable or + # clearing denominators are now recognized (homogeneous form, numerator up to + # a nonzero constant). + for pred, ref in [ + ("F = m a", "a = F/m"), + ("E = m c^2", "m = E/c^2"), + ("1/f = 1/u + 1/v", "f = (u v)/(u + v)"), + ]: + result = compare_protocol_answers(_relation(pred), _relation(ref)) + assert result.equivalent is True, (pred, ref) + assert result.comparison_mode == "relation" + + +def test_protocol_relation_rearrangement_rejects_distinct_equations() -> None: + # The "up to a nonzero constant" condition rejects spurious polynomial factors: + # these equations have genuinely different solution sets. + for pred, ref in [ + ("x = 0", "x*y = 0"), + ("x = 1", "x^2 = 1"), + ("F = m a", "F = m/a"), + ]: + result = compare_protocol_answers(_relation(pred), _relation(ref)) + assert result.equivalent is False, (pred, ref) + + +def test_protocol_relation_rearrangement_does_not_relax_inequalities() -> None: + # Clearing a symbol-signed denominator can flip an inequality, so the + # rearrangement fallback is gated to equalities only. + result = compare_protocol_answers(_relation("F < m a"), _relation("a < F/m")) + assert result.equivalent is False + + +def test_protocol_relation_functional_form_lhs_is_normalized() -> None: + # "target as a function of its variable" notation on the LHS no longer blocks a + # match: r(t) = ... is compared as r = ... + for pred, ref in [ + ("r(t) = r_0 e^{-2 alpha t/m}", "Eq(r, r_0*exp(-2*alpha*t/m))"), + ("v(x) = a x + b", "v = a x + b"), + ]: + result = compare_protocol_answers(_relation(pred), _relation(ref)) + assert result.equivalent is True, (pred, ref) + assert result.comparison_mode == "relation" + + +def test_protocol_relation_functional_form_lhs_rejects_distinct() -> None: + for pred, ref in [ + ("x = a + b", "y = a + b"), # different target + ("f(x) = x^2", "g(x) = x^2"), # different function name + ("r(t) = a t", "r(t) = b t"), # same target, different RHS + ("f(2) = 3", "f = 3"), # numeric evaluation must not be collapsed + ]: + result = compare_protocol_answers(_relation(pred), _relation(ref)) + assert result.equivalent is False, (pred, ref) + + +def test_protocol_relation_summation_bound_does_not_corrupt_parsing() -> None: + # The "=" inside a summation limit must not be split as a top-level equality. + # Two surface forms of the same summation compare equal once parsing is intact. + pred = "V = sum_{n=1}^{N} (m V_r)/(M + n m)" + ref = r"V = \sum_{n=1}^{N} \frac{m V_r}{M + n m}" + assert compare_protocol_answers(_relation(pred), _relation(ref)).equivalent is True + + +def test_protocol_relation_summation_different_bodies_stay_distinct() -> None: + # Folding the limit into an opaque token keeps distinct summations distinct. + pred = "V = sum_{n=1}^{N} (m V_r)/(M + n m)" + ref = "V = sum_{n=1}^{N} (m V_r)/(M - n m)" + assert compare_protocol_answers(_relation(pred), _relation(ref)).equivalent is False + + +def test_protocol_relation_compact_product_with_subscript_is_expanded() -> None: + # NmV_r expands to N*m*V_r, the same as the spaced form. + pred = "V = NmV_r/(M + Nm)" + ref = "V = N m V_r/(M + N m)" + assert compare_protocol_answers(_relation(pred), _relation(ref)).equivalent is True diff --git a/tests/prkit/verify/test_verify.py b/tests/prkit/verify/test_verify.py index 9dd1e0d..2a5df4b 100644 --- a/tests/prkit/verify/test_verify.py +++ b/tests/prkit/verify/test_verify.py @@ -39,6 +39,13 @@ def test_symbolic_commutativity_equivalent(self): assert v.correct is True assert v.symbolic_equiv is True + def test_symbolic_relation_rearrangement_equivalent(self): + # Equalities equivalent only after algebraic rearrangement across "=". + v = verify("F = m a", "a = F/m") + assert v.correct is True + assert v.symbolic_equiv is True + assert v.comparison_mode == "relation" + def test_number_vs_fraction_equivalent(self): v = verify("0.5", "1/2") assert v.correct is True From 7dcee0531f680c2f946f54951099971ca988177e Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Thu, 18 Jun 2026 23:25:37 -0400 Subject: [PATCH 05/28] Document the physics-semantics equivalence judgement Add EQUIVALENCE.md, a detailed reference for compare_protocol_answers: the pipeline, the per-object-kind criteria, the cross-kind bridges, the policy modes and numeric tolerance, with diagrams and a worked example for every condition. Add METHODOLOGY.md, the precision-preserving design discipline for changing the judgement (canonical forms and principled per-kind criteria, not fallback rescues). Point the answer-semantics enum taxonomy comment at the new reference. Co-Authored-By: Claude Opus 4.8 --- src/prkit/semantics/comparison/EQUIVALENCE.md | 380 ++++++++++++++++++ src/prkit/semantics/comparison/METHODOLOGY.md | 144 +++++++ src/prkit/semantics/schema/enums.py | 4 +- 3 files changed, 527 insertions(+), 1 deletion(-) create mode 100644 src/prkit/semantics/comparison/EQUIVALENCE.md create mode 100644 src/prkit/semantics/comparison/METHODOLOGY.md diff --git a/src/prkit/semantics/comparison/EQUIVALENCE.md b/src/prkit/semantics/comparison/EQUIVALENCE.md new file mode 100644 index 0000000..8b112e0 --- /dev/null +++ b/src/prkit/semantics/comparison/EQUIVALENCE.md @@ -0,0 +1,380 @@ +# Physics-semantics equivalence judgement — detailed reference + +How `compare_protocol_answers` decides whether two physics answers express the same +physical meaning. This is the **reference** for the judgement; the precision-preserving +**design discipline** for changing it lives in [`METHODOLOGY.md`](METHODOLOGY.md). + +Every example below is a real engine result. Notation: `pred ≡ ref` means equivalent, +`pred ≢ ref` means not, and `→ mode` is the resulting `comparison_mode`. + +- Entry point: `compare_protocol_answers(pred, ref, *, contract=None, context=None, policy_mode=None)` — [engine.py:39](engine.py) +- It is the deterministic equivalence relation `Eq(a_pred, a_ref ; q)` of the physics-semantics framework: a typed, question-conditioned judgement, not string overlap. + +--- + +## 1. Inputs: the answer-semantics record + +Each side is an `PhysicsAnswerSemantics` record (raw strings are coerced into one): + +- **`object_kind`** — one of 8 atomic kinds (`AnswerObjectKind`): `number`, + `physical_quantity`, `expression`, `relation`, `qualitative_label`, `choice`, + `boolean`, `sign_direction`. +- **`structure`** — one of 9 (`AnswerStructure`): `atomic`, `multi_part`, `tuple`, + `set`, `interval`, `vector`, `matrix`, `tensor`, `piecewise`. +- **`canonical_text`** + typed fields (`numeric_value`, `numeric_text`, `unit`, + `choice_label`, `boolean_value`, `sign_value`, `children`, `cases`, …). + +The **question semantics** `q` (`PhysicsQuestionSemantics`, passed as `context`) supply +the conditioning: `target_variable`, `symbol_aliases`, unit/sign policy, ordering policy, +and the numeric `tolerance`. The judgement is *under* `q` — e.g. a required unit lets a +bare `5` be read as `5 m/s²`. + +--- + +## 2. Pipeline overview + +```mermaid +flowchart TD + A["Eq( a_pred , a_ref ; q )
compare_protocol_answers"] --> B["Normalize + repair"] + B --> C{"Contract gate
admitted / coercible / violating"} + C -->|"violating, or strict + coercible"| X["non-equivalent
contract_violation"] + C -->|"ok"| D{"Structure"} + D -->|"pred ≠ ref structure"| Y["structure_mismatch"] + D -->|"structured"| R["align by structure
recurse per element"] + D -->|"atomic"| E{"Atomic dispatch"} + R --> E + E -->|"same object kind"| F["one criterion per kind"] + E -->|"different object kind"| G["tiered bridges
gated by policy"] + F --> Z["AnswerComparison → Verdict"] + G --> Z +``` + +The four gates run before any kind-specific logic. Sections 3–6 walk them; section 7 is +the heart (per-kind criteria); sections 8–9 cover bridges and policy. + +--- + +## 3. Stage 1 — normalize & repair + +`coerce_protocol_answer` builds a typed record from a dict/string; then +`_repair_answer_for_comparison` re-parses ambiguous symbolic surfaces and re-hydrates +structured answers. The most common repair: a prediction stored as `expression` whose +text is really a relation (`d = sqrt(P L)`) is reclassified to `relation` so the right +criterion applies. + +This stage also applies the **symbolic canonicalizations** that make later comparison +stable (all in `preprocess_symbolic_text` / `parse_relation_clauses`): + +| Canonicalization | Effect | Code | +|---|---|---| +| LaTeX → ASCII math | `\frac{a}{b}`, `\sqrt{x}`, Greek, accents, subscripts | `_replace_simple_latex`, `_normalize_latex_*` | +| Question-scoped symbol aliases | `q.symbol_aliases` rewrite (`r(t)→r`) | `_canonicalize_symbol_alias_surfaces` | +| Functional-form relation LHS | `r(t) = … → r = …` (relations only) | `_collapse_functional_form_lhs` | +| Big-operator limits | fold `\sum_{n=1}^{N}` so its `=` can't corrupt parsing | `_normalize_big_operator_bounds` | +| Compact products | `NmV_r → N*m*V_r` | `_normalize_symbol_products` | + +--- + +## 4. Stage 2 — contract gate & policy + +A `PhysicsEvaluationContract` is derived from the reference answer + `q` (expected kind, +structure, target variable, unit policy, symbolic mode, choice space, ordering, enabled +bridges). Each side is classified by `validate_answer_against_contract` +([contract.py:69](contract.py)): + +- **admitted** — satisfies the expected kind and question-side policies directly. +- **coercible** — differs in a limited, possibly-meaningful way. +- **violating** — fails the contract. + +The three **policy modes** (`ComparisonPolicyMode`) control how strict this is and which +bridges may fire: + +| Policy | Contract validation | Coercible pred | Cross-kind bridges | +|---|---|---|---| +| `strict` | enforced | rejected (`contract_violation`) | **none** | +| `audited` | enforced | allowed | only bridge tiers in `contract.enabled_bridge_tiers`, precondition must hold | +| `permissive` | skipped | allowed | **all** | + +A violating reference short-circuits to `reference_contract_violation`; a violating +prediction to `contract_violation`. (See §9 for a worked policy example.) + +--- + +## 5. Stage 3 — structure routing + +If the two structures differ → `structure_mismatch`. Otherwise atomic goes to §6; every +structured form aligns by its structure and **recurses into `compare_protocol_answers` +per element**, ultimately reducing to atomic comparisons. + +| Structure | Routing | Example | +|---|---|---| +| `atomic` | §6 atomic dispatch | — | +| `multi_part` | ordered / unordered / per-part by `q.ordering` | — | +| `tuple` | positional | `(1, 2) ≡ (1, 2)` → `tuple` | +| `set` | order-insensitive | `{1, 2} ≡ {2, 1}` → `set` | +| `interval` | endpoint + boundary check | — | +| `vector` / `matrix` / `tensor` | shape + per-cell | — | +| `piecewise` | align cases + conditions | — | + +`1 ≢ (1)` → `structure_mismatch` (atomic vs tuple). + +--- + +## 6. Stage 4 — atomic dispatch + +```mermaid +flowchart TD + S{"pred.kind == ref.kind ?"} -->|"yes"| K["compare_same_object_kind
(per-kind criterion)"] + K -->|"equivalent"| OK["return equivalent"] + K -->|"no"| LF["label_family_fallback (T3)"] + LF -->|"miss"| IT{"identical canonical text?"} + IT -->|"no"| NEQ["non-equivalent"] + S -->|"no"| BR["compare_different_object_kinds
tiered bridges"] + BR -->|"hit"| BP["bridge policy gate"] + BR -->|"miss"| LF2["label_family_fallback (T3)"] + LF2 -->|"miss"| MM["object_kind_mismatch"] + LF --> OK + IT --> OK + BP --> OK + LF2 --> OK +``` + +`_compare_atomic` ([engine.py:314](engine.py)): same kind → the §7 criterion; on a miss, +a Tier-3 label-family fallback and an identical-text check. Different kind → the §8 +bridges, then the label-family fallback, else `object_kind_mismatch`. + +--- + +## 7. Same object kind — one criterion per kind + +`compare_same_object_kind` ([same_object_kind.py:33](same_object_kind.py)) dispatches on +`object_kind`. Each kind has a **canonical form** and **one decision criterion**. + +### 7.1 `number` + +Parse the scalar; accept on relative closeness within `q.tolerance`, else on a +**reference-precision** match (see §10). The reference defines the required precision, so +the relation is asymmetric in pred vs ref. + +``` +0.5 ≡ 1/2 → number (exact) +0.5 ≢ 0.7 → number (outside tolerance) +9.8 ≡ 9.81 → number (pred less precise; consistent with the reference) +9.81 ≢ 9.8 → number (pred MORE precise than a coarser reference) +0.333 ≡ 1/3 → number (ref is an exact non-terminating rational) +1/3 ≢ 0.333 → number (reference fixes 3 decimals; 1/3 is not that number) +``` + +### 7.2 `physical_quantity` + +Canonical form is `(coefficient, symbolic factor, unit)`. Convert the prediction's unit +to the reference unit (`unit_conversion_factor`), require the **symbolic factor** to +match, then apply the §10 numeric criterion to the coefficients. Dimensionally +incompatible units fail. + +``` +5 m/s ≡ 18 km/h → physical_quantity (unit conversion) +100 cm ≡ 1 m → physical_quantity +9.8 m/s^2 ≡ 9.8 m/s² → physical_quantity (unicode / suffix unit normalization) +3 m/s ≢ 3 m → physical_quantity (dimension mismatch) +``` + +### 7.3 `expression` + +Parse both to SymPy; equivalent iff `simplify(a − b) == 0` (with `trigsimp` and a +numeric-`N` fallback). A prediction written as `x = …` is reduced to its solved side +before comparison (`_prediction_rhs_matches_expression`). + +``` +v t ≡ t v → expression +sqrt(lambda P L/(L+P)) ≡ sqrt(lambda L P/(L+P)) → expression +x^2 ≢ x^3 → expression +``` + +### 7.4 `relation` — the algebraic core + +A relation is parsed into a **canonical clause set** and matched order-insensitively +(`relations_equivalent`). Per clause, two layered criteria +(`_relation_clause_equivalent`): + +```mermaid +flowchart TD + P["parse to canonical clauses
(functional-form LHS folded: r(t) → r)"] --> M["order-insensitive clause-set match"] + M --> CE["per clause"] + CE --> SE{"surface equality
(sides equivalent, direct or reversed)"} + SE -->|"yes"| OK["clause matches"] + SE -->|"no"| HF["homogeneous form H = L − R"] + HF --> OP{"operator class"} + OP -->|"equality ="| NUM["numerators of H equal
up to a nonzero CONSTANT"] + OP -->|"inequality"| SGN["H a signed-constant multiple
(sign sets operator direction)"] +``` + +**Equality criterion** (`_equalities_equivalent`) — clear denominators from `H = L − R` +and require the numerators equal up to a nonzero *constant*. This admits rearrangement +across `=` and rejects equations with extra roots: + +``` +F = m a ≡ a = F/m → relation (solve for another variable) +E = m c^2 ≡ m = E/c^2 → relation +1/f = 1/u + 1/v ≡ f = (u v)/(u + v) → relation (clear fractions) +v = a t ≡ v = t a → relation (commutative RHS) +F = m a ≡ m a = F → relation (reversed sides) +r(t) = a x ≡ r = a x → relation (functional-form LHS) +x = 0 ≢ x*y = 0 → relation (factor y enlarges the root set) +x = 1 ≢ x^2 = 1 → relation (extra root x = -1) +F = m a ≢ F = m/a → relation +``` + +**Inequality criterion** (`_inequalities_equivalent`) — the homogeneous forms must be a +signed-*constant* multiple, sign-consistent with the operator directions. Denominators +are **not** cleared (an unknown-sign denominator could flip the inequality), so +rearrangement that needs division is *not* applied: + +``` +2 <= k < 3 ≡ k >= 2 and k < 3 → relation (chained / conjunction, order-insensitive) +F < m a ≢ a < F/m → relation (would need ÷m; sign unknown → not merged) +``` + +Two parser canonicalizations keep relations robust: + +``` +V = sum_{n=1}^{N} (m V_r)/(M + n m) ≡ V = \sum_{n=1}^{N} \frac{m V_r}{M + n m} → relation +V = NmV_r/(M + Nm) ≡ V = N m V_r/(M + N m) → relation +``` + +### 7.5 categorical kinds — `choice`, `boolean`, `sign_direction`, `qualitative_label` + +Canonicalize to a controlled label, then compare for equality (qualitative also matches +on a shared alias-group candidate). + +``` +B ≡ (B) → choice (uppercase token) +yes ≡ true → boolean +clockwise ≡ clockwise → sign_direction +increases ≡ goes up → qualitative_label (alias group) +``` + +--- + +## 8. Different object kind — tiered bridges + +When kinds differ, `compare_different_object_kinds` +([different_object_kind.py:47](different_object_kind.py)) tries a coercion bridge. Each +bridge carries a **risk tier**; the tier and policy decide whether it is allowed (§9). + +| Tier | Bridge (`comparison_mode`) | Coercion | Example | +|---|---|---|---| +| **T1** | `relation_to_expression` | project `target = …` to its solved side | `v = a + b` ≡ `a + b` (target `v`) | +| **T1** | `relation_rhs` | relation vs number/quantity via the relation's RHS | `E = 5` ≡ `5` | +| **T1** | `expression_to_number` | evaluate an expression to a number | `2 + 3` ≡ `5` | +| **T2** | `quantity_to_number` | quantity vs number when `q` fixes the unit | `5` ≡ `5 m/s²` (unit from `q`) | +| **T2** | `expression_quantity` | expression vs physical quantity | — | +| **T2** | `choice`, `terminal_polarity_choice`, `terminal_polarity` | choice ↔ label ↔ sign | — | +| **T3** | `relation_to_qualitative_label` | relation vs a qualitative outcome | — | +| **T3** | `qualitative_zero` | "no change" ↔ a zero value | `0` ≡ `no change` | +| **T3** | `label_family_fallback` | last-resort same-/cross-kind label family | — | + +A bridged result records `bridge_id` and `bridge_tier` on the `AnswerComparison`. If no +bridge fires → `object_kind_mismatch` (e.g. `5` vs choice `B`). + +--- + +## 9. Policy in action + +The same coercible pair behaves differently by policy (`_apply_bridge_policy` → +`bridge_enabled_for_policy`). Number `0` vs qualitative `no change` (a Tier-3 bridge): + +``` +strict → 0 ≢ no change → contract_violation (no bridges; coercible rejected) +audited → 0 ≢ no change → bridge_blocked (T3 not in enabled tiers) +permissive → 0 ≡ no change → qualitative_zero (bridge fires) +``` + +So: `strict` is the same-kind criteria only; `audited` admits exactly the bridge tiers a +contract opts into; `permissive` is the most lenient (and the legacy default). + +--- + +## 10. Numeric tolerance & reference precision + +`numbers_match_with_reference_precision` ([semantics.py](semantics.py)) decides numeric +agreement in order: + +1. **Relative closeness** — `numbers_close(pred, ref, tolerance)` (`q.tolerance`, + relative; handles NaN/inf and sign). +2. **Significant figures** — if the prediction has at least the reference's significant + figures, round both to the reference's sig-figs and require a match whose raw + difference sits strictly inside the rounding interval (half-quantum). +3. **Decimal places** — analogous fallback for fixed-point literals. +4. **Exact non-terminating reference** — if the reference is an exact rational like `1/3`, + accept a prediction that matches at the prediction's own stated precision. + +The reference sets the bar, so the relation is asymmetric (see the `9.8`/`9.81` and +`0.333`/`1/3` pairs in §7.1). For the exact half-quantum boundary, read +`_difference_is_strictly_within_half_quantum`. + +--- + +## 11. `comparison_mode` catalogue + +The `AnswerComparison.comparison_mode` names the path taken: + +- **Same-kind criteria:** `number`, `physical_quantity`, `expression`, `relation`, + `choice`, `boolean`, `sign_direction`, `qualitative_label`. +- **Cross-kind bridges:** `relation_to_expression`, `relation_rhs`, + `expression_to_number`, `quantity_to_number`, `expression_quantity`, `choice`, + `terminal_polarity`, `terminal_polarity_choice`, `relation_to_qualitative_label`, + `qualitative_zero`, `label_family_fallback`. +- **Structured:** `tuple`, `set`, `multi_part`, `interval`, `vector`/`matrix`/`tensor`, + `piecewise`. +- **Non-equivalent / control:** `structure_mismatch`, `object_kind_mismatch`, + `contract_violation`, `reference_contract_violation`, `bridge_blocked`, + `unsupported_structure`, `unsupported_object_kind`. + +--- + +## 12. Output → `Verdict` + +The engine returns an `AnswerComparison` (`equivalent`, `comparison_mode`, `diagnostics`, +`validation_status`, `bridge_id`/`bridge_tier`, `policy_mode`). `scoring/_adapt.py` maps +it losslessly into the public `Verdict`: + +| Verdict field | Source | +|---|---| +| `correct` / `equivalent` | `AnswerComparison.equivalent` | +| `score` | `1.0` / `0.0` (graded scorers fill `partial_credit`) | +| `comparison_mode` | passthrough | +| `symbolic_equiv` | `equivalent` for symbolic modes (`expression`, `relation`, …) | +| `units_ok`, `numeric_within_tol` | derived for numeric modes | +| `diagnostics`, `scorer_version` | passthrough / version stamp | + +`prkit.verify.verify(pred, ref)` is the light-import facade over this whole pipeline. + +--- + +## 13. End-to-end traces + +- `verify("F = m a", "a = F/m")` → normalize → both `relation`, `atomic` → same-kind + relation criterion → clause `F = m a` vs `a = F/m`: surface no; homogeneous numerators + `F − m a` and `a m − F` differ by `−1` → **equivalent**, mode `relation`, + `symbolic_equiv = True`. +- `verify("5 m/s", "18 km/h")` → both `physical_quantity` → convert `18 km/h = 5 m/s`, + symbolic factor `1` matches, coefficients close → **equivalent**, mode + `physical_quantity`, `units_ok = True`. +- number `0` vs qualitative `no change` under `audited` → kinds differ → `qualitative_zero` + is Tier-3 → not in enabled tiers → **bridge_blocked**, not equivalent. + +--- + +## 14. Where to look + +- Dispatch & gates: [engine.py](engine.py); contract: [contract.py](contract.py). +- Same-kind criteria: [same_object_kind.py](same_object_kind.py), + [numeric.py](numeric.py), [semantics.py](semantics.py). +- Bridges & tiers: [different_object_kind.py](different_object_kind.py), + [bridge_registry.py](bridge_registry.py). +- Verdict mapping: [`../../scoring/_adapt.py`](../../scoring/_adapt.py). +- Design discipline for changes: [METHODOLOGY.md](METHODOLOGY.md). +- Tests (every example here has a counterpart): + [`tests/prkit/semantics/test_protocol_comparison.py`](../../../../tests/prkit/semantics/test_protocol_comparison.py), + [`tests/prkit/verify/test_verify.py`](../../../../tests/prkit/verify/test_verify.py). diff --git a/src/prkit/semantics/comparison/METHODOLOGY.md b/src/prkit/semantics/comparison/METHODOLOGY.md new file mode 100644 index 0000000..0a5b240 --- /dev/null +++ b/src/prkit/semantics/comparison/METHODOLOGY.md @@ -0,0 +1,144 @@ +# Deterministic physics-semantics equivalence: methodology and design discipline + +This document is the discipline for **improving the equivalence judgement without +breaking it**. For *how the judgement works* in detail — the pipeline, the per-kind +criteria, the bridges, and a worked example for each condition — see the reference +[`EQUIVALENCE.md`](EQUIVALENCE.md). It is the engineering companion to the comparison +engine in this package (`engine.py`, `same_object_kind.py`, `different_object_kind.py`, +`numeric.py`, `semantics.py`). + +## 1. The frame: Physics Semantics + +A free-form physics answer is not compared as a string. It is first parsed into an +**answer-semantics record** — `(object kind, canonical content, metadata)` — and judged +under the **question semantics** `q` that say what the problem asked for (target +quantity, expected answer type, unit/sign/frame policies). Equivalence is the predicate + +``` +Eq(a_pred, a_ref ; q) -> bool +``` + +evaluated on canonicalized *meaning*, not surface form. The same predicate serves two +uses: reference-based correctness (`a_ref` is the gold answer) and reference-free +clustering (both records are model predictions). This is the engine's contract; the +research framing is *"Uncertainty Quantification for Open-Ended LLM Physics Reasoning +via Physics Semantics"* (the engine is that paper's `Eq(·,·;q_i)`). + +### Concept → code map + +| Concept | Code | +|---|---| +| `Eq(a_pred, a_ref ; q)` | `compare_protocol_answers(pred, ref, context)` — `engine.py` | +| 8 answer object kinds | `AnswerObjectKind` — `semantics/schema/enums.py` | +| 9 answer structures | `AnswerStructure` | +| admitted / coercible / violating | `ContractValidationStatus` | +| cross-kind bridges + risk tiers | `compare_different_object_kinds` + `BridgeTier` | +| strict / audited / permissive | `ComparisonPolicyMode` | +| question semantics `q` | `PhysicsQuestionSemantics` (target_variable, symbol_aliases, unit/sign policy) | +| answer semantics `a` | `PhysicsAnswerSemantics` | + +`compare_protocol_answers` first repairs/reparses the records, routes structured answers +by `AnswerStructure` (ordered → positional, set/unordered → collection match, interval / +shaped / piecewise → their own routes), then reduces to **atomic** comparison by object +kind via `compare_same_object_kind` (or a cross-kind bridge when kinds differ). + +## 2. Atomic comparison rules, by object kind + +| Object kind | Rule (canonicalize → decide) | Maturity | +|---|---|---| +| `number` | parse value, compare within tolerance honoring reference printed precision (`numeric.py`, `numbers_match_with_reference_precision`) | mature | +| `physical_quantity` | resolve units, convert, then numeric tolerance | mature | +| `expression` | `simplify(a - b) == 0`, with trig/numeric fallbacks and a prediction-side RHS rescue (`expressions_equivalent`, `_prediction_rhs_matches_expression`) | strong, conservative | +| `relation` | parse to clauses; match as an order-insensitive set; per clause try exact/reversed surfaces, then homogeneous scalar/rational-multiple equivalence (`relations_equivalent`, `_relation_clause_equivalent`, `_proportional_ratio`) | strong, conservative | +| `choice` / `boolean` / `sign_direction` / `qualitative_label` | canonical-label equality (curated alias groups) | mature | + +The SymPy substrate (`parse_symbolic_expression`, `expressions_equivalent`, +`parse_relation_clauses`, `_proportional_ratio`, `preprocess_symbolic_text`) is the +moat. **Extend it; do not reimplement it.** + +## 3. The governing principle: high precision is the product + +The deterministic judge's value is that **when it accepts, the acceptance is reliable** +(audited at ~95.6% precision, near-zero false positives). Its measured weakness is the +opposite — **recall on symbolic answers** (it under-accepts algebraically equivalent +expressions/relations). The way to raise recall here is **not** to bolt looser "rescue" +checks after a strict one. A rescue that fires only when the strict check fails is, by +construction, a relaxation — and an unjustified relaxation is exactly what erodes +precision. Instead: + +> **Equivalence is decided by comparing canonical forms under one principled criterion +> per object kind.** Improve recall by strengthening the *canonical form* or by +> *sharpening the criterion* — both of which stay precise because they are +> meaning-preserving and mathematically justified, applied symmetrically to both answers. + +Two levers, both stable: + +- **Canonical normalization.** A deterministic, meaning-preserving rewrite applied to + *every* answer of a kind before comparison — so equal answers reach one shared form + regardless of surface choice. It cannot fabricate equivalences (it is applied to both + sides identically and changes no meaning). Examples here: functional-form LHS + (`r(t)=… → r=…`, `_collapse_functional_form_lhs` inside `parse_relation_clauses`), + summation-bound folding and compact-product expansion (both in + `preprocess_symbolic_text`). +- **Principled equivalence criterion.** One mathematically justified rule per object kind + (and operator class), stated once — not a primary check plus fallbacks. For relations: + `_relation_clause_equivalent` decides surface equality, then an algebraic criterion on + the homogeneous form `H = L − R` (`_equalities_equivalent` / `_inequalities_equivalent`). + +### Five rules for any equivalence change + +1. **Make it a canonical form or a criterion, not a rescue.** New equivalence is either a + meaning-preserving normalization applied to both sides, or a sharpening of the single + per-kind criterion — never an "if strict failed, try looser" branch. +2. **Justify it mathematically.** The criterion must admit *exactly* the intended class. + The equality criterion is "homogeneous numerators equal up to a nonzero **constant**" + because that is provably the same solution variety up to nonzero scale; the + weaker "up to a rational function" would admit extra-root equations and lose precision. +3. **Restrict to the class it is proven for.** The equality criterion is stated for + equalities only (clearing a symbol-signed denominator could flip an inequality, so + inequalities use the signed-constant criterion). A criterion needing context — e.g. a + sign-convention rule — is gated on explicit `q` metadata, not applied blindly. +4. **Adversarial rejects + bounded cost.** Ship reject cases proving the criterion + excludes the near-miss class (the in-repo proxy for precision), and keep SymPy work + bounded — the verifier is a hot path (RL rewards, harness adapters). +5. **Measure Δrecall at fixed precision.** The audit set (human-labeled correctness) + lives in the **consumer repo**, not PRKit (toolkit independence). A change is good + only if recall rises with precision held. + +## 4. Recall-gap inventory + +Catalogued from real false negatives. Each is addressed by a canonical form or a +criterion (not a rescue); the last is deferred pending a justified, gated criterion. + +| Gap | Example | Mechanism | +|---|---|---| +| **Algebraic rearrangement** | `F=ma` ↔ `a=F/m`, `E=mc²` ↔ `m=E/c²`, `1/f=1/u+1/v` ↔ `f=uv/(u+v)` | equality **criterion**: homogeneous numerators equal up to a nonzero constant — implemented | +| **Functional-form LHS** | `r(t)=…` ↔ `r=…` | canonical **normalization** of the relation clause (numeric args like `f(2)` excluded) — implemented | +| **Parser corruption** | `=` inside `\sum_{n=1}^{N}`; compact products `NmV_r` | canonical **normalization** in `preprocess_symbolic_text` (parsing correctness) — implemented | +| **Sign convention** | global `−` sign on a directional quantity | needs a **gated criterion** — axis choice *or* real error; only directional quantities, only when frame/sign metadata absent, as an *audited* bridge — deferred | + +### Why "up to a nonzero constant" is the correct equality criterion + +For an equality `L = R`, the homogeneous form is `H = L − R`; `H = 0` is the equation. +Two equalities are the same constraint iff their homogeneous forms are equal up to +rescaling. The criterion clears denominators (`together`/`fraction`) and compares the +**numerators up to a nonzero constant** (`_equalities_equivalent`). This is exact, not a +heuristic: a rearrangement (solving for another variable, clearing fractions) only ever +changes the numerator by a constant, whereas a genuinely different equation differs by a +*symbolic* factor — so `x=0` vs `x·y=0` (factor `y`) and `x=1` vs `x²=1` (factor `x+1`) +are correctly excluded. It does **not** cover radical-introducing solves (`c=√(E/m)`) or +inequalities, by design. + +## 5. Checklist for an equivalence change + +1. **Decide the lever:** a canonical-form normalization (§3) or a sharper per-kind + criterion. If you find yourself adding an "if it still failed, also try…" branch, stop + — fold it into one of the two instead. +2. **Reuse the SymPy substrate** (§2) — parse/normalize with existing helpers. +3. **Justify the class admitted** (a short proof/argument in the docstring or comment), + and restrict the criterion to that class; keep `comparison_mode` accurate. +4. **Write accept + adversarial-reject tests** in + `tests/prkit/semantics/test_protocol_comparison.py` (and an end-to-end `verify()` + assertion in `tests/prkit/verify/test_verify.py` when it changes a verdict). +5. **Confirm the hot path** isn't materially slowed and the full suite stays green. +6. **Note the residual limit** so the next gap is discoverable. diff --git a/src/prkit/semantics/schema/enums.py b/src/prkit/semantics/schema/enums.py index 00050b3..631cb08 100644 --- a/src/prkit/semantics/schema/enums.py +++ b/src/prkit/semantics/schema/enums.py @@ -1,6 +1,8 @@ """Enumerations for physics answer semantics. -See ``TAXONOMY.md`` in this package for the full human-readable taxonomy. +See ``../comparison/EQUIVALENCE.md`` for the detailed equivalence-judgement reference +(object kinds, structures, per-kind criteria, bridges, examples) and +``../comparison/METHODOLOGY.md`` for the design discipline behind it. """ from __future__ import annotations From 6fdef843838d65e0d2a874aaa38c0466fb626e9f Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Fri, 19 Jun 2026 13:54:36 -0400 Subject: [PATCH 06/28] Judge symbolic equivalence over declared symbol domains Carry per-symbol real-domain assumptions into SymPy parsing so expression/relation equivalence is decided over the intended real-physical domain instead of the generic complex default: - symbol assumptions: realness derived in-engine (precision-safe), positivity/nonnegativity declared via q.symbol_assumptions; threaded to Symbol(token, **assumptions). - de-radicalization: a solved even root (c = sqrt(E/m)) canonicalizes to its squared form when the non-radical side is provably nonnegative, falling into the existing exact equality criterion. - numeric identity testing: domain-honoring multi-point evaluation decides 'is a - b the zero function', rejecting on any disagreement (exact) and guarding assumption-empowered symbolic accepts. Also rename the previously dormant symbol_domains -> symbol_assumptions (enum SymbolAssumption, model PhysicsSymbolAssumptionSemantics, inner field assumption; coercion still accepts the legacy 'domain' key). Co-Authored-By: Claude Opus 4.8 --- src/prkit/semantics/__init__.py | 4 + src/prkit/semantics/comparison/EQUIVALENCE.md | 81 ++- src/prkit/semantics/comparison/METHODOLOGY.md | 39 +- src/prkit/semantics/comparison/coercion.py | 28 + src/prkit/semantics/comparison/common.py | 30 ++ .../semantics/comparison/same_object_kind.py | 17 + src/prkit/semantics/comparison/semantics.py | 502 ++++++++++++++++-- src/prkit/semantics/schema/__init__.py | 4 + src/prkit/semantics/schema/enums.py | 17 + src/prkit/semantics/schema/models.py | 22 + 10 files changed, 700 insertions(+), 44 deletions(-) diff --git a/src/prkit/semantics/__init__.py b/src/prkit/semantics/__init__.py index bb71b68..a381283 100644 --- a/src/prkit/semantics/__init__.py +++ b/src/prkit/semantics/__init__.py @@ -71,8 +71,10 @@ PhysicsEvaluationContract, PhysicsQuestionSemantics, PhysicsSymbolAliasSemantics, + PhysicsSymbolAssumptionSemantics, QuestionSymbolicMode, QuestionUnitPolicy, + SymbolAssumption, ) QuestionContext = PhysicsQuestionSemantics @@ -100,6 +102,8 @@ "PhysicsEvaluationContract", "PhysicsQuestionSemantics", "PhysicsSymbolAliasSemantics", + "PhysicsSymbolAssumptionSemantics", + "SymbolAssumption", "build_evaluation_contract", "QuestionContext", "QuestionSymbolicMode", diff --git a/src/prkit/semantics/comparison/EQUIVALENCE.md b/src/prkit/semantics/comparison/EQUIVALENCE.md index 8b112e0..3fc590d 100644 --- a/src/prkit/semantics/comparison/EQUIVALENCE.md +++ b/src/prkit/semantics/comparison/EQUIVALENCE.md @@ -25,9 +25,10 @@ Each side is an `PhysicsAnswerSemantics` record (raw strings are coerced into on `choice_label`, `boolean_value`, `sign_value`, `children`, `cases`, …). The **question semantics** `q` (`PhysicsQuestionSemantics`, passed as `context`) supply -the conditioning: `target_variable`, `symbol_aliases`, unit/sign policy, ordering policy, -and the numeric `tolerance`. The judgement is *under* `q` — e.g. a required unit lets a -bare `5` be read as `5 m/s²`. +the conditioning: `target_variable`, `symbol_aliases`, `symbol_assumptions`, unit/sign policy, +ordering policy, and the numeric `tolerance`. The judgement is *under* `q` — e.g. a +required unit lets a bare `5` be read as `5 m/s²`, and a `symbol_assumptions` declaration +(`c`, `E`, `m` positive) lets `c = √(E/m)` be read as `E = m c²` (§7.3–7.4). --- @@ -159,8 +160,8 @@ the relation is asymmetric in pred vs ref. ``` 0.5 ≡ 1/2 → number (exact) 0.5 ≢ 0.7 → number (outside tolerance) -9.8 ≡ 9.81 → number (pred less precise; consistent with the reference) -9.81 ≢ 9.8 → number (pred MORE precise than a coarser reference) +9.81 ≡ 9.8 → number (pred MORE precise; rounds to the reference) +9.8 ≢ 9.81 → number (pred coarser than the reference; cannot supply the required digit) 0.333 ≡ 1/3 → number (ref is an exact non-terminating rational) 1/3 ≢ 0.333 → number (reference fixes 3 decimals; 1/3 is not that number) ``` @@ -181,14 +182,27 @@ incompatible units fail. ### 7.3 `expression` -Parse both to SymPy; equivalent iff `simplify(a − b) == 0` (with `trigsimp` and a -numeric-`N` fallback). A prediction written as `x = …` is reduced to its solved side -before comparison (`_prediction_rhs_matches_expression`). +The criterion is one predicate: **is `a − b` the zero function over the symbols' domain?** +It is decided symbolically first (`simplify(a − b) == 0`, with `trigsimp`), then by +**numeric identity testing** (`_numeric_identity_equivalent`) when that is inconclusive — +multi-point high-precision evaluation that *rejects on the first clear disagreement* (an +exact disproof) and accepts on agreement at many generic points (§10.1). A prediction +written as `x = …` is reduced to its solved side first +(`_prediction_rhs_matches_expression`). + +Symbols are parsed **with domain assumptions** (§10.2), so the judgement is decided over +the intended *real* domain rather than the generic complex default. Identities that hold +only over the reals/nonnegatives are accepted exactly when the domain supports them, and +rejected otherwise: ``` v t ≡ t v → expression sqrt(lambda P L/(L+P)) ≡ sqrt(lambda L P/(L+P)) → expression +sqrt(x^2) ≡ |x| → expression (real x; derived) x^2 ≢ x^3 → expression +sqrt(a b) ≢ sqrt(a) sqrt(b) → expression (generic real: differ at a,b<0) +sqrt(a b) ≡ sqrt(a) sqrt(b) → expression (q: a,b nonnegative) +log(a b) ≡ log a + log b → expression (q: a,b positive) ``` ### 7.4 `relation` — the algebraic core @@ -225,6 +239,19 @@ x = 1 ≢ x^2 = 1 → relation (extra root x = -1) F = m a ≢ F = m/a → relation ``` +**De-radicalization** (`_deradicalize_clause`) — a solved even root is the same constraint +as its squared form *when the non-radical side is provably nonnegative* (squaring is +injective on the nonnegative reals, so it adds no spurious branch). This is a canonical +normalization, gated on `q.symbol_assumptions`, applied before the equality criterion; it is +withheld otherwise, since `c = √(E/m)` (the `c ≥ 0` branch) is genuinely *not* `E = m c²` +(both branches) over generic reals: + +``` +E = m c^2 ≡ c = sqrt(E/m) → relation (q: c,E,m positive → c**2 = E/m) +v^2 = u^2 + 2 a s ≡ v = sqrt(u^2 + 2 a s) → relation (q: v,… nonnegative) +E = m c^2 ≢ c = sqrt(E/m) → relation (generic real: gate off) +``` + **Inequality criterion** (`_inequalities_equivalent`) — the homogeneous forms must be a signed-*constant* multiple, sign-consistent with the operator directions. Denominators are **not** cleared (an unknown-sign denominator could flip the inequality), so @@ -313,6 +340,38 @@ The reference sets the bar, so the relation is asymmetric (see the `9.8`/`9.81` `0.333`/`1/3` pairs in §7.1). For the exact half-quantum boundary, read `_difference_is_strictly_within_half_quantum`. +### 10.1 Numeric identity testing (`_numeric_identity_equivalent`) + +`simplify` is incomplete (no canonical form exists for transcendental/nested-radical +expressions), so symbolic comparison under-accepts genuine identities. The same predicate +— *is `a − b` the zero function?* — is also decided numerically: sample the free symbols at +many deterministic **generic** points (wide range, no special values), evaluate both sides +at high working precision, and + +- **reject** on the first point whose relative difference clearly exceeds noise — a single + disagreement at a valid point is an *exact* disproof of a function identity; +- **accept** on agreement at enough points (Schwartz–Zippel: distinct functions cannot + coincide at many generic points); +- return *undecidable* if too few points evaluate (every sample singular), falling back to + the legacy constant-residual check. + +Each symbol is sampled **over its declared domain** (positive → positive samples, generic +real → both signs, complex → complex), and signs vary independently across points so +products like `√(a·b)` vs `√a·√b` are rejected (they agree unless several symbols are +simultaneously negative). Because rejection is exact, the test also **guards** a symbolic +acceptance reached under a strengthening assumption: a numeric disagreement vetoes it. + +### 10.2 Symbol-domain assumptions (`build_symbol_assumption_map`) + +Symbols are parsed as `Symbol(token, **assumptions)` so equivalence is decided over the +intended real domain. The map merges two sources: the **authoritative** `q.symbol_assumptions` +declaration (`SymbolAssumption`: `real`/`nonzero`/`nonnegative`/`positive`/`complex`) and a +**conservative** in-engine derivation that adds only the realness default — every symbol is +`real` unless an explicit imaginary marker (standalone `I`, `\imath`) appears. Positivity is +*never* derived from surface form (it would flip truth values — see METHODOLOGY.md §4); it +must be declared. Realness alone is precision-safe and unlocks real-only identities +(`√(x²) = |x|`). + --- ## 11. `comparison_mode` catalogue @@ -363,6 +422,9 @@ it losslessly into the public `Verdict`: `physical_quantity`, `units_ok = True`. - number `0` vs qualitative `no change` under `audited` → kinds differ → `qualitative_zero` is Tier-3 → not in enabled tiers → **bridge_blocked**, not equivalent. +- `verify` with `q.symbol_assumptions` declaring `c, E, m` positive: `E = m c²` vs `c = √(E/m)` + → both `relation` → `c = √(E/m)` de-radicalizes to `c² = E/m` (non-radical side `c ≥ 0`) + → homogeneous numerators `c² m − E` and `E − m c²` differ by `−1` → **equivalent**. --- @@ -371,6 +433,9 @@ it losslessly into the public `Verdict`: - Dispatch & gates: [engine.py](engine.py); contract: [contract.py](contract.py). - Same-kind criteria: [same_object_kind.py](same_object_kind.py), [numeric.py](numeric.py), [semantics.py](semantics.py). +- Domain assumptions, de-radicalization, numeric identity testing (§7.3–7.4, §10.1–10.2): + `build_symbol_assumption_map`, `_deradicalize_clause`, `_numeric_identity_equivalent` in + [semantics.py](semantics.py); `symbol_assumptions` in [`../schema/models.py`](../schema/models.py). - Bridges & tiers: [different_object_kind.py](different_object_kind.py), [bridge_registry.py](bridge_registry.py). - Verdict mapping: [`../../scoring/_adapt.py`](../../scoring/_adapt.py). diff --git a/src/prkit/semantics/comparison/METHODOLOGY.md b/src/prkit/semantics/comparison/METHODOLOGY.md index 0a5b240..c85734e 100644 --- a/src/prkit/semantics/comparison/METHODOLOGY.md +++ b/src/prkit/semantics/comparison/METHODOLOGY.md @@ -48,8 +48,8 @@ kind via `compare_same_object_kind` (or a cross-kind bridge when kinds differ). |---|---|---| | `number` | parse value, compare within tolerance honoring reference printed precision (`numeric.py`, `numbers_match_with_reference_precision`) | mature | | `physical_quantity` | resolve units, convert, then numeric tolerance | mature | -| `expression` | `simplify(a - b) == 0`, with trig/numeric fallbacks and a prediction-side RHS rescue (`expressions_equivalent`, `_prediction_rhs_matches_expression`) | strong, conservative | -| `relation` | parse to clauses; match as an order-insensitive set; per clause try exact/reversed surfaces, then homogeneous scalar/rational-multiple equivalence (`relations_equivalent`, `_relation_clause_equivalent`, `_proportional_ratio`) | strong, conservative | +| `expression` | decide "is `a - b` the zero function over the symbols' domain": `simplify(a - b) == 0` (with trig), then **numeric identity testing** over domain-honoring sample points (`expressions_equivalent`, `_numeric_identity_equivalent`); symbols carry **domain assumptions** from `q` so the test is exact over the real-physical domain | strong, conservative | +| `relation` | parse to clauses; **de-radicalize** a solved even root when sign-safe (`c = sqrt(E/m)` → `c**2 = E/m`); match as an order-insensitive set; per clause try exact/reversed surfaces, then homogeneous scalar/rational-multiple equivalence (`relations_equivalent`, `_relation_clause_equivalent`, `_deradicalize_clause`, `_proportional_ratio`) | strong, conservative | | `choice` / `boolean` / `sign_direction` / `qualitative_label` | canonical-label equality (curated alias groups) | mature | The SymPy substrate (`parse_symbolic_expression`, `expressions_equivalent`, @@ -71,19 +71,32 @@ precision. Instead: > *sharpening the criterion* — both of which stay precise because they are > meaning-preserving and mathematically justified, applied symmetrically to both answers. -Two levers, both stable: +Three levers, all stable: - **Canonical normalization.** A deterministic, meaning-preserving rewrite applied to *every* answer of a kind before comparison — so equal answers reach one shared form regardless of surface choice. It cannot fabricate equivalences (it is applied to both sides identically and changes no meaning). Examples here: functional-form LHS - (`r(t)=… → r=…`, `_collapse_functional_form_lhs` inside `parse_relation_clauses`), - summation-bound folding and compact-product expansion (both in + (`r(t)=… → r=…`, `_collapse_functional_form_lhs`), de-radicalization of a solved even + root (`c = sqrt(E/m) → c**2 = E/m`, `_deradicalize_clause`, gated on a nonnegative + side), summation-bound folding and compact-product expansion (in `preprocess_symbolic_text`). +- **Domain enrichment.** A physics answer denotes a real, often nonnegative, quantity; the + generic-complex default makes SymPy *correctly* refuse real-only identities + (`sqrt(a*b) = √a·√b`, `sqrt(x²) = |x|`). Carrying each symbol's real domain into the + parse (`build_symbol_assumption_map` → `Symbol(token, **assumptions)`) decides + equivalence over the *intended* domain while staying exact — it is applied symmetrically + and changes no truth value. The domain is **authoritatively declared** in + `q.symbol_assumptions`; the in-engine derivation adds only the precision-safe realness + default (never positivity, which surface form cannot justify — see §4). - **Principled equivalence criterion.** One mathematically justified rule per object kind (and operator class), stated once — not a primary check plus fallbacks. For relations: `_relation_clause_equivalent` decides surface equality, then an algebraic criterion on the homogeneous form `H = L − R` (`_equalities_equivalent` / `_inequalities_equivalent`). + For expressions the criterion is "is `a − b` the zero function over the domain": SymPy + `simplify`, then **numeric identity testing** (`_numeric_identity_equivalent`) — two + sound implementations of one predicate, where numeric *disagreement at any domain point + is an exact disproof* (so it also guards an assumption-empowered symbolic accept). ### Five rules for any equivalence change @@ -115,8 +128,24 @@ criterion (not a rescue); the last is deferred pending a justified, gated criter | **Algebraic rearrangement** | `F=ma` ↔ `a=F/m`, `E=mc²` ↔ `m=E/c²`, `1/f=1/u+1/v` ↔ `f=uv/(u+v)` | equality **criterion**: homogeneous numerators equal up to a nonzero constant — implemented | | **Functional-form LHS** | `r(t)=…` ↔ `r=…` | canonical **normalization** of the relation clause (numeric args like `f(2)` excluded) — implemented | | **Parser corruption** | `=` inside `\sum_{n=1}^{N}`; compact products `NmV_r` | canonical **normalization** in `preprocess_symbolic_text` (parsing correctness) — implemented | +| **Real-only identities** | `sqrt(a·b)`↔`√a·√b`, `sqrt(x²)`↔`\|x\|`, `log(ab)`↔`log a+log b` | **domain enrichment**: carry the symbols' real domain into the parse (`build_symbol_assumption_map`); positivity from `q.symbol_assumptions`, realness derived — implemented | +| **`simplify` incompleteness** | nested-radical / transcendental identities `simplify` cannot crack | **criterion**: domain-honoring numeric identity testing (`_numeric_identity_equivalent`), exact on rejection — implemented | +| **Solved radical** | `E=mc²` ↔ `c=√(E/m)`, `v²=u²+2as` ↔ `v=√(u²+2as)` | canonical **normalization**: de-radicalize when the non-radical side is nonnegative (`_deradicalize_clause`), gated on `q.symbol_assumptions` — implemented | | **Sign convention** | global `−` sign on a directional quantity | needs a **gated criterion** — axis choice *or* real error; only directional quantities, only when frame/sign metadata absent, as an *audited* bridge — deferred | +### Why positivity is declared, not derived from surface form + +The domain-enrichment lever derives only **realness** in-engine; positivity / nonnegativity +must be declared in `q.symbol_assumptions`. The reason is precision. Writing `sqrt(a·b)` or +`log(x²)` does *not* presuppose any individual symbol is nonnegative — only that a product +or an even power is. So a surface heuristic ("symbol appears under a root → assume it +nonnegative") would manufacture sign assumptions that flip truth values: it would wrongly +accept `sqrt(a·b)` vs `√a·√b` (which differ at `a,b<0`) and `log(x²)` vs `2·log(x)` +(which differ at `x<0`). Under generic reals those pairs are correctly **rejected** (numeric +identity testing samples both signs); they become equivalent only when the domain is +declared. Symbol-name whitelists are avoided for the same reason — physics reuses letters +(`m` mass vs metre, `T` period vs temperature, signed coordinate `x`). + ### Why "up to a nonzero constant" is the correct equality criterion For an equality `L = R`, the homogeneous form is `H = L − R`; `H = 0` is the equation. diff --git a/src/prkit/semantics/comparison/coercion.py b/src/prkit/semantics/comparison/coercion.py index 50404ff..02a0fd2 100644 --- a/src/prkit/semantics/comparison/coercion.py +++ b/src/prkit/semantics/comparison/coercion.py @@ -18,8 +18,10 @@ PhysicsEvaluationContract, PhysicsQuestionSemantics, PhysicsSymbolAliasSemantics, + PhysicsSymbolAssumptionSemantics, QuestionSymbolicMode, QuestionUnitPolicy, + SymbolAssumption, ) _LEGACY_ANSWER_FIELDS = frozenset( @@ -56,6 +58,10 @@ def coerce_question_semantics( symbol_aliases=tuple( _coerce_symbol_alias(alias) for alias in data.get("symbol_aliases", ()) ), + symbol_assumptions=tuple( + _coerce_symbol_assumption(entry) + for entry in data.get("symbol_assumptions", ()) + ), allowed_object_kinds=_enum_tuple( AnswerObjectKind, data.get("allowed_object_kinds"), @@ -162,6 +168,28 @@ def _coerce_symbol_alias( ) +def _coerce_symbol_assumption( + value: PhysicsSymbolAssumptionSemantics | Mapping[str, Any], +) -> PhysicsSymbolAssumptionSemantics: + """Coerce one symbol-assumption declaration into the schema model.""" + + if isinstance(value, PhysicsSymbolAssumptionSemantics): + return value + if isinstance(value, Mapping): + symbol = _optional_text(value.get("symbol") or value.get("canonical_symbol")) + if symbol is None: + raise TypeError("Symbol assumption mappings must define symbol.") + # Accept both the canonical "assumption" key and the legacy "domain" alias. + raw = value.get("assumption", value.get("domain")) + return PhysicsSymbolAssumptionSemantics( + symbol=symbol, + assumption=_enum_value(SymbolAssumption, raw, SymbolAssumption.REAL), + ) + raise TypeError( + "Symbol assumptions must be PhysicsSymbolAssumptionSemantics or mappings." + ) + + def coerce_protocol_answer( value: PhysicsAnswerSemantics | Mapping[str, Any], ) -> PhysicsAnswerSemantics: diff --git a/src/prkit/semantics/comparison/common.py b/src/prkit/semantics/comparison/common.py index d268bc3..0bb33da 100644 --- a/src/prkit/semantics/comparison/common.py +++ b/src/prkit/semantics/comparison/common.py @@ -9,8 +9,20 @@ PhysicsAnswerSemantics, PhysicsQuestionSemantics, QuestionUnitPolicy, + SymbolAssumption, ) +# SymPy ``Symbol`` assumption kwargs for each declared real-domain. These describe the +# *real* domain the symbol ranges over; equivalence stays exact because it is decided over +# the declared domain rather than the generic complex default. +_SYMBOL_ASSUMPTION_KWARGS: Mapping[SymbolAssumption, Mapping[str, bool]] = { + SymbolAssumption.COMPLEX: {}, + SymbolAssumption.REAL: {"real": True}, + SymbolAssumption.NONZERO: {"real": True, "nonzero": True}, + SymbolAssumption.NONNEGATIVE: {"nonnegative": True}, + SymbolAssumption.POSITIVE: {"positive": True}, +} + def available_texts(*texts: str | None) -> tuple[str, ...]: """Return unique non-empty text surfaces in priority order.""" @@ -95,6 +107,24 @@ def context_symbol_alias_map(context: PhysicsQuestionSemantics) -> Mapping[str, return alias_map +def context_symbol_assumption_map( + context: PhysicsQuestionSemantics, +) -> Mapping[str, Mapping[str, bool]]: + """Return question-declared SymPy assumption kwargs keyed by canonical symbol token. + + This is the *authoritative* source for symbol assumptions: a declaration here always + wins over the conservative in-engine derivation. Keys are canonical (post-alias) tokens. + """ + + declared: dict[str, Mapping[str, bool]] = {} + for entry in context.symbol_assumptions: + symbol = entry.symbol.strip() + if not symbol: + continue + declared[symbol] = dict(_SYMBOL_ASSUMPTION_KWARGS.get(entry.assumption, {})) + return declared + + def resolved_unit( answer: PhysicsAnswerSemantics, *, context: PhysicsQuestionSemantics ) -> str | None: diff --git a/src/prkit/semantics/comparison/same_object_kind.py b/src/prkit/semantics/comparison/same_object_kind.py index 307c9ee..a83a881 100644 --- a/src/prkit/semantics/comparison/same_object_kind.py +++ b/src/prkit/semantics/comparison/same_object_kind.py @@ -18,6 +18,7 @@ ) from .numeric import compare_numeric_like_answers from .semantics import ( + build_symbol_assumption_map, canonicalize_boolean_value, canonicalize_choice_label, canonicalize_qualitative_label, @@ -106,12 +107,17 @@ def _compare_symbolic_answers( if not pred_primary or not ref_primary: return AnswerComparison(False, mode, ("missing_symbolic_text",)) + assumptions_map = build_symbol_assumption_map( + pred_primary, ref_primary, context=context, alias_map=alias_map + ) + if _symbolic_compare( mode, pred_primary, ref_primary, tolerance=context.tolerance, alias_map=alias_map, + assumptions_map=assumptions_map, ): return AnswerComparison(True, mode) @@ -127,6 +133,7 @@ def _compare_symbolic_answers( ref_fallback, tolerance=context.tolerance, alias_map=alias_map, + assumptions_map=assumptions_map, ) ): return AnswerComparison(True, mode) @@ -138,6 +145,7 @@ def _compare_symbolic_answers( ref_fallback, tolerance=context.tolerance, alias_map=alias_map, + assumptions_map=assumptions_map, ): return AnswerComparison(True, mode) @@ -147,6 +155,7 @@ def _compare_symbolic_answers( ref_fallback, context=context, alias_map=alias_map, + assumptions_map=assumptions_map, ): return AnswerComparison(True, mode) @@ -160,6 +169,7 @@ def _symbolic_compare( *, tolerance: float, alias_map: Mapping[str, str] | None, + assumptions_map: Mapping[str, Mapping[str, bool]] | None = None, ) -> bool: """Dispatch symbolic comparison to expression or relation matching.""" @@ -169,12 +179,14 @@ def _symbolic_compare( right, tolerance, alias_map=alias_map, + assumptions_map=assumptions_map, ) return relations_equivalent( left, right, tolerance, alias_map=alias_map, + assumptions_map=assumptions_map, ) @@ -185,6 +197,7 @@ def _prediction_rhs_matches_expression( *, context: PhysicsQuestionSemantics, alias_map: Mapping[str, str] | None, + assumptions_map: Mapping[str, Mapping[str, bool]] | None = None, ) -> bool: """Retry expression comparison against a prediction-side extracted RHS.""" @@ -201,6 +214,7 @@ def _prediction_rhs_matches_expression( ref_primary, context.tolerance, alias_map=alias_map, + assumptions_map=assumptions_map, ): return True if ( @@ -211,6 +225,7 @@ def _prediction_rhs_matches_expression( ref_fallback, context.tolerance, alias_map=alias_map, + assumptions_map=assumptions_map, ) ): return True @@ -225,6 +240,7 @@ def _relation_alternatives_match( *, tolerance: float, alias_map: Mapping[str, str] | None, + assumptions_map: Mapping[str, Mapping[str, bool]] | None = None, ) -> bool: """Retry relation comparison across explicitly signposted equivalent forms.""" @@ -247,6 +263,7 @@ def _relation_alternatives_match( ref_text, tolerance, alias_map=alias_map, + assumptions_map=assumptions_map, ): return True return False diff --git a/src/prkit/semantics/comparison/semantics.py b/src/prkit/semantics/comparison/semantics.py index ef31612..a46f5e5 100644 --- a/src/prkit/semantics/comparison/semantics.py +++ b/src/prkit/semantics/comparison/semantics.py @@ -17,6 +17,7 @@ Float, Ge, Gt, + I, Integer, Le, Lt, @@ -24,6 +25,7 @@ Min, N, Piecewise, + Pow, Rational, Symbol, acos, @@ -59,6 +61,7 @@ from ..units import convert_numeric_value as _shared_convert_numeric_value from ..units import normalize_unit_text as _shared_normalize_unit_text from ..units import unit_conversion_factor as _shared_unit_conversion_factor +from .common import context_symbol_assumption_map _TRANSFORMATIONS = standard_transformations + ( implicit_multiplication_application, @@ -175,6 +178,17 @@ r"(?[A-Za-z]{2,})(?P_[A-Za-z0-9]+)?\b" ) _TRIG_FUNCTION_RE = re.compile(r"\b(?:sin|cos|tan|asin|acos|atan|sinh|cosh|tanh)\b") +# Explicit imaginary-unit markers. Lower-case ``i``/``j`` are intentionally *not* markers: +# they are overwhelmingly summation indices / ordinary symbols in physics answers, so +# treating them as imaginary would needlessly disable the (precision-safe) realness +# default. A standalone capital ``I`` or a LaTeX imaginary command withholds realness so +# such answers stay generic complex (no recall gain, but never a wrong accept). +_IMAGINARY_MARKER_RE = re.compile( + r"(?=!])\b[\w]+\s*=") _DECORATION_SUFFIXES = ("_ddot", "_dot", "_hat", "_vec", "_bar", "_tilde") @@ -829,8 +843,15 @@ def parse_symbolic_expression( text: str, *, alias_map: Mapping[str, str] | None = None, + assumptions_map: Mapping[str, Mapping[str, bool]] | None = None, ) -> Any | None: - """Parse algebraic text into a SymPy expression after light canonicalization.""" + """Parse algebraic text into a SymPy expression after light canonicalization. + + ``assumptions_map`` carries per-symbol SymPy assumption kwargs (e.g. + ``{"m": {"positive": True}}``) so the equivalence judgement is decided over the + intended physical domain rather than the generic complex default. Keys are the + canonical (post-alias) tokens that appear after preprocessing. + """ candidate = preprocess_symbolic_text(text, alias_map=alias_map) if not candidate: @@ -844,7 +865,12 @@ def parse_symbolic_expression( local_dict = dict(_EXPRESSION_FUNCTIONS) for token in _SYMBOL_TOKEN_RE.findall(parse_candidate): if token not in local_dict: - local_dict[token] = Symbol(token) + token_assumptions = assumptions_map.get(token) if assumptions_map else None + local_dict[token] = ( + Symbol(token, **token_assumptions) + if token_assumptions + else Symbol(token) + ) try: return parse_expr( @@ -870,23 +896,274 @@ def parse_scalar_symbolic_expression( text: str, *, alias_map: Mapping[str, str] | None = None, + assumptions_map: Mapping[str, Mapping[str, bool]] | None = None, ) -> Any | None: """Parse one scalar symbolic expression and reject tuple/set-like parses.""" - expression = parse_symbolic_expression(text, alias_map=alias_map) + expression = parse_symbolic_expression( + text, alias_map=alias_map, assumptions_map=assumptions_map + ) if not _is_scalar_symbolic_object(expression): return None return expression +def build_symbol_assumption_map( + pred_text: str | None, + ref_text: str | None, + *, + context: Any, + alias_map: Mapping[str, str] | None = None, +) -> Mapping[str, Mapping[str, bool]]: + """Build the per-symbol assumption map used to decide symbolic equivalence. + + Two sources are merged. The **authoritative** source is the question's declared + ``symbol_assumptions`` (``context_symbol_assumption_map``); whatever it declares wins. The + gaps are filled by a **conservative** in-engine derivation that fires only on + self-justifying, symmetric signals (see ``_derive_symbol_assumptions``). The result is + meaning-preserving: it never invents a domain that the answers themselves do not + already presuppose, so it raises recall without eroding precision. + """ + + declared = context_symbol_assumption_map(context) + derived = _derive_symbol_assumptions(pred_text, ref_text, alias_map=alias_map) + if not declared: + return derived + merged: dict[str, Mapping[str, bool]] = dict(derived) + merged.update(declared) + return merged + + +def _derive_symbol_assumptions( + pred_text: str | None, + ref_text: str | None, + *, + alias_map: Mapping[str, str] | None = None, +) -> dict[str, Mapping[str, bool]]: + """Conservatively infer real-domain assumptions from the two answer surfaces. + + The only derived signal is the **realness default**: every free symbol is assumed + ``real`` unless an explicit imaginary-unit marker appears in either surface (in which + case the answers may be complex and nothing is derived). Physics expression answers + are real-valued functions of real variables, so this is precision-safe -- it unlocks + identities that hold over the reals (e.g. ``sqrt(x**2) == Abs(x)``) without changing + any truth value. + + Positivity / nonnegativity is deliberately **not** derived from surface appearance. + Writing ``sqrt(a*b)`` or ``log(x**2)`` does not presuppose any individual symbol is + nonnegative (only that a product / even power is), so a surface heuristic would + manufacture sign assumptions that flip truth values -- e.g. wrongly accepting + ``sqrt(a*b)`` vs ``sqrt(a)*sqrt(b)`` or ``log(x**2)`` vs ``2*log(x)``, which differ at + negative arguments. Those accepts require an explicit ``symbol_assumptions`` declaration, + which is the authoritative source. Symbol-name whitelists are likewise avoided + (physics reuses letters: ``m`` mass vs metre, ``T`` period vs temperature, signed + coordinates ``x``). + """ + + if not pred_text or not ref_text: + return {} + if _has_imaginary_marker(pred_text) or _has_imaginary_marker(ref_text): + return {} + left = parse_symbolic_expression(pred_text, alias_map=alias_map) + right = parse_symbolic_expression(ref_text, alias_map=alias_map) + if left is None or right is None: + return {} + + all_names = _expression_free_symbol_names(left) | _expression_free_symbol_names( + right + ) + return {name: {"real": True} for name in all_names} + + +def _has_imaginary_marker(text: str | None) -> bool: + """Whether a surface carries an explicit imaginary-unit marker (see the regex).""" + + if not text: + return False + return _IMAGINARY_MARKER_RE.search(text) is not None + + +def _expression_free_symbol_names(expression: Any) -> set[str]: + """Return the names of an expression's free symbols, tolerant of odd parses.""" + + try: + return {symbol.name for symbol in expression.free_symbols} + except Exception: + return set() + + +def _has_strengthening_assumption( + assumptions_map: Mapping[str, Mapping[str, bool]] | None, +) -> bool: + """Whether any symbol carries an assumption stronger than plain realness.""" + + if not assumptions_map: + return False + for kwargs in assumptions_map.values(): + if any(kwargs.get(key) for key in ("positive", "nonnegative", "nonzero")): + return True + return False + + +# Deterministic generic sample magnitudes for numeric identity testing. They are spread +# over a wide range and deliberately avoid integers / simple fractions / special constants +# (0, 1, pi, e) so a genuinely different function cannot coincide with the reference at all +# of them, while a true identity matches to full working precision. +_PIT_SAMPLE_MAGNITUDES = ( + 0.6431, + 1.2719, + 2.3137, + 0.8923, + 3.7211, + 1.9043, + 4.5317, + 0.4129, + 2.8761, + 5.3119, + 1.4567, + 3.1409, + 0.7321, + 6.2237, + 2.0173, + 4.1287, + 0.5519, + 1.6633, + 3.9041, + 7.1129, + 2.5503, + 0.3697, + 5.7919, + 1.1087, + 4.8231, + 2.2391, + 0.9817, + 3.4523, + 6.6133, + 1.8219, + 5.0317, + 0.7129, + 2.6611, + 4.3719, + 1.3313, + 8.2237, +) +_PIT_WORKING_PRECISION = 30 +_PIT_MAX_POINTS = 36 +_PIT_MIN_AGREEMENTS = 12 +_PIT_AGREE_REL = 1e-12 +_PIT_REJECT_REL = 1e-6 + + +def _pit_sample_value(symbol: Any, point_index: int, symbol_index: int) -> Any: + """Pick a deterministic generic sample for ``symbol`` honoring its real-domain. + + Positive / nonnegative symbols sample positive; generic real symbols sample both + signs (so domain-sensitive identities such as ``sqrt(x**2) == x`` are rejected when + ``x`` is not nonnegative); symbols with no real assumption sample a generic complex + value (so identities that hold only over the reals are not falsely accepted). + + Each symbol's sign is bit ``symbol_index`` of ``point_index``, so signs vary + *independently* across points: as ``point_index`` increases, every sign combination of + the first ``log2(point_count)`` symbols is visited. That coverage is what rejects + products like ``sqrt(a*b)`` vs ``sqrt(a)*sqrt(b)``, which agree unless several symbols + are simultaneously negative. + """ + + pool = _PIT_SAMPLE_MAGNITUDES + size = len(pool) + magnitude = pool[(point_index * 7 + symbol_index * 13) % size] + if symbol.is_nonnegative: + return Float(magnitude) + if symbol.is_real: + negative = bool((point_index >> symbol_index) & 1) + return Float(-magnitude if negative else magnitude) + imaginary = pool[(point_index * 5 + symbol_index * 11 + 3) % size] + return Float(magnitude) + Float(imaginary) * I + + +def _numeric_identity_equivalent( + left_expr: Any, + right_expr: Any, + tolerance: float, +) -> bool | None: + """Decide ``left == right`` as functions by multi-point numeric identity testing. + + Returns ``True`` when the two evaluate equal at enough generic domain points, + ``False`` on the first clear disagreement (an exact disproof of a function identity), + and ``None`` when too few points could be evaluated to decide. Sampling honors each + symbol's domain assumptions (see ``_pit_sample_value``); evaluation uses high working + precision and skips singular / ill-conditioned points so numeric noise never produces + a false verdict. + """ + + try: + symbols = sorted( + left_expr.free_symbols | right_expr.free_symbols, + key=lambda symbol: symbol.name, + ) + except Exception: + return None + + point_count = 1 if not symbols else _PIT_MAX_POINTS + required = 1 if not symbols else _PIT_MIN_AGREEMENTS + agreements = 0 + for point_index in range(point_count): + subs = { + symbol: _pit_sample_value(symbol, point_index, symbol_index) + for symbol_index, symbol in enumerate(symbols) + } + try: + left_value = left_expr.evalf(_PIT_WORKING_PRECISION, subs=subs) + right_value = right_expr.evalf(_PIT_WORKING_PRECISION, subs=subs) + except Exception: + continue + if getattr(left_value, "free_symbols", set()) or getattr( + right_value, "free_symbols", set() + ): + continue + try: + left_complex = complex(left_value) + right_complex = complex(right_value) + except (TypeError, ValueError): + continue + if not _is_finite_complex(left_complex) or not _is_finite_complex( + right_complex + ): + continue + scale = max(abs(left_complex), abs(right_complex), 1.0) + relative = abs(left_complex - right_complex) / scale + if relative > _PIT_REJECT_REL: + return False + if relative <= _PIT_AGREE_REL: + agreements += 1 + if agreements >= required: + return True + return True if agreements >= required else None + + +def _is_finite_complex(value: complex) -> bool: + """Whether a Python complex value is fully finite.""" + + return math.isfinite(value.real) and math.isfinite(value.imag) + + def expressions_equivalent( left_text: str | None, right_text: str | None, tolerance: float, *, alias_map: Mapping[str, str] | None = None, + assumptions_map: Mapping[str, Mapping[str, bool]] | None = None, ) -> bool: - """Check symbolic equivalence between two expression texts.""" + """Check symbolic equivalence between two expression texts. + + Equivalence is the single predicate "is ``left - right`` the zero function over the + declared domain?". It is decided symbolically first (canonical text, ``simplify``, + ``trigsimp``) and, when that is inconclusive, by ``_numeric_identity_equivalent`` -- + multi-point evaluation over the symbols' domain. Numeric *disagreement* is an exact + disproof, so it also guards a symbolic acceptance reached under a strengthening + assumption. + """ if not left_text or not right_text: return False @@ -906,8 +1183,12 @@ def expressions_equivalent( ): return True - left_expr = parse_symbolic_expression(left_text, alias_map=alias_map) - right_expr = parse_symbolic_expression(right_text, alias_map=alias_map) + left_expr = parse_symbolic_expression( + left_text, alias_map=alias_map, assumptions_map=assumptions_map + ) + right_expr = parse_symbolic_expression( + right_text, alias_map=alias_map, assumptions_map=assumptions_map + ) if left_expr is None or right_expr is None: return normalize_plain_text(left_text) == normalize_plain_text(right_text) if isinstance(left_expr, Relational) or isinstance(right_expr, Relational): @@ -916,6 +1197,7 @@ def expressions_equivalent( right_text, tolerance, alias_map=alias_map, + assumptions_map=assumptions_map, ) if not _is_scalar_symbolic_object(left_expr) or not _is_scalar_symbolic_object( right_expr @@ -925,27 +1207,52 @@ def expressions_equivalent( try: diff = simplify(left_expr - right_expr) except Exception: - return normalize_plain_text(left_text) == normalize_plain_text(right_text) - if diff == 0 or diff.is_zero is True: - return True - if _TRIG_FUNCTION_RE.search(left_text) or _TRIG_FUNCTION_RE.search(right_text): + diff = None + + symbolic_equal = diff is not None and (diff == 0 or diff.is_zero is True) + if not symbolic_equal and ( + _TRIG_FUNCTION_RE.search(left_text) or _TRIG_FUNCTION_RE.search(right_text) + ): try: trig_diff = trigsimp(left_expr - right_expr) if trig_diff == 0 or trig_diff.is_zero is True: - return True + symbolic_equal = True except Exception: pass + if not symbolic_equal: + try: + if trigsimp(left_expr) == trigsimp(right_expr): + symbolic_equal = True + except Exception: + pass + if not symbolic_equal and diff is not None: try: - if trigsimp(left_expr) == trigsimp(right_expr): - return True + if diff.equals(0): + symbolic_equal = True except Exception: pass - try: - if diff.equals(0): - return True - except Exception: - pass - if diff.is_number: + + if symbolic_equal: + # Guard: a symbolic acceptance reached under a strengthening assumption + # (positive/nonnegative/nonzero) is confirmed numerically over that domain; an + # exact numeric disproof vetoes it. Plain realness and assumption-free accepts are + # already exact, so they skip the (hot-path) guard. + if _has_strengthening_assumption(assumptions_map) and ( + _numeric_identity_equivalent(left_expr, right_expr, tolerance) is False + ): + return False + return True + + # Symbolic test inconclusive: decide by numeric identity testing over the domain. + verdict = _numeric_identity_equivalent(left_expr, right_expr, tolerance) + if verdict is True: + return True + if verdict is False: + return False + + # Numerically undecidable (e.g. every sample hit a singularity): preserve the legacy + # constant-residual fallback, then fail closed. + if diff is not None and diff.is_number: try: return numbers_close(float(N(diff)), 0.0, tolerance) except (TypeError, ValueError): @@ -957,13 +1264,15 @@ def parse_relation_clauses( text: str | None, *, alias_map: Mapping[str, str] | None = None, + assumptions_map: Mapping[str, Mapping[str, bool]] | None = None, ) -> tuple[RelationClause, ...] | None: """Parse relation text into a flat tuple of canonical binary clauses. Canonicalization folds a single-symbol functional-form left-hand side (``r(t) = ...`` -> ``r = ...``) -- the standard "target as a function of its - variable" notation -- so a relation has one canonical clause form independent of - that surface choice. + variable" notation -- and de-radicalizes a solved even root (``c = sqrt(E/m)`` -> + ``c**2 = E/m``) when the symbol assumptions make squaring sign-safe, so a relation has + one canonical clause form independent of those surface choices. """ candidate = preprocess_symbolic_text(text, alias_map=alias_map) @@ -974,7 +1283,9 @@ def parse_relation_clauses( if relation_object is not None: relation_clauses = _clauses_from_relation_object(relation_object) if relation_clauses: - return _canonical_relation_clauses(relation_clauses) + return _canonical_relation_clauses( + relation_clauses, alias_map=alias_map, assumptions_map=assumptions_map + ) clauses: list[RelationClause] = [] for segment in _split_top_level_conjunctions(candidate): @@ -982,7 +1293,11 @@ def parse_relation_clauses( if parsed is None: return None clauses.extend(parsed) - return _canonical_relation_clauses(tuple(clauses)) if clauses else None + if not clauses: + return None + return _canonical_relation_clauses( + tuple(clauses), alias_map=alias_map, assumptions_map=assumptions_map + ) def relations_equivalent( @@ -991,6 +1306,7 @@ def relations_equivalent( tolerance: float, *, alias_map: Mapping[str, str] | None = None, + assumptions_map: Mapping[str, Mapping[str, bool]] | None = None, ) -> bool: """Check whether two relation strings encode the same constraint set.""" @@ -1005,13 +1321,21 @@ def relations_equivalent( ): return True - left_clauses = parse_relation_clauses(left_text, alias_map=alias_map) - right_clauses = parse_relation_clauses(right_text, alias_map=alias_map) + left_clauses = parse_relation_clauses( + left_text, alias_map=alias_map, assumptions_map=assumptions_map + ) + right_clauses = parse_relation_clauses( + right_text, alias_map=alias_map, assumptions_map=assumptions_map + ) if left_clauses is None or right_clauses is None: return normalize_plain_text(left_text) == normalize_plain_text(right_text) return _relation_clause_sets_equivalent( - left_clauses, right_clauses, tolerance, alias_map=alias_map + left_clauses, + right_clauses, + tolerance, + alias_map=alias_map, + assumptions_map=assumptions_map, ) @@ -1021,6 +1345,7 @@ def _relation_clause_sets_equivalent( tolerance: float, *, alias_map: Mapping[str, str] | None = None, + assumptions_map: Mapping[str, Mapping[str, bool]] | None = None, ) -> bool: """Match two clause sets as an order-insensitive collection of equivalent clauses.""" @@ -1036,6 +1361,7 @@ def _relation_clause_sets_equivalent( right_clause, tolerance, alias_map=alias_map, + assumptions_map=assumptions_map, ): matched_index = index break @@ -1052,10 +1378,21 @@ def _relation_clause_sets_equivalent( def _canonical_relation_clauses( clauses: tuple[RelationClause, ...], + *, + alias_map: Mapping[str, str] | None = None, + assumptions_map: Mapping[str, Mapping[str, bool]] | None = None, ) -> tuple[RelationClause, ...]: """Return the canonical clause form used for all relation comparison.""" - return tuple(_collapse_functional_form_lhs(clause) for clause in clauses) + canonical = [] + for clause in clauses: + collapsed = _collapse_functional_form_lhs(clause) + canonical.append( + _deradicalize_clause( + collapsed, alias_map=alias_map, assumptions_map=assumptions_map + ) + ) + return tuple(canonical) def _collapse_functional_form_lhs(clause: RelationClause) -> RelationClause: @@ -1078,6 +1415,96 @@ def _collapse_functional_form_lhs(clause: RelationClause) -> RelationClause: return RelationClause(match.group("name"), clause.operator, clause.rhs_text) +def _deradicalize_clause( + clause: RelationClause, + *, + alias_map: Mapping[str, str] | None = None, + assumptions_map: Mapping[str, Mapping[str, bool]] | None = None, +) -> RelationClause: + """Square a solved even-root equality into polynomial form when sign-safe. + + A solved radical equality such as ``c = sqrt(E/m)`` or ``v = sqrt(u**2 + 2*a*s)`` is + the same constraint as its squared form (``c**2 = E/m``) *provided the non-radical side + is known nonnegative* -- squaring is injective on the nonnegative reals, so it + introduces no spurious branch. This is a meaning-preserving canonical form (applied to + every relation, gated on the assumptions), so the squared clause is then matched by the + existing polynomial equality criterion (``_equalities_equivalent``). The gate is + essential: without a nonnegative non-radical side, squaring would wrongly merge + ``c = sqrt(E/m)`` (the ``c >= 0`` branch) with ``E = m*c**2`` (both branches), so the + rewrite is skipped and the clause is left untouched. + """ + + if clause.operator != "=": + return clause + # Squaring needs a provably-nonnegative non-radical side, which only a strengthening + # assumption (declared positive/nonnegative/nonzero) supplies -- realness alone never + # does. Gate on that and on a cheap radical hint so the common no-radical relation + # comparison does no extra parsing. + if not _has_strengthening_assumption(assumptions_map): + return clause + if not _RADICAL_HINT_RE.search(clause.lhs_text) and not _RADICAL_HINT_RE.search( + clause.rhs_text + ): + return clause + lhs = parse_scalar_symbolic_expression( + clause.lhs_text, alias_map=alias_map, assumptions_map=assumptions_map + ) + rhs = parse_scalar_symbolic_expression( + clause.rhs_text, alias_map=alias_map, assumptions_map=assumptions_map + ) + if lhs is None or rhs is None: + return clause + + for plain_side, radical_side in ((lhs, rhs), (rhs, lhs)): + squared = _square_isolated_even_root(plain_side, radical_side) + if squared is not None: + new_lhs, new_rhs = squared + return RelationClause(str(new_lhs), "=", str(new_rhs)) + return clause + + +def _square_isolated_even_root( + plain_side: Any, radical_side: Any +) -> tuple[Any, Any] | None: + """Return ``(plain_side**2, radical_side**2)`` when squaring is justified, else ``None``. + + Justified iff the ``plain_side`` is provably nonnegative (the sign gate), the + ``radical_side`` carries an even root, and squaring removes that root (so the result is + rational and the polynomial equality criterion applies). + """ + + if plain_side.is_nonnegative is not True: + return None + if not _contains_even_root(radical_side): + return None + try: + radical_squared = simplify(radical_side**2) + plain_squared = simplify(plain_side**2) + except Exception: + return None + if _contains_even_root(radical_squared) or _contains_even_root(plain_squared): + return None + return plain_squared, radical_squared + + +def _contains_even_root(expression: Any) -> bool: + """Whether the expression contains a non-integer power with an even denominator.""" + + try: + powers = expression.atoms(Pow) + except Exception: + return False + for power in powers: + exponent = power.exp + if ( + getattr(exponent, "is_Rational", False) + and not exponent.is_Integer + and int(exponent.q) % 2 == 0 + ): + return True + return False + + def relation_compare_candidates( text: str | None, *, @@ -2106,6 +2533,7 @@ def _relation_clause_equivalent( tolerance: float, *, alias_map: Mapping[str, str] | None = None, + assumptions_map: Mapping[str, Mapping[str, bool]] | None = None, ) -> bool: """Whether two relation clauses denote the same constraint. @@ -2121,12 +2549,14 @@ def _relation_clause_equivalent( right.lhs_text, tolerance, alias_map=alias_map, + assumptions_map=assumptions_map, ) and expressions_equivalent( left.rhs_text, right.rhs_text, tolerance, alias_map=alias_map, + assumptions_map=assumptions_map, ) ): return True @@ -2139,20 +2569,30 @@ def _relation_clause_equivalent( right.rhs_text, tolerance, alias_map=alias_map, + assumptions_map=assumptions_map, ) and expressions_equivalent( left.rhs_text, right.lhs_text, tolerance, alias_map=alias_map, + assumptions_map=assumptions_map, ) ): return True - left_lhs = parse_symbolic_expression(left.lhs_text, alias_map=alias_map) - left_rhs = parse_symbolic_expression(left.rhs_text, alias_map=alias_map) - right_lhs = parse_symbolic_expression(right.lhs_text, alias_map=alias_map) - right_rhs = parse_symbolic_expression(right.rhs_text, alias_map=alias_map) + left_lhs = parse_symbolic_expression( + left.lhs_text, alias_map=alias_map, assumptions_map=assumptions_map + ) + left_rhs = parse_symbolic_expression( + left.rhs_text, alias_map=alias_map, assumptions_map=assumptions_map + ) + right_lhs = parse_symbolic_expression( + right.lhs_text, alias_map=alias_map, assumptions_map=assumptions_map + ) + right_rhs = parse_symbolic_expression( + right.rhs_text, alias_map=alias_map, assumptions_map=assumptions_map + ) if left_lhs is None or left_rhs is None or right_lhs is None or right_rhs is None: return False if not all( diff --git a/src/prkit/semantics/schema/__init__.py b/src/prkit/semantics/schema/__init__.py index b6d620c..418ce38 100644 --- a/src/prkit/semantics/schema/__init__.py +++ b/src/prkit/semantics/schema/__init__.py @@ -15,6 +15,7 @@ OrderingPolicy, QuestionSymbolicMode, QuestionUnitPolicy, + SymbolAssumption, ) from .models import ( DEFAULT_NUMERIC_TOLERANCE, @@ -27,6 +28,7 @@ PhysicsEvaluationContract, PhysicsQuestionSemantics, PhysicsSymbolAliasSemantics, + PhysicsSymbolAssumptionSemantics, ) __all__ = [ @@ -46,6 +48,8 @@ "PhysicsEvaluationContract", "PhysicsQuestionSemantics", "PhysicsSymbolAliasSemantics", + "PhysicsSymbolAssumptionSemantics", "QuestionSymbolicMode", "QuestionUnitPolicy", + "SymbolAssumption", ] diff --git a/src/prkit/semantics/schema/enums.py b/src/prkit/semantics/schema/enums.py index 631cb08..1d2316d 100644 --- a/src/prkit/semantics/schema/enums.py +++ b/src/prkit/semantics/schema/enums.py @@ -91,3 +91,20 @@ class BridgeTier(_StrEnum): TIER1 = "tier1" TIER2 = "tier2" TIER3 = "tier3" + + +class SymbolAssumption(_StrEnum): + """Real-domain a free symbol ranges over during symbolic comparison. + + Physics answers denote real, often nonnegative, quantities; declaring this lets the + SymPy substrate decide equivalence over the *intended* domain instead of the generic + complex default (e.g. ``sqrt(a*b) == sqrt(a)*sqrt(b)`` holds for ``a, b >= 0`` but not + over the complex plane). The judgement stays exact -- it is decided over the declared + domain, not relaxed. + """ + + COMPLEX = "complex" + REAL = "real" + NONZERO = "nonzero" + NONNEGATIVE = "nonnegative" + POSITIVE = "positive" diff --git a/src/prkit/semantics/schema/models.py b/src/prkit/semantics/schema/models.py index d4a9cd2..27e13ce 100644 --- a/src/prkit/semantics/schema/models.py +++ b/src/prkit/semantics/schema/models.py @@ -15,6 +15,7 @@ OrderingPolicy, QuestionSymbolicMode, QuestionUnitPolicy, + SymbolAssumption, ) DEFAULT_NUMERIC_TOLERANCE = 1e-10 @@ -37,6 +38,10 @@ class PhysicsQuestionSemantics(_SemanticsModel): default_factory=tuple, description="Question-conditioned alias groups that map alternate symbol names onto a canonical symbol for comparison.", ) + symbol_assumptions: tuple[PhysicsSymbolAssumptionSemantics, ...] = Field( + default_factory=tuple, + description="Question-conditioned real-domain declarations for free symbols, used to decide symbolic equivalence over the intended physical domain.", + ) allowed_object_kinds: tuple[AnswerObjectKind, ...] = Field( default_factory=lambda: tuple(AnswerObjectKind), description="Semantic answer kinds admitted by the question.", @@ -240,6 +245,23 @@ class PhysicsSymbolAliasSemantics(_SemanticsModel): ) +class PhysicsSymbolAssumptionSemantics(_SemanticsModel): + """One question-conditioned real-domain assumption for a symbol. + + The ``symbol`` is the canonical (post-alias) token; ``assumption`` constrains the + values it ranges over (real / nonzero / nonnegative / positive / complex) so symbolic + comparison is decided over the intended real-physical domain rather than the generic + complex default. See ``SymbolAssumption``. + """ + + symbol: str = Field( + description="Canonical (post-alias) symbol token the assumption applies to.", + ) + assumption: SymbolAssumption = Field( + description="Real-domain assumption the symbol ranges over during symbolic comparison.", + ) + + class PhysicsAnswerSemantics(_SemanticsModel): """Normalized physics-aware final answer semantics.""" From 337799ef1cbee438ba15951466b13f90b29c5d72 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Fri, 19 Jun 2026 13:55:30 -0400 Subject: [PATCH 07/28] Add the answer-structure decision layer (canonicalize, then gate to atomic) Structure misclassification is an unrecoverable false negative (the structure-mismatch gate has no bridge), so reliably reach genuine atomic-vs-atomic before the atomic judgement: - canonicalize_structure: a symmetric, idempotent, precision-safe pre-comparison pass that collapses structural degeneracies to atomic (1-element tuple/set/vector, closed point-interval [a,a], single-case trivially-conditioned piecewise), inserted last in _repair_answer_for_comparison. - gate non-atomic comparison to proven-sound accepts only: ordered (tuple/interval/vector with atomic cells), piecewise, and label-aligned per-part are enabled; set/unordered accept only an exact multiset match (sound tolerant matching is a deferred roadmap milestone); matrix/tensor, per-part positional fallback, one-sided coordinate frame, and unparsed shaped payloads return a TBD sentinel (comparison_mode=not_implemented), or raise under STRICT_STRUCTURE_COMPARISON. - contract admits ATOMIC whenever a collapsible structure is admitted, so a collapsed answer is not a spurious contract violation. Adds STRUCTURE.md, structure decision guidance in the inference prompt role, a gold structure corpus + confusion harness, and per-comparator adversarial-reject batteries (incl. the {1.0,1.0} != {1.0,1.1} set-tolerance regression). Also carries the atomic-equivalence test battery (shared test file). Co-Authored-By: Claude Opus 4.8 --- src/prkit/semantics/comparison/STRUCTURE.md | 77 +++++++ src/prkit/semantics/comparison/contract.py | 26 ++- src/prkit/semantics/comparison/engine.py | 143 ++++++++---- .../comparison/structure_canonicalization.py | 145 ++++++++++++ src/prkit/semantics/inference/prompts.py | 9 +- tests/prkit/semantics/__init__.py | 0 tests/prkit/semantics/fixtures/__init__.py | 1 + .../semantics/fixtures/structure_gold.py | 118 ++++++++++ .../semantics/test_protocol_comparison.py | 210 +++++++++++++++++- .../test_structure_canonicalization.py | 183 +++++++++++++++ .../semantics/test_structure_decision.py | 93 ++++++++ .../prkit/semantics/test_structure_gating.py | 201 +++++++++++++++++ 12 files changed, 1156 insertions(+), 50 deletions(-) create mode 100644 src/prkit/semantics/comparison/STRUCTURE.md create mode 100644 src/prkit/semantics/comparison/structure_canonicalization.py create mode 100644 tests/prkit/semantics/__init__.py create mode 100644 tests/prkit/semantics/fixtures/__init__.py create mode 100644 tests/prkit/semantics/fixtures/structure_gold.py create mode 100644 tests/prkit/semantics/test_structure_canonicalization.py create mode 100644 tests/prkit/semantics/test_structure_decision.py create mode 100644 tests/prkit/semantics/test_structure_gating.py diff --git a/src/prkit/semantics/comparison/STRUCTURE.md b/src/prkit/semantics/comparison/STRUCTURE.md new file mode 100644 index 0000000..7a79965 --- /dev/null +++ b/src/prkit/semantics/comparison/STRUCTURE.md @@ -0,0 +1,77 @@ +# Answer structure — decision, canonicalization, and gating + +The companion to [`EQUIVALENCE.md`](EQUIVALENCE.md) for the **structure** axis +(`AnswerStructure`), which sits *above* the atomic object-kind judgement. Structure +classification is load-bearing: `compare_protocol_answers` returns `structure_mismatch` the +instant `pred.structure != ref.structure` ([engine.py](engine.py)), and there are no +structural bridges — so a misclassification is an *unrecoverable* false negative. The goal +of this layer is to **reliably reach genuine atomic-vs-atomic** before the atomic judgement +runs. + +## 1. Per-structure signatures + +Each structure is defined by `⟨denotation, cardinality, ordering, surface evidence⟩`. + +| Structure | Denotation | Card. | Ordering | Surface evidence | +|---|---|---|---|---| +| `atomic` | one indivisible value | 1 | — | the default; target of every collapse | +| `multi_part` | answers to several question-defined sub-questions | ≥1 | from `q.ordering` | `required_parts`/enumerated `(1)(2)`, `;`, newlines | +| `tuple` | one ordered coordinate of a single object `(x, y)` | ≥2 | positional | parenthesized ≥2 finite parts, no `required_parts` match | +| `set` | unordered collection of distinct solutions `{x₁, x₂}` | ≥2 | none | brace-delimited ≥2 parts | +| `interval` | a connected range of one variable | 2 endpoints | — | bracket form `[a,b]`/`(a,b]`/…, an `∞` token, or `Interval()` | +| `vector` / `matrix` / `tensor` | a shaped array of rank 1 / 2 / ≥3 | shape | positional | `<…>`, basis sums, LaTeX matrix env, nested brackets of depth 1/2/≥3 | +| `piecewise` | a function defined by (expression, condition) branches | cases | by case | `\begin{cases}…`, `Piecewise(…)` | + +## 2. Boundary tie-break rules (parser + LLM + canonicalizer share these) + +- **Rule A — interval vs tuple.** A bare *finite* `(a, b)` is a **tuple**. It is an + **interval** only with a bracket boundary (`[`/`]`), an `∞` token, or explicit + `Interval()`/range wording. +- **Rule B — tuple vs multi_part vs vector.** Promote `(…)` to **multi_part** iff + `q.ordering == PER_PART` and the part count matches `q.required_parts`; otherwise it is a + **tuple**. A tuple of uniform atomics is promoted to **vector** only at repair time, never + at classification time. +- **Rule C — vector vs matrix vs tensor.** Rank is bracket-nesting depth: depth 1 = vector, + depth 2 with uniform rows = matrix, depth ≥3 or non-uniform = tensor. A `(n,)` vector and + an `(n, 1)` matrix are **different denotations** and are **not** auto-reconciled + (precision guard). + +## 3. Canonicalization (precision-preserving, symmetric, idempotent) + +`canonicalize_structure` ([structure_canonicalization.py](structure_canonicalization.py)) +runs as the last step of `_repair_answer_for_comparison`, on **both** pred and ref, before +the structure gate. It applies only **denotational identities** — meaning-preserving +rewrites that can reach atomic-vs-atomic but never equate distinct answers: + +1. a 1-element `tuple`/`set`/`vector` → its sole element (a 1-coordinate is its scalar); +2. a **closed** point-interval `[a, a]` → the point `a` (open `(a,a)`/`[a,a)`/`(a,a]` denote + the empty set and are left intact); +3. a single-case `piecewise` whose condition is syntactically trivial (`True`/`otherwise`/…) + → its expression. + +It deliberately does **not** collapse `multi_part` (a one-part answer may carry a +part-structure the contract enforces) and does **not** reconcile shapes. + +## 4. Comparison gating — only proven-sound accepts pass + +The equivalence judgement runs for a non-atomic structure only through a comparator path +whose *accept* is proven 100%-equivalence-sound (each behind an adversarial-reject battery, +mirroring the atomic methodology). Other non-atomic cases raise `NotImplementedError`/`TBD` +rather than returning a silent verdict. See the per-comparator audit and gating in W1d of +the implementation plan and the batteries in +`tests/prkit/semantics/`. + +## 5. Deferred (named gaps — not silently dropped) + +- interval ↔ 2-clause-conjunction reconciliation (open/closed-boundary precision hazard); +- folding `subject_to` bound-pairs into an interval (conflates a side-condition with an + interval-valued answer); +- `(n,)` ↔ `(n, 1)` shape reconciliation; +- **sound unordered matching algorithm** (`set` / unordered `multi_part`) under tolerance and + symbolic equivalence — **roadmap milestone**. Today only *exact* multiset matches are + accepted (exact numeric value or normalized text); everything requiring a tolerant/symbolic + bijection is TBD, because a greedy/tolerant match is unsound under non-transitive tolerance + (it accepts `{1.0, 1.0}` vs `{1.0, 1.1}`); +- legitimate ref/pred structure disagreements the precision guards keep distinct (e.g. + roots-as-`set` vs roots-as-`multi_part`) — recorded as a known false-negative inventory; +- full structured-comparison recall (matrix/tensor, richer per-structure comparators). diff --git a/src/prkit/semantics/comparison/contract.py b/src/prkit/semantics/comparison/contract.py index a0ae06a..6a8c3b6 100644 --- a/src/prkit/semantics/comparison/contract.py +++ b/src/prkit/semantics/comparison/contract.py @@ -66,6 +66,20 @@ def build_evaluation_contract( ) +# Structures that `canonicalize_structure` may collapse to ATOMIC (a 1-element collection, +# a closed point-interval, a single-case piecewise). When any of these is admitted, ATOMIC +# is admitted too (it is the collapse target). +_STRUCTURES_COLLAPSIBLE_TO_ATOMIC = frozenset( + { + AnswerStructure.TUPLE, + AnswerStructure.SET, + AnswerStructure.VECTOR, + AnswerStructure.INTERVAL, + AnswerStructure.PIECEWISE, + } +) + + def validate_answer_against_contract( answer: PhysicsAnswerSemantics | dict[str, Any], contract: PhysicsEvaluationContract | dict[str, Any], @@ -82,7 +96,17 @@ def validate_answer_against_contract( violating = False coercible = False - if resolved_answer.structure not in question.allowed_structures: + if resolved_answer.structure not in question.allowed_structures and not ( + resolved_answer.structure == AnswerStructure.ATOMIC + and any( + admitted in _STRUCTURES_COLLAPSIBLE_TO_ATOMIC + for admitted in question.allowed_structures + ) + ): + # Structure canonicalization collapses a degenerate collapsible structure (1-element + # tuple/set/vector, [a,a] interval, single-case piecewise) to ATOMIC before this + # check, so admit ATOMIC whenever such a structure is admitted — else a collapsed + # answer would spuriously violate the contract in strict/audited modes. diagnostics.append(f"structure_not_admitted:{resolved_answer.structure.value}") violating = True diff --git a/src/prkit/semantics/comparison/engine.py b/src/prkit/semantics/comparison/engine.py index b2d0ecf..9cdd868 100644 --- a/src/prkit/semantics/comparison/engine.py +++ b/src/prkit/semantics/comparison/engine.py @@ -33,7 +33,12 @@ from .different_object_kind import compare_different_object_kinds from .label_family_fallback import compare_label_family_fallback from .same_object_kind import compare_same_object_kind -from .semantics import canonicalize_qualitative_label, normalize_plain_text +from .semantics import ( + canonicalize_qualitative_label, + normalize_plain_text, + parse_numeric_value, +) +from .structure_canonicalization import canonicalize_structure def compare_protocol_answers( @@ -388,6 +393,52 @@ def _compare_identical_atomic_text( return None +# Non-atomic comparison runs only when it provably reduces to atomic-vs-atomic element +# comparisons; anything else is TBD. By default TBD is a distinct non-equivalent sentinel so +# the hot path (batch eval / RL rewards) does not crash; flip this to raise instead. +STRICT_STRUCTURE_COMPARISON = False + + +def _structure_tbd(mode: str, *diagnostics: str) -> AnswerComparison: + """Signal that a non-atomic comparison cannot yet be certified (TBD).""" + + if STRICT_STRUCTURE_COMPARISON: + raise NotImplementedError( + f"structure comparison not implemented ({mode}): {', '.join(diagnostics)}" + ) + return AnswerComparison(False, "not_implemented", (mode,) + diagnostics) + + +def _children_all_atomic(*answers: PhysicsAnswerSemantics) -> bool: + """Whether every child of every given answer is atomic.""" + + return all( + child.structure == AnswerStructure.ATOMIC + for answer in answers + for child in answer.children + ) + + +def _exact_element_key(answer: PhysicsAnswerSemantics) -> tuple: + """A conservative exact-identity key for an atomic element (no tolerance). + + Numbers use their parsed value so ``1/2`` and ``0.5`` match exactly while ``1/3`` and + ``0.3333`` do not (exact float equality, never a tolerance window). Everything else uses + normalized canonical text. + """ + + if answer.object_kind in { + AnswerObjectKind.NUMBER, + AnswerObjectKind.PHYSICAL_QUANTITY, + }: + value = answer.numeric_value + if value is None: + value = parse_numeric_value(answer.numeric_text or answer.canonical_text) + if value is not None: + return ("num", value, answer.unit or "") + return ("text", normalize_plain_text(answer.canonical_text or "")) + + def _compare_ordered_children( pred: PhysicsAnswerSemantics, ref: PhysicsAnswerSemantics, @@ -397,10 +448,12 @@ def _compare_ordered_children( policy_mode: ComparisonPolicyMode, mode: str, ) -> AnswerComparison: - """Compare structured children position by position.""" + """Compare structured children position by position (atomic elements only).""" if len(pred.children) != len(ref.children): return AnswerComparison(False, mode, ("different_child_count",)) + if not _children_all_atomic(pred, ref): + return _structure_tbd(mode, "non_atomic_element") for index, (pred_child, ref_child) in enumerate(zip(pred.children, ref.children)): result = compare_protocol_answers( @@ -429,30 +482,24 @@ def _compare_unordered_children( policy_mode: ComparisonPolicyMode, mode: str, ) -> AnswerComparison: - """Compare structured children as an order-insensitive multiset.""" + """Compare structured children as an order-insensitive multiset. + + Only the certain case is accepted: an *exact* multiset match of atomic elements (no + tolerance, no ambiguous matching). A greedy/tolerant bijection is unsound under + non-transitive tolerance (it accepts ``{1.0, 1.0}`` vs ``{1.0, 1.1}``), so every other + case is TBD pending the sound matching algorithm (roadmap milestone; see STRUCTURE.md). + """ if len(pred.children) != len(ref.children): return AnswerComparison(False, mode, ("different_child_count",)) + if not _children_all_atomic(pred, ref): + return _structure_tbd(mode, "non_atomic_element") - unused = list(pred.children) - for ref_child in ref.children: - match_index = None - for index, pred_child in enumerate(unused): - result = compare_protocol_answers( - pred_child, - ref_child, - contract=contract, - context=context, - policy_mode=policy_mode, - _validate_top_level=False, - ) - if result.equivalent: - match_index = index - break - if match_index is None: - return AnswerComparison(False, mode, ("unmatched_child",)) - unused.pop(match_index) - return AnswerComparison(True, mode) + pred_keys = sorted(_exact_element_key(child) for child in pred.children) + ref_keys = sorted(_exact_element_key(child) for child in ref.children) + if pred_keys == ref_keys: + return AnswerComparison(True, mode) + return _structure_tbd(mode, "inexact_unordered_match") def _compare_per_part_children( @@ -468,18 +515,15 @@ def _compare_per_part_children( if len(pred.children) != len(ref.children): return AnswerComparison(False, mode, ("different_child_count",)) + if not _children_all_atomic(pred, ref): + return _structure_tbd(mode, "non_atomic_part") pred_map = _part_child_map(pred, context=context) ref_map = _part_child_map(ref, context=context) - if pred_map is None or ref_map is None or tuple(pred_map) != tuple(ref_map): - return _compare_ordered_children( - pred, - ref, - context=context, - contract=contract, - policy_mode=policy_mode, - mode=mode, - ) + # Require an explicit, aligned label set on both sides. The previous positional fallback + # when labels did not align could compare mismatched parts, so it is retired to TBD. + if pred_map is None or ref_map is None or set(pred_map) != set(ref_map): + return _structure_tbd(mode, "part_labels_unaligned") for label in pred_map: result = compare_protocol_answers( @@ -539,22 +583,17 @@ def _compare_shaped( (f"shape_mismatch:{pred.shape}!={ref.shape}",), ) + # An unparsed shaped payload cannot be certified per-cell; the text-equality fallback is + # retired to TBD. if not pred.children or not ref.children: - if pred.children != ref.children: - return AnswerComparison( - False, - pred.structure.value, - ("unparsed_shaped_answer",), - ) - matched = pred.canonical_text == ref.canonical_text - return AnswerComparison( - matched, - pred.structure.value, - () if matched else ("unparsed_shaped_answer",), - ) + return _structure_tbd(pred.structure.value, "unparsed_shaped_answer") + # Coordinate frame: both-unset ⇒ the problem's implicit shared frame (proceed); a + # one-sided declaration is unresolved ⇒ TBD; both-set-incompatible is a real mismatch. pred_frame = pred.coordinate_frame or context.coordinate_frame ref_frame = ref.coordinate_frame or context.coordinate_frame + if bool(pred_frame) != bool(ref_frame): + return _structure_tbd(pred.structure.value, "coordinate_frame_unresolved") if ( pred_frame and ref_frame @@ -566,11 +605,15 @@ def _compare_shaped( pred_sign = pred.sign_convention or context.sign_convention ref_sign = ref.sign_convention or context.sign_convention + if bool(pred_sign) != bool(ref_sign): + return _structure_tbd(pred.structure.value, "sign_convention_unresolved") if pred_sign and ref_sign and not _metadata_text_compatible(pred_sign, ref_sign): return AnswerComparison( False, pred.structure.value, ("sign_convention_mismatch",) ) + # Per-cell comparison via the ordered path, which itself gates non-atomic cells: a vector + # of atomic cells is certified; a matrix/tensor (rows are non-atomic) falls to TBD. return _compare_ordered_children( pred, ref, @@ -595,6 +638,16 @@ def _compare_piecewise( return AnswerComparison(False, "piecewise", ("different_case_count",)) for index, (pred_case, ref_case) in enumerate(zip(pred.cases, ref.cases)): + if any( + part.structure != AnswerStructure.ATOMIC + for part in ( + pred_case.expression, + pred_case.condition, + ref_case.expression, + ref_case.condition, + ) + ): + return _structure_tbd("piecewise", f"non_atomic_case_{index}") expr_result = compare_protocol_answers( pred_case.expression, ref_case.expression, @@ -645,6 +698,10 @@ def _repair_answer_for_comparison( repaired = enrich_answer_quantity_views(answer, context=context) repaired = _hydrate_structured_answer(repaired, context=context) repaired = _backfill_subject_to(repaired, context=context) + # Collapse structural degeneracies LAST (dominates the tuple→vector promotion above), so + # a degenerate wrapper reaches atomic before the structure gate; if it does, fall through + # to the atomic repair. + repaired = canonicalize_structure(repaired, context=context) if repaired.structure != AnswerStructure.ATOMIC: return repaired return _repair_atomic_answer(repaired, context=context) diff --git a/src/prkit/semantics/comparison/structure_canonicalization.py b/src/prkit/semantics/comparison/structure_canonicalization.py new file mode 100644 index 0000000..c915808 --- /dev/null +++ b/src/prkit/semantics/comparison/structure_canonicalization.py @@ -0,0 +1,145 @@ +"""Pre-comparison structure canonicalization. + +Collapses structural *degeneracies* to their canonical representative so that +semantically-equal answers reach one structure before the unrecoverable structure-mismatch +gate (``engine.py``). Three reduce-to-atomic identities are applied: + +1. a 1-element ``tuple``/``set``/``vector`` is its sole element (a 1-coordinate is its scalar); +2. a closed point-interval ``[a, a]`` is the point ``a``; +3. a single-case ``piecewise`` with a trivially-true condition is its expression. + +Each rule is a *denotational identity* — it preserves meaning and is applied symmetrically +to both answers — so it can only raise recall (by reaching atomic-vs-atomic), never fabricate +equivalence between genuinely distinct answers. It deliberately does NOT collapse +``multi_part`` (a one-part answer may carry a part-structure the contract enforces) nor +reconcile shapes (``(n,)`` vs ``(n,1)``). + +Runs as the last step of ``_repair_answer_for_comparison`` (so it dominates the earlier +``_coerce_tuple_shaped_answer`` tuple→vector promotion) and is idempotent. +""" + +from __future__ import annotations + +from ..schema import ( + AnswerStructure, + PhysicsAnswerSemantics, + PhysicsQuestionSemantics, +) + +_COLLAPSIBLE_COLLECTIONS = frozenset( + {AnswerStructure.TUPLE, AnswerStructure.SET, AnswerStructure.VECTOR} +) +_TRIVIAL_CONDITION_TEXTS = frozenset( + {"", "true", "otherwise", "else", "always", "all", "any"} +) + + +def canonicalize_structure( + answer: PhysicsAnswerSemantics, + *, + context: PhysicsQuestionSemantics, +) -> PhysicsAnswerSemantics: + """Return ``answer`` with degenerate structures collapsed to canonical form. + + Bottom-up (children/cases are canonicalized first), then a single degeneracy rule is + applied; collapsing recurses so nested degeneracies fully reduce. Idempotent: re-running + on the result returns it unchanged. + """ + + answer = _with_canonical_descendants(answer, context=context) + collapsed = _collapse_once(answer, context=context) + if collapsed is not answer: + return canonicalize_structure(collapsed, context=context) + return answer + + +def _with_canonical_descendants( + answer: PhysicsAnswerSemantics, *, context: PhysicsQuestionSemantics +) -> PhysicsAnswerSemantics: + """Canonicalize children and piecewise cases before considering the parent.""" + + updates: dict[str, object] = {} + if answer.children: + updates["children"] = tuple( + canonicalize_structure(child, context=context) for child in answer.children + ) + if answer.cases: + updates["cases"] = tuple( + case.model_copy( + update={ + "expression": canonicalize_structure( + case.expression, context=context + ), + "condition": canonicalize_structure( + case.condition, context=context + ), + } + ) + for case in answer.cases + ) + return answer.model_copy(update=updates) if updates else answer + + +def _collapse_once( + answer: PhysicsAnswerSemantics, *, context: PhysicsQuestionSemantics +) -> PhysicsAnswerSemantics: + """Apply the first applicable degeneracy collapse, else return ``answer`` unchanged.""" + + structure = answer.structure + + if structure == AnswerStructure.PIECEWISE and len(answer.cases) == 1: + case = answer.cases[0] + if _is_trivial_condition(case.condition): + return _promote_to(answer, case.expression) + + if ( + structure == AnswerStructure.INTERVAL + and len(answer.children) == 2 + and answer.interval_open_left is False + and answer.interval_open_right is False + and _endpoints_equal(answer.children[0], answer.children[1]) + ): + return _promote_to(answer, answer.children[0]) + + if structure in _COLLAPSIBLE_COLLECTIONS and len(answer.children) == 1: + return _promote_to(answer, answer.children[0]) + + return answer + + +def _promote_to( + parent: PhysicsAnswerSemantics, child: PhysicsAnswerSemantics +) -> PhysicsAnswerSemantics: + """Promote ``child`` to the top level, carrying parent-only metadata it lacks.""" + + updates: dict[str, object] = {} + if parent.subject_to and not child.subject_to: + updates["subject_to"] = parent.subject_to + if parent.target_variable and not child.target_variable: + updates["target_variable"] = parent.target_variable + if parent.coordinate_frame and not child.coordinate_frame: + updates["coordinate_frame"] = parent.coordinate_frame + if parent.sign_convention and not child.sign_convention: + updates["sign_convention"] = parent.sign_convention + return child.model_copy(update=updates) if updates else child + + +def _is_trivial_condition(condition: PhysicsAnswerSemantics) -> bool: + """Whether a piecewise condition imposes no real restriction (syntactic only).""" + + if condition.boolean_value is True: + return True + text = (condition.canonical_text or "").strip().lower().rstrip(".") + return text in _TRIVIAL_CONDITION_TEXTS + + +def _endpoints_equal( + left: PhysicsAnswerSemantics, right: PhysicsAnswerSemantics +) -> bool: + """Whether two interval endpoints denote the same value (conservative check).""" + + if left.numeric_value is not None and right.numeric_value is not None: + return left.numeric_value == right.numeric_value + left_text = (left.canonical_text or "").strip() + right_text = (right.canonical_text or "").strip() + return bool(left_text) and left_text == right_text diff --git a/src/prkit/semantics/inference/prompts.py b/src/prkit/semantics/inference/prompts.py index 09271f4..f692b6b 100644 --- a/src/prkit/semantics/inference/prompts.py +++ b/src/prkit/semantics/inference/prompts.py @@ -31,7 +31,14 @@ - In `symbol_aliases`, use plain token-style symbol names like `y`, `y_s`, `theta_dot`, not full equations or wrapped LaTeX snippets. - `reference_answer_semantics` or `prediction_answer_semantics` must represent only the final answer. - Use `object_kind` from: number, physical_quantity, expression, relation, qualitative_label, choice, boolean, sign_direction. -- Use `structure` from: atomic, multi_part, tuple, set, interval, vector, matrix, tensor, piecewise. +- Use `structure` from: atomic, multi_part, tuple, set, interval, vector, matrix, tensor, piecewise. Decide structure by denotation, not surface punctuation: + - `atomic` = one indivisible value (the default). Prefer it: a single coordinate is atomic, not a 1-tuple; a closed point-range `[a, a]` is the atom `a`. + - `tuple` = one ordered coordinate of a single object, `(x, y)`; a bare finite `(a, b)` is a tuple, NOT an interval. + - `interval` = a connected range; use it only for bracket forms `[a, b]`/`(a, b]`/… or a range containing `∞`. + - `set` = an unordered collection of distinct solutions `{x1, x2}`. + - `multi_part` = answers to several question-defined sub-questions; use it only when the question defines parts (`required_parts`) or the answer is explicitly enumerated. + - `vector`/`matrix`/`tensor` = a shaped array of rank 1/2/≥3; keep a `(n,)` vector distinct from an `(n, 1)` matrix. + - `piecewise` = a function with (expression, condition) branches. - Fill `numeric_value`, `numeric_text`, and `unit` for physical quantities when possible. - Use `children` for structured answers and `cases` only for true piecewise answers. - Use `subject_to` for global constraints on one answer object, such as `x>0`, `a", "vector", "number"), + StructureGoldRow(r"2\hat{i} - 3\hat{j}", "vector", "number"), + StructureGoldRow( + r"\begin{pmatrix} 1 & 2 \\ 3 & 4 \end{pmatrix}", "matrix", "number" + ), + # --- multi_part --- + StructureGoldRow( + "3 m/s, east", + "multi_part", + "physical_quantity", + context=_PARTS_CTX, + gate_kind=False, + note="aggregate kind over heterogeneous parts", + ), + StructureGoldRow( + "x = 1; y = 2", + "multi_part", + "relation", + gate_kind=False, + note="aggregate kind over parts", + ), + # --- subject_to (atomic answer carrying a side condition) --- + StructureGoldRow( + "E = k/r, a < r < b", + "atomic", + "relation", + note="trailing constraint parsed as subject_to, answer stays atomic", + ), +) + + +# Adversarial pairs that MUST stay non-equivalent (precision floor). Each holds today and +# must keep holding after structure canonicalization lands. +ADVERSARIAL_DISTINCT: tuple[tuple[str, str, str], ...] = ( + ("{1, 2}", "(1, 2)", "set vs tuple: different structure"), + ("(1, 2)", "(2, 1)", "tuple is ordered"), + ("[2, 3]", "(2, 3)", "interval vs tuple: different structure"), + ("[2, 3]", "[2, 3)", "closed vs half-open interval"), + ("{1, 2}", "{1, 3}", "different set elements"), + ("(1, 2, 3)", "(1, 2)", "different cardinality"), +) diff --git a/tests/prkit/semantics/test_protocol_comparison.py b/tests/prkit/semantics/test_protocol_comparison.py index a29e8dd..18d4c08 100644 --- a/tests/prkit/semantics/test_protocol_comparison.py +++ b/tests/prkit/semantics/test_protocol_comparison.py @@ -1101,9 +1101,11 @@ def test_protocol_subject_to_records_require_matching_constraints() -> None: assert "subject_to_count_mismatch" in result.diagnostics -def test_protocol_multi_part_respects_per_part_order() -> None: +def test_protocol_multi_part_respects_ordered_order() -> None: + # Order-sensitivity for multi-part answers is provided by the sound ORDERED path; the + # old per_part positional fallback (when labels did not align) is retired to TBD. context = { - "ordering": "per_part", + "ordering": "ordered", "question_unit_policy": "optional_if_question_fixed_unit", "question_unit": "T", } @@ -1863,7 +1865,7 @@ def test_protocol_matrix_records_are_rehydrated_from_text() -> None: "object_kind": "expression", "structure": "matrix", "canonical_text": ( - "In the x,y coordinate system shown, the tensor is " "[[2*a, 0], [0, 2*b]]." + "In the x,y coordinate system shown, the tensor is [[2*a, 0], [0, 2*b]]." ), "shape": [2, 2], "coordinate_frame": "x,y as shown; origin at center", @@ -1879,7 +1881,10 @@ def test_protocol_matrix_records_are_rehydrated_from_text() -> None: result = compare_protocol_answers(pred, ref) - assert result.equivalent is True + # Matrix/tensor comparison is deferred (cells are non-atomic rows): gated to TBD this + # pass rather than returning an uncertified verdict. (Rehydration still runs.) + assert result.equivalent is False + assert result.comparison_mode == "not_implemented" def test_protocol_matrix_rehydration_prefers_canonical_text_over_raw_latex() -> None: @@ -1899,7 +1904,9 @@ def test_protocol_matrix_rehydration_prefers_canonical_text_over_raw_latex() -> result = compare_protocol_answers(pred, ref) - assert result.equivalent is True + # Matrix comparison is deferred (non-atomic cells) → TBD this pass. + assert result.equivalent is False + assert result.comparison_mode == "not_implemented" def test_protocol_near_zero_opposite_signed_quantities_do_not_match() -> None: @@ -2357,3 +2364,196 @@ def test_protocol_relation_compact_product_with_subscript_is_expanded() -> None: pred = "V = NmV_r/(M + Nm)" ref = "V = N m V_r/(M + N m)" assert compare_protocol_answers(_relation(pred), _relation(ref)).equivalent is True + + +# --------------------------------------------------------------------------- +# Symbol-domain assumptions, de-radicalization, and numeric identity testing. +# +# These three composed levers raise recall by deciding equivalence over the +# answers' intended *real* domain. The discipline (METHODOLOGY.md) is that they +# must not erode precision, so the reject battery below is the contract: every +# domain-sensitive identity must stay non-equivalent under generic reals and +# become equivalent only when the domain is declared. The reject set runs first +# (precision proof); the accept set documents the recall wins. +# --------------------------------------------------------------------------- + + +def _expression(canonical_text: str) -> dict[str, str]: + return {"object_kind": "expression", "canonical_text": canonical_text} + + +def _assumption_context(assumption: str, *symbols: str) -> dict[str, object]: + return { + "symbol_assumptions": [{"symbol": s, "assumption": assumption} for s in symbols] + } + + +# --- Reject battery: domain-sensitive identities, generic real, must NOT match. --- + + +@pytest.mark.parametrize( + "pred, ref", + [ + # sqrt(x**2) == x only for x >= 0; over generic reals it is |x|. + ("sqrt(x**2)", "x"), + # log(x**2) == 2*log(x) only for x > 0; over reals it is 2*log|x|. + ("log(x**2)", "2*log(x)"), + # sqrt(a*b) == sqrt(a)*sqrt(b) only for a, b >= 0 (differs at a, b < 0). + ("sqrt(a*b)", "sqrt(a)*sqrt(b)"), + # log(a*b) == log(a) + log(b) only for a, b > 0. + ("log(a*b)", "log(a) + log(b)"), + # A global sign flip under a shared radical is never an identity. + ("sqrt(x)", "-sqrt(x)"), + ], +) +def test_protocol_expression_domain_sensitive_pairs_reject_under_generic_real( + pred: str, ref: str +) -> None: + result = compare_protocol_answers(_expression(pred), _expression(ref)) + assert result.equivalent is False + + +def test_protocol_expression_pit_rejects_near_miss_that_agrees_near_zero() -> None: + # sin(x) ~ x near 0 but differs generically: wide-range sampling rejects it. + assert ( + compare_protocol_answers(_expression("sin(x)"), _expression("x")).equivalent + is False + ) + + +def test_protocol_expression_pit_rejects_distinct_powers() -> None: + assert ( + compare_protocol_answers(_expression("x**2"), _expression("x**3")).equivalent + is False + ) + + +def test_protocol_relation_deradicalization_gated_off_without_declaration() -> None: + # c = sqrt(E/m) is the c >= 0 branch only; without a nonnegative declaration it is + # NOT the same constraint as E = m c^2 (which admits c < 0), so squaring is withheld. + result = compare_protocol_answers( + _relation("E = m*c**2"), _relation("c = sqrt(E/m)") + ) + assert result.equivalent is False + + +def test_protocol_expression_complex_marker_withholds_realness_default() -> None: + # Without an imaginary marker, realness makes sqrt(x**2) == Abs(x). + assert ( + compare_protocol_answers( + _expression("sqrt(x**2)"), _expression("Abs(x)") + ).equivalent + is True + ) + # A standalone imaginary unit keeps the symbols complex, where the identity fails. + assert ( + compare_protocol_answers( + _expression("I*sqrt(x**2)"), _expression("I*Abs(x)") + ).equivalent + is False + ) + + +# --- Existing documented rejects still reject (precision regression guard). --- + + +@pytest.mark.parametrize( + "pred, ref", + [ + ("x = 0", "x*y = 0"), + ("x = 1", "x**2 = 1"), + ("F = m*a", "F = m/a"), + ], +) +def test_protocol_relation_documented_rejects_unchanged(pred: str, ref: str) -> None: + assert compare_protocol_answers(_relation(pred), _relation(ref)).equivalent is False + + +# --- Accept set: recall wins, sound over the declared (or real) domain. --- + + +def test_protocol_expression_sqrt_of_square_equals_abs_over_reals() -> None: + # Realness alone (derived) is enough: sqrt(x**2) == |x| for real x. + assert ( + compare_protocol_answers( + _expression("sqrt(x**2)"), _expression("Abs(x)") + ).equivalent + is True + ) + + +@pytest.mark.parametrize( + "pred, ref, domain, symbols", + [ + ("sqrt(a*b)", "sqrt(a)*sqrt(b)", "nonnegative", ("a", "b")), + ("sqrt(x**2)", "x", "nonnegative", ("x",)), + ("log(x**2)", "2*log(x)", "positive", ("x",)), + ("log(a*b)", "log(a) + log(b)", "positive", ("a", "b")), + ("atan(x) + atan(1/x)", "pi/2", "positive", ("x",)), + ], +) +def test_protocol_expression_domain_sensitive_pairs_accept_when_declared( + pred: str, ref: str, domain: str, symbols: tuple[str, ...] +) -> None: + context = _assumption_context(domain, *symbols) + result = compare_protocol_answers( + _expression(pred), _expression(ref), context=context + ) + assert result.equivalent is True + + +@pytest.mark.parametrize( + "pred, ref, symbols", + [ + ("E = m*c**2", "c = sqrt(E/m)", ("c", "E", "m")), + ("v**2 = u**2 + 2*a*s", "v = sqrt(u**2 + 2*a*s)", ("v", "u", "a", "s")), + ], +) +def test_protocol_relation_deradicalization_accepts_when_declared_nonnegative( + pred: str, ref: str, symbols: tuple[str, ...] +) -> None: + context = _assumption_context("positive", *symbols) + result = compare_protocol_answers(_relation(pred), _relation(ref), context=context) + assert result.equivalent is True + assert result.comparison_mode == "relation" + + +# --- Unit-level contract for the assumption map and numeric identity test. --- + + +def test_derive_symbol_assumptions_is_realness_only() -> None: + from prkit.semantics.comparison.semantics import _derive_symbol_assumptions + + # Realness is derived for every symbol; positivity/nonnegativity never is. + assert _derive_symbol_assumptions("sqrt(a*b)", "sqrt(a)*sqrt(b)") == { + "a": {"real": True}, + "b": {"real": True}, + } + # An imaginary marker withholds the realness default entirely. + assert _derive_symbol_assumptions("sqrt(a) + I", "sqrt(a)") == {} + + +def test_build_symbol_assumption_map_declared_overrides_derived() -> None: + from prkit.semantics.comparison.semantics import build_symbol_assumption_map + + context = coerce_question_semantics(_assumption_context("positive", "x")) + built = build_symbol_assumption_map("sqrt(x*y)", "sqrt(x)*sqrt(y)", context=context) + # Declared positivity wins for x; y falls back to the derived realness default. + assert built["x"] == {"positive": True} + assert built["y"] == {"real": True} + + +def test_numeric_identity_equivalent_rejection_is_exact() -> None: + from prkit.semantics.comparison.semantics import ( + _numeric_identity_equivalent, + parse_scalar_symbolic_expression, + ) + + left = parse_scalar_symbolic_expression("x + y") + right = parse_scalar_symbolic_expression("x*y") + # Distinct functions disagree at sampled points -> exact False. + assert _numeric_identity_equivalent(left, right, 1e-9) is False + + same_left = parse_scalar_symbolic_expression("(x + 1)**2") + same_right = parse_scalar_symbolic_expression("x**2 + 2*x + 1") + assert _numeric_identity_equivalent(same_left, same_right, 1e-9) is True diff --git a/tests/prkit/semantics/test_structure_canonicalization.py b/tests/prkit/semantics/test_structure_canonicalization.py new file mode 100644 index 0000000..caf9935 --- /dev/null +++ b/tests/prkit/semantics/test_structure_canonicalization.py @@ -0,0 +1,183 @@ +"""Structure canonicalization: degeneracy collapses, idempotence, precision guards. + +These assert the (degenerate, canonical) pairs collapse to equivalent and that the collapses +are idempotent and precision-safe (they never collapse a non-degenerate structure). +""" + +from __future__ import annotations + +import pytest + +from prkit.semantics import ( + coerce_protocol_answer, + coerce_question_semantics, + compare_protocol_answers, +) +from prkit.semantics.comparison.structure_canonicalization import canonicalize_structure +from prkit.semantics.schema.enums import AnswerStructure + +_CTX = coerce_question_semantics({}) + + +def _num(value: float) -> dict: + return { + "object_kind": "number", + "structure": "atomic", + "numeric_value": float(value), + "numeric_text": str(value), + "canonical_text": str(value), + } + + +def _true() -> dict: + return { + "object_kind": "boolean", + "structure": "atomic", + "boolean_value": True, + "canonical_text": "True", + } + + +def _one_tuple(value: float) -> dict: + return { + "object_kind": "number", + "structure": "tuple", + "children": [_num(value)], + "canonical_text": f"({value})", + } + + +def _one_set(value: float) -> dict: + return { + "object_kind": "number", + "structure": "set", + "children": [_num(value)], + "canonical_text": f"{{{value}}}", + } + + +def _one_vector(value: float) -> dict: + return { + "object_kind": "number", + "structure": "vector", + "shape": (1,), + "children": [_num(value)], + "canonical_text": f"<{value}>", + } + + +def _point_interval(value: float, *, open_left=False, open_right=False) -> dict: + return { + "object_kind": "number", + "structure": "interval", + "children": [_num(value), _num(value)], + "interval_open_left": open_left, + "interval_open_right": open_right, + "canonical_text": f"[{value}, {value}]", + } + + +def _single_case_piecewise(value: float, condition: dict) -> dict: + return { + "object_kind": "number", + "structure": "piecewise", + "canonical_text": str(value), + "cases": [{"expression": _num(value), "condition": condition}], + } + + +# --- collapses reduce to atomic and compare equivalent to the bare value --- + + +@pytest.mark.parametrize( + "degenerate", + [ + _one_tuple(5), + _one_set(5), + _one_vector(5), + _point_interval(5), + _single_case_piecewise(5, _true()), + _single_case_piecewise( + 5, + { + "object_kind": "qualitative_label", + "structure": "atomic", + "canonical_text": "otherwise", + }, + ), + ], + ids=[ + "1-tuple", + "1-set", + "1-vector", + "[a,a]", + "piecewise-True", + "piecewise-otherwise", + ], +) +def test_degeneracy_collapses_to_atomic(degenerate: dict) -> None: + canon = canonicalize_structure(coerce_protocol_answer(degenerate), context=_CTX) + assert canon.structure == AnswerStructure.ATOMIC + assert compare_protocol_answers(degenerate, _num(5)).equivalent is True + + +def test_collapse_is_idempotent() -> None: + for degenerate in ( + _one_tuple(7), + _point_interval(7), + _single_case_piecewise(7, _true()), + ): + once = canonicalize_structure(coerce_protocol_answer(degenerate), context=_CTX) + twice = canonicalize_structure(once, context=_CTX) + assert once == twice + + +# --- precision guards: non-degenerate structures must NOT collapse --- + + +def test_open_point_interval_does_not_collapse() -> None: + # (a, a) / [a, a) / (a, a] denote the empty set, not the point a — must not collapse. + for open_left, open_right in [(True, True), (True, False), (False, True)]: + canon = canonicalize_structure( + coerce_protocol_answer( + _point_interval(5, open_left=open_left, open_right=open_right) + ), + context=_CTX, + ) + assert canon.structure == AnswerStructure.INTERVAL + + +def test_distinct_endpoint_interval_does_not_collapse() -> None: + interval = { + "object_kind": "number", + "structure": "interval", + "children": [_num(2), _num(3)], + "interval_open_left": False, + "interval_open_right": False, + "canonical_text": "[2, 3]", + } + canon = canonicalize_structure(coerce_protocol_answer(interval), context=_CTX) + assert canon.structure == AnswerStructure.INTERVAL + + +def test_multi_element_collection_does_not_collapse() -> None: + two_tuple = { + "object_kind": "number", + "structure": "tuple", + "children": [_num(1), _num(2)], + "canonical_text": "(1, 2)", + } + canon = canonicalize_structure(coerce_protocol_answer(two_tuple), context=_CTX) + assert canon.structure == AnswerStructure.TUPLE + + +def test_one_part_multi_part_does_not_collapse() -> None: + # A 1-part multi_part may carry a part-structure denotation the contract enforces. + one_part = { + "object_kind": "number", + "structure": "multi_part", + "children": [_num(5)], + "canonical_text": "5", + } + canon = canonicalize_structure(coerce_protocol_answer(one_part), context=_CTX) + assert canon.structure == AnswerStructure.MULTI_PART diff --git a/tests/prkit/semantics/test_structure_decision.py b/tests/prkit/semantics/test_structure_decision.py new file mode 100644 index 0000000..f682f83 --- /dev/null +++ b/tests/prkit/semantics/test_structure_decision.py @@ -0,0 +1,93 @@ +"""Structure-decision eval: classification gates, confusion report, precision floor. + +This is the Workstream-0 foundation for the structure-decision work. It measures the +deterministic structure classifier (`normalize_physics_answer`) against a gold corpus and +locks the precision floor (answers that must stay distinct). The collapse-equivalence pairs +that canonicalization must satisfy live with that work (`test_structure_canonicalization`). +""" + +from __future__ import annotations + +import pytest + +from prkit.semantics import ( + coerce_question_semantics, + compare_protocol_answers, + normalize_physics_answer, +) +from prkit.semantics.schema.enums import AnswerStructure + +from .fixtures.structure_gold import ( + ADVERSARIAL_DISTINCT, + STRUCTURE_GOLD, +) + + +def _classify(row): + context = coerce_question_semantics(row.context or {}) + return normalize_physics_answer(row.answer, context=context) + + +@pytest.mark.parametrize( + "row", [r for r in STRUCTURE_GOLD if r.gate_structure], ids=lambda r: r.answer +) +def test_gold_structure_classification(row) -> None: + """Gated rows must classify to their expected structure (regression lock).""" + assert _classify(row).structure.value == row.expected_structure + + +@pytest.mark.parametrize( + "row", [r for r in STRUCTURE_GOLD if r.gate_kind], ids=lambda r: r.answer +) +def test_gold_object_kind_classification(row) -> None: + """Gated rows must classify to their expected object kind (regression lock).""" + assert _classify(row).object_kind.value == row.expected_object_kind + + +def test_structure_confusion_report(capsys) -> None: + """Report the full structure confusion matrix; assert no gated regressions. + + Off-diagonal (non-gated) cells are reported, not gated — recall on the harder rows is + visibility, not a build gate (per the methodology: structure recall is deferred). + """ + labels = [s.value for s in AnswerStructure] + confusion = {(a, b): 0 for a in labels for b in labels} + misses: list[str] = [] + for row in STRUCTURE_GOLD: + got = _classify(row).structure.value + confusion[(row.expected_structure, got)] += 1 + if got != row.expected_structure: + misses.append( + f"{row.answer!r}: expected {row.expected_structure}, got {got}" + ) + + lines = ["structure confusion (expected → got):"] + for a in labels: + row_counts = {b: confusion[(a, b)] for b in labels if confusion[(a, b)]} + if row_counts: + lines.append(f" {a:11} -> {row_counts}") + if misses: + lines.append("non-gated misclassifications (known gaps):") + lines.extend(f" {m}" for m in misses) + with capsys.disabled(): + print("\n".join(lines)) + + # Gated rows must never regress. + for row in STRUCTURE_GOLD: + if row.gate_structure: + assert _classify(row).structure.value == row.expected_structure + + +@pytest.mark.parametrize( + "pred, ref, why", ADVERSARIAL_DISTINCT, ids=[w for *_, w in ADVERSARIAL_DISTINCT] +) +def test_adversarial_distinct_stay_non_equivalent( + pred: str, ref: str, why: str +) -> None: + """Precision floor: structurally/elementwise distinct answers must not compare equal.""" + context = coerce_question_semantics({}) + pred_ans = normalize_physics_answer(pred, context=context) + ref_ans = normalize_physics_answer(ref, context=context) + assert ( + compare_protocol_answers(pred_ans, ref_ans, context=context).equivalent is False + ) diff --git a/tests/prkit/semantics/test_structure_gating.py b/tests/prkit/semantics/test_structure_gating.py new file mode 100644 index 0000000..9656dfe --- /dev/null +++ b/tests/prkit/semantics/test_structure_gating.py @@ -0,0 +1,201 @@ +"""Non-atomic comparison gating: only proven-sound accepts pass; the rest are TBD. + +The equivalence judgement runs for a non-atomic structure only when it provably reduces to +atomic-vs-atomic element comparisons. Everything else returns the ``not_implemented`` (TBD) +sentinel by default, or raises ``NotImplementedError`` under the strict toggle. +""" + +from __future__ import annotations + +import pytest + +from prkit.semantics import compare_protocol_answers +from prkit.semantics.comparison import engine + + +def _num(value, text=None): + return { + "object_kind": "number", + "structure": "atomic", + "numeric_value": float(value), + "numeric_text": text if text is not None else str(value), + "canonical_text": text if text is not None else str(value), + } + + +def _coll(structure, children, **extra): + return { + "object_kind": "number", + "structure": structure, + "children": children, + "canonical_text": "", + **extra, + } + + +def _vector(children, **extra): + return _coll("vector", children, shape=[len(children)], **extra) + + +# --- ordered: tuple / vector with atomic cells are enabled --- + + +def test_tuple_atomic_exact_equivalent(): + r = compare_protocol_answers( + _coll("tuple", [_num(1), _num(2)]), _coll("tuple", [_num(1), _num(2)]) + ) + assert r.equivalent is True and r.comparison_mode == "tuple" + + +def test_tuple_order_sensitive_real_reject(): + r = compare_protocol_answers( + _coll("tuple", [_num(1), _num(2)]), _coll("tuple", [_num(2), _num(1)]) + ) + assert r.equivalent is False and r.comparison_mode == "tuple" + + +def test_tuple_non_atomic_element_is_tbd(): + nested = _coll("tuple", [_coll("tuple", [_num(1), _num(2)]), _num(3)]) + r = compare_protocol_answers(nested, nested) + assert r.equivalent is False and r.comparison_mode == "not_implemented" + + +# --- unordered set: only exact multiset matches pass --- + + +def test_set_exact_match_equivalent(): + r = compare_protocol_answers( + _coll("set", [_num(1), _num(2)]), _coll("set", [_num(2), _num(1)]) + ) + assert r.equivalent is True and r.comparison_mode == "set" + + +def test_set_exact_numeric_equivalence_half_vs_decimal(): + # 1/2 and 0.5 are exactly equal numbers (no tolerance) — accepted. + r = compare_protocol_answers( + _coll("set", [_num(0), _num(0.5, "1/2")]), + _coll("set", [_num(0.5, "0.5"), _num(0)]), + ) + assert r.equivalent is True + + +def test_set_tolerance_fuzz_is_tbd(): + # The classic non-transitive-tolerance false positive must NOT pass — it is TBD now. + ctx = {"tolerance": 0.2} + r = compare_protocol_answers( + _coll("set", [_num(1.0), _num(1.0)]), + _coll("set", [_num(1.0), _num(1.1)]), + context=ctx, + ) + assert r.equivalent is False and r.comparison_mode == "not_implemented" + + +def test_set_inexact_elements_is_tbd(): + # Two-element sets (a 1-element set would correctly collapse to its atom): 9.8 vs 9.81 + # are not exactly equal, so the set is TBD rather than tolerance-matched. + r = compare_protocol_answers( + _coll("set", [_num(9.8), _num(5)]), _coll("set", [_num(9.81), _num(5)]) + ) + assert r.equivalent is False and r.comparison_mode == "not_implemented" + + +# --- matrix / tensor are deferred (non-atomic cells) --- + + +def test_matrix_is_tbd(): + matrix = _coll( + "matrix", + [_vector([_num(1), _num(2)]), _vector([_num(3), _num(4)])], + shape=[2, 2], + ) + r = compare_protocol_answers(matrix, matrix) + assert r.equivalent is False and r.comparison_mode == "not_implemented" + + +# --- vector frames --- + + +def test_vector_both_unset_frame_equivalent(): + r = compare_protocol_answers( + _vector([_num(1), _num(2), _num(3)]), _vector([_num(1), _num(2), _num(3)]) + ) + assert r.equivalent is True and r.comparison_mode == "vector" + + +def test_vector_one_sided_frame_is_tbd(): + r = compare_protocol_answers( + _vector([_num(1), _num(2)], coordinate_frame="x,y at center"), + _vector([_num(1), _num(2)]), + ) + assert r.equivalent is False and r.comparison_mode == "not_implemented" + + +def test_vector_incompatible_frames_real_reject(): + r = compare_protocol_answers( + _vector([_num(1), _num(2)], coordinate_frame="polar r,theta"), + _vector([_num(1), _num(2)], coordinate_frame="cartesian x,y"), + ) + assert r.equivalent is False and r.comparison_mode == "vector" + + +# --- per_part: aligned labels only --- + + +def test_per_part_aligned_labels_equivalent(): + ctx = {"ordering": "per_part", "required_parts": ("a", "b")} + pred = _coll( + "multi_part", + [ + {**_num(1), "part_label": "a"}, + {**_num(2), "part_label": "b"}, + ], + ) + ref = _coll( + "multi_part", + [ + {**_num(2), "part_label": "b"}, + {**_num(1), "part_label": "a"}, + ], + ) + r = compare_protocol_answers(pred, ref, context=ctx) + assert r.equivalent is True and r.comparison_mode == "multi_part" + + +def test_per_part_unaligned_labels_is_tbd(): + ctx = {"ordering": "per_part"} + pred = _coll("multi_part", [_num(1), _num(2)]) + ref = _coll("multi_part", [_num(1), _num(2)]) + r = compare_protocol_answers(pred, ref, context=ctx) + assert r.equivalent is False and r.comparison_mode == "not_implemented" + + +# --- contract reconciliation: a collapsed structure stays admitted --- + + +@pytest.mark.parametrize("policy", ["strict", "audited", "permissive"]) +def test_collapsed_structure_admitted_under_restricted_allowed_structures(policy): + # A 1-tuple collapses to ATOMIC; with allowed_structures=(tuple,) both the collapsed + # prediction and the atomic reference must stay admitted (no contract violation). + one_tuple = _coll("tuple", [_num(5)]) + atomic = _num(5) + ctx = {"allowed_structures": ["tuple"]} + result = compare_protocol_answers( + one_tuple, atomic, context=ctx, policy_mode=policy + ) + assert result.comparison_mode not in { + "contract_violation", + "reference_contract_violation", + } + assert result.equivalent is True + + +def test_strict_mode_raises(monkeypatch): + monkeypatch.setattr(engine, "STRICT_STRUCTURE_COMPARISON", True) + # 2-element rows stay non-atomic (a 1-element row would collapse to its atom). + matrix = _coll( + "matrix", + [_vector([_num(1), _num(2)]), _vector([_num(3), _num(4)])], + shape=[2, 2], + ) + with pytest.raises(NotImplementedError): + compare_protocol_answers(matrix, matrix) From f006e85f5846bf4b5f1824d06124c5ddb536f5f9 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Fri, 19 Jun 2026 14:17:04 -0400 Subject: [PATCH 08/28] Explain the per-structure signature columns in STRUCTURE.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spell out cardinality (was abbreviated 'Card.') and add a short explanation of what each signature column means — denotation (meaning, drives equivalence), cardinality (part count, drives length checks and degeneracy collapses), ordering, and surface evidence (textual cues, drives classification) — and why keeping denotation separate from surface evidence is the point. Co-Authored-By: Claude Opus 4.8 --- src/prkit/semantics/comparison/STRUCTURE.md | 25 +++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/prkit/semantics/comparison/STRUCTURE.md b/src/prkit/semantics/comparison/STRUCTURE.md index 7a79965..9564ec7 100644 --- a/src/prkit/semantics/comparison/STRUCTURE.md +++ b/src/prkit/semantics/comparison/STRUCTURE.md @@ -10,9 +10,30 @@ runs. ## 1. Per-structure signatures -Each structure is defined by `⟨denotation, cardinality, ordering, surface evidence⟩`. +Each structure is defined by `⟨denotation, cardinality, ordering, surface evidence⟩`. The +four columns answer four different questions — *what it means*, *how many parts*, *does order +matter*, and *how to recognize it*: -| Structure | Denotation | Card. | Ordering | Surface evidence | +- **Denotation** — what the structure *means* (the mathematical object it stands for), + independent of how it is written. This is the axis equivalence and canonicalization reason + about: two answers with the same denotation are the same answer regardless of surface — so + a degenerate wrapper may collapse to its content (a 1-element tuple *is* its scalar), while + a `set` and a `tuple` (unordered collection vs. ordered coordinate) must stay distinct. +- **Cardinality** — how many parts the structure holds (children, endpoints, cases, or + shape). A comparison requires the counts to match, and a degeneracy collapses to `atomic` + exactly when its cardinality drops to 1. +- **Ordering** — whether element order is semantically significant (positional, none, or + driven by `q.ordering`). +- **Surface evidence** — the textual cues used to *recognize* the structure from the raw + answer (brackets, braces, `\begin{cases}`, an `∞` token, …). It is distinct from + denotation (meaning) and can be ambiguous — a bare `(a, b)` looks like both a tuple and an + open interval — which is exactly what the §2 tie-break rules resolve. + +The design hinges on keeping these separate: classification reads **surface evidence** to +assign a structure, but equivalence judges the **denotation** — so the same denotation +written two different ways should canonicalize to one structure. + +| Structure | Denotation | Cardinality | Ordering | Surface evidence | |---|---|---|---|---| | `atomic` | one indivisible value | 1 | — | the default; target of every collapse | | `multi_part` | answers to several question-defined sub-questions | ≥1 | from `q.ordering` | `required_parts`/enumerated `(1)(2)`, `;`, newlines | From 8a3937f670ec095d6f4e5f2bf8a1fb67b5a5132e Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Fri, 19 Jun 2026 18:46:16 -0400 Subject: [PATCH 09/28] Add the objective semantics-build ecosystem: q_ref/q_prob, a_ref, a_pred, reference-free judgement Build the question/answer semantics the equivalence judgement consumes with the same precision discipline the engine enforces, and expose the three steps as independent capabilities. Reference creation: a staged builder produces q_ref + a_ref (problem + golden) and q_prob (problem only). The deterministic backbone (normalize_physics_answer + canonicalize_structure) is authoritative for structure/object_kind; advisory LLM calls run best-effort and only clean surfaces, fill question policy, and declare justified symbol assumptions, each cross-checked (round-trip, contract self-consistency, q_ref<->a_ref mutual consistency) with a provenance build report. Symbol assumptions are declared-not-derived and keyed by canonical post-alias tokens; numeric tolerance is relative (never absolute); allowed_* are widen-only. Replaces the single fused reference call (old body kept commented for review). Answer generation: problem-only isolated solve with a leakage guard, producing an LLM-structured record (a_pred_llm) and a deterministically-extracted one (a_pred_ext); it degrades to a_pred_ext alone when native structured output is unavailable. Equivalence judgement: add a symmetric reference-free compare_predictions(q_prob) that builds an explicit contract and requires agreement both directions, so neither prediction's printed precision nor kind/structure decides and no self-contract-violation arises; the reference-based path is unchanged. The verify/scoring facade gains an optional, duck-typed q_ref context so a caller can supply reference-built assumptions without the light facade importing the inference layer (backward compatible). The three steps are independent: the judgement core imports nothing from the build/generation layer at runtime, generation never calls the reference build, and each entry takes plain problem/semantics inputs. Methodology, the three-step ecosystem, and the engine-compatibility rules are documented in the comparison reference docs. Co-Authored-By: Claude Opus 4.8 --- src/prkit/scoring/partial_credit_scorer.py | 11 +- src/prkit/scoring/semantics_scorer.py | 11 +- src/prkit/semantics/__init__.py | 2 + src/prkit/semantics/comparison/EQUIVALENCE.md | 51 + src/prkit/semantics/comparison/METHODOLOGY.md | 184 ++++ src/prkit/semantics/comparison/STRUCTURE.md | 23 + src/prkit/semantics/comparison/__init__.py | 2 + src/prkit/semantics/comparison/engine.py | 148 ++- src/prkit/semantics/inference/__init__.py | 22 + src/prkit/semantics/inference/artifacts.py | 126 ++- src/prkit/semantics/inference/calls.py | 940 +++++++++++++++++- src/prkit/semantics/inference/prompts.py | 159 ++- .../semantics/inference/semantics_build.py | 558 +++++++++++ .../semantics/inference/strict_models.py | 62 ++ .../normalization/question_inference.py | 1 + src/prkit/verify/__init__.py | 52 +- .../semantics/test_compare_predictions.py | 252 +++++ .../prkit/semantics/test_inference_prompts.py | 138 ++- .../test_prediction_isolated_build.py | 199 ++++ tests/prkit/semantics/test_semantics_build.py | 274 +++++ tests/prkit/semantics/test_staged_build.py | 172 ++++ tests/prkit/verify/test_verify.py | 81 ++ 22 files changed, 3390 insertions(+), 78 deletions(-) create mode 100644 src/prkit/semantics/inference/semantics_build.py create mode 100644 tests/prkit/semantics/test_compare_predictions.py create mode 100644 tests/prkit/semantics/test_prediction_isolated_build.py create mode 100644 tests/prkit/semantics/test_semantics_build.py create mode 100644 tests/prkit/semantics/test_staged_build.py diff --git a/src/prkit/scoring/partial_credit_scorer.py b/src/prkit/scoring/partial_credit_scorer.py index 36eb24b..b0dc9bd 100644 --- a/src/prkit/scoring/partial_credit_scorer.py +++ b/src/prkit/scoring/partial_credit_scorer.py @@ -51,11 +51,20 @@ def __str__(self) -> str: def _coerce_context( context: PhysicsQuestionSemantics | dict[str, Any] | None, ) -> PhysicsQuestionSemantics | None: - """Coerce an optional context into validated ``PhysicsQuestionSemantics``.""" + """Coerce an optional context into validated ``PhysicsQuestionSemantics``. + + A reference-built artifact (anything exposing ``.question_semantics``, e.g. a + ``ReferenceSemanticsArtifact``) is unwrapped to its ``q_ref`` — duck-typed so the scorer + does not depend on the heavy inference artifact type — mirroring ``SemanticsScorer`` so + both scorers accept the same context kinds. + """ if context is None: return None if isinstance(context, PhysicsQuestionSemantics): return context + question_semantics = getattr(context, "question_semantics", None) + if isinstance(question_semantics, PhysicsQuestionSemantics): + return question_semantics return PhysicsQuestionSemantics.model_validate(context) diff --git a/src/prkit/scoring/semantics_scorer.py b/src/prkit/scoring/semantics_scorer.py index 980e8b7..26680d6 100644 --- a/src/prkit/scoring/semantics_scorer.py +++ b/src/prkit/scoring/semantics_scorer.py @@ -47,11 +47,20 @@ def _coerce_context( context: PhysicsQuestionSemantics | dict[str, Any] | None, ) -> PhysicsQuestionSemantics | None: - """Coerce an optional context into validated ``PhysicsQuestionSemantics``.""" + """Coerce an optional context into validated ``PhysicsQuestionSemantics``. + + A reference-built artifact (anything exposing ``.question_semantics``, e.g. a + ``ReferenceSemanticsArtifact``) is unwrapped to its ``q_ref`` — duck-typed so the + scorer does not depend on the heavy inference artifact type. A + ``PhysicsQuestionSemantics`` passes through; a mapping is validated. + """ if context is None: return None if isinstance(context, PhysicsQuestionSemantics): return context + question_semantics = getattr(context, "question_semantics", None) + if isinstance(question_semantics, PhysicsQuestionSemantics): + return question_semantics return PhysicsQuestionSemantics.model_validate(context) diff --git a/src/prkit/semantics/__init__.py b/src/prkit/semantics/__init__.py index a381283..5fac497 100644 --- a/src/prkit/semantics/__init__.py +++ b/src/prkit/semantics/__init__.py @@ -18,6 +18,7 @@ coerce_protocol_answer, coerce_question_semantics, compare_physics_answers, + compare_predictions, compare_protocol_answers, compare_protocol_answers_legacy, validate_answer_against_contract, @@ -116,6 +117,7 @@ "SemanticsGeneratorInfo", "SemanticsProblemRecord", "compare_physics_answers", + "compare_predictions", "compare_protocol_answers", "compare_protocol_answers_legacy", "compare_saved_semantics", diff --git a/src/prkit/semantics/comparison/EQUIVALENCE.md b/src/prkit/semantics/comparison/EQUIVALENCE.md index 3fc590d..403f046 100644 --- a/src/prkit/semantics/comparison/EQUIVALENCE.md +++ b/src/prkit/semantics/comparison/EQUIVALENCE.md @@ -30,6 +30,25 @@ ordering policy, and the numeric `tolerance`. The judgement is *under* `q` — e required unit lets a bare `5` be read as `5 m/s²`, and a `symbol_assumptions` declaration (`c`, `E`, `m` positive) lets `c = √(E/m)` be read as `E = m c²` (§7.3–7.4). +### 1.1 Where `q` comes from, and the two judgement modes + +The same predicate `Eq(·, · ; q)` serves two modes, which differ only in **which `q`** is +supplied as `context` (see METHODOLOGY.md §6 for how each is built): + +| Mode | Call | `q` | Built from | +|---|---|---|---| +| **reference-based** (correctness) | `compare_protocol_answers(a_pred, a_ref, context=q_ref)` | `q_ref` | problem **+ golden** | +| **reference-free** (clustering) | `compare_predictions(a_pred_i, a_pred_j, context=q_prob)` | `q_prob` | **problem only** (answer-blind) | + +In the reference-based mode the second argument is the gold answer `a_ref`, and `q_ref` is +co-constructed with it so the contract admits exactly that answer's kind/structure. In the +reference-free mode **neither side is gold**, so a symmetric entry point +(`compare_predictions`) is used instead of `compare_protocol_answers` — it derives an +explicit `q_prob` contract rather than inferring the expected kind/structure from one of the +two predictions (see the reference-free subsection after §10). `q_prob` agrees with `q_ref` +on every problem-only-determinable field but declares only `allowed_*` and policy fields, not +a realized answer. + --- ## 2. Pipeline overview @@ -340,6 +359,17 @@ The reference sets the bar, so the relation is asymmetric (see the `9.8`/`9.81` `0.333`/`1/3` pairs in §7.1). For the exact half-quantum boundary, read `_difference_is_strictly_within_half_quantum`. +**`q.tolerance` is relative, not absolute.** Step 1 reads it as a *relative* tolerance: +`numbers_close` compares the difference against `tolerance·max(|pred|, |ref|)`, falling back +to an absolute comparison **only at the zero boundary** (when either value is exactly `0.0`; +the both-zero case is already an exact-equality short-circuit). The N-significant-figures +agreement (steps 2–4) is a **separate** path keyed off `a_ref`'s *printed* precision — it is +**not** driven by `q.tolerance`. So the build (METHODOLOGY.md §6) sets `q.tolerance` +relative for an explicit relative instruction ("within 1%" ⇒ `0.01`) and **never converts it +to absolute**; for "N sig figs" / displayed precision it **preserves `a_ref`'s printed +precision** in the numeric surface and lets steps 2–4 do the work, rather than tightening +`q.tolerance`. The default `DEFAULT_NUMERIC_TOLERANCE` is likewise relative. + ### 10.1 Numeric identity testing (`_numeric_identity_equivalent`) `simplify` is incomplete (no canonical form exists for transcendental/nested-radical @@ -372,6 +402,27 @@ declaration (`SymbolAssumption`: `real`/`nonzero`/`nonnegative`/`positive`/`comp must be declared. Realness alone is precision-safe and unlocks real-only identities (`√(x²) = |x|`). +**Tokens must be canonical (post-alias).** `build_symbol_assumption_map` +(`context_symbol_assumption_map`) keys assumptions by the **canonical** token — the one that +survives `q.symbol_aliases` rewriting (§3, `_canonicalize_symbol_alias_surfaces`). An +assumption keyed by a raw *alias* token never matches a parsed symbol and is silently +dropped. So a `q` author (and the staged build, METHODOLOGY.md §6) must emit every +`symbol_assumptions.symbol` as a canonical token, resolved through the alias map, and must +not key an assumption on an alias *source*. + +### 10.3 Reference-free numeric criterion (`compare_predictions`) + +In the reference-free mode (§1.1) neither side is gold, so the asymmetric +reference-precision rule above (steps 2–4) is **not** applicable — there is no reference +whose printed precision should "set the bar." `compare_predictions(a_i, a_j, context=q_prob)` +therefore uses a **symmetric** numeric criterion: relative `q_prob.tolerance` only, with +neither side's printed precision tightening the threshold. It also builds an explicit +`q_prob`-derived contract (expected kind/structure from `q_prob.allowed_*`, permissive when +unconstrained) and passes it via `contract=`, so the engine does not infer the expected +kind/structure from one of the two predictions and cannot raise +`reference_contract_violation` against a self-derived contract. Everything else — structure +routing, the per-kind criteria, the bridges — is already symmetric and is reused unchanged. + --- ## 11. `comparison_mode` catalogue diff --git a/src/prkit/semantics/comparison/METHODOLOGY.md b/src/prkit/semantics/comparison/METHODOLOGY.md index c85734e..befca0b 100644 --- a/src/prkit/semantics/comparison/METHODOLOGY.md +++ b/src/prkit/semantics/comparison/METHODOLOGY.md @@ -171,3 +171,187 @@ inequalities, by design. assertion in `tests/prkit/verify/test_verify.py` when it changes a verdict). 5. **Confirm the hot path** isn't materially slowed and the full suite stays green. 6. **Note the residual limit** so the next gap is discoverable. + +## 6. Build-time methodology: constructing `q` and `a` for the judgement + +Everything above is about *running* `Eq(a_pred, a_ref ; q)`. This section is the matching +discipline for *building* the records the judgement consumes — the question semantics `q` +and the answer-semantics records `a` — so that they are constructed with the **same +precision authority** the engine enforces, not by ad-hoc heuristics. The build is offline +and one-time per data point; its deterministic core lives in +[`../inference/semantics_build.py`](../inference/semantics_build.py) and is wrapped by the +staged calls in [`../inference/calls.py`](../inference/calls.py). + +### Vocabulary (build outputs) + +The build produces distinct, named records. These names are the shared vocabulary across +the docs, the artifact types, and the builder signatures. + +| Name | Built from | Role | +|---|---|---| +| `q_ref` | problem **+ golden** | the contract for reference-based `Eq(a_pred, a_ref ; q_ref)` | +| `q_prob` | **problem only** (answer-blind) | the contract for reference-free `Eq(a_pred_i, a_pred_j ; q_prob)` | +| `a_ref` | golden answer under `q_ref` | the gold answer record the contract judges against | +| `a_pred_llm` | LLM structured output during solve | a prediction record used directly | +| `a_pred_ext` | plain text → deterministic extraction | a prediction record (same authority as `a_ref`); also the A/B baseline | + +`q_ref` and `a_ref` are **co-constructed in one pass and never built independently**: the +build returns the *pair* and validates them for mutual consistency +(`reference_pair_consistency`) — `q_ref`'s allowed sets must admit `a_ref`'s realized +kind/structure, any shared `target_variable` must agree, and every assumption token must be +canonical. That mutual check is what guarantees the contract actually describes the gold +answer it will judge. + +### The three-step semantics ecosystem (and where native structured output matters) + +The build outputs feed a three-step pipeline: + +1. **Reference creation** — `(problem, golden) → (q_ref, a_ref)`. +2. **Answer generation** — `problem → a_pred` (one or both of `a_pred_ext` / `a_pred_llm`). +3. **Equivalence judgement** — `Eq(a_pred, a_ref ; q_ref)` (reference-based), or + `Eq(a_pred_i, a_pred_j ; q_prob)` (reference-free clustering). + +Native provider-enforced structured output is a **Step-2 output-form concern only**: + +- **Step 1 is unaffected.** Its advisory LLM calls run *best-effort* (native when the + provider supports it, otherwise plain text parsed back), so a provider lacking native + structured output still yields a full `(q_ref, a_ref)`. Lacking it is a normal route, not a + defect — it does not set `review_required` (only a genuine cross-check failure does). +- **Step 2 yields one form instead of two.** With native structured output the solve returns + `a_pred_llm` (and the `a_pred_ext` disagreement audit); without it there is **one route** — + the output is plain text and `a_pred_ext = canonicalize_structure(normalize_physics_answer(...))` + is the deterministic extraction. Never a failure; just one record instead of two. +- **Step 3 is provenance-agnostic.** The judgement consumes a `PhysicsAnswerSemantics` + regardless of whether it came from `a_pred_llm` or `a_pred_ext` — both are simply + "generated answer semantics." *Which* form a caller feeds in is out of this toolkit's scope. + +**The three steps are independent; the codebase must keep them so.** PRKit exposes each step +as a standalone capability for users and downstream applications to invoke à la carte — judge +with their own references and predictions, build only references, or only extract answer +semantics. **No step may depend on another inside the toolkit.** Concretely: the judgement +core (`prkit.semantics.comparison`, `prkit.verify`, `prkit.scoring`) imports **nothing** from +the build/generation layer (`prkit.semantics.inference`) at runtime — `verify(...)` accepts a +`q_ref` by *duck-typing* `.question_semantics` (a `TYPE_CHECKING`-only annotation), so it never +pulls in the inference layer; generation never calls the reference build; and every step's +entry point takes plain `problem` / `PhysicsAnswerSemantics` / `PhysicsQuestionSemantics` +inputs rather than requiring another step's output. A new feature must not introduce a runtime +import or a mandatory call from one step into another. + +### Deterministic authority vs. LLM advisory + +The build mirrors the engine's authority discipline (§3), lifted to construction time: + +> **The deterministic pipeline is authoritative for the contract and the gold record; the +> LLM is advisory.** `normalize_physics_answer` + `canonicalize_structure` decide +> `structure`/`object_kind` symmetrically (the *same* helpers, so `a_ref` and `a_pred_ext` +> classify identically). The LLM may *clean a messy surface*, *declare* a domain/policy +> field, and *flag* a disagreement — it never overrides the deterministic classification. + +There is **no "fallback to the LLM draft on error"** — the exact analogue of the engine's +"no rescue branch" rule. An LLM edit that fails a cross-check is simply *not adopted*, +because the deterministic value already stood; the inconsistency is recorded as a flag, not +silently reconciled. `a_pred_llm` is the one deliberate exception (an LLM-structured +prediction the user wants for direct use and head-to-head comparison); its risk is contained +by the §B4 disagreement flag against `a_pred_ext`, never by reconciliation. + +Multiple focused LLM calls are expected (surface cleanup, then question policy, then symbol +assumptions), each schema-strict with structure/kind **pinned** and each individually +cross-checked. This is decomposition for accuracy — **not** N-sample majority voting, which +would be a statistical patch rather than a methodological one. + +### Declared, not derived — at build time + +§4's rule stands unchanged at build time: domain positivity/nonnegativity is only ever a +**justified declaration**, never a heuristic guess. Dimension-priors are **not a source** +(they over-constrain signed quantities, and the `common.py` consumer is live). On a genuine +conflict between sources, the build declares the **least-restrictive sound** assumption — +asserting an unjustified one would manufacture false accepts, exactly the failure §4 guards +against on the engine side. + +### `symbol_assumptions` — source precedence and the canonical-token requirement + +Assumptions are synthesized with a provenance-tagged precedence +(`assumptions_from_subject_to` → `merge_symbol_assumptions`): + +| Precedence | Source | Rule | +|---|---|---| +| **A (authoritative)** | `subject_to` / problem-text constraints | a logical consequence of an explicit constraint — `x>0`→`positive`, `x>=0`→`nonnegative`, `x!=0`→`nonzero`, `x∈ℝ`→`real` (the `SymbolAssumption` lattice). `q_ref` may use the golden's `subject_to`; `q_prob` uses only problem-text constraints. | +| **B (advisory)** | LLM declaration **with justification** | adopted only when consistent with (A) or strictly refining it; cross-checked against (A). On conflict, declare the least-restrictive sound assumption and flag. | +| — | dimension-priors, symbol-name whitelists, surface heuristics | **never a source** (precision hazard — §4). | + +Where two sound constraints touch the same symbol, they are combined by intersecting the +denoted real domains (`meet_assumptions`: `x!=0` and `x>=0` together ⇒ `x>0`), so combining +sound sources stays sound. + +**Canonical-token requirement (compat fix #2).** The engine looks up assumptions by the +**canonical (post-alias) token** — `context_symbol_assumption_map` keys by the token that +survives alias rewriting, so an assumption keyed by a raw *alias* token is silently dropped +at parse time. The build therefore resolves every assumption symbol through the question's +alias map *before* emitting it (`resolve_to_canonical`), and a cross-check +(`alias_source_violations`) asserts that no emitted `symbol_assumptions.symbol` is an alias +*source*. This couples assumption synthesis to the alias map the same build pass produces. + +### `tolerance` — relative, never absolute (compat fix #1) + +The engine reads `q.tolerance` as a **relative** tolerance (`numbers_close` = +`tol·max(|a|,|b|)`; absolute only at the zero boundary, i.e. when either side is exactly +zero), and **N-significant-figures is a +separate path** keyed off the reference's *printed* precision +(`numbers_match_with_reference_precision`, EQUIVALENCE.md §10). So the build +(`infer_answer_tolerance` / `parse_relative_tolerance_instruction`): + +- maps an explicit **relative** instruction ("within 1%", "±2%") to a relative + `q.tolerance` (`0.01`, `0.02`) and **never converts it to absolute**; +- for "N sig figs" / displayed-precision phrasing, **preserves `a_ref`'s printed precision** + in its numeric surface rather than tightening `q.tolerance` — letting the engine's + reference-precision logic do the work. (`_significant_figures` is used only to *validate* + that the preserved precision matches the stated one.) +- otherwise keeps the relative `DEFAULT_NUMERIC_TOLERANCE`. + +### `allowed_*` — a justified, widen-only precision lever (compat fix #3) + +`allowed_object_kinds` / `allowed_structures` express **question-level admissibility**, not +"what the gold happens to be." Two facts shape how the build populates them: + +- they are **hard violating-gates** in `validate_answer_against_contract` (no bridge rescue, + unlike an `expected_*` mismatch which preserves bridges), so an over-narrow set turns a + cross-kind-equivalent or degenerate-collapsed prediction into a false `contract_violation` + — converting a recall win into a false reject; +- so the build is **permissive by default** and `reconcile_allowed_sets` only ever *widens*: + it admits `a_ref`'s realized kind/structure **and the closure** under the contract's + enabled cross-kind bridges and the structure collapses (`_STRUCTURES_COLLAPSIBLE_TO_ATOMIC` + ⇒ also admit `ATOMIC`). See STRUCTURE.md §4. + +Narrowing is a precision choice exactly like an equivalence criterion: it is made only on +explicit question evidence (e.g. MCQ ⇒ `choice`), never by default. Over-narrowing is the +build-time analogue of an unjustified relaxation — it silently destroys recall. + +### Cross-checks are validation, not rescue + +The build's cross-checks **validate the authority** rather than rescuing a failed attempt +(the §3 distinction, restated): a round-trip (re-normalize `canonical_text` ⇒ same +structure/kind/numeric), contract self-consistency (`build_evaluation_contract` ⇒ no +self-violation), and `q_ref`↔`a_ref` mutual consistency (`reference_pair_consistency`). On +failure the build does **not** adopt the inconsistent LLM edit — the deterministic value +stands — and flags for review. A cross-check is never the thing that *enables* an accept; +it is the thing that can *veto* an advisory edit. + +### Build report and provenance + +Every build attaches an additive `SemanticsBuildReport` (`build_report` on +`ReferenceSemanticsArtifact` / `ProblemSemanticsArtifact`) so the result is auditable and +reproducible: + +- `build_method` (e.g. `reference_3call` / `problem_3call`), `temperature` (0 for + reproducibility); +- `field_provenance` — per-field source: `deterministic` / `subject_to` / `llm_declared` / + `default`; +- `assumption_provenance` — per-symbol `SymbolAssumptionProvenance` + (canonical `symbol`, adopted `assumption`, `source`, LLM `justification`); +- `flags` (disagreements, advisory-strengthening, cross-check reverts), + `cross_checks_passed`, `review_required`. + +The advisory stages **degrade gracefully**: if a provider lacks native structured output, +the deterministic backbone is authoritative and the advisory failure is recorded as +`review_required` rather than raising. The cache key is +`(problem_id, model, prompt_version, build_method)`. diff --git a/src/prkit/semantics/comparison/STRUCTURE.md b/src/prkit/semantics/comparison/STRUCTURE.md index 9564ec7..61fb2f8 100644 --- a/src/prkit/semantics/comparison/STRUCTURE.md +++ b/src/prkit/semantics/comparison/STRUCTURE.md @@ -73,6 +73,16 @@ rewrites that can reach atomic-vs-atomic but never equate distinct answers: It deliberately does **not** collapse `multi_part` (a one-part answer may carry a part-structure the contract enforces) and does **not** reconcile shapes. +**Safe to apply at build time too.** `canonicalize_structure` is **idempotent** +(re-applying it to its own output is a no-op — every rewrite reaches a fixed point) and +**context-insensitive** (it reads only the record, never `q`). So the staged builder +(METHODOLOGY.md §6) may apply it when pinning `a_ref` / `a_pred_ext` / `a_pred_llm` even +though the engine applies it again inside `_repair_answer_for_comparison`: the second +application changes nothing. This is what lets the build and the engine share one +denotational classifier without a double-collapse hazard, and is why `a_ref` and +`a_pred_ext` — normalized by the *same* helper — classify identically (the +`structure_mismatch` defense). + ## 4. Comparison gating — only proven-sound accepts pass The equivalence judgement runs for a non-atomic structure only through a comparator path @@ -82,6 +92,19 @@ rather than returning a silent verdict. See the per-comparator audit and gating the implementation plan and the batteries in `tests/prkit/semantics/`. +**`allowed_structures` from the build must admit the collapse closure.** When a built `q` +(METHODOLOGY.md §6) populates `allowed_structures`, the set is a **hard violating-gate** in +`validate_answer_against_contract` — there is no bridge or collapse rescue for a structure +the contract excludes. So if the contract admits a structure that `canonicalize_structure` +can reduce (anything in `_STRUCTURES_COLLAPSIBLE_TO_ATOMIC`), it must **also admit +`ATOMIC`**: otherwise a legitimate prediction that the §3 canonicalizer collapses to a +scalar (a 1-element tuple, a `[a,a]` point-interval, a trivial single-case piecewise) would +hit `contract_violation` for being the very atom it was reduced to. The builder's +`reconcile_allowed_sets` enforces this widen-only closure (it admits `a_ref`'s realized +structure, and adds `ATOMIC` whenever a collapsible structure is admitted), so it never +narrows recall away — see METHODOLOGY.md §6, "`allowed_*` — a justified, widen-only +precision lever." + ## 5. Deferred (named gaps — not silently dropped) - interval ↔ 2-clause-conjunction reconciliation (open/closed-boundary precision hazard); diff --git a/src/prkit/semantics/comparison/__init__.py b/src/prkit/semantics/comparison/__init__.py index ed5e79d..62aae5d 100644 --- a/src/prkit/semantics/comparison/__init__.py +++ b/src/prkit/semantics/comparison/__init__.py @@ -24,6 +24,7 @@ ) from .engine import ( compare_physics_answers, + compare_predictions, compare_protocol_answers, compare_protocol_answers_legacy, ) @@ -35,6 +36,7 @@ "coerce_policy_mode", "coerce_question_semantics", "compare_physics_answers", + "compare_predictions", "compare_protocol_answers", "compare_protocol_answers_legacy", "validate_answer_against_contract", diff --git a/src/prkit/semantics/comparison/engine.py b/src/prkit/semantics/comparison/engine.py index 9cdd868..4fdf4fc 100644 --- a/src/prkit/semantics/comparison/engine.py +++ b/src/prkit/semantics/comparison/engine.py @@ -10,6 +10,7 @@ AnswerComparison, AnswerObjectKind, AnswerStructure, + BridgeTier, ComparisonPolicyMode, ContractValidationStatus, OrderingPolicy, @@ -18,7 +19,11 @@ PhysicsQuestionSemantics, QuestionSymbolicMode, ) -from .bridge_registry import bridge_enabled_for_policy, bridge_spec_for +from .bridge_registry import ( + BRIDGE_REGISTRY, + bridge_enabled_for_policy, + bridge_spec_for, +) from .coercion import ( coerce_evaluation_contract, coerce_protocol_answer, @@ -316,6 +321,147 @@ def compare_protocol_answers_legacy( ) +def compare_predictions( + a_i: PhysicsAnswerSemantics | dict, + a_j: PhysicsAnswerSemantics | dict, + *, + context: PhysicsQuestionSemantics | dict | None = None, + policy_mode: ComparisonPolicyMode | str | None = None, +) -> AnswerComparison: + """Symmetric reference-free equivalence ``Eq(a_i, a_j ; q_prob)`` for clustering. + + Unlike :func:`compare_protocol_answers` — whose contract is co-built with a *gold* + second argument — neither argument here is a reference, so deriving the expected + kind/structure or the numeric precision bar from one of the two predictions is + unsound (it makes the verdict depend on argument order and lets a prediction reject + the very contract it defined). This entry point removes both hazards: + + - **Explicit ``q_prob`` contract.** A single contract is built from ``context`` + (``q_prob``) and shared by both directions, so the engine never infers + ``expected_object_kind`` / ``expected_structure`` from one prediction. Its + ``allowed_*`` and ``expected_*`` come from the question's declared admissibility + (permissive when ``q_prob`` is unconstrained), so a self-derived + ``reference_contract_violation`` cannot arise. + - **Symmetrization.** The verdict is ``equivalent`` iff the engine accepts in *both* + directions under that shared contract. This neutralizes the only remaining + asymmetry — the reference-printed-precision steps of + ``numbers_match_with_reference_precision`` (an order-sensitive accept that fires + only one way is rejected) — without letting either side's printed precision set the + bar. Relative ``q_prob.tolerance`` (symmetric by construction) still applies in both + directions, so genuine numeric agreement is accepted both ways. + + The structure routing, per-kind criteria, and bridges are reused unchanged because + they are already symmetric. Defaults to the permissive policy (no gold to gate + against); an explicit ``policy_mode`` is honored and uses the shared ``q_prob`` + contract for any contract enforcement. + + See ``EQUIVALENCE.md`` §1.1 and §10.3 for the methodology. + """ + + resolved_policy = coerce_policy_mode(policy_mode) + resolved_context = coerce_question_semantics(context) + contract = _build_reference_free_contract(resolved_context) + + forward = _normalize_reference_free_mode( + compare_protocol_answers( + a_i, + a_j, + contract=contract, + context=resolved_context, + policy_mode=resolved_policy, + ) + ) + if not forward.equivalent: + return forward + + backward = compare_protocol_answers( + a_j, + a_i, + contract=contract, + context=resolved_context, + policy_mode=resolved_policy, + ) + if backward.equivalent: + return forward + return AnswerComparison( + equivalent=False, + comparison_mode=forward.comparison_mode, + diagnostics=("asymmetric_match",) + backward.diagnostics, + validation_status=forward.validation_status, + policy_mode=resolved_policy, + ) + + +def _normalize_reference_free_mode(result: AnswerComparison) -> AnswerComparison: + """Drop the incoherent ``reference_contract_violation`` mode when neither side is gold. + + ``compare_protocol_answers`` validates its *second* argument as the reference and labels + a violation ``reference_contract_violation``. In reference-free comparison neither side + is gold, so a contract violation of one prediction is just a ``contract_violation`` (a + prediction failing the ``q_prob`` contract), never a *reference* violation. Remapping + keeps the verdict symmetric and avoids the self-rejection audit #4 calls out — the + contract was derived from ``q_prob``, not from the answer it rejects. + """ + + if result.comparison_mode != "reference_contract_violation": + return result + return result.model_copy(update={"comparison_mode": "contract_violation"}) + + +def _build_reference_free_contract( + context: PhysicsQuestionSemantics, +) -> PhysicsEvaluationContract: + """Build a symmetric, ``q_prob``-derived contract for reference-free comparison. + + Expected kind/structure are taken from the question's *declared* admissibility + (``allowed_object_kinds`` / ``allowed_structures``) rather than from either + prediction: a singleton allowed set pins the expectation, while an unconstrained + (default-permissive) set yields a permissive expectation that admits every kind / + structure. This is the build-time analogue of ``q_prob`` carrying only ``allowed_*`` + and policy fields, never a realized answer. + """ + + expected_object_kind = _single_or_default( + context.allowed_object_kinds, + full=tuple(AnswerObjectKind), + default=AnswerObjectKind.EXPRESSION, + ) + expected_structure = _single_or_default( + context.allowed_structures, + full=tuple(AnswerStructure), + default=AnswerStructure.ATOMIC, + ) + return PhysicsEvaluationContract( + question_semantics=context, + expected_object_kind=expected_object_kind, + expected_structure=expected_structure, + target_variable=context.target_variable, + enabled_bridge_ids=tuple(BRIDGE_REGISTRY), + enabled_bridge_tiers=(BridgeTier.TIER1, BridgeTier.TIER2, BridgeTier.TIER3), + ) + + +def _single_or_default( + values: tuple[Any, ...], *, full: tuple[Any, ...], default: Any +) -> Any: + """Pin a singleton declared set, else fall back to a permissive default. + + A singleton ``allowed_*`` is an explicit, symmetric expectation. The default-permissive + set (every member) carries no expectation, so a neutral ``default`` is used — under the + permissive policy (the reference-free default) the contract's ``expected_*`` is never + read, and under an explicit strict/audited override the neutral default keeps the gate + from favoring one prediction's shape over the other's. + """ + + if len(values) == 1: + return values[0] + if values and set(values) != set(full): + # A narrowed-but-not-singleton declaration; pick a member deterministically so the + # shared contract is identical for both directions. + return next(iter(values)) + return default + + def _compare_atomic( pred: PhysicsAnswerSemantics, ref: PhysicsAnswerSemantics, diff --git a/src/prkit/semantics/inference/__init__.py b/src/prkit/semantics/inference/__init__.py index a782de9..c163173 100644 --- a/src/prkit/semantics/inference/__init__.py +++ b/src/prkit/semantics/inference/__init__.py @@ -11,14 +11,18 @@ from .artifacts import ( PredictionSemanticsArtifact, PredictionSemanticsResponse, + ProblemSemanticsArtifact, ReferenceSemanticsArtifact, ReferenceSemanticsResponse, SemanticsArtifact, + SemanticsBuildReport, SemanticsComparisonInputs, SemanticsEvaluationRecord, SemanticsGeneratorInfo, SemanticsProblemRecord, + SymbolAssumptionProvenance, load_prediction_semantics_artifact, + load_problem_semantics_artifact, load_reference_semantics_artifact, load_semantics_artifact, load_semantics_evaluation_record, @@ -26,15 +30,22 @@ ) from .calls import ( PredictionSemanticsInferenceSpec, + build_extracted_prediction_semantics_artifact, build_prediction_semantics_artifact, + build_problem_semantics, + build_reference_semantics, compare_saved_semantics, evaluate_saved_semantics, + extract_prediction_answer_semantics, infer_prediction_semantics, infer_reference_semantics, parse_prediction_semantics_response_text, parse_reference_semantics_response_text, + prepare_isolated_prediction_semantics_inference_spec, prepare_prediction_semantics_inference_spec, prepare_semantics_comparison, + resolve_isolated_prediction_response_model, + resolve_prediction_response_model, ) from .prompts import ( PREDICTION_PROMPT_NAME, @@ -52,30 +63,41 @@ "PredictionSemanticsArtifact", "PredictionSemanticsInferenceSpec", "PredictionSemanticsResponse", + "ProblemSemanticsArtifact", "REFERENCE_PROMPT_NAME", "REFERENCE_PROMPT_VERSION", "ReferenceSemanticsArtifact", "ReferenceSemanticsResponse", "SemanticsArtifact", + "SemanticsBuildReport", "SemanticsComparisonInputs", "SemanticsEvaluationRecord", "SemanticsGeneratorInfo", "SemanticsProblemRecord", + "SymbolAssumptionProvenance", "answer_like_to_text", + "build_extracted_prediction_semantics_artifact", "build_prediction_semantics_prompt", + "build_problem_semantics", + "build_reference_semantics", "build_reference_semantics_prompt", "build_prediction_semantics_artifact", "compare_saved_semantics", "evaluate_saved_semantics", + "extract_prediction_answer_semantics", "infer_prediction_semantics", "infer_reference_semantics", "load_prediction_semantics_artifact", + "load_problem_semantics_artifact", "load_reference_semantics_artifact", "load_semantics_artifact", "load_semantics_evaluation_record", "parse_prediction_semantics_response_text", "parse_reference_semantics_response_text", + "prepare_isolated_prediction_semantics_inference_spec", "prepare_prediction_semantics_inference_spec", "prepare_semantics_comparison", + "resolve_isolated_prediction_response_model", + "resolve_prediction_response_model", "save_semantics_json", ] diff --git a/src/prkit/semantics/inference/artifacts.py b/src/prkit/semantics/inference/artifacts.py index 5fc9f63..9298a5f 100644 --- a/src/prkit/semantics/inference/artifacts.py +++ b/src/prkit/semantics/inference/artifacts.py @@ -15,6 +15,7 @@ PhysicsAnswerSemantics, PhysicsEvaluationContract, PhysicsQuestionSemantics, + SymbolAssumption, ) @@ -90,6 +91,61 @@ class SemanticsGeneratorInfo(_InferenceModel): ) +class SymbolAssumptionProvenance(_InferenceModel): + """Provenance for one synthesized symbol assumption in a build report.""" + + symbol: str = Field(description="Canonical (post-alias) symbol token.") + assumption: SymbolAssumption = Field( + description="The real-domain assumption adopted for the symbol.", + ) + source: str = Field( + description="Where the assumption came from: subject_to, llm_declared, or merged.", + ) + justification: str | None = Field( + default=None, + description="LLM-stated justification for a declared assumption, when available.", + ) + + +class SemanticsBuildReport(_InferenceModel): + """Objective provenance + confidence record for one semantics build. + + Attached additively to reference/problem artifacts so a build is auditable and + reproducible: which fields are deterministic vs LLM-advisory, why each symbol + assumption was adopted, what disagreements/cross-check reverts occurred, and whether a + human should review the result. + """ + + build_method: str = Field( + description="Identifier for the build pipeline, e.g. reference_3call or problem_3call.", + ) + temperature: float = Field( + default=0.0, + description="LLM sampling temperature used for advisory calls (0 for reproducibility).", + ) + field_provenance: dict[str, str] = Field( + default_factory=dict, + description="Per-field source: deterministic, subject_to, llm_declared, or default.", + json_schema_extra={"additionalProperties": False}, + ) + assumption_provenance: tuple[SymbolAssumptionProvenance, ...] = Field( + default_factory=tuple, + description="Provenance for each adopted symbol assumption.", + ) + flags: tuple[str, ...] = Field( + default_factory=tuple, + description="Disagreement, strengthening, and cross-check-revert flags for review.", + ) + cross_checks_passed: bool = Field( + default=True, + description="Whether round-trip, contract self-consistency, and pair-consistency held.", + ) + review_required: bool = Field( + default=False, + description="Whether the build surfaced a low-confidence signal warranting review.", + ) + + class ReferenceSemanticsResponse(_InferenceModel): """Structured model output for reference-semantics generation.""" @@ -136,14 +192,50 @@ class ReferenceSemanticsArtifact(_InferenceModel): description="Ground-truth answer surface supplied to the model.", ) question_semantics: PhysicsQuestionSemantics = Field( - description="Question semantics returned by the model.", + description="Reference-conditioned question semantics (q_ref), built from problem + golden.", ) reference_answer_semantics: PhysicsAnswerSemantics = Field( - description="Reference answer semantics returned by the model.", + description="Reference answer semantics (a_ref) for the ground-truth final answer.", ) generator: SemanticsGeneratorInfo = Field( description="Generation metadata.", ) + build_report: SemanticsBuildReport | None = Field( + default=None, + description="Objective build provenance/confidence record when built by the staged pipeline.", + ) + + +class ProblemSemanticsArtifact(_InferenceModel): + """Saved JSON artifact for a problem-only question semantics (q_prob). + + Built from the problem text alone (answer-blind), this is the contract for the + reference-free clustering judgement ``Eq(a_pred_i, a_pred_j; q_prob)``. It carries no + answer semantics: there is no golden answer to realize a structure/kind, so the contract + only declares admissible answer forms and policy. + """ + + artifact_type: str = Field( + default="problem_semantics", + description="Artifact discriminator.", + ) + created_at: str = Field( + default_factory=_utc_now_iso, + description="UTC timestamp for artifact creation.", + ) + problem: SemanticsProblemRecord = Field( + description="Problem snapshot used for the problem-only semantics call.", + ) + question_semantics: PhysicsQuestionSemantics = Field( + description="Problem-only question semantics (q_prob), built answer-blind.", + ) + generator: SemanticsGeneratorInfo = Field( + description="Generation metadata.", + ) + build_report: SemanticsBuildReport | None = Field( + default=None, + description="Objective build provenance/confidence record when built by the staged pipeline.", + ) class PredictionSemanticsArtifact(_InferenceModel): @@ -175,6 +267,14 @@ class PredictionSemanticsArtifact(_InferenceModel): generator: SemanticsGeneratorInfo = Field( description="Generation metadata.", ) + build_report: SemanticsBuildReport | None = Field( + default=None, + description=( + "Objective build provenance/confidence record. For isolated solves it carries " + "the a_pred_llm-vs-a_pred_ext structure disagreement audit; for the extracted " + "path it records deterministic provenance." + ), + ) class SemanticsComparisonInputs(_InferenceModel): @@ -227,7 +327,9 @@ class SemanticsEvaluationRecord(_InferenceModel): ) -SemanticsArtifact = ReferenceSemanticsArtifact | PredictionSemanticsArtifact +SemanticsArtifact = ( + ReferenceSemanticsArtifact | PredictionSemanticsArtifact | ProblemSemanticsArtifact +) def save_semantics_json(record: BaseModel, path: str | Path) -> Path: @@ -255,8 +357,16 @@ def load_prediction_semantics_artifact(path: str | Path) -> PredictionSemanticsA ) +def load_problem_semantics_artifact(path: str | Path) -> ProblemSemanticsArtifact: + """Load a problem-only (q_prob) semantics artifact from JSON.""" + + return ProblemSemanticsArtifact.model_validate_json( + Path(path).read_text(encoding="utf-8") + ) + + def load_semantics_artifact(path: str | Path) -> SemanticsArtifact: - """Load either a reference or prediction semantics artifact from JSON.""" + """Load a reference, prediction, or problem semantics artifact from JSON.""" raw_text = Path(path).read_text(encoding="utf-8") payload = json.loads(raw_text) @@ -266,10 +376,12 @@ def load_semantics_artifact(path: str | Path) -> SemanticsArtifact: return ReferenceSemanticsArtifact.model_validate(payload) if artifact_type == "prediction_semantics": return PredictionSemanticsArtifact.model_validate(payload) + if artifact_type == "problem_semantics": + return ProblemSemanticsArtifact.model_validate(payload) raise ValueError( "Semantics artifact JSON must define artifact_type as " - "'reference_semantics' or 'prediction_semantics'." + "'reference_semantics', 'prediction_semantics', or 'problem_semantics'." ) @@ -284,14 +396,18 @@ def load_semantics_evaluation_record(path: str | Path) -> SemanticsEvaluationRec __all__ = [ "PredictionSemanticsArtifact", "PredictionSemanticsResponse", + "ProblemSemanticsArtifact", "ReferenceSemanticsArtifact", "ReferenceSemanticsResponse", "SemanticsArtifact", + "SemanticsBuildReport", "SemanticsComparisonInputs", "SemanticsEvaluationRecord", "SemanticsGeneratorInfo", "SemanticsProblemRecord", + "SymbolAssumptionProvenance", "load_prediction_semantics_artifact", + "load_problem_semantics_artifact", "load_reference_semantics_artifact", "load_semantics_artifact", "load_semantics_evaluation_record", diff --git a/src/prkit/semantics/inference/calls.py b/src/prkit/semantics/inference/calls.py index 6a617b7..5c82463 100644 --- a/src/prkit/semantics/inference/calls.py +++ b/src/prkit/semantics/inference/calls.py @@ -26,7 +26,8 @@ build_evaluation_contract, compare_protocol_answers, ) -from ..comparison.contract import coerce_policy_mode +from ..comparison.contract import coerce_policy_mode, validate_answer_against_contract +from ..comparison.structure_canonicalization import canonicalize_structure from ..normalization import ( enrich_answer_quantity_views, infer_prediction_question_semantics, @@ -38,35 +39,59 @@ AnswerObjectKind, AnswerStructure, ComparisonPolicyMode, + ContractValidationStatus, PhysicsAnswerSemantics, PhysicsQuestionSemantics, + SymbolAssumption, ) from .artifacts import ( PredictionSemanticsArtifact, + ProblemSemanticsArtifact, ReferenceSemanticsArtifact, + SemanticsBuildReport, SemanticsComparisonInputs, SemanticsEvaluationRecord, SemanticsGeneratorInfo, SemanticsProblemRecord, + SymbolAssumptionProvenance, load_prediction_semantics_artifact, load_reference_semantics_artifact, ) from .prompts import ( PREDICTION_PROMPT_NAME, PREDICTION_PROMPT_VERSION, + PROBLEM_PROMPT_NAME, + PROBLEM_PROMPT_VERSION, REFERENCE_PROMPT_NAME, REFERENCE_PROMPT_VERSION, answer_like_to_text, + build_answer_surface_cleanup_prompt, build_prediction_semantics_prompt, - build_reference_semantics_prompt, + build_question_policy_prompt, + build_symbol_assumptions_prompt, +) +from .semantics_build import ( + alias_source_violations, + assumptions_from_subject_to, + assumptions_to_semantics, + build_alias_map, + extract_candidate_symbols, + infer_answer_tolerance, + meet_assumptions, + merge_symbol_assumptions, + reconcile_allowed_sets, + reference_pair_consistency, + resolve_to_canonical, ) from .strict_models import ( StrictPhysicsAnswerCaseSemantics, StrictPhysicsAnswerSemantics, StrictPhysicsQuestionSemantics, StrictPredictionFinalAnswerResponse, + StrictPredictionIsolatedResponse, StrictPredictionSemanticsResponse, StrictReferenceSemanticsResponse, + StrictSymbolAssumptionsResponse, ) logger = logging.getLogger(__name__) @@ -145,6 +170,102 @@ def resolve_prediction_response_model( return StrictPredictionSemanticsResponse +def resolve_isolated_prediction_response_model( + model_client: BaseModelClient, +) -> type[BaseModel]: + """Pick the provider-facing model for the ISOLATED problem-only solve (a_pred_llm). + + Prefers :class:`StrictPredictionIsolatedResponse` (reasoning + final_answer + + prediction_answer_semantics, no question_semantics). Providers that cannot enforce it + fall back to the compact :class:`StrictPredictionFinalAnswerResponse`, in which case only + ``a_pred_ext`` is available (the deterministic reconstruction). + """ + + isolated_plan = model_client.resolve_structured_output_plan( + StrictPredictionIsolatedResponse, + structured_policy="best_effort", + ) + if isolated_plan.native_schema_enforced: + return StrictPredictionIsolatedResponse + + compact_plan = model_client.resolve_structured_output_plan( + StrictPredictionFinalAnswerResponse, + structured_policy="best_effort", + ) + if compact_plan.native_schema_enforced: + logger.warning( + "Model %s (%s) cannot natively enforce the isolated prediction semantics schema; " + "using compact final-answer response schema (a_pred_ext only).", + getattr(model_client, "model", "unknown"), + getattr(model_client, "provider", "unknown"), + ) + return StrictPredictionFinalAnswerResponse + + return StrictPredictionIsolatedResponse + + +# ---------------------------------------------------------------------------------------- +# DEPRECATED single-call reference build (kept commented for manual review/comparison +# against the staged `build_reference_semantics` that replaced it). The old path made ONE +# fused LLM call returning both question and answer semantics with no determinism control, +# no structure/kind pinning, and no cross-checks. See `build_reference_semantics` below. +# ---------------------------------------------------------------------------------------- +# def infer_reference_semantics( +# problem: PhysicsProblem, +# model_client: BaseModelClient, +# *, +# max_output_tokens: int | None = None, +# **chat_kwargs: Any, +# ) -> ReferenceSemanticsArtifact: +# """Infer and package reference semantics for a problem's ground-truth answer.""" +# +# if problem.answer is None: +# raise ValueError( +# f"Problem {problem.problem_id} does not provide `problem.answer`." +# ) +# +# require_native_json_schema = _semantics_should_require_native_json_schema( +# model_client, +# StrictReferenceSemanticsResponse, +# ) +# ground_truth_answer_text = answer_like_to_text(problem.answer) +# draft_question_semantics = infer_reference_question_semantics(problem) +# prompt = build_reference_semantics_prompt( +# problem, +# draft_question_semantics=draft_question_semantics, +# ) +# response, structured_result = _run_structured_inference( +# model_client, +# prompt=prompt, +# response_model=StrictReferenceSemanticsResponse, +# image_paths=tuple(problem.image_path or ()), +# max_output_tokens=max_output_tokens, +# require_native_json_schema=require_native_json_schema, +# **chat_kwargs, +# ) +# +# merged_question_semantics = _merge_question_semantics_fallbacks( +# response.question_semantics.to_canonical(), +# draft_question_semantics, +# ) +# return ReferenceSemanticsArtifact( +# problem=_problem_record_from_problem(problem), +# ground_truth_answer=ground_truth_answer_text, +# question_semantics=merged_question_semantics, +# reference_answer_semantics=enrich_answer_quantity_views( +# response.reference_answer_semantics.to_canonical(), +# context=merged_question_semantics, +# ), +# generator=_generator_info( +# model_client, +# prompt_name=REFERENCE_PROMPT_NAME, +# prompt_version=REFERENCE_PROMPT_VERSION, +# structured_output_mode=structured_result.structured_output_mode, +# structured_output_strategy=structured_result.structured_output_strategy, +# ), +# ) + + def infer_reference_semantics( problem: PhysicsProblem, model_client: BaseModelClient, @@ -152,52 +273,502 @@ def infer_reference_semantics( max_output_tokens: int | None = None, **chat_kwargs: Any, ) -> ReferenceSemanticsArtifact: - """Infer and package reference semantics for a problem's ground-truth answer.""" + """Infer and package reference semantics (q_ref + a_ref) for a problem's golden answer. + + Back-compatible entry point: delegates to the staged :func:`build_reference_semantics`. + """ + + return build_reference_semantics( + problem, + model_client, + max_output_tokens=max_output_tokens, + **chat_kwargs, + ) + + +# ---------------------------------------------------------------------------------------- +# Staged objective semantics build (WS A). Deterministic backbone is authoritative for +# structure/object_kind/tolerance/allowed_*; three advisory LLM calls (temperature 0) clean +# the answer surface, fill question policy, and declare justified symbol assumptions. See +# `semantics_build.py` for the deterministic methodology and the build's compatibility rules. +# ---------------------------------------------------------------------------------------- +_ANSWER_SURFACE_FILL_FIELDS = ( + "canonical_latex", + "unit", + "dimension", + "choice_label", + "boolean_value", + "sign_value", +) + + +def _advisory_inference( + model_client: BaseModelClient, + *, + prompt: str, + response_model: type[ResponseModelT], + image_paths: tuple[str, ...], + max_output_tokens: int | None, + **chat_kwargs: Any, +) -> ResponseModelT | None: + """Run one advisory build call, returning ``None`` on any failure. + + Advisory calls only refine fields the deterministic backbone has already decided, so a + failed call degrades to the deterministic value rather than crashing the build. + + These calls run **best-effort** (``require_native_json_schema=False``): native structured + output is used when the provider supports it, otherwise the LLM still enriches via the + plain-text / ``json_object`` route and the result is parsed back. So lacking native + structured output does **not** reduce the reference build (Step 1) to deterministic-only — + native structured output is a Step-2 (answer-generation output-form) concern, not a Step-1 + one. Only a genuine inference/parse failure degrades to the deterministic value. + """ + + try: + response, _ = _run_structured_inference( + model_client, + prompt=prompt, + response_model=response_model, + image_paths=image_paths, + max_output_tokens=max_output_tokens, + require_native_json_schema=False, + **chat_kwargs, + ) + return response + except Exception as exc: # noqa: BLE001 - advisory; the deterministic value stands + logger.warning( + "Advisory build call (%s) failed; using the deterministic value instead: %s", + response_model.__name__, + exc, + ) + return None + + +def _adopt_answer_cleanup( + deterministic: PhysicsAnswerSemantics, + cleaned: PhysicsAnswerSemantics, +) -> tuple[PhysicsAnswerSemantics, list[str], dict[str, str]]: + """Fill empty surface fields from the cleanup call; structure/kind stay pinned. + + Only presentational/gap fields are adopted (and only when the deterministic value is + empty); ``canonical_text``/``numeric_text``/``numeric_value`` stay deterministic so the + golden's printed precision is preserved (compat #1). + """ + + flags: list[str] = [] + if ( + cleaned.structure != deterministic.structure + or cleaned.object_kind != deterministic.object_kind + ): + flags.append( + "answer_cleanup_structure_disagreement:" + f"{cleaned.structure.value}/{cleaned.object_kind.value}" + ) + updates: dict[str, Any] = {} + provenance: dict[str, str] = {} + for field in _ANSWER_SURFACE_FILL_FIELDS: + det_value = getattr(deterministic, field) + llm_value = getattr(cleaned, field) + if not det_value and llm_value: + updates[field] = llm_value + provenance[field] = "llm_declared" + adopted = deterministic.model_copy(update=updates) if updates else deterministic + return adopted, flags, provenance + + +def _adopt_question_policy( + draft: PhysicsQuestionSemantics, + policy: PhysicsQuestionSemantics, +) -> tuple[PhysicsQuestionSemantics, dict[str, str]]: + """Adopt LLM question-policy fields onto the draft. + + ``allowed_*``, ``tolerance``, and ``symbol_assumptions`` from the policy call are ignored + here; the build sets them deterministically. + """ + + updates: dict[str, Any] = {} + provenance: dict[str, str] = {} + + # Enum policy fields are decisions; adopt them as-is. + for field in ("question_symbolic_mode", "question_unit_policy", "ordering"): + updates[field] = getattr(policy, field) + provenance[field] = "llm_declared" + + # Optional / collection fields: adopt only when the LLM provided a value. + if policy.target_variable: + updates["target_variable"] = policy.target_variable + provenance["target_variable"] = "llm_declared" + if policy.symbol_aliases: + updates["symbol_aliases"] = policy.symbol_aliases + provenance["symbol_aliases"] = "llm_declared" + if policy.question_unit: + updates["question_unit"] = policy.question_unit + provenance["question_unit"] = "llm_declared" + if policy.dimension: + updates["dimension"] = policy.dimension + provenance["dimension"] = "llm_declared" + if policy.required_parts: + updates["required_parts"] = policy.required_parts + provenance["required_parts"] = "llm_declared" + if policy.coordinate_frame: + updates["coordinate_frame"] = policy.coordinate_frame + provenance["coordinate_frame"] = "llm_declared" + if policy.sign_convention: + updates["sign_convention"] = policy.sign_convention + provenance["sign_convention"] = "llm_declared" + if policy.choice_space: + updates["choice_space"] = policy.choice_space + provenance["choice_space"] = "llm_declared" + + return draft.merged(updates), provenance + + +def _collect_declared_assumptions( + response: StrictSymbolAssumptionsResponse | None, + alias_map: dict[str, str], +) -> tuple[dict[str, SymbolAssumption], dict[str, str]]: + """Resolve declared assumptions to canonical tokens, returning (map, justifications).""" + + declared: dict[str, SymbolAssumption] = {} + justifications: dict[str, str] = {} + if response is None: + return declared, justifications + for entry in response.assumptions: + canonical = resolve_to_canonical(entry.symbol, alias_map) + if canonical in declared: + declared[canonical] = meet_assumptions( + declared[canonical], entry.assumption + ) + else: + declared[canonical] = entry.assumption + if entry.justification: + justifications[canonical] = entry.justification + return declared, justifications + + +def _assumption_provenance( + merged: dict[str, SymbolAssumption], + subject_to: dict[str, SymbolAssumption], + declared: dict[str, SymbolAssumption], + justifications: dict[str, str], +) -> tuple[SymbolAssumptionProvenance, ...]: + """Tag each adopted assumption with its source for the build report.""" + + provenance: list[SymbolAssumptionProvenance] = [] + for symbol in sorted(merged): + in_st = symbol in subject_to + in_llm = symbol in declared + source = ( + "merged" + if (in_st and in_llm) + else ("subject_to" if in_st else "llm_declared") + ) + provenance.append( + SymbolAssumptionProvenance( + symbol=symbol, + assumption=merged[symbol], + source=source, + justification=justifications.get(symbol), + ) + ) + return tuple(provenance) + + +def build_reference_semantics( + problem: PhysicsProblem, + model_client: BaseModelClient, + *, + golden: str | None = None, + max_output_tokens: int | None = None, + temperature: float = 0.0, + **chat_kwargs: Any, +) -> ReferenceSemanticsArtifact: + """Build reference semantics (q_ref + a_ref) from a problem and its golden answer. + + Deterministic backbone (authoritative): ``normalize_physics_answer`` + + ``canonicalize_structure`` pin ``a_ref``'s structure/object_kind; ``subject_to`` + constraints seed ``symbol_assumptions``; tolerance and ``allowed_*`` are deterministic. + Three advisory LLM calls (temperature 0) then clean the answer surface, fill question + policy, and declare justified symbol assumptions; their outputs are validated and never + override the deterministic decisions. A build report records provenance and cross-checks. + """ - if problem.answer is None: + resolved_golden = ( + golden + if golden is not None + else ( + answer_like_to_text(problem.answer) if problem.answer is not None else None + ) + ) + if not resolved_golden: raise ValueError( - f"Problem {problem.problem_id} does not provide `problem.answer`." + f"Problem {problem.problem_id} has no golden answer to build reference semantics." ) - require_native_json_schema = _semantics_should_require_native_json_schema( + call_kwargs = dict(chat_kwargs) + call_kwargs.setdefault("temperature", temperature) + image_paths = tuple(problem.image_path or ()) + flags: list[str] = [] + provenance: dict[str, str] = { + "structure": "deterministic", + "object_kind": "deterministic", + "tolerance": "deterministic", + } + + # Stage 0 - deterministic backbone (authoritative for structure/kind). + q_draft = infer_reference_question_semantics(problem) + a_det = canonicalize_structure( + normalize_physics_answer(resolved_golden, context=q_draft), + context=q_draft, + ) + + # Stage 1a - answer-surface cleanup (advisory; structure/kind pinned). + a_ref = a_det + cleanup = _advisory_inference( model_client, - StrictReferenceSemanticsResponse, + prompt=build_answer_surface_cleanup_prompt( + problem, golden_text=resolved_golden, draft_answer=a_det + ), + response_model=StrictPhysicsAnswerSemantics, + image_paths=image_paths, + max_output_tokens=max_output_tokens, + **call_kwargs, ) - ground_truth_answer_text = answer_like_to_text(problem.answer) - draft_question_semantics = infer_reference_question_semantics(problem) - prompt = build_reference_semantics_prompt( - problem, - draft_question_semantics=draft_question_semantics, + if cleanup is not None: + a_ref, cleanup_flags, cleanup_provenance = _adopt_answer_cleanup( + a_det, cleanup.to_canonical() + ) + flags.extend(cleanup_flags) + provenance.update(cleanup_provenance) + else: + flags.append("answer_cleanup_call_unavailable") + + # Stage 1b - question policy (advisory). + policy = _advisory_inference( + model_client, + prompt=build_question_policy_prompt(problem, answer_draft=a_ref), + response_model=StrictPhysicsQuestionSemantics, + image_paths=image_paths, + max_output_tokens=max_output_tokens, + **call_kwargs, ) - response, structured_result = _run_structured_inference( + if policy is not None: + q_ref, policy_provenance = _adopt_question_policy( + q_draft, policy.to_canonical() + ) + provenance.update(policy_provenance) + else: + q_ref = q_draft + flags.append("question_policy_call_unavailable") + + alias_map = build_alias_map(q_ref) + + # Stage 1c - justified symbol-assumption declaration (advisory). + assumptions_response = _advisory_inference( model_client, - prompt=prompt, - response_model=StrictReferenceSemanticsResponse, - image_paths=tuple(problem.image_path or ()), + prompt=build_symbol_assumptions_prompt( + problem, + candidate_symbols=extract_candidate_symbols( + a_ref.canonical_text, alias_map=alias_map + ), + answer_draft=a_ref, + ), + response_model=StrictSymbolAssumptionsResponse, + image_paths=image_paths, max_output_tokens=max_output_tokens, - require_native_json_schema=require_native_json_schema, - **chat_kwargs, + **call_kwargs, + ) + if assumptions_response is None: + flags.append("symbol_assumptions_call_unavailable") + declared, justifications = _collect_declared_assumptions( + assumptions_response, alias_map ) - merged_question_semantics = _merge_question_semantics_fallbacks( - response.question_semantics.to_canonical(), - draft_question_semantics, + # Stage 2 - deterministic synthesis & reconciliation. + subject_to_assumptions = assumptions_from_subject_to( + a_ref.subject_to, alias_map=alias_map + ) + merged_assumptions, merge_flags = merge_symbol_assumptions( + subject_to_assumptions, declared ) + flags.extend(merge_flags) + for symbol in alias_source_violations(merged_assumptions, alias_map): + merged_assumptions.pop(symbol, None) + flags.append(f"dropped_alias_source_assumption:{symbol}") + + q_ref = q_ref.merged( + { + "symbol_assumptions": assumptions_to_semantics(merged_assumptions), + "tolerance": infer_answer_tolerance(instruction_text=problem.question), + } + ) + q_ref = reconcile_allowed_sets(q_ref, a_ref) + a_ref = enrich_answer_quantity_views(a_ref, context=q_ref) + + # Stage 3 - cross-checks (validate authority; never rescue). + cross_ok = True + try: + round_trip = normalize_physics_answer(a_ref.canonical_text, context=q_ref) + if ( + round_trip.structure != a_ref.structure + or round_trip.object_kind != a_ref.object_kind + ): + flags.append( + "roundtrip_drift:" + f"{round_trip.structure.value}/{round_trip.object_kind.value}" + ) + cross_ok = False + except Exception as exc: # noqa: BLE001 - cross-check is best-effort, never fatal + flags.append(f"roundtrip_error:{exc.__class__.__name__}") + cross_ok = False + + contract = build_evaluation_contract( + question_semantics=q_ref, reference_answer_semantics=a_ref + ) + if ( + validate_answer_against_contract(a_ref, contract).status + == ContractValidationStatus.VIOLATING + ): + flags.append("reference_self_contract_violation") + cross_ok = False + + pair_issues = reference_pair_consistency(q_ref, a_ref) + if pair_issues: + flags.extend(f"pair:{issue}" for issue in pair_issues) + cross_ok = False + if ( + any(issue.startswith("target_variable_mismatch") for issue in pair_issues) + and a_ref.target_variable + ): + q_ref = q_ref.merged({"target_variable": a_ref.target_variable}) + provenance["target_variable"] = "deterministic" + flags.append("reverted_target_variable_to_deterministic") + + report = SemanticsBuildReport( + build_method="reference_3call", + temperature=temperature, + field_provenance=provenance, + assumption_provenance=_assumption_provenance( + merged_assumptions, subject_to_assumptions, declared, justifications + ), + flags=tuple(flags), + cross_checks_passed=cross_ok, + # Only a genuine cross-check failure warrants review. An advisory call being + # unavailable (e.g. no native structured output) is a normal route, not a defect — + # it is recorded as an informational flag but does not force review. + review_required=not cross_ok, + ) + return ReferenceSemanticsArtifact( problem=_problem_record_from_problem(problem), - ground_truth_answer=ground_truth_answer_text, - question_semantics=merged_question_semantics, - reference_answer_semantics=enrich_answer_quantity_views( - response.reference_answer_semantics.to_canonical(), - context=merged_question_semantics, - ), + ground_truth_answer=resolved_golden, + question_semantics=q_ref, + reference_answer_semantics=a_ref, generator=_generator_info( model_client, prompt_name=REFERENCE_PROMPT_NAME, prompt_version=REFERENCE_PROMPT_VERSION, - structured_output_mode=structured_result.structured_output_mode, - structured_output_strategy=structured_result.structured_output_strategy, + structured_output_mode="staged_3call", ), + build_report=report, + ) + + +def build_problem_semantics( + problem: PhysicsProblem, + model_client: BaseModelClient, + *, + max_output_tokens: int | None = None, + temperature: float = 0.0, + **chat_kwargs: Any, +) -> ProblemSemanticsArtifact: + """Build problem-only question semantics (q_prob), answer-blind. + + Used as the contract for reference-free clustering. There is no golden answer, so no + answer record and no structure/kind realization: ``allowed_*`` stay permissive and + symbol assumptions come only from LLM declarations (cross-checked for canonical tokens). + """ + + call_kwargs = dict(chat_kwargs) + call_kwargs.setdefault("temperature", temperature) + image_paths = tuple(problem.image_path or ()) + flags: list[str] = [] + provenance: dict[str, str] = {"tolerance": "deterministic"} + + q_draft = infer_prediction_question_semantics(problem) + + policy = _advisory_inference( + model_client, + prompt=build_question_policy_prompt(problem, answer_draft=None), + response_model=StrictPhysicsQuestionSemantics, + image_paths=image_paths, + max_output_tokens=max_output_tokens, + **call_kwargs, + ) + if policy is not None: + q_prob, policy_provenance = _adopt_question_policy( + q_draft, policy.to_canonical() + ) + provenance.update(policy_provenance) + else: + q_prob = q_draft + flags.append("question_policy_call_unavailable") + + alias_map = build_alias_map(q_prob) + + assumptions_response = _advisory_inference( + model_client, + prompt=build_symbol_assumptions_prompt( + problem, + candidate_symbols=extract_candidate_symbols( + problem.question, alias_map=alias_map + ), + answer_draft=None, + ), + response_model=StrictSymbolAssumptionsResponse, + image_paths=image_paths, + max_output_tokens=max_output_tokens, + **call_kwargs, + ) + declared, justifications = _collect_declared_assumptions( + assumptions_response, alias_map + ) + for symbol in alias_source_violations(declared, alias_map): + declared.pop(symbol, None) + flags.append(f"dropped_alias_source_assumption:{symbol}") + if any(value != SymbolAssumption.REAL for value in declared.values()): + # Stronger-than-real assumptions have no golden/subject_to to cross-check here. + flags.append("problem_only_assumptions_unverified") + + q_prob = q_prob.merged( + { + "symbol_assumptions": assumptions_to_semantics(declared), + "tolerance": infer_answer_tolerance(instruction_text=problem.question), + } + ) + + report = SemanticsBuildReport( + build_method="problem_3call", + temperature=temperature, + field_provenance=provenance, + assumption_provenance=_assumption_provenance( + declared, {}, declared, justifications + ), + flags=tuple(flags), + cross_checks_passed=True, + review_required=bool(flags), + ) + + return ProblemSemanticsArtifact( + problem=_problem_record_from_problem(problem), + question_semantics=q_prob, + generator=_generator_info( + model_client, + prompt_name=PROBLEM_PROMPT_NAME, + prompt_version=PROBLEM_PROMPT_VERSION, + structured_output_mode="staged_3call", + ), + build_report=report, ) @@ -205,23 +776,38 @@ def infer_prediction_semantics( problem: PhysicsProblem, model_client: BaseModelClient, *, + isolated_solve: bool = True, max_output_tokens: int | None = None, allow_non_native_structured_output: bool = False, **chat_kwargs: Any, ) -> PredictionSemanticsArtifact: - """Let a model solve a problem and package the predicted answer semantics.""" + """Let a model solve a problem and package the predicted answer semantics. + + By default (``isolated_solve=True``) the model solves the problem **answer-blind and + contract-blind**: the solve prompt is problem + options + context only, with no embedded + question-semantics draft (the leakage guard). The artifact's ``prediction_answer_semantics`` + is ``a_pred_llm`` (the LLM-structured record, canonicalized), and the build report carries + the ``a_pred_llm``-vs-``a_pred_ext`` structure-disagreement audit. Set + ``isolated_solve=False`` to keep the legacy fused path (the draft is injected and the + model authors a prediction-side ``question_semantics``). + """ - response_model = resolve_prediction_response_model(model_client) - require_native_json_schema = ( - _semantics_should_require_native_json_schema( + if isolated_solve: + # The isolated solve is always graceful (`allow_non_native_structured_output` applies + # only to the legacy fused path): missing native structured output yields a_pred_ext, + # never a failure (the user's "plain output -> deterministic extraction" route). + return _infer_isolated_prediction_semantics( + problem, model_client, - response_model, + max_output_tokens=max_output_tokens, + **chat_kwargs, ) - if not allow_non_native_structured_output - else model_client.resolve_structured_output_plan( - response_model, - structured_policy="best_effort", - ).native_schema_enforced + + response_model = resolve_prediction_response_model(model_client) + require_native_json_schema = _resolve_prediction_native_requirement( + model_client, + response_model, + allow_non_native_structured_output=allow_non_native_structured_output, ) spec = prepare_prediction_semantics_inference_spec( problem, @@ -252,12 +838,131 @@ def infer_prediction_semantics( ) +def _resolve_prediction_native_requirement( + model_client: BaseModelClient, + response_model: type[BaseModel], + *, + allow_non_native_structured_output: bool, +) -> bool: + """Decide whether native structured output is required for a prediction call.""" + + if allow_non_native_structured_output: + return model_client.resolve_structured_output_plan( + response_model, + structured_policy="best_effort", + ).native_schema_enforced + return _semantics_should_require_native_json_schema(model_client, response_model) + + +def _infer_isolated_prediction_semantics( + problem: PhysicsProblem, + model_client: BaseModelClient, + *, + max_output_tokens: int | None, + **chat_kwargs: Any, +) -> PredictionSemanticsArtifact: + """Problem-only isolated solve building a_pred_llm (+ the a_pred_ext disagreement audit). + + Always graceful: native structured output is used when the provider supports it (so + ``a_pred_llm`` is produced), otherwise the solve falls back to plain text and only + ``a_pred_ext`` (deterministic extraction) is produced — lacking native structured output + never fails this step, it just yields one answer-semantics form instead of two. + """ + + response_model = resolve_isolated_prediction_response_model(model_client) + # Best-effort: prefer native when available, never require it. + require_native_json_schema = model_client.resolve_structured_output_plan( + response_model, + structured_policy="best_effort", + ).native_schema_enforced + spec = prepare_isolated_prediction_semantics_inference_spec( + problem, + response_model=response_model, + ) + response, structured_result = _run_structured_inference( + model_client, + prompt=spec.prompt, + response_model=spec.response_model, + image_paths=spec.image_paths, + max_output_tokens=max_output_tokens, + require_native_json_schema=require_native_json_schema, + **chat_kwargs, + ) + if not isinstance( + response, + (StrictPredictionIsolatedResponse, StrictPredictionFinalAnswerResponse), + ): + raise TypeError( + "Isolated prediction response must be either StrictPredictionIsolatedResponse " + f"or StrictPredictionFinalAnswerResponse. Got {type(response)!r}." + ) + + # a_pred_ext: deterministic extraction from the final-answer surface (same authority as + # a_ref). Always computable, so it is the A/B baseline and the disagreement reference. + a_pred_ext = extract_prediction_answer_semantics(response.final_answer) + + flags: list[str] = [] + if isinstance(response, StrictPredictionIsolatedResponse): + # a_pred_llm: the LLM-structured record, canonicalized for symmetric structure. + a_pred_llm = canonicalize_structure( + response.prediction_answer_semantics.to_canonical(), + context=PhysicsQuestionSemantics(), + ) + adopted = a_pred_llm + build_method = "prediction_isolated_llm" + if ( + a_pred_llm.structure != a_pred_ext.structure + or a_pred_llm.object_kind != a_pred_ext.object_kind + ): + # Disagreement audit (compat: do NOT reconcile; feeds the A/B comparison). + flags.append( + "a_pred_llm_vs_ext_disagreement:" + f"{a_pred_llm.structure.value}/{a_pred_llm.object_kind.value}" + f"!={a_pred_ext.structure.value}/{a_pred_ext.object_kind.value}" + ) + else: + # Provider could not enforce the isolated schema; only a_pred_ext is available. + adopted = a_pred_ext + build_method = "prediction_isolated_extracted" + flags.append("a_pred_llm_unavailable") + + report = SemanticsBuildReport( + build_method=build_method, + field_provenance={ + "structure": "deterministic" if adopted is a_pred_ext else "llm_declared", + "object_kind": "deterministic" if adopted is a_pred_ext else "llm_declared", + }, + flags=tuple(flags), + cross_checks_passed=True, + review_required=bool(flags), + ) + + return PredictionSemanticsArtifact( + problem=_problem_record_from_problem(problem), + reasoning=response.reasoning, + final_answer=response.final_answer, + question_semantics=PhysicsQuestionSemantics(), + prediction_answer_semantics=enrich_answer_quantity_views( + adopted, context=PhysicsQuestionSemantics() + ), + generator=_generator_info_from_metadata( + provider=getattr(model_client, "provider", None), + model_name=getattr(model_client, "model", None), + prompt_name=PREDICTION_PROMPT_NAME, + prompt_version=PREDICTION_PROMPT_VERSION, + structured_output_mode=structured_result.structured_output_mode, + structured_output_strategy=structured_result.structured_output_strategy, + ), + build_report=report, + ) + + def prepare_prediction_semantics_inference_spec( problem: PhysicsProblem, *, response_model: type[BaseModel] = StrictPredictionSemanticsResponse, ) -> PredictionSemanticsInferenceSpec: - """Build the prompt/schema bundle used for prediction-semantics inference.""" + """Build the prompt/schema bundle for the legacy fused prediction-semantics inference.""" draft_question_semantics = infer_prediction_question_semantics(problem) return PredictionSemanticsInferenceSpec( @@ -275,6 +980,33 @@ def prepare_prediction_semantics_inference_spec( ) +def prepare_isolated_prediction_semantics_inference_spec( + problem: PhysicsProblem, + *, + response_model: type[BaseModel] = StrictPredictionIsolatedResponse, +) -> PredictionSemanticsInferenceSpec: + """Build the prompt/schema bundle for the ISOLATED problem-only solve. + + The prompt suppresses the embedded question-semantics draft (the leakage guard): it is + problem + options + context only. ``draft_question_semantics`` is left at the empty default + because nothing reference/contract-side reaches the solver here. + """ + + return PredictionSemanticsInferenceSpec( + prompt=build_prediction_semantics_prompt( + problem, + include_prediction_answer_semantics=( + response_model is StrictPredictionIsolatedResponse + ), + suppress_question_semantics_draft=True, + ), + image_paths=tuple(problem.image_path or ()), + draft_question_semantics=PhysicsQuestionSemantics(), + response_model=response_model, + response_format=normalize_response_format(response_model), + ) + + def parse_prediction_semantics_response_text( raw_response: str, *, @@ -298,12 +1030,83 @@ def parse_reference_semantics_response_text( return _parse_response_model(StrictReferenceSemanticsResponse, raw_response) +def extract_prediction_answer_semantics( + answer_text: str, + *, + context: PhysicsQuestionSemantics | None = None, +) -> PhysicsAnswerSemantics: + """Build the deterministic ``a_pred_ext`` record from a plain-text answer surface. + + ``a_pred_ext = canonicalize_structure(normalize_physics_answer(final_answer))`` — the + same deterministic authority used for ``a_ref``, so the two classify identically (the + structure-mismatch defense). No generation happens here, so this is also the entry point + for externally-supplied answers and the A/B baseline. + """ + + resolved_context = context or PhysicsQuestionSemantics() + return canonicalize_structure( + normalize_physics_answer(answer_text, context=resolved_context), + context=resolved_context, + ) + + +def build_extracted_prediction_semantics_artifact( + problem: PhysicsProblem, + answer_text: str, + *, + context: PhysicsQuestionSemantics | None = None, + provider: str | None = None, + model_name: str | None = None, + reasoning: str = "", +) -> PredictionSemanticsArtifact: + """Package ``a_pred_ext`` for an externally-supplied plain-text answer (no generation). + + This is the standalone, generation-free path: it deterministically extracts the + prediction answer semantics from ``answer_text`` and is also the A/B baseline against the + LLM-structured ``a_pred_llm``. The artifact carries no question semantics (the prediction + side never authors a contract); ``question_semantics`` stays the empty default. + """ + + a_pred_ext = enrich_answer_quantity_views( + extract_prediction_answer_semantics(answer_text, context=context), + context=context or PhysicsQuestionSemantics(), + ) + return PredictionSemanticsArtifact( + problem=_problem_record_from_problem(problem), + reasoning=reasoning, + final_answer=answer_text, + question_semantics=PhysicsQuestionSemantics(), + prediction_answer_semantics=a_pred_ext, + generator=_generator_info_from_metadata( + provider=provider, + model_name=model_name, + prompt_name=PREDICTION_PROMPT_NAME, + prompt_version=PREDICTION_PROMPT_VERSION, + structured_output_mode="extracted", + ), + build_report=SemanticsBuildReport( + build_method="prediction_extracted", + field_provenance={ + "structure": "deterministic", + "object_kind": "deterministic", + }, + cross_checks_passed=True, + review_required=False, + ), + ) + + def _coerce_prediction_response_to_strict( response: BaseModel, *, draft_question_semantics: PhysicsQuestionSemantics | None = None, ) -> StrictPredictionSemanticsResponse: - """Lift compact provider-facing responses into the full strict response model.""" + """Lift compact provider-facing responses into the full strict response model. + + The compact response carries only a ``final_answer`` surface, so the prediction answer + semantics are reconstructed deterministically as ``a_pred_ext`` — the same + ``canonicalize_structure(normalize_physics_answer(...))`` authority used for ``a_ref``. + """ if isinstance(response, StrictPredictionSemanticsResponse): return response @@ -315,7 +1118,7 @@ def _coerce_prediction_response_to_strict( resolved_question_semantics = draft_question_semantics or PhysicsQuestionSemantics() strict_answer_payload = _strict_answer_payload( - normalize_physics_answer( + extract_prediction_answer_semantics( response.final_answer, context=resolved_question_semantics, ) @@ -659,6 +1462,47 @@ def _normalize_response_payload( "Prompt-only prediction parsing dropped top-level fields: %s", ", ".join(dropped_top_level), ) + elif response_model is StrictPredictionIsolatedResponse: + # Isolated solve: no question_semantics; keep only reasoning/final_answer/answer. + if _looks_like_prediction_answer_semantics_payload(normalized): + normalized = {"prediction_answer_semantics": normalized} + normalized.pop("question_semantics", None) + normalized.pop("reference_answer_semantics", None) + reasoning_summary = normalized.pop("reasoning_summary", None) + if "reasoning" not in normalized and isinstance(reasoning_summary, str): + normalized["reasoning"] = reasoning_summary + normalized.setdefault("reasoning", "") + answer_semantics = normalized.get("prediction_answer_semantics") + if isinstance(answer_semantics, dict): + normalized["prediction_answer_semantics"] = ( + _normalize_answer_semantics_payload( + answer_semantics, + path="prediction_answer_semantics", + ) + ) + if "final_answer" not in normalized: + final_answer = _infer_final_answer_from_answer_semantics( + normalized["prediction_answer_semantics"] + ) + if final_answer is not None: + normalized["final_answer"] = final_answer + elif isinstance(normalized.get("final_answer"), str): + normalized["prediction_answer_semantics"] = _strict_answer_payload( + normalize_physics_answer( + normalized["final_answer"], + context=PhysicsQuestionSemantics(), + ) + ) + dropped_top_level = sorted( + set(payload) + - set(normalized) + - {"question_semantics", "reference_answer_semantics", "reasoning_summary"} + ) + if dropped_top_level: + logger.debug( + "Prompt-only isolated prediction parsing dropped top-level fields: %s", + ", ".join(dropped_top_level), + ) elif response_model is StrictPredictionFinalAnswerResponse: normalized = { key: value @@ -1063,7 +1907,10 @@ def _build_non_native_json_retry_prompt( "Do not include any prose, analysis, markdown fences, or comments before or after the JSON.", "Keep every string field concise.", ] - if response_model is StrictPredictionSemanticsResponse: + if response_model in { + StrictPredictionSemanticsResponse, + StrictPredictionIsolatedResponse, + }: extra_lines.append( "Keep `reasoning` to a brief 1-3 sentence summary, not a full derivation." ) @@ -1124,15 +1971,22 @@ def _coerce_prediction_artifact( __all__ = [ "PredictionSemanticsInferenceSpec", + "build_extracted_prediction_semantics_artifact", "build_prediction_semantics_artifact", + "build_problem_semantics", + "build_reference_semantics", "compare_saved_semantics", "ensure_semantics_native_structured_output_support", "ensure_semantics_native_json_schema_support", "evaluate_saved_semantics", + "extract_prediction_answer_semantics", "infer_prediction_semantics", "infer_reference_semantics", "parse_prediction_semantics_response_text", "parse_reference_semantics_response_text", + "prepare_isolated_prediction_semantics_inference_spec", "prepare_prediction_semantics_inference_spec", "prepare_semantics_comparison", + "resolve_isolated_prediction_response_model", + "resolve_prediction_response_model", ] diff --git a/src/prkit/semantics/inference/prompts.py b/src/prkit/semantics/inference/prompts.py index f692b6b..867e5c7 100644 --- a/src/prkit/semantics/inference/prompts.py +++ b/src/prkit/semantics/inference/prompts.py @@ -13,9 +13,17 @@ from ..schema import PhysicsAnswerSemantics, PhysicsQuestionSemantics REFERENCE_PROMPT_NAME = "reference_semantics" -REFERENCE_PROMPT_VERSION = "v4" +# v5: staged 3-call build (answer-surface cleanup / question policy / symbol assumptions) +# replaces the single fused reference call; structure/kind are deterministically pinned. +REFERENCE_PROMPT_VERSION = "v5" PREDICTION_PROMPT_NAME = "prediction_semantics" -PREDICTION_PROMPT_VERSION = "v3" +# v4: isolated problem-only solve flag (suppresses the embedded question-semantics draft) + +# STRUCTURE.md section-2 surface conventions added to the answer-format guidance. +PREDICTION_PROMPT_VERSION = "v4" +PROBLEM_PROMPT_NAME = "problem_semantics" +# Problem-only (answer-blind) q_prob build, shares the staged question-policy / assumption +# prompts with no answer context. +PROBLEM_PROMPT_VERSION = "v1" # Shared instructions keep the two prompt families aligned on the protocol # schema and on the constraint that only the final answer object should be @@ -49,6 +57,21 @@ - Keep `diagnostics` empty unless there is real uncertainty. """ +# STRUCTURE.md section-2 surface conventions, restated for the answer-format guidance so a +# plain `final_answer` surface is unambiguous to the deterministic parser (a_pred_ext). These +# are the same boundary tie-break rules the parser/canonicalizer share; stating them in the +# prompt keeps the model's surface and the parser's reading of it aligned. +_ANSWER_SURFACE_CONVENTIONS = """Write the final-answer surface using these conventions so it parses unambiguously: +- One indivisible value: write it bare (e.g. `5 m`, `x**2/2`). A single coordinate is one value, not a 1-tuple. +- Ordered coordinate of one object `(x, y)`: parentheses with >=2 finite parts, e.g. `(3, 4)`. A bare finite `(a, b)` is a tuple, NOT an interval. +- Unordered set of distinct solutions: braces, e.g. `{2, -2}`. +- Connected range (interval): bracket form only, e.g. `[a, b]`, `(a, b]`, or a range containing `inf`/`-inf`. Never use bare parentheses for a range. +- Several question-defined parts: label them, e.g. `(a) 5 m; (b) 2 s`. +- Shaped array: a vector as `<...>` or `[a, b, c]` (depth 1); a matrix as nested brackets (depth 2); keep a `(n,)` vector distinct from an `(n, 1)` matrix. +- Piecewise function: use `\\begin{cases}...\\end{cases}` or `Piecewise(...)`. +- Equation/relation: write the full relation, e.g. `F = m*a`, `v >= 0`. +""" + def build_reference_semantics_prompt( problem: PhysicsProblem, @@ -95,12 +118,19 @@ def build_prediction_semantics_prompt( *, draft_question_semantics: PhysicsQuestionSemantics | None = None, include_prediction_answer_semantics: bool = True, + suppress_question_semantics_draft: bool = False, ) -> str: - """Build the prompt for prediction-semantics generation.""" - - question_draft = draft_question_semantics or infer_prediction_question_semantics( - problem - ) + """Build the prompt for prediction-semantics generation. + + By default the prompt injects a toolkit draft of the question semantics as a hint (the + fused path). For the isolated problem-only solve (``a_pred_llm`` / ``a_pred_ext``), pass + ``suppress_question_semantics_draft=True``: the solve prompt is then problem text + + options + context only, with no embedded question-semantics draft — closing a leakage + surface (the draft is reference-conditioned policy the solver should not see) and keeping + the contract side strictly separate from the solve side. The STRUCTURE.md section-2 + surface conventions are added to the answer-format guidance so a plain ``final_answer`` + surface is unambiguous to the deterministic parser regardless of the draft hint. + """ sections = [ _COMMON_ROLE, @@ -114,10 +144,25 @@ def build_prediction_semantics_prompt( ) ), _format_problem(problem, include_reference_context=False), - "Toolkit heuristic draft question semantics:", - question_draft.model_dump_json(indent=2), - "Your `final_answer` must be only the final answer text.", ] + + if not suppress_question_semantics_draft: + question_draft = ( + draft_question_semantics or infer_prediction_question_semantics(problem) + ) + sections.extend( + [ + "Toolkit heuristic draft question semantics:", + question_draft.model_dump_json(indent=2), + ] + ) + + sections.extend( + [ + "Your `final_answer` must be only the final answer text.", + _ANSWER_SURFACE_CONVENTIONS, + ] + ) if include_prediction_answer_semantics: sections.append( "Make `prediction_answer_semantics` match that final answer exactly." @@ -129,6 +174,95 @@ def build_prediction_semantics_prompt( return "\n\n".join(sections) +# ---------------------------------------------------------------------------------------- +# Staged build prompts (3 focused calls). Each is advisory: the deterministic backbone pins +# structure/object_kind, tolerance, and allowed_*; these calls only clean/declare fields. +# ---------------------------------------------------------------------------------------- +_STAGED_BUILD_NOTE = ( + "The toolkit decides `structure`, `object_kind`, `tolerance`, and `allowed_*` " + "deterministically; do not try to change them. Only provide the fields this task asks for." +) + + +def build_answer_surface_cleanup_prompt( + problem: PhysicsProblem, + *, + golden_text: str, + draft_answer: PhysicsAnswerSemantics, +) -> str: + """Call A: clean the golden answer surface; structure/object_kind are pinned.""" + + return "\n\n".join( + [ + _COMMON_ROLE, + "Task: clean the canonical surface of the ground-truth answer semantics below.", + _STAGED_BUILD_NOTE, + "Keep `structure` and `object_kind` EXACTLY as in the draft (pinned by the toolkit). " + "Only improve `canonical_text`, `canonical_latex`, `unit`, `numeric_text`, " + "`choice_label`, and similar surface fields; preserve the answer's printed numeric " + "precision (do not round or add digits).", + _format_problem(problem, include_reference_context=True), + "Ground-truth answer surface:\n" + golden_text, + "Toolkit deterministic draft answer semantics (authoritative for structure/kind):", + draft_answer.model_dump_json(indent=2), + ] + ) + + +def build_question_policy_prompt( + problem: PhysicsProblem, + *, + answer_draft: PhysicsAnswerSemantics | None = None, +) -> str: + """Call B: question-side policy fields (q_ref when answer_draft given, else q_prob).""" + + sections = [ + _COMMON_ROLE, + "Task: return the question-side policy semantics that constrain acceptable answers.", + _STAGED_BUILD_NOTE, + "Provide `target_variable`, `symbol_aliases`, `question_unit_policy`, `question_unit`, " + "`dimension`, `ordering`, `required_parts`, `coordinate_frame`, `sign_convention`, and " + "`choice_space` when applicable. Leave `symbol_assumptions` empty here (declared separately).", + _format_problem(problem, include_reference_context=answer_draft is not None), + ] + if answer_draft is not None: + sections.append( + "Cleaned answer semantics for context (do not restate it; infer policy against it):" + ) + sections.append(answer_draft.model_dump_json(indent=2)) + return "\n\n".join(sections) + + +def build_symbol_assumptions_prompt( + problem: PhysicsProblem, + *, + candidate_symbols: tuple[str, ...], + answer_draft: PhysicsAnswerSemantics | None = None, +) -> str: + """Call C: justified real-domain declarations for the answer's free symbols.""" + + symbols_line = ( + ", ".join(candidate_symbols) + if candidate_symbols + else "(infer the free symbols from the problem)" + ) + sections = [ + _COMMON_ROLE, + "Task: declare the real-domain assumption for each free symbol, with justification.", + "Use ONLY canonical (post-alias) symbol tokens. Allowed assumptions: " + "real, nonzero, nonnegative, positive, complex.", + "Declare a stronger-than-real domain (positive/nonnegative/nonzero) ONLY when the " + "problem text explicitly justifies it (e.g. a stated constraint, a physical bound). " + "Never infer positivity from surface form; when unsure, declare `real` or omit the symbol.", + f"Candidate canonical symbols: {symbols_line}", + _format_problem(problem, include_reference_context=answer_draft is not None), + ] + if answer_draft is not None: + sections.append("Answer semantics for context:") + sections.append(answer_draft.model_dump_json(indent=2)) + return "\n\n".join(sections) + + def answer_like_to_text(answer: object) -> str: """Convert an answer-like object into a single answer surface string.""" @@ -194,9 +328,14 @@ def _problem_solution_text(problem: PhysicsProblem) -> str: __all__ = [ "PREDICTION_PROMPT_NAME", "PREDICTION_PROMPT_VERSION", + "PROBLEM_PROMPT_NAME", + "PROBLEM_PROMPT_VERSION", "REFERENCE_PROMPT_NAME", "REFERENCE_PROMPT_VERSION", "answer_like_to_text", + "build_answer_surface_cleanup_prompt", "build_prediction_semantics_prompt", + "build_question_policy_prompt", "build_reference_semantics_prompt", + "build_symbol_assumptions_prompt", ] diff --git a/src/prkit/semantics/inference/semantics_build.py b/src/prkit/semantics/inference/semantics_build.py new file mode 100644 index 0000000..55de5d7 --- /dev/null +++ b/src/prkit/semantics/inference/semantics_build.py @@ -0,0 +1,558 @@ +"""Deterministic helpers for objective question/answer semantics building. + +This module holds the *methodological core* of reference (`q_ref` + `a_ref`) and +problem-only (`q_prob`) semantics building: the deterministic, side-effect-free decisions +that the (advisory) LLM stages wrap. Everything here is offline-testable and embodies the +same precision discipline as the comparison engine (see +``../comparison/METHODOLOGY.md`` and ``../comparison/EQUIVALENCE.md``). + +Two engine-compatibility rules are enforced here (both surfaced by auditing the live +judgement before this lane was built): + +* **Tolerance is relative.** ``q.tolerance`` is consumed by ``numbers_close`` as a + *relative* tolerance (``tol * max(|a|, |b|)``; absolute only at zero), and + significant-figure agreement is handled separately from the reference's *printed* + precision. So we synthesize a relative tolerance and never convert it to absolute, and + precision is preserved by keeping ``a_ref``'s printed numeric surface intact. +* **Symbol assumptions use canonical (post-alias) tokens.** The engine looks up + ``q.symbol_assumptions`` by the canonical token that survives alias rewriting + (``context_symbol_assumption_map``). An assumption keyed by a raw alias token is silently + dropped at parse time, so every assumption symbol is resolved through the alias map here. + +Symbol assumptions are **declared, not derived** (METHODOLOGY.md §4): positivity / +nonnegativity is emitted only as a logical consequence of an explicit ``subject_to`` +constraint, or from an LLM declaration that is cross-checked against those constraints. +Dimension-priors and surface heuristics are never a source. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping, Sequence + +from ..comparison.contract import _STRUCTURES_COLLAPSIBLE_TO_ATOMIC +from ..schema import ( + DEFAULT_NUMERIC_TOLERANCE, + AnswerObjectKind, + AnswerStructure, + PhysicsAnswerSemantics, + PhysicsQuestionSemantics, + PhysicsSymbolAssumptionSemantics, + SymbolAssumption, +) + +# -------------------------------------------------------------------------------------- +# Symbol-assumption lattice (real-domain meet) +# -------------------------------------------------------------------------------------- +# +# Each assumption denotes a subset of the reals (or the complex plane for COMPLEX). We +# represent it by the set of SymPy-style flags it implies and combine two constraints on +# the same symbol by *intersecting* the denoted sets (the union of implied flags) -- the +# "most restrictive consistent" rule. The flags collapse back to the strongest single +# ``SymbolAssumption`` representing that intersection. +_ASSUMPTION_FLAGS: Mapping[SymbolAssumption, frozenset[str]] = { + SymbolAssumption.COMPLEX: frozenset(), + SymbolAssumption.REAL: frozenset({"real"}), + SymbolAssumption.NONZERO: frozenset({"real", "nonzero"}), + SymbolAssumption.NONNEGATIVE: frozenset({"real", "nonnegative"}), + SymbolAssumption.POSITIVE: frozenset( + {"real", "nonzero", "nonnegative", "positive"} + ), +} + + +def _assumption_from_flags(flags: frozenset[str]) -> SymbolAssumption: + """Collapse a flag set to the strongest single assumption it represents.""" + + if "positive" in flags or {"nonnegative", "nonzero"} <= flags: + return SymbolAssumption.POSITIVE + if "nonnegative" in flags: + return SymbolAssumption.NONNEGATIVE + if "nonzero" in flags: + return SymbolAssumption.NONZERO + if "real" in flags: + return SymbolAssumption.REAL + return SymbolAssumption.COMPLEX + + +def meet_assumptions( + left: SymbolAssumption, right: SymbolAssumption +) -> SymbolAssumption: + """Return the most-restrictive assumption consistent with both inputs. + + This is the intersection of the denoted domains (``x != 0`` and ``x >= 0`` together + mean ``x > 0``), so combining sound constraints stays sound. + """ + + return _assumption_from_flags(_ASSUMPTION_FLAGS[left] | _ASSUMPTION_FLAGS[right]) + + +# -------------------------------------------------------------------------------------- +# Alias resolution (canonical-token requirement) +# -------------------------------------------------------------------------------------- +def build_alias_map(question: PhysicsQuestionSemantics) -> dict[str, str]: + """Map every alias token to its canonical symbol for the question.""" + + alias_map: dict[str, str] = {} + for group in question.symbol_aliases: + canonical = group.canonical_symbol.strip() + if not canonical: + continue + for alias in group.aliases: + token = alias.strip() + if token: + alias_map[token] = canonical + return alias_map + + +def resolve_to_canonical(symbol: str, alias_map: Mapping[str, str]) -> str: + """Resolve ``symbol`` to its canonical token (identity when it is not an alias).""" + + return alias_map.get(symbol.strip(), symbol.strip()) + + +def alias_source_violations( + assumptions: Mapping[str, SymbolAssumption], + alias_map: Mapping[str, str], +) -> list[str]: + """Return assumption symbols that are alias *sources* (would be dropped at parse time).""" + + return sorted(symbol for symbol in assumptions if symbol in alias_map) + + +_CANDIDATE_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9_]*") +_FUNCTION_TOKENS = frozenset( + { + "sin", + "cos", + "tan", + "cot", + "sec", + "csc", + "exp", + "log", + "ln", + "sqrt", + "abs", + "pi", + "e", + } +) + + +def extract_candidate_symbols( + text: str | None, + *, + alias_map: Mapping[str, str] | None = None, +) -> tuple[str, ...]: + """Extract canonical candidate symbol tokens from an answer surface (Call C hints). + + These are only hints handed to the LLM; over-inclusion is harmless. Common math + function names are filtered so they are not offered as free symbols. + """ + + if not text: + return () + resolved_alias_map = dict(alias_map or {}) + ordered: list[str] = [] + for token in _CANDIDATE_TOKEN_RE.findall(text): + if token.lower() in _FUNCTION_TOKENS: + continue + canonical = resolve_to_canonical(token, resolved_alias_map) + if canonical not in ordered: + ordered.append(canonical) + return tuple(ordered) + + +# -------------------------------------------------------------------------------------- +# subject_to -> symbol assumptions (authoritative, deterministic) +# -------------------------------------------------------------------------------------- +_SYMBOL_RE = r"[A-Za-z\\][A-Za-z0-9_]*(?:_\{?[A-Za-z0-9]+\}?)?" +_NUMBER_RE = r"[+-]?(?:\d+\.?\d*|\.\d+)" +_BOUND_RE = rf"(?:{_NUMBER_RE}|{_SYMBOL_RE})" +_OP_RE = r"<=|>=|<|>|!=|==" + +_SYMBOL_OP_NUMBER_RE = re.compile( + rf"^\s*(?P{_SYMBOL_RE})\s*(?P{_OP_RE})\s*(?P{_NUMBER_RE})\s*$" +) +_NUMBER_OP_SYMBOL_RE = re.compile( + rf"^\s*(?P{_NUMBER_RE})\s*(?P{_OP_RE})\s*(?P{_SYMBOL_RE})\s*$" +) +# A chained ``lo OP sym OP hi`` constraint; each bound may be a number or a symbol, and a +# domain assumption is derived only from the numeric bound(s). +_CHAINED_RE = re.compile( + rf"^\s*(?P{_BOUND_RE})\s*(?P{_OP_RE})\s*(?P{_SYMBOL_RE})\s*" + rf"(?P{_OP_RE})\s*(?P{_BOUND_RE})\s*$" +) +_REAL_MEMBERSHIP_RE = re.compile( + rf"^\s*(?P{_SYMBOL_RE})\s*(?:\\in|∈|in)\s*(?:\\mathbb\{{R\}}|ℝ|R)\s*$" +) + +_UNICODE_OPS = { + "≥": ">=", + "⩾": ">=", + "≤": "<=", + "⩽": "<=", + "≠": "!=", + "＝": "==", +} + + +def _normalize_constraint_text(text: str) -> str: + """Normalize unicode comparison operators to ASCII for matching.""" + + cleaned = text.strip() + for unicode_op, ascii_op in _UNICODE_OPS.items(): + cleaned = cleaned.replace(unicode_op, ascii_op) + return cleaned + + +def _maybe_float(text: str) -> float | None: + """Parse ``text`` as a float, or ``None`` when it is a symbol (not numeric).""" + + try: + return float(text) + except ValueError: + return None + + +def _assumption_from_lower_bound( + bound: float, *, strict: bool +) -> SymbolAssumption | None: + """Sound assumption from ``symbol > bound`` (strict) or ``symbol >= bound``.""" + + if strict: + # symbol > bound; if bound >= 0 then symbol > 0. + return SymbolAssumption.POSITIVE if bound >= 0 else None + # symbol >= bound + if bound > 0: + return SymbolAssumption.POSITIVE + if bound == 0: + return SymbolAssumption.NONNEGATIVE + return None + + +def _assumption_from_upper_bound( + bound: float, *, strict: bool +) -> SymbolAssumption | None: + """Sound assumption from ``symbol < bound`` (strict) or ``symbol <= bound``. + + The lattice has no negative/nonpositive member, so a clearly-negative symbol is + expressed only as ``nonzero`` (real and nonzero); ``symbol <= 0`` yields only ``real``. + """ + + if strict: + # symbol < bound; if bound <= 0 then symbol < 0 (nonzero real). + return SymbolAssumption.NONZERO if bound <= 0 else None + # symbol <= bound + if bound < 0: + return SymbolAssumption.NONZERO + if bound == 0: + return SymbolAssumption.REAL + return None + + +def _assumption_from_comparison( + op: str, number: float, *, symbol_on_left: bool +) -> SymbolAssumption | None: + """Sound assumption for one ``symbol OP number`` (or reversed) comparison.""" + + # Normalize so the operator always reads "symbol OP number". + if not symbol_on_left: + op = {"<": ">", "<=": ">=", ">": "<", ">=": "<="}.get(op, op) + + if op == "!=": + return SymbolAssumption.NONZERO if number == 0 else None + if op == "==": + return None # an equality fixes a value, not a domain + if op in {">", ">="}: + return _assumption_from_lower_bound(number, strict=op == ">") + if op in {"<", "<="}: + return _assumption_from_upper_bound(number, strict=op == "<") + return None + + +def _parse_constraint(text: str) -> tuple[str, SymbolAssumption] | None: + """Parse one constraint surface into ``(symbol, assumption)`` when sound.""" + + cleaned = _normalize_constraint_text(text) + if not cleaned: + return None + + membership = _REAL_MEMBERSHIP_RE.match(cleaned) + if membership is not None: + return membership.group("sym"), SymbolAssumption.REAL + + chained = _CHAINED_RE.match(cleaned) + if chained is not None: + symbol = chained.group("sym") + lo_value = _maybe_float(chained.group("lo")) + hi_value = _maybe_float(chained.group("hi")) + lower = ( + _assumption_from_comparison( + chained.group("op1"), lo_value, symbol_on_left=False + ) + if lo_value is not None + else None + ) + upper = ( + _assumption_from_comparison( + chained.group("op2"), hi_value, symbol_on_left=True + ) + if hi_value is not None + else None + ) + combined: SymbolAssumption | None = None + for part in (lower, upper): + if part is not None: + combined = ( + part if combined is None else meet_assumptions(combined, part) + ) + return (symbol, combined) if combined is not None else None + + single = _SYMBOL_OP_NUMBER_RE.match(cleaned) + if single is not None: + assumption = _assumption_from_comparison( + single.group("op"), float(single.group("num")), symbol_on_left=True + ) + return (single.group("sym"), assumption) if assumption is not None else None + + reversed_single = _NUMBER_OP_SYMBOL_RE.match(cleaned) + if reversed_single is not None: + assumption = _assumption_from_comparison( + reversed_single.group("op"), + float(reversed_single.group("num")), + symbol_on_left=False, + ) + return ( + (reversed_single.group("sym"), assumption) + if assumption is not None + else None + ) + + return None + + +def _constraint_texts( + subject_to: Sequence[PhysicsAnswerSemantics], +) -> Iterable[str]: + """Yield each side-condition's most informative text surface.""" + + for constraint in subject_to: + text = (constraint.canonical_text or constraint.raw_text or "").strip() + if text: + yield text + + +def assumptions_from_subject_to( + subject_to: Sequence[PhysicsAnswerSemantics], + *, + alias_map: Mapping[str, str] | None = None, +) -> dict[str, SymbolAssumption]: + """Derive authoritative real-domain assumptions from ``subject_to`` constraints. + + Each emitted assumption is a logical consequence of an explicit constraint (e.g. + ``x > 0`` -> ``positive``), keyed by the *canonical* (post-alias) symbol token. + Multiple constraints on one symbol are combined with :func:`meet_assumptions`. + """ + + resolved_alias_map = dict(alias_map or {}) + derived: dict[str, SymbolAssumption] = {} + for text in _constraint_texts(subject_to): + parsed = _parse_constraint(text) + if parsed is None: + continue + raw_symbol, assumption = parsed + symbol = resolve_to_canonical(raw_symbol, resolved_alias_map) + if symbol in derived: + derived[symbol] = meet_assumptions(derived[symbol], assumption) + else: + derived[symbol] = assumption + return derived + + +def merge_symbol_assumptions( + authoritative: Mapping[str, SymbolAssumption], + advisory: Mapping[str, SymbolAssumption], +) -> tuple[dict[str, SymbolAssumption], list[str]]: + """Merge ``subject_to``-derived (authoritative) and LLM-declared (advisory) maps. + + Most-restrictive-wins where the two sources are consistent (their domains intersect to + a nonempty set, which always holds in this lattice). The returned flag list records + symbols where the advisory source *strengthened* an authoritative constraint, for + provenance review -- the merge still adopts the (sound) intersection. + """ + + merged: dict[str, SymbolAssumption] = dict(authoritative) + flags: list[str] = [] + for symbol, advised in advisory.items(): + if symbol in merged: + combined = meet_assumptions(merged[symbol], advised) + if combined != merged[symbol]: + flags.append( + f"advisory_strengthened:{symbol}:{merged[symbol].value}->{combined.value}" + ) + merged[symbol] = combined + else: + merged[symbol] = advised + return merged, flags + + +def assumptions_to_semantics( + assumptions: Mapping[str, SymbolAssumption], +) -> tuple[PhysicsSymbolAssumptionSemantics, ...]: + """Render an assumption map into the schema tuple (sorted for determinism).""" + + return tuple( + PhysicsSymbolAssumptionSemantics(symbol=symbol, assumption=assumptions[symbol]) + for symbol in sorted(assumptions) + ) + + +# -------------------------------------------------------------------------------------- +# Tolerance synthesis (relative; never absolute) +# -------------------------------------------------------------------------------------- +_RELATIVE_TOLERANCE_RE = re.compile( + r"(?:within|to within|±|\+/-|relative\s+error|accuracy|tolerance|precision)" + r"[^%\d]{0,12}?(?P\d+(?:\.\d+)?)\s*%", + re.IGNORECASE, +) + + +def parse_relative_tolerance_instruction(text: str | None) -> float | None: + """Return a relative tolerance from an explicit percentage instruction, else ``None``. + + Recognizes phrasings like "within 1%", "to within 0.5 %", "±2%". Significant-figure / + decimal-place phrasing is intentionally *not* mapped here: the engine derives that from + the reference's printed precision, so the build preserves ``a_ref``'s numeric surface + instead of tightening ``q.tolerance``. + """ + + if not text: + return None + match = _RELATIVE_TOLERANCE_RE.search(text) + if match is None: + return None + percent = float(match.group("pct")) + if percent <= 0: + return None + return percent / 100.0 + + +def infer_answer_tolerance( + *, + instruction_text: str | None = None, + relative_tolerance: float | None = None, + default: float = DEFAULT_NUMERIC_TOLERANCE, +) -> float: + """Synthesize a **relative** ``q.tolerance``. + + Precedence: an explicit relative tolerance argument, then a parsed percentage + instruction, then the default. Never converts to absolute and never tightens past the + reference's printed precision (which the engine handles separately). + """ + + if relative_tolerance is not None and relative_tolerance > 0: + return relative_tolerance + parsed = parse_relative_tolerance_instruction(instruction_text) + if parsed is not None: + return parsed + return default + + +# -------------------------------------------------------------------------------------- +# allowed_* reconciliation (compat #3) and q_ref <-> a_ref mutual consistency +# -------------------------------------------------------------------------------------- +def reconcile_allowed_sets( + question: PhysicsQuestionSemantics, + reference_answer: PhysicsAnswerSemantics, +) -> PhysicsQuestionSemantics: + """Ensure ``question`` admits the gold answer's kind/structure and its collapse target. + + ``allowed_object_kinds`` / ``allowed_structures`` express *question-level* admissibility + and are permissive by default. The contract gate treats them as hard violating-gates (no + bridge rescue), so an over-narrow set turns a cross-kind-equivalent or + degenerate-collapsed prediction into a false ``contract_violation``. This helper only ever + *widens*: it admits ``a_ref``'s realized kind and structure, and -- when a structure that + ``canonicalize_structure`` can reduce is admitted -- admits ``ATOMIC`` too (mirroring the + contract's collapse reconciliation). It never narrows; narrowing is a justified precision + choice the builder makes elsewhere only on explicit question evidence. + """ + + kinds = set(question.allowed_object_kinds) + kinds.add(reference_answer.object_kind) + structures = set(question.allowed_structures) + structures.add(reference_answer.structure) + if structures & _STRUCTURES_COLLAPSIBLE_TO_ATOMIC: + structures.add(AnswerStructure.ATOMIC) + + return question.merged( + { + "allowed_object_kinds": tuple( + kind for kind in AnswerObjectKind if kind in kinds + ), + "allowed_structures": tuple( + structure for structure in AnswerStructure if structure in structures + ), + } + ) + + +def reference_pair_consistency( + question: PhysicsQuestionSemantics, + reference_answer: PhysicsAnswerSemantics, +) -> list[str]: + """Return ``q_ref`` <-> ``a_ref`` inconsistencies (empty when mutually consistent). + + A co-constructed pair must satisfy: the contract admits the gold answer's kind/structure + (honoring the collapse target), any shared ``target_variable`` agrees, and every + ``symbol_assumptions`` token is canonical (post-alias) so the engine will not silently + drop it. + """ + + issues: list[str] = [] + + if reference_answer.object_kind not in question.allowed_object_kinds: + issues.append(f"kind_not_admitted:{reference_answer.object_kind.value}") + + structure_admitted = reference_answer.structure in question.allowed_structures or ( + reference_answer.structure == AnswerStructure.ATOMIC + and bool(set(question.allowed_structures) & _STRUCTURES_COLLAPSIBLE_TO_ATOMIC) + ) + if not structure_admitted: + issues.append(f"structure_not_admitted:{reference_answer.structure.value}") + + if ( + question.target_variable + and reference_answer.target_variable + and question.target_variable != reference_answer.target_variable + ): + issues.append( + "target_variable_mismatch:" + f"{question.target_variable}!={reference_answer.target_variable}" + ) + + alias_map = build_alias_map(question) + declared = {entry.symbol: entry.assumption for entry in question.symbol_assumptions} + issues.extend( + f"assumption_alias_source:{symbol}" + for symbol in alias_source_violations(declared, alias_map) + ) + + return issues + + +__all__ = [ + "alias_source_violations", + "assumptions_from_subject_to", + "assumptions_to_semantics", + "build_alias_map", + "extract_candidate_symbols", + "infer_answer_tolerance", + "meet_assumptions", + "merge_symbol_assumptions", + "parse_relative_tolerance_instruction", + "reconcile_allowed_sets", + "reference_pair_consistency", + "resolve_to_canonical", +] diff --git a/src/prkit/semantics/inference/strict_models.py b/src/prkit/semantics/inference/strict_models.py index cc3d2a3..6de8e3e 100644 --- a/src/prkit/semantics/inference/strict_models.py +++ b/src/prkit/semantics/inference/strict_models.py @@ -18,8 +18,10 @@ PhysicsAnswerSemantics, PhysicsQuestionSemantics, PhysicsSymbolAliasSemantics, + PhysicsSymbolAssumptionSemantics, QuestionSymbolicMode, QuestionUnitPolicy, + SymbolAssumption, ) @@ -40,6 +42,14 @@ class StrictPhysicsQuestionSemantics(_StrictInferenceModel): default_factory=tuple, description="Question-conditioned symbol alias groups.", ) + symbol_assumptions: tuple[PhysicsSymbolAssumptionSemantics, ...] = Field( + default_factory=tuple, + description=( + "Question-conditioned real-domain declarations for free symbols. Each `symbol` " + "must be the canonical (post-alias) token. Declare a stronger-than-real domain " + "(positive/nonnegative/nonzero) only when the problem justifies it; never guess." + ), + ) allowed_object_kinds: tuple[AnswerObjectKind, ...] = Field( default_factory=lambda: tuple(AnswerObjectKind), description="Semantic answer kinds admitted by the question.", @@ -234,6 +244,27 @@ class StrictPredictionSemanticsResponse(_StrictInferenceModel): ) +class StrictPredictionIsolatedResponse(_StrictInferenceModel): + """Provider-facing response for the ISOLATED problem-only solve (a_pred_llm path). + + Unlike :class:`StrictPredictionSemanticsResponse`, this model deliberately OMITS + ``question_semantics``: the prediction side never authors a judgement contract (that is + ``q_ref`` / ``q_prob``, built separately), and a prediction-side ``q`` is both unused and + a leakage surface. The model returns only its reasoning, the final-answer surface, and the + predicted answer semantics, from which ``a_pred_llm`` is built. + """ + + reasoning: str = Field( + description="Concise reasoning summary used to produce the final answer.", + ) + final_answer: str = Field( + description="Final answer surface form only.", + ) + prediction_answer_semantics: StrictPhysicsAnswerSemantics = Field( + description="Canonical semantics for the predicted final answer.", + ) + + class StrictPredictionFinalAnswerResponse(_StrictInferenceModel): """Compact provider-facing response for models with strict schema limits.""" @@ -245,18 +276,49 @@ class StrictPredictionFinalAnswerResponse(_StrictInferenceModel): ) +class StrictSymbolAssumptionDeclaration(_StrictInferenceModel): + """One justified symbol-domain declaration (Call C of the staged build).""" + + symbol: str = Field( + description="Canonical (post-alias) symbol token the assumption applies to.", + ) + assumption: SymbolAssumption = Field( + description="Real-domain the symbol ranges over: real/nonzero/nonnegative/positive/complex.", + ) + justification: str = Field( + description=( + "Why the problem justifies this domain. Required for any stronger-than-real " + "assumption (positive/nonnegative/nonzero); never guess from surface form." + ), + ) + + +class StrictSymbolAssumptionsResponse(_StrictInferenceModel): + """Provider-facing response for the symbol-assumption declaration call.""" + + assumptions: tuple[StrictSymbolAssumptionDeclaration, ...] = Field( + default_factory=tuple, + description="Justified real-domain declarations for the answer's free symbols.", + ) + + StrictPhysicsAnswerCaseSemantics.model_rebuild() StrictPhysicsAnswerSemantics.model_rebuild() StrictReferenceSemanticsResponse.model_rebuild() StrictPredictionSemanticsResponse.model_rebuild() +StrictPredictionIsolatedResponse.model_rebuild() StrictPredictionFinalAnswerResponse.model_rebuild() +StrictSymbolAssumptionsResponse.model_rebuild() __all__ = [ "StrictPhysicsAnswerCaseSemantics", "StrictPhysicsAnswerSemantics", "StrictPredictionFinalAnswerResponse", + "StrictPredictionIsolatedResponse", "StrictPhysicsQuestionSemantics", "StrictPredictionSemanticsResponse", "StrictReferenceSemanticsResponse", + "StrictSymbolAssumptionDeclaration", + "StrictSymbolAssumptionsResponse", ] diff --git a/src/prkit/semantics/normalization/question_inference.py b/src/prkit/semantics/normalization/question_inference.py index d091407..c559b53 100644 --- a/src/prkit/semantics/normalization/question_inference.py +++ b/src/prkit/semantics/normalization/question_inference.py @@ -158,6 +158,7 @@ "answer_parts", "source_answer_text", "symbol_aliases", + "symbol_assumptions", } ) diff --git a/src/prkit/verify/__init__.py b/src/prkit/verify/__init__.py index 2d4f5a7..cb2f266 100644 --- a/src/prkit/verify/__init__.py +++ b/src/prkit/verify/__init__.py @@ -16,13 +16,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, cast from prkit.core.verdict import Verdict if TYPE_CHECKING: # annotations only — never imported at runtime by this module from prkit.core.domain.answer import Answer - from prkit.semantics import PhysicsAnswerSemantics + from prkit.semantics import PhysicsAnswerSemantics, PhysicsQuestionSemantics + from prkit.semantics.inference import ReferenceSemanticsArtifact __all__ = ["parse", "verify", "Verdict"] @@ -33,6 +34,37 @@ _RECOGNIZED_UNIT_POLICIES = ("strict", "audited", "permissive") +def _resolve_question_context( + context: ( + PhysicsQuestionSemantics | ReferenceSemanticsArtifact | dict[str, Any] | None + ), +) -> PhysicsQuestionSemantics | dict[str, Any] | None: + """Coerce a caller-supplied judgement context into question semantics (``q_ref``). + + The judgement contract ``q`` is read only from the ``context`` arg of the engine, + so this is how a caller threads a reference-built ``q_ref`` (e.g. with + ``symbol_assumptions`` that unlock a domain-gated symbolic accept) into a plain + ``verify(...)`` call. ``None`` preserves the historical default (empty context). + + A :class:`~prkit.semantics.inference.ReferenceSemanticsArtifact` (or anything else + exposing ``.question_semantics``) is accepted and unwrapped to its + ``question_semantics`` — duck-typed so this light-import facade never has to import + the heavy inference layer that defines the artifact. A ``PhysicsQuestionSemantics`` + or a plain dict is passed straight through to the scorer's own coercion. + """ + if context is None: + return None + question_semantics: PhysicsQuestionSemantics | None = getattr( + context, "question_semantics", None + ) + if question_semantics is not None: + return question_semantics + # Not an artifact wrapper: a PhysicsQuestionSemantics or a plain dict. mypy + # cannot statically rule out the (duck-typed) artifact branch, so cast to the + # scorer-accepted shape. + return cast("PhysicsQuestionSemantics | dict[str, Any]", context) + + def parse(text: str, *, category: object | None = None) -> PhysicsAnswerSemantics: """Normalize a raw answer surface into typed physics semantics. @@ -58,6 +90,9 @@ def verify( tolerance: float | None = None, unit_policy: str = "strict", partial_credit: bool = False, + context: ( + PhysicsQuestionSemantics | ReferenceSemanticsArtifact | dict[str, Any] | None + ) = None, ) -> Verdict: """Verify a predicted physics answer against the gold answer. @@ -75,6 +110,13 @@ def verify( partial_credit: when ``True``, score with the graded EED/SEED :class:`~prkit.scoring.PartialCreditScorer` (which populates ``Verdict.partial_credit``) instead of the binary deterministic engine. + context: optional question contract (``q_ref``) supplying the judgement with + the question's domain/policy fields — e.g. ``symbol_assumptions`` that + unlock a domain-gated symbolic accept. May be a + :class:`~prkit.semantics.PhysicsQuestionSemantics`, a dict, or a + :class:`~prkit.semantics.inference.ReferenceSemanticsArtifact` (its + ``question_semantics`` is used). Defaults to ``None`` (empty context), + preserving the historical behavior. Raises: ValueError: if ``unit_policy`` is not a recognized value. @@ -85,6 +127,8 @@ def verify( f"got {unit_policy!r}" ) + question_context = _resolve_question_context(context) + # Lazy: keeps anthropic/openai/google.genai/datasets/pandas/sympy off the # bare ``import prkit.verify`` path (provider SDKs are lazy in model_clients). # math-verify is verify(gold, pred); the Scorer scores prediction vs reference, @@ -93,9 +137,9 @@ def verify( from prkit.scoring import PartialCreditScorer pc_scorer = PartialCreditScorer(tolerance=tolerance, policy_mode=unit_policy) - return pc_scorer.score(pred, gold) + return pc_scorer.score(pred, gold, context=question_context) from prkit.scoring import SemanticsScorer scorer = SemanticsScorer(tolerance=tolerance, policy_mode=unit_policy) - return scorer.score(pred, gold) + return scorer.score(pred, gold, context=question_context) diff --git a/tests/prkit/semantics/test_compare_predictions.py b/tests/prkit/semantics/test_compare_predictions.py new file mode 100644 index 0000000..165fc64 --- /dev/null +++ b/tests/prkit/semantics/test_compare_predictions.py @@ -0,0 +1,252 @@ +"""Tests for the symmetric reference-free entry point ``compare_predictions``. + +``compare_predictions(a_i, a_j; q_prob)`` is the reference-free judgement +``Eq(a_pred_i, a_pred_j; q_prob)`` used for clustering, where *neither* side is gold. +These tests pin the three soundness properties the reference-based +``compare_protocol_answers`` cannot provide here (audit #4): + +1. **Symmetry** — ``compare(a, b)`` and ``compare(b, a)`` always agree on equivalence, + across a battery of pairs (numeric, symbolic, categorical, cross-kind, structured). +2. **No self-rejection** — a contract derived from ``q_prob`` never yields + ``reference_contract_violation`` (that mode presupposes a gold second argument). +3. **No printed-precision bias** — an accept driven by one side's printed precision in + the reference-based path (e.g. ``9.81`` rounding to a coarser ``9.8`` reference) is + *not* accepted reference-free, because neither side may set the precision bar; genuine + agreement within the relative ``q_prob.tolerance`` is accepted in both directions. +""" + +from __future__ import annotations + +import itertools + +import pytest + +from prkit.semantics import ( + ComparisonPolicyMode, + compare_predictions, + normalize_physics_answer, +) + + +def _cp(a: str, b: str, **kwargs: object): + """Compare two plain-text answers reference-free via normalized records.""" + + return compare_predictions( + normalize_physics_answer(a), + normalize_physics_answer(b), + **kwargs, # type: ignore[arg-type] + ) + + +# A battery of representative answer surfaces spanning every routing path. +_BATTERY = ( + "0.5", + "1/2", + "0.7", + "9.8", + "9.81", + "1/3", + "0.333", + "5", + "E = 5", + "F = m a", + "a = F/m", + "F = m/a", + "v t", + "t v", + "x^2", + "x^3", + "no change", + "0", + "increases", + "goes up", + "{1, 2}", + "{2, 1}", + "(1, 2)", +) + + +@pytest.mark.parametrize( + "left,right", itertools.combinations_with_replacement(_BATTERY, 2) +) +def test_compare_predictions_is_symmetric(left: str, right: str) -> None: + forward = _cp(left, right) + backward = _cp(right, left) + assert forward.equivalent == backward.equivalent, ( + left, + right, + forward.comparison_mode, + backward.comparison_mode, + ) + + +@pytest.mark.parametrize( + "left,right", itertools.combinations_with_replacement(_BATTERY, 2) +) +def test_compare_predictions_never_reference_contract_violation( + left: str, right: str +) -> None: + # The contract is derived from q_prob, not from a gold answer, so the + # reference-violation mode is incoherent and must never surface. + for first, second in ((left, right), (right, left)): + result = _cp(first, second) + assert result.comparison_mode != "reference_contract_violation" + + +def test_genuine_match_is_equivalent_both_directions() -> None: + forward = _cp("F = m a", "a = F/m") + backward = _cp("a = F/m", "F = m a") + assert forward.equivalent is True + assert backward.equivalent is True + assert forward.comparison_mode == "relation" + + +def test_commutative_expression_match_both_directions() -> None: + left = {"object_kind": "expression", "canonical_text": "v t", "structure": "atomic"} + right = { + "object_kind": "expression", + "canonical_text": "t v", + "structure": "atomic", + } + forward = compare_predictions(left, right) + backward = compare_predictions(right, left) + assert forward.equivalent is True + assert backward.equivalent is True + assert forward.comparison_mode == "expression" + + +def test_distinct_numbers_rejected_both_directions() -> None: + assert _cp("0.5", "0.7").equivalent is False + assert _cp("0.7", "0.5").equivalent is False + + +def test_exact_rational_decimal_match_is_symmetric() -> None: + # 0.5 and 1/2 denote the same number, accepted either way. + assert _cp("0.5", "1/2").equivalent is True + assert _cp("1/2", "0.5").equivalent is True + + +def test_printed_precision_does_not_set_the_bar() -> None: + # In the reference-based path 9.81 rounds to a coarser 9.8 *reference* and is accepted, + # but the reverse is not -- an order-sensitive accept. Reference-free, neither side is + # gold, so the precision-driven accept must be withheld in BOTH directions. + forward = _cp("9.81", "9.8") + backward = _cp("9.8", "9.81") + assert forward.equivalent is False + assert backward.equivalent is False + assert forward.equivalent == backward.equivalent + + +def test_relative_tolerance_accepts_close_pair_both_directions() -> None: + # A genuine agreement within the relative q_prob.tolerance is symmetric and accepted. + context = {"tolerance": 0.01} + forward = _cp("9.81", "9.8", context=context) + backward = _cp("9.8", "9.81", context=context) + assert forward.equivalent is True + assert backward.equivalent is True + + +def test_relative_tolerance_rejects_far_pair_both_directions() -> None: + context = {"tolerance": 0.01} + forward = _cp("10.0", "9.8", context=context) + backward = _cp("9.8", "10.0", context=context) + assert forward.equivalent is False + assert backward.equivalent is False + + +def test_cross_kind_bridge_is_symmetric_under_permissive_default() -> None: + # relation_rhs bridge: `E = 5` vs `5`. Reference-free defaults to permissive, so the + # bridge fires; the wrapper enforces it both ways. + forward = _cp("E = 5", "5") + backward = _cp("5", "E = 5") + assert forward.equivalent == backward.equivalent + assert forward.equivalent is True + + +def test_set_order_insensitive_match_is_symmetric() -> None: + assert _cp("{1, 2}", "{2, 1}").equivalent is True + assert _cp("{2, 1}", "{1, 2}").equivalent is True + + +def test_adversarial_reject_stays_non_equivalent() -> None: + # x^2 vs x^3 are genuinely different functions; rejected both ways. + assert _cp("x^2", "x^3").equivalent is False + assert _cp("x^3", "x^2").equivalent is False + + +def test_self_derived_contract_violation_is_not_reference_violation() -> None: + # An out-of-space choice would be a `reference_contract_violation` in the reference-based + # path (second arg validated as gold). Reference-free, the contract is self-derived from + # q_prob, so the violation is a plain `contract_violation`, symmetric in both directions. + context = { + "allowed_object_kinds": ["choice"], + "allowed_structures": ["atomic"], + "choice_space": ["A", "B", "C"], + } + out_of_space = { + "object_kind": "choice", + "canonical_text": "Z", + "choice_label": "Z", + "structure": "atomic", + } + forward = compare_predictions( + out_of_space, + out_of_space, + context=context, + policy_mode=ComparisonPolicyMode.STRICT, + ) + assert forward.equivalent is False + assert forward.comparison_mode == "contract_violation" + assert forward.comparison_mode != "reference_contract_violation" + + +def test_in_space_choice_accepts_under_strict_self_derived_contract() -> None: + context = { + "allowed_object_kinds": ["choice"], + "allowed_structures": ["atomic"], + "choice_space": ["A", "B", "C"], + } + in_space = { + "object_kind": "choice", + "canonical_text": "B", + "choice_label": "B", + "structure": "atomic", + } + forward = compare_predictions( + in_space, + in_space, + context=context, + policy_mode=ComparisonPolicyMode.STRICT, + ) + backward = compare_predictions( + in_space, + in_space, + context=context, + policy_mode=ComparisonPolicyMode.STRICT, + ) + assert forward.equivalent is True + assert backward.equivalent is True + assert forward.comparison_mode == "choice" + + +def test_single_allowed_kind_pins_expected_without_favoring_a_side() -> None: + # When q_prob pins a single allowed kind/structure, both predictions are judged against + # that pinned expectation -- not against either prediction's own kind. + context = { + "allowed_object_kinds": ["number"], + "allowed_structures": ["atomic"], + } + forward = compare_predictions( + normalize_physics_answer("5"), + normalize_physics_answer("5"), + context=context, + policy_mode=ComparisonPolicyMode.STRICT, + ) + assert forward.equivalent is True + assert forward.comparison_mode == "number" + + +def test_top_level_export_is_reachable() -> None: + from prkit.semantics.comparison import compare_predictions as via_comparison + + assert via_comparison is compare_predictions diff --git a/tests/prkit/semantics/test_inference_prompts.py b/tests/prkit/semantics/test_inference_prompts.py index a0afbc0..95b3ad5 100644 --- a/tests/prkit/semantics/test_inference_prompts.py +++ b/tests/prkit/semantics/test_inference_prompts.py @@ -189,9 +189,9 @@ def response( ) -def test_infer_prediction_semantics_uses_native_json_schema_with_strict_response_model() -> ( - None -): +def test_infer_prediction_semantics_isolated_uses_isolated_response_model() -> None: + # The default isolated solve requests the isolated response model (no question_semantics) + # and the solve prompt suppresses the embedded question-semantics draft. model_client = _PredictionStubModelClient() artifact = infer_prediction_semantics( @@ -203,11 +203,37 @@ def test_infer_prediction_semantics_uses_native_json_schema_with_strict_response assert model_client.last_response_format is not None assert model_client.last_response_format["type"] == "json_schema" assert ( - model_client.last_response_format["name"] == "StrictPredictionSemanticsResponse" + model_client.last_response_format["name"] == "StrictPredictionIsolatedResponse" + ) + assert "Toolkit heuristic draft question semantics:" not in ( + model_client.last_prompt or "" ) assert "Return ONLY a JSON object matching this JSON Schema:" not in ( model_client.last_prompt or "" ) + # Prediction side authors no contract in the isolated artifact. + assert artifact.question_semantics == PhysicsQuestionSemantics() + + +def test_infer_prediction_semantics_fused_uses_strict_response_model() -> None: + # The legacy fused path (isolated_solve=False) still uses the full response model and + # injects the question-semantics draft. + model_client = _PredictionStubModelClient() + + artifact = infer_prediction_semantics( + _build_problem(), + model_client, + isolated_solve=False, + ) + + assert artifact.generator.structured_output_mode == "json_schema" + assert model_client.last_response_format is not None + assert ( + model_client.last_response_format["name"] == "StrictPredictionSemanticsResponse" + ) + assert "Toolkit heuristic draft question semantics:" in ( + model_client.last_prompt or "" + ) class _AnthropicPlanStubModelClient(_PredictionStubModelClient): @@ -734,20 +760,106 @@ def response( **kwargs: Any, ) -> str: del input, image_paths, response_format, kwargs - raise AssertionError( - "chat should not be called when native json_schema is unsupported" + raise RuntimeError("stub model cannot produce any output") + + +class _PlainTextPredictionStubModelClient(BaseModelClient): + """A provider that can chat (plain text) but cannot enforce native structured output. + + Models the realistic "lacks native structured output" case from the three-step + ecosystem: the isolated solve should degrade to ``a_pred_ext`` (plain text -> deterministic + extraction), never fail. + """ + + def __init__(self) -> None: + super().__init__(model="stub-model") + self.provider = "stub" + + def _resolve_structured_output_plan( + self, + spec, + *, + structured_policy, + ) -> StructuredOutputPlan: + # Can enforce the simple compact schema natively, but NOT the complex isolated + # answer-semantics schema -> the isolated solve falls back to a_pred_ext only. + del structured_policy + if getattr(spec, "name", "") == "StrictPredictionFinalAnswerResponse": + return StructuredOutputPlan( + mode="json_schema", + strategy="stub_native", + native_schema_enforced=True, + accepted_artifact_modes=("json_schema",), + accepted_artifact_strategies=("stub_native",), + response_format={ + "type": "json_schema", + "name": spec.name, + "schema": spec.schema, + }, + prompt_suffix=None, + ) + return StructuredOutputPlan( + mode="prompt_only", + strategy="stub_prompt_only", + native_schema_enforced=False, + accepted_artifact_modes=("prompt_only",), + accepted_artifact_strategies=("stub_prompt_only",), + response_format=None, + prompt_suffix="", ) + def response( + self, + input: str, + image_paths: list[str] | None = None, + response_format: dict[str, Any] | type | None = None, + **kwargs: Any, + ) -> str: + del input, image_paths, response_format, kwargs + return json.dumps( + {"reasoning": "Use Newton's second law.", "final_answer": "5 N"} + ) -def test_infer_prediction_semantics_requires_native_json_schema_support() -> None: - with pytest.raises( - ValueError, match="requires native provider-enforced structured output support" - ): - infer_prediction_semantics(_build_problem(), _NoStructuredOutputModelClient()) + +def test_infer_prediction_semantics_isolated_degrades_to_extracted_without_native() -> ( + None +): + # Step 2 without native structured output: only a_pred_ext (plain -> deterministic + # extraction) is produced, never a failure. + artifact = infer_prediction_semantics( + _build_problem(), _PlainTextPredictionStubModelClient() + ) + + assert artifact.prediction_answer_semantics.canonical_text + assert artifact.build_report is not None + assert artifact.build_report.build_method == "prediction_isolated_extracted" + assert "a_pred_llm_unavailable" in artifact.build_report.flags -def test_infer_reference_semantics_requires_native_json_schema_support() -> None: +def test_infer_prediction_semantics_fused_requires_native_json_schema_support() -> None: + # The legacy fused path still hard-requires native structured output. with pytest.raises( ValueError, match="requires native provider-enforced structured output support" ): - infer_reference_semantics(_build_problem(), _NoStructuredOutputModelClient()) + infer_prediction_semantics( + _build_problem(), + _NoStructuredOutputModelClient(), + isolated_solve=False, + ) + + +def test_infer_reference_semantics_degrades_without_native_structured_output() -> None: + # Step 1 (reference build) never fails on native-output lack. The advisory calls run + # best-effort; even when the provider can produce no parseable output at all, the + # deterministic backbone still yields a valid q_ref + a_ref. Lacking native structured + # output is a normal route (a Step-2 concern), not a defect — so it does NOT force review, + # though the unavailable advisory calls are recorded as informational flags. + artifact = infer_reference_semantics( + _build_problem(), _NoStructuredOutputModelClient() + ) + + assert artifact.reference_answer_semantics.canonical_text + assert artifact.build_report is not None + assert artifact.build_report.review_required is False + assert artifact.build_report.cross_checks_passed is True + assert any("unavailable" in flag for flag in artifact.build_report.flags) diff --git a/tests/prkit/semantics/test_prediction_isolated_build.py b/tests/prkit/semantics/test_prediction_isolated_build.py new file mode 100644 index 0000000..97b0555 --- /dev/null +++ b/tests/prkit/semantics/test_prediction_isolated_build.py @@ -0,0 +1,199 @@ +"""Tests for the isolated problem-only prediction build (a_pred_llm + a_pred_ext). + +Covers WS B: the solve prompt suppresses the embedded question-semantics draft and carries +the STRUCTURE.md section-2 surface conventions; ``a_pred_llm`` is the LLM-structured record +(canonicalized) and ``a_pred_ext`` is the deterministic +``canonicalize_structure(normalize_physics_answer(...))`` extraction; their structure +disagreement is flagged (never reconciled); a standalone plain-text answer builds +``a_pred_ext`` with no generation; and a gold ``subject_to`` does not leak into the solve +prompt. A fake structured-output client returns canned responses keyed by the response schema +name so the orchestration is exercised offline. +""" + +from __future__ import annotations + +import json +from typing import Any + +from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem +from prkit.core.model_clients import BaseModelClient +from prkit.semantics.inference.calls import ( + build_extracted_prediction_semantics_artifact, + extract_prediction_answer_semantics, + infer_prediction_semantics, + resolve_isolated_prediction_response_model, +) +from prkit.semantics.inference.prompts import build_prediction_semantics_prompt +from prkit.semantics.inference.strict_models import ( + StrictPredictionIsolatedResponse, +) +from prkit.semantics.normalization.question_inference import ( + infer_prediction_question_semantics, +) +from prkit.semantics.schema import AnswerObjectKind, AnswerStructure + + +def _problem() -> PhysicsProblem: + return PhysicsProblem( + problem_id="pred-iso-1", + question="Find the speed v.", + answer=Answer(value="sqrt(E/m), m > 0", answer_category=AnswerCategory.FORMULA), + solution="Use conservation of energy.", + domain="mechanics", + additional_fields={ + "symbol_assumptions": [{"symbol": "m", "assumption": "positive"}], + }, + ) + + +class _IsolatedSolveStubModelClient(BaseModelClient): + """Returns a canned isolated-solve response (reasoning + final_answer + answer).""" + + supports_response_format_json_schema = True + + def __init__(self, *, answer_payload: dict[str, Any] | None = None) -> None: + super().__init__(model="stub-model") + self.provider = "stub" + self.prompts: list[str] = [] + self.response_formats: list[Any] = [] + self._answer_payload = answer_payload or { + "canonical_text": "sqrt(E/m)", + "object_kind": "expression", + "structure": "atomic", + } + + def response( + self, + input: str, + image_paths: list[str] | None = None, + response_format: dict[str, Any] | type | None = None, + **kwargs: Any, + ) -> str: + del image_paths, kwargs + self.prompts.append(input) + self.response_formats.append(response_format) + name = ( + response_format.get("name") if isinstance(response_format, dict) else None + ) + if name == "StrictPredictionIsolatedResponse": + return json.dumps( + { + "reasoning": "Energy conservation.", + "final_answer": "sqrt(E/m)", + "prediction_answer_semantics": self._answer_payload, + } + ) + raise AssertionError(f"unexpected response schema: {name}") + + +def test_isolated_solve_prompt_suppresses_question_semantics_draft() -> None: + prompt = build_prediction_semantics_prompt( + _problem(), suppress_question_semantics_draft=True + ) + + assert "Toolkit heuristic draft question semantics:" not in prompt + # Surface conventions are present so a plain surface parses unambiguously. + assert "parses unambiguously" in prompt + assert "braces" in prompt or "{2, -2}" in prompt + # Solve prompt is problem + options + context only (no golden answer/solution). + assert "Answer:\nsqrt(E/m)" not in prompt + assert "Solution:" not in prompt + + +def test_fused_solve_prompt_still_injects_draft_by_default() -> None: + prompt = build_prediction_semantics_prompt(_problem()) + + assert "Toolkit heuristic draft question semantics:" in prompt + + +def test_isolated_solve_builds_a_pred_llm_and_records_provenance() -> None: + client = _IsolatedSolveStubModelClient() + artifact = infer_prediction_semantics(_problem(), client) + + # a_pred_llm adopted (LLM structure, deterministic-canonicalized). + a_pred = artifact.prediction_answer_semantics + assert a_pred.object_kind == AnswerObjectKind.EXPRESSION + assert a_pred.structure == AnswerStructure.ATOMIC + # Prediction side never authors a contract. + assert artifact.question_semantics.target_variable is None + assert artifact.build_report is not None + assert artifact.build_report.build_method == "prediction_isolated_llm" + # Agreeing structures -> no disagreement flag. + assert not any( + "a_pred_llm_vs_ext_disagreement" in flag for flag in artifact.build_report.flags + ) + + +def test_isolated_solve_flags_structure_disagreement_without_reconciling() -> None: + # The LLM claims a `set` structure, but the deterministic a_pred_ext reads `sqrt(E/m)` + # as an atomic expression: the disagreement is flagged, never reconciled. + client = _IsolatedSolveStubModelClient( + answer_payload={ + "canonical_text": "sqrt(E/m)", + "object_kind": "expression", + "structure": "set", + } + ) + artifact = infer_prediction_semantics(_problem(), client) + + assert artifact.build_report is not None + assert any( + flag.startswith("a_pred_llm_vs_ext_disagreement") + for flag in artifact.build_report.flags + ) + assert artifact.build_report.review_required is True + # a_pred_llm is adopted as-is (no silent reconciliation to a_pred_ext). + assert artifact.prediction_answer_semantics.structure == AnswerStructure.SET + + +def test_isolated_solve_prompt_has_no_golden_or_assumptions_leak() -> None: + client = _IsolatedSolveStubModelClient() + infer_prediction_semantics(_problem(), client) + + solve_prompt = client.prompts[0] + assert "Toolkit heuristic draft question semantics:" not in solve_prompt + # Gold subject_to / domain declarations must not reach the solver. + assert "positive" not in solve_prompt + assert "symbol_assumptions" not in solve_prompt + assert "m > 0" not in solve_prompt + assert "Solution:" not in solve_prompt + + +def test_a_pred_ext_classifies_like_a_ref_on_shared_surface() -> None: + # The deterministic extraction uses the same authority as a_ref, so a set surface reads + # as a set and a tuple surface as a tuple. + set_ext = extract_prediction_answer_semantics("{2, -2}") + assert set_ext.structure == AnswerStructure.SET + tuple_ext = extract_prediction_answer_semantics("(3, 4)") + assert tuple_ext.structure == AnswerStructure.TUPLE + + +def test_standalone_extracted_artifact_needs_no_generation() -> None: + artifact = build_extracted_prediction_semantics_artifact( + _problem(), "sqrt(E/m)", provider="external", model_name="human" + ) + + assert artifact.final_answer == "sqrt(E/m)" + assert ( + artifact.prediction_answer_semantics.object_kind == AnswerObjectKind.EXPRESSION + ) + assert artifact.question_semantics.target_variable is None + assert artifact.build_report is not None + assert artifact.build_report.build_method == "prediction_extracted" + assert artifact.generator.structured_output_mode == "extracted" + + +def test_prediction_question_view_redacts_symbol_assumptions() -> None: + # The leakage guard: gold symbol_assumptions must never reach the prediction-side draft. + semantics = infer_prediction_question_semantics(_problem()) + + assert semantics.symbol_assumptions == () + + +def test_resolve_isolated_prediction_response_model_prefers_isolated_schema() -> None: + client = _IsolatedSolveStubModelClient() + + assert ( + resolve_isolated_prediction_response_model(client) + is StrictPredictionIsolatedResponse + ) diff --git a/tests/prkit/semantics/test_semantics_build.py b/tests/prkit/semantics/test_semantics_build.py new file mode 100644 index 0000000..31bf864 --- /dev/null +++ b/tests/prkit/semantics/test_semantics_build.py @@ -0,0 +1,274 @@ +"""Tests for the deterministic semantics-build helpers (WS A core). + +These cover the methodology + engine-compatibility rules that the (advisory) LLM build +stages wrap: relative tolerance synthesis, ``subject_to`` -> symbol-assumption derivation +with canonical-token resolution, the assumption lattice, and the merge policy. +""" + +from __future__ import annotations + +import pytest + +from prkit.semantics.inference.semantics_build import ( + alias_source_violations, + assumptions_from_subject_to, + build_alias_map, + infer_answer_tolerance, + meet_assumptions, + merge_symbol_assumptions, + parse_relative_tolerance_instruction, + reconcile_allowed_sets, + reference_pair_consistency, + resolve_to_canonical, +) +from prkit.semantics.schema import ( + DEFAULT_NUMERIC_TOLERANCE, + AnswerObjectKind, + AnswerStructure, + PhysicsAnswerSemantics, + PhysicsQuestionSemantics, + PhysicsSymbolAliasSemantics, + PhysicsSymbolAssumptionSemantics, + SymbolAssumption, +) + + +def _relation(text: str) -> PhysicsAnswerSemantics: + return PhysicsAnswerSemantics( + canonical_text=text, + raw_text=text, + object_kind=AnswerObjectKind.RELATION, + ) + + +# -------------------------------------------------------------------------------------- +# Lattice +# -------------------------------------------------------------------------------------- +@pytest.mark.parametrize( + ("left", "right", "expected"), + [ + ( + SymbolAssumption.NONZERO, + SymbolAssumption.NONNEGATIVE, + SymbolAssumption.POSITIVE, + ), + ( + SymbolAssumption.REAL, + SymbolAssumption.NONNEGATIVE, + SymbolAssumption.NONNEGATIVE, + ), + (SymbolAssumption.REAL, SymbolAssumption.NONZERO, SymbolAssumption.NONZERO), + (SymbolAssumption.COMPLEX, SymbolAssumption.REAL, SymbolAssumption.REAL), + (SymbolAssumption.POSITIVE, SymbolAssumption.REAL, SymbolAssumption.POSITIVE), + ( + SymbolAssumption.NONNEGATIVE, + SymbolAssumption.NONNEGATIVE, + SymbolAssumption.NONNEGATIVE, + ), + ], +) +def test_meet_assumptions( + left: SymbolAssumption, right: SymbolAssumption, expected: SymbolAssumption +) -> None: + assert meet_assumptions(left, right) == expected + assert meet_assumptions(right, left) == expected # commutative + + +# -------------------------------------------------------------------------------------- +# subject_to -> assumptions +# -------------------------------------------------------------------------------------- +@pytest.mark.parametrize( + ("constraint", "expected"), + [ + ("x > 0", SymbolAssumption.POSITIVE), + ("x >= 0", SymbolAssumption.NONNEGATIVE), + ("x ≥ 0", SymbolAssumption.NONNEGATIVE), + ("x != 0", SymbolAssumption.NONZERO), + ("x ≠ 0", SymbolAssumption.NONZERO), + ("0 < x", SymbolAssumption.POSITIVE), + ("x < 0", SymbolAssumption.NONZERO), + ("x <= 0", SymbolAssumption.REAL), + ("x ∈ R", SymbolAssumption.REAL), + ("x > 5", SymbolAssumption.POSITIVE), + ("x >= 3", SymbolAssumption.POSITIVE), + ], +) +def test_assumptions_from_subject_to_single( + constraint: str, expected: SymbolAssumption +) -> None: + derived = assumptions_from_subject_to([_relation(constraint)]) + assert derived == {"x": expected} + + +def test_assumptions_from_subject_to_chained_positive_lower_bound() -> None: + # 0 < r < L => r is positive (lower bound 0, strict) + derived = assumptions_from_subject_to([_relation("0 < r < L")]) + assert derived == {"r": SymbolAssumption.POSITIVE} + + +def test_assumptions_from_subject_to_unparsable_is_skipped() -> None: + # An equality fixes a value, not a domain; a free-form clause yields nothing. + assert assumptions_from_subject_to([_relation("x = 1")]) == {} + assert assumptions_from_subject_to([_relation("n is an integer")]) == {} + + +def test_assumptions_from_subject_to_combines_constraints_on_one_symbol() -> None: + derived = assumptions_from_subject_to([_relation("x != 0"), _relation("x >= 0")]) + assert derived == {"x": SymbolAssumption.POSITIVE} + + +def test_assumptions_from_subject_to_resolves_alias_to_canonical() -> None: + # The constraint names the alias `y_s`; the engine looks up the canonical `y`. + alias_map = {"y_s": "y"} + derived = assumptions_from_subject_to([_relation("y_s > 0")], alias_map=alias_map) + assert derived == {"y": SymbolAssumption.POSITIVE} + + +def test_build_alias_map_and_resolve() -> None: + question = PhysicsQuestionSemantics( + symbol_aliases=( + PhysicsSymbolAliasSemantics(canonical_symbol="y", aliases=("y_s", "y0")), + ) + ) + alias_map = build_alias_map(question) + assert alias_map == {"y_s": "y", "y0": "y"} + assert resolve_to_canonical("y_s", alias_map) == "y" + assert resolve_to_canonical("z", alias_map) == "z" + + +def test_alias_source_violations_flags_raw_alias_tokens() -> None: + alias_map = {"y_s": "y"} + # A correctly canonicalized map has no violations. + assert alias_source_violations({"y": SymbolAssumption.POSITIVE}, alias_map) == [] + # A map keyed by the raw alias would be dropped by the engine -> violation. + assert alias_source_violations({"y_s": SymbolAssumption.POSITIVE}, alias_map) == [ + "y_s" + ] + + +# -------------------------------------------------------------------------------------- +# merge policy +# -------------------------------------------------------------------------------------- +def test_merge_symbol_assumptions_fills_gaps_from_advisory() -> None: + merged, flags = merge_symbol_assumptions( + {"x": SymbolAssumption.POSITIVE}, + {"y": SymbolAssumption.REAL}, + ) + assert merged == {"x": SymbolAssumption.POSITIVE, "y": SymbolAssumption.REAL} + assert flags == [] + + +def test_merge_symbol_assumptions_most_restrictive_wins_and_flags() -> None: + # subject_to says nonnegative; the LLM refines to positive -> meet is positive, flagged. + merged, flags = merge_symbol_assumptions( + {"x": SymbolAssumption.NONNEGATIVE}, + {"x": SymbolAssumption.POSITIVE}, + ) + assert merged == {"x": SymbolAssumption.POSITIVE} + assert flags == ["advisory_strengthened:x:nonnegative->positive"] + + +def test_merge_symbol_assumptions_consistent_no_flag() -> None: + merged, flags = merge_symbol_assumptions( + {"x": SymbolAssumption.POSITIVE}, + {"x": SymbolAssumption.REAL}, + ) + assert merged == {"x": SymbolAssumption.POSITIVE} + assert flags == [] + + +# -------------------------------------------------------------------------------------- +# tolerance (relative; sig-figs not mapped here) +# -------------------------------------------------------------------------------------- +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("Answer within 1%", 0.01), + ("accurate to within 0.5 %", 0.005), + ("±2%", 0.02), + ("give 3 significant figures", None), + ("round to 2 decimal places", None), + ("no precision stated", None), + ], +) +def test_parse_relative_tolerance_instruction( + text: str, expected: float | None +) -> None: + assert parse_relative_tolerance_instruction(text) == expected + + +def test_infer_answer_tolerance_precedence_is_relative() -> None: + # explicit relative arg wins + assert infer_answer_tolerance(relative_tolerance=0.01) == 0.01 + # else parse the instruction + assert infer_answer_tolerance(instruction_text="within 2%") == 0.02 + # else default (still relative, never absolute-converted) + assert infer_answer_tolerance() == DEFAULT_NUMERIC_TOLERANCE + # sig-fig phrasing does NOT tighten tolerance (handled by printed precision) + assert ( + infer_answer_tolerance(instruction_text="3 significant figures") + == DEFAULT_NUMERIC_TOLERANCE + ) + + +# -------------------------------------------------------------------------------------- +# allowed_* reconciliation (compat #3) + mutual consistency +# -------------------------------------------------------------------------------------- +def _answer( + kind: AnswerObjectKind, + structure: AnswerStructure = AnswerStructure.ATOMIC, + **extra: object, +) -> PhysicsAnswerSemantics: + return PhysicsAnswerSemantics( + canonical_text="x", object_kind=kind, structure=structure, **extra + ) + + +def test_reconcile_allowed_sets_widens_to_admit_gold_kind_and_structure() -> None: + # An over-narrow question that admits only `choice`/`atomic`... + question = PhysicsQuestionSemantics( + allowed_object_kinds=(AnswerObjectKind.CHOICE,), + allowed_structures=(AnswerStructure.ATOMIC,), + ) + gold = _answer(AnswerObjectKind.NUMBER, AnswerStructure.TUPLE) + + reconciled = reconcile_allowed_sets(question, gold) + + # ...is widened to admit the gold's number/tuple, plus ATOMIC (tuple collapse target). + assert AnswerObjectKind.NUMBER in reconciled.allowed_object_kinds + assert AnswerStructure.TUPLE in reconciled.allowed_structures + assert AnswerStructure.ATOMIC in reconciled.allowed_structures + + +def test_reconcile_allowed_sets_never_narrows_permissive_default() -> None: + question = PhysicsQuestionSemantics() # permissive: all kinds, all structures + reconciled = reconcile_allowed_sets(question, _answer(AnswerObjectKind.NUMBER)) + assert set(reconciled.allowed_object_kinds) == set(AnswerObjectKind) + assert set(reconciled.allowed_structures) == set(AnswerStructure) + + +def test_reference_pair_consistency_clean_pair_has_no_issues() -> None: + gold = _answer(AnswerObjectKind.EXPRESSION, target_variable="v") + question = reconcile_allowed_sets( + PhysicsQuestionSemantics(target_variable="v"), gold + ) + assert reference_pair_consistency(question, gold) == [] + + +def test_reference_pair_consistency_flags_target_mismatch_and_alias_source() -> None: + gold = _answer(AnswerObjectKind.EXPRESSION, target_variable="v") + question = PhysicsQuestionSemantics( + target_variable="w", + symbol_aliases=( + PhysicsSymbolAliasSemantics(canonical_symbol="y", aliases=("y_s",)), + ), + # `y_s` is an alias *source* -> the engine would drop this assumption. + symbol_assumptions=( + PhysicsSymbolAssumptionSemantics( + symbol="y_s", assumption=SymbolAssumption.POSITIVE + ), + ), + ) + issues = reference_pair_consistency(question, gold) + assert "target_variable_mismatch:w!=v" in issues + assert "assumption_alias_source:y_s" in issues diff --git a/tests/prkit/semantics/test_staged_build.py b/tests/prkit/semantics/test_staged_build.py new file mode 100644 index 0000000..0d8dd27 --- /dev/null +++ b/tests/prkit/semantics/test_staged_build.py @@ -0,0 +1,172 @@ +"""Happy-path tests for the staged objective semantics build (WS A orchestration). + +A fake structured-output client returns canned per-call responses so the orchestration's +guarantees can be asserted offline: structure/kind pinning, LLM ``allowed_*``/``tolerance`` +ignored in favor of the deterministic decisions, ``subject_to`` + LLM symbol-assumption +merge with canonical-token resolution, build-report provenance, and determinism. +""" + +from __future__ import annotations + +import json +from typing import Any + +from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem +from prkit.core.model_clients import BaseModelClient +from prkit.semantics.inference.calls import ( + build_problem_semantics, + build_reference_semantics, +) +from prkit.semantics.schema import ( + DEFAULT_NUMERIC_TOLERANCE, + AnswerObjectKind, + AnswerStructure, + SymbolAssumption, +) + + +def _problem() -> PhysicsProblem: + return PhysicsProblem( + problem_id="staged-1", + question="Find the energy E.", + answer=Answer(value="x**2/2, x > 0", answer_category=AnswerCategory.FORMULA), + domain="mechanics", + ) + + +class _StagedBuildStubModelClient(BaseModelClient): + """Returns a canned response per staged-build call, keyed by the response schema name. + + Call A (answer cleanup) deliberately reports a *wrong* structure/kind to prove the + deterministic pin holds; Call B sets ``allowed_*``/``tolerance`` to prove they are + ignored; Call C declares an assumption keyed by an alias token to prove canonical + resolution. + """ + + supports_response_format_json_schema = True + + def __init__(self) -> None: + super().__init__(model="stub-model") + self.provider = "stub" + self.prompts: list[str] = [] + + def response( + self, + input: str, + image_paths: list[str] | None = None, + response_format: dict[str, Any] | type | None = None, + **kwargs: Any, + ) -> str: + del image_paths, kwargs + self.prompts.append(input) + name = ( + response_format.get("name") if isinstance(response_format, dict) else None + ) + if name == "StrictPhysicsAnswerSemantics": + return json.dumps( + { + # wrong structure/kind on purpose -> must be pinned + flagged + "canonical_text": "x**2/2", + "object_kind": "number", + "structure": "set", + "canonical_latex": "\\frac{x^2}{2}", + } + ) + if name == "StrictPhysicsQuestionSemantics": + return json.dumps( + { + "target_variable": "E", + "symbol_aliases": [{"canonical_symbol": "y", "aliases": ["y_s"]}], + # allowed_* and tolerance below must be ignored by the build + "allowed_object_kinds": ["choice"], + "allowed_structures": ["atomic"], + "tolerance": 0.5, + } + ) + if name == "StrictSymbolAssumptionsResponse": + return json.dumps( + { + "assumptions": [ + { + "symbol": "y_s", # alias source -> must resolve to canonical y + "assumption": "positive", + "justification": "the problem states y_s > 0", + } + ] + } + ) + raise AssertionError(f"unexpected response schema: {name}") + + +def test_build_reference_semantics_pins_structure_and_kind() -> None: + artifact = build_reference_semantics(_problem(), _StagedBuildStubModelClient()) + + a_ref = artifact.reference_answer_semantics + # Deterministic pin holds despite Call A reporting number/set. + assert a_ref.object_kind == AnswerObjectKind.EXPRESSION + assert a_ref.structure == AnswerStructure.ATOMIC + assert artifact.build_report is not None + assert any( + "answer_cleanup_structure_disagreement" in flag + for flag in artifact.build_report.flags + ) + + +def test_build_reference_semantics_ignores_llm_allowed_and_tolerance() -> None: + artifact = build_reference_semantics(_problem(), _StagedBuildStubModelClient()) + q_ref = artifact.question_semantics + + # LLM tolerance 0.5 is ignored; relative default stands (no % instruction in question). + assert q_ref.tolerance == DEFAULT_NUMERIC_TOLERANCE + # LLM narrowed allowed_* to choice/atomic, but the build keeps the gold kind admitted. + assert AnswerObjectKind.EXPRESSION in q_ref.allowed_object_kinds + # LLM policy fields that ARE adopted: + assert q_ref.target_variable == "E" + + +def test_build_reference_semantics_merges_subject_to_and_llm_assumptions() -> None: + artifact = build_reference_semantics(_problem(), _StagedBuildStubModelClient()) + declared = { + entry.symbol: entry.assumption + for entry in artifact.question_semantics.symbol_assumptions + } + + # x is positive from the golden's `subject_to`; y is positive from the LLM declaration + # (declared as alias `y_s`, resolved to canonical `y`); the raw alias never survives. + assert declared.get("x") == SymbolAssumption.POSITIVE + assert declared.get("y") == SymbolAssumption.POSITIVE + assert "y_s" not in declared + + sources = { + entry.symbol: entry.source + for entry in artifact.build_report.assumption_provenance + } + assert sources["x"] == "subject_to" + assert sources["y"] == "llm_declared" + + +def test_build_reference_semantics_is_deterministic() -> None: + client = _StagedBuildStubModelClient() + first = build_reference_semantics(_problem(), client) + second = build_reference_semantics(_problem(), client) + + assert first.question_semantics == second.question_semantics + assert first.reference_answer_semantics == second.reference_answer_semantics + assert first.build_report == second.build_report + + +def test_build_problem_semantics_is_answer_blind() -> None: + client = _StagedBuildStubModelClient() + artifact = build_problem_semantics(_problem(), client) + + # No golden answer surface should appear in any problem-only prompt. + assert all("Answer:\n" not in prompt for prompt in client.prompts) + assert artifact.artifact_type == "problem_semantics" + assert artifact.build_report is not None + assert artifact.build_report.build_method == "problem_3call" + # The LLM-declared assumption is adopted (canonical token), with no subject_to source. + declared = { + entry.symbol: entry.assumption + for entry in artifact.question_semantics.symbol_assumptions + } + assert declared.get("y") == SymbolAssumption.POSITIVE diff --git a/tests/prkit/verify/test_verify.py b/tests/prkit/verify/test_verify.py index 2a5df4b..4cc4800 100644 --- a/tests/prkit/verify/test_verify.py +++ b/tests/prkit/verify/test_verify.py @@ -95,6 +95,87 @@ def test_tolerance_passthrough(self): assert loose.correct is True +def _positive_x_context(): + """A ``q_ref`` declaring ``x`` positive, unlocking domain-gated symbolic identities. + + Built lazily inside the test so the module-level import stays light (the schema + types live behind the heavier ``prkit.semantics`` import). + """ + from prkit.semantics import PhysicsQuestionSemantics + from prkit.semantics.schema.enums import SymbolAssumption + from prkit.semantics.schema.models import PhysicsSymbolAssumptionSemantics + + return PhysicsQuestionSemantics( + symbol_assumptions=( + PhysicsSymbolAssumptionSemantics( + symbol="x", assumption=SymbolAssumption.POSITIVE + ), + ) + ) + + +class TestVerifyThreadsQuestionContext: + """WS C: ``verify(context=q_ref)`` reaches the judgement and can change a verdict. + + ``log(x**2) == 2*log(x)`` holds only for ``x > 0``. Under the empty default + context (``context=None``) the engine cannot assume positivity and rejects the + pair; supplying a ``q_ref`` that declares ``x`` positive unlocks the + domain-gated symbolic accept. This is the wiring WS C closes — previously + ``verify`` passed ``context=None`` unconditionally so a rich ``q_ref`` was inert. + """ + + GOLD = "log(x**2)" + PRED = "2*log(x)" + + def test_default_context_rejects_domain_gated_identity(self): + v = verify(self.GOLD, self.PRED) + assert v.correct is False + + def test_supplying_q_ref_unlocks_domain_gated_accept(self): + v = verify(self.GOLD, self.PRED, context=_positive_x_context()) + assert v.correct is True + assert v.symbolic_equiv is True + + def test_verdict_changes_with_vs_without_context(self): + without = verify(self.GOLD, self.PRED) + with_q_ref = verify(self.GOLD, self.PRED, context=_positive_x_context()) + assert without.correct != with_q_ref.correct + + def test_dict_context_accepted(self): + v = verify( + self.GOLD, + self.PRED, + context={"symbol_assumptions": [{"symbol": "x", "assumption": "positive"}]}, + ) + assert v.correct is True + + def test_reference_artifact_context_unwrapped_to_question_semantics(self): + # Anything exposing ``.question_semantics`` (e.g. a ReferenceSemanticsArtifact) + # is duck-typed and unwrapped to its q_ref — without importing the heavy + # inference artifact type into the light verify facade. + class _ArtifactLike: + question_semantics = _positive_x_context() + + v = verify(self.GOLD, self.PRED, context=_ArtifactLike()) + assert v.correct is True + + def test_none_context_preserves_default_behavior(self): + explicit_none = verify("3 m/s", "3 m/s", context=None) + implicit = verify("3 m/s", "3 m/s") + assert explicit_none.correct == implicit.correct is True + + def test_partial_credit_path_also_threads_context(self): + # The graded path must accept and thread q_ref too (it reaches the same + # context-coercion plumbing); supplying it must not error and yields a + # valid graded Verdict. The verdict-change assertion lives on the binary + # path above, where the domain gate is decisive. + v = verify( + self.GOLD, self.PRED, partial_credit=True, context=_positive_x_context() + ) + assert isinstance(v, Verdict) + assert v.partial_credit == v.score + + class TestParse: def test_returns_physics_answer_semantics(self): parsed = parse("9.8 m/s^2") From 8c47f935fa74484ed8be9d9c3f8e1ae31623f71b Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Fri, 19 Jun 2026 21:50:04 -0400 Subject: [PATCH 10/28] Reconcile sign-convention flips between directional answers Two directional answers can differ by a global sign because each was expressed under an opposite, unstated axis/sign-convention choice (a reference velocity -20 m/s right-as-positive vs a prediction +20 m/s left-as-positive describe the identical motion). These were false negatives. Add a convention-aware criterion that reconciles two *stated* conventions to a common frame. The convention is concrete per-answer data, not a toggle: a global flip is forgiven iff the question fixes no convention (q.sign_convention/q.coordinate_frame absent), both answers declare conventions whose positive axes are a provable global reversal (antonym directions), and the values are an exact global -1. Covers number / physical_quantity / sign_direction (atomic) and vector (component-wise) answers; reuses _proportional_ratio (already returns -1 for a negation) and numbers_close. New comparison/sign_convention.py holds the pure criteria; engine wiring runs it before the strict same-kind criterion (so it can also override a plain-equality false accept) and refines the _compare_shaped both-set- incompatible frame branch (opposite global-reversal frames reconcile, genuinely-incompatible frames still reject, partial flips reject). Registered as the audited sign_convention TIER2 bridge: on under audited, blocked under strict, with bridge_id/tier/evidence recorded. Precision is held two ways. A flip is forgiven only with both stated opposite conventions, so a bare intrinsic sign (charge -5 C vs +5 C) is never reconciled -- directional intent is declared, not derived. And the precision dual ships alongside the accept: opposite conventions with equal values denote physically opposite quantities and are rejected under every policy, avoiding an asymmetric relaxation. Also preserve answer-level coordinate_frame/sign_convention through _repair_atomic_answer's reparse so the lane can read them. Co-Authored-By: Claude Opus 4.8 --- .../semantics/comparison/bridge_registry.py | 23 + src/prkit/semantics/comparison/engine.py | 98 +++ .../semantics/comparison/sign_convention.py | 309 ++++++++++ .../semantics/test_protocol_comparison.py | 568 ++++++++++++++++++ tests/prkit/verify/test_verify.py | 97 +++ 5 files changed, 1095 insertions(+) create mode 100644 src/prkit/semantics/comparison/sign_convention.py diff --git a/src/prkit/semantics/comparison/bridge_registry.py b/src/prkit/semantics/comparison/bridge_registry.py index 8ffee51..326971d 100644 --- a/src/prkit/semantics/comparison/bridge_registry.py +++ b/src/prkit/semantics/comparison/bridge_registry.py @@ -72,6 +72,16 @@ def _has_choice_space( return bool(contract.question_semantics.choice_space) +def _sign_convention_reconcilable( + pred: PhysicsAnswerSemantics, + ref: PhysicsAnswerSemantics, + contract: PhysicsEvaluationContract, +) -> bool: + del pred, ref + question = contract.question_semantics + return question.sign_convention is None and question.coordinate_frame is None + + def _looks_like_change_question( pred: PhysicsAnswerSemantics, ref: PhysicsAnswerSemantics, @@ -170,6 +180,13 @@ def _always( predicate=_looks_like_change_question, description="Treat no-change language as zero only for change-oriented questions.", ), + "sign_convention": BridgeSpec( + bridge_id="sign_convention", + tier=BridgeTier.TIER2, + predicate=_sign_convention_reconcilable, + description="Reconcile a global sign flip between directional answers that declare " + "opposite conventions, only when the question fixes none.", + ), } @@ -202,6 +219,12 @@ def _bridge_candidate_ids_for_atomic_kinds( kinds = {pred_kind, ref_kind} candidates: list[str] = [] + if pred_kind == ref_kind and pred_kind in { + AnswerObjectKind.NUMBER, + AnswerObjectKind.PHYSICAL_QUANTITY, + AnswerObjectKind.SIGN_DIRECTION, + }: + candidates.append("sign_convention") if kinds == {AnswerObjectKind.NUMBER, AnswerObjectKind.PHYSICAL_QUANTITY}: candidates.append("quantity_to_number") if kinds == {AnswerObjectKind.NUMBER, AnswerObjectKind.EXPRESSION}: diff --git a/src/prkit/semantics/comparison/engine.py b/src/prkit/semantics/comparison/engine.py index 4fdf4fc..684ef43 100644 --- a/src/prkit/semantics/comparison/engine.py +++ b/src/prkit/semantics/comparison/engine.py @@ -43,6 +43,12 @@ normalize_plain_text, parse_numeric_value, ) +from .sign_convention import ( + answer_directional_convention, + compare_sign_convention, + orientation_relation, + vectors_exact_negation, +) from .structure_canonicalization import canonicalize_structure @@ -474,6 +480,22 @@ def _compare_atomic( """Compare two atomic answers using strict, bridged, and fallback logic.""" if pred.object_kind == ref.object_kind: + # Sign-convention reconciliation runs first for directional kinds so it can both + # accept a global flip between opposite stated conventions and override a plain + # equality that would otherwise miss the precision dual (opposite conventions, + # equal values => physically opposite). It owns the verdict only on that concrete + # evidence; otherwise it declines and the normal criterion decides. + sign_convention = compare_sign_convention(pred, ref, context=context) + if sign_convention is not None: + if sign_convention.equivalent: + return _apply_bridge_policy( + sign_convention, + pred=pred, + ref=ref, + contract=contract, + policy_mode=policy_mode, + ) + return sign_convention strict = compare_same_object_kind(pred, ref, context=context) if strict.equivalent: return strict @@ -712,6 +734,54 @@ def _compare_interval( ) +def _reconcile_shaped_sign_convention( + pred: PhysicsAnswerSemantics, + ref: PhysicsAnswerSemantics, + *, + context: PhysicsQuestionSemantics, + contract: PhysicsEvaluationContract, + policy_mode: ComparisonPolicyMode, +) -> AnswerComparison | None: + """Reconcile a shaped (vector) pair under opposite stated frames; ``None`` to defer. + + Owns the verdict only when the question fixes no convention, both answers declare + opposite (globally-reversed) directional conventions, and every cell is atomic. An exact + component-wise negation accepts via the audited ``sign_convention`` bridge; a partial / + non-negation rejects (the precision dual). Otherwise it declines so the existing frame + gates (one-sided ⇒ TBD, genuinely-incompatible ⇒ mismatch, both-unset ⇒ per-cell) run. + """ + + if context.sign_convention or context.coordinate_frame: + return None + pred_convention = answer_directional_convention(pred) + ref_convention = answer_directional_convention(ref) + if not pred_convention or not ref_convention: + return None + if orientation_relation(pred_convention, ref_convention) != "opposite": + return None + if not _children_all_atomic(pred, ref): + return None + + if vectors_exact_negation(pred, ref, context.tolerance): + result = AnswerComparison( + True, + "sign_convention", + ("global_sign_flip", f"kind={pred.structure.value}"), + ) + return _apply_bridge_policy( + result, + pred=pred, + ref=ref, + contract=contract, + policy_mode=policy_mode, + ) + return AnswerComparison( + False, + "sign_convention", + ("opposite_convention_not_global_negation",), + ) + + def _compare_shaped( pred: PhysicsAnswerSemantics, ref: PhysicsAnswerSemantics, @@ -734,6 +804,21 @@ def _compare_shaped( if not pred.children or not ref.children: return _structure_tbd(pred.structure.value, "unparsed_shaped_answer") + # Sign-convention reconciliation (vectors): when the question fixes no convention and the + # two answers declare opposite (globally-reversed) frames, an exact component-wise + # negation is the same physical vector. This refines the both-set-incompatible branch + # below — only the *opposite* sub-case reconciles; genuinely-incompatible frames still + # reject, partial flips still reject. + reconciled = _reconcile_shaped_sign_convention( + pred, + ref, + context=context, + contract=contract, + policy_mode=policy_mode, + ) + if reconciled is not None: + return reconciled + # Coordinate frame: both-unset ⇒ the problem's implicit shared frame (proceed); a # one-sided declaration is unresolved ⇒ TBD; both-set-incompatible is a real mismatch. pred_frame = pred.coordinate_frame or context.coordinate_frame @@ -1039,6 +1124,10 @@ def _repair_atomic_answer( "provenance": dict(answer.provenance) or dict(reparsed.provenance), "diagnostics": answer.diagnostics or reparsed.diagnostics, "subject_to": answer.subject_to or reparsed.subject_to, + # Preserve the answer's stated directional convention through the reparse so the + # sign-convention lane can still read it (mirrors the structured reparse merge). + "coordinate_frame": answer.coordinate_frame or reparsed.coordinate_frame, + "sign_convention": answer.sign_convention or reparsed.sign_convention, } if reparsed.object_kind == AnswerObjectKind.PHYSICAL_QUANTITY: update.update( @@ -1402,6 +1491,15 @@ def _bridge_evidence( and contract.question_semantics.choice_space ): evidence["choice_space"] = ",".join(contract.question_semantics.choice_space) + if bridge_id == "sign_convention": + pred_convention = answer_directional_convention(pred) + ref_convention = answer_directional_convention(ref) + if pred_convention: + evidence["pred_convention"] = pred_convention + if ref_convention: + evidence["ref_convention"] = ref_convention + evidence["orientation"] = "opposite" + evidence["reconciliation"] = "global_-1" return evidence diff --git a/src/prkit/semantics/comparison/sign_convention.py b/src/prkit/semantics/comparison/sign_convention.py new file mode 100644 index 0000000..d9f39d5 --- /dev/null +++ b/src/prkit/semantics/comparison/sign_convention.py @@ -0,0 +1,309 @@ +"""Sign-convention reconciliation for directional answers. + +Two directional answers can differ by a *global* sign because each was expressed under an +opposite, unstated axis/sign-convention choice (a reference velocity ``-20 m/s`` declared +*right-as-positive* vs a prediction ``+20 m/s`` declared *left-as-positive* describe the +identical motion). This module reconciles such a pair, but **only** on concrete evidence: + +* the question fixes no convention (``q.sign_convention`` and ``q.coordinate_frame`` both + absent) -- if it did, the axis is pinned and a flip is a real error; +* **both** answers carry a stated convention whose positive axis is a *global reversal* of + the other (a provable antonym, e.g. right vs left, up vs down); and +* the values are an exact global ``-1`` of one another. + +The stated conventions *are* the evidence, so there is no on/off switch. The lane is a +meaning-preserving re-expression to a common frame applied symmetrically (the METHODOLOGY's +preferred lever), not a "rescue". It is symmetric in precision: opposite conventions with +*equal* values denote physically opposite quantities and are rejected -- consuming the +convention to accept a flip while ignoring it to reject a coincidence would be an +asymmetric relaxation. The orchestration of the resulting accept through the bridge policy +(``comparison_mode="sign_convention"``, blocked under ``strict``) lives in ``engine.py``; +this module supplies the pure criteria. +""" + +from __future__ import annotations + +from ..schema import ( + AnswerComparison, + AnswerObjectKind, + PhysicsAnswerSemantics, + PhysicsQuestionSemantics, +) +from .common import context_symbol_alias_map +from .numeric import _aligned_pred_numeric_value, extract_numeric_comparable_answer +from .semantics import ( + _SIGN_DIRECTION_CANONICAL, + canonicalize_sign_direction, + expressions_equivalent, + normalize_plain_text, + numbers_close, + parse_numeric_value, +) + +# Antonyms over the canonical direction vocabulary. Two conventions are reconcilable only +# when their positive axes are a *global* reversal of one another; orthogonal or unrelated +# directions (right vs up) are indeterminate, not opposite. +_DIRECTION_OPPOSITE: dict[str, str] = { + "right": "left", + "left": "right", + "up": "down", + "down": "up", + "into_page": "out_of_page", + "out_of_page": "into_page", + "inward": "outward", + "outward": "inward", + "clockwise": "counterclockwise", + "counterclockwise": "clockwise", + "up_in_plane": "down_in_plane", + "down_in_plane": "up_in_plane", + "positive": "negative", + "negative": "positive", +} + +# The polarity labels that are *axis-relative* (their sign meaning flips with the axis); the +# absolute direction words (up, clockwise, ...) are not -- a flip there is a real change. +_POLARITY_LABELS = frozenset({"positive", "negative"}) + +# Direction phrases (longest first) used to read a positive-axis orientation out of a +# free-text convention such as "right-as-positive" or "taking up as positive". The pure +# sign words (+/-/positive/negative) are excluded: alone they name a role, not a physical +# direction, so a convention without a named direction is indeterminate (declines, never +# guesses). +_DIRECTION_PHRASES: tuple[tuple[str, str], ...] = tuple( + sorted( + ( + (normalize_plain_text(phrase), orientation) + for phrase, orientation in _SIGN_DIRECTION_CANONICAL.items() + if orientation not in _POLARITY_LABELS and not phrase.startswith(("+", "-")) + ), + key=lambda item: len(item[0]), + reverse=True, + ) +) + +_DIRECTIONAL_ATOMIC_KINDS = frozenset( + { + AnswerObjectKind.NUMBER, + AnswerObjectKind.PHYSICAL_QUANTITY, + AnswerObjectKind.SIGN_DIRECTION, + } +) + + +def answer_directional_convention(answer: PhysicsAnswerSemantics) -> str | None: + """Return an answer's own stated directional convention (frame preferred).""" + + return answer.coordinate_frame or answer.sign_convention + + +def _convention_orientation(text: str | None) -> str | None: + """Read the canonical positive-axis direction out of a free-text convention.""" + + if not text: + return None + normalized = normalize_plain_text(text) + if not normalized: + return None + padded = f" {normalized} " + for phrase, orientation in _DIRECTION_PHRASES: + if phrase and f" {phrase} " in padded: + return orientation + return None + + +def orientation_relation( + pred_convention: str | None, ref_convention: str | None +) -> str | None: + """Classify two conventions as ``"same"``, ``"opposite"``, or ``None`` (indeterminate).""" + + pred_orientation = _convention_orientation(pred_convention) + ref_orientation = _convention_orientation(ref_convention) + if pred_orientation is None or ref_orientation is None: + return None + if pred_orientation == ref_orientation: + return "same" + if _DIRECTION_OPPOSITE.get(pred_orientation) == ref_orientation: + return "opposite" + return None + + +def compare_sign_convention( + pred: PhysicsAnswerSemantics, + ref: PhysicsAnswerSemantics, + *, + context: PhysicsQuestionSemantics, +) -> AnswerComparison | None: + """Reconcile two atomic directional answers under opposite stated conventions. + + Returns ``None`` when the lane is inactive (so the normal criterion decides), an + accepting ``AnswerComparison`` (``comparison_mode="sign_convention"``) for a provable + global ``-1`` between opposite conventions, or a rejecting one for the precision dual + (opposite conventions, non-negated values). It *owns* the verdict only when both sides + carry stated, provably-opposite conventions; otherwise it declines. + """ + + if context.sign_convention or context.coordinate_frame: + return None + if not pred.is_atomic or not ref.is_atomic: + return None + kind = pred.object_kind + if kind != ref.object_kind or kind not in _DIRECTIONAL_ATOMIC_KINDS: + return None + + if kind == AnswerObjectKind.SIGN_DIRECTION: + return _compare_sign_direction(pred, ref) + return _compare_scalar(pred, ref, context=context) + + +def _compare_scalar( + pred: PhysicsAnswerSemantics, + ref: PhysicsAnswerSemantics, + *, + context: PhysicsQuestionSemantics, +) -> AnswerComparison | None: + """Reconcile a signed number / physical-quantity pair under opposite conventions.""" + + if ( + orientation_relation( + answer_directional_convention(pred), + answer_directional_convention(ref), + ) + != "opposite" + ): + return None + + pred_numeric = extract_numeric_comparable_answer(pred, context=context) + ref_numeric = extract_numeric_comparable_answer(ref, context=context) + if pred_numeric is None or ref_numeric is None: + return AnswerComparison( + False, "sign_convention", ("unparsable_directional_value",) + ) + + if not expressions_equivalent( + pred_numeric.symbolic_factor_text, + ref_numeric.symbolic_factor_text, + context.tolerance, + alias_map=context_symbol_alias_map(context), + ): + return AnswerComparison(False, "sign_convention", ("symbolic_factor_mismatch",)) + + aligned_pred_value, unit_diagnostics, _ = _aligned_pred_numeric_value( + pred_numeric, ref_numeric, context=context + ) + if aligned_pred_value is None: + return AnswerComparison(False, "sign_convention", unit_diagnostics) + + ref_value = ref_numeric.coefficient_value + if aligned_pred_value == 0.0 or ref_value == 0.0: + # No sign to flip: a zero quantity is convention-invariant. + return AnswerComparison(False, "sign_convention", ("degenerate_zero",)) + + if numbers_close(aligned_pred_value, -ref_value, context.tolerance): + return AnswerComparison( + True, + "sign_convention", + ("global_sign_flip", f"kind={pred.object_kind.value}"), + ) + if numbers_close(aligned_pred_value, ref_value, context.tolerance): + # Opposite conventions but equal values => physically opposite (precision dual). + return AnswerComparison( + False, "sign_convention", ("opposite_convention_same_value",) + ) + return AnswerComparison( + False, "sign_convention", ("opposite_convention_value_mismatch",) + ) + + +def _resolve_polarity(answer: PhysicsAnswerSemantics) -> str | None: + """Return an answer's axis-relative polarity (``positive``/``negative``) when it has one.""" + + polarity = answer.sign_value or canonicalize_sign_direction( + answer.canonical_text or "" + ) + return polarity if polarity in _POLARITY_LABELS else None + + +def _resolve_sign_direction(answer: PhysicsAnswerSemantics) -> str | None: + """Resolve ``(polarity, convention)`` to an absolute physical direction.""" + + polarity = _resolve_polarity(answer) + if polarity is None: + return None + orientation = _convention_orientation(answer_directional_convention(answer)) + if orientation is None: + return None + return ( + orientation if polarity == "positive" else _DIRECTION_OPPOSITE.get(orientation) + ) + + +def _compare_sign_direction( + pred: PhysicsAnswerSemantics, ref: PhysicsAnswerSemantics +) -> AnswerComparison | None: + """Reconcile two polarity ``sign_direction`` answers via their stated conventions.""" + + if ( + orientation_relation( + answer_directional_convention(pred), + answer_directional_convention(ref), + ) + != "opposite" + ): + return None + + pred_direction = _resolve_sign_direction(pred) + ref_direction = _resolve_sign_direction(ref) + if pred_direction is None or ref_direction is None: + return None + + if pred_direction == ref_direction: + return AnswerComparison(True, "sign_convention", ("resolved_direction",)) + return AnswerComparison( + False, "sign_convention", ("opposite_convention_direction_mismatch",) + ) + + +def vectors_exact_negation( + pred: PhysicsAnswerSemantics, + ref: PhysicsAnswerSemantics, + tolerance: float, +) -> bool: + """Whether two equal-shape vectors are an exact component-wise global negation. + + Every component pair must satisfy ``p_i == -r_i`` (numeric within tolerance, or symbolic + via expression negation) and at least one component must be nonzero. A *partial* flip + (some components equal, some negated) fails -- a single global axis reversal flips every + axis, so a partial change is a different vector, not a convention artifact. + """ + + if len(pred.children) != len(ref.children) or not pred.children: + return False + + saw_nonzero = False + for pred_child, ref_child in zip(pred.children, ref.children): + pred_value = _child_numeric_value(pred_child) + ref_value = _child_numeric_value(ref_child) + if pred_value is not None and ref_value is not None: + if pred_value != 0.0 or ref_value != 0.0: + saw_nonzero = True + if not numbers_close(pred_value, -ref_value, tolerance): + return False + continue + + pred_text = pred_child.canonical_text + ref_text = ref_child.canonical_text + if not pred_text or not ref_text: + return False + if not expressions_equivalent(pred_text, f"-({ref_text})", tolerance): + return False + saw_nonzero = True + + return saw_nonzero + + +def _child_numeric_value(child: PhysicsAnswerSemantics) -> float | None: + """Return a vector cell's numeric value when it is a plain scalar.""" + + if child.numeric_value is not None: + return child.numeric_value + return parse_numeric_value(child.numeric_text or child.canonical_text) diff --git a/tests/prkit/semantics/test_protocol_comparison.py b/tests/prkit/semantics/test_protocol_comparison.py index 18d4c08..92ccc3e 100644 --- a/tests/prkit/semantics/test_protocol_comparison.py +++ b/tests/prkit/semantics/test_protocol_comparison.py @@ -2557,3 +2557,571 @@ def test_numeric_identity_equivalent_rejection_is_exact() -> None: same_left = parse_scalar_symbolic_expression("(x + 1)**2") same_right = parse_scalar_symbolic_expression("x**2 + 2*x + 1") assert _numeric_identity_equivalent(same_left, same_right, 1e-9) is True + + +# --------------------------------------------------------------------------- +# Sign-convention equivalence lane +# +# A global sign flip between two directional answers is reconciled only on concrete +# evidence: the question fixes no convention, both answers declare opposite (globally +# reversed) conventions, and the values are an exact global -1. The stated conventions are +# the evidence; the lane is an audited TIER2 bridge (blocked under strict). These batteries +# span many real open-form physics answers and prove the precision boundary. +# --------------------------------------------------------------------------- + + +def _directional_quantity( + canonical_text: str, + *, + numeric_value: float, + unit: str, + convention: str | None = None, +) -> dict[str, object]: + """Build a signed physical-quantity answer optionally carrying a stated convention.""" + + payload: dict[str, object] = { + "object_kind": "physical_quantity", + "canonical_text": canonical_text, + "numeric_value": numeric_value, + "numeric_text": str(numeric_value), + "unit": unit, + } + if convention is not None: + payload["coordinate_frame"] = convention + return payload + + +def _directional_number( + value: float, *, convention: str | None = None +) -> dict[str, object]: + """Build a signed bare-number answer optionally carrying a stated convention.""" + + payload: dict[str, object] = { + "object_kind": "number", + "canonical_text": ("+" if value >= 0 else "") + str(value), + "numeric_value": value, + "numeric_text": str(value), + } + if convention is not None: + payload["sign_convention"] = convention + return payload + + +def _directional_vector( + components: tuple[float, ...], + *, + convention: str | None = None, + unit: str | None = None, +) -> dict[str, object]: + """Build a numeric vector answer optionally carrying a stated frame.""" + + payload: dict[str, object] = { + "object_kind": "number", + "structure": "vector", + "canonical_text": "(" + ", ".join(str(c) for c in components) + ")", + "shape": [len(components)], + "children": [ + { + "object_kind": "number", + "structure": "atomic", + "canonical_text": str(c), + "numeric_value": float(c), + "numeric_text": str(c), + **({"unit": unit} if unit else {}), + } + for c in components + ], + } + if convention is not None: + payload["coordinate_frame"] = convention + return payload + + +def _sign_label(label: str, *, convention: str | None = None) -> dict[str, object]: + """Build a sign_direction answer optionally carrying a stated convention.""" + + payload: dict[str, object] = { + "object_kind": "sign_direction", + "canonical_text": label, + "sign_value": label, + } + if convention is not None: + payload["sign_convention"] = convention + return payload + + +# A pair of opposite axis conventions written in several natural surfaces. +_RIGHT = "right-as-positive" +_LEFT = "taking left as positive" +_UP = "up is positive" +_DOWN = "down positive" +_OUT = "out of the page positive" +_INTO = "into the page positive" + + +@pytest.mark.parametrize( + "pred, ref", + [ + # Velocity reported under opposite axis choices. + ( + _directional_quantity( + "+20 m/s", numeric_value=20.0, unit="m/s", convention=_LEFT + ), + _directional_quantity( + "-20 m/s", numeric_value=-20.0, unit="m/s", convention=_RIGHT + ), + ), + # Acceleration: up-positive vs down-positive. + ( + _directional_quantity( + "-9.8 m/s^2", numeric_value=-9.8, unit="m/s^2", convention=_UP + ), + _directional_quantity( + "+9.8 m/s^2", numeric_value=9.8, unit="m/s^2", convention=_DOWN + ), + ), + # Force. + ( + _directional_quantity( + "-15 N", numeric_value=-15.0, unit="N", convention=_RIGHT + ), + _directional_quantity( + "+15 N", numeric_value=15.0, unit="N", convention=_LEFT + ), + ), + # Momentum. + ( + _directional_quantity( + "+4 kg*m/s", numeric_value=4.0, unit="kg*m/s", convention=_LEFT + ), + _directional_quantity( + "-4 kg*m/s", numeric_value=-4.0, unit="kg*m/s", convention=_RIGHT + ), + ), + # A unit conversion rides along (72 km/h == 20 m/s). + ( + _directional_quantity( + "-72 km/h", numeric_value=-72.0, unit="km/h", convention=_RIGHT + ), + _directional_quantity( + "+20 m/s", numeric_value=20.0, unit="m/s", convention=_LEFT + ), + ), + # Bare numbers. + ( + _directional_number(5.0, convention=_RIGHT), + _directional_number(-5.0, convention=_LEFT), + ), + ( + _directional_number(-3.0, convention=_UP), + _directional_number(3.0, convention=_DOWN), + ), + ], +) +def test_protocol_sign_convention_scalar_accepts_under_audited( + pred: dict[str, object], ref: dict[str, object] +) -> None: + result = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.AUDITED + ) + assert result.equivalent is True, (pred, ref) + assert result.comparison_mode == "sign_convention" + assert result.bridge_id == "sign_convention" + assert result.bridge_tier is not None + assert result.bridge_evidence.get("orientation") == "opposite" + assert result.bridge_evidence.get("reconciliation") == "global_-1" + + +@pytest.mark.parametrize( + "pred, ref", + [ + # 2D global reversal. + ( + _directional_vector((3.0, -4.0), convention="x to the right"), + _directional_vector((-3.0, 4.0), convention="x to the left"), + ), + # 3D global reversal. + ( + _directional_vector((1.0, 2.0, -2.0), convention=_RIGHT), + _directional_vector((-1.0, -2.0, 2.0), convention=_LEFT), + ), + # E-field with a zero component (still has a nonzero one). + ( + _directional_vector((0.0, -5.0, 0.0), convention=_UP), + _directional_vector((0.0, 5.0, 0.0), convention=_DOWN), + ), + # Into/out-of-page reversal. + ( + _directional_vector((2.0, -1.0), convention=_OUT), + _directional_vector((-2.0, 1.0), convention=_INTO), + ), + ], +) +def test_protocol_sign_convention_vector_accepts_under_audited( + pred: dict[str, object], ref: dict[str, object] +) -> None: + result = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.AUDITED + ) + assert result.equivalent is True, (pred, ref) + assert result.comparison_mode == "sign_convention" + assert result.bridge_id == "sign_convention" + + +def test_protocol_sign_convention_symbolic_vector_accepts() -> None: + pred = { + "object_kind": "expression", + "structure": "vector", + "canonical_text": "(a, -b)", + "shape": [2], + "children": [ + {"object_kind": "expression", "structure": "atomic", "canonical_text": "a"}, + { + "object_kind": "expression", + "structure": "atomic", + "canonical_text": "-b", + }, + ], + "coordinate_frame": "x to the right", + } + ref = { + "object_kind": "expression", + "structure": "vector", + "canonical_text": "(-a, b)", + "shape": [2], + "children": [ + { + "object_kind": "expression", + "structure": "atomic", + "canonical_text": "-a", + }, + {"object_kind": "expression", "structure": "atomic", "canonical_text": "b"}, + ], + "coordinate_frame": "x to the left", + } + result = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.AUDITED + ) + assert result.equivalent is True + assert result.comparison_mode == "sign_convention" + + +@pytest.mark.parametrize( + "pred, ref", + [ + # negative under right-positive == left, positive under left-positive == left. + ( + _sign_label("negative", convention=_RIGHT), + _sign_label("positive", convention=_LEFT), + ), + ( + _sign_label("positive", convention=_UP), + _sign_label("negative", convention=_DOWN), + ), + ( + _sign_label("negative", convention=_OUT), + _sign_label("positive", convention=_INTO), + ), + ], +) +def test_protocol_sign_convention_sign_direction_resolves_to_same_physical_direction( + pred: dict[str, object], ref: dict[str, object] +) -> None: + result = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.AUDITED + ) + assert result.equivalent is True, (pred, ref) + assert result.comparison_mode == "sign_convention" + + +def test_protocol_sign_convention_accept_is_audited_but_not_strict() -> None: + pred = _directional_quantity( + "+20 m/s", numeric_value=20.0, unit="m/s", convention=_LEFT + ) + ref = _directional_quantity( + "-20 m/s", numeric_value=-20.0, unit="m/s", convention=_RIGHT + ) + + audited = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.AUDITED + ) + strict = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.STRICT + ) + permissive = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.PERMISSIVE + ) + + assert audited.equivalent is True + assert audited.bridge_tier is not None + assert strict.equivalent is False + assert strict.comparison_mode == "bridge_blocked" + assert permissive.equivalent is True + + +# --- Adversarial-reject battery (the precision proof) ---------------------- + + +@pytest.mark.parametrize( + "pred, ref", + [ + # Intrinsic signs never reconcile (no axis convention at all). + ( + _directional_quantity("+5 C", numeric_value=5.0, unit="C"), + _directional_quantity("-5 C", numeric_value=-5.0, unit="C"), + ), + ( + _directional_quantity("+30 J", numeric_value=30.0, unit="J"), + _directional_quantity("-30 J", numeric_value=-30.0, unit="J"), + ), + ( + _directional_quantity("-2 K", numeric_value=-2.0, unit="K"), + _directional_quantity("+2 K", numeric_value=2.0, unit="K"), + ), + ( + _directional_quantity("-12 V", numeric_value=-12.0, unit="V"), + _directional_quantity("+12 V", numeric_value=12.0, unit="V"), + ), + (_directional_number(5.0), _directional_number(-5.0)), + # Flipped side missing a convention: never assume the opposite axis. + ( + _directional_quantity("+20 m/s", numeric_value=20.0, unit="m/s"), + _directional_quantity( + "-20 m/s", numeric_value=-20.0, unit="m/s", convention=_RIGHT + ), + ), + # Opposite orientation, magnitude mismatch. + ( + _directional_quantity( + "-20 m/s", numeric_value=-20.0, unit="m/s", convention=_LEFT + ), + _directional_quantity( + "+15 m/s", numeric_value=15.0, unit="m/s", convention=_RIGHT + ), + ), + # Same orientation, opposite value: a real disagreement. + ( + _directional_quantity( + "+20 m/s", numeric_value=20.0, unit="m/s", convention=_RIGHT + ), + _directional_quantity( + "-20 m/s", numeric_value=-20.0, unit="m/s", convention=_RIGHT + ), + ), + # Zero has no sign to flip. + ( + _directional_quantity( + "0 m/s", numeric_value=0.0, unit="m/s", convention=_LEFT + ), + _directional_quantity( + "0 m/s", numeric_value=0.0, unit="m/s", convention=_RIGHT + ), + ), + # Non-opposite (orthogonal) conventions are indeterminate, not flippable. + ( + _directional_number(5.0, convention="x-axis positive"), + _directional_number(-5.0, convention="y-axis positive"), + ), + ], +) +def test_protocol_sign_convention_scalar_rejects( + pred: dict[str, object], ref: dict[str, object] +) -> None: + result = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.AUDITED + ) + assert result.equivalent is False, (pred, ref) + + +def test_protocol_sign_convention_precision_dual_rejects_under_every_policy() -> None: + # Opposite conventions but EQUAL values => physically opposite quantities. + pred = _directional_quantity( + "-20 m/s", numeric_value=-20.0, unit="m/s", convention=_LEFT + ) + ref = _directional_quantity( + "-20 m/s", numeric_value=-20.0, unit="m/s", convention=_RIGHT + ) + for policy in ( + ComparisonPolicyMode.STRICT, + ComparisonPolicyMode.AUDITED, + ComparisonPolicyMode.PERMISSIVE, + ): + result = compare_protocol_answers(pred, ref, context={}, policy_mode=policy) + assert result.equivalent is False, policy + + +@pytest.mark.parametrize( + "pred, ref", + [ + # Partial flip: only x negated (a global axis reversal flips every axis). + ( + _directional_vector((3.0, -4.0, 0.0), convention="x to the right"), + _directional_vector((-3.0, -4.0, 0.0), convention="x to the left"), + ), + # 3D one-axis-only flip. + ( + _directional_vector((1.0, 2.0, 3.0), convention=_RIGHT), + _directional_vector((-1.0, 2.0, 3.0), convention=_LEFT), + ), + # Constant ratio != -1 (scaling, not a sign flip). + ( + _directional_vector((3.0, -4.0), convention=_RIGHT), + _directional_vector((-6.0, 8.0), convention=_LEFT), + ), + # Zero vector: nothing to flip. + ( + _directional_vector((0.0, 0.0), convention=_RIGHT), + _directional_vector((0.0, 0.0), convention=_LEFT), + ), + ], +) +def test_protocol_sign_convention_vector_rejects( + pred: dict[str, object], ref: dict[str, object] +) -> None: + result = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.AUDITED + ) + assert result.equivalent is False, (pred, ref) + + +def test_protocol_sign_convention_blocked_when_question_fixes_convention() -> None: + pred = _directional_quantity( + "+20 m/s", numeric_value=20.0, unit="m/s", convention=_LEFT + ) + ref = _directional_quantity( + "-20 m/s", numeric_value=-20.0, unit="m/s", convention=_RIGHT + ) + context = {"sign_convention": "rightward is positive"} + result = compare_protocol_answers( + pred, ref, context=context, policy_mode=ComparisonPolicyMode.AUDITED + ) + assert result.equivalent is False + + +def test_protocol_sign_convention_absolute_labels_use_normal_path() -> None: + # Absolute directions (up vs down) carry no axis convention -> real disagreement. + up_vs_down = compare_protocol_answers( + _sign_label("up"), + _sign_label("down"), + context={}, + policy_mode=ComparisonPolicyMode.AUDITED, + ) + assert up_vs_down.equivalent is False + assert up_vs_down.comparison_mode == "sign_direction" + same = compare_protocol_answers( + _sign_label("up"), + _sign_label("up"), + context={}, + policy_mode=ComparisonPolicyMode.AUDITED, + ) + assert same.equivalent is True + assert same.comparison_mode == "sign_direction" + + +def test_protocol_sign_convention_no_convention_same_value_still_matches() -> None: + # The "accepted before sign comparison" case: equal values, one side unmarked. + pred = _directional_quantity("-20 m/s", numeric_value=-20.0, unit="m/s") + ref = _directional_quantity( + "-20 m/s", numeric_value=-20.0, unit="m/s", convention=_RIGHT + ) + result = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.AUDITED + ) + assert result.equivalent is True + assert result.comparison_mode == "physical_quantity" + + +def test_protocol_sign_convention_opposite_frame_vector_not_global_negation_rejects() -> ( + None +): + # Both frames opposite but values are identical (not a negation) -> precision reject, + # not the legacy coordinate_frame_mismatch. + pred = _directional_vector((3.0, -4.0), convention="x to the right") + ref = _directional_vector((3.0, -4.0), convention="x to the left") + result = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.AUDITED + ) + assert result.equivalent is False + assert result.comparison_mode == "sign_convention" + + +def test_protocol_sign_convention_non_opposite_vector_frames_still_mismatch() -> None: + # Genuinely-incompatible (non-opposite) frames keep the existing reject path. + pred = _directional_vector((3.0, 4.0), convention="x-axis positive") + ref = _directional_vector((3.0, 4.0), convention="y-axis positive") + result = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.AUDITED + ) + assert result.equivalent is False + assert "coordinate_frame_mismatch" in result.diagnostics + + +def test_protocol_sign_convention_sign_direction_precision_dual_rejects() -> None: + # Same polarity under opposite conventions resolves to opposite physical directions. + pred = _sign_label("positive", convention=_RIGHT) + ref = _sign_label("positive", convention=_LEFT) + result = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.AUDITED + ) + assert result.equivalent is False + assert result.comparison_mode == "sign_convention" + + +def test_protocol_sign_convention_non_polarity_labels_with_conventions_use_normal_path() -> ( + None +): + # "up"/"down" are absolute, not axis-relative: a stated convention cannot flip them. + pred = _sign_label("up", convention=_RIGHT) + ref = _sign_label("down", convention=_LEFT) + result = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.AUDITED + ) + assert result.equivalent is False + assert result.comparison_mode == "sign_direction" + + +def test_protocol_sign_convention_recognized_orthogonal_frames_decline() -> None: + # Both frames name a known direction but they are orthogonal (right vs up), not a + # global reversal -> the lane declines and the existing frame gate rejects. + pred = _directional_vector((3.0, 4.0), convention="x to the right") + ref = _directional_vector((3.0, 4.0), convention="up is positive") + result = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.AUDITED + ) + assert result.equivalent is False + assert "coordinate_frame_mismatch" in result.diagnostics + + +def test_protocol_sign_convention_symbolic_vector_partial_flip_rejects() -> None: + # Symbolic cells, only one component negated -> not a global negation. + pred = { + "object_kind": "expression", + "structure": "vector", + "canonical_text": "(a, b)", + "shape": [2], + "children": [ + {"object_kind": "expression", "structure": "atomic", "canonical_text": "a"}, + {"object_kind": "expression", "structure": "atomic", "canonical_text": "b"}, + ], + "coordinate_frame": "x to the right", + } + ref = { + "object_kind": "expression", + "structure": "vector", + "canonical_text": "(-a, b)", + "shape": [2], + "children": [ + { + "object_kind": "expression", + "structure": "atomic", + "canonical_text": "-a", + }, + {"object_kind": "expression", "structure": "atomic", "canonical_text": "b"}, + ], + "coordinate_frame": "x to the left", + } + result = compare_protocol_answers( + pred, ref, context={}, policy_mode=ComparisonPolicyMode.AUDITED + ) + assert result.equivalent is False + assert result.comparison_mode == "sign_convention" diff --git a/tests/prkit/verify/test_verify.py b/tests/prkit/verify/test_verify.py index 4cc4800..bad1b64 100644 --- a/tests/prkit/verify/test_verify.py +++ b/tests/prkit/verify/test_verify.py @@ -185,3 +185,100 @@ def test_returns_physics_answer_semantics(self): def test_category_not_supported_yet(self): with pytest.raises(NotImplementedError): parse("9.8", category="number") + + +def _answer(payload): + """Coerce a protocol-answer mapping into ``PhysicsAnswerSemantics`` for ``verify``.""" + from prkit.semantics import coerce_protocol_answer + + return coerce_protocol_answer(payload) + + +def _directional_quantity(text, value, unit, convention=None): + payload = { + "object_kind": "physical_quantity", + "canonical_text": text, + "numeric_value": value, + "numeric_text": str(value), + "unit": unit, + } + if convention is not None: + payload["coordinate_frame"] = convention + return _answer(payload) + + +def _directional_vector(components, convention=None): + payload = { + "object_kind": "number", + "structure": "vector", + "canonical_text": "(" + ", ".join(str(c) for c in components) + ")", + "shape": [len(components)], + "children": [ + { + "object_kind": "number", + "structure": "atomic", + "canonical_text": str(c), + "numeric_value": float(c), + "numeric_text": str(c), + } + for c in components + ], + } + if convention is not None: + payload["coordinate_frame"] = convention + return _answer(payload) + + +def _sign_label(label, convention=None): + payload = { + "object_kind": "sign_direction", + "canonical_text": label, + "sign_value": label, + } + if convention is not None: + payload["sign_convention"] = convention + return _answer(payload) + + +class TestVerifySignConvention: + """End-to-end: ``verify`` reconciles a global sign flip between opposite conventions. + + The lane fires only under an enforcement policy that enables bridges + (``unit_policy="audited"``) and only when both answers declare opposite, + globally-reversed conventions and the question fixes none. The default + (``"strict"``) keeps it off. Bridge metadata surfaces under ``Verdict.details``. + """ + + def test_velocity_flip_accepts_under_audited(self): + gold = _directional_quantity("-20 m/s", -20.0, "m/s", "right-as-positive") + pred = _directional_quantity("+20 m/s", 20.0, "m/s", "taking left as positive") + v = verify(gold, pred, unit_policy="audited") + assert v.correct is True + assert v.comparison_mode == "sign_convention" + assert v.details["bridge_id"] == "sign_convention" + assert v.details["bridge_tier"] is not None + + def test_vector_global_negation_accepts_under_audited(self): + gold = _directional_vector((-3.0, 4.0), "x to the left") + pred = _directional_vector((3.0, -4.0), "x to the right") + v = verify(gold, pred, unit_policy="audited") + assert v.correct is True + assert v.comparison_mode == "sign_convention" + + def test_sign_direction_resolves_under_audited(self): + gold = _sign_label("positive", "taking left as positive") + pred = _sign_label("negative", "right-as-positive") + v = verify(gold, pred, unit_policy="audited") + assert v.correct is True + assert v.comparison_mode == "sign_convention" + + def test_default_strict_policy_does_not_reconcile(self): + gold = _directional_quantity("-20 m/s", -20.0, "m/s", "right-as-positive") + pred = _directional_quantity("+20 m/s", 20.0, "m/s", "taking left as positive") + assert verify(gold, pred).correct is False + + def test_intrinsic_sign_never_reconciled(self): + # Charge sign is intrinsic, not an axis convention: +5 C != -5 C. + gold = _directional_quantity("+5 C", 5.0, "C") + pred = _directional_quantity("-5 C", -5.0, "C") + assert verify(gold, pred, unit_policy="audited").correct is False From de33a01bb44764c116d236d1985882cca6e1a6c0 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Fri, 19 Jun 2026 21:51:45 -0400 Subject: [PATCH 11/28] Document the sign-convention equivalence lane Record the implemented lane across the three comparison references: METHODOLOGY.md flips the section 4 recall-gap row to implemented and adds the discipline (convention as concrete per-answer data, the canonical- reframe lever, the admitted-class proof, and the precision dual) plus the section 5 residual. EQUIVALENCE.md documents the sign_direction resolution (7.5), the sign_convention bridge row and same-kind note (8), a policy example (9), the mode catalogue entry (11), and an end-to-end trace (13). STRUCTURE.md documents the refined _compare_shaped frame branch for vectors (both-unset / one-sided / opposite / incompatible) and adds the per-axis-frame residual. Co-Authored-By: Claude Opus 4.8 --- src/prkit/semantics/comparison/EQUIVALENCE.md | 37 ++++++++++++++++++ src/prkit/semantics/comparison/METHODOLOGY.md | 38 ++++++++++++++++++- src/prkit/semantics/comparison/STRUCTURE.md | 28 ++++++++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) diff --git a/src/prkit/semantics/comparison/EQUIVALENCE.md b/src/prkit/semantics/comparison/EQUIVALENCE.md index 403f046..4725148 100644 --- a/src/prkit/semantics/comparison/EQUIVALENCE.md +++ b/src/prkit/semantics/comparison/EQUIVALENCE.md @@ -300,6 +300,14 @@ clockwise ≡ clockwise → sign_direction increases ≡ goes up → qualitative_label (alias group) ``` +A `sign_direction` polarity (`positive`/`negative`) is *axis-relative*: paired with a +**stated** convention it resolves to an absolute physical direction, so two polarity answers +under opposite conventions can denote the same direction (`negative` taking right-as-positive +≡ `positive` taking left-as-positive — both *left*). That reconciliation is the +sign-convention lane (§8, `comparison_mode = sign_convention`), gated on the question fixing +no convention. Absolute labels (`up`, `clockwise`, `into_page`) are **not** axis-relative — +a flip there is a real disagreement and stays on the plain `sign_direction` path. + --- ## 8. Different object kind — tiered bridges @@ -316,6 +324,7 @@ bridge carries a **risk tier**; the tier and policy decide whether it is allowed | **T2** | `quantity_to_number` | quantity vs number when `q` fixes the unit | `5` ≡ `5 m/s²` (unit from `q`) | | **T2** | `expression_quantity` | expression vs physical quantity | — | | **T2** | `choice`, `terminal_polarity_choice`, `terminal_polarity` | choice ↔ label ↔ sign | — | +| **T2** | `sign_convention` | reconcile a global `−1` between directional answers (`number`/`physical_quantity`/`vector`/`sign_direction`) that declare **opposite** conventions, only when `q` fixes none | `−20 m/s` (right-as-positive) ≡ `+20 m/s` (left-as-positive) | | **T3** | `relation_to_qualitative_label` | relation vs a qualitative outcome | — | | **T3** | `qualitative_zero` | "no change" ↔ a zero value | `0` ≡ `no change` | | **T3** | `label_family_fallback` | last-resort same-/cross-kind label family | — | @@ -323,6 +332,14 @@ bridge carries a **risk tier**; the tier and policy decide whether it is allowed A bridged result records `bridge_id` and `bridge_tier` on the `AnswerComparison`. If no bridge fires → `object_kind_mismatch` (e.g. `5` vs choice `B`). +`sign_convention` is the one bridge that fires on **same-kind** (and same-structure, for +vectors) pairs rather than across kinds: it is tried before the same-kind criterion for +directional answers (and inside the shaped/vector path), but uses the identical tier/policy +machinery — `bridge_id`/`bridge_tier`/`bridge_evidence`, blocked under `strict`, enabled +under `audited`. Its evidence records the two stated conventions and the `global_-1` +reconciliation. See §7.4's `_proportional_ratio` (which already returns `−1` for a negation) +and METHODOLOGY.md §4 for the criterion and its precision dual. + --- ## 9. Policy in action @@ -339,6 +356,19 @@ permissive → 0 ≡ no change → qualitative_zero (bridge fires) So: `strict` is the same-kind criteria only; `audited` admits exactly the bridge tiers a contract opts into; `permissive` is the most lenient (and the legacy default). +The same gating governs the sign-convention lane. `−20 m/s` (right-as-positive) vs +`+20 m/s` (left-as-positive), with `q` fixing no convention (a TIER2 bridge): + +``` +strict → −20 m/s ≢ +20 m/s → bridge_blocked (no bridges under strict) +audited → −20 m/s ≡ +20 m/s → sign_convention (TIER2 enabled; opposite stated conventions, global −1) +permissive → −20 m/s ≡ +20 m/s → sign_convention +``` + +The precision dual is policy-independent: `−20 m/s` (left-as-positive) vs `−20 m/s` +(right-as-positive) → `sign_convention` **non-equivalent** under every policy (opposite +conventions, equal values ⇒ physically opposite). + --- ## 10. Numeric tolerance & reference precision @@ -435,6 +465,8 @@ The `AnswerComparison.comparison_mode` names the path taken: `expression_to_number`, `quantity_to_number`, `expression_quantity`, `choice`, `terminal_polarity`, `terminal_polarity_choice`, `relation_to_qualitative_label`, `qualitative_zero`, `label_family_fallback`. +- **Same-kind / structured bridge:** `sign_convention` (global `−1` reconciliation between + opposite stated conventions; also used by the vector/shaped path). - **Structured:** `tuple`, `set`, `multi_part`, `interval`, `vector`/`matrix`/`tensor`, `piecewise`. - **Non-equivalent / control:** `structure_mismatch`, `object_kind_mismatch`, @@ -476,6 +508,11 @@ it losslessly into the public `Verdict`: - `verify` with `q.symbol_assumptions` declaring `c, E, m` positive: `E = m c²` vs `c = √(E/m)` → both `relation` → `c = √(E/m)` de-radicalizes to `c² = E/m` (non-radical side `c ≥ 0`) → homogeneous numerators `c² m − E` and `E − m c²` differ by `−1` → **equivalent**. +- `verify("−20 m/s" right-as-positive, "+20 m/s" left-as-positive, unit_policy="audited")` + with `q` fixing no convention → both `physical_quantity` → sign-convention lane: conventions + are opposite (right vs left) and `+20 = −(−20)` → **equivalent**, mode `sign_convention`, + `details.bridge_id = "sign_convention"`. Under the default `strict` policy the same pair is + `bridge_blocked` → **non-equivalent**. --- diff --git a/src/prkit/semantics/comparison/METHODOLOGY.md b/src/prkit/semantics/comparison/METHODOLOGY.md index befca0b..8cc0482 100644 --- a/src/prkit/semantics/comparison/METHODOLOGY.md +++ b/src/prkit/semantics/comparison/METHODOLOGY.md @@ -131,7 +131,7 @@ criterion (not a rescue); the last is deferred pending a justified, gated criter | **Real-only identities** | `sqrt(a·b)`↔`√a·√b`, `sqrt(x²)`↔`\|x\|`, `log(ab)`↔`log a+log b` | **domain enrichment**: carry the symbols' real domain into the parse (`build_symbol_assumption_map`); positivity from `q.symbol_assumptions`, realness derived — implemented | | **`simplify` incompleteness** | nested-radical / transcendental identities `simplify` cannot crack | **criterion**: domain-honoring numeric identity testing (`_numeric_identity_equivalent`), exact on rejection — implemented | | **Solved radical** | `E=mc²` ↔ `c=√(E/m)`, `v²=u²+2as` ↔ `v=√(u²+2as)` | canonical **normalization**: de-radicalize when the non-radical side is nonnegative (`_deradicalize_clause`), gated on `q.symbol_assumptions` — implemented | -| **Sign convention** | global `−` sign on a directional quantity | needs a **gated criterion** — axis choice *or* real error; only directional quantities, only when frame/sign metadata absent, as an *audited* bridge — deferred | +| **Sign convention** | a directional answer flipped by a global `−` under an opposite, unstated axis choice (`−20 m/s` right-as-positive ↔ `+20 m/s` left-as-positive; a vector ↔ its negation) | **criterion** reconciling two *stated* conventions to a common frame (`sign_convention`, `_compare_sign_convention`); gated on the question fixing none, both answers declaring **opposite** conventions, and a provable global `−1` — implemented (audited bridge, see below) | ### Why positivity is declared, not derived from surface form @@ -158,6 +158,35 @@ changes the numerator by a constant, whereas a genuinely different equation diff are correctly excluded. It does **not** cover radical-introducing solves (`c=√(E/m)`) or inequalities, by design. +### Sign convention — concrete data, not a toggle + +A free axis/sign-convention choice acts on a directional quantity as exactly one +transformation: negating the chosen positive axis (a single global `−1` over the whole +quantity). The lane forgives that flip, but the convention is **concrete per-answer data**, +not an on/off switch — the reference's convention is determined at build time (the LLM, +from problem + figure + golden) and a prediction's comes from its own record; the judgement +then **reconciles the two stated conventions** to a common frame +(`sign_convention.py::compare_sign_convention`, and `_reconcile_shaped_sign_convention` for +vectors). The criterion admits a pair **iff**: (1) the question fixes no convention +(`q.sign_convention` and `q.coordinate_frame` both absent — else the axis is pinned and a +flip is a real error); (2) **both** answers declare conventions whose positive axes are a +provable *global* reversal (antonym directions — `_DIRECTION_OPPOSITE`); and (3) the values +are an exact global `−1` (reusing `_proportional_ratio`/`numbers_close`; for vectors, an +exact *component-wise* negation — a **partial** flip is a different vector, not a +convention). This is the §3 *canonical-reframe* lever applied symmetrically, never a +rescue. + +The change is **precision-symmetric** (the §3 rule against asymmetric relaxation): the same +machinery that *accepts* a flip also *rejects* the dual — opposite conventions with **equal** +values denote physically opposite quantities (`−20` left-as-positive ≠ `−20` +right-as-positive). That reject is policy-independent (rejecting is never a precision risk); +only the *accept* is the audited `sign_convention` bridge (TIER2 — fires under `audited` by +default because the stated-opposite-conventions evidence is the strong gate, blocked under +`strict`, recorded with `bridge_id`/`bridge_tier`/`bridge_evidence`). Because the lane +activates only when **both** answers carry stated conventions, a bare signed scalar with no +declared axis (a charge `−5 C` vs `+5 C`, a work `−30 J`) is never reconciled — directional +intent is *declared, not derived* (§4), exactly as positivity is. + ## 5. Checklist for an equivalence change 1. **Decide the lever:** a canonical-form normalization (§3) or a sharper per-kind @@ -172,6 +201,13 @@ inequalities, by design. 5. **Confirm the hot path** isn't materially slowed and the full suite stays green. 6. **Note the residual limit** so the next gap is discoverable. +The §4 recall-gap inventory is now fully addressed. The known residual frontier for the +sign-convention lane: (a) only a *global* axis reversal is reconciled — a single-axis frame +difference (which would flip one vector component) is deliberately rejected, pending a sound +per-axis frame algebra; and (b) the lane consumes answer-level conventions, so it stays +dormant until the build routes the reference's inferred convention onto `a_ref` (with +`q_ref` kept convention-free when the problem fixes none) — a build-side follow-up. + ## 6. Build-time methodology: constructing `q` and `a` for the judgement Everything above is about *running* `Eq(a_pred, a_ref ; q)`. This section is the matching diff --git a/src/prkit/semantics/comparison/STRUCTURE.md b/src/prkit/semantics/comparison/STRUCTURE.md index 61fb2f8..2e91688 100644 --- a/src/prkit/semantics/comparison/STRUCTURE.md +++ b/src/prkit/semantics/comparison/STRUCTURE.md @@ -92,6 +92,31 @@ rather than returning a silent verdict. See the per-comparator audit and gating the implementation plan and the batteries in `tests/prkit/semantics/`. +### Vector / shaped frames and sign-convention reconciliation + +A shaped answer (`vector` / `matrix` / `tensor`) carries an optional coordinate frame, and +`_compare_shaped` gates on it before the per-cell comparison: + +- **both frames unset** ⇒ assume the problem's implicit shared frame, proceed per-cell; +- **one-sided** (exactly one of pred/ref states a frame) ⇒ unresolved ⇒ **TBD** + (`coordinate_frame_unresolved`); +- **both set, the same** ⇒ proceed per-cell; +- **both set, opposite** (a *global* axis reversal — antonym positive axes, e.g. *x-right* vs + *x-left*) and the question fixes **no** convention ⇒ the **sign-convention** lane: an exact + **component-wise** negation is the same physical vector under a reversed frame ⇒ accept + (`comparison_mode = sign_convention`, the audited TIER2 bridge — blocked under `strict`); a + **partial** flip (some components negated, some not) or a non-`−1` scaling ⇒ reject + (`opposite_convention_not_global_negation`) — a single global reversal flips *every* axis, + so a partial change is a different vector; +- **both set, genuinely incompatible** (non-opposite, e.g. *x-axis* vs *y-axis*) ⇒ real + mismatch (`coordinate_frame_mismatch`). + +This refines the former flat "both-set-incompatible ⇒ reject": only the *opposite* sub-case +now reconciles, and only on the concrete evidence of two stated, globally-reversed frames +(see METHODOLOGY.md §4, "Sign convention — concrete data, not a toggle", and EQUIVALENCE.md +§8). The reconciliation reuses the atomic numeric/`_proportional_ratio` negation check +cell-by-cell, and at least one cell must be nonzero (a zero vector has no sign to flip). + **`allowed_structures` from the build must admit the collapse closure.** When a built `q` (METHODOLOGY.md §6) populates `allowed_structures`, the set is a **hard violating-gate** in `validate_answer_against_contract` — there is no bridge or collapse rescue for a structure @@ -111,6 +136,9 @@ precision lever." - folding `subject_to` bound-pairs into an interval (conflates a side-condition with an interval-valued answer); - `(n,)` ↔ `(n, 1)` shape reconciliation; +- **per-axis frame reconciliation** for vectors — only a *global* axis reversal is + reconciled today (§4); a single-axis frame difference (which would flip one component) is + rejected, pending a sound per-axis frame algebra; - **sound unordered matching algorithm** (`set` / unordered `multi_part`) under tolerance and symbolic equivalence — **roadmap milestone**. Today only *exact* multiset matches are accepted (exact numeric value or normalized text); everything requiring a tolerant/symbolic From a3158ad6fbc5b924518f57554efa24528405b79b Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sat, 20 Jun 2026 00:32:57 -0400 Subject: [PATCH 12/28] Capture sign conventions at build time so the equivalence lane fires on built data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The committed sign-convention lane reconciles a global sign flip on a directional answer only when both answers carry stated, opposite conventions and the question fixes none. It was correct-but-dormant on built data: the build never populated answer-level conventions and routed any inferred convention onto q_ref, which closes the lane's gate. Split q vs a so each record carries its convention where the judge reads it: - a_ref: Call A declares the convention the golden is expressed in (free axis); sign_convention added to _ANSWER_SURFACE_FILL_FIELDS, adopted fill-only. - q_ref: Call B sets a convention only when the problem text itself fixes one. - a_pred: a solver-declared field for a_pred_llm; a conservative, declaration-only parser (_extract_sign_convention_declaration) for a_pred_ext that reads an explicit " as positive" clause, strips it before value parsing, and never guesses from a bare sign. reference_pair_consistency flags a q-fixed-vs-a_ref-opposite convention. Conventions are recorded in sign_convention at every capture point (coordinate_frame stays for a named frame) so the field-specific vector judge path cannot manufacture a spurious one-sided TBD. Prompt versions bumped v5->v6 and v4->v5; no schema change. Also make the prediction answer-semantics form a consumer choice rather than a provider-capability decision: infer_prediction_semantics gains answer_semantics — "structured" requires native structured output and raises otherwise, "extracted" uses the plain-text deterministic route, and "auto" (default) keeps the prior best-effort behavior. Tests: extended staged-build and isolated-prediction suites; new parser adversarial battery, built-records->verify integration cases (flip accepts; no-convention / precision-dual / q-fixed reject; vector both-sided accept + one-sided TBD), and an opt-in live smoke. Full gate green. Co-Authored-By: Claude Opus 4.8 --- src/prkit/semantics/comparison/METHODOLOGY.md | 53 +++- src/prkit/semantics/inference/calls.py | 89 ++++-- src/prkit/semantics/inference/prompts.py | 29 +- .../semantics/inference/semantics_build.py | 25 +- .../normalization/answer_normalization.py | 146 +++++++++- .../test_prediction_isolated_build.py | 122 +++++++- .../test_sign_convention_build_integration.py | 269 ++++++++++++++++++ .../test_sign_convention_build_live.py | 55 ++++ .../test_sign_convention_declaration.py | 103 +++++++ tests/prkit/semantics/test_staged_build.py | 71 ++++- 10 files changed, 910 insertions(+), 52 deletions(-) create mode 100644 tests/prkit/semantics/test_sign_convention_build_integration.py create mode 100644 tests/prkit/semantics/test_sign_convention_build_live.py create mode 100644 tests/prkit/semantics/test_sign_convention_declaration.py diff --git a/src/prkit/semantics/comparison/METHODOLOGY.md b/src/prkit/semantics/comparison/METHODOLOGY.md index 8cc0482..526b090 100644 --- a/src/prkit/semantics/comparison/METHODOLOGY.md +++ b/src/prkit/semantics/comparison/METHODOLOGY.md @@ -204,9 +204,16 @@ intent is *declared, not derived* (§4), exactly as positivity is. The §4 recall-gap inventory is now fully addressed. The known residual frontier for the sign-convention lane: (a) only a *global* axis reversal is reconciled — a single-axis frame difference (which would flip one vector component) is deliberately rejected, pending a sound -per-axis frame algebra; and (b) the lane consumes answer-level conventions, so it stays -dormant until the build routes the reference's inferred convention onto `a_ref` (with -`q_ref` kept convention-free when the problem fixes none) — a build-side follow-up. +per-axis frame algebra; and (b) for a *vector* answer a **one-sided** convention (declared on +one side only) is a deliberate TBD (precision-safe — never a false accept), so populating a +vector `a_ref`'s frame can move a previously per-cell-accepted pair to TBD until the prediction +also declares its frame. + +The lane is now **live on built data** (it was previously correct-but-dormant): the build +routes conventions to exactly where the judgement reads them — `a_ref` carries the golden's +expressed convention, `a_pred` carries the prediction's, and `q_ref` stays convention-free +unless the problem text itself fixes one (see §6, "Directional conventions"). Δrecall at fixed +precision is measured in the consumer (`uq`) repo. ## 6. Build-time methodology: constructing `q` and `a` for the judgement @@ -253,10 +260,14 @@ Native provider-enforced structured output is a **Step-2 output-form concern onl provider supports it, otherwise plain text parsed back), so a provider lacking native structured output still yields a full `(q_ref, a_ref)`. Lacking it is a normal route, not a defect — it does not set `review_required` (only a genuine cross-check failure does). -- **Step 2 yields one form instead of two.** With native structured output the solve returns - `a_pred_llm` (and the `a_pred_ext` disagreement audit); without it there is **one route** — - the output is plain text and `a_pred_ext = canonicalize_structure(normalize_physics_answer(...))` - is the deterministic extraction. Never a failure; just one record instead of two. +- **Step 2's form is the consumer's choice, not the toolkit's.** `infer_prediction_semantics` + takes `answer_semantics`: `"structured"` returns `a_pred_llm` (native provider-enforced output; + it **raises** if the provider cannot enforce it — no silent substitution), `"extracted"` returns + `a_pred_ext = canonicalize_structure(normalize_physics_answer(...))` (plain-text solve, needs no + native support), and `"auto"` (the default) picks by provider capability — `a_pred_llm` when + supported, else `a_pred_ext`. The toolkit is **neutral**: it does exactly what is asked, and + `"auto"` is the only capability-driven mode (the consumer explicitly leaves the choice to it). + Whichever single record a form yields, the judgement consumes it identically. - **Step 3 is provenance-agnostic.** The judgement consumes a `PhysicsAnswerSemantics` regardless of whether it came from `a_pred_llm` or `a_pred_ext` — both are simply "generated answer semantics." *Which* form a caller feeds in is out of this toolkit's scope. @@ -304,6 +315,34 @@ conflict between sources, the build declares the **least-restrictive sound** ass asserting an unjustified one would manufacture false accepts, exactly the failure §4 guards against on the engine side. +### Directional conventions — `q` fixes, `a` expresses + +The sign-convention lane (§4) reads *answer-level* conventions and is gated on the question +fixing none. The build populates them on that exact split — the same declared-not-derived +discipline as `symbol_assumptions`: + +- **`q_ref.sign_convention`/`coordinate_frame`** — set (Call B) **only** when the *problem text* + itself fixes a convention every answer must follow. That pins the axis, so a flip is a real + error and the lane's gate closes. If the problem leaves the axis free, `q_ref` stays + convention-free. +- **`a_ref.sign_convention`** — set (Call A, adopted fill-only with provenance `llm_declared`) + to the convention the *golden is expressed in*, when the golden is a directional quantity on a + free axis. This is the lane's evidence, not a question policy. +- **`a_pred.sign_convention`** — the prediction's own convention: a solver-declared field on + `a_pred_llm`, or a **conservative, declaration-only** parse of an explicit + "`` as positive" clause in the `a_pred_ext` surface (`_extract_sign_convention_declaration`). + A bare sign (`+20`) is never read as a convention — directional intent is *declared, not + derived*, exactly as positivity is. + +Two engineering invariants keep capture and judgement aligned: (1) a positive-direction choice +is recorded in **`sign_convention`** at every capture point (`coordinate_frame` is reserved for +an explicitly *named* frame), because the vector judge path reads the two fields +field-specifically — mixing them would manufacture a spurious one-sided TBD; and (2) the captured +string reuses the engine's direction vocabulary (`_SIGN_DIRECTION_CANONICAL`) so +`_convention_orientation` reads one orientation. A co-construction cross-check +(`reference_pair_consistency`) flags the build inconsistency where `q_ref` fixes a convention but +`a_ref` is expressed in a provably-*opposite* one. + ### `symbol_assumptions` — source precedence and the canonical-token requirement Assumptions are synthesized with a provenance-tagged precedence diff --git a/src/prkit/semantics/inference/calls.py b/src/prkit/semantics/inference/calls.py index 5c82463..7708d30 100644 --- a/src/prkit/semantics/inference/calls.py +++ b/src/prkit/semantics/inference/calls.py @@ -5,7 +5,7 @@ import logging from dataclasses import dataclass from pathlib import Path -from typing import Any, TypeVar +from typing import Any, Literal, TypeVar from pydantic import BaseModel, ValidationError @@ -299,6 +299,8 @@ def infer_reference_semantics( "choice_label", "boolean_value", "sign_value", + "coordinate_frame", + "sign_convention", ) @@ -772,11 +774,20 @@ def build_problem_semantics( ) +PredictionAnswerForm = Literal["structured", "extracted", "auto"] +_PREDICTION_ANSWER_FORMS: tuple[PredictionAnswerForm, ...] = ( + "structured", + "extracted", + "auto", +) + + def infer_prediction_semantics( problem: PhysicsProblem, model_client: BaseModelClient, *, isolated_solve: bool = True, + answer_semantics: PredictionAnswerForm = "auto", max_output_tokens: int | None = None, allow_non_native_structured_output: bool = False, **chat_kwargs: Any, @@ -790,15 +801,41 @@ def infer_prediction_semantics( the ``a_pred_llm``-vs-``a_pred_ext`` structure-disagreement audit. Set ``isolated_solve=False`` to keep the legacy fused path (the draft is injected and the model authors a prediction-side ``question_semantics``). + + ``answer_semantics`` selects which prediction record the isolated solve yields — the + **consumer's** choice, not a toolkit decision (the toolkit is neutral and does exactly + what is asked): + + * ``"structured"`` — return ``a_pred_llm`` (native provider-enforced structured output); + raises if the provider cannot enforce it (no silent substitution). + * ``"extracted"`` — return ``a_pred_ext`` (plain-text solve, then deterministic + ``canonicalize_structure(normalize_physics_answer(...))`` extraction); needs no native + structured output. + * ``"auto"`` (default) — pick by provider capability (``a_pred_llm`` when the provider + supports native structured output, otherwise ``a_pred_ext``). The only capability-driven + mode, and only because the consumer left the choice to the toolkit. + + ``answer_semantics`` governs the isolated path only; passing a non-``"auto"`` value with + ``isolated_solve=False`` raises. """ + if answer_semantics not in _PREDICTION_ANSWER_FORMS: + raise ValueError( + f"answer_semantics must be one of {list(_PREDICTION_ANSWER_FORMS)}, " + f"got {answer_semantics!r}." + ) + if not isolated_solve and answer_semantics != "auto": + raise ValueError( + "answer_semantics is only supported on the isolated solve path " + "(isolated_solve=True). The legacy fused path always returns a_pred_llm with " + "deterministic a_pred_ext reconstruction for compact providers." + ) + if isolated_solve: - # The isolated solve is always graceful (`allow_non_native_structured_output` applies - # only to the legacy fused path): missing native structured output yields a_pred_ext, - # never a failure (the user's "plain output -> deterministic extraction" route). return _infer_isolated_prediction_semantics( problem, model_client, + answer_semantics=answer_semantics, max_output_tokens=max_output_tokens, **chat_kwargs, ) @@ -858,23 +895,39 @@ def _infer_isolated_prediction_semantics( problem: PhysicsProblem, model_client: BaseModelClient, *, + answer_semantics: PredictionAnswerForm = "auto", max_output_tokens: int | None, **chat_kwargs: Any, ) -> PredictionSemanticsArtifact: - """Problem-only isolated solve building a_pred_llm (+ the a_pred_ext disagreement audit). + """Problem-only isolated solve producing the consumer-selected prediction record. - Always graceful: native structured output is used when the provider supports it (so - ``a_pred_llm`` is produced), otherwise the solve falls back to plain text and only - ``a_pred_ext`` (deterministic extraction) is produced — lacking native structured output - never fails this step, it just yields one answer-semantics form instead of two. + ``answer_semantics`` decides the form (see :func:`infer_prediction_semantics`): + ``"structured"`` forces ``a_pred_llm`` and **raises** if the provider cannot enforce native + structured output; ``"extracted"`` forces the plain-text ``a_pred_ext`` route; ``"auto"`` + picks by provider capability (``a_pred_llm`` when supported, else ``a_pred_ext``). The + toolkit does exactly what is asked — it never silently substitutes a different form. """ - response_model = resolve_isolated_prediction_response_model(model_client) - # Best-effort: prefer native when available, never require it. - require_native_json_schema = model_client.resolve_structured_output_plan( - response_model, - structured_policy="best_effort", - ).native_schema_enforced + response_model: type[BaseModel] + if answer_semantics == "structured": + # Honor the explicit request: require native structured output; _run_structured_inference + # raises a clear error (ensure_semantics_native_structured_output_support) if unavailable. + response_model = StrictPredictionIsolatedResponse + require_native_json_schema = True + elif answer_semantics == "extracted": + # Plain-text solve, then deterministic extraction. Never needs native structured output. + response_model = StrictPredictionFinalAnswerResponse + require_native_json_schema = model_client.resolve_structured_output_plan( + response_model, + structured_policy="best_effort", + ).native_schema_enforced + else: + # "auto": the only capability-driven mode (the consumer left the choice to the toolkit). + response_model = resolve_isolated_prediction_response_model(model_client) + require_native_json_schema = model_client.resolve_structured_output_plan( + response_model, + structured_policy="best_effort", + ).native_schema_enforced spec = prepare_isolated_prediction_semantics_inference_spec( problem, response_model=response_model, @@ -921,10 +974,12 @@ def _infer_isolated_prediction_semantics( f"!={a_pred_ext.structure.value}/{a_pred_ext.object_kind.value}" ) else: - # Provider could not enforce the isolated schema; only a_pred_ext is available. + # a_pred_ext is the produced record. Under "extracted" it is the requested form (no + # flag); under "auto" it means the provider could not enforce the structured schema. adopted = a_pred_ext build_method = "prediction_isolated_extracted" - flags.append("a_pred_llm_unavailable") + if answer_semantics == "auto": + flags.append("a_pred_llm_unavailable") report = SemanticsBuildReport( build_method=build_method, diff --git a/src/prkit/semantics/inference/prompts.py b/src/prkit/semantics/inference/prompts.py index 867e5c7..54a608a 100644 --- a/src/prkit/semantics/inference/prompts.py +++ b/src/prkit/semantics/inference/prompts.py @@ -13,13 +13,17 @@ from ..schema import PhysicsAnswerSemantics, PhysicsQuestionSemantics REFERENCE_PROMPT_NAME = "reference_semantics" +# v6: answer-level sign-convention declaration — Call A sets a_ref's expressed convention when the +# problem leaves the axis free; Call B sets q_ref's convention only when the problem text fixes one. # v5: staged 3-call build (answer-surface cleanup / question policy / symbol assumptions) # replaces the single fused reference call; structure/kind are deterministically pinned. -REFERENCE_PROMPT_VERSION = "v5" +REFERENCE_PROMPT_VERSION = "v6" PREDICTION_PROMPT_NAME = "prediction_semantics" +# v5: directional answers declare their sign convention — a_pred_llm via +# prediction_answer_semantics.sign_convention, a_pred_ext by stating it in the final-answer surface. # v4: isolated problem-only solve flag (suppresses the embedded question-semantics draft) + # STRUCTURE.md section-2 surface conventions added to the answer-format guidance. -PREDICTION_PROMPT_VERSION = "v4" +PREDICTION_PROMPT_VERSION = "v5" PROBLEM_PROMPT_NAME = "problem_semantics" # Problem-only (answer-blind) q_prob build, shares the staged question-policy / assumption # prompts with no answer context. @@ -70,6 +74,7 @@ - Shaped array: a vector as `<...>` or `[a, b, c]` (depth 1); a matrix as nested brackets (depth 2); keep a `(n,)` vector distinct from an `(n, 1)` matrix. - Piecewise function: use `\\begin{cases}...\\end{cases}` or `Piecewise(...)`. - Equation/relation: write the full relation, e.g. `F = m*a`, `v >= 0`. +- Free-axis sign convention: if your answer is a directional quantity (signed scalar, vector, or direction) whose sign depends on a positive-direction choice the problem left free, state that choice explicitly in the surface, e.g. `-20 m/s (taking rightward as positive)`. """ @@ -165,7 +170,10 @@ def build_prediction_semantics_prompt( ) if include_prediction_answer_semantics: sections.append( - "Make `prediction_answer_semantics` match that final answer exactly." + "Make `prediction_answer_semantics` match that final answer exactly. When your answer " + "is directional and the problem fixed no positive direction, set " + "`prediction_answer_semantics.sign_convention` to the convention you used (e.g. " + "`right-as-positive`)." ) else: sections.append( @@ -201,6 +209,13 @@ def build_answer_surface_cleanup_prompt( "Only improve `canonical_text`, `canonical_latex`, `unit`, `numeric_text`, " "`choice_label`, and similar surface fields; preserve the answer's printed numeric " "precision (do not round or add digits).", + "If the golden is a directional quantity (a signed scalar, a vector, or a directional " + "sign) whose sign depends on a positive-direction / axis choice the problem did NOT fix, " + "set `sign_convention` to the convention this answer is expressed in (e.g. " + "`right-as-positive`, `up-as-positive`), inferred from the problem text, any figure, and " + "the golden's sign. Otherwise leave `sign_convention` null — for a non-directional " + "quantity, or when the problem itself fixes the axis (that is a question policy, captured " + "separately, not an answer attribute).", _format_problem(problem, include_reference_context=True), "Ground-truth answer surface:\n" + golden_text, "Toolkit deterministic draft answer semantics (authoritative for structure/kind):", @@ -221,8 +236,12 @@ def build_question_policy_prompt( "Task: return the question-side policy semantics that constrain acceptable answers.", _STAGED_BUILD_NOTE, "Provide `target_variable`, `symbol_aliases`, `question_unit_policy`, `question_unit`, " - "`dimension`, `ordering`, `required_parts`, `coordinate_frame`, `sign_convention`, and " - "`choice_space` when applicable. Leave `symbol_assumptions` empty here (declared separately).", + "`dimension`, `ordering`, `required_parts`, and `choice_space` when applicable. Leave " + "`symbol_assumptions` empty here (declared separately).", + "Set `coordinate_frame` / `sign_convention` ONLY if the problem statement itself fixes a " + "convention every answer must follow (e.g. it says 'take rightward as positive', or it " + "defines the axes). If the problem leaves the positive direction / axis free, leave both " + "null — the convention the golden answer happens to use is captured on the answer, not here.", _format_problem(problem, include_reference_context=answer_draft is not None), ] if answer_draft is not None: diff --git a/src/prkit/semantics/inference/semantics_build.py b/src/prkit/semantics/inference/semantics_build.py index 55de5d7..d221302 100644 --- a/src/prkit/semantics/inference/semantics_build.py +++ b/src/prkit/semantics/inference/semantics_build.py @@ -31,6 +31,10 @@ from collections.abc import Iterable, Mapping, Sequence from ..comparison.contract import _STRUCTURES_COLLAPSIBLE_TO_ATOMIC +from ..comparison.sign_convention import ( + answer_directional_convention, + orientation_relation, +) from ..schema import ( DEFAULT_NUMERIC_TOLERANCE, AnswerObjectKind, @@ -505,9 +509,10 @@ def reference_pair_consistency( """Return ``q_ref`` <-> ``a_ref`` inconsistencies (empty when mutually consistent). A co-constructed pair must satisfy: the contract admits the gold answer's kind/structure - (honoring the collapse target), any shared ``target_variable`` agrees, and every + (honoring the collapse target), any shared ``target_variable`` agrees, every ``symbol_assumptions`` token is canonical (post-alias) so the engine will not silently - drop it. + drop it, and -- when the problem fixes a convention -- the gold is not expressed in a + provably-opposite one. """ issues: list[str] = [] @@ -539,6 +544,22 @@ def reference_pair_consistency( for symbol in alias_source_violations(declared, alias_map) ) + # When the problem fixes a convention (q_ref carries one), the gold must not be expressed in + # a provably-*opposite* one (a build inconsistency: the golden's stated frame reverses the + # problem's). Flag only the proven-opposite case (declared-not-derived); an indeterminate or + # matching convention is fine. Reuses the engine's orientation reader so capture and + # judgement share one vocabulary. + question_convention = question.coordinate_frame or question.sign_convention + answer_convention = answer_directional_convention(reference_answer) + if ( + question_convention + and answer_convention + and orientation_relation(answer_convention, question_convention) == "opposite" + ): + issues.append( + f"opposite_convention_vs_question:{answer_convention}!~{question_convention}" + ) + return issues diff --git a/src/prkit/semantics/normalization/answer_normalization.py b/src/prkit/semantics/normalization/answer_normalization.py index ca10084..16338f8 100644 --- a/src/prkit/semantics/normalization/answer_normalization.py +++ b/src/prkit/semantics/normalization/answer_normalization.py @@ -95,6 +95,59 @@ "down the plane of the page": "down_in_plane", } +# Kinds for which an explicit " as positive" surface declaration is captured onto the +# answer's ``sign_convention`` (the directional scalars). Vectors/structured answers carry a +# convention via the LLM-structured record (``a_pred_llm``), not this deterministic parser. +_SIGN_CONVENTION_DECLARATION_KINDS = frozenset( + {AnswerObjectKind.NUMBER, AnswerObjectKind.PHYSICAL_QUANTITY} +) + + +def _build_sign_convention_declaration_re() -> re.Pattern[str]: + """Compile the declaration-only convention parser from the direction vocabulary. + + Fires only on an explicit direction word *adjacent to* "positive" (e.g. ``taking rightward + as positive``, ``right-as-positive``, ``positive direction is up``, ``+ve = left``). A bare + sign (``+20``) or a fully-specifying direction phrase (``5 N to the right``, no "positive") + is **not** a convention declaration, so it never matches — the sign-convention lane stays + declared-not-derived. + """ + + directions = sorted( + ( + phrase + for phrase, orientation in _SIGN_DIRECTION_CANONICAL.items() + if orientation not in {"positive", "negative"} + and not phrase.startswith(("+", "-")) + ), + key=len, + reverse=True, + ) + dirs = r"\b(?:" + "|".join(re.escape(phrase) for phrase in directions) + r")\b" + alternatives = "|".join( + ( + rf"(?:taking|with|assuming|measuring|counting|treating|where)\s+" + rf"(?P{dirs})\s+(?:as\s+|is\s+|=\s*|to\s+be\s+)?positive", + rf"(?P{dirs})[\s-]+(?:as|is|=|taken\s+as|counted\s+as)[\s-]*positive", + rf"positive\s+(?:direction\s+is\s+|axis\s+is\s+|is\s+)?(?P{dirs})", + rf"\+\s*ve\s*=\s*(?P{dirs})", + ) + ) + return re.compile( + rf"(?P[\s(,;:]*(?:{alternatives})[\s).;:]*)", re.IGNORECASE + ) + + +_SIGN_CONVENTION_DECLARATION_RE = _build_sign_convention_declaration_re() +_DECLARATION_PHRASE_SEPARATOR_RE = re.compile(r"[\s_-]+") + + +def _normalize_declaration_phrase(phrase: str) -> str: + """Collapse a matched direction phrase to the judge-readable spaced form.""" + + return _DECLARATION_PHRASE_SEPARATOR_RE.sub(" ", phrase.strip().lower()) + + _QUALITATIVE_ALIAS_GROUPS = { "constant_temperature": { "temperature stays constant", @@ -186,27 +239,90 @@ def normalize_physics_answer( resolved_context, ) - structured = _normalize_structured_text(stripped, context=resolved_context) + # Capture an explicit " as positive" convention declaration and strip it before parsing + # (declaration-only; no match leaves `main_text == stripped`, so behavior is unchanged). The + # strip runs before the structured / subject_to parsers so a parenthetical or comma-introduced + # clause is neither misparsed as a tuple nor mis-filed as a subject_to. + main_text, declared_convention = _extract_sign_convention_declaration(stripped) + + structured = _normalize_structured_text(main_text, context=resolved_context) if structured is not None: - structured = structured.model_copy( + outcome = structured.model_copy( update={"provenance": provenance | structured.provenance} ) - return _finalize_outcome(structured, resolved_context) + else: + atomic_with_subject_to = _normalize_atomic_with_subject_to( + main_text, + context=resolved_context, + provenance=provenance, + ) + outcome = ( + atomic_with_subject_to + if atomic_with_subject_to is not None + else _normalize_atomic_text( + main_text, + context=resolved_context, + provenance=provenance, + ) + ) - atomic_with_subject_to = _normalize_atomic_with_subject_to( - stripped, - context=resolved_context, - provenance=provenance, - ) - if atomic_with_subject_to is not None: - return _finalize_outcome(atomic_with_subject_to, resolved_context) + outcome = _apply_declared_sign_convention(outcome, declared_convention) + return _finalize_outcome(outcome, resolved_context) - atomic = _normalize_atomic_text( - stripped, - context=resolved_context, - provenance=provenance, + +def _extract_sign_convention_declaration(text: str) -> tuple[str, str | None]: + """Split off an explicit positive-direction convention declaration from a surface. + + Returns ``(text_without_declaration, sign_convention)`` where ``sign_convention`` is a + judge-readable ``" as positive"`` string (e.g. ``"rightward as positive"``) when an + explicit declaration is present, else ``(text, None)``. The convention vocabulary is shared + with the comparison engine (:data:`_SIGN_DIRECTION_CANONICAL`), so capture and judgement read + one orientation. + """ + + match = _SIGN_CONVENTION_DECLARATION_RE.search(text) + if match is None: + return text, None + direction_phrase = next( + ( + value + for key, value in match.groupdict().items() + if key.startswith("dir") and value + ), + None, ) - return _finalize_outcome(atomic, resolved_context) + if direction_phrase is None: + return text, None + stripped = (text[: match.start("clause")] + text[match.end("clause") :]).strip() + stripped = stripped.strip(" ,;:") + if not stripped: + # The whole surface was the declaration; nothing to attach a value to. + return text, None + convention = f"{_normalize_declaration_phrase(direction_phrase)} as positive" + return stripped, convention + + +def _apply_declared_sign_convention( + outcome: PhysicsAnswerSemantics, declared_convention: str | None +) -> PhysicsAnswerSemantics: + """Attach an explicitly-declared positive-direction convention to a directional scalar. + + Applied only to an atomic number / physical quantity whose convention is otherwise unset, so + a bare signed value (no stated direction) stays convention-free and the sign-convention lane + stays off. Structured answers already inherit context conventions in + :func:`_make_structured_outcome`, so they are deliberately excluded here (no double-write). + """ + + if declared_convention is None: + return outcome + if ( + outcome.is_atomic + and outcome.object_kind in _SIGN_CONVENTION_DECLARATION_KINDS + and outcome.sign_convention is None + and outcome.coordinate_frame is None + ): + return outcome.model_copy(update={"sign_convention": declared_convention}) + return outcome def _normalize_structured_text( diff --git a/tests/prkit/semantics/test_prediction_isolated_build.py b/tests/prkit/semantics/test_prediction_isolated_build.py index 97b0555..043d4cd 100644 --- a/tests/prkit/semantics/test_prediction_isolated_build.py +++ b/tests/prkit/semantics/test_prediction_isolated_build.py @@ -15,6 +15,8 @@ import json from typing import Any +import pytest + from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem from prkit.core.model_clients import BaseModelClient from prkit.semantics.inference.calls import ( @@ -51,11 +53,17 @@ class _IsolatedSolveStubModelClient(BaseModelClient): supports_response_format_json_schema = True - def __init__(self, *, answer_payload: dict[str, Any] | None = None) -> None: + def __init__( + self, + *, + answer_payload: dict[str, Any] | None = None, + final_answer: str = "sqrt(E/m)", + ) -> None: super().__init__(model="stub-model") self.provider = "stub" self.prompts: list[str] = [] self.response_formats: list[Any] = [] + self._final_answer = final_answer self._answer_payload = answer_payload or { "canonical_text": "sqrt(E/m)", "object_kind": "expression", @@ -79,13 +87,27 @@ def response( return json.dumps( { "reasoning": "Energy conservation.", - "final_answer": "sqrt(E/m)", + "final_answer": self._final_answer, "prediction_answer_semantics": self._answer_payload, } ) + if name == "StrictPredictionFinalAnswerResponse": + # The compact (plain-text) route used by the "extracted" form. + return json.dumps( + { + "reasoning": "Energy conservation.", + "final_answer": self._final_answer, + } + ) raise AssertionError(f"unexpected response schema: {name}") +class _NoNativeSolveStubModelClient(_IsolatedSolveStubModelClient): + """A solver that cannot enforce native structured output (so "structured" must raise).""" + + supports_response_format_json_schema = False + + def test_isolated_solve_prompt_suppresses_question_semantics_draft() -> None: prompt = build_prediction_semantics_prompt( _problem(), suppress_question_semantics_draft=True @@ -152,8 +174,11 @@ def test_isolated_solve_prompt_has_no_golden_or_assumptions_leak() -> None: solve_prompt = client.prompts[0] assert "Toolkit heuristic draft question semantics:" not in solve_prompt - # Gold subject_to / domain declarations must not reach the solver. - assert "positive" not in solve_prompt + # Gold subject_to / domain declarations must not reach the solver. (The bare word + # "positive" now appears in the answer-surface sign-convention guidance, e.g. + # "right-as-positive", so assert the *gold's symbol assumption* did not leak rather than + # the word itself.) + assert '"assumption": "positive"' not in solve_prompt assert "symbol_assumptions" not in solve_prompt assert "m > 0" not in solve_prompt assert "Solution:" not in solve_prompt @@ -197,3 +222,92 @@ def test_resolve_isolated_prediction_response_model_prefers_isolated_schema() -> resolve_isolated_prediction_response_model(client) is StrictPredictionIsolatedResponse ) + + +# --- sign-convention capture + consumer-selected answer form --------------------------------- + + +def test_a_pred_llm_carries_solver_declared_sign_convention() -> None: + client = _IsolatedSolveStubModelClient( + answer_payload={ + "canonical_text": "-20 m/s", + "object_kind": "physical_quantity", + "structure": "atomic", + "sign_convention": "right-as-positive", + }, + final_answer="-20 m/s", + ) + artifact = infer_prediction_semantics( + _problem(), client, answer_semantics="structured" + ) + + a_pred = artifact.prediction_answer_semantics + assert a_pred.object_kind == AnswerObjectKind.PHYSICAL_QUANTITY + assert a_pred.sign_convention == "right-as-positive" + assert artifact.build_report.build_method == "prediction_isolated_llm" + + +def test_a_pred_ext_parser_captures_surface_convention() -> None: + ext = extract_prediction_answer_semantics("-20 m/s (taking rightward as positive)") + assert ext.object_kind == AnswerObjectKind.PHYSICAL_QUANTITY + assert ext.sign_convention == "rightward as positive" + assert ext.numeric_value == -20.0 + # A bare signed value with no stated direction stays convention-free. + assert extract_prediction_answer_semantics("-20 m/s").sign_convention is None + + +def test_answer_semantics_extracted_overrides_capability() -> None: + # The stub CAN enforce native structured output, but the consumer asked for "extracted": + # the toolkit honors the choice and yields a_pred_ext from the plain-text surface. + client = _IsolatedSolveStubModelClient( + final_answer="-20 m/s (taking rightward as positive)" + ) + artifact = infer_prediction_semantics( + _problem(), client, answer_semantics="extracted" + ) + + assert artifact.build_report.build_method == "prediction_isolated_extracted" + # Not flagged unavailable: extraction was the requested form, not a capability fallback. + assert not any( + "a_pred_llm_unavailable" in flag for flag in artifact.build_report.flags + ) + assert artifact.build_report.review_required is False + # The surface-stated convention is captured deterministically. + assert ( + artifact.prediction_answer_semantics.sign_convention == "rightward as positive" + ) + # The solver was asked for the compact final-answer schema, not the structured one. + assert any( + isinstance(rf, dict) and rf.get("name") == "StrictPredictionFinalAnswerResponse" + for rf in client.response_formats + ) + + +def test_answer_semantics_structured_raises_without_native_support() -> None: + client = _NoNativeSolveStubModelClient() + + with pytest.raises(ValueError, match="native"): + infer_prediction_semantics(_problem(), client, answer_semantics="structured") + + +def test_answer_semantics_rejects_unknown_value() -> None: + client = _IsolatedSolveStubModelClient() + + with pytest.raises(ValueError, match="answer_semantics"): + infer_prediction_semantics( + _problem(), + client, + answer_semantics="weird", # type: ignore[arg-type] + ) + + +def test_answer_semantics_non_auto_rejected_on_fused_path() -> None: + client = _IsolatedSolveStubModelClient() + + with pytest.raises(ValueError, match="isolated"): + infer_prediction_semantics( + _problem(), + client, + isolated_solve=False, + answer_semantics="structured", + ) diff --git a/tests/prkit/semantics/test_sign_convention_build_integration.py b/tests/prkit/semantics/test_sign_convention_build_integration.py new file mode 100644 index 0000000..748ca2d --- /dev/null +++ b/tests/prkit/semantics/test_sign_convention_build_integration.py @@ -0,0 +1,269 @@ +"""Offline end-to-end: built reference/prediction records drive the sign-convention lane. + +These tests exercise the whole chain — the staged reference build places the golden's +convention on ``a_ref`` (and a problem-fixed convention on ``q_ref``); the prediction build +places the prediction's convention on its record — then ``verify(...)`` reconciles (or +refuses) a global sign flip. They prove the lane is *live on built data*, not just on +hand-constructed records. +""" + +from __future__ import annotations + +import json +from typing import Any + +from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem +from prkit.core.model_clients import BaseModelClient +from prkit.semantics.inference.calls import ( + build_reference_semantics, + extract_prediction_answer_semantics, + infer_prediction_semantics, +) +from prkit.verify import verify + + +def _quantity_problem(golden: str) -> PhysicsProblem: + return PhysicsProblem( + problem_id="signconv-int", + question="Find the block's velocity v.", + answer=Answer(value=golden, answer_category=AnswerCategory.PHYSICAL_QUANTITY), + domain="mechanics", + ) + + +def _vector_problem(golden: str) -> PhysicsProblem: + return PhysicsProblem( + problem_id="signconv-int-vec", + question="Find the displacement vector.", + answer=Answer(value=golden, answer_category=AnswerCategory.PHYSICAL_QUANTITY), + domain="mechanics", + ) + + +class _RefBuildStub(BaseModelClient): + """Staged reference-build stub: Call A declares ``answer_convention`` on the golden, Call B + declares ``question_convention`` (a problem-fixed policy), Call C declares nothing. + """ + + supports_response_format_json_schema = True + + def __init__( + self, + *, + answer_convention: str | None = None, + question_convention: str | None = None, + ) -> None: + super().__init__(model="stub-model") + self.provider = "stub" + self._answer_convention = answer_convention + self._question_convention = question_convention + + def response( + self, + input: str, + image_paths: list[str] | None = None, + response_format: dict[str, Any] | type | None = None, + **kwargs: Any, + ) -> str: + del input, image_paths, kwargs + name = ( + response_format.get("name") if isinstance(response_format, dict) else None + ) + if name == "StrictPhysicsAnswerSemantics": + payload: dict[str, Any] = { + "canonical_text": "value", + "object_kind": "physical_quantity", + "structure": "atomic", + } + if self._answer_convention is not None: + payload["sign_convention"] = self._answer_convention + return json.dumps(payload) + if name == "StrictPhysicsQuestionSemantics": + payload = {} + if self._question_convention is not None: + payload["sign_convention"] = self._question_convention + return json.dumps(payload) + if name == "StrictSymbolAssumptionsResponse": + return json.dumps({"assumptions": []}) + raise AssertionError(f"unexpected response schema: {name}") + + +class _VectorPredStub(BaseModelClient): + """Isolated-solve stub returning a structured vector ``a_pred_llm`` with a convention.""" + + supports_response_format_json_schema = True + + def __init__(self, *, payload: dict[str, Any], final_answer: str) -> None: + super().__init__(model="stub-model") + self.provider = "stub" + self._payload = payload + self._final_answer = final_answer + + def response( + self, + input: str, + image_paths: list[str] | None = None, + response_format: dict[str, Any] | type | None = None, + **kwargs: Any, + ) -> str: + del input, image_paths, kwargs + name = ( + response_format.get("name") if isinstance(response_format, dict) else None + ) + if name == "StrictPredictionIsolatedResponse": + return json.dumps( + { + "reasoning": "vector solve", + "final_answer": self._final_answer, + "prediction_answer_semantics": self._payload, + } + ) + raise AssertionError(f"unexpected response schema: {name}") + + +def test_built_velocity_flip_accepts_under_audited() -> None: + ref = build_reference_semantics( + _quantity_problem("-20 m/s"), + _RefBuildStub(answer_convention="right as positive"), + ) + a_ref = ref.reference_answer_semantics + q_ref = ref.question_semantics + # The build placed the convention on a_ref, leaving q_ref convention-free. + assert a_ref.sign_convention == "right as positive" + assert q_ref.sign_convention is None and q_ref.coordinate_frame is None + + a_pred = extract_prediction_answer_semantics("20 m/s (taking leftward as positive)") + verdict = verify(a_ref, a_pred, unit_policy="audited", context=q_ref) + assert verdict.correct is True + assert verdict.comparison_mode == "sign_convention" + # The accept is audited, not strict (the bridge is blocked under strict). + assert verify(a_ref, a_pred, unit_policy="strict", context=q_ref).correct is False + + +def test_built_prediction_without_convention_rejects() -> None: + ref = build_reference_semantics( + _quantity_problem("-20 m/s"), + _RefBuildStub(answer_convention="right as positive"), + ) + a_ref = ref.reference_answer_semantics + # The prediction states no convention -> the lane never reconciles a bare flip. + a_pred = extract_prediction_answer_semantics("20 m/s") + verdict = verify( + a_ref, a_pred, unit_policy="audited", context=ref.question_semantics + ) + assert verdict.correct is False + assert verdict.comparison_mode != "sign_convention" + + +def test_built_precision_dual_rejects_under_every_policy() -> None: + ref = build_reference_semantics( + _quantity_problem("-20 m/s"), + _RefBuildStub(answer_convention="right as positive"), + ) + a_ref = ref.reference_answer_semantics + # Opposite conventions but EQUAL values -> physically opposite quantities -> reject. + a_pred = extract_prediction_answer_semantics( + "-20 m/s (taking leftward as positive)" + ) + for policy in ("audited", "strict", "permissive"): + verdict = verify( + a_ref, a_pred, unit_policy=policy, context=ref.question_semantics + ) + assert verdict.correct is False, policy + + +def test_built_question_fixed_convention_rejects_flip() -> None: + # The problem itself fixes the axis -> q_ref carries the convention -> the gate is closed, + # so a flipped value is a genuine error, not a convention artifact. + ref = build_reference_semantics( + _quantity_problem("-20 m/s"), + _RefBuildStub(question_convention="rightward is positive"), + ) + q_ref = ref.question_semantics + assert q_ref.sign_convention == "rightward is positive" + + a_pred = extract_prediction_answer_semantics("20 m/s (taking leftward as positive)") + verdict = verify( + ref.reference_answer_semantics, a_pred, unit_policy="audited", context=q_ref + ) + assert verdict.correct is False + assert verdict.comparison_mode != "sign_convention" + + +def test_built_vector_opposite_frames_accepts_and_one_sided_is_tbd() -> None: + ref = build_reference_semantics( + _vector_problem("<-3, 4>"), + _RefBuildStub(answer_convention="right as positive"), + ) + a_ref = ref.reference_answer_semantics + assert a_ref.sign_convention == "right as positive" + + # a_pred_llm: component-wise negation under the opposite frame -> accept. + opposite = infer_prediction_semantics( + _vector_problem("<3, -4>"), + _VectorPredStub( + final_answer="<3, -4>", + payload={ + "canonical_text": "<3, -4>", + "object_kind": "number", + "structure": "vector", + "shape": [2], + "sign_convention": "left as positive", + "children": [ + { + "canonical_text": "3", + "object_kind": "number", + "structure": "atomic", + "numeric_value": 3.0, + }, + { + "canonical_text": "-4", + "object_kind": "number", + "structure": "atomic", + "numeric_value": -4.0, + }, + ], + }, + ), + answer_semantics="structured", + ).prediction_answer_semantics + accept = verify( + a_ref, opposite, unit_policy="audited", context=ref.question_semantics + ) + assert accept.correct is True + assert accept.comparison_mode == "sign_convention" + + # One-sided convention (a_ref declares, a_pred does not): deliberately TBD (precision-safe), + # the committed lane's documented residual — never a false accept. + one_sided = infer_prediction_semantics( + _vector_problem("<-3, 4>"), + _VectorPredStub( + final_answer="<-3, 4>", + payload={ + "canonical_text": "<-3, 4>", + "object_kind": "number", + "structure": "vector", + "shape": [2], + "children": [ + { + "canonical_text": "-3", + "object_kind": "number", + "structure": "atomic", + "numeric_value": -3.0, + }, + { + "canonical_text": "4", + "object_kind": "number", + "structure": "atomic", + "numeric_value": 4.0, + }, + ], + }, + ), + answer_semantics="structured", + ).prediction_answer_semantics + tbd = verify( + a_ref, one_sided, unit_policy="audited", context=ref.question_semantics + ) + assert tbd.correct is not True + assert tbd.comparison_mode == "not_implemented" diff --git a/tests/prkit/semantics/test_sign_convention_build_live.py b/tests/prkit/semantics/test_sign_convention_build_live.py new file mode 100644 index 0000000..b91ed78 --- /dev/null +++ b/tests/prkit/semantics/test_sign_convention_build_live.py @@ -0,0 +1,55 @@ +"""Opt-in live smoke: a real reference build routes a free-axis convention onto ``a_ref``. + +Skipped unless ``OPENAI_API_KEY`` is set. Builds reference semantics for a 1-D kinematics +problem whose golden is a signed velocity on a *free* axis, and asserts the build places the +convention on ``a_ref`` (the lane's evidence) and leaves ``q_ref`` convention-free (the problem +fixes no axis) — the routing this whole change is about. If the model declares no convention at +all this run, the routing cannot be observed and the test skips rather than fails. +""" + +from __future__ import annotations + +import os + +import pytest + +from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem +from prkit.core.model_clients import create_model_client +from prkit.semantics.inference.calls import build_reference_semantics + +pytestmark = pytest.mark.integration + +_LIVE_MODEL = os.environ.get("PRKIT_SIGN_CONVENTION_SMOKE_MODEL", "gpt-5.4-mini") + + +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), reason="No OPENAI_API_KEY set" +) +def test_live_reference_build_routes_free_axis_convention_to_a_ref() -> None: + problem = PhysicsProblem( + problem_id="signconv-live-1", + question=( + "A block slides along a horizontal frictionless track. No positive direction " + "is specified. Taking the block's motion into account, its velocity is found to " + "be 20 m/s directed to the left. Report the velocity as a signed value." + ), + answer=Answer( + value="-20 m/s", answer_category=AnswerCategory.PHYSICAL_QUANTITY + ), + domain="mechanics", + ) + + client = create_model_client(_LIVE_MODEL) + artifact = build_reference_semantics(problem, client) + + a_ref = artifact.reference_answer_semantics + q_ref = artifact.question_semantics + a_convention = a_ref.coordinate_frame or a_ref.sign_convention + q_convention = q_ref.coordinate_frame or q_ref.sign_convention + + # The problem fixes no axis, so the convention must not be routed onto q_ref (the old bug). + assert q_convention is None, f"convention leaked onto q_ref: {q_convention!r}" + if a_convention is None: + pytest.skip("model declared no convention this run; routing not observable") + # The golden's expressed convention landed on a_ref (the lane's evidence). + assert a_convention diff --git a/tests/prkit/semantics/test_sign_convention_declaration.py b/tests/prkit/semantics/test_sign_convention_declaration.py new file mode 100644 index 0000000..af325c3 --- /dev/null +++ b/tests/prkit/semantics/test_sign_convention_declaration.py @@ -0,0 +1,103 @@ +"""Adversarial tests for the deterministic sign-convention declaration parser. + +The parser (``answer_normalization._extract_sign_convention_declaration``, wired into +``normalize_physics_answer``) captures an *explicit* " as positive" convention onto a +directional scalar's ``sign_convention`` and strips the clause before value parsing. It is +**declaration-only**: a bare signed value or a fully-specifying direction phrase (no "positive") +is never treated as a convention, so the sign-convention lane stays declared-not-derived. The +captured string is read by the judge's ``_convention_orientation`` (shared vocabulary). +""" + +from __future__ import annotations + +import pytest + +from prkit.semantics.comparison.sign_convention import _convention_orientation +from prkit.semantics.inference.calls import extract_prediction_answer_semantics +from prkit.semantics.normalization.answer_normalization import ( + _extract_sign_convention_declaration, + normalize_physics_answer, +) +from prkit.semantics.schema import AnswerObjectKind + +# (surface, expected main text after strip, expected orientation the judge reads) +_ACCEPTED = [ + ("-20 m/s (taking rightward as positive)", "-20 m/s", "right"), + ("-20 m/s, taking rightward as positive", "-20 m/s", "right"), + ("20 m/s (right-as-positive)", "20 m/s", "right"), + ("-9.8 m/s^2 (with up as positive)", "-9.8 m/s^2", "up"), + ("5 (positive direction is left)", "5", "left"), + ("3 N (+ve = right)", "3 N", "right"), + ("4 T (into the page as positive)", "4 T", "into_page"), + ("-12 (taking down as positive)", "-12", "down"), + ("7 m/s (counterclockwise is positive)", "7 m/s", "counterclockwise"), +] + +# Surfaces that must NOT be read as a convention declaration (no capture, no strip). +_REJECTED = [ + "5 N to the right", # fully specifies the answer; no "positive" -> not a convention + "+20 m/s", # bare sign, no stated direction + "-20 m/s", + "(3, 4)", # a tuple, not a declaration + "{2, -2}", + "x**2/2", + "sqrt(E/m)", + "increases", + "12 m", + "F = m*a", +] + + +@pytest.mark.parametrize("surface, expected_main, expected_orientation", _ACCEPTED) +def test_declaration_captured_and_orientation_readable( + surface: str, expected_main: str, expected_orientation: str +) -> None: + main, convention = _extract_sign_convention_declaration(surface) + assert main == expected_main + assert convention is not None + # The captured string round-trips through the judge's orientation reader. + assert _convention_orientation(convention) == expected_orientation + + +@pytest.mark.parametrize("surface", _REJECTED) +def test_non_declarations_are_not_captured(surface: str) -> None: + main, convention = _extract_sign_convention_declaration(surface) + assert convention is None + # No declaration -> the surface is returned byte-identical (zero behavior change). + assert main == surface + + +@pytest.mark.parametrize("surface, expected_main, expected_orientation", _ACCEPTED) +def test_normalize_applies_convention_to_directional_scalar( + surface: str, expected_main: str, expected_orientation: str +) -> None: + answer = normalize_physics_answer(surface) + assert answer.object_kind in { + AnswerObjectKind.NUMBER, + AnswerObjectKind.PHYSICAL_QUANTITY, + } + assert answer.sign_convention is not None + assert _convention_orientation(answer.sign_convention) == expected_orientation + # The clause was stripped, so the value still parses. + assert answer.numeric_value is not None + + +@pytest.mark.parametrize("surface", _REJECTED) +def test_normalize_leaves_non_declarations_convention_free(surface: str) -> None: + answer = normalize_physics_answer(surface) + assert answer.sign_convention is None + + +def test_extract_prediction_value_parses_after_strip() -> None: + ext = extract_prediction_answer_semantics("-20 m/s (taking rightward as positive)") + assert ext.object_kind == AnswerObjectKind.PHYSICAL_QUANTITY + assert ext.numeric_value == -20.0 + assert ext.unit == "m/s" + assert _convention_orientation(ext.sign_convention) == "right" + + +def test_declaration_only_surface_is_left_alone() -> None: + # If the whole surface is the declaration (no value), do not strip to empty. + main, convention = _extract_sign_convention_declaration("taking right as positive") + assert main == "taking right as positive" + assert convention is None diff --git a/tests/prkit/semantics/test_staged_build.py b/tests/prkit/semantics/test_staged_build.py index 0d8dd27..c79af24 100644 --- a/tests/prkit/semantics/test_staged_build.py +++ b/tests/prkit/semantics/test_staged_build.py @@ -34,21 +34,40 @@ def _problem() -> PhysicsProblem: ) +def _directional_problem() -> PhysicsProblem: + return PhysicsProblem( + problem_id="staged-dir-1", + question="Find the velocity v of the block.", + answer=Answer( + value="-20 m/s", answer_category=AnswerCategory.PHYSICAL_QUANTITY + ), + domain="mechanics", + ) + + class _StagedBuildStubModelClient(BaseModelClient): """Returns a canned response per staged-build call, keyed by the response schema name. Call A (answer cleanup) deliberately reports a *wrong* structure/kind to prove the deterministic pin holds; Call B sets ``allowed_*``/``tolerance`` to prove they are ignored; Call C declares an assumption keyed by an alias token to prove canonical - resolution. + resolution. ``answer_extra`` / ``policy_extra`` merge extra fields into the Call A / Call B + payloads (e.g. an answer-level ``sign_convention`` or a problem-fixed question convention). """ supports_response_format_json_schema = True - def __init__(self) -> None: + def __init__( + self, + *, + answer_extra: dict[str, Any] | None = None, + policy_extra: dict[str, Any] | None = None, + ) -> None: super().__init__(model="stub-model") self.provider = "stub" self.prompts: list[str] = [] + self._answer_extra = answer_extra or {} + self._policy_extra = policy_extra or {} def response( self, @@ -70,6 +89,7 @@ def response( "object_kind": "number", "structure": "set", "canonical_latex": "\\frac{x^2}{2}", + **self._answer_extra, } ) if name == "StrictPhysicsQuestionSemantics": @@ -81,6 +101,7 @@ def response( "allowed_object_kinds": ["choice"], "allowed_structures": ["atomic"], "tolerance": 0.5, + **self._policy_extra, } ) if name == "StrictSymbolAssumptionsResponse": @@ -155,6 +176,52 @@ def test_build_reference_semantics_is_deterministic() -> None: assert first.build_report == second.build_report +def test_build_reference_captures_answer_convention_on_a_ref() -> None: + # Call A declares the convention the golden is expressed in (free-axis directional answer). + # It lands on a_ref; q_ref stays convention-free (the problem fixed no axis). + client = _StagedBuildStubModelClient( + answer_extra={"sign_convention": "right as positive"} + ) + artifact = build_reference_semantics(_directional_problem(), client) + + a_ref = artifact.reference_answer_semantics + q_ref = artifact.question_semantics + assert a_ref.object_kind == AnswerObjectKind.PHYSICAL_QUANTITY + assert a_ref.sign_convention == "right as positive" + assert q_ref.sign_convention is None + assert q_ref.coordinate_frame is None + # The adopted answer-level convention is recorded as an LLM declaration. + assert ( + artifact.build_report.field_provenance.get("sign_convention") == "llm_declared" + ) + + +def test_build_reference_sets_question_convention_only_when_problem_fixes_it() -> None: + # Call B declares a problem-fixed convention -> it lands on q_ref (gate-closing policy). + client = _StagedBuildStubModelClient( + policy_extra={"sign_convention": "rightward is positive"} + ) + artifact = build_reference_semantics(_directional_problem(), client) + + assert artifact.question_semantics.sign_convention == "rightward is positive" + + +def test_build_reference_flags_opposite_answer_vs_question_convention() -> None: + # Problem fixes right-as-positive (q_ref) but the golden is expressed left-as-positive + # (a_ref): a provably-opposite build inconsistency -> flagged + review_required. + client = _StagedBuildStubModelClient( + answer_extra={"sign_convention": "left as positive"}, + policy_extra={"sign_convention": "rightward is positive"}, + ) + artifact = build_reference_semantics(_directional_problem(), client) + + assert any( + flag.startswith("pair:opposite_convention_vs_question") + for flag in artifact.build_report.flags + ) + assert artifact.build_report.review_required is True + + def test_build_problem_semantics_is_answer_blind() -> None: client = _StagedBuildStubModelClient() artifact = build_problem_semantics(_problem(), client) From 2092877caeae5ebee6fc489267964794009b6f97 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sat, 20 Jun 2026 08:57:31 -0400 Subject: [PATCH 13/28] Add a single-source dataset license registry and gate auto-download License metadata for the bundled benchmarks was duplicated as free-text strings in each loader's get_info() and each downloader's download_info, with no shared source of truth, no SPDX normalization, and several wrong/missing values (phybench, seephys, and jeebench mislabeled "Research use"; physreason a non-SPDX "CC BY-NC-SA / MIT"; ugphysics and tpbench missing the key entirely). PhysReason's load() built a literal info dict that dropped the license, so its two public read paths disagreed, and auto_download fetched any dataset with no license check. Introduce a typed LicenseSpec (SPDX id + machine-readable usage flags) and a registry (license_registry.get_license) that owns one spec per dataset. Loaders and downloaders now embed get_license(...).to_info_dict() at info["license"] plus an info["license_spdx"] shortcut, so every read path reports the same corrected license. Gate DatasetHub.load(auto_download=True) on the spec's redistributable flag (override with allow_nonredistributable=True) and warn on eval_only. Corrected values: phybench, jeebench, and physreason are MIT; seephys is Apache-2.0 (eval-only); ugphysics is CC-BY-NC-SA-4.0; tpbench stays LicenseRef-unknown pending upstream verification. Tests: new registry/gating/parity suite (every key resolves, unknown degrades to non-redistributable, corrected flags per dataset, loader<->downloader parity, the PhysReason load-vs-get_info regression, and the redistributable gate); the two phyx assertions read the dict/shortcut. Full gate green. Co-Authored-By: Claude Opus 4.8 --- src/prkit/core/domain/__init__.py | 2 + src/prkit/core/domain/license_spec.py | 51 ++++ .../downloaders/phybench_downloader.py | 5 +- .../downloaders/physbench_downloader.py | 5 +- .../downloaders/physics_downloader.py | 5 +- .../downloaders/physreason_downloader.py | 5 +- .../datasets/downloaders/phyx_downloader.py | 5 +- .../downloaders/seephys_downloader.py | 5 +- .../downloaders/ugphysics_downloader.py | 4 +- src/prkit/datasets/hub.py | 24 ++ src/prkit/datasets/license_registry.py | 130 +++++++++ src/prkit/datasets/loaders/jeebench_loader.py | 4 +- src/prkit/datasets/loaders/phybench_loader.py | 4 +- .../datasets/loaders/physbench_loader.py | 4 +- src/prkit/datasets/loaders/physics_loader.py | 4 +- .../datasets/loaders/physreason_loader.py | 6 +- src/prkit/datasets/loaders/phyx_loader.py | 4 +- src/prkit/datasets/loaders/seephys_loader.py | 4 +- src/prkit/datasets/loaders/tpbench_loader.py | 3 + .../datasets/loaders/ugphysics_loader.py | 3 + .../downloaders/test_phyx_downloader.py | 3 +- .../datasets/loaders/test_phyx_loader.py | 3 +- tests/prkit/datasets/test_hub.py | 12 +- tests/prkit/datasets/test_license_registry.py | 259 ++++++++++++++++++ 24 files changed, 536 insertions(+), 18 deletions(-) create mode 100644 src/prkit/core/domain/license_spec.py create mode 100644 src/prkit/datasets/license_registry.py create mode 100644 tests/prkit/datasets/test_license_registry.py diff --git a/src/prkit/core/domain/__init__.py b/src/prkit/core/domain/__init__.py index d47baf9..8f174ac 100644 --- a/src/prkit/core/domain/__init__.py +++ b/src/prkit/core/domain/__init__.py @@ -13,6 +13,7 @@ # Domain models (data classes) from .answer import Answer from .answer_category import AnswerCategory +from .license_spec import LicenseSpec from .physics_dataset import PhysicalDataset from .physics_domain import PhysicsDomain from .physics_problem import PhysicsProblem @@ -27,4 +28,5 @@ "PhysicsProblem", "PhysicalDataset", "PhysicsSolution", + "LicenseSpec", ] diff --git a/src/prkit/core/domain/license_spec.py b/src/prkit/core/domain/license_spec.py new file mode 100644 index 0000000..6b61df2 --- /dev/null +++ b/src/prkit/core/domain/license_spec.py @@ -0,0 +1,51 @@ +"""Typed license facts for a bundled dataset. + +``LicenseSpec`` is the single machine-readable record of a dataset's license: an SPDX id, a +human name/url, and boolean usage flags (redistributable / commercial / eval-only / etc.). The +dataset license registry (``prkit.datasets.license_registry``) owns one ``LicenseSpec`` per +dataset; loaders and downloaders embed ``to_info_dict()`` at ``PhysicalDataset.info["license"]`` +so every read path reports the same, normalized license truth. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any + + +@dataclass(frozen=True) +class LicenseSpec: + """One dataset's license, as an SPDX id plus machine-readable usage flags. + + Frozen (hashable) so it can be shared safely from the registry. ``spdx`` uses canonical + SPDX ids (``"MIT"``, ``"Apache-2.0"``, ``"CC-BY-NC-SA-4.0"``) or a ``"LicenseRef-*"`` token + when no SPDX id applies. The flags are advisory facts consumers can gate on: + + - ``redistributable`` — the data may be re-hosted/downloaded (gates ``auto_download``). + - ``commercial_use`` — commercial use is permitted (informational; does not gate download). + - ``eval_only`` — intended for evaluation only (e.g. test answers withheld upstream). + - ``attribution_required`` / ``share_alike`` — attribution / copyleft obligations. + - ``license_unknown`` — the license could not be verified; treat conservatively. + """ + + spdx: str + name: str + url: str | None = None + redistributable: bool = False + commercial_use: bool = False + eval_only: bool = False + attribution_required: bool = False + share_alike: bool = False + license_unknown: bool = False + notes: str | None = None + + def to_info_dict(self) -> dict[str, Any]: + """Return the flat dict embedded at ``PhysicalDataset.info["license"]``.""" + + return asdict(self) + + @property + def is_permissive(self) -> bool: + """Whether the license allows redistribution and commercial use without copyleft.""" + + return self.redistributable and self.commercial_use and not self.share_alike diff --git a/src/prkit/datasets/downloaders/phybench_downloader.py b/src/prkit/datasets/downloaders/phybench_downloader.py index 27f0b46..07dc2c7 100644 --- a/src/prkit/datasets/downloaders/phybench_downloader.py +++ b/src/prkit/datasets/downloaders/phybench_downloader.py @@ -12,6 +12,8 @@ from pathlib import Path from typing import Any +from prkit.datasets.license_registry import get_license + from .base_downloader import BaseDownloader @@ -40,7 +42,8 @@ def download_info(self) -> dict[str, Any]: "format": "JSON", "splits": ["train"], "size_bytes": None, # Size varies - "license": "Research use", + "license": get_license(self.dataset_name).to_info_dict(), + "license_spdx": get_license(self.dataset_name).spdx, "download_method": "datasets-server API", } diff --git a/src/prkit/datasets/downloaders/physbench_downloader.py b/src/prkit/datasets/downloaders/physbench_downloader.py index 1f7bc35..d315fcb 100644 --- a/src/prkit/datasets/downloaders/physbench_downloader.py +++ b/src/prkit/datasets/downloaders/physbench_downloader.py @@ -11,6 +11,8 @@ from pathlib import Path from typing import Any +from prkit.datasets.license_registry import get_license + from .base_downloader import BaseDownloader @@ -38,7 +40,8 @@ def download_info(self) -> dict[str, Any]: "paper_url": "https://arxiv.org/pdf/2501.16411", "huggingface_url": "https://huggingface.co/datasets/USC-PSI-Lab/PhysBench", "homepage": "https://physbench.github.io/", - "license": "apache-2.0", + "license": get_license(self.dataset_name).to_info_dict(), + "license_spdx": get_license(self.dataset_name).spdx, "format": "JSON + optional ZIP media archives", "variants": ["full", "general", "image_only", "image_video"], "splits": ["full", "val", "test"], diff --git a/src/prkit/datasets/downloaders/physics_downloader.py b/src/prkit/datasets/downloaders/physics_downloader.py index bf2bf71..02ff85b 100644 --- a/src/prkit/datasets/downloaders/physics_downloader.py +++ b/src/prkit/datasets/downloaders/physics_downloader.py @@ -12,6 +12,8 @@ from pathlib import Path from typing import Any +from prkit.datasets.license_registry import get_license + from .base_downloader import BaseDownloader @@ -54,7 +56,8 @@ def download_info(self) -> dict[str, Any]: "format": "JSONL", "variants": ["full", "hard", "textonly"], "splits": ["full", "test", "eval"], - "license": "MIT", + "license": get_license(self.dataset_name).to_info_dict(), + "license_spdx": get_license(self.dataset_name).spdx, "download_method": "raw GitHub file download", } diff --git a/src/prkit/datasets/downloaders/physreason_downloader.py b/src/prkit/datasets/downloaders/physreason_downloader.py index e159e12..e740915 100644 --- a/src/prkit/datasets/downloaders/physreason_downloader.py +++ b/src/prkit/datasets/downloaders/physreason_downloader.py @@ -11,6 +11,8 @@ from pathlib import Path from typing import Any +from prkit.datasets.license_registry import get_license + from .base_downloader import BaseDownloader @@ -42,7 +44,8 @@ def download_info(self) -> dict[str, Any]: "variants": ["full", "mini"], "splits": ["train"], "size_bytes": None, # Size varies by variant - "license": "CC BY-NC-SA / MIT", + "license": get_license(self.dataset_name).to_info_dict(), + "license_spdx": get_license(self.dataset_name).spdx, "download_method": "HuggingFace direct download", } diff --git a/src/prkit/datasets/downloaders/phyx_downloader.py b/src/prkit/datasets/downloaders/phyx_downloader.py index 58183f1..74591fd 100644 --- a/src/prkit/datasets/downloaders/phyx_downloader.py +++ b/src/prkit/datasets/downloaders/phyx_downloader.py @@ -12,6 +12,8 @@ from pathlib import Path from typing import Any +from prkit.datasets.license_registry import get_license + from .base_downloader import BaseDownloader # Try to import PIL for image handling @@ -53,7 +55,8 @@ def download_info(self) -> dict[str, Any]: "format": "JSON", "splits": ["test_mini"], "size_bytes": None, # Size varies - "license": "MIT", + "license": get_license(self.dataset_name).to_info_dict(), + "license_spdx": get_license(self.dataset_name).spdx, "download_method": "datasets-server API", } diff --git a/src/prkit/datasets/downloaders/seephys_downloader.py b/src/prkit/datasets/downloaders/seephys_downloader.py index abda96d..9ae8ae6 100644 --- a/src/prkit/datasets/downloaders/seephys_downloader.py +++ b/src/prkit/datasets/downloaders/seephys_downloader.py @@ -14,6 +14,8 @@ import numpy as np import pandas as pd +from prkit.datasets.license_registry import get_license + from .base_downloader import BaseDownloader try: @@ -51,7 +53,8 @@ def download_info(self) -> dict[str, Any]: "format": "Parquet/JSON", "splits": ["train"], "size_bytes": None, # Size varies - "license": "Research use", + "license": get_license(self.dataset_name).to_info_dict(), + "license_spdx": get_license(self.dataset_name).spdx, "download_method": "datasets library", } diff --git a/src/prkit/datasets/downloaders/ugphysics_downloader.py b/src/prkit/datasets/downloaders/ugphysics_downloader.py index 796d953..46eaf46 100644 --- a/src/prkit/datasets/downloaders/ugphysics_downloader.py +++ b/src/prkit/datasets/downloaders/ugphysics_downloader.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import Any +from prkit.datasets.license_registry import get_license from prkit.datasets.ugphysics_common import ( UGPHYSICS_DOMAIN_COUNTS, UGPHYSICS_DOMAIN_VARIANTS, @@ -59,7 +60,8 @@ def download_info(self) -> dict[str, Any]: "domains": self.DOMAINS, "languages": self.LANGUAGES, "size_bytes": None, - "license": "cc-by-nc-sa-4.0", + "license": get_license(self.dataset_name).to_info_dict(), + "license_spdx": get_license(self.dataset_name).spdx, "download_method": "datasets library", "total_problems": { "en": UGPHYSICS_SPLIT_TOTALS["en"], diff --git a/src/prkit/datasets/hub.py b/src/prkit/datasets/hub.py index 12347d2..a468784 100644 --- a/src/prkit/datasets/hub.py +++ b/src/prkit/datasets/hub.py @@ -19,6 +19,7 @@ UGPhysicsDownloader, ) from prkit.datasets.downloaders.base_downloader import BaseDownloader +from prkit.datasets.license_registry import get_license from prkit.datasets.loaders import ( JEEBenchLoader, PHYBenchLoader, @@ -177,6 +178,7 @@ def load( data_dir: str | Path | None = None, sample_size: int | None = None, auto_download: bool = False, + allow_nonredistributable: bool = False, **kwargs: Any, ) -> PhysicalDataset: """ @@ -187,6 +189,8 @@ def load( data_dir: Path to the data directory (None = auto-detect) sample_size: Number of problems to load (None = all) auto_download: If True, automatically download the dataset if it doesn't exist + allow_nonredistributable: If True, permit auto_download of a dataset whose license + is not marked redistributable (default False gates such downloads) **kwargs: Additional arguments for the specific loader (e.g., split, variant, etc.) Returns: @@ -196,6 +200,8 @@ def load( ValueError: If dataset name is unknown, or if variant/split is invalid FileNotFoundError: If data directory doesn't exist and auto_download=False RuntimeError: If auto_download=True but download fails + PermissionError: If auto_download=True for a dataset whose license is not + redistributable and allow_nonredistributable=False Examples: >>> # Load UGPhysics dataset (uses default variant and split) @@ -347,6 +353,24 @@ def load( if split is None: split = loader.get_default_split() + # License gate: do not auto-download (re-host) a dataset that is not marked + # redistributable unless the caller explicitly overrides. + license_spec = get_license(dataset_name) + if not license_spec.redistributable and not allow_nonredistributable: + notes = f" — {license_spec.notes}" if license_spec.notes else "" + raise PermissionError( + f"auto_download for '{dataset_name}' is gated: license " + f"'{license_spec.spdx}' ({license_spec.name}) is not marked " + f"redistributable{notes}. Download it manually and pass data_dir=, " + "or pass allow_nonredistributable=True to override." + ) + if license_spec.eval_only: + cls._logger.warning( + "Dataset '%s' is licensed for evaluation only (%s).", + dataset_name, + license_spec.spdx, + ) + try: # Download the dataset download_kwargs: dict[str, Any] = { diff --git a/src/prkit/datasets/license_registry.py b/src/prkit/datasets/license_registry.py new file mode 100644 index 0000000..1e9303d --- /dev/null +++ b/src/prkit/datasets/license_registry.py @@ -0,0 +1,130 @@ +"""Single source of truth for bundled-dataset licenses. + +One ``LicenseSpec`` per dataset, keyed by the same lowercase name used in +``DatasetHub._loaders`` / ``_downloaders``. Loaders and downloaders read from here instead of +hardcoding free-text strings, so ``PhysicalDataset.info["license"]`` is uniform and correct +across every read path. The facts are hardcoded (not fetched from HF cards at runtime) to keep +the load path network-free and the toolkit independent of external services. +""" + +from __future__ import annotations + +from prkit.core.domain.license_spec import LicenseSpec + +_MIT_URL = "https://opensource.org/license/mit" +_APACHE_URL = "https://www.apache.org/licenses/LICENSE-2.0" +_CC_BY_NC_SA_URL = "https://creativecommons.org/licenses/by-nc-sa/4.0/" + + +# Verified against each dataset's upstream HF card / repo LICENSE (see roadmap N2 references). +_REGISTRY: dict[str, LicenseSpec] = { + "phybench": LicenseSpec( + "MIT", + "MIT License", + _MIT_URL, + redistributable=True, + commercial_use=True, + attribution_required=True, + ), + "physbench": LicenseSpec( + "Apache-2.0", + "Apache License 2.0", + _APACHE_URL, + redistributable=True, + commercial_use=True, + attribution_required=True, + ), + "physics": LicenseSpec( + "MIT", + "MIT License", + _MIT_URL, + redistributable=True, + commercial_use=True, + attribution_required=True, + ), + "phyx": LicenseSpec( + "MIT", + "MIT License", + _MIT_URL, + redistributable=True, + commercial_use=True, + attribution_required=True, + ), + "seephys": LicenseSpec( + "Apache-2.0", + "Apache License 2.0", + _APACHE_URL, + redistributable=True, + commercial_use=True, + attribution_required=True, + eval_only=True, + notes="HF card declares apache-2.0; test-split answers withheld upstream", + ), + "ugphysics": LicenseSpec( + "CC-BY-NC-SA-4.0", + "Creative Commons Attribution-NonCommercial-ShareAlike 4.0", + _CC_BY_NC_SA_URL, + redistributable=True, + commercial_use=False, + attribution_required=True, + share_alike=True, + ), + "jeebench": LicenseSpec( + "MIT", + "MIT License", + _MIT_URL, + redistributable=True, + commercial_use=True, + attribution_required=True, + notes="upstream dair-iitd/jeebench; PRKit ships a local copy", + ), + "tpbench": LicenseSpec( + "LicenseRef-unknown", + "Unknown / unverified", + None, + license_unknown=True, + eval_only=True, + notes="confirm upstream TPBench terms before redistribution", + ), + "physreason": LicenseSpec( + "MIT", + "MIT License", + _MIT_URL, + redistributable=True, + commercial_use=True, + attribution_required=True, + ), +} + +# Map legacy free-text license strings (and casing variants) onto canonical SPDX ids. +_SPDX_ALIASES: dict[str, str] = { + "research use": "LicenseRef-research-use", + "cc by-nc-sa / mit": "MIT", + "cc-by-nc-sa-4.0": "CC-BY-NC-SA-4.0", + "apache-2.0": "Apache-2.0", + "mit": "MIT", +} + + +def get_license(dataset_name: str) -> LicenseSpec: + """Return the ``LicenseSpec`` for a dataset (case-insensitive lookup by registry key). + + An unregistered name returns a conservative ``license_unknown`` / non-redistributable spec + rather than raising, so callers can gate safely on an unknown dataset. + """ + + spec = _REGISTRY.get(dataset_name.lower()) + if spec is None: + return LicenseSpec( + "LicenseRef-unknown", + "Unknown / unregistered", + license_unknown=True, + redistributable=False, + ) + return spec + + +def normalize_spdx(raw: str) -> str: + """Map a legacy free-text license string to a canonical SPDX id (passthrough otherwise).""" + + return _SPDX_ALIASES.get(raw.strip().lower(), raw) diff --git a/src/prkit/datasets/loaders/jeebench_loader.py b/src/prkit/datasets/loaders/jeebench_loader.py index 5a859d4..d5df310 100644 --- a/src/prkit/datasets/loaders/jeebench_loader.py +++ b/src/prkit/datasets/loaders/jeebench_loader.py @@ -38,6 +38,7 @@ from prkit.core import PRKitLogger from prkit.core.domain import PhysicalDataset, PhysicsProblem +from prkit.datasets.license_registry import get_license from .base_loader import BaseDatasetLoader @@ -71,7 +72,8 @@ def get_info(self) -> dict[str, Any]: "difficulty": "JEE Advanced level", "source": "JEE Advanced examination papers", "citation": "JEEBench dataset for JEE Advanced preparation", - "license": "Research use", + "license": get_license(self.name).to_info_dict(), + "license_spdx": get_license(self.name).spdx, "repository": "Local dataset under data/JEEBench/", "modalities": self.modalities, } diff --git a/src/prkit/datasets/loaders/phybench_loader.py b/src/prkit/datasets/loaders/phybench_loader.py index 69d6f86..35a56ab 100644 --- a/src/prkit/datasets/loaders/phybench_loader.py +++ b/src/prkit/datasets/loaders/phybench_loader.py @@ -14,6 +14,7 @@ from prkit.core.domain import PhysicalDataset, PhysicsProblem from prkit.core.domain.physics_domain import PhysicsDomain +from prkit.datasets.license_registry import get_license from .base_loader import BaseDatasetLoader @@ -37,7 +38,8 @@ def get_info(self) -> dict[str, Any]: "paper_url": "https://arxiv.org/pdf/2504.16074", "homepage": "https://www.phybench.cn/", "repository_url": "https://huggingface.co/datasets/Eureka-Lab/PHYBench", - "license": "Research use", + "license": get_license(self.name).to_info_dict(), + "license_spdx": get_license(self.name).spdx, "domains": [ "mechanics", "electricity", diff --git a/src/prkit/datasets/loaders/physbench_loader.py b/src/prkit/datasets/loaders/physbench_loader.py index db2c4c6..be74fa9 100644 --- a/src/prkit/datasets/loaders/physbench_loader.py +++ b/src/prkit/datasets/loaders/physbench_loader.py @@ -12,6 +12,7 @@ from prkit.core import PRKitLogger from prkit.core.domain import PhysicalDataset, PhysicsProblem +from prkit.datasets.license_registry import get_license from .base_loader import BaseDatasetLoader @@ -65,7 +66,8 @@ def get_info(self) -> dict[str, Any]: "paper_url": "https://arxiv.org/pdf/2501.16411", "homepage": "https://physbench.github.io/", "repository_url": "https://huggingface.co/datasets/USC-PSI-Lab/PhysBench", - "license": "apache-2.0", + "license": get_license(self.name).to_info_dict(), + "license_spdx": get_license(self.name).spdx, "languages": ["en"], "variants": list(self.VARIANT_TO_MODE.keys()), "splits": ["full", "val", "test"], diff --git a/src/prkit/datasets/loaders/physics_loader.py b/src/prkit/datasets/loaders/physics_loader.py index 9165e9b..06cab79 100644 --- a/src/prkit/datasets/loaders/physics_loader.py +++ b/src/prkit/datasets/loaders/physics_loader.py @@ -13,6 +13,7 @@ from prkit.core import PRKitLogger from prkit.core.domain import PhysicalDataset, PhysicsDomain, PhysicsProblem +from prkit.datasets.license_registry import get_license from .base_loader import BaseDatasetLoader, detect_answer_category @@ -71,7 +72,8 @@ def get_info(self) -> dict[str, Any]: "paper_url": "https://aclanthology.org/2025.findings-acl.610.pdf", "homepage": "https://github.com/yale-nlp/Physics", "repository_url": "https://github.com/yale-nlp/Physics", - "license": "MIT", + "license": get_license(self.name).to_info_dict(), + "license_spdx": get_license(self.name).spdx, "languages": ["en"], "variants": ["full", "hard", "textonly"], "splits": ["full", "test", "eval"], diff --git a/src/prkit/datasets/loaders/physreason_loader.py b/src/prkit/datasets/loaders/physreason_loader.py index 2450d92..8c578dc 100644 --- a/src/prkit/datasets/loaders/physreason_loader.py +++ b/src/prkit/datasets/loaders/physreason_loader.py @@ -13,6 +13,7 @@ from prkit.core import PRKitLogger from prkit.core.domain import PhysicalDataset, PhysicsProblem +from prkit.datasets.license_registry import get_license from prkit.datasets.loaders.base_loader import BaseDatasetLoader @@ -41,7 +42,8 @@ def get_info(self) -> dict[str, Any]: "paper_url": "https://aclanthology.org/2025.acl-long.811.pdf", "homepage": "https://dxzxy12138.github.io/PhysReason/", "repository_url": "https://huggingface.co/datasets/zhibei1204/PhysReason", - "license": "CC BY-NC-SA / MIT", + "license": get_license(self.name).to_info_dict(), + "license_spdx": get_license(self.name).spdx, "languages": ["en"], "variants": ["full", "mini"], "splits": ["test"], @@ -220,6 +222,8 @@ def load( problems=physics_problems, info={ "name": self.name, + "license": get_license(self.name).to_info_dict(), + "license_spdx": get_license(self.name).spdx, "description": self.description, "variant": variant, "total_problems": len(physics_problems), diff --git a/src/prkit/datasets/loaders/phyx_loader.py b/src/prkit/datasets/loaders/phyx_loader.py index bd929f5..f0bd800 100644 --- a/src/prkit/datasets/loaders/phyx_loader.py +++ b/src/prkit/datasets/loaders/phyx_loader.py @@ -15,6 +15,7 @@ from prkit.core import PRKitLogger from prkit.core.domain import PhysicalDataset, PhysicsProblem from prkit.core.domain.physics_domain import PhysicsDomain +from prkit.datasets.license_registry import get_license from .base_loader import BaseDatasetLoader @@ -48,7 +49,8 @@ def get_info(self) -> dict[str, Any]: "paper_url": "https://arxiv.org/pdf/2505.15929v2", "homepage": "https://phyx-bench.github.io/", "repository_url": "https://huggingface.co/datasets/Cloudriver/PhyX", - "license": "MIT", + "license": get_license(self.name).to_info_dict(), + "license_spdx": get_license(self.name).spdx, "domains": [ "mechanics", "electromagnetism", diff --git a/src/prkit/datasets/loaders/seephys_loader.py b/src/prkit/datasets/loaders/seephys_loader.py index 8808dcb..2a507d2 100644 --- a/src/prkit/datasets/loaders/seephys_loader.py +++ b/src/prkit/datasets/loaders/seephys_loader.py @@ -8,6 +8,7 @@ from prkit.core import PRKitLogger from prkit.core.domain import PhysicalDataset, PhysicsProblem +from prkit.datasets.license_registry import get_license from .base_loader import BaseDatasetLoader @@ -38,7 +39,8 @@ def get_info(self) -> dict[str, Any]: "name": self.name, "description": self.description, "repository_url": "https://huggingface.co/datasets/SeePhys/SeePhys", - "license": "Research use", + "license": get_license(self.name).to_info_dict(), + "license_spdx": get_license(self.name).spdx, "homepage": "https://seephys.github.io/", "paper_url": "https://openreview.net/pdf?id=APNWmytTCS", "languages": ["en", "zh"], diff --git a/src/prkit/datasets/loaders/tpbench_loader.py b/src/prkit/datasets/loaders/tpbench_loader.py index 0ca66f8..7cff49c 100644 --- a/src/prkit/datasets/loaders/tpbench_loader.py +++ b/src/prkit/datasets/loaders/tpbench_loader.py @@ -14,6 +14,7 @@ from prkit.core import PRKitLogger from prkit.core.domain import PhysicalDataset, PhysicsDomain, PhysicsProblem +from prkit.datasets.license_registry import get_license from prkit.datasets.loaders.base_loader import BaseDatasetLoader @@ -39,6 +40,8 @@ def get_info(self) -> dict[str, Any]: """Get dataset information.""" return { "name": self.name, + "license": get_license(self.name).to_info_dict(), + "license_spdx": get_license(self.name).spdx, "description": self.description, "domains": [ "quantum_mechanics", diff --git a/src/prkit/datasets/loaders/ugphysics_loader.py b/src/prkit/datasets/loaders/ugphysics_loader.py index 435b0c3..3188f4e 100644 --- a/src/prkit/datasets/loaders/ugphysics_loader.py +++ b/src/prkit/datasets/loaders/ugphysics_loader.py @@ -13,6 +13,7 @@ from prkit.core import PRKitLogger from prkit.core.domain import PhysicalDataset, PhysicsDomain, PhysicsProblem +from prkit.datasets.license_registry import get_license from prkit.datasets.loaders.base_loader import BaseDatasetLoader from prkit.datasets.ugphysics_common import ( UGPHYSICS_DEFAULT_SUBDIR, @@ -54,6 +55,8 @@ def get_info(self) -> dict[str, Any]: """Get dataset information.""" return { "name": self.name, + "license": get_license(self.name).to_info_dict(), + "license_spdx": get_license(self.name).spdx, "description": self.description, "domains": [ "atomic_physics", diff --git a/tests/prkit/datasets/downloaders/test_phyx_downloader.py b/tests/prkit/datasets/downloaders/test_phyx_downloader.py index 7b7f243..8e5c311 100644 --- a/tests/prkit/datasets/downloaders/test_phyx_downloader.py +++ b/tests/prkit/datasets/downloaders/test_phyx_downloader.py @@ -36,7 +36,8 @@ def test_download_info(self): assert "paper_url" in info assert "homepage" in info assert "license" in info - assert info["license"] == "MIT" + assert info["license"]["spdx"] == "MIT" + assert info["license_spdx"] == "MIT" def test_resolve_download_dir(self, temp_dir, monkeypatch): """Test resolve_download_dir method.""" diff --git a/tests/prkit/datasets/loaders/test_phyx_loader.py b/tests/prkit/datasets/loaders/test_phyx_loader.py index 7a712db..10893dc 100644 --- a/tests/prkit/datasets/loaders/test_phyx_loader.py +++ b/tests/prkit/datasets/loaders/test_phyx_loader.py @@ -48,7 +48,8 @@ def test_get_info(self): assert "homepage" in info assert "repository_url" in info assert "license" in info - assert info["license"] == "MIT" + assert info["license"]["spdx"] == "MIT" + assert info["license_spdx"] == "MIT" assert "domains" in info assert "splits" in info assert "test_mini" in info["splits"] diff --git a/tests/prkit/datasets/test_hub.py b/tests/prkit/datasets/test_hub.py index 0983cd3..a23bae8 100644 --- a/tests/prkit/datasets/test_hub.py +++ b/tests/prkit/datasets/test_hub.py @@ -555,7 +555,11 @@ def load(self, data_dir=None, **kwargs): DatasetHub.register_downloader("mock_auto_download", mock_downloader_class) try: - dataset = DatasetHub.load("mock_auto_download", auto_download=True) + # Unregistered mock key is non-redistributable by default; this test exercises the + # download mechanism, not the license gate, so override it. + dataset = DatasetHub.load( + "mock_auto_download", auto_download=True, allow_nonredistributable=True + ) assert dataset is not None mock_downloader.download.assert_called_once() assert MockLoader.last_loaded_data_dir == Path("/tmp/downloaded_mock_data") @@ -646,7 +650,11 @@ def load(self, data_dir=None, **kwargs): try: with pytest.raises(RuntimeError, match="Auto-download failed"): - DatasetHub.load("mock_download_fail", auto_download=True) + DatasetHub.load( + "mock_download_fail", + auto_download=True, + allow_nonredistributable=True, + ) finally: if "mock_download_fail" in DatasetHub._loaders: del DatasetHub._loaders["mock_download_fail"] diff --git a/tests/prkit/datasets/test_license_registry.py b/tests/prkit/datasets/test_license_registry.py new file mode 100644 index 0000000..c3d15fc --- /dev/null +++ b/tests/prkit/datasets/test_license_registry.py @@ -0,0 +1,259 @@ +"""Tests for the single-source dataset license registry (N2). + +Covers: every registered key resolves; unknown keys degrade conservatively; corrected SPDX + +usage flags per dataset; loader/downloader license parity; the PhysReason load-vs-get_info +parity regression (the loaded dataset used to drop the license); ``auto_download`` gating on +redistributability; ``normalize_spdx`` aliasing; and the frozen-dataclass contract. +""" + +from __future__ import annotations + +import json + +import pytest + +from prkit.core.domain import LicenseSpec +from prkit.datasets import license_registry +from prkit.datasets.hub import DatasetHub +from prkit.datasets.license_registry import get_license, normalize_spdx +from prkit.datasets.loaders import PhysReasonLoader + +_ALL_KEYS = [ + "phybench", + "physbench", + "physics", + "phyx", + "seephys", + "ugphysics", + "jeebench", + "tpbench", + "physreason", +] + +# Datasets that ship a downloader (loader<->downloader parity applies to these). +_KEYS_WITH_DOWNLOADER = [ + "phybench", + "physbench", + "physics", + "phyx", + "seephys", + "ugphysics", + "physreason", +] + +# (key, spdx, redistributable, commercial_use, eval_only, share_alike, license_unknown) +_EXPECTED = [ + ("phybench", "MIT", True, True, False, False, False), + ("physbench", "Apache-2.0", True, True, False, False, False), + ("physics", "MIT", True, True, False, False, False), + ("phyx", "MIT", True, True, False, False, False), + ("seephys", "Apache-2.0", True, True, True, False, False), + ("ugphysics", "CC-BY-NC-SA-4.0", True, False, False, True, False), + ("jeebench", "MIT", True, True, False, False, False), + ("tpbench", "LicenseRef-unknown", False, False, True, False, True), + ("physreason", "MIT", True, True, False, False, False), +] + + +@pytest.mark.parametrize("key", _ALL_KEYS) +def test_every_registered_key_resolves(key: str) -> None: + spec = get_license(key) + assert isinstance(spec, LicenseSpec) + assert spec.spdx + assert get_license(key.upper()) == spec # case-insensitive + + +def test_unknown_key_degrades_conservatively() -> None: + spec = get_license("not-a-real-dataset") + assert spec.license_unknown is True + assert spec.redistributable is False + + +@pytest.mark.parametrize( + "key, spdx, redist, commercial, eval_only, share_alike, unknown", _EXPECTED +) +def test_corrected_spdx_and_flags( + key: str, + spdx: str, + redist: bool, + commercial: bool, + eval_only: bool, + share_alike: bool, + unknown: bool, +) -> None: + spec = get_license(key) + assert spec.spdx == spdx + assert spec.redistributable is redist + assert spec.commercial_use is commercial + assert spec.eval_only is eval_only + assert spec.share_alike is share_alike + assert spec.license_unknown is unknown + + +def test_no_legacy_freetext_values_survive() -> None: + # The wrong/imprecise legacy strings must be gone everywhere. + for key in _ALL_KEYS: + spdx = get_license(key).spdx + assert spdx not in { + "Research use", + "CC BY-NC-SA / MIT", + "apache-2.0", + "cc-by-nc-sa-4.0", + } + + +@pytest.mark.parametrize("key", _ALL_KEYS) +def test_hub_get_info_carries_registry_license(key: str) -> None: + info = DatasetHub.get_info(key) + assert info["license"] == get_license(key).to_info_dict() + assert info["license_spdx"] == get_license(key).spdx + + +@pytest.mark.parametrize("key", _KEYS_WITH_DOWNLOADER) +def test_loader_downloader_license_parity(key: str) -> None: + loader_license = DatasetHub.get_info(key)["license"] + downloader = DatasetHub._get_downloader(key) + assert downloader is not None + assert downloader.download_info["license"] == loader_license + assert downloader.download_info["license_spdx"] == get_license(key).spdx + + +def test_physreason_loaded_dataset_carries_license(temp_dir) -> None: + # Regression: the literal info={...} built in PhysReasonLoader.load() used to omit the + # license, so the loaded dataset diverged from get_info(). It must now carry MIT. + loader = PhysReasonLoader() + data_dir = temp_dir / "physreason" + problem_dir = data_dir / "PhysReason_full" / "problem_001" + problem_dir.mkdir(parents=True) + (problem_dir / "problem.json").write_text( + json.dumps( + { + "problem_id": "problem_001", + "question_structure": { + "context": "A ball is thrown upward.", + "sub_question_1": "What is the velocity at the top?", + }, + "answer": ["0 m/s"], + "explanation_steps": {"sub_question_1": {"step1": "zero"}}, + "difficulty": "easy", + } + ), + encoding="utf-8", + ) + + dataset = loader.load(data_dir=str(data_dir), variant="full", split="test") + info = dataset.get_info() + assert info["license"]["spdx"] == "MIT" + # The two public read paths now agree. + assert info["license"] == DatasetHub.get_info("physreason")["license"] + + +def test_normalize_spdx_maps_legacy_strings() -> None: + assert normalize_spdx("Research use") == "LicenseRef-research-use" + assert normalize_spdx("CC BY-NC-SA / MIT") == "MIT" + assert normalize_spdx("cc-by-nc-sa-4.0") == "CC-BY-NC-SA-4.0" + assert normalize_spdx("apache-2.0") == "Apache-2.0" + assert normalize_spdx("MIT") == "MIT" + assert normalize_spdx("Some-Future-License") == "Some-Future-License" # passthrough + + +def test_license_spec_is_frozen_and_round_trips() -> None: + spec = get_license("ugphysics") + assert hash(spec) is not None # frozen => hashable + with pytest.raises(Exception): + spec.spdx = "MIT" # type: ignore[misc] # frozen => immutable + d = spec.to_info_dict() + assert d["spdx"] == "CC-BY-NC-SA-4.0" + assert set(d) == { + "spdx", + "name", + "url", + "redistributable", + "commercial_use", + "eval_only", + "attribution_required", + "share_alike", + "license_unknown", + "notes", + } + + +# --- auto_download gating ---------------------------------------------------------------- + + +class _GateLoader: + """A loader with valid defaults whose data is always 'missing' (forces the download path).""" + + def get_default_variant(self) -> str: + return "full" + + def get_default_split(self) -> str: + return "test" + + def validate_variant(self, variant: str) -> None: + return None + + def validate_split(self, split: str) -> None: + return None + + def load(self, **kwargs: object) -> object: + raise FileNotFoundError("no data on disk") + + +class _GateDownloadReached(Exception): + """Sentinel proving the gate let execution reach ``downloader.download``.""" + + +class _GateDownloader: + def resolve_download_dir(self, data_dir: object) -> str: + return "/tmp/gated" + + def download(self, **kwargs: object) -> str: + raise _GateDownloadReached("download reached") + + +def _register_gated_fake(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(DatasetHub._loaders, "gated_fake", _GateLoader) + monkeypatch.setitem(DatasetHub._downloaders, "gated_fake", _GateDownloader) + monkeypatch.setitem( + license_registry._REGISTRY, + "gated_fake", + LicenseSpec( + "LicenseRef-x", "X license", license_unknown=True, redistributable=False + ), + ) + + +def test_auto_download_gated_for_nonredistributable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _register_gated_fake(monkeypatch) + with pytest.raises(PermissionError, match="not marked\\s+redistributable"): + DatasetHub.load("gated_fake", data_dir="/nonexistent", auto_download=True) + + +def test_allow_nonredistributable_override_reaches_download( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _register_gated_fake(monkeypatch) + # Gate passes -> download() is reached (our sentinel surfaces, wrapped as RuntimeError). + with pytest.raises(RuntimeError, match="download reached"): + DatasetHub.load( + "gated_fake", + data_dir="/nonexistent", + auto_download=True, + allow_nonredistributable=True, + ) + + +def test_redistributable_dataset_not_gated(monkeypatch: pytest.MonkeyPatch) -> None: + # A redistributable license must NOT be gated: execution reaches download(). + monkeypatch.setitem(DatasetHub._loaders, "gated_fake", _GateLoader) + monkeypatch.setitem(DatasetHub._downloaders, "gated_fake", _GateDownloader) + monkeypatch.setitem( + license_registry._REGISTRY, + "gated_fake", + LicenseSpec("MIT", "MIT License", redistributable=True, commercial_use=True), + ) + with pytest.raises(RuntimeError, match="download reached"): + DatasetHub.load("gated_fake", data_dir="/nonexistent", auto_download=True) From 84b7601cb8fcc3555fb5f4a0df9ef995afe5758e Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sat, 20 Jun 2026 14:44:48 -0400 Subject: [PATCH 14/28] Ignore the incidental uv.lock (project standardizes on pip/.venv) Co-Authored-By: Claude Opus 4.8 --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index efbd527..390ec2a 100644 --- a/.gitignore +++ b/.gitignore @@ -251,3 +251,6 @@ uncertainty_*/ **/perturbations/ + +# uv lockfile (incidental; project standardizes on pip + .venv) +uv.lock From 90d37768ca88a8978fac5e1ed33df47852cd1262 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sat, 20 Jun 2026 15:29:19 -0400 Subject: [PATCH 15/28] Restructure the semantics build/extraction public API Rename the semantics.inference subpackage to semantics.build (the layer creates references and generates predictions, it does not only "infer"); keep semantics.inference as a package-level deprecation shim that re-exports from build and warns on import. Rename the action functions infer_reference_semantics -> create_reference_semantics and infer_prediction_semantics -> generate_prediction_semantics, keeping the old names as deprecated aliases. create_reference_semantics gains a deterministic mode: when model_client is None the three advisory LLM calls are skipped (None-guard in _advisory_inference) and the build records the existing *_call_unavailable flags, so it yields a (q_ref, a_ref) bundle with no LLM call. model_client stays positional-or-keyword so the alias keeps working for positional callers. Surface extract_prediction_answer_semantics on prkit.semantics, and remove the duplicate prkit.verify.parse (callers use the extractor instead). Anchor the ruff and git "build" excludes to the repo root (/build) so the new semantics/build package is linted and tracked rather than silently skipped. Repoint all internal test and cookbook imports to prkit.semantics.build. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 5 +- cookbooks/enrich_quantity_views.py | 2 +- cookbooks/generate_reference_semantics.py | 6 +- cookbooks/physics_reasoning_with_semantics.py | 4 +- pyproject.toml | 5 +- src/prkit/scoring/semantics_scorer.py | 3 +- src/prkit/semantics/__init__.py | 32 +++-- src/prkit/semantics/build/__init__.py | 111 ++++++++++++++++ .../{inference => build}/artifacts.py | 0 .../semantics/{inference => build}/calls.py | 52 ++++++-- .../semantics/{inference => build}/prompts.py | 0 .../{inference => build}/semantics_build.py | 0 .../{inference => build}/strict_models.py | 0 src/prkit/semantics/inference/__init__.py | 119 +++--------------- src/prkit/verify/__init__.py | 33 ++--- .../prkit/core/model_clients/test_prompts.py | 2 +- .../model_clients/test_structured_output.py | 2 +- .../prkit/semantics/test_inference_prompts.py | 6 +- .../test_prediction_isolated_build.py | 6 +- tests/prkit/semantics/test_quantity_views.py | 4 +- tests/prkit/semantics/test_semantics_build.py | 2 +- .../test_sign_convention_build_integration.py | 2 +- .../test_sign_convention_build_live.py | 2 +- .../test_sign_convention_declaration.py | 2 +- tests/prkit/semantics/test_staged_build.py | 2 +- tests/prkit/semantics/test_strict_models.py | 6 +- tests/prkit/verify/test_import_isolation.py | 7 +- tests/prkit/verify/test_verify.py | 16 ++- 28 files changed, 245 insertions(+), 186 deletions(-) create mode 100644 src/prkit/semantics/build/__init__.py rename src/prkit/semantics/{inference => build}/artifacts.py (100%) rename src/prkit/semantics/{inference => build}/calls.py (97%) rename src/prkit/semantics/{inference => build}/prompts.py (100%) rename src/prkit/semantics/{inference => build}/semantics_build.py (100%) rename src/prkit/semantics/{inference => build}/strict_models.py (100%) diff --git a/.gitignore b/.gitignore index 390ec2a..c7a016e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,10 @@ # Distribution / packaging .Python -build/ +# Anchored to the repo root so the setuptools build/ artifact is ignored WITHOUT +# also ignoring the src/prkit/semantics/build package (a bare "build/" matches any +# directory of that name at any depth). +/build/ develop-eggs/ dist/ downloads/ diff --git a/cookbooks/enrich_quantity_views.py b/cookbooks/enrich_quantity_views.py index 88184fb..353710c 100644 --- a/cookbooks/enrich_quantity_views.py +++ b/cookbooks/enrich_quantity_views.py @@ -12,7 +12,7 @@ from typing import Any from prkit.core import PRKitLogger -from prkit.semantics.inference import ( +from prkit.semantics.build import ( load_prediction_semantics_artifact, load_reference_semantics_artifact, load_semantics_artifact, diff --git a/cookbooks/generate_reference_semantics.py b/cookbooks/generate_reference_semantics.py index 2d18287..0f341e0 100644 --- a/cookbooks/generate_reference_semantics.py +++ b/cookbooks/generate_reference_semantics.py @@ -14,7 +14,7 @@ from prkit.core import PRKitLogger from prkit.core.model_clients import create_model_client from prkit.datasets import DatasetHub -from prkit.semantics import infer_reference_semantics, save_semantics_json +from prkit.semantics import create_reference_semantics, save_semantics_json logger = PRKitLogger.get_logger(__name__) @@ -83,9 +83,9 @@ def main() -> None: artifact_path = output_dir / f"{_safe_filename(problem.problem_id)}.json" try: - artifact = infer_reference_semantics( + artifact = create_reference_semantics( problem, - client, + model_client=client, max_output_tokens=args.max_output_tokens, ) save_semantics_json(artifact, artifact_path) diff --git a/cookbooks/physics_reasoning_with_semantics.py b/cookbooks/physics_reasoning_with_semantics.py index f34e426..209d622 100644 --- a/cookbooks/physics_reasoning_with_semantics.py +++ b/cookbooks/physics_reasoning_with_semantics.py @@ -14,7 +14,7 @@ from prkit.core import PRKitLogger from prkit.core.model_clients import create_model_client from prkit.datasets import DatasetHub -from prkit.semantics import infer_prediction_semantics, save_semantics_json +from prkit.semantics import generate_prediction_semantics, save_semantics_json logger = PRKitLogger.get_logger(__name__) @@ -83,7 +83,7 @@ def main() -> None: artifact_path = output_dir / f"{_safe_filename(problem.problem_id)}.json" try: - artifact = infer_prediction_semantics( + artifact = generate_prediction_semantics( problem, client, max_output_tokens=args.max_output_tokens, diff --git a/pyproject.toml b/pyproject.toml index dcc4b34..88ad15a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,7 +105,10 @@ target-version = ['py310', 'py311', 'py312'] [tool.ruff] line-length = 88 target-version = "py310" -extend-exclude = ["legacy", "build", "dist", "htmlcov"] +# "/build" is anchored to the repo root so it excludes the setuptools build/ artifact +# dir WITHOUT excluding the src/prkit/semantics/build package (gitignore semantics: a +# bare "build" would match any directory of that name at any depth). +extend-exclude = ["legacy", "/build", "dist", "htmlcov"] [tool.ruff.lint] # E501 (line length) is owned by black. Bugbear (B) is deferred to a later pass. diff --git a/src/prkit/scoring/semantics_scorer.py b/src/prkit/scoring/semantics_scorer.py index 26680d6..b009b0b 100644 --- a/src/prkit/scoring/semantics_scorer.py +++ b/src/prkit/scoring/semantics_scorer.py @@ -143,7 +143,8 @@ def score( ``prediction`` / ``reference`` may be raw strings, :class:`Answer` objects, or already-normalized :class:`PhysicsAnswerSemantics` (e.g. from - :func:`prkit.verify.parse`); all three are accepted by the normalizer. The + :func:`prkit.semantics.extract_prediction_answer_semantics`); all three are + accepted by the normalizer. The wider input type stays compatible with the narrower :class:`prkit.api.Scorer` protocol by parameter contravariance. diff --git a/src/prkit/semantics/__init__.py b/src/prkit/semantics/__init__.py index 5fac497..4643b6d 100644 --- a/src/prkit/semantics/__init__.py +++ b/src/prkit/semantics/__init__.py @@ -11,19 +11,7 @@ For the operational comparison flow, read ``PROTOCOL_COMPARISON.md``. """ -from .comparison import ( - build_evaluation_contract, - coerce_evaluation_contract, - coerce_policy_mode, - coerce_protocol_answer, - coerce_question_semantics, - compare_physics_answers, - compare_predictions, - compare_protocol_answers, - compare_protocol_answers_legacy, - validate_answer_against_contract, -) -from .inference import ( +from .build import ( PREDICTION_PROMPT_NAME, PREDICTION_PROMPT_VERSION, REFERENCE_PROMPT_NAME, @@ -37,7 +25,10 @@ SemanticsProblemRecord, build_prediction_semantics_artifact, compare_saved_semantics, + create_reference_semantics, evaluate_saved_semantics, + extract_prediction_answer_semantics, + generate_prediction_semantics, infer_prediction_semantics, infer_reference_semantics, load_prediction_semantics_artifact, @@ -49,6 +40,18 @@ prepare_semantics_comparison, save_semantics_json, ) +from .comparison import ( + build_evaluation_contract, + coerce_evaluation_contract, + coerce_policy_mode, + coerce_protocol_answer, + coerce_question_semantics, + compare_physics_answers, + compare_predictions, + compare_protocol_answers, + compare_protocol_answers_legacy, + validate_answer_against_contract, +) from .normalization import ( infer_prediction_question_semantics, infer_question_semantics, @@ -121,11 +124,14 @@ "compare_protocol_answers", "compare_protocol_answers_legacy", "compare_saved_semantics", + "create_reference_semantics", "coerce_evaluation_contract", "coerce_policy_mode", "coerce_protocol_answer", "coerce_question_semantics", "evaluate_saved_semantics", + "extract_prediction_answer_semantics", + "generate_prediction_semantics", "infer_prediction_question_context", "infer_prediction_question_semantics", "infer_question_semantics", diff --git a/src/prkit/semantics/build/__init__.py b/src/prkit/semantics/build/__init__.py new file mode 100644 index 0000000..0c50b24 --- /dev/null +++ b/src/prkit/semantics/build/__init__.py @@ -0,0 +1,111 @@ +"""Convenience exports for the semantics-build layer. + +The :mod:`prkit.semantics.build` package *builds* the records the comparison engine +judges — it creates references and generates predictions. It wraps: + +- building stable prompts for reference and prediction semantics calls, +- creating/generating + validating + persisting the resulting artifacts, and +- comparing saved artifacts with the protocol comparator. + +The public build actions are :func:`create_reference_semantics`, +:func:`generate_prediction_semantics`, and :func:`extract_prediction_answer_semantics` +(the deprecated ``infer_*`` names remain as aliases for one release). +""" + +from .artifacts import ( + PredictionSemanticsArtifact, + PredictionSemanticsResponse, + ProblemSemanticsArtifact, + ReferenceSemanticsArtifact, + ReferenceSemanticsResponse, + SemanticsArtifact, + SemanticsBuildReport, + SemanticsComparisonInputs, + SemanticsEvaluationRecord, + SemanticsGeneratorInfo, + SemanticsProblemRecord, + SymbolAssumptionProvenance, + load_prediction_semantics_artifact, + load_problem_semantics_artifact, + load_reference_semantics_artifact, + load_semantics_artifact, + load_semantics_evaluation_record, + save_semantics_json, +) +from .calls import ( + PredictionSemanticsInferenceSpec, + build_extracted_prediction_semantics_artifact, + build_prediction_semantics_artifact, + build_problem_semantics, + build_reference_semantics, + compare_saved_semantics, + create_reference_semantics, + evaluate_saved_semantics, + extract_prediction_answer_semantics, + generate_prediction_semantics, + infer_prediction_semantics, + infer_reference_semantics, + parse_prediction_semantics_response_text, + parse_reference_semantics_response_text, + prepare_isolated_prediction_semantics_inference_spec, + prepare_prediction_semantics_inference_spec, + prepare_semantics_comparison, + resolve_isolated_prediction_response_model, + resolve_prediction_response_model, +) +from .prompts import ( + PREDICTION_PROMPT_NAME, + PREDICTION_PROMPT_VERSION, + REFERENCE_PROMPT_NAME, + REFERENCE_PROMPT_VERSION, + answer_like_to_text, + build_prediction_semantics_prompt, + build_reference_semantics_prompt, +) + +__all__ = [ + "PREDICTION_PROMPT_NAME", + "PREDICTION_PROMPT_VERSION", + "PredictionSemanticsArtifact", + "PredictionSemanticsInferenceSpec", + "PredictionSemanticsResponse", + "ProblemSemanticsArtifact", + "REFERENCE_PROMPT_NAME", + "REFERENCE_PROMPT_VERSION", + "ReferenceSemanticsArtifact", + "ReferenceSemanticsResponse", + "SemanticsArtifact", + "SemanticsBuildReport", + "SemanticsComparisonInputs", + "SemanticsEvaluationRecord", + "SemanticsGeneratorInfo", + "SemanticsProblemRecord", + "SymbolAssumptionProvenance", + "answer_like_to_text", + "build_extracted_prediction_semantics_artifact", + "build_prediction_semantics_prompt", + "build_problem_semantics", + "build_reference_semantics", + "build_reference_semantics_prompt", + "build_prediction_semantics_artifact", + "compare_saved_semantics", + "create_reference_semantics", + "evaluate_saved_semantics", + "extract_prediction_answer_semantics", + "generate_prediction_semantics", + "infer_prediction_semantics", + "infer_reference_semantics", + "load_prediction_semantics_artifact", + "load_problem_semantics_artifact", + "load_reference_semantics_artifact", + "load_semantics_artifact", + "load_semantics_evaluation_record", + "parse_prediction_semantics_response_text", + "parse_reference_semantics_response_text", + "prepare_isolated_prediction_semantics_inference_spec", + "prepare_prediction_semantics_inference_spec", + "prepare_semantics_comparison", + "resolve_isolated_prediction_response_model", + "resolve_prediction_response_model", + "save_semantics_json", +] diff --git a/src/prkit/semantics/inference/artifacts.py b/src/prkit/semantics/build/artifacts.py similarity index 100% rename from src/prkit/semantics/inference/artifacts.py rename to src/prkit/semantics/build/artifacts.py diff --git a/src/prkit/semantics/inference/calls.py b/src/prkit/semantics/build/calls.py similarity index 97% rename from src/prkit/semantics/inference/calls.py rename to src/prkit/semantics/build/calls.py index 7708d30..c031af8 100644 --- a/src/prkit/semantics/inference/calls.py +++ b/src/prkit/semantics/build/calls.py @@ -266,16 +266,24 @@ def resolve_isolated_prediction_response_model( # ) -def infer_reference_semantics( +def create_reference_semantics( problem: PhysicsProblem, - model_client: BaseModelClient, + model_client: BaseModelClient | None = None, *, max_output_tokens: int | None = None, **chat_kwargs: Any, ) -> ReferenceSemanticsArtifact: - """Infer and package reference semantics (q_ref + a_ref) for a problem's golden answer. + """Create reference semantics (q_ref + a_ref) for a problem's golden answer. + + The public entry to the reference-build step; delegates to the staged + :func:`build_reference_semantics`. **Deterministic** when ``model_client is None`` + (the three advisory LLM calls are skipped and the build records the same + ``*_call_unavailable`` flags as a degraded call, so ``review_required`` is set); + **LLM-assisted** otherwise. - Back-compatible entry point: delegates to the staged :func:`build_reference_semantics`. + ``model_client`` is positional-or-keyword (default ``None``) so the deprecated + ``infer_reference_semantics`` alias keeps working for callers that pass it + positionally. """ return build_reference_semantics( @@ -286,6 +294,11 @@ def infer_reference_semantics( ) +#: Deprecated alias for :func:`create_reference_semantics` (renamed in the build/API +#: restructure). Retained for one release; prefer ``create_reference_semantics``. +infer_reference_semantics = create_reference_semantics + + # ---------------------------------------------------------------------------------------- # Staged objective semantics build (WS A). Deterministic backbone is authoritative for # structure/object_kind/tolerance/allowed_*; three advisory LLM calls (temperature 0) clean @@ -305,7 +318,7 @@ def infer_reference_semantics( def _advisory_inference( - model_client: BaseModelClient, + model_client: BaseModelClient | None, *, prompt: str, response_model: type[ResponseModelT], @@ -316,7 +329,9 @@ def _advisory_inference( """Run one advisory build call, returning ``None`` on any failure. Advisory calls only refine fields the deterministic backbone has already decided, so a - failed call degrades to the deterministic value rather than crashing the build. + failed call degrades to the deterministic value rather than crashing the build. When + ``model_client is None`` (the deterministic build mode) the call is skipped entirely and + ``None`` is returned, so the same "unavailable" degradation path runs with no LLM call. These calls run **best-effort** (``require_native_json_schema=False``): native structured output is used when the provider supports it, otherwise the LLM still enriches via the @@ -326,6 +341,9 @@ def _advisory_inference( one. Only a genuine inference/parse failure degrades to the deterministic value. """ + if model_client is None: + return None + try: response, _ = _run_structured_inference( model_client, @@ -478,7 +496,7 @@ def _assumption_provenance( def build_reference_semantics( problem: PhysicsProblem, - model_client: BaseModelClient, + model_client: BaseModelClient | None = None, *, golden: str | None = None, max_output_tokens: int | None = None, @@ -493,6 +511,9 @@ def build_reference_semantics( Three advisory LLM calls (temperature 0) then clean the answer surface, fill question policy, and declare justified symbol assumptions; their outputs are validated and never override the deterministic decisions. A build report records provenance and cross-checks. + When ``model_client is None`` the advisory calls are skipped and each records its + ``*_call_unavailable`` flag, yielding a fully deterministic ``(q_ref, a_ref)`` bundle + (``review_required``) with no LLM call. """ resolved_golden = ( @@ -782,7 +803,7 @@ def build_problem_semantics( ) -def infer_prediction_semantics( +def generate_prediction_semantics( problem: PhysicsProblem, model_client: BaseModelClient, *, @@ -875,6 +896,11 @@ def infer_prediction_semantics( ) +#: Deprecated alias for :func:`generate_prediction_semantics` (renamed in the build/API +#: restructure). Retained for one release; prefer ``generate_prediction_semantics``. +infer_prediction_semantics = generate_prediction_semantics + + def _resolve_prediction_native_requirement( model_client: BaseModelClient, response_model: type[BaseModel], @@ -1824,14 +1850,18 @@ def _merge_question_semantics_fallbacks( def _generator_info( - model_client: BaseModelClient, + model_client: BaseModelClient | None, *, prompt_name: str, prompt_version: str, structured_output_mode: str, structured_output_strategy: str | None = None, ) -> SemanticsGeneratorInfo: - """Capture lightweight provenance for one inference call.""" + """Capture lightweight provenance for one inference call. + + ``model_client is None`` (the deterministic build) yields a generator record with + no provider/model — the ``getattr`` fallbacks resolve to ``None``. + """ return _generator_info_from_metadata( provider=getattr(model_client, "provider", None), @@ -2031,10 +2061,12 @@ def _coerce_prediction_artifact( "build_problem_semantics", "build_reference_semantics", "compare_saved_semantics", + "create_reference_semantics", "ensure_semantics_native_structured_output_support", "ensure_semantics_native_json_schema_support", "evaluate_saved_semantics", "extract_prediction_answer_semantics", + "generate_prediction_semantics", "infer_prediction_semantics", "infer_reference_semantics", "parse_prediction_semantics_response_text", diff --git a/src/prkit/semantics/inference/prompts.py b/src/prkit/semantics/build/prompts.py similarity index 100% rename from src/prkit/semantics/inference/prompts.py rename to src/prkit/semantics/build/prompts.py diff --git a/src/prkit/semantics/inference/semantics_build.py b/src/prkit/semantics/build/semantics_build.py similarity index 100% rename from src/prkit/semantics/inference/semantics_build.py rename to src/prkit/semantics/build/semantics_build.py diff --git a/src/prkit/semantics/inference/strict_models.py b/src/prkit/semantics/build/strict_models.py similarity index 100% rename from src/prkit/semantics/inference/strict_models.py rename to src/prkit/semantics/build/strict_models.py diff --git a/src/prkit/semantics/inference/__init__.py b/src/prkit/semantics/inference/__init__.py index c163173..3fbe05b 100644 --- a/src/prkit/semantics/inference/__init__.py +++ b/src/prkit/semantics/inference/__init__.py @@ -1,103 +1,24 @@ -"""Convenience exports for semantics-generation workflows. +"""Deprecated import alias for :mod:`prkit.semantics.build`. -The :mod:`prkit.semantics.inference` package wraps three related -tasks: - -- building stable prompts for reference and prediction semantics calls, -- validating and persisting the resulting artifacts, and -- comparing saved artifacts with the protocol comparator. +.. deprecated:: + This subpackage was renamed to :mod:`prkit.semantics.build`: the layer *builds* + reference and prediction records (it **creates** references — not only "infers"), + so ``build`` names it for what it does. Importing ``prkit.semantics.inference`` + re-exports the full ``prkit.semantics.build`` surface and emits a + :class:`DeprecationWarning`; this alias will be removed in a future release. + Import from ``prkit.semantics.build`` (or the ``prkit.semantics`` public surface) + instead. See ``prkit/CONTRACT.md``. """ -from .artifacts import ( - PredictionSemanticsArtifact, - PredictionSemanticsResponse, - ProblemSemanticsArtifact, - ReferenceSemanticsArtifact, - ReferenceSemanticsResponse, - SemanticsArtifact, - SemanticsBuildReport, - SemanticsComparisonInputs, - SemanticsEvaluationRecord, - SemanticsGeneratorInfo, - SemanticsProblemRecord, - SymbolAssumptionProvenance, - load_prediction_semantics_artifact, - load_problem_semantics_artifact, - load_reference_semantics_artifact, - load_semantics_artifact, - load_semantics_evaluation_record, - save_semantics_json, -) -from .calls import ( - PredictionSemanticsInferenceSpec, - build_extracted_prediction_semantics_artifact, - build_prediction_semantics_artifact, - build_problem_semantics, - build_reference_semantics, - compare_saved_semantics, - evaluate_saved_semantics, - extract_prediction_answer_semantics, - infer_prediction_semantics, - infer_reference_semantics, - parse_prediction_semantics_response_text, - parse_reference_semantics_response_text, - prepare_isolated_prediction_semantics_inference_spec, - prepare_prediction_semantics_inference_spec, - prepare_semantics_comparison, - resolve_isolated_prediction_response_model, - resolve_prediction_response_model, -) -from .prompts import ( - PREDICTION_PROMPT_NAME, - PREDICTION_PROMPT_VERSION, - REFERENCE_PROMPT_NAME, - REFERENCE_PROMPT_VERSION, - answer_like_to_text, - build_prediction_semantics_prompt, - build_reference_semantics_prompt, -) +import warnings + +from prkit.semantics.build import * # noqa: F403 - re-export the renamed surface +from prkit.semantics.build import __all__ as __all__ -__all__ = [ - "PREDICTION_PROMPT_NAME", - "PREDICTION_PROMPT_VERSION", - "PredictionSemanticsArtifact", - "PredictionSemanticsInferenceSpec", - "PredictionSemanticsResponse", - "ProblemSemanticsArtifact", - "REFERENCE_PROMPT_NAME", - "REFERENCE_PROMPT_VERSION", - "ReferenceSemanticsArtifact", - "ReferenceSemanticsResponse", - "SemanticsArtifact", - "SemanticsBuildReport", - "SemanticsComparisonInputs", - "SemanticsEvaluationRecord", - "SemanticsGeneratorInfo", - "SemanticsProblemRecord", - "SymbolAssumptionProvenance", - "answer_like_to_text", - "build_extracted_prediction_semantics_artifact", - "build_prediction_semantics_prompt", - "build_problem_semantics", - "build_reference_semantics", - "build_reference_semantics_prompt", - "build_prediction_semantics_artifact", - "compare_saved_semantics", - "evaluate_saved_semantics", - "extract_prediction_answer_semantics", - "infer_prediction_semantics", - "infer_reference_semantics", - "load_prediction_semantics_artifact", - "load_problem_semantics_artifact", - "load_reference_semantics_artifact", - "load_semantics_artifact", - "load_semantics_evaluation_record", - "parse_prediction_semantics_response_text", - "parse_reference_semantics_response_text", - "prepare_isolated_prediction_semantics_inference_spec", - "prepare_prediction_semantics_inference_spec", - "prepare_semantics_comparison", - "resolve_isolated_prediction_response_model", - "resolve_prediction_response_model", - "save_semantics_json", -] +warnings.warn( + "prkit.semantics.inference has been renamed to prkit.semantics.build; import " + "from prkit.semantics.build (or the prkit.semantics public surface) instead. " + "This alias will be removed in a future release.", + DeprecationWarning, + stacklevel=2, +) diff --git a/src/prkit/verify/__init__.py b/src/prkit/verify/__init__.py index cb2f266..718b4e7 100644 --- a/src/prkit/verify/__init__.py +++ b/src/prkit/verify/__init__.py @@ -3,9 +3,12 @@ This is the headline public surface for third parties who just want to verify a physics answer:: - from prkit.verify import parse, verify + from prkit.verify import verify verdict = verify("9.81 m/s^2", "9.8 m/s²") # verify(gold, pred) -> Verdict +To turn a raw answer string into typed physics semantics (the former +``prkit.verify.parse``), use ``prkit.semantics.extract_prediction_answer_semantics``. + Import discipline (the whole point of this subpackage): ``import prkit.verify`` must NOT pull in provider SDKs (anthropic / openai / google.genai), the dataset hub, the ``datasets`` library, or pandas. The heavy :class:`~prkit.scoring.SemanticsScorer` @@ -23,9 +26,9 @@ if TYPE_CHECKING: # annotations only — never imported at runtime by this module from prkit.core.domain.answer import Answer from prkit.semantics import PhysicsAnswerSemantics, PhysicsQuestionSemantics - from prkit.semantics.inference import ReferenceSemanticsArtifact + from prkit.semantics.build import ReferenceSemanticsArtifact -__all__ = ["parse", "verify", "Verdict"] +__all__ = ["verify", "Verdict"] # A ``verify(unit_policy=...)`` value maps onto the engine's enforcement-strictness # axis (``ComparisonPolicyMode``). Finer-grained per-question unit rules @@ -46,10 +49,10 @@ def _resolve_question_context( ``symbol_assumptions`` that unlock a domain-gated symbolic accept) into a plain ``verify(...)`` call. ``None`` preserves the historical default (empty context). - A :class:`~prkit.semantics.inference.ReferenceSemanticsArtifact` (or anything else + A :class:`~prkit.semantics.build.ReferenceSemanticsArtifact` (or anything else exposing ``.question_semantics``) is accepted and unwrapped to its ``question_semantics`` — duck-typed so this light-import facade never has to import - the heavy inference layer that defines the artifact. A ``PhysicsQuestionSemantics`` + the heavy build layer that defines the artifact. A ``PhysicsQuestionSemantics`` or a plain dict is passed straight through to the scorer's own coercion. """ if context is None: @@ -65,24 +68,6 @@ def _resolve_question_context( return cast("PhysicsQuestionSemantics | dict[str, Any]", context) -def parse(text: str, *, category: object | None = None) -> PhysicsAnswerSemantics: - """Normalize a raw answer surface into typed physics semantics. - - Mirrors ``math_verify.parse``. ``category`` is reserved for a future - answer-category hint; it is not yet wired into the deterministic normalizer, so - passing a non-``None`` value raises ``NotImplementedError`` rather than being - silently ignored. - """ - if category is not None: - raise NotImplementedError( - "parse(category=...) is not supported yet; pass category=None." - ) - # Lazy: defers sympy / the semantics layer off the import path. - from prkit.semantics import normalize_physics_answer - - return normalize_physics_answer(text) - - def verify( gold: Answer | str | PhysicsAnswerSemantics, pred: Answer | str | PhysicsAnswerSemantics, @@ -114,7 +99,7 @@ def verify( the question's domain/policy fields — e.g. ``symbol_assumptions`` that unlock a domain-gated symbolic accept. May be a :class:`~prkit.semantics.PhysicsQuestionSemantics`, a dict, or a - :class:`~prkit.semantics.inference.ReferenceSemanticsArtifact` (its + :class:`~prkit.semantics.build.ReferenceSemanticsArtifact` (its ``question_semantics`` is used). Defaults to ``None`` (empty context), preserving the historical behavior. diff --git a/tests/prkit/core/model_clients/test_prompts.py b/tests/prkit/core/model_clients/test_prompts.py index 6a7581e..cdad8c5 100644 --- a/tests/prkit/core/model_clients/test_prompts.py +++ b/tests/prkit/core/model_clients/test_prompts.py @@ -51,7 +51,7 @@ def test_build_plain_question_prompt_matches_context(): def test_header_parity_with_semantics_format_problem(): """The core header must match the semantics layer's prediction header.""" - from prkit.semantics.inference.prompts import _format_problem + from prkit.semantics.build.prompts import _format_problem problem = PhysicsProblem( problem_id="p5", diff --git a/tests/prkit/core/model_clients/test_structured_output.py b/tests/prkit/core/model_clients/test_structured_output.py index 8fe08cc..2b1844a 100644 --- a/tests/prkit/core/model_clients/test_structured_output.py +++ b/tests/prkit/core/model_clients/test_structured_output.py @@ -22,7 +22,7 @@ strip_schema_keywords, ) from prkit.core.model_clients.xai import XAIModel -from prkit.semantics.inference.strict_models import ( +from prkit.semantics.build.strict_models import ( StrictPredictionFinalAnswerResponse, StrictPredictionSemanticsResponse, ) diff --git a/tests/prkit/semantics/test_inference_prompts.py b/tests/prkit/semantics/test_inference_prompts.py index 95b3ad5..1ab0b3f 100644 --- a/tests/prkit/semantics/test_inference_prompts.py +++ b/tests/prkit/semantics/test_inference_prompts.py @@ -9,7 +9,7 @@ from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem from prkit.core.model_clients import BaseModelClient from prkit.core.model_clients.structured_output import StructuredOutputPlan -from prkit.semantics.inference.calls import ( +from prkit.semantics.build.calls import ( _merge_question_semantics_fallbacks, _parse_response_model, _resolve_max_output_tokens, @@ -18,11 +18,11 @@ infer_reference_semantics, resolve_prediction_response_model, ) -from prkit.semantics.inference.prompts import ( +from prkit.semantics.build.prompts import ( build_prediction_semantics_prompt, build_reference_semantics_prompt, ) -from prkit.semantics.inference.strict_models import ( +from prkit.semantics.build.strict_models import ( StrictPredictionFinalAnswerResponse, StrictPredictionSemanticsResponse, StrictReferenceSemanticsResponse, diff --git a/tests/prkit/semantics/test_prediction_isolated_build.py b/tests/prkit/semantics/test_prediction_isolated_build.py index 043d4cd..ae6935c 100644 --- a/tests/prkit/semantics/test_prediction_isolated_build.py +++ b/tests/prkit/semantics/test_prediction_isolated_build.py @@ -19,14 +19,14 @@ from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem from prkit.core.model_clients import BaseModelClient -from prkit.semantics.inference.calls import ( +from prkit.semantics.build.calls import ( build_extracted_prediction_semantics_artifact, extract_prediction_answer_semantics, infer_prediction_semantics, resolve_isolated_prediction_response_model, ) -from prkit.semantics.inference.prompts import build_prediction_semantics_prompt -from prkit.semantics.inference.strict_models import ( +from prkit.semantics.build.prompts import build_prediction_semantics_prompt +from prkit.semantics.build.strict_models import ( StrictPredictionIsolatedResponse, ) from prkit.semantics.normalization.question_inference import ( diff --git a/tests/prkit/semantics/test_quantity_views.py b/tests/prkit/semantics/test_quantity_views.py index cab18b1..c0e8342 100644 --- a/tests/prkit/semantics/test_quantity_views.py +++ b/tests/prkit/semantics/test_quantity_views.py @@ -11,8 +11,7 @@ _backfill_evaluation_dir, _backfill_prediction_dir, ) -from prkit.semantics.comparison import build_evaluation_contract -from prkit.semantics.inference.artifacts import ( +from prkit.semantics.build.artifacts import ( PredictionSemanticsArtifact, SemanticsEvaluationRecord, SemanticsGeneratorInfo, @@ -21,6 +20,7 @@ load_semantics_evaluation_record, save_semantics_json, ) +from prkit.semantics.comparison import build_evaluation_contract from prkit.semantics.normalization import ( enrich_answer_quantity_views, materialize_quantity_view, diff --git a/tests/prkit/semantics/test_semantics_build.py b/tests/prkit/semantics/test_semantics_build.py index 31bf864..eb50dc2 100644 --- a/tests/prkit/semantics/test_semantics_build.py +++ b/tests/prkit/semantics/test_semantics_build.py @@ -9,7 +9,7 @@ import pytest -from prkit.semantics.inference.semantics_build import ( +from prkit.semantics.build.semantics_build import ( alias_source_violations, assumptions_from_subject_to, build_alias_map, diff --git a/tests/prkit/semantics/test_sign_convention_build_integration.py b/tests/prkit/semantics/test_sign_convention_build_integration.py index 748ca2d..4f79759 100644 --- a/tests/prkit/semantics/test_sign_convention_build_integration.py +++ b/tests/prkit/semantics/test_sign_convention_build_integration.py @@ -14,7 +14,7 @@ from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem from prkit.core.model_clients import BaseModelClient -from prkit.semantics.inference.calls import ( +from prkit.semantics.build.calls import ( build_reference_semantics, extract_prediction_answer_semantics, infer_prediction_semantics, diff --git a/tests/prkit/semantics/test_sign_convention_build_live.py b/tests/prkit/semantics/test_sign_convention_build_live.py index b91ed78..ca5c181 100644 --- a/tests/prkit/semantics/test_sign_convention_build_live.py +++ b/tests/prkit/semantics/test_sign_convention_build_live.py @@ -15,7 +15,7 @@ from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem from prkit.core.model_clients import create_model_client -from prkit.semantics.inference.calls import build_reference_semantics +from prkit.semantics.build.calls import build_reference_semantics pytestmark = pytest.mark.integration diff --git a/tests/prkit/semantics/test_sign_convention_declaration.py b/tests/prkit/semantics/test_sign_convention_declaration.py index af325c3..6f1e60a 100644 --- a/tests/prkit/semantics/test_sign_convention_declaration.py +++ b/tests/prkit/semantics/test_sign_convention_declaration.py @@ -12,8 +12,8 @@ import pytest +from prkit.semantics.build.calls import extract_prediction_answer_semantics from prkit.semantics.comparison.sign_convention import _convention_orientation -from prkit.semantics.inference.calls import extract_prediction_answer_semantics from prkit.semantics.normalization.answer_normalization import ( _extract_sign_convention_declaration, normalize_physics_answer, diff --git a/tests/prkit/semantics/test_staged_build.py b/tests/prkit/semantics/test_staged_build.py index c79af24..ca67b9c 100644 --- a/tests/prkit/semantics/test_staged_build.py +++ b/tests/prkit/semantics/test_staged_build.py @@ -13,7 +13,7 @@ from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem from prkit.core.model_clients import BaseModelClient -from prkit.semantics.inference.calls import ( +from prkit.semantics.build.calls import ( build_problem_semantics, build_reference_semantics, ) diff --git a/tests/prkit/semantics/test_strict_models.py b/tests/prkit/semantics/test_strict_models.py index 9507a6d..04cd212 100644 --- a/tests/prkit/semantics/test_strict_models.py +++ b/tests/prkit/semantics/test_strict_models.py @@ -3,9 +3,9 @@ import pytest from pydantic import ValidationError -from prkit.semantics.inference.artifacts import PredictionSemanticsResponse -from prkit.semantics.inference.calls import _response_schema_has_open_objects -from prkit.semantics.inference.strict_models import ( +from prkit.semantics.build.artifacts import PredictionSemanticsResponse +from prkit.semantics.build.calls import _response_schema_has_open_objects +from prkit.semantics.build.strict_models import ( StrictPhysicsAnswerSemantics, StrictPhysicsQuestionSemantics, StrictPredictionSemanticsResponse, diff --git a/tests/prkit/verify/test_import_isolation.py b/tests/prkit/verify/test_import_isolation.py index b5a3a4a..3ab2425 100644 --- a/tests/prkit/verify/test_import_isolation.py +++ b/tests/prkit/verify/test_import_isolation.py @@ -1,7 +1,7 @@ """Import-boundary guard: ``prkit.verify`` must stay light-import-clean. Runs in a fresh subprocess (so the host test process's own imports cannot mask a -leak) and asserts that importing the facade — and exercising ``parse``/``verify`` — +leak) and asserts that importing the facade — and exercising ``verify`` — never pulls in provider SDKs, the dataset hub, the ``datasets`` library, or pandas. This is the contract that makes ``prkit.verify`` a ``pip install``-and-call verifier. """ @@ -30,12 +30,11 @@ def test_verify_path_does_not_import_heavy_deps(): code = textwrap.dedent(f""" import sys import prkit.verify - from prkit.verify import parse, verify + from prkit.verify import verify - # Exercise the full lazy path: these trigger the SemanticsScorer/sympy + # Exercise the full lazy path: this triggers the SemanticsScorer/sympy # imports, which still must not drag in the forbidden modules. verify("3 m/s", "3 m/s") - parse("9.8 m/s^2") forbidden = {_FORBIDDEN!r} leaked = [name for name in forbidden if name in sys.modules] diff --git a/tests/prkit/verify/test_verify.py b/tests/prkit/verify/test_verify.py index bad1b64..816be3e 100644 --- a/tests/prkit/verify/test_verify.py +++ b/tests/prkit/verify/test_verify.py @@ -1,12 +1,12 @@ -"""Tests for the ``prkit.verify`` light-import facade (parse / verify).""" +"""Tests for the ``prkit.verify`` light-import facade (verify).""" from __future__ import annotations import pytest from prkit.core.verdict import Verdict -from prkit.semantics import PhysicsAnswerSemantics -from prkit.verify import parse, verify +from prkit.semantics import PhysicsAnswerSemantics, extract_prediction_answer_semantics +from prkit.verify import verify class TestVerify: @@ -176,16 +176,14 @@ def test_partial_credit_path_also_threads_context(self): assert v.partial_credit == v.score -class TestParse: +class TestExtractPredictionAnswerSemantics: + """The deterministic extractor replaces the removed ``prkit.verify.parse``.""" + def test_returns_physics_answer_semantics(self): - parsed = parse("9.8 m/s^2") + parsed = extract_prediction_answer_semantics("9.8 m/s^2") assert isinstance(parsed, PhysicsAnswerSemantics) assert parsed.unit == "m/s^2" - def test_category_not_supported_yet(self): - with pytest.raises(NotImplementedError): - parse("9.8", category="number") - def _answer(payload): """Coerce a protocol-answer mapping into ``PhysicsAnswerSemantics`` for ``verify``.""" From ce26e206e8e726932e8204fa571ca70253a1ef70 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sat, 20 Jun 2026 15:30:59 -0400 Subject: [PATCH 16/28] Refresh the docs for the physics-semantics layer Replace docs/EVALUATION.md (which named deleted comparator/metric symbols) with a scorer-focused stub, and add docs/PHYSICS_SEMANTICS.md: a narrative on-ramp covering question/answer semantics, the five build/judge steps and their entry points, the judgement at a glance, and the canonical doc map. Add a physics-semantics section to the README and repoint the package overview at prkit.scoring / prkit.verify. Fix the stale code anchors in EQUIVALENCE.md, extend its comparison_mode catalogue (identical_text, not_implemented, and an asymmetric_match note), repoint METHODOLOGY.md / CONTRACT.md / the semantics README at the build package and the new function names, and drop parse from the contract examples. Co-Authored-By: Claude Opus 4.8 --- README.md | 30 ++++- docs/EVALUATION.md | 81 +++++------- docs/PHYSICS_SEMANTICS.md | 119 ++++++++++++++++++ src/prkit/CONTRACT.md | 7 +- src/prkit/semantics/README.md | 14 ++- src/prkit/semantics/comparison/EQUIVALENCE.md | 20 ++- src/prkit/semantics/comparison/METHODOLOGY.md | 14 +-- 7 files changed, 207 insertions(+), 78 deletions(-) create mode 100644 docs/PHYSICS_SEMANTICS.md diff --git a/README.md b/README.md index e40e117..30a9a78 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ verifier—a `math-verify`-shaped API that, unlike `math-verify`, is unit- and symbolic-aware and imports no model clients, dataset hub, or provider SDKs: ```python -from prkit.verify import parse, verify +from prkit.verify import verify v = verify("9.8 m/s^2", "9.8 m/s²") # verify(gold, pred) -> Verdict v.correct # True — the unit suffix normalizes (math-verify strips units) @@ -46,12 +46,29 @@ v.symbolic_equiv # None (numeric case); True for e.g. verify("v = a t", "v = v.scorer_version # stamped so a stored score is attributable to its scorer ``` +#### Physics semantics (`prkit.semantics`) + +Underneath `verify` is the **physics-semantics** layer. It models a question's contract +`q` and an answer's typed semantics `a`, and judges equivalence as a question-conditioned +relation `Eq(a_pred, a_ref ; q)` — deterministically, not by string match. It exposes three +build actions and two judge entry points, all importable from `prkit.semantics`: + +- `extract_prediction_answer_semantics(answer_text)` — deterministically type a prediction; +- `create_reference_semantics(problem, model_client=None)` — build a reference `(q_ref, a_ref)` + (deterministic when `model_client` is omitted, LLM-assisted otherwise); +- `generate_prediction_semantics(problem, solver_client, ...)` — solve, then type the answer; +- `compare_protocol_answers(pred, ref, ...)` — reference-based judgement; +- `compare_predictions(a_i, a_j, ...)` — reference-free (symmetric) judgement. + +See **[PHYSICS_SEMANTICS.md](docs/PHYSICS_SEMANTICS.md)** for the full story and doc map. + ### 📖 Documentation **Quick Links:** - 🔧 **[CORE.md](docs/CORE.md)** - Core components: domain model, model client, logger, and definitions - 📚 **[DATASETS.md](docs/DATASETS.md)** - Complete guide to supported datasets and benchmarks -- 📊 **[EVALUATION.md](docs/EVALUATION.md)** - Evaluation metrics and comparison strategies +- 🧪 **[PHYSICS_SEMANTICS.md](docs/PHYSICS_SEMANTICS.md)** - Physics-semantics layer: `q`/`a`, the five build/judge steps, and the doc map +- 📊 **[EVALUATION.md](docs/EVALUATION.md)** - The deterministic physics-semantics scorer (`verify` / `SemanticsScorer` → `Verdict`) - 🏷️ **[ANNOTATION.md](docs/ANNOTATION.md)** - Human annotation tasks (gold, correctness) - 📝 **[CHANGELOG.md](CHANGELOG.md)** - Version history and release notes @@ -205,10 +222,13 @@ The essential building blocks of the physical-reasoning-toolkit. All datasets, i 📖 See [CORE.md](docs/CORE.md) for the full domain model, entity relationships, subpackage dependency diagram, and import reference. -### prkit.evaluation 📈 -Answer comparators (symbolic, numerical, textual, option-based), accuracy evaluator, and physics-focused assessment protocols. +### prkit.scoring / prkit.verify 📈 +The deterministic physics-semantics scorer: `prkit.verify.verify` (light-import, one-call) +and `prkit.scoring.SemanticsScorer` / `PartialCreditScorer`, all returning the canonical +`Verdict`. Wraps the `prkit.semantics.comparison` engine. (The legacy `prkit.evaluation` +comparator/evaluator stack is deprecated; `prkit.evaluation.llm_judge` stays.) -📖 [EVALUATION.md](docs/EVALUATION.md) +📖 [EVALUATION.md](docs/EVALUATION.md) · [PHYSICS_SEMANTICS.md](docs/PHYSICS_SEMANTICS.md) ### prkit.datasets 📊 Dataset hub with a Datasets-like interface: `DatasetHub.load()` for PHYBench, PhysReason, UGPhysics, SeePhys, PhyX (plus JEEBench, TPBench loaders). Auto-download, variant selection, and reproducible sampling. diff --git a/docs/EVALUATION.md b/docs/EVALUATION.md index 366d9c1..02bcacb 100644 --- a/docs/EVALUATION.md +++ b/docs/EVALUATION.md @@ -1,66 +1,41 @@ # Evaluation -`prkit.evaluation` provides physics-oriented evaluation utilities for physical reasoning benchmarks. It focuses on comparisons that are common in this domain (e.g., symbolic expressions, numerical answers with units, multiple-choice options), and is designed to expand to richer evaluation signals over time. +Evaluation in `prkit` is the **deterministic physics-semantics scorer**: it judges +whether a predicted answer expresses the same physical meaning as the reference, and +returns a canonical [`Verdict`](../src/prkit/core/verdict.py). -## Quick Start +## Use it ```python -from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation import AccuracyMetric +from prkit.verify import verify -predictions = [ - Answer(value=r"x^2 + 2x + 1", answer_category=AnswerCategory.FORMULA), - Answer(value=3.14, answer_category=AnswerCategory.PHYSICAL_QUANTITY, unit="m/s"), - Answer(value="A", answer_category=AnswerCategory.OPTION), -] - -ground_truths = [ - Answer(value=r"(x+1)^2", answer_category=AnswerCategory.FORMULA), - Answer(value=3.14159, answer_category=AnswerCategory.PHYSICAL_QUANTITY, unit="m/s"), - Answer(value="A", answer_category=AnswerCategory.OPTION), -] - -metric = AccuracyMetric() -result = metric.compute(predictions=predictions, ground_truths=ground_truths) -print(result["accuracy"]) +v = verify("9.8 m/s^2", "9.8 m/s²") # verify(gold, pred) -> Verdict +v.correct # True — the unit suffix normalizes (math-verify strips units) +v.units_ok # True +v.scorer_version # stamped so a stored score is attributable to its scorer ``` -## Answer Representation - -PRKit uses a single `Answer` dataclass with an `AnswerCategory` enum: - -- **Number**: dimensionless numeric values -- **Physical Quantity**: numbers with `unit` -- **Equation**: single-equation form (e.g., F = ma) -- **Formula**: mathematical expressions (often LaTeX) -- **Text**: free-form strings -- **Option**: strings like `"A"` or `"AC"` (multi-select supported via normalization) - -## Comparators - -Comparators live in `prkit.evaluation.comparison` and return structured results (not just booleans): - -- **`SymbolicComparator`**: parses/normalizes LaTeX and checks equivalence via SymPy -- **`NumericalComparator`**: compares numbers with significant-figure handling; may compare units -- **`OptionComparator`**: normalizes option strings and supports multi-select comparisons -- **`TextualComparator`**: fuzzy/semantic matching (implementation-dependent) -- **`SmartAnswerComparator`**: routes to the right comparator based on `AnswerCategory` - -## Metrics - -Metrics live in `prkit.evaluation.metrics`. Currently: - -- **`AccuracyMetric`**: accuracy over a list of predictions vs ground truths using `SmartAnswerComparator` +- **`prkit.verify.verify`** — the light-import, `math-verify`-shaped one-call facade. + Imports no provider SDKs, dataset hub, `datasets`, or pandas. +- **`prkit.scoring.SemanticsScorer`** — the reference `Scorer` (binary pass/fail) that + `verify` wraps. Use it directly when you want the `prkit.api.Scorer` object. +- **`prkit.scoring.PartialCreditScorer`** — graded EED/SEED partial credit (populates + `Verdict.partial_credit`); reachable via `verify(..., partial_credit=True)`. -## Working with Datasets +All three return the same canonical `Verdict`. The judgement itself lives in the +deterministic engine `prkit.semantics.comparison`. -Datasets loaded via `DatasetHub` yield `PhysicsProblem` objects whose `.answer` field (when present) is an `Answer`. To evaluate model outputs, convert your model’s responses into `Answer` objects and compare against the dataset’s ground truth answers. +## Learn more -## Roadmap (Planned) +- [PHYSICS_SEMANTICS.md](PHYSICS_SEMANTICS.md) — the narrative on-ramp: the concept, + the five build/judge steps and their entry points, and the doc map. +- [`src/prkit/semantics/README.md`](../src/prkit/semantics/README.md) — the PASEC + protocol and the full semantics API. +- [`src/prkit/CONTRACT.md`](../src/prkit/CONTRACT.md) — the version-stable public surface + (`prkit.api`, `prkit.verify`, `Verdict`). -The evaluation package is designed to grow beyond final-answer correctness, with support for physics-specific signals such as: +## Deprecated -- theorem / principle usage checks -- intermediate-step validation -- rubric-based or structured reasoning assessments +The legacy comparator/evaluator stack (`prkit.evaluation.comparator`, +`prkit.evaluation.evaluator`) is **deprecated** in favor of the scorer above and is slated +for removal. The model-graded `prkit.evaluation.llm_judge` is **not** deprecated and stays. diff --git a/docs/PHYSICS_SEMANTICS.md b/docs/PHYSICS_SEMANTICS.md new file mode 100644 index 0000000..d0bc3e2 --- /dev/null +++ b/docs/PHYSICS_SEMANTICS.md @@ -0,0 +1,119 @@ +# Physics semantics in PRKit + +A narrative on-ramp to PRKit's physics-semantics layer: what it is, the steps you call, +and where to read next. This is the **map**; the authoritative depth lives in the +technical docs linked at the bottom — this page does not duplicate them. + +## 1. The concept: `q` and `a`, not strings + +Scoring a physics answer by string match is wrong in both directions. `9.8 m/s^2` and +`9.8 m/s²` are the same answer; `v = a·t` and `v = t·a` are the same relation; `{1, 2}` +and `{2, 1}` are the same set — yet they differ as strings. Conversely, `5` and `5 m` are +*not* the same answer unless the question already fixes the unit. + +PRKit therefore models two typed objects: + +- **Question semantics `q`** — what the question *asks for*: the expected object kind and + structure, the target variable, the unit policy, the allowed symbolic forms, the choice + space, sign/coordinate conventions, and symbol-domain assumptions. `q` is the **contract** + the answer is judged against. +- **Answer semantics `a`** — a *typed, canonicalized* representation of an answer surface: + its object kind, structure, canonical text/number, unit, and so on. + +Equivalence is then a question-conditioned relation **`Eq(a_pred, a_ref ; q)`** — a typed +judgement, not string overlap. + +### The 8 object kinds + +`number`, `physical_quantity`, `expression`, `relation`, `qualitative_label`, `choice`, +`boolean`, `sign_direction`. Each kind has its own canonical form and **one** decision +criterion (see [EQUIVALENCE.md](../src/prkit/semantics/comparison/EQUIVALENCE.md) §7). + +> A 9th kind (`descriptive_text`) and a taxonomy unification are planned but **not yet +> implemented**; this page describes the current eight. + +### The 9 structures + +`atomic`, `multi_part`, `tuple`, `set`, `interval`, `vector`, `matrix`, `tensor`, +`piecewise`. The structure axis is orthogonal to the object kind. Atomic and the structures +that collapse to atomic (a 1-element collection, a closed point-interval, a single-case +piecewise) are fully judged; the genuinely structured cases are handled conservatively — +see [STRUCTURE.md](../src/prkit/semantics/comparison/STRUCTURE.md). + +## 2. The five steps and their entry points + +PRKit splits the work into five single-goal steps with no overlap. The first three +**build** records (`prkit.semantics.build`); the last two **judge** them +(`prkit.semantics.comparison`). + +| # | Step | Entry point | Returns | +|---|------|-------------|---------| +| 1 | Extract a prediction | `extract_prediction_answer_semantics(answer_text, *, context=None)` | `PhysicsAnswerSemantics` | +| 2 | Create a reference | `create_reference_semantics(problem, model_client=None)` | `ReferenceSemanticsArtifact` (q_ref + a_ref) | +| 3 | Generate a prediction | `generate_prediction_semantics(problem, solver_client, ...)` | `PredictionSemanticsArtifact` | +| 4 | Judge — reference-based | `compare_protocol_answers(pred, ref, *, contract, context, policy_mode)` | `AnswerComparison` | +| 5 | Judge — reference-free | `compare_predictions(a_i, a_j, *, context)` | `AnswerComparison` | + +All are importable from `prkit.semantics`. Notes: + +- **Step 1 is deterministic and answer-blind** — you already have the answer string; it is + `canonicalize_structure(normalize_physics_answer(...))`, the same authority that classifies + the reference, so prediction and reference classify identically. +- **Step 2 is deterministic when `model_client is None`** (the advisory LLM calls are skipped + and recorded as `*_call_unavailable`), LLM-assisted otherwise. It is the *only* way to make + a reference — there is no separate reference-side `extract_*`; the reference is the + `(q_ref, a_ref)` bundle. +- **Step 5 is symmetric**: it compares two predictions both ways and only accepts when both + directions agree (an asymmetric match is recorded and rejected). Used for reference-free + clustering. + +### Which doorway do I reach for? + +- **Just want a score?** Use **`prkit.verify.verify(gold, pred)`** — the light-import, + `math-verify`-shaped one-call facade returning a canonical `Verdict`. It pulls in no + provider SDKs, dataset hub, `datasets`, or pandas. +- **Want a `Scorer` object** (e.g. to plug into a runner or for partial credit)? Use + **`prkit.scoring.SemanticsScorer`** (binary) or **`prkit.scoring.PartialCreditScorer`** + (graded EED/SEED). Both return the same `Verdict`. +- **Want the raw mechanism** (the rich `AnswerComparison` with `comparison_mode`, + `bridge_*`, `diagnostics`)? Call **`compare_protocol_answers`** (reference-based) or + **`compare_predictions`** (reference-free) directly. + +`AnswerComparison` (the engine-native mechanism) is projected losslessly to `Verdict` +(the stable public contract) by `prkit.scoring._adapt.verdict_from_comparison`. They are two +layers, kept separate on purpose. + +## 3. The judgement at a glance + +Given `a_pred`, `a_ref`, and `q`, `compare_protocol_answers`: + +1. **Builds a contract** from `a_ref` + `q` (expected kind/structure, target variable, unit + policy, symbolic mode, choice space, ordering, enabled bridges) and **classifies each + side** as *admitted*, *coercible*, or *violating* + ([contract.py](../src/prkit/semantics/comparison/contract.py)). +2. **Routes by structure**, then dispatches atomic comparisons to the **one criterion per + object kind** (§7 of EQUIVALENCE.md). Same kind → that criterion; different kind → a + **tiered, policy-gated bridge** (e.g. relation→expression, quantity→number, sign + conventions), never an ad-hoc rescue. +3. Returns a deterministic `AnswerComparison` whose `comparison_mode` names the path taken. + +It is **deterministic** — no model call in the judgement — and the design discipline is +*equivalence = canonical forms + one principled criterion per kind, never rescue branches* +(see [METHODOLOGY.md](../src/prkit/semantics/comparison/METHODOLOGY.md)). The +`policy_mode` (`strict` / `audited` / `permissive`) only tunes enforcement strictness; it +never invents acceptances. + +## 4. Doc map (the canonical hierarchy) + +This page is the **narrative**. For depth, read the authoritative technical references: + +- [EQUIVALENCE.md](../src/prkit/semantics/comparison/EQUIVALENCE.md) — the judgement + reference: contract gate, per-kind criteria, bridges, the `comparison_mode` catalogue. +- [METHODOLOGY.md](../src/prkit/semantics/comparison/METHODOLOGY.md) — the design + discipline (precision-preserving recall; the build methodology). +- [STRUCTURE.md](../src/prkit/semantics/comparison/STRUCTURE.md) — the structure axis and + what is/ isn't yet judged for structured answers. +- [`semantics/README.md`](../src/prkit/semantics/README.md) — the PASEC protocol and the + full semantics API (build → judge → artifacts). +- [`CONTRACT.md`](../src/prkit/CONTRACT.md) — the version-stable public surface + (`prkit.api`, `prkit.verify`, `Verdict`). diff --git a/src/prkit/CONTRACT.md b/src/prkit/CONTRACT.md index e09ed5c..9099656 100644 --- a/src/prkit/CONTRACT.md +++ b/src/prkit/CONTRACT.md @@ -9,12 +9,15 @@ what is stable, how it is versioned, and how things get deprecated. If all you want is to verify a physics answer, use the light-import facade: ```python -from prkit.verify import parse, verify +from prkit.verify import verify v = verify("9.8 m/s^2", "9.8 m/s²") # verify(gold, pred) -> Verdict v.correct # True v.units_ok # True (the unit suffix normalizes; math-verify would strip it) ``` +To turn a raw answer string into typed physics semantics (the former +`prkit.verify.parse`), use `prkit.semantics.extract_prediction_answer_semantics`. + `prkit.verify` imports **no** provider SDKs, dataset hub, `datasets`, or pandas — the boundary is enforced by `tests/prkit/verify/test_import_isolation.py`. It is a thin, `math-verify`-shaped wrapper over the reference `prkit.scoring.SemanticsScorer` @@ -23,7 +26,7 @@ and returns the same canonical `Verdict`. ## Stable surface - **Only names exported in `prkit.api.__all__` are stable**, plus the - `prkit.verify` facade (`parse`, `verify`). Everything else — module paths, + `prkit.verify` facade (`verify`). Everything else — module paths, private helpers, subpackage internals — may change without notice. - The conformance suite in `prkit.testing` (`check_dataset`, `check_scorer`, `check_model_client`, `ConformanceTestMixin`) is a stable companion: use it to diff --git a/src/prkit/semantics/README.md b/src/prkit/semantics/README.md index bea5540..bc6158e 100644 --- a/src/prkit/semantics/README.md +++ b/src/prkit/semantics/README.md @@ -354,14 +354,15 @@ policy, and final result. ```python from prkit.semantics import ( + create_reference_semantics, evaluate_saved_semantics, - infer_prediction_semantics, - infer_reference_semantics, + generate_prediction_semantics, save_semantics_json, ) -reference_artifact = infer_reference_semantics(problem, reference_model_client) -prediction_artifact = infer_prediction_semantics(problem, solver_model_client) +# create_reference_semantics is deterministic when model_client is omitted. +reference_artifact = create_reference_semantics(problem, model_client=reference_model_client) +prediction_artifact = generate_prediction_semantics(problem, solver_model_client) save_semantics_json(reference_artifact, "reference/demo-speed.json") save_semantics_json(prediction_artifact, "prediction/demo-speed.json") @@ -476,8 +477,9 @@ from prkit.semantics import ( normalize_problem_answer, build_evaluation_contract, compare_protocol_answers, - infer_reference_semantics, - infer_prediction_semantics, + create_reference_semantics, + generate_prediction_semantics, + extract_prediction_answer_semantics, evaluate_saved_semantics, ) ``` diff --git a/src/prkit/semantics/comparison/EQUIVALENCE.md b/src/prkit/semantics/comparison/EQUIVALENCE.md index 4725148..d1350fb 100644 --- a/src/prkit/semantics/comparison/EQUIVALENCE.md +++ b/src/prkit/semantics/comparison/EQUIVALENCE.md @@ -7,7 +7,7 @@ physical meaning. This is the **reference** for the judgement; the precision-pre Every example below is a real engine result. Notation: `pred ≡ ref` means equivalent, `pred ≢ ref` means not, and `→ mode` is the resulting `comparison_mode`. -- Entry point: `compare_protocol_answers(pred, ref, *, contract=None, context=None, policy_mode=None)` — [engine.py:39](engine.py) +- Entry point: `compare_protocol_answers(pred, ref, *, contract=None, context=None, policy_mode=None)` — [engine.py:55](engine.py) - It is the deterministic equivalence relation `Eq(a_pred, a_ref ; q)` of the physics-semantics framework: a typed, question-conditioned judgement, not string overlap. --- @@ -100,7 +100,7 @@ stable (all in `preprocess_symbolic_text` / `parse_relation_clauses`): A `PhysicsEvaluationContract` is derived from the reference answer + `q` (expected kind, structure, target variable, unit policy, symbolic mode, choice space, ordering, enabled bridges). Each side is classified by `validate_answer_against_contract` -([contract.py:69](contract.py)): +([contract.py:83](contract.py)): - **admitted** — satisfies the expected kind and question-side policies directly. - **coercible** — differs in a limited, possibly-meaningful way. @@ -159,7 +159,7 @@ flowchart TD LF2 --> OK ``` -`_compare_atomic` ([engine.py:314](engine.py)): same kind → the §7 criterion; on a miss, +`_compare_atomic` ([engine.py:471](engine.py)): same kind → the §7 criterion; on a miss, a Tier-3 label-family fallback and an identical-text check. Different kind → the §8 bridges, then the label-family fallback, else `object_kind_mismatch`. @@ -167,7 +167,7 @@ bridges, then the label-family fallback, else `object_kind_mismatch`. ## 7. Same object kind — one criterion per kind -`compare_same_object_kind` ([same_object_kind.py:33](same_object_kind.py)) dispatches on +`compare_same_object_kind` ([same_object_kind.py:34](same_object_kind.py)) dispatches on `object_kind`. Each kind has a **canonical form** and **one decision criterion**. ### 7.1 `number` @@ -461,6 +461,9 @@ The `AnswerComparison.comparison_mode` names the path taken: - **Same-kind criteria:** `number`, `physical_quantity`, `expression`, `relation`, `choice`, `boolean`, `sign_direction`, `qualitative_label`. +- **Surface shortcut:** `identical_text` ([engine.py:560](engine.py)) — byte-identical + atomic surfaces accept directly, with `surface_shortcut_used=True`. Same-kind by default; + also used cross-kind when the dispatcher allows it. - **Cross-kind bridges:** `relation_to_expression`, `relation_rhs`, `expression_to_number`, `quantity_to_number`, `expression_quantity`, `choice`, `terminal_polarity`, `terminal_polarity_choice`, `relation_to_qualitative_label`, @@ -471,7 +474,14 @@ The `AnswerComparison.comparison_mode` names the path taken: `piecewise`. - **Non-equivalent / control:** `structure_mismatch`, `object_kind_mismatch`, `contract_violation`, `reference_contract_violation`, `bridge_blocked`, - `unsupported_structure`, `unsupported_object_kind`. + `unsupported_structure`, `unsupported_object_kind`, `not_implemented` + ([engine.py:577](engine.py)) — the TBD sentinel returned when a structured comparison is + not yet proven sound (see [STRUCTURE.md](STRUCTURE.md)). + +`asymmetric_match` is **not** a `comparison_mode`: it is a diagnostic tag added by the +symmetric reference-free path (`compare_predictions`) when the forward comparison accepts +but the backward does not. The pair is then judged **non-equivalent**, and +`comparison_mode` keeps the value the forward criterion produced. --- diff --git a/src/prkit/semantics/comparison/METHODOLOGY.md b/src/prkit/semantics/comparison/METHODOLOGY.md index 526b090..fd4d8a6 100644 --- a/src/prkit/semantics/comparison/METHODOLOGY.md +++ b/src/prkit/semantics/comparison/METHODOLOGY.md @@ -166,8 +166,8 @@ quantity). The lane forgives that flip, but the convention is **concrete per-ans not an on/off switch — the reference's convention is determined at build time (the LLM, from problem + figure + golden) and a prediction's comes from its own record; the judgement then **reconciles the two stated conventions** to a common frame -(`sign_convention.py::compare_sign_convention`, and `_reconcile_shaped_sign_convention` for -vectors). The criterion admits a pair **iff**: (1) the question fixes no convention +(`sign_convention.py::compare_sign_convention`, and `engine.py::_reconcile_shaped_sign_convention` +for vectors). The criterion admits a pair **iff**: (1) the question fixes no convention (`q.sign_convention` and `q.coordinate_frame` both absent — else the axis is pinned and a flip is a real error); (2) **both** answers declare conventions whose positive axes are a provable *global* reversal (antonym directions — `_DIRECTION_OPPOSITE`); and (3) the values @@ -222,8 +222,8 @@ discipline for *building* the records the judgement consumes — the question se and the answer-semantics records `a` — so that they are constructed with the **same precision authority** the engine enforces, not by ad-hoc heuristics. The build is offline and one-time per data point; its deterministic core lives in -[`../inference/semantics_build.py`](../inference/semantics_build.py) and is wrapped by the -staged calls in [`../inference/calls.py`](../inference/calls.py). +[`../build/semantics_build.py`](../build/semantics_build.py) and is wrapped by the +staged calls in [`../build/calls.py`](../build/calls.py). ### Vocabulary (build outputs) @@ -260,7 +260,7 @@ Native provider-enforced structured output is a **Step-2 output-form concern onl provider supports it, otherwise plain text parsed back), so a provider lacking native structured output still yields a full `(q_ref, a_ref)`. Lacking it is a normal route, not a defect — it does not set `review_required` (only a genuine cross-check failure does). -- **Step 2's form is the consumer's choice, not the toolkit's.** `infer_prediction_semantics` +- **Step 2's form is the consumer's choice, not the toolkit's.** `generate_prediction_semantics` takes `answer_semantics`: `"structured"` returns `a_pred_llm` (native provider-enforced output; it **raises** if the provider cannot enforce it — no silent substitution), `"extracted"` returns `a_pred_ext = canonicalize_structure(normalize_physics_answer(...))` (plain-text solve, needs no @@ -277,9 +277,9 @@ as a standalone capability for users and downstream applications to invoke à la with their own references and predictions, build only references, or only extract answer semantics. **No step may depend on another inside the toolkit.** Concretely: the judgement core (`prkit.semantics.comparison`, `prkit.verify`, `prkit.scoring`) imports **nothing** from -the build/generation layer (`prkit.semantics.inference`) at runtime — `verify(...)` accepts a +the build/generation layer (`prkit.semantics.build`) at runtime — `verify(...)` accepts a `q_ref` by *duck-typing* `.question_semantics` (a `TYPE_CHECKING`-only annotation), so it never -pulls in the inference layer; generation never calls the reference build; and every step's +pulls in the build layer; generation never calls the reference build; and every step's entry point takes plain `problem` / `PhysicsAnswerSemantics` / `PhysicsQuestionSemantics` inputs rather than requiring another step's output. A new feature must not introduce a runtime import or a mandatory call from one step into another. From 0c6b7d0a34f09656058b0375ec5a52da80504b7b Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sat, 20 Jun 2026 17:31:42 -0400 Subject: [PATCH 17/28] Unify the answer taxonomy on AnswerObjectKind and retire AnswerCategory Move AnswerObjectKind/AnswerStructure (and the _StrEnum base) into prkit.core.domain as the toolkit's single canonical answer taxonomy, re-exported from prkit.semantics.schema for back-compat and promoted onto prkit.api. Add a 9th object kind, descriptive_text, for free-form answers, judged by conservative normalized-text equality (curated controlled-vocabulary phrases still classify as qualitative_label; sign/boolean detection now tolerates trailing punctuation). Retire the legacy AnswerCategory enum: Answer.answer_category becomes answer_kind: AnswerObjectKind (mapping NUMBER->number, EQUATION->relation, PHYSICAL_QUANTITY->physical_quantity, FORMULA->expression, OPTION->choice, TEXT->descriptive_text), with the loaders, conformance suite, physics_problem, and the llm_judge payload migrated and the serialized key renamed to answer_kind. Delete the deprecated evaluation comparator/evaluator/utils/similarities stacks (only llm_judge remains; the semantics layer is the canonical replacement). Breaking change: bump API_VERSION to 2.0 and record the removal in CONTRACT.md. Co-Authored-By: Claude Opus 4.8 --- docs/PHYSICS_SEMANTICS.md | 15 +- src/prkit/CONTRACT.md | 39 +- src/prkit/__init__.py | 16 +- src/prkit/api.py | 15 +- src/prkit/core/domain/__init__.py | 8 +- src/prkit/core/domain/answer.py | 84 +- src/prkit/core/domain/answer_category.py | 30 - src/prkit/core/domain/answer_kinds.py | 53 ++ src/prkit/core/domain/physics_problem.py | 16 +- src/prkit/datasets/loaders/base_loader.py | 53 +- src/prkit/datasets/loaders/physics_loader.py | 10 +- src/prkit/evaluation/__init__.py | 16 +- src/prkit/evaluation/comparator/__init__.py | 43 - src/prkit/evaluation/comparator/base.py | 95 -- src/prkit/evaluation/comparator/by_module.py | 101 --- .../evaluation/comparator/category_match.py | 127 --- .../evaluation/comparator/exact_match.py | 44 - .../evaluation/comparator/normalized_match.py | 82 -- .../evaluation/comparator/record_match.py | 156 ---- .../evaluation/comparator/similarity_match.py | 110 --- src/prkit/evaluation/comparator/smart_llm.py | 123 --- .../evaluation/comparator/smart_match.py | 407 --------- .../evaluation/comparator/smart_pipeline.py | 121 --- src/prkit/evaluation/comparator/typed_llm.py | 458 ---------- src/prkit/evaluation/evaluator/__init__.py | 14 - src/prkit/evaluation/evaluator/accuracy.py | 300 ------- src/prkit/evaluation/evaluator/base.py | 76 -- src/prkit/evaluation/llm_judge/payload.py | 4 +- src/prkit/evaluation/similarities/__init__.py | 5 - src/prkit/evaluation/similarities/rouge_l.py | 57 -- src/prkit/evaluation/utils/NORMALIZATION.md | 407 --------- src/prkit/evaluation/utils/__init__.py | 38 - src/prkit/evaluation/utils/answer_utils.py | 14 - .../evaluation/utils/category_dispatch.py | 52 -- .../evaluation/utils/compare_cross_type.py | 117 --- .../evaluation/utils/compare_same_type.py | 319 ------- .../utils/latex_symbol_preprocess.py | 71 -- src/prkit/evaluation/utils/normalization.py | 184 ---- src/prkit/evaluation/utils/number_utils.py | 48 - .../utils/type_specific_processing.py | 42 - src/prkit/semantics/README.md | 6 +- src/prkit/semantics/build/prompts.py | 3 +- src/prkit/semantics/comparison/EQUIVALENCE.md | 25 +- .../semantics/comparison/same_object_kind.py | 9 + src/prkit/semantics/comparison/semantics.py | 19 + .../normalization/answer_normalization.py | 73 +- src/prkit/semantics/schema/enums.py | 63 +- src/prkit/testing/conformance.py | 8 +- tests/conftest.py | 16 +- tests/prkit/core/domain/test_answer.py | 139 +-- tests/prkit/core/domain/test_definitions.py | 78 +- .../prkit/core/domain/test_physics_problem.py | 14 +- .../loaders/test_base_loader_additional.py | 17 +- .../datasets/loaders/test_ugphysics_loader.py | 6 +- tests/prkit/datasets/test_utils.py | 40 +- tests/prkit/datasets/test_utils_functions.py | 10 +- tests/prkit/evaluation/comparator/__init__.py | 0 .../evaluation/comparator/test_by_module.py | 60 -- .../comparator/test_category_match.py | 503 ----------- .../comparator/test_normalized_match.py | 28 - .../comparator/test_record_match.py | 264 ------ .../comparator/test_similarity_match.py | 42 - .../evaluation/comparator/test_smart_llm.py | 46 - .../evaluation/comparator/test_smart_match.py | 375 -------- .../evaluation/comparator/test_typed_llm.py | 543 ------------ .../evaluation/evaluator/test_accuracy.py | 123 --- .../evaluation/llm_judge/test_payload.py | 10 +- .../evaluation/similarities/test_rouge_l.py | 19 - tests/prkit/evaluation/utils/__init__.py | 1 - .../evaluation/utils/test_answer_utils.py | 90 -- .../utils/test_category_dispatch.py | 59 -- .../utils/test_compare_cross_type.py | 112 --- .../utils/test_latex_symbol_preprocess.py | 215 ----- .../evaluation/utils/test_normalization.py | 832 ------------------ .../evaluation/utils/test_number_utils.py | 120 --- tests/prkit/scoring/test_semantics_scorer.py | 6 +- .../prkit/semantics/test_inference_prompts.py | 6 +- tests/prkit/semantics/test_outcome_space.py | 50 +- .../test_prediction_isolated_build.py | 6 +- .../semantics/test_protocol_comparison.py | 35 +- .../test_sign_convention_build_integration.py | 6 +- .../test_sign_convention_build_live.py | 6 +- tests/prkit/semantics/test_staged_build.py | 8 +- tests/prkit/test_api.py | 3 +- 84 files changed, 609 insertions(+), 7455 deletions(-) delete mode 100644 src/prkit/core/domain/answer_category.py create mode 100644 src/prkit/core/domain/answer_kinds.py delete mode 100644 src/prkit/evaluation/comparator/__init__.py delete mode 100644 src/prkit/evaluation/comparator/base.py delete mode 100644 src/prkit/evaluation/comparator/by_module.py delete mode 100644 src/prkit/evaluation/comparator/category_match.py delete mode 100644 src/prkit/evaluation/comparator/exact_match.py delete mode 100644 src/prkit/evaluation/comparator/normalized_match.py delete mode 100644 src/prkit/evaluation/comparator/record_match.py delete mode 100644 src/prkit/evaluation/comparator/similarity_match.py delete mode 100644 src/prkit/evaluation/comparator/smart_llm.py delete mode 100644 src/prkit/evaluation/comparator/smart_match.py delete mode 100644 src/prkit/evaluation/comparator/smart_pipeline.py delete mode 100644 src/prkit/evaluation/comparator/typed_llm.py delete mode 100644 src/prkit/evaluation/evaluator/__init__.py delete mode 100644 src/prkit/evaluation/evaluator/accuracy.py delete mode 100644 src/prkit/evaluation/evaluator/base.py delete mode 100644 src/prkit/evaluation/similarities/__init__.py delete mode 100644 src/prkit/evaluation/similarities/rouge_l.py delete mode 100644 src/prkit/evaluation/utils/NORMALIZATION.md delete mode 100644 src/prkit/evaluation/utils/__init__.py delete mode 100644 src/prkit/evaluation/utils/answer_utils.py delete mode 100644 src/prkit/evaluation/utils/category_dispatch.py delete mode 100644 src/prkit/evaluation/utils/compare_cross_type.py delete mode 100644 src/prkit/evaluation/utils/compare_same_type.py delete mode 100644 src/prkit/evaluation/utils/latex_symbol_preprocess.py delete mode 100644 src/prkit/evaluation/utils/normalization.py delete mode 100644 src/prkit/evaluation/utils/number_utils.py delete mode 100644 src/prkit/evaluation/utils/type_specific_processing.py delete mode 100644 tests/prkit/evaluation/comparator/__init__.py delete mode 100644 tests/prkit/evaluation/comparator/test_by_module.py delete mode 100644 tests/prkit/evaluation/comparator/test_category_match.py delete mode 100644 tests/prkit/evaluation/comparator/test_normalized_match.py delete mode 100644 tests/prkit/evaluation/comparator/test_record_match.py delete mode 100644 tests/prkit/evaluation/comparator/test_similarity_match.py delete mode 100644 tests/prkit/evaluation/comparator/test_smart_llm.py delete mode 100644 tests/prkit/evaluation/comparator/test_smart_match.py delete mode 100644 tests/prkit/evaluation/comparator/test_typed_llm.py delete mode 100644 tests/prkit/evaluation/evaluator/test_accuracy.py delete mode 100644 tests/prkit/evaluation/similarities/test_rouge_l.py delete mode 100644 tests/prkit/evaluation/utils/__init__.py delete mode 100644 tests/prkit/evaluation/utils/test_answer_utils.py delete mode 100644 tests/prkit/evaluation/utils/test_category_dispatch.py delete mode 100644 tests/prkit/evaluation/utils/test_compare_cross_type.py delete mode 100644 tests/prkit/evaluation/utils/test_latex_symbol_preprocess.py delete mode 100644 tests/prkit/evaluation/utils/test_normalization.py delete mode 100644 tests/prkit/evaluation/utils/test_number_utils.py diff --git a/docs/PHYSICS_SEMANTICS.md b/docs/PHYSICS_SEMANTICS.md index d0bc3e2..52fadb7 100644 --- a/docs/PHYSICS_SEMANTICS.md +++ b/docs/PHYSICS_SEMANTICS.md @@ -23,14 +23,17 @@ PRKit therefore models two typed objects: Equivalence is then a question-conditioned relation **`Eq(a_pred, a_ref ; q)`** — a typed judgement, not string overlap. -### The 8 object kinds +### The 9 object kinds `number`, `physical_quantity`, `expression`, `relation`, `qualitative_label`, `choice`, -`boolean`, `sign_direction`. Each kind has its own canonical form and **one** decision -criterion (see [EQUIVALENCE.md](../src/prkit/semantics/comparison/EQUIVALENCE.md) §7). - -> A 9th kind (`descriptive_text`) and a taxonomy unification are planned but **not yet -> implemented**; this page describes the current eight. +`boolean`, `sign_direction`, `descriptive_text`. Each kind has its own canonical form and +**one** decision criterion (see +[EQUIVALENCE.md](../src/prkit/semantics/comparison/EQUIVALENCE.md) §7). + +> `descriptive_text` (free-form "explain/why" answers) is a deliberate extension beyond the +> v1 paper's eight kinds; its criterion is conservative normalized-text equality (no semantic +> rescue). `AnswerObjectKind` / `AnswerStructure` are the toolkit's single canonical taxonomy, +> defined in `prkit.core.domain` and re-exported from `prkit.semantics.schema`. ### The 9 structures diff --git a/src/prkit/CONTRACT.md b/src/prkit/CONTRACT.md index 9099656..117a909 100644 --- a/src/prkit/CONTRACT.md +++ b/src/prkit/CONTRACT.md @@ -93,10 +93,8 @@ hub backfills it in `DatasetHub.get_loader_info`). Re-routing an existing implementation **that is part of `prkit.api.__all__`** in a way that changes its observable behavior is a **major** change and must bump -`API_VERSION`. This does **not** apply to already-deprecated classes outside the -contract surface (e.g. `AccuracyEvaluator`, whose default was repointed at the -`Scorer` contract — see *Current deprecations*); such changes are documented in the -package release notes, not the contract version. +`API_VERSION`. This does **not** apply to names **outside** the contract surface; +changes to those are documented in the package release notes, not the contract version. ## Deprecation policy @@ -105,19 +103,20 @@ package release notes, not the contract version. - Precedent: `BaseModelClient.chat()` / `chat_structured()` (see `core/model_clients/base.py`). -### Current deprecations - -- **`prkit.evaluation.comparator.*`**, **`BaseComparator`**, **`BaseEvaluator`**, - **`AccuracyEvaluator`** are deprecated in favor of - `prkit.scoring.SemanticsScorer` (the `Scorer` / `Verdict` contract), which wraps - the deterministic semantics comparison engine. Constructing any of them emits a - `DeprecationWarning`. They will be removed no earlier than the next minor release - — but not before downstream consumers migrate off the comparator stack. -- **Behavior change (0.2.0):** `AccuracyEvaluator` now takes an injectable - `scorer=` and **defaults to `SemanticsScorer`** instead of `ExactMatchComparator`. - `evaluate()` shapes its legacy result dict from the returned `Verdict` - (`accuracy_score=score`, `comparison_result=equivalent`, plus `scorer_version` / - `comparison_mode` in `details`). Passing a `comparator=` still selects the old, - unchanged comparator path; passing both `scorer=` and `comparator=` raises. -- `prkit.evaluation.llm_judge` (model-graded scoring) is **not** deprecated — it - is a distinct capability, not a duplicate of the deterministic scoring path. +### Removed in 2.0 + +- **Taxonomy unification (MAJOR).** The legacy `AnswerCategory` enum was **removed**. + `Answer.answer_category: AnswerCategory` is now `Answer.answer_kind: AnswerObjectKind`, + and the canonical ontology enums `AnswerObjectKind` / `AnswerStructure` are promoted + onto `prkit.api.__all__`. Migration mapping for the old `AnswerCategory` members: + `NUMBER → number`, `PHYSICAL_QUANTITY → physical_quantity`, `FORMULA → expression`, + `EQUATION → relation`, `OPTION → choice`, `TEXT → descriptive_text` (a new 9th object + kind for free-form answers). Serialized answers now carry `"answer_kind"` instead of + `"answer_category"`. +- **Deprecated scoring stack deleted.** `prkit.evaluation.comparator.*`, + `prkit.evaluation.evaluator.*` (`BaseComparator`, `ExactMatchComparator`, + `BaseEvaluator`, `AccuracyEvaluator`, …) were removed. Use + `prkit.scoring.SemanticsScorer` (the `Scorer` / `Verdict` contract), or the + light-import facade `prkit.verify`, for deterministic scoring. +- `prkit.evaluation.llm_judge` (model-graded scoring) is **retained** — it is a distinct + capability, not a duplicate of the deterministic scoring path. diff --git a/src/prkit/__init__.py b/src/prkit/__init__.py index 58d857a..ec66582 100644 --- a/src/prkit/__init__.py +++ b/src/prkit/__init__.py @@ -20,8 +20,9 @@ - :mod:`prkit.scoring` — reference scorers (``SemanticsScorer``). - :mod:`prkit.testing` — conformance suite (``check_dataset``/``check_scorer``/…). - :mod:`prkit.semantics` — physics-aware answer normalization & comparison. - - :mod:`prkit.evaluation` — comparators, evaluators, LLM judge (deprecated; - superseded by :mod:`prkit.scoring`). + - :mod:`prkit.evaluation` — model-graded LLM judge (``llm_judge``). The legacy + comparator/evaluator stacks were removed in ``API_VERSION`` 2.0; use + :mod:`prkit.scoring` for deterministic scoring. - :mod:`prkit.annotation` — human annotation tasks (gold, correctness). """ @@ -33,7 +34,13 @@ __version__ = "0.0.0.dev0" from .core import PRKitLogger -from .core.domain import AnswerCategory, PhysicalDataset, PhysicsDomain, PhysicsProblem +from .core.domain import ( + AnswerObjectKind, + AnswerStructure, + PhysicalDataset, + PhysicsDomain, + PhysicsProblem, +) __all__ = [ "__version__", @@ -41,5 +48,6 @@ "PhysicsProblem", "PhysicalDataset", "PhysicsDomain", - "AnswerCategory", + "AnswerObjectKind", + "AnswerStructure", ] diff --git a/src/prkit/api.py b/src/prkit/api.py index 5915198..87274a1 100644 --- a/src/prkit/api.py +++ b/src/prkit/api.py @@ -30,7 +30,8 @@ # --- re-export EXISTING concrete contract anchors ------------------------- from prkit.core.domain import ( - AnswerCategory, + AnswerObjectKind, + AnswerStructure, PhysicalDataset, PhysicsDomain, PhysicsProblem, @@ -43,9 +44,11 @@ # --- contract version (independent of prkit.__version__) ------------------ # Bump per CONTRACT.md: additive change -> minor, breaking change -> major. -# The 0.2.0 additions (the Verdict superset fields + the prkit.verify facade) are -# fully backward compatible; the contract version is held at 1.0 by decision. -API_VERSION = "1.0" +# 2.0 is the taxonomy-unification MAJOR: the legacy ``AnswerCategory`` field on +# ``Answer`` was removed in favor of the canonical ``AnswerObjectKind`` (with +# ``AnswerObjectKind``/``AnswerStructure`` promoted onto the contract), and the +# deprecated ``evaluation`` comparator/evaluator stack was deleted. +API_VERSION = "2.0" # --- the four nouns as structural Protocols ------------------------------- @@ -121,9 +124,11 @@ def run( "Scorer", "Runner", "Verdict", + # canonical answer ontology + "AnswerObjectKind", + "AnswerStructure", # re-exported concrete anchors "Answer", - "AnswerCategory", "PhysicsDomain", "PhysicsProblem", "PhysicalDataset", diff --git a/src/prkit/core/domain/__init__.py b/src/prkit/core/domain/__init__.py index 8f174ac..9e3a05e 100644 --- a/src/prkit/core/domain/__init__.py +++ b/src/prkit/core/domain/__init__.py @@ -6,13 +6,13 @@ It consolidates: - Domain models: Answer, PhysicsProblem, PhysicalDataset, PhysicsSolution -- Domain definitions: AnswerCategory, PhysicsDomain +- Domain definitions: AnswerObjectKind, AnswerStructure, PhysicsDomain """ # Domain definitions (enums/constants) # Domain models (data classes) from .answer import Answer -from .answer_category import AnswerCategory +from .answer_kinds import AnswerObjectKind, AnswerStructure from .license_spec import LicenseSpec from .physics_dataset import PhysicalDataset from .physics_domain import PhysicsDomain @@ -20,9 +20,11 @@ from .physics_solution import PhysicsSolution __all__ = [ + # Canonical answer ontology (single contract for the whole toolkit) + "AnswerObjectKind", + "AnswerStructure", # Definitions "PhysicsDomain", - "AnswerCategory", # Models "Answer", "PhysicsProblem", diff --git a/src/prkit/core/domain/answer.py b/src/prkit/core/domain/answer.py index 0ae91ea..2f7df09 100644 --- a/src/prkit/core/domain/answer.py +++ b/src/prkit/core/domain/answer.py @@ -1,24 +1,29 @@ """ Answer models for physical reasoning evaluation. -This module provides a unified Answer class that handles all answer categories +This module provides a unified Answer class that handles all answer kinds through composition rather than inheritance. """ from dataclasses import dataclass, field from typing import Any -from .answer_category import AnswerCategory +from .answer_kinds import AnswerObjectKind AnswerValue = int | float | str @dataclass class Answer: - """Unified answer class that handles all answer categories through composition.""" + """Unified answer class that handles all answer kinds through composition. - value: AnswerValue # NUMBER: number; PHYSICAL_QUANTITY/OPTION/EQUATION/FORMULA/TEXT: text - answer_category: AnswerCategory + ``answer_kind`` is the canonical :class:`AnswerObjectKind` (the toolkit-wide + answer ontology). This is a coarse ingestion-time tag; the physics-semantics + engine re-derives the precise ``object_kind`` independently when judging. + """ + + value: AnswerValue # NUMBER: number; all other kinds: text + answer_kind: AnswerObjectKind unit: str | None = None # Used only for PHYSICAL_QUANTITY (e.g., "m/s²", "N") metadata: dict[str, Any] = field(default_factory=dict) @@ -28,16 +33,19 @@ def __post_init__(self) -> None: self.metadata = {} def validate(self) -> bool: - """Validate the answer based on its category.""" + """Validate the answer based on its kind.""" validators = { - AnswerCategory.NUMBER: self._validate_number, - AnswerCategory.EQUATION: self._validate_string, - AnswerCategory.PHYSICAL_QUANTITY: self._validate_string, - AnswerCategory.FORMULA: self._validate_string, - AnswerCategory.TEXT: self._validate_string, - AnswerCategory.OPTION: self._validate_option, + AnswerObjectKind.NUMBER: self._validate_number, + AnswerObjectKind.PHYSICAL_QUANTITY: self._validate_string, + AnswerObjectKind.EXPRESSION: self._validate_string, + AnswerObjectKind.RELATION: self._validate_string, + AnswerObjectKind.QUALITATIVE_LABEL: self._validate_string, + AnswerObjectKind.BOOLEAN: self._validate_string, + AnswerObjectKind.SIGN_DIRECTION: self._validate_string, + AnswerObjectKind.DESCRIPTIVE_TEXT: self._validate_string, + AnswerObjectKind.CHOICE: self._validate_option, } - validator = validators.get(self.answer_category) + validator = validators.get(self.answer_kind) return validator() if validator else False def _validate_number(self) -> bool: @@ -55,41 +63,41 @@ def _validate_option(self) -> bool: # Type checking methods def is_number(self) -> bool: """Check if this is a dimensionless number answer.""" - return self.answer_category == AnswerCategory.NUMBER + return self.answer_kind == AnswerObjectKind.NUMBER def is_equation(self) -> bool: - """Check if this is an equation answer.""" - return self.answer_category == AnswerCategory.EQUATION + """Check if this is an equation/relation answer.""" + return self.answer_kind == AnswerObjectKind.RELATION def is_physical_quantity(self) -> bool: """Check if this is a physical quantity (number + units) answer.""" - return self.answer_category == AnswerCategory.PHYSICAL_QUANTITY + return self.answer_kind == AnswerObjectKind.PHYSICAL_QUANTITY def is_formula(self) -> bool: - """Check if this is a formula answer.""" - return self.answer_category == AnswerCategory.FORMULA + """Check if this is a formula/expression answer.""" + return self.answer_kind == AnswerObjectKind.EXPRESSION def is_text(self) -> bool: - """Check if this is a text answer.""" - return self.answer_category == AnswerCategory.TEXT + """Check if this is a free-form descriptive text answer.""" + return self.answer_kind == AnswerObjectKind.DESCRIPTIVE_TEXT def is_option(self) -> bool: - """Check if this is an option answer.""" - return self.answer_category == AnswerCategory.OPTION + """Check if this is an option/choice answer.""" + return self.answer_kind == AnswerObjectKind.CHOICE def is_numerical(self) -> bool: """Check if this has a numeric component (number or physical_quantity).""" - return self.answer_category in ( - AnswerCategory.NUMBER, - AnswerCategory.PHYSICAL_QUANTITY, + return self.answer_kind in ( + AnswerObjectKind.NUMBER, + AnswerObjectKind.PHYSICAL_QUANTITY, ) def is_symbolic(self) -> bool: - """Check if this is a symbolic/math answer (equation, formula, or physical_quantity).""" - return self.answer_category in ( - AnswerCategory.EQUATION, - AnswerCategory.FORMULA, - AnswerCategory.PHYSICAL_QUANTITY, + """Check if this is a symbolic/math answer (relation, expression, or physical_quantity).""" + return self.answer_kind in ( + AnswerObjectKind.RELATION, + AnswerObjectKind.EXPRESSION, + AnswerObjectKind.PHYSICAL_QUANTITY, ) # Numerical-specific methods @@ -223,13 +231,13 @@ def __str__(self) -> str: def __repr__(self) -> str: """Detailed string representation for debugging.""" - return f"Answer(value={repr(self.value)}, answer_category={self.answer_category.value}, unit={repr(self.unit)})" + return f"Answer(value={repr(self.value)}, answer_kind={self.answer_kind.value}, unit={repr(self.unit)})" def to_dict(self) -> dict[str, Any]: """Convert to dictionary for serialization.""" result: dict[str, Any] = { "value": self.value, - "answer_category": self.answer_category.value, + "answer_kind": self.answer_kind.value, } if self.unit: result["unit"] = self.unit @@ -241,10 +249,10 @@ def get_value(self) -> AnswerValue: """Get the answer value.""" return self.value - def get_type(self) -> AnswerCategory: - """Get the answer category.""" - return self.answer_category + def get_type(self) -> AnswerObjectKind: + """Get the answer kind.""" + return self.answer_kind def get_type_name(self) -> str: - """Get the answer category as a string.""" - return self.answer_category.value + """Get the answer kind as a string.""" + return self.answer_kind.value diff --git a/src/prkit/core/domain/answer_category.py b/src/prkit/core/domain/answer_category.py deleted file mode 100644 index f7b86aa..0000000 --- a/src/prkit/core/domain/answer_category.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -Answer category definitions for physical reasoning evaluation. - -This module defines the different categories of answers that can be compared. -AnswerCategory provides granular semantics (number, equation, physical_quantity, -formula, text, option) that cover both content-based and format-based classification. -""" - -from enum import Enum - - -class AnswerCategory(Enum): - """ - Enumeration of answer categories for normalization and comparison. - - Covers both content semantics (from normalization) and format (option): - - number: Dimensionless numeric value (e.g., 42, 3.14) - - equation: Single-equation form (e.g., F = ma) - - physical_quantity: Number with units (e.g., 9.8 m/s^2) - - formula: Mathematical expression (e.g., x^2 + 1) - - text: Text-based descriptive answer - - option: Multiple choice selection (e.g., A, B, 1, 2) - """ - - NUMBER = "number" - EQUATION = "equation" - PHYSICAL_QUANTITY = "physical_quantity" - FORMULA = "formula" - TEXT = "text" - OPTION = "option" diff --git a/src/prkit/core/domain/answer_kinds.py b/src/prkit/core/domain/answer_kinds.py new file mode 100644 index 0000000..a06d593 --- /dev/null +++ b/src/prkit/core/domain/answer_kinds.py @@ -0,0 +1,53 @@ +"""Canonical answer-ontology enumerations for PRKit. + +``AnswerObjectKind`` (what kind of object the final answer is) and +``AnswerStructure`` (how the answer is shaped) are the single, version-stable +taxonomy the whole toolkit targets — normalization, the judgement engine, saved +artifacts, and the public contract all agree on these names. They live in +``prkit.core.domain`` (not ``semantics``) because they are *ontology*, not +judgement mechanism; the policy enums (unit policy, comparison mode, bridge tier, +…) stay in ``prkit.semantics.schema``. + +See ``../../semantics/comparison/EQUIVALENCE.md`` for the per-kind equivalence +criteria and ``../../semantics/comparison/METHODOLOGY.md`` for the design +discipline behind the taxonomy. +""" + +from __future__ import annotations + +from enum import Enum + + +class _StrEnum(str, Enum): + """Enum subclass with string values and friendly ``str()`` output.""" + + def __str__(self) -> str: + return str(self.value) + + +class AnswerObjectKind(_StrEnum): + """What kind of answer object the normalized final answer is.""" + + NUMBER = "number" + PHYSICAL_QUANTITY = "physical_quantity" + EXPRESSION = "expression" + RELATION = "relation" + QUALITATIVE_LABEL = "qualitative_label" + CHOICE = "choice" + BOOLEAN = "boolean" + SIGN_DIRECTION = "sign_direction" + DESCRIPTIVE_TEXT = "descriptive_text" + + +class AnswerStructure(_StrEnum): + """How the answer is structured.""" + + ATOMIC = "atomic" + MULTI_PART = "multi_part" + TUPLE = "tuple" + SET = "set" + INTERVAL = "interval" + VECTOR = "vector" + MATRIX = "matrix" + TENSOR = "tensor" + PIECEWISE = "piecewise" diff --git a/src/prkit/core/domain/physics_problem.py b/src/prkit/core/domain/physics_problem.py index e988df7..061f296 100644 --- a/src/prkit/core/domain/physics_problem.py +++ b/src/prkit/core/domain/physics_problem.py @@ -14,7 +14,7 @@ from ..logging_config import PRKitLogger from .answer import Answer, AnswerValue -from .answer_category import AnswerCategory +from .answer_kinds import AnswerObjectKind from .physics_domain import PhysicsDomain # Get logger for this module @@ -324,7 +324,7 @@ def from_dict(cls, data: dict[str, Any]) -> "PhysicsProblem": "image_path", "options", "correct_option", - "answer_category", + "answer_kind", ] core_data: dict[str, Any] = {} @@ -339,22 +339,22 @@ def from_dict(cls, data: dict[str, Any]) -> "PhysicsProblem": answer_value: AnswerValue = raw_answer_value else: answer_value = str(raw_answer_value) - answer_category_str = value.get("answer_category") + answer_kind_str = value.get("answer_kind") answer_unit = value.get("unit") answer_metadata = value.get("metadata", {}) - if answer_category_str: + if answer_kind_str: try: - answer_category = AnswerCategory(answer_category_str) + answer_kind = AnswerObjectKind(answer_kind_str) except ValueError: - answer_category = AnswerCategory.TEXT + answer_kind = AnswerObjectKind.DESCRIPTIVE_TEXT else: - answer_category = AnswerCategory.TEXT + answer_kind = AnswerObjectKind.DESCRIPTIVE_TEXT # Create Answer object core_data[key] = Answer( value=answer_value, - answer_category=answer_category, + answer_kind=answer_kind, unit=answer_unit, metadata=answer_metadata, ) diff --git a/src/prkit/datasets/loaders/base_loader.py b/src/prkit/datasets/loaders/base_loader.py index abdfc12..cd08969 100644 --- a/src/prkit/datasets/loaders/base_loader.py +++ b/src/prkit/datasets/loaders/base_loader.py @@ -10,7 +10,7 @@ from prkit.core import PRKitLogger from prkit.core.domain import PhysicalDataset, PhysicsProblem from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_category import AnswerCategory +from prkit.core.domain.answer_kinds import AnswerObjectKind # Try to import PIL/Pillow for image loading PILImageModule: Any | None @@ -59,14 +59,17 @@ def raw_answer_to_text(value: Any) -> str: return str(value).strip() -def detect_answer_category(value: str) -> AnswerCategory: +def detect_answer_category(value: str) -> AnswerObjectKind: """ - Infer answer category from a string value when dataset does not specify it. + Infer the coarse answer kind from a string value when a dataset does not specify it. + + This is an ingestion-time hint only; the physics-semantics engine re-derives the + precise ``object_kind`` independently when judging. Strategy: 1. Try to parse as pure number first -> NUMBER - 2. Check for mathematical expression patterns -> FORMULA - 3. Fall back to TEXT if unclear + 2. Check for mathematical expression patterns -> EXPRESSION + 3. Fall back to DESCRIPTIVE_TEXT if unclear """ value = str(value).strip() @@ -79,14 +82,14 @@ def detect_answer_category(value: str) -> AnswerCategory: # Step 1: Check if it's a pure number (including scientific notation) if is_pure_number(value): - return AnswerCategory.NUMBER + return AnswerObjectKind.NUMBER # Step 2: Check if it's a mathematical expression if is_mathematical_expression(value): - return AnswerCategory.FORMULA + return AnswerObjectKind.EXPRESSION - # Step 3: Default to text - return AnswerCategory.TEXT + # Step 3: Default to free-form descriptive text + return AnswerObjectKind.DESCRIPTIVE_TEXT def is_pure_number(value: str) -> bool: @@ -583,9 +586,12 @@ def _create_answer_from_raw( if "MC" in problem_type: return Answer( value=raw_answer_to_text(answer), - answer_category=AnswerCategory.OPTION, + answer_kind=AnswerObjectKind.CHOICE, ) + # The metadata tag is a coarse ingestion hint; accept both the canonical + # AnswerObjectKind spellings and legacy dataset tags ("formula"/"equation"/ + # "text"/"option"). The semantics engine re-derives object_kind anyway. if answer_category in ("number", "physical_quantity"): if isinstance(answer, dict): value = raw_answer_to_text(answer.get("value")) @@ -601,30 +607,35 @@ def _create_answer_from_raw( value = re.sub(r"\$\$(.*?)\$\$", r"\1", value) value = re.sub(r"\$([^$]+)\$", r"\1", value) - category = ( - AnswerCategory.PHYSICAL_QUANTITY if unit else AnswerCategory.NUMBER + kind = ( + AnswerObjectKind.PHYSICAL_QUANTITY if unit else AnswerObjectKind.NUMBER + ) + return Answer(value=value, answer_kind=kind, unit=unit or None) + elif answer_category in ("expression", "formula"): + return Answer( + value=raw_answer_to_text(answer), + answer_kind=AnswerObjectKind.EXPRESSION, ) - return Answer(value=value, answer_category=category, unit=unit or None) - elif answer_category in ("formula", "equation"): + elif answer_category in ("relation", "equation"): return Answer( value=raw_answer_to_text(answer), - answer_category=AnswerCategory.FORMULA, + answer_kind=AnswerObjectKind.RELATION, ) - elif answer_category == "text": + elif answer_category in ("descriptive_text", "text"): return Answer( value=raw_answer_to_text(answer), - answer_category=AnswerCategory.TEXT, + answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT, ) - elif answer_category == "option": + elif answer_category in ("choice", "option"): return Answer( value=raw_answer_to_text(answer), - answer_category=AnswerCategory.OPTION, + answer_kind=AnswerObjectKind.CHOICE, ) else: - # fallback to auto-detect when answer_category not specified + # fallback to auto-detect when answer kind not specified answer_text = raw_answer_to_text(answer) detected = detect_answer_category(answer_text) - return Answer(value=answer_text, answer_category=detected) + return Answer(value=answer_text, answer_kind=detected) def create_physics_problem( self, diff --git a/src/prkit/datasets/loaders/physics_loader.py b/src/prkit/datasets/loaders/physics_loader.py index 06cab79..111fb97 100644 --- a/src/prkit/datasets/loaders/physics_loader.py +++ b/src/prkit/datasets/loaders/physics_loader.py @@ -322,7 +322,7 @@ def _normalize_answers(self, raw_answers: Any) -> tuple[str, list[str], str]: ) if not answer_parts: - return "", [], "text" + return "", [], "descriptive_text" if len(answer_parts) == 1: answer_value = answer_parts[0] @@ -335,9 +335,11 @@ def _normalize_answers(self, raw_answers: Any) -> tuple[str, list[str], str]: detected_categories = [ detect_answer_category(answer) for answer in answer_parts ] - if all(category.value == "text" for category in detected_categories): - return answer_value, answer_parts, "text" - return answer_value, answer_parts, "formula" + if all( + category.value == "descriptive_text" for category in detected_categories + ): + return answer_value, answer_parts, "descriptive_text" + return answer_value, answer_parts, "expression" def _decode_graphs( self, diff --git a/src/prkit/evaluation/__init__.py b/src/prkit/evaluation/__init__.py index 0ac77da..a80e76f 100644 --- a/src/prkit/evaluation/__init__.py +++ b/src/prkit/evaluation/__init__.py @@ -1,11 +1,9 @@ -"""Answer comparators and evaluators for physical reasoning tasks.""" +"""Evaluation utilities for physical reasoning tasks. -from prkit.evaluation.comparator import BaseComparator, ExactMatchComparator -from prkit.evaluation.evaluator import AccuracyEvaluator, BaseEvaluator +The deprecated comparator/evaluator stacks were **removed** in ``API_VERSION`` 2.0; +use :class:`prkit.scoring.SemanticsScorer` (the ``Scorer`` / ``Verdict`` contract) for +deterministic scoring. The model-graded :mod:`prkit.evaluation.llm_judge` remains a +distinct, supported capability. +""" -__all__ = [ - "BaseComparator", - "ExactMatchComparator", - "BaseEvaluator", - "AccuracyEvaluator", -] +__all__: list[str] = [] diff --git a/src/prkit/evaluation/comparator/__init__.py b/src/prkit/evaluation/comparator/__init__.py deleted file mode 100644 index d9bfdea..0000000 --- a/src/prkit/evaluation/comparator/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Answer comparators for physical reasoning evaluation: exact, normalized, category, smart, LLM, and similarity.""" - -from .base import BaseComparator -from .by_module import ( - build_comparator, - comparator_module_names, - resolve_comparator_module, - uses_openai_model, - uses_rouge_threshold, -) -from .category_match import CategoryComparator -from .exact_match import ExactMatchComparator -from .normalized_match import NormalizedMatchComparator -from .record_match import RecordMatchComparator -from .similarity_match import SimilarityMatchComparator -from .smart_llm import SmartLLMComparator -from .smart_match import SmartMatchComparator -from .smart_pipeline import ( - SmartMatchPipelineHost, - SmartPipelineResult, - run_smart_pipeline, -) -from .typed_llm import TypedLLMComparator - -__all__ = [ - "BaseComparator", - "build_comparator", - "CategoryComparator", - "comparator_module_names", - "resolve_comparator_module", - "ExactMatchComparator", - "NormalizedMatchComparator", - "RecordMatchComparator", - "SimilarityMatchComparator", - "SmartLLMComparator", - "SmartMatchComparator", - "SmartMatchPipelineHost", - "SmartPipelineResult", - "TypedLLMComparator", - "uses_openai_model", - "uses_rouge_threshold", - "run_smart_pipeline", -] diff --git a/src/prkit/evaluation/comparator/base.py b/src/prkit/evaluation/comparator/base.py deleted file mode 100644 index b7c4675..0000000 --- a/src/prkit/evaluation/comparator/base.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Abstract base class for all answer comparators in PRKit. - -.. deprecated:: - The comparator/evaluator stacks are superseded by the unified, version-stamped - :class:`prkit.scoring.SemanticsScorer` (the :class:`prkit.api.Scorer` / - :class:`prkit.api.Verdict` contract), which wraps the deterministic semantics - comparison engine. Constructing any comparator emits a ``DeprecationWarning``; - these classes will be removed in a future release. See ``prkit/CONTRACT.md``. -""" - -import warnings -from abc import ABC, abstractmethod -from typing import Any - -from prkit.core.domain.answer import Answer - -#: Shared pointer to the replacement, reused by the evaluator stack. -DEPRECATION_HINT = ( - "use prkit.scoring.SemanticsScorer (the prkit.api.Scorer / Verdict contract) " - "instead; see prkit/CONTRACT.md" -) - - -class BaseComparator(ABC): - """Base class for answer comparison strategies. - - .. deprecated:: superseded by :class:`prkit.scoring.SemanticsScorer`. - """ - - def __init__(self) -> None: - warnings.warn( - f"{type(self).__name__} is deprecated and will be removed in a future " - f"release; {DEPRECATION_HINT}.", - DeprecationWarning, - stacklevel=2, - ) - - @abstractmethod - def compare( - self, answer1: str | Answer, answer2: str | Answer, **kwargs: Any - ) -> Any: - """ - Compare two answers and return comparison result. - - For exact match comparators, returns a boolean (True/False). - For distance-based comparators, returns a numeric value (distance). - - Args: - answer1: First answer to compare (typically predicted/student answer) - answer2: Second answer to compare (typically ground truth/correct answer) - - Returns: - Comparison result: - - bool: True if answers match exactly, False otherwise - - float: Numeric distance/score for distance-based comparison - """ - pass - - @abstractmethod - def accuracy_score( - self, answer1: str | Answer, answer2: str | Answer, **kwargs: Any - ) -> float: - """ - Compute a normalized accuracy score between two answers in [0, 1]. - - For exact match comparators, returns 1.0 if equal, 0.0 otherwise. - For distance-based comparators, scales the distance to [0, 1]. - - Args: - answer1: First answer to compare - answer2: Second answer to compare - - Returns: - Accuracy score in [0, 1] where: - - 1.0 means perfect match - - 0.0 means no match - - Values in between indicate partial accuracy - """ - pass - - def can_compare(self, answer1: Answer, answer2: Answer) -> bool: - """ - Check if this comparator can handle the given answer types. - - Default implementation returns True. Subclasses can override - to restrict which answer types they can handle. - - Args: - answer1: First answer to check - answer2: Second answer to check - - Returns: - True if this comparator can handle the answer types, False otherwise - """ - return True diff --git a/src/prkit/evaluation/comparator/by_module.py b/src/prkit/evaluation/comparator/by_module.py deleted file mode 100644 index 250a10c..0000000 --- a/src/prkit/evaluation/comparator/by_module.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -Construct a :class:`~prkit.evaluation.comparator.base.BaseComparator` from a -package submodule name (the Python module stem, e.g. ``\"smart_match\"`` for -``smart_match.py``). -""" - -from __future__ import annotations - -import importlib -from typing import Any - -from prkit.evaluation.comparator.base import BaseComparator - -# Submodule name (``*.py`` stem) -> comparator class name in that module. -_MODULE_CLASS: dict[str, str] = { - "category_match": "CategoryComparator", - "exact_match": "ExactMatchComparator", - "normalized_match": "NormalizedMatchComparator", - "record_match": "RecordMatchComparator", - "similarity_match": "SimilarityMatchComparator", - "smart_match": "SmartMatchComparator", - "smart_llm": "SmartLLMComparator", - "typed_llm": "TypedLLMComparator", -} - -# Legacy / convenience aliases (e.g. old scripts referred to an LLM judge comparator). -_ALIASES: dict[str, str] = { - "llm_judge": "typed_llm", -} - -_MODULES_ACCEPTING_MODEL: frozenset[str] = frozenset({"typed_llm", "smart_llm"}) -_MODULES_WITH_ROUGE_THRESHOLD: frozenset[str] = frozenset({"similarity_match"}) - - -def comparator_module_names() -> tuple[str, ...]: - """Sorted valid submodule names (excluding aliases).""" - return tuple(sorted(_MODULE_CLASS.keys())) - - -def resolve_comparator_module(name: str) -> str: - """Map an alias or module name to the canonical submodule name.""" - n = name.strip() - if not n: - raise ValueError("Comparator name must be non-empty") - return _ALIASES.get(n, n) - - -def uses_openai_model(module_name: str) -> bool: - """Whether ``build_comparator(..., model=...)`` passes ``model`` to the class.""" - return resolve_comparator_module(module_name) in _MODULES_ACCEPTING_MODEL - - -def uses_rouge_threshold(module_name: str) -> bool: - """Whether ``build_comparator(..., rouge_threshold=...)`` passes the threshold to the class.""" - return resolve_comparator_module(module_name) in _MODULES_WITH_ROUGE_THRESHOLD - - -def build_comparator( - module_name: str, - *, - model: str | None = None, - rouge_threshold: float = 0.5, -) -> BaseComparator: - """ - Import ``prkit.evaluation.comparator.`` and instantiate the - registered comparator class. - - Parameters - ---------- - module_name - Submodule stem, e.g. ``smart_match``, or alias ``llm_judge`` → ``typed_llm``. - model - Passed to :class:`~prkit.evaluation.comparator.typed_llm.TypedLLMComparator` - and :class:`~prkit.evaluation.comparator.smart_llm.SmartLLMComparator`. - If omitted, ``prkit.evaluation.llm_judge.DEFAULT_MODEL`` is used. - rouge_threshold - Passed to :class:`~prkit.evaluation.comparator.similarity_match.SimilarityMatchComparator`. - """ - from prkit.evaluation.llm_judge import DEFAULT_MODEL - - name = resolve_comparator_module(module_name) - if name not in _MODULE_CLASS: - valid = ", ".join(comparator_module_names()) - raise ValueError( - f"Unknown comparator module {module_name!r}. Choose one of: {valid}" - ) - - mod = importlib.import_module(f"prkit.evaluation.comparator.{name}") - cls: type[Any] = getattr(mod, _MODULE_CLASS[name]) - - if name in _MODULES_ACCEPTING_MODEL: - m = model if model is not None else DEFAULT_MODEL - comparator = cls(model=m) - elif name in _MODULES_WITH_ROUGE_THRESHOLD: - comparator = cls(rouge_threshold=rouge_threshold) - else: - comparator = cls() - - if not isinstance(comparator, BaseComparator): - raise TypeError(f"{cls.__name__} is not a BaseComparator") - return comparator diff --git a/src/prkit/evaluation/comparator/category_match.py b/src/prkit/evaluation/comparator/category_match.py deleted file mode 100644 index db609b2..0000000 --- a/src/prkit/evaluation/comparator/category_match.py +++ /dev/null @@ -1,127 +0,0 @@ -""" -Category-Based Comparator for same-type answer comparison. - -``compare`` / ``accuracy_score`` resolve same-type pairs via the shared -:func:`compare_by_category` dispatch. Cross-type pairs are compared as normalized -plain text only (GT-as-substring semantics via :func:`compare_plain_text`). - -For cross-type deterministic matching (e.g. PQ vs NUMBER), see -:class:`SmartMatchComparator`. - -Subclass hooks: overriding ``_comparators`` affects :meth:`_compare_by_category` -and :meth:`compare`. -""" - -from typing import Any - -from prkit.core import PRKitLogger -from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation.utils.answer_utils import same_comparison_category -from prkit.evaluation.utils.category_dispatch import ( - SameCategoryCompareFn, - compare_by_category, -) -from prkit.evaluation.utils.compare_same_type import ( - compare_formula, - compare_number, - compare_option, - compare_physical_quantity, - compare_plain_text, -) -from prkit.evaluation.utils.normalization import normalize_answer, normalize_text - -from .base import BaseComparator - - -def _typed_category_and_value( - answer: str | Answer, -) -> tuple[AnswerCategory | None, float | str]: - """Return ``(category, normalized_value)`` for *answer*, or ``(None, raw_text)`` when normalization fails.""" - if isinstance(answer, Answer): - return answer.answer_category, str(answer.value) - try: - category, normalized = normalize_answer(answer) - return category, normalized - except (ValueError, TypeError, RuntimeError): - return None, str(answer).strip() - - -class CategoryComparator(BaseComparator): - """ - Comparator restricted to same-type category dispatch plus plain-text - fallback for cross-type pairs. - - ``answer1`` is treated as the model prediction and ``answer2`` as ground truth, - matching :class:`TypedLLMComparator`. Optional ``kwargs`` (e.g. ``question``, - ``symbolic_answer_is_expression``) are accepted for API compatibility with - the LLM judge but are ignored — this comparator does not call an LLM. - """ - - DEFAULT_COMPARATORS: dict[AnswerCategory, SameCategoryCompareFn] = { - AnswerCategory.NUMBER: compare_number, - AnswerCategory.EQUATION: compare_plain_text, - AnswerCategory.PHYSICAL_QUANTITY: compare_physical_quantity, - AnswerCategory.FORMULA: compare_formula, - AnswerCategory.TEXT: compare_plain_text, - AnswerCategory.OPTION: compare_option, - } - - def __init__(self) -> None: - """Initialize with default category comparators.""" - super().__init__() - self._comparators = dict(self.DEFAULT_COMPARATORS) - self.logger = PRKitLogger.get_logger(__name__) - - def _compare_by_category( - self, - category: AnswerCategory, - predicted_norm: float | str, - ground_truth_norm: float | str, - ) -> bool: - """Compare two normalized values using the category-specific strategy.""" - return compare_by_category( - category, - predicted_norm, - ground_truth_norm, - self._comparators, - self.logger, - ) - - def compare( - self, - answer1: str | Answer, - answer2: str | Answer, - **kwargs: Any, - ) -> bool: - """ - True when same-type :func:`compare_by_category` matches, or when - cross-type plain-text comparison matches after normalization. - - Cross-type pairs are not graded with category-specific logic; they are - compared only as normalized text. For richer cross-type rules, use - :class:`SmartMatchComparator`. - """ - pred_cat, pred_value = _typed_category_and_value(answer1) - gt_cat, gt_value = _typed_category_and_value(answer2) - if pred_cat is None or gt_cat is None: - return False - - if same_comparison_category(gt_cat, pred_cat): - return self._compare_by_category(gt_cat, pred_value, gt_value) - - pred_text = normalize_text(str(pred_value)) - gt_text = normalize_text(str(gt_value)) - return compare_plain_text(pred_text, gt_text) - - def accuracy_score( - self, - answer1: str | Answer, - answer2: str | Answer, - **kwargs: Any, - ) -> float: - """ - 1.0 if :meth:`compare` is True, else 0.0. - """ - is_match = self.compare(answer1, answer2, **kwargs) - return 1.0 if is_match else 0.0 diff --git a/src/prkit/evaluation/comparator/exact_match.py b/src/prkit/evaluation/comparator/exact_match.py deleted file mode 100644 index 2b0c030..0000000 --- a/src/prkit/evaluation/comparator/exact_match.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Exact-string match comparator for answer comparison.""" - -from typing import Any - -from prkit.core.domain.answer import Answer -from prkit.evaluation.utils.answer_utils import to_str - -from .base import BaseComparator - - -class ExactMatchComparator(BaseComparator): - """Comparator that performs exact string matching between answers.""" - - def compare( - self, answer1: str | Answer, answer2: str | Answer, **kwargs: Any - ) -> bool: - """ - Compare two answers exactly. - - Args: - answer1: First answer to compare (string or Answer) - answer2: Second answer to compare (string or Answer) - - Returns: - True if answers match exactly, False otherwise - """ - return to_str(answer1) == to_str(answer2) - - def accuracy_score( - self, answer1: str | Answer, answer2: str | Answer, **kwargs: Any - ) -> float: - """ - Compute accuracy score for exact match comparison. - - Returns 1.0 if answers match exactly, 0.0 otherwise. - - Args: - answer1: First answer to compare - answer2: Second answer to compare - - Returns: - 1.0 if answers match exactly, 0.0 otherwise - """ - return 1.0 if self.compare(answer1, answer2) else 0.0 diff --git a/src/prkit/evaluation/comparator/normalized_match.py b/src/prkit/evaluation/comparator/normalized_match.py deleted file mode 100644 index f61b555..0000000 --- a/src/prkit/evaluation/comparator/normalized_match.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Normalized-string match comparator: normalizes answers by category before comparing.""" - -from typing import Any - -from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation.utils.answer_utils import same_comparison_category, to_str -from prkit.evaluation.utils.normalization import normalize_answer, normalize_text -from prkit.evaluation.utils.number_utils import DEFAULT_NUMBER_EPSILON - -from .base import BaseComparator - - -class NormalizedMatchComparator(BaseComparator): - """Comparator that normalizes answers before exact matching.""" - - def compare( - self, answer1: str | Answer, answer2: str | Answer, **kwargs: Any - ) -> bool: - """ - Compare two answers after normalization. - - If the answers are in different categories, both are normalized as text - and compared as strings. - - Args: - answer1: First answer to compare (string or Answer) - answer2: Second answer to compare (string or Answer) - - Returns: - True if answers match after normalization, False otherwise - """ - ans1_str = to_str(answer1) - ans2_str = to_str(answer2) - - # Option answers: direct comparison (case-insensitive) when both are Answer with OPTION - if isinstance(answer1, Answer) and isinstance(answer2, Answer): - if ( - answer1.answer_category == AnswerCategory.OPTION - and answer2.answer_category == AnswerCategory.OPTION - ): - return ans1_str.strip().upper() == ans2_str.strip().upper() - - # Use auto-detection normalization (categorizes as number, equation, - # physical_quantity, formula, or text) - cat1, norm1 = normalize_answer(ans1_str) - cat2, norm2 = normalize_answer(ans2_str) - - # If categories differ, treat both as text and compare as strings - if not same_comparison_category(cat1, cat2): - # Normalize both as text and compare - text1 = normalize_text(ans1_str) - text2 = normalize_text(ans2_str) - return text1 == text2 - - # Compare based on category - if cat1 == AnswerCategory.NUMBER: - if not (isinstance(norm1, float) and isinstance(norm2, float)): - return False - return abs(norm1 - norm2) < DEFAULT_NUMBER_EPSILON - else: - # equation, formula, physical_quantity, text: identical string comparison - if not (isinstance(norm1, str) and isinstance(norm2, str)): - return False - return norm1 == norm2 - - def accuracy_score( - self, answer1: str | Answer, answer2: str | Answer, **kwargs: Any - ) -> float: - """ - Compute accuracy score for normalized match comparison. - - Returns 1.0 if answers match after normalization, 0.0 otherwise. - - Args: - answer1: First answer to compare (string or Answer) - answer2: Second answer to compare (string or Answer) - - Returns: - 1.0 if answers match after normalization, 0.0 otherwise - """ - return 1.0 if self.compare(answer1, answer2, **kwargs) else 0.0 diff --git a/src/prkit/evaluation/comparator/record_match.py b/src/prkit/evaluation/comparator/record_match.py deleted file mode 100644 index cf86daf..0000000 --- a/src/prkit/evaluation/comparator/record_match.py +++ /dev/null @@ -1,156 +0,0 @@ -""" -Record-aware comparator for canonical typed final-answer records. - -This comparator accepts canonical-answer records (mapping-like objects or -objects exposing the same attributes), maps them into -``prkit.core.domain.answer.Answer``, and then delegates to the -deterministic SmartMatch pipeline. This mirrors SmartLLM's local typed / -cross-type behavior without any LLM fallback. -""" - -from __future__ import annotations - -from collections.abc import Mapping -from typing import Any - -from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_category import AnswerCategory - -from .base import BaseComparator -from .smart_match import SmartMatchComparator - -RecordLike = object - -_RECORD_TYPE_TO_CATEGORY = { - "number": AnswerCategory.NUMBER, - "physical_quantity": AnswerCategory.PHYSICAL_QUANTITY, - "formula": AnswerCategory.FORMULA, - "equation": AnswerCategory.EQUATION, - "short_text": AnswerCategory.TEXT, - "option": AnswerCategory.OPTION, - "invalid": AnswerCategory.TEXT, -} - - -def _is_record_like(answer: object) -> bool: - """Return ``True`` when *answer* looks like a typed final-answer record (has status/answer_type/final_answer).""" - if isinstance(answer, Mapping): - return any(key in answer for key in ("status", "answer_type", "final_answer")) - return any( - hasattr(answer, key) for key in ("status", "answer_type", "final_answer") - ) - - -def _record_field(record: RecordLike, field_name: str) -> Any: - """Return the value of *field_name* from a record, supporting both Mapping and attribute access.""" - if isinstance(record, Mapping): - return record.get(field_name) - return getattr(record, field_name, None) - - -def _record_string(record: RecordLike, field_name: str) -> str | None: - """Return the string value of *field_name* from a record, or ``None`` when absent.""" - value = _record_field(record, field_name) - if value is None: - return None - return str(value).strip() - - -def _is_usable_record(record: RecordLike) -> bool: - """Return ``True`` when the record has ``status == "ok"`` and ``answer_type != "invalid"``.""" - status = (_record_string(record, "status") or "").lower() - answer_type = (_record_string(record, "answer_type") or "").lower() - return status == "ok" and answer_type != "invalid" - - -def _record_to_answer(record: RecordLike) -> Answer: - """Convert a typed final-answer record to an ``Answer`` domain object.""" - answer_type = (_record_string(record, "answer_type") or "short_text").lower() - category = _RECORD_TYPE_TO_CATEGORY.get(answer_type, AnswerCategory.TEXT) - - final_answer = _record_string(record, "final_answer") or "" - final_answer_latex = _record_string(record, "final_answer_latex") - value = _record_string(record, "value") - unit = _record_string(record, "unit") - - if category in (AnswerCategory.FORMULA, AnswerCategory.EQUATION): - return Answer( - value=final_answer_latex or final_answer, - answer_category=category, - ) - - if category == AnswerCategory.NUMBER: - return Answer( - value=value or final_answer, - answer_category=category, - ) - - if category == AnswerCategory.PHYSICAL_QUANTITY: - quantity_value = value or final_answer - if quantity_value and unit: - quantity_value = f"{quantity_value} {unit}" - return Answer( - value=quantity_value, - answer_category=category, - unit=unit, - ) - - return Answer( - value=( - final_answer - if category != AnswerCategory.OPTION - else (final_answer or (_record_string(record, "option_label") or "")) - ), - answer_category=category, - ) - - -class RecordMatchComparator(BaseComparator): - """ - Comparator for typed final-answer records. - - Records are compared only when both sides are usable final answers - (``status == "ok"`` and ``answer_type != "invalid"``). The mapped - ``Answer`` objects are then delegated to :class:`SmartMatchComparator`, - which runs same-type comparison, equation-RHS extraction, equation-from-text - rescue, and deterministic cross-type matching, but never calls an LLM. - """ - - def __init__(self) -> None: - super().__init__() - self._delegate = SmartMatchComparator() - - def _coerce_answer(self, answer: str | Answer | RecordLike) -> str | Answer: - """Return *answer* as a ``str`` or ``Answer``, converting record-like objects via ``_record_to_answer``.""" - if isinstance(answer, (str, Answer)): - return answer - if _is_record_like(answer): - return _record_to_answer(answer) - raise TypeError( - "RecordMatchComparator expects an Answer, string, or record-like object " - "with status/answer_type/final_answer fields." - ) - - def compare( - self, - answer1: str | Answer | RecordLike, - answer2: str | Answer | RecordLike, - **kwargs: Any, - ) -> bool: - if _is_record_like(answer1) and not _is_usable_record(answer1): - return False - if _is_record_like(answer2) and not _is_usable_record(answer2): - return False - - pred = self._coerce_answer(answer1) - gt = self._coerce_answer(answer2) - _ = kwargs - return self._delegate.compare(pred, gt) - - def accuracy_score( - self, - answer1: str | Answer | RecordLike, - answer2: str | Answer | RecordLike, - **kwargs: Any, - ) -> float: - return 1.0 if self.compare(answer1, answer2, **kwargs) else 0.0 diff --git a/src/prkit/evaluation/comparator/similarity_match.py b/src/prkit/evaluation/comparator/similarity_match.py deleted file mode 100644 index ebf9727..0000000 --- a/src/prkit/evaluation/comparator/similarity_match.py +++ /dev/null @@ -1,110 +0,0 @@ -""" -Comparator: quick typed match from :class:`TypedLLMComparator` when it returns a -definite bool; otherwise word-level ROUGE-L F1 on plain text (no LLM). -""" - -from __future__ import annotations - -from typing import Any - -from prkit.core import PRKitLogger -from prkit.core.domain.answer import Answer -from prkit.evaluation.similarities import rouge_l_f1 - -from .base import BaseComparator -from .typed_llm import TypedLLMComparator, _typed_category_and_value - - -def _text_for_rouge(answer: str | Answer) -> str: - """Return the plain text representation of *answer* used as ROUGE-L input.""" - if isinstance(answer, Answer): - return str(answer.value).strip() - return str(answer).strip() - - -class SimilarityMatchComparator(BaseComparator): - """ - If :meth:`TypedLLMComparator._quick_typed_match` returns a definite ``bool``, - :meth:`compare` / :meth:`accuracy_score` use it. Otherwise similarity is - word-level ROUGE-L F1 between the two answers as plain text. - - :meth:`compare` returns ``True`` if ROUGE-L ≥ ``rouge_threshold``; - :meth:`accuracy_score` returns the ROUGE-L value in ``[0, 1]`` on that path. - - Optional ``kwargs``: ``question`` and ``symbolic_answer_is_expression`` are - forwarded to :meth:`TypedLLMComparator._quick_typed_match` only. - """ - - def __init__( - self, - *, - rouge_threshold: float = 0.5, - ) -> None: - super().__init__() - self.logger = PRKitLogger.get_logger(__name__) - self._rouge_threshold = rouge_threshold - self._last_rouge_score: float | None = None - - @property - def last_rouge_score(self) -> float | None: - """ROUGE-L F1 from the last comparison that used the ROUGE fallback, if any.""" - return self._last_rouge_score - - def compare( - self, - answer1: str | Answer, - answer2: str | Answer, - **kwargs: Any, - ) -> bool: - self._last_rouge_score = None - quick = TypedLLMComparator._quick_typed_match( - answer1, - answer2, - question=kwargs.get("question"), - symbolic_answer_is_expression=kwargs.get("symbolic_answer_is_expression"), - ) - if quick is not None: - return bool(quick) - - pred_text = _text_for_rouge(answer1) - gt_text = _text_for_rouge(answer2) - score = rouge_l_f1(pred_text, gt_text) - self._last_rouge_score = score - pred_cat, _ = _typed_category_and_value(answer1) - gt_cat, _ = _typed_category_and_value(answer2) - self.logger.debug( - "ROUGE-L fallback F1=%.4f (threshold=%.4f) pred_cat=%s gt_cat=%s", - score, - self._rouge_threshold, - pred_cat, - gt_cat, - ) - return score >= self._rouge_threshold - - def accuracy_score( - self, - answer1: str | Answer, - answer2: str | Answer, - **kwargs: Any, - ) -> float: - self._last_rouge_score = None - quick = TypedLLMComparator._quick_typed_match( - answer1, - answer2, - question=kwargs.get("question"), - symbolic_answer_is_expression=kwargs.get("symbolic_answer_is_expression"), - ) - note = ( - "falling back to rouge-l" - if quick is None - else "returning typed match score" - ) - self.logger.debug("quick=%s, %s", quick, note) - if quick is not None: - return 1.0 if quick else 0.0 - - pred_text = _text_for_rouge(answer1) - gt_text = _text_for_rouge(answer2) - score = rouge_l_f1(pred_text, gt_text) - self._last_rouge_score = score - return score diff --git a/src/prkit/evaluation/comparator/smart_llm.py b/src/prkit/evaluation/comparator/smart_llm.py deleted file mode 100644 index 7a8ec60..0000000 --- a/src/prkit/evaluation/comparator/smart_llm.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -Smart-Match + LLM hybrid comparator. - -Runs :func:`~prkit.evaluation.comparator.smart_pipeline.run_smart_pipeline` -(same-type, RHS, equation-from-text rescue, then cross-type matching). -If cross-type resolution is inconclusive (``None``), optionally calls the shared -LLM judge; otherwise the deterministic outcome is final. -""" - -from __future__ import annotations - -from typing import Any - -from openai import OpenAI -from typing_extensions import assert_never - -from prkit.core.domain.answer import Answer -from prkit.evaluation.llm_judge import ( - DEFAULT_MODEL, - RESULT_SOURCE_SKIPPED_LLM, - RESULT_SOURCE_SMART_MATCH, - LLMJudgeResult, - OpenAIJudgeRunner, - build_standard_answer_judge_payload, -) - -from .smart_match import SmartMatchComparator -from .smart_pipeline import SmartPipelineResult, run_smart_pipeline - - -class SmartLLMComparator(SmartMatchComparator): - """Deterministic SmartMatch pipeline; LLM judge only when cross-type is inconclusive.""" - - def __init__( - self, - model: str = DEFAULT_MODEL, - *, - instructions: str | None = None, - client: OpenAI | None = None, - ) -> None: - super().__init__() - self._runner = OpenAIJudgeRunner( - model=model, - instructions=instructions, - client=client, - logger=self.logger, - ) - self._last_result: LLMJudgeResult | None = None - - def _result_smart(self, *, correct: bool) -> LLMJudgeResult: - """Build an ``LLMJudgeResult`` representing a deterministic SmartMatch verdict.""" - return LLMJudgeResult( - verdict="correct" if correct else "incorrect", - confidence=1.0, - expected_answer_type="other", - reasoning=( - "SmartMatchComparator deterministic path " - "(same-type, RHS extraction, equation-from-text rescue, or cross-type)." - ), - raw_response="local_smart_match", - verdict_type=RESULT_SOURCE_SMART_MATCH, - ) - - def compare( - self, - answer1: str | Answer, - answer2: str | Answer, - *, - skip_llm: bool = False, - **kwargs: Any, - ) -> bool: - outcome: SmartPipelineResult = run_smart_pipeline(self, answer1, answer2) - if outcome == "inconclusive": - if skip_llm: - self._last_result = LLMJudgeResult( - verdict="incorrect", - confidence=0.0, - expected_answer_type="other", - reasoning=( - "Cross-type matching returned no deterministic verdict; " - "LLM judge skipped (skip_llm=True)." - ), - raw_response="skipped_llm", - verdict_type=RESULT_SOURCE_SKIPPED_LLM, - ) - return False - - payload = build_standard_answer_judge_payload( - answer1, - answer2, - kwargs.get("question"), - ) - result = self._runner.judge(payload) - self._last_result = result - return result.verdict == "correct" - - if outcome == "match": - self._last_result = self._result_smart(correct=True) - return True - if outcome == "no_match": - self._last_result = self._result_smart(correct=False) - return False - - assert_never(outcome) - - def accuracy_score( - self, - answer1: str | Answer, - answer2: str | Answer, - *, - skip_llm: bool = False, - **kwargs: Any, - ) -> float: - ok = self.compare(answer1, answer2, skip_llm=skip_llm, **kwargs) - return 1.0 if ok else 0.0 - - @property - def last_result(self) -> LLMJudgeResult | None: - return self._last_result - - @property - def model_name(self) -> str: - return self._runner.model_name diff --git a/src/prkit/evaluation/comparator/smart_match.py b/src/prkit/evaluation/comparator/smart_match.py deleted file mode 100644 index 8b02bcb..0000000 --- a/src/prkit/evaluation/comparator/smart_match.py +++ /dev/null @@ -1,407 +0,0 @@ -""" -Smart-Match Comparator for answer comparison. - -``compare`` / ``accuracy_score`` first attempt same-type comparison, then -try equation RHS extraction to reduce equations to their value types, and -finally attempt deterministic cross-type matching for pairs that neither -path could resolve. - -A match is ``True`` when any of the three paths resolves to ``True``. -If none resolves the pair, this comparator returns ``False`` / score ``0.0``. -""" - -import re -from typing import Any - -from prkit.core import PRKitLogger -from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation.utils.answer_utils import same_comparison_category -from prkit.evaluation.utils.category_dispatch import ( - SameCategoryCompareFn, - compare_by_category, -) -from prkit.evaluation.utils.compare_cross_type import ( - compare_text_against_formula_or_equation_gt, - extract_rhs_and_category, -) -from prkit.evaluation.utils.compare_same_type import ( - compare_formula, - compare_number, - compare_option, - compare_physical_quantity, - compare_plain_text, - parse_physical_quantity, -) -from prkit.evaluation.utils.normalization import normalize_answer, normalize_text - -from .base import BaseComparator -from .smart_pipeline import run_smart_pipeline - -# Matches LaTeX-delimited math expressions in free text. -# Alternation order matters: $$...$$ must precede $...$ to avoid partial matches. -_LATEX_DELIMITED_RE = re.compile( - r"\$\$(.+?)\$\$|\$(.+?)\$|\\\[(.+?)\\\]|\\\((.+?)\\\)", - re.DOTALL, -) - - -def _extract_latex_equations(text: str) -> list[str]: - """Extract LaTeX-delimited math expressions from free text. - - Returns the full matched strings (including delimiters) so that - ``normalize_answer`` receives proper LaTeX context. - """ - return [m.group(0) for m in _LATEX_DELIMITED_RE.finditer(text)] - - -def _typed_category_and_value( - answer: str | Answer, -) -> tuple[AnswerCategory | None, float | str]: - """Return ``(category, normalized_value)`` for *answer*, or ``(None, raw_text)`` when normalization fails.""" - if isinstance(answer, Answer): - return answer.answer_category, str(answer.value) - try: - category, normalized = normalize_answer(answer) - return category, normalized - except (ValueError, TypeError, RuntimeError): - return None, str(answer).strip() - - -class SmartMatchComparator(BaseComparator): - """ - Comparator that combines same-type comparison, equation RHS extraction, - and deterministic cross-type matching. - - ``answer1`` is treated as the model prediction and ``answer2`` as ground - truth. - """ - - DEFAULT_COMPARATORS: dict[AnswerCategory, SameCategoryCompareFn] = { - AnswerCategory.NUMBER: compare_number, - AnswerCategory.PHYSICAL_QUANTITY: compare_physical_quantity, - AnswerCategory.FORMULA: compare_formula, - AnswerCategory.TEXT: compare_plain_text, - AnswerCategory.OPTION: compare_option, - } - - def __init__(self) -> None: - """Initialize with default category comparators.""" - super().__init__() - self._comparators = dict(self.DEFAULT_COMPARATORS) - self.logger = PRKitLogger.get_logger(__name__) - - # ------------------------------------------------------------------ - # Cross-type matching helpers - # ------------------------------------------------------------------ - - @staticmethod - def _extract_equation_rhs_raw(raw_text: str) -> str | None: - """Extract RHS from a raw equation string (e.g. ``$T_B = 355\\,K$``). - - For multi-line text only the first line is considered, so subsidiary - definitions (``where omega^2 = ...``) do not contaminate the RHS. - """ - s = raw_text.strip() - for opening, closing in [("$$", "$$"), ("\\(", "\\)"), ("\\[", "\\]")]: - if s.startswith(opening) and s.endswith(closing): - s = s[len(opening) : -len(closing)].strip() - break - s = s.strip("$").strip() - - if "\n" in s: - s = s.split("\n", 1)[0].strip() - - if "=" not in s: - return None - rhs = s.rsplit("=", 1)[1].strip() - return rhs if rhs else None - - @staticmethod - def _compare_numeric_with_renormalized( - numeric_cat: AnswerCategory, - numeric_value: float | str, - rhs_raw: str, - ) -> bool | None: - """Compare a NUMBER or PHYSICAL_QUANTITY value against a re-normalized - raw RHS string extracted from an equation. - - When *numeric_cat* is NUMBER, matching against a PQ-typed RHS is - intentionally skipped: a bare number missing its unit should not be - accepted when the ground truth carries units. - """ - try: - rhs_cat, rhs_norm = normalize_answer(rhs_raw) - except (ValueError, TypeError, RuntimeError): - return None - - if numeric_cat == AnswerCategory.PHYSICAL_QUANTITY: - if rhs_cat == AnswerCategory.PHYSICAL_QUANTITY: - try: - return compare_physical_quantity(str(numeric_value), str(rhs_norm)) - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - pass - elif rhs_cat == AnswerCategory.NUMBER: - try: - num, _, num_str = parse_physical_quantity(str(numeric_value)) - if num is not None: - return compare_number(num_str, str(rhs_norm)) - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - pass - elif numeric_cat == AnswerCategory.NUMBER: - if rhs_cat == AnswerCategory.NUMBER: - try: - return compare_number(str(numeric_value), str(rhs_norm)) - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - pass - # NUMBER vs PQ intentionally omitted: missing unit = wrong. - return None - - @staticmethod - def _compare_formula_with_renormalized( - formula_value: float | str, - rhs_raw: str, - ) -> bool | None: - """Compare a FORMULA value against a re-normalized raw RHS string - extracted from an equation on the other side.""" - try: - rhs_cat, rhs_norm = normalize_answer(rhs_raw) - except (ValueError, TypeError, RuntimeError): - return None - - if rhs_cat not in ( - AnswerCategory.FORMULA, - AnswerCategory.EQUATION, - AnswerCategory.NUMBER, - ): - return None - - try: - if compare_formula(str(formula_value), str(rhs_norm)): - return True - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - pass - try: - if compare_plain_text(str(formula_value), str(rhs_norm)): - return True - except (ValueError, TypeError): - pass - return None - - # ------------------------------------------------------------------ - # Equation-from-text extraction - # ------------------------------------------------------------------ - - def _try_equation_from_text( - self, - text_raw: str, - gt_norm: float | str, - gt_raw: str, - ) -> bool: - """Try matching by extracting an embedded LaTeX equation from text. - - 1. Extract LaTeX-delimited equations from *text_raw*. - 2. Normalize each and compare against *gt_norm* (direct, RHS, formula). - 3. If nothing matched, check whether *gt_norm* or *gt_raw* appears as a - substring of the prediction text. - """ - equations = _extract_latex_equations(text_raw) - for eq_str in equations: - try: - eq_cat, eq_norm = normalize_answer(eq_str) - except (ValueError, TypeError, RuntimeError): - continue - # Direct same-type comparison - if same_comparison_category(eq_cat, AnswerCategory.EQUATION): - if compare_by_category( - AnswerCategory.EQUATION, - eq_norm, - gt_norm, - self._comparators, - self.logger, - ): - return True - # RHS extraction and comparison - eq_rhs, eq_rhs_cat = extract_rhs_and_category(eq_norm, eq_cat) - gt_rhs, gt_rhs_cat = extract_rhs_and_category( - gt_norm, AnswerCategory.EQUATION - ) - if same_comparison_category(eq_rhs_cat, gt_rhs_cat): - if compare_by_category( - eq_rhs_cat, eq_rhs, gt_rhs, self._comparators, self.logger - ): - return True - # Formula-level comparison - try: - if compare_formula(str(eq_norm), str(gt_norm)): - return True - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - pass - - # Substring fallback: check if gt_norm or gt_raw appears in pred text - pred_text = normalize_text(text_raw) - gt_norm_text = normalize_text(str(gt_norm)) - if gt_norm_text and gt_norm_text in pred_text: - return True - gt_raw_text = normalize_text(gt_raw) - if gt_raw_text and gt_raw_text in pred_text: - return True - return False - - # ------------------------------------------------------------------ - # Main cross-type dispatch - # ------------------------------------------------------------------ - - def _cross_type_match( - self, - answer1: str | Answer, - answer2: str | Answer, - ) -> bool | None: - """Positive-only cross-type matching for pairs that same-type - comparison could not resolve. - - Returns ``True`` when a confident deterministic match is found, - ``None`` otherwise. Never returns ``False``, so callers can safely - fall back. - """ - pred_cat, pred_value = _typed_category_and_value(answer1) - gt_cat, gt_value = _typed_category_and_value(answer2) - - if pred_cat is None or gt_cat is None: - return None - - # --- PHYSICAL_QUANTITY (pred) vs NUMBER (gt) --- - # Model gives value+unit while GT is just a number -> model has more - # information; compare the numeric parts. - # The reverse direction (NUMBER pred vs PQ gt) is intentionally - # excluded: a bare number missing its unit is wrong for physics. - if ( - pred_cat == AnswerCategory.PHYSICAL_QUANTITY - and gt_cat == AnswerCategory.NUMBER - ): - try: - pred_num, _, pred_num_str = parse_physical_quantity(str(pred_value)) - if pred_num is not None and compare_number(pred_num_str, gt_value): - return True - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - pass - # --- NUMBER (pred) vs PHYSICAL_QUANTITY (gt) --- - if ( - pred_cat == AnswerCategory.NUMBER - and gt_cat == AnswerCategory.PHYSICAL_QUANTITY - ): - try: - gt_num, _, gt_num_str = parse_physical_quantity(str(gt_value)) - if gt_num is None or not compare_number(pred_value, gt_num_str): - return False - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - pass - - # --- TEXT(pred) vs FORMULA / EQUATION (gt) --- - if pred_cat == AnswerCategory.TEXT and gt_cat in ( - AnswerCategory.FORMULA, - AnswerCategory.EQUATION, - ): - pred_str = str(pred_value) - gt_str = str(gt_value) - try: - if compare_text_against_formula_or_equation_gt(pred_str, gt_value): - return True - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - pass - try: - if compare_plain_text(pred_str, gt_str): - return True - except (ValueError, TypeError): - pass - try: - if compare_formula(pred_str, gt_str): - return True - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - pass - - # --- EQUATION (GT) vs TEXT (pred) --- - if gt_cat == AnswerCategory.EQUATION and pred_cat == AnswerCategory.TEXT: - pred_raw = str(answer1).strip() - gt_raw = str(answer2).strip() - if self._try_equation_from_text(pred_raw, gt_value, gt_raw): - return True - - # --- EQUATION (GT) vs PHYSICAL_QUANTITY/NUMBER (pred) --- - if gt_cat == AnswerCategory.EQUATION and pred_cat in ( - AnswerCategory.PHYSICAL_QUANTITY, - AnswerCategory.NUMBER, - ): - gt_raw = str(answer2).strip() - rhs = self._extract_equation_rhs_raw(gt_raw) - if rhs is not None: - match = self._compare_numeric_with_renormalized( - pred_cat, pred_value, rhs - ) - if match is True: - return True - - # --- EQUATION (GT) vs FORMULA (pred) --- - if gt_cat == AnswerCategory.EQUATION and pred_cat == AnswerCategory.FORMULA: - gt_raw = str(answer2).strip() - rhs = self._extract_equation_rhs_raw(gt_raw) - if rhs is not None: - match = self._compare_formula_with_renormalized(pred_value, rhs) - if match is True: - return True - - # --- EQUATION (pred) vs PHYSICAL_QUANTITY/NUMBER (gt) --- - if pred_cat == AnswerCategory.EQUATION and gt_cat in ( - AnswerCategory.PHYSICAL_QUANTITY, - AnswerCategory.NUMBER, - ): - pred_raw = str(answer1).strip() - rhs = self._extract_equation_rhs_raw(pred_raw) - if rhs is not None: - match = self._compare_numeric_with_renormalized(gt_cat, gt_value, rhs) - if match is True: - return True - - # --- EQUATION (pred) vs FORMULA (gt) --- - if pred_cat == AnswerCategory.EQUATION and gt_cat == AnswerCategory.FORMULA: - pred_raw = str(answer1).strip() - rhs = self._extract_equation_rhs_raw(pred_raw) - if rhs is not None: - match = self._compare_formula_with_renormalized(gt_value, rhs) - if match is True: - return True - - return None - - # ------------------------------------------------------------------ - # Public interface - # ------------------------------------------------------------------ - - def compare( - self, - answer1: str | Answer, - answer2: str | Answer, - **kwargs: Any, - ) -> bool: - """ - True when either same-type comparison, RHS-extracted same-type - comparison, or cross-type matching resolves to True. - """ - _ = kwargs - outcome = run_smart_pipeline(self, answer1, answer2) - if outcome == "match": - return True - if outcome == "no_match": - return False - # Cross-type inconclusive: pure SmartMatch still returns False. - return False - - def accuracy_score( - self, - answer1: str | Answer, - answer2: str | Answer, - **kwargs: Any, - ) -> float: - """1.0 if :meth:`compare` is True, else 0.0.""" - is_match = self.compare(answer1, answer2, **kwargs) - return 1.0 if is_match else 0.0 diff --git a/src/prkit/evaluation/comparator/smart_pipeline.py b/src/prkit/evaluation/comparator/smart_pipeline.py deleted file mode 100644 index 87fa75b..0000000 --- a/src/prkit/evaluation/comparator/smart_pipeline.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -Shared deterministic pipeline for SmartMatch-style comparison. - -Same-type comparison, equation RHS extraction, equation-from-text rescue, then -:class:`~prkit.evaluation.comparator.smart_match.SmartMatchComparator` -cross-type matching. Used by :class:`~prkit.evaluation.comparator.smart_match.SmartMatchComparator` -and :class:`~prkit.evaluation.comparator.smart_llm.SmartLLMComparator`. -""" - -from __future__ import annotations - -import logging -from collections.abc import Mapping -from typing import Literal, Protocol - -from prkit.core.domain.answer import Answer, AnswerValue -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation.utils.answer_utils import same_comparison_category -from prkit.evaluation.utils.category_dispatch import ( - SameCategoryCompareFn, - compare_by_category, -) -from prkit.evaluation.utils.compare_cross_type import extract_rhs_and_category -from prkit.evaluation.utils.normalization import normalize_answer - -SmartPipelineResult = Literal["match", "no_match", "inconclusive"] - - -class SmartMatchPipelineHost(Protocol): - """Minimum interface required to run :func:`run_smart_pipeline`.""" - - @property - def _comparators(self) -> Mapping[AnswerCategory, SameCategoryCompareFn]: ... - - @property - def logger(self) -> logging.Logger: ... - - def _try_equation_from_text( - self, - text_raw: str, - gt_norm: AnswerValue, - gt_raw: str, - ) -> bool: ... - - def _cross_type_match( - self, - answer1: str | Answer, - answer2: str | Answer, - ) -> bool | None: ... - - -def run_smart_pipeline( - comparator: SmartMatchPipelineHost, - answer1: str | Answer, - answer2: str | Answer, -) -> SmartPipelineResult: - """Same-type, RHS, equation-from-text rescue, then cross-type. - - Returns ``match`` / ``no_match`` / ``inconclusive`` (cross-type unresolved). - - Implemented against :class:`~prkit.evaluation.comparator.smart_match.SmartMatchComparator` - internals (protected members); callers should pass that comparator or a compatible host. - """ - # pylint: disable=protected-access - if isinstance(answer1, Answer): - pred_norm: AnswerValue = answer1.value - pred_cat = answer1.answer_category - else: - pred_cat, pred_norm = normalize_answer(answer1) - if isinstance(answer2, Answer): - gt_norm: AnswerValue = answer2.value - gt_cat = answer2.answer_category - else: - gt_cat, gt_norm = normalize_answer(answer2) - - # 1. Same-type comparison - if same_comparison_category(gt_cat, pred_cat): - if compare_by_category( - gt_cat, pred_norm, gt_norm, comparator._comparators, comparator.logger - ): - return "match" - # Non-EQUATION same-type failures are final. - if gt_cat != AnswerCategory.EQUATION: - return "no_match" - # EQUATION same-type failure: the normalised pred may be garbage - # (e.g. preamble text contaminating the SymPy parse). Fall through - # to RHS extraction and equation-from-text rescue below. - - # 2. Extract RHS from equation-like answers, then retry same-type - pred_rhs_norm, pred_rhs_cat = extract_rhs_and_category(pred_norm, pred_cat) - gt_rhs_norm, gt_rhs_cat = extract_rhs_and_category(gt_norm, gt_cat) - if same_comparison_category(gt_rhs_cat, pred_rhs_cat): - comparator.logger.debug( - "Same RHS category - answer: %s and model answer: %s", - gt_rhs_norm, - pred_rhs_norm, - ) - if compare_by_category( - gt_rhs_cat, - pred_rhs_norm, - gt_rhs_norm, - comparator._comparators, - comparator.logger, - ): - return "match" - - # 2b. Same-type EQUATION rescue: extract embedded LaTeX equations - # from the raw prediction string and compare against GT. - if gt_cat == AnswerCategory.EQUATION and pred_cat == AnswerCategory.EQUATION: - pred_raw = str(answer1).strip() - gt_raw = str(answer2).strip() - if comparator._try_equation_from_text(pred_raw, gt_norm, gt_raw): - return "match" - - # 3. Cross-type matching - cross = comparator._cross_type_match(answer1, answer2) - if cross is True: - return "match" - if cross is False: - return "no_match" - return "inconclusive" diff --git a/src/prkit/evaluation/comparator/typed_llm.py b/src/prkit/evaluation/comparator/typed_llm.py deleted file mode 100644 index dca0bd0..0000000 --- a/src/prkit/evaluation/comparator/typed_llm.py +++ /dev/null @@ -1,458 +0,0 @@ -""" -Typed-path + LLM comparator using OpenAI Responses API. - -Quick typed match, LLM judge fallback, payload schema, and overrides. -Reusable judge primitives live in :mod:`prkit.evaluation.llm_judge`. -""" - -from __future__ import annotations - -import re -from typing import Any - -from openai import OpenAI -from sympy import Eq, sympify -from sympy.core.relational import Relational -from sympy.core.sympify import SympifyError - -from prkit.core import PRKitLogger -from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation.llm_judge import ( - DEFAULT_MODEL, - RESULT_SOURCE_SKIPPED_LLM, - RESULT_SOURCE_TYPED_MATCH, - LLMJudgeResult, - OpenAIJudgeRunner, - build_standard_answer_judge_payload, -) -from prkit.evaluation.llm_judge.payload import answer_to_text_and_category -from prkit.evaluation.utils.answer_utils import same_comparison_category -from prkit.evaluation.utils.category_dispatch import compare_by_category -from prkit.evaluation.utils.compare_same_type import ( - compare_formula, - compare_number, - compare_option, - compare_physical_quantity, - compare_plain_text, - parse_physical_quantity, -) -from prkit.evaluation.utils.normalization import normalize_answer - -from .base import BaseComparator - -_OPTION_ONLY_COMPARATORS = {AnswerCategory.OPTION: compare_option} - - -def _typed_category_and_value( - answer: str | Answer, -) -> tuple[AnswerCategory | None, float | str]: - """Return ``(category, normalized_value)`` for *answer*, or ``(None, raw_text)`` when normalization fails.""" - if isinstance(answer, Answer): - return answer.answer_category, str(answer.value) - try: - category, normalized = normalize_answer(answer) - return category, normalized - except (ValueError, TypeError, RuntimeError): - return None, str(answer).strip() - - -def _contains_latex_text_macro(s: str) -> bool: - """Return ``True`` when *s* contains a ``\\text{…}`` macro, indicating a prose formula.""" - return bool(re.search(r"\\text\s*\{", s)) - - -def infer_symbolic_answer_is_expression(question: str | None) -> bool | None: - """ - Heuristic: whether the question asks for a *symbolic expression* (scalar / - formula) vs a full *equation* as the graded object. - - When this returns ``True``, formula vs equation pairs may be compared by RHS - (see :meth:`TypedLLMComparator._quick_typed_match`). When ``False``, compare - full equations (no RHS-only shortcut). When ``None``, the comparator does not - apply that shortcut (falls back to LLM judge). - - Tuned using seephys-style wording (English + Chinese). Callers can override - with ``symbolic_answer_is_expression=`` in ``compare()``. - """ - if not question or not question.strip(): - return None - q_en = question.lower() - - if re.search(r"\bin terms of\b", q_en): - return True - if "的表达式" in question: - return True - if re.search( - r"(?:写出|试求|试计算|求|计算)[^。\n;；]{0,55}表达式", - question, - ): - return True - - _equation_false_en = ( - r"\bwrite (?:down )?the (?:time[- ])?wave equation\b", - r"\bwrite (?:down )?the wave equation\b", - r"\bwrite (?:down )?the equation\b", - r"\bderive (?:an |the )?equations?\s+describing\b", - r"\bderive (?:an |the )?equations?\s+of\s+motion\b", - r"\bderive (?:an |the )?equations?\b", - r"\bderive (?:an |the )?equation\s+for\s+the\s+distribution\b", - r"\bderive (?:an |the )?equation\s+for\s+the\s+(?:phase|intensity)\b", - r"\bderive an equation for\b", - r"\bequation-involving\b", - r"\bequations?\s+describing\b", - r"\bequations?\s+of\s+motion\b", - r"\bequation\s+describing\b", - r"\bequation\s+which\s+describes\b", - r"\bdifferential equation\b", - r"\bstate lagrange", - r"\blagrange'?s\s+equations?\b", - r"\bhamilton'?s\s+equations?\b", - r"\bwhat is the equation\b", - r"\bfind the equation\b", - r"\bstate the equation\b", - r"\bshow that the equation\b", - r"\bequation of motion\b", - r"\bthe equation (?:of|relating)\b", - r"\bequation for (?:the\s+)?(?:phase|distribution|field)\b", - ) - for pat in _equation_false_en: - if re.search(pat, q_en): - return False - - _equation_false_cn = ( - r"(?:试求|写出|试分别写出|试写出|推导)[^。\n;；]{0,160}" - r"(?:轨迹|轨道|迹线|包络线|谐振动|简谐振动|运动学|漏水|抛物线|彗星|椭圆|飞船|" - r"薛定谔|麦克斯韦|哈密顿|拉格朗日|微分|振动|运动)方程(?!的表达式)" - r"|(?:试求|写出|试分别写出|试写出|推导)[^。\n;；]{0,80}波动方程(?!的表达式)" - ) - if re.search(_equation_false_cn, question): - return False - - _expr_true_en = ( - r"\bwhat is\b", - r"\bwhat are\b", - r"\bhow much\b", - r"\bhow many\b", - r"\bhow long\b", - r"\bhow fast\b", - r"\bfind the\b", - r"\bcalculate\b", - r"\bdetermine\b", - r"\bobtain\b", - r"\bcompute\b", - r"\bgive (?:me )?the\b", - r"\bevaluate\b", - r"\bexpress\b", - r"\bmagnitude of\b", - r"\bvalue of\b", - r"\bspeed of\b", - r"\bvelocity of\b", - r"\bacceleration of\b", - r"\bangular velocity of\b", - r"\bangular frequency of\b", - r"\bforce on\b", - r"\bforce between\b", - r"\bfrequency of\b", - r"\bperiod of\b", - r"\bpotential (?:at|of)\b", - r"\bcurrent through\b", - r"\bcurrent in\b", - r"\bresistance of\b", - r"\bvoltage across\b", - r"\belectric field (?:at|of)\b", - r"\bmagnetic field (?:at|of)\b", - r"\bcharge (?:on|of)\b", - r"\bcapacitance of\b", - r"\binductance of\b", - r"\benergy (?:of|stored)\b", - r"\bpower (?:dissipated|delivered|of)\b", - r"\bmaximum\b.*\bof\b", - r"\bminimum\b.*\bof\b", - ) - for pat in _expr_true_en: - if re.search(pat, q_en): - return True - - if re.search( - r"(?:^|[\s,，、。；;])(?:试求|试计算|试导出|计算|求|问|多大|多少|何值)", - question, - ): - return True - if re.search(r"用[^，。；\n]{0,40}表示", question): - return True - - return None - - -def _normalized_equation_rhs_string(normalized_value: str) -> str | None: - """Parse *normalized_value* as a SymPy equation and return its RHS as a string, or ``None``.""" - try: - e = sympify(str(normalized_value)) - except (SympifyError, TypeError, ValueError, AttributeError): - return None - if isinstance(e, Eq) or ( - isinstance(e, Relational) and getattr(e, "rhs", None) is not None - ): - try: - return str(e.rhs) - except (ValueError, TypeError, AttributeError): - return None - return None - - -def _symbolic_operand_for_expression_compare( - category: AnswerCategory, - normalized_value: float | str, -) -> str | None: - """Extract the symbolic operand used in expression-level comparison from a normalized value. - - Returns the value as-is for FORMULA, the equation RHS for EQUATION, or ``None`` for other categories. - """ - if category == AnswerCategory.FORMULA: - return str(normalized_value) - if category == AnswerCategory.EQUATION: - return _normalized_equation_rhs_string(str(normalized_value)) - return None - - -def _compare_formula_or_equation_as_expressions( - pred_cat: AnswerCategory, - pred_value: float | str, - gt_cat: AnswerCategory, - gt_value: float | str, - pred_raw: str, - gt_raw: str, -) -> bool | None: - """Compare a FORMULA/EQUATION prediction and ground truth as symbolic expressions via ``compare_formula``.""" - if pred_cat not in (AnswerCategory.FORMULA, AnswerCategory.EQUATION): - return None - if gt_cat not in (AnswerCategory.FORMULA, AnswerCategory.EQUATION): - return None - if _contains_latex_text_macro(pred_raw) or _contains_latex_text_macro(gt_raw): - return None - pe = _symbolic_operand_for_expression_compare(pred_cat, pred_value) - ge = _symbolic_operand_for_expression_compare(gt_cat, gt_value) - if pe is None or ge is None: - return None - try: - return compare_formula(pe, ge) - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - return None - - -def _plain_text_true_else_llm( - predicted_norm: float | str, - ground_truth_norm: float | str, -) -> bool | None: - try: - return True if compare_plain_text(predicted_norm, ground_truth_norm) else None - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - return None - - -def _compare_physical_quantity_same_unit_pool_placeholder( - pred_num_str: str, - pred_unit: str, - gt_num_str: str, - gt_unit: str, -) -> tuple[bool, bool]: - _ = (pred_num_str, pred_unit, gt_num_str, gt_unit) - return False, False - - -class TypedLLMComparator(BaseComparator): - """OpenAI Responses API comparator with question-aware judging.""" - - def __init__( - self, - model: str = DEFAULT_MODEL, - *, - instructions: str | None = None, - client: OpenAI | None = None, - ) -> None: - super().__init__() - self.logger = PRKitLogger.get_logger(__name__) - self._runner = OpenAIJudgeRunner( - model=model, - instructions=instructions, - client=client, - logger=self.logger, - ) - self._last_result: LLMJudgeResult | None = None - - def _judge(self, payload: dict[str, Any]) -> LLMJudgeResult: - return self._runner.judge(payload) - - @staticmethod - def _quick_typed_match( - predicted: str | Answer, - ground_truth: str | Answer, - *, - question: str | None = None, - symbolic_answer_is_expression: bool | None = None, - ) -> bool | None: - pred_raw, _ = answer_to_text_and_category(predicted) - gt_raw, _ = answer_to_text_and_category(ground_truth) - pred_cat, pred_value = _typed_category_and_value(predicted) - gt_cat, gt_value = _typed_category_and_value(ground_truth) - if pred_cat is None or gt_cat is None: - return None - - if same_comparison_category(gt_cat, pred_cat): - if pred_cat == AnswerCategory.OPTION: - return compare_by_category( - pred_cat, - pred_value, - gt_value, - _OPTION_ONLY_COMPARATORS, - None, - ) - - if pred_cat == AnswerCategory.NUMBER: - try: - return compare_number(pred_value, gt_value) - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - return None - - if pred_cat == AnswerCategory.PHYSICAL_QUANTITY: - try: - _pred_num, pred_unit, pred_num_str = parse_physical_quantity( - str(pred_value) - ) - _gt_num, gt_unit, gt_num_str = parse_physical_quantity( - str(gt_value) - ) - if pred_unit == gt_unit: - return compare_physical_quantity(pred_value, gt_value) - is_same_unit_pool, is_equal_value = ( - _compare_physical_quantity_same_unit_pool_placeholder( - pred_num_str, pred_unit, gt_num_str, gt_unit - ) - ) - if is_same_unit_pool and is_equal_value: - return True - return _plain_text_true_else_llm(str(pred_value), str(gt_value)) - - except ( - ValueError, - TypeError, - ZeroDivisionError, - RuntimeError, - AttributeError, - ): - return None - - if pred_cat == AnswerCategory.FORMULA: - if _contains_latex_text_macro(pred_raw) or _contains_latex_text_macro( - gt_raw - ): - return _plain_text_true_else_llm(pred_value, gt_value) - try: - if compare_formula(pred_value, gt_value): - return True - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - pass - return _plain_text_true_else_llm(pred_value, gt_value) - - if pred_cat == AnswerCategory.EQUATION: - pred_rhs = _normalized_equation_rhs_string(str(pred_value)) - gt_rhs = _normalized_equation_rhs_string(str(gt_value)) - if _contains_latex_text_macro(pred_raw) or _contains_latex_text_macro( - gt_raw - ): - return _plain_text_true_else_llm(pred_value, gt_value) - if pred_rhs is not None and gt_rhs is not None: - try: - if compare_formula(pred_rhs, gt_rhs): - return True - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - pass - return _plain_text_true_else_llm(pred_value, gt_value) - - if pred_cat == AnswerCategory.TEXT: - return _plain_text_true_else_llm(pred_value, gt_value) - - return None - - formula_equation_pair = pred_cat in ( - AnswerCategory.FORMULA, - AnswerCategory.EQUATION, - ) and gt_cat in (AnswerCategory.FORMULA, AnswerCategory.EQUATION) - if formula_equation_pair: - resolved_expr: bool | None = symbolic_answer_is_expression - if resolved_expr is None and question: - resolved_expr = infer_symbolic_answer_is_expression(question) - if resolved_expr is True: - qm = _compare_formula_or_equation_as_expressions( - pred_cat, pred_value, gt_cat, gt_value, pred_raw, gt_raw - ) - if qm is True: - return True - return _plain_text_true_else_llm(pred_value, gt_value) - - return None - - def compare( - self, - answer1: str | Answer, - answer2: str | Answer, - *, - skip_llm: bool = False, - **kwargs: Any, - ) -> bool: - quick = self._quick_typed_match( - answer1, - answer2, - question=kwargs.get("question"), - symbolic_answer_is_expression=kwargs.get("symbolic_answer_is_expression"), - ) - if quick is not None: - self._last_result = LLMJudgeResult( - verdict="correct" if quick else "incorrect", - confidence=1.0, - expected_answer_type="other", - reasoning="Quick typed match path.", - raw_response="local_shortcut", - verdict_type=RESULT_SOURCE_TYPED_MATCH, - ) - return quick - - if skip_llm: - self._last_result = LLMJudgeResult( - verdict="incorrect", - confidence=0.0, - expected_answer_type="other", - reasoning="LLM judge skipped (skip_llm=True); no API call was made.", - raw_response="skipped_llm", - verdict_type=RESULT_SOURCE_SKIPPED_LLM, - ) - return False - - payload = build_standard_answer_judge_payload( - answer1, - answer2, - kwargs.get("question"), - ) - result = self._judge(payload) - self._last_result = result - return result.verdict == "correct" - - def accuracy_score( - self, - answer1: str | Answer, - answer2: str | Answer, - *, - skip_llm: bool = False, - **kwargs: Any, - ) -> float: - is_correct = self.compare(answer1, answer2, skip_llm=skip_llm, **kwargs) - return 1.0 if is_correct else 0.0 - - @property - def last_result(self) -> LLMJudgeResult | None: - return self._last_result - - @property - def model_name(self) -> str: - return self._runner.model_name diff --git a/src/prkit/evaluation/evaluator/__init__.py b/src/prkit/evaluation/evaluator/__init__.py deleted file mode 100644 index 903d2dd..0000000 --- a/src/prkit/evaluation/evaluator/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Evaluator module for physical reasoning evaluation. - -This module provides evaluators that use different comparators to evaluate -answers in physical reasoning tasks. -""" - -from .accuracy import AccuracyEvaluator -from .base import BaseEvaluator - -__all__ = [ - "BaseEvaluator", - "AccuracyEvaluator", -] diff --git a/src/prkit/evaluation/evaluator/accuracy.py b/src/prkit/evaluation/evaluator/accuracy.py deleted file mode 100644 index 388375f..0000000 --- a/src/prkit/evaluation/evaluator/accuracy.py +++ /dev/null @@ -1,300 +0,0 @@ -"""Accuracy evaluator backed by the canonical ``Scorer`` contract (or a legacy comparator). - -By default the evaluator scores answers through :class:`~prkit.scoring.SemanticsScorer` -— the :class:`prkit.api.Scorer` / :class:`~prkit.core.verdict.Verdict` contract — and -shapes its long-standing result dict from the returned ``Verdict``. A different -``Scorer`` can be injected (e.g. a future semantics+LLM scorer); passing a legacy -``comparator`` instead selects the deprecated comparator path unchanged. -""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import TYPE_CHECKING, Any - -from prkit.core.domain.answer import Answer -from prkit.core.domain.physics_dataset import PhysicalDataset -from prkit.core.domain.physics_problem import PhysicsProblem -from prkit.evaluation.comparator.base import BaseComparator -from prkit.scoring import SemanticsScorer - -from .base import BaseEvaluator - -if TYPE_CHECKING: # typing-only; avoids importing the api surface at runtime - from prkit.api import Scorer - - -class AccuracyEvaluator(BaseEvaluator): - """Evaluator that scores answers via an injectable ``Scorer`` (or legacy comparator). - - The default (no ``comparator``, no ``scorer``) is backed by - :class:`~prkit.scoring.SemanticsScorer`, so results conform to the canonical - ``Verdict`` contract. Only the ``SemanticsScorer`` path is exercised today; the - seam is intentionally open for other ``Scorer`` implementations later. - """ - - def __init__( - self, - comparator: BaseComparator | None = None, - *, - scorer: Scorer | None = None, - ) -> None: - """Initialize the accuracy evaluator. - - Args: - comparator: Legacy comparator (the deprecated path). Mutually exclusive - with ``scorer``. - scorer: ``Scorer`` to back evaluation. When neither ``comparator`` nor - ``scorer`` is given, defaults to :class:`~prkit.scoring.SemanticsScorer`. - """ - if comparator is not None and scorer is not None: - raise ValueError("Pass either scorer= or comparator=, not both.") - if comparator is None and scorer is None: - scorer = SemanticsScorer() - # BaseEvaluator stores the (possibly None) comparator and emits the stack's - # DeprecationWarning; the dataset-level harness is slated for the N4 Runner. - super().__init__(comparator) - self.scorer = scorer - - @staticmethod - def _describe_answer(answer: str | Answer) -> tuple[str, str]: - """Return ``(value, type)`` surface strings for a result's ``details`` block.""" - if isinstance(answer, Answer): - return str(answer.value), answer.answer_category.value - return str(answer), "string" - - def evaluate( - self, - predicted_answer: str | Answer, - ground_truth_answer: str | Answer, - **kwargs: Any, - ) -> dict[str, Any]: - """ - Evaluate a predicted answer against a ground truth answer. - - Args: - predicted_answer: The predicted/student answer (string or Answer) - ground_truth_answer: The ground truth/correct answer (string or Answer) - **kwargs: Additional arguments (currently unused) - - Returns: - Dictionary containing evaluation results: - - accuracy_score: Accuracy score in [0, 1] - - comparison_result: Raw pass/fail comparison result - - details: Additional evaluation details - """ - pred_val, pred_type = self._describe_answer(predicted_answer) - gt_val, gt_type = self._describe_answer(ground_truth_answer) - - if self.scorer is not None: - # Scorer contract: score(prediction, reference) -> Verdict, then shape - # the legacy dict from it (accuracy_score=score, comparison=equivalent). - verdict = self.scorer.score(predicted_answer, ground_truth_answer) - return { - "accuracy_score": verdict.score, - "comparison_result": verdict.equivalent, - "details": { - "predicted_value": pred_val, - "ground_truth_value": gt_val, - "predicted_type": pred_type, - "ground_truth_type": gt_type, - "scorer_type": type(self.scorer).__name__, - "scorer_version": verdict.scorer_version, - "comparison_mode": verdict.comparison_mode, - }, - } - - # Legacy comparator path (deprecated) — unchanged behavior. - if self.comparator is None: - raise ValueError("Comparator must be set before evaluation") - - comparison_result = self.comparator.compare( - predicted_answer, ground_truth_answer - ) - accuracy_score = self.comparator.accuracy_score( - predicted_answer, ground_truth_answer - ) - - return { - "accuracy_score": accuracy_score, - "comparison_result": comparison_result, - "details": { - "predicted_value": pred_val, - "ground_truth_value": gt_val, - "predicted_type": pred_type, - "ground_truth_type": gt_type, - "comparator_type": type(self.comparator).__name__, - }, - } - - def evaluate_dataset( - self, - dataset: PhysicalDataset, - predicted_answers: dict[str, Answer] | None = None, - answer_extractor: Callable[[PhysicsProblem], Answer] | None = None, - **kwargs: Any, - ) -> dict[str, Any]: - """ - Evaluate a dataset and return dataset-level statistics. - - The method can work in two modes: - 1. If `predicted_answers` is provided: uses a dictionary mapping problem_id to Answer - 2. If `answer_extractor` is provided: extracts predicted answers from each problem - - Args: - dataset: PhysicalDataset to evaluate - predicted_answers: Optional dictionary mapping problem_id to predicted Answer - answer_extractor: Optional function that takes a PhysicsProblem and returns Answer - **kwargs: Additional arguments passed to individual evaluations - - Returns: - Dictionary containing dataset-level statistics: - - overall_accuracy: Average accuracy score across all problems - - total_problems: Total number of problems evaluated - - evaluated_problems: Number of problems successfully evaluated - - failed_problems: Number of problems that failed evaluation - - per_problem_results: List of individual evaluation results - - statistics: Additional statistics (by domain, problem_type, etc.) - """ - if self.scorer is None and self.comparator is None: - raise ValueError("A scorer or comparator must be set before evaluation") - - if predicted_answers is None and answer_extractor is None: - raise ValueError( - "Either predicted_answers or answer_extractor must be provided" - ) - - per_problem_results: list[dict[str, Any]] = [] - total_problems = len(dataset) - evaluated_problems = 0 - failed_problems = 0 - accuracy_scores: list[float] = [] - - # Statistics by domain and problem type - domain_stats: dict[str, list[float]] = {} - problem_type_stats: dict[str, list[float]] = {} - - for problem in dataset: - problem_id = problem.problem_id - ground_truth_answer = problem.answer - - # Skip if no ground truth answer - if ground_truth_answer is None: - failed_problems += 1 - per_problem_results.append( - { - "problem_id": problem_id, - "accuracy_score": 0.0, - "status": "no_ground_truth", - "details": {"error": "No ground truth answer available"}, - } - ) - continue - - # Get predicted answer - try: - if predicted_answers is not None: - if problem_id not in predicted_answers: - failed_problems += 1 - per_problem_results.append( - { - "problem_id": problem_id, - "accuracy_score": 0.0, - "status": "missing_prediction", - "details": { - "error": f"No predicted answer for problem_id: {problem_id}" - }, - } - ) - continue - predicted_answer = predicted_answers[problem_id] - else: - # Use answer_extractor - assert answer_extractor is not None - predicted_answer = answer_extractor(problem) - if predicted_answer is None: - failed_problems += 1 - per_problem_results.append( - { - "problem_id": problem_id, - "accuracy_score": 0.0, - "status": "extraction_failed", - "details": {"error": "Answer extractor returned None"}, - } - ) - continue - - # Evaluate the answer - eval_result = self.evaluate( - predicted_answer, ground_truth_answer, **kwargs - ) - accuracy_score = eval_result["accuracy_score"] - accuracy_scores.append(accuracy_score) - evaluated_problems += 1 - - # Add problem metadata to result - result = { - "problem_id": problem_id, - "accuracy_score": accuracy_score, - "status": "success", - **eval_result, - } - per_problem_results.append(result) - - # Update domain statistics - domain = problem.get_domain_name() - if domain not in domain_stats: - domain_stats[domain] = [] - domain_stats[domain].append(accuracy_score) - - # Update problem type statistics - problem_type = problem.problem_type or "unknown" - if problem_type not in problem_type_stats: - problem_type_stats[problem_type] = [] - problem_type_stats[problem_type].append(accuracy_score) - - except Exception as e: - failed_problems += 1 - per_problem_results.append( - { - "problem_id": problem_id, - "accuracy_score": 0.0, - "status": "error", - "details": {"error": str(e)}, - } - ) - - # Calculate overall accuracy - overall_accuracy = ( - sum(accuracy_scores) / len(accuracy_scores) if accuracy_scores else 0.0 - ) - - # Calculate domain-level statistics - domain_accuracy = { - domain: sum(scores) / len(scores) if scores else 0.0 - for domain, scores in domain_stats.items() - } - - # Calculate problem type statistics - problem_type_accuracy = { - ptype: sum(scores) / len(scores) if scores else 0.0 - for ptype, scores in problem_type_stats.items() - } - - return { - "overall_accuracy": overall_accuracy, - "total_problems": total_problems, - "evaluated_problems": evaluated_problems, - "failed_problems": failed_problems, - "per_problem_results": per_problem_results, - "statistics": { - "by_domain": domain_accuracy, - "by_problem_type": problem_type_accuracy, - "domain_counts": { - domain: len(scores) for domain, scores in domain_stats.items() - }, - "problem_type_counts": { - ptype: len(scores) for ptype, scores in problem_type_stats.items() - }, - }, - } diff --git a/src/prkit/evaluation/evaluator/base.py b/src/prkit/evaluation/evaluator/base.py deleted file mode 100644 index 590be1e..0000000 --- a/src/prkit/evaluation/evaluator/base.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Abstract base evaluator for physical reasoning evaluation, delegating comparisons to a :class:`BaseComparator`. - -.. deprecated:: - The evaluator stack is superseded by :class:`prkit.scoring.SemanticsScorer` - (the :class:`prkit.api.Scorer` / :class:`prkit.api.Verdict` contract). - Constructing an evaluator emits a ``DeprecationWarning``; this stack will be - removed in a future release. See ``prkit/CONTRACT.md``. -""" - -import warnings -from abc import ABC, abstractmethod -from typing import Any - -from prkit.core.domain.answer import Answer -from prkit.evaluation.comparator.base import DEPRECATION_HINT, BaseComparator - - -class BaseEvaluator(ABC): - """Base class for evaluators that use comparators. - - .. deprecated:: superseded by :class:`prkit.scoring.SemanticsScorer`. - """ - - def __init__(self, comparator: BaseComparator | None = None): - """ - Initialize the evaluator with a comparator. - - Args: - comparator: Comparator instance to use for comparing answers. - If None, a default comparator will be used. - """ - warnings.warn( - f"{type(self).__name__} is deprecated and will be removed in a future " - f"release; {DEPRECATION_HINT}.", - DeprecationWarning, - stacklevel=2, - ) - self.comparator = comparator - - @abstractmethod - def evaluate( - self, predicted_answer: Answer, ground_truth_answer: Answer, **kwargs: Any - ) -> dict[str, Any]: - """ - Evaluate a predicted answer against a ground truth answer. - - Args: - predicted_answer: The predicted/student answer to evaluate - ground_truth_answer: The ground truth/correct answer - **kwargs: Additional arguments for evaluation - - Returns: - Dictionary containing evaluation results with keys such as: - - accuracy_score: Accuracy score in [0, 1] - - comparison_result: Raw comparison result from comparator - - details: Additional evaluation details - """ - pass - - def set_comparator(self, comparator: BaseComparator) -> None: - """ - Set or change the comparator used by this evaluator. - - Args: - comparator: Comparator instance to use - """ - self.comparator = comparator - - def get_comparator(self) -> BaseComparator | None: - """ - Get the current comparator used by this evaluator. - - Returns: - The current comparator instance, or None if not set - """ - return self.comparator diff --git a/src/prkit/evaluation/llm_judge/payload.py b/src/prkit/evaluation/llm_judge/payload.py index 0d25799..5ffac3a 100644 --- a/src/prkit/evaluation/llm_judge/payload.py +++ b/src/prkit/evaluation/llm_judge/payload.py @@ -9,9 +9,9 @@ def answer_to_text_and_category(answer: str | Answer) -> tuple[str, str]: - """Plain text and category label for embedding in a judge JSON payload.""" + """Plain text and answer-kind label for embedding in a judge JSON payload.""" if isinstance(answer, Answer): - return str(answer).strip(), answer.answer_category.value + return str(answer).strip(), answer.answer_kind.value return str(answer).strip(), "unknown" diff --git a/src/prkit/evaluation/similarities/__init__.py b/src/prkit/evaluation/similarities/__init__.py deleted file mode 100644 index db5fa09..0000000 --- a/src/prkit/evaluation/similarities/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Text similarity helpers for evaluation (e.g. cross-typed matching).""" - -from .rouge_l import rouge_l_f1 - -__all__ = ["rouge_l_f1"] diff --git a/src/prkit/evaluation/similarities/rouge_l.py b/src/prkit/evaluation/similarities/rouge_l.py deleted file mode 100644 index 3247dec..0000000 --- a/src/prkit/evaluation/similarities/rouge_l.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Word-level ROUGE-L (F1) using longest common subsequence length. - -Reference: Lin, ROUGE (2004). Suitable for comparing predicted vs reference text -without extra dependencies. -""" - -from __future__ import annotations - -import re - - -def _tokenize(text: str) -> list[str]: - """Lowercase and whitespace-split *text* into word tokens.""" - s = text.strip().lower() - s = re.sub(r"\s+", " ", s) - if not s: - return [] - return s.split() - - -def _lcs_length(a: list[str], b: list[str]) -> int: - """Length of longest common subsequence (dynamic programming).""" - if not a or not b: - return 0 - n, m = len(a), len(b) - # Two rows to limit memory - prev = [0] * (m + 1) - curr = [0] * (m + 1) - for i in range(1, n + 1): - for j in range(1, m + 1): - if a[i - 1] == b[j - 1]: - curr[j] = prev[j - 1] + 1 - else: - curr[j] = max(prev[j], curr[j - 1]) - prev, curr = curr, prev - return prev[m] - - -def rouge_l_f1(candidate: str, reference: str) -> float: - """ - ROUGE-L F1 between candidate and reference (word-level). - - Returns a value in [0, 1]. Empty candidate or reference yields 0.0. - """ - c = _tokenize(candidate) - r = _tokenize(reference) - if not c or not r: - return 0.0 - lcs = _lcs_length(c, r) - if lcs == 0: - return 0.0 - p = lcs / len(c) - rec = lcs / len(r) - if p + rec == 0: - return 0.0 - return 2.0 * p * rec / (p + rec) diff --git a/src/prkit/evaluation/utils/NORMALIZATION.md b/src/prkit/evaluation/utils/NORMALIZATION.md deleted file mode 100644 index 9b93ea5..0000000 --- a/src/prkit/evaluation/utils/NORMALIZATION.md +++ /dev/null @@ -1,407 +0,0 @@ -# Answer Normalization & Comparison Workflow - -This document describes the full pipeline from raw answer string to final -match verdict, covering both **normalization** (`normalize_answer`) and -**comparison** (`compare_formula`, `compare_physical_quantity`, etc.). - -Source files: - -- `normalization.py` — normalization (classification + canonical form) -- `compare_same_type.py` — same-type comparison functions -- `compare_cross_type.py` — cross-type helpers (used by SmartMatch) - ---- - -## 1 Normalization: `normalize_answer(answer_str)` - -Returns `(AnswerCategory, normalized_value)`. - -- **category**: `NUMBER` | `EQUATION` | `PHYSICAL_QUANTITY` | `FORMULA` | `TEXT` -- **normalized_value**: `float` for NUMBER, canonical string for all others - -### Main Flow - -Steps are ordered; first successful path wins. - -``` - answer_str - │ - ▼ -┌──────────────────────────────────────────────────────────────────────────────┐ -│ STEP 1: normalize_number(answer_str) │ -│ - strips LaTeX wrappers │ -│ - parses numeric forms │ -│ Returns: float or NaN │ -└──────────────────────────┬───────────────────────────────────────────────────┘ - │ - not NaN │ NaN - │ - ▼ - RETURN ("number", float) - │ - ▼ -┌──────────────────────────────────────────────────────────────────────────────┐ -│ STEP 2: _starts_with_latex_delimiter(answer_str)? │ -└──────────────────────────┬───────────────────────────────────────────────────┘ - │ - NO │ YES - │ - ▼ -┌──────────────────────────────────┐ ┌──────────────────────────────────────┐ -│ STEP 2A: plain-string branch │ │ STEP 2B: LaTeX-expression branch │ -│ - clean = _extract_math_content │ │ - clean = _extract_math_content │ -│ - classify_expression(clean) │ │ - classify_expression(clean) │ -└───────────────┬─────────────────┘ └───────────────┬──────────────────────┘ - │ │ - physical_quantity? │ - YES / NO │ - │ │ - YES -> RETURN ("physical_quantity", │ - _normalize_physical_quantity(clean)) │ - NO -> unicode-math-to-LaTeX conversion, │ - if LaTeX commands found -> expression path│ - else -> RETURN ("text", normalize_text) │ - │ - ┌─────────┴─────────┐ - │ category result │ - │ equation / pq / │ - │ formula │ - └─────────┬─────────┘ - │ - ┌────────────────────────────┼────────────────────────────┐ - │ │ │ - ▼ ▼ ▼ - equation -> symbolic physical_quantity -> pq normalize formula - return ("equation", norm) return ("physical_quantity", norm) │ - STEP 2B-R - formula rescue: - 1) retry as number - 2) retry as physical quantity - 3) else symbolic formula - return formula/equation fallback -``` - ---- - -### Parsing Priorities - -#### Step 1: Number-first - -`normalize_number(answer_str)` tries numeric-only parse before any category routing. - -Supported numeric forms: - -- plain: `12`, `-3.5`, `.5`, `1.` -- scientific `e`: `1.4e-4`, `2E+3` -- scientific `*10^`: `1*10^3`, `-2.5*10^-4` -- fractions: `3/4`, `1e-3/2` -- LaTeX fraction: `\frac{2}{3}` - -Success → immediate `("number", float)`. - -#### Step 2A: Non-LaTeX-start inputs - -If input does not start with math delimiters: - -1. Clean via `_extract_math_content` and classify. -2. If classified as `physical_quantity` → PQ normalization. -3. If classified as `equation` AND `_looks_like_math_expression()` → symbolic - normalization (with `_contains_latex_commands` detection for SymPy parsing - of bare LaTeX like `z = \frac{...}`) → return equation. - The `_looks_like_math_expression()` guard rejects prose containing `=` - (e.g. "the answer is x = 5") by scanning for common English function words. -4. Otherwise, convert Unicode math symbols (Greek letters, `√`, superscripts) - to LaTeX commands. If the result contains LaTeX → attempt expression - normalization. If that succeeds → return the parsed category. -5. Else → text normalization. - -#### Step 2B: LaTeX-start inputs - -If input starts with math delimiters (`$`, `$$`, `\(`, `\[`, `\boxed{`, `\frac{`, etc.): - -- Extract math content. -- Classify to equation / physical quantity / formula. -- Equation and formula go through symbolic normalization (latex2sympy). -- Physical quantities go through quantity normalization. - ---- - -### Rescue - -When initial classification is `"formula"`, run a second-pass rescue before finalizing: - -1. **Number rescue** — canonicalize numeric text, retry numeric parse, must be - pure numeric (no unit suffix). If success → `("number", value)`. -2. **Physical quantity rescue** — canonicalize for unit-aware parsing, retry - split into numeric part + unit part. If success → - `("physical_quantity", normalized_string)`. -3. **Otherwise keep formula** — symbolic normalization output remains. - ---- - -### Canonicalization Rules for Number Path - -- Unicode whitespace normalization and trim -- Unicode minus normalization (`−`, `–`, `—` → `-`) -- Exponent brace flattening (`10^{ -5 }` → `10^-5`) -- Superscript exponent normalization (`10⁻⁵` → `10^-5`) -- Scientific multiplication marker normalization (`1 × 10^3`, `1·10^3`, `1 x 10^3` → `1*10^3`) -- Comma removal in numeric text when used as separators (`1,000` → `1000`) -- LaTeX wrapper removal (`$...$`, `\(...\)`, `\[...\]`, `\boxed{...}`, `\text{...}`, `\mathrm{...}`) -- LaTeX numeric fraction normalization for simple forms (`\frac{a}{b}` where both sides are numeric) - -### Canonicalization Rules for Physical Quantity Path - -- Unicode whitespace/minus normalization -- Superscript exponent normalization (`m/s²` → `m/s^2`) -- Scientific notations (`e`, `*10^`, LaTeX multiplication) -- Unit alias normalization (`meter` → `m`, `ohm` → `Ω`, `°` → `deg`) -- Unit scaling to canonical base units (e.g., `cm` → `m`, `g` → `kg`) -- Combined unit expression normalization (`g/cm^3` → `kg/m^3` with scaled value) - -### LaTeX Spacing Commands - -LaTeX spacing commands (`\,`, `\;`, `\:`, `\!`, and backslash-space `\ `) are -replaced with a regular space during `_extract_math_content` so that adjacent -tokens are not accidentally merged (e.g., `\mathrm{rad}\,\mathrm{s}^{-1}` -becomes `rad s^{-1}`, not `rads^{-1}`). - -### `_normalize_unicode` (early pass) - -Runs at the start of `_extract_math_content`. Text punctuation (quotes, -fullwidth digits/letters) is normalized to ASCII. **Math symbols** are mapped to -LaTeX commands so SymPy / `latex2sympy` see valid math tokens. Highlights: - -| Unicode | Meaning | Replacement | -|---------|---------|----------------| -| `−` `–` `—` … | minus / dashes | `-` | -| `×` · `⋅` `∙` | multiply | ` \times ` / ` \cdot ` | -| `÷` | divide | ` \div ` (`_canonicalize_quantity_string` maps `\div` → `/`) | -| `≤` `≥` `≠` | inequalities | ` \leq ` ` \geq ` ` \neq ` | -| `≈` | approx. | `\approx ` (leading space only when paired with following token) | -| `∝` | proportional to | ` \propto ` | -| `∞` | infinity | `\infty` | -| `±` | plus-minus | ` \pm ` | -| `°` | degree | ` deg` (canonical **unit** token for quantity parsing, not `^\circ`) | - -**Classification:** Inequalities and `\propto` without `=` still count as -**equation** via `_LATEX_BINARY_RELATION_MARKERS` in `classify_expression`. -`\approx` is **not** in that list so strings like `\approx 355\,\mathrm{K}` can -still be improved toward `PHYSICAL_QUANTITY` in a later pass. - ---- - -## 2 Comparison: Formula Equivalence Cascade - -`compare_formula()` in `compare_same_type.py` uses a four-stage cascade. -The first stage that returns `True` wins; the first that returns a definitive -`False` short-circuits. - -``` - pred_sym, gt_sym = sympify(...) - │ - ▼ - ┌───────────────────────────────┐ - │ Stage 1: SymPy equals() │ Quick random-point numerical check - │ pred_sym.equals(gt_sym) │ (SymPy built-in, ~5 random points) - └───────────┬───────────────────┘ - True │ exception/False - ▼ - ┌───────────────────────────────┐ - │ Stage 2: Multi-strategy │ Try five simplification strategies - │ symbolic simplification │ on pred_sym and gt_sym: - │ │ - │ a) simplify(pred - gt) == 0 │ General-purpose simplification - │ b) expand(pred) == expand(gt)│ Distribute / collect terms - │ c) factor(pred) == factor(gt)│ Polynomial factorization - │ d) trigsimp(pred) == │ Trig identities (sin²+cos²=1 etc.) - │ trigsimp(gt) │ - │ e) cancel(pred) == cancel(gt)│ Rational function simplification - └───────────┬───────────────────┘ - True │ all False - ▼ - ┌───────────────────────────────┐ - │ Stage 3: Numerical │ Evaluate (pred - gt) at 20 random - │ equivalence │ points. If all within tolerance - │ │ (1e-6) → True. If any large - │ │ deviation → False. If inconclusive - │ │ (too many errors) → fall through. - └───────────┬───────────────────┘ - True/False│ None (inconclusive) - ▼ - ┌───────────────────────────────┐ - │ Stage 4: Normalized text │ Exact string match after - │ fallback │ whitespace/LaTeX cleanup - └───────────────────────────────┘ -``` - -### Stage details - -**Stage 1 — `equals()`**: SymPy's built-in method. Evaluates both expressions at -a small number of random points and checks numerical closeness. Fast but can -give false negatives for complex expressions or expressions with branch cuts. - -**Stage 2 — Multi-strategy symbolic**: Applies five algebraic transformations. -Each is independent and catches different equivalence classes: - -| Strategy | What it catches | -|------------|--------------------------------------------------------| -| `simplify` | General identities, constant folding | -| `expand` | Distributed vs. factored polynomials | -| `factor` | Factored vs. expanded polynomials | -| `trigsimp` | Pythagorean identities, double-angle formulas | -| `cancel` | Rational expressions like `(x²-1)/(x-1)` vs. `x+1` | - -**Stage 3 — Numerical equivalence**: Evaluates `pred - gt` at 20 random points -(each variable sampled from ±[0.5, 5.0] with a fixed seed for determinism). -Returns `True` only if at least half the trials succeed and all show -`|diff| < 1e-6`. Returns `False` on any trial with a large deviation. Returns -`None` (inconclusive) if too many trials error out, allowing the text fallback. - -**Stage 4 — Normalized text fallback**: Strips LaTeX styling commands -(`\left`, `\right`, `\displaystyle`, spacing commands), normalizes `\dfrac` → -`\frac`, collapses whitespace, then checks exact string equality. - ---- - -## 3 Comparison: Other Types - -| Function | Type | Logic | -|-----------------------------|---------------|-----------------------------------------------------------------------| -| `compare_number` | NUMBER | Precision-aware: round pred to GT's decimal places, check `|diff| < ε` | -| `compare_physical_quantity` | PHYSICAL_QUANTITY | Parse value+unit, if units match → `compare_number`, else text fallback | -| `compare_plain_text` | TEXT | Exact match, or GT is a substring of pred | -| `compare_formula` | FORMULA / EQUATION | Four-stage cascade described above | - ---- - -## 4 Category Definitions - -| Category | Meaning | Normalized Value Type | -|---------------------|-----------------------------------------------------|--------------------------------------| -| `NUMBER` | Numeric-only answer | `float` | -| `PHYSICAL_QUANTITY` | Numeric value with units | canonical string (`"{num} {unit}"`) | -| `EQUATION` | Expression with single `=` | symbolic/text string | -| `FORMULA` | Symbolic expression not rescued as number/quantity | symbolic/text string | -| `TEXT` | Prose / non-math answer | stripped/collapsed string | - ---- - -## 5 Example Outcomes - -| Input | Final Category | Notes | -|------------------------------|----------------------|---------------------------------------| -| `"500"` | `NUMBER` | Step 1 direct number | -| `"1.4e-4 A/s"` | `PHYSICAL_QUANTITY` | Non-LaTeX quantity path | -| `"\[3.14 \\mathrm{A/s}\]"` | `PHYSICAL_QUANTITY` | LaTeX path + quantity classify | -| `"$$1.0 \\times 10^{-5}$$"` | `NUMBER` (via rescue)| Formula rescue re-parses numeric form | -| `"$$x^2 + 1$$"` | `FORMULA` | Remains symbolic formula | -| `"$F = ma$"` | `EQUATION` | Equation path | -| `"from $B$ to $A$"` | `TEXT` | Non-LaTeX-start prose | - ---- - -## 6 Future Improvements - -### 6.1 Canonical form pre-normalization - -Before comparing, convert both expressions to a canonical algebraic form to -reduce the surface area for equivalence checking: - -```python -from sympy import expand_trig, powsimp, radsimp - -def canonicalize(expr): - expr = expand(expr) - expr = powsimp(expr) # consolidate power terms - expr = expand_trig(expr) # decompose to sin/cos basis - expr = radsimp(expr) # simplify radical expressions - return expr -``` - -This can be applied as a pre-pass before the multi-strategy cascade so that -each strategy starts from a more uniform representation. Particularly useful -for nested trigonometric and radical expressions. - -### 6.2 External CAS backends - -For the hardest symbolic equivalences where SymPy's heuristics fail, a -second CAS engine can serve as an oracle: - -- **SageMath** (Python, wraps Maxima + Singular + PARI): - `sage.symbolic.expression.Expression.is_zero()` uses Maxima's simplifier - which handles some identities SymPy cannot. Available as a local install. -- **Wolfram Alpha API** (cloud, free tier available): send `simplify(pred - gt)` - and check if the result is `0`. Best symbolic simplifier available, but - requires network access and has rate limits. -- **Mathematica** via `wolframclient` Python package: local alternative to the - API for those with a Mathematica license. No rate limits. - -Recommended integration pattern: use an external CAS only as a **last-resort -fallback** after all SymPy strategies and numerical checks fail. This keeps -latency low for the 95%+ of cases that SymPy handles natively. - -### 6.3 LLM-assisted formula comparison - -For the irreducible tail of cases where no CAS can confirm equivalence -(domain-specific notation, physics-convention equivalences, non-standard -representations), an LLM call can serve as the final arbiter: - -- Prompt: *"Are the following two mathematical expressions equivalent? - Expression A: `...` Expression B: `...` Answer only YES or NO."* -- Use a fast, cheap model (e.g., GPT-4.1-mini) with temperature 0. -- Gate behind a confidence threshold: only accept if the LLM is confident. - -This is already partially supported via `TypedLLMComparator` in the existing -architecture. The integration point would be adding an optional Stage 5 to -`compare_formula` that calls the LLM when Stages 1–4 are all inconclusive. - -Trade-offs: -- **Pros**: Handles semantic equivalences, physics conventions, notation - variants that no CAS can resolve. -- **Cons**: Non-deterministic, adds latency and cost, requires API access. - Should not be used in tight evaluation loops without caching. - -### 6.4 Bare numeric answer vs ground truth with explicit SI unit (SeePhys-style) - -**Motivation (e.g. problem 660):** The question may ask for a length “in meters” while the -reference answer includes the unit in LaTeX, e.g. `$0.020 \text{m}$` → normalized -`PHYSICAL_QUANTITY` `"0.020 m"`. A model may answer `0.020` or `0.02`, which normalizes -to `NUMBER` `0.02`. The numeric value matches, but **categories differ** (`NUMBER` vs -`PHYSICAL_QUANTITY`), and the comparator stack intentionally treats a bare number as -not substitutable for a quantity when the ground truth carries a unit. - -**Future work:** - -- Optional **question-conditioned** coercion: when the stem specifies the required - unit (meters, seconds, …), map a pure-number prediction to that unit for comparison. -- Or **GT-side relaxation** for single-unit answers: compare numeric parts under an - explicit “this problem expects length in m” flag. -- Document dataset convention: either always include units in GT, or always omit them, - to reduce mixed signals. - - -### 6.7 Angles: `\approx`, `\operatorname{arcsec}`, and radians (e.g. problem 1734) - -**Categorization today:** - -- `$\approx 2 \operatorname{arcsec}$` → **`TEXT`** (e.g. stripped `"\approx 2 arcsec"`), - not `PHYSICAL_QUANTITY`. -- The same value without LaTeX fluff, `2 arcsec`, → **`PHYSICAL_QUANTITY`** `"2 arcsec"`. -- `1.0×10⁻⁵ rad` → **`PHYSICAL_QUANTITY`** `"1e-05 rad"`. - -So from a **physics** standpoint both are angular quantities, but the pipeline often -assigns **TEXT** vs **PHYSICAL_QUANTITY**. Even when both sides are quantities, -`compare_physical_quantity` compares unit strings as given: **`arcsec` and `rad` are -different units** with no automatic conversion, so equivalence must rely on numeric -comparison after conversion to a common angular unit (not implemented in the basic -PQ comparator). - -**Future work:** - -- Strip `\approx` / Unicode `≈` (and similar) **before** category classification so - `2 \operatorname{arcsec}` can normalize like `2 arcsec`. -- Map `\operatorname{arcsec}` (and `°`, `arcmin`, …) to a single **angle** normal form - or extend **unit-pool** comparison for angular measures. -- Optionally convert **arcsecond** and **radian** to a canonical unit for numerical - tolerance checks when humans treat them as interchangeable at fixed precision. diff --git a/src/prkit/evaluation/utils/__init__.py b/src/prkit/evaluation/utils/__init__.py deleted file mode 100644 index 4c6ac70..0000000 --- a/src/prkit/evaluation/utils/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Shared evaluation helpers: answer normalization, comparison dispatch, and numeric utilities.""" - -from prkit.core.domain.answer_category import AnswerCategory - -from .category_dispatch import compare_by_category -from .compare_same_type import ( - compare_formula, - compare_number, - compare_physical_quantity, - compare_plain_text, -) -from .normalization import ( - classify_expression, - normalize_answer, - normalize_expression, - normalize_number, - normalize_text, -) -from .number_utils import ( - decimal_places, - round_to_decimal_places, -) - -__all__ = [ - "AnswerCategory", - "classify_expression", - "compare_by_category", - "compare_formula", - "compare_number", - "compare_physical_quantity", - "compare_plain_text", - "decimal_places", - "normalize_answer", - "normalize_expression", - "normalize_number", - "normalize_text", - "round_to_decimal_places", -] diff --git a/src/prkit/evaluation/utils/answer_utils.py b/src/prkit/evaluation/utils/answer_utils.py deleted file mode 100644 index a4b01dd..0000000 --- a/src/prkit/evaluation/utils/answer_utils.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Helpers for converting ``Answer`` objects to strings and categorising comparison pairs.""" - -from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_category import AnswerCategory - - -def to_str(a: str | Answer) -> str: - """Extract string from Answer or return str as-is, with leading/trailing whitespace stripped.""" - return str(a.value).strip() if isinstance(a, Answer) else str(a).strip() - - -def same_comparison_category(cat1: AnswerCategory, cat2: AnswerCategory) -> bool: - """True if both answers should be compared using the same comparison strategy.""" - return cat1 == cat2 diff --git a/src/prkit/evaluation/utils/category_dispatch.py b/src/prkit/evaluation/utils/category_dispatch.py deleted file mode 100644 index 9881825..0000000 --- a/src/prkit/evaluation/utils/category_dispatch.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Dispatch normalized prediction/ground-truth to the per-category compare function.""" - -from __future__ import annotations - -import logging -from collections.abc import Callable, Mapping - -from prkit.core.domain.answer import AnswerValue -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation.utils.compare_same_type import compare_plain_text -from prkit.evaluation.utils.normalization import normalize_text - -SameCategoryCompareFn = Callable[[AnswerValue, AnswerValue], bool] - - -def compare_by_category( - category: AnswerCategory, - predicted_norm: AnswerValue, - ground_truth_norm: AnswerValue, - compare_fn_by_category: Mapping[AnswerCategory, SameCategoryCompareFn], - logger: logging.Logger | None = None, -) -> bool: - """Compare two normalized values using the category-specific compare function. - - For :attr:`AnswerCategory.TEXT`, applies :func:`~prkit.evaluation.utils.normalization.normalize_text` - to both sides before dispatching. Unknown categories fall back to - :func:`~prkit.evaluation.utils.compare_same_type.compare_plain_text`. - - On compare-function exception, logs a warning when *logger* is provided, then - falls back to plain-text comparison (SmartMatch behavior). - - Args: - category: Shared answer category for both sides. - predicted_norm: Normalized model answer. - ground_truth_norm: Normalized reference answer. - compare_fn_by_category: Mapping from category to a same-type compare callable - (e.g. :data:`SmartMatchComparator.DEFAULT_COMPARATORS`). - logger: Optional logger with ``warning`` for fallback diagnostics. - """ - if category == AnswerCategory.TEXT: - predicted_norm = normalize_text(str(predicted_norm)) - ground_truth_norm = normalize_text(str(ground_truth_norm)) - compare_fn = compare_fn_by_category.get(category, compare_plain_text) - try: - return compare_fn(predicted_norm, ground_truth_norm) - except Exception as e: - if logger is not None: - logger.warning( - f"{category} comparator failed: {e}. " - "Falling back to plain text comparison." - ) - return compare_plain_text(predicted_norm, ground_truth_norm) diff --git a/src/prkit/evaluation/utils/compare_cross_type.py b/src/prkit/evaluation/utils/compare_cross_type.py deleted file mode 100644 index 4622a1d..0000000 --- a/src/prkit/evaluation/utils/compare_cross_type.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Helpers for cross-category answer matching (e.g. formula vs text, equation vs number).""" - -from __future__ import annotations - -import re - -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation.utils.compare_same_type import ( - compare_formula, - compare_physical_quantity, -) -from prkit.evaluation.utils.normalization import normalize_answer -from prkit.evaluation.utils.type_specific_processing import extract_rhs_and_category - -_EQ_PATTERN = re.compile(r"^Eq\((.+),\s*(.+)\)$", re.DOTALL) - - -def split_respecting_parens(s: str, delimiter: str = ",") -> list[str]: - """Split on delimiter only at depth-0 (outside nested parentheses).""" - parts: list[str] = [] - depth = 0 - buf: list[str] = [] - for ch in s: - if ch in "({[": - depth += 1 - elif ch in ")}]": - depth = max(depth - 1, 0) - if ch == delimiter and depth == 0: - parts.append("".join(buf)) - buf = [] - else: - buf.append(ch) - parts.append("".join(buf)) - return parts - - -def expand_gt_set(gt_norm: float | str) -> list[str]: - """Split a SymPy-style set literal ``{a, b, …}`` into individual candidate strings.""" - s = str(gt_norm).strip() - if s.startswith("{") and s.endswith("}"): - inner = s[1:-1] - parts = split_respecting_parens(inner) - return [p.strip() for p in parts if p.strip()] - return [s] - - -def strip_unbalanced_parens(s: str) -> str: - """Strip leading/trailing unmatched parentheses and brackets.""" - s = s.strip() - for _open, _close in [("(", ")"), ("[", "]")]: - while s.endswith(_close) and s.count(_close) > s.count(_open): - s = s[:-1].rstrip() - while s.startswith(_open) and s.count(_open) > s.count(_close): - s = s[1:].lstrip() - return s - - -def extract_formula_candidates(text: str) -> list[str]: - """Split *text* on commas/semicolons and collect RHS substrings for formula matching.""" - candidates: list[str] = [] - separators = re.compile(r"[,;，；]") - parts = separators.split(text) - for part in parts: - part = part.strip() - if not part: - continue - candidates.append(part) - cleaned = strip_unbalanced_parens(part) - if cleaned != part: - candidates.append(cleaned) - if "=" in part: - rhs = part.rsplit("=", 1)[1].strip() - if rhs: - rhs_clean = strip_unbalanced_parens(rhs) - candidates.append(rhs) - if rhs_clean != rhs: - candidates.append(rhs_clean) - return candidates - - -def compare_text_against_formula_or_equation_gt( - pred_text: str, - gt_norm: float | str, -) -> bool: - """Try symbolic/quantity matching of formula candidates extracted from text.""" - gt_norm_str = str(gt_norm) - eq_m = _EQ_PATTERN.match(gt_norm_str) - if eq_m: - gt_norm_str = eq_m.group(2).strip() - - gt_norms = expand_gt_set(gt_norm_str) - candidates = extract_formula_candidates(pred_text) - for cand_str in candidates: - cand_str = cand_str.strip() - if not cand_str: - continue - try: - cand_cat, cand_norm = normalize_answer(cand_str) - except (ValueError, TypeError, RuntimeError): - continue - if cand_cat == AnswerCategory.EQUATION: - cand_norm, cand_cat = extract_rhs_and_category(cand_norm, cand_cat) - if cand_cat == AnswerCategory.TEXT: - continue - for gn in gt_norms: - try: - if compare_formula(cand_norm, gn): - return True - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - pass - if cand_cat == AnswerCategory.PHYSICAL_QUANTITY: - try: - if compare_physical_quantity(cand_norm, gn): - return True - except (ValueError, TypeError, ZeroDivisionError, AttributeError): - pass - return False diff --git a/src/prkit/evaluation/utils/compare_same_type.py b/src/prkit/evaluation/utils/compare_same_type.py deleted file mode 100644 index 8de0a1a..0000000 --- a/src/prkit/evaluation/utils/compare_same_type.py +++ /dev/null @@ -1,319 +0,0 @@ -"""Same-type answer comparison functions: number, formula, physical quantity, text, and option.""" - -import logging -import random -from collections.abc import Callable - -from sympy import ( - Derivative, - Integral, - N, - Product, - Sum, - cancel, - expand, - simplify, - sympify, - trigsimp, -) -from sympy.core.basic import Basic -from sympy.core.function import count_ops -from sympy.core.sympify import SympifyError - -from prkit.core.domain.answer import Answer, AnswerValue -from prkit.evaluation.utils.number_utils import ( - DEFAULT_NUMBER_EPSILON, - decimal_places, - round_to_decimal_places, -) -from prkit.evaluation.utils.type_specific_processing import parse_physical_quantity - -_log = logging.getLogger(__name__) - - -ComparableAnswerValue = Answer | AnswerValue -CategoryCompareFn = Callable[[AnswerValue, AnswerValue], bool] - - -def _raw_answer_value(value: ComparableAnswerValue) -> AnswerValue: - """Unwrap an ``Answer`` object to its raw value, or return non-Answer values unchanged.""" - return value.value if isinstance(value, Answer) else value - - -def _answer_text(value: ComparableAnswerValue) -> str: - """Return the string representation of the underlying answer value.""" - return str(_raw_answer_value(value)) - - -def compare_number( - predicted_norm: ComparableAnswerValue, - ground_truth_norm: ComparableAnswerValue, - epsilon: float = DEFAULT_NUMBER_EPSILON, -) -> bool: - """Compare two normalized numbers with precision-aware logic.""" - predicted_value = _raw_answer_value(predicted_norm) - ground_truth_value = _raw_answer_value(ground_truth_norm) - - pred = float(predicted_value) - gt = float(ground_truth_value) - - gt_dp = decimal_places(ground_truth_value) - pred_dp = decimal_places(predicted_value) - if pred_dp > gt_dp: - pred = round_to_decimal_places(pred, gt_dp) - - return abs(pred - gt) < epsilon - - -def compare_plain_text( - predicted_norm: ComparableAnswerValue, - ground_truth_norm: ComparableAnswerValue, -) -> bool: - """Compare two normalized strings with GT-as-substring acceptance.""" - pred_str = _answer_text(predicted_norm) - gt_str = _answer_text(ground_truth_norm) - if pred_str == gt_str: - return True - if gt_str and gt_str in pred_str: - return True - return False - - -def compare_option( - predicted_norm: ComparableAnswerValue, - ground_truth_norm: ComparableAnswerValue, -) -> bool: - """Compare multiple-choice / option labels (case-insensitive, stripped).""" - return ( - _answer_text(predicted_norm).strip().upper() - == _answer_text(ground_truth_norm).strip().upper() - ) - - -def _parse_physical_quantity(s: str) -> tuple[float | None, str]: - """Backward-compatible two-tuple alias for :func:`parse_physical_quantity`.""" - num, unit, _ = parse_physical_quantity(s) - return num, unit - - -def compare_physical_quantity( - predicted_norm: ComparableAnswerValue, - ground_truth_norm: ComparableAnswerValue, - epsilon: float = DEFAULT_NUMBER_EPSILON, -) -> bool: - """Compare two normalized physical quantities (value + unit).""" - pred_num: AnswerValue | None - gt_num: AnswerValue | None - if isinstance(predicted_norm, Answer): - pred_num = predicted_norm.value - pred_num_str = str(predicted_norm.value) - pred_unit = predicted_norm.unit - pred_string = ( - f"{pred_num} {pred_unit}" if pred_unit is not None else str(pred_num) - ) - else: - pred_string = _answer_text(predicted_norm) - pred_num, pred_unit, pred_num_str = parse_physical_quantity(pred_string) - - if isinstance(ground_truth_norm, Answer): - gt_num = ground_truth_norm.value - gt_num_str = str(ground_truth_norm.value) - gt_unit = ground_truth_norm.unit - gt_string = f"{gt_num} {gt_unit}" if gt_unit is not None else str(gt_num) - else: - gt_string = _answer_text(ground_truth_norm) - gt_num, gt_unit, gt_num_str = parse_physical_quantity(gt_string) - - if pred_unit == gt_unit and pred_num is not None and gt_num is not None: - return compare_number(pred_num_str, gt_num_str, epsilon) - return compare_plain_text(pred_string, gt_string) - - -def _formula_to_sympify(s: str) -> str: - """Convert a formula string to a form SymPy can parse (``^`` → ``**``).""" - return str(s).strip().replace("^", "**") - - -def _normalize_formula_text(s: str) -> str: - """Strip LaTeX display commands and normalize whitespace for a text-level formula comparison.""" - import re - - s = str(s).strip() - s = re.sub(r"\\(?:left|right|[bB]ig[glr]?)\b", "", s) - s = re.sub(r"\\(?:displaystyle|phantom)\b", "", s) - s = re.sub(r"\\[;,:!]", "", s) - s = s.replace(r"\dfrac", r"\frac") - s = re.sub(r"\s+", " ", s).strip() - return s - - -_NUM_EQUIV_TRIALS = 20 -_NUM_EQUIV_TOL = 1e-6 -_RNG = random.Random(42) - -# SymPy can hang for a long time on ``equals()``, ``simplify()``, and ``trigsimp()`` -# for moderate-sized trigonometric expressions (e.g. SeePhys ``problem_335`` pairs). -_MAX_SYM_EQUALS_OPS = 50 -_MAX_HEAVY_SIMPLIFY_OPS = 55 -_NUM_EQUIV_EXPENSIVE_NODES = (Integral, Derivative, Sum, Product) - - -def _sympy_multi_strategy_equal(pred_sym: Basic, gt_sym: Basic) -> bool: - """Try multiple SymPy simplification strategies to test equivalence. - - Returns True as soon as any strategy confirms equality, False only if - all strategies either fail or report inequality. - """ - # ``sympify`` can yield sets/intervals/tuples (e.g. interval notation); those - # types do not support subtraction against ordinary expressions. - try: - diff = pred_sym - gt_sym - except TypeError: - return False - - try: - diff_ops = int(count_ops(diff)) - except Exception: - diff_ops = 0 - heavy = diff_ops > _MAX_HEAVY_SIMPLIFY_OPS - - # Cheap structural checks first. ``expand`` / ``cancel`` are usually fast. - try: - if expand(pred_sym) == expand(gt_sym): - return True - except Exception: - pass - - try: - if cancel(pred_sym) == cancel(gt_sym): - return True - except Exception: - pass - - # No ``factor()`` here: SymPy's polynomial factorization can take effectively - # forever on some valid expressions (GCD/factor in extension fields). - - # Heavy simplifications: skip when ``pred - gt`` is large — ``simplify`` and - # ``trigsimp`` can stall on valid physics formulas. - if not heavy: - try: - if simplify(diff) == 0: - return True - except Exception: - pass - - try: - if trigsimp(pred_sym) == trigsimp(gt_sym): - return True - except Exception: - pass - - return False - - -def _numerical_equivalence(pred_sym: Basic, gt_sym: Basic) -> bool | None: - """Check equivalence by evaluating at multiple random points. - - Returns True if all trials match within tolerance, False if any trial - shows a definitive mismatch, or None if the check is inconclusive - (e.g. every trial raised an error). - """ - if pred_sym.has(*_NUM_EQUIV_EXPENSIVE_NODES) or gt_sym.has( - *_NUM_EQUIV_EXPENSIVE_NODES - ): - # SymPy numerical evaluation on integrals / derivatives / symbolic sums - # can take effectively unbounded time on otherwise valid formulas. - return None - - free_vars = sorted(pred_sym.free_symbols | gt_sym.free_symbols, key=str) - - if not free_vars: - try: - diff_val = complex(N(pred_sym - gt_sym)) - return abs(diff_val) < _NUM_EQUIV_TOL - except Exception: - return None - - passes = 0 - for _ in range(_NUM_EQUIV_TRIALS): - subs = {v: _RNG.uniform(0.5, 5.0) * _RNG.choice((1, -1)) for v in free_vars} - try: - diff_val = complex(N((pred_sym - gt_sym).subs(subs))) - except Exception: - continue - if abs(diff_val) > _NUM_EQUIV_TOL: - return False - passes += 1 - - if passes >= min(_NUM_EQUIV_TRIALS // 2, 5): - return True - return None - - -def compare_formula( - predicted_norm: ComparableAnswerValue, - ground_truth_norm: ComparableAnswerValue, -) -> bool: - """Compare normalized formulas using a cascade of strategies. - - The cascade (first True wins): - 1. SymPy ``equals()`` — fast random-point numerical check. - 2. Multi-strategy symbolic — ``simplify``, ``expand``, ``trigsimp``, - ``cancel`` applied to the difference or both sides (no ``factor()`` — - it can hang on some inputs). - 3. Robust numerical equivalence — evaluate at many random points to - confirm equality when symbolic methods are inconclusive. - 4. Normalized text fallback — exact string match after whitespace / - LaTeX cleanup (last resort). - """ - pred_expression = _answer_text(predicted_norm) - gt_expression = _answer_text(ground_truth_norm) - - try: - pred_sym = sympify(_formula_to_sympify(pred_expression)) - gt_sym = sympify(_formula_to_sympify(gt_expression)) - except (SympifyError, ValueError, TypeError, AttributeError): - pred_clean = _normalize_formula_text(pred_expression) - gt_clean = _normalize_formula_text(gt_expression) - return pred_clean == gt_clean - - # ``sympify`` can return a plain Python ``set`` for braced notation (e.g. ``"{1,2}"``). - # Those objects lack ``.free_symbols`` and break symbolic strategies; use text fallback. - if not isinstance(pred_sym, Basic) or not isinstance(gt_sym, Basic): - pred_clean = _normalize_formula_text(pred_expression) - gt_clean = _normalize_formula_text(gt_expression) - return pred_clean == gt_clean - - # --- Strategy 1: SymPy equals() (random-point / structural check) --- - try: - pred_ops = int(count_ops(pred_sym)) - gt_ops = int(count_ops(gt_sym)) - except Exception: - pred_ops = gt_ops = 0 - if pred_ops <= _MAX_SYM_EQUALS_OPS and gt_ops <= _MAX_SYM_EQUALS_OPS: - try: - if pred_sym.equals(gt_sym): - return True - except Exception: - pass - - # --- Strategy 2: multi-strategy symbolic simplification --- - if _sympy_multi_strategy_equal(pred_sym, gt_sym): - return True - - # --- Strategy 3: robust numerical equivalence --- - num_result = _numerical_equivalence(pred_sym, gt_sym) - if num_result is True: - _log.debug( - "Formula match via numerical equivalence: %s ≡ %s", - pred_expression, - gt_expression, - ) - return True - if num_result is False: - return False - - # --- Strategy 4: normalized text fallback --- - pred_clean = _normalize_formula_text(pred_expression) - gt_clean = _normalize_formula_text(gt_expression) - return pred_clean == gt_clean diff --git a/src/prkit/evaluation/utils/latex_symbol_preprocess.py b/src/prkit/evaluation/utils/latex_symbol_preprocess.py deleted file mode 100644 index 0b2e559..0000000 --- a/src/prkit/evaluation/utils/latex_symbol_preprocess.py +++ /dev/null @@ -1,71 +0,0 @@ -"""LaTeX preprocessing to protect physics symbols that break ``latex2sympy2_extended``.""" - -import re - -# ONLY include commands that are NOT natively handled as atomic symbols -# or those that include subscripts/decorations that break the parser. -PROTECTED_PHYSICS_SYMBOLS = { - r"\hbar": "hbar", - r"\mu_0": "mu0", - r"\epsilon_0": "eps0", - r"\varepsilon_0": "eps0", - r"\ell": "ell", - r"\square": "dalembert", - r"\angstrom": "angstrom", - r"\degree": "deg", -} - - -def _preprocess_latex(latex_str: str) -> str: - """Protect physics symbols known to break ``latex2sympy2_extended`` before parsing.""" - if not latex_str: - return "" - - processed = latex_str - - # 1. Clean spacing (Essential: these often cause "Unexpected Token" errors) - spacings = [r"\,", r"\:", r"\;", r"\!", r"\quad", r"\qquad"] - for space in spacings: - processed = processed.replace(space, " ") - - # 2. Protect specific breaking symbols - # Standard Greek (\alpha, \omega, etc.) are REMOVED from here - # because the parser handles them natively. - # Sort by length descending so longer keys (e.g. \varepsilon_0) are - # replaced before shorter prefixes (e.g. \varepsilon). - for cmd, name in sorted( - PROTECTED_PHYSICS_SYMBOLS.items(), key=lambda x: -len(x[0]) - ): - replacement = f"\\mathrm{{{name}}}" - if cmd in processed: - processed = processed.replace(cmd, replacement) - # Also handle braced subscript form: \foo_{0} alongside \foo_0 - if "_" in cmd: - base, sub = cmd.rsplit("_", 1) - braced = f"{base}_{{{sub}}}" - if braced in processed: - processed = processed.replace(braced, replacement) - - # 3. Strip decoration commands that latex2sympy doesn't reduce to the - # inner symbol (\dot{r} -> "dot{r}" instead of "r"). Stripping the - # wrapper lets the underlying variable parse cleanly. - for deco in ( - r"\vec", - r"\hat", - r"\dot", - r"\ddot", - r"\bar", - r"\tilde", - r"\overline", - r"\underline", - ): - processed = re.sub(re.escape(deco) + r"\{([^}]*)\}", r"\1", processed) - - # 4. Standardize differentials - # Many physics equations use \text{d}x; converting to 'd x' helps SymPy. - processed = processed.replace(r"\mathrm{d}", " d ").replace(r"\text{d}", " d ") - - # 5. Final cleanup - processed = re.sub(r"\s+", " ", processed).strip() - - return processed diff --git a/src/prkit/evaluation/utils/normalization.py b/src/prkit/evaluation/utils/normalization.py deleted file mode 100644 index 08ba33e..0000000 --- a/src/prkit/evaluation/utils/normalization.py +++ /dev/null @@ -1,184 +0,0 @@ -"""Backward-compatible evaluation normalization wrappers. - -`prkit.semantics` owns the normalization logic. This module preserves the -legacy evaluation-facing API by re-exporting semantics normalization helpers and mapping -its answer kinds back to legacy `AnswerCategory` / string labels. -""" - -from __future__ import annotations - -import re - -from latex2sympy2_extended import latex2sympy - -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation.utils.latex_symbol_preprocess import _preprocess_latex -from prkit.semantics.normalization.atomic_kinds import ( - NormalizedAtomicKind, -) -from prkit.semantics.normalization.atomic_normalization import ( - normalize_answer as _semantics_normalize_answer, -) -from prkit.semantics.normalization.atomic_normalization import ( - normalize_expression as _semantics_normalize_expression, -) -from prkit.semantics.normalization.atomic_normalization import ( - normalize_number, -) -from prkit.semantics.normalization.math_text_normalization import ( - _UNICODE_WHITESPACE, - _extract_math_content, - _match_balanced_braces, - _normalize_unicode, - _starts_with_latex_delimiter, - normalize_text, -) -from prkit.semantics.normalization.physical_quantity_normalization import ( - _FRAC_LATEX_PATTERN, - _FRACTION_RE, - _FRACTION_TOKEN, - _NUM_TOKEN, - _NUMERIC_PREFIX_RE, - _POWER_RE, - _POWER_TOKEN, - _QUANTITY_PATTERN, - _SCI_10_RE, - _SCI_10_TOKEN, - _SCI_E_TOKEN, - _SIGNED_NUM_OR_E_TOKEN, - _SIGNED_NUM_TOKEN, - _SIMPLE_NUMBER_RE, - _SUPERSCRIPT_TRANSLATION, - _UNIT_ALIASES, - _UNIT_TO_BASE, - _canonicalize_quantity_string, - _canonicalize_unit_alias, - _evaluate_numeric_expression, - _format_numeric_value, - _normalize_physical_quantity, - _normalize_unit_expression, - _parse_exponent, - _parse_numeric_base, - _parse_unit_expression, - _replace_superscript_exponents, - _split_numeric_and_unit, - _split_unit_exponent, - _try_parse_number_only, - _try_parse_physical_quantity, -) - - -def _legacy_expression_kind(kind: NormalizedAtomicKind) -> str: - """Convert a semantics ``NormalizedAtomicKind`` to the legacy string label used by the evaluation API.""" - mapping = { - NormalizedAtomicKind.NUMBER: "number", - NormalizedAtomicKind.PHYSICAL_QUANTITY: "physical_quantity", - NormalizedAtomicKind.RELATION: "equation", - NormalizedAtomicKind.EXPRESSION: "formula", - } - return mapping.get(kind, "formula") - - -def _legacy_answer_category(kind: NormalizedAtomicKind) -> AnswerCategory: - """Convert a semantics ``NormalizedAtomicKind`` to the evaluation-layer ``AnswerCategory``.""" - mapping = { - NormalizedAtomicKind.NUMBER: AnswerCategory.NUMBER, - NormalizedAtomicKind.PHYSICAL_QUANTITY: AnswerCategory.PHYSICAL_QUANTITY, - NormalizedAtomicKind.RELATION: AnswerCategory.EQUATION, - NormalizedAtomicKind.EXPRESSION: AnswerCategory.FORMULA, - NormalizedAtomicKind.TEXT: AnswerCategory.TEXT, - } - return mapping[kind] - - -def classify_expression(clean_str: str) -> str: - """Classify a cleaned expression using the legacy string labels.""" - - from prkit.semantics.normalization.expression_normalization import ( - classify_expression as _classify_expression, - ) - - return _legacy_expression_kind(_classify_expression(clean_str)) - - -def normalize_expression( - answer_str: str, -) -> tuple[float | str, bool, str]: - """Normalize an expression while preserving legacy category labels.""" - - normalized, success, kind = _semantics_normalize_expression(answer_str) - return normalized, success, _legacy_expression_kind(kind) - - -def normalize_answer( - answer_str: str, -) -> tuple[AnswerCategory, float | str]: - """Normalize an answer string via the semantics-owned implementation.""" - - kind, normalized = _semantics_normalize_answer(answer_str) - return _legacy_answer_category(kind), normalized - - -def _normalize_symbolic_expression( - clean_math: str, had_latex_patterns: bool -) -> tuple[str, bool]: - """Legacy-compatible symbolic normalization helper for tests and aliases.""" - - if had_latex_patterns: - preprocessed = _preprocess_latex(clean_math) - try: - symbolic_expr = latex2sympy(preprocessed) - normalized = re.sub(r"\s+", " ", str(symbolic_expr)).strip() - return normalized, True - except Exception: - return preprocessed, False - - normalized = re.sub(r"\s+", " ", clean_math).strip() - return normalized, True - - -__all__ = [ - "_FRAC_LATEX_PATTERN", - "_FRACTION_RE", - "_FRACTION_TOKEN", - "_NUMERIC_PREFIX_RE", - "_NUM_TOKEN", - "_POWER_RE", - "_POWER_TOKEN", - "_QUANTITY_PATTERN", - "_SCI_10_RE", - "_SCI_10_TOKEN", - "_SCI_E_TOKEN", - "_SIGNED_NUM_OR_E_TOKEN", - "_SIGNED_NUM_TOKEN", - "_SIMPLE_NUMBER_RE", - "_SUPERSCRIPT_TRANSLATION", - "_UNICODE_WHITESPACE", - "_UNIT_ALIASES", - "_UNIT_TO_BASE", - "_canonicalize_quantity_string", - "_canonicalize_unit_alias", - "_evaluate_numeric_expression", - "_extract_math_content", - "_format_numeric_value", - "_match_balanced_braces", - "_normalize_physical_quantity", - "_normalize_symbolic_expression", - "_normalize_unicode", - "_normalize_unit_expression", - "_parse_exponent", - "_parse_numeric_base", - "_parse_unit_expression", - "_replace_superscript_exponents", - "_split_numeric_and_unit", - "_split_unit_exponent", - "_starts_with_latex_delimiter", - "_try_parse_number_only", - "_try_parse_physical_quantity", - "classify_expression", - "latex2sympy", - "normalize_answer", - "normalize_expression", - "normalize_number", - "normalize_text", -] diff --git a/src/prkit/evaluation/utils/number_utils.py b/src/prkit/evaluation/utils/number_utils.py deleted file mode 100644 index ef40bf4..0000000 --- a/src/prkit/evaluation/utils/number_utils.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Numeric precision helpers: decimal-place inference, significant digits, and epsilon-aware rounding.""" - -import math - -# Default epsilon for number comparison -DEFAULT_NUMBER_EPSILON = 1e-10 - - -def decimal_places(x: int | float | str) -> int: - """ - Infer the number of decimal places. - - Accepts a float **or** a string. When a string is provided, trailing - zeros are preserved (e.g. ``"0.50"`` → 2). When a float is provided, - trailing zeros are indistinguishable so we strip them (``0.5`` → 1). - - Args: - x: Float or string representation of a number - - Returns: - Number of digits after the decimal point (0 for integers) - - Examples: - ``9.8`` → 1, ``"0.50"`` → 2, ``500.0`` → 0, ``0.00123`` → 5 - """ - if isinstance(x, str): - s = x.strip() - if "." in s: - return len(s.split(".")[1]) - return 0 - - if x == 0 or math.isnan(x) or math.isinf(x): - return 0 - s = format(x, ".15g") - if "e" in s.lower(): - s = format(x, ".15f").rstrip("0").rstrip(".") - else: - s = s.rstrip("0").rstrip(".") - if "." in s: - return len(s.split(".")[1]) - return 0 - - -def round_to_decimal_places(x: float, n: int) -> float: - """Round a float to n decimal places.""" - if n < 0: - return x - return round(x, n) diff --git a/src/prkit/evaluation/utils/type_specific_processing.py b/src/prkit/evaluation/utils/type_specific_processing.py deleted file mode 100644 index 9487fe0..0000000 --- a/src/prkit/evaluation/utils/type_specific_processing.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Physical quantity parsing and equation RHS extraction shared across comparators.""" - -from __future__ import annotations - -import re - -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation.utils.normalization import normalize_answer -from prkit.semantics.normalization.physical_quantity_normalization import ( - parse_physical_quantity as _parse_semantics_physical_quantity, -) - -_EQ_PATTERN = re.compile(r"^Eq\((.+),\s*(.+)\)$", re.DOTALL) - - -def parse_physical_quantity(s: str) -> tuple[float | None, str, str]: - """Parse normalized physical quantity as ``(numeric_value, unit, num_str)``.""" - return _parse_semantics_physical_quantity(s) - - -def extract_rhs_and_category( - norm_value: float | str, - category: AnswerCategory, -) -> tuple[float | str, AnswerCategory]: - """Extract equation RHS and re-normalize when input is equation-like.""" - if category != AnswerCategory.EQUATION: - s = str(norm_value) - if "=" not in s: - return norm_value, category - - s = str(norm_value) - rhs: str | None = None - eq_m = _EQ_PATTERN.match(s) - if eq_m: - rhs = eq_m.group(2).strip() - elif "=" in s: - rhs = s.rsplit("=", 1)[1].strip() - - if rhs: - new_cat, new_norm = normalize_answer(rhs) - return new_norm, new_cat - return norm_value, category diff --git a/src/prkit/semantics/README.md b/src/prkit/semantics/README.md index bc6158e..648a191 100644 --- a/src/prkit/semantics/README.md +++ b/src/prkit/semantics/README.md @@ -192,6 +192,7 @@ AnswerObjectKind: choice boolean sign_direction + descriptive_text ``` The answer-structure enum is: @@ -291,6 +292,7 @@ PASEC-Base covers common final-answer forms in physics: - global side conditions using `subject_to` - multiple-choice labels and bounded discrete outcomes - booleans, signs, directions, and curated qualitative labels +- free-form descriptive ("explain/why") answers, judged by conservative normalized-text equality - question-scoped symbol aliases and notation variants - coordinate-frame and sign-convention metadata @@ -303,7 +305,7 @@ The smallest reproducible path is deterministic question inference, answer normalization, contract construction, and evaluation. ```python -from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem +from prkit.core.domain import Answer, AnswerObjectKind, PhysicsProblem from prkit.semantics import ( ComparisonPolicyMode, build_evaluation_contract, @@ -315,7 +317,7 @@ from prkit.semantics import ( problem = PhysicsProblem( problem_id="demo-speed", question="Find the speed in m/s.", - answer=Answer(value="18", unit="km/h", answer_category=AnswerCategory.PHYSICAL_QUANTITY), + answer=Answer(value="18", unit="km/h", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), ) question_semantics = infer_reference_question_semantics(problem) diff --git a/src/prkit/semantics/build/prompts.py b/src/prkit/semantics/build/prompts.py index 54a608a..d89d834 100644 --- a/src/prkit/semantics/build/prompts.py +++ b/src/prkit/semantics/build/prompts.py @@ -42,7 +42,8 @@ - Add `question_semantics.symbol_aliases` when the problem and answer use different names for the same symbol, such as `y_s` versus `y`. - In `symbol_aliases`, use plain token-style symbol names like `y`, `y_s`, `theta_dot`, not full equations or wrapped LaTeX snippets. - `reference_answer_semantics` or `prediction_answer_semantics` must represent only the final answer. -- Use `object_kind` from: number, physical_quantity, expression, relation, qualitative_label, choice, boolean, sign_direction. +- Use `object_kind` from: number, physical_quantity, expression, relation, qualitative_label, choice, boolean, sign_direction, descriptive_text. + - `qualitative_label` = a short curated controlled-vocabulary label (e.g. "increases", "isothermal"); `descriptive_text` = a free-form explanatory/"why" answer in prose. - Use `structure` from: atomic, multi_part, tuple, set, interval, vector, matrix, tensor, piecewise. Decide structure by denotation, not surface punctuation: - `atomic` = one indivisible value (the default). Prefer it: a single coordinate is atomic, not a 1-tuple; a closed point-range `[a, a]` is the atom `a`. - `tuple` = one ordered coordinate of a single object, `(x, y)`; a bare finite `(a, b)` is a tuple, NOT an interval. diff --git a/src/prkit/semantics/comparison/EQUIVALENCE.md b/src/prkit/semantics/comparison/EQUIVALENCE.md index d1350fb..2004e67 100644 --- a/src/prkit/semantics/comparison/EQUIVALENCE.md +++ b/src/prkit/semantics/comparison/EQUIVALENCE.md @@ -16,9 +16,9 @@ Every example below is a real engine result. Notation: `pred ≡ ref` means equi Each side is an `PhysicsAnswerSemantics` record (raw strings are coerced into one): -- **`object_kind`** — one of 8 atomic kinds (`AnswerObjectKind`): `number`, +- **`object_kind`** — one of 9 atomic kinds (`AnswerObjectKind`): `number`, `physical_quantity`, `expression`, `relation`, `qualitative_label`, `choice`, - `boolean`, `sign_direction`. + `boolean`, `sign_direction`, `descriptive_text`. - **`structure`** — one of 9 (`AnswerStructure`): `atomic`, `multi_part`, `tuple`, `set`, `interval`, `vector`, `matrix`, `tensor`, `piecewise`. - **`canonical_text`** + typed fields (`numeric_value`, `numeric_text`, `unit`, @@ -308,6 +308,27 @@ sign-convention lane (§8, `comparison_mode = sign_convention`), gated on the qu no convention. Absolute labels (`up`, `clockwise`, `into_page`) are **not** axis-relative — a flip there is a real disagreement and stays on the plain `sign_direction` path. +### 7.6 `descriptive_text` — free-form prose + +For free-form descriptive ("explain/why") answers the criterion is **conservative +normalized-text equality**: canonicalize the surface (math/text wrappers, case, +punctuation, whitespace) via `canonicalize_descriptive_text`, then accept only if the +two canonical forms are identical. There is **no** alias/synonym mapping — that is what +separates `descriptive_text` (free prose) from `qualitative_label` (a curated controlled +vocabulary). This honors the precision-first discipline (canonical form + one criterion, +no rescue branches; §1, `METHODOLOGY.md` §3). + +``` +"net force is nonzero." ≡ "Net force is nonzero" → descriptive_text (surface-equal) +"forces are balanced" ≢ "net force is nonzero" → reject (genuinely different prose) +``` + +> Richer recall for free-form answers — semantic similarity or a gated model-judge — is a +> deliberately deferred v2 research lever, **not** implemented here: it would break +> determinism and introduce unprincipled thresholds. The deterministic normalizer routes +> only genuinely free-form text to `descriptive_text`; curated controlled-vocabulary +> phrases stay `qualitative_label`. + --- ## 8. Different object kind — tiered bridges diff --git a/src/prkit/semantics/comparison/same_object_kind.py b/src/prkit/semantics/comparison/same_object_kind.py index a83a881..1f84576 100644 --- a/src/prkit/semantics/comparison/same_object_kind.py +++ b/src/prkit/semantics/comparison/same_object_kind.py @@ -21,6 +21,7 @@ build_symbol_assumption_map, canonicalize_boolean_value, canonicalize_choice_label, + canonicalize_descriptive_text, canonicalize_qualitative_label, canonicalize_sign_direction, equality_like_rhs_expression_text, @@ -89,6 +90,14 @@ def compare_same_object_kind( ) return AnswerComparison(matched, "qualitative_label") + if kind == AnswerObjectKind.DESCRIPTIVE_TEXT: + # Free-form prose: conservative normalized-text equality (surface canonical + # form, one criterion, no semantic rescue). See canonicalize_descriptive_text. + matched = canonicalize_descriptive_text( + pred.canonical_text + ) == canonicalize_descriptive_text(ref.canonical_text) + return AnswerComparison(matched, "descriptive_text") + return AnswerComparison(False, "unsupported_object_kind", (kind.value,)) diff --git a/src/prkit/semantics/comparison/semantics.py b/src/prkit/semantics/comparison/semantics.py index a46f5e5..772da53 100644 --- a/src/prkit/semantics/comparison/semantics.py +++ b/src/prkit/semantics/comparison/semantics.py @@ -462,6 +462,25 @@ def canonicalize_qualitative_label(text: str) -> str: return normalized +def canonicalize_descriptive_text(text: str | None) -> str: + """Conservatively canonicalize a free-form descriptive ("explain/why") answer. + + Surface-only normalization (math/text wrappers, case, punctuation, whitespace) + with **no** semantic alias or synonym mapping. That is exactly what separates + ``descriptive_text`` (free prose, judged equivalent only when the wording is + essentially identical) from ``qualitative_label`` (a curated controlled + vocabulary canonicalized through :data:`_QUALITATIVE_ALIAS_GROUPS`). + + Richer recall for free-form answers — semantic similarity or a gated + model-judge — is a deliberately deferred v2 research lever, **not** added here: + it would break determinism and introduce unprincipled thresholds, violating the + "canonical form + one principled criterion, no rescue branches" discipline + (``METHODOLOGY.md`` §3). + """ + + return normalize_plain_text(text) + + def qualitative_label_candidates(text: str) -> tuple[str, ...]: """Return canonical qualitative labels that are explicitly asserted in ``text``.""" diff --git a/src/prkit/semantics/normalization/answer_normalization.py b/src/prkit/semantics/normalization/answer_normalization.py index 16338f8..b8862cb 100644 --- a/src/prkit/semantics/normalization/answer_normalization.py +++ b/src/prkit/semantics/normalization/answer_normalization.py @@ -148,6 +148,10 @@ def _normalize_declaration_phrase(phrase: str) -> str: return _DECLARATION_PHRASE_SEPARATOR_RE.sub(" ", phrase.strip().lower()) +# Curated controlled vocabulary for the ``qualitative_label`` kind. Text the +# deterministic normalizer cannot place in a structured kind is classified as +# ``qualitative_label`` only when it matches one of these curated alias groups; +# any other free-form prose is ``descriptive_text`` (see _classify_text_kind). _QUALITATIVE_ALIAS_GROUPS = { "constant_temperature": { "temperature stays constant", @@ -492,13 +496,7 @@ def _normalize_atomic_text( provenance=provenance, ) - canonical_text = _canonicalize_qualitative_text(raw_text) - return PhysicsAnswerSemantics( - canonical_text=canonical_text, - raw_text=raw_text, - object_kind=AnswerObjectKind.QUALITATIVE_LABEL, - provenance=provenance, - ) + return _classify_text_kind(raw_text, provenance=provenance) def _normalize_atomic_with_subject_to( @@ -1280,17 +1278,64 @@ def _strip_math_wrappers(text: str) -> str: def _canonicalize_boolean(text: str) -> bool | None: """Return the canonical boolean value encoded by ``text``.""" - normalized = _normalize_phrase(text) + normalized = _normalize_phrase(text).strip(" .,;:!?") return _BOOLEAN_CANONICAL.get(normalized) def _canonicalize_sign_direction(text: str) -> str | None: """Return the canonical sign/direction label encoded by ``text``.""" - normalized = _normalize_phrase(text) + # Tolerate trailing sentence punctuation (e.g. "to the right.") so a directional + # answer is detected as SIGN_DIRECTION instead of falling through to text. + normalized = _normalize_phrase(text).strip(" .,;:!?") return _SIGN_DIRECTION_CANONICAL.get(normalized) +def _curated_qualitative_label(text: str) -> str | None: + """Return the curated controlled-vocabulary label for ``text``, else ``None``. + + Only the phrases in :data:`_QUALITATIVE_ALIAS_GROUPS` are ``qualitative_label``; + any other free-form text is descriptive prose (``descriptive_text``). + """ + + normalized = _normalize_phrase(text) + for canonical, aliases in _QUALITATIVE_ALIAS_GROUPS.items(): + if normalized == canonical or normalized in aliases: + return canonical + return None + + +def _classify_text_kind( + raw_text: str, + *, + provenance: dict[str, Any], +) -> PhysicsAnswerSemantics: + """Classify catch-all text as a curated qualitative label or free-form prose. + + Curated controlled-vocabulary phrases become ``qualitative_label`` (canonicalized + through the alias groups). Everything else is ``descriptive_text``, judged later + by conservative normalized-text equality only — no semantic alias rescue. Richer + recall for free-form answers (semantic similarity / a gated model-judge) is a + deferred v2 lever, intentionally not added here (it would break determinism). + """ + + qualitative_label = _curated_qualitative_label(raw_text) + if qualitative_label is not None: + return PhysicsAnswerSemantics( + canonical_text=qualitative_label, + raw_text=raw_text, + object_kind=AnswerObjectKind.QUALITATIVE_LABEL, + provenance=provenance, + ) + + return PhysicsAnswerSemantics( + canonical_text=_normalize_phrase(raw_text), + raw_text=raw_text, + object_kind=AnswerObjectKind.DESCRIPTIVE_TEXT, + provenance=provenance, + ) + + def _canonicalize_choice( text: str, choice_space: tuple[str, ...], @@ -1308,16 +1353,6 @@ def _canonicalize_choice( return None -def _canonicalize_qualitative_text(text: str) -> str: - """Normalize qualitative text using the same alias groups as comparison.""" - - normalized = _normalize_phrase(text) - for canonical, aliases in _QUALITATIVE_ALIAS_GROUPS.items(): - if normalized in aliases: - return canonical - return normalized - - def _try_expression_rescue( raw_text: str, *, diff --git a/src/prkit/semantics/schema/enums.py b/src/prkit/semantics/schema/enums.py index 1d2316d..c5bc104 100644 --- a/src/prkit/semantics/schema/enums.py +++ b/src/prkit/semantics/schema/enums.py @@ -1,5 +1,12 @@ """Enumerations for physics answer semantics. +The answer *ontology* enums (:class:`AnswerObjectKind`, :class:`AnswerStructure`) +and the :class:`_StrEnum` base now live in :mod:`prkit.core.domain.answer_kinds` +as the toolkit's canonical taxonomy; they are re-exported here so existing +``from prkit.semantics.schema import AnswerObjectKind`` import sites keep working. +The *judgement-policy* enums below (unit policy, comparison mode, bridge tier, …) +stay in semantics — they are mechanism, not ontology. + See ``../comparison/EQUIVALENCE.md`` for the detailed equivalence-judgement reference (object kinds, structures, per-kind criteria, bridges, examples) and ``../comparison/METHODOLOGY.md`` for the design discipline behind it. @@ -7,41 +14,27 @@ from __future__ import annotations -from enum import Enum - - -class _StrEnum(str, Enum): - """Enum subclass with string values and friendly ``str()`` output.""" - - def __str__(self) -> str: - return str(self.value) - - -class AnswerObjectKind(_StrEnum): - """What kind of answer object the normalized final answer is.""" - - NUMBER = "number" - PHYSICAL_QUANTITY = "physical_quantity" - EXPRESSION = "expression" - RELATION = "relation" - QUALITATIVE_LABEL = "qualitative_label" - CHOICE = "choice" - BOOLEAN = "boolean" - SIGN_DIRECTION = "sign_direction" - - -class AnswerStructure(_StrEnum): - """How the answer is structured.""" - - ATOMIC = "atomic" - MULTI_PART = "multi_part" - TUPLE = "tuple" - SET = "set" - INTERVAL = "interval" - VECTOR = "vector" - MATRIX = "matrix" - TENSOR = "tensor" - PIECEWISE = "piecewise" +# Re-exported canonical ontology enums (defined in prkit.core.domain). ``_StrEnum`` +# is the shared base for the judgement-policy enums defined in this module. +from prkit.core.domain.answer_kinds import ( + AnswerObjectKind, + AnswerStructure, + _StrEnum, +) + +__all__ = [ + # Re-exported canonical ontology (defined in prkit.core.domain) + "AnswerObjectKind", + "AnswerStructure", + # Judgement-policy enums (defined below) + "QuestionSymbolicMode", + "QuestionUnitPolicy", + "OrderingPolicy", + "ContractValidationStatus", + "ComparisonPolicyMode", + "BridgeTier", + "SymbolAssumption", +] class QuestionSymbolicMode(_StrEnum): diff --git a/src/prkit/testing/conformance.py b/src/prkit/testing/conformance.py index dc3cced..c4c7ab3 100644 --- a/src/prkit/testing/conformance.py +++ b/src/prkit/testing/conformance.py @@ -17,7 +17,7 @@ from pydantic import BaseModel from prkit.api import DatasetProvider, ModelClient, Scorer, Verdict -from prkit.core.domain import AnswerCategory, PhysicsProblem +from prkit.core.domain import AnswerObjectKind, PhysicsProblem from prkit.core.model_clients.structured_output import StructuredOutputPlan from prkit.datasets.loaders.base_loader import BaseDatasetLoader @@ -122,9 +122,9 @@ def check_dataset( problem, PhysicsProblem ), f"load() must yield PhysicsProblem, got {type(problem)!r}" if problem.answer is not None: - assert problem.answer.answer_category in AnswerCategory, ( - f"problem {problem.problem_id!r} has invalid answer_category " - f"{problem.answer.answer_category!r}" + assert problem.answer.answer_kind in AnswerObjectKind, ( + f"problem {problem.problem_id!r} has invalid answer_kind " + f"{problem.answer.answer_kind!r}" ) diff --git a/tests/conftest.py b/tests/conftest.py index 7d54633..65584db 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ from prkit.core.domain import ( Answer, - AnswerCategory, + AnswerObjectKind, PhysicalDataset, PhysicsDomain, PhysicsProblem, @@ -20,14 +20,14 @@ def sample_answer_numerical(): """Create a sample numerical answer.""" return Answer( - value=42.0, answer_category=AnswerCategory.PHYSICAL_QUANTITY, unit="m/s" + value=42.0, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, unit="m/s" ) @pytest.fixture def sample_answer_symbolic(): """Create a sample symbolic answer.""" - return Answer(value="x^2 + 2x + 1", answer_category=AnswerCategory.FORMULA) + return Answer(value="x^2 + 2x + 1", answer_kind=AnswerObjectKind.EXPRESSION) @pytest.fixture @@ -35,14 +35,14 @@ def sample_answer_textual(): """Create a sample textual answer.""" return Answer( value="The force is equal to mass times acceleration", - answer_category=AnswerCategory.TEXT, + answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT, ) @pytest.fixture def sample_answer_option(): """Create a sample option answer.""" - return Answer(value="A", answer_category=AnswerCategory.OPTION) + return Answer(value="A", answer_kind=AnswerObjectKind.CHOICE) @pytest.fixture @@ -52,7 +52,7 @@ def sample_physics_problem(): problem_id="test_001", question="What is the speed of light?", answer=Answer( - value=3e8, answer_category=AnswerCategory.PHYSICAL_QUANTITY, unit="m/s" + value=3e8, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, unit="m/s" ), solution="The speed of light in vacuum is approximately 3 × 10^8 m/s", domain=PhysicsDomain.CLASSICAL_MECHANICS, @@ -67,7 +67,7 @@ def sample_physics_problem_mc(): return PhysicsProblem( problem_id="test_002", question="What is F = ma?", - answer=Answer(value="A", answer_category=AnswerCategory.OPTION), + answer=Answer(value="A", answer_kind=AnswerObjectKind.CHOICE), options=[ "Newton's second law", "Newton's first law", @@ -104,7 +104,7 @@ def sample_problems_list(): problem = PhysicsProblem( problem_id=f"test_{i:03d}", question=f"Test question {i}", - answer=Answer(value=i, answer_category=AnswerCategory.NUMBER), + answer=Answer(value=i, answer_kind=AnswerObjectKind.NUMBER), domain=( PhysicsDomain.CLASSICAL_MECHANICS if i % 2 == 0 diff --git a/tests/prkit/core/domain/test_answer.py b/tests/prkit/core/domain/test_answer.py index 1d11db4..26dbcb5 100644 --- a/tests/prkit/core/domain/test_answer.py +++ b/tests/prkit/core/domain/test_answer.py @@ -2,7 +2,7 @@ Tests for Answer model. """ -from prkit.core.domain import Answer, AnswerCategory +from prkit.core.domain import Answer, AnswerObjectKind class TestAnswer: @@ -10,76 +10,78 @@ class TestAnswer: def test_answer_creation_numerical(self): """Test creating a numerical answer.""" - answer = Answer(value=42.0, answer_category=AnswerCategory.NUMBER, unit="m/s") + answer = Answer(value=42.0, answer_kind=AnswerObjectKind.NUMBER, unit="m/s") assert answer.value == 42.0 - assert answer.answer_category == AnswerCategory.NUMBER + assert answer.answer_kind == AnswerObjectKind.NUMBER assert answer.unit == "m/s" assert answer.metadata == {} def test_answer_creation_symbolic(self): """Test creating a symbolic answer.""" - answer = Answer(value="x^2 + 1", answer_category=AnswerCategory.FORMULA) + answer = Answer(value="x^2 + 1", answer_kind=AnswerObjectKind.EXPRESSION) assert answer.value == "x^2 + 1" - assert answer.answer_category == AnswerCategory.FORMULA + assert answer.answer_kind == AnswerObjectKind.EXPRESSION assert answer.unit is None def test_answer_creation_textual(self): """Test creating a textual answer.""" - answer = Answer(value="The answer is 42", answer_category=AnswerCategory.TEXT) + answer = Answer( + value="The answer is 42", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT + ) assert answer.value == "The answer is 42" - assert answer.answer_category == AnswerCategory.TEXT + assert answer.answer_kind == AnswerObjectKind.DESCRIPTIVE_TEXT def test_answer_creation_option(self): """Test creating an option answer.""" - answer = Answer(value="A", answer_category=AnswerCategory.OPTION) + answer = Answer(value="A", answer_kind=AnswerObjectKind.CHOICE) assert answer.value == "A" - assert answer.answer_category == AnswerCategory.OPTION + assert answer.answer_kind == AnswerObjectKind.CHOICE def test_answer_metadata_initialization(self): """Test that metadata is initialized as empty dict.""" - answer = Answer(value=1, answer_category=AnswerCategory.NUMBER) + answer = Answer(value=1, answer_kind=AnswerObjectKind.NUMBER) assert answer.metadata == {} def test_answer_metadata_custom(self): """Test custom metadata.""" metadata = {"source": "test", "confidence": 0.9} - answer = Answer( - value=1, answer_category=AnswerCategory.NUMBER, metadata=metadata - ) + answer = Answer(value=1, answer_kind=AnswerObjectKind.NUMBER, metadata=metadata) assert answer.metadata == metadata def test_answer_validation_numerical(self): """Test numerical answer validation.""" - valid_answer = Answer(value=42.0, answer_category=AnswerCategory.NUMBER) + valid_answer = Answer(value=42.0, answer_kind=AnswerObjectKind.NUMBER) assert valid_answer.validate() is True invalid_answer = Answer( - value="not a number", answer_category=AnswerCategory.NUMBER + value="not a number", answer_kind=AnswerObjectKind.NUMBER ) assert invalid_answer.validate() is False def test_answer_validation_symbolic(self): """Test symbolic answer validation.""" - valid_answer = Answer(value="x^2", answer_category=AnswerCategory.FORMULA) + valid_answer = Answer(value="x^2", answer_kind=AnswerObjectKind.EXPRESSION) assert valid_answer.validate() is True - invalid_answer = Answer(value="", answer_category=AnswerCategory.FORMULA) + invalid_answer = Answer(value="", answer_kind=AnswerObjectKind.EXPRESSION) assert invalid_answer.validate() is False def test_answer_validation_textual(self): """Test textual answer validation.""" - valid_answer = Answer(value="Some text", answer_category=AnswerCategory.TEXT) + valid_answer = Answer( + value="Some text", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT + ) assert valid_answer.validate() is True - invalid_answer = Answer(value="", answer_category=AnswerCategory.TEXT) + invalid_answer = Answer(value="", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) assert invalid_answer.validate() is False def test_answer_category_checking(self): """Test answer type checking methods.""" - numerical = Answer(value=1, answer_category=AnswerCategory.NUMBER) - symbolic = Answer(value="x", answer_category=AnswerCategory.FORMULA) - textual = Answer(value="text", answer_category=AnswerCategory.TEXT) - option = Answer(value="A", answer_category=AnswerCategory.OPTION) + numerical = Answer(value=1, answer_kind=AnswerObjectKind.NUMBER) + symbolic = Answer(value="x", answer_kind=AnswerObjectKind.EXPRESSION) + textual = Answer(value="text", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) + option = Answer(value="A", answer_kind=AnswerObjectKind.CHOICE) assert numerical.is_numerical() is True assert numerical.is_symbolic() is False @@ -89,18 +91,18 @@ def test_answer_category_checking(self): def test_answer_numerical_methods(self): """Test numerical-specific methods.""" - answer = Answer(value=42, answer_category=AnswerCategory.NUMBER, unit="m/s") + answer = Answer(value=42, answer_kind=AnswerObjectKind.NUMBER, unit="m/s") assert answer.get_unit() == "m/s" assert answer.has_unit() is True assert answer.is_integer() is True assert answer.is_positive() is True - negative_answer = Answer(value=-5, answer_category=AnswerCategory.NUMBER) + negative_answer = Answer(value=-5, answer_kind=AnswerObjectKind.NUMBER) assert negative_answer.is_negative() is True def test_answer_symbolic_methods(self): """Test symbolic-specific methods.""" - latex_answer = Answer(value="$x^2$", answer_category=AnswerCategory.FORMULA) + latex_answer = Answer(value="$x^2$", answer_kind=AnswerObjectKind.EXPRESSION) assert latex_answer.is_latex() is True clean = latex_answer.get_clean_expression() @@ -109,7 +111,7 @@ def test_answer_symbolic_methods(self): def test_answer_textual_methods(self): """Test textual-specific methods.""" answer = Answer( - value="This is a test answer", answer_category=AnswerCategory.TEXT + value="This is a test answer", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT ) assert answer.word_count() == 5 assert answer.char_count() == 21 # "This is a test answer" = 21 chars @@ -119,34 +121,34 @@ def test_answer_textual_methods(self): def test_answer_option_methods(self): """Test option-specific methods.""" - letter_answer = Answer(value="A", answer_category=AnswerCategory.OPTION) + letter_answer = Answer(value="A", answer_kind=AnswerObjectKind.CHOICE) assert letter_answer.is_letter_option() is True assert letter_answer.get_option_index() == 0 - numeric_answer = Answer(value="1", answer_category=AnswerCategory.OPTION) + numeric_answer = Answer(value="1", answer_kind=AnswerObjectKind.CHOICE) assert numeric_answer.is_numeric_option() is True - yes_answer = Answer(value="YES", answer_category=AnswerCategory.OPTION) + yes_answer = Answer(value="YES", answer_kind=AnswerObjectKind.CHOICE) assert yes_answer.is_yes_no() is True def test_answer_to_dict(self): """Test answer serialization to dictionary.""" answer = Answer( value=42, - answer_category=AnswerCategory.NUMBER, + answer_kind=AnswerObjectKind.NUMBER, unit="m/s", metadata={"test": True}, ) result = answer.to_dict() assert result["value"] == 42 - assert result["answer_category"] == "number" + assert result["answer_kind"] == "number" assert result["unit"] == "m/s" assert result["metadata"]["test"] is True def test_answer_str_repr(self): """Test string representations.""" - answer = Answer(value=42, answer_category=AnswerCategory.NUMBER, unit="m/s") + answer = Answer(value=42, answer_kind=AnswerObjectKind.NUMBER, unit="m/s") assert "42" in str(answer) assert "m/s" in str(answer) @@ -155,22 +157,22 @@ def test_answer_str_repr(self): def test_answer_validation_numerical_bool_false(self): """Test that boolean False is not valid for numerical.""" - answer = Answer(value=False, answer_category=AnswerCategory.NUMBER) + answer = Answer(value=False, answer_kind=AnswerObjectKind.NUMBER) assert answer.validate() is False def test_answer_validation_numerical_bool_true(self): """Test that boolean True is not valid for numerical.""" - answer = Answer(value=True, answer_category=AnswerCategory.NUMBER) + answer = Answer(value=True, answer_kind=AnswerObjectKind.NUMBER) assert answer.validate() is False def test_answer_validation_symbolic_whitespace_only(self): """Test that whitespace-only string is invalid for symbolic.""" - answer = Answer(value=" \n\t ", answer_category=AnswerCategory.FORMULA) + answer = Answer(value=" \n\t ", answer_kind=AnswerObjectKind.EXPRESSION) assert answer.validate() is False def test_answer_numerical_zero(self): """Test numerical answer with zero value.""" - answer = Answer(value=0, answer_category=AnswerCategory.NUMBER) + answer = Answer(value=0, answer_kind=AnswerObjectKind.NUMBER) assert answer.is_numerical() is True assert answer.is_positive() is False assert answer.is_negative() is False @@ -178,47 +180,50 @@ def test_answer_numerical_zero(self): def test_answer_numerical_float_integer(self): """Test numerical answer with float that is integer.""" - answer = Answer(value=42.0, answer_category=AnswerCategory.NUMBER) + answer = Answer(value=42.0, answer_kind=AnswerObjectKind.NUMBER) assert answer.is_integer() is True def test_answer_symbolic_latex_double_dollar(self): """Test symbolic answer with double dollar LaTeX.""" - answer = Answer(value="$$x^2 + y^2$$", answer_category=AnswerCategory.FORMULA) + answer = Answer(value="$$x^2 + y^2$$", answer_kind=AnswerObjectKind.EXPRESSION) clean = answer.get_clean_expression() assert "$$" not in clean or clean == "$$x^2 + y^2$$" def test_answer_symbolic_latex_single_dollar(self): """Test symbolic answer with single dollar LaTeX.""" - answer = Answer(value="$x^2$", answer_category=AnswerCategory.FORMULA) + answer = Answer(value="$x^2$", answer_kind=AnswerObjectKind.EXPRESSION) clean = answer.get_clean_expression() assert "$" not in clean or clean == "$x^2$" def test_answer_symbolic_backslash_latex(self): """Test symbolic answer with backslash LaTeX.""" - answer = Answer(value="\\frac{1}{2}", answer_category=AnswerCategory.FORMULA) + answer = Answer(value="\\frac{1}{2}", answer_kind=AnswerObjectKind.EXPRESSION) assert answer.is_latex() is True def test_answer_textual_word_count_empty(self): """Test word count for empty textual answer.""" - answer = Answer(value="", answer_category=AnswerCategory.TEXT) + answer = Answer(value="", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) assert answer.word_count() == 0 def test_answer_textual_word_count_multiple_spaces(self): """Test word count with multiple spaces.""" answer = Answer( - value="word1 word2 word3", answer_category=AnswerCategory.TEXT + value="word1 word2 word3", + answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT, ) assert answer.word_count() == 3 def test_answer_textual_is_long(self): """Test is_long method for textual answer.""" long_text = " ".join(["word"] * 60) # 60 words - answer = Answer(value=long_text, answer_category=AnswerCategory.TEXT) + answer = Answer(value=long_text, answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) assert answer.is_long() is True def test_answer_textual_contains_keywords_case_insensitive(self): """Test contains_keywords is case insensitive.""" - answer = Answer(value="This is a TEST", answer_category=AnswerCategory.TEXT) + answer = Answer( + value="This is a TEST", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT + ) assert answer.contains_keywords(["test"]) is True assert answer.contains_keywords(["TEST"]) is True assert answer.contains_keywords(["Test"]) is True @@ -226,70 +231,74 @@ def test_answer_textual_contains_keywords_case_insensitive(self): def test_answer_option_all_letters(self): """Test option methods for all letter options.""" for letter in ["A", "B", "C", "D", "E"]: - answer = Answer(value=letter, answer_category=AnswerCategory.OPTION) + answer = Answer(value=letter, answer_kind=AnswerObjectKind.CHOICE) assert answer.is_letter_option() is True assert answer.get_option_index() is not None def test_answer_option_numeric_strings(self): """Test option methods for numeric option strings.""" for num_str in ["1", "2", "3", "4", "5"]: - answer = Answer(value=num_str, answer_category=AnswerCategory.OPTION) + answer = Answer(value=num_str, answer_kind=AnswerObjectKind.CHOICE) assert answer.is_numeric_option() is True assert answer.get_option_index() is not None def test_answer_option_invalid_letter(self): """Test option with invalid letter.""" - answer = Answer(value="F", answer_category=AnswerCategory.OPTION) + answer = Answer(value="F", answer_kind=AnswerObjectKind.CHOICE) assert answer.is_letter_option() is False def test_answer_option_yes_no_variants(self): """Test yes/no option variants.""" for variant in ["YES", "yes", "Yes", "NO", "no", "No"]: - answer = Answer(value=variant, answer_category=AnswerCategory.OPTION) + answer = Answer(value=variant, answer_kind=AnswerObjectKind.CHOICE) assert answer.is_yes_no() is True def test_answer_option_true_false_variants(self): """Test true/false option variants.""" for variant in ["TRUE", "true", "True", "FALSE", "false", "False"]: - answer = Answer(value=variant, answer_category=AnswerCategory.OPTION) + answer = Answer(value=variant, answer_kind=AnswerObjectKind.CHOICE) assert answer.is_true_false() is True def test_answer_to_dict_without_unit(self): """Test to_dict without unit.""" - answer = Answer(value="test", answer_category=AnswerCategory.TEXT) + answer = Answer(value="test", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) result = answer.to_dict() assert "unit" not in result def test_answer_to_dict_without_metadata(self): """Test to_dict with empty metadata.""" - answer = Answer(value=1, answer_category=AnswerCategory.NUMBER) + answer = Answer(value=1, answer_kind=AnswerObjectKind.NUMBER) answer.metadata = {} result = answer.to_dict() # Metadata may or may not be included if empty assert "value" in result - assert "answer_category" in result + assert "answer_kind" in result def test_answer_get_value(self): """Test get_value method.""" - answer = Answer(value=42, answer_category=AnswerCategory.NUMBER) + answer = Answer(value=42, answer_kind=AnswerObjectKind.NUMBER) assert answer.get_value() == 42 def test_answer_get_type(self): """Test get_type method.""" - answer = Answer(value="test", answer_category=AnswerCategory.TEXT) - assert answer.get_type() == AnswerCategory.TEXT + answer = Answer(value="test", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) + assert answer.get_type() == AnswerObjectKind.DESCRIPTIVE_TEXT def test_answer_get_type_name(self): """Test get_type_name method.""" - answer = Answer(value="test", answer_category=AnswerCategory.TEXT) - assert answer.get_type_name() == "text" + answer = Answer(value="test", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) + assert answer.get_type_name() == "descriptive_text" def test_answer_additional_false_paths_and_option_validation(self): - text_answer = Answer(value="text", answer_category=AnswerCategory.TEXT) - option_answer = Answer(value=" ", answer_category=AnswerCategory.OPTION) - numeric_answer = Answer(value=3.5, answer_category=AnswerCategory.NUMBER) - symbolic_answer = Answer(value="plain", answer_category=AnswerCategory.TEXT) - invalid_option = Answer(value="Z", answer_category=AnswerCategory.OPTION) + text_answer = Answer( + value="text", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT + ) + option_answer = Answer(value=" ", answer_kind=AnswerObjectKind.CHOICE) + numeric_answer = Answer(value=3.5, answer_kind=AnswerObjectKind.NUMBER) + symbolic_answer = Answer( + value="plain", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT + ) + invalid_option = Answer(value="Z", answer_kind=AnswerObjectKind.CHOICE) assert option_answer.validate() is False assert text_answer.is_number() is False @@ -306,10 +315,10 @@ def test_answer_additional_false_paths_and_option_validation(self): def test_answer_str_without_unit(self): """Test __str__ without unit.""" - answer = Answer(value=42, answer_category=AnswerCategory.NUMBER) + answer = Answer(value=42, answer_kind=AnswerObjectKind.NUMBER) assert str(answer) == "42" def test_answer_str_non_numerical(self): """Test __str__ for non-numerical answer.""" - answer = Answer(value="test", answer_category=AnswerCategory.TEXT) + answer = Answer(value="test", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) assert str(answer) == "test" diff --git a/tests/prkit/core/domain/test_definitions.py b/tests/prkit/core/domain/test_definitions.py index 7a4f16a..9410d87 100644 --- a/tests/prkit/core/domain/test_definitions.py +++ b/tests/prkit/core/domain/test_definitions.py @@ -1,8 +1,8 @@ """ -Tests for definitions: PhysicsDomain, AnswerCategory. +Tests for definitions: PhysicsDomain, AnswerObjectKind. """ -from prkit.core.domain import AnswerCategory, PhysicsDomain +from prkit.core.domain import AnswerObjectKind, PhysicsDomain class TestPhysicsDomain: @@ -79,35 +79,41 @@ def test_all_domains_accessible(self): assert all(isinstance(d, PhysicsDomain) for d in domains) -class TestAnswerCategory: - """Test cases for AnswerCategory enum.""" +class TestAnswerObjectKind: + """Test cases for the canonical AnswerObjectKind enum.""" - def test_answer_category_enum_values(self): - """Test that answer category enum has expected values.""" - assert AnswerCategory.NUMBER.value == "number" - assert AnswerCategory.EQUATION.value == "equation" - assert AnswerCategory.PHYSICAL_QUANTITY.value == "physical_quantity" - assert AnswerCategory.FORMULA.value == "formula" - assert AnswerCategory.TEXT.value == "text" - assert AnswerCategory.OPTION.value == "option" + def test_answer_kind_enum_values(self): + """Test that the answer-kind enum has expected canonical values.""" + assert AnswerObjectKind.NUMBER.value == "number" + assert AnswerObjectKind.RELATION.value == "relation" + assert AnswerObjectKind.PHYSICAL_QUANTITY.value == "physical_quantity" + assert AnswerObjectKind.EXPRESSION.value == "expression" + assert AnswerObjectKind.QUALITATIVE_LABEL.value == "qualitative_label" + assert AnswerObjectKind.BOOLEAN.value == "boolean" + assert AnswerObjectKind.SIGN_DIRECTION.value == "sign_direction" + assert AnswerObjectKind.DESCRIPTIVE_TEXT.value == "descriptive_text" + assert AnswerObjectKind.CHOICE.value == "choice" - def test_all_answer_categories_accessible(self): - """Test that all answer categories are accessible.""" + def test_all_answer_kinds_accessible(self): + """Test that all nine canonical answer kinds are accessible.""" types = [ - AnswerCategory.NUMBER, - AnswerCategory.EQUATION, - AnswerCategory.PHYSICAL_QUANTITY, - AnswerCategory.FORMULA, - AnswerCategory.TEXT, - AnswerCategory.OPTION, + AnswerObjectKind.NUMBER, + AnswerObjectKind.PHYSICAL_QUANTITY, + AnswerObjectKind.EXPRESSION, + AnswerObjectKind.RELATION, + AnswerObjectKind.QUALITATIVE_LABEL, + AnswerObjectKind.CHOICE, + AnswerObjectKind.BOOLEAN, + AnswerObjectKind.SIGN_DIRECTION, + AnswerObjectKind.DESCRIPTIVE_TEXT, ] - assert len(types) == 6 - assert all(isinstance(t, AnswerCategory) for t in types) + assert len(types) == 9 + assert all(isinstance(t, AnswerObjectKind) for t in types) - def test_answer_category_str(self): + def test_answer_kind_str(self): """Test string representation.""" - assert AnswerCategory.NUMBER.value == "number" - assert AnswerCategory.FORMULA.value == "formula" + assert AnswerObjectKind.NUMBER.value == "number" + assert AnswerObjectKind.EXPRESSION.value == "expression" def test_domain_from_string_lowercase(self): """Test from_string with lowercase input.""" @@ -163,14 +169,14 @@ def test_domain_repr_method(self): assert "CLASSICAL_MECHANICS" in repr_str assert "PhysicsDomain" in repr_str - def test_answer_category_enum_comparison(self): - """Test AnswerCategory enum comparison.""" - assert AnswerCategory.NUMBER == AnswerCategory.NUMBER - assert AnswerCategory.NUMBER != AnswerCategory.FORMULA - - def test_answer_category_value_access(self): - """Test accessing AnswerCategory values.""" - assert AnswerCategory.NUMBER.value == "number" - assert AnswerCategory.FORMULA.value == "formula" - assert AnswerCategory.TEXT.value == "text" - assert AnswerCategory.OPTION.value == "option" + def test_answer_kind_enum_comparison(self): + """Test AnswerObjectKind enum comparison.""" + assert AnswerObjectKind.NUMBER == AnswerObjectKind.NUMBER + assert AnswerObjectKind.NUMBER != AnswerObjectKind.EXPRESSION + + def test_answer_kind_value_access(self): + """Test accessing AnswerObjectKind values.""" + assert AnswerObjectKind.NUMBER.value == "number" + assert AnswerObjectKind.EXPRESSION.value == "expression" + assert AnswerObjectKind.DESCRIPTIVE_TEXT.value == "descriptive_text" + assert AnswerObjectKind.CHOICE.value == "choice" diff --git a/tests/prkit/core/domain/test_physics_problem.py b/tests/prkit/core/domain/test_physics_problem.py index 6900b38..e4cb507 100644 --- a/tests/prkit/core/domain/test_physics_problem.py +++ b/tests/prkit/core/domain/test_physics_problem.py @@ -7,7 +7,7 @@ import pytest -from prkit.core.domain import Answer, AnswerCategory, PhysicsDomain, PhysicsProblem +from prkit.core.domain import Answer, AnswerObjectKind, PhysicsDomain, PhysicsProblem from prkit.core.domain import physics_problem as physics_problem_module @@ -25,7 +25,7 @@ def test_problem_creation_minimal(self): def test_problem_creation_full(self): """Test creating a full physics problem.""" - answer = Answer(value=42, answer_category=AnswerCategory.NUMBER) + answer = Answer(value=42, answer_kind=AnswerObjectKind.NUMBER) problem = PhysicsProblem( problem_id="test_001", question="What is the answer?", @@ -222,7 +222,7 @@ def test_problem_update(self): def test_problem_to_dict(self): """Test problem serialization.""" - answer = Answer(value=42, answer_category=AnswerCategory.NUMBER) + answer = Answer(value=42, answer_kind=AnswerObjectKind.NUMBER) problem = PhysicsProblem( problem_id="test_001", question="Test", @@ -239,24 +239,24 @@ def test_problem_from_dict(self): data = { "problem_id": "test_001", "question": "Test question", - "answer": {"value": 42, "answer_category": "number", "unit": "m/s"}, + "answer": {"value": 42, "answer_kind": "number", "unit": "m/s"}, "domain": "classical_mechanics", } problem = PhysicsProblem.from_dict(data) assert problem.problem_id == "test_001" assert problem.question == "Test question" assert problem.answer.value == 42 - assert problem.answer.answer_category == AnswerCategory.NUMBER + assert problem.answer.answer_kind == AnswerObjectKind.NUMBER fallback = PhysicsProblem.from_dict( { "problem_id": "test_002", "question": "Q", - "answer": {"value": "hello", "answer_category": "not-real"}, + "answer": {"value": "hello", "answer_kind": "not-real"}, "custom_field": "custom", } ) - assert fallback.answer.answer_category == AnswerCategory.TEXT + assert fallback.answer.answer_kind == AnswerObjectKind.DESCRIPTIVE_TEXT assert fallback.additional_fields["custom_field"] == "custom" def test_problem_copy(self): diff --git a/tests/prkit/datasets/loaders/test_base_loader_additional.py b/tests/prkit/datasets/loaders/test_base_loader_additional.py index 96e1fd8..32fca43 100644 --- a/tests/prkit/datasets/loaders/test_base_loader_additional.py +++ b/tests/prkit/datasets/loaders/test_base_loader_additional.py @@ -1,4 +1,4 @@ -from prkit.core.domain import AnswerCategory, PhysicalDataset, PhysicsDomain +from prkit.core.domain import AnswerObjectKind, PhysicalDataset, PhysicsDomain from prkit.datasets.loaders.base_loader import ( BaseDatasetLoader, detect_answer_category, @@ -32,9 +32,12 @@ def test_base_loader_numeric_and_math_detection(): assert is_pure_number("3/4") is True assert is_pure_number("not-a-number") is False assert is_mathematical_expression("x + y") is True - assert detect_answer_category("9.8") == AnswerCategory.NUMBER - assert detect_answer_category("F = ma") == AnswerCategory.FORMULA - assert detect_answer_category("descriptive answer") == AnswerCategory.TEXT + assert detect_answer_category("9.8") == AnswerObjectKind.NUMBER + assert detect_answer_category("F = ma") == AnswerObjectKind.EXPRESSION + assert ( + detect_answer_category("descriptive answer") + == AnswerObjectKind.DESCRIPTIVE_TEXT + ) def test_base_loader_defaults_validation_and_metadata(tmp_path, monkeypatch): @@ -82,7 +85,7 @@ def test_base_loader_creates_problem_and_loads_images(tmp_path): ) assert problem.problem_id == "p1" - assert problem.answer.answer_category == AnswerCategory.NUMBER + assert problem.answer.answer_kind == AnswerObjectKind.NUMBER assert problem.image_path == [str(image_file.resolve())] assert problem.additional_fields["extra"] == "meta" assert loader._determine_problem_type({"options": ["A", "B"]}) == "MC" @@ -102,8 +105,8 @@ def test_base_loader_handles_invalid_image_inputs_and_answer_types(): number_answer = loader._create_answer_from_raw( {"answer": {"value": "5", "unit": "m"}, "answer_category": "physical_quantity"} ) - assert number_answer.answer_category == AnswerCategory.PHYSICAL_QUANTITY + assert number_answer.answer_kind == AnswerObjectKind.PHYSICAL_QUANTITY assert number_answer.unit == "m" fallback_answer = loader._create_answer_from_raw({"answer": "F = ma"}) - assert fallback_answer.answer_category == AnswerCategory.FORMULA + assert fallback_answer.answer_kind == AnswerObjectKind.EXPRESSION diff --git a/tests/prkit/datasets/loaders/test_ugphysics_loader.py b/tests/prkit/datasets/loaders/test_ugphysics_loader.py index c4c2744..42b528d 100644 --- a/tests/prkit/datasets/loaders/test_ugphysics_loader.py +++ b/tests/prkit/datasets/loaders/test_ugphysics_loader.py @@ -8,7 +8,7 @@ import pytest -from prkit.core.domain.answer_category import AnswerCategory +from prkit.core.domain.answer_kinds import AnswerObjectKind from prkit.datasets.loaders import UGPhysicsLoader @@ -249,7 +249,7 @@ def test_load_maps_mc_answers_to_option_category(self, temp_dir): problem = dataset[0] assert problem.problem_type == "MC" - assert problem.answer.answer_category == AnswerCategory.OPTION + assert problem.answer.answer_kind == AnswerObjectKind.CHOICE assert problem.answer.value == "B" def test_load_preserves_multi_answer_metadata(self, temp_dir): @@ -281,7 +281,7 @@ def test_load_preserves_multi_answer_metadata(self, temp_dir): ) problem = dataset[0] - assert problem.answer.answer_category == AnswerCategory.TEXT + assert problem.answer.answer_kind == AnswerObjectKind.DESCRIPTIVE_TEXT assert problem.additional_fields["answer_parts"] == [ {"value": "2", "unit": None}, {"value": "3", "unit": None}, diff --git a/tests/prkit/datasets/test_utils.py b/tests/prkit/datasets/test_utils.py index 1e0889a..c3353bd 100644 --- a/tests/prkit/datasets/test_utils.py +++ b/tests/prkit/datasets/test_utils.py @@ -2,7 +2,7 @@ Tests for utility functions and helper modules. """ -from prkit.core.domain import AnswerCategory +from prkit.core.domain import AnswerObjectKind from prkit.datasets.loaders.base_loader import ( detect_answer_category, is_mathematical_expression, @@ -10,52 +10,52 @@ ) -class TestAnswerCategoryDetection: - """Test cases for answer category detection utilities.""" +class TestAnswerKindDetection: + """Test cases for answer-kind detection utilities.""" def test_detect_answer_category_numerical(self): """Test detecting number answer category.""" - assert detect_answer_category("42") == AnswerCategory.NUMBER - assert detect_answer_category("3.14") == AnswerCategory.NUMBER - assert detect_answer_category("1e-5") == AnswerCategory.NUMBER - assert detect_answer_category("1.23e+10") == AnswerCategory.NUMBER + assert detect_answer_category("42") == AnswerObjectKind.NUMBER + assert detect_answer_category("3.14") == AnswerObjectKind.NUMBER + assert detect_answer_category("1e-5") == AnswerObjectKind.NUMBER + assert detect_answer_category("1.23e+10") == AnswerObjectKind.NUMBER def test_detect_answer_category_fraction(self): """Test detecting fractions as number.""" - assert detect_answer_category("3/4") == AnswerCategory.NUMBER - assert detect_answer_category("1/2") == AnswerCategory.NUMBER + assert detect_answer_category("3/4") == AnswerObjectKind.NUMBER + assert detect_answer_category("1/2") == AnswerObjectKind.NUMBER def test_detect_answer_category_formula(self): """Test detecting formula/symbolic answer category.""" - assert detect_answer_category("x^2 + 1") == AnswerCategory.FORMULA - assert detect_answer_category("\\frac{a}{b}") == AnswerCategory.FORMULA - assert detect_answer_category("$x^2$") == AnswerCategory.FORMULA - assert detect_answer_category("\\boxed{x^2}") == AnswerCategory.FORMULA + assert detect_answer_category("x^2 + 1") == AnswerObjectKind.EXPRESSION + assert detect_answer_category("\\frac{a}{b}") == AnswerObjectKind.EXPRESSION + assert detect_answer_category("$x^2$") == AnswerObjectKind.EXPRESSION + assert detect_answer_category("\\boxed{x^2}") == AnswerObjectKind.EXPRESSION def test_detect_answer_category_text(self): """Test detecting text answer category.""" assert ( detect_answer_category("This is a descriptive answer") - == AnswerCategory.TEXT + == AnswerObjectKind.DESCRIPTIVE_TEXT ) assert ( detect_answer_category("The solution involves multiple steps") - == AnswerCategory.TEXT + == AnswerObjectKind.DESCRIPTIVE_TEXT ) assert ( detect_answer_category("Explanation of the physics concept") - == AnswerCategory.TEXT + == AnswerObjectKind.DESCRIPTIVE_TEXT ) def test_detect_answer_category_with_boxed(self): """Test detecting answer category with \\boxed{} wrapper.""" - assert detect_answer_category("\\boxed{42}") == AnswerCategory.NUMBER - assert detect_answer_category("\\boxed{x^2}") == AnswerCategory.FORMULA + assert detect_answer_category("\\boxed{42}") == AnswerObjectKind.NUMBER + assert detect_answer_category("\\boxed{x^2}") == AnswerObjectKind.EXPRESSION def test_detect_answer_category_with_dollar_signs(self): """Test detecting answer category with $ delimiters.""" - assert detect_answer_category("$42$") == AnswerCategory.NUMBER - assert detect_answer_category("$$x^2$$") == AnswerCategory.FORMULA + assert detect_answer_category("$42$") == AnswerObjectKind.NUMBER + assert detect_answer_category("$$x^2$$") == AnswerObjectKind.EXPRESSION class TestIsPureNumber: diff --git a/tests/prkit/datasets/test_utils_functions.py b/tests/prkit/datasets/test_utils_functions.py index e20a8f4..4ff23af 100644 --- a/tests/prkit/datasets/test_utils_functions.py +++ b/tests/prkit/datasets/test_utils_functions.py @@ -4,7 +4,7 @@ import json -from prkit.core.domain import Answer, AnswerCategory, PhysicalDataset, PhysicsProblem +from prkit.core.domain import Answer, AnswerObjectKind, PhysicalDataset, PhysicsProblem from prkit.datasets import utils @@ -130,7 +130,7 @@ def test_filter_by_keywords_in_question(self, sample_problems_list): problem_id="keyword_test", question="What is the speed of light?", answer=Answer( - value=3e8, answer_category=AnswerCategory.PHYSICAL_QUANTITY, unit="m/s" + value=3e8, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, unit="m/s" ), ) all_problems = list(sample_problems_list) + [problem_with_keyword] @@ -147,7 +147,7 @@ def test_filter_by_keywords_case_insensitive(self, sample_problems_list): problem_id="test_case", question="What is the SPEED of light?", answer=Answer( - value=3e8, answer_category=AnswerCategory.PHYSICAL_QUANTITY, unit="m/s" + value=3e8, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, unit="m/s" ), ) all_problems = list(sample_problems_list) + [problem] @@ -165,7 +165,7 @@ def test_filter_by_keywords_case_sensitive(self, sample_problems_list): problem_id="test_case", question="What is the speed of light?", answer=Answer( - value=3e8, answer_category=AnswerCategory.PHYSICAL_QUANTITY, unit="m/s" + value=3e8, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, unit="m/s" ), ) all_problems = list(sample_problems_list) + [problem] @@ -184,7 +184,7 @@ def test_filter_by_keywords_multiple_fields(self, sample_problems_list): problem_id="test_multi", question="Test question", solution="The answer involves force calculation", - answer=Answer(value=1, answer_category=AnswerCategory.NUMBER), + answer=Answer(value=1, answer_kind=AnswerObjectKind.NUMBER), ) all_problems = list(sample_problems_list) + [problem] dataset = PhysicalDataset(problems=all_problems) diff --git a/tests/prkit/evaluation/comparator/__init__.py b/tests/prkit/evaluation/comparator/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/prkit/evaluation/comparator/test_by_module.py b/tests/prkit/evaluation/comparator/test_by_module.py deleted file mode 100644 index 1ab9da1..0000000 --- a/tests/prkit/evaluation/comparator/test_by_module.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Tests for :mod:`prkit.evaluation.comparator.by_module`.""" - -from __future__ import annotations - -from unittest.mock import patch - -import pytest - -from prkit.evaluation.comparator.by_module import ( - build_comparator, - comparator_module_names, - resolve_comparator_module, -) -from prkit.evaluation.comparator.category_match import CategoryComparator -from prkit.evaluation.comparator.exact_match import ExactMatchComparator -from prkit.evaluation.comparator.record_match import RecordMatchComparator -from prkit.evaluation.comparator.smart_match import SmartMatchComparator -from prkit.evaluation.comparator.typed_llm import TypedLLMComparator - - -def test_comparator_module_names_sorted_unique() -> None: - names = comparator_module_names() - assert names == tuple(sorted(names)) - assert len(names) == len(set(names)) - - -def test_resolve_llm_judge_alias() -> None: - assert resolve_comparator_module("llm_judge") == "typed_llm" - - -def test_build_exact_match() -> None: - c = build_comparator("exact_match") - assert isinstance(c, ExactMatchComparator) - - -@patch("prkit.evaluation.llm_judge.runner.OpenAI") -def test_build_typed_llm_uses_model(mock_openai) -> None: - c = build_comparator("typed_llm", model="gpt-4.1-mini") - assert isinstance(c, TypedLLMComparator) - assert c.model_name == "gpt-4.1-mini" - - -def test_build_category_match() -> None: - c = build_comparator("category_match") - assert isinstance(c, CategoryComparator) - - -def test_build_smart_match() -> None: - c = build_comparator("smart_match") - assert isinstance(c, SmartMatchComparator) - - -def test_build_record_match() -> None: - c = build_comparator("record_match") - assert isinstance(c, RecordMatchComparator) - - -def test_unknown_module_raises() -> None: - with pytest.raises(ValueError, match="Unknown comparator"): - build_comparator("not_a_real_comparator") diff --git a/tests/prkit/evaluation/comparator/test_category_match.py b/tests/prkit/evaluation/comparator/test_category_match.py deleted file mode 100644 index 1a4a52d..0000000 --- a/tests/prkit/evaluation/comparator/test_category_match.py +++ /dev/null @@ -1,503 +0,0 @@ -""" -Unit tests for category_match module. - -Tests cover: -- same_comparison_category (from answer_utils) -- compare_same_type functions (compare_number, compare_plain_text, etc.) -- CategoryComparator (init, compare, accuracy_score, mixed input types) -""" - -from prkit.core.domain import Answer, AnswerCategory -from prkit.evaluation.comparator.category_match import CategoryComparator -from prkit.evaluation.utils.answer_utils import same_comparison_category -from prkit.evaluation.utils.compare_same_type import ( - _formula_to_sympify, - _parse_physical_quantity, - compare_formula, - compare_number, - compare_physical_quantity, - compare_plain_text, -) -from prkit.evaluation.utils.number_utils import DEFAULT_NUMBER_EPSILON - - -class TestSameComparisonCategory: - """Tests for same_comparison_category.""" - - def test_same_category_returns_true(self): - """Same category should return True.""" - assert ( - same_comparison_category(AnswerCategory.NUMBER, AnswerCategory.NUMBER) - is True - ) - assert ( - same_comparison_category(AnswerCategory.TEXT, AnswerCategory.TEXT) is True - ) - - def test_different_category_returns_false(self): - """Different categories should return False.""" - assert ( - same_comparison_category(AnswerCategory.NUMBER, AnswerCategory.TEXT) - is False - ) - assert ( - same_comparison_category(AnswerCategory.FORMULA, AnswerCategory.EQUATION) - is False - ) - - -class TestCompareNumber: - """Tests for compare_number.""" - - def test_equal_floats(self): - """Equal floats should match.""" - assert compare_number(3.14, 3.14) is True - assert compare_number(0.0, 0.0) is True - - def test_floats_within_epsilon(self): - """Floats within epsilon should match.""" - assert compare_number(1.0, 1.0 + DEFAULT_NUMBER_EPSILON / 2) is True - assert compare_number(1e-10, 1e-10 + 1e-20) is True - - def test_floats_outside_epsilon(self): - """Floats outside epsilon should not match.""" - assert compare_number(1.0, 1.0 + 1e-5) is False - assert compare_number(0.0, 0.001) is False - - def test_custom_epsilon(self): - """Custom epsilon should be respected.""" - assert compare_number(1.0, 1.0001, epsilon=0.001) is True - assert compare_number(1.0, 1.0001, epsilon=0.00001) is False - - def test_decimal_place_rounding(self): - """Predicted with more decimals should be rounded to GT precision.""" - # 9.8 has 1 decimal place; 9.86 rounds to 9.9, which matches 9.9 - assert compare_number(9.86, 9.9) is True - # 9.8 vs 9.9 - different - assert compare_number(9.8, 9.9) is False - # 9.84 vs 9.8 - 9.84 rounds to 9.8 (1 decimal) - assert compare_number(9.84, 9.8) is True - - def test_with_answer_objects(self): - """Should handle Answer objects by extracting value.""" - ans1 = Answer(value=3.14, answer_category=AnswerCategory.NUMBER) - ans2 = Answer(value=3.14, answer_category=AnswerCategory.NUMBER) - assert compare_number(ans1, ans2) is True - assert compare_number(ans1, 3.14) is True - assert compare_number(3.14, ans2) is True - - -class TestComparePlainText: - """Tests for compare_plain_text.""" - - def test_equal_strings(self): - """Equal strings should match.""" - assert compare_plain_text("hello", "hello") is True - - def test_unequal_strings(self): - """Unequal strings should not match.""" - assert compare_plain_text("hello", "world") is False - - def test_with_answer_objects(self): - """Should handle Answer objects by extracting value.""" - a1 = Answer(value="foo", answer_category=AnswerCategory.TEXT) - a2 = Answer(value="foo", answer_category=AnswerCategory.TEXT) - assert compare_plain_text(a1, a2) is True - assert compare_plain_text(a1, "foo") is True - assert compare_plain_text("foo", a2) is True - - -class TestParsePhysicalQuantity: - """Tests for _parse_physical_quantity.""" - - def test_simple_number_unit(self): - """Parse 'number unit' format.""" - num, unit = _parse_physical_quantity("9.8 m/s^2") - assert num == 9.8 - assert unit == "m/s^2" - - def test_negative_number_unit(self): - """Parse negative number with unit.""" - num, unit = _parse_physical_quantity("-10000 A/s") - assert num == -10000.0 - assert unit == "A/s" - - def test_fraction_unit(self): - """Parse fraction format (e.g. 500/11).""" - num, unit = _parse_physical_quantity("500/11 kg") - assert abs(num - (500 / 11)) < 1e-10 - assert unit == "kg" - - def test_number_only(self): - """Number only, no unit.""" - num, unit = _parse_physical_quantity("42") - assert num == 42.0 - assert unit == "" - - def test_parse_failure_returns_none(self): - """Parse failure returns (None, full_string).""" - num, full = _parse_physical_quantity("not-a-number m/s") - assert num is None - assert full == "not-a-number m/s" - - def test_division_by_zero(self): - """Fraction with zero denominator returns (None, full_string).""" - num, full = _parse_physical_quantity("1/0 m") - assert num is None - assert full == "1/0 m" - - def test_whitespace_handling(self): - """Leading/trailing whitespace is stripped.""" - num, unit = _parse_physical_quantity(" 3.14 rad ") - assert num == 3.14 - assert "rad" in unit - - def test_number_with_comma(self): - """Parse number with comma as thousands separator.""" - num, unit = _parse_physical_quantity("1,000.5 kg") - assert num == 1000.5 - assert unit == "kg" - - -class TestFormulaToSympify: - """Tests for _formula_to_sympify.""" - - def test_caret_replaced_with_double_star(self): - """^ is replaced with ** for sympify compatibility.""" - assert _formula_to_sympify("x^2") == "x**2" - assert _formula_to_sympify("a^b + c^d") == "a**b + c**d" - - def test_strips_whitespace(self): - """Leading/trailing whitespace is stripped.""" - assert _formula_to_sympify(" x^2 ") == "x**2" - - def test_no_caret_unchanged(self): - """Expressions without ^ are unchanged except strip.""" - assert _formula_to_sympify(" x**2 + 1 ") == "x**2 + 1" - - -class TestComparePhysicalQuantity: - """Tests for compare_physical_quantity.""" - - def test_same_units_numeric_match(self): - """Same units: compare numeric part with epsilon.""" - assert compare_physical_quantity("9.8 m/s^2", "9.8 m/s^2") is True - # Values within epsilon (1e-10) - assert compare_physical_quantity("9.8 m/s^2", "9.80000000001 m/s^2") is True - - def test_same_units_numeric_mismatch(self): - """Same units but different numeric values (even after decimal rounding).""" - # 9.8 vs 15 - no rounding makes them equal - assert compare_physical_quantity("9.8 m/s^2", "15 m/s^2") is False - # With GT precision 1 decimal (10.0), 9.8 should not be rounded to 10. - assert compare_physical_quantity("9.8 m/s^2", "10.0 m/s^2") is False - # With GT precision 0 decimals (10), 9.8 rounds to 10 and matches. - assert compare_physical_quantity("9.8 m/s^2", "10 m/s^2") is True - - def test_different_units_fallback_to_text(self): - """Different units: fall back to plain text (no unit conversion).""" - # Same number, different units - text compare fails - assert compare_physical_quantity("9.8 m/s^2", "9.8 km/h") is False - # Same string would match - assert compare_physical_quantity("9.8 m/s^2", "9.8 m/s^2") is True - - def test_with_answer_objects(self): - """Answer objects with value and unit.""" - ans1 = Answer( - value=-10000, - answer_category=AnswerCategory.PHYSICAL_QUANTITY, - unit="A/s", - ) - ans2 = Answer( - value=-10000.0, - answer_category=AnswerCategory.PHYSICAL_QUANTITY, - unit="A/s", - ) - assert compare_physical_quantity(ans1, ans2) is True - - def test_parse_failure_fallback(self): - """When parse fails, fall back to plain text comparison.""" - # Both unparseable - compare as text - result = compare_physical_quantity("invalid m", "invalid m") - assert result is True - - def test_mixed_answer_and_string(self): - """Answer + string (either order).""" - ans = Answer( - value=-10000, - answer_category=AnswerCategory.PHYSICAL_QUANTITY, - unit="A/s", - ) - assert compare_physical_quantity(ans, "-10000 A/s") is True - assert compare_physical_quantity("-10000 A/s", ans) is True - - def test_custom_epsilon(self): - """Custom epsilon is passed to numeric comparison.""" - assert compare_physical_quantity("1.0 m", "1.001 m", epsilon=0.01) is True - assert compare_physical_quantity("1.0 m", "1.001 m", epsilon=0.0001) is False - - def test_answer_with_unit_none_fallback_to_text(self): - """Answer with unit=None falls back to plain text comparison.""" - ans = Answer( - value=9.8, - answer_category=AnswerCategory.PHYSICAL_QUANTITY, - unit=None, - ) - # pred_unit is None, gt_unit is "m/s^2" - differ, so fall back to text - # pred_string = "9.8 None", gt_string = "9.8 m/s^2" - assert compare_physical_quantity(ans, "9.8 m/s^2") is False - - -class TestCompareFormula: - """Tests for compare_formula.""" - - def test_string_args_equal(self): - """Two equal strings should match.""" - assert compare_formula("x^2 + 1", "x^2 + 1") is True - - def test_caret_converted_to_double_star(self): - """Formula with ^ is converted and compares correctly via sympify.""" - # x^2 becomes x**2; x**2 equals x*x - assert compare_formula("x^2", "x**2") is True - assert compare_formula("x^2", "x*x") is True - - def test_string_args_unequal(self): - """Two different strings should not match.""" - assert compare_formula("x^2", "y^2") is False - - def test_with_answer_objects(self): - """Answer objects: extract value and compare.""" - a1 = Answer(value="a + b", answer_category=AnswerCategory.FORMULA) - a2 = Answer(value="a + b", answer_category=AnswerCategory.FORMULA) - assert compare_formula(a1, a2) is True - - def test_mixed_string_and_answer(self): - """One string, one Answer.""" - a = Answer(value="x + 1", answer_category=AnswerCategory.FORMULA) - assert compare_formula("x + 1", a) is True - assert compare_formula(a, "x + 1") is True - - def test_sympy_structurally_equivalent_expressions(self): - """Mathematically equivalent but structurally different expressions should match.""" - # Reordered terms (commutativity) - assert compare_formula("x + y", "y + x") is True - assert compare_formula("a*b", "b*a") is True - # Expanded vs factored form - assert compare_formula("(x + 1)**2", "x**2 + 2*x + 1") is True - # Simplified form (x/2 vs 0.5*x) - assert compare_formula("x/2", "0.5*x") is True - # Same expression with different representation - assert compare_formula("x**2", "x*x") is True - - def test_sympy_different_expressions(self): - """Mathematically different expressions should not match.""" - assert compare_formula("x + 1", "x + 2") is False - assert compare_formula("x**2", "x**3") is False - assert compare_formula("x + y", "x - y") is False - - def test_sympy_parse_failure_fallback(self): - """On parse failure, fall back to plain text comparison.""" - # Invalid sympy syntax - falls back to text compare - result = compare_formula("invalid formula {{", "invalid formula {{") - assert result is True - result = compare_formula("valid", "invalid formula {{") - assert result is False - - def test_sympify_brace_set_uses_text_fallback(self): - """Braces like ``{1,2}`` sympify to a Python ``set`` (not :class:`~sympy.core.basic.Basic`).""" - assert compare_formula("{1, 2}", "{1, 2}") is True - assert compare_formula("{1, 2}", "{2, 1}") is False - - def test_sympy_interval_vs_expression_no_crash(self): - """SymPy may parse one side as Interval; ``Expr - Interval`` is invalid — no exception.""" - assert compare_formula("x + 1", "Interval(0, 1)") is False - - # --- Multi-strategy symbolic tests (Strategy 2) --- - - def test_trig_identity_sin2_cos2(self): - """sin²(x) + cos²(x) == 1 via trigsimp.""" - assert compare_formula("sin(x)**2 + cos(x)**2", "1") is True - - def test_cancel_rational(self): - """(x^2 - 1)/(x - 1) == x + 1 via cancel.""" - assert compare_formula("(x**2 - 1)/(x - 1)", "x + 1") is True - - def test_factor_vs_expand(self): - """x^3 - x == x*(x-1)*(x+1) via factor/expand.""" - assert compare_formula("x**3 - x", "x*(x - 1)*(x + 1)") is True - - def test_simplify_combined(self): - """(a+b)^2 - (a^2 + 2*a*b + b^2) == 0.""" - assert compare_formula("(a + b)**2 - a**2 - 2*a*b - b**2", "0") is True - - def test_different_after_simplify(self): - """Expressions that are NOT equal should still return False.""" - assert compare_formula("sin(x)**2 + cos(x)**2", "2") is False - - # --- Numerical equivalence tests (Strategy 3) --- - - def test_numerical_constant_equivalence(self): - """Pure-constant expressions with no free variables.""" - assert compare_formula("2**10", "1024") is True - assert compare_formula("3**3", "28") is False - - def test_numerical_multivar(self): - """Multi-variable expression equality checked numerically.""" - assert compare_formula("(x+y)**2", "x**2 + 2*x*y + y**2") is True - - def test_numerical_equivalence_skips_integrals(self): - """Integral-bearing formulas should bypass numeric sampling.""" - from prkit.evaluation.utils import compare_same_type as compare_same_type_module - - pred = compare_same_type_module.sympify("Integral(exp(-x**2), (x, 0, a))") - gt = compare_same_type_module.sympify("Integral(exp(-x**2), (x, 0, b))") - - assert compare_same_type_module._numerical_equivalence(pred, gt) is None - - -class TestCategoryComparator: - """Tests for CategoryComparator class.""" - - def test_init_default(self): - """Default init uses DEFAULT_COMPARATORS.""" - comp = CategoryComparator() - assert AnswerCategory.NUMBER in comp._comparators - assert AnswerCategory.TEXT in comp._comparators - - def test_compare_two_answers_same_category(self): - """Two Answer objects, same category -> category compare.""" - a1 = Answer(value=42.0, answer_category=AnswerCategory.NUMBER) - a2 = Answer(value=42.0, answer_category=AnswerCategory.NUMBER) - comp = CategoryComparator() - assert comp.compare(a1, a2) is True - - def test_compare_two_answers_different_category(self): - """Cross-category pairs use normalized plain-text comparison only.""" - a1 = Answer(value="42", answer_category=AnswerCategory.NUMBER) - a2 = Answer(value="42", answer_category=AnswerCategory.TEXT) - comp = CategoryComparator() - assert comp.compare(a1, a2) is True - a3 = Answer(value="43", answer_category=AnswerCategory.TEXT) - assert comp.compare(a1, a3) is False - - def test_compare_two_strings(self): - """Two string inputs: normalize and compare by category.""" - comp = CategoryComparator() - assert comp.compare("42", "42") is True - assert comp.compare("42", "43") is False - assert comp.compare("hello", "hello") is True - - def test_compare_answer_and_string(self): - """One Answer, one string: should work (normalize string path).""" - comp = CategoryComparator() - ans = Answer(value=42.0, answer_category=AnswerCategory.NUMBER) - # Answer + string: goes to else branch, needs ans1_str/ans2_str - assert comp.compare(ans, "42") is True - assert comp.compare("42", ans) is True - - def test_compare_answer_string_text_category(self): - """TEXT same-type uses shared compare_by_category (normalize_text then plain text).""" - comp = CategoryComparator() - ans = Answer(value=" hello ", answer_category=AnswerCategory.TEXT) - assert comp.compare(ans, "hello") is True - assert comp.compare("hello", ans) is True - - def test_compare_physical_quantity_strings(self): - """String physical quantities.""" - comp = CategoryComparator() - assert comp.compare("9.8 m/s^2", "9.8 m/s^2") is True - # 9.8 rounds to 10 when GT has 0 decimals, so these match - assert comp.compare("9.8 m/s^2", "10 m/s^2") is True - # Clearly different values - assert comp.compare("9.8 m/s^2", "15 m/s^2") is False - - def test_compare_different_categories_text_fallback(self): - """When categories differ, compare as normalized text.""" - comp = CategoryComparator() - # "42" (NUMBER) vs "hello" (TEXT) - different categories, text compare - assert comp.compare("42", "hello") is False - # Same category (both TEXT) - strip and compare - assert comp.compare(" foo ", "foo") is True - - def test_compare_by_category_unknown_fallback(self): - """Unknown category falls back to compare_plain_text.""" - comp = CategoryComparator() - # Remove NUMBER from comparators to simulate unknown - comp._comparators = {AnswerCategory.TEXT: compare_plain_text} - # NUMBER not in comparators -> fallback - result = comp._compare_by_category(AnswerCategory.NUMBER, "42", "42") - assert result is True - - def test_compare_by_category_equation(self): - """EQUATION category uses _compare_plain_text.""" - comp = CategoryComparator() - assert ( - comp._compare_by_category(AnswerCategory.EQUATION, "x = 1", "x = 1") is True - ) - assert ( - comp._compare_by_category(AnswerCategory.EQUATION, "x = 1", "x = 2") - is False - ) - - def test_accuracy_score_match(self): - """accuracy_score returns 1.0 when match.""" - comp = CategoryComparator() - assert comp.accuracy_score("42", "42") == 1.0 - assert ( - comp.accuracy_score( - Answer(value=1, answer_category=AnswerCategory.NUMBER), - Answer(value=1, answer_category=AnswerCategory.NUMBER), - ) - == 1.0 - ) - - def test_accuracy_score_mismatch(self): - """accuracy_score returns 0.0 when no match.""" - comp = CategoryComparator() - assert comp.accuracy_score("42", "43") == 0.0 - - def test_option_answers_case_insensitive_like_typed_llm(self): - """OPTION quick path uses case-insensitive match (same as LLM judge).""" - comp = CategoryComparator() - a1 = Answer(value="A", answer_category=AnswerCategory.OPTION) - a2 = Answer(value="a", answer_category=AnswerCategory.OPTION) - assert comp.compare(a1, a2) is True - - def test_compare_formula_answer_objects(self): - """Answer objects with FORMULA category use compare_formula.""" - comp = CategoryComparator() - a1 = Answer(value="x + y", answer_category=AnswerCategory.FORMULA) - a2 = Answer(value="y + x", answer_category=AnswerCategory.FORMULA) - assert comp.compare(a1, a2) is True - - def test_option_answers_exact_match(self): - """Option answers with same value match.""" - comp = CategoryComparator() - a1 = Answer(value="A", answer_category=AnswerCategory.OPTION) - a2 = Answer(value="A", answer_category=AnswerCategory.OPTION) - assert comp.compare(a1, a2) is True - - def test_can_compare_inherited_from_base(self): - """can_compare returns True (default from BaseComparator).""" - comp = CategoryComparator() - a1 = Answer(value=1, answer_category=AnswerCategory.NUMBER) - a2 = Answer(value=2, answer_category=AnswerCategory.NUMBER) - assert comp.can_compare(a1, a2) is True - - -class TestCategoryComparatorSubclass: - """Tests for CategoryComparator subclass customizing comparators.""" - - def test_subclass_override_comparators(self): - """Subclass can customize _comparators; unknown category falls back to plain text.""" - - class CustomComparator(CategoryComparator): - def __init__(self): - super().__init__() - # Only TEXT comparator - NUMBER falls back to compare_plain_text - self._comparators = {AnswerCategory.TEXT: compare_plain_text} - - comp = CustomComparator() - # NUMBER not in comparators -> fallback to _compare_plain_text - result = comp._compare_by_category(AnswerCategory.NUMBER, "42", "42") - assert result is True diff --git a/tests/prkit/evaluation/comparator/test_normalized_match.py b/tests/prkit/evaluation/comparator/test_normalized_match.py deleted file mode 100644 index 466e4ae..0000000 --- a/tests/prkit/evaluation/comparator/test_normalized_match.py +++ /dev/null @@ -1,28 +0,0 @@ -from prkit.core.domain import Answer, AnswerCategory -from prkit.evaluation.comparator.normalized_match import ( - NormalizedMatchComparator, -) - - -def test_normalized_match_comparator_compares_numbers_by_normalized_value(): - comparator = NormalizedMatchComparator() - assert comparator.compare("4.0", "4") is True - assert comparator.accuracy_score("4.0", "4") == 1.0 - - -def test_normalized_match_comparator_compares_options_case_insensitively(): - comparator = NormalizedMatchComparator() - answer1 = Answer(value="a", answer_category=AnswerCategory.OPTION) - answer2 = Answer(value="A", answer_category=AnswerCategory.OPTION) - assert comparator.compare(answer1, answer2) is True - - -def test_normalized_match_comparator_falls_back_to_text_for_mixed_categories(): - comparator = NormalizedMatchComparator() - assert comparator.compare("Energy", r"\text{Energy}") is False - - -def test_normalized_match_comparator_returns_false_for_mismatched_numbers(): - comparator = NormalizedMatchComparator() - assert comparator.compare("4", "5") is False - assert comparator.accuracy_score("4", "5") == 0.0 diff --git a/tests/prkit/evaluation/comparator/test_record_match.py b/tests/prkit/evaluation/comparator/test_record_match.py deleted file mode 100644 index c6c8db6..0000000 --- a/tests/prkit/evaluation/comparator/test_record_match.py +++ /dev/null @@ -1,264 +0,0 @@ -"""Tests for :mod:`prkit.evaluation.comparator.record_match`.""" - -from __future__ import annotations - -from types import SimpleNamespace - -from prkit.evaluation.comparator.record_match import RecordMatchComparator - - -def _record(**overrides): - record = { - "schema_version": "typed_final_answer.v1", - "status": "ok", - "answer_type": "short_text", - "final_answer": "", - "final_answer_latex": None, - "value": None, - "unit": None, - "option_label": None, - "notes": "", - } - record.update(overrides) - return record - - -def test_formula_uses_final_answer_latex() -> None: - comparator = RecordMatchComparator() - - predicted = _record( - answer_type="formula", - final_answer="unused predicted text", - final_answer_latex="x^2 + y^2", - ) - ground_truth = _record( - answer_type="formula", - final_answer="unused ground truth text", - final_answer_latex="x^2 + y^2", - ) - - assert comparator.compare(predicted, ground_truth) is True - - -def test_equation_uses_final_answer_latex() -> None: - comparator = RecordMatchComparator() - - predicted = _record( - answer_type="equation", - final_answer="unused predicted text", - final_answer_latex="F=ma", - ) - ground_truth = _record( - answer_type="equation", - final_answer="unused ground truth text", - final_answer_latex="F=ma", - ) - - assert comparator.compare(predicted, ground_truth) is True - - -def test_number_uses_value_field() -> None: - comparator = RecordMatchComparator() - - predicted = _record( - answer_type="number", - final_answer="not used", - value="4.0", - ) - ground_truth = _record( - answer_type="number", - final_answer="also not used", - value="4", - ) - - assert comparator.compare(predicted, ground_truth) is True - - -def test_physical_quantity_uses_value_and_unit_fields() -> None: - comparator = RecordMatchComparator() - - predicted = _record( - answer_type="physical_quantity", - final_answer="wrong text", - value="4.5e3", - unit="N", - ) - ground_truth = _record( - answer_type="physical_quantity", - final_answer="another text", - value="4500", - unit="N", - ) - - assert comparator.compare(predicted, ground_truth) is True - - -def test_physical_quantity_unit_mismatch_is_rejected() -> None: - comparator = RecordMatchComparator() - - predicted = _record( - answer_type="physical_quantity", - final_answer="wrong text", - value="9.8", - unit="m/s^2", - ) - ground_truth = _record( - answer_type="physical_quantity", - final_answer="another text", - value="9.8", - unit="km/h", - ) - - assert comparator.compare(predicted, ground_truth) is False - - -def test_short_text_and_option_use_final_answer() -> None: - comparator = RecordMatchComparator() - - predicted_text = _record( - answer_type="short_text", - final_answer="The temperature stays constant.", - ) - ground_truth_text = _record( - answer_type="short_text", - final_answer="temperature stays constant", - ) - predicted_option = _record( - answer_type="option", - final_answer="b", - option_label="B", - ) - ground_truth_option = _record( - answer_type="option", - final_answer="B", - option_label="B", - ) - - assert comparator.compare(predicted_text, ground_truth_text) is True - assert comparator.compare(predicted_option, ground_truth_option) is True - - -def test_cross_type_pq_pred_vs_number_gt() -> None: - comparator = RecordMatchComparator() - - predicted = _record( - answer_type="physical_quantity", - final_answer="9.8 m/s^2", - value="9.8", - unit="m/s^2", - ) - ground_truth = _record( - answer_type="number", - final_answer="9.8", - value="9.8", - ) - - assert comparator.compare(predicted, ground_truth) is True - - -def test_cross_type_number_pred_vs_pq_gt_is_false() -> None: - comparator = RecordMatchComparator() - - predicted = _record( - answer_type="number", - final_answer="9.8", - value="9.8", - ) - ground_truth = _record( - answer_type="physical_quantity", - final_answer="9.8 m/s^2", - value="9.8", - unit="m/s^2", - ) - - assert comparator.compare(predicted, ground_truth) is False - - -def test_cross_type_text_pred_vs_formula_gt() -> None: - comparator = RecordMatchComparator() - - predicted = _record( - answer_type="short_text", - final_answer="v^2", - ) - ground_truth = _record( - answer_type="formula", - final_answer="unused formula text", - final_answer_latex="v**2", - ) - - assert comparator.compare(predicted, ground_truth) is True - - -def test_cross_type_equation_gt_vs_number_pred() -> None: - comparator = RecordMatchComparator() - - predicted = _record( - answer_type="number", - final_answer="355", - value="355", - ) - ground_truth = _record( - answer_type="equation", - final_answer="T_B = 355", - final_answer_latex="T_B = 355", - ) - - assert comparator.compare(predicted, ground_truth) is True - - -def test_cross_type_equation_pred_vs_formula_gt() -> None: - comparator = RecordMatchComparator() - - predicted = _record( - answer_type="equation", - final_answer="f = omega^2", - final_answer_latex=r"f = \omega^2", - ) - ground_truth = _record( - answer_type="formula", - final_answer="omega^2", - final_answer_latex="omega**2", - ) - - assert comparator.compare(predicted, ground_truth) is True - - -def test_attribute_based_records_are_supported() -> None: - comparator = RecordMatchComparator() - - predicted = SimpleNamespace( - **_record( - answer_type="number", - final_answer="unused", - value="3.14", - ) - ) - ground_truth = SimpleNamespace( - **_record( - answer_type="number", - final_answer="unused", - value="3.14", - ) - ) - - assert comparator.compare(predicted, ground_truth) is True - - -def test_non_ok_record_is_incorrect() -> None: - comparator = RecordMatchComparator() - - predicted = _record( - status="unfinished", - answer_type="number", - final_answer="4", - value="4", - ) - ground_truth = _record( - answer_type="number", - final_answer="4", - value="4", - ) - - assert comparator.compare(predicted, ground_truth) is False - assert comparator.accuracy_score(predicted, ground_truth) == 0.0 diff --git a/tests/prkit/evaluation/comparator/test_similarity_match.py b/tests/prkit/evaluation/comparator/test_similarity_match.py deleted file mode 100644 index ba99d48..0000000 --- a/tests/prkit/evaluation/comparator/test_similarity_match.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Tests for SimilarityMatchComparator (quick path, ROUGE-L fallback).""" - -import pytest - -from prkit.core.domain import Answer, AnswerCategory -from prkit.evaluation.comparator.similarity_match import SimilarityMatchComparator - - -class TestSimilarityMatchComparator: - def test_same_type_number_uses_quick_path(self): - comp = SimilarityMatchComparator() - a1 = Answer(value=42.0, answer_category=AnswerCategory.NUMBER) - a2 = Answer(value=42.0, answer_category=AnswerCategory.NUMBER) - assert comp.compare(a1, a2) is True - assert comp.accuracy_score(a1, a2) == 1.0 - assert comp.last_rouge_score is None - - def test_cross_type_high_rouge_matches(self): - comp = SimilarityMatchComparator(rouge_threshold=0.5) - pred = Answer(value="42", answer_category=AnswerCategory.NUMBER) - gt = Answer(value="42", answer_category=AnswerCategory.TEXT) - assert comp.compare(pred, gt) is True - assert comp.accuracy_score(pred, gt) == pytest.approx(1.0) - assert comp.last_rouge_score == pytest.approx(1.0) - - def test_cross_type_low_rouge_no_match(self): - comp = SimilarityMatchComparator(rouge_threshold=0.99) - pred = Answer(value="1", answer_category=AnswerCategory.NUMBER) - gt = Answer( - value="unrelated explanation with many different words", - answer_category=AnswerCategory.TEXT, - ) - assert comp.compare(pred, gt) is False - assert 0.0 <= (comp.last_rouge_score or 0.0) < 0.99 - - def test_same_type_text_uses_rouge_when_quick_none(self): - comp = SimilarityMatchComparator(rouge_threshold=0.5) - pred = Answer(value="aaa", answer_category=AnswerCategory.TEXT) - gt = Answer(value="bbb", answer_category=AnswerCategory.TEXT) - assert comp.compare(pred, gt) is False - assert 0.0 <= comp.accuracy_score(pred, gt) < 0.5 - assert comp.last_rouge_score is not None diff --git a/tests/prkit/evaluation/comparator/test_smart_llm.py b/tests/prkit/evaluation/comparator/test_smart_llm.py deleted file mode 100644 index 0d0cb06..0000000 --- a/tests/prkit/evaluation/comparator/test_smart_llm.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Tests for SmartLLMComparator: deterministic SmartMatch path + LLM fallback metadata.""" - -from unittest.mock import patch - -import pytest - -from prkit.evaluation.comparator.smart_llm import SmartLLMComparator -from prkit.evaluation.llm_judge import ( - RESULT_SOURCE_SKIPPED_LLM, - RESULT_SOURCE_SMART_MATCH, -) - - -class TestSmartLLMComparator: - @pytest.fixture(autouse=True) - def _mock_openai(self): - with patch("prkit.evaluation.llm_judge.runner.OpenAI"): - yield - - def test_match_records_smart_match_source(self): - comp = SmartLLMComparator() - assert comp.compare("42", "42") is True - assert comp.last_result is not None - assert comp.last_result.verdict_type == RESULT_SOURCE_SMART_MATCH - assert comp.last_result.verdict == "correct" - - def test_no_match_records_smart_match_source(self): - comp = SmartLLMComparator() - assert comp.compare("42", "43") is False - assert comp.last_result is not None - assert comp.last_result.verdict_type == RESULT_SOURCE_SMART_MATCH - assert comp.last_result.verdict == "incorrect" - - @pytest.mark.parametrize( - ("pred", "gt"), - [ - ("unrelated text", "3.14"), - ("3.14", "unrelated text"), - ], - ) - def test_cross_inconclusive_skip_llm(self, pred, gt): - """Pairs that reach cross-type with no deterministic verdict defer to LLM.""" - comp = SmartLLMComparator() - assert comp.compare(pred, gt, skip_llm=True) is False - assert comp.last_result is not None - assert comp.last_result.verdict_type == RESULT_SOURCE_SKIPPED_LLM diff --git a/tests/prkit/evaluation/comparator/test_smart_match.py b/tests/prkit/evaluation/comparator/test_smart_match.py deleted file mode 100644 index a452354..0000000 --- a/tests/prkit/evaluation/comparator/test_smart_match.py +++ /dev/null @@ -1,375 +0,0 @@ -""" -Unit tests for smart_match module. - -Tests cover SmartMatchComparator: -- Same-category comparison (number, physical_quantity, formula, text) -- Equation RHS extraction and re-normalization -- Cross-category comparison (e.g. PQ vs NUMBER, TEXT vs FORMULA/EQUATION) -""" - -from prkit.core.domain import Answer, AnswerCategory -from prkit.evaluation.comparator import smart_match as smart_match_module -from prkit.evaluation.comparator.smart_match import ( - SmartMatchComparator, - _extract_latex_equations, - _typed_category_and_value, -) - - -class TestSmartMatchSameType: - """Tests for same-type comparison path.""" - - def test_init_default(self): - """Default init uses DEFAULT_COMPARATORS.""" - comp = SmartMatchComparator() - assert AnswerCategory.NUMBER in comp._comparators - assert AnswerCategory.TEXT in comp._comparators - assert AnswerCategory.PHYSICAL_QUANTITY in comp._comparators - - def test_compare_same_category_number(self): - """Same category NUMBER: compare_number used.""" - comp = SmartMatchComparator() - assert comp.compare("42", "42") is True - assert comp.compare("42", "43") is False - assert comp.compare("3.14", "3.14") is True - - def test_compare_same_category_physical_quantity(self): - """Same category PHYSICAL_QUANTITY: compare_physical_quantity used.""" - comp = SmartMatchComparator() - assert comp.compare("9.8 m/s^2", "9.8 m/s^2") is True - assert comp.compare("9.8 m/s^2", "15 m/s^2") is False - - def test_compare_same_category_text(self): - """Same category TEXT: compare_plain_text used.""" - comp = SmartMatchComparator() - assert comp.compare("hello", "hello") is True - assert comp.compare("hello", "world") is False - - def test_compare_answer_objects(self): - """Answer objects with same category.""" - comp = SmartMatchComparator() - a1 = Answer(value=42.0, answer_category=AnswerCategory.NUMBER) - a2 = Answer(value=42.0, answer_category=AnswerCategory.NUMBER) - assert comp.compare(a1, a2) is True - - def test_compare_formula_commutativity(self): - """Mathematically equivalent formulas should match.""" - comp = SmartMatchComparator() - a1 = Answer(value="x + y", answer_category=AnswerCategory.FORMULA) - a2 = Answer(value="y + x", answer_category=AnswerCategory.FORMULA) - assert comp.compare(a1, a2) is True - - -class TestSmartMatchRHSExtraction: - """Tests for equation RHS extraction path.""" - - def test_equation_extracts_rhs_to_number(self): - """Equation RHS extraction enables match against a number.""" - comp = SmartMatchComparator() - assert comp.compare("x = 42", "42") is True - - def test_equation_answer_extracts_rhs_to_number(self): - """Equation Answer object: RHS extraction enables match against a number.""" - comp = SmartMatchComparator() - pred = Answer(value="T_B = 355", answer_category=AnswerCategory.EQUATION) - gt = Answer(value="355", answer_category=AnswerCategory.NUMBER) - assert comp.compare(pred, gt) is True - - -class TestSmartMatchCrossType: - """Tests for cross-type matching path (moved from CategoryComparator).""" - - def test_pq_pred_vs_number_gt(self): - """PQ(pred) vs NUMBER(gt): compare numeric part.""" - comp = SmartMatchComparator() - assert comp.compare("9.8 m/s^2", "9.8") is True - assert comp.compare("15 m/s^2", "15") is True - assert comp.compare("9.8 m/s^2", "10") is True # rounding - - def test_number_pred_vs_pq_gt_is_false(self): - """NUMBER(pred) vs PQ(gt): missing unit is rejected.""" - comp = SmartMatchComparator() - assert comp.compare("9.8", "9.8 m/s^2") is False - - def test_text_pred_vs_formula_gt(self): - """TEXT(pred) vs FORMULA(gt): formula extraction from text.""" - comp = SmartMatchComparator() - a1 = Answer(value="v^2", answer_category=AnswerCategory.TEXT) - a2 = Answer(value="v**2", answer_category=AnswerCategory.FORMULA) - assert comp.compare(a1, a2) is True - - def test_equation_gt_vs_number_pred(self): - """EQUATION(gt) vs NUMBER(pred): extract RHS from GT equation.""" - comp = SmartMatchComparator() - pred = Answer(value="355", answer_category=AnswerCategory.NUMBER) - gt = Answer(value="T_B = 355", answer_category=AnswerCategory.EQUATION) - assert comp.compare(pred, gt) is True - - def test_equation_gt_vs_pq_pred(self): - """EQUATION(gt) vs PQ(pred): extract RHS from GT equation.""" - comp = SmartMatchComparator() - pred = Answer(value="355 K", answer_category=AnswerCategory.PHYSICAL_QUANTITY) - gt = Answer(value="T_B = 355 K", answer_category=AnswerCategory.EQUATION) - assert comp.compare(pred, gt) is True - - def test_equation_gt_vs_formula_pred(self): - """EQUATION(gt) vs FORMULA(pred): extract RHS, compare formulas.""" - comp = SmartMatchComparator() - pred = Answer(value="omega**2", answer_category=AnswerCategory.FORMULA) - gt = Answer(value=r"f = \omega^2", answer_category=AnswerCategory.EQUATION) - assert comp.compare(pred, gt) is True - - def test_equation_pred_vs_number_gt(self): - """EQUATION(pred) vs NUMBER(gt): extract RHS from pred equation.""" - comp = SmartMatchComparator() - pred = Answer(value="T_B = 355", answer_category=AnswerCategory.EQUATION) - gt = Answer(value="355", answer_category=AnswerCategory.NUMBER) - assert comp.compare(pred, gt) is True - - def test_equation_pred_vs_formula_gt(self): - """EQUATION(pred) vs FORMULA(gt): extract RHS from pred, compare formulas.""" - comp = SmartMatchComparator() - pred = Answer(value=r"f = \omega^2", answer_category=AnswerCategory.EQUATION) - gt = Answer(value="omega**2", answer_category=AnswerCategory.FORMULA) - assert comp.compare(pred, gt) is True - - def test_cross_type_no_false_positive_number_vs_text(self): - """Unrelated cross-type pair returns False (no spurious match).""" - comp = SmartMatchComparator() - a1 = Answer(value="42", answer_category=AnswerCategory.NUMBER) - a2 = Answer(value="hello world", answer_category=AnswerCategory.TEXT) - assert comp.compare(a1, a2) is False - - -class TestSmartMatchEquationFromText: - """Tests for equation-from-text extraction (regression fix + cross-type).""" - - def test_equation_with_preamble_text_regression(self): - """Regression: pred has preamble text that contaminates SymPy parse. - - Both normalize as EQUATION, but the preamble "Paraboloid of revolution:" - produces garbage in the pred SymPy output. The fix extracts the embedded - LaTeX equation and compares its RHS against the GT RHS. - """ - comp = SmartMatchComparator() - pred = r"Paraboloid of revolution: $z(r) = \frac{\omega^2 r^2}{2g}$" - gt = r"$z=\frac{\omega^{2}r^{2}}{2g}$" - assert comp.compare(pred, gt) is True - - def test_text_pred_with_embedded_equation_vs_equation_gt(self): - """TEXT(pred) vs EQUATION(gt): LaTeX equation extracted from free text.""" - comp = SmartMatchComparator() - pred = "The shape is described by $F = ma$" - gt = "$F = ma$" - assert comp.compare(pred, gt) is True - - def test_text_pred_no_equation_substring_fallback(self): - """TEXT(pred) vs EQUATION(gt): substring fallback when no LaTeX found.""" - comp = SmartMatchComparator() - pred = Answer( - value="The answer is Eq(F, a*m)", - answer_category=AnswerCategory.TEXT, - ) - gt = Answer( - value="Eq(F, a*m)", - answer_category=AnswerCategory.EQUATION, - ) - assert comp.compare(pred, gt) is True - - def test_text_pred_no_match_returns_false(self): - """TEXT(pred) vs EQUATION(gt): unrelated text returns False.""" - comp = SmartMatchComparator() - pred = Answer( - value="completely unrelated text", - answer_category=AnswerCategory.TEXT, - ) - gt = Answer( - value="Eq(z, omega**2*r**2/(2*g))", - answer_category=AnswerCategory.EQUATION, - ) - assert comp.compare(pred, gt) is False - - -class TestSmartMatchHelpers: - """Tests for internal helper methods.""" - - def test_extract_latex_equations_and_typed_category_fallback(self, monkeypatch): - assert _extract_latex_equations(r"text $x=1$ and \[y=2\]") == [ - "$x=1$", - r"\[y=2\]", - ] - - monkeypatch.setattr( - smart_match_module, - "normalize_answer", - lambda _answer: (_ for _ in ()).throw(ValueError("bad")), - ) - assert _typed_category_and_value(" ?? ") == (None, "??") - - def test_extract_equation_rhs_raw_simple(self): - """Simple equation RHS extraction.""" - assert SmartMatchComparator._extract_equation_rhs_raw("x = 42") == "42" - assert SmartMatchComparator._extract_equation_rhs_raw("T_B = 355 K") == "355 K" - - def test_extract_equation_rhs_raw_latex_delimiters(self): - """LaTeX delimiters are stripped before extraction.""" - assert SmartMatchComparator._extract_equation_rhs_raw("$T = 300$") == "300" - assert SmartMatchComparator._extract_equation_rhs_raw("$$v = 10$$") == "10" - - def test_extract_equation_rhs_raw_no_equals(self): - """No equals sign returns None.""" - assert SmartMatchComparator._extract_equation_rhs_raw("42") is None - - def test_extract_equation_rhs_raw_multiline(self): - """Only first line is used (ignoring 'where' clauses).""" - raw = "T = 300 K\nwhere T is temperature" - assert SmartMatchComparator._extract_equation_rhs_raw(raw) == "300 K" - - def test_compare_numeric_with_renormalized_variants(self, monkeypatch): - monkeypatch.setattr( - smart_match_module, - "normalize_answer", - lambda rhs: (AnswerCategory.NUMBER, rhs), - ) - assert ( - SmartMatchComparator._compare_numeric_with_renormalized( - AnswerCategory.NUMBER, - "42", - "42", - ) - is True - ) - assert ( - SmartMatchComparator._compare_numeric_with_renormalized( - AnswerCategory.PHYSICAL_QUANTITY, - "42 m", - "42", - ) - is True - ) - - monkeypatch.setattr( - smart_match_module, - "normalize_answer", - lambda _rhs: (_ for _ in ()).throw(ValueError("bad")), - ) - assert ( - SmartMatchComparator._compare_numeric_with_renormalized( - AnswerCategory.NUMBER, - "42", - "oops", - ) - is None - ) - - def test_compare_formula_with_renormalized_fallbacks(self, monkeypatch): - monkeypatch.setattr( - smart_match_module, - "normalize_answer", - lambda _rhs: (AnswerCategory.FORMULA, "x + y"), - ) - monkeypatch.setattr( - smart_match_module, - "compare_formula", - lambda *_args: (_ for _ in ()).throw(ValueError("boom")), - ) - monkeypatch.setattr( - smart_match_module, "compare_plain_text", lambda *_args: True - ) - assert ( - SmartMatchComparator._compare_formula_with_renormalized("x + y", "rhs") - is True - ) - - monkeypatch.setattr( - smart_match_module, - "normalize_answer", - lambda _rhs: (AnswerCategory.TEXT, "words"), - ) - assert ( - SmartMatchComparator._compare_formula_with_renormalized("x + y", "rhs") - is None - ) - - def test_try_equation_from_text_and_cross_type_branches(self, monkeypatch): - comparator = SmartMatchComparator() - - monkeypatch.setattr( - smart_match_module, - "_extract_latex_equations", - lambda _text: ["$bad$", "$good$"], - ) - - def fake_normalize_answer(text): - if text == "$bad$": - raise ValueError("bad equation") - return AnswerCategory.EQUATION, "Eq(x, 2)" - - monkeypatch.setattr( - smart_match_module, "normalize_answer", fake_normalize_answer - ) - monkeypatch.setattr( - smart_match_module, - "compare_by_category", - lambda *_args: False, - ) - monkeypatch.setattr( - smart_match_module, - "extract_rhs_and_category", - lambda _norm, _cat: ("2", AnswerCategory.NUMBER), - ) - monkeypatch.setattr(smart_match_module, "compare_formula", lambda *_args: True) - assert ( - comparator._try_equation_from_text("text", "Eq(x, 2)", "Eq(x, 2)") is True - ) - - monkeypatch.setattr( - comparator, - "_try_equation_from_text", - lambda *_args: False, - ) - monkeypatch.setattr( - comparator, - "_compare_numeric_with_renormalized", - lambda *_args: True, - ) - monkeypatch.setattr( - comparator, - "_compare_formula_with_renormalized", - lambda *_args: True, - ) - assert ( - comparator._cross_type_match( - Answer(value="355", answer_category=AnswerCategory.NUMBER), - Answer(value="T = 355", answer_category=AnswerCategory.EQUATION), - ) - is True - ) - assert ( - comparator._cross_type_match( - Answer(value="x + y", answer_category=AnswerCategory.FORMULA), - Answer(value="z = x + y", answer_category=AnswerCategory.EQUATION), - ) - is True - ) - assert ( - comparator._cross_type_match( - Answer(value="z = x + y", answer_category=AnswerCategory.EQUATION), - Answer(value="x + y", answer_category=AnswerCategory.FORMULA), - ) - is True - ) - - -class TestSmartMatchAccuracyScore: - """Tests for accuracy_score.""" - - def test_accuracy_score_match(self): - """accuracy_score returns 1.0 for match.""" - comp = SmartMatchComparator() - assert comp.accuracy_score("42", "42") == 1.0 - - def test_accuracy_score_mismatch(self): - """accuracy_score returns 0.0 for mismatch.""" - comp = SmartMatchComparator() - assert comp.accuracy_score("42", "43") == 0.0 diff --git a/tests/prkit/evaluation/comparator/test_typed_llm.py b/tests/prkit/evaluation/comparator/test_typed_llm.py deleted file mode 100644 index 5ccbf52..0000000 --- a/tests/prkit/evaluation/comparator/test_typed_llm.py +++ /dev/null @@ -1,543 +0,0 @@ -from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation.comparator import typed_llm as typed_llm_module -from prkit.evaluation.comparator.typed_llm import ( - DEFAULT_MODEL, - TypedLLMComparator, - _compare_formula_or_equation_as_expressions, - _compare_physical_quantity_same_unit_pool_placeholder, - _contains_latex_text_macro, - _normalized_equation_rhs_string, - _plain_text_true_else_llm, - _symbolic_operand_for_expression_compare, - _typed_category_and_value, - infer_symbolic_answer_is_expression, -) -from prkit.evaluation.llm_judge import ( - RESULT_SOURCE_LLM_JUDGE, - RESULT_SOURCE_SKIPPED_LLM, - RESULT_SOURCE_TYPED_MATCH, - parse_judge_response, -) - - -class _DummyResponses: - def __init__(self, response_text: str): - self.response_text = response_text - self.last_params = None - - def create(self, **kwargs): - self.last_params = kwargs - - class _Resp: - output_text = self.response_text - - return _Resp() - - -class _DummyClient: - def __init__(self, response_text: str): - self.responses = _DummyResponses(response_text) - - -def test_default_model_name(): - dummy = _DummyClient( - '{"verdict":"correct","confidence":0.9,"expected_answer_type":"numeric_value","reasoning":"ok"}' - ) - comp = TypedLLMComparator(client=dummy) - assert comp.model_name == DEFAULT_MODEL - - -def test_compare_uses_question_in_payload(): - dummy = _DummyClient( - '{"verdict":"correct","confidence":0.8,"expected_answer_type":"physical_quantity","reasoning":"unit inferred"}' - ) - comp = TypedLLMComparator(client=dummy) - - result = comp.compare("10", "10 N", question="Find the force in N") - assert result is True - payload_text = dummy.responses.last_params["input"][0]["content"][0]["text"] - assert '"expectations_from_question"' not in payload_text - assert "Find the force in N" in payload_text - assert "instructions" in dummy.responses.last_params - - -def test_quick_option_case_insensitive_shortcut(): - dummy = _DummyClient( - '{"verdict":"incorrect","confidence":0.1,"expected_answer_type":"multiple_choice","reasoning":"should not be called"}' - ) - comp = TypedLLMComparator(client=dummy) - pred = Answer(value="A", answer_category=AnswerCategory.OPTION) - gt = Answer(value="a", answer_category=AnswerCategory.OPTION) - assert comp.compare(pred, gt) is True - assert comp.last_result is not None - assert comp.last_result.raw_response == "local_shortcut" - assert comp.last_result.expected_answer_type == "other" - assert comp.last_result.verdict_type == RESULT_SOURCE_TYPED_MATCH - - -def test_same_type_mismatch_uses_quick_typed_path_not_llm(): - dummy = _DummyClient( - '{"verdict":"correct","confidence":0.99,"expected_answer_type":"numeric_value","reasoning":"should not be called"}' - ) - comp = TypedLLMComparator(client=dummy) - assert comp.compare("10", "11") is False - assert comp.last_result is not None - assert comp.last_result.raw_response == "local_shortcut" - assert dummy.responses.last_params is None - - -def test_text_answers_route_to_llm_not_quick_path(): - dummy = _DummyClient( - '{"verdict":"incorrect","confidence":0.77,"expected_answer_type":"textual_concept","reasoning":"extra items included"}' - ) - comp = TypedLLMComparator(client=dummy) - assert ( - comp.compare( - "fine structure, lamb shift", - "fine structure, hyperfine structure", - question="Which corrections apply? Answer in the name of the corrections.", - ) - is False - ) - assert comp.last_result is not None - assert comp.last_result.raw_response != "local_shortcut" - assert dummy.responses.last_params is not None - - -def test_text_exact_match_uses_local_plaintext_shortcut(): - dummy = _DummyClient( - '{"verdict":"incorrect","confidence":0.2,"expected_answer_type":"textual_concept","reasoning":"should not be called"}' - ) - comp = TypedLLMComparator(client=dummy) - pred = Answer( - value="mechanical energy is conserved", answer_category=AnswerCategory.TEXT - ) - gt = Answer( - value="mechanical energy is conserved", answer_category=AnswerCategory.TEXT - ) - assert comp.compare(pred, gt) is True - assert comp.last_result is not None - assert comp.last_result.raw_response == "local_shortcut" - assert dummy.responses.last_params is None - - -def test_formula_with_text_macro_routes_to_llm_not_quick_path(): - dummy = _DummyClient( - '{"verdict":"correct","confidence":0.88,"expected_answer_type":"symbolic_expression","reasoning":"gradient and slope are equivalent labels"}' - ) - comp = TypedLLMComparator(client=dummy) - assert ( - comp.compare(r"M = \frac{\text{slope}}{G}", r"M=\frac{\text{gradient}}{G}") - is True - ) - assert comp.last_result is not None - assert comp.last_result.raw_response != "local_shortcut" - assert dummy.responses.last_params is not None - - -def test_formula_compare_false_then_plaintext_true_shortcuts_locally(monkeypatch): - dummy = _DummyClient( - '{"verdict":"incorrect","confidence":0.2,"expected_answer_type":"symbolic_expression","reasoning":"should not be called"}' - ) - comp = TypedLLMComparator(client=dummy) - - def _always_false_formula(*_args, **_kwargs): - return False - - monkeypatch.setattr(typed_llm_module, "compare_formula", _always_false_formula) - - pred = Answer(value="v = a + b", answer_category=AnswerCategory.FORMULA) - gt = Answer(value="a + b", answer_category=AnswerCategory.FORMULA) - assert comp.compare(pred, gt) is True - assert comp.last_result is not None - assert comp.last_result.raw_response == "local_shortcut" - assert dummy.responses.last_params is None - - -def test_parse_response_with_json_and_fallback(): - parsed = parse_judge_response( - '{"verdict":"incorrect","confidence":0.2,"expected_answer_type":"direction_or_sign","reasoning":"wrong sign"}' - ) - assert parsed.verdict == "incorrect" - assert parsed.confidence == 0.2 - assert parsed.expected_answer_type == "direction_or_sign" - assert parsed.verdict_type == RESULT_SOURCE_LLM_JUDGE - - fallback = parse_judge_response("This is correct.") - assert fallback.verdict == "correct" - assert fallback.expected_answer_type == "other" - assert fallback.verdict_type == RESULT_SOURCE_LLM_JUDGE - - -def test_parse_response_contradictory_reasoning_forces_incorrect(): - parsed = parse_judge_response( - '{"verdict":"correct","confidence":0.99,"expected_answer_type":"symbolic_expression","reasoning":"The model answer is incorrect because it is missing a factor of 2."}' - ) - assert parsed.verdict == "incorrect" - assert parsed.confidence <= 0.35 - - -def test_accuracy_score_is_binary_from_verdict(): - dummy = _DummyClient( - '{"verdict":"correct","confidence":0.73,"expected_answer_type":"symbolic_expression","reasoning":"equivalent"}' - ) - comp = TypedLLMComparator(client=dummy) - score = comp.accuracy_score("x=2", "2", question="Find x") - assert score == 1.0 - - dummy_bad = _DummyClient( - '{"verdict":"incorrect","confidence":0.9,"expected_answer_type":"symbolic_expression","reasoning":"no"}' - ) - comp_bad = TypedLLMComparator(client=dummy_bad) - assert comp_bad.accuracy_score("x=2", "3", question="Find x") == 0.0 - - -def test_parse_unknown_expected_type_falls_back_to_other(): - parsed = parse_judge_response( - '{"verdict":"correct","confidence":0.95,"expected_answer_type":"free_text_blob","reasoning":"ok"}' - ) - assert parsed.expected_answer_type == "other" - - -def test_expression_question_formula_vs_equation_matches_without_llm(): - """Ground-truth expression vs model with v=...; question asks for expression in terms of.""" - dummy = _DummyClient( - '{"verdict":"incorrect","confidence":0.1,"expected_answer_type":"symbolic_expression","reasoning":"should not be called"}' - ) - comp = TypedLLMComparator(client=dummy) - gt = r"$\sqrt{C(T_{1}+T_{2}-2 T)}$" - pred = r"$v = \sqrt{C(T_1 + T_2 - 2T)}$" - q = ( - "What is the speed of the jet in terms of $T_{1}, T_{2}$ and $T$, where $T$ is the " - "temperature of water in the jet?" - ) - assert comp.compare(pred, gt, question=q) is True - assert comp.last_result is not None - assert comp.last_result.raw_response == "local_shortcut" - assert dummy.responses.last_params is None - - -def test_equation_question_formula_vs_equation_can_shortcut_via_plaintext(): - """For equation-style question, if plaintext fallback is affirmative, keep local shortcut.""" - dummy = _DummyClient( - '{"verdict":"correct","confidence":0.9,"expected_answer_type":"symbolic_expression","reasoning":"ok"}' - ) - comp = TypedLLMComparator(client=dummy) - gt = r"$\sqrt{C(T_{1}+T_{2}-2 T)}$" - pred = r"$v = \sqrt{C(T_1 + T_2 - 2T)}$" - q = "Derive the equation relating the jet speed to $T_1$, $T_2$, and $T$." - assert comp.compare(pred, gt, question=q) is True - assert comp.last_result is not None - assert comp.last_result.raw_response == "local_shortcut" - assert dummy.responses.last_params is None - - -def test_equation_question_formula_vs_equation_routes_to_llm_when_plaintext_not_affirmative(): - dummy = _DummyClient( - '{"verdict":"incorrect","confidence":0.9,"expected_answer_type":"symbolic_expression","reasoning":"not equivalent"}' - ) - comp = TypedLLMComparator(client=dummy) - gt = r"$\sqrt{C(T_{1}+T_{2}-2 T)}$" - pred = r"$v = \sqrt{C(T_1 + T_2 + 2T)}$" - q = "Derive the equation relating the jet speed to $T_1$, $T_2$, and $T$." - assert comp.compare(pred, gt, question=q) is False - assert comp.last_result is not None - assert comp.last_result.raw_response != "local_shortcut" - assert dummy.responses.last_params is not None - - -def test_infer_symbolic_answer_is_expression(): - assert ( - infer_symbolic_answer_is_expression( - "What is the speed in terms of $a$ and $b$?" - ) - is True - ) - assert ( - infer_symbolic_answer_is_expression( - "Write the equation of motion for the system." - ) - is False - ) - assert infer_symbolic_answer_is_expression("Solve the problem.") is None - assert ( - infer_symbolic_answer_is_expression( - "Determine the equation for $V$ in terms of $t$, where $V$ is in volts." - ) - is True - ) - assert ( - infer_symbolic_answer_is_expression( - "Derive an equation for the distribution of intensity $I(x, y)$ in the plane." - ) - is False - ) - assert ( - infer_symbolic_answer_is_expression( - "图中（I）是 $t=0$ 时的波形图，写出波动方程的表达式。" - ) - is True - ) - assert ( - infer_symbolic_answer_is_expression( - "图示为两个简谐振动的 $x-t$ 曲线，试分别写出其简谐振动方程。" - ) - is False - ) - - -def test_symbolic_answer_is_expression_kwarg_overrides_question(): - dummy = _DummyClient( - '{"verdict":"incorrect","confidence":0.1,"expected_answer_type":"symbolic_expression","reasoning":"should not be called"}' - ) - comp = TypedLLMComparator(client=dummy) - gt = r"$\sqrt{C(T_{1}+T_{2}-2 T)}$" - pred = r"$v = \sqrt{C(T_1 + T_2 - 2T)}$" - q = "Derive the equation relating the jet speed to temperatures." - assert ( - comp.compare( - pred, - gt, - question=q, - symbolic_answer_is_expression=True, - ) - is True - ) - assert comp.last_result.raw_response == "local_shortcut" - assert dummy.responses.last_params is None - - -def test_ambiguous_physical_quantity_units_route_to_llm(): - dummy = _DummyClient( - '{"verdict":"correct","confidence":0.81,"expected_answer_type":"physical_quantity","reasoning":"equivalent units"}' - ) - comp = TypedLLMComparator(client=dummy) - assert comp.compare("22 rad/s", "22 1/rads", question="Angular frequency") is True - assert comp.last_result is not None - assert comp.last_result.raw_response != "local_shortcut" - assert dummy.responses.last_params is not None - - -def test_skip_llm_returns_dummy_without_api_call(): - dummy = _DummyClient( - '{"verdict":"correct","confidence":0.9,"expected_answer_type":"textual_concept","reasoning":"should not be called"}' - ) - comp = TypedLLMComparator(client=dummy) - assert ( - comp.compare( - "a", - "b", - question="Which corrections apply?", - skip_llm=True, - ) - is False - ) - assert comp.last_result is not None - assert comp.last_result.verdict_type == RESULT_SOURCE_SKIPPED_LLM - assert comp.last_result.raw_response == "skipped_llm" - assert dummy.responses.last_params is None - - -def test_skip_llm_still_uses_typed_match_shortcut(): - dummy = _DummyClient( - '{"verdict":"incorrect","confidence":0.1,"expected_answer_type":"multiple_choice","reasoning":"should not be called"}' - ) - comp = TypedLLMComparator(client=dummy) - pred = Answer(value="A", answer_category=AnswerCategory.OPTION) - gt = Answer(value="a", answer_category=AnswerCategory.OPTION) - assert comp.compare(pred, gt, skip_llm=True) is True - assert comp.last_result.verdict_type == RESULT_SOURCE_TYPED_MATCH - assert dummy.responses.last_params is None - - -def test_accuracy_score_forwards_skip_llm(): - dummy = _DummyClient( - '{"verdict":"correct","confidence":0.9,"expected_answer_type":"textual_concept","reasoning":"should not be called"}' - ) - comp = TypedLLMComparator(client=dummy) - assert ( - comp.accuracy_score( - "a", - "b", - question="Which corrections apply?", - skip_llm=True, - ) - == 0.0 - ) - assert comp.last_result is not None - assert comp.last_result.verdict_type == RESULT_SOURCE_SKIPPED_LLM - assert dummy.responses.last_params is None - - -def test_comparator_exception_routes_to_llm(monkeypatch): - dummy = _DummyClient( - '{"verdict":"correct","confidence":0.7,"expected_answer_type":"numeric_value","reasoning":"LLM fallback on comparator exception"}' - ) - comp = TypedLLMComparator(client=dummy) - - def _raise_compare_number(*_args, **_kwargs): - raise ValueError("forced test error") - - monkeypatch.setattr(typed_llm_module, "compare_number", _raise_compare_number) - - pred = Answer(value="10", answer_category=AnswerCategory.NUMBER) - gt = Answer(value="10", answer_category=AnswerCategory.NUMBER) - assert comp.compare(pred, gt, question="Compute the value") is True - assert comp.last_result is not None - assert comp.last_result.raw_response != "local_shortcut" - assert dummy.responses.last_params is not None - - -def test_helper_functions_cover_fallback_paths(monkeypatch): - assert _contains_latex_text_macro(r"\text{speed}") is True - assert _contains_latex_text_macro("plain text") is False - assert _normalized_equation_rhs_string("Eq(x, 2)") == "2" - assert _normalized_equation_rhs_string("not an equation") is None - assert ( - _symbolic_operand_for_expression_compare(AnswerCategory.FORMULA, "a + b") - == "a + b" - ) - assert ( - _symbolic_operand_for_expression_compare(AnswerCategory.EQUATION, "Eq(v, t)") - == "t" - ) - assert ( - _symbolic_operand_for_expression_compare(AnswerCategory.TEXT, "ignored") is None - ) - assert _compare_physical_quantity_same_unit_pool_placeholder( - "1", "m", "100", "cm" - ) == ( - False, - False, - ) - - monkeypatch.setattr( - typed_llm_module, - "normalize_answer", - lambda _answer: (_ for _ in ()).throw(ValueError("bad")), - ) - assert _typed_category_and_value(" ?? ") == (None, "??") - - -def test_compare_formula_or_equation_as_expressions_handles_non_symbolic_cases( - monkeypatch, -): - assert ( - _compare_formula_or_equation_as_expressions( - AnswerCategory.TEXT, - "x", - AnswerCategory.FORMULA, - "x", - "x", - "x", - ) - is None - ) - assert ( - _compare_formula_or_equation_as_expressions( - AnswerCategory.FORMULA, - "x", - AnswerCategory.TEXT, - "x", - "x", - "x", - ) - is None - ) - assert ( - _compare_formula_or_equation_as_expressions( - AnswerCategory.FORMULA, - "x", - AnswerCategory.EQUATION, - "Eq(y, x)", - r"\text{x}", - "Eq(y, x)", - ) - is None - ) - - monkeypatch.setattr(typed_llm_module, "compare_formula", lambda *_args: True) - assert ( - _compare_formula_or_equation_as_expressions( - AnswerCategory.FORMULA, - "x + y", - AnswerCategory.EQUATION, - "Eq(z, y + x)", - "x + y", - "Eq(z, y + x)", - ) - is True - ) - - monkeypatch.setattr( - typed_llm_module, - "compare_formula", - lambda *_args: (_ for _ in ()).throw(ValueError("boom")), - ) - assert ( - _compare_formula_or_equation_as_expressions( - AnswerCategory.FORMULA, - "x", - AnswerCategory.FORMULA, - "x", - "x", - "x", - ) - is None - ) - - -def test_plain_text_true_else_llm_returns_none_for_false_or_errors(monkeypatch): - assert _plain_text_true_else_llm("x", "y") is None - - monkeypatch.setattr( - typed_llm_module, - "compare_plain_text", - lambda *_args: (_ for _ in ()).throw(TypeError("boom")), - ) - assert _plain_text_true_else_llm("x", "x") is None - - -def test_quick_typed_match_helper_branches(monkeypatch): - monkeypatch.setattr( - typed_llm_module, - "normalize_answer", - lambda _answer: (_ for _ in ()).throw(ValueError("bad")), - ) - assert TypedLLMComparator._quick_typed_match("bad", "still bad") is None - - monkeypatch.setattr( - typed_llm_module, - "_compare_physical_quantity_same_unit_pool_placeholder", - lambda *_args: (True, True), - ) - assert ( - TypedLLMComparator._quick_typed_match( - Answer(value="22 rad/s", answer_category=AnswerCategory.PHYSICAL_QUANTITY), - Answer(value="22 1/rads", answer_category=AnswerCategory.PHYSICAL_QUANTITY), - ) - is True - ) - - monkeypatch.setattr( - typed_llm_module, - "compare_formula", - lambda *_args: (_ for _ in ()).throw(ValueError("bad formula")), - ) - assert ( - TypedLLMComparator._quick_typed_match( - Answer(value="x + y", answer_category=AnswerCategory.FORMULA), - Answer(value="x + y", answer_category=AnswerCategory.FORMULA), - ) - is True - ) - - monkeypatch.setattr(typed_llm_module, "compare_plain_text", lambda *_args: False) - assert ( - TypedLLMComparator._quick_typed_match( - Answer(value="Eq(x, 1)", answer_category=AnswerCategory.EQUATION), - Answer(value="Eq(x, 2)", answer_category=AnswerCategory.EQUATION), - ) - is None - ) diff --git a/tests/prkit/evaluation/evaluator/test_accuracy.py b/tests/prkit/evaluation/evaluator/test_accuracy.py deleted file mode 100644 index 7dbe0ce..0000000 --- a/tests/prkit/evaluation/evaluator/test_accuracy.py +++ /dev/null @@ -1,123 +0,0 @@ -import pytest - -from prkit.core.domain import ( - Answer, - AnswerCategory, - PhysicalDataset, - PhysicsDomain, - PhysicsProblem, -) -from prkit.evaluation.comparator.exact_match import ExactMatchComparator -from prkit.evaluation.evaluator.accuracy import AccuracyEvaluator -from prkit.scoring import SemanticsScorer - - -def _make_problem( - problem_id: str, - answer: Answer | None, - *, - domain=PhysicsDomain.CLASSICAL_MECHANICS, - problem_type="OE", -) -> PhysicsProblem: - return PhysicsProblem( - problem_id=problem_id, - question=f"Question {problem_id}", - answer=answer, - domain=domain, - problem_type=problem_type, - ) - - -def test_accuracy_evaluator_defaults_to_semantics_scorer(): - evaluator = AccuracyEvaluator() - assert isinstance(evaluator.scorer, SemanticsScorer) - assert evaluator.comparator is None - - -def test_accuracy_evaluator_rejects_both_scorer_and_comparator(): - with pytest.raises(ValueError, match="not both"): - AccuracyEvaluator(comparator=ExactMatchComparator(), scorer=SemanticsScorer()) - - -def test_accuracy_evaluator_scorer_path_returns_verdict_backed_details(): - evaluator = AccuracyEvaluator() - result = evaluator.evaluate("4", "4") - - assert result["accuracy_score"] == 1.0 - assert result["comparison_result"] is True - assert result["details"]["scorer_type"] == "SemanticsScorer" - assert result["details"]["scorer_version"] == SemanticsScorer.version - assert result["details"]["comparison_mode"] == "number" - assert result["details"]["predicted_type"] == "string" - - -def test_accuracy_evaluator_legacy_comparator_path_unchanged(): - evaluator = AccuracyEvaluator(comparator=ExactMatchComparator()) - assert evaluator.scorer is None - - result = evaluator.evaluate("4", "4") - - assert result["accuracy_score"] == 1.0 - assert result["comparison_result"] is True - assert result["details"]["comparator_type"] == "ExactMatchComparator" - assert result["details"]["predicted_type"] == "string" - - -def test_accuracy_evaluator_evaluate_dataset_with_predicted_answers(): - dataset = PhysicalDataset( - problems=[ - _make_problem( - "p1", Answer(value="4", answer_category=AnswerCategory.NUMBER) - ), - _make_problem("p2", None), - _make_problem( - "p3", - Answer(value="B", answer_category=AnswerCategory.OPTION), - problem_type="MC", - ), - ] - ) - evaluator = AccuracyEvaluator() - - result = evaluator.evaluate_dataset( - dataset, - predicted_answers={ - "p1": Answer(value="4", answer_category=AnswerCategory.NUMBER), - "p3": Answer(value="A", answer_category=AnswerCategory.OPTION), - }, - ) - - assert result["total_problems"] == 3 - assert result["evaluated_problems"] == 2 - assert result["failed_problems"] == 1 - assert result["overall_accuracy"] == 0.5 - assert result["statistics"]["domain_counts"]["classical_mechanics"] == 2 - assert result["statistics"]["problem_type_counts"]["OE"] == 1 - assert result["statistics"]["problem_type_counts"]["MC"] == 1 - assert result["per_problem_results"][1]["status"] == "no_ground_truth" - - -def test_accuracy_evaluator_evaluate_dataset_with_answer_extractor_and_errors(): - dataset = PhysicalDataset( - problems=[ - _make_problem( - "p1", Answer(value="4", answer_category=AnswerCategory.NUMBER) - ), - _make_problem( - "p2", Answer(value="5", answer_category=AnswerCategory.NUMBER) - ), - ] - ) - evaluator = AccuracyEvaluator() - - def extractor(problem: PhysicsProblem): - if problem.problem_id == "p1": - return Answer(value="4", answer_category=AnswerCategory.NUMBER) - raise RuntimeError("boom") - - result = evaluator.evaluate_dataset(dataset, answer_extractor=extractor) - - assert result["evaluated_problems"] == 1 - assert result["failed_problems"] == 1 - assert result["per_problem_results"][1]["status"] == "error" - assert result["per_problem_results"][1]["details"]["error"] == "boom" diff --git a/tests/prkit/evaluation/llm_judge/test_payload.py b/tests/prkit/evaluation/llm_judge/test_payload.py index f8a0f2c..a109726 100644 --- a/tests/prkit/evaluation/llm_judge/test_payload.py +++ b/tests/prkit/evaluation/llm_judge/test_payload.py @@ -1,5 +1,5 @@ from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_category import AnswerCategory +from prkit.core.domain.answer_kinds import AnswerObjectKind from prkit.evaluation.llm_judge.payload import ( answer_to_text_and_category, build_standard_answer_judge_payload, @@ -9,17 +9,15 @@ def test_answer_to_text_and_category_for_answers_and_plain_strings(): - answer = Answer(value=" 42 ", answer_category=AnswerCategory.NUMBER) + answer = Answer(value=" 42 ", answer_kind=AnswerObjectKind.NUMBER) assert answer_to_text_and_category(answer) == ("42", "number") assert answer_to_text_and_category(" free text ") == ("free text", "unknown") def test_build_standard_answer_judge_payload_cleans_fields(): payload = build_standard_answer_judge_payload( - Answer( - value=" 10\u00a0 m/s ", answer_category=AnswerCategory.PHYSICAL_QUANTITY - ), - Answer(value=" 10\tm/s ", answer_category=AnswerCategory.PHYSICAL_QUANTITY), + Answer(value=" 10\u00a0 m/s ", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), + Answer(value=" 10\tm/s ", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), " What is the speed? ", ) diff --git a/tests/prkit/evaluation/similarities/test_rouge_l.py b/tests/prkit/evaluation/similarities/test_rouge_l.py deleted file mode 100644 index 18f9d3b..0000000 --- a/tests/prkit/evaluation/similarities/test_rouge_l.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Tests for word-level ROUGE-L F1.""" - -from prkit.evaluation.similarities.rouge_l import rouge_l_f1 - - -class TestRougeLF1: - def test_identical(self): - assert rouge_l_f1("hello world", "hello world") == 1.0 - - def test_empty(self): - assert rouge_l_f1("", "a") == 0.0 - assert rouge_l_f1("a", "") == 0.0 - - def test_partial_overlap(self): - s = rouge_l_f1("the cat sat", "the dog sat") - assert 0.0 < s < 1.0 - - def test_case_insensitive(self): - assert rouge_l_f1("Hello WORLD", "hello world") == 1.0 diff --git a/tests/prkit/evaluation/utils/__init__.py b/tests/prkit/evaluation/utils/__init__.py deleted file mode 100644 index 7659e05..0000000 --- a/tests/prkit/evaluation/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Tests for prkit.evaluation.utils.""" diff --git a/tests/prkit/evaluation/utils/test_answer_utils.py b/tests/prkit/evaluation/utils/test_answer_utils.py deleted file mode 100644 index 99b4738..0000000 --- a/tests/prkit/evaluation/utils/test_answer_utils.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -Unit tests for answer_utils module. - -Tests cover: -- to_str (Answer, str, whitespace stripping) -- same_comparison_category (same category, different categories) -""" - -from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation.utils.answer_utils import same_comparison_category, to_str - - -class TestToStr: - """Tests for to_str function.""" - - def test_string_input(self): - """Plain string returns stripped value.""" - assert to_str("hello") == "hello" - assert to_str(" hello ") == "hello" - - def test_answer_with_string_value(self): - """Answer with string value returns stripped value.""" - ans = Answer(value=" foo ", answer_category=AnswerCategory.TEXT) - assert to_str(ans) == "foo" - - def test_answer_with_number_value(self): - """Answer with number value returns string representation.""" - ans = Answer(value=42, answer_category=AnswerCategory.NUMBER) - assert to_str(ans) == "42" - - def test_answer_with_float_value(self): - """Answer with float value returns string representation.""" - ans = Answer(value=3.14, answer_category=AnswerCategory.NUMBER) - assert to_str(ans) == "3.14" - - def test_answer_with_whitespace_value(self): - """Answer value with leading/trailing whitespace is stripped.""" - ans = Answer(value=" x^2 + 1 ", answer_category=AnswerCategory.FORMULA) - assert to_str(ans) == "x^2 + 1" - - def test_empty_string(self): - """Empty string returns empty string.""" - assert to_str("") == "" - assert to_str(" ") == "" - - def test_answer_empty_value(self): - """Answer with empty string value returns empty after strip.""" - ans = Answer(value=" ", answer_category=AnswerCategory.TEXT) - assert to_str(ans) == "" - - -class TestSameComparisonCategory: - """Tests for same_comparison_category function.""" - - def test_same_category_returns_true(self): - """Same category should return True.""" - assert ( - same_comparison_category(AnswerCategory.NUMBER, AnswerCategory.NUMBER) - is True - ) - assert ( - same_comparison_category(AnswerCategory.TEXT, AnswerCategory.TEXT) is True - ) - assert ( - same_comparison_category(AnswerCategory.FORMULA, AnswerCategory.FORMULA) - is True - ) - - def test_different_categories_return_false(self): - """Different categories should return False.""" - assert ( - same_comparison_category(AnswerCategory.NUMBER, AnswerCategory.TEXT) - is False - ) - assert ( - same_comparison_category(AnswerCategory.FORMULA, AnswerCategory.EQUATION) - is False - ) - assert ( - same_comparison_category( - AnswerCategory.PHYSICAL_QUANTITY, AnswerCategory.OPTION - ) - is False - ) - - def test_all_categories_self_match(self): - """Each category matches itself.""" - for cat in AnswerCategory: - assert same_comparison_category(cat, cat) is True diff --git a/tests/prkit/evaluation/utils/test_category_dispatch.py b/tests/prkit/evaluation/utils/test_category_dispatch.py deleted file mode 100644 index 3148a21..0000000 --- a/tests/prkit/evaluation/utils/test_category_dispatch.py +++ /dev/null @@ -1,59 +0,0 @@ -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation.utils.category_dispatch import compare_by_category - - -def test_compare_by_category_normalizes_text_before_dispatch(): - seen = {} - - def compare_text(predicted, ground_truth): - seen["values"] = (predicted, ground_truth) - return predicted == ground_truth - - assert ( - compare_by_category( - AnswerCategory.TEXT, - " HELLO,\nWorld ", - "HELLO, World", - {AnswerCategory.TEXT: compare_text}, - ) - is True - ) - assert seen["values"] == ("HELLO, World", "HELLO, World") - - -def test_compare_by_category_falls_back_to_plain_text_on_exception(): - class Logger: - def __init__(self): - self.messages = [] - - def warning(self, message): - self.messages.append(message) - - logger = Logger() - - def exploding_compare(_predicted, _ground_truth): - raise RuntimeError("boom") - - assert ( - compare_by_category( - AnswerCategory.NUMBER, - "42", - "42", - {AnswerCategory.NUMBER: exploding_compare}, - logger, - ) - is True - ) - assert logger.messages - - -def test_compare_by_category_uses_plain_text_for_unknown_categories(): - assert ( - compare_by_category( - AnswerCategory.EQUATION, - "Eq(x, 2)", - "Eq(x, 2)", - {}, - ) - is True - ) diff --git a/tests/prkit/evaluation/utils/test_compare_cross_type.py b/tests/prkit/evaluation/utils/test_compare_cross_type.py deleted file mode 100644 index 38608fe..0000000 --- a/tests/prkit/evaluation/utils/test_compare_cross_type.py +++ /dev/null @@ -1,112 +0,0 @@ -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation.utils import compare_cross_type as compare_cross_type_module - - -def test_split_respecting_parens_and_expand_gt_set(): - assert compare_cross_type_module.split_respecting_parens("a,(b,c),[d,e]") == [ - "a", - "(b,c)", - "[d,e]", - ] - assert compare_cross_type_module.expand_gt_set("{ a , Eq(x, y) }") == [ - "a", - "Eq(x, y)", - ] - assert compare_cross_type_module.expand_gt_set("single") == ["single"] - - -def test_strip_unbalanced_parens_and_extract_formula_candidates(): - assert compare_cross_type_module.strip_unbalanced_parens("((x+y)") == "(x+y)" - assert compare_cross_type_module.strip_unbalanced_parens("[value]]") == "[value]" - assert compare_cross_type_module.extract_formula_candidates(", F = (ma ; [p]]") == [ - "F = (ma", - "(ma", - "ma", - "[p]]", - "[p]", - ] - - -def test_compare_text_against_formula_or_equation_gt_matches_symbolic_equation( - monkeypatch, -): - monkeypatch.setattr( - compare_cross_type_module, - "extract_formula_candidates", - lambda _text: ["3*t"], - ) - monkeypatch.setattr( - compare_cross_type_module, - "normalize_answer", - lambda _candidate: (AnswerCategory.FORMULA, "3*t"), - ) - monkeypatch.setattr( - compare_cross_type_module, - "compare_formula", - lambda candidate_norm, gt_norm: candidate_norm == gt_norm == "3*t", - ) - - assert ( - compare_cross_type_module.compare_text_against_formula_or_equation_gt( - "The conserved quantity is 3*t.", - "Eq(v, 3*t)", - ) - is True - ) - - -def test_compare_text_against_formula_or_equation_gt_handles_errors_and_quantity_path( - monkeypatch, -): - monkeypatch.setattr( - compare_cross_type_module, - "extract_formula_candidates", - lambda _text: ["", "bad", "Eq(x, 2)", "10 m/s", "words"], - ) - - def fake_normalize_answer(candidate: str): - if candidate == "bad": - raise ValueError("bad candidate") - if candidate == "Eq(x, 2)": - return AnswerCategory.EQUATION, "Eq(x, 2)" - if candidate == "10 m/s": - return AnswerCategory.PHYSICAL_QUANTITY, "10 m/s" - return AnswerCategory.TEXT, candidate - - monkeypatch.setattr( - compare_cross_type_module, - "normalize_answer", - fake_normalize_answer, - ) - monkeypatch.setattr( - compare_cross_type_module, - "extract_rhs_and_category", - lambda normalized, _category: ( - normalized.rsplit(",", 1)[-1].strip(" )"), - AnswerCategory.PHYSICAL_QUANTITY, - ), - ) - - def fake_compare_formula(candidate_norm: str, gt_norm: str) -> bool: - if gt_norm == "2": - raise ValueError("force quantity fallback") - return False - - monkeypatch.setattr( - compare_cross_type_module, - "compare_formula", - fake_compare_formula, - ) - monkeypatch.setattr( - compare_cross_type_module, - "compare_physical_quantity", - lambda candidate_norm, gt_norm: candidate_norm == gt_norm == "10 m/s", - ) - - assert ( - compare_cross_type_module.compare_text_against_formula_or_equation_gt( - "unused", - "{2, 10 m/s}", - ) - is True - ) diff --git a/tests/prkit/evaluation/utils/test_latex_symbol_preprocess.py b/tests/prkit/evaluation/utils/test_latex_symbol_preprocess.py deleted file mode 100644 index 3af27a7..0000000 --- a/tests/prkit/evaluation/utils/test_latex_symbol_preprocess.py +++ /dev/null @@ -1,215 +0,0 @@ -""" -Unit tests for latex_symbol_preprocess module. - -Tests cover: -- _preprocess_latex: empty input, spacing cleanup, protected symbols, - vector/hat decorations, differential standardization, final cleanup -- PROTECTED_PHYSICS_SYMBOLS constant - -Target: 100% coverage. -""" - -from prkit.evaluation.utils.latex_symbol_preprocess import ( - PROTECTED_PHYSICS_SYMBOLS, - _preprocess_latex, -) - - -class TestProtectedPhysicsSymbols: - """Tests for PROTECTED_PHYSICS_SYMBOLS constant.""" - - def test_contains_expected_symbols(self): - """Constant should contain known physics symbols that break parser.""" - expected = { - r"\hbar": "hbar", - r"\mu_0": "mu0", - r"\epsilon_0": "eps0", - r"\varepsilon_0": "eps0", - r"\ell": "ell", - r"\square": "dalembert", - r"\angstrom": "angstrom", - r"\degree": "deg", - } - assert PROTECTED_PHYSICS_SYMBOLS == expected - - -class TestPreprocessLatexEmptyInput: - """Tests for _preprocess_latex with empty/falsy input.""" - - def test_empty_string_returns_empty(self): - """Empty string should return empty string.""" - assert _preprocess_latex("") == "" - - -class TestPreprocessLatexSpacing: - """Tests for LaTeX spacing command replacement.""" - - def test_thin_space(self): - """\\, (thin space) should be replaced with regular space.""" - assert _preprocess_latex(r"a\,b") == "a b" - - def test_medium_space(self): - """\\: (medium space) should be replaced with regular space.""" - assert _preprocess_latex(r"a\:b") == "a b" - - def test_thick_space(self): - """\\; (thick space) should be replaced with regular space.""" - assert _preprocess_latex(r"a\;b") == "a b" - - def test_negative_space(self): - """\\! (negative space) should be replaced with regular space.""" - assert _preprocess_latex(r"a\!b") == "a b" - - def test_quad_space(self): - """\\quad should be replaced with regular space.""" - assert _preprocess_latex(r"a\quad b") == "a b" - - def test_qquad_space(self): - """\\qquad should be replaced with regular space.""" - assert _preprocess_latex(r"a\qquad b") == "a b" - - def test_multiple_spacings_combined(self): - """Multiple spacing commands should all be replaced.""" - result = _preprocess_latex(r"x\,y\:z\;w\!a\quad b\qquad c") - assert result == "x y z w a b c" - - -class TestPreprocessLatexProtectedSymbols: - """Tests for protected physics symbol replacement.""" - - def test_hbar(self): - """\\hbar should become \\mathrm{hbar}.""" - assert _preprocess_latex(r"\hbar") == r"\mathrm{hbar}" - - def test_mu_0(self): - """\\mu_0 should become \\mathrm{mu0}.""" - assert _preprocess_latex(r"\mu_0") == r"\mathrm{mu0}" - - def test_epsilon_0(self): - """\\epsilon_0 should become \\mathrm{eps0}.""" - assert _preprocess_latex(r"\epsilon_0") == r"\mathrm{eps0}" - - def test_ell(self): - """\\ell should become \\mathrm{ell}.""" - assert _preprocess_latex(r"\ell") == r"\mathrm{ell}" - - def test_square(self): - """\\square should become \\mathrm{dalembert}.""" - assert _preprocess_latex(r"\square") == r"\mathrm{dalembert}" - - def test_angstrom(self): - """\\angstrom should become \\mathrm{angstrom}.""" - assert _preprocess_latex(r"\angstrom") == r"\mathrm{angstrom}" - - def test_degree(self): - """\\degree should become \\mathrm{deg}.""" - assert _preprocess_latex(r"\degree") == r"\mathrm{deg}" - - def test_multiple_protected_symbols(self): - """Multiple protected symbols in one string.""" - result = _preprocess_latex(r"E = \hbar \omega, \mu_0") - assert result == r"E = \mathrm{hbar} \omega, \mathrm{mu0}" - - -class TestPreprocessLatexVectorAndHat: - """Tests for vector and hat decoration stripping.""" - - def test_vec_single_char(self): - """\\vec{v} should become v.""" - assert _preprocess_latex(r"\vec{v}") == "v" - - def test_vec_multi_char(self): - """\\vec{F} with multi-char content.""" - assert _preprocess_latex(r"\vec{F}") == "F" - - def test_vec_with_subscript(self): - """\\vec with subscript-like content.""" - assert _preprocess_latex(r"\vec{v_x}") == "v_x" - - def test_hat_single_char(self): - """\\hat{x} should become x.""" - assert _preprocess_latex(r"\hat{x}") == "x" - - def test_hat_multi_char(self): - """\\hat with multi-char content.""" - assert _preprocess_latex(r"\hat{abc}") == "abc" - - def test_vec_and_hat_combined(self): - """Both vec and hat in same string.""" - result = _preprocess_latex(r"\vec{v} + \hat{x}") - assert result == "v + x" - - def test_nested_braces_in_vec(self): - """Vec with inner braces - only first level is matched.""" - # \vec{...} matches up to first } - assert _preprocess_latex(r"\vec{v}") == "v" - # Complex: \vec{a_b} -> a_b (braces inside don't nest for this simple pattern) - assert _preprocess_latex(r"\vec{a_b}") == "a_b" - - -class TestPreprocessLatexDifferentials: - """Tests for differential standardization.""" - - def test_mathrm_d(self): - """\\mathrm{d} should become ' d ' (collapsed to 'd' after cleanup when alone).""" - assert _preprocess_latex(r"\mathrm{d}") == "d" - - def test_text_d(self): - """\\text{d} should become ' d ' (collapsed to 'd' after cleanup when alone).""" - assert _preprocess_latex(r"\text{d}") == "d" - - def test_mathrm_d_in_integral(self): - """\\mathrm{d} in integral context.""" - result = _preprocess_latex(r"\int f(x) \mathrm{d}x") - assert " d " in result or "d" in result - assert "x" in result - - def test_text_d_in_integral(self): - """\\text{d} in integral context.""" - result = _preprocess_latex(r"\int \text{d}t") - assert "d" in result - assert "t" in result - - -class TestPreprocessLatexFinalCleanup: - """Tests for final whitespace cleanup.""" - - def test_multiple_spaces_collapsed(self): - """Multiple spaces should be collapsed to single space.""" - assert _preprocess_latex("a b c") == "a b c" - - def test_leading_trailing_whitespace_stripped(self): - """Leading and trailing whitespace should be stripped.""" - assert _preprocess_latex(" x + y ") == "x + y" - - def test_tabs_and_newlines_become_space(self): - r"""Tabs and newlines in \s+ are collapsed to single space.""" - result = _preprocess_latex("a\t\tb\n\nc") - assert result == "a b c" - - -class TestPreprocessLatexCombined: - """Integration tests with multiple transformations.""" - - def test_full_physics_equation(self): - """Combined: spacing, protected symbols, vec, differentials, cleanup.""" - latex = r"E\,=\ \hbar\omega\quad\text{and}\quad\int\vec{F}\cdot\text{d}x" - result = _preprocess_latex(latex) - assert r"\mathrm{hbar}" in result - assert "vec" not in result # \vec{F} stripped to F - assert "d" in result # \text{d} replaced - assert " " not in result # final cleanup collapses whitespace - - def test_protected_then_vec_order(self): - """Protected symbols are replaced before vec/hat (order in code).""" - # \hbar then \vec{v} - both should be processed - result = _preprocess_latex(r"\hbar \vec{v}") - assert r"\mathrm{hbar}" in result - assert "vec" not in result - assert "v" in result - - def test_no_modification_when_clean(self): - """Simple LaTeX with no special symbols passes through mostly unchanged.""" - result = _preprocess_latex(r"\alpha + \beta") - assert r"\alpha" in result - assert r"\beta" in result diff --git a/tests/prkit/evaluation/utils/test_normalization.py b/tests/prkit/evaluation/utils/test_normalization.py deleted file mode 100644 index 694b440..0000000 --- a/tests/prkit/evaluation/utils/test_normalization.py +++ /dev/null @@ -1,832 +0,0 @@ -""" -Unit tests for normalization module. - -Tests provide full coverage of: -- _parse_numeric_base, _format_numeric_value -- normalize_number -- _match_balanced_braces, _extract_math_content -- _parse_exponent, _normalize_physical_quantity -- _normalize_symbolic_expression, normalize_expression -- _starts_with_latex_delimiter, classify_expression -- normalize_text, normalize_answer -""" - -import math -from unittest.mock import patch - -import pytest - -from prkit.core.domain.answer_category import AnswerCategory -from prkit.evaluation.utils.normalization import ( - _extract_math_content, - _format_numeric_value, - _match_balanced_braces, - _normalize_physical_quantity, - _normalize_symbolic_expression, - _normalize_unicode, - _parse_exponent, - _parse_numeric_base, - _starts_with_latex_delimiter, - classify_expression, - normalize_answer, - normalize_expression, - normalize_number, - normalize_text, -) - -# ============================================================================= -# _normalize_unicode -# ============================================================================= - - -class TestNormalizeUnicode: - """Tests for _normalize_unicode.""" - - def test_unicode_minus_signs(self): - assert _normalize_unicode("5\u2212x") == "5-x" # − MINUS SIGN - assert _normalize_unicode("5\u2013x") == "5-x" # – EN DASH - assert _normalize_unicode("5\u2014x") == "5-x" # — EM DASH - assert _normalize_unicode("5\u2010x") == "5-x" # ‐ HYPHEN - assert _normalize_unicode("5\u2011x") == "5-x" # ‑ NON-BREAKING HYPHEN - assert _normalize_unicode("5\uff0dx") == "5-x" # ﹣ FULLWIDTH - - def test_multiplication_division(self): - assert _normalize_unicode("3\u00d74") == r"3 \times 4" # × - assert _normalize_unicode("3\u00b74") == r"3 \cdot 4" # · MIDDLE DOT - assert _normalize_unicode("3\u22c54") == r"3 \cdot 4" # ⋅ DOT OPERATOR - assert _normalize_unicode("3\u22194") == r"3 \cdot 4" # ∙ BULLET OPERATOR - assert _normalize_unicode("6\u00f72") == r"6 \div 2" # ÷ - - def test_vulgar_fractions(self): - assert _normalize_unicode("\u00bd") == "1/2" # ½ - assert _normalize_unicode("\u2153") == "1/3" # ⅓ - assert _normalize_unicode("\u00bc") == "1/4" # ¼ - assert _normalize_unicode("\u00be") == "3/4" # ¾ - assert _normalize_unicode("\u215e") == "7/8" # ⅞ - - def test_smart_quotes(self): - assert _normalize_unicode("\u201chello\u201d") == '"hello"' - assert _normalize_unicode("\u2018x\u2019") == "'x'" - assert _normalize_unicode("\u00abhi\u00bb") == '"hi"' - - def test_micro_sign_to_mu(self): - assert _normalize_unicode("\u00b5m") == "\u03bcm" # µm → μm - - def test_fullwidth_digits(self): - assert _normalize_unicode("\uff11\uff12\uff13") == "123" - - def test_fullwidth_letters(self): - assert _normalize_unicode("\uff21\uff22\uff23") == "ABC" - assert _normalize_unicode("\uff41\uff42\uff43") == "abc" - - def test_fullwidth_punctuation(self): - assert _normalize_unicode("\uff08x\uff09") == "(x)" - assert _normalize_unicode("\uff5bk\uff5d") == "{k}" - assert _normalize_unicode("a\uff1db") == "a=b" - - def test_subscript_digits(self): - assert _normalize_unicode("x\u2082") == "x_2" # x₂ → x_2 - assert _normalize_unicode("H\u2082O") == "H_2O" # H₂O → H_2O - assert _normalize_unicode("v\u2080") == "v_0" # v₀ → v_0 - - def test_math_relational(self): - assert _normalize_unicode("a\u2264b") == r"a \leq b" # ≤ - assert _normalize_unicode("a\u2265b") == r"a \geq b" # ≥ - assert _normalize_unicode("a\u2260b") == r"a \neq b" # ≠ - assert _normalize_unicode("a\u2248b") == r"a\approx b" # ≈ - - def test_math_operators_and_propto(self): - assert _normalize_unicode("a\u00d7b") == r"a \times b" # × - assert _normalize_unicode("a\u00f7b") == r"a \div b" # ÷ - assert _normalize_unicode("a\u221db") == r"a \propto b" # ∝ - - def test_infinity_and_pm(self): - assert _normalize_unicode("\u221e") == r"\infty" # ∞ - assert _normalize_unicode("5\u00b13") == r"5 \pm 3" # ± - - def test_degree_sign(self): - assert _normalize_unicode("30\u00b0") == "30 deg" # 30° - - def test_idempotent(self): - s = "3.14 m/s^2" - assert _normalize_unicode(_normalize_unicode(s)) == _normalize_unicode(s) - - def test_plain_ascii_unchanged(self): - s = "F = m * a" - assert _normalize_unicode(s) == s - - -class TestUnicodeIntegrationNormalizeNumber: - """Verify Unicode is eliminated before number parsing.""" - - def test_unicode_minus_number(self): - assert ( - normalize_number("5\u22123") == pytest.approx(-53) - or normalize_number("\u22125") == -5.0 - ) - - def test_unicode_fraction_char(self): - assert normalize_number("\u00bd") == pytest.approx(0.5) - - def test_fullwidth_digits(self): - assert normalize_number("\uff14\uff12") == 42.0 - - -class TestUnicodeIntegrationNormalizeText: - """Verify Unicode is eliminated in text normalization.""" - - def test_smart_quotes_in_text(self): - assert normalize_text("\u201chello\u201d") == '"hello"' - - def test_fullwidth_in_text(self): - assert normalize_text("\uff28ello") == "Hello" - - -class TestUnicodeIntegrationNormalizeAnswer: - """End-to-end: Unicode strings categorized and normalized correctly.""" - - def test_vulgar_fraction_as_number(self): - cat, val = normalize_answer("\u00bd") - assert cat == AnswerCategory.NUMBER - assert val == pytest.approx(0.5) - - def test_fullwidth_integer(self): - cat, val = normalize_answer("\uff14\uff12") - assert cat == AnswerCategory.NUMBER - assert val == 42.0 - - def test_unicode_quantity(self): - cat, val = normalize_answer("9.8 m/s\u00b2") # superscript ² - assert cat == AnswerCategory.PHYSICAL_QUANTITY - assert "9.8" in str(val) - - def test_unicode_minus_in_quantity(self): - cat, val = normalize_answer("\u221210 m/s") # −10 m/s - assert cat == AnswerCategory.PHYSICAL_QUANTITY - assert "-10" in str(val) - - -# ============================================================================= -# _parse_numeric_base -# ============================================================================= - - -class TestParseNumericBase: - """Tests for _parse_numeric_base.""" - - def test_simple_integers(self): - assert _parse_numeric_base("500") == 500.0 - assert _parse_numeric_base("-10") == -10.0 - assert _parse_numeric_base("0") == 0.0 - - def test_decimals(self): - assert _parse_numeric_base("9.8") == 9.8 - assert _parse_numeric_base("-3.14") == -3.14 - - def test_fractions(self): - assert _parse_numeric_base("500/11") == pytest.approx(500 / 11) - assert _parse_numeric_base("1/3") == pytest.approx(1 / 3) - assert _parse_numeric_base("-5/2") == pytest.approx(-2.5) - - def test_fraction_with_spaces(self): - assert _parse_numeric_base(" 2 / 3 ") == pytest.approx(2 / 3) - - def test_division_by_zero_returns_none(self): - assert _parse_numeric_base("1/0") is None - assert _parse_numeric_base("5/0") is None - - def test_fraction_value_error_returns_none(self): - assert _parse_numeric_base("abc/3") is None - assert _parse_numeric_base("3/def") is None - - def test_multiple_slashes_not_fraction(self): - """Strings with multiple slashes fall through to float(); 1/2/3 invalid.""" - assert _parse_numeric_base("1/2/3") is None - - def test_whitespace_and_comma_handling(self): - assert _parse_numeric_base(" 500 ") == 500.0 - assert _parse_numeric_base("1,000") == 1000.0 - assert _parse_numeric_base("1,000.5") == 1000.5 - - def test_invalid_returns_none(self): - assert _parse_numeric_base("not a number") is None - assert _parse_numeric_base("") is None - - def test_scientific_notation_via_float(self): - """float() handles scientific notation.""" - assert _parse_numeric_base("1e10") == 1e10 - assert _parse_numeric_base("-3.5e-2") == pytest.approx(-0.035) - - -# ============================================================================= -# _format_numeric_value -# ============================================================================= - - -class TestFormatNumericValue: - """Tests for _format_numeric_value.""" - - def test_integer_values_formatted_as_int(self): - assert _format_numeric_value(5.0) == "5" - assert _format_numeric_value(-10.0) == "-10" - assert _format_numeric_value(0.0) == "0" - - def test_float_values_preserve_decimals(self): - assert _format_numeric_value(3.14) == "3.14" - assert _format_numeric_value(0.5) == "0.5" - - def test_large_whole_number(self): - assert _format_numeric_value(1000000.0) == "1000000" - - -# ============================================================================= -# normalize_number -# ============================================================================= - - -class TestNormalizeNumber: - """Tests for normalize_number.""" - - def test_simple_integers(self): - assert normalize_number("42") == 42.0 - assert normalize_number("-5") == -5.0 - - def test_decimals(self): - assert normalize_number("3.14") == 3.14 - - def test_scientific_notation(self): - assert normalize_number("1e5") == 100000.0 - - def test_fraction_direct_format(self): - assert normalize_number("2/3") == pytest.approx(2 / 3) - - def test_fraction_latex_format(self): - assert normalize_number(r"\frac{2}{3}") == pytest.approx(2 / 3) - assert normalize_number(r"\frac{-5}{2}") == pytest.approx(-2.5) - assert normalize_number(r"\frac{500}{11}") == pytest.approx(500 / 11) - - def test_fraction_latex_with_spaces(self): - """LaTeX \\frac with spaces inside braces - regex requires digits only, returns NaN.""" - result = normalize_number(r"\frac{ 2 }{ 3 }") - assert math.isnan(result) - - def test_fraction_in_boxed(self): - assert normalize_number(r"\boxed{\frac{2}{3}}") == pytest.approx(2 / 3) - - def test_number_in_dollar_delimiters(self): - assert normalize_number(r"$42$") == 42.0 - assert normalize_number(r"$$\frac{1}{2}$$") == pytest.approx(0.5) - - def test_division_by_zero_returns_nan(self): - result = normalize_number(r"\frac{1}{0}") - assert isinstance(result, float) and math.isnan(result) - - def test_invalid_returns_nan(self): - result = normalize_number("not a number") - assert isinstance(result, float) and math.isnan(result) - - def test_frac_pattern_matches_but_parse_fails_returns_nan(self): - """\frac{1}{0} matches pattern but parse returns None -> NaN.""" - result = normalize_number(r"\frac{1}{0}") - assert math.isnan(result) - - def test_number_with_comma(self): - assert normalize_number("1,000.5") == 1000.5 - - -# ============================================================================= -# _match_balanced_braces -# ============================================================================= - - -class TestMatchBalancedBraces: - """Tests for _match_balanced_braces.""" - - def test_simple_balanced_braces(self): - assert _match_balanced_braces("x{y}z", 1) == 3 - - def test_nested_braces(self): - assert _match_balanced_braces("x{y{z}}w", 1) == 6 - # "a{b{c{d}}}e": { at 1 matches } at 9 (indices 0-10) - assert _match_balanced_braces("a{b{c{d}}}e", 1) == 9 - - def test_empty_braces(self): - assert _match_balanced_braces("{}", 0) == 1 - - def test_single_char_in_braces(self): - assert _match_balanced_braces("{x}", 0) == 2 - - def test_start_pos_out_of_range(self): - assert _match_balanced_braces("abc", 3) == -1 - assert _match_balanced_braces("abc", 5) == -1 - - def test_char_at_start_not_open_brace(self): - assert _match_balanced_braces("x{y}z", 0) == -1 - - def test_unbalanced_braces(self): - assert _match_balanced_braces("x{y", 1) == -1 - # "x{y}z}" - { at 1 matches } at 3; extra } at end doesn't affect this - assert _match_balanced_braces("x{y}z}", 1) == 3 - # Start at 0 for "x}y" - pos 0 is 'x', not '{' - assert _match_balanced_braces("x}y", 0) == -1 - - def test_unclosed_brace(self): - assert _match_balanced_braces("x{yyy", 1) == -1 - - def test_custom_open_close_chars(self): - assert _match_balanced_braces("a(b)c", 1, "(", ")") == 3 - assert _match_balanced_braces("a(b(c))d", 1, "(", ")") == 6 - - -# ============================================================================= -# _extract_math_content -# ============================================================================= - - -class TestExtractMathContent: - """Tests for _extract_math_content.""" - - def test_double_dollar_delimiters(self): - text, had = _extract_math_content(r"$$x^2 + 1$$") - assert text == r"x^2 + 1" - assert had is True - - def test_single_dollar_delimiters(self): - text, had = _extract_math_content(r"$F = ma$") - assert text == "F = ma" - assert had is True - - def test_bracket_delimiters(self): - text1, had1 = _extract_math_content(r"\[E = mc^2\]") - assert text1 == "E = mc^2" - assert had1 is True - - text2, had2 = _extract_math_content(r"\(a + b\)") - assert text2 == "a + b" - assert had2 is True - - def test_boxed_delimiter(self): - text, had = _extract_math_content(r"\boxed{42}") - assert text == "42" - assert had is True - - def test_boxed_with_nested_content(self): - text, had = _extract_math_content(r"\boxed{\frac{1}{2}}") - assert "1" in text and "2" in text - assert had is True - - def test_text_delimiter(self): - # \text{} is intentionally preserved — latex2sympy handles it natively - # and stripping it causes multi-letter words to be split into - # implicit-multiplication factors (e.g. gradient -> g*r*a*d*i*e*n*t). - text, had = _extract_math_content(r"\text{hello}") - assert text == r"\text{hello}" - assert had is False - - def test_mathrm_delimiter(self): - text, had = _extract_math_content(r"\mathrm{A}") - assert text == "A" - assert had is True - - def test_nested_delimiters_iterative(self): - text, had = _extract_math_content(r"$$\boxed{x + 1}$$") - assert text == "x + 1" - assert had is True - - def test_text_inside_boxed(self): - text, had = _extract_math_content(r"\boxed{\text{result}}") - assert "result" in text - assert had is True - - def test_latex_spacing_commands_removed(self): - text, _ = _extract_math_content(r"a\;b\,c\:d\!e") - assert "\\;" not in text - assert "\\," not in text - assert "\\:" not in text - assert "\\!" not in text - - def test_no_latex_patterns(self): - text, had = _extract_math_content("plain text") - assert text == "plain text" - assert had is False - - def test_max_iterations_prevents_infinite_loop(self): - nested = r"\boxed{" * 25 + "x" + "}" * 25 - text, had = _extract_math_content(nested) - assert had is True - # Should complete without hanging - - def test_prose_with_inline_math(self): - """Prose like 'from $B$ to $A$' - $...$ stripped.""" - text, had = _extract_math_content("from $B$ to $A$") - assert had is True - assert "B" in text and "A" in text - - -# ============================================================================= -# _parse_exponent -# ============================================================================= - - -class TestParseExponent: - """Tests for _parse_exponent.""" - - def test_none_returns_none(self): - assert _parse_exponent(None) is None - - def test_plain_integer(self): - assert _parse_exponent("4") == 4 - assert _parse_exponent("-2") == -2 - assert _parse_exponent("0") == 0 - - def test_parenthesized_balanced(self): - assert _parse_exponent("((-2))") == -2 - assert _parse_exponent("(2)") == 2 - - def test_parenthesized_unbalanced(self): - assert _parse_exponent("((2") is None - - def test_simple_arithmetic(self): - assert _parse_exponent("2+3") == 5 - assert _parse_exponent("2*3") == 6 - - def test_invalid_returns_none(self): - assert _parse_exponent("abc") is None - - def test_division_by_zero_in_eval(self): - assert _parse_exponent("1/0") is None - - def test_exponent_with_curly_braces_stripped(self): - """Exponent like ^{4} - the caller passes just "4" typically. Test plain.""" - assert _parse_exponent("4") == 4 - - def test_whitespace_stripped(self): - assert _parse_exponent(" 4 ") == 4 - - -# ============================================================================= -# _normalize_physical_quantity -# ============================================================================= - - -class TestNormalizePhysicalQuantity: - """Tests for _normalize_physical_quantity.""" - - def test_plain_quantity(self): - assert _normalize_physical_quantity("9.8 m/s^2") == "9.8 m/s^2" - - def test_with_caret_exponent(self): - assert _normalize_physical_quantity("-10^4 A/s") == "-10000 A/s" - - def test_with_brace_exponent(self): - assert _normalize_physical_quantity("-10^{4} A/s") == "-10000 A/s" - - def test_with_double_star_exponent(self): - assert _normalize_physical_quantity("10**2 m") == "100 m" - - def test_fraction_base(self): - result = _normalize_physical_quantity("500/11 kg") - num_part, unit_part = result.split(None, 1) - assert unit_part == "kg" - assert abs(float(num_part) - 500 / 11) < 1e-10 - - def test_negative_exponent(self): - """**-1 format is supported; ^(-1) has parens that may not match.""" - result = _normalize_physical_quantity("10**-1 s") - assert "0.1" in result and "s" in result - - def test_latex_mathrm_units(self): - result = _normalize_physical_quantity(r"-10^{4} \mathrm{~A}/\mathrm{s}") - assert "mathrm" not in result - assert "-10000" in result - - def test_unicode_whitespace_normalized(self): - result = _normalize_physical_quantity("-10\u00a0\u3000m/s") - assert "\u00a0" not in result - - def test_frac_in_units(self): - result = _normalize_physical_quantity(r"1 \frac{\mathrm{kg}}{\mathrm{m}^3}") - assert "/" in result - - def test_no_match_returns_stripped(self): - assert _normalize_physical_quantity("just text") == "just text" - - def test_base_parse_failure_returns_stripped(self): - result = _normalize_physical_quantity("0/0 m/s") - assert "m/s" in result - - def test_dot_cdot_replaced(self): - result = _normalize_physical_quantity(r"1 \cdot m") - assert "\\cdot" not in result - - def test_tilde_replaced_with_space(self): - result = _normalize_physical_quantity(r"10 ~ m/s") - assert "~" not in result or result.strip() - - @pytest.mark.parametrize( - ("raw_text", "expected"), - [ - (r"3.2 \Omega", "3.2 ohm"), - (r"1100 \ohm", "1100 ohm"), - (r"1000 \AA", "1e-07 m"), - (r"1000 \angstrom", "1e-07 m"), - ], - ) - def test_latex_unit_aliases(self, raw_text, expected): - assert _normalize_physical_quantity(raw_text) == expected - - -# ============================================================================= -# _normalize_symbolic_expression -# ============================================================================= - - -class TestNormalizeSymbolicExpression: - """Tests for _normalize_symbolic_expression.""" - - def test_plain_string_no_latex(self): - result, success = _normalize_symbolic_expression(" x^2 + 1 ", False) - assert result == "x^2 + 1" - assert success is True - - def test_latex_success(self): - result, success = _normalize_symbolic_expression("x + y", True) - assert success is True - assert "x" in result and "y" in result - - @patch("prkit.evaluation.utils.normalization.latex2sympy") - def test_latex_parse_failure_returns_false(self, mock_latex2sympy): - mock_latex2sympy.side_effect = Exception("parse error") - result, success = _normalize_symbolic_expression("invalid \\latex", True) - assert success is False - assert result # Returns preprocessed_math - - -# ============================================================================= -# normalize_expression -# ============================================================================= - - -class TestNormalizeExpression: - """Tests for normalize_expression.""" - - def test_physical_quantity(self): - result, success, cat = normalize_expression("9.8 m/s^2") - assert success is True - assert cat == "physical_quantity" - assert "9.8" in result - - def test_equation(self): - result, success, cat = normalize_expression("F = ma") - assert success is True - assert cat == "equation" - - def test_formula(self): - result, success, cat = normalize_expression("x^2 + 1") - assert success is True - assert cat == "formula" - - def test_formula_multiple_equals(self): - result, success, cat = normalize_expression("a=0, b=1") - assert cat == "formula" - - def test_physical_quantity_with_latex(self): - """LaTeX-wrapped physical quantity.""" - result, success, cat = normalize_expression(r"$-10^{4} \mathrm{A}/\mathrm{s}$") - assert success is True - assert cat == "physical_quantity" - - @pytest.mark.parametrize( - ("raw_text", "expected"), - [ - (r"\lambda \approx 1000 \mathring{\mathrm{A}}", "1e-07 m"), - (r"f = 1.8 \mathrm{Hz}", "1.8 Hz"), - (r"$t=5$ s", "5 s"), - ], - ) - def test_relation_wrapped_quantity_with_latex(self, raw_text, expected): - result, success, cat = normalize_expression(raw_text) - assert success is True - assert cat == "physical_quantity" - assert result == expected - - -# ============================================================================= -# _starts_with_latex_delimiter -# ============================================================================= - - -class TestStartsWithLatexDelimiter: - """Tests for _starts_with_latex_delimiter.""" - - def test_starts_with_double_dollar(self): - assert _starts_with_latex_delimiter("$$ x $$") is True - - def test_starts_with_single_dollar(self): - assert _starts_with_latex_delimiter("$x$") is True - - def test_starts_with_backslash_bracket(self): - assert _starts_with_latex_delimiter(r"\[x\]") is True - assert _starts_with_latex_delimiter(r"\(x\)") is True - - def test_starts_with_boxed(self): - assert _starts_with_latex_delimiter(r"\boxed{42}") is True - - def test_starts_with_frac(self): - assert _starts_with_latex_delimiter(r"\frac{1}{2}") is True - - def test_starts_with_text(self): - assert _starts_with_latex_delimiter(r"\text{hello}") is True - - def test_starts_with_mathrm(self): - assert _starts_with_latex_delimiter(r"\mathrm{A}") is True - - def test_plain_text_returns_false(self): - assert _starts_with_latex_delimiter("plain text") is False - assert _starts_with_latex_delimiter("9.8 m/s^2") is False - - def test_leading_spaces_then_dollar(self): - assert _starts_with_latex_delimiter(" $B$") is True - - def test_dollar_not_at_start_returns_false(self): - """'from $B$ to $A$' - $ is not at start (after optional whitespace, it's 'from').""" - assert _starts_with_latex_delimiter("from $B$ to $A$") is False - - -# ============================================================================= -# classify_expression -# ============================================================================= - - -class TestClassifyExpression: - """Tests for classify_expression.""" - - def test_equation_single_equals(self): - assert classify_expression("F = ma") == "equation" - assert classify_expression("x = 1") == "equation" - - def test_physical_quantity_plain(self): - assert classify_expression("9.8 m/s^2") == "physical_quantity" - - def test_physical_quantity_with_exponent(self): - assert classify_expression("-10^4 A/s") == "physical_quantity" - assert classify_expression("10**2 m") == "physical_quantity" - - def test_physical_quantity_with_fraction_base(self): - assert classify_expression("500/11 kg") == "physical_quantity" - - def test_physical_quantity_with_mathrm(self): - r"""\mathrm{...} in units is replaced for pattern match.""" - assert classify_expression(r"-10 \mathrm{m}/\mathrm{s}") == "physical_quantity" - - @pytest.mark.parametrize( - "raw_text", - [ - r"3.2 \Omega", - r"1100 \ohm", - r"\lambda \approx 1000 \mathring{\mathrm{A}}", - r"1000 \AA", - r"1000 \angstrom", - r"f = 1.8 \mathrm{Hz}", - r"$t=5$ s", - ], - ) - def test_physical_quantity_alias_and_relation_wrapped_cases(self, raw_text): - assert classify_expression(raw_text) == "physical_quantity" - - def test_formula_multiple_equals(self): - assert classify_expression("a=0, b=1") == "formula" - - def test_formula_no_equals(self): - assert classify_expression("x^2 + y^2") == "formula" - - def test_equation_latex_inequalities(self): - """Unicode → LaTeX in _normalize_unicode; still classify as equation.""" - assert classify_expression(r"a \leq b") == "equation" - assert classify_expression(r"x \geq y") == "equation" - assert classify_expression(r"p \neq q") == "equation" - assert classify_expression(r"F \propto a") == "equation" - - -# ============================================================================= -# normalize_text -# ============================================================================= - - -class TestNormalizeText: - """Tests for normalize_text.""" - - def test_strips_whitespace(self): - assert normalize_text(" hello ") == "hello" - - def test_preserves_inner_space(self): - assert normalize_text("hello world") == "hello world" - - def test_empty_after_strip(self): - assert normalize_text(" ") == "" - - -# ============================================================================= -# normalize_answer -# ============================================================================= - - -class TestNormalizeAnswer: - """Tests for normalize_answer.""" - - def test_number_success(self): - cat, val = normalize_answer("42") - assert cat == AnswerCategory.NUMBER - assert val == 42.0 - - def test_number_fraction_success(self): - cat, val = normalize_answer("2/3") - assert cat == AnswerCategory.NUMBER - assert val == pytest.approx(2 / 3) - - def test_number_latex_frac_success(self): - cat, val = normalize_answer(r"\frac{1}{2}") - assert cat == AnswerCategory.NUMBER - assert val == pytest.approx(0.5) - - def test_plain_physical_quantity_no_latex(self): - """No LaTeX prefix, but matches physical_quantity -> PHYSICAL_QUANTITY.""" - cat, val = normalize_answer("-10^4 A/s") - assert cat == AnswerCategory.PHYSICAL_QUANTITY - assert "-10000" in str(val) - - def test_plain_physical_quantity_9_8_ms2(self): - """'9.8 m/s^2' without LaTeX -> physical_quantity (Step 2 path).""" - cat, val = normalize_answer("9.8 m/s^2") - assert cat == AnswerCategory.PHYSICAL_QUANTITY - assert "9.8" in str(val) - - def test_no_latex_not_physical_quantity_returns_text(self): - cat, val = normalize_answer("some prose answer") - assert cat == AnswerCategory.TEXT - assert val == "some prose answer" - - def test_prose_with_inline_math_returns_text(self): - """'from $B$ to $A$' - no LaTeX at start -> TEXT.""" - cat, val = normalize_answer("from $B$ to $A$") - assert cat == AnswerCategory.TEXT - assert "B" in val and "A" in val - - def test_latex_equation_success(self): - cat, val = normalize_answer(r"$F = ma$") - assert cat == AnswerCategory.EQUATION - - def test_latex_physical_quantity_success(self): - cat, val = normalize_answer(r"$-10^{4} \mathrm{A}/\mathrm{s}$") - assert cat == AnswerCategory.PHYSICAL_QUANTITY - - def test_latex_formula_success(self): - cat, val = normalize_answer(r"$x^2 + 1$") - assert cat == AnswerCategory.FORMULA - - @patch("prkit.evaluation.utils.normalization.latex2sympy") - def test_latex_expression_fail_fallback_to_text(self, mock_latex2sympy): - mock_latex2sympy.side_effect = Exception("parse error") - cat, val = normalize_answer(r"$\invalid\latex$") - assert cat == AnswerCategory.TEXT - assert isinstance(val, str) - - def test_number_zero(self): - cat, val = normalize_answer("0") - assert cat == AnswerCategory.NUMBER - assert val == 0.0 - - def test_latex_thin_space_does_not_merge_unit_tokens(self): - r"""$22 \mathrm{rad}\,\mathrm{s}^{-1}$ must not merge 'rad'+'s' into 'rads'.""" - cat, val = normalize_answer(r"$22 \mathrm{rad}\,\mathrm{s}^{-1}$") - assert cat == AnswerCategory.PHYSICAL_QUANTITY - assert val == "22 rad/s" - - def test_latex_thick_space_does_not_merge_unit_tokens(self): - r"""Units separated by \; must stay separate.""" - cat, val = normalize_answer(r"$5\;\mathrm{m}\,\mathrm{s}^{-2}$") - assert cat == AnswerCategory.PHYSICAL_QUANTITY - assert val == "5 m/s^2" - - def test_latex_backslash_space_before_text_unit(self): - r"""$3.8\ \text{Hz}$ (SeePhys-style): \text{Hz} must classify as quantity, not formula.""" - cat, val = normalize_answer(r"$3.8\ \text{Hz}$") - assert cat == AnswerCategory.PHYSICAL_QUANTITY - assert val == "3.8 Hz" - - @pytest.mark.parametrize( - ("raw_text", "expected"), - [ - (r"3.2 \Omega", "3.2 ohm"), - (r"1100 \ohm", "1100 ohm"), - (r"\lambda \approx 1000 \mathring{\mathrm{A}}", "1e-07 m"), - (r"1000 \AA", "1e-07 m"), - (r"1000 \angstrom", "1e-07 m"), - (r"f = 1.8 \mathrm{Hz}", "1.8 Hz"), - (r"$t=5$ s", "5 s"), - ], - ) - def test_quantity_alias_and_relation_wrapped_regressions(self, raw_text, expected): - cat, val = normalize_answer(raw_text) - assert cat == AnswerCategory.PHYSICAL_QUANTITY - assert val == expected diff --git a/tests/prkit/evaluation/utils/test_number_utils.py b/tests/prkit/evaluation/utils/test_number_utils.py deleted file mode 100644 index e25956c..0000000 --- a/tests/prkit/evaluation/utils/test_number_utils.py +++ /dev/null @@ -1,120 +0,0 @@ -""" -Unit tests for number_utils module. - -Tests cover: -- DEFAULT_NUMBER_EPSILON -- decimal_places -- round_to_decimal_places - -Target: 100% coverage. -""" - -import math - -from prkit.evaluation.utils.number_utils import ( - DEFAULT_NUMBER_EPSILON, - decimal_places, - round_to_decimal_places, -) - - -class TestDefaultNumberEpsilon: - """Tests for DEFAULT_NUMBER_EPSILON constant.""" - - def test_value(self): - """Default epsilon should be 1e-10.""" - assert DEFAULT_NUMBER_EPSILON == 1e-10 - - -class TestDecimalPlaces: - """Tests for decimal_places function.""" - - def test_zero_returns_zero(self): - """Zero should return 0 decimal places.""" - assert decimal_places(0) == 0 - assert decimal_places(0.0) == 0 - - def test_nan_returns_zero(self): - """NaN should return 0 decimal places.""" - assert decimal_places(float("nan")) == 0 - assert decimal_places(math.nan) == 0 - - def test_inf_returns_zero(self): - """Positive and negative infinity should return 0 decimal places.""" - assert decimal_places(float("inf")) == 0 - assert decimal_places(math.inf) == 0 - assert decimal_places(float("-inf")) == 0 - assert decimal_places(-math.inf) == 0 - - def test_single_decimal_place(self): - """Floats with one decimal place.""" - assert decimal_places(9.8) == 1 - assert decimal_places(-9.8) == 1 - assert decimal_places(1.0) == 0 # 1.0 formats as "1" -> 0 - - def test_multiple_decimal_places(self): - """Floats with multiple decimal places.""" - assert decimal_places(9.87) == 2 - assert decimal_places(3.14159) == 5 - assert decimal_places(0.00123) == 5 - - def test_integers_and_whole_numbers(self): - """Integers and floats that represent whole numbers should return 0.""" - assert decimal_places(500.0) == 0 - assert decimal_places(100) == 0 - assert decimal_places(-42.0) == 0 - - def test_scientific_notation(self): - """Numbers in scientific notation should infer decimal places from f format.""" - # 1.5e-05 -> format .15g gives "1.5e-05", triggers e branch - assert decimal_places(1.5e-05) == 6 - # 1e-10 -> format .15g gives "1e-10" - assert decimal_places(1e-10) == 10 - - def test_large_numbers_no_scientific(self): - """Large numbers that format without 'e' (e.g. 1e10 -> 10000000000).""" - assert decimal_places(1e10) == 0 - assert decimal_places(10000000000.0) == 0 - - def test_trailing_zeros_stripped(self): - """Trailing zeros should be stripped before counting.""" - # 1.200 -> "1.2" after rstrip -> 1 decimal place - assert decimal_places(1.2) == 1 - assert decimal_places(9.80) == 1 - - -class TestRoundToDecimalPlaces: - """Tests for round_to_decimal_places function.""" - - def test_positive_n_rounds(self): - """When n >= 0, should round to n decimal places.""" - assert round_to_decimal_places(3.14159, 2) == 3.14 - assert round_to_decimal_places(3.14159, 0) == 3.0 - assert round_to_decimal_places(3.14159, 4) == 3.1416 - assert round_to_decimal_places(9.876, 1) == 9.9 - - def test_n_zero(self): - """n=0 should round to nearest integer.""" - assert round_to_decimal_places(3.7, 0) == 4.0 - assert round_to_decimal_places(3.4, 0) == 3.0 - - def test_negative_n_returns_unchanged(self): - """When n < 0, should return x unchanged.""" - x = 3.14159 - assert round_to_decimal_places(x, -1) == x - assert round_to_decimal_places(x, -5) == x - assert round_to_decimal_places(9.8, -1) == 9.8 - - def test_negative_numbers(self): - """Negative numbers should round correctly.""" - assert round_to_decimal_places(-3.14159, 2) == -3.14 - assert round_to_decimal_places(-9.876, 1) == -9.9 - - def test_zero_and_special(self): - """Zero and special values should round as expected.""" - assert round_to_decimal_places(0, 2) == 0 - assert round_to_decimal_places(0.0, 5) == 0.0 - # NaN and Inf propagate through round() - assert math.isnan(round_to_decimal_places(float("nan"), 2)) - assert round_to_decimal_places(float("inf"), 2) == float("inf") - assert round_to_decimal_places(float("-inf"), 2) == float("-inf") diff --git a/tests/prkit/scoring/test_semantics_scorer.py b/tests/prkit/scoring/test_semantics_scorer.py index f71398f..94b7ead 100644 --- a/tests/prkit/scoring/test_semantics_scorer.py +++ b/tests/prkit/scoring/test_semantics_scorer.py @@ -8,7 +8,7 @@ from prkit.api import Scorer, Verdict from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_category import AnswerCategory +from prkit.core.domain.answer_kinds import AnswerObjectKind from prkit.scoring import SemanticsScorer # Empirically validated against the deterministic engine (see plan step 4). @@ -55,10 +55,10 @@ def test_identity_equivalent(self, value): def test_accepts_answer_objects(self): pred = Answer( - value=3.0, answer_category=AnswerCategory.PHYSICAL_QUANTITY, unit="m/s" + value=3.0, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, unit="m/s" ) ref = Answer( - value=3, answer_category=AnswerCategory.PHYSICAL_QUANTITY, unit="m/s" + value=3, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, unit="m/s" ) v = SemanticsScorer().score(pred, ref) assert v.equivalent is True diff --git a/tests/prkit/semantics/test_inference_prompts.py b/tests/prkit/semantics/test_inference_prompts.py index 1ab0b3f..ecba4e2 100644 --- a/tests/prkit/semantics/test_inference_prompts.py +++ b/tests/prkit/semantics/test_inference_prompts.py @@ -6,7 +6,7 @@ import pytest from pydantic import ValidationError -from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem +from prkit.core.domain import Answer, AnswerObjectKind, PhysicsProblem from prkit.core.model_clients import BaseModelClient from prkit.core.model_clients.structured_output import StructuredOutputPlan from prkit.semantics.build.calls import ( @@ -71,7 +71,7 @@ def _build_problem() -> PhysicsProblem: answer=Answer( value="5", unit="N", - answer_category=AnswerCategory.PHYSICAL_QUANTITY, + answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, ), solution="Use Newton's second law.", domain="mechanics", @@ -115,7 +115,7 @@ def test_build_prediction_semantics_prompt_uses_answer_blind_question_draft() -> problem = PhysicsProblem( problem_id="prob-gold-split", question="Give both values: the displacement value and the time value.", - answer=Answer(value="F = ma", answer_category=AnswerCategory.EQUATION), + answer=Answer(value="F = ma", answer_kind=AnswerObjectKind.RELATION), additional_fields={ "answer_parts": [ {"part_label": "speed_slot", "raw_text": "1 m"}, diff --git a/tests/prkit/semantics/test_outcome_space.py b/tests/prkit/semantics/test_outcome_space.py index 6623241..ddeb397 100644 --- a/tests/prkit/semantics/test_outcome_space.py +++ b/tests/prkit/semantics/test_outcome_space.py @@ -2,7 +2,7 @@ import pytest -from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem +from prkit.core.domain import Answer, PhysicsProblem from prkit.semantics import ( AnswerObjectKind, AnswerStructure, @@ -123,6 +123,20 @@ def test_atomic_boolean_sign_and_qualitative_aliases() -> None: assert qualitative.canonical_text == "constant_temperature" +def test_atomic_free_form_prose_is_descriptive_text() -> None: + # Free-form prose that is not curated controlled vocabulary classifies as the + # descriptive_text kind, with a conservative surface canonical form (no aliasing). + prose = normalize_physics_answer( + "The block slides because the applied force exceeds friction." + ) + + assert prose.object_kind == AnswerObjectKind.DESCRIPTIVE_TEXT + assert ( + prose.canonical_text + == "block slides because the applied force exceeds friction." + ) + + def test_structured_tuple_set_interval_and_vector_normalization() -> None: tuple_answer = normalize_physics_answer("(1, 2)") set_answer = normalize_physics_answer("{2, 1}") @@ -266,7 +280,7 @@ def test_fixed_question_unit_allows_bare_number_but_required_unit_does_not() -> PhysicsProblem( problem_id="p1", question="Find the speed in m/s.", - answer=Answer(value="5", answer_category=AnswerCategory.NUMBER), + answer=Answer(value="5", answer_kind=AnswerObjectKind.NUMBER), ) ) required_unit_context = QuestionContext( @@ -316,7 +330,7 @@ def test_infer_question_context_rejects_prose_after_in_keyword( question=question, answer=Answer( value=answer_value, - answer_category=AnswerCategory.PHYSICAL_QUANTITY, + answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, ), ) ) @@ -332,29 +346,23 @@ def test_infer_question_context_drops_stopword_targets_but_keeps_symbol_targets( PhysicsProblem( problem_id="p_stopword_the", question="What is the magnitude of the force on the block?", - answer=Answer( - value="25 N", answer_category=AnswerCategory.PHYSICAL_QUANTITY - ), + answer=Answer(value="25 N", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), ), PhysicsProblem( problem_id="p_stopword_all", question="What is all?", - answer=Answer( - value="25 N", answer_category=AnswerCategory.PHYSICAL_QUANTITY - ), + answer=Answer(value="25 N", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), ), PhysicsProblem( problem_id="p_stopword_which", question="What is which?", - answer=Answer( - value="25 N", answer_category=AnswerCategory.PHYSICAL_QUANTITY - ), + answer=Answer(value="25 N", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), ), ] symbol_problem = PhysicsProblem( problem_id="p_symbol_target", question="What is T?", - answer=Answer(value="0.78 s", answer_category=AnswerCategory.PHYSICAL_QUANTITY), + answer=Answer(value="0.78 s", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), ) for problem in prose_problems: @@ -368,7 +376,7 @@ def test_question_semantics_split_uses_gold_target_only_for_reference() -> None: question="Give the final expression for the magnetic field.", answer=Answer( value="B = \\mu_0 I / (2\\pi r)", - answer_category=AnswerCategory.EQUATION, + answer_kind=AnswerObjectKind.RELATION, ), ) @@ -388,7 +396,7 @@ def test_question_semantics_split_uses_gold_unit_policy_only_for_reference() -> answer=Answer( value="5", unit="m/s", - answer_category=AnswerCategory.PHYSICAL_QUANTITY, + answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, ), ) @@ -404,7 +412,7 @@ def test_prediction_question_semantics_ignores_answer_parts_metadata() -> None: problem = PhysicsProblem( problem_id="p_answer_parts_split", question="Give both values: the displacement value and the time value.", - answer=Answer(value="ignored", answer_category=AnswerCategory.TEXT), + answer=Answer(value="ignored", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT), additional_fields={ "answer_parts": [ {"part_label": "speed_slot", "raw_text": "1 m"}, @@ -479,7 +487,7 @@ def test_prediction_question_semantics_ignores_symbol_alias_metadata() -> None: problem = PhysicsProblem( problem_id="p_symbol_alias_split", question="Give the final expression for the displacement.", - answer=Answer(value="x_final = v*t", answer_category=AnswerCategory.EQUATION), + answer=Answer(value="x_final = v*t", answer_kind=AnswerObjectKind.RELATION), additional_fields={ "symbol_aliases": [ { @@ -503,7 +511,7 @@ def test_problem_answer_parts_take_precedence() -> None: problem = PhysicsProblem( problem_id="p2", question="Give both values.", - answer=Answer(value="ignored", answer_category=AnswerCategory.TEXT), + answer=Answer(value="ignored", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT), additional_fields={"answer_parts": ["1 m", "2 m"]}, ) @@ -520,7 +528,7 @@ def test_dataset_backed_relation_and_multi_part_strings() -> None: PhysicsProblem( problem_id="ugphysics-628", question="Give the effect type and the field strength.", - answer=Answer(value="C, 7.77", answer_category=AnswerCategory.OPTION), + answer=Answer(value="C, 7.77", answer_kind=AnswerObjectKind.CHOICE), additional_fields={"answer_parts": ["C", "7.77"]}, ) ) @@ -529,8 +537,10 @@ def test_dataset_backed_relation_and_multi_part_strings() -> None: assert relation_answer.structure == AnswerStructure.ATOMIC assert multipart_answer.structure == AnswerStructure.MULTI_PART assert multipart_answer.diagnostics == () + # "C" is not curated controlled vocabulary, so the deterministic normalizer + # classifies it as free-form descriptive_text (not qualitative_label). assert [child.object_kind for child in multipart_answer.children] == [ - AnswerObjectKind.QUALITATIVE_LABEL, + AnswerObjectKind.DESCRIPTIVE_TEXT, AnswerObjectKind.NUMBER, ] diff --git a/tests/prkit/semantics/test_prediction_isolated_build.py b/tests/prkit/semantics/test_prediction_isolated_build.py index ae6935c..8c15b3f 100644 --- a/tests/prkit/semantics/test_prediction_isolated_build.py +++ b/tests/prkit/semantics/test_prediction_isolated_build.py @@ -17,7 +17,7 @@ import pytest -from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem +from prkit.core.domain import Answer, PhysicsProblem from prkit.core.model_clients import BaseModelClient from prkit.semantics.build.calls import ( build_extracted_prediction_semantics_artifact, @@ -39,7 +39,9 @@ def _problem() -> PhysicsProblem: return PhysicsProblem( problem_id="pred-iso-1", question="Find the speed v.", - answer=Answer(value="sqrt(E/m), m > 0", answer_category=AnswerCategory.FORMULA), + answer=Answer( + value="sqrt(E/m), m > 0", answer_kind=AnswerObjectKind.EXPRESSION + ), solution="Use conservation of energy.", domain="mechanics", additional_fields={ diff --git a/tests/prkit/semantics/test_protocol_comparison.py b/tests/prkit/semantics/test_protocol_comparison.py index 92ccc3e..eee1208 100644 --- a/tests/prkit/semantics/test_protocol_comparison.py +++ b/tests/prkit/semantics/test_protocol_comparison.py @@ -4,7 +4,7 @@ import pytest -from prkit.core.domain import PhysicsProblem +from prkit.core.domain import AnswerObjectKind, PhysicsProblem from prkit.semantics import ( ComparisonPolicyMode, PhysicsQuestionSemantics, @@ -103,6 +103,39 @@ def test_protocol_number_record_comparison() -> None: assert result.comparison_mode == "number" +def test_protocol_descriptive_text_accepts_surface_equivalent_prose() -> None: + pred = normalize_physics_answer( + "The object accelerates because the net force is nonzero." + ) + ref = normalize_physics_answer( + "the object accelerates because the net force is nonzero" + ) + + assert pred.object_kind == AnswerObjectKind.DESCRIPTIVE_TEXT + assert ref.object_kind == AnswerObjectKind.DESCRIPTIVE_TEXT + + result = compare_protocol_answers(pred, ref) + + assert result.equivalent is True + assert result.comparison_mode == "descriptive_text" + + +def test_protocol_descriptive_text_rejects_different_prose() -> None: + # Adversarial: conservative normalized-text equality must NOT treat two + # genuinely different explanations as equivalent (no semantic rescue). + pred = normalize_physics_answer( + "The object accelerates because the net force is nonzero." + ) + ref = normalize_physics_answer( + "The object stays at rest because the forces are balanced." + ) + + result = compare_protocol_answers(pred, ref) + + assert result.equivalent is False + assert result.comparison_mode == "descriptive_text" + + def test_protocol_quantity_comparison_converts_units() -> None: pred = { "object_kind": "physical_quantity", diff --git a/tests/prkit/semantics/test_sign_convention_build_integration.py b/tests/prkit/semantics/test_sign_convention_build_integration.py index 4f79759..07197e4 100644 --- a/tests/prkit/semantics/test_sign_convention_build_integration.py +++ b/tests/prkit/semantics/test_sign_convention_build_integration.py @@ -12,7 +12,7 @@ import json from typing import Any -from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem +from prkit.core.domain import Answer, AnswerObjectKind, PhysicsProblem from prkit.core.model_clients import BaseModelClient from prkit.semantics.build.calls import ( build_reference_semantics, @@ -26,7 +26,7 @@ def _quantity_problem(golden: str) -> PhysicsProblem: return PhysicsProblem( problem_id="signconv-int", question="Find the block's velocity v.", - answer=Answer(value=golden, answer_category=AnswerCategory.PHYSICAL_QUANTITY), + answer=Answer(value=golden, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), domain="mechanics", ) @@ -35,7 +35,7 @@ def _vector_problem(golden: str) -> PhysicsProblem: return PhysicsProblem( problem_id="signconv-int-vec", question="Find the displacement vector.", - answer=Answer(value=golden, answer_category=AnswerCategory.PHYSICAL_QUANTITY), + answer=Answer(value=golden, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), domain="mechanics", ) diff --git a/tests/prkit/semantics/test_sign_convention_build_live.py b/tests/prkit/semantics/test_sign_convention_build_live.py index ca5c181..62f51da 100644 --- a/tests/prkit/semantics/test_sign_convention_build_live.py +++ b/tests/prkit/semantics/test_sign_convention_build_live.py @@ -13,7 +13,7 @@ import pytest -from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem +from prkit.core.domain import Answer, AnswerObjectKind, PhysicsProblem from prkit.core.model_clients import create_model_client from prkit.semantics.build.calls import build_reference_semantics @@ -33,9 +33,7 @@ def test_live_reference_build_routes_free_axis_convention_to_a_ref() -> None: "is specified. Taking the block's motion into account, its velocity is found to " "be 20 m/s directed to the left. Report the velocity as a signed value." ), - answer=Answer( - value="-20 m/s", answer_category=AnswerCategory.PHYSICAL_QUANTITY - ), + answer=Answer(value="-20 m/s", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), domain="mechanics", ) diff --git a/tests/prkit/semantics/test_staged_build.py b/tests/prkit/semantics/test_staged_build.py index ca67b9c..8289258 100644 --- a/tests/prkit/semantics/test_staged_build.py +++ b/tests/prkit/semantics/test_staged_build.py @@ -11,7 +11,7 @@ import json from typing import Any -from prkit.core.domain import Answer, AnswerCategory, PhysicsProblem +from prkit.core.domain import Answer, PhysicsProblem from prkit.core.model_clients import BaseModelClient from prkit.semantics.build.calls import ( build_problem_semantics, @@ -29,7 +29,7 @@ def _problem() -> PhysicsProblem: return PhysicsProblem( problem_id="staged-1", question="Find the energy E.", - answer=Answer(value="x**2/2, x > 0", answer_category=AnswerCategory.FORMULA), + answer=Answer(value="x**2/2, x > 0", answer_kind=AnswerObjectKind.EXPRESSION), domain="mechanics", ) @@ -38,9 +38,7 @@ def _directional_problem() -> PhysicsProblem: return PhysicsProblem( problem_id="staged-dir-1", question="Find the velocity v of the block.", - answer=Answer( - value="-20 m/s", answer_category=AnswerCategory.PHYSICAL_QUANTITY - ), + answer=Answer(value="-20 m/s", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), domain="mechanics", ) diff --git a/tests/prkit/test_api.py b/tests/prkit/test_api.py index 1c3fe6a..9576dbf 100644 --- a/tests/prkit/test_api.py +++ b/tests/prkit/test_api.py @@ -27,8 +27,9 @@ def test_all_is_frozen_surface(self): "Scorer", "Runner", "Verdict", + "AnswerObjectKind", + "AnswerStructure", "Answer", - "AnswerCategory", "PhysicsDomain", "PhysicsProblem", "PhysicalDataset", From f45b50612719b55fb66122b62cf0a02f1eb59607 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sun, 21 Jun 2026 15:51:15 -0400 Subject: [PATCH 18/28] Streamline the Answer record, contract version, and edit-distance core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce Answer to a thin observed-data record (value/unit/source_type/ metadata) and keep the answer ontology (AnswerObjectKind/AnswerStructure) solely in the semantics layer; simplify the dataset loaders that built the heavier record. Mark the integration contract as provisional API_VERSION "1.0" — breaking changes are tracked in the delta doc rather than via a major bump — and drop the removed verify.parse from the facade docs. Split the front-end-free EED/SEED tree-edit core (tree/zss/score/timeout) out of semantics/edit_distance/ into evaluation/edit_distance/, leaving the SEED dispatch glue under semantics/ to keep the package graph acyclic, and narrow the import-isolation guard to prkit.evaluation.llm_judge. Refresh the docs and tests to match. Co-Authored-By: Claude Opus 4.8 --- README.md | 5 +- docs/CORE.md | 90 ++-- docs/DATASETS.md | 37 +- src/prkit/CONTRACT.md | 28 +- src/prkit/__init__.py | 4 +- src/prkit/api.py | 18 +- src/prkit/core/domain/answer.py | 274 +++-------- src/prkit/core/domain/physics_problem.py | 54 +-- src/prkit/datasets/loaders/base_loader.py | 107 +---- src/prkit/datasets/loaders/jeebench_loader.py | 9 +- src/prkit/datasets/loaders/phybench_loader.py | 4 - .../datasets/loaders/physbench_loader.py | 1 - src/prkit/datasets/loaders/physics_loader.py | 22 +- src/prkit/datasets/loaders/tpbench_loader.py | 2 - .../datasets/loaders/ugphysics_loader.py | 11 +- .../evaluation/edit_distance/__init__.py | 35 ++ .../edit_distance/score.py | 0 .../edit_distance/timeout.py | 0 .../edit_distance/tree.py | 0 .../edit_distance/zss.py | 2 +- src/prkit/evaluation/llm_judge/payload.py | 6 +- src/prkit/semantics/README.md | 4 +- src/prkit/semantics/edit_distance/__init__.py | 25 +- src/prkit/semantics/edit_distance/pipeline.py | 15 +- src/prkit/semantics/inference/__init__.py | 24 - src/prkit/testing/conformance.py | 14 +- tests/conftest.py | 22 +- tests/prkit/core/domain/test_answer.py | 439 +++++------------- .../prkit/core/domain/test_physics_problem.py | 23 +- .../loaders/test_base_loader_additional.py | 55 ++- .../datasets/loaders/test_jeebench_loader.py | 4 +- .../datasets/loaders/test_phybench_loader.py | 3 +- .../datasets/loaders/test_physbench_loader.py | 3 +- .../datasets/loaders/test_tpbench_loader.py | 3 +- .../datasets/loaders/test_ugphysics_loader.py | 6 +- tests/prkit/datasets/test_utils.py | 62 --- tests/prkit/datasets/test_utils_functions.py | 16 +- .../edit_distance/test_edit_distance_score.py | 4 +- .../edit_distance/test_edit_distance_tree.py | 2 +- .../edit_distance/test_edit_distance_zss.py | 6 +- .../evaluation/llm_judge/test_payload.py | 24 +- tests/prkit/scoring/test_semantics_scorer.py | 9 +- .../test_edit_distance_robustness.py | 5 +- .../prkit/semantics/test_inference_prompts.py | 10 +- tests/prkit/semantics/test_outcome_space.py | 21 +- .../test_prediction_isolated_build.py | 4 +- .../test_sign_convention_build_integration.py | 6 +- .../test_sign_convention_build_live.py | 4 +- tests/prkit/semantics/test_staged_build.py | 4 +- tests/prkit/test_api.py | 62 +++ tests/prkit/verify/test_import_isolation.py | 6 +- 51 files changed, 568 insertions(+), 1026 deletions(-) create mode 100644 src/prkit/evaluation/edit_distance/__init__.py rename src/prkit/{semantics => evaluation}/edit_distance/score.py (100%) rename src/prkit/{semantics => evaluation}/edit_distance/timeout.py (100%) rename src/prkit/{semantics => evaluation}/edit_distance/tree.py (100%) rename src/prkit/{semantics => evaluation}/edit_distance/zss.py (98%) delete mode 100644 src/prkit/semantics/inference/__init__.py rename tests/prkit/{semantics => evaluation}/edit_distance/test_edit_distance_score.py (96%) rename tests/prkit/{semantics => evaluation}/edit_distance/test_edit_distance_tree.py (98%) rename tests/prkit/{semantics => evaluation}/edit_distance/test_edit_distance_zss.py (94%) diff --git a/README.md b/README.md index 30a9a78..e7c1572 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ PRKit applies a “unified interface” idea to the full physical-reasoning loop PRKit centers on **core components** that define the physical reasoning ontology. Three integrated subpackages build on this foundation: -- **Core components**: `PhysicsDomain`, `AnswerCategory`, `PhysicsProblem`, `Answer`, `PhysicalDataset`, `PhysicsSolution`, `BaseModelClient`, `create_model_client`, `PRKitLogger`—the shared abstractions used across the toolkit. +- **Core components**: `PhysicsDomain`, `PhysicsProblem`, `Answer`, `PhysicalDataset`, `PhysicsSolution`, `BaseModelClient`, `create_model_client`, `PRKitLogger`—the shared abstractions used across the toolkit. - **`prkit.datasets`**: A Datasets-like hub that downloads/loads benchmarks into the unified schema (`PhysicsProblem`, `PhysicalDataset`). - **`prkit.annotation`**: Workflow-oriented tools for structured, lower-level labels (e.g., domain/subdomain, theorem usage). - **`prkit.evaluation`**: Evaluate-like components for physics-oriented scoring and comparison (e.g., symbolic/numerical answer matching). @@ -211,9 +211,8 @@ The toolkit is organized around **core components** and three subpackages that u The essential building blocks of the physical-reasoning-toolkit. All datasets, inference, evaluation, and annotation workflows use these components. * **PhysicsDomain** — Enumeration of physics subfields (mechanics, thermodynamics, quantum mechanics, optics, etc.) for problem classification. Aligned with UGPhysics, PHYBench, TPBench. Use `PhysicsDomain.from_string()` for flexible parsing. -* **AnswerCategory** — Enumeration of answer types for normalization and evaluation: `NUMBER`, `PHYSICAL_QUANTITY`, `EQUATION`, `FORMULA`, `TEXT`, `OPTION`. Drives how answers are compared (numerical precision, symbolic equivalence, exact match). * **PhysicsProblem** — The canonical representation of a physics problem. Required: `problem_id`, `question`. Optional: `answer` (Answer), `solution`, `domain`, `image_path`, `problem_type` (MC/OE), `options`, `correct_option`. Supports dictionary-like access and `load_images()` for visual problems. -* **Answer** — Unified answer model. `value` holds the number (NUMBER), numeric part (PHYSICAL_QUANTITY), option string (OPTION), or plain string (EQUATION, FORMULA, TEXT). `unit` is optional and used only for PHYSICAL_QUANTITY. Type checks, unit helpers, LaTeX handling, option indexing. +* **Answer** — Thin observation record: `value` (str, verbatim), optional `unit` (observed unit string), optional `source_type` (dataset-native type tag, verbatim), and `metadata` dict. The canonical answer kind (`AnswerObjectKind`, 9 object kinds) is derived on demand by the `prkit.semantics` layer — it is not stored on `Answer`. * **PhysicalDataset** — Collection of `PhysicsProblem` instances. Indexing, slicing, `get_by_id()`, `filter_by_domain()`, `take()`, `sample()`, `save_to_json()` / `from_json()`. Provides `get_statistics()` for domain and problem-type distribution. * **PhysicsSolution** — Bundles a `PhysicsProblem`, model `agent_answer`, and optional `intermediate_steps`. Captures the full solution trace for evaluation and analysis. * **BaseModelClient** — Abstract base for model clients. Subclasses implement `chat(user_prompt, image_paths=None)`. diff --git a/docs/CORE.md b/docs/CORE.md index 0f981cf..672d574 100644 --- a/docs/CORE.md +++ b/docs/CORE.md @@ -44,19 +44,6 @@ domain = PhysicsDomain.from_string("unknown") # → PhysicsDomain.OTHE str(domain) # → "quantum_mechanics" ``` -### AnswerCategory - -Enumeration of answer semantic types used for normalization and comparison. - -| Category | Description | Example | -|----------|-------------|---------| -| `NUMBER` | Dimensionless numeric value | `42`, `3.14` | -| `PHYSICAL_QUANTITY` | Number with units | `9.8 m/s²`, `5 N` | -| `EQUATION` | Single-equation form | `F = ma` | -| `FORMULA` | Mathematical expression | `x² + 1`, `e^(−t/τ)` | -| `TEXT` | Descriptive text | "The ball accelerates downward." | -| `OPTION` | Multiple-choice selection | `A`, `B`, `(1)` | - ### PhysicsProblem The core unit of a physics problem. Works both standalone and as a dataset-compatible object (dictionary-like access). @@ -89,31 +76,21 @@ The core unit of a physics problem. Works both standalone and as a dataset-compa ### Answer -Unified answer representation via composition over `AnswerCategory`. Handles all answer semantics in one class. +A thin observation record: the verbatim answer string, an optional unit, an optional dataset-native type label, and a metadata dict. It captures exactly what the dataset provides—nothing more. **Fields:** -- `value`: The answer content; semantics depend on category: - - `NUMBER`: Actual number (int or float) - - `PHYSICAL_QUANTITY`: The numeric part only (int or float) - - `OPTION`: The option string (e.g., `"A"`, `"B"`, `"Yes"`) - - `EQUATION`, `FORMULA`, `TEXT`: Plain string -- `answer_category`: `AnswerCategory` -- `unit`: Optional; used only for `PHYSICAL_QUANTITY` (e.g., `"m/s²"`, `"N"`) -- `metadata`: Extra key-value data - -**Type checks:** -- `is_number()`, `is_physical_quantity()`, `is_equation()`, `is_formula()`, `is_text()`, `is_option()` -- `is_numerical()` → number or physical quantity -- `is_symbolic()` → equation, formula, or physical quantity +- `value`: `str` — verbatim answer string (always a string; numeric answers remain as strings) +- `unit`: `str | None` — observed unit, when the dataset provides one (e.g. `"m/s²"`, `"N"`); `None` otherwise +- `source_type`: `str | None` — dataset-native answer-type label, verbatim (e.g. `"MC"`, `"NV"`, `"EX"`, `"Integer"`); `None` when the dataset provides none; **never fabricated by heuristics** +- `metadata`: `dict` — extra dataset-provided key-value data -**Numerical helpers:** -- `get_unit()`, `has_unit()`, `is_integer()`, `is_positive()`, `is_negative()` +> **Canonical answer kind is derived, not stored.** The `AnswerObjectKind` ontology (9 object kinds: `number`, `physical_quantity`, `expression`, `relation`, `choice`, `qualitative_label`, `assertion`, `structured`, `descriptive_text`) lives in `prkit.semantics` and is returned as `object_kind` on `PhysicsAnswerSemantics`. It is not a field on `Answer`. -**Symbolic helpers:** -- `is_latex()`, `get_clean_expression()` - -**Option helpers:** -- `is_letter_option()`, `is_yes_no()`, `is_true_false()`, `get_option_index()` +**Access helpers:** +- `get_value()` → `str` +- `get_unit()` → `str | None` +- `has_unit()` → `bool` +- `__str__()` → `"{value} {unit}"` when unit is set, else `value` --- @@ -349,7 +326,7 @@ logger.info("Message") - **PhysicalDataset** = a collection of physics problems - **PhysicsProblem** = one problem (question + optional ground-truth answer + optional domain) -- **Answer** = ground-truth or predicted answer, with a category (number, option, text, etc.) +- **Answer** = thin observation record: `value` (str) + optional `unit` + optional `source_type` + `metadata` - **PhysicsSolution** = a problem plus model output (agent_answer), used for evaluation ### Core Domain Model @@ -362,7 +339,7 @@ logger.info("Message") │ └── answer: Answer (optional, ground truth) │ - └── answer_category: AnswerCategory (enum) + └── { value: str, unit: str|None, source_type: str|None, metadata: dict } PhysicsSolution (separate: one per model run) ├── problem: PhysicsProblem @@ -374,12 +351,12 @@ logger.info("Message") ``` PhysicalDataset PhysicsProblem ┌──────────────┐ ┌──────────────────┐ Answer - │ _problems │────────►│ problem_id │ ┌──────────────────┐ - │ _info │ 1:N │ question │ │ value │ - │ _split │ │ domain ──────────┼──┐ │ answer_category ─┼─► AnswerCategory - └──────────────┘ │ answer ──────────┼──┼───►│ unit │ - model call │ solution │ │ └──────────────────┘ - ┌─────────────────└──────────────────┘ │ + │ _problems │────────►│ problem_id │ ┌──────────────────────┐ + │ _info │ 1:N │ question │ │ value: str │ + │ _split │ │ domain ──────────┼──┐ │ unit: str|None │ + └──────────────┘ │ answer ──────────┼──┼───►│ source_type: str|None│ + model call │ solution │ │ │ metadata: dict │ + ┌─────────────────└──────────────────┘ │ └──────────────────────┘ ▼ ▲ └── PhysicsDomain PhysicsSolution │ ┌──────────────┐ │ problem @@ -394,7 +371,7 @@ logger.info("Message") |--------|----------------| | PhysicalDataset | Many PhysicsProblem (_problems list) | | PhysicsProblem | problem_id, question, domain (PhysicsDomain), answer (Answer), solution, image_path, ... | -| Answer | value, answer_category (AnswerCategory), unit | +| Answer | value (str), unit (str\|None), source_type (str\|None), metadata (dict) | | PhysicsSolution | problem (PhysicsProblem), agent_answer (string). Evaluation compares agent_answer to problem.answer | ### Subpackage Dependencies @@ -417,8 +394,11 @@ flowchart TB end subgraph evaluation["prkit.evaluation"] - EV[Evaluator] - CMP[Comparator] + LLJ[LLMJudge] + end + + subgraph scoring["prkit.scoring / prkit.verify"] + SC[SemanticsScorer] end subgraph datasets["prkit.datasets"] @@ -439,9 +419,10 @@ flowchart TB PP -->|has| AN PP -->|has| PS - EV --> CMP - AN -->|ground truth| CMP - MO[/model output/] -->|model answer| CMP + AN -->|ground truth| SC + MO[/model output/] -->|model answer| SC + AN -.->|model-graded path| LLJ + MO -.->|model-graded path| LLJ ``` @@ -449,8 +430,9 @@ flowchart TB | Package | Uses from Core | Produces / Operates On | |---------|----------------|------------------------| -| prkit.datasets | PhysicalDataset, PhysicsProblem, Answer, AnswerCategory, PhysicsDomain, PRKitLogger | PhysicalDataset (via DatasetLoader.load) | -| prkit.evaluation | PhysicsProblem, Answer, AnswerCategory, Comparator | Accuracy scores (via AccuracyEvaluator.evaluate) | +| prkit.datasets | PhysicalDataset, PhysicsProblem, Answer, PhysicsDomain, PRKitLogger | PhysicalDataset (via DatasetLoader.load) | +| prkit.scoring / prkit.verify | PhysicsProblem, Answer, Verdict | Verdict (via SemanticsScorer / verify) | +| prkit.evaluation | PhysicsProblem, Answer | Model-graded scores (via LLMJudge; deterministic scoring is in prkit.scoring) | --- @@ -460,13 +442,15 @@ flowchart TB # Core components from prkit.core.domain import ( PhysicsDomain, - AnswerCategory, Answer, PhysicsProblem, PhysicalDataset, PhysicsSolution, ) +# Semantics-layer canonical kind (derived, not stored on Answer) +from prkit.semantics.schema import AnswerObjectKind # 9 object kinds + # Utility components from prkit.core import PRKitLogger from prkit.core.model_clients import create_model_client, BaseModelClient @@ -477,6 +461,6 @@ from prkit.core.model_clients import create_model_client, BaseModelClient ## Design Principles 1. **Unified schema:** All supported benchmarks map to `PhysicsProblem` and `PhysicalDataset`. -2. **Answer semantics:** `AnswerCategory` drives normalization and evaluation. -3. **Composition over inheritance:** `Answer` uses a category field rather than subclassing. +2. **Observed data vs. derived interpretation:** `Answer` is a thin observation record (`value`/`unit`/`source_type`/`metadata`). The canonical answer kind (`AnswerObjectKind`) is derived on demand by `prkit.semantics`, not stored on `Answer`. +3. **Composition over inheritance:** `Answer` is a flat dataclass; type interpretation is a semantics-layer concern, not a subclass hierarchy. 4. **Dataset compatibility:** `PhysicsProblem` supports dict-like access and `additional_fields`. diff --git a/docs/DATASETS.md b/docs/DATASETS.md index 1a73f67..1afc57c 100644 --- a/docs/DATASETS.md +++ b/docs/DATASETS.md @@ -97,9 +97,12 @@ for problem in dataset[:5]: print(f"Problem {problem.problem_id}: {problem.question[:100]}...") print(f"Domain: {problem.domain}") if problem.answer is not None: - print( - f"Answer: {problem.answer.value} (Category: {problem.answer.answer_category.value})" - ) + parts = [f"Answer: {problem.answer.value}"] + if problem.answer.unit: + parts.append(f"unit={problem.answer.unit}") + if problem.answer.source_type: + parts.append(f"source_type={problem.answer.source_type}") + print(", ".join(parts)) ``` ## Problem Representation and Data Contract @@ -117,31 +120,23 @@ This section documents each structure in dependency order, starting with the fou The `Answer` class is the foundational building block for representing answers in physics problems. It handles all answer types through composition rather than inheritance. -#### Initialization +#### Fields ```python Answer( - value: Any, # The answer value (type depends on answer_category) - answer_category: AnswerCategory, # AnswerCategory enum (see below) - unit: Optional[str] = None, # Unit string for number/physical_quantity answers - metadata: Dict[str, Any] = {} # Additional metadata + value: str, # Verbatim answer string (always str) + unit: Optional[str] = None, # Observed unit from the dataset, e.g. "m/s²" + source_type: Optional[str] = None, # Dataset-native type label, verbatim, e.g. "MC", "NV", "EX" + metadata: Dict[str, Any] = {} # Additional dataset-provided key-value data ) ``` -#### Answer Categories - -The `answer_category` field uses the `AnswerCategory` enum with the following values: - -| Category | Enum Value | Description | Example | -|----------|------------|-------------|---------| -| **Number** | `AnswerCategory.NUMBER` | Dimensionless numeric value | `42.5`, `3.14` | -| **Physical Quantity** | `AnswerCategory.PHYSICAL_QUANTITY` | Number with units | `"9.8 m/s^2"`, `42.5` + `"m/s"` | -| **Equation** | `AnswerCategory.EQUATION` | Single-equation form | `"F = ma"` | -| **Formula** | `AnswerCategory.FORMULA` | Mathematical expression | `"\\frac{mv^2}{2}"`, `"E = mc^2"` | -| **Text** | `AnswerCategory.TEXT` | Text-based answers | `"The force is upward"` | -| **Option** | `AnswerCategory.OPTION` | Multiple choice selection | `"A"`, `"B"`, `"1"` | +`Answer` is a **thin observation record** — it stores exactly what the dataset provides. The canonical answer kind (`AnswerObjectKind` — one of 9 semantic object kinds such as `number`, `physical_quantity`, `choice`, `expression`, etc.) is **derived on demand by `prkit.semantics`**, not stored on `Answer`. -**Note**: Answer category detection is automatic during dataset loading, but can be explicitly set. +- `value` is always a `str` (the verbatim answer string). +- `unit` is the observed unit when present; `None` otherwise. It is consumed by the semantics engine. +- `source_type` is the dataset's own native type tag (e.g. `"MC"` for multiple-choice in UGPhysics, `"Integer"` in JEEBench). It is a verbatim free string, never fabricated by heuristics, and never mapped to `AnswerObjectKind`. The semantics engine ignores it. +- `metadata` holds any remaining dataset-provided fields. ### Physics Domains diff --git a/src/prkit/CONTRACT.md b/src/prkit/CONTRACT.md index 117a909..e5b398e 100644 --- a/src/prkit/CONTRACT.md +++ b/src/prkit/CONTRACT.md @@ -85,6 +85,10 @@ hub backfills it in `DatasetHub.get_loader_info`). ## `API_VERSION` semver policy +The contract is currently **provisional / pre-stable** at `1.0`. Breaking changes are +allowed and are tracked in `internal/PAPER_V1_TO_V2_DELTA.md`, not by bumping a major +version number. Once the surface stabilizes, semver bumps will follow this policy: + - **PATCH** (`1.0` → `1.0.1`): documentation/typo only. - **MINOR** (`1.0` → `1.1`): purely additive — a new protocol, a new optional method, a new re-export. Backward compatible. @@ -103,20 +107,24 @@ changes to those are documented in the package release notes, not the contract v - Precedent: `BaseModelClient.chat()` / `chat_structured()` (see `core/model_clients/base.py`). -### Removed in 2.0 - -- **Taxonomy unification (MAJOR).** The legacy `AnswerCategory` enum was **removed**. - `Answer.answer_category: AnswerCategory` is now `Answer.answer_kind: AnswerObjectKind`, - and the canonical ontology enums `AnswerObjectKind` / `AnswerStructure` are promoted - onto `prkit.api.__all__`. Migration mapping for the old `AnswerCategory` members: - `NUMBER → number`, `PHYSICAL_QUANTITY → physical_quantity`, `FORMULA → expression`, - `EQUATION → relation`, `OPTION → choice`, `TEXT → descriptive_text` (a new 9th object - kind for free-form answers). Serialized answers now carry `"answer_kind"` instead of - `"answer_category"`. +### Removed during provisional 1.0 shaping + +- **`Answer` reshaped to a thin observation record.** `Answer` now carries only + `value: str`, `unit: str | None`, `source_type: str | None`, and `metadata: dict`. + The former `answer_kind: AnswerObjectKind` field (and all predicate helpers such as + `is_number()`, `is_option()`, `get_type()`, etc.) are **removed**. The canonical + answer ontology (`AnswerObjectKind` / `AnswerStructure`, 9 object kinds) lives only + in `prkit.semantics` and is returned as `object_kind` on `PhysicsAnswerSemantics` + — it is never stored on `Answer`. `source_type` is the dataset's verbatim native + type label (e.g. `"MC"`, `"NV"`, `"EX"`, `"Integer"`); it is never fabricated and + never read by the semantics engine. Serialized answers migrate via: + `source_type = value.get("source_type") or value.get("answer_kind") or value.get("answer_category")`. - **Deprecated scoring stack deleted.** `prkit.evaluation.comparator.*`, `prkit.evaluation.evaluator.*` (`BaseComparator`, `ExactMatchComparator`, `BaseEvaluator`, `AccuracyEvaluator`, …) were removed. Use `prkit.scoring.SemanticsScorer` (the `Scorer` / `Verdict` contract), or the light-import facade `prkit.verify`, for deterministic scoring. +- **`prkit.semantics.inference` alias removed.** The `prkit.semantics.inference` + compatibility shim is deleted; import from `prkit.semantics.build` directly. - `prkit.evaluation.llm_judge` (model-graded scoring) is **retained** — it is a distinct capability, not a duplicate of the deterministic scoring path. diff --git a/src/prkit/__init__.py b/src/prkit/__init__.py index ec66582..640d125 100644 --- a/src/prkit/__init__.py +++ b/src/prkit/__init__.py @@ -21,8 +21,8 @@ - :mod:`prkit.testing` — conformance suite (``check_dataset``/``check_scorer``/…). - :mod:`prkit.semantics` — physics-aware answer normalization & comparison. - :mod:`prkit.evaluation` — model-graded LLM judge (``llm_judge``). The legacy - comparator/evaluator stacks were removed in ``API_VERSION`` 2.0; use - :mod:`prkit.scoring` for deterministic scoring. + comparator/evaluator stacks were removed while shaping the provisional + contract; use :mod:`prkit.scoring` for deterministic scoring. - :mod:`prkit.annotation` — human annotation tasks (gold, correctness). """ diff --git a/src/prkit/api.py b/src/prkit/api.py index 87274a1..51ed282 100644 --- a/src/prkit/api.py +++ b/src/prkit/api.py @@ -11,9 +11,10 @@ protocols are grounded in. For the headline "just verify a physics answer" use case, integrators should -reach for the light-import facade :mod:`prkit.verify` (``parse`` / ``verify``), -which returns the same :class:`Verdict` without importing clients, the hub, or -provider SDKs. +reach for the light-import facade :mod:`prkit.verify` (``verify``), which +returns the same :class:`Verdict` without importing clients, the hub, or +provider SDKs. Answer parsing is handled by +:func:`prkit.semantics.extract_prediction_answer_semantics`. .. note:: ``@runtime_checkable`` only verifies that the named **methods/attributes @@ -43,12 +44,11 @@ from prkit.datasets.loaders.base_loader import BaseDatasetLoader # --- contract version (independent of prkit.__version__) ------------------ -# Bump per CONTRACT.md: additive change -> minor, breaking change -> major. -# 2.0 is the taxonomy-unification MAJOR: the legacy ``AnswerCategory`` field on -# ``Answer`` was removed in favor of the canonical ``AnswerObjectKind`` (with -# ``AnswerObjectKind``/``AnswerStructure`` promoted onto the contract), and the -# deprecated ``evaluation`` comparator/evaluator stack was deleted. -API_VERSION = "2.0" +# The contract is PROVISIONAL at 1.0: breaking changes are allowed and are +# tracked in src/prkit/CONTRACT.md and internal/PAPER_V1_TO_V2_DELTA.md rather +# than via a major-version bump. When the contract stabilises, semver policy +# (additive → minor, breaking → major) will apply. +API_VERSION = "1.0" # --- the four nouns as structural Protocols ------------------------------- diff --git a/src/prkit/core/domain/answer.py b/src/prkit/core/domain/answer.py index 2f7df09..db1ffe8 100644 --- a/src/prkit/core/domain/answer.py +++ b/src/prkit/core/domain/answer.py @@ -1,258 +1,92 @@ """ -Answer models for physical reasoning evaluation. +Thin observation record for a physics problem's ground-truth answer. -This module provides a unified Answer class that handles all answer kinds -through composition rather than inheritance. +An ``Answer`` captures only what a dataset directly provides: + +* ``value`` — verbatim answer string (always ``str``) +* ``unit`` — observed unit string when present (e.g. ``"m/s²"``, ``"N"``) +* ``source_type`` — dataset-native answer-type label, verbatim (e.g. ``"MC"``, + ``"NV"``, ``"Integer"``); ``None`` when the dataset provides + none. **Never fabricated from heuristics.** +* ``metadata`` — arbitrary extra fields passed through from the loader. + +The canonical answer ontology (``AnswerObjectKind``) and structural +classification live **only** in :mod:`prkit.semantics` (as ``object_kind`` on +:class:`~prkit.semantics.schema.models.PhysicsAnswerSemantics`). They are +derived on demand by the semantics engine and are **not** stored here. """ from dataclasses import dataclass, field from typing import Any -from .answer_kinds import AnswerObjectKind - -AnswerValue = int | float | str - @dataclass class Answer: - """Unified answer class that handles all answer kinds through composition. + """Thin observed-data record for a physics answer. + + ``value`` is always a plain string — the verbatim answer text after + stripping LaTeX wrappers (``\\boxed{}``, ``$…$``, ``$$…$$``). + + ``unit`` is first-class observed data consumed by the equivalence engine; + it lives *only* here, never duplicated in ``metadata``. - ``answer_kind`` is the canonical :class:`AnswerObjectKind` (the toolkit-wide - answer ontology). This is a coarse ingestion-time tag; the physics-semantics - engine re-derives the precise ``object_kind`` independently when judging. + ``source_type`` is the dataset's own native answer-type label stored + verbatim (e.g. ``"MC"``, ``"NV"``, ``"EX"``, ``"Integer"``). The + semantics engine does **not** read it. ``None`` when the dataset provides + no such label. """ - value: AnswerValue # NUMBER: number; all other kinds: text - answer_kind: AnswerObjectKind - unit: str | None = None # Used only for PHYSICAL_QUANTITY (e.g., "m/s²", "N") + value: str + unit: str | None = None + source_type: str | None = None metadata: dict[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: - """Initialize metadata if not provided.""" if self.metadata is None: self.metadata = {} - def validate(self) -> bool: - """Validate the answer based on its kind.""" - validators = { - AnswerObjectKind.NUMBER: self._validate_number, - AnswerObjectKind.PHYSICAL_QUANTITY: self._validate_string, - AnswerObjectKind.EXPRESSION: self._validate_string, - AnswerObjectKind.RELATION: self._validate_string, - AnswerObjectKind.QUALITATIVE_LABEL: self._validate_string, - AnswerObjectKind.BOOLEAN: self._validate_string, - AnswerObjectKind.SIGN_DIRECTION: self._validate_string, - AnswerObjectKind.DESCRIPTIVE_TEXT: self._validate_string, - AnswerObjectKind.CHOICE: self._validate_option, - } - validator = validators.get(self.answer_kind) - return validator() if validator else False - - def _validate_number(self) -> bool: - """Validate number answers.""" - return isinstance(self.value, (int, float)) and not isinstance(self.value, bool) - - def _validate_string(self) -> bool: - """Validate string-based answers (equation, formula, physical_quantity, text).""" - return isinstance(self.value, str) and len(self.value.strip()) > 0 - - def _validate_option(self) -> bool: - """Validate option answers.""" - return isinstance(self.value, str) and len(self.value.strip()) > 0 - - # Type checking methods - def is_number(self) -> bool: - """Check if this is a dimensionless number answer.""" - return self.answer_kind == AnswerObjectKind.NUMBER - - def is_equation(self) -> bool: - """Check if this is an equation/relation answer.""" - return self.answer_kind == AnswerObjectKind.RELATION - - def is_physical_quantity(self) -> bool: - """Check if this is a physical quantity (number + units) answer.""" - return self.answer_kind == AnswerObjectKind.PHYSICAL_QUANTITY + # ------------------------------------------------------------------ + # Accessors (kept for convenience; no kind guards) + # ------------------------------------------------------------------ - def is_formula(self) -> bool: - """Check if this is a formula/expression answer.""" - return self.answer_kind == AnswerObjectKind.EXPRESSION - - def is_text(self) -> bool: - """Check if this is a free-form descriptive text answer.""" - return self.answer_kind == AnswerObjectKind.DESCRIPTIVE_TEXT - - def is_option(self) -> bool: - """Check if this is an option/choice answer.""" - return self.answer_kind == AnswerObjectKind.CHOICE - - def is_numerical(self) -> bool: - """Check if this has a numeric component (number or physical_quantity).""" - return self.answer_kind in ( - AnswerObjectKind.NUMBER, - AnswerObjectKind.PHYSICAL_QUANTITY, - ) - - def is_symbolic(self) -> bool: - """Check if this is a symbolic/math answer (relation, expression, or physical_quantity).""" - return self.answer_kind in ( - AnswerObjectKind.RELATION, - AnswerObjectKind.EXPRESSION, - AnswerObjectKind.PHYSICAL_QUANTITY, - ) + def get_value(self) -> str: + """Return the answer value string.""" + return self.value - # Numerical-specific methods def get_unit(self) -> str | None: - """Get the unit for numerical/physical quantity answers.""" - return self.unit if self.is_numerical() else None + """Return the unit string, or ``None`` when absent.""" + return self.unit def has_unit(self) -> bool: - """Check if the answer has a unit (physical quantity).""" - return self.is_physical_quantity() or ( - self.is_number() and self.unit is not None - ) - - def is_integer(self) -> bool: - """Check if the numerical value is an integer.""" - if not self.is_numerical(): - return False - return isinstance(self.value, int) or ( - isinstance(self.value, float) and self.value.is_integer() - ) + """Return ``True`` when a unit is present.""" + return self.unit is not None - def is_positive(self) -> bool: - """Check if the numerical value is positive.""" - if not self.is_numerical(): - return False - return ( - isinstance(self.value, (int, float)) - and not isinstance(self.value, bool) - and self.value > 0 - ) + # ------------------------------------------------------------------ + # Dunder helpers + # ------------------------------------------------------------------ - def is_negative(self) -> bool: - """Check if the numerical value is negative.""" - if not self.is_numerical(): - return False - return ( - isinstance(self.value, (int, float)) - and not isinstance(self.value, bool) - and self.value < 0 - ) - - # Symbolic-specific methods - def is_latex(self) -> bool: - """Check if the symbolic answer contains LaTeX formatting.""" - if not self.is_symbolic(): - return False - value = str(self.value) - return "$" in value or "\\" in value - - def get_clean_expression(self) -> str: - """Get the mathematical expression without LaTeX delimiters.""" - if not self.is_symbolic(): - return str(self.value) - clean = str(self.value).strip() - if clean.startswith("$$") and clean.endswith("$$"): - clean = clean[2:-2].strip() - elif clean.startswith("$") and clean.endswith("$"): - clean = clean[1:-1].strip() - return clean - - # Textual-specific methods - def word_count(self) -> int: - """Get the number of words in the text.""" - if not self.is_text(): - return 0 - return len(str(self.value).split()) - - def char_count(self) -> int: - """Get the number of characters in the text.""" - if not self.is_text(): - return 0 - return len(str(self.value)) - - def is_short(self) -> bool: - """Check if the text is short (less than 10 words).""" - return self.word_count() < 10 - - def is_long(self) -> bool: - """Check if the text is long (more than 50 words).""" - return self.word_count() > 50 - - def contains_keywords(self, keywords: list[str]) -> bool: - """Check if the text contains any of the specified keywords.""" - if not self.is_text(): - return False - text_lower = str(self.value).lower() - return any(keyword.lower() in text_lower for keyword in keywords) - - # Option-specific methods - def is_letter_option(self) -> bool: - """Check if the option is a letter (A, B, C, D, E).""" - if not self.is_option(): - return False - return str(self.value).upper() in ["A", "B", "C", "D", "E"] - - def is_yes_no(self) -> bool: - """Check if the option is Yes/No.""" - if not self.is_option(): - return False - return str(self.value).upper() in ["YES", "NO"] - - def is_true_false(self) -> bool: - """Check if the option is True/False.""" - if not self.is_option(): - return False - return str(self.value).upper() in ["TRUE", "FALSE"] - - def is_numeric_option(self) -> bool: - """Check if the option is a number (1, 2, 3, 4, 5).""" - if not self.is_option(): - return False - return str(self.value) in ["1", "2", "3", "4", "5"] - - def get_option_index(self) -> int | None: - """Get the numeric index of the option if applicable.""" - if not self.is_option(): - return None - value = str(self.value).upper() - if self.is_letter_option(): - return ord(value) - ord("A") # A=0, B=1, C=2, etc. - elif self.is_numeric_option(): - return int(value) - 1 # 1=0, 2=1, 3=2, etc. - return None - - # Utility methods def __str__(self) -> str: - """String representation of the answer.""" - if self.is_numerical() and self.unit: + if self.unit: return f"{self.value} {self.unit}" return str(self.value) def __repr__(self) -> str: - """Detailed string representation for debugging.""" - return f"Answer(value={repr(self.value)}, answer_kind={self.answer_kind.value}, unit={repr(self.unit)})" + return ( + f"Answer(value={self.value!r}, unit={self.unit!r}, " + f"source_type={self.source_type!r})" + ) + + # ------------------------------------------------------------------ + # Serialization + # ------------------------------------------------------------------ def to_dict(self) -> dict[str, Any]: - """Convert to dictionary for serialization.""" - result: dict[str, Any] = { - "value": self.value, - "answer_kind": self.answer_kind.value, - } + """Serialize to a JSON-safe dict (omits falsy optional fields).""" + result: dict[str, Any] = {"value": self.value} if self.unit: result["unit"] = self.unit + if self.source_type: + result["source_type"] = self.source_type if self.metadata: result["metadata"] = self.metadata return result - - def get_value(self) -> AnswerValue: - """Get the answer value.""" - return self.value - - def get_type(self) -> AnswerObjectKind: - """Get the answer kind.""" - return self.answer_kind - - def get_type_name(self) -> str: - """Get the answer kind as a string.""" - return self.answer_kind.value diff --git a/src/prkit/core/domain/physics_problem.py b/src/prkit/core/domain/physics_problem.py index 061f296..c9d86e7 100644 --- a/src/prkit/core/domain/physics_problem.py +++ b/src/prkit/core/domain/physics_problem.py @@ -13,8 +13,7 @@ from typing import TYPE_CHECKING, Any from ..logging_config import PRKitLogger -from .answer import Answer, AnswerValue -from .answer_kinds import AnswerObjectKind +from .answer import Answer from .physics_domain import PhysicsDomain # Get logger for this module @@ -312,8 +311,13 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls, data: dict[str, Any]) -> "PhysicsProblem": - """Create PhysicsProblem from dictionary.""" - # Extract core fields + """Create PhysicsProblem from dictionary. + + Legacy migration: previously-cached answer dicts may carry + ``answer_kind`` or ``answer_category`` in place of ``source_type``. + Both are accepted and folded into ``source_type`` so stored datasets + survive the reshape without data loss. + """ core_fields = [ "question", "problem_id", @@ -324,7 +328,6 @@ def from_dict(cls, data: dict[str, Any]) -> "PhysicsProblem": "image_path", "options", "correct_option", - "answer_kind", ] core_data: dict[str, Any] = {} @@ -333,48 +336,39 @@ def from_dict(cls, data: dict[str, Any]) -> "PhysicsProblem": for key, value in data.items(): if key in core_fields: if key == "answer" and isinstance(value, dict): - # Convert answer dictionary to Answer object - raw_answer_value = value.get("value", "") - if isinstance(raw_answer_value, (int, float, str)): - answer_value: AnswerValue = raw_answer_value - else: - answer_value = str(raw_answer_value) - answer_kind_str = value.get("answer_kind") - answer_unit = value.get("unit") - answer_metadata = value.get("metadata", {}) - - if answer_kind_str: - try: - answer_kind = AnswerObjectKind(answer_kind_str) - except ValueError: - answer_kind = AnswerObjectKind.DESCRIPTIVE_TEXT - else: - answer_kind = AnswerObjectKind.DESCRIPTIVE_TEXT - - # Create Answer object + answer_value = str(value.get("value", "")) + answer_unit = value.get("unit") or None + # Legacy migration: accept answer_kind / answer_category as source_type + source_type = ( + value.get("source_type") + or value.get("answer_kind") + or value.get("answer_category") + or None + ) + if source_type is not None: + source_type = str(source_type) + answer_metadata = value.get("metadata") or {} core_data[key] = Answer( value=answer_value, - answer_kind=answer_kind, unit=answer_unit, + source_type=source_type, metadata=answer_metadata, ) else: core_data[key] = value + elif key in ("answer_kind", "answer_category"): + # Top-level legacy keys: ignore (they belonged to the old schema) + pass elif key == "additional_fields": - # Handle additional fields separately if value: core_data["additional_fields"] = value elif key == "domain": - # Handle domain - will be converted in __post_init__ core_data["domain"] = value else: - # Store as custom field custom_data[key] = value - # Create instance problem = cls(**core_data) - # Add custom fields if custom_data: problem.additional_fields.update(custom_data) diff --git a/src/prkit/datasets/loaders/base_loader.py b/src/prkit/datasets/loaders/base_loader.py index cd08969..e497df0 100644 --- a/src/prkit/datasets/loaders/base_loader.py +++ b/src/prkit/datasets/loaders/base_loader.py @@ -10,7 +10,6 @@ from prkit.core import PRKitLogger from prkit.core.domain import PhysicalDataset, PhysicsProblem from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_kinds import AnswerObjectKind # Try to import PIL/Pillow for image loading PILImageModule: Any | None @@ -33,7 +32,7 @@ "problem_type", # problem type in OE, MC, MMC, etc. "domain", # domain in physics "language", # language - "answer_category", # answer category for comparison + "source_type", # dataset-native answer-type label (verbatim, may be None) "image_paths", # paths to associated image files (for visual problems) "options", # MC answer choices "correct_option", # MC index or key (dataset-specific) @@ -59,39 +58,6 @@ def raw_answer_to_text(value: Any) -> str: return str(value).strip() -def detect_answer_category(value: str) -> AnswerObjectKind: - """ - Infer the coarse answer kind from a string value when a dataset does not specify it. - - This is an ingestion-time hint only; the physics-semantics engine re-derives the - precise ``object_kind`` independently when judging. - - Strategy: - 1. Try to parse as pure number first -> NUMBER - 2. Check for mathematical expression patterns -> EXPRESSION - 3. Fall back to DESCRIPTIVE_TEXT if unclear - """ - value = str(value).strip() - - # remove \\boxed{} that wraps the value if present - value = re.sub(r"\\boxed\{([^}]+)\}", r"\1", value) - - # remove $$ that wraps the value if present - value = re.sub(r"\$\$(.*?)\$\$", r"\1", value) - value = re.sub(r"\$([^$]+)\$", r"\1", value) - - # Step 1: Check if it's a pure number (including scientific notation) - if is_pure_number(value): - return AnswerObjectKind.NUMBER - - # Step 2: Check if it's a mathematical expression - if is_mathematical_expression(value): - return AnswerObjectKind.EXPRESSION - - # Step 3: Default to free-form descriptive text - return AnswerObjectKind.DESCRIPTIVE_TEXT - - def is_pure_number(value: str) -> bool: """Check if value represents a single concrete number.""" # Remove common number formatting @@ -577,65 +543,31 @@ def _create_answer_from_raw( metadata: dict[str, Any], ) -> Answer | None: answer = metadata.get("answer") - answer_category = str(metadata.get("answer_category", "")) - problem_type = str(metadata.get("problem_type", "")) if answer is None: return None - if "MC" in problem_type: - return Answer( - value=raw_answer_to_text(answer), - answer_kind=AnswerObjectKind.CHOICE, - ) + # Extract unit from dict-shaped answers; None otherwise + if isinstance(answer, dict): + raw_value = answer.get("value") + unit: str | None = answer.get("unit") or None + else: + raw_value = answer + unit = None - # The metadata tag is a coarse ingestion hint; accept both the canonical - # AnswerObjectKind spellings and legacy dataset tags ("formula"/"equation"/ - # "text"/"option"). The semantics engine re-derives object_kind anyway. - if answer_category in ("number", "physical_quantity"): - if isinstance(answer, dict): - value = raw_answer_to_text(answer.get("value")) - unit = answer.get("unit", "") or "" - else: - value = raw_answer_to_text(answer) - unit = "" + value = raw_answer_to_text(raw_value) - # remove \\boxed{} that wraps the value if present - value = re.sub(r"\\boxed\{([^}]+)\}", r"\1", value) + # Strip LaTeX wrappers universally (\\boxed{}, $…$, $$…$$) + value = re.sub(r"\\boxed\{([^}]+)\}", r"\1", value) + value = re.sub(r"\$\$(.*?)\$\$", r"\1", value, flags=re.DOTALL) + value = re.sub(r"\$([^$]+)\$", r"\1", value) - # remove $$ that wraps the value if present - value = re.sub(r"\$\$(.*?)\$\$", r"\1", value) - value = re.sub(r"\$([^$]+)\$", r"\1", value) + # source_type is set by the loader from the dataset's native type field + source_type: str | None = metadata.get("source_type") or None + if source_type is not None: + source_type = str(source_type) - kind = ( - AnswerObjectKind.PHYSICAL_QUANTITY if unit else AnswerObjectKind.NUMBER - ) - return Answer(value=value, answer_kind=kind, unit=unit or None) - elif answer_category in ("expression", "formula"): - return Answer( - value=raw_answer_to_text(answer), - answer_kind=AnswerObjectKind.EXPRESSION, - ) - elif answer_category in ("relation", "equation"): - return Answer( - value=raw_answer_to_text(answer), - answer_kind=AnswerObjectKind.RELATION, - ) - elif answer_category in ("descriptive_text", "text"): - return Answer( - value=raw_answer_to_text(answer), - answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT, - ) - elif answer_category in ("choice", "option"): - return Answer( - value=raw_answer_to_text(answer), - answer_kind=AnswerObjectKind.CHOICE, - ) - else: - # fallback to auto-detect when answer kind not specified - answer_text = raw_answer_to_text(answer) - detected = detect_answer_category(answer_text) - return Answer(value=answer_text, answer_kind=detected) + return Answer(value=value, unit=unit, source_type=source_type) def create_physics_problem( self, @@ -725,7 +657,8 @@ def create_physics_problem( # Create Answer object from answer answer_obj = self._create_answer_from_raw(metadata) metadata.pop("answer", None) - metadata.pop("answer_category", None) + metadata.pop("answer_category", None) # defensive: loaders may still set it + metadata.pop("source_type", None) # consumed into Answer; don't leak metadata.pop("unit", None) # collect all other fields as additional fields diff --git a/src/prkit/datasets/loaders/jeebench_loader.py b/src/prkit/datasets/loaders/jeebench_loader.py index d5df310..254987f 100644 --- a/src/prkit/datasets/loaders/jeebench_loader.py +++ b/src/prkit/datasets/loaders/jeebench_loader.py @@ -240,13 +240,8 @@ def _process_metadata(self, metadata: dict[str, Any]) -> dict[str, Any]: # Set language to English (JEEBench is primarily in English) metadata["language"] = "en" - # Set answer category based on question type - if original_type in ["Integer", "Numeric"]: - metadata["answer_category"] = "number" - elif metadata["problem_type"] in ["MC", "MultipleMC"]: - metadata["answer_category"] = "option" - else: - metadata["answer_category"] = "text" + # source_type carries the dataset's own native type label verbatim + metadata["source_type"] = original_type or None return metadata diff --git a/src/prkit/datasets/loaders/phybench_loader.py b/src/prkit/datasets/loaders/phybench_loader.py index 35a56ab..ed7622b 100644 --- a/src/prkit/datasets/loaders/phybench_loader.py +++ b/src/prkit/datasets/loaders/phybench_loader.py @@ -80,11 +80,7 @@ def DOMAIN_MAPPING(self) -> dict[str, PhysicsDomain]: def _process_metadata(self, metadata: dict[str, Any]) -> dict[str, Any]: """Process metadata to create standardized problem fields.""" - - metadata["answer_category"] = "formula" - self._map_domain(metadata) - return metadata def load( diff --git a/src/prkit/datasets/loaders/physbench_loader.py b/src/prkit/datasets/loaders/physbench_loader.py index be74fa9..537b1e8 100644 --- a/src/prkit/datasets/loaders/physbench_loader.py +++ b/src/prkit/datasets/loaders/physbench_loader.py @@ -235,7 +235,6 @@ def _process_metadata( answer = metadata.get("answer") if isinstance(answer, str) and answer.strip().upper() in {"A", "B", "C", "D"}: metadata["correct_option"] = ord(answer.strip().upper()) - ord("A") - metadata["answer_category"] = "option" file_names = metadata.get("file_name") or [] image_paths, video_paths, missing_media_count = self._resolve_media_paths( diff --git a/src/prkit/datasets/loaders/physics_loader.py b/src/prkit/datasets/loaders/physics_loader.py index 111fb97..e0ccda1 100644 --- a/src/prkit/datasets/loaders/physics_loader.py +++ b/src/prkit/datasets/loaders/physics_loader.py @@ -15,7 +15,7 @@ from prkit.core.domain import PhysicalDataset, PhysicsDomain, PhysicsProblem from prkit.datasets.license_registry import get_license -from .base_loader import BaseDatasetLoader, detect_answer_category +from .base_loader import BaseDatasetLoader class PhysicsLoader(BaseDatasetLoader): @@ -257,7 +257,7 @@ def _process_metadata( source_file: str, decode_images: bool, ) -> dict[str, Any]: - answer_value, answer_parts, answer_category = self._normalize_answers( + answer_value, answer_parts = self._normalize_answers( metadata.pop("final_answers", None) ) graphs = metadata.pop("graphs", None) @@ -291,7 +291,6 @@ def _process_metadata( ) metadata["answer"] = answer_value - metadata["answer_category"] = answer_category metadata["problem_type"] = "OE" metadata["domain"] = ( self.DOMAIN_MAPPING.get(resolved_domain, PhysicsDomain.OTHER) @@ -309,7 +308,7 @@ def _process_metadata( metadata["source_file"] = source_file return metadata - def _normalize_answers(self, raw_answers: Any) -> tuple[str, list[str], str]: + def _normalize_answers(self, raw_answers: Any) -> tuple[str, list[str]]: if raw_answers is None: answer_parts: list[str] = [] elif isinstance(raw_answers, list): @@ -322,24 +321,15 @@ def _normalize_answers(self, raw_answers: Any) -> tuple[str, list[str], str]: ) if not answer_parts: - return "", [], "descriptive_text" + return "", [] if len(answer_parts) == 1: - answer_value = answer_parts[0] - answer_category = detect_answer_category(answer_value).value - return answer_value, answer_parts, answer_category + return answer_parts[0], answer_parts answer_value = "\n".join( f"({index + 1}) {answer}" for index, answer in enumerate(answer_parts) ) - detected_categories = [ - detect_answer_category(answer) for answer in answer_parts - ] - if all( - category.value == "descriptive_text" for category in detected_categories - ): - return answer_value, answer_parts, "descriptive_text" - return answer_value, answer_parts, "expression" + return answer_value, answer_parts def _decode_graphs( self, diff --git a/src/prkit/datasets/loaders/tpbench_loader.py b/src/prkit/datasets/loaders/tpbench_loader.py index 7cff49c..8652d12 100644 --- a/src/prkit/datasets/loaders/tpbench_loader.py +++ b/src/prkit/datasets/loaders/tpbench_loader.py @@ -90,9 +90,7 @@ def DOMAIN_MAPPING(self) -> dict[str, PhysicsDomain]: def _process_metadata(self, metadata: dict[str, Any]) -> dict[str, Any]: """Process metadata to create standardized problem fields.""" - metadata["answer_category"] = "formula" self._map_domain(metadata) - return metadata def load( diff --git a/src/prkit/datasets/loaders/ugphysics_loader.py b/src/prkit/datasets/loaders/ugphysics_loader.py index 3188f4e..a40d818 100644 --- a/src/prkit/datasets/loaders/ugphysics_loader.py +++ b/src/prkit/datasets/loaders/ugphysics_loader.py @@ -144,6 +144,9 @@ def _process_metadata( metadata["raw_answers"] = raw_answers metadata["raw_unit"] = raw_unit + # source_type carries the dataset's own native label verbatim + metadata["source_type"] = raw_answer_type or None + if "MC" in raw_answer_type: option_answers = self._split_answer_parts(raw_answers) option_answers = [ @@ -152,7 +155,6 @@ def _process_metadata( if answer ] metadata["problem_type"] = "MultipleMC" if is_multiple_answer else "MC" - metadata["answer_category"] = "option" if is_multiple_answer: metadata["answer"] = ", ".join(option_answers) metadata["answer_parts"] = option_answers @@ -180,31 +182,24 @@ def _process_metadata( "unit": unit_part, } ) - metadata["answer_category"] = "text" metadata["answer"] = "; ".join( self._format_physical_answer(str(part["value"] or ""), part["unit"]) for part in normalized_parts ) metadata["answer_parts"] = normalized_parts else: - metadata["answer_category"] = ( - "physical_quantity" if normalized_unit else "number" - ) metadata["answer"] = { "value": self._clean_answer_text(raw_answers), "unit": normalized_unit, } elif "EX" in raw_answer_type: - metadata["answer_category"] = "formula" if is_multiple_answer: answer_parts = self._split_answer_parts(raw_answers) - metadata["answer_category"] = "text" metadata["answer"] = "; ".join(answer_parts) metadata["answer_parts"] = answer_parts else: metadata["answer"] = self._clean_answer_text(raw_answers) else: - metadata["answer_category"] = "text" if is_multiple_answer: answer_parts = self._split_answer_parts(raw_answers) metadata["answer"] = "; ".join(answer_parts) diff --git a/src/prkit/evaluation/edit_distance/__init__.py b/src/prkit/evaluation/edit_distance/__init__.py new file mode 100644 index 0000000..312337d --- /dev/null +++ b/src/prkit/evaluation/edit_distance/__init__.py @@ -0,0 +1,35 @@ +"""Pure Expression Edit Distance (EED / SEED) algorithm core — related-work methods. + +A self-contained reimplementation of PHYBench's Expression Edit Distance and +CMPhysBench's Scalable EED *tree-edit* machinery. These modules are deliberately +**free of any PRKit semantics dependency** — they import only ``sympy`` + stdlib: + +* :mod:`.tree` — SymPy expression → :class:`ExprNode` tree builder +* :mod:`.zss` — extended Zhang-Shasha tree edit distance +* :mod:`.score` — EED cost model and distance → score map +* :mod:`.timeout` — bounded ``simplify`` helper + +The physics-aware *dispatch* that runs these on PRKit's normalized semantics (the +SEED answer-kind/unit/symbolic glue) lives separately in +:mod:`prkit.semantics.edit_distance.pipeline` (``eed_compare``), which imports this +core. Keeping the algorithm here means the related-work method has no dependency on +PRKit's own semantics layer. +""" + +from __future__ import annotations + +from .score import EditCosts, eed_score +from .timeout import SimplifyTimeout, run_with_timeout +from .tree import ExprNode, UnsupportedExpressionError, sympy_to_tree +from .zss import tree_edit_distance + +__all__ = [ + "EditCosts", + "ExprNode", + "SimplifyTimeout", + "UnsupportedExpressionError", + "eed_score", + "run_with_timeout", + "sympy_to_tree", + "tree_edit_distance", +] diff --git a/src/prkit/semantics/edit_distance/score.py b/src/prkit/evaluation/edit_distance/score.py similarity index 100% rename from src/prkit/semantics/edit_distance/score.py rename to src/prkit/evaluation/edit_distance/score.py diff --git a/src/prkit/semantics/edit_distance/timeout.py b/src/prkit/evaluation/edit_distance/timeout.py similarity index 100% rename from src/prkit/semantics/edit_distance/timeout.py rename to src/prkit/evaluation/edit_distance/timeout.py diff --git a/src/prkit/semantics/edit_distance/tree.py b/src/prkit/evaluation/edit_distance/tree.py similarity index 100% rename from src/prkit/semantics/edit_distance/tree.py rename to src/prkit/evaluation/edit_distance/tree.py diff --git a/src/prkit/semantics/edit_distance/zss.py b/src/prkit/evaluation/edit_distance/zss.py similarity index 98% rename from src/prkit/semantics/edit_distance/zss.py rename to src/prkit/evaluation/edit_distance/zss.py index 96cbb54..e784f00 100644 --- a/src/prkit/semantics/edit_distance/zss.py +++ b/src/prkit/evaluation/edit_distance/zss.py @@ -1,7 +1,7 @@ """Extended Zhang-Shasha tree-edit distance with subtree cluster discount. A clean reimplementation of the ordered-tree edit distance (Zhang & Shasha 1989) -parametrized by :class:`~prkit.semantics.edit_distance.score.EditCosts`, extended +parametrized by :class:`~prkit.evaluation.edit_distance.score.EditCosts`, extended with PHYBench's whole-subtree "cluster discount": deleting or inserting an entire subtree can cost less than the per-node sum, so a large wrong sub-formula is not penalized linearly. diff --git a/src/prkit/evaluation/llm_judge/payload.py b/src/prkit/evaluation/llm_judge/payload.py index 5ffac3a..807425c 100644 --- a/src/prkit/evaluation/llm_judge/payload.py +++ b/src/prkit/evaluation/llm_judge/payload.py @@ -9,10 +9,10 @@ def answer_to_text_and_category(answer: str | Answer) -> tuple[str, str]: - """Plain text and answer-kind label for embedding in a judge JSON payload.""" + """Plain text and native-type label for embedding in a judge JSON payload.""" if isinstance(answer, Answer): - return str(answer).strip(), answer.answer_kind.value - return str(answer).strip(), "unknown" + return str(answer).strip(), answer.source_type or "" + return str(answer).strip(), "" def clean_answer_text(answer_text: str) -> str: diff --git a/src/prkit/semantics/README.md b/src/prkit/semantics/README.md index 648a191..bae4d37 100644 --- a/src/prkit/semantics/README.md +++ b/src/prkit/semantics/README.md @@ -305,7 +305,7 @@ The smallest reproducible path is deterministic question inference, answer normalization, contract construction, and evaluation. ```python -from prkit.core.domain import Answer, AnswerObjectKind, PhysicsProblem +from prkit.core.domain import Answer, PhysicsProblem from prkit.semantics import ( ComparisonPolicyMode, build_evaluation_contract, @@ -317,7 +317,7 @@ from prkit.semantics import ( problem = PhysicsProblem( problem_id="demo-speed", question="Find the speed in m/s.", - answer=Answer(value="18", unit="km/h", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), + answer=Answer(value="18", unit="km/h", source_type="physical_quantity"), ) question_semantics = infer_reference_question_semantics(problem) diff --git a/src/prkit/semantics/edit_distance/__init__.py b/src/prkit/semantics/edit_distance/__init__.py index 3008af0..dc597d4 100644 --- a/src/prkit/semantics/edit_distance/__init__.py +++ b/src/prkit/semantics/edit_distance/__init__.py @@ -1,11 +1,13 @@ -"""Expression Edit Distance (EED / SEED) partial-credit algorithm on PRKit's substrate. +"""SEED dispatch (``eed_compare``) — the physics-aware glue over the EED core. -A self-contained reimplementation of PHYBench's Expression Edit Distance and -CMPhysBench's Scalable EED, run on PRKit's existing SymPy parser, LaTeX normalizer, -and unit backend instead of vendoring ``latex2sympy2`` + ``pint``. The pure pieces -(:mod:`.tree`, :mod:`.zss`, :mod:`.score`, :mod:`.timeout`) depend only on -``sympy`` + stdlib; :mod:`.pipeline` adds the SEED dispatch that reuses the -comparison engine's parsing/unit primitives. +This subpackage holds only the integration layer: :mod:`.pipeline` reproduces +CMPhysBench SEED's answer-kind dispatch on top of PRKit's normalized +:class:`~prkit.semantics.PhysicsAnswerSemantics` (numeric/unit/symbolic primitives +from :mod:`prkit.semantics.comparison`). It depends on the semantics layer **by +design**. + +The pure tree-edit algorithm core (the related-work method, with no semantics +dependency) lives in :mod:`prkit.evaluation.edit_distance`, which this module imports. See :class:`prkit.scoring.PartialCreditScorer` for the ``Scorer`` wrapper that maps :class:`EedResult` onto :class:`prkit.core.verdict.Verdict`. @@ -14,18 +16,9 @@ from __future__ import annotations from .pipeline import EedConfig, EedResult, eed_compare -from .score import EditCosts, eed_score -from .tree import ExprNode, UnsupportedExpressionError, sympy_to_tree -from .zss import tree_edit_distance __all__ = [ - "EditCosts", "EedConfig", "EedResult", - "ExprNode", - "UnsupportedExpressionError", "eed_compare", - "eed_score", - "sympy_to_tree", - "tree_edit_distance", ] diff --git a/src/prkit/semantics/edit_distance/pipeline.py b/src/prkit/semantics/edit_distance/pipeline.py index c706c5c..63a48f8 100644 --- a/src/prkit/semantics/edit_distance/pipeline.py +++ b/src/prkit/semantics/edit_distance/pipeline.py @@ -24,6 +24,17 @@ from sympy import simplify +# Pure tree-edit algorithm core (related-work EED/SEED), no semantics dependency. +from prkit.evaluation.edit_distance import ( + EditCosts, + SimplifyTimeout, + UnsupportedExpressionError, + eed_score, + run_with_timeout, + sympy_to_tree, + tree_edit_distance, +) + from ..comparison.common import available_texts, context_symbol_alias_map from ..comparison.numeric import ( NumericComparableAnswer, @@ -43,10 +54,6 @@ PhysicsQuestionSemantics, QuestionUnitPolicy, ) -from .score import EditCosts, eed_score -from .timeout import SimplifyTimeout, run_with_timeout -from .tree import UnsupportedExpressionError, sympy_to_tree -from .zss import tree_edit_distance #: Operators PRKit's tree grammar cannot represent; mirror EED's hard 0-score guard. _UNSUPPORTED_RE = re.compile(r"\\(?:i{1,3}nt|oint|sum|prod)") diff --git a/src/prkit/semantics/inference/__init__.py b/src/prkit/semantics/inference/__init__.py deleted file mode 100644 index 3fbe05b..0000000 --- a/src/prkit/semantics/inference/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Deprecated import alias for :mod:`prkit.semantics.build`. - -.. deprecated:: - This subpackage was renamed to :mod:`prkit.semantics.build`: the layer *builds* - reference and prediction records (it **creates** references — not only "infers"), - so ``build`` names it for what it does. Importing ``prkit.semantics.inference`` - re-exports the full ``prkit.semantics.build`` surface and emits a - :class:`DeprecationWarning`; this alias will be removed in a future release. - Import from ``prkit.semantics.build`` (or the ``prkit.semantics`` public surface) - instead. See ``prkit/CONTRACT.md``. -""" - -import warnings - -from prkit.semantics.build import * # noqa: F403 - re-export the renamed surface -from prkit.semantics.build import __all__ as __all__ - -warnings.warn( - "prkit.semantics.inference has been renamed to prkit.semantics.build; import " - "from prkit.semantics.build (or the prkit.semantics public surface) instead. " - "This alias will be removed in a future release.", - DeprecationWarning, - stacklevel=2, -) diff --git a/src/prkit/testing/conformance.py b/src/prkit/testing/conformance.py index c4c7ab3..910e629 100644 --- a/src/prkit/testing/conformance.py +++ b/src/prkit/testing/conformance.py @@ -17,7 +17,7 @@ from pydantic import BaseModel from prkit.api import DatasetProvider, ModelClient, Scorer, Verdict -from prkit.core.domain import AnswerObjectKind, PhysicsProblem +from prkit.core.domain import PhysicsProblem from prkit.core.model_clients.structured_output import StructuredOutputPlan from prkit.datasets.loaders.base_loader import BaseDatasetLoader @@ -122,9 +122,15 @@ def check_dataset( problem, PhysicsProblem ), f"load() must yield PhysicsProblem, got {type(problem)!r}" if problem.answer is not None: - assert problem.answer.answer_kind in AnswerObjectKind, ( - f"problem {problem.problem_id!r} has invalid answer_kind " - f"{problem.answer.answer_kind!r}" + assert isinstance(problem.answer.value, str), ( + f"problem {problem.problem_id!r}: answer.value must be str, " + f"got {type(problem.answer.value)!r}" + ) + assert problem.answer.source_type is None or isinstance( + problem.answer.source_type, str + ), ( + f"problem {problem.problem_id!r}: answer.source_type must be " + f"str or None, got {type(problem.answer.source_type)!r}" ) diff --git a/tests/conftest.py b/tests/conftest.py index 65584db..8ab5c4e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,6 @@ from prkit.core.domain import ( Answer, - AnswerObjectKind, PhysicalDataset, PhysicsDomain, PhysicsProblem, @@ -19,30 +18,25 @@ @pytest.fixture def sample_answer_numerical(): """Create a sample numerical answer.""" - return Answer( - value=42.0, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, unit="m/s" - ) + return Answer(value="42.0", unit="m/s", source_type="NV") @pytest.fixture def sample_answer_symbolic(): """Create a sample symbolic answer.""" - return Answer(value="x^2 + 2x + 1", answer_kind=AnswerObjectKind.EXPRESSION) + return Answer(value="x^2 + 2x + 1") @pytest.fixture def sample_answer_textual(): """Create a sample textual answer.""" - return Answer( - value="The force is equal to mass times acceleration", - answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT, - ) + return Answer(value="The force is equal to mass times acceleration") @pytest.fixture def sample_answer_option(): """Create a sample option answer.""" - return Answer(value="A", answer_kind=AnswerObjectKind.CHOICE) + return Answer(value="A", source_type="MC") @pytest.fixture @@ -51,9 +45,7 @@ def sample_physics_problem(): return PhysicsProblem( problem_id="test_001", question="What is the speed of light?", - answer=Answer( - value=3e8, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, unit="m/s" - ), + answer=Answer(value="3e8", unit="m/s"), solution="The speed of light in vacuum is approximately 3 × 10^8 m/s", domain=PhysicsDomain.CLASSICAL_MECHANICS, language="en", @@ -67,7 +59,7 @@ def sample_physics_problem_mc(): return PhysicsProblem( problem_id="test_002", question="What is F = ma?", - answer=Answer(value="A", answer_kind=AnswerObjectKind.CHOICE), + answer=Answer(value="A", source_type="MCQ"), options=[ "Newton's second law", "Newton's first law", @@ -104,7 +96,7 @@ def sample_problems_list(): problem = PhysicsProblem( problem_id=f"test_{i:03d}", question=f"Test question {i}", - answer=Answer(value=i, answer_kind=AnswerObjectKind.NUMBER), + answer=Answer(value=str(i)), domain=( PhysicsDomain.CLASSICAL_MECHANICS if i % 2 == 0 diff --git a/tests/prkit/core/domain/test_answer.py b/tests/prkit/core/domain/test_answer.py index 26dbcb5..c8c8d0e 100644 --- a/tests/prkit/core/domain/test_answer.py +++ b/tests/prkit/core/domain/test_answer.py @@ -1,324 +1,117 @@ -""" -Tests for Answer model. -""" +"""Tests for the thin Answer observation record.""" -from prkit.core.domain import Answer, AnswerObjectKind - - -class TestAnswer: - """Test cases for Answer model.""" - - def test_answer_creation_numerical(self): - """Test creating a numerical answer.""" - answer = Answer(value=42.0, answer_kind=AnswerObjectKind.NUMBER, unit="m/s") - assert answer.value == 42.0 - assert answer.answer_kind == AnswerObjectKind.NUMBER - assert answer.unit == "m/s" - assert answer.metadata == {} - - def test_answer_creation_symbolic(self): - """Test creating a symbolic answer.""" - answer = Answer(value="x^2 + 1", answer_kind=AnswerObjectKind.EXPRESSION) - assert answer.value == "x^2 + 1" - assert answer.answer_kind == AnswerObjectKind.EXPRESSION - assert answer.unit is None - - def test_answer_creation_textual(self): - """Test creating a textual answer.""" - answer = Answer( - value="The answer is 42", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT - ) - assert answer.value == "The answer is 42" - assert answer.answer_kind == AnswerObjectKind.DESCRIPTIVE_TEXT - - def test_answer_creation_option(self): - """Test creating an option answer.""" - answer = Answer(value="A", answer_kind=AnswerObjectKind.CHOICE) - assert answer.value == "A" - assert answer.answer_kind == AnswerObjectKind.CHOICE - - def test_answer_metadata_initialization(self): - """Test that metadata is initialized as empty dict.""" - answer = Answer(value=1, answer_kind=AnswerObjectKind.NUMBER) - assert answer.metadata == {} - - def test_answer_metadata_custom(self): - """Test custom metadata.""" - metadata = {"source": "test", "confidence": 0.9} - answer = Answer(value=1, answer_kind=AnswerObjectKind.NUMBER, metadata=metadata) - assert answer.metadata == metadata - - def test_answer_validation_numerical(self): - """Test numerical answer validation.""" - valid_answer = Answer(value=42.0, answer_kind=AnswerObjectKind.NUMBER) - assert valid_answer.validate() is True - - invalid_answer = Answer( - value="not a number", answer_kind=AnswerObjectKind.NUMBER - ) - assert invalid_answer.validate() is False - - def test_answer_validation_symbolic(self): - """Test symbolic answer validation.""" - valid_answer = Answer(value="x^2", answer_kind=AnswerObjectKind.EXPRESSION) - assert valid_answer.validate() is True - - invalid_answer = Answer(value="", answer_kind=AnswerObjectKind.EXPRESSION) - assert invalid_answer.validate() is False - - def test_answer_validation_textual(self): - """Test textual answer validation.""" - valid_answer = Answer( - value="Some text", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT - ) - assert valid_answer.validate() is True - - invalid_answer = Answer(value="", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) - assert invalid_answer.validate() is False - - def test_answer_category_checking(self): - """Test answer type checking methods.""" - numerical = Answer(value=1, answer_kind=AnswerObjectKind.NUMBER) - symbolic = Answer(value="x", answer_kind=AnswerObjectKind.EXPRESSION) - textual = Answer(value="text", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) - option = Answer(value="A", answer_kind=AnswerObjectKind.CHOICE) - - assert numerical.is_numerical() is True - assert numerical.is_symbolic() is False - assert symbolic.is_symbolic() is True - assert textual.is_text() is True - assert option.is_option() is True - - def test_answer_numerical_methods(self): - """Test numerical-specific methods.""" - answer = Answer(value=42, answer_kind=AnswerObjectKind.NUMBER, unit="m/s") - assert answer.get_unit() == "m/s" - assert answer.has_unit() is True - assert answer.is_integer() is True - assert answer.is_positive() is True - - negative_answer = Answer(value=-5, answer_kind=AnswerObjectKind.NUMBER) - assert negative_answer.is_negative() is True - - def test_answer_symbolic_methods(self): - """Test symbolic-specific methods.""" - latex_answer = Answer(value="$x^2$", answer_kind=AnswerObjectKind.EXPRESSION) - assert latex_answer.is_latex() is True - - clean = latex_answer.get_clean_expression() - assert "$" not in clean or clean.startswith("$") - - def test_answer_textual_methods(self): - """Test textual-specific methods.""" - answer = Answer( - value="This is a test answer", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT - ) - assert answer.word_count() == 5 - assert answer.char_count() == 21 # "This is a test answer" = 21 chars - assert answer.is_short() is True - assert answer.is_long() is False - assert answer.contains_keywords(["test", "answer"]) is True - - def test_answer_option_methods(self): - """Test option-specific methods.""" - letter_answer = Answer(value="A", answer_kind=AnswerObjectKind.CHOICE) - assert letter_answer.is_letter_option() is True - assert letter_answer.get_option_index() == 0 - - numeric_answer = Answer(value="1", answer_kind=AnswerObjectKind.CHOICE) - assert numeric_answer.is_numeric_option() is True - - yes_answer = Answer(value="YES", answer_kind=AnswerObjectKind.CHOICE) - assert yes_answer.is_yes_no() is True - - def test_answer_to_dict(self): - """Test answer serialization to dictionary.""" - answer = Answer( - value=42, - answer_kind=AnswerObjectKind.NUMBER, - unit="m/s", - metadata={"test": True}, - ) - result = answer.to_dict() - - assert result["value"] == 42 - assert result["answer_kind"] == "number" - assert result["unit"] == "m/s" - assert result["metadata"]["test"] is True - - def test_answer_str_repr(self): - """Test string representations.""" - answer = Answer(value=42, answer_kind=AnswerObjectKind.NUMBER, unit="m/s") - assert "42" in str(answer) - assert "m/s" in str(answer) - - assert "Answer" in repr(answer) - assert "number" in repr(answer) - - def test_answer_validation_numerical_bool_false(self): - """Test that boolean False is not valid for numerical.""" - answer = Answer(value=False, answer_kind=AnswerObjectKind.NUMBER) - assert answer.validate() is False - - def test_answer_validation_numerical_bool_true(self): - """Test that boolean True is not valid for numerical.""" - answer = Answer(value=True, answer_kind=AnswerObjectKind.NUMBER) - assert answer.validate() is False - - def test_answer_validation_symbolic_whitespace_only(self): - """Test that whitespace-only string is invalid for symbolic.""" - answer = Answer(value=" \n\t ", answer_kind=AnswerObjectKind.EXPRESSION) - assert answer.validate() is False - - def test_answer_numerical_zero(self): - """Test numerical answer with zero value.""" - answer = Answer(value=0, answer_kind=AnswerObjectKind.NUMBER) - assert answer.is_numerical() is True - assert answer.is_positive() is False - assert answer.is_negative() is False - assert answer.is_integer() is True - - def test_answer_numerical_float_integer(self): - """Test numerical answer with float that is integer.""" - answer = Answer(value=42.0, answer_kind=AnswerObjectKind.NUMBER) - assert answer.is_integer() is True - - def test_answer_symbolic_latex_double_dollar(self): - """Test symbolic answer with double dollar LaTeX.""" - answer = Answer(value="$$x^2 + y^2$$", answer_kind=AnswerObjectKind.EXPRESSION) - clean = answer.get_clean_expression() - assert "$$" not in clean or clean == "$$x^2 + y^2$$" - - def test_answer_symbolic_latex_single_dollar(self): - """Test symbolic answer with single dollar LaTeX.""" - answer = Answer(value="$x^2$", answer_kind=AnswerObjectKind.EXPRESSION) - clean = answer.get_clean_expression() - assert "$" not in clean or clean == "$x^2$" - - def test_answer_symbolic_backslash_latex(self): - """Test symbolic answer with backslash LaTeX.""" - answer = Answer(value="\\frac{1}{2}", answer_kind=AnswerObjectKind.EXPRESSION) - assert answer.is_latex() is True - - def test_answer_textual_word_count_empty(self): - """Test word count for empty textual answer.""" - answer = Answer(value="", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) - assert answer.word_count() == 0 - - def test_answer_textual_word_count_multiple_spaces(self): - """Test word count with multiple spaces.""" - answer = Answer( - value="word1 word2 word3", - answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT, - ) - assert answer.word_count() == 3 - - def test_answer_textual_is_long(self): - """Test is_long method for textual answer.""" - long_text = " ".join(["word"] * 60) # 60 words - answer = Answer(value=long_text, answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) - assert answer.is_long() is True - - def test_answer_textual_contains_keywords_case_insensitive(self): - """Test contains_keywords is case insensitive.""" - answer = Answer( - value="This is a TEST", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT - ) - assert answer.contains_keywords(["test"]) is True - assert answer.contains_keywords(["TEST"]) is True - assert answer.contains_keywords(["Test"]) is True - - def test_answer_option_all_letters(self): - """Test option methods for all letter options.""" - for letter in ["A", "B", "C", "D", "E"]: - answer = Answer(value=letter, answer_kind=AnswerObjectKind.CHOICE) - assert answer.is_letter_option() is True - assert answer.get_option_index() is not None - - def test_answer_option_numeric_strings(self): - """Test option methods for numeric option strings.""" - for num_str in ["1", "2", "3", "4", "5"]: - answer = Answer(value=num_str, answer_kind=AnswerObjectKind.CHOICE) - assert answer.is_numeric_option() is True - assert answer.get_option_index() is not None - - def test_answer_option_invalid_letter(self): - """Test option with invalid letter.""" - answer = Answer(value="F", answer_kind=AnswerObjectKind.CHOICE) - assert answer.is_letter_option() is False - - def test_answer_option_yes_no_variants(self): - """Test yes/no option variants.""" - for variant in ["YES", "yes", "Yes", "NO", "no", "No"]: - answer = Answer(value=variant, answer_kind=AnswerObjectKind.CHOICE) - assert answer.is_yes_no() is True - - def test_answer_option_true_false_variants(self): - """Test true/false option variants.""" - for variant in ["TRUE", "true", "True", "FALSE", "false", "False"]: - answer = Answer(value=variant, answer_kind=AnswerObjectKind.CHOICE) - assert answer.is_true_false() is True - - def test_answer_to_dict_without_unit(self): - """Test to_dict without unit.""" - answer = Answer(value="test", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) - result = answer.to_dict() - assert "unit" not in result - - def test_answer_to_dict_without_metadata(self): - """Test to_dict with empty metadata.""" - answer = Answer(value=1, answer_kind=AnswerObjectKind.NUMBER) - answer.metadata = {} - result = answer.to_dict() - # Metadata may or may not be included if empty - assert "value" in result - assert "answer_kind" in result - - def test_answer_get_value(self): - """Test get_value method.""" - answer = Answer(value=42, answer_kind=AnswerObjectKind.NUMBER) - assert answer.get_value() == 42 - - def test_answer_get_type(self): - """Test get_type method.""" - answer = Answer(value="test", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) - assert answer.get_type() == AnswerObjectKind.DESCRIPTIVE_TEXT - - def test_answer_get_type_name(self): - """Test get_type_name method.""" - answer = Answer(value="test", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) - assert answer.get_type_name() == "descriptive_text" - - def test_answer_additional_false_paths_and_option_validation(self): - text_answer = Answer( - value="text", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT - ) - option_answer = Answer(value=" ", answer_kind=AnswerObjectKind.CHOICE) - numeric_answer = Answer(value=3.5, answer_kind=AnswerObjectKind.NUMBER) - symbolic_answer = Answer( - value="plain", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT - ) - invalid_option = Answer(value="Z", answer_kind=AnswerObjectKind.CHOICE) - - assert option_answer.validate() is False - assert text_answer.is_number() is False - assert text_answer.is_equation() is False - assert text_answer.is_physical_quantity() is False - assert text_answer.is_formula() is False - assert text_answer.get_unit() is None - assert numeric_answer.has_unit() is False - assert numeric_answer.is_integer() is False - assert symbolic_answer.is_latex() is False - assert symbolic_answer.get_clean_expression() == "plain" - assert invalid_option.is_true_false() is False - assert invalid_option.get_option_index() is None - - def test_answer_str_without_unit(self): - """Test __str__ without unit.""" - answer = Answer(value=42, answer_kind=AnswerObjectKind.NUMBER) - assert str(answer) == "42" - - def test_answer_str_non_numerical(self): - """Test __str__ for non-numerical answer.""" - answer = Answer(value="test", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT) - assert str(answer) == "test" +from prkit.core.domain.answer import Answer + + +class TestAnswerCreation: + def test_value_only(self): + a = Answer(value="42") + assert a.value == "42" + assert a.unit is None + assert a.source_type is None + assert a.metadata == {} + + def test_with_unit(self): + a = Answer(value="9.81", unit="m/s^2") + assert a.value == "9.81" + assert a.unit == "m/s^2" + + def test_with_source_type(self): + a = Answer(value="A", source_type="MC") + assert a.source_type == "MC" + + def test_with_all_fields(self): + a = Answer(value="5", unit="N", source_type="NV", metadata={"raw": True}) + assert a.value == "5" + assert a.unit == "N" + assert a.source_type == "NV" + assert a.metadata["raw"] is True + + def test_metadata_default_not_shared(self): + a = Answer(value="x") + b = Answer(value="y") + a.metadata["k"] = 1 + assert "k" not in b.metadata + + def test_metadata_none_normalized(self): + a = Answer(value="x", metadata=None) # type: ignore[arg-type] + assert a.metadata == {} + + +class TestAnswerAccessors: + def test_get_value(self): + a = Answer(value="hello") + assert a.get_value() == "hello" + + def test_get_unit_present(self): + a = Answer(value="3", unit="m") + assert a.get_unit() == "m" + + def test_get_unit_absent(self): + a = Answer(value="3") + assert a.get_unit() is None + + def test_has_unit_true(self): + a = Answer(value="3", unit="m") + assert a.has_unit() is True + + def test_has_unit_false(self): + a = Answer(value="3") + assert a.has_unit() is False + + +class TestAnswerDunder: + def test_str_with_unit(self): + a = Answer(value="9.81", unit="m/s^2") + assert str(a) == "9.81 m/s^2" + + def test_str_without_unit(self): + a = Answer(value="42") + assert str(a) == "42" + + def test_repr_contains_key_fields(self): + a = Answer(value="5", unit="N", source_type="NV") + r = repr(a) + assert "Answer(" in r + assert "'5'" in r + assert "'N'" in r + assert "'NV'" in r + + def test_repr_no_answer_kind(self): + a = Answer(value="x") + assert "answer_kind" not in repr(a) + + +class TestAnswerToDict: + def test_value_only(self): + d = Answer(value="42").to_dict() + assert d == {"value": "42"} + + def test_with_unit(self): + d = Answer(value="3", unit="m").to_dict() + assert d == {"value": "3", "unit": "m"} + + def test_with_source_type(self): + d = Answer(value="A", source_type="MCQ").to_dict() + assert d == {"value": "A", "source_type": "MCQ"} + + def test_with_metadata(self): + d = Answer(value="x", metadata={"raw": "1"}).to_dict() + assert d["metadata"] == {"raw": "1"} + + def test_no_answer_kind_key(self): + d = Answer(value="x").to_dict() + assert "answer_kind" not in d + assert "answer_category" not in d + + def test_empty_unit_omitted(self): + d = Answer(value="x", unit=None).to_dict() + assert "unit" not in d + + def test_none_source_type_omitted(self): + d = Answer(value="x", source_type=None).to_dict() + assert "source_type" not in d + + def test_empty_metadata_omitted(self): + d = Answer(value="x", metadata={}).to_dict() + assert "metadata" not in d diff --git a/tests/prkit/core/domain/test_physics_problem.py b/tests/prkit/core/domain/test_physics_problem.py index e4cb507..7a0e829 100644 --- a/tests/prkit/core/domain/test_physics_problem.py +++ b/tests/prkit/core/domain/test_physics_problem.py @@ -7,7 +7,7 @@ import pytest -from prkit.core.domain import Answer, AnswerObjectKind, PhysicsDomain, PhysicsProblem +from prkit.core.domain import Answer, PhysicsDomain, PhysicsProblem from prkit.core.domain import physics_problem as physics_problem_module @@ -25,7 +25,7 @@ def test_problem_creation_minimal(self): def test_problem_creation_full(self): """Test creating a full physics problem.""" - answer = Answer(value=42, answer_kind=AnswerObjectKind.NUMBER) + answer = Answer(value="42") problem = PhysicsProblem( problem_id="test_001", question="What is the answer?", @@ -222,7 +222,7 @@ def test_problem_update(self): def test_problem_to_dict(self): """Test problem serialization.""" - answer = Answer(value=42, answer_kind=AnswerObjectKind.NUMBER) + answer = Answer(value="42") problem = PhysicsProblem( problem_id="test_001", question="Test", @@ -239,25 +239,28 @@ def test_problem_from_dict(self): data = { "problem_id": "test_001", "question": "Test question", - "answer": {"value": 42, "answer_kind": "number", "unit": "m/s"}, + "answer": {"value": "42", "unit": "m/s"}, "domain": "classical_mechanics", } problem = PhysicsProblem.from_dict(data) assert problem.problem_id == "test_001" assert problem.question == "Test question" - assert problem.answer.value == 42 - assert problem.answer.answer_kind == AnswerObjectKind.NUMBER + assert problem.answer.value == "42" + assert problem.answer.unit == "m/s" - fallback = PhysicsProblem.from_dict( + # Legacy migration: answer_kind in serialized dict → preserved in source_type + legacy = PhysicsProblem.from_dict( { "problem_id": "test_002", "question": "Q", - "answer": {"value": "hello", "answer_kind": "not-real"}, + "answer": {"value": "hello", "answer_kind": "descriptive_text"}, "custom_field": "custom", } ) - assert fallback.answer.answer_kind == AnswerObjectKind.DESCRIPTIVE_TEXT - assert fallback.additional_fields["custom_field"] == "custom" + assert legacy.answer.value == "hello" + assert legacy.answer.source_type == "descriptive_text" + assert not hasattr(legacy.answer, "answer_kind") + assert legacy.additional_fields["custom_field"] == "custom" def test_problem_copy(self): """Test problem copying.""" diff --git a/tests/prkit/datasets/loaders/test_base_loader_additional.py b/tests/prkit/datasets/loaders/test_base_loader_additional.py index 32fca43..5489f28 100644 --- a/tests/prkit/datasets/loaders/test_base_loader_additional.py +++ b/tests/prkit/datasets/loaders/test_base_loader_additional.py @@ -1,7 +1,6 @@ -from prkit.core.domain import AnswerObjectKind, PhysicalDataset, PhysicsDomain +from prkit.core.domain import PhysicalDataset, PhysicsDomain from prkit.datasets.loaders.base_loader import ( BaseDatasetLoader, - detect_answer_category, is_mathematical_expression, is_pure_number, ) @@ -32,12 +31,6 @@ def test_base_loader_numeric_and_math_detection(): assert is_pure_number("3/4") is True assert is_pure_number("not-a-number") is False assert is_mathematical_expression("x + y") is True - assert detect_answer_category("9.8") == AnswerObjectKind.NUMBER - assert detect_answer_category("F = ma") == AnswerObjectKind.EXPRESSION - assert ( - detect_answer_category("descriptive answer") - == AnswerObjectKind.DESCRIPTIVE_TEXT - ) def test_base_loader_defaults_validation_and_metadata(tmp_path, monkeypatch): @@ -76,7 +69,6 @@ def test_base_loader_creates_problem_and_loads_images(tmp_path): "problem_id": "p1", "question": "What is 2 + 2?", "answer": "4", - "answer_category": "number", "domain": PhysicsDomain.CLASSICAL_MECHANICS, "image_paths": [image_file.name], "extra": "meta", @@ -85,7 +77,9 @@ def test_base_loader_creates_problem_and_loads_images(tmp_path): ) assert problem.problem_id == "p1" - assert problem.answer.answer_kind == AnswerObjectKind.NUMBER + assert isinstance(problem.answer.value, str) + assert problem.answer.value == "4" + assert problem.answer.source_type is None assert problem.image_path == [str(image_file.resolve())] assert problem.additional_fields["extra"] == "meta" assert loader._determine_problem_type({"options": ["A", "B"]}) == "MC" @@ -102,11 +96,40 @@ def test_base_loader_handles_invalid_image_inputs_and_answer_types(): assert loader.load_images_from_paths(None) == [] assert loader.load_images_from_paths(123) == [] - number_answer = loader._create_answer_from_raw( - {"answer": {"value": "5", "unit": "m"}, "answer_category": "physical_quantity"} + # dict answer with unit → unit is preserved in Answer + unit_answer = loader._create_answer_from_raw( + {"answer": {"value": "5", "unit": "m"}} ) - assert number_answer.answer_kind == AnswerObjectKind.PHYSICAL_QUANTITY - assert number_answer.unit == "m" + assert unit_answer is not None + assert unit_answer.unit == "m" + assert unit_answer.value == "5" + + # plain expression answer → no kind needed; engine derives + plain_answer = loader._create_answer_from_raw({"answer": "F = ma"}) + assert plain_answer is not None + assert plain_answer.value == "F = ma" + assert plain_answer.source_type is None + + +def test_base_loader_source_type_from_metadata(): + loader = DummyLoader() + + answer = loader._create_answer_from_raw({"answer": "42", "source_type": "Integer"}) + assert answer is not None + assert answer.source_type == "Integer" + + +def test_base_loader_strips_latex_wrappers(): + loader = DummyLoader() + + boxed = loader._create_answer_from_raw({"answer": "\\boxed{9.81}"}) + assert boxed is not None + assert boxed.value == "9.81" + + dollar = loader._create_answer_from_raw({"answer": "$x^2$"}) + assert dollar is not None + assert dollar.value == "x^2" - fallback_answer = loader._create_answer_from_raw({"answer": "F = ma"}) - assert fallback_answer.answer_kind == AnswerObjectKind.EXPRESSION + ddollar = loader._create_answer_from_raw({"answer": "$$42$$"}) + assert ddollar is not None + assert ddollar.value == "42" diff --git a/tests/prkit/datasets/loaders/test_jeebench_loader.py b/tests/prkit/datasets/loaders/test_jeebench_loader.py index 0032446..0340405 100644 --- a/tests/prkit/datasets/loaders/test_jeebench_loader.py +++ b/tests/prkit/datasets/loaders/test_jeebench_loader.py @@ -63,11 +63,11 @@ def test_process_metadata_preserves_subject_and_numeric_type(self): ) assert mcq["problem_type"] == "MC" - assert mcq["answer_category"] == "option" + assert mcq["source_type"] == "MCQ" assert mcq["subject"] == "phy" assert mcq["type"] == "MCQ" assert numeric["problem_type"] == "OE" - assert numeric["answer_category"] == "number" + assert numeric["source_type"] == "Numeric" assert numeric["subject"] == "phy" assert numeric["type"] == "Numeric" diff --git a/tests/prkit/datasets/loaders/test_phybench_loader.py b/tests/prkit/datasets/loaders/test_phybench_loader.py index 46331eb..405ca64 100644 --- a/tests/prkit/datasets/loaders/test_phybench_loader.py +++ b/tests/prkit/datasets/loaders/test_phybench_loader.py @@ -205,5 +205,6 @@ def test_process_metadata(self): processed = loader._process_metadata( metadata ) # pylint: disable=protected-access - assert processed["answer_category"] == "formula" + assert processed.get("answer_category") is None + assert processed.get("source_type") is None assert "domain" in processed diff --git a/tests/prkit/datasets/loaders/test_physbench_loader.py b/tests/prkit/datasets/loaders/test_physbench_loader.py index 38cd05e..6fded61 100644 --- a/tests/prkit/datasets/loaders/test_physbench_loader.py +++ b/tests/prkit/datasets/loaders/test_physbench_loader.py @@ -123,7 +123,8 @@ def test_load_success(self, temp_dir): assert first_problem.problem_type == "MC" assert first_problem.correct_option == 2 assert first_problem.answer is not None - assert first_problem.answer.is_option() + assert first_problem.answer.value == "C" + assert first_problem.answer.source_type is None assert len(first_problem.options) == 4 assert len(first_problem.image_path) == 4 assert Path(first_problem.image_path[0]).exists() diff --git a/tests/prkit/datasets/loaders/test_tpbench_loader.py b/tests/prkit/datasets/loaders/test_tpbench_loader.py index 2e1dff0..3f5d3a7 100644 --- a/tests/prkit/datasets/loaders/test_tpbench_loader.py +++ b/tests/prkit/datasets/loaders/test_tpbench_loader.py @@ -210,7 +210,8 @@ def test_process_metadata(self): processed = loader._process_metadata( metadata ) # pylint: disable=protected-access - assert processed["answer_category"] == "formula" + assert processed.get("answer_category") is None + assert processed.get("source_type") is None assert "domain" in processed def test_load_empty_json_file(self, temp_dir): diff --git a/tests/prkit/datasets/loaders/test_ugphysics_loader.py b/tests/prkit/datasets/loaders/test_ugphysics_loader.py index 42b528d..5fab81a 100644 --- a/tests/prkit/datasets/loaders/test_ugphysics_loader.py +++ b/tests/prkit/datasets/loaders/test_ugphysics_loader.py @@ -8,7 +8,6 @@ import pytest -from prkit.core.domain.answer_kinds import AnswerObjectKind from prkit.datasets.loaders import UGPhysicsLoader @@ -249,7 +248,7 @@ def test_load_maps_mc_answers_to_option_category(self, temp_dir): problem = dataset[0] assert problem.problem_type == "MC" - assert problem.answer.answer_kind == AnswerObjectKind.CHOICE + assert problem.answer.source_type == "MC" assert problem.answer.value == "B" def test_load_preserves_multi_answer_metadata(self, temp_dir): @@ -281,7 +280,8 @@ def test_load_preserves_multi_answer_metadata(self, temp_dir): ) problem = dataset[0] - assert problem.answer.answer_kind == AnswerObjectKind.DESCRIPTIVE_TEXT + assert isinstance(problem.answer.value, str) + assert problem.answer.source_type == "NV" assert problem.additional_fields["answer_parts"] == [ {"value": "2", "unit": None}, {"value": "3", "unit": None}, diff --git a/tests/prkit/datasets/test_utils.py b/tests/prkit/datasets/test_utils.py index c3353bd..078910c 100644 --- a/tests/prkit/datasets/test_utils.py +++ b/tests/prkit/datasets/test_utils.py @@ -2,95 +2,39 @@ Tests for utility functions and helper modules. """ -from prkit.core.domain import AnswerObjectKind from prkit.datasets.loaders.base_loader import ( - detect_answer_category, is_mathematical_expression, is_pure_number, ) -class TestAnswerKindDetection: - """Test cases for answer-kind detection utilities.""" - - def test_detect_answer_category_numerical(self): - """Test detecting number answer category.""" - assert detect_answer_category("42") == AnswerObjectKind.NUMBER - assert detect_answer_category("3.14") == AnswerObjectKind.NUMBER - assert detect_answer_category("1e-5") == AnswerObjectKind.NUMBER - assert detect_answer_category("1.23e+10") == AnswerObjectKind.NUMBER - - def test_detect_answer_category_fraction(self): - """Test detecting fractions as number.""" - assert detect_answer_category("3/4") == AnswerObjectKind.NUMBER - assert detect_answer_category("1/2") == AnswerObjectKind.NUMBER - - def test_detect_answer_category_formula(self): - """Test detecting formula/symbolic answer category.""" - assert detect_answer_category("x^2 + 1") == AnswerObjectKind.EXPRESSION - assert detect_answer_category("\\frac{a}{b}") == AnswerObjectKind.EXPRESSION - assert detect_answer_category("$x^2$") == AnswerObjectKind.EXPRESSION - assert detect_answer_category("\\boxed{x^2}") == AnswerObjectKind.EXPRESSION - - def test_detect_answer_category_text(self): - """Test detecting text answer category.""" - assert ( - detect_answer_category("This is a descriptive answer") - == AnswerObjectKind.DESCRIPTIVE_TEXT - ) - assert ( - detect_answer_category("The solution involves multiple steps") - == AnswerObjectKind.DESCRIPTIVE_TEXT - ) - assert ( - detect_answer_category("Explanation of the physics concept") - == AnswerObjectKind.DESCRIPTIVE_TEXT - ) - - def test_detect_answer_category_with_boxed(self): - """Test detecting answer category with \\boxed{} wrapper.""" - assert detect_answer_category("\\boxed{42}") == AnswerObjectKind.NUMBER - assert detect_answer_category("\\boxed{x^2}") == AnswerObjectKind.EXPRESSION - - def test_detect_answer_category_with_dollar_signs(self): - """Test detecting answer category with $ delimiters.""" - assert detect_answer_category("$42$") == AnswerObjectKind.NUMBER - assert detect_answer_category("$$x^2$$") == AnswerObjectKind.EXPRESSION - - class TestIsPureNumber: """Test cases for is_pure_number utility.""" def test_is_pure_number_integers(self): - """Test integer detection.""" assert is_pure_number("42") is True assert is_pure_number("0") is True assert is_pure_number("-5") is True def test_is_pure_number_decimals(self): - """Test decimal detection.""" assert is_pure_number("3.14") is True assert is_pure_number("0.5") is True assert is_pure_number("-2.5") is True def test_is_pure_number_scientific_notation(self): - """Test scientific notation detection.""" assert is_pure_number("1e5") is True assert is_pure_number("1.23e-4") is True assert is_pure_number("2.5E+6") is True def test_is_pure_number_with_commas(self): - """Test numbers with comma separators.""" assert is_pure_number("1,000") is True assert is_pure_number("1,234.56") is True def test_is_pure_number_fractions(self): - """Test fraction detection.""" assert is_pure_number("3/4") is True assert is_pure_number("1/2") is True def test_is_pure_number_not_numbers(self): - """Test non-number strings.""" assert is_pure_number("x") is False assert is_pure_number("x^2") is False assert is_pure_number("text") is False @@ -101,37 +45,31 @@ class TestIsMathematicalExpression: """Test cases for is_mathematical_expression utility.""" def test_is_mathematical_expression_with_operators(self): - """Test expressions with operators.""" assert is_mathematical_expression("x + y") is True assert is_mathematical_expression("a * b") is True assert is_mathematical_expression("x^2") is True def test_is_mathematical_expression_with_functions(self): - """Test expressions with functions.""" assert is_mathematical_expression("sin(x)") is True assert is_mathematical_expression("log(x)") is True assert is_mathematical_expression("sqrt(x)") is True def test_is_mathematical_expression_latex(self): - """Test LaTeX expressions.""" assert is_mathematical_expression("\\frac{a}{b}") is True assert is_mathematical_expression("$x^2$") is True assert is_mathematical_expression("\\sqrt{x}") is True def test_is_mathematical_expression_with_symbols(self): - """Test expressions with mathematical symbols.""" assert is_mathematical_expression("π") is True assert is_mathematical_expression("∞") is True assert is_mathematical_expression("≤") is True def test_is_mathematical_expression_not_expressions(self): - """Test non-mathematical strings.""" assert is_mathematical_expression("42") is False # Pure number assert is_mathematical_expression("text") is False assert is_mathematical_expression("") is False def test_is_mathematical_expression_with_variables(self): - """Test expressions with variables.""" assert is_mathematical_expression("x") is True assert is_mathematical_expression("a_1") is True assert is_mathematical_expression("x_i") is True diff --git a/tests/prkit/datasets/test_utils_functions.py b/tests/prkit/datasets/test_utils_functions.py index 4ff23af..e04dca9 100644 --- a/tests/prkit/datasets/test_utils_functions.py +++ b/tests/prkit/datasets/test_utils_functions.py @@ -4,7 +4,7 @@ import json -from prkit.core.domain import Answer, AnswerObjectKind, PhysicalDataset, PhysicsProblem +from prkit.core.domain import Answer, PhysicalDataset, PhysicsProblem from prkit.datasets import utils @@ -129,9 +129,7 @@ def test_filter_by_keywords_in_question(self, sample_problems_list): problem_with_keyword = PhysicsProblem( problem_id="keyword_test", question="What is the speed of light?", - answer=Answer( - value=3e8, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, unit="m/s" - ), + answer=Answer(value="3e8", unit="m/s"), ) all_problems = list(sample_problems_list) + [problem_with_keyword] dataset = PhysicalDataset(problems=all_problems) @@ -146,9 +144,7 @@ def test_filter_by_keywords_case_insensitive(self, sample_problems_list): problem = PhysicsProblem( problem_id="test_case", question="What is the SPEED of light?", - answer=Answer( - value=3e8, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, unit="m/s" - ), + answer=Answer(value="3e8", unit="m/s"), ) all_problems = list(sample_problems_list) + [problem] dataset = PhysicalDataset(problems=all_problems) @@ -164,9 +160,7 @@ def test_filter_by_keywords_case_sensitive(self, sample_problems_list): problem = PhysicsProblem( problem_id="test_case", question="What is the speed of light?", - answer=Answer( - value=3e8, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, unit="m/s" - ), + answer=Answer(value="3e8", unit="m/s"), ) all_problems = list(sample_problems_list) + [problem] dataset = PhysicalDataset(problems=all_problems) @@ -184,7 +178,7 @@ def test_filter_by_keywords_multiple_fields(self, sample_problems_list): problem_id="test_multi", question="Test question", solution="The answer involves force calculation", - answer=Answer(value=1, answer_kind=AnswerObjectKind.NUMBER), + answer=Answer(value="1"), ) all_problems = list(sample_problems_list) + [problem] dataset = PhysicalDataset(problems=all_problems) diff --git a/tests/prkit/semantics/edit_distance/test_edit_distance_score.py b/tests/prkit/evaluation/edit_distance/test_edit_distance_score.py similarity index 96% rename from tests/prkit/semantics/edit_distance/test_edit_distance_score.py rename to tests/prkit/evaluation/edit_distance/test_edit_distance_score.py index 9b12beb..e66c890 100644 --- a/tests/prkit/semantics/edit_distance/test_edit_distance_score.py +++ b/tests/prkit/evaluation/edit_distance/test_edit_distance_score.py @@ -2,7 +2,7 @@ from __future__ import annotations -from prkit.semantics.edit_distance.score import ( +from prkit.evaluation.edit_distance.score import ( EditCosts, delete_cost, eed_score, @@ -10,7 +10,7 @@ subtree_discount, update_cost, ) -from prkit.semantics.edit_distance.tree import ExprNode +from prkit.evaluation.edit_distance.tree import ExprNode class TestEditCosts: diff --git a/tests/prkit/semantics/edit_distance/test_edit_distance_tree.py b/tests/prkit/evaluation/edit_distance/test_edit_distance_tree.py similarity index 98% rename from tests/prkit/semantics/edit_distance/test_edit_distance_tree.py rename to tests/prkit/evaluation/edit_distance/test_edit_distance_tree.py index 27eca8a..39460b4 100644 --- a/tests/prkit/semantics/edit_distance/test_edit_distance_tree.py +++ b/tests/prkit/evaluation/edit_distance/test_edit_distance_tree.py @@ -5,7 +5,7 @@ import pytest import sympy as sp -from prkit.semantics.edit_distance.tree import ( +from prkit.evaluation.edit_distance.tree import ( ExprNode, UnsupportedExpressionError, sympy_to_tree, diff --git a/tests/prkit/semantics/edit_distance/test_edit_distance_zss.py b/tests/prkit/evaluation/edit_distance/test_edit_distance_zss.py similarity index 94% rename from tests/prkit/semantics/edit_distance/test_edit_distance_zss.py rename to tests/prkit/evaluation/edit_distance/test_edit_distance_zss.py index 9c9deee..3c43016 100644 --- a/tests/prkit/semantics/edit_distance/test_edit_distance_zss.py +++ b/tests/prkit/evaluation/edit_distance/test_edit_distance_zss.py @@ -4,9 +4,9 @@ import sympy as sp -from prkit.semantics.edit_distance.score import EditCosts, eed_score -from prkit.semantics.edit_distance.tree import ExprNode, sympy_to_tree -from prkit.semantics.edit_distance.zss import tree_edit_distance +from prkit.evaluation.edit_distance.score import EditCosts, eed_score +from prkit.evaluation.edit_distance.tree import ExprNode, sympy_to_tree +from prkit.evaluation.edit_distance.zss import tree_edit_distance _COSTS = EditCosts() diff --git a/tests/prkit/evaluation/llm_judge/test_payload.py b/tests/prkit/evaluation/llm_judge/test_payload.py index a109726..ab97b58 100644 --- a/tests/prkit/evaluation/llm_judge/test_payload.py +++ b/tests/prkit/evaluation/llm_judge/test_payload.py @@ -1,5 +1,4 @@ from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_kinds import AnswerObjectKind from prkit.evaluation.llm_judge.payload import ( answer_to_text_and_category, build_standard_answer_judge_payload, @@ -9,24 +8,31 @@ def test_answer_to_text_and_category_for_answers_and_plain_strings(): - answer = Answer(value=" 42 ", answer_kind=AnswerObjectKind.NUMBER) - assert answer_to_text_and_category(answer) == ("42", "number") - assert answer_to_text_and_category(" free text ") == ("free text", "unknown") + # Answer with source_type → category is the source_type string + answer = Answer(value=" 42 ", source_type="NV") + assert answer_to_text_and_category(answer) == ("42", "NV") + + # Answer without source_type → empty string + answer_no_type = Answer(value=" 42 ") + assert answer_to_text_and_category(answer_no_type) == ("42", "") + + # Plain string → empty string category + assert answer_to_text_and_category(" free text ") == ("free text", "") def test_build_standard_answer_judge_payload_cleans_fields(): payload = build_standard_answer_judge_payload( - Answer(value=" 10\u00a0 m/s ", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), - Answer(value=" 10\tm/s ", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), + Answer(value=" 10  m/s "), + Answer(value=" 10\tm/s "), " What is the speed? ", ) assert payload == { "question": "What is the speed?", - "ground_truth": {"text": "10 m/s", "category": "physical_quantity"}, - "model_answer": {"text": "10 m/s", "category": "physical_quantity"}, + "ground_truth": {"text": "10 m/s", "category": ""}, + "model_answer": {"text": "10 m/s", "category": ""}, } - assert clean_answer_text(" a\u00a0 \t b ") == "a b" + assert clean_answer_text(" a  \t b ") == "a b" def test_truncate_judge_payload_trims_long_fields_and_preserves_shape(): diff --git a/tests/prkit/scoring/test_semantics_scorer.py b/tests/prkit/scoring/test_semantics_scorer.py index 94b7ead..6395aec 100644 --- a/tests/prkit/scoring/test_semantics_scorer.py +++ b/tests/prkit/scoring/test_semantics_scorer.py @@ -8,7 +8,6 @@ from prkit.api import Scorer, Verdict from prkit.core.domain.answer import Answer -from prkit.core.domain.answer_kinds import AnswerObjectKind from prkit.scoring import SemanticsScorer # Empirically validated against the deterministic engine (see plan step 4). @@ -54,12 +53,8 @@ def test_identity_equivalent(self, value): assert SemanticsScorer().score(value, value).equivalent is True def test_accepts_answer_objects(self): - pred = Answer( - value=3.0, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, unit="m/s" - ) - ref = Answer( - value=3, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, unit="m/s" - ) + pred = Answer(value="3.0", unit="m/s") + ref = Answer(value="3", unit="m/s") v = SemanticsScorer().score(pred, ref) assert v.equivalent is True diff --git a/tests/prkit/semantics/edit_distance/test_edit_distance_robustness.py b/tests/prkit/semantics/edit_distance/test_edit_distance_robustness.py index 3303486..850f737 100644 --- a/tests/prkit/semantics/edit_distance/test_edit_distance_robustness.py +++ b/tests/prkit/semantics/edit_distance/test_edit_distance_robustness.py @@ -7,9 +7,10 @@ import pytest import sympy as sp +from prkit.evaluation.edit_distance import sympy_to_tree +from prkit.evaluation.edit_distance.timeout import SimplifyTimeout, run_with_timeout from prkit.semantics import normalize_physics_answer -from prkit.semantics.edit_distance import EedConfig, eed_compare, sympy_to_tree -from prkit.semantics.edit_distance.timeout import SimplifyTimeout, run_with_timeout +from prkit.semantics.edit_distance import EedConfig, eed_compare class TestTimeout: diff --git a/tests/prkit/semantics/test_inference_prompts.py b/tests/prkit/semantics/test_inference_prompts.py index ecba4e2..c132b8e 100644 --- a/tests/prkit/semantics/test_inference_prompts.py +++ b/tests/prkit/semantics/test_inference_prompts.py @@ -6,7 +6,7 @@ import pytest from pydantic import ValidationError -from prkit.core.domain import Answer, AnswerObjectKind, PhysicsProblem +from prkit.core.domain import Answer, PhysicsProblem from prkit.core.model_clients import BaseModelClient from prkit.core.model_clients.structured_output import StructuredOutputPlan from prkit.semantics.build.calls import ( @@ -68,11 +68,7 @@ def _build_problem() -> PhysicsProblem: problem = PhysicsProblem( problem_id="prob-1", question="What is the force?", - answer=Answer( - value="5", - unit="N", - answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, - ), + answer=Answer(value="5", unit="N"), solution="Use Newton's second law.", domain="mechanics", image_path=["/tmp/img1.png", "/tmp/img2.png"], @@ -115,7 +111,7 @@ def test_build_prediction_semantics_prompt_uses_answer_blind_question_draft() -> problem = PhysicsProblem( problem_id="prob-gold-split", question="Give both values: the displacement value and the time value.", - answer=Answer(value="F = ma", answer_kind=AnswerObjectKind.RELATION), + answer=Answer(value="F = ma"), additional_fields={ "answer_parts": [ {"part_label": "speed_slot", "raw_text": "1 m"}, diff --git a/tests/prkit/semantics/test_outcome_space.py b/tests/prkit/semantics/test_outcome_space.py index ddeb397..0e7cf6c 100644 --- a/tests/prkit/semantics/test_outcome_space.py +++ b/tests/prkit/semantics/test_outcome_space.py @@ -280,7 +280,7 @@ def test_fixed_question_unit_allows_bare_number_but_required_unit_does_not() -> PhysicsProblem( problem_id="p1", question="Find the speed in m/s.", - answer=Answer(value="5", answer_kind=AnswerObjectKind.NUMBER), + answer=Answer(value="5"), ) ) required_unit_context = QuestionContext( @@ -330,7 +330,6 @@ def test_infer_question_context_rejects_prose_after_in_keyword( question=question, answer=Answer( value=answer_value, - answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, ), ) ) @@ -346,23 +345,23 @@ def test_infer_question_context_drops_stopword_targets_but_keeps_symbol_targets( PhysicsProblem( problem_id="p_stopword_the", question="What is the magnitude of the force on the block?", - answer=Answer(value="25 N", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), + answer=Answer(value="25 N"), ), PhysicsProblem( problem_id="p_stopword_all", question="What is all?", - answer=Answer(value="25 N", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), + answer=Answer(value="25 N"), ), PhysicsProblem( problem_id="p_stopword_which", question="What is which?", - answer=Answer(value="25 N", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), + answer=Answer(value="25 N"), ), ] symbol_problem = PhysicsProblem( problem_id="p_symbol_target", question="What is T?", - answer=Answer(value="0.78 s", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), + answer=Answer(value="0.78 s"), ) for problem in prose_problems: @@ -376,7 +375,6 @@ def test_question_semantics_split_uses_gold_target_only_for_reference() -> None: question="Give the final expression for the magnetic field.", answer=Answer( value="B = \\mu_0 I / (2\\pi r)", - answer_kind=AnswerObjectKind.RELATION, ), ) @@ -396,7 +394,6 @@ def test_question_semantics_split_uses_gold_unit_policy_only_for_reference() -> answer=Answer( value="5", unit="m/s", - answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY, ), ) @@ -412,7 +409,7 @@ def test_prediction_question_semantics_ignores_answer_parts_metadata() -> None: problem = PhysicsProblem( problem_id="p_answer_parts_split", question="Give both values: the displacement value and the time value.", - answer=Answer(value="ignored", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT), + answer=Answer(value="ignored"), additional_fields={ "answer_parts": [ {"part_label": "speed_slot", "raw_text": "1 m"}, @@ -487,7 +484,7 @@ def test_prediction_question_semantics_ignores_symbol_alias_metadata() -> None: problem = PhysicsProblem( problem_id="p_symbol_alias_split", question="Give the final expression for the displacement.", - answer=Answer(value="x_final = v*t", answer_kind=AnswerObjectKind.RELATION), + answer=Answer(value="x_final = v*t"), additional_fields={ "symbol_aliases": [ { @@ -511,7 +508,7 @@ def test_problem_answer_parts_take_precedence() -> None: problem = PhysicsProblem( problem_id="p2", question="Give both values.", - answer=Answer(value="ignored", answer_kind=AnswerObjectKind.DESCRIPTIVE_TEXT), + answer=Answer(value="ignored"), additional_fields={"answer_parts": ["1 m", "2 m"]}, ) @@ -528,7 +525,7 @@ def test_dataset_backed_relation_and_multi_part_strings() -> None: PhysicsProblem( problem_id="ugphysics-628", question="Give the effect type and the field strength.", - answer=Answer(value="C, 7.77", answer_kind=AnswerObjectKind.CHOICE), + answer=Answer(value="C, 7.77"), additional_fields={"answer_parts": ["C", "7.77"]}, ) ) diff --git a/tests/prkit/semantics/test_prediction_isolated_build.py b/tests/prkit/semantics/test_prediction_isolated_build.py index 8c15b3f..6129d59 100644 --- a/tests/prkit/semantics/test_prediction_isolated_build.py +++ b/tests/prkit/semantics/test_prediction_isolated_build.py @@ -39,9 +39,7 @@ def _problem() -> PhysicsProblem: return PhysicsProblem( problem_id="pred-iso-1", question="Find the speed v.", - answer=Answer( - value="sqrt(E/m), m > 0", answer_kind=AnswerObjectKind.EXPRESSION - ), + answer=Answer(value="sqrt(E/m), m > 0"), solution="Use conservation of energy.", domain="mechanics", additional_fields={ diff --git a/tests/prkit/semantics/test_sign_convention_build_integration.py b/tests/prkit/semantics/test_sign_convention_build_integration.py index 07197e4..4de3f93 100644 --- a/tests/prkit/semantics/test_sign_convention_build_integration.py +++ b/tests/prkit/semantics/test_sign_convention_build_integration.py @@ -12,7 +12,7 @@ import json from typing import Any -from prkit.core.domain import Answer, AnswerObjectKind, PhysicsProblem +from prkit.core.domain import Answer, PhysicsProblem from prkit.core.model_clients import BaseModelClient from prkit.semantics.build.calls import ( build_reference_semantics, @@ -26,7 +26,7 @@ def _quantity_problem(golden: str) -> PhysicsProblem: return PhysicsProblem( problem_id="signconv-int", question="Find the block's velocity v.", - answer=Answer(value=golden, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), + answer=Answer(value=golden), domain="mechanics", ) @@ -35,7 +35,7 @@ def _vector_problem(golden: str) -> PhysicsProblem: return PhysicsProblem( problem_id="signconv-int-vec", question="Find the displacement vector.", - answer=Answer(value=golden, answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), + answer=Answer(value=golden), domain="mechanics", ) diff --git a/tests/prkit/semantics/test_sign_convention_build_live.py b/tests/prkit/semantics/test_sign_convention_build_live.py index 62f51da..e298d01 100644 --- a/tests/prkit/semantics/test_sign_convention_build_live.py +++ b/tests/prkit/semantics/test_sign_convention_build_live.py @@ -13,7 +13,7 @@ import pytest -from prkit.core.domain import Answer, AnswerObjectKind, PhysicsProblem +from prkit.core.domain import Answer, PhysicsProblem from prkit.core.model_clients import create_model_client from prkit.semantics.build.calls import build_reference_semantics @@ -33,7 +33,7 @@ def test_live_reference_build_routes_free_axis_convention_to_a_ref() -> None: "is specified. Taking the block's motion into account, its velocity is found to " "be 20 m/s directed to the left. Report the velocity as a signed value." ), - answer=Answer(value="-20 m/s", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), + answer=Answer(value="-20 m/s"), domain="mechanics", ) diff --git a/tests/prkit/semantics/test_staged_build.py b/tests/prkit/semantics/test_staged_build.py index 8289258..ad80626 100644 --- a/tests/prkit/semantics/test_staged_build.py +++ b/tests/prkit/semantics/test_staged_build.py @@ -29,7 +29,7 @@ def _problem() -> PhysicsProblem: return PhysicsProblem( problem_id="staged-1", question="Find the energy E.", - answer=Answer(value="x**2/2, x > 0", answer_kind=AnswerObjectKind.EXPRESSION), + answer=Answer(value="x**2/2, x > 0"), domain="mechanics", ) @@ -38,7 +38,7 @@ def _directional_problem() -> PhysicsProblem: return PhysicsProblem( problem_id="staged-dir-1", question="Find the velocity v of the block.", - answer=Answer(value="-20 m/s", answer_kind=AnswerObjectKind.PHYSICAL_QUANTITY), + answer=Answer(value="-20 m/s"), domain="mechanics", ) diff --git a/tests/prkit/test_api.py b/tests/prkit/test_api.py index 9576dbf..b2d0cba 100644 --- a/tests/prkit/test_api.py +++ b/tests/prkit/test_api.py @@ -11,11 +11,17 @@ Verdict, create_model_client, ) +from prkit.core.domain.answer import Answer +from prkit.core.domain.physics_problem import PhysicsProblem from prkit.core.verdict import Verdict as CoreVerdict from prkit.datasets.hub import DatasetHub +from prkit.scoring import SemanticsScorer class TestContractSurface: + def test_api_version_is_provisional_1_0(self): + assert api.API_VERSION == "1.0" + def test_api_version_present(self): assert isinstance(api.API_VERSION, str) and api.API_VERSION @@ -44,6 +50,62 @@ def test_verdict_reexport_identity(self): assert Verdict is CoreVerdict +class TestLegacyRoundTrip: + """Legacy-serialized answer dicts (answer_kind / answer_category) survive reshape.""" + + def test_answer_kind_migrated_to_source_type(self): + data = { + "problem_id": "legacy_001", + "question": "Q", + "answer": {"value": "9.81", "answer_kind": "number", "unit": "m/s^2"}, + } + problem = PhysicsProblem.from_dict(data) + assert problem.answer is not None + assert problem.answer.value == "9.81" + assert problem.answer.unit == "m/s^2" + assert problem.answer.source_type == "number" + assert not hasattr(problem.answer, "answer_kind") + + def test_answer_category_migrated_to_source_type(self): + data = { + "problem_id": "legacy_002", + "question": "Q", + "answer": {"value": "F = ma", "answer_category": "expression"}, + } + problem = PhysicsProblem.from_dict(data) + assert problem.answer.source_type == "expression" + assert not hasattr(problem.answer, "answer_kind") + + def test_unit_preserved_alongside_legacy_label(self): + data = { + "problem_id": "legacy_003", + "question": "Q", + "answer": {"value": "5", "unit": "N", "answer_kind": "physical_quantity"}, + } + problem = PhysicsProblem.from_dict(data) + assert problem.answer.unit == "N" + assert problem.answer.source_type == "physical_quantity" + + def test_thin_answer_has_no_answer_kind_attribute(self): + a = Answer(value="x") + assert not hasattr(a, "answer_kind") + + +class TestUnitEquivalenceNoRegression: + """Unit-bearing answers still score correctly via the equivalence engine.""" + + def test_unit_aware_numeric_equivalence(self): + scorer = SemanticsScorer() + verdict = scorer.score("9.81 m/s^2", "9.8 m/s^2") + assert isinstance(verdict, Verdict) + assert verdict.equivalent is True + + def test_unit_aware_numeric_inequivalence(self): + scorer = SemanticsScorer() + verdict = scorer.score("3 m/s", "5 m/s") + assert verdict.equivalent is False + + class TestRuntimeCheckableProtocols: def test_registered_loaders_satisfy_dataset_provider(self): for name in DatasetHub.list_available(): diff --git a/tests/prkit/verify/test_import_isolation.py b/tests/prkit/verify/test_import_isolation.py index 3ab2425..3de2476 100644 --- a/tests/prkit/verify/test_import_isolation.py +++ b/tests/prkit/verify/test_import_isolation.py @@ -22,7 +22,11 @@ "datasets", "pandas", "prkit.datasets", - "prkit.evaluation", + # The model-graded judge (pulls ``openai``) must stay off the verify path. We + # forbid the ``llm_judge`` subpackage specifically — ``prkit.evaluation`` itself is + # now legitimately reachable, since the pure EED/SEED algorithm core lives in + # ``prkit.evaluation.edit_distance`` and the partial-credit scorer imports it. + "prkit.evaluation.llm_judge", ] From 2f366a538ea79e4279093f279a702856fc3e5f3c Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sun, 21 Jun 2026 15:53:00 -0400 Subject: [PATCH 19/28] Add the model-graded LLMJudgeScorer wrapping the LLM-judge engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce prkit.scoring.LLMJudgeScorer, the one non-deterministic reference scorer. It adapts the OpenAIJudgeRunner (which stays in evaluation/llm_judge) into the canonical Verdict: a "correct" judgement maps to score 1.0 / equivalent True, the model's reasoning lands in rationale, and confidence / expected_answer_type / raw_response / verdict_type / model are recorded in details. partial_credit stays None (the judge is binary) and deterministic False is reported via get_info(). All judge imports are deferred to method bodies (with TYPE_CHECKING-only annotations) so that re-exporting the scorer keeps import prkit.scoring free of openai and prkit.evaluation.llm_judge — verified by the import-isolation test. Reframe the evaluation package docstring as the engine home the thin scoring wrappers adapt. Tests inject a FakeJudgeRunner (no network/client). Co-Authored-By: Claude Opus 4.8 --- src/prkit/evaluation/__init__.py | 18 ++- src/prkit/scoring/__init__.py | 17 +- src/prkit/scoring/llm_judge_scorer.py | 124 +++++++++++++++ tests/prkit/scoring/test_llm_judge_scorer.py | 158 +++++++++++++++++++ 4 files changed, 309 insertions(+), 8 deletions(-) create mode 100644 src/prkit/scoring/llm_judge_scorer.py create mode 100644 tests/prkit/scoring/test_llm_judge_scorer.py diff --git a/src/prkit/evaluation/__init__.py b/src/prkit/evaluation/__init__.py index a80e76f..5b3637b 100644 --- a/src/prkit/evaluation/__init__.py +++ b/src/prkit/evaluation/__init__.py @@ -1,9 +1,17 @@ -"""Evaluation utilities for physical reasoning tasks. +"""Scoring *engines* that back PRKit's scorers — the related-work reference home. -The deprecated comparator/evaluator stacks were **removed** in ``API_VERSION`` 2.0; -use :class:`prkit.scoring.SemanticsScorer` (the ``Scorer`` / ``Verdict`` contract) for -deterministic scoring. The model-graded :mod:`prkit.evaluation.llm_judge` remains a -distinct, supported capability. +The thin :mod:`prkit.scoring` wrappers adapt the heavier engines that live here +into the ``Scorer`` / ``Verdict`` contract: + +* :mod:`prkit.evaluation.llm_judge` — the model-graded OpenAI physics judge, + wrapped by :class:`prkit.scoring.LLMJudgeScorer`. +* :mod:`prkit.evaluation.edit_distance` — the pure (front-end-free) tree-edit + core used by the EED/SEED edit-distance scorers. + +This package is deliberately import-light: importing it pulls no provider SDK or +heavy dependency. The legacy comparator/evaluator stacks were removed while +shaping the provisional contract; use :class:`prkit.scoring.SemanticsScorer` for +deterministic scoring. """ __all__: list[str] = [] diff --git a/src/prkit/scoring/__init__.py b/src/prkit/scoring/__init__.py index fc0a5e5..2d6dc06 100644 --- a/src/prkit/scoring/__init__.py +++ b/src/prkit/scoring/__init__.py @@ -3,11 +3,22 @@ ``SemanticsScorer`` is the canonical, version-stamped scorer wrapping the deterministic (binary) semantics comparison engine. ``PartialCreditScorer`` is its graded counterpart: an EED/SEED edit-distance scorer that populates -``Verdict.partial_credit``. Both structurally satisfy :class:`prkit.api.Scorer` and -emit :class:`prkit.api.Verdict`. +``Verdict.partial_credit``. ``LLMJudgeScorer`` is the model-graded scorer wrapping +the ``prkit.evaluation.llm_judge`` engine. All structurally satisfy +:class:`prkit.api.Scorer` and emit :class:`prkit.api.Verdict`. + +Import discipline: re-exporting ``LLMJudgeScorer`` here must not pull ``openai`` or +``prkit.evaluation.llm_judge`` onto ``import prkit.scoring`` — its judge imports are +deferred to method bodies (see ``llm_judge_scorer``). """ +from .llm_judge_scorer import LLMJudgeScorer from .partial_credit_scorer import PartialCreditMode, PartialCreditScorer from .semantics_scorer import SemanticsScorer -__all__ = ["PartialCreditMode", "PartialCreditScorer", "SemanticsScorer"] +__all__ = [ + "LLMJudgeScorer", + "PartialCreditMode", + "PartialCreditScorer", + "SemanticsScorer", +] diff --git a/src/prkit/scoring/llm_judge_scorer.py b/src/prkit/scoring/llm_judge_scorer.py new file mode 100644 index 0000000..2f304c5 --- /dev/null +++ b/src/prkit/scoring/llm_judge_scorer.py @@ -0,0 +1,124 @@ +"""Model-graded :class:`prkit.api.Scorer` wrapping the OpenAI LLM-judge engine. + +``LLMJudgeScorer`` is the one non-deterministic scorer in PRKit's reference set. +It adapts :class:`prkit.evaluation.llm_judge.OpenAIJudgeRunner` (the model-graded +physics judge engine, which stays in ``prkit.evaluation``) into the canonical +:class:`~prkit.core.verdict.Verdict`. + +Import discipline (enforced by ``tests/prkit/verify/test_import_isolation.py``): +``import prkit.scoring`` must stay free of ``openai`` and the +``prkit.evaluation.llm_judge`` subpackage. Every judge import is therefore +deferred to a method body (``__init__`` / ``score``); annotations that need the +judge types reference them only under :data:`typing.TYPE_CHECKING`. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from prkit.core.domain.answer import Answer +from prkit.core.verdict import Verdict + +if TYPE_CHECKING: # annotations only — never imported at runtime by this module + from prkit.evaluation.llm_judge.runner import OpenAIJudgeRunner + from prkit.evaluation.llm_judge.types import LLMJudgeResult + +#: Local scorer-wiring revision (the ``wrapN`` slot of the canonical provenance +#: format). The judged model is recorded per-verdict in ``details["model"]`` and +#: surfaced by ``get_info()``, not baked into this stamp. +_VERSION = "llm-judge/openai+wrap1" + + +class LLMJudgeScorer: + """Model-graded :class:`prkit.api.Scorer` over the OpenAI physics judge. + + Args: + model: The judge model name. Required to build the default runner; it may + be omitted only when an explicit *runner* is injected. + instructions: Optional grading-instruction override; the runner defaults + it to the standard physics grading prompt when ``None``. + runner: An injectable judge runner — anything exposing ``judge(payload) + -> LLMJudgeResult`` and a ``model_name`` property. Defaults to a real + :class:`~prkit.evaluation.llm_judge.OpenAIJudgeRunner`; tests inject a + fake to avoid constructing an OpenAI client or hitting the network. + """ + + version: str = _VERSION + + def __init__( + self, + *, + model: str | None = None, + instructions: str | None = None, + runner: OpenAIJudgeRunner | Any | None = None, + ) -> None: + if runner is None: + if not model: + raise ValueError( + "LLMJudgeScorer requires `model` when no `runner` is injected" + ) + # Lazy import keeps ``import prkit.scoring`` free of openai / + # prkit.evaluation.llm_judge (enforced by test_import_isolation). + from prkit.evaluation.llm_judge.runner import OpenAIJudgeRunner + + runner = OpenAIJudgeRunner(model=model, instructions=instructions) + self._runner = runner + + def score( + self, + prediction: Answer | str, + reference: Answer | str, + *, + question: str | None = None, + **kwargs: Any, + ) -> Verdict: + """Grade ``prediction`` against ``reference`` with the LLM judge. + + ``question``, when supplied, is embedded in the judge payload so the model + has the problem context. Returns a binary :class:`Verdict` (``score`` is + ``1.0`` for a ``"correct"`` judgement, else ``0.0``); the judge's + natural-language explanation is surfaced in ``rationale``. + """ + # Lazy import: must not pull prkit.evaluation.llm_judge at module load. + from prkit.evaluation.llm_judge.payload import ( + build_standard_answer_judge_payload, + ) + + payload = build_standard_answer_judge_payload(prediction, reference, question) + result = self._runner.judge(payload) + return self._verdict_from_result(result) + + def _verdict_from_result(self, result: LLMJudgeResult) -> Verdict: + """Map an :class:`LLMJudgeResult` onto the canonical :class:`Verdict`. + + ``Verdict`` is ``frozen`` / ``extra="forbid"``; every ``details`` value + here is a JSON-serializable scalar/string. ``deterministic=False`` lives + in ``get_info()``, never on the Verdict. + """ + equivalent = result.verdict == "correct" + return Verdict( + equivalent=equivalent, + correct=equivalent, + score=1.0 if equivalent else 0.0, + comparison_mode=f"llm_judge:{result.expected_answer_type}", + scorer_version=self.version, + partial_credit=None, + rationale=result.reasoning, + details={ + "confidence": result.confidence, + "expected_answer_type": result.expected_answer_type, + "raw_response": result.raw_response, + "verdict_type": result.verdict_type, + "model": self._runner.model_name, + }, + ) + + def get_info(self) -> dict[str, Any]: + """Return scorer metadata; always includes ``version`` (non-deterministic).""" + return { + "name": "LLMJudgeScorer", + "version": self.version, + "engine": "openai_judge", + "deterministic": False, + "model": self._runner.model_name, + } diff --git a/tests/prkit/scoring/test_llm_judge_scorer.py b/tests/prkit/scoring/test_llm_judge_scorer.py new file mode 100644 index 0000000..6384f7f --- /dev/null +++ b/tests/prkit/scoring/test_llm_judge_scorer.py @@ -0,0 +1,158 @@ +"""Tests for the model-graded LLMJudgeScorer. + +No network and no OpenAI client: every scoring test injects a ``FakeJudgeRunner`` +via ``LLMJudgeScorer(runner=...)``. The one default-runner test stubs the OpenAI +client class so construction needs neither an API key nor a live connection. +""" + +from __future__ import annotations + +import json + +import pytest + +from prkit.api import Scorer, Verdict +from prkit.core.domain.answer import Answer +from prkit.evaluation.llm_judge.types import LLMJudgeResult +from prkit.scoring import LLMJudgeScorer + + +class FakeJudgeRunner: + """Deterministic stand-in for OpenAIJudgeRunner (no OpenAI client / network).""" + + def __init__( + self, result: LLMJudgeResult, *, model: str = "fake-judge-model" + ) -> None: + self._result = result + self._model_name = model + self.calls: list[dict] = [] + + @property + def model_name(self) -> str: + return self._model_name + + def judge(self, payload: dict) -> LLMJudgeResult: + self.calls.append(payload) + return self._result + + +def _result(verdict: str, **overrides) -> LLMJudgeResult: + base = dict( + verdict=verdict, + confidence=0.9, + expected_answer_type="numeric", + reasoning="because the magnitudes agree", + raw_response=json.dumps({"verdict": verdict}), + verdict_type="llm_judge", + ) + base.update(overrides) + return LLMJudgeResult(**base) # type: ignore[arg-type] + + +def _scorer(runner: FakeJudgeRunner) -> LLMJudgeScorer: + return LLMJudgeScorer(model="gpt-judge", runner=runner) + + +class TestProtocolConformance: + def test_satisfies_scorer_protocol(self): + assert isinstance(_scorer(FakeJudgeRunner(_result("correct"))), Scorer) + + def test_version_non_empty(self): + assert isinstance(LLMJudgeScorer.version, str) and LLMJudgeScorer.version + + def test_get_info_matches_version_and_reports_non_deterministic(self): + s = _scorer(FakeJudgeRunner(_result("correct"))) + info = s.get_info() + assert info["version"] == s.version + assert info["name"] == "LLMJudgeScorer" + assert info["engine"] == "openai_judge" + assert info["deterministic"] is False + # Model is pulled from the runner, not the constructor's `model` arg. + assert info["model"] == "fake-judge-model" + + +class TestScoring: + def test_correct_verdict_maps_to_pass(self): + v = _scorer(FakeJudgeRunner(_result("correct"))).score("3 m/s", "3.0 m/s") + assert isinstance(v, Verdict) + assert v.equivalent is True + assert v.correct is True + assert v.score == 1.0 + + def test_incorrect_verdict_maps_to_fail(self): + v = _scorer(FakeJudgeRunner(_result("incorrect"))).score("3 m/s", "5 m/s") + assert v.equivalent is False + assert v.correct is False + assert v.score == 0.0 + + def test_comparison_mode_embeds_expected_answer_type(self): + runner = FakeJudgeRunner(_result("correct", expected_answer_type="expression")) + v = _scorer(runner).score("x+1", "1+x") + assert v.comparison_mode == "llm_judge:expression" + + def test_rationale_is_the_judge_reasoning(self): + v = _scorer(FakeJudgeRunner(_result("correct"))).score("a", "a") + assert v.rationale == "because the magnitudes agree" + + def test_partial_credit_is_none(self): + v = _scorer(FakeJudgeRunner(_result("correct"))).score("a", "a") + assert v.partial_credit is None + + def test_scorer_version_propagated(self): + s = _scorer(FakeJudgeRunner(_result("correct"))) + assert s.score("a", "a").scorer_version == s.version + + def test_details_carry_judge_evidence_and_runner_model(self): + runner = FakeJudgeRunner(_result("correct"), model="gpt-judge-xl") + v = _scorer(runner).score("a", "a") + assert v.details == { + "confidence": 0.9, + "expected_answer_type": "numeric", + "raw_response": json.dumps({"verdict": "correct"}), + "verdict_type": "llm_judge", + "model": "gpt-judge-xl", + } + + def test_details_json_serializable(self): + v = _scorer(FakeJudgeRunner(_result("correct"))).score("a", "a") + json.dumps(v.model_dump()) + + def test_question_is_threaded_into_the_payload(self): + runner = FakeJudgeRunner(_result("correct")) + _scorer(runner).score("3 m/s", "3 m/s", question="What is the speed?") + assert len(runner.calls) == 1 + assert runner.calls[0]["question"] == "What is the speed?" + assert runner.calls[0]["model_answer"]["text"] == "3 m/s" + assert runner.calls[0]["ground_truth"]["text"] == "3 m/s" + + def test_accepts_answer_objects(self): + runner = FakeJudgeRunner(_result("correct")) + pred = Answer(value="3.0", unit="m/s") + ref = Answer(value="3", unit="m/s", source_type="NV") + v = _scorer(runner).score(pred, ref) + assert v.equivalent is True + # Answer.source_type flows into the payload category. + assert runner.calls[0]["ground_truth"]["category"] == "NV" + + +class TestConstruction: + def test_runner_can_be_injected_without_model(self): + # Matches the §IV fixture form: LLMJudgeScorer(runner=fake). + s = LLMJudgeScorer(runner=FakeJudgeRunner(_result("correct"))) + assert s.score("a", "a").equivalent is True + + def test_requires_model_when_no_runner_injected(self): + with pytest.raises(ValueError, match="requires `model`"): + LLMJudgeScorer() + + def test_default_runner_is_built_lazily_from_model(self, monkeypatch): + # Stub the OpenAI client class so no API key / network is needed; this also + # proves the lazy `from prkit.evaluation.llm_judge.runner import ...` path. + pytest.importorskip("openai") + import prkit.evaluation.llm_judge.runner as runner_mod + + monkeypatch.setattr(runner_mod, "OpenAI", lambda **kwargs: object()) + s = LLMJudgeScorer(model="gpt-default") + info = s.get_info() + assert info["model"] == "gpt-default" + assert info["deterministic"] is False From 3789281270cac9cad9a6975575af726b81cb8d6d Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sun, 21 Jun 2026 15:56:00 -0400 Subject: [PATCH 20/28] Drop the dead "legacy" entry from the ruff extend-exclude The legacy/ directory was removed, so excluding it from ruff is no longer needed. "/build", "dist", and "htmlcov" remain. Co-Authored-By: Claude Opus 4.8 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 88ad15a..954b60c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,7 +108,7 @@ target-version = "py310" # "/build" is anchored to the repo root so it excludes the setuptools build/ artifact # dir WITHOUT excluding the src/prkit/semantics/build package (gitignore semantics: a # bare "build" would match any directory of that name at any depth). -extend-exclude = ["legacy", "/build", "dist", "htmlcov"] +extend-exclude = ["/build", "dist", "htmlcov"] [tool.ruff.lint] # E501 (line length) is owned by black. Bugbear (B) is deferred to a later pass. From a87a0ce5515976ed0c1088f2cb2a99272fca6de8 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sun, 21 Jun 2026 16:27:38 -0400 Subject: [PATCH 21/28] Vendor the PHYBench EED and CMPhysBench SEED edit-distance baselines Add faithful, lightly-modified forks of the two upstream edit-distance scorers under evaluation/baselines/, each split into a front-end-free pure core and a LaTeX front-end, with the verbatim upstream LICENSE/NOTICE/PROVENANCE shipped alongside: - phybench_eed/ (github.com/phybench-official/phybench@706feb4, MIT) - cmphysbench_seed/ (github.com/CMPhysBench/CMPhysBench@b2cd857, Apache-2.0; attribution chain to PHYBench EED MIT + the zss BSD license preserved in NOTICE) Local modifications (documented per-file and in each PROVENANCE.md): lift the top-level latex front-end import out of the core so it imports without latex2sympy2_extended; make pint a lazy singleton in the SEED core; and replace the SIGALRM timeout_decorator bounds with the thread-safe prkit.evaluation.edit_distance.timeout.run_with_timeout. Importing either pure core pulls no latex2sympy2_extended/pint/timeout_decorator. Wire the baselines into the Scorer contract with two thin wrappers, EedScorer and SeedScorer, that lazily import the vendored core inside score() so import prkit.scoring stays free of pint and the LaTeX front-end. SeedScorer resolves its answer_type from the kwarg, then reference.source_type (validated against the SEED enum), then default Expression, with an opt-in classifier (default off) recorded in get_info(). Add the [baselines] extra (pint only) plus its all-aggregate entry, ship the vendored licenses as package data, and exclude the vendored tree from ruff/black/mypy and the whitespace pre-commit hooks so it stays auditable against upstream. Create the CMPhysBench loader (HF weidawang/CMPhysBench) mapping the curated answer_type column into Answer.source_type as one of the five SEED tokens, and register it in DatasetHub with an Apache-2.0 license-registry entry. Co-Authored-By: Claude Opus 4.8 --- .pre-commit-config.yaml | 6 +- pyproject.toml | 27 +- src/prkit/datasets/hub.py | 2 + src/prkit/datasets/license_registry.py | 9 + src/prkit/datasets/loaders/__init__.py | 2 + .../datasets/loaders/cmphysbench_loader.py | 202 ++++ src/prkit/evaluation/baselines/__init__.py | 16 + .../baselines/cmphysbench_seed/LICENSE | 201 ++++ .../baselines/cmphysbench_seed/NOTICE | 79 ++ .../baselines/cmphysbench_seed/PROVENANCE.md | 44 + .../baselines/cmphysbench_seed/__init__.py | 8 + .../cmphysbench_seed/core/__init__.py | 6 + .../cmphysbench_seed/core/extended_zss.py | 158 +++ .../baselines/cmphysbench_seed/core/seed.py | 916 ++++++++++++++++++ .../cmphysbench_seed/frontend/__init__.py | 5 + .../frontend/latex_pre_process.py | 897 +++++++++++++++++ .../evaluation/baselines/phybench_eed/LICENSE | 21 + .../baselines/phybench_eed/PROVENANCE.md | 38 + .../baselines/phybench_eed/__init__.py | 6 + .../baselines/phybench_eed/core/__init__.py | 5 + .../baselines/phybench_eed/core/eed.py | 367 +++++++ .../phybench_eed/core/extended_zss.py | 162 ++++ .../phybench_eed/frontend/__init__.py | 5 + .../frontend/latex_pre_process.py | 526 ++++++++++ src/prkit/scoring/__init__.py | 17 +- src/prkit/scoring/eed_scorer.py | 129 +++ src/prkit/scoring/seed_scorer.py | 284 ++++++ .../loaders/test_cmphysbench_loader.py | 117 +++ tests/prkit/scoring/test_eed_scorer.py | 62 ++ tests/prkit/scoring/test_seed_scorer.py | 117 +++ tests/prkit/test_conformance.py | 18 +- 31 files changed, 4440 insertions(+), 12 deletions(-) create mode 100644 src/prkit/datasets/loaders/cmphysbench_loader.py create mode 100644 src/prkit/evaluation/baselines/__init__.py create mode 100644 src/prkit/evaluation/baselines/cmphysbench_seed/LICENSE create mode 100644 src/prkit/evaluation/baselines/cmphysbench_seed/NOTICE create mode 100644 src/prkit/evaluation/baselines/cmphysbench_seed/PROVENANCE.md create mode 100644 src/prkit/evaluation/baselines/cmphysbench_seed/__init__.py create mode 100644 src/prkit/evaluation/baselines/cmphysbench_seed/core/__init__.py create mode 100644 src/prkit/evaluation/baselines/cmphysbench_seed/core/extended_zss.py create mode 100644 src/prkit/evaluation/baselines/cmphysbench_seed/core/seed.py create mode 100644 src/prkit/evaluation/baselines/cmphysbench_seed/frontend/__init__.py create mode 100644 src/prkit/evaluation/baselines/cmphysbench_seed/frontend/latex_pre_process.py create mode 100644 src/prkit/evaluation/baselines/phybench_eed/LICENSE create mode 100644 src/prkit/evaluation/baselines/phybench_eed/PROVENANCE.md create mode 100644 src/prkit/evaluation/baselines/phybench_eed/__init__.py create mode 100644 src/prkit/evaluation/baselines/phybench_eed/core/__init__.py create mode 100644 src/prkit/evaluation/baselines/phybench_eed/core/eed.py create mode 100644 src/prkit/evaluation/baselines/phybench_eed/core/extended_zss.py create mode 100644 src/prkit/evaluation/baselines/phybench_eed/frontend/__init__.py create mode 100644 src/prkit/evaluation/baselines/phybench_eed/frontend/latex_pre_process.py create mode 100644 src/prkit/scoring/eed_scorer.py create mode 100644 src/prkit/scoring/seed_scorer.py create mode 100644 tests/prkit/datasets/loaders/test_cmphysbench_loader.py create mode 100644 tests/prkit/scoring/test_eed_scorer.py create mode 100644 tests/prkit/scoring/test_seed_scorer.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df1f19f..74b1308 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,10 +18,10 @@ repos: rev: v4.6.0 hooks: - id: end-of-file-fixer - # Vendored, minified third-party assets are shipped as-is. - exclude: ^src/prkit/annotation/tasks/correctness/ui/vendor/ + # Vendored third-party assets/forks are shipped as-is (verbatim upstream). + exclude: ^src/prkit/(annotation/tasks/correctness/ui/vendor|evaluation/baselines)/ - id: trailing-whitespace - exclude: ^src/prkit/annotation/tasks/correctness/ui/vendor/ + exclude: ^src/prkit/(annotation/tasks/correctness/ui/vendor|evaluation/baselines)/ # Local hooks mirror the CI checks (.github/workflows/ci.yml) so type and # test failures surface at commit time instead of in CI. They invoke the diff --git a/pyproject.toml b/pyproject.toml index 954b60c..000fbc0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,8 +72,11 @@ docs = [ annotation = [ "streamlit>=1.28.0", ] +baselines = [ + "pint>=0.23", +] all = [ - "physical-reasoning-toolkit[dev,docs,annotation]", + "physical-reasoning-toolkit[dev,docs,annotation,baselines]", ] [project.urls] @@ -96,19 +99,30 @@ prkit = ["py.typed"] "vendor/katex/*", "vendor/katex/fonts/*", ] +# Ship the verbatim upstream attribution alongside each vendored baseline (the +# extension-less LICENSE/NOTICE are not covered by the "*.md/.txt/.rst" glob below). +"prkit.evaluation.baselines.phybench_eed" = ["LICENSE", "PROVENANCE.md"] +"prkit.evaluation.baselines.cmphysbench_seed" = ["LICENSE", "NOTICE", "PROVENANCE.md"] "*" = ["*.txt", "*.md", "*.rst"] [tool.black] line-length = 88 target-version = ['py310', 'py311', 'py312'] +# Vendored baselines are upstream forks kept close to source for auditability — +# never reformatted. force-exclude applies even to explicitly-passed (pre-commit) paths. +force-exclude = "src/prkit/evaluation/baselines/" [tool.ruff] line-length = 88 target-version = "py310" # "/build" is anchored to the repo root so it excludes the setuptools build/ artifact # dir WITHOUT excluding the src/prkit/semantics/build package (gitignore semantics: a -# bare "build" would match any directory of that name at any depth). -extend-exclude = ["/build", "dist", "htmlcov"] +# bare "build" would match any directory of that name at any depth). The vendored +# baselines are upstream forks kept close to source for auditability — never linted. +extend-exclude = ["/build", "dist", "htmlcov", "src/prkit/evaluation/baselines"] +# force-exclude makes the excludes authoritative even when pre-commit passes the +# vendored paths explicitly (they live under the linted src/prkit/ tree). +force-exclude = true [tool.ruff.lint] # E501 (line length) is owned by black. Bugbear (B) is deferred to a later pass. @@ -125,6 +139,13 @@ warn_return_any = true warn_unused_configs = true disallow_untyped_defs = true +# Vendored baselines are upstream forks (untyped, star-imports) kept close to source +# for auditability — not type-checked. Errors there are suppressed even when the +# scorers import the pure core. +[[tool.mypy.overrides]] +module = "prkit.evaluation.baselines.*" +ignore_errors = true + [[tool.mypy.overrides]] module = [ "datasets", diff --git a/src/prkit/datasets/hub.py b/src/prkit/datasets/hub.py index a468784..673d961 100644 --- a/src/prkit/datasets/hub.py +++ b/src/prkit/datasets/hub.py @@ -21,6 +21,7 @@ from prkit.datasets.downloaders.base_downloader import BaseDownloader from prkit.datasets.license_registry import get_license from prkit.datasets.loaders import ( + CMPhysBenchLoader, JEEBenchLoader, PHYBenchLoader, PhysBenchLoader, @@ -79,6 +80,7 @@ def _register_default_loaders(cls) -> None: cls._loaders.setdefault("jeebench", JEEBenchLoader) cls._loaders.setdefault("tpbench", TPBenchLoader) cls._loaders.setdefault("physreason", PhysReasonLoader) + cls._loaders.setdefault("cmphysbench", CMPhysBenchLoader) @classmethod def _register_default_downloaders(cls) -> None: diff --git a/src/prkit/datasets/license_registry.py b/src/prkit/datasets/license_registry.py index 1e9303d..325bd6b 100644 --- a/src/prkit/datasets/license_registry.py +++ b/src/prkit/datasets/license_registry.py @@ -94,6 +94,15 @@ commercial_use=True, attribution_required=True, ), + "cmphysbench": LicenseSpec( + "Apache-2.0", + "Apache License 2.0", + _APACHE_URL, + redistributable=True, + commercial_use=True, + attribution_required=True, + notes="upstream weidawang/CMPhysBench (Apache-2.0)", + ), } # Map legacy free-text license strings (and casing variants) onto canonical SPDX ids. diff --git a/src/prkit/datasets/loaders/__init__.py b/src/prkit/datasets/loaders/__init__.py index c1d7068..22965b3 100644 --- a/src/prkit/datasets/loaders/__init__.py +++ b/src/prkit/datasets/loaders/__init__.py @@ -3,6 +3,7 @@ """ from .base_loader import BaseDatasetLoader +from .cmphysbench_loader import CMPhysBenchLoader from .jeebench_loader import JEEBenchLoader from .phybench_loader import PHYBenchLoader from .physbench_loader import PhysBenchLoader @@ -15,6 +16,7 @@ __all__ = [ "BaseDatasetLoader", + "CMPhysBenchLoader", "PhysBenchLoader", "PHYBenchLoader", "PhysicsLoader", diff --git a/src/prkit/datasets/loaders/cmphysbench_loader.py b/src/prkit/datasets/loaders/cmphysbench_loader.py new file mode 100644 index 0000000..e10309c --- /dev/null +++ b/src/prkit/datasets/loaders/cmphysbench_loader.py @@ -0,0 +1,202 @@ +""" +CMPhysBench Dataset Loader + +CMPhysBench is a condensed-matter-physics benchmark whose every item carries a +curated ``answer_type`` annotation — one of ``Expression``, ``Equation``, +``Tuple``, ``Interval``, ``Numeric`` — that the SEED scorer +(:class:`prkit.scoring.SeedScorer`) dispatches on. This loader lifts that label +verbatim into ``Answer.source_type`` (mapped to exactly those five SEED tokens) so a +faithful SEED run needs no inference. + +Upstream HF dataset: ``weidawang/CMPhysBench`` (Apache-2.0). Native columns used: + +- ``id`` → ``problem_id`` +- ``context`` + ``question`` → combined ``question`` text +- ``final_answer`` → ``answer`` (ground-truth LaTeX) +- ``answer_type`` → ``source_type`` (one of the five SEED tokens) +- ``topic`` → preserved as an additional field + +Like the other loaders, ``load()`` reads a local copy of the dataset from the +resolved data directory (``dataset.json``); it does not fetch from HuggingFace at +runtime, keeping the load path network-free. +""" + +import json +import random +from pathlib import Path +from typing import Any + +from prkit.core import PRKitLogger +from prkit.core.domain import PhysicalDataset, PhysicsProblem +from prkit.datasets.license_registry import get_license + +from .base_loader import BaseDatasetLoader + +#: The five SEED dispatch tokens that ``answer_type`` is normalized into. +_SEED_ANSWER_TYPES = ("Expression", "Equation", "Tuple", "Interval", "Numeric") +_SEED_ANSWER_TYPE_BY_LOWER = {token.lower(): token for token in _SEED_ANSWER_TYPES} + + +class CMPhysBenchLoader(BaseDatasetLoader): + """Loader for the CMPhysBench condensed-matter-physics dataset.""" + + def __init__(self) -> None: + """Initialize the CMPhysBench loader with a logger.""" + super().__init__() + self.logger = PRKitLogger.get_logger(__name__) + + @property + def name(self) -> str: + return "cmphysbench" + + @property + def description(self) -> str: + return ( + "CMPhysBench: a condensed-matter-physics benchmark with curated " + "per-item answer types (Expression/Equation/Tuple/Interval/Numeric)" + ) + + def get_info(self) -> dict[str, Any]: + return { + "name": self.name, + "description": self.description, + "repository_url": "https://huggingface.co/datasets/weidawang/CMPhysBench", + "homepage": "https://github.com/CMPhysBench/CMPhysBench", + "license": get_license(self.name).to_info_dict(), + "license_spdx": get_license(self.name).spdx, + "languages": ["en"], + "variants": ["full"], + "splits": ["test"], + "problem_types": ["OE"], + "answer_types": list(_SEED_ANSWER_TYPES), + "source": "CMPhysBench dataset from HuggingFace (weidawang/CMPhysBench)", + "modalities": self.modalities, + } + + @property + def field_mapping(self) -> dict[str, str]: + """Map CMPhysBench native columns onto standard PRKit fields. + + ``context`` + ``question`` are combined in :meth:`_process_metadata`, so + ``question`` is intentionally left unmapped here. + """ + return { + "id": "problem_id", + "final_answer": "answer", + } + + def get_default_variant(self) -> str | None: + """Return default variant 'full'.""" + return "full" + + def get_default_split(self) -> str | None: + """Return default split 'test'.""" + return "test" + + @staticmethod + def _normalize_answer_type(raw: Any) -> str | None: + """Normalize a native ``answer_type`` into one of the five SEED tokens. + + A recognized value (case-insensitively) is canonicalized to its SEED token; + an unrecognized value is passed through verbatim (``Answer.source_type`` is a + free-form label, and ``SeedScorer`` validates it before dispatch); a + missing/empty value yields ``None``. + """ + if raw is None: + return None + text = str(raw).strip() + if not text: + return None + return _SEED_ANSWER_TYPE_BY_LOWER.get(text.lower(), text) + + def _process_metadata(self, metadata: dict[str, Any]) -> dict[str, Any]: + """Combine context+question and lift ``answer_type`` into ``source_type``.""" + context = metadata.pop("context", "") or "" + question = metadata.get("question", "") or "" + metadata["question"] = (str(context) + str(question)).strip() + + # source_type carries the dataset's curated answer-type label (a SEED token). + metadata["source_type"] = self._normalize_answer_type( + metadata.get("answer_type") + ) + + metadata["problem_type"] = "OE" + metadata["language"] = "en" + return metadata + + def load( + self, + data_dir: str | Path | None = None, + variant: str | None = None, + split: str | None = None, + sample_size: int | None = None, + **kwargs: Any, + ) -> PhysicalDataset: + """ + Load the CMPhysBench dataset from a local copy. + + Args: + data_dir: Path to the CMPhysBench dataset (defaults to + ~/PHYSICAL_REASONING_DATASETS/CMPhysBench). + variant: Dataset variant. Defaults to "full". + split: Dataset split. Defaults to "test". + sample_size: Number of problems to sample (None for all). + **kwargs: Additional loading parameters. + + Returns: + PhysicalDataset instance. + + Raises: + ValueError: If an unsupported split or variant is requested, or the JSON + is invalid. + FileNotFoundError: If the dataset file is not found. + """ + if split is None: + split = self.get_default_split() or "test" + if variant is None: + variant = self.get_default_variant() or "full" + + self.validate_variant(variant) + self.validate_split(split) + + data_dir = self.resolve_data_dir(data_dir, "CMPhysBench") + self.logger.debug(f"Using data directory: {data_dir}") + + if not data_dir.exists(): + raise FileNotFoundError(f"Data directory not found: {data_dir}") + + dataset_file = data_dir / "dataset.json" + if not dataset_file.exists(): + raise FileNotFoundError( + f"CMPhysBench dataset file not found: {dataset_file}" + ) + + try: + with open(dataset_file, encoding="utf-8") as handle: + data = json.load(handle) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON in CMPhysBench dataset: {exc}") + + problems: list[PhysicsProblem] = [] + for problem_data in data: + try: + metadata = self.initialize_metadata(problem_data) + metadata = self._process_metadata(metadata) + problems.append(self.create_physics_problem(metadata=metadata)) + except Exception as exc: + self.logger.warning( + f"Skipping problem {problem_data.get('id', 'unknown')}: {exc}" + ) + continue + + if sample_size is not None and sample_size < len(problems): + problems = random.sample(problems, sample_size) + + info = self.get_info() + info["total_problems"] = len(problems) + + self.logger.info( + f"Successfully loaded {len(problems)} problems from CMPhysBench dataset" + ) + + return PhysicalDataset(problems, info, split=split) diff --git a/src/prkit/evaluation/baselines/__init__.py b/src/prkit/evaluation/baselines/__init__.py new file mode 100644 index 0000000..8715cda --- /dev/null +++ b/src/prkit/evaluation/baselines/__init__.py @@ -0,0 +1,16 @@ +"""Vendored related-work baselines — faithful forks of upstream scoring code. + +Each subpackage is a lightly-modified vendor of an upstream edit-distance scorer, +split into a front-end-free **pure core** (``core/``) and a LaTeX front-end +(``frontend/``), with the upstream ``LICENSE``/``NOTICE``/``PROVENANCE.md`` shipped +verbatim alongside: + +* :mod:`.phybench_eed` — PHYBench Expression Edit Distance (EED), MIT. +* :mod:`.cmphysbench_seed` — CMPhysBench Scalable Expression Edit Distance (SEED), + Apache-2.0. + +The :class:`prkit.scoring.EedScorer` / :class:`prkit.scoring.SeedScorer` thin +wrappers adapt these into the ``Scorer`` / ``Verdict`` contract. This package is +import-light: importing it pulls no ``latex2sympy2_extended``/``pint`` — those are +loaded lazily on the scoring path. +""" diff --git a/src/prkit/evaluation/baselines/cmphysbench_seed/LICENSE b/src/prkit/evaluation/baselines/cmphysbench_seed/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/src/prkit/evaluation/baselines/cmphysbench_seed/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/prkit/evaluation/baselines/cmphysbench_seed/NOTICE b/src/prkit/evaluation/baselines/cmphysbench_seed/NOTICE new file mode 100644 index 0000000..45f602b --- /dev/null +++ b/src/prkit/evaluation/baselines/cmphysbench_seed/NOTICE @@ -0,0 +1,79 @@ +CMPhysBench SEED — attribution chain +==================================== + +This directory vendors the Scalable Expression Edit Distance (SEED) scorer from +CMPhysBench, which is itself a derivative work. The complete attribution chain is +preserved below. The primary license for this vendored code is the Apache License +2.0 (see the adjacent LICENSE file). + +SEED derives from two upstream works: + + 1. PHYBench Expression Edit Distance (EED) — MIT License + 2. The zss / Zhang-Shasha tree-edit-distance package — BSD-style License + +---------------------------------------------------------------------- +1. PHYBench EED — MIT License +---------------------------------------------------------------------- + +MIT License + +Copyright (c) 2025 phybench-official + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +2. zss / Zhang-Shasha — BSD-style License +---------------------------------------------------------------------- + +The ``extended_zss.py`` module is a modified version of the zss package. + +Zhang-Shasha Tree Edit Distance Implementation is licensed under a BSD style +license + +Copyright (c) 2012 + Tim Henderson (tim.tadh@gmail.com) + Stephen Johnson (steve@steveasleep.com) +Copyright (c) 2015 + Gustavo Sousa (gu_ludo@yahoo.com.br) +Copyright (c) 2017 + Erick R. Fonseca (erickrfonseca@gmail.com) +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of this software nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/prkit/evaluation/baselines/cmphysbench_seed/PROVENANCE.md b/src/prkit/evaluation/baselines/cmphysbench_seed/PROVENANCE.md new file mode 100644 index 0000000..c44202e --- /dev/null +++ b/src/prkit/evaluation/baselines/cmphysbench_seed/PROVENANCE.md @@ -0,0 +1,44 @@ +# CMPhysBench SEED — vendoring provenance + +- **Upstream:** https://github.com/CMPhysBench/CMPhysBench +- **Path:** `SEED/` +- **Commit:** `b2cd8571279450f0861759f47d98e9fc577aa993` (`b2cd857`) +- **License:** Apache-2.0 (see `LICENSE`, copied verbatim from the upstream repo + root). SEED derives from PHYBench EED (MIT) and the `zss` package (BSD); the full + attribution chain is preserved verbatim in `NOTICE`. +- **Vendored on:** 2026-06-21 (commit re-verified at vendoring time; `b2cd857` is + the upstream `HEAD` at that date) + +## Layout + +``` +cmphysbench_seed/ + core/ + extended_zss.py # pure tree-edit core (numpy/stdlib) — verbatim + seed.py # dispatch / numeric_score_calc / score_calc / SEED() — modified + frontend/ + latex_pre_process.py # master_convert() → latex2sympy2_extended — modified + LICENSE # upstream Apache-2.0, verbatim + NOTICE # full attribution chain (Apache-2.0 + PHYBench MIT + zss BSD) + PROVENANCE.md # this file +``` + +## Local modifications + +- **`core/seed.py`** + - Lifted the top-level `from .latex_pre_process import *` front-end import so the + pure core imports without `latex2sympy2_extended`. `master_convert` is imported + lazily inside `SEED()` from `..frontend.latex_pre_process`. + - Made `pint` a lazy singleton via `_get_ureg()` (replacing the module-level + `ureg = pint.UnitRegistry()`), so importing this core pulls no `pint`; it is + loaded only on the unit-aware Numeric path. + - Removed `import timeout_decorator` and replaced the `@timeout_decorator.timeout` + (SIGALRM-based) bounds on `simplify_with_timeout` / `equal_with_timeout` and the + nested `subtract_and_simplify_with_timeout` with the thread-safe + `prkit.evaluation.edit_distance.timeout.run_with_timeout`. +- **`frontend/latex_pre_process.py`** + - Removed `import timeout_decorator`; replaced the `@timeout_decorator.timeout` + bound on `master_convert` with `run_with_timeout`. +- **`core/extended_zss.py`** — verbatim apart from the vendoring header comment. + +No `__pycache__`/`*.pyc` are vendored. diff --git a/src/prkit/evaluation/baselines/cmphysbench_seed/__init__.py b/src/prkit/evaluation/baselines/cmphysbench_seed/__init__.py new file mode 100644 index 0000000..d5ea33c --- /dev/null +++ b/src/prkit/evaluation/baselines/cmphysbench_seed/__init__.py @@ -0,0 +1,8 @@ +"""Vendored CMPhysBench Scalable Expression Edit Distance (SEED) — Apache-2.0. + +Upstream: https://github.com/CMPhysBench/CMPhysBench @ ``b2cd857`` (``SEED/``). +SEED derives from PHYBench EED (MIT) and the ``zss`` package (BSD); the full +attribution chain is preserved in ``LICENSE`` + ``NOTICE`` (see :doc:`PROVENANCE.md`). +Split into the front-end-free :mod:`.core` (pure ``sympy``/``numpy``; ``pint`` is +lazy) and the :mod:`.frontend` LaTeX pipeline. +""" diff --git a/src/prkit/evaluation/baselines/cmphysbench_seed/core/__init__.py b/src/prkit/evaluation/baselines/cmphysbench_seed/core/__init__.py new file mode 100644 index 0000000..92a2926 --- /dev/null +++ b/src/prkit/evaluation/baselines/cmphysbench_seed/core/__init__.py @@ -0,0 +1,6 @@ +"""Front-end-free CMPhysBench SEED core (pure ``sympy``/``numpy``/stdlib). + +Importing this package pulls no ``latex2sympy2_extended`` and no ``pint``: the LaTeX +front-end is imported lazily by :func:`.seed.SEED`, and ``pint`` is a lazy singleton +(:func:`.seed._get_ureg`) loaded only on the unit-aware Numeric path. +""" diff --git a/src/prkit/evaluation/baselines/cmphysbench_seed/core/extended_zss.py b/src/prkit/evaluation/baselines/cmphysbench_seed/core/extended_zss.py new file mode 100644 index 0000000..eb41a63 --- /dev/null +++ b/src/prkit/evaluation/baselines/cmphysbench_seed/core/extended_zss.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Vendored from https://github.com/CMPhysBench/CMPhysBench@b2cd857 (SEED/extended_zss.py), +# Apache-2.0 (derives from the zss package, BSD). Local modification: none (verbatim). +# See ../LICENSE, ../NOTICE and ../PROVENANCE.md. +#Original Authors: Tim Henderson and Steve Johnson +#Email: tim.tadh@gmail.com, steve@steveasleep.com +#For licensing see the LICENSE file in the top level directory. + +# This is a modified version of zss package. + + +import collections +import numpy as np +from numpy import zeros,ones + +class Node(object): + + + def __init__(self, label, children=None): + self.label = label + self.children = children or list() + + + @staticmethod + def get_children(node): + return node.children + + @staticmethod + def get_label(node): + return node.label + + def addkid(self, node, before=False): + + if before: self.children.insert(0, node) + else: self.children.append(node) + return self + + def get(self, label): + + if self.label == label: return self + for c in self.children: + if label in c: return c.get(label) + +class AnnotatedTree(object): + + def __init__(self, root, get_children): + self.get_children = get_children + + self.root = root + self.nodes = list() # a post-order enumeration of the nodes in the tree + self.ids = list() # a matching list of ids + self.lmds = list() # left most descendents of each nodes + self.keyroots = None + # the keyroots in the original paper + + + stack = list() + pstack = list() + stack.append((root, collections.deque())) + j = 0 + while len(stack) > 0: + n, anc = stack.pop() + nid = j + for c in self.get_children(n): + a = collections.deque(anc) + a.appendleft(nid) + stack.append((c, a)) + pstack.append(((n, nid), anc)) + j += 1 + lmds = dict() + keyroots = dict() + i = 0 + while len(pstack) > 0: + (n, nid), anc = pstack.pop() + self.nodes.append(n) + self.ids.append(nid) + if not self.get_children(n): + lmd = i + for a in anc: + if a not in lmds: lmds[a] = i + else: break + else: + try: lmd = lmds[nid] + except: + import pdb + pdb.set_trace() + self.lmds.append(lmd) + keyroots[lmd] = i + i += 1 + self.keyroots = sorted(keyroots.values()) + + +def ext_distance(A, B, get_children, single_insert_cost,insert_cost,single_remove_cost, remove_cost, update_cost): + '''Computes the extended tree edit distance between trees A and B with extended-zss algorithm + Args: + A(Node): Root node of tree 1 + B(Node): Root node of tree 2 + get_children(Func): the get_children method of tree + single_insert_cost(Func): cost of inserting single node + insert_cost(Func): cost of inserting a subtree + update_cost(Func): cost of updating A to B + + + Return: + Distance(float):the tree editing distance + ''' + A, B = AnnotatedTree(A, get_children), AnnotatedTree(B, get_children) + size_a = len(A.nodes) + size_b = len(B.nodes) + treedists = zeros((size_a, size_b), float) + fd=1000*ones((size_a+1,size_b+1),float) + operations = [[[] for _ in range(size_b)] for _ in range(size_a)] + + + def treedist(x, y): + Al = A.lmds + Bl = B.lmds + An = A.nodes + Bn = B.nodes + + m = size_a + n = size_b + + fd[Al[x]][Bl[y]]=0 + for i in range(Al[x], x+1): + node = An[i] + fd[i+1][Bl[y]] = fd[Al[i]][Bl[y]] + remove_cost(node) + + for j in range(Bl[y], y+1): + node = Bn[j] + + fd[Al[x]][j+1] = fd[Al[x]][Bl[j]] + insert_cost(node) + + for i in range(Al[x], x+1): + for j in range(Bl[y], y+1): + + node1 = An[i] + node2 = Bn[j] + costs = [fd[i][j+1] + single_remove_cost(node1), + fd[i+1][j] + single_insert_cost(node2), + fd[Al[i]][j+1]+ remove_cost(node1), + fd[i+1][Bl[j]]+ insert_cost(node2)] + m=min(costs) + + if Al[x] == Al[i] and Bl[y] == Bl[j]: + treedists[i][j]=min(m,fd[i][j]+update_cost(node1,node2)) + fd[i+1][j+1]=treedists[i][j] + else: + fd[i+1][j+1]=min(m,fd[Al[i]][Bl[j]]+treedists[i][j]) + + + for x in A.keyroots: + for y in B.keyroots: + treedist(x, y) + + return treedists[-1][-1] + diff --git a/src/prkit/evaluation/baselines/cmphysbench_seed/core/seed.py b/src/prkit/evaluation/baselines/cmphysbench_seed/core/seed.py new file mode 100644 index 0000000..8a79392 --- /dev/null +++ b/src/prkit/evaluation/baselines/cmphysbench_seed/core/seed.py @@ -0,0 +1,916 @@ +# Vendored from https://github.com/CMPhysBench/CMPhysBench@b2cd857 (SEED/SEED.py), +# Apache-2.0 (derives from PHYBench EED, MIT, and the zss package, BSD). +# Local modifications (see ../PROVENANCE.md and ../NOTICE): +# * lifted the top-level `from .latex_pre_process import *` front-end import so the +# pure core imports without `latex2sympy2_extended`; `master_convert` is imported +# lazily inside `SEED()`. +# * made `pint` a lazy singleton (`_get_ureg()`) so importing this core pulls no +# `pint`; it is loaded only on the unit-aware Numeric path. +# * replaced the SIGALRM `timeout_decorator` bounds with the thread-safe +# `prkit.evaluation.edit_distance.timeout.run_with_timeout`. +from sympy import * +from sympy.core.function import AppliedUndef +from sympy.core.numbers import Pi, Exp1,ImaginaryUnit,Infinity,NegativeInfinity,NaN,ComplexInfinity +from sympy.matrices import MatrixBase +from sympy.core.relational import Relational +from sympy import Derivative +from sympy.logic.boolalg import And, Or, Not + +import re +import numpy as np +from .extended_zss import ext_distance +from sympy.simplify import * + +from prkit.evaluation.edit_distance.timeout import SimplifyTimeout, run_with_timeout +# from graphviz import Digraph + +""" +There are four main categories: + +Constants: such as integers, decimals, or mathematical constants like π and e. +Variables: letters like x, y, z, or specified terms in problems (e.g., ħ, c, G). +Functions: sine, cosine, exponential, logarithm, etc. +Operators: basic binary operations including addition, multiplication, and exponentiation. +""" +# The costs can be modified if you think their values are different +insert_cost={"number":1,"symbol":1,"operator":1,"function":1,"matrix":1,"relation":1} +delete_cost={"number":1,"symbol":1,"operator":1,"function":1,"matrix":1,"relation":1} +update_cost={"number":1,"symbol":1,"operator":1,"function":1,"matrix":1,"relation":1} + +change_type_cost=1 #the cost of an update between different types,can be set to higher + +bar_size=5 # the minimum size of triggering cluster discount +discount_slope=0.6 #discount + +simplify_time_limit=30 #set the time limit of simplify +equals_time_limit=10 #set the time limit of equals + +def update_func(x,y): + + if x.label==y.label: + return 0 + + elif x.label.split("_")[0]==y.label.split("_")[0]: + return update_cost[x.label.split("_")[0]] + return change_type_cost +def remove_func(x): + return delete_cost[x.label.split("_")[0]] + +def remove_tree_func(x): + if not x.children: + return remove_func(x) + s=calc_tree_size(x) + return min(s,discount_slope*(s-bar_size)+bar_size) + +def insert_func(x): + return insert_cost[x.label.split("_")[0]] +def insert_tree_func(x): + return remove_tree_func(x) + +def calc_tree_size(node): + """ + Calculate the size of a subtree based on its total insertion cost. + + The function computes the size of a subtree by summing up the insertion + costs of the current node and all its descendant nodes. If the subtree + size has already been calculated and stored in `node.subtree_size`, it + returns the cached value to avoid redundant computation. + + Args: + node (Node): The root node of the subtree for which the size is to + be calculated + Returns: + int: The total size of the subtree, calculated as the sum of the + insertion costs of the current node and all its descendants. + Notes: + - The `insert_cost` dictionary is assumed to be globally defined + and maps node labels to their respective insertion costs. + - The function modifies the `subtree_size` attribute of the input + node to store the calculated subtree size for future use. + """ + """The size of a subtree equals to its total insertion cost""" + + total = insert_cost[node.label.split("_")[0]] + + if node.children and node.subtree_size !=0: + + return node.subtree_size + + for child in node.children: + total += calc_tree_size(child) + + node.subtree_size=total + + return total +""" +Scoring function from relative distance +""" +def score_calc(tree_dist,tree_size): + + if tree_dist==0.: + return 100 + return max(0,100*discount_slope-100*tree_dist/tree_size) + +def numeric_score_calc(student_answer_exp, ground_truth_exp): + """ + Specialized scoring function for numeric types + Scores based on combined criteria of absolute and relative errors with configurable thresholds + Features + - Multi-tier scoring: 100pts (0.5% tolerance), 90pts (1%), 80pts (2%) + - Sign consistency checking to catch conceptual errors + - Special handling for zero values + - Graceful fallback to tree-based scoring on conversion failures + """ + # Parameter Setting Section (Adjust scoring strictness) + + # 100-point standard (strictest) + RelTol_100_strict = 0.01 # 1% + + # 90-point standard (moderately strict) + RelTol_90 = 0.02 # 2% + + # 80-point standard (more lenient) + RelTol_80 = 0.04 # 4% + + try: + # If ground_truth_exp is an equation, extract the right-hand side value + if hasattr(ground_truth_exp, 'rhs'): + ground_truth_value = ground_truth_exp.rhs + print(f"Detected equation, using rhs: {ground_truth_value}") + else: + ground_truth_value = ground_truth_exp + + # Try to convert SymPy expressions to numerical values + ground_truth = float(ground_truth_value.evalf()) + student_answer = float(student_answer_exp.evalf()) + + # Preprocessing: Handle special case where correct answer is 0 + if ground_truth == 0: + if student_answer == 0: + return 100 + else: + return 0 + + # Sign consistency check + if ground_truth * student_answer < 0: + return 0 + + # Calculate errors + absolute_error = abs(student_answer - ground_truth) + relative_error = absolute_error / abs(ground_truth) + + + # Judge + is_extremely_close = (relative_error <= RelTol_100_strict) + if is_extremely_close: + return 100 + elif relative_error <= RelTol_90: + return 90 + elif relative_error <= RelTol_80: + return 80 + # None of the standards are met + else: + return 0 + + except Exception as e: + print(f" -> numeric_score_calc error: {e}") + # If numerical conversion fails, fall back to the original scoring method + return 0 + +def simplify_with_timeout(expr): + return run_with_timeout(lambda: simplify(expr), timeout_s=simplify_time_limit) +def time_simplify(expr): + try: + result=simplify_with_timeout(expr) + return result + except SimplifyTimeout: + return expr + +def equal_with_timeout(expr1,expr2): + return run_with_timeout(lambda: expr1.equals(expr2), timeout_s=equals_time_limit) +def time_equal(expr1,expr2): + try: + result=equal_with_timeout(expr1,expr2) + return result + except SimplifyTimeout: + return False + + +def sympy_to_tree(expr): + """ + Convert a SymPy expression into a tree structure. + This function takes a SymPy expression and recursively converts it into a tree + representation using `TreeNode` objects. Each node in the tree is labeled based + on the type of the SymPy expression (e.g., number, symbol, operator, or function), + and its children represent the arguments of the expression. + Args: + expr (sympy.Basic): The SymPy expression to be converted. + Returns: + TreeNode: The root node of the tree representation of the SymPy expression. + Raises: + ValueError: If the SymPy expression contains an unsupported type. + Supported Types: + - Numbers: Integer, Pi, Exp1, Float, Rational, Infinity, NegativeInfinity + - Symbols: Symbol + - Binary Operators: Add, Mul, Pow + - Functions: Any subclass of `sympy.Function` + Example: + >>> from sympy import symbols, sin, pi + >>> x, y = symbols('x y') + >>> expr = x + y * sin(pi) + >>> tree = sympy_to_tree(expr) + >>> print(tree) + """ + + + """Convert the sympy expression to a tree""" + if isinstance(expr, MatrixBase): + children = [] + for i in range(expr.rows): + for j in range(expr.cols): + children.append(sympy_to_tree(expr[i, j])) + return TreeNode(label=f"matrix_{expr.rows}x{expr.cols}", children=children) + + elif isinstance(expr, (Integer, Pi, Exp1, ImaginaryUnit, Float, Rational, Infinity, NegativeInfinity, NaN, ComplexInfinity)): + return TreeNode(label="number_" + str(expr), children=[]) + elif isinstance(expr, Symbol): + return TreeNode(label="symbol_" + str(expr), children=[]) + elif isinstance(expr, (Add, Mul, Pow)): + op_name = type(expr).__name__ + children = [sympy_to_tree(arg) for arg in expr.args] + return TreeNode(label="operator_" + op_name, children=children) + elif isinstance(expr, Function): + func_name = expr.func.__name__ + children = [sympy_to_tree(arg) for arg in expr.args] + return TreeNode(label="function_" + func_name, children=children) + elif isinstance(expr, Relational): + op_name = type(expr).__name__ + children = [sympy_to_tree(expr.lhs), sympy_to_tree(expr.rhs)] + return TreeNode(label="relation_" + op_name, children=children) + elif isinstance(expr, Derivative): + children = [sympy_to_tree(expr.expr)] + [sympy_to_tree(v) for v in expr.variables] + return TreeNode(label="function_Derivative", children=children) + elif isinstance(expr, And): + children = [sympy_to_tree(arg) for arg in expr.args] + return TreeNode(label="logic_And", children=children) + elif isinstance(expr, Or): + children = [sympy_to_tree(arg) for arg in expr.args] + return TreeNode(label="logic_Or", children=children) + elif isinstance(expr, Not): + children = [sympy_to_tree(expr.args[0])] + return TreeNode(label="logic_Not", children=children) + else: + raise ValueError(f"Unsupported SymPy type: {type(expr)} Expression: {expr}") + +class TreeNode: + def __init__(self, label, children=None,node_type='other'): + self.label = label + self.children = children if children is not None else [] + self.node_type=node_type + self.subtree_size=0 + def get_children(self): + return self.children + + def __str__(self): + return self.label + +def print_tree(node, indent=0): + """Print a tree structure""" + print(' ' * indent + f'└─ {node.label}') + for child in node.children: + print_tree(child, indent + 1) + +class LaTeXError(Exception): + def __init__(self, message="LaTeXError"): + super().__init__(message) + +class SymPyError(Exception): + def __init__(self, message="SymPyError"): + super().__init__(message) + +class TreeError(Exception): + def __init__(self, message="TreeError"): + super().__init__(message) + +class DistError(Exception): + def __init__(self, message="DistanceError"): + super().__init__(message) + +def Equation_standardize(latex): + """ + Standardize equation by converting it to difference form + """ + return latex.args[0] - latex.args[1] + +def extract_interval(latex): + """ + Extract interval notation from LaTeX string + Use regular strings (not raw strings), so all backslashes are escaped with \\ + """ + interval_pattern = re.compile( + r"^\s*" # Leading whitespace + r"(?:\\left)?\s*" # Optional \left + r"([\(\[])\s*" # Group 1: left bracket + r"(.*?)\s*,\s*" # Group 2: lower bound + r"(.*?)\s*" # Group 3: upper bound + r"(?:\\right)?\s*" # Optional \right + r"([\)\]])\s*$" # Group 4: right bracket + ) + match = interval_pattern.match(latex) + if match: + left_bracket, lower_bound, upper_bound, right_bracket = match.groups() + return True, left_bracket, lower_bound, upper_bound, right_bracket + else: + return False, None, None, None, None + +def judge_interval(latex): + """ + Judge if a LaTeX string represents an interval + """ + latex=latex.replace('$','') + match, left_bracket, lower_bound, upper_bound, right_bracket = extract_interval(latex) + if match: + # Judge whether it's open/closed interval + is_left_closed = left_bracket == "[" + is_right_closed = right_bracket == "]" + left_type = "l_c" if is_left_closed else "l_o" + right_type = "r_c" if is_right_closed else "r_o" + return True, left_type + lower_bound + "+" + upper_bound + right_type + else: + return False, latex + +def check_latex_wrap(s): + s = s.strip() + pattern = r''' + ^( + \(.*\) | # Regular parentheses ( ) + \[.*\] | # Regular square brackets [ ] + \\\(.*\\\) | # LaTeX inline math: \( \) + \\\[.*\\\] | # LaTeX display math: \[ \] + \\\\left\(.*\\\\right\) | # LaTeX \left( \right) + \\\\left\[.*\\\\right\] | # LaTeX \left[ \right] + \$.*\$ # LaTeX inline math with $...$ + )$ + ''' + return re.match(pattern, s, re.VERBOSE) is not None + +def parse_bracketed_string(s): + # Remove surrounding brackets: supports (), \left( \right) + s = s.strip() + s = re.sub(r'^\\left\(|^\(', '', s) + s = re.sub(r'\\right\)$|\)$', '', s) + parts = [item.strip() for item in s.split(',')] + return parts + +def strip_dollar_signs(s): + s = s.strip() + if s.startswith("$$") and s.endswith("$$"): + return s[2:-2].strip() + elif s.startswith("$") and s.endswith("$"): + return s[1:-1].strip() + return s + +def extract_numeric_part(latex_str: str) -> str: + """ + Numeric extractor + Intelligently extracts and returns a clean string containing only numbers and basic operators + from a complex LaTeX string that may contain units, variables, equations. + """ + if not isinstance(latex_str, str) or not latex_str: + return "" + + s = latex_str.strip() + + # Strip outer LaTeX math environment delimiters + if s.startswith('$') and s.endswith('$'): + s = s.strip('$').strip() + if s.startswith('\\(') and s.endswith('\\)'): + s = s[2:-2].strip() + if s.startswith('\\[') and s.endswith('\\]'): + s = s[2:-2].strip() + """ + If there's an equation or approximately equal sign, take only the right side + Use non-greedy matching .*? to ensure it doesn't accidentally match too much + Support various forms like a = b, a \\approx b, etc. + """ + equal_sign_pattern = r'.*(?:=|\\approx|\\sim|\\simeq|\\propto)\s*(.*)' + match = re.search(equal_sign_pattern, s) + if match: + s = match.group(1).strip() + + # Remove LaTeX whitespace commands so signs adjacent to numbers are preserved + try: + s = _remove_latex_whitespace_commands(s) + except Exception: + pass + # Normalize percent: turn "number\%" or "number%" into "number/100" + s = re.sub(r"(\d(?:[\d\.]*)?)\s*\\%", r"(\1/100)", s) + s = re.sub(r"(\d(?:[\d\.]*)?)\s*%", r"(\1/100)", s) + # Remove stray backslashes directly before a sign or digit (e.g., \, -\,2.14 -> -2.14) + s = re.sub(r'\\(?=[\d\+\-])', '', s) + + """ + Actively match and extract scientific notation or regular numbers + This regex can match various forms like -1.28, 1.28e-5, -1.28 \\times 10^{-5}, -1.28 \\\\times 10^{-5}, etc. + Also normalize common \\frac forms into a/b for rational parsing. + """ + # Normalize \\frac forms to a/b to support rational parsing + # \\frac{a}{b} + s = re.sub(r"\\frac\s*\{\s*([^{}]+)\s*\}\s*\{\s*([^{}]+)\s*\}", r"(\1)/(\2)", s) + # \\frac a b (brace-less) for simple numeric tokens + s = re.sub(r"\\frac\s*([+-]?(?:\d+(?:\.\d+)?|\.\d+))\s*([+-]?(?:\d+(?:\.\d+)?|\.\d+))", r"\1/\2", s) + # \\frac12 (compact) -> 1/2 + s = re.sub(r"\\frac\s*([0-9])\s*([0-9])", r"\1/\2", s) + + # Prefer fraction pattern a/b first to avoid capturing only the numerator + frac_match = re.search(r"[-+]?\s*(?:\(?\s*(?:\d+\.?\d*|\.\d+)\s*\)?\s*/\s*\(?\s*(?:\d+\.?\d*|\.\d+)\s*\)?)", s) + if frac_match: + return frac_match.group(0).strip() + + # Fall back to scientific/regular number + numeric_pattern = re.compile( + r"([-+]?\s*(?:\d+\.?\d*|\.\d+)\s*(?:(?:e|E)\s*[-+]?\s*\d+|\\\\?times\s*10\^\{?[-+]?\d+\}?)?)" + ) + + match = numeric_pattern.search(s) + + if match: + # If successful match, directly return the core numeric string + numeric_part = match.group(0) + # Clean up by replacing both \\times and \\\\times with * + cleaned_part = numeric_part.replace('\\\\times', '*').replace('\\times', '*') + return cleaned_part.strip() + return s + +def extract_tuple(latex): + """ + A tuple/key-value pair parser. + Core strategy: + 1. If the expression is in the form `(keys) = (values)`, [ignore] the left `(keys) =` part, + only take the right `(values)` as the parsing target. + 2. If the expression is just a tuple `(values)`, parse it directly. + 3. Always return a dictionary with numeric indices as keys, like {'0': val1, '1': val2, ...}. + """ + latex = strip_dollar_signs(latex.strip()) + latex = latex.replace(r'\left', '') + latex = latex.replace(r'\right', '') + + # Check if there's a top-level '(keys) = (values)' structure + paren_level = 0 + top_level_equal_index = -1 + for i, char in enumerate(latex): + if char in '({[': paren_level += 1 + elif char in ')}]': paren_level -= 1 + elif char == '=' and paren_level == 0: + top_level_equal_index = i + break + # If found this structure, we only focus on the right side of the equals sign + if top_level_equal_index != -1: + left_part = latex[:top_level_equal_index].strip() + right_part = latex[top_level_equal_index+1:].strip() + # Do a sanity check to ensure both sides of the equals sign look like tuples + if check_latex_wrap(left_part) and check_latex_wrap(right_part): + # override the entire expression with the right side + latex = right_part + + # Parse the final tuple string + if not check_latex_wrap(latex): + return {} + + # remove brackets and split by commas + values = parse_bracketed_string(latex) + + # If it's an empty tuple "()", values will be an empty list after parsing + if not values: + # Here we return empty dict, the logic in EED will handle it correctly + return {} + + # Convert value list to dictionary with numeric indices as keys + return {str(i): v for i, v in enumerate(values)} + +# Unit processing related functions +_UREG = None + +def _get_ureg(): + """Lazily construct the ``pint`` unit registry (keeps ``pint`` off import).""" + global _UREG + if _UREG is None: + import pint # local import: only the unit-aware Numeric path needs pint + _UREG = pint.UnitRegistry() + return _UREG + +def _remove_latex_whitespace_commands(text: str) -> str: + """Remove common LaTeX whitespace commands from text (no regex side-effects).""" + if not text: + return text + commands = [ + "\\,", "\\;", "\\:", "\\!", "\\quad", "\\qquad", "\\thinspace", "\\enspace", "\\ ", + ] + for cmd in commands: + text = text.replace(cmd, "") + return text + +def _safe_parse_numeric_string(numeric_str: str) -> float: + """ + Safely parse a numeric string that may be in forms like: + - 1.23 + - -0.5 + - 1e-3 / 1E+6 + - 1.2*10^3 / 1.2 * 10^{3} + Never uses eval. Returns float or raises ValueError. + """ + if not isinstance(numeric_str, str): + raise ValueError("numeric_str must be a string") + s = numeric_str.strip() + # Normalize spacing and variants + s = s.replace("\\times", "*").replace("\\\\times", "*") + s = re.sub(r"\s+", "", s) + # Expand percent to division by 100 if trailing + s = re.sub(r"^(.*?)(\d(?:[\d\.]*)?)/?100\)?$", r"\1(\2/100)", s) if False else s + if s.endswith('%'): + s = s[:-1] + "/100" + if s.endswith('\\%'): + s = s[:-2] + "/100" + # Normalize *10^{n} to *10**n + s = re.sub(r"\*10\^\{?([+-]?\d+)\}?", r"*10**\1", s) + # Pattern a*10**b + m = re.fullmatch(r"([+-]?(?:\d+(?:\.\d+)?|\.\d+))\*10\*\*([+-]?\d+)", s) + if m: + base = float(m.group(1)) + exp = int(m.group(2)) + return base * (10 ** exp) + # Pattern scientific e/E + m = re.fullmatch(r"([+-]?(?:\d+(?:\.\d+)?|\.\d+))[eE]([+-]?\d+)", s) + if m: + base = float(m.group(1)) + exp = int(m.group(2)) + return base * (10 ** exp) + # Fraction a/b (allow simple parentheses around parts), only when exactly one '/' + if s.count('/') == 1: + num_str, den_str = s.split('/', 1) + # strip one layer of parentheses if present + num_str = re.sub(r"^\((.*)\)$", r"\1", num_str) + den_str = re.sub(r"^\((.*)\)$", r"\1", den_str) + num = _safe_parse_numeric_string(num_str) + den = _safe_parse_numeric_string(den_str) + if den == 0: + raise ValueError("Division by zero in fraction") + return num / den + # Plain number + m = re.fullmatch(r"[+-]?(?:\d+(?:\.\d+)?|\.\d+)", s) + if m: + return float(s) + raise ValueError(f"Unrecognized numeric format: {numeric_str}") + +def clean_latex_unit(unit_str): + r""" + Clean LaTeX unit string for pint parsing + Recursively clean LaTeX wrapping like \mathrm{}, \text{}, \operatorname{} from unit strings, + extract plain text units while preserving braces in exponent parts. + """ + pattern = re.compile(r"\\(mathrm|text|operatorname)\{([^{}]*(\{[^{}]*\}[^{}]*)*)\}") + prev_str = None + while prev_str != unit_str: + prev_str = unit_str + unit_str = pattern.sub(r"\2", unit_str) + if unit_str.startswith("{") and unit_str.endswith("}"): + unit_str = unit_str[1:-1] + unit_str = unit_str.strip() + unit_str = _remove_latex_whitespace_commands(unit_str) + return unit_str + +def parse_latex_quantity_general(latex_str): + r""" + Generically parse LaTeX-formatted quantity strings to extract numeric values and units. + Supports: + - Numbers (including decimals, negative signs, scientific notation, and LaTeX-style scientific notation) + - Units wrapped in \mathrm{} or \text{}, or without any wrapper + - Removal of all LaTeX whitespace commands + Returns: (float value, unit string) + """ + numeric_part = extract_numeric_part(latex_str) + try: + number = _safe_parse_numeric_string(numeric_part) + except Exception as e: + raise ValueError(f"Failed to compute numeric value from: {numeric_part}, error: {e}") + + original_numeric = re.search(r"[-+]?\s*(?:\d+\.?\d*|\.\d+)\s*(?:(?:e|E)\s*[-+]?\s*\d+|\\\\?times\s*10\^\{?[-+]?\d+\}?)?", latex_str) + if original_numeric: + unit_part = latex_str[original_numeric.end():].strip() + else: + unit_part = "" + + unit_part = clean_latex_unit(unit_part) + return number, unit_part + +def convert_and_output_general(latex_qty1, latex_qty2, target_unit=None): + """ + Parse two generalized LaTeX-formatted quantity strings, convert them to the target unit, and output the result. + If target_unit is empty, convert to the unit of the first quantity. + """ + n1, u1 = parse_latex_quantity_general(latex_qty1) + n2, u2 = parse_latex_quantity_general(latex_qty2) + + ureg = _get_ureg() + q1 = n1 * ureg(u1) + q2 = n2 * ureg(u2) + + if target_unit is None: + target_unit = u1 + + q1_converted = q1.to(target_unit) + q2_converted = q2.to(target_unit) + + out1 = f"{q1_converted.magnitude} {target_unit}" + out2 = f"{q2_converted.magnitude} {target_unit}" + + return out1, out2 + +def SEED(answer_latex,test_latex,expr_type,debug_mode=False): + """ + SEED (Scalable Expression Edit Distance) - Enhanced version of EED + NEW FEATURES in SEED vs EED: + Multi-type expression support: Expression, Equation, Tuple, Interval, Numeric + Advanced numeric scoring with relative/absolute error thresholds + Physical unit conversion and comparison using Pint library + Intelligent tuple/key-value pair parsing and comparison + Interval notation support with open/closed bracket distinction + Improved equation standardization (A=B → A-B) + Robust error handling + + Computes the similarity score and distance metrics between two LaTeX expressions. + + This function evaluates the equivalence of two mathematical expressions represented + in LaTeX format. It uses symbolic computation and tree-based distance metrics to + calculate a similarity score and other related metrics. + + Args: + answer_latex: The latex expression of answer expression + test_latex: The latex expression of test expression + t: Expression type (Expression, Equation, Tuple, Interval, Numeric) + debug_mode: Whether it raise errors or just skip it + + Returns: + tuple: A tuple containing the following elements: + - score (float): The similarity score between the two expressions (0 to 100). + - relative_distance (float): The normalized distance between the two expressions. + - answer_tree_size (int): The size of the expression tree for the answer. + - distance (float): The raw distance between the two expression trees. + + Notes: + - If either input contains unsupported LaTeX constructs (e.g., integrals or sums), + the function returns default values indicating failure. + - If the test expression is significantly longer than the answer expression, + the function assumes they are not equivalent. + - The function uses symbolic simplification and tree-based distance metrics to + evaluate equivalence. + - In case of errors during processing, the function returns default values unless + `debug_mode` is enabled, in which case it raises specific exceptions. + + Exceptions: + - LaTeXError: Raised when LaTeX conversion to symbolic expressions fails (if `debug_mode` is True). + - SymPyError: Raised when symbolic simplification or tree construction fails (if `debug_mode` is True). + - DistError: Raised when distance calculation fails (if `debug_mode` is True). + """ + + if not test_latex: + return 0,-1,-1,-1 + if '\\int' in test_latex or '\\int' in answer_latex: + return 0,-1,-1,-1 + if '\\sum' in test_latex or '\\sum' in answer_latex: + return 0,-1,-1,1 + if answer_latex==test_latex: + return 100,0.0,-1,0 + # if len(test_latex)>3*len(answer_latex): + # return 0,-1,-1,-1 + + # Front-end is loaded lazily so importing this pure core stays free of + # latex2sympy2_extended (see the module-header note). + from ..frontend.latex_pre_process import master_convert + + try: + if expr_type == 'Tuple': + answer_dict = extract_tuple(answer_latex) + test_dict = extract_tuple(test_latex) + + if not answer_dict or not test_dict: + return 0, -1, -1, -1 + + try: + norm_answer_dict = {master_convert(k, 'Expression'): v for k, v in answer_dict.items()} + norm_test_dict = {master_convert(k, 'Expression'): v for k, v in test_dict.items()} + except Exception as e: + if debug_mode: print(f"Error normalizing tuple keys: {e}") + return 0, -1, -1, -1 + + if set(norm_answer_dict.keys()) != set(norm_test_dict.keys()): + return 0, -1, -1, -1 + + scores, rel_distances, tree_sizes, distance_numbers = 0, 0, 0, 0 + size = len(norm_answer_dict) + if size == 0: + return 100, 0.0, 0, 0 + + for sympy_key, answer_v_latex in norm_answer_dict.items(): + test_v_latex = norm_test_dict[sympy_key] + + # Recursively call to compare SEED values + score, rel_distance, tree_size, distance_number = SEED(answer_v_latex, test_v_latex, 'Expression') + scores += score + + if rel_distance != -1: rel_distances += rel_distance + if tree_size != -1: tree_sizes += tree_size + if distance_number != -1: distance_numbers += distance_number + + return scores / size, rel_distances / size, tree_sizes / size, distance_numbers / size + + elif expr_type=='Interval': + is_interval, answer_latex= judge_interval(answer_latex) + is_interval, test_latex= judge_interval(test_latex) + # if is_interval:t='Interval' + elif expr_type=='Numeric': + # Numeric path: directly compute numeric values first using SymPy on RHS, then try units, then fallback + def _rhs_or_self(s: str) -> str: + ss = s.strip() + if ss.startswith('$') and ss.endswith('$'): + ss = ss.strip('$').strip() + if ss.startswith('\\(') and ss.endswith('\\)'): + ss = ss[2:-2].strip() + if ss.startswith('\\[') and ss.endswith('\\]'): + ss = ss[2:-2].strip() + m = re.search(r'.*(?:=|\\approx|\\sim|\\simeq|\\propto)\s*(.*)', ss) + if m: + return m.group(1).strip() + return ss + + def _normalize_numeric_rhs(s: str) -> str: + # Replace common LaTeX multiply operators and remove whitespace commands + s = re.sub(r'\\+times', '*', s) + s = re.sub(r'\\+cdot', '*', s) + s = _remove_latex_whitespace_commands(s) + return s + + try: + ans_rhs = _normalize_numeric_rhs(_rhs_or_self(answer_latex)) + tst_rhs = _normalize_numeric_rhs(_rhs_or_self(test_latex)) + ans_exp_try = master_convert(ans_rhs, 'Expression') + test_exp_try = master_convert(tst_rhs, 'Expression') + if ans_exp_try is not None and test_exp_try is not None: + try: + if getattr(ans_exp_try, 'free_symbols', set()) or getattr(test_exp_try, 'free_symbols', set()): + pass # fall through to unit-aware parsing + else: + score = numeric_score_calc(test_exp_try, ans_exp_try) + return score, -1, -1, -1 + except Exception: + pass + except Exception: + pass + + def _try_parse_quantity(s): + try: + return parse_latex_quantity_general(s) + except Exception: + return None, None + a_val, a_unit = _try_parse_quantity(answer_latex) + t_val, t_unit = _try_parse_quantity(test_latex) + + if a_val is not None and t_val is not None and a_unit and t_unit: + try: + ureg = _get_ureg() + qa = a_val * ureg(a_unit) + qt = t_val * ureg(t_unit) + qt_conv = qt.to(qa.units) + score = numeric_score_calc(Float(qt_conv.magnitude), Float(qa.magnitude)) + return score, -1, -1, -1 + except Exception: + pass + + try: + if a_val is None: + a_val = _safe_parse_numeric_string(extract_numeric_part(answer_latex)) + if t_val is None: + t_val = _safe_parse_numeric_string(extract_numeric_part(test_latex)) + print(a_val) + print(t_val) + score = numeric_score_calc(Float(t_val), Float(a_val)) + return score, -1, -1, -1 + except Exception: + return 0, -1, -1, -1 + + answer_exp = master_convert(answer_latex, expr_type) + test_exp = master_convert(test_latex, expr_type) + if expr_type =='Equation': + answer_exp = Equation_standardize(answer_exp) + test_exp = Equation_standardize(test_exp) + + except Exception as e: + if debug_mode: + raise LaTeXError(f"Fail to convert latex.\n GT:{answer_latex}\n GEN:{test_latex}") + return 0,-1,-1,-1 + + try: + if answer_exp is None or test_exp is None: + return 0,-1,-1,-1 + answer_exp,rep1=posify(answer_exp) + answer_exp=time_simplify(answer_exp) + + test_exp,rep2=posify(test_exp) + test_exp=time_simplify(test_exp) + + answer_exp=answer_exp.subs(rep1) + test_exp=test_exp.subs(rep2) + + # if False: + def _subtract_and_simplify(a, b): + if isinstance(a, Expr) and isinstance(b, Expr): + return simplify(expand(a - b)) + elif isinstance(a, Matrix) and isinstance(b, Matrix): + if a.shape == b.shape: + return simplify(expand(a - b)) + else: + return 1 # Matrix dimensions do not match + else: + return 1 + + def subtract_and_simplify_with_timeout(a, b): + return run_with_timeout(lambda: _subtract_and_simplify(a, b), timeout_s=10) + + def safe_subtract_and_simplify(a, b): + try: + return subtract_and_simplify_with_timeout(a, b) + except SimplifyTimeout: + print(" -> subtract_and_simplify timeout, returning 1") + return 1 # Treat as unequal if a timeout occurs + except Exception as e: + print(f" -> subtract_and_simplify error: {e}") + return 1 + zero_exp=safe_subtract_and_simplify(answer_exp,test_exp) + # zero_exp=time_simplify(expand(answer_exp-test_exp)) + + if expr_type == "Equation": + if answer_exp == test_exp or zero_exp == 0 or answer_exp + test_exp == 0: + return 100, 0., 0, 0 + + if answer_exp == test_exp or zero_exp == 0: + return 100, 0., 0, 0 + + if time_equal(answer_exp, test_exp): + return 100, 0., 0, 0 + + except Exception as e: + if debug_mode: + raise SymPyError(f"Failed to simplify the sympy expression. Expressions: answer_exp={answer_exp}, test_exp={test_exp}") + return 0,-1,-1,-1 + + try: + tree_answer=sympy_to_tree(answer_exp) + tree_test=sympy_to_tree(test_exp) + + except Exception as e: + if debug_mode: + raise SymPyError(f"Failed to build the sympy expression tree.\n GT:{answer_exp}\n GEN:{test_exp}") + return 0,-1,-1,-1 + + distance=ext_distance( + tree_test, + tree_answer, + get_children=lambda x:x.get_children(), + single_insert_cost=insert_func, + insert_cost=insert_tree_func, + single_remove_cost=remove_func, + remove_cost=remove_tree_func, + update_cost=update_func) + + tree_size=calc_tree_size(tree_answer) + distance_number=distance + + rel_distance=distance/tree_size + + # Non-numeric types use tree-based scoring + score = score_calc(distance_number, tree_size) + return score,rel_distance,tree_size,distance_number + +if __name__ == "__main__": + # Example usage of SEED scoring + # ----------------------------------------------------------- + # Fill in the variables below to test SEED: + # gt : Ground truth LaTeX expression string + # pred : Model-predicted LaTeX expression string + # t : Expression type (choose one of): + # "Expression", "Equation", "Tuple", "Interval", "Numeric" + # ----------------------------------------------------------- + + gt = "4.08 \\times 10^{-5}(\\mathrm{~cm})" # Ground truth LaTeX expression + pred = "4.08 \\times 10^{-7}(\\mathrm{~m})" # Predicted LaTeX expression + expr_type = "Numeric" # Answer type + + score, rel_distance, tree_size, dist = SEED(gt, pred, expr_type) + + print("\n=== Test Result ===") + print(f"GT LaTeX: {gt}") + print(f"Predicted: {pred}") + print(f"Score: {score}") # Final SEED score + print(f"Rel Distance: {rel_distance}") # Relative edit distance + print(f"Tree Size: {tree_size}") # Number of nodes in the ground truth expression tree + print(f"Raw Distance: {dist}") # Raw node edit distance \ No newline at end of file diff --git a/src/prkit/evaluation/baselines/cmphysbench_seed/frontend/__init__.py b/src/prkit/evaluation/baselines/cmphysbench_seed/frontend/__init__.py new file mode 100644 index 0000000..dd0a01e --- /dev/null +++ b/src/prkit/evaluation/baselines/cmphysbench_seed/frontend/__init__.py @@ -0,0 +1,5 @@ +"""CMPhysBench SEED LaTeX front-end (``latex2sympy2_extended``). + +Imported lazily by :func:`..core.seed.SEED`; importing it pulls +``latex2sympy2_extended`` + its ``antlr4`` runtime. +""" diff --git a/src/prkit/evaluation/baselines/cmphysbench_seed/frontend/latex_pre_process.py b/src/prkit/evaluation/baselines/cmphysbench_seed/frontend/latex_pre_process.py new file mode 100644 index 0000000..9f08592 --- /dev/null +++ b/src/prkit/evaluation/baselines/cmphysbench_seed/frontend/latex_pre_process.py @@ -0,0 +1,897 @@ +# Vendored from https://github.com/CMPhysBench/CMPhysBench@b2cd857 +# (SEED/latex_pre_process.py), Apache-2.0. Local modification: replaced the SIGALRM +# `timeout_decorator` bound on `master_convert` with the thread-safe +# `prkit.evaluation.edit_distance.timeout.run_with_timeout`. See ../LICENSE, +# ../NOTICE and ../PROVENANCE.md. +#This file is used to pre-process input latex expressions +#You only need a "master_convert()" +from latex2sympy2_extended import * +from sympy import simplify + +import re + +from prkit.evaluation.edit_distance.timeout import SimplifyTimeout, run_with_timeout + +def convert_caret_to_derivative(latex_str): + # Match multiple consecutive ^ after variable names (2 or more) + def repl(m): + var = m.group(1) + carets = m.group(2) + n = len(carets) + if n == 2: + return f"{var}''" # Second order uses double prime notation + else: + return f"{var}^{{({n})}}" # Higher orders use ^{(n)} notation + pattern = r'([a-zA-Z]+)(\^{2,})' + return re.sub(pattern, repl, latex_str) + +def preprocess_special_superscripts(latex_str): + # Define general variable pattern: variable name + optional subscript + optional existing superscript + var_pattern = r'([a-zA-Z0-9_\\]+(?:_\{[^}]+\})?(?:\^\{[^}]+\})?)' + + # 1. Replace ^+ -> ^{+} + latex_str = re.sub(fr'{var_pattern}\^\+', r'\1^{+}', latex_str) + + # 2. Replace ^- -> ^{-} + latex_str = re.sub(fr'{var_pattern}\^\-', r'\1^{-}', latex_str) + + # 3. Replace ^* -> ^{star} + latex_str = re.sub(fr'{var_pattern}\^\*', r'\1^{star}', latex_str) + latex_str = re.sub(fr'{var_pattern}\^\{{(\\ast|\*)\}}', r'\1^{star}', latex_str) + latex_str = re.sub(r'\^\{(\\ast|\*)\}', r'^{star}', latex_str) + # 4. Replace invalid empty exponents with ^{prime} + latex_str = re.sub(fr'{var_pattern}\^(?![\{{\\a-zA-Z0-9])', r'\1^{prime}', latex_str) + + return latex_str + +def brackets_balanced(s: str) -> bool: + """ + Check if the brackets in a LaTeX string are balanced + Args: + s(str): the input string + Return: + bool: True if the brackets are balanced, False otherwise + """ + stack = [] + bracket_pairs = {')': '(', ']': '[', '}': '{'} + + for char in s: + if char in bracket_pairs.values(): + stack.append(char) + elif char in bracket_pairs: + if not stack or stack[-1] != bracket_pairs[char]: + return False + stack.pop() + return len(stack) == 0 + +def remove_non_ascii(text): + """Remove non-ASCII characters from text""" + return text.encode("ascii", errors="ignore").decode() + +def extract_bracket_content(s: str, bracket_position: int) -> str: + """Extract content within braces starting from given position""" + start_idx=bracket_position + + stack = [] + content = [] + escaped = False + brace_start=start_idx+1 + brace_depth = 0 + for i in range(brace_start, len(s)): + char = s[i] + if escaped: + content.append(char) + escaped = False + continue + if char == '\\': + escaped = True + content.append(char) + continue + if char == '{': + brace_depth += 1 + content.append(char) + elif char == '}': + if brace_depth == 0: + return ''.join(content),i + brace_depth -= 1 + content.append(char) + else: + content.append(char) + + return None,-1 +def find_first_unescaped_brace(s: str) -> int: + """Find the position of the first unescaped opening brace""" + escaped = False + for i, c in enumerate(s): + if c == '\\' and not escaped: + escaped = True + continue + if c == '{' and not escaped: + return i + escaped = False + return -1 + +def extract_command(s: str, brace_pos: int) -> str | None: + """extract the command name from a bracket""" + i = brace_pos - 1 + parameter_mode=False + while i >= 0: + if not parameter_mode and s[i] in ('^','_'): + return s[i] + if not parameter_mode and not s[i] in (' ','\t',']','['): + break + if s[i]==']': + parameter_mode=True + if s[i]=='[' and parameter_mode: + parameter_mode=False + i -= 1 + + # Start point + if i < 0 or s[i] == '\\': + return None + + # Extract command name + command_end = i + i -= 1 + while i >= 0 and s[i].isalpha(): + i -= 1 + if i<-1 or s[i]!='\\': + return None + return s[i+1:command_end+1] + +def remove_command(s, command, keep_inside=False): + """ + Removes all occurrences of a specified LaTeX-style command from a string using an iterative approach. + + This function is more robust and efficient than a recursive solution, avoiding recursion depth limits + and excessive string copying. + + Args: + s (str): The input string. + command (str): The LaTeX-style command to remove (e.g., "\\textbf"). + keep_inside (bool, optional): If True, keeps the content inside the braces. Defaults to False. + + Returns: + str: The modified string. + + Examples: + >>> remove_command("This is \\textbf{bold text}.", "\\textbf") + 'This is ' + >>> remove_command("This is \\textbf{bold text}.", "\\textbf", keep_inside=True) + 'This is bold text.' + >>> remove_command("Nested \\textbf{bold \\textit{italic text}} example.", "\\textbf", keep_inside=True) + 'Nested bold \\textit{italic text} example.' + >>> remove_command("No braces \\here.", "\\here") + 'No braces .' + >>> remove_command("Mismatched \\textbf{braces", "\\textbf") + 'Mismatched \\textbf{braces' # No replacement if brace is not closed + """ + result_parts = [] + current_pos = 0 + while True: + pos = s.find(command, current_pos) + + # If no more commands are found, end the loop + if pos == -1: + result_parts.append(s[current_pos:]) + break + + # 1. Add the part before the command + result_parts.append(s[current_pos:pos]) + + # Find the first character after the command, check if it's '{' + brace_start_pos = pos + len(command) + + if brace_start_pos < len(s) and s[brace_start_pos] == '{': + # Find the matching '}' + level = 0 + brace_end_pos = -1 + for i in range(brace_start_pos, len(s)): + if s[i] == '{': + level += 1 + elif s[i] == '}': + level -= 1 + if level == 0: + brace_end_pos = i + break + + if brace_end_pos != -1: # Successfully found matching bracket + if keep_inside: + # Keep the content inside the brackets + result_parts.append(s[brace_start_pos + 1 : brace_end_pos]) + # Update next search start position, skip the entire command and its content + current_pos = brace_end_pos + 1 + else: # No matching bracket found, don't process + # Add the command itself back, then start searching from after the command + result_parts.append(s[pos:brace_start_pos + 1]) + current_pos = brace_start_pos + 1 + + else: # No bracket after command, only remove the command itself + current_pos = brace_start_pos + + return "".join(result_parts) + +def convert_latex_fractions(latex_str): + """Convert non-standard fractions to standard format""" + pattern = r'\\frac((?:\\[a-zA-Z]+|\d|[a-zA-Z]|{[^{}]*}))((?:\\[a-zA-Z]+|\d|[a-zA-Z]|{[^{}]*}))' + + def replacer(match): + numerator, denominator = match.group(1), match.group(2) + wrap_num = f'{{{numerator}}}' if not (numerator.startswith('{') and numerator.endswith('}')) else numerator + wrap_den = f'{{{denominator}}}' if not (denominator.startswith('{') and denominator.endswith('}')) else denominator + return fr'\frac{wrap_num}{wrap_den}' + + return re.sub(pattern, replacer, latex_str) + + +def get_first_brace_command(s: str) -> str | None: + """ Find the position of the first unescaped opening brace and extract the command before it """ + brace_pos = find_first_unescaped_brace(s) + if brace_pos == -1: + return None + return extract_command(s, brace_pos) +def remove_overall_brace(s: str) -> str: + """Remove the outermost brace pair if it wraps the entire string""" + pos=find_first_unescaped_brace(s) + if pos==-1: + return s,0 + command=get_first_brace_command(s) + if not command: + + content,final=extract_bracket_content(s,pos) + if final==len(s) or not '}' in s[final+1:]: + return content,1 + return s,0 + +def exp_frac(s): + """Add braces around exponentiated fractions""" + + def exp_frac_single(s): + position=s.find("^\\frac")+1 + if position == 0: + return s + level=0 + cnt=0 + idx=position + while idx>> convert_vec_syntax(r"\vec x + \vec\alpha + \vec\Gamma") + '\\vec{x} + \\vec{\\alpha} + \\vec{\\Gamma}' + """ + + pattern = r'\\vec(\s*)(\\?[a-zA-Zα-ωΑ-Ω]+)' + replacement = r'\\vec{\2}' + return re.sub(pattern, replacement, text) + +def remove_outer_braces(tex_str): + """ + Convert {base}_{subscript} to base_{subscript} + Example + {a}_{xyz} → a_{xyz} + {\theta}_{0} → \theta_{0} + """ + + pattern = r'\{(\\(?:[a-zA-Z]+|.)|[^{}])+\}_\{([^}]+)\}' + return re.sub(pattern, r'\1_{\2}', tex_str) + +def extract_last_equal_content(s: str, strip_whitespace: bool = True) -> str: + """ + Extract the content after the last occurrence of specific mathematical comparison or assignment operators. + + :param strip_whitespace: If True, removes leading and trailing whitespace from the extracted content. Defaults to True. + (e.g., '=', '\\approx', '\\ge', '\\le', etc.) within the input string `s`. It then extracts + and returns the content that follows the operator. If no operator is found, the entire string + is returned. Optionally, leading and trailing whitespace can be stripped from the extracted content. + + Args: + s (str): The input string to process. + strip_whitespace (bool): Whether to strip leading and trailing whitespace from the extracted content. Defaults to True. + + Returns: + str: The content after the last matching operator, or the entire string if no operator is found. + """ + comparison_operators=('\\approx','\\ge','\\le','\\geq','\\leq','=') +#'\\approx','\\ge','\\le','\\geq','\\leq','<','>', + content=s + for sign in comparison_operators: + if sign in s: + rfind_index = s.rfind(sign) + if s[rfind_index:rfind_index+5]=="\\left" and sign=='\\le': + continue + if rfind_index != -1: + content = s[rfind_index + 1:] + if content =="0": + print("") + if strip_whitespace: + return content.strip() + return content + +def first_pre_process(s,t,extract_box=True): + """ + Perform the first stage of LaTeX string preprocessing. + + if not brackets_balanced(s): + raise ValueError("The input string has unbalanced brackets. Please check the LaTeX expression.") + equality or comparison operator. + + Args: + s (str): The input LaTeX string to preprocess. + extract_box (bool): If True, extracts the content inside a '\\boxed' command. Defaults to True. + + Returns: + str: The preprocessed LaTeX string. + """ + #s=remove_non_ascii(s) + s=s.replace('\\{','(') + s=s.replace('\\}',')') + + if t == "Expression" or t == "Equation": + s = s.replace('\\approx', '=') + + if not brackets_balanced(s): + return s + if extract_box: + boxed_content=remove_command(s,'\\boxed',keep_inside=True) + else: + boxed_content=s + exist_overall_brace=True + cnt=0 + while exist_overall_brace and cnt<10: + boxed_content,exist_overall_brace=remove_overall_brace(boxed_content) + cnt+=1 + + if '\\quad' in boxed_content: + boxed_content = boxed_content.split('\\quad')[0] + + if '\\qquad' in boxed_content: + boxed_content = boxed_content.split('\\qquad')[0] + + boxed_content = boxed_content.strip(' \\') + + if t == "Equation": + last_equal_content = boxed_content + else: + last_equal_content = extract_last_equal_content(boxed_content) + + + # last_equal_content=extract_last_equal_content(boxed_content) + + exist_overall_brace=True + cnt=0 + while exist_overall_brace and cnt<10: + last_equal_content,exist_overall_brace=remove_overall_brace(last_equal_content) + cnt+=1 + return last_equal_content + +def remove_text_from_latex(expr: str) -> str: + """Replace Chinese characters with '1' characters""" + def repl(match): + length = len(match.group()) + return '1' * length + return re.sub(r'[\u4e00-\u9fa5]+', repl, expr) + +def extract_bracket_subscript_pairs(expr): + """Extract bracket-subscript pairs from expression""" + matches = [] + stack = [] + i = 0 + n = len(expr) + + while i < n: + if expr[i] in '({[': + stack.append((i, expr[i])) + elif expr[i] in ')}]': + if not stack: + i += 1 + continue + start, open_br = stack.pop() + close_br = expr[i] + if (open_br, close_br) not in [('(', ')'), ('[', ']'), ('{', '}')]: + i += 1 + continue + + j = i + 1 + if j < n and expr[j] == '_': + k = j + 1 + if k < n and expr[k] == '{': + k += 1 + while k < n and expr[k] != '}': + k += 1 + k += 1 + else: + k += 1 + matches.append((start, k, expr[start:k])) + i += 1 + return matches + +def add_number_to_bracket_subscripts(expr): + """Add numbering to bracket subscripts""" + matches = extract_bracket_subscript_pairs(expr) + if not matches: + return expr + + matches.sort(reverse=True) + counter = 1 + for start, end, content in matches: + new_content = re.sub(r'(_)', f'{counter}\\1', content, count=1) + expr = expr[:start] + new_content + expr[end:] + counter += 1 + return expr + +def insert_multiplication_symbols(expr): + """ + Automatically insert \cdot in LaTeX expressions where needed, handling implicit multiplication cases. + Example: \frac{1}{2}\bar{E}1_a^i → \frac{1}{2} \cdot \bar{E} \cdot 1_a^i + """ + + # Add \cdot after \frac{...}{...} if directly followed by variables or functions + expr = re.sub(r'(\\frac\{[^}]+\}\{[^}]+\})(?=\\[a-zA-Z]|[a-zA-Z0-9])', r'\1 \\cdot ', expr) + + # Insert \cdot between a symbol (like \bar{E}) and another variable + expr = re.sub(r'(\})((\d|[a-zA-Z])_?[a-zA-Z]?\^?[a-zA-Z]?)', r'\1 \\cdot \2', expr) + + return expr + +def remove_all_text_commands(latex_str): + """ + Remove all \text{...} commands and their content from LaTeX. + Args: + latex_str (str): Input LaTeX string + Returns: + str: String after removing \text{...} + """ + pattern = r'\\text\{[^{}]*\}' + return re.sub(pattern, '1', latex_str) +def convert_general_exp_format(latex_str): + # Match patterns like x^{*2}, f(x)^{*3}, \alpha^{*4}, etc. + pattern = r"([a-zA-Z\\]+|\([^)]+\)|\{[^}]+\})\^\{\*(\d+)\}" + + # Convert to (base^*)^n format + return re.sub(pattern, r"(\1^*)^\2", latex_str) +def modify_latex_expression(expr: str) -> str: + # Replace V_{CKM}^{ji*} with V_{CKM}^ji^* + expr = re.sub(r'V_\{CKM\}\^\{([^\}]*?)\*\}', r'V_{CKM}^\1', expr) + + # Remove + appearing before \text + expr = re.sub(r'\+\s*(\\text)', r'\1', expr) + + return expr + +def wrap_single_subscripts(s: str) -> str: + """ + Convert subscripts like xxx_Y or xxx_y to xxx_{Y}/xxx_{y}. + + - Only handle single English letters + - If subscript is already _{...} or followed by \command, don't modify + """ + # Negative lookahead (?![{\\]): exclude _{ already bracketed and _\command cases + pattern = re.compile(r'_(?![{\\])([A-Za-z])') + return pattern.sub(r'_{\1}', s) + +def replace_hc_text(s: str) -> str: + """ + Replace \text{h.c.} (case and space insensitive) with h_c, + keep other \text{...} unchanged. + """ + pattern = re.compile(r'\\text\s*{([^{}]*)}') + + def repl(m): + content = m.group(1).strip() + norm = content.lower().replace(' ', '') + if norm in ('h.c.', 'h.c'): + return 'h_c' + return m.group(0) + + return pattern.sub(repl, s) + +def standardize_dE_notation(s: str) -> str: + s = re.sub(r'd\*([A-Z])_({?[a-zA-Z0-9]+}?)', r'd{\1}_\2', s) + return s + +def replace_arrow_expression(s: str) -> str: + """ + Replace W(i arrow f) with W(iRf), i.e., change 'i arrow f' to 'iRf' in parentheses. + """ + return re.sub(r'W\(\s*(\w+)\s+arrow\s+(\w+)\s*\)', r'W(\1R\2)', s) + +def preprocess_feynman_slash(latex_str: str) -> str: + """ + Converts Feynman slash notation like \not{k} into a plain variable `kslash`. + This helps latex2sympy to parse specialized physics notations. + Example: \not{k}_0 -> kslash_0 + """ + pattern = r'\\not\{([^{}]+)\}' + + replacement = r'\\bar{\1slash}' + + return re.sub(pattern, replacement, latex_str) + +def fix_subscript_on_parentheses(s: str) -> str: + + # Match pattern: (content)_{subscript} + pattern = r'\(([^)]+)\)_\{([^}]+)\}' + + # Replacement rule: keep only "content" and "subscript", remove outer () + replacement = r'\1_{\2}' + + return re.sub(pattern, replacement, s) + + +def reorder_super_sub(latex_str: str) -> str: + """ + Reorder base^{super}_{sub} form to base_{sub}^{super}. + Example: M^{-1}_{j_1 i_1} -> M_{j_1 i_1}^{-1} + This function can handle single letters, multiple letters, and LaTeX commands as base symbols. + """ + # Pattern: (base symbol)(superscript)(subscript) + # Base symbol: one or more letters, possibly starting with backslash + # Superscript: ^{...} + # Subscript: _{...} + pattern = r'([a-zA-Z\\]+)(\^\{[^}]+\})(_\{[^}]+\})' + replacement = r'\1\3\2' + + # Continuously apply replacement until the string no longer changes + # This is a safer approach for handling more complex cases (though not needed in this example) + while True: + new_str = re.sub(pattern, replacement, latex_str) + if new_str == latex_str: + break + latex_str = new_str + + return latex_str + +def second_pre_process(s): + """ + Perform the second stage of LaTeX string preprocessing. + + This function removes or modifies specific LaTeX commands and content to standardize + the input string for further processing. It handles commands like '\\text', '\\mathbf', + and '\\mathrm', removes unnecessary content, and applies transformations such as + converting fractions and vector syntax. + + Args: + s (str): The input LaTeX string to preprocess. + + Returns: + str: The preprocessed LaTeX string. + """ + + s = reorder_super_sub(s) + + kill_commands=[ + '\\begin', + '\\end' + ] + remove_commands=[ + '\\text', + '\\mathbf', + '\\mathrm', + '\\mathscr', + '\\mathcal', + '\\mathfrak', + '\\pmb', + '\\hat', + '\\overline', + '\\boldsymbol', + '\\mathbb', + ] + + + remove_content=[ + '\\,','$',',','`','latex','\\left','\\right','\\text','\\mathrm','\\Bigr','\\Bigl','\n','\\]','\\[', + '\\Big','\\bigl','\\bigr','\\biggl','\\biggr','\\displaystyle','\\boldsymbol','\\infty' + ] + replace_content=[ + ('\\operatorname{asin}','\\asin'), + ('\\operatorname{sech}','\\sech'), + ('\\operatorname{acos}','\\acos'), + ('\\operatorname{sinh}','\\sinh'), + ('\\operatorname{rot}','\\bar{rot}'), + ('\\dfrac','\\frac'), + ('\\tfrac','\\frac'), + ('\\Exp','\\exp'), + ('\\gg','>'), + ('\\ll','<'), + ('\\times','\\bar{times}'), + ('\\dagger','\\bar{dagger}'), + ('\\operatorname{dim}','\\bar{dim}'), + ('\\overleftarrow','\\bar{overleftarrow}'), + ('\;',' '), + (';','\\bar{CD}'), + ('\\partial','\\bar{partial}'), + ('\\perp','\\bar{perp}'), + ('\\parallel','\\bar{parallel}'), + ('\\|','\\bar{parallel}'), + ('\\epsilon','\\varepsilon'), + ('\\varOmega','\\Omega'), + ('I','\\bar{I}'), + ('_e','_{e}'), + ('e_','\\bar{e}_'), + ('E_','\\bar{E}_'), + ('\\pm','+'), + ('\\mp','-'), + ('{+}','{p}'), + ("{-}",'{m}'), + ("_+",'_p'), + ('_-',"_m"), + # ('\\infty', 'oo') + ] + + # More precise handling of single quotes: distinguish derivatives and physics symbols + # Handle function derivatives: f'(x) -> f^{prime}(x) + s = re.sub(r'([a-zA-Z]+)\'(?=\()', r'\1^{prime}', s) + s = re.sub(r'([a-zA-Z]+)\'(?=\s|$|[^a-zA-Z(])', r'\1^{prime}', s) + # Handle single quotes in braces: {k}' -> {k}^{prime} + s = re.sub(r'(\{[a-zA-Z]+\})\'', r'\1^{prime}', s) + s = re.sub(r'·', '', s) + # s = s.replace(r'\dagger', 'dagger') + # s = re.sub(r'\|(.+?)\\rangle', r'\1', s) + s = s.replace(r'\operatorname{Im}', 'Im') + # Remove angle brackets from Dirac symbols or inner product symbols + s = re.sub(r'\\langle\s*(.+?)\s*\\rangle', r'{\1}', s) + s = re.sub(r'\|\s*(.+?)\s*\\rangle', r'\1', s) + s = s.replace(r'\sim', 'Symbol("sim")') + s = re.sub(r'\\bar\{([^{}]+)\}', r'\1', s) + s = replace_hc_text(s) + s = convert_general_exp_format(s) + s = convert_caret_to_derivative(s) + s = preprocess_special_superscripts(s) + s = wrap_single_subscripts(s) + s = modify_latex_expression(s) + s = remove_all_text_commands(s) + s = fix_subscript_on_parentheses(s) + + # s=remove_outer_braces(s) + # Special case: protect differential forms, avoid E_ replacement affecting dE_{k} + # Handle normal form: dE_{k} + s = re.sub(r'\bd([A-Z])_', r'd\1UNDERSCORE', s) + # Handle mathbf form: d\mathbf{E}_{k} + s = re.sub(r'\bd\\mathbf\{([A-Z])\}_', r'd\\mathbf{\1}UNDERSCORE', s) + + s = re.sub(r'\\ddot\{([^}]+)\}', r'\1_{ddot}', s) + s = re.sub(r'\\ddot([A-Za-z]+)', r'\1_{ddot}', s) + # Similarly handle \dot + s = re.sub(r'\\dot\{([^}]+)\}', r'\1_{dot}', s) + s = re.sub(r'\\dot([A-Za-z]+)', r'\1_{dot}', s) + # If the string contains matrix environment keywords, skip kill_commands processing + if not ('\\begin{pmatrix}' in s or '\\end{pmatrix}' in s or + '\\begin{bmatrix}' in s or '\\end{bmatrix}' in s or + '\\begin{matrix}' in s or '\\end{matrix}' in s or + '\\begin{vmatrix}' in s or '\\end{vmatrix}' in s or + '\\begin{Vmatrix}' in s or '\\end{Vmatrix}' in s): + + for command in kill_commands: + s=remove_command(s,command,keep_inside=False) + for command in remove_commands: + s=remove_command(s,command,keep_inside=True) + for content in remove_content: + s=s.replace(content,'') + for content in replace_content: + s=s.replace(content[0],content[1]) + # Restore protected differential forms and add multiplication signs for latex2sympy recognition + if '\\lim' in s: + s = s.replace(r'arrow', r'\rightarrow') + else: + s = re.sub(r'\barrow\b', r'\\bar{arrow}', s) + s = re.sub(r'd([A-Z])UNDERSCORE', r'd*\1_', s) + s = re.sub(r'd\\mathbf\{([A-Z])\}UNDERSCORE', r'd*\\mathbf{\1}_', s) + s = preprocess_feynman_slash(s) + s= convert_latex_fractions(s) + s = standardize_dE_notation(s) + # s = replace_arrow_expression(s) + s=bar_inside_vec(s) + s=vec_lower_idx(s) + s=convert_vec_syntax(s) + s=exp_frac(s) + if s and s[-1] == '.': + s = s[:-1] + s = s.replace(r'\varkappa', r'\kappa') + # First replace derivative forms to avoid parsing errors + s = replace_derivative_frac_preserve_frac(s) + s = remove_text_from_latex(s) + s = add_parentheses_to_d(s) + s = add_number_to_bracket_subscripts(s) + s = insert_multiplication_symbols(s) + s = s.replace('Å', 'A') + return s + +def add_parentheses_to_d(expr): + """ + Pattern: match a 'd', but ensure it's not preceded by \frac{ + (? 1 + interpret_contains_as_eq (bool): Whether to interpret contains as equality x \\in {1,2,3} -> x = {1,2,3} + lowercase_symbols (bool): Whether to lowercase all symbols + """ +class MyNormalization: + """Configuration for latex normalization. + + Each field controls a group of related normalizations: + - basic_latex: Basic latex command replacements (mathrm, displaystyle, etc.) + - units: Remove units and their variations + - malformed_operators: Fix malformed operators (sqrt, frac, etc.) + - nits: Small formatting fixes (spaces, dots, etc.) + - boxed: Extract content from boxed environments + - equations: Handle equation splitting and approximations (deprecated) + """ + + basic_latex: bool = True + units: bool = False + malformed_operators: bool = True + nits: bool = True + boxed = "all" + equations: bool = False + + +def replace_derivative_frac_preserve_frac(expr: str) -> str: + """ + Convert d in \frac{d}{d} to symbol names, preserve \frac structure, + preserve underscores _. + """ + pattern = r''' + \\frac\{ + d + (\\?[a-zA-Z]+) + (_\{?[a-zA-Z0-9]+\}?)? + \}\{ + d + (\\?[a-zA-Z]+) + (_\{?[a-zA-Z0-9]+\}?)? + \} + ''' + + def clean(s): + return s.replace('\\', '').replace('{', '').replace('}', '') + + def repl(m): + var1 = clean(m.group(1)) + sub1 = clean(m.group(2) or '') + var2 = clean(m.group(3)) + sub2 = clean(m.group(4) or '') + + return f'\\frac{{D{var1}{sub1}}}{{D{var2}{sub2}}}' + + return re.sub(pattern, repl, expr, flags=re.VERBOSE) + +def _master_convert(s, t): + """Master convert with timeout protection""" + s = re.sub(r'~', '', s) + preprocessed_stage1 = first_pre_process(s, t) + preprocessed_stage2 = second_pre_process(preprocessed_stage1) + Sym = latex2sympy(preprocessed_stage2, normalization_config=MyNormalization(), conversion_config=MyConfig()) + return Sym + +def master_convert_with_timeout(s, t): + return run_with_timeout(lambda: _master_convert(s, t), timeout_s=10) + +def master_convert(s,t): + """ + The only function needed to convert a LaTeX string into a SymPy expression. + + Args: + s (str): The input LaTeX string. It should be a valid LaTeX mathematical expression, + such as equations, fractions, or symbols, and must have balanced brackets. + + Returns: + Sym (Sympy Expression): A SymPy expression representing the mathematical content of the input string. + The returned object can be used for symbolic computation, simplification, + or evaluation using SymPy's functionality. + + Example: + >>> master_convert("\\frac{1}{2} + x") + 1/2 + x + """ + try: + return master_convert_with_timeout(s, t) + except SimplifyTimeout: + print(f" -> master_convert timeout for LaTeX: {s[:100]}...") + return None + except Exception as e: + print(f" -> master_convert error: {e}") + return None \ No newline at end of file diff --git a/src/prkit/evaluation/baselines/phybench_eed/LICENSE b/src/prkit/evaluation/baselines/phybench_eed/LICENSE new file mode 100644 index 0000000..a4baaf6 --- /dev/null +++ b/src/prkit/evaluation/baselines/phybench_eed/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 phybench-official + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/prkit/evaluation/baselines/phybench_eed/PROVENANCE.md b/src/prkit/evaluation/baselines/phybench_eed/PROVENANCE.md new file mode 100644 index 0000000..2a79751 --- /dev/null +++ b/src/prkit/evaluation/baselines/phybench_eed/PROVENANCE.md @@ -0,0 +1,38 @@ +# PHYBench EED — vendoring provenance + +- **Upstream:** https://github.com/phybench-official/phybench +- **Path:** `EED/` +- **Commit:** `706feb418ea13f5dec3934dab1ce956208dd73c3` (`706feb4`) +- **License:** MIT (see `LICENSE`, copied verbatim from the upstream repo root) +- **Vendored on:** 2026-06-21 (commit re-verified at vendoring time; `EED/` is + byte-identical between `706feb4` and the upstream `HEAD` at that date) + +## Layout + +``` +phybench_eed/ + core/ + extended_zss.py # pure tree-edit core (numpy/stdlib) — verbatim + eed.py # sympy_to_tree / score_calc / cost funcs / EED() — modified + frontend/ + latex_pre_process.py # master_convert() → latex2sympy2_extended — verbatim + LICENSE # upstream MIT, verbatim + PROVENANCE.md # this file +``` + +## Local modifications + +- **`core/eed.py`** + - Lifted the top-level `from latex_pre_process import *` front-end import so the + pure core imports without `latex2sympy2_extended`. `master_convert` is imported + lazily inside `EED()` from `..frontend.latex_pre_process`. + - Changed `from extended_zss import ext_distance` to the package-relative + `from .extended_zss import ext_distance`. + - Removed `import timeout_decorator` (both occurrences) and replaced the + `@timeout_decorator.timeout(...)` (SIGALRM-based; unsafe under threaded/batch + runners and on Windows) bounds on `simplify_with_timeout` / `equal_with_timeout` + with the thread-safe `prkit.evaluation.edit_distance.timeout.run_with_timeout`. +- **`core/extended_zss.py`**, **`frontend/latex_pre_process.py`** — verbatim apart + from the top-of-file vendoring header comment. + +No `__pycache__`/`*.pyc` are vendored. diff --git a/src/prkit/evaluation/baselines/phybench_eed/__init__.py b/src/prkit/evaluation/baselines/phybench_eed/__init__.py new file mode 100644 index 0000000..b890d7c --- /dev/null +++ b/src/prkit/evaluation/baselines/phybench_eed/__init__.py @@ -0,0 +1,6 @@ +"""Vendored PHYBench Expression Edit Distance (EED) — MIT. + +Upstream: https://github.com/phybench-official/phybench @ ``706feb4`` (``EED/``). +See :doc:`PROVENANCE.md`, ``LICENSE``. Split into the front-end-free +:mod:`.core` (pure ``sympy``/``numpy``) and the :mod:`.frontend` LaTeX pipeline. +""" diff --git a/src/prkit/evaluation/baselines/phybench_eed/core/__init__.py b/src/prkit/evaluation/baselines/phybench_eed/core/__init__.py new file mode 100644 index 0000000..1cf01f2 --- /dev/null +++ b/src/prkit/evaluation/baselines/phybench_eed/core/__init__.py @@ -0,0 +1,5 @@ +"""Front-end-free PHYBench EED core (pure ``sympy``/``numpy``/stdlib). + +Importing this package pulls no ``latex2sympy2_extended``; the LaTeX front-end is +imported lazily by :func:`.eed.EED`. +""" diff --git a/src/prkit/evaluation/baselines/phybench_eed/core/eed.py b/src/prkit/evaluation/baselines/phybench_eed/core/eed.py new file mode 100644 index 0000000..2b7cc48 --- /dev/null +++ b/src/prkit/evaluation/baselines/phybench_eed/core/eed.py @@ -0,0 +1,367 @@ +# Vendored from https://github.com/phybench-official/phybench@706feb4 (EED/EED.py), MIT. +# Local modifications (see ../PROVENANCE.md): +# * lifted the top-level `from latex_pre_process import *` front-end import so the +# pure core imports without `latex2sympy2_extended`; `master_convert` is imported +# lazily inside `EED()`. +# * replaced the SIGALRM `timeout_decorator` bounds with the thread-safe +# `prkit.evaluation.edit_distance.timeout.run_with_timeout`. +from sympy import * +from sympy.core.function import AppliedUndef +from sympy.core.numbers import Pi, Exp1,I,Infinity,NegativeInfinity +import numpy as np +from .extended_zss import ext_distance +from sympy.simplify import * + +from prkit.evaluation.edit_distance.timeout import SimplifyTimeout, run_with_timeout +""" +Guide: +You only need to use EED and install the following packages: +- sympy +- numpy +- latex2sympy2_extended +""" + +""" +There are four main categories: + +Constants: such as integers, decimals, or mathematical constants like π and e. +Variables: letters like x, y, z, or specified terms in problems (e.g., ħ, c, G). +Functions: sine, cosine, exponential, logarithm, etc. +Operators: basic binary operations including addition, multiplication, and exponentiation. +""" +# The costs can be modified if you think their values are different +insert_cost={"number":1,"symbol":1,"operator":1,"function":1} +delete_cost={"number":1,"symbol":1,"operator":1,"function":1} +update_cost={"number":1,"symbol":1,"operator":1,"function":1} + +change_type_cost=1 #the cost of an update between different types,can be set to higher + +bar_size=5 # the minimum size of triggering cluster discount +discount_slope=0.6 #discount + +simplify_time_limit=30 #set the time limit of simplify +equals_time_limit=10 #set the time limit of equals + +def update_func(x,y): + + if x.label==y.label: + return 0 + + elif x.label.split("_")[0]==y.label.split("_")[0]: + return update_cost[x.label.split("_")[0]] + return change_type_cost +def remove_func(x): + return delete_cost[x.label.split("_")[0]] + +def remove_tree_func(x): + if not x.children: + return remove_func(x) + s=calc_tree_size(x) + return min(s,discount_slope*(s-bar_size)+bar_size) + + +def insert_func(x): + return insert_cost[x.label.split("_")[0]] +def insert_tree_func(x): + return remove_tree_func(x) + + + +def calc_tree_size(node): + """ + Calculate the size of a subtree based on its total insertion cost. + The function computes the size of a subtree by summing up the insertion + costs of the current node and all its descendant nodes. If the subtree + size has already been calculated and stored in `node.subtree_size`, it + returns the cached value to avoid redundant computation. + Args: + node (Node): The root node of the subtree for which the size is to + be calculated + Returns: + int: The total size of the subtree, calculated as the sum of the + insertion costs of the current node and all its descendants. + Notes: + - The `insert_cost` dictionary is assumed to be globally defined + and maps node labels to their respective insertion costs. + - The function modifies the `subtree_size` attribute of the input + node to store the calculated subtree size for future use. + """ + """The size of a subtree equals to its total insertion cost""" + + total = insert_cost[node.label.split("_")[0]] + + if node.children and node.subtree_size !=0: + + return node.subtree_size + + for child in node.children: + total += calc_tree_size(child) + + node.subtree_size=total + + return total +""" +Scoring function from relative distance +""" +def score_calc(tree_dist,tree_size): + + if tree_dist==0.: + return 100 + return max(0,100*discount_slope-100*tree_dist/tree_size) + + + + +def simplify_with_timeout(expr): + return run_with_timeout(lambda: simplify(expr), timeout_s=simplify_time_limit) +def time_simplify(expr): + try: + result=simplify_with_timeout(expr) + return result + except SimplifyTimeout: + return expr + +def equal_with_timeout(expr1,expr2): + return run_with_timeout(lambda: expr1.equals(expr2), timeout_s=equals_time_limit) +def time_equal(expr1,expr2): + try: + result=equal_with_timeout(expr1,expr2) + return result + except SimplifyTimeout: + return False + + +def sympy_to_tree(expr): + """ + Convert a SymPy expression into a tree structure. + This function takes a SymPy expression and recursively converts it into a tree + representation using `TreeNode` objects. Each node in the tree is labeled based + on the type of the SymPy expression (e.g., number, symbol, operator, or function), + and its children represent the arguments of the expression. + Args: + expr (sympy.Basic): The SymPy expression to be converted. + Returns: + TreeNode: The root node of the tree representation of the SymPy expression. + Raises: + ValueError: If the SymPy expression contains an unsupported type. + Supported Types: + - Numbers: Integer, Pi, Exp1, Float, Rational, Infinity, NegativeInfinity + - Symbols: Symbol + - Binary Operators: Add, Mul, Pow + - Functions: Any subclass of `sympy.Function` + Example: + >>> from sympy import symbols, sin, pi + >>> x, y = symbols('x y') + >>> expr = x + y * sin(pi) + >>> tree = sympy_to_tree(expr) + >>> print(tree) + """ + #print(expr) + + """Convert the sympy expression to a tree""" + # Symbols and constants + if isinstance(expr, (Integer, Pi, Exp1, Float, Rational, Infinity, NegativeInfinity)): + return TreeNode(label="number_"+str(expr), children=[]) + elif isinstance(expr, (Symbol,)): + + return TreeNode(label="symbol_"+str(expr),children=[]) + + + # Binary operators + elif isinstance(expr, (Add, Mul, Pow)): + + op_name = type(expr).__name__ + children = [sympy_to_tree(arg) for arg in expr.args] + return TreeNode(label="operator_"+op_name, children=children) + + + elif isinstance(expr, (Function)): + # Functions + + func_name = expr.func.__name__ + children = [sympy_to_tree(arg) for arg in expr.args] + return TreeNode(label="function_"+func_name, children=children) + + else: + #print(expr) + print(f"Unsupported Sympy type: {type(expr).__name__}, Expression: {expr}") + raise ValueError(f"Unsupported SymPy type: {type(expr)}") + +class TreeNode: + def __init__(self, label, children=None,node_type='other'): + self.label = label + self.children = children if children is not None else [] + self.node_type=node_type + self.subtree_size=0 + def get_children(self): + return self.children + + def __str__(self): + return self.label + + + + +def print_tree(node, indent=0): + """Print a tree structure""" + print(' ' * indent + f'└─ {node.label}') + for child in node.children: + print_tree(child, indent + 1) + + + +class LaTeXError(Exception): + def __init__(self, message="LaTeXError"): + super().__init__(message) +class SymPyError(Exception): + def __init__(self, message="SymPyError"): + super().__init__(message) + + +class TreeError(Exception): + def __init__(self, message="TreeError"): + super().__init__(message) + + +class DistError(Exception): + def __init__(self, message="DistanceError"): + super().__init__(message) + +def EED(answer_latex,test_latex,debug_mode=False): + """ + Computes the similarity score and distance metrics between two LaTeX expressions. + This function evaluates the equivalence of two mathematical expressions represented + in LaTeX format. It uses symbolic computation and tree-based distance metrics to + calculate a similarity score and other related metrics. + + tuple: A tuple containing the following elements: + - score (float): The similarity score between the two expressions (0 to 100). + - relative_distance (float): The normalized distance between the two expressions. + - answer_tree_size (int): The size of the expression tree for the answer. + - distance (float): The raw distance between the two expression trees. + Notes: + - If either input contains unsupported LaTeX constructs (e.g., integrals or sums), + the function returns default values indicating failure. + - If the test expression is significantly longer than the answer expression, + the function assumes they are not equivalent. + - The function uses symbolic simplification and tree-based distance metrics to + evaluate equivalence. + - In case of errors during processing, the function returns default values unless + `debug_mode` is enabled, in which case it raises specific exceptions. + Exceptions: + - LaTeXError: Raised when LaTeX conversion to symbolic expressions fails (if `debug_mode` is True). + - SymPyError: Raised when symbolic simplification or tree construction fails (if `debug_mode` is True). + - DistError: Raised when distance calculation fails (if `debug_mode` is True). + Args: + answer_latex: the latex expression of answer expression + test_latex: the latex expression of test expression + debug_mode: whether it raise errors or just skip it + Returns: + tuple: A tuple containing the following elements: + - score (float): The similarity score between the two expressions (0 to 100). + - relative_distance (float): The normalized distance between the two expressions. + - answer_tree_size (int): The size of the expression tree for the answer. + - distance (float): The raw distance between the two expression trees. + """ + + if not test_latex: + return 0,-1,-1,-1 + if '\\int' in test_latex or '\\int' in answer_latex: + return 0,-1,-1,-1 + if '\\sum' in test_latex or '\\sum' in answer_latex: + return 0,-1,-1,1 + if answer_latex==test_latex: + return 100,0.0,-1,0 + if len(test_latex)>3*len(answer_latex): + return 0,-1,-1,-1 + + # Front-end is loaded lazily so importing this pure core stays free of + # latex2sympy2_extended (see the module-header note). + from ..frontend.latex_pre_process import master_convert + + try: + + answer_exp=master_convert(answer_latex) + test_exp=master_convert(test_latex) + except: + print(f"Failed to convert input latex to sympy expression,please check it") + if debug_mode: + raise LaTeXError(f"Fail to convert latex.\n GT:{answer_latex}\n GEN:{test_latex}") + return 0,-1,-1,-1 + + try: + + answer_exp,rep1=posify(answer_exp) + + answer_exp=time_simplify(answer_exp) + + + test_exp,rep2=posify(test_exp) + test_exp=time_simplify(test_exp) + + + + answer_exp=answer_exp.subs(rep1) + test_exp=test_exp.subs(rep2) + + zero_exp=time_simplify(expand(answer_exp-test_exp)) + + + if answer_exp==test_exp or zero_exp==0: + return 100,0.,0,0 + + if time_equal(answer_exp,test_exp): + return 100,0.,0,0 + + except: + print("Something happened during simplification,returning zero") + if debug_mode: + raise SymPyError(f"Failed to simplify the sympy expression. Expressions: answer_exp={answer_exp}, test_exp={test_exp}") + return 0,-1,-1,-1 + + try: + tree_answer=sympy_to_tree(answer_exp) + tree_test=sympy_to_tree(test_exp) + + except: + + print("Failed to build expression tree,returning zero") + if debug_mode: + raise SymPyError(f"Failed to build the sympy expression tree.\n GT:{answer_exp}\n GEN:{test_exp}") + return 0,-1,-1,-1 + + distance=ext_distance( + tree_test, + tree_answer, + get_children=lambda x:x.get_children(), + single_insert_cost=insert_func, + insert_cost=insert_tree_func, + single_remove_cost=remove_func, + remove_cost=remove_tree_func, + update_cost=update_func) + try: + + + distance=ext_distance( + tree_test, + tree_answer, + get_children=lambda x:x.get_children(), + single_insert_cost=insert_func, + insert_cost=insert_tree_func, + single_remove_cost=remove_func, + remove_cost=remove_tree_func, + update_cost=update_func + ) + except: + print("Failed to calculate distance") + if debug_mode: + raise DistError(f"Failed to calculate the distance between trees.\n GT:{answer_latex}\n GEN:{test_latex}") + return 0,-1,calc_tree_size(tree_answer),-1 + tree_size=calc_tree_size(tree_answer) + distance_number=distance + + rel_distance=distance/tree_size + + score=score_calc(distance_number,tree_size) + + return score,rel_distance,tree_size,distance_number diff --git a/src/prkit/evaluation/baselines/phybench_eed/core/extended_zss.py b/src/prkit/evaluation/baselines/phybench_eed/core/extended_zss.py new file mode 100644 index 0000000..306cf85 --- /dev/null +++ b/src/prkit/evaluation/baselines/phybench_eed/core/extended_zss.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Vendored from https://github.com/phybench-official/phybench@706feb4 (EED/extended_zss.py), +# MIT. Local modification: none (verbatim). See ../LICENSE and ../PROVENANCE.md. +#Original Authors: Tim Henderson and Steve Johnson +#Email: tim.tadh@gmail.com, steve@steveasleep.com +#For licensing see the LICENSE file in the top level directory. + +# This is a modified version of zss package. + + +import collections +import numpy as np +from numpy import zeros,ones + +class Node(object): + + + def __init__(self, label, children=None): + self.label = label + self.children = children or list() + + + @staticmethod + def get_children(node): + return node.children + + @staticmethod + def get_label(node): + return node.label + + def addkid(self, node, before=False): + + if before: self.children.insert(0, node) + else: self.children.append(node) + return self + + def get(self, label): + + if self.label == label: return self + for c in self.children: + if label in c: return c.get(label) + + + + + + +class AnnotatedTree(object): + + def __init__(self, root, get_children): + self.get_children = get_children + + self.root = root + self.nodes = list() # a post-order enumeration of the nodes in the tree + self.ids = list() # a matching list of ids + self.lmds = list() # left most descendents of each nodes + self.keyroots = None + # the keyroots in the original paper + + + stack = list() + pstack = list() + stack.append((root, collections.deque())) + j = 0 + while len(stack) > 0: + n, anc = stack.pop() + nid = j + for c in self.get_children(n): + a = collections.deque(anc) + a.appendleft(nid) + stack.append((c, a)) + pstack.append(((n, nid), anc)) + j += 1 + lmds = dict() + keyroots = dict() + i = 0 + while len(pstack) > 0: + (n, nid), anc = pstack.pop() + self.nodes.append(n) + self.ids.append(nid) + if not self.get_children(n): + lmd = i + for a in anc: + if a not in lmds: lmds[a] = i + else: break + else: + try: lmd = lmds[nid] + except: + import pdb + pdb.set_trace() + self.lmds.append(lmd) + keyroots[lmd] = i + i += 1 + self.keyroots = sorted(keyroots.values()) + + +def ext_distance(A, B, get_children, single_insert_cost,insert_cost,single_remove_cost, remove_cost, update_cost): + '''Computes the extended tree edit distance between trees A and B with extended-zss algorithm + Args: + A(Node): Root node of tree 1 + B(Node): Root node of tree 2 + get_children(Func): the get_children method of tree + single_insert_cost(Func): cost of inserting single node + insert_cost(Func): cost of inserting a subtree + update_cost(Func): cost of updating A to B + + + Return: + Distance(float):the tree editing distance + ''' + A, B = AnnotatedTree(A, get_children), AnnotatedTree(B, get_children) + size_a = len(A.nodes) + size_b = len(B.nodes) + treedists = zeros((size_a, size_b), float) + fd=1000*ones((size_a+1,size_b+1),float) + operations = [[[] for _ in range(size_b)] for _ in range(size_a)] + + + def treedist(x, y): + Al = A.lmds + Bl = B.lmds + An = A.nodes + Bn = B.nodes + + m = size_a + n = size_b + + fd[Al[x]][Bl[y]]=0 + for i in range(Al[x], x+1): + node = An[i] + fd[i+1][Bl[y]] = fd[Al[i]][Bl[y]] + remove_cost(node) + + for j in range(Bl[y], y+1): + node = Bn[j] + + fd[Al[x]][j+1] = fd[Al[x]][Bl[j]] + insert_cost(node) + + for i in range(Al[x], x+1): + for j in range(Bl[y], y+1): + + node1 = An[i] + node2 = Bn[j] + costs = [fd[i][j+1] + single_remove_cost(node1), + fd[i+1][j] + single_insert_cost(node2), + fd[Al[i]][j+1]+ remove_cost(node1), + fd[i+1][Bl[j]]+ insert_cost(node2)] + m=min(costs) + + if Al[x] == Al[i] and Bl[y] == Bl[j]: + treedists[i][j]=min(m,fd[i][j]+update_cost(node1,node2)) + fd[i+1][j+1]=treedists[i][j] + else: + fd[i+1][j+1]=min(m,fd[Al[i]][Bl[j]]+treedists[i][j]) + + + for x in A.keyroots: + for y in B.keyroots: + treedist(x, y) + + return treedists[-1][-1] + diff --git a/src/prkit/evaluation/baselines/phybench_eed/frontend/__init__.py b/src/prkit/evaluation/baselines/phybench_eed/frontend/__init__.py new file mode 100644 index 0000000..c8f0226 --- /dev/null +++ b/src/prkit/evaluation/baselines/phybench_eed/frontend/__init__.py @@ -0,0 +1,5 @@ +"""PHYBench EED LaTeX front-end (``latex2sympy2_extended``). + +Imported lazily by :func:`..core.eed.EED`; importing it pulls +``latex2sympy2_extended`` + its ``antlr4`` runtime. +""" diff --git a/src/prkit/evaluation/baselines/phybench_eed/frontend/latex_pre_process.py b/src/prkit/evaluation/baselines/phybench_eed/frontend/latex_pre_process.py new file mode 100644 index 0000000..c7d4a54 --- /dev/null +++ b/src/prkit/evaluation/baselines/phybench_eed/frontend/latex_pre_process.py @@ -0,0 +1,526 @@ +# Vendored from https://github.com/phybench-official/phybench@706feb4 +# (EED/latex_pre_process.py), MIT. Local modification: none (verbatim). +# See ../LICENSE and ../PROVENANCE.md. +#This file is used to pre-process input latex expressions +#You only need a "master_convert()" +from latex2sympy2_extended import * +from sympy import simplify + + + +def brackets_balanced(s: str) -> bool: + """ + Check if the brackets in a LaTeX string are balanced + Args: + s(str): the input string + Return: + bool: True if the brackets are balanced, False otherwise + """ + stack = [] + bracket_pairs = {')': '(', ']': '[', '}': '{'} + + for char in s: + if char in bracket_pairs.values(): + stack.append(char) + elif char in bracket_pairs: + if not stack or stack[-1] != bracket_pairs[char]: + return False + stack.pop() + return len(stack) == 0 + + + +def remove_non_ascii(text): + return text.encode("ascii", errors="ignore").decode() + +import re +def extract_bracket_content(s:str,bracket_position:int) -> str: + start_idx=bracket_position + + stack = [] + content = [] + escaped = False + brace_start=start_idx+1 + brace_depth = 0 + for i in range(brace_start, len(s)): + char = s[i] + if escaped: + content.append(char) + escaped = False + continue + if char == '\\': + escaped = True + content.append(char) + continue + if char == '{': + brace_depth += 1 + content.append(char) + elif char == '}': + if brace_depth == 0: + return ''.join(content),i + brace_depth -= 1 + content.append(char) + else: + content.append(char) + + return None,-1 +def find_first_unescaped_brace(s: str) -> int: + escaped = False + for i, c in enumerate(s): + if c == '\\' and not escaped: + escaped = True + continue + if c == '{' and not escaped: + return i + escaped = False + return -1 + +def extract_command(s: str, brace_pos: int) -> str | None: + """extract the command name from a bracket""" + i = brace_pos - 1 + parameter_mode=False + while i >= 0: + if not parameter_mode and s[i] in ('^','_'): + return s[i] + if not parameter_mode and not s[i] in (' ','\t',']','['): + break + if s[i]==']': + parameter_mode=True + if s[i]=='[' and parameter_mode: + parameter_mode=False + i -= 1 + + # Start point + if i < 0 or s[i] == '\\': + return None + + # Extract command name + command_end = i + i -= 1 + while i >= 0 and s[i].isalpha(): + i -= 1 + if i<-1 or s[i]!='\\': + return None + return s[i+1:command_end+1] + + +def remove_command(s,command,keep_inside=False): + def remove_command(s, command, keep_inside=False): + """ + Removes all occurrences of a specified LaTeX-style command from a string. + + This function searches for a given command in the input string `s` and removes it, + along with its associated content enclosed in curly braces `{}`. If `keep_inside` + is set to `True`, the content inside the braces is preserved, and only the command + itself is removed. The function handles nested braces correctly. + + Args: + s (str): The input string from which the command should be removed. + command (str): The LaTeX-style command to be removed (e.g., "\\textbf"). + keep_inside (bool, optional): If `True`, preserves the content inside the braces + while removing the command. Defaults to `False`. + + Returns: + str: The modified string with the specified command removed. + + Examples: + >>> remove_command("This is \\textbf{bold text}.", "\\textbf") + 'This is bold text.' + + >>> remove_command("This is \\textbf{bold text}.", "\\textbf", keep_inside=True) + 'This is bold text.' + + >>> remove_command("Nested \\textbf{bold \\textit{italic text}} example.", "\\textbf") + 'Nested bold \\textit{italic text} example.' + """ + pos=s.find(command) + if pos<0: + return s + end_index=pos+len(command) + level=0 + escaped=False + #print(end_index,s[end_index]) + if end_index < len(s) and s[end_index] == "{": + while end_index str | None: + """ Find the first brace """ + brace_pos = find_first_unescaped_brace(s) + if brace_pos == -1: + return None + return extract_command(s, brace_pos) +def remove_overall_brace(s:str) -> str: + """ + Remove the overall {xxx} brace + """ + pos=find_first_unescaped_brace(s) + if pos==-1: + return s,0 + command=get_first_brace_command(s) + if not command: + + content,final=extract_bracket_content(s,pos) + #print(s[final]) + if final==len(s) or not '}' in s[final+1:]: + return content,1 + return s,0 + + +def exp_frac(s): + + def exp_frac_single(s): + position=s.find("^\\frac")+1 + if position == 0: + return s + level=0 + cnt=0 + idx=position + while idx>> convert_vec_syntax(r"\vec x + \vec\alpha + \vec\Gamma") + '\\vec{x} + \\vec{\\alpha} + \\vec{\\Gamma}' + """ + + pattern = r'\\vec(\s*)(\\?[a-zA-Zα-ωΑ-Ω]+)' + replacement = r'\\vec{\2}' + return re.sub(pattern, replacement, text) + +def remove_outer_braces(tex_str): + """ + convert {base}_{subscript} to base_{subscript} + Example： + {a}_{xyz} → a_{xyz} + {\theta}_{0} → \theta_{0} + """ + pattern = r'\{(\\(?:[a-zA-Z]+|.)|[^{}])+\}_\{([^}]+)\}' + return re.sub(pattern, r'\1_{\2}', tex_str) + +def extract_last_equal_content(s: str, strip_whitespace: bool = True) -> str: + """ + Extract the content after the last occurrence of specific mathematical comparison or assignment operators. + + :param strip_whitespace: If True, removes leading and trailing whitespace from the extracted content. Defaults to True. + (e.g., '=', '\\approx', '\\ge', '\\le', etc.) within the input string `s`. It then extracts + and returns the content that follows the operator. If no operator is found, the entire string + is returned. Optionally, leading and trailing whitespace can be stripped from the extracted content. + + Args: + s (str): The input string to process. + strip_whitespace (bool): Whether to strip leading and trailing whitespace from the extracted content. Defaults to True. + + Returns: + str: The content after the last matching operator, or the entire string if no operator is found. + """ + comparison_operators=('=','\\approx','\\ge','\\le','\\geq','\\leq','<','>') + + content=s + for sign in comparison_operators: + if sign in s: + rfind_index = s.rfind(sign) + if rfind_index != -1: + content = s[rfind_index + 1:] + if strip_whitespace: + return content.strip() + return content + + +def first_pre_process(s,extrac_box=True): + """ + Perform the first stage of LaTeX string preprocessing. + + if not brackets_balanced(s): + raise ValueError("The input string has unbalanced brackets. Please check the LaTeX expression.") + equality or comparison operator. + + Args: + s (str): The input LaTeX string to preprocess. + extrac_box (bool): If True, extracts the content inside a '\\boxed' command. Defaults to True. + + Returns: + str: The preprocessed LaTeX string. + """ + #s=remove_non_ascii(s) + s=s.replace('\\{','(') + s=s.replace('\\}',')') + if not brackets_balanced(s): + return s + if extrac_box: + boxed_content=remove_command(s,'\\boxed',keep_inside=True) + else: + boxed_content=s + exist_overall_brace=True + cnt=0 + while exist_overall_brace and cnt<10: + boxed_content,exist_overall_brace=remove_overall_brace(boxed_content) + cnt+=1 + + if '\\quad' in boxed_content: + boxed_content = boxed_content.split('\\quad')[0] + + last_equal_content=extract_last_equal_content(boxed_content) + + exist_overall_brace=True + cnt=0 + while exist_overall_brace and cnt<10: + last_equal_content,exist_overall_brace=remove_overall_brace(last_equal_content) + cnt+=1 + return last_equal_content +def second_pre_process(s): + """ + Perform the second stage of LaTeX string preprocessing. + + This function removes or modifies specific LaTeX commands and content to standardize + the input string for further processing. It handles commands like '\\text', '\\mathbf', + and '\\mathrm', removes unnecessary content, and applies transformations such as + converting fractions and vector syntax. + + Args: + s (str): The input LaTeX string to preprocess. + + Returns: + str: The preprocessed LaTeX string. + """ + + + kill_commands=[ + '\\begin', + '\\end' + ] + remove_commands=[ + '\\text', + '\\mathbf', + '\\mathrm', + '\\pmb', + '\\hat', + '\\overline', + '\\boldsymbol', + ] + + + remove_content=[ + '\\,','$',',','`','latex','\\left','\\right','\\text','\\mathrm','\\Bigr','\\Bigl','\n','\\]','\\[', + '\\Big','\\bigl','\\bigr','\\biggl','\\biggr','\\displaystyle','\\boldsymbol','\\infty' + ] + replace_content=[ + ('\\operatorname{asin}','\\asin'), + ('\\operatorname{sech}','\\sech'), + ('\\operatorname{acos}','\\acos'), + ('\\operatorname{sinh}','\\sinh'), + ('\\dfrac','\\frac'), + ('\\tfrac','\\frac'), + ('\\Exp','\\exp'), + ('\\times','\\bar{times}'), + ('\\partial','\\bar{partial}'), + ('\\perp','\\bar{perp}'), + ('\\epsilon','\\varepsilon'), + ('\\varOmega','\\Omega'), + ('I','\\bar{I}'), + ('_e','_{e}'), + ('e_','\\bar{e}_'), + ('E_','\\bar{E}_'), + ('\\pm','+'), + ('\\mp','-'), + ('{+}','{p}'), + ("{-}",'{m}'), + ("_+",'_p'), + ('_-',"_m") + ] + for command in kill_commands: + s=remove_command(s,command,keep_inside=False) + for command in remove_commands: + s=remove_command(s,command,keep_inside=True) + for content in remove_content: + s=s.replace(content,'') + for content in replace_content: + s=s.replace(content[0],content[1]) + s=convert_latex_fractions(s) + #print(s) + s=bar_inside_vec(s) + s=vec_lower_idx(s) + s=convert_vec_syntax(s) + s=exp_frac(s) + #s=remove_outer_braces(s) + if s and s[-1] == '.': + return s[:-1] + return s + + +class MyConfig: + + interpret_as_mixed_fractions: bool = False + interpret_simple_eq_as_assignment: bool = False + interpret_contains_as_eq: bool = True + lowercase_symbols: bool = False + """ + Args: + interpret_as_mixed_fractions (bool): Whether to interpert 2 \frac{1}{2} as 2/2 or 2 + 1/2 + interpret_simple_eq_as_assignment (bool): Whether to interpret simple equations as assignments k=1 -> 1 + interpret_contains_as_eq (bool): Whether to interpret contains as equality x \\in {1,2,3} -> x = {1,2,3} + lowercase_symbols (bool): Whether to lowercase all symbols + """ +class MyNormalization: + """Configuration for latex normalization. + + Each field controls a group of related normalizations: + - basic_latex: Basic latex command replacements (mathrm, displaystyle, etc.) + - units: Remove units and their variations + - malformed_operators: Fix malformed operators (sqrt, frac, etc.) + - nits: Small formatting fixes (spaces, dots, etc.) + - boxed: Extract content from boxed environments + - equations: Handle equation splitting and approximations (deprecated) + """ + basic_latex: bool = True + units: bool = False + malformed_operators: bool = True + nits: bool = True + boxed = "all" + equations: bool = False + +def master_convert(s): + """ + The only function needed to convert a LaTeX string into a SymPy expression. + + Args: + s (str): The input LaTeX string. It should be a valid LaTeX mathematical expression, + such as equations, fractions, or symbols, and must have balanced brackets. + + Returns: + Sym (Sympy Expression): A SymPy expression representing the mathematical content of the input string. + The returned object can be used for symbolic computation, simplification, + or evaluation using SymPy's functionality. + + Example: + >>> master_convert("\\frac{1}{2} + x") + 1/2 + x + """ + preprocessed_stage1=first_pre_process(s) + + preprocessed_stage2=second_pre_process(preprocessed_stage1) + + Sym=latex2sympy(preprocessed_stage2,normalization_config=MyNormalization(),conversion_config=MyConfig()) + return Sym diff --git a/src/prkit/scoring/__init__.py b/src/prkit/scoring/__init__.py index 2d6dc06..133a609 100644 --- a/src/prkit/scoring/__init__.py +++ b/src/prkit/scoring/__init__.py @@ -1,24 +1,31 @@ """Reference scoring implementations for PRKit's ``Scorer`` contract. ``SemanticsScorer`` is the canonical, version-stamped scorer wrapping the -deterministic (binary) semantics comparison engine. ``PartialCreditScorer`` is its -graded counterpart: an EED/SEED edit-distance scorer that populates +deterministic (binary) semantics comparison engine. ``EedScorer`` / ``SeedScorer`` +are the faithful PHYBench-EED / CMPhysBench-SEED edit-distance *baselines* (vendor +LaTeX front-end + the front-end-free pure core). ``PartialCreditScorer`` is the +graded EED/SEED scorer over PRKit's own semantics front-end that populates ``Verdict.partial_credit``. ``LLMJudgeScorer`` is the model-graded scorer wrapping the ``prkit.evaluation.llm_judge`` engine. All structurally satisfy :class:`prkit.api.Scorer` and emit :class:`prkit.api.Verdict`. -Import discipline: re-exporting ``LLMJudgeScorer`` here must not pull ``openai`` or -``prkit.evaluation.llm_judge`` onto ``import prkit.scoring`` — its judge imports are -deferred to method bodies (see ``llm_judge_scorer``). +Import discipline: re-exporting these scorers here must not pull ``openai``, +``prkit.evaluation.llm_judge``, ``pint``, or the vendored LaTeX front-end onto +``import prkit.scoring`` — the judge, vendored-core, and front-end imports are all +deferred to method bodies (see ``llm_judge_scorer`` / ``eed_scorer`` / ``seed_scorer``). """ +from .eed_scorer import EedScorer from .llm_judge_scorer import LLMJudgeScorer from .partial_credit_scorer import PartialCreditMode, PartialCreditScorer +from .seed_scorer import SeedScorer from .semantics_scorer import SemanticsScorer __all__ = [ + "EedScorer", "LLMJudgeScorer", "PartialCreditMode", "PartialCreditScorer", + "SeedScorer", "SemanticsScorer", ] diff --git a/src/prkit/scoring/eed_scorer.py b/src/prkit/scoring/eed_scorer.py new file mode 100644 index 0000000..6aa3186 --- /dev/null +++ b/src/prkit/scoring/eed_scorer.py @@ -0,0 +1,129 @@ +"""Faithful PHYBench EED baseline :class:`prkit.api.Scorer` (vendor front-end). + +``EedScorer`` is the *baseline* edit-distance scorer: it runs the vendored PHYBench +Expression Edit Distance pipeline end-to-end on raw answer strings — the upstream +LaTeX front-end (``latex2sympy2_extended``) feeding the front-end-free pure core +(:mod:`prkit.evaluation.baselines.phybench_eed`). It exists so a PRKit number is +provably comparable to the published PHYBench metric; the *our-semantics* counterpart +is :class:`prkit.scoring.SemanticsEedScorer`. + +Import discipline: the vendored core + LaTeX front-end are imported lazily inside +:meth:`score`, so re-exporting this scorer from :mod:`prkit.scoring` keeps +``import prkit.scoring`` free of the front-end import side effects. + +PRKit policy (not upstream-faithful): upstream EED has no non-expression path — +every answer is treated as an expression (parse + tree-diff), scoring ``0`` when the +input is unparseable. ``EedScorer`` preserves that behavior; it takes no +``answer_type``. Use :class:`prkit.scoring.SeedScorer` for typed answers. +""" + +from __future__ import annotations + +from typing import Any + +from prkit.core.domain.answer import Answer +from prkit.core.verdict import Verdict + +#: Provenance stamp: ``/@+frontend-+wrap``. +_VERSION = "eed/phybench@706feb4+frontend-vendor+wrap1" + + +def _as_text(value: Answer | str | Any) -> str: + """Render an ``Answer``/string answer as the raw text the front-end expects. + + ``Answer.__str__`` appends the unit when present (``"3 m/s"``), which is exactly + the surface the vendored LaTeX front-end parses. + """ + if isinstance(value, str): + return value + return str(value) + + +class EedScorer: + """Faithful PHYBench EED baseline; vendor LaTeX front-end + pure core. + + Args: + tolerance: Recorded for provenance/``get_info()`` only. The vendored EED + scoring tiers are fixed upstream constants and are **not** overridden, + so a baseline number stays faithful to the published metric. + config: Reserved opaque passthrough for future vendor-core tunables. + """ + + version: str = _VERSION + + def __init__( + self, + *, + tolerance: float | None = None, + config: Any | None = None, + ) -> None: + self._tolerance = None if tolerance is None else float(tolerance) + self._config = config + + def score( + self, + prediction: Answer | str, + reference: Answer | str, + **kwargs: Any, + ) -> Verdict: + """Score ``prediction`` against ``reference`` with the vendored EED pipeline. + + Both inputs may be raw strings or :class:`Answer` objects. The reference is + the EED ``answer`` (ground truth) and the prediction is the EED ``test``. + """ + # Lazy import keeps latex2sympy2_extended off `import prkit.scoring`. + from prkit.evaluation.baselines.phybench_eed.core.eed import EED + + pred_text = _as_text(prediction) + ref_text = _as_text(reference) + + raw, rel_distance, tree_size, distance = EED(ref_text, pred_text) + return _eed_verdict(self.version, raw, rel_distance, tree_size, distance) + + def get_info(self) -> dict[str, Any]: + """Return scorer metadata; always includes ``version``.""" + return { + "name": "EedScorer", + "version": self.version, + "engine": "phybench_eed", + "deterministic": True, + "front_end": "vendor", + "tolerance": self._tolerance, + } + + +def _eed_verdict( + version: str, + raw: Any, + rel_distance: Any, + tree_size: Any, + distance: Any, +) -> Verdict: + """Map a raw EED ``(score, rel_dist, tree_size, distance)`` tuple onto a Verdict. + + The raw score is ``0..100`` (``100`` exact, else ``max(0, 60 − 100·dist/size)``); + it is normalized to ``[0, 1]`` for :attr:`Verdict.score`, and ``equivalent`` is + reserved for an exact (raw ``100``) match. ``relative_distance``/``tree_size``/ + ``distance`` carry the algorithm's ``-1`` "not computed" markers verbatim in + ``details`` (these are EED-internal, distinct from any Verdict sentinel). + """ + raw_score = float(raw) + normalized = min(1.0, max(0.0, raw_score / 100.0)) + equivalent = raw_score >= 100.0 + return Verdict( + equivalent=equivalent, + score=normalized, + comparison_mode="eed", + scorer_version=version, + details={ + "front_end": "vendor", + "raw_score": raw_score, + "relative_distance": float(rel_distance), + "tree_size": float(tree_size), + "distance": float(distance), + }, + partial_credit=normalized, + ) + + +__all__ = ["EedScorer"] diff --git a/src/prkit/scoring/seed_scorer.py b/src/prkit/scoring/seed_scorer.py new file mode 100644 index 0000000..be276f3 --- /dev/null +++ b/src/prkit/scoring/seed_scorer.py @@ -0,0 +1,284 @@ +"""Faithful CMPhysBench SEED baseline :class:`prkit.api.Scorer` (vendor front-end). + +``SeedScorer`` is the *baseline* multi-type edit-distance scorer: it runs the vendored +CMPhysBench Scalable Expression Edit Distance pipeline end-to-end on raw answer strings +— the upstream LaTeX/``pint`` front-end feeding the front-end-free pure core +(:mod:`prkit.evaluation.baselines.cmphysbench_seed`). It exists so a PRKit number is +provably comparable to the published CMPhysBench metric; the *our-semantics* +counterpart is :class:`prkit.scoring.SemanticsSeedScorer`. + +SEED dispatches on a per-item ``answer_type`` ∈ ``{Expression, Equation, Tuple, +Interval, Numeric}`` — upstream a curated dataset annotation, never inferred. +``SeedScorer`` resolves it from (in order) the ``answer_type=`` kwarg, the +``reference.source_type`` dataset label (validated against the SEED enum), then the +``default_answer_type`` (``"Expression"``, upstream-faithful). Only when +``enable_classifier=True`` (off by default) does it fall back to PRKit's own opt-in +classifier — so a default-config number is provably CMPhysBench-faithful, and +``get_info()`` records whether classification was used. + +Import discipline: the vendored core + front-end are imported lazily inside +:meth:`score`, so re-exporting this scorer from :mod:`prkit.scoring` keeps +``import prkit.scoring`` free of ``pint``/the LaTeX front-end. +""" + +from __future__ import annotations + +import re +from typing import Any + +from prkit.core.domain.answer import Answer +from prkit.core.verdict import Verdict + +#: Provenance stamp: ``/@+frontend-+wrap``. +_VERSION = "seed/cmphysbench@b2cd857+frontend-vendor+wrap1" + +#: The five SEED dispatch tokens (upstream ``answer_type`` enum). +SEED_ANSWER_TYPES: tuple[str, ...] = ( + "Expression", + "Equation", + "Tuple", + "Interval", + "Numeric", +) +_SEED_TYPE_SET = frozenset(SEED_ANSWER_TYPES) + + +def _as_text(value: Answer | str | Any) -> str: + """Render an ``Answer``/string answer as the raw text the front-end expects.""" + if isinstance(value, str): + return value + return str(value) + + +class SeedScorer: + """Faithful CMPhysBench SEED baseline; vendor front-end + pure core. + + Args: + tolerance: Recorded for provenance/``get_info()`` only. The vendored SEED + scoring tiers are fixed upstream constants and are **not** overridden, + so a baseline number stays faithful to the published metric. + default_answer_type: SEED token used when neither the kwarg, the reference + label, nor (if enabled) the classifier resolves a type. Defaults to the + upstream-faithful ``"Expression"``. + enable_classifier: When ``True``, unlabeled pairs are classified by PRKit's + own opt-in heuristic (returns ``"Expression"`` on ambiguity). Off by + default so a baseline number is provably CMPhysBench-faithful. + config: Reserved opaque passthrough for future vendor-core tunables. + """ + + version: str = _VERSION + + def __init__( + self, + *, + tolerance: float | None = None, + default_answer_type: str = "Expression", + enable_classifier: bool = False, + config: Any | None = None, + ) -> None: + if default_answer_type not in _SEED_TYPE_SET: + raise ValueError( + f"default_answer_type must be one of {SEED_ANSWER_TYPES}, " + f"got {default_answer_type!r}" + ) + self._tolerance = None if tolerance is None else float(tolerance) + self._default_answer_type = default_answer_type + self._enable_classifier = bool(enable_classifier) + self._config = config + #: Whether the opt-in classifier was used for the most recent dispatch. + self._classifier_used = False + + def score( + self, + prediction: Answer | str, + reference: Answer | str, + *, + answer_type: str | None = None, + **kwargs: Any, + ) -> Verdict: + """Score ``prediction`` against ``reference`` with the vendored SEED pipeline. + + ``answer_type`` (when a valid SEED token) overrides the resolved type; + otherwise the type is taken from ``reference.source_type`` (validated), then + the classifier (if enabled), then ``default_answer_type``. + """ + # Lazy import keeps pint + the LaTeX front-end off `import prkit.scoring`. + from prkit.evaluation.baselines.cmphysbench_seed.core.seed import SEED + + pred_text = _as_text(prediction) + ref_text = _as_text(reference) + + resolved, classifier_used = self._resolve_answer_type( + answer_type, reference, ref_text + ) + self._classifier_used = classifier_used + + raw, rel_distance, tree_size, distance = SEED(ref_text, pred_text, resolved) + return _seed_verdict( + self.version, + resolved, + classifier_used, + raw, + rel_distance, + tree_size, + distance, + ) + + def _resolve_answer_type( + self, + answer_type: str | None, + reference: Answer | str, + ref_text: str, + ) -> tuple[str, bool]: + """Resolve the SEED dispatch token; return ``(token, classifier_used)``. + + Order: a valid ``answer_type`` kwarg → a valid ``reference.source_type`` → + (if ``enable_classifier``) the classifier → ``default_answer_type``. Any + unrecognized value is ignored rather than passed to the SEED core. + """ + if answer_type is not None and answer_type in _SEED_TYPE_SET: + return answer_type, False + + source_type = getattr(reference, "source_type", None) + if isinstance(source_type, str) and source_type in _SEED_TYPE_SET: + return source_type, False + + if self._enable_classifier: + return self._classify_answer_type(ref_text), True + + return self._default_answer_type, False + + @staticmethod + def _classify_answer_type(text: str) -> str: + """PRKit's opt-in answer-type triage (returns ``"Expression"`` on ambiguity). + + Borrows the *shape* of SEED's own ``judge_interval``/``extract_tuple`` cues: + a top-level ``=`` → ``"Equation"``; a bracketed comma-list of 3+ elements → + ``"Tuple"`` (a 2-element bracket is Tuple/Interval-ambiguous → ``"Expression"``); + a bare number (optionally with a unit) → ``"Numeric"``; else ``"Expression"``. + """ + s = _strip_math_wrappers(text) + if _has_top_level_equals(s): + return "Equation" + items = _top_level_bracket_items(s) + if items is not None and len(items) >= 3: + return "Tuple" + if _looks_numeric(s): + return "Numeric" + return "Expression" + + def get_info(self) -> dict[str, Any]: + """Return scorer metadata; always includes ``version``.""" + return { + "name": "SeedScorer", + "version": self.version, + "engine": "cmphysbench_seed", + "deterministic": True, + "front_end": "vendor", + "default_answer_type": self._default_answer_type, + "enable_classifier": self._enable_classifier, + "classifier_used": self._classifier_used, + "tolerance": self._tolerance, + } + + +def _seed_verdict( + version: str, + answer_type: str, + classifier_used: bool, + raw: Any, + rel_distance: Any, + tree_size: Any, + distance: Any, +) -> Verdict: + """Map a raw SEED ``(score, rel_dist, tree_size, distance)`` tuple onto a Verdict. + + The raw score is ``0..100`` (``100`` exact / within the tightest numeric tier); + it is normalized to ``[0, 1]`` for :attr:`Verdict.score`, and ``equivalent`` is + reserved for a raw ``100``. The SEED-internal ``-1`` "not computed" markers are + carried verbatim in ``details`` (distinct from any Verdict sentinel). + """ + raw_score = float(raw) + normalized = min(1.0, max(0.0, raw_score / 100.0)) + equivalent = raw_score >= 100.0 + return Verdict( + equivalent=equivalent, + score=normalized, + comparison_mode=f"seed:{answer_type}", + scorer_version=version, + details={ + "front_end": "vendor", + "answer_type": answer_type, + "classifier_used": classifier_used, + "raw_score": raw_score, + "relative_distance": float(rel_distance), + "tree_size": float(tree_size), + "distance": float(distance), + }, + partial_credit=normalized, + ) + + +def _strip_math_wrappers(text: str) -> str: + """Strip a single layer of ``$$…$$``/``$…$`` math delimiters and whitespace.""" + s = text.strip() + if s.startswith("$$") and s.endswith("$$"): + return s[2:-2].strip() + if s.startswith("$") and s.endswith("$"): + return s[1:-1].strip() + return s + + +def _has_top_level_equals(s: str) -> bool: + """Return ``True`` when ``s`` has a ``=`` at bracket depth 0 (an equation).""" + depth = 0 + for ch in s: + if ch in "([{": + depth += 1 + elif ch in ")]}": + depth = max(0, depth - 1) + elif ch == "=" and depth == 0: + return True + return False + + +def _top_level_bracket_items(s: str) -> list[str] | None: + """If ``s`` is wrapped in one matching bracket pair, split its top-level commas. + + Returns the list of comma-separated items (depth-1) for a ``(...)``/``[...]`` + wrapper, or ``None`` when ``s`` is not a single bracketed group. + """ + s = s.strip().replace("\\left", "").replace("\\right", "").strip() + if len(s) < 2 or s[0] not in "([" or s[-1] not in ")]": + return None + inner = s[1:-1] + items: list[str] = [] + depth = 0 + current: list[str] = [] + for ch in inner: + if ch in "([{": + depth += 1 + current.append(ch) + elif ch in ")]}": + depth -= 1 + current.append(ch) + elif ch == "," and depth == 0: + items.append("".join(current).strip()) + current = [] + else: + current.append(ch) + items.append("".join(current).strip()) + return items + + +_NUMERIC_RE = re.compile( + r"^[-+]?(?:\d+\.?\d*|\.\d+)(?:[eE][-+]?\d+|\s*\\times\s*10\^\{?[-+]?\d+\}?)?" +) + + +def _looks_numeric(s: str) -> bool: + """Return ``True`` when ``s`` begins with a bare number (optionally + a unit).""" + return bool(_NUMERIC_RE.match(s.strip())) + + +__all__ = ["SeedScorer", "SEED_ANSWER_TYPES"] diff --git a/tests/prkit/datasets/loaders/test_cmphysbench_loader.py b/tests/prkit/datasets/loaders/test_cmphysbench_loader.py new file mode 100644 index 0000000..7e57186 --- /dev/null +++ b/tests/prkit/datasets/loaders/test_cmphysbench_loader.py @@ -0,0 +1,117 @@ +"""Unit tests for the CMPhysBench loader (answer_type → SEED-token source_type).""" + +from __future__ import annotations + +import json + +import pytest + +from prkit.datasets.hub import DatasetHub +from prkit.datasets.loaders import CMPhysBenchLoader + + +def _write_dataset_json(path, rows): + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + json.dump(rows, handle) + + +class TestCMPhysBenchLoader: + def test_metadata_and_field_mapping(self): + loader = CMPhysBenchLoader() + assert loader.name == "cmphysbench" + assert loader.field_mapping == {"id": "problem_id", "final_answer": "answer"} + info = loader.get_info() + assert info["answer_types"] == [ + "Expression", + "Equation", + "Tuple", + "Interval", + "Numeric", + ] + assert info["license_spdx"] == "Apache-2.0" + + def test_registered_in_hub(self): + assert "cmphysbench" in DatasetHub.list_available() + assert isinstance(DatasetHub._get_loader("cmphysbench"), CMPhysBenchLoader) + + @pytest.mark.parametrize( + ("raw", "expected"), + [ + ("Expression", "Expression"), + ("numeric", "Numeric"), # case-insensitive canonicalization + ("Tuple", "Tuple"), + ("", None), + (None, None), + ("WeirdType", "WeirdType"), # unrecognized passes through verbatim + ], + ) + def test_normalize_answer_type(self, raw, expected): + assert CMPhysBenchLoader._normalize_answer_type(raw) == expected + + def test_process_metadata_combines_question_and_lifts_source_type(self): + loader = CMPhysBenchLoader() + meta = loader._process_metadata( + { + "problem_id": "p1", + "context": "Given a lattice. ", + "question": "What is the energy?", + "answer": "E = k", + "answer_type": "equation", + } + ) + assert meta["question"] == "Given a lattice. What is the energy?" + assert meta["source_type"] == "Equation" + assert meta["problem_type"] == "OE" + + def test_load_maps_answer_type_into_source_type(self, temp_dir): + loader = CMPhysBenchLoader() + data_dir = temp_dir / "CMPhysBench" + _write_dataset_json( + data_dir / "dataset.json", + [ + { + "id": "cmp_1", + "context": "A crystal. ", + "question": "Find x.", + "final_answer": "x = 1", + "answer_type": "Equation", + "topic": "lattice", + }, + { + "id": "cmp_2", + "question": "Compute the value.", + "final_answer": "3.14", + "answer_type": "Numeric", + "topic": "thermo", + }, + ], + ) + + dataset = loader.load(data_dir=str(data_dir)) + assert len(dataset) == 2 + + first = dataset[0] + assert first.problem_id == "cmp_1" + assert first.question == "A crystal. Find x." + assert first.answer is not None + assert first.answer.value == "x = 1" + assert first.answer.source_type == "Equation" + assert first.additional_fields["topic"] == "lattice" + + second = dataset[1] + assert second.answer.source_type == "Numeric" + assert dataset.get_info()["total_problems"] == 2 + + def test_load_raises_for_missing_dir(self, temp_dir): + loader = CMPhysBenchLoader() + with pytest.raises(FileNotFoundError): + loader.load(data_dir=str(temp_dir / "does_not_exist")) + + def test_load_raises_for_invalid_json(self, temp_dir): + loader = CMPhysBenchLoader() + data_dir = temp_dir / "CMPhysBench" + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "dataset.json").write_text("{ not valid json", encoding="utf-8") + with pytest.raises(ValueError, match="Invalid JSON"): + loader.load(data_dir=str(data_dir)) diff --git a/tests/prkit/scoring/test_eed_scorer.py b/tests/prkit/scoring/test_eed_scorer.py new file mode 100644 index 0000000..8c4c473 --- /dev/null +++ b/tests/prkit/scoring/test_eed_scorer.py @@ -0,0 +1,62 @@ +"""Unit tests for the faithful PHYBench EED baseline scorer.""" + +from __future__ import annotations + +import pytest + +# The vendored EED front-end needs latex2sympy2_extended (a core dep, but guard so +# the suite degrades gracefully if it is ever made optional). +pytest.importorskip("latex2sympy2_extended") + +from prkit.core.domain.answer import Answer # noqa: E402 +from prkit.core.verdict import Verdict # noqa: E402 +from prkit.scoring import EedScorer # noqa: E402 + + +class TestEedScorer: + def test_equivalent_pair_scores_one(self): + verdict = EedScorer().score("x + 1", "1 + x") + assert isinstance(verdict, Verdict) + assert verdict.equivalent is True + assert verdict.score == 1.0 + assert verdict.comparison_mode == "eed" + assert verdict.scorer_version == EedScorer.version + + def test_different_pair_scores_lower_and_not_equivalent(self): + verdict = EedScorer().score("3 m/s", "5 m/s") + assert verdict.equivalent is False + assert 0.0 <= verdict.score < 1.0 + # graded edit-distance signal is surfaced as partial credit + assert verdict.partial_credit == verdict.score + + def test_accepts_answer_objects(self): + verdict = EedScorer().score( + Answer(value="3.0", unit="m/s"), Answer(value="3", unit="m/s") + ) + assert verdict.equivalent is True + assert verdict.score == 1.0 + + def test_score_always_in_unit_interval(self): + # An unparseable / wildly different prediction must still land in [0, 1]. + verdict = EedScorer().score("\\int x dx", "y") + assert 0.0 <= verdict.score <= 1.0 + assert verdict.equivalent is False + + def test_get_info_keys(self): + info = EedScorer(tolerance=1e-3).get_info() + assert info["name"] == "EedScorer" + assert info["version"] == EedScorer.version + assert info["engine"] == "phybench_eed" + assert info["deterministic"] is True + assert info["front_end"] == "vendor" + assert info["tolerance"] == pytest.approx(1e-3) + + def test_deterministic(self): + scorer = EedScorer() + assert scorer.score("x + 1", "1 + x") == scorer.score("x + 1", "1 + x") + + def test_details_are_plain_floats(self): + details = EedScorer().score("3 m/s", "5 m/s").details + assert details["front_end"] == "vendor" + for key in ("raw_score", "relative_distance", "tree_size", "distance"): + assert isinstance(details[key], float) diff --git a/tests/prkit/scoring/test_seed_scorer.py b/tests/prkit/scoring/test_seed_scorer.py new file mode 100644 index 0000000..722406d --- /dev/null +++ b/tests/prkit/scoring/test_seed_scorer.py @@ -0,0 +1,117 @@ +"""Unit tests for the faithful CMPhysBench SEED baseline scorer.""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("latex2sympy2_extended") + +from prkit.core.domain.answer import Answer # noqa: E402 +from prkit.core.verdict import Verdict # noqa: E402 +from prkit.scoring import SeedScorer # noqa: E402 +from prkit.scoring.seed_scorer import SEED_ANSWER_TYPES # noqa: E402 + + +class TestSeedScorer: + def test_equivalent_pair_scores_one(self): + verdict = SeedScorer().score("x + 1", "1 + x") + assert isinstance(verdict, Verdict) + assert verdict.equivalent is True + assert verdict.score == 1.0 + assert verdict.comparison_mode == "seed:Expression" + + def test_different_pair_scores_lower(self): + verdict = SeedScorer().score("3 m/s", "5 m/s") + assert verdict.equivalent is False + assert 0.0 <= verdict.score < 1.0 + + def test_explicit_answer_type_kwarg_dispatches(self): + # Within the tightest numeric tier (<=1%) → equivalent, mode records the type. + verdict = SeedScorer().score("4.10", "4.08", answer_type="Numeric") + assert verdict.comparison_mode == "seed:Numeric" + assert verdict.equivalent is True + assert verdict.details["answer_type"] == "Numeric" + assert verdict.details["classifier_used"] is False + + def test_source_type_fallback_for_tuple(self): + verdict = SeedScorer().score( + "(1, 3)", Answer(value="(1, 2)", source_type="Tuple") + ) + assert verdict.comparison_mode == "seed:Tuple" + assert verdict.details["answer_type"] == "Tuple" + + def test_non_seed_source_type_falls_back_to_expression(self): + verdict = SeedScorer().score("x + 1", Answer(value="1 + x", source_type="MC")) + assert verdict.comparison_mode == "seed:Expression" + assert verdict.equivalent is True + + def test_invalid_kwarg_falls_back_to_default(self): + verdict = SeedScorer().score("x + 1", "1 + x", answer_type="NotAType") + assert verdict.comparison_mode == "seed:Expression" + + def test_default_answer_type_override(self): + scorer = SeedScorer(default_answer_type="Equation") + assert scorer.get_info()["default_answer_type"] == "Equation" + verdict = scorer.score("E = m c^2", "E = c^2 m") + assert verdict.comparison_mode == "seed:Equation" + + def test_invalid_default_answer_type_rejected(self): + with pytest.raises(ValueError, match="default_answer_type"): + SeedScorer(default_answer_type="bogus") + + def test_classifier_disabled_by_default(self): + scorer = SeedScorer() + # Even an equation-looking reference dispatches as the default (no classify). + verdict = scorer.score("x = 2", "x = 1") + assert verdict.comparison_mode == "seed:Expression" + assert scorer.get_info()["classifier_used"] is False + + def test_classifier_when_enabled(self): + scorer = SeedScorer(enable_classifier=True) + verdict = scorer.score("x = 2", "x = 1") + assert verdict.comparison_mode == "seed:Equation" + assert verdict.details["classifier_used"] is True + assert scorer.get_info()["classifier_used"] is True + + @pytest.mark.parametrize( + ("text", "expected"), + [ + ("x = 1", "Equation"), + ("(1, 2, 3)", "Tuple"), + ("4.08 m", "Numeric"), + ("x + 1", "Expression"), + ("(0, 1)", "Expression"), # 2-element bracket: Tuple/Interval ambiguous + ], + ) + def test_classifier_triage(self, text, expected): + assert SeedScorer._classify_answer_type(text) == expected + assert expected in SEED_ANSWER_TYPES + + def test_get_info_keys(self): + info = SeedScorer(tolerance=0.01, enable_classifier=True).get_info() + assert info["name"] == "SeedScorer" + assert info["version"] == SeedScorer.version + assert info["engine"] == "cmphysbench_seed" + assert info["deterministic"] is True + assert info["front_end"] == "vendor" + assert info["enable_classifier"] is True + assert info["tolerance"] == pytest.approx(0.01) + + def test_deterministic(self): + scorer = SeedScorer() + assert scorer.score("x + 1", "1 + x") == scorer.score("x + 1", "1 + x") + + +class TestSeedScorerUnits: + """The unit-aware Numeric path is the only one that needs ``pint``.""" + + def test_unit_conversion_numeric(self): + pytest.importorskip("pint") + scorer = SeedScorer() + verdict = scorer.score( + "4.08 \\times 10^{-7}(\\mathrm{~m})", + "4.08 \\times 10^{-5}(\\mathrm{~cm})", + answer_type="Numeric", + ) + assert verdict.comparison_mode == "seed:Numeric" + assert verdict.equivalent is True # 1e-5 cm == 1e-7 m diff --git a/tests/prkit/test_conformance.py b/tests/prkit/test_conformance.py index 2ade4a5..638f79d 100644 --- a/tests/prkit/test_conformance.py +++ b/tests/prkit/test_conformance.py @@ -14,7 +14,12 @@ from prkit.api import Verdict from prkit.core.model_clients.base import BaseModelClient from prkit.datasets.hub import DatasetHub -from prkit.scoring import PartialCreditScorer, SemanticsScorer +from prkit.scoring import ( + EedScorer, + PartialCreditScorer, + SeedScorer, + SemanticsScorer, +) from prkit.testing import check_dataset, check_model_client, check_scorer @@ -46,6 +51,17 @@ def test_partial_credit_scorer_conforms(): check_scorer(PartialCreditScorer()) +def test_eed_scorer_conforms(): + pytest.importorskip("latex2sympy2_extended") + check_scorer(EedScorer()) + + +def test_seed_scorer_conforms(): + pytest.importorskip("latex2sympy2_extended") + pytest.importorskip("pint") + check_scorer(SeedScorer()) + + def test_stub_model_client_conforms_offline(): check_model_client(_StubClient("stub-model"), live=False) From 0d5a5a1efca6e97d4c029c48eca51028826693ac Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sun, 21 Jun 2026 16:31:42 -0400 Subject: [PATCH 22/28] Forbid pint on the verify/scoring import path Add pint to the import-isolation guard's forbidden list. The vendored CMPhysBench SEED baseline pulls pint only on its unit-aware Numeric path, and the EedScorer / SeedScorer wrappers import the vendored core lazily inside score(), so pint must never reach import prkit.scoring or the verify facade. latex2sympy2_extended and its antlr4 runtime are a required core dep already on this path by design and stay deliberately unforbidden. Co-Authored-By: Claude Opus 4.8 --- tests/prkit/verify/test_import_isolation.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/prkit/verify/test_import_isolation.py b/tests/prkit/verify/test_import_isolation.py index 3de2476..53256e1 100644 --- a/tests/prkit/verify/test_import_isolation.py +++ b/tests/prkit/verify/test_import_isolation.py @@ -27,6 +27,12 @@ # now legitimately reachable, since the pure EED/SEED algorithm core lives in # ``prkit.evaluation.edit_distance`` and the partial-credit scorer imports it. "prkit.evaluation.llm_judge", + # SEED units. The vendored CMPhysBench baseline pulls ``pint`` only on its + # unit-aware Numeric path; ``EedScorer``/``SeedScorer`` import the vendored core + # lazily inside ``score()`` so ``pint`` stays off ``import prkit.scoring``/``verify``. + # (``latex2sympy2_extended``/``antlr4`` are a required core dep already on this + # path by design — deliberately NOT forbidden.) + "pint", ] From ce1eb2c7442f58473a3854e13b8c025bb40e4d38 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sun, 21 Jun 2026 16:56:22 -0400 Subject: [PATCH 23/28] Add the our-semantics EED/SEED scorers and the N/A score sentinel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire PRKit's own answer normalization to the vendored PHYBench-EED and CMPhysBench-SEED pure cores via a new semantics→pure-core adapter (scoring/_edit_distance_adapt.py), shipping SemanticsEedScorer and SemanticsSeedScorer. The adapter parses each answer with the semantics parser (never latex2sympy2) and the object_kind+structure → SEED-type map, then calls the cores' own tree-edit / numeric-tier algorithm verbatim, so a SemanticsEed/SeedScorer number stays directly comparable to its vendor Eed/SeedScorer baseline. The semantics path never loads pint. Reserve score == -1.0 as the not-applicable sentinel: Verdict accepts it (comparison_mode="not_applicable") for kinds/structures with no SEED type, check_scorer accepts it and skips the equivalence/identity expectations for it, and CONTRACT records that aggregators must exclude it. Remove PartialCreditScorer (and PartialCreditMode), which the two semantics edit-distance scorers supersede; verify(partial_credit=True) now uses SemanticsSeedScorer. The PHYBench/CMPhysBench reimplementations under evaluation/edit_distance and semantics/edit_distance are relabeled as parked reference (eed-reimpl / seed-reimpl), no longer wired to any Scorer. Co-Authored-By: Claude Opus 4.8 --- README.md | 8 +- docs/EVALUATION.md | 13 +- docs/PHYSICS_SEMANTICS.md | 5 +- src/prkit/CONTRACT.md | 8 +- src/prkit/__init__.py | 4 +- src/prkit/core/verdict.py | 10 +- .../evaluation/edit_distance/__init__.py | 10 +- src/prkit/scoring/__init__.py | 32 +- src/prkit/scoring/_edit_distance_adapt.py | 661 ++++++++++++++++++ src/prkit/scoring/partial_credit_scorer.py | 249 ------- src/prkit/scoring/semantics_eed_scorer.py | 169 +++++ src/prkit/scoring/semantics_seed_scorer.py | 182 +++++ src/prkit/semantics/edit_distance/__init__.py | 19 +- src/prkit/semantics/edit_distance/pipeline.py | 9 + src/prkit/testing/conformance.py | 24 +- src/prkit/verify/__init__.py | 9 +- tests/prkit/core/test_verdict.py | 18 +- .../scoring/test_partial_credit_scorer.py | 132 ---- .../scoring/test_semantics_eed_scorer.py | 89 +++ .../scoring/test_semantics_seed_scorer.py | 123 ++++ tests/prkit/test_conformance.py | 19 +- 21 files changed, 1359 insertions(+), 434 deletions(-) create mode 100644 src/prkit/scoring/_edit_distance_adapt.py delete mode 100644 src/prkit/scoring/partial_credit_scorer.py create mode 100644 src/prkit/scoring/semantics_eed_scorer.py create mode 100644 src/prkit/scoring/semantics_seed_scorer.py delete mode 100644 tests/prkit/scoring/test_partial_credit_scorer.py create mode 100644 tests/prkit/scoring/test_semantics_eed_scorer.py create mode 100644 tests/prkit/scoring/test_semantics_seed_scorer.py diff --git a/README.md b/README.md index e7c1572..fe676f5 100644 --- a/README.md +++ b/README.md @@ -223,9 +223,11 @@ The essential building blocks of the physical-reasoning-toolkit. All datasets, i ### prkit.scoring / prkit.verify 📈 The deterministic physics-semantics scorer: `prkit.verify.verify` (light-import, one-call) -and `prkit.scoring.SemanticsScorer` / `PartialCreditScorer`, all returning the canonical -`Verdict`. Wraps the `prkit.semantics.comparison` engine. (The legacy `prkit.evaluation` -comparator/evaluator stack is deprecated; `prkit.evaluation.llm_judge` stays.) +and the `prkit.scoring` family — `SemanticsScorer` (binary), the `EedScorer`/`SeedScorer` +edit-distance baselines, the graded `SemanticsEedScorer`/`SemanticsSeedScorer`, and the +model-graded `LLMJudgeScorer` — all returning the canonical `Verdict`. Wraps the +`prkit.semantics.comparison` engine. (The legacy `prkit.evaluation` comparator/evaluator +stack is deprecated; `prkit.evaluation.llm_judge` stays.) 📖 [EVALUATION.md](docs/EVALUATION.md) · [PHYSICS_SEMANTICS.md](docs/PHYSICS_SEMANTICS.md) diff --git a/docs/EVALUATION.md b/docs/EVALUATION.md index 02bcacb..e8574a5 100644 --- a/docs/EVALUATION.md +++ b/docs/EVALUATION.md @@ -19,11 +19,14 @@ v.scorer_version # stamped so a stored score is attributable to its scorer Imports no provider SDKs, dataset hub, `datasets`, or pandas. - **`prkit.scoring.SemanticsScorer`** — the reference `Scorer` (binary pass/fail) that `verify` wraps. Use it directly when you want the `prkit.api.Scorer` object. -- **`prkit.scoring.PartialCreditScorer`** — graded EED/SEED partial credit (populates - `Verdict.partial_credit`); reachable via `verify(..., partial_credit=True)`. - -All three return the same canonical `Verdict`. The judgement itself lives in the -deterministic engine `prkit.semantics.comparison`. +- **`prkit.scoring.SemanticsSeedScorer`** — graded partial credit (our-semantics + front-end over the CMPhysBench-SEED edit-distance pure core; populates + `Verdict.partial_credit`); reachable via `verify(..., partial_credit=True)`. Its + EED-algorithm sibling is `SemanticsEedScorer`, and the faithful vendor-front-end + baselines are `EedScorer` / `SeedScorer`. + +All return the same canonical `Verdict`. The deterministic binary judgement itself +lives in the engine `prkit.semantics.comparison`. ## Learn more diff --git a/docs/PHYSICS_SEMANTICS.md b/docs/PHYSICS_SEMANTICS.md index 52fadb7..c167898 100644 --- a/docs/PHYSICS_SEMANTICS.md +++ b/docs/PHYSICS_SEMANTICS.md @@ -76,8 +76,9 @@ All are importable from `prkit.semantics`. Notes: `math-verify`-shaped one-call facade returning a canonical `Verdict`. It pulls in no provider SDKs, dataset hub, `datasets`, or pandas. - **Want a `Scorer` object** (e.g. to plug into a runner or for partial credit)? Use - **`prkit.scoring.SemanticsScorer`** (binary) or **`prkit.scoring.PartialCreditScorer`** - (graded EED/SEED). Both return the same `Verdict`. + **`prkit.scoring.SemanticsScorer`** (binary) or **`prkit.scoring.SemanticsSeedScorer`** + (graded, our-semantics over the CMPhysBench-SEED edit-distance core). Both return the + same `Verdict`. - **Want the raw mechanism** (the rich `AnswerComparison` with `comparison_mode`, `bridge_*`, `diagnostics`)? Call **`compare_protocol_answers`** (reference-based) or **`compare_predictions`** (reference-free) directly. diff --git a/src/prkit/CONTRACT.md b/src/prkit/CONTRACT.md index e5b398e..e24708c 100644 --- a/src/prkit/CONTRACT.md +++ b/src/prkit/CONTRACT.md @@ -38,7 +38,7 @@ The contract pins four structural (`typing.Protocol`) nouns plus one result type |------|----------|--------------------------| | Dataset loader | `DatasetProvider` | `BaseDatasetLoader` subclasses | | Inference client | `ModelClient` | `BaseModelClient` subclasses | -| Scorer | `Scorer` | `prkit.scoring.SemanticsScorer` (binary); `prkit.scoring.PartialCreditScorer` (graded EED/SEED) | +| Scorer | `Scorer` | `prkit.scoring.SemanticsScorer` (binary); `EedScorer`/`SeedScorer` (vendor edit-distance baselines); `SemanticsEedScorer`/`SemanticsSeedScorer` (our-semantics edit distance, graded); `LLMJudgeScorer` (model-graded) | | Runner | `Runner` | *(reserved; no implementation yet)* | | Result | `Verdict` | `prkit.core.verdict.Verdict` | @@ -56,7 +56,7 @@ when not applicable (or not yet produced): | Field | Kind | Meaning | |-------|------|---------| | `equivalent` / `correct` | core | primary pass/fail (`correct` mirrors `equivalent` by default) | -| `score` | core | continuous score in `[0,1]`; binary scorers emit `1.0`/`0.0` | +| `score` | core | continuous score in `[0,1]`; binary scorers emit `1.0`/`0.0`; `-1.0` is the reserved not-applicable sentinel (`comparison_mode="not_applicable"`, emitted by the edit-distance scorers for kinds/structures with no SEED type) and MUST be excluded from aggregation (filter `score >= 0`) | | `comparison_mode` | core | how the verdict was reached (`number`, `expression`, …) | | `scorer_version` | core | Gymnasium-style stamp of the scorer revision | | `diagnostics` | core | machine-readable mismatch/fallback tags | @@ -65,8 +65,8 @@ when not applicable (or not yet produced): | `symbolic_equiv` | enriched | equivalence decided symbolically; `None` for non-symbolic modes | | `numeric_within_tol` | enriched | numeric/quantity match within tolerance; `None` otherwise | | `extracted_answer` | enriched | parsed prediction surface, when available | -| `partial_credit` | enriched | continuous partial-credit signal; `None` from the binary `SemanticsScorer`, populated by the graded `PartialCreditScorer` (EED/SEED) — also via `verify(..., partial_credit=True)` | -| `rationale` | enriched | human-readable explanation; `None` from the deterministic engine, populated by `PartialCreditScorer` | +| `partial_credit` | enriched | continuous partial-credit signal; `None` from the binary `SemanticsScorer`, populated by the graded edit-distance scorers (`EedScorer`/`SeedScorer`/`SemanticsEedScorer`/`SemanticsSeedScorer`) — `verify(..., partial_credit=True)` uses `SemanticsSeedScorer` | +| `rationale` | enriched | human-readable explanation; `None` from the deterministic engine, populated by the model-graded `LLMJudgeScorer` | ## Three independent version axes diff --git a/src/prkit/__init__.py b/src/prkit/__init__.py index 640d125..4e00afb 100644 --- a/src/prkit/__init__.py +++ b/src/prkit/__init__.py @@ -17,7 +17,9 @@ - :mod:`prkit.api` — frozen public contract (protocols + ``Verdict``). - :mod:`prkit.core` — domain models, model clients, logging. - :mod:`prkit.datasets` — dataset hub, loaders, downloaders. - - :mod:`prkit.scoring` — reference scorers (``SemanticsScorer``). + - :mod:`prkit.scoring` — reference scorers (``SemanticsScorer``; the + ``Eed``/``Seed`` edit-distance baselines + ``Semantics`` front-end variants; + ``LLMJudgeScorer``). - :mod:`prkit.testing` — conformance suite (``check_dataset``/``check_scorer``/…). - :mod:`prkit.semantics` — physics-aware answer normalization & comparison. - :mod:`prkit.evaluation` — model-graded LLM judge (``llm_judge``). The legacy diff --git a/src/prkit/core/verdict.py b/src/prkit/core/verdict.py index f9ba32f..87efa43 100644 --- a/src/prkit/core/verdict.py +++ b/src/prkit/core/verdict.py @@ -86,7 +86,15 @@ def _default_correct_to_equivalent(cls, data: Any) -> Any: @field_validator("score") @classmethod def _score_in_range(cls, value: float) -> float: - """Reject scores outside the closed unit interval ``[0, 1]``.""" + """Reject scores outside ``[0, 1]`` except the reserved N/A sentinel. + + ``-1.0`` is the reserved *not-applicable* sentinel (emitted by the + edit-distance scorers when an answer kind/structure has no SEED type, with + ``comparison_mode="not_applicable"``). It is an honest "N/A", distinct from a + ``0.0`` "wrong"; numeric aggregators MUST exclude it (filter ``score >= 0``). + """ + if value == -1.0: + return value if not (0.0 <= value <= 1.0): raise ValueError(f"score must be in [0, 1], got {value!r}") return value diff --git a/src/prkit/evaluation/edit_distance/__init__.py b/src/prkit/evaluation/edit_distance/__init__.py index 312337d..8b42096 100644 --- a/src/prkit/evaluation/edit_distance/__init__.py +++ b/src/prkit/evaluation/edit_distance/__init__.py @@ -1,8 +1,12 @@ """Pure Expression Edit Distance (EED / SEED) algorithm core — related-work methods. -A self-contained reimplementation of PHYBench's Expression Edit Distance and -CMPhysBench's Scalable EED *tree-edit* machinery. These modules are deliberately -**free of any PRKit semantics dependency** — they import only ``sympy`` + stdlib: +PARKED REFERENCE (``eed-reimpl``). A self-contained reimplementation of PHYBench's +Expression Edit Distance and CMPhysBench's Scalable EED *tree-edit* machinery. The +shipped edit-distance scorers run the **vendored** upstream cores under +:mod:`prkit.evaluation.baselines` instead; this reimpl is retained as a readable, +semantics-free reference and as a differential-testing oracle. Its modules are +deliberately **free of any PRKit semantics dependency** — they import only ``sympy`` ++ stdlib: * :mod:`.tree` — SymPy expression → :class:`ExprNode` tree builder * :mod:`.zss` — extended Zhang-Shasha tree edit distance diff --git a/src/prkit/scoring/__init__.py b/src/prkit/scoring/__init__.py index 133a609..ef26f33 100644 --- a/src/prkit/scoring/__init__.py +++ b/src/prkit/scoring/__init__.py @@ -1,31 +1,39 @@ """Reference scoring implementations for PRKit's ``Scorer`` contract. -``SemanticsScorer`` is the canonical, version-stamped scorer wrapping the -deterministic (binary) semantics comparison engine. ``EedScorer`` / ``SeedScorer`` -are the faithful PHYBench-EED / CMPhysBench-SEED edit-distance *baselines* (vendor -LaTeX front-end + the front-end-free pure core). ``PartialCreditScorer`` is the -graded EED/SEED scorer over PRKit's own semantics front-end that populates -``Verdict.partial_credit``. ``LLMJudgeScorer`` is the model-graded scorer wrapping -the ``prkit.evaluation.llm_judge`` engine. All structurally satisfy -:class:`prkit.api.Scorer` and emit :class:`prkit.api.Verdict`. +The scorer family is a 2×2 edit-distance matrix (front-end × algorithm) plus two +standalone scorers, all structurally satisfying :class:`prkit.api.Scorer` and +emitting :class:`prkit.api.Verdict`: + +- ``SemanticsScorer`` — deterministic binary equivalence (the ``semantics/comparison`` + engine); the canonical, version-stamped scorer. +- ``EedScorer`` / ``SeedScorer`` — faithful PHYBench-EED / CMPhysBench-SEED baselines + (vendor LaTeX front-end + the front-end-free pure core). +- ``SemanticsEedScorer`` / ``SemanticsSeedScorer`` — the *our-semantics* front-end + (``normalize_physics_answer``) over those same pure cores; they populate + ``Verdict.partial_credit`` and ``SemanticsSeedScorer`` backs + ``verify(..., partial_credit=True)``. +- ``LLMJudgeScorer`` — the model-graded scorer wrapping the ``prkit.evaluation.llm_judge`` + engine. Import discipline: re-exporting these scorers here must not pull ``openai``, ``prkit.evaluation.llm_judge``, ``pint``, or the vendored LaTeX front-end onto ``import prkit.scoring`` — the judge, vendored-core, and front-end imports are all -deferred to method bodies (see ``llm_judge_scorer`` / ``eed_scorer`` / ``seed_scorer``). +deferred to method bodies (see ``llm_judge_scorer`` / ``eed_scorer`` / ``seed_scorer`` +/ ``semantics_eed_scorer`` / ``semantics_seed_scorer``). """ from .eed_scorer import EedScorer from .llm_judge_scorer import LLMJudgeScorer -from .partial_credit_scorer import PartialCreditMode, PartialCreditScorer from .seed_scorer import SeedScorer +from .semantics_eed_scorer import SemanticsEedScorer from .semantics_scorer import SemanticsScorer +from .semantics_seed_scorer import SemanticsSeedScorer __all__ = [ "EedScorer", "LLMJudgeScorer", - "PartialCreditMode", - "PartialCreditScorer", "SeedScorer", + "SemanticsEedScorer", "SemanticsScorer", + "SemanticsSeedScorer", ] diff --git a/src/prkit/scoring/_edit_distance_adapt.py b/src/prkit/scoring/_edit_distance_adapt.py new file mode 100644 index 0000000..8c5f2f3 --- /dev/null +++ b/src/prkit/scoring/_edit_distance_adapt.py @@ -0,0 +1,661 @@ +"""Adapter: ``PhysicsAnswerSemantics`` → the vendored EED/SEED **pure cores**. + +This is the *our-semantics* front-end for the edit-distance scorers. It bridges +:func:`prkit.semantics.normalize_physics_answer`'s output to the front-end-free +PHYBench-EED / CMPhysBench-SEED pure cores +(:mod:`prkit.evaluation.baselines.phybench_eed` / ``cmphysbench_seed``) **without** +the LaTeX front-end (``latex2sympy2``) or ``pint``: the semantics layer has already +parsed, classified, and unit-normalized the answer, so the adapter feeds parsed +SymPy expressions (and canonical numeric magnitudes) straight into the cores' +post-conversion scoring tail. + +Ablation discipline: the *algorithm* (tree-edit distance, ``score_calc`` tiers, +``numeric_score_calc``) is the cores' own, called verbatim — only the front-end +(text → SymPy via our parser, plus the ``object_kind``+``structure`` → SEED-type +classification) differs from the vendor baselines. That keeps a +``SemanticsEed/SeedScorer`` number directly comparable to its ``Eed/SeedScorer`` +baseline. + +Acyclic-DAG note: this module lives under ``scoring/`` (not ``evaluation/``) +precisely because it imports *both* ``prkit.semantics`` and the +``prkit.evaluation.baselines`` cores; ``evaluation.*`` must never import +``prkit.semantics``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from prkit.core.domain import AnswerObjectKind, AnswerStructure +from prkit.core.verdict import Verdict + +# The vendored pure cores are front-end-free (no latex2sympy2; pint is lazy), so +# importing them here pulls only SymPy + the thread-safe timeout shim. The scorers +# import *this* adapter lazily inside ``score()``, so ``import prkit.scoring`` stays +# free of these imports until a score is actually computed. +from prkit.evaluation.baselines.cmphysbench_seed.core import seed as _seed_core +from prkit.evaluation.baselines.phybench_eed.core import eed as _eed_core +from prkit.semantics import ( + PhysicsAnswerSemantics, + PhysicsQuestionSemantics, + QuestionUnitPolicy, +) +from prkit.semantics.comparison.common import context_symbol_alias_map +from prkit.semantics.comparison.numeric import ( + NumericComparableAnswer, + extract_numeric_comparable_answer, +) +from prkit.semantics.comparison.semantics import ( + convert_numeric_value, + normalize_unit_text, + parse_relation_clauses, + parse_scalar_symbolic_expression, +) + +# --------------------------------------------------------------------------- # +# object_kind + structure → SEED type map (§II.8). A container ``structure`` +# wins over ``object_kind``; ``ATOMIC`` falls through to the object_kind map. +# --------------------------------------------------------------------------- # + +#: ``structure`` → SEED type (checked before ``object_kind``). +STRUCTURE_TO_SEED: dict[AnswerStructure, str] = { + AnswerStructure.TUPLE: "Tuple", + AnswerStructure.SET: "Tuple", # unordered → pre-sorted before positional compare + AnswerStructure.MULTI_PART: "Tuple", # sub-answers → Tuple branch (per-element) + AnswerStructure.INTERVAL: "Interval", +} + +#: ``structure`` values with no SEED representation (the tree grammar can't encode +#: them) → not-applicable. +NA_STRUCTURES: frozenset[AnswerStructure] = frozenset( + { + AnswerStructure.VECTOR, + AnswerStructure.MATRIX, + AnswerStructure.TENSOR, + AnswerStructure.PIECEWISE, + } +) + +#: ``object_kind`` → SEED type (used when ``structure`` is ``ATOMIC``/non-container). +OBJECT_KIND_TO_SEED: dict[AnswerObjectKind, str] = { + AnswerObjectKind.NUMBER: "Numeric", + AnswerObjectKind.PHYSICAL_QUANTITY: "Numeric", + AnswerObjectKind.EXPRESSION: "Expression", + AnswerObjectKind.RELATION: "Equation", +} + +#: Non-symbolic ``object_kind`` values → not-applicable. +NA_KINDS: frozenset[AnswerObjectKind] = frozenset( + { + AnswerObjectKind.QUALITATIVE_LABEL, + AnswerObjectKind.CHOICE, + AnswerObjectKind.BOOLEAN, + AnswerObjectKind.SIGN_DIRECTION, + AnswerObjectKind.DESCRIPTIVE_TEXT, + } +) + +#: ``object_kind`` values the expression-only EED front-end can score (all treated +#: as a single SymPy expression; a ``RELATION`` is reduced to its ``lhs − rhs``). +_EED_APPLICABLE_KINDS: frozenset[AnswerObjectKind] = frozenset( + { + AnswerObjectKind.NUMBER, + AnswerObjectKind.PHYSICAL_QUANTITY, + AnswerObjectKind.EXPRESSION, + AnswerObjectKind.RELATION, + } +) + + +def seed_type(sem: PhysicsAnswerSemantics) -> tuple[str | None, str | None]: + """Resolve a reference answer's SEED type from its kind + structure. + + Returns ``(seed_type, na_reason)``: exactly one is non-``None``. ``na_reason`` is + the offending kind/structure value (for the ``not_applicable:`` diagnostic). + """ + structure = sem.structure + if structure in NA_STRUCTURES: + return None, str(structure) + mapped = STRUCTURE_TO_SEED.get(structure) + if mapped is not None: + return mapped, None + kind = sem.object_kind + if kind in NA_KINDS: + return None, str(kind) + mapped = OBJECT_KIND_TO_SEED.get(kind) + if mapped is not None: + return mapped, None + return None, str(kind) # defensive: an unmapped kind is treated as N/A + + +def eed_na_reason(sem: PhysicsAnswerSemantics) -> str | None: + """Return the N/A reason for the expression-only EED front-end (``None`` if OK). + + EED has no container/numeric-tier path, so only an ``ATOMIC`` symbolic answer + (number / quantity / expression / relation) is applicable. + """ + if sem.structure != AnswerStructure.ATOMIC: + return str(sem.structure) + if sem.object_kind not in _EED_APPLICABLE_KINDS: + return str(sem.object_kind) + return None + + +@dataclass(frozen=True) +class CoreScore: + """Pure-core result for one pair (before ``Verdict`` normalization). + + ``raw`` is the cores' ``0..100`` score; the SEED-internal ``-1`` "not computed" + markers are carried verbatim on the distance fields. + """ + + raw: float + answer_type: str + relative_distance: float = -1.0 + tree_size: float = -1.0 + distance: float = -1.0 + units_ok: bool | None = None + symbolic_equiv: bool | None = None + numeric_within_tol: bool | None = None + degraded: bool = False + diagnostics: tuple[str, ...] = () + + +# --------------------------------------------------------------------------- # +# Front-end parsing (our SymPy parser; never latex2sympy2). +# --------------------------------------------------------------------------- # +def _texts(sem: PhysicsAnswerSemantics) -> tuple[str, ...]: + """Surfaces to parse, in preference order: canonical_text → latex → raw.""" + return tuple( + t for t in (sem.canonical_text, sem.canonical_latex, sem.raw_text) if t + ) + + +def _parse_scalar(sem: PhysicsAnswerSemantics, alias_map: Any) -> Any | None: + """Parse the first parseable surface into a scalar SymPy expression.""" + for text in _texts(sem): + expr = parse_scalar_symbolic_expression(text, alias_map=alias_map) + if expr is not None: + return expr + return None + + +def _relation_residual(sem: PhysicsAnswerSemantics, alias_map: Any) -> Any | None: + """Build a relation's ``lhs − rhs`` residual; fall back to a scalar parse.""" + for text in _texts(sem): + clauses = parse_relation_clauses(text, alias_map=alias_map) + if clauses: + clause = clauses[0] + lhs = parse_scalar_symbolic_expression(clause.lhs_text, alias_map=alias_map) + rhs = parse_scalar_symbolic_expression(clause.rhs_text, alias_map=alias_map) + if lhs is not None and rhs is not None: + return lhs - rhs + return _parse_scalar(sem, alias_map) + + +# --------------------------------------------------------------------------- # +# Pure-core scoring tails (the cores' own algorithm, called verbatim). +# --------------------------------------------------------------------------- # +def _ext_distance(pred_tree: Any, gold_tree: Any, core: Any) -> float: + """Run the core's tree-edit distance with the core's own cost functions.""" + return float( + core.ext_distance( + pred_tree, + gold_tree, + get_children=lambda node: node.get_children(), + single_insert_cost=core.insert_func, + insert_cost=core.insert_tree_func, + single_remove_cost=core.remove_func, + remove_cost=core.remove_tree_func, + update_cost=core.update_func, + ) + ) + + +def _expr_tail( + pred_expr: Any, gold_expr: Any, core: Any, *, equation: bool = False +) -> tuple[float, float, float, float, bool, bool]: + """Mirror the cores' post-``master_convert`` expression tail on parsed exprs. + + Returns ``(raw, rel_distance, tree_size, distance, symbolic_equiv, degraded)``, + replicating the vendored ``EED``/``SEED`` simplify → equality short-circuit → + tree-edit-distance → ``score_calc`` sequence verbatim (only the parsing front-end + upstream of this is ours). + """ + try: + gold_exp, rep_gold = core.posify(gold_expr) + gold_exp = core.time_simplify(gold_exp) + test_exp, rep_test = core.posify(pred_expr) + test_exp = core.time_simplify(test_exp) + gold_exp = gold_exp.subs(rep_gold) + test_exp = test_exp.subs(rep_test) + zero_exp = core.time_simplify(core.expand(gold_exp - test_exp)) + if gold_exp == test_exp or zero_exp == 0: + return 100.0, 0.0, 0.0, 0.0, True, False + # SEED's Equation branch also accepts a sign-flipped residual (A == −B). + if equation and gold_exp + test_exp == 0: + return 100.0, 0.0, 0.0, 0.0, True, False + if core.time_equal(gold_exp, test_exp): + return 100.0, 0.0, 0.0, 0.0, True, False + except Exception: + return 0.0, -1.0, -1.0, -1.0, False, True + + try: + gold_tree = core.sympy_to_tree(gold_exp) + test_tree = core.sympy_to_tree(test_exp) + except Exception: + return 0.0, -1.0, -1.0, -1.0, False, True + + distance = _ext_distance(test_tree, gold_tree, core) + tree_size = float(core.calc_tree_size(gold_tree)) + rel = distance / tree_size if tree_size else 0.0 + raw = float(core.score_calc(distance, tree_size)) + return raw, rel, tree_size, distance, distance == 0, False + + +def _from_tail( + tail: tuple[float, float, float, float, bool, bool], + answer_type: str, + *, + degraded: bool = False, + units_ok: bool | None = None, + numeric_within_tol: bool | None = None, + diagnostics: tuple[str, ...] = (), +) -> CoreScore: + """Build a :class:`CoreScore` from an ``_expr_tail`` result.""" + raw, rel, size, dist, sym, tail_degraded = tail + return CoreScore( + raw=raw, + answer_type=answer_type, + relative_distance=rel, + tree_size=size, + distance=dist, + units_ok=units_ok, + symbolic_equiv=sym, + numeric_within_tol=numeric_within_tol, + degraded=degraded or tail_degraded, + diagnostics=diagnostics, + ) + + +def _score_expr_pair( + pred_sem: PhysicsAnswerSemantics, + ref_sem: PhysicsAnswerSemantics, + core: Any, + alias_map: Any, + *, + answer_type: str, + degraded: bool = False, +) -> CoreScore: + """Parse both answers to scalar exprs and run the expression tail.""" + pred_expr = _parse_scalar(pred_sem, alias_map) + gold_expr = _parse_scalar(ref_sem, alias_map) + if pred_expr is None or gold_expr is None: + return CoreScore( + raw=0.0, + answer_type=answer_type, + symbolic_equiv=False, + degraded=True, + diagnostics=("parse_failed",), + ) + return _from_tail( + _expr_tail(pred_expr, gold_expr, core), answer_type, degraded=degraded + ) + + +def _score_equation( + pred_sem: PhysicsAnswerSemantics, + ref_sem: PhysicsAnswerSemantics, + core: Any, + alias_map: Any, + *, + answer_type: str, +) -> CoreScore: + """Score a relation pair by diffing ``lhs − rhs`` residual trees (sign-robust).""" + pred_res = _relation_residual(pred_sem, alias_map) + gold_res = _relation_residual(ref_sem, alias_map) + if pred_res is None or gold_res is None: + return CoreScore( + raw=0.0, + answer_type=answer_type, + symbolic_equiv=False, + degraded=True, + diagnostics=("relation_parse_failed",), + ) + return _from_tail(_expr_tail(pred_res, gold_res, core, equation=True), answer_type) + + +# --------------------------------------------------------------------------- # +# Numeric leaf path (SEED ``numeric_score_calc``; unit alignment via our backend). +# --------------------------------------------------------------------------- # +def _align_units( + pred: NumericComparableAnswer, + ref: NumericComparableAnswer, + ctx: PhysicsQuestionSemantics, +) -> tuple[float | None, bool | None, tuple[str, ...]]: + """Align ``pred`` into ``ref``'s unit space using our unit backend (no ``pint``). + + Returns ``(aligned_value, units_ok, diagnostics)``; ``units_ok`` is ``None`` when + units do not participate, ``True`` on alignment, ``False`` on mismatch (with a + ``None`` aligned value). + """ + pred_unit = pred.unit + ref_unit = ref.unit + if pred_unit is None and ref_unit is None: + return pred.coefficient_value, None, () + + implicit_unit: str | None = None + if ( + ctx.question_unit_policy == QuestionUnitPolicy.OPTIONAL_IF_QUESTION_FIXED_UNIT + and ctx.question_unit + ): + implicit_unit = ctx.question_unit + if pred_unit is None: + pred_unit = implicit_unit + if ref_unit is None: + ref_unit = implicit_unit + if pred_unit is None or ref_unit is None: + return None, False, ("question_unit_mismatch",) + + if normalize_unit_text(pred_unit) == normalize_unit_text(ref_unit): + return pred.coefficient_value, True, () + + converted = convert_numeric_value(pred.coefficient_value, pred_unit, ref_unit) + if converted is None: + return None, False, ("unit_mismatch",) + return converted, True, () + + +def _score_numeric( + pred_sem: PhysicsAnswerSemantics, + ref_sem: PhysicsAnswerSemantics, + ctx: PhysicsQuestionSemantics, + core: Any, + alias_map: Any, +) -> CoreScore: + """Score a numeric/quantity pair via SEED's ``numeric_score_calc`` tiers.""" + pred_num = extract_numeric_comparable_answer(pred_sem, context=ctx) + ref_num = extract_numeric_comparable_answer(ref_sem, context=ctx) + # A symbolic coefficient (e.g. "3π") or a failed numeric extraction is not a clean + # rel-error comparison — defer to expression tree scoring. + if ( + pred_num is None + or ref_num is None + or pred_num.symbolic_factor_text != ref_num.symbolic_factor_text + ): + return _score_expr_pair( + pred_sem, ref_sem, core, alias_map, answer_type="Numeric", degraded=True + ) + + aligned, units_ok, unit_diag = _align_units(pred_num, ref_num, ctx) + if aligned is None: + return CoreScore( + raw=0.0, + answer_type="Numeric", + units_ok=False, + numeric_within_tol=False, + diagnostics=unit_diag, + ) + + raw = float( + core.numeric_score_calc( + core.Float(aligned), core.Float(ref_num.coefficient_value) + ) + ) + return CoreScore( + raw=raw, + answer_type="Numeric", + units_ok=units_ok, + numeric_within_tol=raw >= 100.0, + ) + + +# --------------------------------------------------------------------------- # +# Container paths (Tuple / Set / Multi-part; Interval). +# --------------------------------------------------------------------------- # +def _sort_key(sem: PhysicsAnswerSemantics) -> str: + """Canonical ordering key for set elements (stable, parse-free).""" + return sem.canonical_text or sem.raw_text or "" + + +def _score_tuple( + pred_sem: PhysicsAnswerSemantics, + ref_sem: PhysicsAnswerSemantics, + core: Any, + alias_map: Any, + *, + sort: bool, +) -> CoreScore: + """Average per-element Expression scores over a tuple/set/multi-part pair. + + SEED compares tuples positionally; for an unordered ``SET`` both sides are + canonically pre-sorted so element order is not penalized. + """ + pred_children = list(pred_sem.children) + gold_children = list(ref_sem.children) + if not pred_children or not gold_children: + return _score_expr_pair( + pred_sem, ref_sem, core, alias_map, answer_type="Tuple", degraded=True + ) + if len(pred_children) != len(gold_children): + return CoreScore( + raw=0.0, + answer_type="Tuple", + symbolic_equiv=False, + diagnostics=("tuple_arity_mismatch",), + ) + if sort: + pred_children.sort(key=_sort_key) + gold_children.sort(key=_sort_key) + + total = 0.0 + degraded = False + for pred_child, gold_child in zip(pred_children, gold_children): + element = _score_expr_pair( + pred_child, gold_child, core, alias_map, answer_type="Expression" + ) + total += element.raw + degraded = degraded or element.degraded + raw = total / len(gold_children) + return CoreScore( + raw=raw, + answer_type="Tuple", + symbolic_equiv=raw >= 100.0, + degraded=degraded, + ) + + +def _interval_expr( + sem: PhysicsAnswerSemantics, core: Any, alias_map: Any +) -> Any | None: + """Encode an interval as an order-preserving, open/closed-aware SymPy node. + + The two endpoints become parsed exprs and the bracket type becomes a marker + symbol, all under an undefined ``Interval(...)`` function the core's tree path can + diff — so two intervals match iff endpoints *and* open/closed flags agree. + """ + children = list(sem.children) + if len(children) != 2: + return None + lower = _parse_scalar(children[0], alias_map) + upper = _parse_scalar(children[1], alias_map) + if lower is None or upper is None: + return None + left = core.Symbol("IntervalOpenL" if sem.interval_open_left else "IntervalClosedL") + right = core.Symbol( + "IntervalOpenR" if sem.interval_open_right else "IntervalClosedR" + ) + return core.Function("Interval")(left, lower, upper, right) + + +def _tree_only_score(pred_expr: Any, gold_expr: Any, core: Any) -> CoreScore: + """Tree-edit score two structural exprs (no simplify; structural equality).""" + if pred_expr == gold_expr: + return CoreScore( + raw=100.0, + answer_type="Interval", + relative_distance=0.0, + tree_size=0.0, + distance=0.0, + symbolic_equiv=True, + ) + try: + gold_tree = core.sympy_to_tree(gold_expr) + test_tree = core.sympy_to_tree(pred_expr) + except Exception: + return CoreScore( + raw=0.0, answer_type="Interval", symbolic_equiv=False, degraded=True + ) + distance = _ext_distance(test_tree, gold_tree, core) + tree_size = float(core.calc_tree_size(gold_tree)) + rel = distance / tree_size if tree_size else 0.0 + raw = float(core.score_calc(distance, tree_size)) + return CoreScore( + raw=raw, + answer_type="Interval", + relative_distance=rel, + tree_size=tree_size, + distance=distance, + symbolic_equiv=distance == 0, + ) + + +def _score_interval( + pred_sem: PhysicsAnswerSemantics, + ref_sem: PhysicsAnswerSemantics, + core: Any, + alias_map: Any, +) -> CoreScore: + """Score an interval pair on endpoints + open/closed flags.""" + pred_expr = _interval_expr(pred_sem, core, alias_map) + gold_expr = _interval_expr(ref_sem, core, alias_map) + if pred_expr is None or gold_expr is None: + return _score_expr_pair( + pred_sem, ref_sem, core, alias_map, answer_type="Interval", degraded=True + ) + return _tree_only_score(pred_expr, gold_expr, core) + + +# --------------------------------------------------------------------------- # +# Public dispatch entry points (one per scorer). +# --------------------------------------------------------------------------- # +def score_eed( + pred_sem: PhysicsAnswerSemantics, + ref_sem: PhysicsAnswerSemantics, + *, + context: PhysicsQuestionSemantics | None = None, +) -> CoreScore: + """Score an applicable pair with the PHYBench-EED pure core (expression-only).""" + ctx = context or PhysicsQuestionSemantics() + alias_map = context_symbol_alias_map(ctx) + if ref_sem.object_kind == AnswerObjectKind.RELATION: + return _score_equation( + pred_sem, ref_sem, _eed_core, alias_map, answer_type="relation" + ) + return _score_expr_pair( + pred_sem, ref_sem, _eed_core, alias_map, answer_type="expression" + ) + + +def score_seed( + pred_sem: PhysicsAnswerSemantics, + ref_sem: PhysicsAnswerSemantics, + resolved_type: str, + *, + context: PhysicsQuestionSemantics | None = None, +) -> CoreScore: + """Score a pair with the CMPhysBench-SEED pure core, dispatched on ``resolved_type``.""" + ctx = context or PhysicsQuestionSemantics() + alias_map = context_symbol_alias_map(ctx) + core = _seed_core + if resolved_type == "Numeric": + return _score_numeric(pred_sem, ref_sem, ctx, core, alias_map) + if resolved_type == "Equation": + return _score_equation( + pred_sem, ref_sem, core, alias_map, answer_type="Equation" + ) + if resolved_type == "Tuple": + return _score_tuple( + pred_sem, + ref_sem, + core, + alias_map, + sort=ref_sem.structure == AnswerStructure.SET, + ) + if resolved_type == "Interval": + return _score_interval(pred_sem, ref_sem, core, alias_map) + return _score_expr_pair( + pred_sem, ref_sem, core, alias_map, answer_type="Expression" + ) + + +# --------------------------------------------------------------------------- # +# CoreScore → Verdict (shared by both semantics edit-distance scorers). +# --------------------------------------------------------------------------- # +def not_applicable_verdict(version: str, reason: str, engine: str) -> Verdict: + """Build the reserved not-applicable verdict (``score=-1.0``).""" + return Verdict( + equivalent=False, + correct=False, + score=-1.0, + comparison_mode="not_applicable", + scorer_version=version, + diagnostics=(f"not_applicable:{reason}",), + details={"front_end": "semantics", "engine": engine, "reason": reason}, + partial_credit=None, + ) + + +def verdict_from_core( + version: str, + result: CoreScore, + pred_sem: PhysicsAnswerSemantics, + *, + comparison_mode: str, +) -> Verdict: + """Map a :class:`CoreScore` onto a canonical :class:`Verdict`. + + ``raw`` is normalized to ``[0, 1]`` for both ``score`` and ``partial_credit``; + ``equivalent`` is reserved for an exact (raw ``100``) match. + """ + normalized = min(1.0, max(0.0, result.raw / 100.0)) + equivalent = result.raw >= 100.0 + extracted = pred_sem.canonical_text or pred_sem.raw_text + return Verdict( + equivalent=equivalent, + score=normalized, + comparison_mode=comparison_mode, + scorer_version=version, + diagnostics=result.diagnostics, + details={ + "front_end": "semantics", + "answer_type": result.answer_type, + "raw_score": result.raw, + "relative_distance": result.relative_distance, + "tree_size": result.tree_size, + "distance": result.distance, + "degraded": result.degraded, + }, + correct=equivalent, + units_ok=result.units_ok, + symbolic_equiv=result.symbolic_equiv, + numeric_within_tol=result.numeric_within_tol, + extracted_answer=extracted, + partial_credit=normalized, + ) + + +__all__ = [ + "CoreScore", + "STRUCTURE_TO_SEED", + "NA_STRUCTURES", + "OBJECT_KIND_TO_SEED", + "NA_KINDS", + "seed_type", + "eed_na_reason", + "score_eed", + "score_seed", + "not_applicable_verdict", + "verdict_from_core", +] diff --git a/src/prkit/scoring/partial_credit_scorer.py b/src/prkit/scoring/partial_credit_scorer.py deleted file mode 100644 index b0dc9bd..0000000 --- a/src/prkit/scoring/partial_credit_scorer.py +++ /dev/null @@ -1,249 +0,0 @@ -"""Partial-credit :class:`prkit.api.Scorer` over the EED/SEED edit-distance engine. - -``PartialCreditScorer`` is the scorer that finally populates ``Verdict.partial_credit`` -(the deterministic :class:`SemanticsScorer` is strictly binary). It wraps -:func:`prkit.semantics.edit_distance.eed_compare` — PHYBench EED + CMPhysBench SEED on -PRKit's own parser/unit substrate — and maps its graded :class:`EedResult` onto the -canonical :class:`~prkit.core.verdict.Verdict`. - -Configuration is constructor/keyword-only (no magic strings), satisfying the -``Scorer`` protocol and the sklearn "inspection" principle. ``mode`` selects whether -the graded score is surfaced (``PARTIAL_CREDIT``), collapsed to a pass/fail -(``BINARY``), or reduced to a reference-precision/symbolic tolerance check -(``TOLERANCE``). -""" - -from __future__ import annotations - -from enum import Enum -from typing import Any - -from prkit.core.domain.answer import Answer -from prkit.core.verdict import Verdict -from prkit.semantics import ( - ComparisonPolicyMode, - PhysicsAnswerSemantics, - PhysicsQuestionSemantics, - QuestionUnitPolicy, - normalize_physics_answer, -) -from prkit.semantics.edit_distance import EedConfig, EedResult, eed_compare - -#: Revision of the EED/SEED scorer wiring. Bump when the algorithm or its mapping -#: onto ``Verdict`` changes in a way that can alter scores. -ENGINE_VERSION = "1" -_VERSION = f"eed-seed/engine{ENGINE_VERSION}" - - -class PartialCreditMode(str, Enum): - """How the graded EED/SEED signal is rendered into a :class:`Verdict`.""" - - BINARY = ( - "binary" # collapse score >= binary_threshold -> 1.0/0.0, partial_credit None - ) - PARTIAL_CREDIT = "partial" # graded EED/SEED score in [0, 1], partial_credit set - TOLERANCE = "tolerance" # numeric reference-precision / symbolic equality, no grade - - def __str__(self) -> str: - return str(self.value) - - -def _coerce_context( - context: PhysicsQuestionSemantics | dict[str, Any] | None, -) -> PhysicsQuestionSemantics | None: - """Coerce an optional context into validated ``PhysicsQuestionSemantics``. - - A reference-built artifact (anything exposing ``.question_semantics``, e.g. a - ``ReferenceSemanticsArtifact``) is unwrapped to its ``q_ref`` — duck-typed so the scorer - does not depend on the heavy inference artifact type — mirroring ``SemanticsScorer`` so - both scorers accept the same context kinds. - """ - if context is None: - return None - if isinstance(context, PhysicsQuestionSemantics): - return context - question_semantics = getattr(context, "question_semantics", None) - if isinstance(question_semantics, PhysicsQuestionSemantics): - return question_semantics - return PhysicsQuestionSemantics.model_validate(context) - - -def _policy_to_str(policy_mode: ComparisonPolicyMode | str | None) -> str | None: - """Render a policy mode as a plain string for ``get_info()`` (or ``None``).""" - if policy_mode is None: - return None - if isinstance(policy_mode, ComparisonPolicyMode): - return str(policy_mode) - return str(ComparisonPolicyMode(policy_mode)) - - -class PartialCreditScorer: - """Graded EED/SEED :class:`prkit.api.Scorer`; fills ``Verdict.partial_credit``. - - Args: - mode: how the graded score is surfaced (see :class:`PartialCreditMode`). - tolerance: numeric comparison tolerance; plumbs to - ``PhysicsQuestionSemantics.tolerance``. - unit_policy: how units must appear; plumbs to - ``PhysicsQuestionSemantics.question_unit_policy``. - policy_mode: enforcement strictness accepted for facade compatibility; the - deterministic edit-distance algorithm does not branch on it, so it is - recorded in ``get_info()`` but otherwise unused. - binary_threshold: score at/above which a verdict counts as ``equivalent`` - (and, in ``BINARY`` mode, collapses to ``1.0``). - context: advanced base question semantics; instance-level ``tolerance`` / - ``unit_policy`` overrides are merged on top of it. - config: advanced edit-distance algorithm tunables. - """ - - version: str = _VERSION - - def __init__( - self, - *, - mode: PartialCreditMode = PartialCreditMode.PARTIAL_CREDIT, - tolerance: float | None = None, - unit_policy: QuestionUnitPolicy | str | None = None, - policy_mode: ComparisonPolicyMode | str | None = None, - binary_threshold: float = 1.0, - context: PhysicsQuestionSemantics | dict[str, Any] | None = None, - config: EedConfig | None = None, - ) -> None: - self._mode = mode - self._binary_threshold = float(binary_threshold) - self._policy_mode = policy_mode - self._config = config or EedConfig() - self._base_context = _coerce_context(context) - - overrides: dict[str, Any] = {} - if tolerance is not None: - overrides["tolerance"] = float(tolerance) - if unit_policy is not None: - overrides["question_unit_policy"] = ( - unit_policy - if isinstance(unit_policy, QuestionUnitPolicy) - else QuestionUnitPolicy(unit_policy) - ) - self._context_overrides = overrides - - def _effective_context( - self, call_context: PhysicsQuestionSemantics | dict[str, Any] | None - ) -> PhysicsQuestionSemantics | None: - """Merge instance knob overrides over the per-call or base context.""" - base = ( - _coerce_context(call_context) - if call_context is not None - else self._base_context - ) - if base is None: - if not self._context_overrides: - return None - base = PhysicsQuestionSemantics() - return base.merged(self._context_overrides) - - def score( - self, - prediction: Answer | str | PhysicsAnswerSemantics, - reference: Answer | str | PhysicsAnswerSemantics, - *, - context: PhysicsQuestionSemantics | dict[str, Any] | None = None, - **kwargs: Any, - ) -> Verdict: - """Score ``prediction`` against ``reference`` and return a graded Verdict. - - ``prediction`` / ``reference`` may be raw strings, :class:`Answer` objects, - or already-normalized :class:`PhysicsAnswerSemantics`. The wider input type - stays compatible with the narrower :class:`prkit.api.Scorer` protocol by - parameter contravariance. - """ - effective_context = self._effective_context(context) - pred_sem = normalize_physics_answer(prediction, context=effective_context) - ref_sem = normalize_physics_answer(reference, context=effective_context) - - result = eed_compare( - pred_sem, ref_sem, context=effective_context, config=self._config - ) - return self._verdict_from_result(result, pred_sem) - - def _verdict_from_result( - self, result: EedResult, pred_sem: PhysicsAnswerSemantics - ) -> Verdict: - """Map an :class:`EedResult` onto a :class:`Verdict`, applying the mode policy.""" - graded = result.score - equivalent = graded >= self._binary_threshold - partial_credit: float | None - - if self._mode is PartialCreditMode.PARTIAL_CREDIT: - score = graded - partial_credit = graded - elif self._mode is PartialCreditMode.BINARY: - score = 1.0 if equivalent else 0.0 - partial_credit = None - else: # TOLERANCE: pass/fail by reference precision or symbolic equality - if result.numeric_within_tol is not None: - passed = result.numeric_within_tol - elif result.symbolic_equiv is not None: - passed = result.symbolic_equiv - else: - passed = equivalent - score = 1.0 if passed else 0.0 - partial_credit = None - equivalent = passed - - extracted = pred_sem.canonical_text or pred_sem.raw_text - return Verdict( - equivalent=equivalent, - score=score, - comparison_mode=f"eed_{result.answer_type}", - scorer_version=self.version, - diagnostics=result.diagnostics, - details={ - "answer_type": result.answer_type, - "graded_score": graded, - "raw_distance": result.raw_distance, - "gt_tree_size": result.gt_tree_size, - "relative_distance": result.relative_distance, - "degraded": result.degraded, - "mode": str(self._mode), - }, - correct=equivalent, - units_ok=result.units_ok, - symbolic_equiv=result.symbolic_equiv, - numeric_within_tol=result.numeric_within_tol, - extracted_answer=extracted, - partial_credit=partial_credit, - rationale=_rationale(result, score), - ) - - def get_info(self) -> dict[str, Any]: - """Return scorer metadata; always includes ``version``.""" - unit_policy = self._context_overrides.get("question_unit_policy") - return { - "name": "PartialCreditScorer", - "version": self.version, - "engine": "eed_compare", - "deterministic": True, - "mode": str(self._mode), - "binary_threshold": self._binary_threshold, - "tolerance": self._context_overrides.get("tolerance"), - "unit_policy": (str(unit_policy) if unit_policy is not None else None), - "policy_mode": _policy_to_str(self._policy_mode), - } - - -def _rationale(result: EedResult, final_score: float) -> str: - """Build a deterministic, human-readable explanation of a graded result.""" - parts = [f"score={final_score:.3f}"] - if result.raw_distance is not None and result.gt_tree_size: - parts.append(f"tree_edit_distance={result.raw_distance:g}") - parts.append(f"gt_tree_size={result.gt_tree_size}") - if result.relative_distance is not None: - parts.append(f"rel_dist={result.relative_distance:.4f}") - if result.degraded: - parts.append("degraded") - if result.diagnostics: - parts.append(", ".join(result.diagnostics)) - return "EED/SEED " + "; ".join(parts) - - -__all__ = ["PartialCreditMode", "PartialCreditScorer"] diff --git a/src/prkit/scoring/semantics_eed_scorer.py b/src/prkit/scoring/semantics_eed_scorer.py new file mode 100644 index 0000000..a074ee1 --- /dev/null +++ b/src/prkit/scoring/semantics_eed_scorer.py @@ -0,0 +1,169 @@ +"""Our-semantics PHYBench-EED :class:`prkit.api.Scorer` (semantics front-end). + +``SemanticsEedScorer`` pairs PRKit's own answer normalization +(:func:`prkit.semantics.normalize_physics_answer`) with the front-end-free PHYBench +Expression Edit Distance **pure core** — the *our-semantics* counterpart of the +vendor-front-end :class:`prkit.scoring.EedScorer`. Varying just the front-end with +the EED algorithm fixed is the ablation it exists for. + +EED is expression-only: an ``ATOMIC`` number / quantity / expression / relation is +scored (a relation as its ``lhs − rhs`` residual); every other kind/structure has no +EED representation and yields the reserved not-applicable verdict +(``score=-1.0``, ``comparison_mode="not_applicable"``). + +Import discipline: the vendored core (and the adapter that wires it) are imported +lazily inside :meth:`score`, so re-exporting this scorer from :mod:`prkit.scoring` +keeps ``import prkit.scoring`` free of those import side effects (``pint`` never +loads on this path). +""" + +from __future__ import annotations + +from typing import Any + +from prkit.core.domain.answer import Answer +from prkit.core.verdict import Verdict +from prkit.semantics import ( + ComparisonPolicyMode, + PhysicsAnswerSemantics, + PhysicsQuestionSemantics, + QuestionUnitPolicy, + normalize_physics_answer, +) + +#: Provenance stamp: ``/@+frontend-+wrap``. +_VERSION = "eed/phybench@706feb4+frontend-semantics+wrap1" + + +def _coerce_context( + context: PhysicsQuestionSemantics | dict[str, Any] | None, +) -> PhysicsQuestionSemantics | None: + """Coerce an optional context into validated ``PhysicsQuestionSemantics``. + + Mirrors :class:`SemanticsScorer` so both scorers accept the same context kinds: a + reference-built artifact (anything exposing ``.question_semantics``) is unwrapped, + a ``PhysicsQuestionSemantics`` passes through, and a mapping is validated. + """ + if context is None: + return None + if isinstance(context, PhysicsQuestionSemantics): + return context + question_semantics = getattr(context, "question_semantics", None) + if isinstance(question_semantics, PhysicsQuestionSemantics): + return question_semantics + return PhysicsQuestionSemantics.model_validate(context) + + +def _policy_to_str(policy_mode: ComparisonPolicyMode | str | None) -> str | None: + """Render a policy mode as a plain string for ``get_info()`` (or ``None``).""" + if policy_mode is None: + return None + if isinstance(policy_mode, ComparisonPolicyMode): + return str(policy_mode) + return str(ComparisonPolicyMode(policy_mode)) + + +class SemanticsEedScorer: + """Our-semantics front-end + PHYBench-EED pure core; fills ``partial_credit``. + + Args: + tolerance: numeric comparison tolerance; plumbs to + ``PhysicsQuestionSemantics.tolerance`` (used during normalization). + unit_policy: how units must appear; plumbs to + ``PhysicsQuestionSemantics.question_unit_policy``. + policy_mode: enforcement strictness accepted for facade compatibility; the + edit-distance algorithm does not branch on it, so it is recorded in + ``get_info()`` but otherwise unused. + context: advanced base question semantics; instance-level ``tolerance`` / + ``unit_policy`` overrides are merged on top of it. + """ + + version: str = _VERSION + + def __init__( + self, + *, + tolerance: float | None = None, + unit_policy: QuestionUnitPolicy | str | None = None, + policy_mode: ComparisonPolicyMode | str | None = None, + context: PhysicsQuestionSemantics | dict[str, Any] | None = None, + ) -> None: + self._policy_mode = policy_mode + self._base_context = _coerce_context(context) + + overrides: dict[str, Any] = {} + if tolerance is not None: + overrides["tolerance"] = float(tolerance) + if unit_policy is not None: + overrides["question_unit_policy"] = ( + unit_policy + if isinstance(unit_policy, QuestionUnitPolicy) + else QuestionUnitPolicy(unit_policy) + ) + self._context_overrides = overrides + + def _effective_context( + self, call_context: PhysicsQuestionSemantics | dict[str, Any] | None + ) -> PhysicsQuestionSemantics | None: + """Merge instance knob overrides over the per-call or base context.""" + base = ( + _coerce_context(call_context) + if call_context is not None + else self._base_context + ) + if base is None: + if not self._context_overrides: + return None + base = PhysicsQuestionSemantics() + return base.merged(self._context_overrides) + + def score( + self, + prediction: Answer | str | PhysicsAnswerSemantics, + reference: Answer | str | PhysicsAnswerSemantics, + *, + context: PhysicsQuestionSemantics | dict[str, Any] | None = None, + **kwargs: Any, + ) -> Verdict: + """Score ``prediction`` against ``reference`` with our-semantics + EED core. + + Inputs may be raw strings, :class:`Answer` objects, or already-normalized + :class:`PhysicsAnswerSemantics`. A non-applicable reference kind/structure + yields the reserved ``score=-1.0`` not-applicable verdict. + """ + # Lazy: keeps the vendored core (and pint, via its lazy unit path) off + # ``import prkit.scoring``. + from ._edit_distance_adapt import ( + eed_na_reason, + not_applicable_verdict, + score_eed, + verdict_from_core, + ) + + effective_context = self._effective_context(context) + pred_sem = normalize_physics_answer(prediction, context=effective_context) + ref_sem = normalize_physics_answer(reference, context=effective_context) + + na_reason = eed_na_reason(ref_sem) + if na_reason is not None: + return not_applicable_verdict(self.version, na_reason, "phybench_eed") + + result = score_eed(pred_sem, ref_sem, context=effective_context) + return verdict_from_core(self.version, result, pred_sem, comparison_mode="eed") + + def get_info(self) -> dict[str, Any]: + """Return scorer metadata; always includes ``version``.""" + unit_policy = self._context_overrides.get("question_unit_policy") + return { + "name": "SemanticsEedScorer", + "version": self.version, + "engine": "phybench_eed", + "deterministic": True, + "front_end": "semantics", + "tolerance": self._context_overrides.get("tolerance"), + "unit_policy": (str(unit_policy) if unit_policy is not None else None), + "policy_mode": _policy_to_str(self._policy_mode), + } + + +__all__ = ["SemanticsEedScorer"] diff --git a/src/prkit/scoring/semantics_seed_scorer.py b/src/prkit/scoring/semantics_seed_scorer.py new file mode 100644 index 0000000..0f64920 --- /dev/null +++ b/src/prkit/scoring/semantics_seed_scorer.py @@ -0,0 +1,182 @@ +"""Our-semantics CMPhysBench-SEED :class:`prkit.api.Scorer` (semantics front-end). + +``SemanticsSeedScorer`` pairs PRKit's own answer normalization +(:func:`prkit.semantics.normalize_physics_answer`) with the front-end-free +CMPhysBench Scalable Expression Edit Distance **pure core** — the *our-semantics* +counterpart of the vendor-front-end :class:`prkit.scoring.SeedScorer`, and the +general (super-set) edit-distance scorer that :func:`prkit.verify.verify` selects +for ``partial_credit=True``. + +Unlike the vendor :class:`SeedScorer`, the SEED dispatch type is **derived** from +the reference's normalized ``object_kind`` + ``structure`` (the §II.8 map), not read +from a dataset annotation: containers (``TUPLE``/``SET``/``MULTI_PART`` → ``Tuple``, +``INTERVAL`` → ``Interval``), symbolic atoms (``EXPRESSION`` → ``Expression``, +``RELATION`` → ``Equation``, ``NUMBER``/``PHYSICAL_QUANTITY`` → ``Numeric``), and +everything else (``VECTOR``/``MATRIX``/``TENSOR``/``PIECEWISE`` and the non-symbolic +kinds) → the reserved not-applicable verdict (``score=-1.0``, +``comparison_mode="not_applicable"``). + +Import discipline: the vendored core (and the adapter that wires it) are imported +lazily inside :meth:`score`, so re-exporting this scorer from :mod:`prkit.scoring` +keeps ``import prkit.scoring`` free of those side effects — ``pint`` never loads on +this path (the semantics layer handles unit alignment). +""" + +from __future__ import annotations + +from typing import Any + +from prkit.core.domain.answer import Answer +from prkit.core.verdict import Verdict +from prkit.semantics import ( + ComparisonPolicyMode, + PhysicsAnswerSemantics, + PhysicsQuestionSemantics, + QuestionUnitPolicy, + normalize_physics_answer, +) + +#: Provenance stamp: ``/@+frontend-+wrap``. +_VERSION = "seed/cmphysbench@b2cd857+frontend-semantics+wrap1" + + +def _coerce_context( + context: PhysicsQuestionSemantics | dict[str, Any] | None, +) -> PhysicsQuestionSemantics | None: + """Coerce an optional context into validated ``PhysicsQuestionSemantics``. + + Mirrors :class:`SemanticsScorer` so both scorers accept the same context kinds: a + reference-built artifact (anything exposing ``.question_semantics``) is unwrapped, + a ``PhysicsQuestionSemantics`` passes through, and a mapping is validated. + """ + if context is None: + return None + if isinstance(context, PhysicsQuestionSemantics): + return context + question_semantics = getattr(context, "question_semantics", None) + if isinstance(question_semantics, PhysicsQuestionSemantics): + return question_semantics + return PhysicsQuestionSemantics.model_validate(context) + + +def _policy_to_str(policy_mode: ComparisonPolicyMode | str | None) -> str | None: + """Render a policy mode as a plain string for ``get_info()`` (or ``None``).""" + if policy_mode is None: + return None + if isinstance(policy_mode, ComparisonPolicyMode): + return str(policy_mode) + return str(ComparisonPolicyMode(policy_mode)) + + +class SemanticsSeedScorer: + """Our-semantics front-end + CMPhysBench-SEED pure core; fills ``partial_credit``. + + Args: + tolerance: numeric comparison tolerance; plumbs to + ``PhysicsQuestionSemantics.tolerance`` (used during normalization). + unit_policy: how units must appear; plumbs to + ``PhysicsQuestionSemantics.question_unit_policy``. + policy_mode: enforcement strictness accepted for facade compatibility; the + edit-distance algorithm does not branch on it, so it is recorded in + ``get_info()`` but otherwise unused. + context: advanced base question semantics; instance-level ``tolerance`` / + ``unit_policy`` overrides are merged on top of it. + """ + + version: str = _VERSION + + def __init__( + self, + *, + tolerance: float | None = None, + unit_policy: QuestionUnitPolicy | str | None = None, + policy_mode: ComparisonPolicyMode | str | None = None, + context: PhysicsQuestionSemantics | dict[str, Any] | None = None, + ) -> None: + self._policy_mode = policy_mode + self._base_context = _coerce_context(context) + + overrides: dict[str, Any] = {} + if tolerance is not None: + overrides["tolerance"] = float(tolerance) + if unit_policy is not None: + overrides["question_unit_policy"] = ( + unit_policy + if isinstance(unit_policy, QuestionUnitPolicy) + else QuestionUnitPolicy(unit_policy) + ) + self._context_overrides = overrides + + def _effective_context( + self, call_context: PhysicsQuestionSemantics | dict[str, Any] | None + ) -> PhysicsQuestionSemantics | None: + """Merge instance knob overrides over the per-call or base context.""" + base = ( + _coerce_context(call_context) + if call_context is not None + else self._base_context + ) + if base is None: + if not self._context_overrides: + return None + base = PhysicsQuestionSemantics() + return base.merged(self._context_overrides) + + def score( + self, + prediction: Answer | str | PhysicsAnswerSemantics, + reference: Answer | str | PhysicsAnswerSemantics, + *, + context: PhysicsQuestionSemantics | dict[str, Any] | None = None, + **kwargs: Any, + ) -> Verdict: + """Score ``prediction`` against ``reference`` with our-semantics + SEED core. + + Inputs may be raw strings, :class:`Answer` objects, or already-normalized + :class:`PhysicsAnswerSemantics`. A non-applicable reference kind/structure + yields the reserved ``score=-1.0`` not-applicable verdict; otherwise the SEED + dispatch type is derived from the reference and the matching core path runs. + """ + # Lazy: keeps the vendored core (and pint, via its lazy unit path) off + # ``import prkit.scoring``. + from ._edit_distance_adapt import ( + not_applicable_verdict, + score_seed, + seed_type, + verdict_from_core, + ) + + effective_context = self._effective_context(context) + pred_sem = normalize_physics_answer(prediction, context=effective_context) + ref_sem = normalize_physics_answer(reference, context=effective_context) + + resolved_type, na_reason = seed_type(ref_sem) + if resolved_type is None: + return not_applicable_verdict( + self.version, na_reason or str(ref_sem.object_kind), "cmphysbench_seed" + ) + + result = score_seed(pred_sem, ref_sem, resolved_type, context=effective_context) + return verdict_from_core( + self.version, + result, + pred_sem, + comparison_mode=f"seed:{resolved_type}", + ) + + def get_info(self) -> dict[str, Any]: + """Return scorer metadata; always includes ``version``.""" + unit_policy = self._context_overrides.get("question_unit_policy") + return { + "name": "SemanticsSeedScorer", + "version": self.version, + "engine": "cmphysbench_seed", + "deterministic": True, + "front_end": "semantics", + "tolerance": self._context_overrides.get("tolerance"), + "unit_policy": (str(unit_policy) if unit_policy is not None else None), + "policy_mode": _policy_to_str(self._policy_mode), + } + + +__all__ = ["SemanticsSeedScorer"] diff --git a/src/prkit/semantics/edit_distance/__init__.py b/src/prkit/semantics/edit_distance/__init__.py index dc597d4..d5a61b4 100644 --- a/src/prkit/semantics/edit_distance/__init__.py +++ b/src/prkit/semantics/edit_distance/__init__.py @@ -1,16 +1,19 @@ """SEED dispatch (``eed_compare``) — the physics-aware glue over the EED core. -This subpackage holds only the integration layer: :mod:`.pipeline` reproduces -CMPhysBench SEED's answer-kind dispatch on top of PRKit's normalized -:class:`~prkit.semantics.PhysicsAnswerSemantics` (numeric/unit/symbolic primitives -from :mod:`prkit.semantics.comparison`). It depends on the semantics layer **by -design**. +PARKED REFERENCE (``seed-reimpl``). This subpackage holds only the integration +layer: :mod:`.pipeline` reproduces CMPhysBench SEED's answer-kind dispatch on top of +PRKit's normalized :class:`~prkit.semantics.PhysicsAnswerSemantics` (numeric/unit/ +symbolic primitives from :mod:`prkit.semantics.comparison`). It depends on the +semantics layer **by design**. The pure tree-edit algorithm core (the related-work method, with no semantics -dependency) lives in :mod:`prkit.evaluation.edit_distance`, which this module imports. +dependency) lives in :mod:`prkit.evaluation.edit_distance` (the ``eed-reimpl``), which +this module imports. -See :class:`prkit.scoring.PartialCreditScorer` for the ``Scorer`` wrapper that maps -:class:`EedResult` onto :class:`prkit.core.verdict.Verdict`. +It is **no longer wired to any** ``Scorer``: the shipped graded scorers +(:class:`prkit.scoring.SemanticsEedScorer` / :class:`~prkit.scoring.SemanticsSeedScorer`) +run our-semantics over the *vendored* PHYBench-EED / CMPhysBench-SEED pure cores +instead. This reimpl is retained for differential testing against those cores. """ from __future__ import annotations diff --git a/src/prkit/semantics/edit_distance/pipeline.py b/src/prkit/semantics/edit_distance/pipeline.py index 63a48f8..b1e8107 100644 --- a/src/prkit/semantics/edit_distance/pipeline.py +++ b/src/prkit/semantics/edit_distance/pipeline.py @@ -1,5 +1,14 @@ """SEED-style per-pair dispatch that ties the EED algorithm to PRKit's substrate. +PARKED REFERENCE (``seed-reimpl``) — not wired to any ``Scorer``. After +``PartialCreditScorer`` was replaced by the +:class:`~prkit.scoring.SemanticsSeedScorer` (our-semantics front-end over the +*vendored* CMPhysBench-SEED pure core), this module's ``eed_compare`` is no longer +on any scoring path. It is retained for differential testing against that vendored +core and as a readable record of the SEED dispatch. It stays under ``semantics/`` +(not ``evaluation/``) because it imports ``prkit.semantics.comparison.*``; relocating +it would create a package-level ``evaluation⇄semantics`` import cycle. + ``eed_compare`` reproduces CMPhysBench SEED's answer-type dispatch on top of PRKit's existing parser / unit backend instead of vendoring ``latex2sympy2`` + ``pint``: diff --git a/src/prkit/testing/conformance.py b/src/prkit/testing/conformance.py index 910e629..f2b4491 100644 --- a/src/prkit/testing/conformance.py +++ b/src/prkit/testing/conformance.py @@ -145,6 +145,12 @@ def check_scorer( matches; for each case a frozen :class:`Verdict` with ``score`` in ``[0, 1]``, propagated ``scorer_version``, expected ``equivalent``; determinism; identity. + A ``score`` of ``-1.0`` is the reserved *not-applicable* sentinel (a kind/structure + with no SEED type, ``comparison_mode == "not_applicable"``): it is accepted by the + range check, and the ``equivalent``/identity expectations are skipped for it (an + N/A answer is neither a match nor a mismatch). Pass an expression-only ``cases`` + battery to a semantics edit-distance scorer to avoid N/A cases entirely. + Raises: AssertionError: on any non-conformance. """ @@ -166,17 +172,23 @@ def check_scorer( assert isinstance( verdict, Verdict ), f"score({pred!r}, {ref!r}) must return a Verdict, got {type(verdict)!r}" + not_applicable = ( + verdict.score == -1.0 or verdict.comparison_mode == "not_applicable" + ) assert ( - 0.0 <= verdict.score <= 1.0 + not_applicable or 0.0 <= verdict.score <= 1.0 ), f"score out of range for ({pred!r}, {ref!r}): {verdict.score!r}" assert verdict.scorer_version == scorer.version, ( f"verdict.scorer_version {verdict.scorer_version!r} != scorer.version " f"{scorer.version!r}" ) - assert verdict.equivalent is expect, ( - f"score({pred!r}, {ref!r}).equivalent expected {expect}, " - f"got {verdict.equivalent}" - ) + # An N/A verdict is neither a match nor a mismatch — skip the equivalence + # expectation for it (the identity check below skips it too). + if not not_applicable: + assert verdict.equivalent is expect, ( + f"score({pred!r}, {ref!r}).equivalent expected {expect}, " + f"got {verdict.equivalent}" + ) # Determinism: same inputs -> equal Verdict. assert ( scorer.score(pred, ref) == verdict @@ -185,6 +197,8 @@ def check_scorer( for pred in seen_predictions: identity = scorer.score(pred, pred) + if identity.score == -1.0 or identity.comparison_mode == "not_applicable": + continue # N/A kinds have no identity expectation assert ( identity.equivalent is True ), f"identity case failed: score({pred!r}, {pred!r}).equivalent is not True" diff --git a/src/prkit/verify/__init__.py b/src/prkit/verify/__init__.py index 718b4e7..0187e06 100644 --- a/src/prkit/verify/__init__.py +++ b/src/prkit/verify/__init__.py @@ -92,8 +92,9 @@ def verify( tolerance: numeric comparison tolerance (engine default when ``None``). unit_policy: enforcement strictness — one of ``"strict"`` / ``"audited"`` / ``"permissive"`` (maps to the engine's ``ComparisonPolicyMode``). - partial_credit: when ``True``, score with the graded EED/SEED - :class:`~prkit.scoring.PartialCreditScorer` (which populates + partial_credit: when ``True``, score with the graded + :class:`~prkit.scoring.SemanticsSeedScorer` (our-semantics front-end over + the CMPhysBench-SEED edit-distance core, which populates ``Verdict.partial_credit``) instead of the binary deterministic engine. context: optional question contract (``q_ref``) supplying the judgement with the question's domain/policy fields — e.g. ``symbol_assumptions`` that @@ -119,9 +120,9 @@ def verify( # math-verify is verify(gold, pred); the Scorer scores prediction vs reference, # so prediction=pred and reference=gold — do not swap. if partial_credit: - from prkit.scoring import PartialCreditScorer + from prkit.scoring import SemanticsSeedScorer - pc_scorer = PartialCreditScorer(tolerance=tolerance, policy_mode=unit_policy) + pc_scorer = SemanticsSeedScorer(tolerance=tolerance, policy_mode=unit_policy) return pc_scorer.score(pred, gold, context=question_context) from prkit.scoring import SemanticsScorer diff --git a/tests/prkit/core/test_verdict.py b/tests/prkit/core/test_verdict.py index e5f23e9..7c85922 100644 --- a/tests/prkit/core/test_verdict.py +++ b/tests/prkit/core/test_verdict.py @@ -36,15 +36,29 @@ def test_all_fields(self): class TestScoreValidator: - @pytest.mark.parametrize("score", [0.0, 0.5, 1.0]) + @pytest.mark.parametrize("score", [0.0, 0.5, 1.0, -1.0]) def test_in_range_accepted(self, score): + # -1.0 is the reserved not-applicable sentinel (accepted alongside [0, 1]). v = Verdict( equivalent=True, score=score, comparison_mode="number", scorer_version="x" ) assert v.score == score - @pytest.mark.parametrize("score", [-0.01, 1.01, 2.0, -1.0]) + def test_na_sentinel_round_trips(self): + v = Verdict( + equivalent=False, + correct=False, + score=-1.0, + comparison_mode="not_applicable", + scorer_version="x", + ) + assert v.score == -1.0 + # Survives a dump/reload round trip (the validator runs again on load). + assert Verdict.model_validate(v.model_dump()).score == -1.0 + + @pytest.mark.parametrize("score", [-0.01, 1.01, 1.5, 2.0, -0.5, -2.0]) def test_out_of_range_rejected(self, score): + # Negatives other than the -1.0 sentinel stay rejected. with pytest.raises(ValidationError): Verdict( equivalent=True, diff --git a/tests/prkit/scoring/test_partial_credit_scorer.py b/tests/prkit/scoring/test_partial_credit_scorer.py deleted file mode 100644 index 43fc10a..0000000 --- a/tests/prkit/scoring/test_partial_credit_scorer.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Tests for :class:`prkit.scoring.PartialCreditScorer` and its Verdict mapping.""" - -from __future__ import annotations - -import pytest - -from prkit.api import Scorer, Verdict -from prkit.scoring import PartialCreditMode, PartialCreditScorer -from prkit.testing import check_scorer - - -class TestProtocol: - def test_satisfies_scorer_protocol(self) -> None: - scorer = PartialCreditScorer() - assert isinstance(scorer, Scorer) - assert scorer.version - assert scorer.get_info()["version"] == scorer.version - - @pytest.mark.parametrize( - "mode", - [ - PartialCreditMode.PARTIAL_CREDIT, - PartialCreditMode.BINARY, - PartialCreditMode.TOLERANCE, - ], - ) - def test_conformance_battery(self, mode: PartialCreditMode) -> None: - check_scorer(PartialCreditScorer(mode=mode)) - - -class TestPartialCreditMode: - def test_near_miss_is_graded_and_fills_partial_credit(self) -> None: - scorer = PartialCreditScorer() - verdict = scorer.score("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") - assert 0.0 < verdict.score < 1.0 - assert verdict.partial_credit == verdict.score - assert verdict.equivalent is False - assert verdict.rationale is not None - - def test_exact_match_full_credit(self) -> None: - verdict = PartialCreditScorer().score("3 m/s", "3 m/s") - assert verdict.score == 1.0 - assert verdict.partial_credit == 1.0 - assert verdict.equivalent is True - assert verdict.units_ok is True - - def test_returns_canonical_verdict(self) -> None: - verdict = PartialCreditScorer().score("F = 2*m*a", "F = m*a") - assert isinstance(verdict, Verdict) - assert verdict.comparison_mode == "eed_relation" - assert verdict.scorer_version == PartialCreditScorer.version - - -class TestBinaryMode: - def test_collapses_to_pass_fail_and_nulls_partial_credit(self) -> None: - scorer = PartialCreditScorer(mode=PartialCreditMode.BINARY) - near_miss = scorer.score("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") - assert near_miss.score == 0.0 - assert near_miss.partial_credit is None - assert near_miss.equivalent is False - - def test_threshold_controls_equivalence(self) -> None: - lenient = PartialCreditScorer( - mode=PartialCreditMode.BINARY, binary_threshold=0.4 - ) - verdict = lenient.score("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") - assert verdict.equivalent is True - assert verdict.score == 1.0 - - -class TestToleranceMode: - def test_numeric_within_reference_precision_passes(self) -> None: - scorer = PartialCreditScorer(mode=PartialCreditMode.TOLERANCE) - verdict = scorer.score("3.005", "3") - assert verdict.score == 1.0 - assert verdict.partial_credit is None - assert verdict.equivalent is True - - def test_expression_uses_symbolic_equivalence(self) -> None: - scorer = PartialCreditScorer(mode=PartialCreditMode.TOLERANCE) - assert scorer.score("x + y", "y + x").equivalent is True - assert scorer.score("x + y", "x - y").equivalent is False - - -class TestGuards: - def test_empty_prediction_surfaces_diagnostics(self) -> None: - verdict = PartialCreditScorer().score("", "3") - assert verdict.score == 0.0 - assert verdict.partial_credit == 0.0 - assert "empty_prediction" in verdict.diagnostics - assert "empty_prediction" in verdict.rationale - - -class TestDeterminism: - def test_repeated_score_is_equal(self) -> None: - scorer = PartialCreditScorer() - first = scorer.score("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") - second = scorer.score("2*m*g + 4*m*v0**2/l", "2*m*g + 2*m*v0**2/l") - assert first == second - - -class TestConfig: - def test_tolerance_passthrough_flips_near_miss(self) -> None: - # In TOLERANCE mode the pass/fail uses the reference-precision check, which - # honors the configured tolerance; a generous value flips a clear near-miss. - strict = PartialCreditScorer(mode=PartialCreditMode.TOLERANCE).score( - "150", "100" - ) - loose = PartialCreditScorer( - mode=PartialCreditMode.TOLERANCE, tolerance=0.6 - ).score("150", "100") - assert strict.equivalent is False - assert loose.equivalent is True - - def test_get_info_reports_configuration(self) -> None: - info = PartialCreditScorer(mode=PartialCreditMode.BINARY).get_info() - assert info["mode"] == "binary" - assert info["engine"] == "eed_compare" - assert info["deterministic"] is True - - def test_context_dict_and_policies_reported(self) -> None: - scorer = PartialCreditScorer( - tolerance=0.01, - unit_policy="required", - policy_mode="strict", - context={"target_variable": "x"}, # dict context exercises coercion - ) - info = scorer.get_info() - assert info["unit_policy"] == "required" - assert info["policy_mode"] == "strict" - assert info["tolerance"] == 0.01 - assert isinstance(scorer.score("3 m/s", "3 m/s"), Verdict) diff --git a/tests/prkit/scoring/test_semantics_eed_scorer.py b/tests/prkit/scoring/test_semantics_eed_scorer.py new file mode 100644 index 0000000..325b030 --- /dev/null +++ b/tests/prkit/scoring/test_semantics_eed_scorer.py @@ -0,0 +1,89 @@ +"""Tests for :class:`prkit.scoring.SemanticsEedScorer` and its Verdict mapping.""" + +from __future__ import annotations + +from prkit.api import Scorer, Verdict +from prkit.core.domain import AnswerObjectKind, AnswerStructure +from prkit.core.domain.answer import Answer +from prkit.scoring import SemanticsEedScorer +from prkit.semantics import PhysicsAnswerSemantics +from prkit.testing import check_scorer + +_CASES = [ + ("3 m/s", "3 m/s", True), + ("x+1", "1+x", True), + ("x+1", "x+2", False), +] + + +class TestProtocol: + def test_satisfies_scorer_protocol(self) -> None: + scorer = SemanticsEedScorer() + assert isinstance(scorer, Scorer) + assert scorer.version + assert scorer.get_info()["version"] == scorer.version + + def test_conformance_battery(self) -> None: + check_scorer(SemanticsEedScorer(), cases=_CASES) + + def test_get_info_reports_semantics_front_end(self) -> None: + info = SemanticsEedScorer().get_info() + assert info["name"] == "SemanticsEedScorer" + assert info["engine"] == "phybench_eed" + assert info["front_end"] == "semantics" + assert info["deterministic"] is True + + +class TestScoring: + def test_exact_match_full_credit(self) -> None: + verdict = SemanticsEedScorer().score("x+1", "1+x") + assert verdict.score == 1.0 + assert verdict.partial_credit == 1.0 + assert verdict.equivalent is True + + def test_near_miss_is_graded(self) -> None: + verdict = SemanticsEedScorer().score("x+1", "x+2") + assert 0.0 < verdict.score < 1.0 + assert verdict.partial_credit == verdict.score + assert verdict.equivalent is False + + def test_relation_residual_is_sign_robust(self) -> None: + verdict = SemanticsEedScorer().score("F = m*a", "m*a = F") + assert verdict.score == 1.0 + assert verdict.equivalent is True + + def test_returns_canonical_verdict(self) -> None: + verdict = SemanticsEedScorer().score("x+1", "x+2") + assert isinstance(verdict, Verdict) + assert verdict.comparison_mode == "eed" + assert verdict.scorer_version == SemanticsEedScorer.version + assert verdict.details["front_end"] == "semantics" + + +class TestNotApplicable: + def test_choice_answer_is_not_applicable(self) -> None: + verdict = SemanticsEedScorer().score(Answer(value="A"), Answer(value="B")) + assert verdict.score == -1.0 + assert verdict.correct is False + assert verdict.equivalent is False + assert verdict.comparison_mode == "not_applicable" + assert verdict.partial_credit is None + + def test_vector_structure_is_not_applicable(self) -> None: + vec = PhysicsAnswerSemantics( + canonical_text="(1, 0, 0)", + object_kind=AnswerObjectKind.EXPRESSION, + structure=AnswerStructure.VECTOR, + ) + verdict = SemanticsEedScorer().score(vec, vec) + assert verdict.score == -1.0 + assert verdict.comparison_mode == "not_applicable" + assert verdict.diagnostics == ("not_applicable:vector",) + + +class TestDeterminism: + def test_repeated_score_is_equal(self) -> None: + scorer = SemanticsEedScorer() + first = scorer.score("x+1", "x+2") + second = scorer.score("x+1", "x+2") + assert first == second diff --git a/tests/prkit/scoring/test_semantics_seed_scorer.py b/tests/prkit/scoring/test_semantics_seed_scorer.py new file mode 100644 index 0000000..f39bfa8 --- /dev/null +++ b/tests/prkit/scoring/test_semantics_seed_scorer.py @@ -0,0 +1,123 @@ +"""Tests for :class:`prkit.scoring.SemanticsSeedScorer` and its Verdict mapping.""" + +from __future__ import annotations + +from prkit.api import Scorer +from prkit.core.domain import AnswerObjectKind, AnswerStructure +from prkit.core.domain.answer import Answer +from prkit.scoring import SemanticsSeedScorer +from prkit.semantics import PhysicsAnswerSemantics +from prkit.testing import check_scorer + +_CASES = [ + ("3 m/s", "3 m/s", True), + ("x+1", "1+x", True), + ("x+1", "x+2", False), +] + + +class TestProtocol: + def test_satisfies_scorer_protocol(self) -> None: + scorer = SemanticsSeedScorer() + assert isinstance(scorer, Scorer) + assert scorer.version + assert scorer.get_info()["version"] == scorer.version + + def test_conformance_battery(self) -> None: + check_scorer(SemanticsSeedScorer(), cases=_CASES) + + def test_get_info_reports_semantics_front_end(self) -> None: + info = SemanticsSeedScorer().get_info() + assert info["name"] == "SemanticsSeedScorer" + assert info["engine"] == "cmphysbench_seed" + assert info["front_end"] == "semantics" + assert info["deterministic"] is True + + +class TestExpressionAndEquation: + def test_expression_exact_match(self) -> None: + verdict = SemanticsSeedScorer().score("x+1", "1+x") + assert verdict.score == 1.0 + assert verdict.equivalent is True + assert verdict.comparison_mode == "seed:Expression" + + def test_equation_is_sign_robust(self) -> None: + verdict = SemanticsSeedScorer().score("F = m*a", "m*a = F") + assert verdict.score == 1.0 + assert verdict.equivalent is True + assert verdict.comparison_mode == "seed:Equation" + + def test_equation_near_miss_is_graded(self) -> None: + verdict = SemanticsSeedScorer().score("F = 2*m*a", "F = m*a") + assert 0.0 < verdict.score < 1.0 + assert verdict.partial_credit == verdict.score + assert verdict.equivalent is False + + +class TestNumeric: + def test_within_tightest_tier_is_equivalent(self) -> None: + verdict = SemanticsSeedScorer().score("9.81 m/s^2", "9.8 m/s^2") + assert verdict.score == 1.0 + assert verdict.equivalent is True + assert verdict.units_ok is True + assert verdict.comparison_mode == "seed:Numeric" + + def test_outer_tier_is_partial(self) -> None: + verdict = SemanticsSeedScorer().score("103", "100") + assert verdict.score == 0.8 + assert verdict.equivalent is False + assert verdict.numeric_within_tol is False + + def test_unit_mismatch_scores_zero(self) -> None: + verdict = SemanticsSeedScorer().score("3 m", "3 s") + assert verdict.score == 0.0 + assert verdict.units_ok is False + assert verdict.equivalent is False + + +class TestContainers: + def test_tuple_exact_and_partial(self) -> None: + scorer = SemanticsSeedScorer() + assert scorer.score("(1, 2)", "(1, 2)").score == 1.0 + partial = scorer.score("(1, 2)", "(1, 3)") + assert 0.0 < partial.score < 1.0 + assert partial.comparison_mode == "seed:Tuple" + + def test_set_is_order_insensitive(self) -> None: + verdict = SemanticsSeedScorer().score("{1, 2}", "{2, 1}") + assert verdict.score == 1.0 + assert verdict.equivalent is True + + def test_interval_exact_match(self) -> None: + verdict = SemanticsSeedScorer().score("[0, 5]", "[0, 5]") + assert verdict.score == 1.0 + assert verdict.equivalent is True + assert verdict.comparison_mode == "seed:Interval" + + +class TestNotApplicable: + def test_choice_answer_is_not_applicable(self) -> None: + verdict = SemanticsSeedScorer().score(Answer(value="A"), Answer(value="B")) + assert verdict.score == -1.0 + assert verdict.correct is False + assert verdict.equivalent is False + assert verdict.comparison_mode == "not_applicable" + assert verdict.partial_credit is None + + def test_matrix_structure_is_not_applicable(self) -> None: + mat = PhysicsAnswerSemantics( + canonical_text="[[1, 0], [0, 1]]", + object_kind=AnswerObjectKind.EXPRESSION, + structure=AnswerStructure.MATRIX, + ) + verdict = SemanticsSeedScorer().score(mat, mat) + assert verdict.score == -1.0 + assert verdict.diagnostics == ("not_applicable:matrix",) + + +class TestDeterminism: + def test_repeated_score_is_equal(self) -> None: + scorer = SemanticsSeedScorer() + first = scorer.score("F = 2*m*a", "F = m*a") + second = scorer.score("F = 2*m*a", "F = m*a") + assert first == second diff --git a/tests/prkit/test_conformance.py b/tests/prkit/test_conformance.py index 638f79d..027b7b1 100644 --- a/tests/prkit/test_conformance.py +++ b/tests/prkit/test_conformance.py @@ -16,12 +16,21 @@ from prkit.datasets.hub import DatasetHub from prkit.scoring import ( EedScorer, - PartialCreditScorer, SeedScorer, + SemanticsEedScorer, SemanticsScorer, + SemanticsSeedScorer, ) from prkit.testing import check_dataset, check_model_client, check_scorer +#: Expression/number-only battery for the semantics edit-distance scorers (the +#: default battery has a CHOICE case that normalizes to a not-applicable verdict). +_SEMANTICS_EDIT_DISTANCE_CASES = [ + ("3 m/s", "3 m/s", True), + ("x+1", "1+x", True), + ("x+1", "x+2", False), # genuine expression mismatch (bare "x" alone is N/A) +] + class _StubClient(BaseModelClient): """Offline client with no native structured output (base defaults).""" @@ -47,8 +56,12 @@ def test_reference_scorer_conforms(): check_scorer(SemanticsScorer()) -def test_partial_credit_scorer_conforms(): - check_scorer(PartialCreditScorer()) +def test_semantics_eed_scorer_conforms(): + check_scorer(SemanticsEedScorer(), cases=_SEMANTICS_EDIT_DISTANCE_CASES) + + +def test_semantics_seed_scorer_conforms(): + check_scorer(SemanticsSeedScorer(), cases=_SEMANTICS_EDIT_DISTANCE_CASES) def test_eed_scorer_conforms(): From 775b2793c1f3c798e98c8c7d85cee75623bf2c15 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sun, 21 Jun 2026 17:34:26 -0400 Subject: [PATCH 24/28] Load only the toolkit's own .env via the nearest pyproject ancestor Replace the consumer-repo-locating machinery in core/project_env.py (the PRKIT_TOOLKIT_ROOT env var, the src/prkit marker walk, and the named-sibling fallback) with a single nearest-ancestor-with-pyproject.toml lookup, so the toolkit resolves only its own project .env and never reaches into a sibling or consumer repository. Off-repo callers (e.g. an installed wheel with no pyproject.toml ancestor) get a best-effort empty result. find_toolkit_root is dropped from the public surface; load_project_dotenv and ensure_openai_api_key keep their signatures, so the model-client and llm-judge importers are unaffected. The tests are reworked to key off pyproject.toml. Co-Authored-By: Claude Opus 4.8 --- src/prkit/core/project_env.py | 85 ++++++++-------------------- tests/prkit/core/test_project_env.py | 77 +++++++++++-------------- 2 files changed, 59 insertions(+), 103 deletions(-) diff --git a/src/prkit/core/project_env.py b/src/prkit/core/project_env.py index d6e5663..8379abc 100644 --- a/src/prkit/core/project_env.py +++ b/src/prkit/core/project_env.py @@ -1,81 +1,47 @@ -"""Helpers for loading project-local environment files with deterministic precedence.""" +"""Helpers for loading the toolkit's own project ``.env`` with deterministic precedence. + +The toolkit loads only its *own* project ``.env`` — the one beside its +``pyproject.toml``. It deliberately does **not** locate or read consumer repositories' +environment files (that would violate toolkit-independence): a consumer is responsible +for loading its own ``.env`` before calling into the toolkit (via each consumer's +``scripts/project_env.py`` bridge). +""" from __future__ import annotations import os -from collections.abc import Iterator from os import PathLike from pathlib import Path -_TOOLKIT_ENV_VAR = "PRKIT_TOOLKIT_ROOT" - - -def _anchor_dir(anchor: str | PathLike[str] | Path | None = None) -> Path: - """Return the resolved directory for *anchor*, defaulting to this file's directory.""" - target = Path(anchor).resolve() if anchor is not None else Path(__file__).resolve() - return target if target.is_dir() else target.parent - -def _iter_search_dirs( +def _nearest_pyproject_dir( anchor: str | PathLike[str] | Path | None = None, -) -> Iterator[Path]: - """Yield the anchor directory and all of its ancestors, bottom-up.""" - start = _anchor_dir(anchor) - yield start - yield from start.parents - - -def _resolve_env_root(env_var: str, *, marker_relpath: tuple[str, ...]) -> Path | None: - """Return the path in *env_var* if it exists and contains the marker file, else ``None``.""" - raw = os.environ.get(env_var) - if not raw: - return None - candidate = Path(raw).expanduser().resolve() - if (candidate / Path(*marker_relpath)).exists(): - return candidate - return None - - -def _find_named_sibling( - anchor: str | PathLike[str] | Path | None, - sibling_name: str, - *, - marker_relpath: tuple[str, ...], ) -> Path | None: - """Walk up from *anchor* looking for a sibling directory named *sibling_name* that contains the marker path.""" - for candidate in _iter_search_dirs(anchor): - sibling = candidate / sibling_name - if (sibling / Path(*marker_relpath)).exists(): - return sibling - return None - - -def find_toolkit_root(anchor: str | PathLike[str] | Path | None = None) -> Path | None: - """Return the toolkit repo root for nested or sibling repo layouts.""" - env_root = _resolve_env_root(_TOOLKIT_ENV_VAR, marker_relpath=("src", "prkit")) - if env_root is not None: - return env_root + """Return the nearest ancestor directory containing ``pyproject.toml``. - for candidate in _iter_search_dirs(anchor): - if (candidate / "src" / "prkit").is_dir(): + Resolves *anchor* (defaulting to this module's location) to a directory and walks + upward. Returns ``None`` when no ancestor holds a ``pyproject.toml`` (e.g. an + installed wheel), so off-repo callers get a best-effort empty result rather than + reaching into an unrelated directory. + """ + target = Path(anchor).resolve() if anchor is not None else Path(__file__).resolve() + start = target if target.is_dir() else target.parent + for candidate in (start, *start.parents): + if (candidate / "pyproject.toml").is_file(): return candidate - - return _find_named_sibling( - anchor, - "physical_reasoning_toolkit", - marker_relpath=("src", "prkit"), - ) + return None def project_dotenv_paths( anchor: str | PathLike[str] | Path | None = None, ) -> tuple[Path, ...]: - """Return the toolkit's own `.env` path, when present. + """Return the toolkit's own ``.env`` path, when present. - The toolkit loads only its own project `.env`. Consumer repositories are - responsible for locating and loading their own environment files. + The toolkit's project root is the nearest ancestor of *anchor* holding a + ``pyproject.toml``; its ``.env`` (when a file) is the only env file returned. + Consumer repositories load their own environment files themselves. """ - toolkit_root = find_toolkit_root(anchor) + toolkit_root = _nearest_pyproject_dir(anchor) if toolkit_root is not None: repo_env = toolkit_root / ".env" if repo_env.is_file(): @@ -127,7 +93,6 @@ def ensure_openai_api_key( __all__ = [ "ensure_openai_api_key", - "find_toolkit_root", "load_project_dotenv", "project_dotenv_paths", ] diff --git a/tests/prkit/core/test_project_env.py b/tests/prkit/core/test_project_env.py index 1b0a887..c5bfeef 100644 --- a/tests/prkit/core/test_project_env.py +++ b/tests/prkit/core/test_project_env.py @@ -1,7 +1,8 @@ """Tests for project-local environment loading helpers. -The toolkit loads only its OWN ``.env``; it must never reach into consumer repos. -Consumer-side ``.env`` precedence is tested in the consumer repositories instead. +The toolkit loads only its OWN ``.env`` — the one beside its ``pyproject.toml`` — and +must never reach into consumer repos. Consumer-side ``.env`` precedence is tested in +the consumer repositories instead. """ from __future__ import annotations @@ -18,74 +19,64 @@ ) -def _make_toolkit_layout(tmp_path: Path) -> tuple[Path, Path, Path]: - """Build a toolkit root with a nested consumer repo (used as the ignored sibling).""" +def _make_toolkit_layout(tmp_path: Path) -> tuple[Path, Path]: + """Build a toolkit root (marked by ``pyproject.toml``) with a nested anchor dir.""" toolkit_root = tmp_path / "toolkit" - (toolkit_root / "src" / "prkit").mkdir(parents=True) - consumer_root = toolkit_root / "consumer_repo" - (consumer_root / "scripts").mkdir(parents=True) - toolkit_anchor = toolkit_root / "src" / "prkit" - return toolkit_root, consumer_root, toolkit_anchor + anchor = toolkit_root / "src" / "prkit" / "core" + anchor.mkdir(parents=True) + (toolkit_root / "pyproject.toml").write_text("[project]\n", encoding="utf-8") + return toolkit_root, anchor -def test_project_dotenv_paths_returns_only_toolkit_env( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.delenv("PRKIT_TOOLKIT_ROOT", raising=False) - toolkit_root, consumer_root, toolkit_anchor = _make_toolkit_layout(tmp_path) +def test_project_dotenv_paths_returns_toolkit_env(tmp_path: Path) -> None: + toolkit_root, anchor = _make_toolkit_layout(tmp_path) repo_env = toolkit_root / ".env" repo_env.write_text("OPENAI_API_KEY=repo-key\n", encoding="utf-8") - # A consumer .env must be ignored entirely. - (consumer_root / ".env").write_text( - "OPENAI_API_KEY=consumer-key\n", encoding="utf-8" - ) - assert project_dotenv_paths(toolkit_anchor) == (repo_env.resolve(),) + # Resolved from the nearest ``pyproject.toml`` ancestor of the anchor. + assert project_dotenv_paths(anchor) == (repo_env.resolve(),) + + +def test_project_dotenv_paths_empty_when_no_env(tmp_path: Path) -> None: + _, anchor = _make_toolkit_layout(tmp_path) + # Project root exists but has no ``.env`` → nothing to load. + assert project_dotenv_paths(anchor) == () + + +def test_project_dotenv_paths_empty_off_repo(tmp_path: Path) -> None: + # No ``pyproject.toml`` anywhere up the tree → best-effort empty (installed-wheel + # case); a stray ``.env`` in an unrelated directory is never reached into. + off_repo = tmp_path / "no_project" / "deep" + off_repo.mkdir(parents=True) + (off_repo / ".env").write_text("OPENAI_API_KEY=stray\n", encoding="utf-8") + + assert project_dotenv_paths(off_repo) == () def test_load_project_dotenv_overrides_shell_with_toolkit_value( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.delenv("PRKIT_TOOLKIT_ROOT", raising=False) - toolkit_root, _, toolkit_anchor = _make_toolkit_layout(tmp_path) + toolkit_root, anchor = _make_toolkit_layout(tmp_path) (toolkit_root / ".env").write_text("OPENAI_API_KEY=repo-key\n", encoding="utf-8") monkeypatch.setenv("OPENAI_API_KEY", "shell-key") - loaded = load_project_dotenv(toolkit_anchor, include_cwd_fallback=False) + loaded = load_project_dotenv(anchor, include_cwd_fallback=False) assert loaded == ((toolkit_root / ".env").resolve(),) assert os.environ["OPENAI_API_KEY"] == "repo-key" -def test_load_project_dotenv_ignores_sibling_consumer_env( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.delenv("PRKIT_TOOLKIT_ROOT", raising=False) - _, consumer_root, toolkit_anchor = _make_toolkit_layout(tmp_path) - # Only the consumer has an .env; the toolkit does not. - (consumer_root / ".env").write_text( - "OPENAI_API_KEY=consumer-key\n", encoding="utf-8" - ) - monkeypatch.delenv("OPENAI_API_KEY", raising=False) - - loaded = load_project_dotenv(toolkit_anchor, include_cwd_fallback=False) - - assert loaded == () - assert "OPENAI_API_KEY" not in os.environ - - def test_ensure_openai_api_key_handles_missing_value( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.delenv("PRKIT_TOOLKIT_ROOT", raising=False) - _, _, toolkit_anchor = _make_toolkit_layout(tmp_path) + _, anchor = _make_toolkit_layout(tmp_path) monkeypatch.delenv("OPENAI_API_KEY", raising=False) - assert ensure_openai_api_key(toolkit_anchor, include_cwd_fallback=False) is None + assert ensure_openai_api_key(anchor, include_cwd_fallback=False) is None with pytest.raises(RuntimeError, match="OPENAI_API_KEY is not set"): ensure_openai_api_key( - toolkit_anchor, + anchor, required=True, include_cwd_fallback=False, ) From 5800424c06c590ed528fb4c54faf5f00dbdb3033 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sun, 21 Jun 2026 17:38:49 -0400 Subject: [PATCH 25/28] Collapse the prkit.verify single-file package into a module prkit/verify/ held only __init__.py, so flatten it to prkit/verify.py. The import path prkit.verify is unchanged, so no caller, test, or doc edits are needed; the light-import isolation guarantee and its test are unaffected. Co-Authored-By: Claude Opus 4.8 --- src/prkit/{verify/__init__.py => verify.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/prkit/{verify/__init__.py => verify.py} (98%) diff --git a/src/prkit/verify/__init__.py b/src/prkit/verify.py similarity index 98% rename from src/prkit/verify/__init__.py rename to src/prkit/verify.py index 0187e06..36575d1 100644 --- a/src/prkit/verify/__init__.py +++ b/src/prkit/verify.py @@ -9,7 +9,7 @@ To turn a raw answer string into typed physics semantics (the former ``prkit.verify.parse``), use ``prkit.semantics.extract_prediction_answer_semantics``. -Import discipline (the whole point of this subpackage): ``import prkit.verify`` +Import discipline (the whole point of this module): ``import prkit.verify`` must NOT pull in provider SDKs (anthropic / openai / google.genai), the dataset hub, the ``datasets`` library, or pandas. The heavy :class:`~prkit.scoring.SemanticsScorer` (and its ``sympy`` dependency) is therefore imported *lazily inside the functions*, From 470819026223da0a3132ac8bd264c1b6c8e2cc49 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sun, 21 Jun 2026 17:44:13 -0400 Subject: [PATCH 26/28] Rename the answer ontology module to answer_taxonomy Move core/domain/answer_kinds.py to answer_taxonomy.py (a clearer name for the canonical AnswerObjectKind/AnswerStructure taxonomy) and repoint its two importers (core/domain/__init__.py and semantics/schema/enums.py, plus the latter's :mod: docstring reference). Symbol names are unchanged, so every import-via-package site is unaffected. Co-Authored-By: Claude Opus 4.8 --- src/prkit/core/domain/__init__.py | 2 +- src/prkit/core/domain/{answer_kinds.py => answer_taxonomy.py} | 0 src/prkit/semantics/schema/enums.py | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename src/prkit/core/domain/{answer_kinds.py => answer_taxonomy.py} (100%) diff --git a/src/prkit/core/domain/__init__.py b/src/prkit/core/domain/__init__.py index 9e3a05e..e4d49f7 100644 --- a/src/prkit/core/domain/__init__.py +++ b/src/prkit/core/domain/__init__.py @@ -12,7 +12,7 @@ # Domain definitions (enums/constants) # Domain models (data classes) from .answer import Answer -from .answer_kinds import AnswerObjectKind, AnswerStructure +from .answer_taxonomy import AnswerObjectKind, AnswerStructure from .license_spec import LicenseSpec from .physics_dataset import PhysicalDataset from .physics_domain import PhysicsDomain diff --git a/src/prkit/core/domain/answer_kinds.py b/src/prkit/core/domain/answer_taxonomy.py similarity index 100% rename from src/prkit/core/domain/answer_kinds.py rename to src/prkit/core/domain/answer_taxonomy.py diff --git a/src/prkit/semantics/schema/enums.py b/src/prkit/semantics/schema/enums.py index c5bc104..fa1070e 100644 --- a/src/prkit/semantics/schema/enums.py +++ b/src/prkit/semantics/schema/enums.py @@ -1,7 +1,7 @@ """Enumerations for physics answer semantics. The answer *ontology* enums (:class:`AnswerObjectKind`, :class:`AnswerStructure`) -and the :class:`_StrEnum` base now live in :mod:`prkit.core.domain.answer_kinds` +and the :class:`_StrEnum` base now live in :mod:`prkit.core.domain.answer_taxonomy` as the toolkit's canonical taxonomy; they are re-exported here so existing ``from prkit.semantics.schema import AnswerObjectKind`` import sites keep working. The *judgement-policy* enums below (unit policy, comparison mode, bridge tier, …) @@ -16,7 +16,7 @@ # Re-exported canonical ontology enums (defined in prkit.core.domain). ``_StrEnum`` # is the shared base for the judgement-policy enums defined in this module. -from prkit.core.domain.answer_kinds import ( +from prkit.core.domain.answer_taxonomy import ( AnswerObjectKind, AnswerStructure, _StrEnum, From 6a3a443cbe9407e2e805e11060cff112a32017c1 Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sun, 21 Jun 2026 18:03:24 -0400 Subject: [PATCH 27/28] Rename the Answer and PhysicalDataset domain classes to Physics* Rename the two core domain classes Answer -> PhysicsAnswer and PhysicalDataset -> PhysicsDataset (the latter also resolves the physics_dataset.py file/class stem mismatch and completes the four-noun Physics* symmetry with PhysicsProblem/PhysicsSolution). The other domain nouns and the AnswerObjectKind/AnswerStructure taxonomy are unchanged. Edits target only files that reference the domain symbols as code; string and comment occurrences (mock-answer fixtures, "Answer:" prompt-section text, the docs' unrelated structured-output Answer(BaseModel) example, the PASEC prose, and historical release notes) are deliberately left untouched. The contract stays provisional at API_VERSION "1.0": the breaking rename is tracked in internal/PAPER_V1_TO_V2_DELTA.md rather than signalled by a major bump, and no deprecation alias is provided. CONTRACT.md, CHANGELOG.md, and the API docs (README, CORE, DATASETS, semantics/README) are updated accordingly. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 5 ++ README.md | 12 ++-- docs/CORE.md | 54 +++++++------- docs/DATASETS.md | 28 ++++---- src/prkit/CONTRACT.md | 29 +++++--- src/prkit/__init__.py | 4 +- src/prkit/api.py | 19 ++--- src/prkit/core/domain/__init__.py | 10 +-- src/prkit/core/domain/answer.py | 6 +- src/prkit/core/domain/license_spec.py | 4 +- src/prkit/core/domain/physics_dataset.py | 72 +++++++++---------- src/prkit/core/domain/physics_problem.py | 6 +- src/prkit/datasets/hub.py | 6 +- src/prkit/datasets/license_registry.py | 2 +- src/prkit/datasets/loaders/base_loader.py | 16 ++--- .../datasets/loaders/cmphysbench_loader.py | 8 +-- src/prkit/datasets/loaders/jeebench_loader.py | 8 +-- src/prkit/datasets/loaders/phybench_loader.py | 8 +-- .../datasets/loaders/physbench_loader.py | 6 +- src/prkit/datasets/loaders/physics_loader.py | 8 +-- .../datasets/loaders/physreason_loader.py | 10 +-- src/prkit/datasets/loaders/phyx_loader.py | 8 +-- src/prkit/datasets/loaders/seephys_loader.py | 10 +-- src/prkit/datasets/loaders/tpbench_loader.py | 8 +-- .../datasets/loaders/ugphysics_loader.py | 8 +-- src/prkit/datasets/utils.py | 24 +++---- src/prkit/evaluation/llm_judge/payload.py | 10 +-- src/prkit/scoring/eed_scorer.py | 14 ++-- src/prkit/scoring/llm_judge_scorer.py | 6 +- src/prkit/scoring/seed_scorer.py | 12 ++-- src/prkit/scoring/semantics_eed_scorer.py | 8 +-- src/prkit/scoring/semantics_scorer.py | 10 +-- src/prkit/scoring/semantics_seed_scorer.py | 8 +-- src/prkit/semantics/README.md | 4 +- src/prkit/semantics/build/prompts.py | 4 +- .../normalization/answer_normalization.py | 4 +- .../normalization/question_inference.py | 4 +- src/prkit/verify.py | 8 +-- tests/conftest.py | 20 +++--- tests/prkit/core/domain/test_answer.py | 54 +++++++------- .../prkit/core/domain/test_physics_dataset.py | 54 +++++++------- .../prkit/core/domain/test_physics_problem.py | 6 +- .../loaders/test_base_loader_additional.py | 6 +- .../loaders/test_base_loader_map_domain.py | 10 +-- tests/prkit/datasets/test_hub.py | 30 ++++---- tests/prkit/datasets/test_utils_functions.py | 58 +++++++-------- .../evaluation/llm_judge/test_payload.py | 14 ++-- tests/prkit/scoring/test_eed_scorer.py | 4 +- tests/prkit/scoring/test_llm_judge_scorer.py | 8 +-- tests/prkit/scoring/test_seed_scorer.py | 8 ++- .../scoring/test_semantics_eed_scorer.py | 6 +- tests/prkit/scoring/test_semantics_scorer.py | 6 +- .../scoring/test_semantics_seed_scorer.py | 6 +- .../prkit/semantics/test_inference_prompts.py | 6 +- tests/prkit/semantics/test_outcome_space.py | 26 +++---- .../test_prediction_isolated_build.py | 4 +- .../test_sign_convention_build_integration.py | 6 +- .../test_sign_convention_build_live.py | 4 +- tests/prkit/semantics/test_staged_build.py | 6 +- tests/prkit/test_api.py | 8 +-- 60 files changed, 418 insertions(+), 403 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0eb2020..6c8d86b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ Production releases follow semantic versioning. TestPyPI validation builds use P ### Added +- **Edit-distance scorer family** in `prkit.scoring`: the faithful PHYBench-EED / CMPhysBench-SEED baselines (`EedScorer`, `SeedScorer`, vendored under `prkit.evaluation.baselines`) and their our-semantics counterparts (`SemanticsEedScorer`, `SemanticsSeedScorer`), plus the model-graded `LLMJudgeScorer` wrapping `prkit.evaluation.llm_judge`. All emit the canonical `Verdict`. A new `[baselines]` optional extra pins `pint` for the SEED unit path; `import prkit.scoring` / `prkit.verify` stay free of `pint`/`openai` (lazy in `score()`). +- **`Verdict.score == -1.0`** reserved as the not-applicable sentinel (`comparison_mode="not_applicable"`), emitted by the edit-distance scorers for answer kinds/structures with no SEED type. It is an honest "N/A" distinct from `0.0`; numeric aggregators must exclude it (`score >= 0`). +- **`cmphysbench` loader** — `DatasetHub` gains the CMPhysBench benchmark loader, mapping the dataset-native `answer_type` into `PhysicsAnswer.source_type` (one of the five SEED tokens) for faithful `SeedScorer` dispatch. - **`BaseModelClient.parse()`** — dedicated typed structured-output entry point mirroring the SDK `.parse()` idiom (OpenAI `client.responses.parse`, Anthropic `client.messages.parse`). `parse(input, *, response_format=, image_paths=None, structured_policy="best_effort", instructions=None, **kwargs)` returns a `StructuredCallResult[T]` (`.parsed`, `.raw_text`, `.validation_error`, `.require_parsed()`). The first parameter is `input` and the schema parameter is `response_format`, unifying naming with `response()`. `response()` remains text-only (passing a Pydantic model still returns the JSON string). Replaces `chat_structured()` (now deprecated). - **Batch API support across OpenAI, Anthropic, and Gemini** — `BaseModelClient` gains a synchronous batch job lifecycle (`submit_batch` → `poll_batch` → `retrieve_batch_results`) plus a free-text request builder `build_batch_request(...)` that mirrors `response()` (same `input`/`instructions` handling, no structured output), complementing the existing structured `build_batch_structured_request`. New provider-agnostic types `BatchState`, `BatchStatus`, `BatchItemStatus`, and `BatchResult` (in `prkit.core.model_clients.batch_types`) normalize each provider's status enum and per-request results. Each provider's request-body construction is now shared between `response()` and the batch builders (`_build_responses_body` / `_build_messages_params`) to prevent drift. OpenAI o-family models drop `temperature` at build time. Unsupported providers raise `NotImplementedError`. Batch processing runs asynchronously at ~50% of synchronous cost. Gemini batches are submitted as an uploaded keyed JSONL file (via the File API, `src=`) rather than as inline requests, so results come back as documented keyed JSONL (`{"key": ..., "response": {...}}`) and correlate reliably to each request — inline responses carry no per-request key and cannot be correlated. - **`OpenAIModel` custom endpoint support** — new keyword-only constructor params `base_url`, `api_key`, and `api_key_env` allow routing to any proxy or gateway that implements the OpenAI Responses API (`POST /v1/responses`) with an explicit key or key from a named environment variable. Backward-compatible: omitting all three preserves existing `OPENAI_API_KEY` + default endpoint behaviour. @@ -26,6 +29,8 @@ Production releases follow semantic versioning. TestPyPI validation builds use P ### Changed - **BREAKING: renamed subpackages** to drop the redundant `prkit_` prefix — import from `prkit.core`, `prkit.datasets`, `prkit.evaluation`, `prkit.annotation`, `prkit.semantics` (previously `prkit.prkit_core`, etc.). The `sys.modules` top-level aliasing hack was removed. +- **BREAKING: renamed domain classes** `Answer` → `PhysicsAnswer` and `PhysicalDataset` → `PhysicsDataset` (the latter also fixes the `physics_dataset.py` file/class stem mismatch). The other domain nouns (`PhysicsProblem`, `PhysicsSolution`, `PhysicsDomain`, `AnswerObjectKind`, `AnswerStructure`, `LicenseSpec`) are unchanged. The contract stays provisional at `API_VERSION "1.0"` (the rename is tracked internally, not signalled by a major bump); no deprecation alias is provided. The answer-ontology module `core/domain/answer_kinds.py` was also renamed to `answer_taxonomy.py` (symbols unchanged). +- **`PartialCreditScorer` removed.** The graded edit-distance scoring it provided is now covered by `SemanticsEedScorer` / `SemanticsSeedScorer`; `verify(..., partial_credit=True)` routes to `SemanticsSeedScorer`. No deprecation alias (it was never part of the frozen `prkit.api` surface). - The model-client factory is now an extensible provider registry (`register_model_client`) instead of an if/elif chain; image/MIME/data-URL helpers are centralized in `prkit.core.model_clients.utils`. - Packaging: removed the erroneous `pip` runtime dependency, expanded trove classifiers (Python 3.11/3.12, Education, OS Independent), and aligned `black` / `requires-python` targets. - Release publishing now uses automated version selection instead of manual version bumps. diff --git a/README.md b/README.md index fe676f5..cc10221 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ PRKit applies a “unified interface” idea to the full physical-reasoning loop PRKit centers on **core components** that define the physical reasoning ontology. Three integrated subpackages build on this foundation: -- **Core components**: `PhysicsDomain`, `PhysicsProblem`, `Answer`, `PhysicalDataset`, `PhysicsSolution`, `BaseModelClient`, `create_model_client`, `PRKitLogger`—the shared abstractions used across the toolkit. -- **`prkit.datasets`**: A Datasets-like hub that downloads/loads benchmarks into the unified schema (`PhysicsProblem`, `PhysicalDataset`). +- **Core components**: `PhysicsDomain`, `PhysicsProblem`, `PhysicsAnswer`, `PhysicsDataset`, `PhysicsSolution`, `BaseModelClient`, `create_model_client`, `PRKitLogger`—the shared abstractions used across the toolkit. +- **`prkit.datasets`**: A Datasets-like hub that downloads/loads benchmarks into the unified schema (`PhysicsProblem`, `PhysicsDataset`). - **`prkit.annotation`**: Workflow-oriented tools for structured, lower-level labels (e.g., domain/subdomain, theorem usage). - **`prkit.evaluation`**: Evaluate-like components for physics-oriented scoring and comparison (e.g., symbolic/numerical answer matching). @@ -19,7 +19,7 @@ PRKit centers on **core components** that define the physical reasoning ontology from prkit.datasets import DatasetHub from prkit.core.model_clients import create_model_client -# Load any benchmark into the unified schema (PhysicsProblem, PhysicalDataset) +# Load any benchmark into the unified schema (PhysicsProblem, PhysicsDataset) dataset = DatasetHub.load("physreason", variant="full", split="test") # Run inference with the unified model client (core component) @@ -211,9 +211,9 @@ The toolkit is organized around **core components** and three subpackages that u The essential building blocks of the physical-reasoning-toolkit. All datasets, inference, evaluation, and annotation workflows use these components. * **PhysicsDomain** — Enumeration of physics subfields (mechanics, thermodynamics, quantum mechanics, optics, etc.) for problem classification. Aligned with UGPhysics, PHYBench, TPBench. Use `PhysicsDomain.from_string()` for flexible parsing. -* **PhysicsProblem** — The canonical representation of a physics problem. Required: `problem_id`, `question`. Optional: `answer` (Answer), `solution`, `domain`, `image_path`, `problem_type` (MC/OE), `options`, `correct_option`. Supports dictionary-like access and `load_images()` for visual problems. -* **Answer** — Thin observation record: `value` (str, verbatim), optional `unit` (observed unit string), optional `source_type` (dataset-native type tag, verbatim), and `metadata` dict. The canonical answer kind (`AnswerObjectKind`, 9 object kinds) is derived on demand by the `prkit.semantics` layer — it is not stored on `Answer`. -* **PhysicalDataset** — Collection of `PhysicsProblem` instances. Indexing, slicing, `get_by_id()`, `filter_by_domain()`, `take()`, `sample()`, `save_to_json()` / `from_json()`. Provides `get_statistics()` for domain and problem-type distribution. +* **PhysicsProblem** — The canonical representation of a physics problem. Required: `problem_id`, `question`. Optional: `answer` (PhysicsAnswer), `solution`, `domain`, `image_path`, `problem_type` (MC/OE), `options`, `correct_option`. Supports dictionary-like access and `load_images()` for visual problems. +* **PhysicsAnswer** — Thin observation record: `value` (str, verbatim), optional `unit` (observed unit string), optional `source_type` (dataset-native type tag, verbatim), and `metadata` dict. The canonical answer kind (`AnswerObjectKind`, 9 object kinds) is derived on demand by the `prkit.semantics` layer — it is not stored on `PhysicsAnswer`. +* **PhysicsDataset** — Collection of `PhysicsProblem` instances. Indexing, slicing, `get_by_id()`, `filter_by_domain()`, `take()`, `sample()`, `save_to_json()` / `from_json()`. Provides `get_statistics()` for domain and problem-type distribution. * **PhysicsSolution** — Bundles a `PhysicsProblem`, model `agent_answer`, and optional `intermediate_steps`. Captures the full solution trace for evaluation and analysis. * **BaseModelClient** — Abstract base for model clients. Subclasses implement `chat(user_prompt, image_paths=None)`. * **PRKitLogger** — Centralized logging with colored output, file logging, and env config (`PRKIT_LOG_LEVEL`, `PRKIT_LOG_FILE`, etc.). diff --git a/docs/CORE.md b/docs/CORE.md index 672d574..c6e97f6 100644 --- a/docs/CORE.md +++ b/docs/CORE.md @@ -53,7 +53,7 @@ The core unit of a physics problem. Works both standalone and as a dataset-compa - `question`: Problem text **Core optional fields:** -- `answer`: `Answer` object (ground truth) +- `answer`: `PhysicsAnswer` object (ground truth) - `solution`: Solution text - `domain`: `PhysicsDomain` or string - `language`: Default `"en"` @@ -74,7 +74,7 @@ The core unit of a physics problem. Works both standalone and as a dataset-compa - `load_images()` → Load PIL `Image` objects (requires Pillow) - `to_dict()` / `from_dict()` → Serialization -### Answer +### PhysicsAnswer A thin observation record: the verbatim answer string, an optional unit, an optional dataset-native type label, and a metadata dict. It captures exactly what the dataset provides—nothing more. @@ -84,7 +84,7 @@ A thin observation record: the verbatim answer string, an optional unit, an opti - `source_type`: `str | None` — dataset-native answer-type label, verbatim (e.g. `"MC"`, `"NV"`, `"EX"`, `"Integer"`); `None` when the dataset provides none; **never fabricated by heuristics** - `metadata`: `dict` — extra dataset-provided key-value data -> **Canonical answer kind is derived, not stored.** The `AnswerObjectKind` ontology (9 object kinds: `number`, `physical_quantity`, `expression`, `relation`, `choice`, `qualitative_label`, `assertion`, `structured`, `descriptive_text`) lives in `prkit.semantics` and is returned as `object_kind` on `PhysicsAnswerSemantics`. It is not a field on `Answer`. +> **Canonical answer kind is derived, not stored.** The `AnswerObjectKind` ontology (9 object kinds: `number`, `physical_quantity`, `expression`, `relation`, `choice`, `qualitative_label`, `assertion`, `structured`, `descriptive_text`) lives in `prkit.semantics` and is returned as `object_kind` on `PhysicsAnswerSemantics`. It is not a field on `PhysicsAnswer`. **Access helpers:** - `get_value()` → `str` @@ -94,13 +94,13 @@ A thin observation record: the verbatim answer string, an optional unit, an opti --- -### PhysicalDataset +### PhysicsDataset Collection of `PhysicsProblem` instances with a Datasets-like interface. **Constructor:** ```python -PhysicalDataset(problems: List[PhysicsProblem], info=None, split="test") +PhysicsDataset(problems: List[PhysicsProblem], info=None, split="test") ``` **Access:** @@ -324,20 +324,20 @@ logger.info("Message") ### Overview -- **PhysicalDataset** = a collection of physics problems +- **PhysicsDataset** = a collection of physics problems - **PhysicsProblem** = one problem (question + optional ground-truth answer + optional domain) -- **Answer** = thin observation record: `value` (str) + optional `unit` + optional `source_type` + `metadata` +- **PhysicsAnswer** = thin observation record: `value` (str) + optional `unit` + optional `source_type` + `metadata` - **PhysicsSolution** = a problem plus model output (agent_answer), used for evaluation ### Core Domain Model ``` - PhysicalDataset + PhysicsDataset └── contains 1:N ───► PhysicsProblem │ ├── domain: PhysicsDomain │ - └── answer: Answer (optional, ground truth) + └── answer: PhysicsAnswer (optional, ground truth) │ └── { value: str, unit: str|None, source_type: str|None, metadata: dict } @@ -349,8 +349,8 @@ logger.info("Message") **Box view:** ``` - PhysicalDataset PhysicsProblem - ┌──────────────┐ ┌──────────────────┐ Answer + PhysicsDataset PhysicsProblem + ┌──────────────┐ ┌──────────────────┐ PhysicsAnswer │ _problems │────────►│ problem_id │ ┌──────────────────────┐ │ _info │ 1:N │ question │ │ value: str │ │ _split │ │ domain ──────────┼──┐ │ unit: str|None │ @@ -361,7 +361,7 @@ logger.info("Message") PhysicsSolution │ ┌──────────────┐ │ problem │ problem ─────┼──────────────────┘ - │ agent_answer ┼─► str (─► optional: parsed to Answer)(compared to problem.answer in evaluation) + │ agent_answer ┼─► str (─► optional: parsed to PhysicsAnswer)(compared to problem.answer in evaluation) └──────────────┘ ``` @@ -369,9 +369,9 @@ logger.info("Message") | Entity | Has / Contains | |--------|----------------| -| PhysicalDataset | Many PhysicsProblem (_problems list) | -| PhysicsProblem | problem_id, question, domain (PhysicsDomain), answer (Answer), solution, image_path, ... | -| Answer | value (str), unit (str\|None), source_type (str\|None), metadata (dict) | +| PhysicsDataset | Many PhysicsProblem (_problems list) | +| PhysicsProblem | problem_id, question, domain (PhysicsDomain), answer (PhysicsAnswer), solution, image_path, ... | +| PhysicsAnswer | value (str), unit (str\|None), source_type (str\|None), metadata (dict) | | PhysicsSolution | problem (PhysicsProblem), agent_answer (string). Evaluation compares agent_answer to problem.answer | ### Subpackage Dependencies @@ -387,9 +387,9 @@ flowchart TB subgraph core["prkit.core"] - PD[PhysicalDataset] + PD[PhysicsDataset] PP[PhysicsProblem] - AN[Answer] + AN[PhysicsAnswer] PS[PhysicsSolution] end @@ -426,13 +426,13 @@ flowchart TB ``` -*Rectangles = classes. Parallelogram = value/role (model output is not a class; it is an `Answer` from inference).* +*Rectangles = classes. Parallelogram = value/role (model output is not a class; it is an `PhysicsAnswer` from inference).* | Package | Uses from Core | Produces / Operates On | |---------|----------------|------------------------| -| prkit.datasets | PhysicalDataset, PhysicsProblem, Answer, PhysicsDomain, PRKitLogger | PhysicalDataset (via DatasetLoader.load) | -| prkit.scoring / prkit.verify | PhysicsProblem, Answer, Verdict | Verdict (via SemanticsScorer / verify) | -| prkit.evaluation | PhysicsProblem, Answer | Model-graded scores (via LLMJudge; deterministic scoring is in prkit.scoring) | +| prkit.datasets | PhysicsDataset, PhysicsProblem, PhysicsAnswer, PhysicsDomain, PRKitLogger | PhysicsDataset (via DatasetLoader.load) | +| prkit.scoring / prkit.verify | PhysicsProblem, PhysicsAnswer, Verdict | Verdict (via SemanticsScorer / verify) | +| prkit.evaluation | PhysicsProblem, PhysicsAnswer | Model-graded scores (via LLMJudge; deterministic scoring is in prkit.scoring) | --- @@ -442,13 +442,13 @@ flowchart TB # Core components from prkit.core.domain import ( PhysicsDomain, - Answer, + PhysicsAnswer, PhysicsProblem, - PhysicalDataset, + PhysicsDataset, PhysicsSolution, ) -# Semantics-layer canonical kind (derived, not stored on Answer) +# Semantics-layer canonical kind (derived, not stored on PhysicsAnswer) from prkit.semantics.schema import AnswerObjectKind # 9 object kinds # Utility components @@ -460,7 +460,7 @@ from prkit.core.model_clients import create_model_client, BaseModelClient ## Design Principles -1. **Unified schema:** All supported benchmarks map to `PhysicsProblem` and `PhysicalDataset`. -2. **Observed data vs. derived interpretation:** `Answer` is a thin observation record (`value`/`unit`/`source_type`/`metadata`). The canonical answer kind (`AnswerObjectKind`) is derived on demand by `prkit.semantics`, not stored on `Answer`. -3. **Composition over inheritance:** `Answer` is a flat dataclass; type interpretation is a semantics-layer concern, not a subclass hierarchy. +1. **Unified schema:** All supported benchmarks map to `PhysicsProblem` and `PhysicsDataset`. +2. **Observed data vs. derived interpretation:** `PhysicsAnswer` is a thin observation record (`value`/`unit`/`source_type`/`metadata`). The canonical answer kind (`AnswerObjectKind`) is derived on demand by `prkit.semantics`, not stored on `PhysicsAnswer`. +3. **Composition over inheritance:** `PhysicsAnswer` is a flat dataclass; type interpretation is a semantics-layer concern, not a subclass hierarchy. 4. **Dataset compatibility:** `PhysicsProblem` supports dict-like access and `additional_fields`. diff --git a/docs/DATASETS.md b/docs/DATASETS.md index 1afc57c..02137e8 100644 --- a/docs/DATASETS.md +++ b/docs/DATASETS.md @@ -109,21 +109,21 @@ for problem in dataset[:5]: All datasets in PRKit are converted to a unified format built on four core data structures that work together: -1. **`Answer`** - Represents answer values with type information -2. **`PhysicsProblem`** - Represents a single physics problem (uses `Answer`) -3. **`PhysicalDataset`** - Container for collections of `PhysicsProblem` objects +1. **`PhysicsAnswer`** - Represents answer values with type information +2. **`PhysicsProblem`** - Represents a single physics problem (uses `PhysicsAnswer`) +3. **`PhysicsDataset`** - Container for collections of `PhysicsProblem` objects 4. **`PhysicsSolution`** - Tracks LLM solutions to problems (uses `PhysicsProblem`) This section documents each structure in dependency order, starting with the foundational building blocks. -### Answer Structure +### PhysicsAnswer Structure -The `Answer` class is the foundational building block for representing answers in physics problems. It handles all answer types through composition rather than inheritance. +The `PhysicsAnswer` class is the foundational building block for representing answers in physics problems. It handles all answer types through composition rather than inheritance. #### Fields ```python -Answer( +PhysicsAnswer( value: str, # Verbatim answer string (always str) unit: Optional[str] = None, # Observed unit from the dataset, e.g. "m/s²" source_type: Optional[str] = None, # Dataset-native type label, verbatim, e.g. "MC", "NV", "EX" @@ -131,7 +131,7 @@ Answer( ) ``` -`Answer` is a **thin observation record** — it stores exactly what the dataset provides. The canonical answer kind (`AnswerObjectKind` — one of 9 semantic object kinds such as `number`, `physical_quantity`, `choice`, `expression`, etc.) is **derived on demand by `prkit.semantics`**, not stored on `Answer`. +`PhysicsAnswer` is a **thin observation record** — it stores exactly what the dataset provides. The canonical answer kind (`AnswerObjectKind` — one of 9 semantic object kinds such as `number`, `physical_quantity`, `choice`, `expression`, etc.) is **derived on demand by `prkit.semantics`**, not stored on `PhysicsAnswer`. - `value` is always a `str` (the verbatim answer string). - `unit` is the observed unit when present; `None` otherwise. It is consumed by the semantics engine. @@ -156,7 +156,7 @@ The `PhysicsDomain` enum is used to classify physics problems by domain. The `do ### PhysicsProblem Structure -The `PhysicsProblem` class is the core data structure representing a physics problem. It uses `Answer` objects and `PhysicsDomain` enums, and provides both object-oriented access and dictionary-like access for dataset compatibility. +The `PhysicsProblem` class is the core data structure representing a physics problem. It uses `PhysicsAnswer` objects and `PhysicsDomain` enums, and provides both object-oriented access and dictionary-like access for dataset compatibility. #### Problem Types @@ -173,7 +173,7 @@ The `PhysicsProblem` class is the core data structure representing a physics pro #### Optional Core Fields -- **`answer`** (`Answer`, optional): Answer object containing the solution +- **`answer`** (`PhysicsAnswer`, optional): PhysicsAnswer object containing the solution - **`solution`** (`str`, optional): Step-by-step solution text - **`domain`** (`PhysicsDomain` enum or `str`, optional): Physics domain classification - **`problem_type`** (`str`, optional): Problem type - `"MC"`, `"MultipleMC"`, or `"OE"` @@ -218,14 +218,14 @@ For problems with images (`image_path` is non-empty): - **`load_images()`**: Method to load PIL Image objects from paths - Images are automatically converted to RGB format for consistency -### PhysicalDataset Structure +### PhysicsDataset Structure -The `PhysicalDataset` class is a container for `PhysicsProblem` objects, providing a unified interface similar to Hugging Face Datasets. It supports iteration, indexing, filtering, and various dataset operations. +The `PhysicsDataset` class is a container for `PhysicsProblem` objects, providing a unified interface similar to Hugging Face Datasets. It supports iteration, indexing, filtering, and various dataset operations. #### Initialization ```python -PhysicalDataset( +PhysicsDataset( problems: List[PhysicsProblem], # List of PhysicsProblem instances info: Optional[Dict[str, Any]] = None, # Optional dataset metadata split: str = "test" # Dataset split ("train", "test", "val", or "eval") @@ -551,7 +551,7 @@ subclass needed. ```python from prkit.datasets import DatasetHub from prkit.datasets.loaders.base_loader import BaseDatasetLoader -from prkit.core.domain import PhysicalDataset, PhysicsProblem +from prkit.core.domain import PhysicsDataset, PhysicsProblem class MyLoader(BaseDatasetLoader): @property @@ -566,7 +566,7 @@ class MyLoader(BaseDatasetLoader): } def load(self, data_dir=None, **kwargs): - # Read from data_dir and return a PhysicalDataset + # Read from data_dir and return a PhysicsDataset ... DatasetHub.register("my_dataset", MyLoader) diff --git a/src/prkit/CONTRACT.md b/src/prkit/CONTRACT.md index e24708c..da665fe 100644 --- a/src/prkit/CONTRACT.md +++ b/src/prkit/CONTRACT.md @@ -107,17 +107,24 @@ changes to those are documented in the package release notes, not the contract v - Precedent: `BaseModelClient.chat()` / `chat_structured()` (see `core/model_clients/base.py`). -### Removed during provisional 1.0 shaping - -- **`Answer` reshaped to a thin observation record.** `Answer` now carries only - `value: str`, `unit: str | None`, `source_type: str | None`, and `metadata: dict`. - The former `answer_kind: AnswerObjectKind` field (and all predicate helpers such as - `is_number()`, `is_option()`, `get_type()`, etc.) are **removed**. The canonical - answer ontology (`AnswerObjectKind` / `AnswerStructure`, 9 object kinds) lives only - in `prkit.semantics` and is returned as `object_kind` on `PhysicsAnswerSemantics` - — it is never stored on `Answer`. `source_type` is the dataset's verbatim native - type label (e.g. `"MC"`, `"NV"`, `"EX"`, `"Integer"`); it is never fabricated and - never read by the semantics engine. Serialized answers migrate via: +### Removed/renamed during provisional 1.0 shaping + +- **Domain classes renamed.** `Answer` → **`PhysicsAnswer`** and `PhysicalDataset` → + **`PhysicsDataset`** (the latter also resolves the `physics_dataset.py` file/class + stem mismatch). The other domain nouns (`PhysicsProblem`, `PhysicsSolution`, + `PhysicsDomain`, `AnswerObjectKind`, `AnswerStructure`, `LicenseSpec`) are unchanged. + Per the `API_VERSION` policy this breaking rename is **tracked in + `internal/PAPER_V1_TO_V2_DELTA.md`, not signalled by a major bump** — the contract + stays provisional at `1.0`. No deprecation alias is provided. +- **`PhysicsAnswer` reshaped to a thin observation record.** `PhysicsAnswer` (formerly + `Answer`) now carries only `value: str`, `unit: str | None`, `source_type: str | None`, + and `metadata: dict`. The former `answer_kind: AnswerObjectKind` field (and all + predicate helpers such as `is_number()`, `is_option()`, `get_type()`, etc.) are + **removed**. The canonical answer ontology (`AnswerObjectKind` / `AnswerStructure`, + 9 object kinds) lives only in `prkit.semantics` and is returned as `object_kind` on + `PhysicsAnswerSemantics` — it is never stored on `PhysicsAnswer`. `source_type` is the + dataset's verbatim native type label (e.g. `"MC"`, `"NV"`, `"EX"`, `"Integer"`); it is + never fabricated and never read by the semantics engine. Serialized answers migrate via: `source_type = value.get("source_type") or value.get("answer_kind") or value.get("answer_category")`. - **Deprecated scoring stack deleted.** `prkit.evaluation.comparator.*`, `prkit.evaluation.evaluator.*` (`BaseComparator`, `ExactMatchComparator`, diff --git a/src/prkit/__init__.py b/src/prkit/__init__.py index 4e00afb..c50d0f0 100644 --- a/src/prkit/__init__.py +++ b/src/prkit/__init__.py @@ -39,7 +39,7 @@ from .core.domain import ( AnswerObjectKind, AnswerStructure, - PhysicalDataset, + PhysicsDataset, PhysicsDomain, PhysicsProblem, ) @@ -48,7 +48,7 @@ "__version__", "PRKitLogger", "PhysicsProblem", - "PhysicalDataset", + "PhysicsDataset", "PhysicsDomain", "AnswerObjectKind", "AnswerStructure", diff --git a/src/prkit/api.py b/src/prkit/api.py index 51ed282..3a58206 100644 --- a/src/prkit/api.py +++ b/src/prkit/api.py @@ -33,11 +33,11 @@ from prkit.core.domain import ( AnswerObjectKind, AnswerStructure, - PhysicalDataset, + PhysicsDataset, PhysicsDomain, PhysicsProblem, ) -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer from prkit.core.model_clients import BaseModelClient, create_model_client from prkit.core.verdict import Verdict from prkit.datasets.hub import DatasetHub @@ -56,7 +56,7 @@ class DatasetProvider(Protocol): """Loader noun. Satisfied today by :class:`BaseDatasetLoader` subclasses.""" - def load(self, data_dir: Any = ..., **kwargs: Any) -> PhysicalDataset: ... + def load(self, data_dir: Any = ..., **kwargs: Any) -> PhysicsDataset: ... def get_info(self) -> dict[str, Any]: ... # MUST include "version" @@ -92,7 +92,10 @@ class Scorer(Protocol): version: str def score( - self, prediction: Answer | str, reference: Answer | str, **kwargs: Any + self, + prediction: PhysicsAnswer | str, + reference: PhysicsAnswer | str, + **kwargs: Any, ) -> Verdict: ... def get_info(self) -> dict[str, Any]: ... # MUST include "version" @@ -101,7 +104,7 @@ def get_info(self) -> dict[str, Any]: ... # MUST include "version" @runtime_checkable class Runner(Protocol): """Orchestration noun: drive a :class:`ModelClient` over a - :class:`PhysicalDataset` and score with a :class:`Scorer`. + :class:`PhysicsDataset` and score with a :class:`Scorer`. No implementation ships today; the contract is reserved for a later orchestration item (roadmap N4). @@ -109,7 +112,7 @@ class Runner(Protocol): def run( self, - dataset: PhysicalDataset, + dataset: PhysicsDataset, model: ModelClient, scorer: Scorer, **kwargs: Any, @@ -128,10 +131,10 @@ def run( "AnswerObjectKind", "AnswerStructure", # re-exported concrete anchors - "Answer", + "PhysicsAnswer", "PhysicsDomain", "PhysicsProblem", - "PhysicalDataset", + "PhysicsDataset", "DatasetHub", "BaseDatasetLoader", "BaseModelClient", diff --git a/src/prkit/core/domain/__init__.py b/src/prkit/core/domain/__init__.py index e4d49f7..c299c4b 100644 --- a/src/prkit/core/domain/__init__.py +++ b/src/prkit/core/domain/__init__.py @@ -5,16 +5,16 @@ for PRKit (physical-reasoning-toolkit). It consolidates: -- Domain models: Answer, PhysicsProblem, PhysicalDataset, PhysicsSolution +- Domain models: PhysicsAnswer, PhysicsProblem, PhysicsDataset, PhysicsSolution - Domain definitions: AnswerObjectKind, AnswerStructure, PhysicsDomain """ # Domain definitions (enums/constants) # Domain models (data classes) -from .answer import Answer +from .answer import PhysicsAnswer from .answer_taxonomy import AnswerObjectKind, AnswerStructure from .license_spec import LicenseSpec -from .physics_dataset import PhysicalDataset +from .physics_dataset import PhysicsDataset from .physics_domain import PhysicsDomain from .physics_problem import PhysicsProblem from .physics_solution import PhysicsSolution @@ -26,9 +26,9 @@ # Definitions "PhysicsDomain", # Models - "Answer", + "PhysicsAnswer", "PhysicsProblem", - "PhysicalDataset", + "PhysicsDataset", "PhysicsSolution", "LicenseSpec", ] diff --git a/src/prkit/core/domain/answer.py b/src/prkit/core/domain/answer.py index db1ffe8..5c7bcc5 100644 --- a/src/prkit/core/domain/answer.py +++ b/src/prkit/core/domain/answer.py @@ -1,7 +1,7 @@ """ Thin observation record for a physics problem's ground-truth answer. -An ``Answer`` captures only what a dataset directly provides: +An ``PhysicsAnswer`` captures only what a dataset directly provides: * ``value`` — verbatim answer string (always ``str``) * ``unit`` — observed unit string when present (e.g. ``"m/s²"``, ``"N"``) @@ -21,7 +21,7 @@ @dataclass -class Answer: +class PhysicsAnswer: """Thin observed-data record for a physics answer. ``value`` is always a plain string — the verbatim answer text after @@ -72,7 +72,7 @@ def __str__(self) -> str: def __repr__(self) -> str: return ( - f"Answer(value={self.value!r}, unit={self.unit!r}, " + f"PhysicsAnswer(value={self.value!r}, unit={self.unit!r}, " f"source_type={self.source_type!r})" ) diff --git a/src/prkit/core/domain/license_spec.py b/src/prkit/core/domain/license_spec.py index 6b61df2..59c738b 100644 --- a/src/prkit/core/domain/license_spec.py +++ b/src/prkit/core/domain/license_spec.py @@ -3,7 +3,7 @@ ``LicenseSpec`` is the single machine-readable record of a dataset's license: an SPDX id, a human name/url, and boolean usage flags (redistributable / commercial / eval-only / etc.). The dataset license registry (``prkit.datasets.license_registry``) owns one ``LicenseSpec`` per -dataset; loaders and downloaders embed ``to_info_dict()`` at ``PhysicalDataset.info["license"]`` +dataset; loaders and downloaders embed ``to_info_dict()`` at ``PhysicsDataset.info["license"]`` so every read path reports the same, normalized license truth. """ @@ -40,7 +40,7 @@ class LicenseSpec: notes: str | None = None def to_info_dict(self) -> dict[str, Any]: - """Return the flat dict embedded at ``PhysicalDataset.info["license"]``.""" + """Return the flat dict embedded at ``PhysicsDataset.info["license"]``.""" return asdict(self) diff --git a/src/prkit/core/domain/physics_dataset.py b/src/prkit/core/domain/physics_dataset.py index d35b441..7a0cb1e 100644 --- a/src/prkit/core/domain/physics_dataset.py +++ b/src/prkit/core/domain/physics_dataset.py @@ -13,7 +13,7 @@ _MapResult = TypeVar("_MapResult") -class PhysicalDataset: +class PhysicsDataset: """ Base class for physical reasoning datasets with a unified interface. @@ -49,16 +49,16 @@ def __len__(self) -> int: def __getitem__(self, idx: int) -> PhysicsProblem: ... @overload - def __getitem__(self, idx: slice) -> "PhysicalDataset": ... + def __getitem__(self, idx: slice) -> "PhysicsDataset": ... - def __getitem__(self, idx: int | slice) -> Union[PhysicsProblem, "PhysicalDataset"]: + def __getitem__(self, idx: int | slice) -> Union[PhysicsProblem, "PhysicsDataset"]: """Get a problem by index or a slice of dataset.""" if isinstance(idx, slice): - # Return a new PhysicalDataset with sliced problems + # Return a new PhysicsDataset with sliced problems sliced_problems = [ self._problems[i] for i in range(*idx.indices(len(self._problems))) ] - return PhysicalDataset(sliced_problems, self._info, self._split) + return PhysicsDataset(sliced_problems, self._info, self._split) return self._problems[idx] def __iter__(self) -> Iterator[PhysicsProblem]: @@ -123,9 +123,7 @@ def get_by_id_safe(self, problem_id: str) -> PhysicsProblem | None: except KeyError: return None - def filter( - self, filter_func: Callable[[PhysicsProblem], bool] - ) -> "PhysicalDataset": + def filter(self, filter_func: Callable[[PhysicsProblem], bool]) -> "PhysicsDataset": """ Filter problems using a filter function. @@ -133,14 +131,14 @@ def filter( filter_func: Function that takes a PhysicsProblem and returns bool Returns: - New PhysicalDataset with filtered problems + New PhysicsDataset with filtered problems """ filtered_problems = [p for p in self._problems if filter_func(p)] - return PhysicalDataset(filtered_problems, self._info, self._split) + return PhysicsDataset(filtered_problems, self._info, self._split) def filter_by_domains( self, domains: list[Union[str, "PhysicsDomain"]] - ) -> "PhysicalDataset": + ) -> "PhysicsDataset": """ Filter problems by physics domains. @@ -148,7 +146,7 @@ def filter_by_domains( domains: List of domain names (strings) or PhysicsDomain enum values Returns: - New PhysicalDataset containing only problems from the specified domains + New PhysicsDataset containing only problems from the specified domains Example: # Filter by domain names @@ -188,7 +186,7 @@ def filter_by_domains( filtered_problems.append(problem) # Create new dataset with filtered problems - filtered_dataset = PhysicalDataset(filtered_problems, self._info, self._split) + filtered_dataset = PhysicsDataset(filtered_problems, self._info, self._split) # Log filtering results logger = PRKitLogger.get_logger(__name__) @@ -199,9 +197,7 @@ def filter_by_domains( return filtered_dataset - def filter_by_domain( - self, domain: Union[str, "PhysicsDomain"] - ) -> "PhysicalDataset": + def filter_by_domain(self, domain: Union[str, "PhysicsDomain"]) -> "PhysicsDataset": """ Filter problems by a single physics domain. @@ -209,7 +205,7 @@ def filter_by_domain( domain: Domain name (string) or PhysicsDomain enum value Returns: - New PhysicalDataset containing only problems from the specified domain + New PhysicsDataset containing only problems from the specified domain Example: # Filter by domain name @@ -221,7 +217,7 @@ def filter_by_domain( """ return self.filter_by_domains([domain]) - def select(self, indices: list[int]) -> "PhysicalDataset": + def select(self, indices: list[int]) -> "PhysicsDataset": """ Select problems by indices. @@ -229,14 +225,14 @@ def select(self, indices: list[int]) -> "PhysicalDataset": indices: List of problem indices to select Returns: - New PhysicalDataset with selected problems + New PhysicsDataset with selected problems """ selected_problems = [ self._problems[i] for i in indices if 0 <= i < len(self._problems) ] - return PhysicalDataset(selected_problems, self._info, self._split) + return PhysicsDataset(selected_problems, self._info, self._split) - def take(self, n: int) -> "PhysicalDataset": + def take(self, n: int) -> "PhysicsDataset": """ Take the first N problems from the dataset. @@ -244,14 +240,14 @@ def take(self, n: int) -> "PhysicalDataset": n: Number of problems to take Returns: - New PhysicalDataset with the first N problems + New PhysicsDataset with the first N problems """ if n <= 0: - return PhysicalDataset([], self._info, self._split) + return PhysicsDataset([], self._info, self._split) n = min(n, len(self._problems)) - return PhysicalDataset(self._problems[:n], self._info, self._split) + return PhysicsDataset(self._problems[:n], self._info, self._split) - def head(self, n: int = 5) -> "PhysicalDataset": + def head(self, n: int = 5) -> "PhysicsDataset": """ Get the first N problems (similar to pandas head). @@ -259,11 +255,11 @@ def head(self, n: int = 5) -> "PhysicalDataset": n: Number of problems to get (default: 5) Returns: - New PhysicalDataset with the first N problems + New PhysicsDataset with the first N problems """ return self.take(n) - def tail(self, n: int = 5) -> "PhysicalDataset": + def tail(self, n: int = 5) -> "PhysicsDataset": """ Get the last N problems (similar to pandas tail). @@ -271,14 +267,14 @@ def tail(self, n: int = 5) -> "PhysicalDataset": n: Number of problems to get (default: 5) Returns: - New PhysicalDataset with the last N problems + New PhysicsDataset with the last N problems """ if n <= 0: - return PhysicalDataset([], self._info, self._split) + return PhysicsDataset([], self._info, self._split) n = min(n, len(self._problems)) - return PhysicalDataset(self._problems[-n:], self._info, self._split) + return PhysicsDataset(self._problems[-n:], self._info, self._split) - def sample(self, n: int) -> "PhysicalDataset": + def sample(self, n: int) -> "PhysicsDataset": """ Sample N problems from the dataset. @@ -286,14 +282,12 @@ def sample(self, n: int) -> "PhysicalDataset": n: Number of problems to sample Returns: - New PhysicalDataset with sampled problems + New PhysicsDataset with sampled problems """ if n <= 0: - return PhysicalDataset([], self._info, self._split) + return PhysicsDataset([], self._info, self._split) n = min(n, len(self._problems)) - return PhysicalDataset( - random.sample(self._problems, n), self._info, self._split - ) + return PhysicsDataset(random.sample(self._problems, n), self._info, self._split) def map(self, map_func: Callable[[PhysicsProblem], _MapResult]) -> list[_MapResult]: """ @@ -341,7 +335,7 @@ def save_to_json(self, filepath: str | Path) -> None: json.dump(data, f, indent=2, ensure_ascii=False) @classmethod - def from_json(cls, filepath: str | Path) -> "PhysicalDataset": + def from_json(cls, filepath: str | Path) -> "PhysicsDataset": """Load dataset from JSON file.""" filepath = Path(filepath) @@ -383,8 +377,8 @@ def get_statistics(self) -> dict[str, Any]: } def __repr__(self) -> str: - return f"PhysicalDataset({len(self._problems)} problems, split='{self._split}')" + return f"PhysicsDataset({len(self._problems)} problems, split='{self._split}')" def __str__(self) -> str: stats = self.get_statistics() - return f"PhysicalDataset with {stats['total_problems']} problems ({stats['split']} split)" + return f"PhysicsDataset with {stats['total_problems']} problems ({stats['split']} split)" diff --git a/src/prkit/core/domain/physics_problem.py b/src/prkit/core/domain/physics_problem.py index c9d86e7..19bf04b 100644 --- a/src/prkit/core/domain/physics_problem.py +++ b/src/prkit/core/domain/physics_problem.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any from ..logging_config import PRKitLogger -from .answer import Answer +from .answer import PhysicsAnswer from .physics_domain import PhysicsDomain # Get logger for this module @@ -44,7 +44,7 @@ class PhysicsProblem: question: str # Optional core fields - answer: Answer | None = None + answer: PhysicsAnswer | None = None solution: str | None = None domain: str | PhysicsDomain | None = None language: str = "en" @@ -348,7 +348,7 @@ def from_dict(cls, data: dict[str, Any]) -> "PhysicsProblem": if source_type is not None: source_type = str(source_type) answer_metadata = value.get("metadata") or {} - core_data[key] = Answer( + core_data[key] = PhysicsAnswer( value=answer_value, unit=answer_unit, source_type=source_type, diff --git a/src/prkit/datasets/hub.py b/src/prkit/datasets/hub.py index 673d961..c5d57f2 100644 --- a/src/prkit/datasets/hub.py +++ b/src/prkit/datasets/hub.py @@ -8,7 +8,7 @@ from typing import Any from prkit.core import PRKitLogger -from prkit.core.domain import PhysicalDataset +from prkit.core.domain import PhysicsDataset from prkit.datasets.downloaders import ( PHYBenchDownloader, PhysBenchDownloader, @@ -182,7 +182,7 @@ def load( auto_download: bool = False, allow_nonredistributable: bool = False, **kwargs: Any, - ) -> PhysicalDataset: + ) -> PhysicsDataset: """ Load a physical reasoning dataset. @@ -196,7 +196,7 @@ def load( **kwargs: Additional arguments for the specific loader (e.g., split, variant, etc.) Returns: - PhysicalDataset: Loaded dataset + PhysicsDataset: Loaded dataset Raises: ValueError: If dataset name is unknown, or if variant/split is invalid diff --git a/src/prkit/datasets/license_registry.py b/src/prkit/datasets/license_registry.py index 325bd6b..9d4c45c 100644 --- a/src/prkit/datasets/license_registry.py +++ b/src/prkit/datasets/license_registry.py @@ -2,7 +2,7 @@ One ``LicenseSpec`` per dataset, keyed by the same lowercase name used in ``DatasetHub._loaders`` / ``_downloaders``. Loaders and downloaders read from here instead of -hardcoding free-text strings, so ``PhysicalDataset.info["license"]`` is uniform and correct +hardcoding free-text strings, so ``PhysicsDataset.info["license"]`` is uniform and correct across every read path. The facts are hardcoded (not fetched from HF cards at runtime) to keep the load path network-free and the toolkit independent of external services. """ diff --git a/src/prkit/datasets/loaders/base_loader.py b/src/prkit/datasets/loaders/base_loader.py index e497df0..ba8accf 100644 --- a/src/prkit/datasets/loaders/base_loader.py +++ b/src/prkit/datasets/loaders/base_loader.py @@ -8,8 +8,8 @@ from typing import Any from prkit.core import PRKitLogger -from prkit.core.domain import PhysicalDataset, PhysicsProblem -from prkit.core.domain.answer import Answer +from prkit.core.domain import PhysicsDataset, PhysicsProblem +from prkit.core.domain.answer import PhysicsAnswer # Try to import PIL/Pillow for image loading PILImageModule: Any | None @@ -216,7 +216,7 @@ def field_mapping(self) -> dict[str, str]: pass @abstractmethod - def load(self, data_dir: str | Path, **kwargs: Any) -> PhysicalDataset: + def load(self, data_dir: str | Path, **kwargs: Any) -> PhysicsDataset: """ Load dataset from the specified directory. @@ -225,7 +225,7 @@ def load(self, data_dir: str | Path, **kwargs: Any) -> PhysicalDataset: **kwargs: Additional loading parameters Returns: - PhysicalDataset instance + PhysicsDataset instance """ pass @@ -541,7 +541,7 @@ def validate_required_fields(self, data: dict[str, Any]) -> list[str]: def _create_answer_from_raw( self, metadata: dict[str, Any], - ) -> Answer | None: + ) -> PhysicsAnswer | None: answer = metadata.get("answer") if answer is None: @@ -567,7 +567,7 @@ def _create_answer_from_raw( if source_type is not None: source_type = str(source_type) - return Answer(value=value, unit=unit, source_type=source_type) + return PhysicsAnswer(value=value, unit=unit, source_type=source_type) def create_physics_problem( self, @@ -654,11 +654,11 @@ def create_physics_problem( if source_answer_text: metadata["source_answer_text"] = source_answer_text - # Create Answer object from answer + # Create PhysicsAnswer object from answer answer_obj = self._create_answer_from_raw(metadata) metadata.pop("answer", None) metadata.pop("answer_category", None) # defensive: loaders may still set it - metadata.pop("source_type", None) # consumed into Answer; don't leak + metadata.pop("source_type", None) # consumed into PhysicsAnswer; don't leak metadata.pop("unit", None) # collect all other fields as additional fields diff --git a/src/prkit/datasets/loaders/cmphysbench_loader.py b/src/prkit/datasets/loaders/cmphysbench_loader.py index e10309c..9e76833 100644 --- a/src/prkit/datasets/loaders/cmphysbench_loader.py +++ b/src/prkit/datasets/loaders/cmphysbench_loader.py @@ -27,7 +27,7 @@ from typing import Any from prkit.core import PRKitLogger -from prkit.core.domain import PhysicalDataset, PhysicsProblem +from prkit.core.domain import PhysicsDataset, PhysicsProblem from prkit.datasets.license_registry import get_license from .base_loader import BaseDatasetLoader @@ -131,7 +131,7 @@ def load( split: str | None = None, sample_size: int | None = None, **kwargs: Any, - ) -> PhysicalDataset: + ) -> PhysicsDataset: """ Load the CMPhysBench dataset from a local copy. @@ -144,7 +144,7 @@ def load( **kwargs: Additional loading parameters. Returns: - PhysicalDataset instance. + PhysicsDataset instance. Raises: ValueError: If an unsupported split or variant is requested, or the JSON @@ -199,4 +199,4 @@ def load( f"Successfully loaded {len(problems)} problems from CMPhysBench dataset" ) - return PhysicalDataset(problems, info, split=split) + return PhysicsDataset(problems, info, split=split) diff --git a/src/prkit/datasets/loaders/jeebench_loader.py b/src/prkit/datasets/loaders/jeebench_loader.py index 254987f..d98322c 100644 --- a/src/prkit/datasets/loaders/jeebench_loader.py +++ b/src/prkit/datasets/loaders/jeebench_loader.py @@ -37,7 +37,7 @@ from typing import Any from prkit.core import PRKitLogger -from prkit.core.domain import PhysicalDataset, PhysicsProblem +from prkit.core.domain import PhysicsDataset, PhysicsProblem from prkit.datasets.license_registry import get_license from .base_loader import BaseDatasetLoader @@ -100,7 +100,7 @@ def load( split: str | None = None, sample_size: int | None = None, **kwargs: Any, - ) -> PhysicalDataset: + ) -> PhysicsDataset: """ Load the JEEBench dataset. @@ -113,7 +113,7 @@ def load( **kwargs: Additional loading parameters Returns: - PhysicalDataset instance + PhysicsDataset instance Raises: ValueError: If unsupported split or variant is requested @@ -181,7 +181,7 @@ def load( f"Successfully loaded {len(problems)} problems from JEEBench dataset" ) - return PhysicalDataset( + return PhysicsDataset( problems, info, split=split, diff --git a/src/prkit/datasets/loaders/phybench_loader.py b/src/prkit/datasets/loaders/phybench_loader.py index ed7622b..d1734af 100644 --- a/src/prkit/datasets/loaders/phybench_loader.py +++ b/src/prkit/datasets/loaders/phybench_loader.py @@ -12,7 +12,7 @@ from pathlib import Path from typing import Any -from prkit.core.domain import PhysicalDataset, PhysicsProblem +from prkit.core.domain import PhysicsDataset, PhysicsProblem from prkit.core.domain.physics_domain import PhysicsDomain from prkit.datasets.license_registry import get_license @@ -90,7 +90,7 @@ def load( sample_size: int | None = None, split: str | None = None, **kwargs: Any, - ) -> PhysicalDataset: + ) -> PhysicsDataset: """ Load PHYBench dataset. @@ -101,7 +101,7 @@ def load( **kwargs: Additional loading parameters (unused, for compatibility) Returns: - PhysicalDataset containing PHYBench problems + PhysicsDataset containing PHYBench problems """ # Use defaults if not provided if variant is None: @@ -157,7 +157,7 @@ def load( info = self.get_info() info["total_problems"] = len(problems) - return PhysicalDataset( + return PhysicsDataset( problems, info, split=split, diff --git a/src/prkit/datasets/loaders/physbench_loader.py b/src/prkit/datasets/loaders/physbench_loader.py index 537b1e8..1f9482a 100644 --- a/src/prkit/datasets/loaders/physbench_loader.py +++ b/src/prkit/datasets/loaders/physbench_loader.py @@ -11,7 +11,7 @@ from typing import Any from prkit.core import PRKitLogger -from prkit.core.domain import PhysicalDataset, PhysicsProblem +from prkit.core.domain import PhysicsDataset, PhysicsProblem from prkit.datasets.license_registry import get_license from .base_loader import BaseDatasetLoader @@ -105,7 +105,7 @@ def load( split: str | None = None, sample_size: int | None = None, **kwargs: Any, - ) -> PhysicalDataset: + ) -> PhysicsDataset: """ Load PhysBench from a local cache directory. @@ -191,7 +191,7 @@ def load( variant, split, ) - return PhysicalDataset(problems, info, split=split) + return PhysicsDataset(problems, info, split=split) def _filter_records( self, diff --git a/src/prkit/datasets/loaders/physics_loader.py b/src/prkit/datasets/loaders/physics_loader.py index e0ccda1..45f9c20 100644 --- a/src/prkit/datasets/loaders/physics_loader.py +++ b/src/prkit/datasets/loaders/physics_loader.py @@ -12,7 +12,7 @@ from typing import Any from prkit.core import PRKitLogger -from prkit.core.domain import PhysicalDataset, PhysicsDomain, PhysicsProblem +from prkit.core.domain import PhysicsDataset, PhysicsDomain, PhysicsProblem from prkit.datasets.license_registry import get_license from .base_loader import BaseDatasetLoader @@ -105,7 +105,7 @@ def load( sample_size: int | None = None, decode_images: bool = True, **kwargs: Any, - ) -> PhysicalDataset: + ) -> PhysicsDataset: """ Load the PHYSICS dataset. @@ -118,7 +118,7 @@ def load( **kwargs: Additional loading parameters (ignored for compatibility) Returns: - PhysicalDataset instance + PhysicsDataset instance """ del kwargs # Unused, kept for loader API compatibility @@ -181,7 +181,7 @@ def load( variant, split, ) - return PhysicalDataset(problems, info, split=split) + return PhysicsDataset(problems, info, split=split) def _validate_variant_split_combo(self, variant: str, split: str) -> None: if (variant, split) not in self.FILE_PATTERNS: diff --git a/src/prkit/datasets/loaders/physreason_loader.py b/src/prkit/datasets/loaders/physreason_loader.py index 8c578dc..aef19bc 100644 --- a/src/prkit/datasets/loaders/physreason_loader.py +++ b/src/prkit/datasets/loaders/physreason_loader.py @@ -12,7 +12,7 @@ from typing import Any from prkit.core import PRKitLogger -from prkit.core.domain import PhysicalDataset, PhysicsProblem +from prkit.core.domain import PhysicsDataset, PhysicsProblem from prkit.datasets.license_registry import get_license from prkit.datasets.loaders.base_loader import BaseDatasetLoader @@ -121,7 +121,7 @@ def load( sample_size: int | None = None, split: str | None = None, **kwargs: Any, - ) -> PhysicalDataset: + ) -> PhysicsDataset: """ Load PhysReason dataset from the specified directory. @@ -133,7 +133,7 @@ def load( **kwargs: Additional loading parameters (ignored for compatibility) Returns: - PhysicalDataset containing PhysReason problems + PhysicsDataset containing PhysReason problems """ # Use defaults if not provided if variant is None: @@ -217,8 +217,8 @@ def load( f"Successfully created {len(physics_problems)} PhysicsProblem objects" ) - # Create PhysicalDataset - dataset = PhysicalDataset( + # Create PhysicsDataset + dataset = PhysicsDataset( problems=physics_problems, info={ "name": self.name, diff --git a/src/prkit/datasets/loaders/phyx_loader.py b/src/prkit/datasets/loaders/phyx_loader.py index f0bd800..2dffc45 100644 --- a/src/prkit/datasets/loaders/phyx_loader.py +++ b/src/prkit/datasets/loaders/phyx_loader.py @@ -13,7 +13,7 @@ from typing import Any from prkit.core import PRKitLogger -from prkit.core.domain import PhysicalDataset, PhysicsProblem +from prkit.core.domain import PhysicsDataset, PhysicsProblem from prkit.core.domain.physics_domain import PhysicsDomain from prkit.datasets.license_registry import get_license @@ -154,7 +154,7 @@ def load( sample_size: int | None = None, split: str | None = None, **kwargs: Any, - ) -> PhysicalDataset: + ) -> PhysicsDataset: """ Load PhyX dataset. @@ -166,7 +166,7 @@ def load( **kwargs: Additional loading parameters (unused, for compatibility) Returns: - PhysicalDataset containing PhyX problems + PhysicsDataset containing PhyX problems """ # Use defaults if not provided if variant is None: @@ -236,7 +236,7 @@ def load( info["variant"] = variant info["split"] = split - return PhysicalDataset( + return PhysicsDataset( problems, info, split=split, diff --git a/src/prkit/datasets/loaders/seephys_loader.py b/src/prkit/datasets/loaders/seephys_loader.py index 2a507d2..8a1fe24 100644 --- a/src/prkit/datasets/loaders/seephys_loader.py +++ b/src/prkit/datasets/loaders/seephys_loader.py @@ -7,7 +7,7 @@ from typing import Any from prkit.core import PRKitLogger -from prkit.core.domain import PhysicalDataset, PhysicsProblem +from prkit.core.domain import PhysicsDataset, PhysicsProblem from prkit.datasets.license_registry import get_license from .base_loader import BaseDatasetLoader @@ -69,7 +69,7 @@ def load( sample_size: int | None = None, split: str | None = None, **kwargs: Any, - ) -> PhysicalDataset: + ) -> PhysicsDataset: """ Load SeePhys dataset. @@ -81,7 +81,7 @@ def load( **kwargs: Additional loading parameters Returns: - PhysicalDataset containing SeePhys problems + PhysicsDataset containing SeePhys problems """ # Use defaults if not provided if split is None: @@ -125,7 +125,7 @@ def _load_from_json_only( split: str, sample_size: int | None, **_kwargs: Any, - ) -> PhysicalDataset: + ) -> PhysicsDataset: """Load from split directory.""" split_dir = data_dir / split @@ -162,7 +162,7 @@ def _load_from_json_only( f"Successfully loaded {len(problems)} problems from SeePhys dataset" ) - return PhysicalDataset(problems, info, split=split) + return PhysicsDataset(problems, info, split=split) def _load_from_json_dir( self, split_dir: Path, data_dir: Path diff --git a/src/prkit/datasets/loaders/tpbench_loader.py b/src/prkit/datasets/loaders/tpbench_loader.py index 8652d12..10077e8 100644 --- a/src/prkit/datasets/loaders/tpbench_loader.py +++ b/src/prkit/datasets/loaders/tpbench_loader.py @@ -13,7 +13,7 @@ import pandas as pd from prkit.core import PRKitLogger -from prkit.core.domain import PhysicalDataset, PhysicsDomain, PhysicsProblem +from prkit.core.domain import PhysicsDataset, PhysicsDomain, PhysicsProblem from prkit.datasets.license_registry import get_license from prkit.datasets.loaders.base_loader import BaseDatasetLoader @@ -102,7 +102,7 @@ def load( per_domain: int | None = None, language: str = "en", **kwargs: Any, - ) -> PhysicalDataset: + ) -> PhysicsDataset: """ Load the TPBench dataset. @@ -115,7 +115,7 @@ def load( language: Language to load ("en" only) Returns: - PhysicalDataset instance + PhysicsDataset instance Raises: ValueError: If unsupported split, variant, or language is requested @@ -207,7 +207,7 @@ def load( f"Successfully loaded {len(all_problems)} problems from TPBench dataset" ) - return PhysicalDataset( + return PhysicsDataset( all_problems, info, split=split, diff --git a/src/prkit/datasets/loaders/ugphysics_loader.py b/src/prkit/datasets/loaders/ugphysics_loader.py index a40d818..368a32b 100644 --- a/src/prkit/datasets/loaders/ugphysics_loader.py +++ b/src/prkit/datasets/loaders/ugphysics_loader.py @@ -12,7 +12,7 @@ from typing import Any from prkit.core import PRKitLogger -from prkit.core.domain import PhysicalDataset, PhysicsDomain, PhysicsProblem +from prkit.core.domain import PhysicsDataset, PhysicsDomain, PhysicsProblem from prkit.datasets.license_registry import get_license from prkit.datasets.loaders.base_loader import BaseDatasetLoader from prkit.datasets.ugphysics_common import ( @@ -284,7 +284,7 @@ def load( per_domain: int | None = None, language: str | None = None, **kwargs: Any, - ) -> PhysicalDataset: + ) -> PhysicsDataset: """ Load the UGPhysics dataset. @@ -297,7 +297,7 @@ def load( language: Backward-compatible alias for split selection Returns: - PhysicalDataset instance + PhysicsDataset instance Raises: ValueError: If unsupported split or variant is requested @@ -406,7 +406,7 @@ def load( f"dataset variant='{variant}' split='{split}'" ) - return PhysicalDataset( + return PhysicsDataset( problems, info, split=split, diff --git a/src/prkit/datasets/utils.py b/src/prkit/datasets/utils.py index 118282a..fc58b71 100644 --- a/src/prkit/datasets/utils.py +++ b/src/prkit/datasets/utils.py @@ -6,16 +6,16 @@ from pathlib import Path from typing import Any -from prkit.core.domain.physics_dataset import PhysicalDataset +from prkit.core.domain.physics_dataset import PhysicsDataset from prkit.core.domain.physics_problem import PhysicsProblem def sample_balanced( - dataset: PhysicalDataset, + dataset: PhysicsDataset, field: str, samples_per_category: int, seed: int | None = None, -) -> PhysicalDataset: +) -> PhysicsDataset: """ Sample a balanced subset from the dataset based on a categorical field. @@ -61,10 +61,10 @@ def sample_balanced( } ) - return PhysicalDataset(balanced_samples, info, dataset.split) + return PhysicsDataset(balanced_samples, info, dataset.split) -def get_statistics(dataset: PhysicalDataset) -> dict[str, Any]: +def get_statistics(dataset: PhysicsDataset) -> dict[str, Any]: """ Get statistics about the dataset. @@ -107,7 +107,7 @@ def get_statistics(dataset: PhysicalDataset) -> dict[str, Any]: def export_to_json( - dataset: PhysicalDataset, output_path: str | Path, include_info: bool = True + dataset: PhysicsDataset, output_path: str | Path, include_info: bool = True ) -> None: """ Export dataset to JSON file. @@ -129,11 +129,11 @@ def export_to_json( def filter_by_keywords( - dataset: PhysicalDataset, + dataset: PhysicsDataset, keywords: list[str], fields: list[str] | None = None, case_sensitive: bool = False, -) -> PhysicalDataset: +) -> PhysicsDataset: """ Filter dataset by keywords in specified fields. @@ -168,8 +168,8 @@ def matches_keywords(sample: PhysicsProblem) -> bool: def create_cross_validation_splits( - dataset: PhysicalDataset, n_splits: int = 5, seed: int | None = None -) -> list[tuple[PhysicalDataset, PhysicalDataset]]: + dataset: PhysicsDataset, n_splits: int = 5, seed: int | None = None +) -> list[tuple[PhysicsDataset, PhysicsDataset]]: """ Create cross-validation splits of the dataset. @@ -189,7 +189,7 @@ def create_cross_validation_splits( random.shuffle(indices) # Create splits - splits: list[tuple[PhysicalDataset, PhysicalDataset]] = [] + splits: list[tuple[PhysicsDataset, PhysicsDataset]] = [] fold_size = len(dataset) // n_splits for i in range(n_splits): @@ -208,7 +208,7 @@ def create_cross_validation_splits( def validate_dataset_format( - dataset: PhysicalDataset, required_fields: list[str] | None = None + dataset: PhysicsDataset, required_fields: list[str] | None = None ) -> dict[str, Any]: """ Validate dataset format and check for consistency. diff --git a/src/prkit/evaluation/llm_judge/payload.py b/src/prkit/evaluation/llm_judge/payload.py index 807425c..5f0e7a9 100644 --- a/src/prkit/evaluation/llm_judge/payload.py +++ b/src/prkit/evaluation/llm_judge/payload.py @@ -5,12 +5,12 @@ import re from typing import Any -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer -def answer_to_text_and_category(answer: str | Answer) -> tuple[str, str]: +def answer_to_text_and_category(answer: str | PhysicsAnswer) -> tuple[str, str]: """Plain text and native-type label for embedding in a judge JSON payload.""" - if isinstance(answer, Answer): + if isinstance(answer, PhysicsAnswer): return str(answer).strip(), answer.source_type or "" return str(answer).strip(), "" @@ -24,8 +24,8 @@ def clean_answer_text(answer_text: str) -> str: def build_standard_answer_judge_payload( - predicted: str | Answer, - ground_truth: str | Answer, + predicted: str | PhysicsAnswer, + ground_truth: str | PhysicsAnswer, question: str | None, ) -> dict[str, Any]: """Standard physics payload: ``question``, ``ground_truth``, ``model_answer``.""" diff --git a/src/prkit/scoring/eed_scorer.py b/src/prkit/scoring/eed_scorer.py index 6aa3186..8f31f31 100644 --- a/src/prkit/scoring/eed_scorer.py +++ b/src/prkit/scoring/eed_scorer.py @@ -21,17 +21,17 @@ from typing import Any -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer from prkit.core.verdict import Verdict #: Provenance stamp: ``/@+frontend-+wrap``. _VERSION = "eed/phybench@706feb4+frontend-vendor+wrap1" -def _as_text(value: Answer | str | Any) -> str: - """Render an ``Answer``/string answer as the raw text the front-end expects. +def _as_text(value: PhysicsAnswer | str | Any) -> str: + """Render an ``PhysicsAnswer``/string answer as the raw text the front-end expects. - ``Answer.__str__`` appends the unit when present (``"3 m/s"``), which is exactly + ``PhysicsAnswer.__str__`` appends the unit when present (``"3 m/s"``), which is exactly the surface the vendored LaTeX front-end parses. """ if isinstance(value, str): @@ -62,13 +62,13 @@ def __init__( def score( self, - prediction: Answer | str, - reference: Answer | str, + prediction: PhysicsAnswer | str, + reference: PhysicsAnswer | str, **kwargs: Any, ) -> Verdict: """Score ``prediction`` against ``reference`` with the vendored EED pipeline. - Both inputs may be raw strings or :class:`Answer` objects. The reference is + Both inputs may be raw strings or :class:`PhysicsAnswer` objects. The reference is the EED ``answer`` (ground truth) and the prediction is the EED ``test``. """ # Lazy import keeps latex2sympy2_extended off `import prkit.scoring`. diff --git a/src/prkit/scoring/llm_judge_scorer.py b/src/prkit/scoring/llm_judge_scorer.py index 2f304c5..058225f 100644 --- a/src/prkit/scoring/llm_judge_scorer.py +++ b/src/prkit/scoring/llm_judge_scorer.py @@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Any -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer from prkit.core.verdict import Verdict if TYPE_CHECKING: # annotations only — never imported at runtime by this module @@ -66,8 +66,8 @@ def __init__( def score( self, - prediction: Answer | str, - reference: Answer | str, + prediction: PhysicsAnswer | str, + reference: PhysicsAnswer | str, *, question: str | None = None, **kwargs: Any, diff --git a/src/prkit/scoring/seed_scorer.py b/src/prkit/scoring/seed_scorer.py index be276f3..de14ad0 100644 --- a/src/prkit/scoring/seed_scorer.py +++ b/src/prkit/scoring/seed_scorer.py @@ -26,7 +26,7 @@ import re from typing import Any -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer from prkit.core.verdict import Verdict #: Provenance stamp: ``/@+frontend-+wrap``. @@ -43,8 +43,8 @@ _SEED_TYPE_SET = frozenset(SEED_ANSWER_TYPES) -def _as_text(value: Answer | str | Any) -> str: - """Render an ``Answer``/string answer as the raw text the front-end expects.""" +def _as_text(value: PhysicsAnswer | str | Any) -> str: + """Render an ``PhysicsAnswer``/string answer as the raw text the front-end expects.""" if isinstance(value, str): return value return str(value) @@ -90,8 +90,8 @@ def __init__( def score( self, - prediction: Answer | str, - reference: Answer | str, + prediction: PhysicsAnswer | str, + reference: PhysicsAnswer | str, *, answer_type: str | None = None, **kwargs: Any, @@ -127,7 +127,7 @@ def score( def _resolve_answer_type( self, answer_type: str | None, - reference: Answer | str, + reference: PhysicsAnswer | str, ref_text: str, ) -> tuple[str, bool]: """Resolve the SEED dispatch token; return ``(token, classifier_used)``. diff --git a/src/prkit/scoring/semantics_eed_scorer.py b/src/prkit/scoring/semantics_eed_scorer.py index a074ee1..ee0daa4 100644 --- a/src/prkit/scoring/semantics_eed_scorer.py +++ b/src/prkit/scoring/semantics_eed_scorer.py @@ -21,7 +21,7 @@ from typing import Any -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer from prkit.core.verdict import Verdict from prkit.semantics import ( ComparisonPolicyMode, @@ -119,15 +119,15 @@ def _effective_context( def score( self, - prediction: Answer | str | PhysicsAnswerSemantics, - reference: Answer | str | PhysicsAnswerSemantics, + prediction: PhysicsAnswer | str | PhysicsAnswerSemantics, + reference: PhysicsAnswer | str | PhysicsAnswerSemantics, *, context: PhysicsQuestionSemantics | dict[str, Any] | None = None, **kwargs: Any, ) -> Verdict: """Score ``prediction`` against ``reference`` with our-semantics + EED core. - Inputs may be raw strings, :class:`Answer` objects, or already-normalized + Inputs may be raw strings, :class:`PhysicsAnswer` objects, or already-normalized :class:`PhysicsAnswerSemantics`. A non-applicable reference kind/structure yields the reserved ``score=-1.0`` not-applicable verdict. """ diff --git a/src/prkit/scoring/semantics_scorer.py b/src/prkit/scoring/semantics_scorer.py index b009b0b..198e913 100644 --- a/src/prkit/scoring/semantics_scorer.py +++ b/src/prkit/scoring/semantics_scorer.py @@ -14,7 +14,7 @@ from typing import Any -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer from prkit.core.verdict import Verdict from prkit.semantics import ( PREDICTION_PROMPT_VERSION, @@ -132,8 +132,8 @@ def _effective_context( def score( self, - prediction: Answer | str | PhysicsAnswerSemantics, - reference: Answer | str | PhysicsAnswerSemantics, + prediction: PhysicsAnswer | str | PhysicsAnswerSemantics, + reference: PhysicsAnswer | str | PhysicsAnswerSemantics, *, context: PhysicsQuestionSemantics | dict[str, Any] | None = None, policy_mode: ComparisonPolicyMode | str | None = None, @@ -141,7 +141,7 @@ def score( ) -> Verdict: """Score ``prediction`` against ``reference`` and return a canonical Verdict. - ``prediction`` / ``reference`` may be raw strings, :class:`Answer` objects, + ``prediction`` / ``reference`` may be raw strings, :class:`PhysicsAnswer` objects, or already-normalized :class:`PhysicsAnswerSemantics` (e.g. from :func:`prkit.semantics.extract_prediction_answer_semantics`); all three are accepted by the normalizer. The @@ -154,7 +154,7 @@ def score( effective_context = self._effective_context(context) effective_policy = policy_mode if policy_mode is not None else self._policy_mode - # normalize_physics_answer accepts str | Answer | PhysicsAnswerSemantics. + # normalize_physics_answer accepts str | PhysicsAnswer | PhysicsAnswerSemantics. pred_sem = normalize_physics_answer(prediction, context=effective_context) ref_sem = normalize_physics_answer(reference, context=effective_context) diff --git a/src/prkit/scoring/semantics_seed_scorer.py b/src/prkit/scoring/semantics_seed_scorer.py index 0f64920..53c6306 100644 --- a/src/prkit/scoring/semantics_seed_scorer.py +++ b/src/prkit/scoring/semantics_seed_scorer.py @@ -26,7 +26,7 @@ from typing import Any -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer from prkit.core.verdict import Verdict from prkit.semantics import ( ComparisonPolicyMode, @@ -124,15 +124,15 @@ def _effective_context( def score( self, - prediction: Answer | str | PhysicsAnswerSemantics, - reference: Answer | str | PhysicsAnswerSemantics, + prediction: PhysicsAnswer | str | PhysicsAnswerSemantics, + reference: PhysicsAnswer | str | PhysicsAnswerSemantics, *, context: PhysicsQuestionSemantics | dict[str, Any] | None = None, **kwargs: Any, ) -> Verdict: """Score ``prediction`` against ``reference`` with our-semantics + SEED core. - Inputs may be raw strings, :class:`Answer` objects, or already-normalized + Inputs may be raw strings, :class:`PhysicsAnswer` objects, or already-normalized :class:`PhysicsAnswerSemantics`. A non-applicable reference kind/structure yields the reserved ``score=-1.0`` not-applicable verdict; otherwise the SEED dispatch type is derived from the reference and the matching core path runs. diff --git a/src/prkit/semantics/README.md b/src/prkit/semantics/README.md index bae4d37..ec9f586 100644 --- a/src/prkit/semantics/README.md +++ b/src/prkit/semantics/README.md @@ -305,7 +305,7 @@ The smallest reproducible path is deterministic question inference, answer normalization, contract construction, and evaluation. ```python -from prkit.core.domain import Answer, PhysicsProblem +from prkit.core.domain import PhysicsAnswer, PhysicsProblem from prkit.semantics import ( ComparisonPolicyMode, build_evaluation_contract, @@ -317,7 +317,7 @@ from prkit.semantics import ( problem = PhysicsProblem( problem_id="demo-speed", question="Find the speed in m/s.", - answer=Answer(value="18", unit="km/h", source_type="physical_quantity"), + answer=PhysicsAnswer(value="18", unit="km/h", source_type="physical_quantity"), ) question_semantics = infer_reference_question_semantics(problem) diff --git a/src/prkit/semantics/build/prompts.py b/src/prkit/semantics/build/prompts.py index d89d834..b419cb4 100644 --- a/src/prkit/semantics/build/prompts.py +++ b/src/prkit/semantics/build/prompts.py @@ -2,7 +2,7 @@ from __future__ import annotations -from prkit.core.domain import Answer, PhysicsProblem +from prkit.core.domain import PhysicsAnswer, PhysicsProblem from prkit.core.model_clients.prompts import format_problem_context from ..normalization import ( @@ -290,7 +290,7 @@ def answer_like_to_text(answer: object) -> str: return "" if isinstance(answer, PhysicsAnswerSemantics): return answer.raw_text or answer.canonical_text - if isinstance(answer, Answer): + if isinstance(answer, PhysicsAnswer): value_text = str(answer.value).strip() unit_text = "" if answer.unit is None else str(answer.unit).strip() if value_text and unit_text: diff --git a/src/prkit/semantics/normalization/answer_normalization.py b/src/prkit/semantics/normalization/answer_normalization.py index b8862cb..383609e 100644 --- a/src/prkit/semantics/normalization/answer_normalization.py +++ b/src/prkit/semantics/normalization/answer_normalization.py @@ -6,7 +6,7 @@ from collections.abc import Iterable from typing import Any -from prkit.core.domain import Answer, PhysicsProblem +from prkit.core.domain import PhysicsAnswer, PhysicsProblem from ..part_labels import canonicalize_part_label, infer_multi_part_part_labels from ..schema import ( @@ -213,7 +213,7 @@ def normalize_problem_answer( def normalize_physics_answer( - answer: str | Answer | PhysicsAnswerSemantics | Any, + answer: str | PhysicsAnswer | PhysicsAnswerSemantics | Any, *, context: PhysicsQuestionSemantics | None = None, ) -> PhysicsAnswerSemantics: diff --git a/src/prkit/semantics/normalization/question_inference.py b/src/prkit/semantics/normalization/question_inference.py index c559b53..94bfaa9 100644 --- a/src/prkit/semantics/normalization/question_inference.py +++ b/src/prkit/semantics/normalization/question_inference.py @@ -5,7 +5,7 @@ import re from typing import Any -from prkit.core.domain import Answer, PhysicsProblem +from prkit.core.domain import PhysicsAnswer, PhysicsProblem from ..part_labels import ( canonicalize_part_label, @@ -607,7 +607,7 @@ def _answer_to_raw_text(answer: Any) -> str: return "" if isinstance(answer, PhysicsAnswerSemantics): return answer.canonical_text - if isinstance(answer, Answer): + if isinstance(answer, PhysicsAnswer): return _join_value_and_unit(answer.value, answer.unit) if isinstance(answer, dict): raw_text = answer.get("raw_text") diff --git a/src/prkit/verify.py b/src/prkit/verify.py index 36575d1..4a2001b 100644 --- a/src/prkit/verify.py +++ b/src/prkit/verify.py @@ -24,7 +24,7 @@ from prkit.core.verdict import Verdict if TYPE_CHECKING: # annotations only — never imported at runtime by this module - from prkit.core.domain.answer import Answer + from prkit.core.domain.answer import PhysicsAnswer from prkit.semantics import PhysicsAnswerSemantics, PhysicsQuestionSemantics from prkit.semantics.build import ReferenceSemanticsArtifact @@ -69,8 +69,8 @@ def _resolve_question_context( def verify( - gold: Answer | str | PhysicsAnswerSemantics, - pred: Answer | str | PhysicsAnswerSemantics, + gold: PhysicsAnswer | str | PhysicsAnswerSemantics, + pred: PhysicsAnswer | str | PhysicsAnswerSemantics, *, tolerance: float | None = None, unit_policy: str = "strict", @@ -83,7 +83,7 @@ def verify( A ``math-verify``-shaped one-call verifier returning the canonical :class:`~prkit.core.verdict.Verdict`. ``gold`` / ``pred`` may be raw strings, - :class:`~prkit.core.domain.answer.Answer` objects, or pre-parsed + :class:`~prkit.core.domain.answer.PhysicsAnswer` objects, or pre-parsed :class:`~prkit.semantics.PhysicsAnswerSemantics`. Args: diff --git a/tests/conftest.py b/tests/conftest.py index 8ab5c4e..8383fc5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,8 +8,8 @@ import pytest from prkit.core.domain import ( - Answer, - PhysicalDataset, + PhysicsAnswer, + PhysicsDataset, PhysicsDomain, PhysicsProblem, ) @@ -18,25 +18,25 @@ @pytest.fixture def sample_answer_numerical(): """Create a sample numerical answer.""" - return Answer(value="42.0", unit="m/s", source_type="NV") + return PhysicsAnswer(value="42.0", unit="m/s", source_type="NV") @pytest.fixture def sample_answer_symbolic(): """Create a sample symbolic answer.""" - return Answer(value="x^2 + 2x + 1") + return PhysicsAnswer(value="x^2 + 2x + 1") @pytest.fixture def sample_answer_textual(): """Create a sample textual answer.""" - return Answer(value="The force is equal to mass times acceleration") + return PhysicsAnswer(value="The force is equal to mass times acceleration") @pytest.fixture def sample_answer_option(): """Create a sample option answer.""" - return Answer(value="A", source_type="MC") + return PhysicsAnswer(value="A", source_type="MC") @pytest.fixture @@ -45,7 +45,7 @@ def sample_physics_problem(): return PhysicsProblem( problem_id="test_001", question="What is the speed of light?", - answer=Answer(value="3e8", unit="m/s"), + answer=PhysicsAnswer(value="3e8", unit="m/s"), solution="The speed of light in vacuum is approximately 3 × 10^8 m/s", domain=PhysicsDomain.CLASSICAL_MECHANICS, language="en", @@ -59,7 +59,7 @@ def sample_physics_problem_mc(): return PhysicsProblem( problem_id="test_002", question="What is F = ma?", - answer=Answer(value="A", source_type="MCQ"), + answer=PhysicsAnswer(value="A", source_type="MCQ"), options=[ "Newton's second law", "Newton's first law", @@ -76,7 +76,7 @@ def sample_physics_problem_mc(): def sample_dataset(sample_physics_problem, sample_physics_problem_mc): """Create a sample dataset with multiple problems.""" problems = [sample_physics_problem, sample_physics_problem_mc] - return PhysicalDataset( + return PhysicsDataset( problems=problems, info={"name": "test_dataset", "version": "1.0"}, split="test" ) @@ -96,7 +96,7 @@ def sample_problems_list(): problem = PhysicsProblem( problem_id=f"test_{i:03d}", question=f"Test question {i}", - answer=Answer(value=str(i)), + answer=PhysicsAnswer(value=str(i)), domain=( PhysicsDomain.CLASSICAL_MECHANICS if i % 2 == 0 diff --git a/tests/prkit/core/domain/test_answer.py b/tests/prkit/core/domain/test_answer.py index c8c8d0e..82448c3 100644 --- a/tests/prkit/core/domain/test_answer.py +++ b/tests/prkit/core/domain/test_answer.py @@ -1,117 +1,117 @@ -"""Tests for the thin Answer observation record.""" +"""Tests for the thin PhysicsAnswer observation record.""" -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer class TestAnswerCreation: def test_value_only(self): - a = Answer(value="42") + a = PhysicsAnswer(value="42") assert a.value == "42" assert a.unit is None assert a.source_type is None assert a.metadata == {} def test_with_unit(self): - a = Answer(value="9.81", unit="m/s^2") + a = PhysicsAnswer(value="9.81", unit="m/s^2") assert a.value == "9.81" assert a.unit == "m/s^2" def test_with_source_type(self): - a = Answer(value="A", source_type="MC") + a = PhysicsAnswer(value="A", source_type="MC") assert a.source_type == "MC" def test_with_all_fields(self): - a = Answer(value="5", unit="N", source_type="NV", metadata={"raw": True}) + a = PhysicsAnswer(value="5", unit="N", source_type="NV", metadata={"raw": True}) assert a.value == "5" assert a.unit == "N" assert a.source_type == "NV" assert a.metadata["raw"] is True def test_metadata_default_not_shared(self): - a = Answer(value="x") - b = Answer(value="y") + a = PhysicsAnswer(value="x") + b = PhysicsAnswer(value="y") a.metadata["k"] = 1 assert "k" not in b.metadata def test_metadata_none_normalized(self): - a = Answer(value="x", metadata=None) # type: ignore[arg-type] + a = PhysicsAnswer(value="x", metadata=None) # type: ignore[arg-type] assert a.metadata == {} class TestAnswerAccessors: def test_get_value(self): - a = Answer(value="hello") + a = PhysicsAnswer(value="hello") assert a.get_value() == "hello" def test_get_unit_present(self): - a = Answer(value="3", unit="m") + a = PhysicsAnswer(value="3", unit="m") assert a.get_unit() == "m" def test_get_unit_absent(self): - a = Answer(value="3") + a = PhysicsAnswer(value="3") assert a.get_unit() is None def test_has_unit_true(self): - a = Answer(value="3", unit="m") + a = PhysicsAnswer(value="3", unit="m") assert a.has_unit() is True def test_has_unit_false(self): - a = Answer(value="3") + a = PhysicsAnswer(value="3") assert a.has_unit() is False class TestAnswerDunder: def test_str_with_unit(self): - a = Answer(value="9.81", unit="m/s^2") + a = PhysicsAnswer(value="9.81", unit="m/s^2") assert str(a) == "9.81 m/s^2" def test_str_without_unit(self): - a = Answer(value="42") + a = PhysicsAnswer(value="42") assert str(a) == "42" def test_repr_contains_key_fields(self): - a = Answer(value="5", unit="N", source_type="NV") + a = PhysicsAnswer(value="5", unit="N", source_type="NV") r = repr(a) - assert "Answer(" in r + assert "PhysicsAnswer(" in r assert "'5'" in r assert "'N'" in r assert "'NV'" in r def test_repr_no_answer_kind(self): - a = Answer(value="x") + a = PhysicsAnswer(value="x") assert "answer_kind" not in repr(a) class TestAnswerToDict: def test_value_only(self): - d = Answer(value="42").to_dict() + d = PhysicsAnswer(value="42").to_dict() assert d == {"value": "42"} def test_with_unit(self): - d = Answer(value="3", unit="m").to_dict() + d = PhysicsAnswer(value="3", unit="m").to_dict() assert d == {"value": "3", "unit": "m"} def test_with_source_type(self): - d = Answer(value="A", source_type="MCQ").to_dict() + d = PhysicsAnswer(value="A", source_type="MCQ").to_dict() assert d == {"value": "A", "source_type": "MCQ"} def test_with_metadata(self): - d = Answer(value="x", metadata={"raw": "1"}).to_dict() + d = PhysicsAnswer(value="x", metadata={"raw": "1"}).to_dict() assert d["metadata"] == {"raw": "1"} def test_no_answer_kind_key(self): - d = Answer(value="x").to_dict() + d = PhysicsAnswer(value="x").to_dict() assert "answer_kind" not in d assert "answer_category" not in d def test_empty_unit_omitted(self): - d = Answer(value="x", unit=None).to_dict() + d = PhysicsAnswer(value="x", unit=None).to_dict() assert "unit" not in d def test_none_source_type_omitted(self): - d = Answer(value="x", source_type=None).to_dict() + d = PhysicsAnswer(value="x", source_type=None).to_dict() assert "source_type" not in d def test_empty_metadata_omitted(self): - d = Answer(value="x", metadata={}).to_dict() + d = PhysicsAnswer(value="x", metadata={}).to_dict() assert "metadata" not in d diff --git a/tests/prkit/core/domain/test_physics_dataset.py b/tests/prkit/core/domain/test_physics_dataset.py index 8ff5867..fa65216 100644 --- a/tests/prkit/core/domain/test_physics_dataset.py +++ b/tests/prkit/core/domain/test_physics_dataset.py @@ -1,27 +1,27 @@ """ -Tests for PhysicalDataset model. +Tests for PhysicsDataset model. """ from unittest.mock import patch import pytest -from prkit.core.domain import PhysicalDataset, PhysicsDomain, PhysicsProblem +from prkit.core.domain import PhysicsDataset, PhysicsDomain, PhysicsProblem from prkit.core.domain import physics_dataset as physics_dataset_module class TestPhysicalDataset: - """Test cases for PhysicalDataset model.""" + """Test cases for PhysicsDataset model.""" def test_dataset_creation(self, sample_problems_list): """Test creating a dataset.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) assert len(dataset) == 5 assert dataset.get_split() == "test" def test_dataset_getitem(self, sample_problems_list): """Test dataset indexing.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) assert dataset[0].problem_id == "test_000" assert dataset[1].problem_id == "test_001" @@ -31,7 +31,7 @@ def test_dataset_getitem(self, sample_problems_list): def test_dataset_iteration(self, sample_problems_list): """Test dataset iteration.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) count = 0 for problem in dataset: assert isinstance(problem, PhysicsProblem) @@ -40,7 +40,7 @@ def test_dataset_iteration(self, sample_problems_list): def test_dataset_get_by_id(self, sample_problems_list): """Test getting problem by ID.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) problem = dataset.get_by_id("test_002") assert problem.problem_id == "test_002" @@ -49,7 +49,7 @@ def test_dataset_get_by_id(self, sample_problems_list): def test_dataset_get_by_id_safe(self, sample_problems_list): """Test safe get by ID.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) problem = dataset.get_by_id_safe("test_002") assert problem is not None assert problem.problem_id == "test_002" @@ -59,14 +59,14 @@ def test_dataset_get_by_id_safe(self, sample_problems_list): def test_dataset_filter(self, sample_problems_list): """Test dataset filtering.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) filtered = dataset.filter(lambda p: p.problem_id == "test_000") assert len(filtered) == 1 assert filtered[0].problem_id == "test_000" def test_dataset_filter_by_domain(self, sample_problems_list): """Test filtering by domain.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) filtered = dataset.filter_by_domain(PhysicsDomain.CLASSICAL_MECHANICS) assert len(filtered) >= 1 @@ -76,27 +76,27 @@ def test_dataset_filter_by_domain(self, sample_problems_list): def test_dataset_filter_by_domains(self, sample_problems_list): """Test filtering by multiple domains.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) domains = [PhysicsDomain.CLASSICAL_MECHANICS, PhysicsDomain.QUANTUM_MECHANICS] filtered = dataset.filter_by_domains(domains) assert len(filtered) >= 1 def test_dataset_select(self, sample_problems_list): """Test selecting problems by indices.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) selected = dataset.select([0, 2, 4]) assert len(selected) == 3 def test_dataset_take(self, sample_problems_list): """Test taking first N problems.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) taken = dataset.take(3) assert len(taken) == 3 assert taken[0].problem_id == "test_000" def test_dataset_head_tail(self, sample_problems_list): """Test head and tail methods.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) head = dataset.head(2) assert len(head) == 2 @@ -105,13 +105,13 @@ def test_dataset_head_tail(self, sample_problems_list): def test_dataset_sample(self, sample_problems_list): """Test sampling problems.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) sampled = dataset.sample(3) assert len(sampled) == 3 def test_dataset_map(self, sample_problems_list): """Test mapping function over problems.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) ids = dataset.map(lambda p: p.problem_id) assert len(ids) == 5 assert all(isinstance(id, str) for id in ids) @@ -119,13 +119,13 @@ def test_dataset_map(self, sample_problems_list): def test_dataset_get_info(self, sample_problems_list): """Test getting dataset info.""" info = {"name": "test", "version": "1.0"} - dataset = PhysicalDataset(problems=sample_problems_list, info=info) + dataset = PhysicsDataset(problems=sample_problems_list, info=info) assert dataset.get_info() == info assert dataset.name == "test" def test_dataset_statistics(self, sample_problems_list): """Test dataset statistics.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) stats = dataset.get_statistics() assert stats["total_problems"] == 5 assert "domains" in stats @@ -133,14 +133,14 @@ def test_dataset_statistics(self, sample_problems_list): def test_dataset_to_list(self, sample_problems_list): """Test converting dataset to list.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) problem_list = dataset.to_list() assert len(problem_list) == 5 assert all(isinstance(p, dict) for p in problem_list) def test_dataset_save_load_json(self, sample_problems_list, temp_dir): """Test saving and loading dataset from JSON.""" - dataset = PhysicalDataset( + dataset = PhysicsDataset( problems=sample_problems_list, info={"name": "test_dataset"}, split="test" ) @@ -148,15 +148,15 @@ def test_dataset_save_load_json(self, sample_problems_list, temp_dir): dataset.save_to_json(filepath) assert filepath.exists() - loaded = PhysicalDataset.from_json(filepath) + loaded = PhysicsDataset.from_json(filepath) assert len(loaded) == 5 assert loaded.get_split() == "test" assert loaded.name == "test_dataset" def test_dataset_repr_str(self, sample_problems_list): """Test string representations.""" - dataset = PhysicalDataset(problems=sample_problems_list) - assert "PhysicalDataset" in repr(dataset) + dataset = PhysicsDataset(problems=sample_problems_list) + assert "PhysicsDataset" in repr(dataset) assert "5" in str(dataset) def test_dataset_duplicate_and_missing_problem_ids_are_indexed(self): @@ -170,13 +170,13 @@ def test_dataset_duplicate_and_missing_problem_ids_are_indexed(self): physics_dataset_module.PRKitLogger.get_logger(__name__), "warning", ) as _: - dataset = PhysicalDataset(problems=problems, info={"name": "demo"}) + dataset = PhysicsDataset(problems=problems, info={"name": "demo"}) assert dataset.get_all_ids() == ["dup", "problem_2"] assert dataset.get_by_id("dup").question == "Q1" def test_dataset_additional_branches(self, sample_problems_list): - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) assert len(dataset.select([-1, 100])) == 0 assert len(dataset.take(0)) == 0 @@ -187,9 +187,9 @@ def test_dataset_additional_branches(self, sample_problems_list): def test_dataset_filter_by_domains_with_strings_invalid_types_and_empty_stats( self, sample_problems_list ): - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) filtered = dataset.filter_by_domains( ["Classical Mechanics", "unknown-domain", 123] ) assert len(filtered) >= 1 - assert PhysicalDataset([]).get_statistics() == {"total_problems": 0} + assert PhysicsDataset([]).get_statistics() == {"total_problems": 0} diff --git a/tests/prkit/core/domain/test_physics_problem.py b/tests/prkit/core/domain/test_physics_problem.py index 7a0e829..72bd74e 100644 --- a/tests/prkit/core/domain/test_physics_problem.py +++ b/tests/prkit/core/domain/test_physics_problem.py @@ -7,7 +7,7 @@ import pytest -from prkit.core.domain import Answer, PhysicsDomain, PhysicsProblem +from prkit.core.domain import PhysicsAnswer, PhysicsDomain, PhysicsProblem from prkit.core.domain import physics_problem as physics_problem_module @@ -25,7 +25,7 @@ def test_problem_creation_minimal(self): def test_problem_creation_full(self): """Test creating a full physics problem.""" - answer = Answer(value="42") + answer = PhysicsAnswer(value="42") problem = PhysicsProblem( problem_id="test_001", question="What is the answer?", @@ -222,7 +222,7 @@ def test_problem_update(self): def test_problem_to_dict(self): """Test problem serialization.""" - answer = Answer(value="42") + answer = PhysicsAnswer(value="42") problem = PhysicsProblem( problem_id="test_001", question="Test", diff --git a/tests/prkit/datasets/loaders/test_base_loader_additional.py b/tests/prkit/datasets/loaders/test_base_loader_additional.py index 5489f28..f4cbc66 100644 --- a/tests/prkit/datasets/loaders/test_base_loader_additional.py +++ b/tests/prkit/datasets/loaders/test_base_loader_additional.py @@ -1,4 +1,4 @@ -from prkit.core.domain import PhysicalDataset, PhysicsDomain +from prkit.core.domain import PhysicsDataset, PhysicsDomain from prkit.datasets.loaders.base_loader import ( BaseDatasetLoader, is_mathematical_expression, @@ -19,8 +19,8 @@ def field_mapping(self): def modalities(self): return ["text", "image"] - def load(self, data_dir, **kwargs) -> PhysicalDataset: - return PhysicalDataset(problems=[]) + def load(self, data_dir, **kwargs) -> PhysicsDataset: + return PhysicsDataset(problems=[]) def get_info(self): return {"variants": ["mini", "full"], "splits": ["train", "full"]} diff --git a/tests/prkit/datasets/loaders/test_base_loader_map_domain.py b/tests/prkit/datasets/loaders/test_base_loader_map_domain.py index cdfed60..9703d7d 100644 --- a/tests/prkit/datasets/loaders/test_base_loader_map_domain.py +++ b/tests/prkit/datasets/loaders/test_base_loader_map_domain.py @@ -2,7 +2,7 @@ from __future__ import annotations -from prkit.core.domain import PhysicalDataset, PhysicsDomain +from prkit.core.domain import PhysicsDataset, PhysicsDomain from prkit.datasets.loaders.base_loader import BaseDatasetLoader @@ -13,8 +13,8 @@ class _LoaderNoDomainMapping(BaseDatasetLoader): def field_mapping(self) -> dict[str, str]: return {} - def load(self, data_dir, **kwargs) -> PhysicalDataset: # type: ignore[override] - return PhysicalDataset(problems=[]) + def load(self, data_dir, **kwargs) -> PhysicsDataset: # type: ignore[override] + return PhysicsDataset(problems=[]) def get_info(self) -> dict: return {} @@ -34,8 +34,8 @@ def DOMAIN_MAPPING(self) -> dict[str, PhysicsDomain]: def field_mapping(self) -> dict[str, str]: return {} - def load(self, data_dir, **kwargs) -> PhysicalDataset: # type: ignore[override] - return PhysicalDataset(problems=[]) + def load(self, data_dir, **kwargs) -> PhysicsDataset: # type: ignore[override] + return PhysicsDataset(problems=[]) def get_info(self) -> dict: return {} diff --git a/tests/prkit/datasets/test_hub.py b/tests/prkit/datasets/test_hub.py index a23bae8..83df7ac 100644 --- a/tests/prkit/datasets/test_hub.py +++ b/tests/prkit/datasets/test_hub.py @@ -7,7 +7,7 @@ import pytest -from prkit.core.domain import PhysicalDataset, PhysicsProblem +from prkit.core.domain import PhysicsDataset, PhysicsProblem from prkit.datasets import DatasetHub from prkit.datasets.loaders.base_loader import BaseDatasetLoader @@ -34,7 +34,7 @@ def test_register_custom_loader(self): class CustomLoader(BaseDatasetLoader): def load(self, data_dir=None, **kwargs): - return PhysicalDataset(problems=[], info={"name": "custom"}) + return PhysicsDataset(problems=[], info={"name": "custom"}) def get_info(self): return {"name": "custom", "description": "Custom dataset"} @@ -84,7 +84,7 @@ def test_load_with_sample_size(self, mock_loader_class): PhysicsProblem(problem_id=f"test_{i}", question=f"Question {i}") for i in range(10) ] - mock_dataset = PhysicalDataset(problems=mock_problems) + mock_dataset = PhysicsDataset(problems=mock_problems) mock_loader.load.return_value = mock_dataset mock_loader_class.return_value = mock_loader @@ -112,7 +112,7 @@ def field_mapping(self): def load(self, data_dir=None, **kwargs): assert "custom_param" in kwargs - return PhysicalDataset(problems=[], info={"name": "mock"}) + return PhysicsDataset(problems=[], info={"name": "mock"}) def get_info(self): return {"name": "mock", "variants": ["full"], "splits": ["train"]} @@ -283,7 +283,7 @@ def resolve_data_dir(self, data_dir, dataset_name=None): def load(self, data_dir=None, **kwargs): assert kwargs.get("variant") == "full" assert kwargs.get("split") == "train" - return PhysicalDataset(problems=[], info={"name": "mock"}) + return PhysicsDataset(problems=[], info={"name": "mock"}) DatasetHub.register("mock_defaults", MockLoader) @@ -331,7 +331,7 @@ def resolve_data_dir(self, data_dir, dataset_name=None): def load(self, data_dir=None, **kwargs): assert kwargs.get("variant") == "mini" assert kwargs.get("split") == "test" - return PhysicalDataset(problems=[], info={"name": "mock"}) + return PhysicsDataset(problems=[], info={"name": "mock"}) DatasetHub.register("mock_explicit", MockLoader) @@ -351,7 +351,7 @@ def field_mapping(self): return {} def load(self, data_dir=None, **kwargs): - return PhysicalDataset(problems=[], info={"name": "mock"}) + return PhysicsDataset(problems=[], info={"name": "mock"}) def get_info(self): return { @@ -396,7 +396,7 @@ def field_mapping(self): return {} def load(self, data_dir=None, **kwargs): - return PhysicalDataset(problems=[], info={"name": "mock"}) + return PhysicsDataset(problems=[], info={"name": "mock"}) def get_info(self): return { @@ -441,7 +441,7 @@ def field_mapping(self): return {} def load(self, data_dir=None, **kwargs): - return PhysicalDataset(problems=[], info={"name": "mock"}) + return PhysicsDataset(problems=[], info={"name": "mock"}) def get_info(self): return {"name": "mock", "variants": [], "splits": ["train"]} @@ -478,7 +478,7 @@ def field_mapping(self): return {} def load(self, data_dir=None, **kwargs): - return PhysicalDataset(problems=[], info={"name": "mock"}) + return PhysicsDataset(problems=[], info={"name": "mock"}) def get_info(self): return {"name": "mock", "variants": ["full"], "splits": []} @@ -544,7 +544,7 @@ def load(self, data_dir=None, **kwargs): raise FileNotFoundError("Dataset not found") # Second call succeeds self.__class__.last_loaded_data_dir = data_dir - return PhysicalDataset(problems=[], info={"name": "mock"}) + return PhysicsDataset(problems=[], info={"name": "mock"}) # Mock downloader mock_downloader = Mock() @@ -767,7 +767,7 @@ def load(self, data_dir=None, **kwargs): from pathlib import Path items = json.loads((Path(data_dir) / "problems.json").read_text()) - return PhysicalDataset( + return PhysicsDataset( problems=[ PhysicsProblem(problem_id=item["id"], question=item["q"]) for item in items @@ -814,7 +814,7 @@ def get_info(self): return {"name": "dummy_guard", "variants": ["full"], "splits": ["full"]} def load(self, data_dir=None, **kwargs): - return PhysicalDataset(problems=[]) + return PhysicsDataset(problems=[]) try: # External loader registered first, with _loaders empty @@ -839,7 +839,7 @@ class TestDatasetLoadersIntegration: @pytest.mark.integration def test_loader_returns_physical_dataset(self): - """Test that loaders return PhysicalDataset instances.""" + """Test that loaders return PhysicsDataset instances.""" # This is an integration test that may require actual data files # Skip if data is not available available = DatasetHub.list_available() @@ -850,7 +850,7 @@ def test_loader_returns_physical_dataset(self): # Note: This may fail if data files don't exist try: dataset = DatasetHub.load(available[0], sample_size=1) - assert isinstance(dataset, PhysicalDataset) + assert isinstance(dataset, PhysicsDataset) except (FileNotFoundError, ValueError) as e: # If data files don't exist, skip the test pytest.skip(f"Data files not available: {e}") diff --git a/tests/prkit/datasets/test_utils_functions.py b/tests/prkit/datasets/test_utils_functions.py index e04dca9..0c1aa77 100644 --- a/tests/prkit/datasets/test_utils_functions.py +++ b/tests/prkit/datasets/test_utils_functions.py @@ -4,7 +4,7 @@ import json -from prkit.core.domain import Answer, PhysicalDataset, PhysicsProblem +from prkit.core.domain import PhysicsAnswer, PhysicsDataset, PhysicsProblem from prkit.datasets import utils @@ -13,7 +13,7 @@ class TestSampleBalanced: def test_sample_balanced_by_domain(self, sample_problems_list): """Test sampling balanced by domain.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) balanced = utils.sample_balanced( dataset, "domain", samples_per_category=1, seed=42 ) @@ -24,7 +24,7 @@ def test_sample_balanced_by_domain(self, sample_problems_list): def test_sample_balanced_insufficient_samples(self, sample_problems_list): """Test sampling when category has fewer samples than requested.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) # Request more samples than available in some categories balanced = utils.sample_balanced( dataset, "domain", samples_per_category=100, seed=42 @@ -36,7 +36,7 @@ def test_sample_balanced_insufficient_samples(self, sample_problems_list): def test_sample_balanced_with_seed(self, sample_problems_list): """Test that seed produces reproducible results.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) balanced1 = utils.sample_balanced( dataset, "domain", samples_per_category=1, seed=42 ) @@ -56,7 +56,7 @@ class TestGetStatistics: def test_get_statistics_basic(self, sample_problems_list): """Test getting basic statistics.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) stats = utils.get_statistics(dataset) assert stats["total_samples"] == len(dataset) @@ -64,7 +64,7 @@ def test_get_statistics_basic(self, sample_problems_list): def test_get_statistics_empty_dataset(self): """Test getting statistics for empty dataset.""" - dataset = PhysicalDataset(problems=[]) + dataset = PhysicsDataset(problems=[]) stats = utils.get_statistics(dataset) assert stats["total_samples"] == 0 @@ -72,7 +72,7 @@ def test_get_statistics_empty_dataset(self): def test_get_statistics_domain_distribution(self, sample_problems_list): """Test domain distribution in statistics.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) stats = utils.get_statistics(dataset) # Should have domain distribution if problems have domain @@ -85,7 +85,7 @@ class TestExportToJson: def test_export_to_json(self, sample_problems_list, temp_dir): """Test exporting dataset to JSON.""" - dataset = PhysicalDataset(problems=sample_problems_list, info={"name": "test"}) + dataset = PhysicsDataset(problems=sample_problems_list, info={"name": "test"}) output_path = temp_dir / "test_export.json" utils.export_to_json(dataset, output_path) @@ -98,7 +98,7 @@ def test_export_to_json(self, sample_problems_list, temp_dir): def test_export_to_json_with_info(self, sample_problems_list, temp_dir): """Test exporting dataset with info.""" - dataset = PhysicalDataset(problems=sample_problems_list, info={"name": "test"}) + dataset = PhysicsDataset(problems=sample_problems_list, info={"name": "test"}) output_path = temp_dir / "test_export_info.json" utils.export_to_json(dataset, output_path, include_info=True) @@ -110,7 +110,7 @@ def test_export_to_json_with_info(self, sample_problems_list, temp_dir): def test_export_to_json_without_info(self, sample_problems_list, temp_dir): """Test exporting dataset without info.""" - dataset = PhysicalDataset(problems=sample_problems_list, info={"name": "test"}) + dataset = PhysicsDataset(problems=sample_problems_list, info={"name": "test"}) output_path = temp_dir / "test_export_no_info.json" utils.export_to_json(dataset, output_path, include_info=False) @@ -129,10 +129,10 @@ def test_filter_by_keywords_in_question(self, sample_problems_list): problem_with_keyword = PhysicsProblem( problem_id="keyword_test", question="What is the speed of light?", - answer=Answer(value="3e8", unit="m/s"), + answer=PhysicsAnswer(value="3e8", unit="m/s"), ) all_problems = list(sample_problems_list) + [problem_with_keyword] - dataset = PhysicalDataset(problems=all_problems) + dataset = PhysicsDataset(problems=all_problems) filtered = utils.filter_by_keywords(dataset, ["speed"], fields=["question"]) @@ -144,10 +144,10 @@ def test_filter_by_keywords_case_insensitive(self, sample_problems_list): problem = PhysicsProblem( problem_id="test_case", question="What is the SPEED of light?", - answer=Answer(value="3e8", unit="m/s"), + answer=PhysicsAnswer(value="3e8", unit="m/s"), ) all_problems = list(sample_problems_list) + [problem] - dataset = PhysicalDataset(problems=all_problems) + dataset = PhysicsDataset(problems=all_problems) filtered = utils.filter_by_keywords( dataset, ["speed"], fields=["question"], case_sensitive=False @@ -160,10 +160,10 @@ def test_filter_by_keywords_case_sensitive(self, sample_problems_list): problem = PhysicsProblem( problem_id="test_case", question="What is the speed of light?", - answer=Answer(value="3e8", unit="m/s"), + answer=PhysicsAnswer(value="3e8", unit="m/s"), ) all_problems = list(sample_problems_list) + [problem] - dataset = PhysicalDataset(problems=all_problems) + dataset = PhysicsDataset(problems=all_problems) filtered = utils.filter_by_keywords( dataset, ["SPEED"], fields=["question"], case_sensitive=True @@ -178,10 +178,10 @@ def test_filter_by_keywords_multiple_fields(self, sample_problems_list): problem_id="test_multi", question="Test question", solution="The answer involves force calculation", - answer=Answer(value="1"), + answer=PhysicsAnswer(value="1"), ) all_problems = list(sample_problems_list) + [problem] - dataset = PhysicalDataset(problems=all_problems) + dataset = PhysicsDataset(problems=all_problems) filtered = utils.filter_by_keywords( dataset, ["force"], fields=["question", "solution"] @@ -195,18 +195,18 @@ class TestCreateCrossValidationSplits: def test_create_cv_splits(self, sample_problems_list): """Test creating cross-validation splits.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) splits = utils.create_cross_validation_splits(dataset, n_splits=3, seed=42) assert len(splits) == 3 for train, val in splits: - assert isinstance(train, PhysicalDataset) - assert isinstance(val, PhysicalDataset) + assert isinstance(train, PhysicsDataset) + assert isinstance(val, PhysicsDataset) assert len(train) + len(val) == len(dataset) def test_create_cv_splits_reproducible(self, sample_problems_list): """Test that CV splits are reproducible with same seed.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) splits1 = utils.create_cross_validation_splits(dataset, n_splits=3, seed=42) splits2 = utils.create_cross_validation_splits(dataset, n_splits=3, seed=42) @@ -217,7 +217,7 @@ def test_create_cv_splits_reproducible(self, sample_problems_list): def test_create_cv_splits_no_overlap(self, sample_problems_list): """Test that train and validation sets don't overlap.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) splits = utils.create_cross_validation_splits(dataset, n_splits=2, seed=42) for train, val in splits: @@ -231,7 +231,7 @@ class TestValidateDatasetFormat: def test_validate_dataset_format_valid(self, sample_problems_list): """Test validating a valid dataset.""" - dataset = PhysicalDataset(problems=sample_problems_list) + dataset = PhysicsDataset(problems=sample_problems_list) report = utils.validate_dataset_format(dataset) assert report["valid"] is True @@ -239,7 +239,7 @@ def test_validate_dataset_format_valid(self, sample_problems_list): def test_validate_dataset_format_empty(self): """Test validating an empty dataset.""" - dataset = PhysicalDataset(problems=[]) + dataset = PhysicsDataset(problems=[]) report = utils.validate_dataset_format(dataset) assert report["valid"] is False @@ -254,14 +254,14 @@ def test_validate_dataset_format_missing_fields(self): del problem_dict["question"] # Create a custom dataset-like object for testing - # Note: This is a simplified test since PhysicalDataset expects PhysicsProblem objects + # Note: This is a simplified test since PhysicsDataset expects PhysicsProblem objects # In practice, this would be caught earlier, but we test the validation logic - dataset = PhysicalDataset(problems=[problem]) + dataset = PhysicsDataset(problems=[problem]) report = utils.validate_dataset_format( dataset, required_fields=["question", "problem_id"] ) - # Should be valid since PhysicalDataset ensures problems have required fields + # Should be valid since PhysicsDataset ensures problems have required fields # This test mainly verifies the function doesn't crash assert "valid" in report @@ -270,7 +270,7 @@ def test_validate_dataset_format_duplicate_ids(self): # Create problems with duplicate IDs problem1 = PhysicsProblem(problem_id="duplicate", question="Question 1") problem2 = PhysicsProblem(problem_id="duplicate", question="Question 2") - dataset = PhysicalDataset(problems=[problem1, problem2]) + dataset = PhysicsDataset(problems=[problem1, problem2]) report = utils.validate_dataset_format(dataset) diff --git a/tests/prkit/evaluation/llm_judge/test_payload.py b/tests/prkit/evaluation/llm_judge/test_payload.py index ab97b58..65c1623 100644 --- a/tests/prkit/evaluation/llm_judge/test_payload.py +++ b/tests/prkit/evaluation/llm_judge/test_payload.py @@ -1,4 +1,4 @@ -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer from prkit.evaluation.llm_judge.payload import ( answer_to_text_and_category, build_standard_answer_judge_payload, @@ -8,12 +8,12 @@ def test_answer_to_text_and_category_for_answers_and_plain_strings(): - # Answer with source_type → category is the source_type string - answer = Answer(value=" 42 ", source_type="NV") + # PhysicsAnswer with source_type → category is the source_type string + answer = PhysicsAnswer(value=" 42 ", source_type="NV") assert answer_to_text_and_category(answer) == ("42", "NV") - # Answer without source_type → empty string - answer_no_type = Answer(value=" 42 ") + # PhysicsAnswer without source_type → empty string + answer_no_type = PhysicsAnswer(value=" 42 ") assert answer_to_text_and_category(answer_no_type) == ("42", "") # Plain string → empty string category @@ -22,8 +22,8 @@ def test_answer_to_text_and_category_for_answers_and_plain_strings(): def test_build_standard_answer_judge_payload_cleans_fields(): payload = build_standard_answer_judge_payload( - Answer(value=" 10  m/s "), - Answer(value=" 10\tm/s "), + PhysicsAnswer(value=" 10  m/s "), + PhysicsAnswer(value=" 10\tm/s "), " What is the speed? ", ) diff --git a/tests/prkit/scoring/test_eed_scorer.py b/tests/prkit/scoring/test_eed_scorer.py index 8c4c473..31d5274 100644 --- a/tests/prkit/scoring/test_eed_scorer.py +++ b/tests/prkit/scoring/test_eed_scorer.py @@ -8,7 +8,7 @@ # the suite degrades gracefully if it is ever made optional). pytest.importorskip("latex2sympy2_extended") -from prkit.core.domain.answer import Answer # noqa: E402 +from prkit.core.domain.answer import PhysicsAnswer # noqa: E402 from prkit.core.verdict import Verdict # noqa: E402 from prkit.scoring import EedScorer # noqa: E402 @@ -31,7 +31,7 @@ def test_different_pair_scores_lower_and_not_equivalent(self): def test_accepts_answer_objects(self): verdict = EedScorer().score( - Answer(value="3.0", unit="m/s"), Answer(value="3", unit="m/s") + PhysicsAnswer(value="3.0", unit="m/s"), PhysicsAnswer(value="3", unit="m/s") ) assert verdict.equivalent is True assert verdict.score == 1.0 diff --git a/tests/prkit/scoring/test_llm_judge_scorer.py b/tests/prkit/scoring/test_llm_judge_scorer.py index 6384f7f..2b7a33c 100644 --- a/tests/prkit/scoring/test_llm_judge_scorer.py +++ b/tests/prkit/scoring/test_llm_judge_scorer.py @@ -12,7 +12,7 @@ import pytest from prkit.api import Scorer, Verdict -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer from prkit.evaluation.llm_judge.types import LLMJudgeResult from prkit.scoring import LLMJudgeScorer @@ -127,11 +127,11 @@ def test_question_is_threaded_into_the_payload(self): def test_accepts_answer_objects(self): runner = FakeJudgeRunner(_result("correct")) - pred = Answer(value="3.0", unit="m/s") - ref = Answer(value="3", unit="m/s", source_type="NV") + pred = PhysicsAnswer(value="3.0", unit="m/s") + ref = PhysicsAnswer(value="3", unit="m/s", source_type="NV") v = _scorer(runner).score(pred, ref) assert v.equivalent is True - # Answer.source_type flows into the payload category. + # PhysicsAnswer.source_type flows into the payload category. assert runner.calls[0]["ground_truth"]["category"] == "NV" diff --git a/tests/prkit/scoring/test_seed_scorer.py b/tests/prkit/scoring/test_seed_scorer.py index 722406d..02dd92f 100644 --- a/tests/prkit/scoring/test_seed_scorer.py +++ b/tests/prkit/scoring/test_seed_scorer.py @@ -6,7 +6,7 @@ pytest.importorskip("latex2sympy2_extended") -from prkit.core.domain.answer import Answer # noqa: E402 +from prkit.core.domain.answer import PhysicsAnswer # noqa: E402 from prkit.core.verdict import Verdict # noqa: E402 from prkit.scoring import SeedScorer # noqa: E402 from prkit.scoring.seed_scorer import SEED_ANSWER_TYPES # noqa: E402 @@ -35,13 +35,15 @@ def test_explicit_answer_type_kwarg_dispatches(self): def test_source_type_fallback_for_tuple(self): verdict = SeedScorer().score( - "(1, 3)", Answer(value="(1, 2)", source_type="Tuple") + "(1, 3)", PhysicsAnswer(value="(1, 2)", source_type="Tuple") ) assert verdict.comparison_mode == "seed:Tuple" assert verdict.details["answer_type"] == "Tuple" def test_non_seed_source_type_falls_back_to_expression(self): - verdict = SeedScorer().score("x + 1", Answer(value="1 + x", source_type="MC")) + verdict = SeedScorer().score( + "x + 1", PhysicsAnswer(value="1 + x", source_type="MC") + ) assert verdict.comparison_mode == "seed:Expression" assert verdict.equivalent is True diff --git a/tests/prkit/scoring/test_semantics_eed_scorer.py b/tests/prkit/scoring/test_semantics_eed_scorer.py index 325b030..44f03ca 100644 --- a/tests/prkit/scoring/test_semantics_eed_scorer.py +++ b/tests/prkit/scoring/test_semantics_eed_scorer.py @@ -4,7 +4,7 @@ from prkit.api import Scorer, Verdict from prkit.core.domain import AnswerObjectKind, AnswerStructure -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer from prkit.scoring import SemanticsEedScorer from prkit.semantics import PhysicsAnswerSemantics from prkit.testing import check_scorer @@ -62,7 +62,9 @@ def test_returns_canonical_verdict(self) -> None: class TestNotApplicable: def test_choice_answer_is_not_applicable(self) -> None: - verdict = SemanticsEedScorer().score(Answer(value="A"), Answer(value="B")) + verdict = SemanticsEedScorer().score( + PhysicsAnswer(value="A"), PhysicsAnswer(value="B") + ) assert verdict.score == -1.0 assert verdict.correct is False assert verdict.equivalent is False diff --git a/tests/prkit/scoring/test_semantics_scorer.py b/tests/prkit/scoring/test_semantics_scorer.py index 6395aec..8789243 100644 --- a/tests/prkit/scoring/test_semantics_scorer.py +++ b/tests/prkit/scoring/test_semantics_scorer.py @@ -7,7 +7,7 @@ import pytest from prkit.api import Scorer, Verdict -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer from prkit.scoring import SemanticsScorer # Empirically validated against the deterministic engine (see plan step 4). @@ -53,8 +53,8 @@ def test_identity_equivalent(self, value): assert SemanticsScorer().score(value, value).equivalent is True def test_accepts_answer_objects(self): - pred = Answer(value="3.0", unit="m/s") - ref = Answer(value="3", unit="m/s") + pred = PhysicsAnswer(value="3.0", unit="m/s") + ref = PhysicsAnswer(value="3", unit="m/s") v = SemanticsScorer().score(pred, ref) assert v.equivalent is True diff --git a/tests/prkit/scoring/test_semantics_seed_scorer.py b/tests/prkit/scoring/test_semantics_seed_scorer.py index f39bfa8..b921a76 100644 --- a/tests/prkit/scoring/test_semantics_seed_scorer.py +++ b/tests/prkit/scoring/test_semantics_seed_scorer.py @@ -4,7 +4,7 @@ from prkit.api import Scorer from prkit.core.domain import AnswerObjectKind, AnswerStructure -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer from prkit.scoring import SemanticsSeedScorer from prkit.semantics import PhysicsAnswerSemantics from prkit.testing import check_scorer @@ -97,7 +97,9 @@ def test_interval_exact_match(self) -> None: class TestNotApplicable: def test_choice_answer_is_not_applicable(self) -> None: - verdict = SemanticsSeedScorer().score(Answer(value="A"), Answer(value="B")) + verdict = SemanticsSeedScorer().score( + PhysicsAnswer(value="A"), PhysicsAnswer(value="B") + ) assert verdict.score == -1.0 assert verdict.correct is False assert verdict.equivalent is False diff --git a/tests/prkit/semantics/test_inference_prompts.py b/tests/prkit/semantics/test_inference_prompts.py index c132b8e..93658fe 100644 --- a/tests/prkit/semantics/test_inference_prompts.py +++ b/tests/prkit/semantics/test_inference_prompts.py @@ -6,7 +6,7 @@ import pytest from pydantic import ValidationError -from prkit.core.domain import Answer, PhysicsProblem +from prkit.core.domain import PhysicsAnswer, PhysicsProblem from prkit.core.model_clients import BaseModelClient from prkit.core.model_clients.structured_output import StructuredOutputPlan from prkit.semantics.build.calls import ( @@ -68,7 +68,7 @@ def _build_problem() -> PhysicsProblem: problem = PhysicsProblem( problem_id="prob-1", question="What is the force?", - answer=Answer(value="5", unit="N"), + answer=PhysicsAnswer(value="5", unit="N"), solution="Use Newton's second law.", domain="mechanics", image_path=["/tmp/img1.png", "/tmp/img2.png"], @@ -111,7 +111,7 @@ def test_build_prediction_semantics_prompt_uses_answer_blind_question_draft() -> problem = PhysicsProblem( problem_id="prob-gold-split", question="Give both values: the displacement value and the time value.", - answer=Answer(value="F = ma"), + answer=PhysicsAnswer(value="F = ma"), additional_fields={ "answer_parts": [ {"part_label": "speed_slot", "raw_text": "1 m"}, diff --git a/tests/prkit/semantics/test_outcome_space.py b/tests/prkit/semantics/test_outcome_space.py index 0e7cf6c..153e45d 100644 --- a/tests/prkit/semantics/test_outcome_space.py +++ b/tests/prkit/semantics/test_outcome_space.py @@ -2,7 +2,7 @@ import pytest -from prkit.core.domain import Answer, PhysicsProblem +from prkit.core.domain import PhysicsAnswer, PhysicsProblem from prkit.semantics import ( AnswerObjectKind, AnswerStructure, @@ -280,7 +280,7 @@ def test_fixed_question_unit_allows_bare_number_but_required_unit_does_not() -> PhysicsProblem( problem_id="p1", question="Find the speed in m/s.", - answer=Answer(value="5"), + answer=PhysicsAnswer(value="5"), ) ) required_unit_context = QuestionContext( @@ -328,7 +328,7 @@ def test_infer_question_context_rejects_prose_after_in_keyword( PhysicsProblem( problem_id=problem_id, question=question, - answer=Answer( + answer=PhysicsAnswer( value=answer_value, ), ) @@ -345,23 +345,23 @@ def test_infer_question_context_drops_stopword_targets_but_keeps_symbol_targets( PhysicsProblem( problem_id="p_stopword_the", question="What is the magnitude of the force on the block?", - answer=Answer(value="25 N"), + answer=PhysicsAnswer(value="25 N"), ), PhysicsProblem( problem_id="p_stopword_all", question="What is all?", - answer=Answer(value="25 N"), + answer=PhysicsAnswer(value="25 N"), ), PhysicsProblem( problem_id="p_stopword_which", question="What is which?", - answer=Answer(value="25 N"), + answer=PhysicsAnswer(value="25 N"), ), ] symbol_problem = PhysicsProblem( problem_id="p_symbol_target", question="What is T?", - answer=Answer(value="0.78 s"), + answer=PhysicsAnswer(value="0.78 s"), ) for problem in prose_problems: @@ -373,7 +373,7 @@ def test_question_semantics_split_uses_gold_target_only_for_reference() -> None: problem = PhysicsProblem( problem_id="p_gold_target", question="Give the final expression for the magnetic field.", - answer=Answer( + answer=PhysicsAnswer( value="B = \\mu_0 I / (2\\pi r)", ), ) @@ -391,7 +391,7 @@ def test_question_semantics_split_uses_gold_unit_policy_only_for_reference() -> problem = PhysicsProblem( problem_id="p_gold_unit", question="Find the speed.", - answer=Answer( + answer=PhysicsAnswer( value="5", unit="m/s", ), @@ -409,7 +409,7 @@ def test_prediction_question_semantics_ignores_answer_parts_metadata() -> None: problem = PhysicsProblem( problem_id="p_answer_parts_split", question="Give both values: the displacement value and the time value.", - answer=Answer(value="ignored"), + answer=PhysicsAnswer(value="ignored"), additional_fields={ "answer_parts": [ {"part_label": "speed_slot", "raw_text": "1 m"}, @@ -484,7 +484,7 @@ def test_prediction_question_semantics_ignores_symbol_alias_metadata() -> None: problem = PhysicsProblem( problem_id="p_symbol_alias_split", question="Give the final expression for the displacement.", - answer=Answer(value="x_final = v*t"), + answer=PhysicsAnswer(value="x_final = v*t"), additional_fields={ "symbol_aliases": [ { @@ -508,7 +508,7 @@ def test_problem_answer_parts_take_precedence() -> None: problem = PhysicsProblem( problem_id="p2", question="Give both values.", - answer=Answer(value="ignored"), + answer=PhysicsAnswer(value="ignored"), additional_fields={"answer_parts": ["1 m", "2 m"]}, ) @@ -525,7 +525,7 @@ def test_dataset_backed_relation_and_multi_part_strings() -> None: PhysicsProblem( problem_id="ugphysics-628", question="Give the effect type and the field strength.", - answer=Answer(value="C, 7.77"), + answer=PhysicsAnswer(value="C, 7.77"), additional_fields={"answer_parts": ["C", "7.77"]}, ) ) diff --git a/tests/prkit/semantics/test_prediction_isolated_build.py b/tests/prkit/semantics/test_prediction_isolated_build.py index 6129d59..f17de97 100644 --- a/tests/prkit/semantics/test_prediction_isolated_build.py +++ b/tests/prkit/semantics/test_prediction_isolated_build.py @@ -17,7 +17,7 @@ import pytest -from prkit.core.domain import Answer, PhysicsProblem +from prkit.core.domain import PhysicsAnswer, PhysicsProblem from prkit.core.model_clients import BaseModelClient from prkit.semantics.build.calls import ( build_extracted_prediction_semantics_artifact, @@ -39,7 +39,7 @@ def _problem() -> PhysicsProblem: return PhysicsProblem( problem_id="pred-iso-1", question="Find the speed v.", - answer=Answer(value="sqrt(E/m), m > 0"), + answer=PhysicsAnswer(value="sqrt(E/m), m > 0"), solution="Use conservation of energy.", domain="mechanics", additional_fields={ diff --git a/tests/prkit/semantics/test_sign_convention_build_integration.py b/tests/prkit/semantics/test_sign_convention_build_integration.py index 4de3f93..5689245 100644 --- a/tests/prkit/semantics/test_sign_convention_build_integration.py +++ b/tests/prkit/semantics/test_sign_convention_build_integration.py @@ -12,7 +12,7 @@ import json from typing import Any -from prkit.core.domain import Answer, PhysicsProblem +from prkit.core.domain import PhysicsAnswer, PhysicsProblem from prkit.core.model_clients import BaseModelClient from prkit.semantics.build.calls import ( build_reference_semantics, @@ -26,7 +26,7 @@ def _quantity_problem(golden: str) -> PhysicsProblem: return PhysicsProblem( problem_id="signconv-int", question="Find the block's velocity v.", - answer=Answer(value=golden), + answer=PhysicsAnswer(value=golden), domain="mechanics", ) @@ -35,7 +35,7 @@ def _vector_problem(golden: str) -> PhysicsProblem: return PhysicsProblem( problem_id="signconv-int-vec", question="Find the displacement vector.", - answer=Answer(value=golden), + answer=PhysicsAnswer(value=golden), domain="mechanics", ) diff --git a/tests/prkit/semantics/test_sign_convention_build_live.py b/tests/prkit/semantics/test_sign_convention_build_live.py index e298d01..1d42390 100644 --- a/tests/prkit/semantics/test_sign_convention_build_live.py +++ b/tests/prkit/semantics/test_sign_convention_build_live.py @@ -13,7 +13,7 @@ import pytest -from prkit.core.domain import Answer, PhysicsProblem +from prkit.core.domain import PhysicsAnswer, PhysicsProblem from prkit.core.model_clients import create_model_client from prkit.semantics.build.calls import build_reference_semantics @@ -33,7 +33,7 @@ def test_live_reference_build_routes_free_axis_convention_to_a_ref() -> None: "is specified. Taking the block's motion into account, its velocity is found to " "be 20 m/s directed to the left. Report the velocity as a signed value." ), - answer=Answer(value="-20 m/s"), + answer=PhysicsAnswer(value="-20 m/s"), domain="mechanics", ) diff --git a/tests/prkit/semantics/test_staged_build.py b/tests/prkit/semantics/test_staged_build.py index ad80626..ed9d4ab 100644 --- a/tests/prkit/semantics/test_staged_build.py +++ b/tests/prkit/semantics/test_staged_build.py @@ -11,7 +11,7 @@ import json from typing import Any -from prkit.core.domain import Answer, PhysicsProblem +from prkit.core.domain import PhysicsAnswer, PhysicsProblem from prkit.core.model_clients import BaseModelClient from prkit.semantics.build.calls import ( build_problem_semantics, @@ -29,7 +29,7 @@ def _problem() -> PhysicsProblem: return PhysicsProblem( problem_id="staged-1", question="Find the energy E.", - answer=Answer(value="x**2/2, x > 0"), + answer=PhysicsAnswer(value="x**2/2, x > 0"), domain="mechanics", ) @@ -38,7 +38,7 @@ def _directional_problem() -> PhysicsProblem: return PhysicsProblem( problem_id="staged-dir-1", question="Find the velocity v of the block.", - answer=Answer(value="-20 m/s"), + answer=PhysicsAnswer(value="-20 m/s"), domain="mechanics", ) diff --git a/tests/prkit/test_api.py b/tests/prkit/test_api.py index b2d0cba..e6ce1a3 100644 --- a/tests/prkit/test_api.py +++ b/tests/prkit/test_api.py @@ -11,7 +11,7 @@ Verdict, create_model_client, ) -from prkit.core.domain.answer import Answer +from prkit.core.domain.answer import PhysicsAnswer from prkit.core.domain.physics_problem import PhysicsProblem from prkit.core.verdict import Verdict as CoreVerdict from prkit.datasets.hub import DatasetHub @@ -35,10 +35,10 @@ def test_all_is_frozen_surface(self): "Verdict", "AnswerObjectKind", "AnswerStructure", - "Answer", + "PhysicsAnswer", "PhysicsDomain", "PhysicsProblem", - "PhysicalDataset", + "PhysicsDataset", "DatasetHub", "BaseDatasetLoader", "BaseModelClient", @@ -87,7 +87,7 @@ def test_unit_preserved_alongside_legacy_label(self): assert problem.answer.source_type == "physical_quantity" def test_thin_answer_has_no_answer_kind_attribute(self): - a = Answer(value="x") + a = PhysicsAnswer(value="x") assert not hasattr(a, "answer_kind") From 5199005da3f88a08af2e567b15306f4882d04c9a Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Sun, 21 Jun 2026 21:08:40 -0400 Subject: [PATCH 28/28] Fix CI gate failures and make CI reproducible against local Two independent "green locally, red in CI" failures on this branch: - test_model_client_satisfies_protocol built a real OpenAI client, which the openai SDK validates a credential for at construction. The key came from the gitignored .env locally; CI has neither .env nor a secret, so construction raised OpenAIError. Inject a dummy OPENAI_API_KEY in the test so it runs offline anywhere. - mypy's Type check failed on Python 3.12 because a fresh install floated numpy to 2.5.0, whose bundled stub uses PEP 695 `type` statements that mypy rejects for a target python_version below 3.12 (we target 3.10). Skip following numpy stubs so dependency stub drift can't break the type gate. Prevent recurrence: pin the gate tools (black/ruff/mypy) to exact versions so a fresh CI install can't silently drift from local, add a `make ci` target that mirrors the workflow with no .env and no key, and fold the previously-missing typecheck into `make check`. Co-Authored-By: Claude Opus 4.8 --- Makefile | 18 ++++++++++++++++-- pyproject.toml | 21 ++++++++++++++++++--- tests/prkit/test_api.py | 7 +++++-- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 8fde03a..a28a516 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: lint format format-check typecheck test test-prkit build check +.PHONY: lint format format-check typecheck test test-prkit build check ci lint: python -m ruff check src/prkit tests/prkit @@ -22,4 +22,18 @@ test-prkit: build: python -m build -check: lint format-check test-prkit +check: lint format-check typecheck test-prkit + +# Faithful local mirror of .github/workflows/ci.yml: runs every gate the same +# way CI does, in a CI-like environment with NO .env and NO OPENAI_API_KEY, so +# tests that secretly rely on a local key fail here instead of in CI. The .env +# is moved aside and restored afterwards even if a gate fails. Run before pushing. +ci: + @bash -c 'set -u; \ + if [ -f .env ]; then mv .env .env.cibak; fi; \ + trap "[ -f .env.cibak ] && mv .env.cibak .env" EXIT; \ + unset OPENAI_API_KEY; \ + python -m ruff check src/prkit tests/prkit && \ + python -m black --check src/prkit tests/prkit && \ + python -m mypy src/prkit && \ + python -m pytest tests/prkit' diff --git a/pyproject.toml b/pyproject.toml index 000fbc0..add4e05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,9 +58,13 @@ dev = [ "build>=1.2.2", "pytest>=7.0.0", "pytest-cov>=4.0.0", - "black>=23.0.0", - "ruff>=0.6.0", - "mypy>=1.0.0", + # Gate tools are pinned exactly: their version decides pass/fail, so a + # floating ``>=`` lets a fresh CI install silently drift from local and + # break the build (a new ruff/black reformats, a new mypy newly-errors). + # Bump these deliberately, not by accident. Keep them in sync with the venv. + "black==26.1.0", + "ruff==0.15.17", + "mypy==1.19.1", "pre-commit>=3.5.0", "twine>=6.0.0", ] @@ -167,6 +171,17 @@ module = [ ] ignore_missing_imports = true +# numpy>=2.5 ships PEP 695 ``type`` statements in its bundled stubs, which mypy +# rejects as a syntax error whenever the target ``python_version`` is below 3.12 +# (we target 3.10). Skip following numpy's stubs so a dependency shipping +# newer-syntax stubs can't break the type gate; numpy call sites become untyped, +# as with the libraries above. (ignore_missing_imports does not help — the stub +# is found; the failure is parsing it under the 3.10 syntax target.) +[[tool.mypy.overrides]] +module = ["numpy", "numpy.*"] +follow_imports = "skip" +follow_imports_for_stubs = true + [tool.pytest.ini_options] testpaths = ["tests"] python_files = ["test_*.py"] diff --git a/tests/prkit/test_api.py b/tests/prkit/test_api.py index e6ce1a3..4ca92f6 100644 --- a/tests/prkit/test_api.py +++ b/tests/prkit/test_api.py @@ -112,8 +112,11 @@ def test_registered_loaders_satisfy_dataset_provider(self): loader = DatasetHub._get_loader(name) assert isinstance(loader, DatasetProvider), name - def test_model_client_satisfies_protocol(self): - # Construction only — no network call, works without an API key. + def test_model_client_satisfies_protocol(self, monkeypatch): + # Construction builds a real provider SDK client, which validates that a + # credential is present (no network call). Inject a dummy key so the + # check runs offline anywhere — CI has no .env to supply OPENAI_API_KEY. + monkeypatch.setenv("OPENAI_API_KEY", "test-key") client = create_model_client("gpt-4.1") assert isinstance(client, ModelClient)