From 5f705854c87e8769bd93edb9e3aa06f40ac916bd Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Mon, 15 Jun 2026 23:17:01 -0400 Subject: [PATCH 1/2] Rename find_canonical_root -> find_uqps_root locator API Renames the project-locator helper and its env var (PRKIT_CANONICAL_ROOT -> PRKIT_UQPS_ROOT), and repoints the hardcoded sibling-directory lookups to uncertainty_quantification_via_physics_semantics (the renamed canonical_answer_protocol project). Leaves unrelated domain uses of 'canonical' (unit canonicalization, canonical_text, the --canonical-sampling-root baseline flag) untouched. Co-Authored-By: Claude Opus 4.8 --- docs/PROJECT_SCRIPT_CONVENTIONS.md | 2 +- src/prkit/core/project_env.py | 36 ++++++++++--------- .../evaluation/utils/sampling_backfill.py | 12 +++---- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/docs/PROJECT_SCRIPT_CONVENTIONS.md b/docs/PROJECT_SCRIPT_CONVENTIONS.md index c4f631d..3535705 100644 --- a/docs/PROJECT_SCRIPT_CONVENTIONS.md +++ b/docs/PROJECT_SCRIPT_CONVENTIONS.md @@ -74,4 +74,4 @@ For multi-step workflows such as `prepare`, `submit`, and `fetch`: ## Reference Implementation -- `canonical_answer_protocol/` is the current reference implementation of these conventions. +- `uncertainty_quantification_via_physics_semantics/` is the current reference implementation of these conventions. diff --git a/src/prkit/core/project_env.py b/src/prkit/core/project_env.py index 526b166..026f82c 100644 --- a/src/prkit/core/project_env.py +++ b/src/prkit/core/project_env.py @@ -8,7 +8,7 @@ from pathlib import Path _TOOLKIT_ENV_VAR = "PRKIT_TOOLKIT_ROOT" -_CANONICAL_ENV_VAR = "PRKIT_CANONICAL_ROOT" +_UQPS_ENV_VAR = "PRKIT_UQPS_ROOT" _UQ_ENV_VAR = "PRKIT_UQ_ROOT" @@ -69,12 +69,12 @@ def find_toolkit_root(anchor: str | PathLike[str] | Path | None = None) -> Path ) -def find_canonical_root( +def find_uqps_root( anchor: str | PathLike[str] | Path | None = None, ) -> Path | None: - """Return the canonical-answer-protocol repo root when present.""" + """Return the uqps (uncertainty-quantification-via-physics-semantics) repo root when present.""" env_root = _resolve_env_root( - _CANONICAL_ENV_VAR, + _UQPS_ENV_VAR, marker_relpath=("scripts", "__init__.py"), ) if env_root is not None: @@ -82,23 +82,25 @@ def find_canonical_root( toolkit_root = find_toolkit_root(anchor) if toolkit_root is not None: - sibling_root = toolkit_root.parent / "canonical_answer_protocol" + sibling_root = ( + toolkit_root.parent / "uncertainty_quantification_via_physics_semantics" + ) if (sibling_root / "scripts").is_dir(): return sibling_root - nested_root = toolkit_root / "canonical_answer_protocol" + nested_root = toolkit_root / "uncertainty_quantification_via_physics_semantics" if (nested_root / "scripts").is_dir(): return nested_root for candidate in _iter_search_dirs(anchor): if ( - candidate.name == "canonical_answer_protocol" + candidate.name == "uncertainty_quantification_via_physics_semantics" and (candidate / "scripts").is_dir() ): return candidate return _find_named_sibling( anchor, - "canonical_answer_protocol", + "uncertainty_quantification_via_physics_semantics", marker_relpath=("scripts", "__init__.py"), ) @@ -114,8 +116,8 @@ def find_repo_root( """ if repo_name == "physical_reasoning_toolkit": return find_toolkit_root(anchor) - if repo_name == "canonical_answer_protocol": - return find_canonical_root(anchor) + if repo_name == "uncertainty_quantification_via_physics_semantics": + return find_uqps_root(anchor) if repo_name == "uncertainty_quantification_physical_reasoning": return find_uq_root(anchor) raise ValueError(f"Unsupported repo name: {repo_name}") @@ -162,7 +164,7 @@ def project_dotenv_paths( Precedence is: 1. toolkit root `.env` - 2. `canonical_answer_protocol/.env` + 2. `uncertainty_quantification_via_physics_semantics/.env` 3. `uncertainty_quantification_physical_reasoning/.env` Later files win because they are loaded with `override=True`. @@ -174,11 +176,11 @@ def project_dotenv_paths( if repo_env.is_file(): paths.append(repo_env) - canonical_root = find_canonical_root(anchor) - if canonical_root is not None: - canonical_env = canonical_root / ".env" - if canonical_env.is_file() and canonical_env not in paths: - paths.append(canonical_env) + uqps_root = find_uqps_root(anchor) + if uqps_root is not None: + uqps_env = uqps_root / ".env" + if uqps_env.is_file() and uqps_env not in paths: + paths.append(uqps_env) uq_root = find_uq_root(anchor) if uq_root is not None: @@ -233,10 +235,10 @@ def ensure_openai_api_key( __all__ = [ "ensure_openai_api_key", - "find_canonical_root", "find_repo_root", "find_toolkit_root", "find_uq_root", + "find_uqps_root", "load_project_dotenv", "project_dotenv_paths", ] diff --git a/src/prkit/evaluation/utils/sampling_backfill.py b/src/prkit/evaluation/utils/sampling_backfill.py index e2d6180..07ab2d1 100644 --- a/src/prkit/evaluation/utils/sampling_backfill.py +++ b/src/prkit/evaluation/utils/sampling_backfill.py @@ -12,9 +12,9 @@ from typing import Any from prkit.core.project_env import ( - find_canonical_root, find_toolkit_root, find_uq_root, + find_uqps_root, ) DEFAULT_REPO_ROOT = find_toolkit_root(__file__) or Path(__file__).resolve().parents[4] @@ -25,13 +25,13 @@ DEFAULT_INFERENCE_ROOT = ( DEFAULT_UQ_ROOT / "experiment_results" / "inference" / "response_with_answer_tag" ) -_default_canonical_root = find_canonical_root(__file__) +_default_uqps_root = find_uqps_root(__file__) DEFAULT_CANONICAL_SAMPLING_ROOT = ( - (_default_canonical_root / "baselines" / "sampling") - if _default_canonical_root is not None + (_default_uqps_root / "baselines" / "sampling") + if _default_uqps_root is not None else ( DEFAULT_REPO_ROOT.parent - / "canonical_answer_protocol" + / "uncertainty_quantification_via_physics_semantics" / "baselines" / "sampling" ) @@ -84,7 +84,7 @@ def parse_args() -> argparse.Namespace: "--canonical-sampling-root", type=Path, default=None, - help="Override canonical_answer_protocol/baselines/sampling root.", + help="Override uncertainty_quantification_via_physics_semantics/baselines/sampling root.", ) parser.add_argument( "--missing-ids-dir", From 3e3848fe192fb95e1840ef9619fd74138b81cfdb Mon Sep 17 00:00:00 2001 From: Yinghuan Zhang Date: Tue, 16 Jun 2026 00:19:28 -0400 Subject: [PATCH 2/2] Decouple prkit toolkit from downstream projects The toolkit must not know about its consumer projects. Remove the cross-repo locators (find_uqps_root/find_uq_root/find_repo_root) and the PRKIT_UQPS_ROOT/PRKIT_UQ_ROOT support from core/project_env.py; the .env loader now resolves only the toolkit's own root. Relocate the project-specific CLIs (sampling_backfill.py, fill_is_match_predicate_physpara.py) to the uq repo, and delete the consumer integration tests (tests/uq/) plus the project-script conventions doc that assumed a monorepo layout. Consumers own cross-repo locating and .env loading via their own bridges; neither breaks. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/PROJECT_SCRIPT_CONVENTIONS.md | 77 ---- src/prkit/core/project_env.py | 121 +---- .../evaluation/utils/sampling_backfill.py | 413 ------------------ tests/prkit/core/test_project_env.py | 70 ++- tests/uq/conftest.py | 14 - tests/uq/test_batch_prepare_common.py | 102 ----- tests/uq/test_extract_answer_parsers.py | 20 - .../test_single_inference_with_answer_tags.py | 117 ----- 8 files changed, 53 insertions(+), 881 deletions(-) delete mode 100644 docs/PROJECT_SCRIPT_CONVENTIONS.md delete mode 100644 src/prkit/evaluation/utils/sampling_backfill.py delete mode 100644 tests/uq/conftest.py delete mode 100644 tests/uq/test_batch_prepare_common.py delete mode 100644 tests/uq/test_extract_answer_parsers.py delete mode 100644 tests/uq/test_single_inference_with_answer_tags.py diff --git a/docs/PROJECT_SCRIPT_CONVENTIONS.md b/docs/PROJECT_SCRIPT_CONVENTIONS.md deleted file mode 100644 index 3535705..0000000 --- a/docs/PROJECT_SCRIPT_CONVENTIONS.md +++ /dev/null @@ -1,77 +0,0 @@ -# Project Script Conventions - -These conventions apply to scripting-oriented subprojects inside `physical_reasoning_toolkit`. - -## Scope - -- Treat each subproject as an independent project root. -- Write documentation and examples assuming commands are executed from that project root. -- Keep one and only one `scripts/` folder inside each project root. - -## Required Layout - -Every scripting-oriented subproject should follow this shape: - -```text -/ -├── README.md -├── documents/ -└── scripts/ - ├── / - ├── script_/ - │ ├── run_.py - │ └── / - └── script_/ -``` - -Rules: - -- Runnable workflow folders inside `scripts/` must start with `script_`. -- Reusable module folders inside `scripts/` must not start with `script_`. -- Reusable code should be grouped by functionality and imported by the runnable wrappers. -- Provider-specific entrypoints should live under the relevant workflow folder, for example: - - `scripts/script_inference/openai/` - - `scripts/script_inference/gemini/` - - `scripts/script_ground_truth_cleanup/openai/` - -## CLI And Workflow Design - -- Keep CLI wrappers thin: parse arguments, call shared logic, print actionable results. -- Put reusable prompt builders, schemas, parsers, runners, and helpers in shared modules. -- Prefer modular code with small, testable functions over monolithic scripts. -- Keep output artifacts machine-readable when possible, and emit manifests for multi-stage workflows. - -## Problem Selection Contract - -Any script that selects a subset of problems must support all of the following: - -- `--problem-id` -- `--problem-ids` -- `--problem-ids-file` -- `-f` as the short alias for `--problem-ids-file` - -File support requirements: - -- Plain-text files with one problem id per line must be supported. -- JSON problem-id files must be supported. -- Project-defined default subset files are allowed, but any explicit selector must override the default. - -## Multi-Step Batch UX - -For multi-step workflows such as `prepare`, `submit`, and `fetch`: - -- `prepare` should print the exact next `submit` command. -- `submit` should print the exact next `fetch` command. -- `fetch` should not raise a noisy error when a remote batch is still running. -- Instead, `fetch` should print the current status, useful progress counts when available, and the exact command to rerun later. -- When a batch completes, `fetch` should print the next downstream command if the workflow has one. - -## Documentation Expectations - -- Each subproject should have a `README.md` with the project-root execution model, layout, and minimal command examples. -- Longer workflow explanations, design notes, and algorithm descriptions should go under `documents/`. -- Documentation examples should use paths and commands exactly as the user is expected to run them. - -## Reference Implementation - -- `uncertainty_quantification_via_physics_semantics/` is the current reference implementation of these conventions. diff --git a/src/prkit/core/project_env.py b/src/prkit/core/project_env.py index 026f82c..d6e5663 100644 --- a/src/prkit/core/project_env.py +++ b/src/prkit/core/project_env.py @@ -8,8 +8,6 @@ from pathlib import Path _TOOLKIT_ENV_VAR = "PRKIT_TOOLKIT_ROOT" -_UQPS_ENV_VAR = "PRKIT_UQPS_ROOT" -_UQ_ENV_VAR = "PRKIT_UQ_ROOT" def _anchor_dir(anchor: str | PathLike[str] | Path | None = None) -> Path: @@ -69,126 +67,20 @@ def find_toolkit_root(anchor: str | PathLike[str] | Path | None = None) -> Path ) -def find_uqps_root( - anchor: str | PathLike[str] | Path | None = None, -) -> Path | None: - """Return the uqps (uncertainty-quantification-via-physics-semantics) repo root when present.""" - env_root = _resolve_env_root( - _UQPS_ENV_VAR, - marker_relpath=("scripts", "__init__.py"), - ) - if env_root is not None: - return env_root - - toolkit_root = find_toolkit_root(anchor) - if toolkit_root is not None: - sibling_root = ( - toolkit_root.parent / "uncertainty_quantification_via_physics_semantics" - ) - if (sibling_root / "scripts").is_dir(): - return sibling_root - nested_root = toolkit_root / "uncertainty_quantification_via_physics_semantics" - if (nested_root / "scripts").is_dir(): - return nested_root - - for candidate in _iter_search_dirs(anchor): - if ( - candidate.name == "uncertainty_quantification_via_physics_semantics" - and (candidate / "scripts").is_dir() - ): - return candidate - - return _find_named_sibling( - anchor, - "uncertainty_quantification_via_physics_semantics", - marker_relpath=("scripts", "__init__.py"), - ) - - -def find_repo_root( - repo_name: str, - anchor: str | PathLike[str] | Path | None = None, -) -> Path | None: - """Return one of the three known sibling repo roots by logical name. - - Raises: - ValueError: If *repo_name* is not one of the three supported repos. - """ - if repo_name == "physical_reasoning_toolkit": - return find_toolkit_root(anchor) - if repo_name == "uncertainty_quantification_via_physics_semantics": - return find_uqps_root(anchor) - if repo_name == "uncertainty_quantification_physical_reasoning": - return find_uq_root(anchor) - raise ValueError(f"Unsupported repo name: {repo_name}") - - -def find_uq_root(anchor: str | PathLike[str] | Path | None = None) -> Path | None: - """Return the uncertainty-quantification package root when present.""" - env_root = _resolve_env_root( - _UQ_ENV_VAR, - marker_relpath=("scripts", "__init__.py"), - ) - if env_root is not None: - return env_root - - toolkit_root = find_toolkit_root(anchor) - if toolkit_root is not None: - uq_root = toolkit_root / "uncertainty_quantification_physical_reasoning" - if uq_root.is_dir(): - return uq_root - sibling_root = ( - toolkit_root.parent / "uncertainty_quantification_physical_reasoning" - ) - if sibling_root.is_dir(): - return sibling_root - - for candidate in _iter_search_dirs(anchor): - if ( - candidate.name == "uncertainty_quantification_physical_reasoning" - and (candidate / "scripts").is_dir() - ): - return candidate - - return _find_named_sibling( - anchor, - "uncertainty_quantification_physical_reasoning", - marker_relpath=("scripts", "__init__.py"), - ) - - def project_dotenv_paths( anchor: str | PathLike[str] | Path | None = None, ) -> tuple[Path, ...]: - """Project `.env` files in load order. - - Precedence is: - 1. toolkit root `.env` - 2. `uncertainty_quantification_via_physics_semantics/.env` - 3. `uncertainty_quantification_physical_reasoning/.env` + """Return the toolkit's own `.env` path, when present. - Later files win because they are loaded with `override=True`. + The toolkit loads only its own project `.env`. Consumer repositories are + responsible for locating and loading their own environment files. """ - paths: list[Path] = [] toolkit_root = find_toolkit_root(anchor) if toolkit_root is not None: repo_env = toolkit_root / ".env" if repo_env.is_file(): - paths.append(repo_env) - - uqps_root = find_uqps_root(anchor) - if uqps_root is not None: - uqps_env = uqps_root / ".env" - if uqps_env.is_file() and uqps_env not in paths: - paths.append(uqps_env) - - uq_root = find_uq_root(anchor) - if uq_root is not None: - uq_env = uq_root / ".env" - if uq_env.is_file() and uq_env not in paths: - paths.append(uq_env) - - return tuple(paths) + return (repo_env,) + return () def load_project_dotenv( @@ -235,10 +127,7 @@ def ensure_openai_api_key( __all__ = [ "ensure_openai_api_key", - "find_repo_root", "find_toolkit_root", - "find_uq_root", - "find_uqps_root", "load_project_dotenv", "project_dotenv_paths", ] diff --git a/src/prkit/evaluation/utils/sampling_backfill.py b/src/prkit/evaluation/utils/sampling_backfill.py deleted file mode 100644 index 07ab2d1..0000000 --- a/src/prkit/evaluation/utils/sampling_backfill.py +++ /dev/null @@ -1,413 +0,0 @@ -#!/usr/bin/env python3 -"""CLI tool to backfill sampling-format problem JSONs from inference outputs for a given problem subset.""" - -from __future__ import annotations - -import argparse -import json -import re -import shlex -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from prkit.core.project_env import ( - find_toolkit_root, - find_uq_root, - find_uqps_root, -) - -DEFAULT_REPO_ROOT = find_toolkit_root(__file__) or Path(__file__).resolve().parents[4] -DEFAULT_UQ_ROOT = find_uq_root(__file__) or ( - DEFAULT_REPO_ROOT.parent / "uncertainty_quantification_physical_reasoning" -) -DEFAULT_SAMPLING_ROOT = DEFAULT_UQ_ROOT / "experiment_results" / "sampling" -DEFAULT_INFERENCE_ROOT = ( - DEFAULT_UQ_ROOT / "experiment_results" / "inference" / "response_with_answer_tag" -) -_default_uqps_root = find_uqps_root(__file__) -DEFAULT_CANONICAL_SAMPLING_ROOT = ( - (_default_uqps_root / "baselines" / "sampling") - if _default_uqps_root is not None - else ( - DEFAULT_REPO_ROOT.parent - / "uncertainty_quantification_via_physics_semantics" - / "baselines" - / "sampling" - ) -) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=( - "Backfill sampling-format problem JSONs from response_with_answer_tag " - "inference outputs for a subset of problem IDs." - ) - ) - parser.add_argument( - "--dataset", - required=True, - help="Dataset prefix used in directory names, e.g. seephys.", - ) - parser.add_argument( - "--subset-path", - type=Path, - required=True, - help="JSON array of target problem IDs.", - ) - parser.add_argument( - "--models", - nargs="+", - required=True, - help="Model names, e.g. gpt-5.4 gpt-4.1.", - ) - parser.add_argument( - "--repo-root", - type=Path, - default=DEFAULT_REPO_ROOT, - help=f"Toolkit repo root for reporting/debugging (default: {DEFAULT_REPO_ROOT}).", - ) - parser.add_argument( - "--sampling-root", - type=Path, - default=None, - help="Override experiment_results/sampling root.", - ) - parser.add_argument( - "--inference-root", - type=Path, - default=None, - help="Override experiment_results/inference/response_with_answer_tag root.", - ) - parser.add_argument( - "--canonical-sampling-root", - type=Path, - default=None, - help="Override uncertainty_quantification_via_physics_semantics/baselines/sampling root.", - ) - parser.add_argument( - "--missing-ids-dir", - type=Path, - default=None, - help="Directory for generated missing-id JSON files (default: subset dir).", - ) - parser.add_argument( - "--report-path", - type=Path, - default=None, - help="Optional JSON report output path.", - ) - parser.add_argument( - "--apply", - action="store_true", - help="Write missing sampling files to the configured destinations.", - ) - parser.add_argument( - "--overwrite", - action="store_true", - help="Overwrite already-existing output files when --apply is set.", - ) - return parser.parse_args() - - -def load_problem_ids(path: Path) -> list[str]: - """Load a JSON array of problem ID strings from *path*.""" - with open(path, encoding="utf-8") as f: - raw = json.load(f) - if not isinstance(raw, list): - raise ValueError(f"{path} must be a JSON array") - return [str(item).strip() for item in raw if str(item).strip()] - - -def problem_ids_from_dir(path: Path) -> set[str]: - """Return the set of problem IDs found as ``problem_.json`` files in *path*.""" - if not path.is_dir(): - return set() - ids: set[str] = set() - for file_path in path.glob("problem_*.json"): - match = re.match(r"problem_(.+)\.json$", file_path.name) - if match: - ids.add(match.group(1)) - return ids - - -def sort_problem_ids(problem_ids: set[str] | list[str]) -> list[str]: - """Sort problem IDs numerically when all-digit, lexicographically otherwise.""" - - def key(value: str) -> tuple[int, Any]: - return (0, int(value)) if value.isdigit() else (1, value) - - return sorted((str(item) for item in problem_ids), key=key) - - -def load_inference_doc(path: Path) -> tuple[str, str | None]: - """Load an inference output JSON and return ``(raw_model_response, extracted_model_answer)``.""" - with open(path, encoding="utf-8") as f: - data = json.load(f) - if not isinstance(data, dict): - raise ValueError(f"{path} is not a JSON object") - - raw = ( - data.get("raw_model_response") - or data.get("model_response") - or data.get("response") - or data.get("full_response") - or "" - ) - if raw is None: - raw = "" - if not isinstance(raw, str): - raw = str(raw) - - answer = data.get("extracted_model_answer") - if answer is None: - answer = data.get("model_answer") - if answer is not None and not isinstance(answer, str): - answer = str(answer) - return raw, answer - - -def build_sampling_doc( - *, - dataset: str, - model: str, - problem_id: str, - raw_model_response: str, - extracted_model_answer: str | None, -) -> dict[str, Any]: - """Build a single-response sampling document dict from inference output fields.""" - return { - "problem_id": problem_id, - "database_name": dataset, - "model": model, - "responses": [ - { - "index_no": 0, - "model": model, - "raw_model_response": raw_model_response, - "extracted_model_answer": extracted_model_answer, - "source_model": "dataset", - } - ], - } - - -def write_json(path: Path, payload: Any) -> None: - """Write *payload* as pretty-printed JSON to *path*, creating parent directories as needed.""" - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w", encoding="utf-8") as f: - json.dump(payload, f, indent=2, ensure_ascii=False) - f.write("\n") - - -def write_sampling_doc(path: Path, payload: dict[str, Any], overwrite: bool) -> bool: - """Write a sampling document to *path*; return ``False`` (skip) when it already exists and *overwrite* is false.""" - if path.exists() and not overwrite: - return False - write_json(path, payload) - return True - - -def command_for_model( - script_path: Path, - dataset: str, - subset_path: Path, - model: str, - report_path: Path | None, -) -> str: - """Build the shell command string to re-run this script for a single model.""" - parts = [ - "python", - str(script_path), - "--dataset", - dataset, - "--subset-path", - str(subset_path), - "--models", - model, - "--apply", - ] - if report_path is not None: - parts.extend(["--report-path", str(report_path)]) - return " ".join(shlex.quote(part) for part in parts) - - -def model_report( - *, - dataset: str, - model: str, - subset_ids: list[str], - subset_path: Path, - missing_ids_dir: Path, - inference_root: Path, - sampling_root: Path, - canonical_sampling_root: Path | None, - apply: bool, - overwrite: bool, - report_path: Path | None, -) -> dict[str, Any]: - """Compute the backfill status for one model and optionally write missing sampling documents.""" - run_name = f"{dataset}_{model}" - inference_dir = inference_root / run_name - sampling_dir = sampling_root / run_name - canonical_dir = ( - canonical_sampling_root / run_name if canonical_sampling_root else None - ) - - subset_set = set(subset_ids) - existing_sampling_ids = problem_ids_from_dir(sampling_dir) - existing_subset_ids = subset_set & existing_sampling_ids - extra_sampling_ids = existing_sampling_ids - subset_set - missing_sampling_ids = sort_problem_ids(subset_set - existing_sampling_ids) - - inference_ids = problem_ids_from_dir(inference_dir) - missing_inference_ids = sort_problem_ids(set(missing_sampling_ids) - inference_ids) - - subset_stem = subset_path.stem - missing_sampling_ids_path = ( - missing_ids_dir / f"{subset_stem}_missing_in_sampling__{model}.json" - ) - write_json(missing_sampling_ids_path, missing_sampling_ids) - - missing_inference_ids_path: Path | None = None - if missing_inference_ids: - missing_inference_ids_path = ( - missing_ids_dir - / f"{subset_stem}_missing_in_response_with_answer_tag__{model}.json" - ) - write_json(missing_inference_ids_path, missing_inference_ids) - - backfilled_sampling = 0 - backfilled_canonical = 0 - skipped_existing_sampling = 0 - skipped_existing_canonical = 0 - - if apply: - for problem_id in missing_sampling_ids: - inference_path = inference_dir / f"problem_{problem_id}.json" - if not inference_path.exists(): - continue - - raw_model_response, extracted_model_answer = load_inference_doc( - inference_path - ) - payload = build_sampling_doc( - dataset=dataset, - model=model, - problem_id=problem_id, - raw_model_response=raw_model_response, - extracted_model_answer=extracted_model_answer, - ) - - sampling_path = sampling_dir / f"problem_{problem_id}.json" - if write_sampling_doc(sampling_path, payload, overwrite=overwrite): - backfilled_sampling += 1 - else: - skipped_existing_sampling += 1 - - if canonical_dir is not None: - canonical_path = canonical_dir / f"problem_{problem_id}.json" - if write_sampling_doc(canonical_path, payload, overwrite=overwrite): - backfilled_canonical += 1 - else: - skipped_existing_canonical += 1 - - return { - "run_name": run_name, - "subset_problem_count": len(subset_ids), - "subset_problem_ids_present_in_sampling_count": len(existing_subset_ids), - "extra_sampling_problem_ids_count": len(extra_sampling_ids), - "extra_sampling_problem_ids": sort_problem_ids(extra_sampling_ids), - "missing_sampling_problem_ids_count": len(missing_sampling_ids), - "missing_sampling_problem_ids": missing_sampling_ids, - "missing_sampling_problem_ids_file": str(missing_sampling_ids_path), - "missing_inference_problem_ids_count": len(missing_inference_ids), - "missing_inference_problem_ids": missing_inference_ids, - "missing_inference_problem_ids_file": ( - str(missing_inference_ids_path) if missing_inference_ids_path else None - ), - "batch_submission_required": bool(missing_inference_ids), - "batch_submission_note": ( - "No new response_with_answer_tag batch is required; inference already exists " - "for every missing sampling problem." - if not missing_inference_ids - else "Response_with_answer_tag inference is still missing for some problems." - ), - "inference_dir": str(inference_dir), - "sampling_dir": str(sampling_dir), - "canonical_sampling_dir": str(canonical_dir) if canonical_dir else None, - "post_parse_sync_command": command_for_model( - Path(__file__).resolve(), - dataset, - subset_path, - model, - report_path, - ), - "backfilled_sampling_count": backfilled_sampling, - "backfilled_canonical_sampling_count": backfilled_canonical, - "skipped_existing_sampling_count": skipped_existing_sampling, - "skipped_existing_canonical_sampling_count": skipped_existing_canonical, - } - - -def main() -> int: - """Entry point: parse arguments, run backfill for each model, and output the report.""" - args = parse_args() - - repo_root = args.repo_root.resolve() - subset_path = args.subset_path.resolve() - sampling_root = (args.sampling_root or DEFAULT_SAMPLING_ROOT).resolve() - inference_root = (args.inference_root or DEFAULT_INFERENCE_ROOT).resolve() - canonical_sampling_root = ( - None - if args.canonical_sampling_root == Path("") - else ( - (args.canonical_sampling_root or DEFAULT_CANONICAL_SAMPLING_ROOT).resolve() - ) - ) - missing_ids_dir = (args.missing_ids_dir or subset_path.parent).resolve() - - subset_ids = load_problem_ids(subset_path) - - report: dict[str, Any] = { - "generated_at": datetime.now(timezone.utc).isoformat(), - "dataset": args.dataset, - "subset_path": str(subset_path), - "subset_problem_count": len(subset_ids), - "apply": args.apply, - "overwrite": args.overwrite, - "repo_root": str(repo_root), - "sampling_root": str(sampling_root), - "inference_root": str(inference_root), - "canonical_sampling_root": ( - str(canonical_sampling_root) if canonical_sampling_root else None - ), - "models": {}, - } - - for model in args.models: - report["models"][model] = model_report( - dataset=args.dataset, - model=model, - subset_ids=subset_ids, - subset_path=subset_path, - missing_ids_dir=missing_ids_dir, - inference_root=inference_root, - sampling_root=sampling_root, - canonical_sampling_root=canonical_sampling_root, - apply=args.apply, - overwrite=args.overwrite, - report_path=args.report_path.resolve() if args.report_path else None, - ) - - if args.report_path is not None: - write_json(args.report_path.resolve(), report) - else: - print(json.dumps(report, indent=2, ensure_ascii=False)) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/prkit/core/test_project_env.py b/tests/prkit/core/test_project_env.py index 3842784..1b0a887 100644 --- a/tests/prkit/core/test_project_env.py +++ b/tests/prkit/core/test_project_env.py @@ -1,4 +1,8 @@ -"""Tests for project-local environment loading helpers.""" +"""Tests for project-local environment loading helpers. + +The toolkit loads only its OWN ``.env``; it must never reach into consumer repos. +Consumer-side ``.env`` precedence is tested in the consumer repositories instead. +""" from __future__ import annotations @@ -15,51 +19,73 @@ def _make_toolkit_layout(tmp_path: Path) -> tuple[Path, Path, Path]: + """Build a toolkit root with a nested consumer repo (used as the ignored sibling).""" toolkit_root = tmp_path / "toolkit" (toolkit_root / "src" / "prkit").mkdir(parents=True) - uq_root = toolkit_root / "uncertainty_quantification_physical_reasoning" - anchor = uq_root / "scripts" / "script_predicate" / "run_example.py" - anchor.parent.mkdir(parents=True) - anchor.write_text("# anchor\n", encoding="utf-8") - return toolkit_root, uq_root, anchor + consumer_root = toolkit_root / "consumer_repo" + (consumer_root / "scripts").mkdir(parents=True) + toolkit_anchor = toolkit_root / "src" / "prkit" + return toolkit_root, consumer_root, toolkit_anchor -def test_project_dotenv_paths_include_repo_then_uq(tmp_path: Path) -> None: - toolkit_root, uq_root, anchor = _make_toolkit_layout(tmp_path) +def test_project_dotenv_paths_returns_only_toolkit_env( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("PRKIT_TOOLKIT_ROOT", raising=False) + toolkit_root, consumer_root, toolkit_anchor = _make_toolkit_layout(tmp_path) repo_env = toolkit_root / ".env" - uq_env = uq_root / ".env" repo_env.write_text("OPENAI_API_KEY=repo-key\n", encoding="utf-8") - uq_env.write_text("OPENAI_API_KEY=uq-key\n", encoding="utf-8") + # A consumer .env must be ignored entirely. + (consumer_root / ".env").write_text( + "OPENAI_API_KEY=consumer-key\n", encoding="utf-8" + ) - assert project_dotenv_paths(anchor) == (repo_env, uq_env) + assert project_dotenv_paths(toolkit_anchor) == (repo_env.resolve(),) -def test_load_project_dotenv_overrides_shell_with_uq_value( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, +def test_load_project_dotenv_overrides_shell_with_toolkit_value( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - toolkit_root, uq_root, anchor = _make_toolkit_layout(tmp_path) + monkeypatch.delenv("PRKIT_TOOLKIT_ROOT", raising=False) + toolkit_root, _, toolkit_anchor = _make_toolkit_layout(tmp_path) (toolkit_root / ".env").write_text("OPENAI_API_KEY=repo-key\n", encoding="utf-8") - (uq_root / ".env").write_text("OPENAI_API_KEY=uq-key\n", encoding="utf-8") monkeypatch.setenv("OPENAI_API_KEY", "shell-key") - loaded = load_project_dotenv(anchor, include_cwd_fallback=False) + loaded = load_project_dotenv(toolkit_anchor, include_cwd_fallback=False) + + assert loaded == ((toolkit_root / ".env").resolve(),) + assert os.environ["OPENAI_API_KEY"] == "repo-key" + + +def test_load_project_dotenv_ignores_sibling_consumer_env( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("PRKIT_TOOLKIT_ROOT", raising=False) + _, consumer_root, toolkit_anchor = _make_toolkit_layout(tmp_path) + # Only the consumer has an .env; the toolkit does not. + (consumer_root / ".env").write_text( + "OPENAI_API_KEY=consumer-key\n", encoding="utf-8" + ) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + + loaded = load_project_dotenv(toolkit_anchor, include_cwd_fallback=False) - assert loaded == (toolkit_root / ".env", uq_root / ".env") - assert os.environ["OPENAI_API_KEY"] == "uq-key" + assert loaded == () + assert "OPENAI_API_KEY" not in os.environ def test_ensure_openai_api_key_handles_missing_value( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - _, _, anchor = _make_toolkit_layout(tmp_path) + monkeypatch.delenv("PRKIT_TOOLKIT_ROOT", raising=False) + _, _, toolkit_anchor = _make_toolkit_layout(tmp_path) monkeypatch.delenv("OPENAI_API_KEY", raising=False) - assert ensure_openai_api_key(anchor, include_cwd_fallback=False) is None + assert ensure_openai_api_key(toolkit_anchor, include_cwd_fallback=False) is None with pytest.raises(RuntimeError, match="OPENAI_API_KEY is not set"): ensure_openai_api_key( - anchor, + toolkit_anchor, required=True, include_cwd_fallback=False, ) diff --git a/tests/uq/conftest.py b/tests/uq/conftest.py deleted file mode 100644 index 28a6e80..0000000 --- a/tests/uq/conftest.py +++ /dev/null @@ -1,14 +0,0 @@ -import importlib.util - -_uq_available = ( - importlib.util.find_spec("uncertainty_quantification_physical_reasoning") - is not None -) - -collect_ignore: list[str] = [] -if not _uq_available: - collect_ignore = [ - "test_batch_prepare_common.py", - "test_extract_answer_parsers.py", - "test_single_inference_with_answer_tags.py", - ] diff --git a/tests/uq/test_batch_prepare_common.py b/tests/uq/test_batch_prepare_common.py deleted file mode 100644 index 2211fc0..0000000 --- a/tests/uq/test_batch_prepare_common.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Tests for shared batch preparation helpers.""" - -from __future__ import annotations - -import json - -from uncertainty_quantification_physical_reasoning.scripts import batch_prepare_common - -from prkit.core.domain import PhysicalDataset, PhysicsProblem - - -def test_infer_dataset_load_kwargs_uses_problem_id_manifest(tmp_path): - problem_ids_file = tmp_path / "problem_ids_for_perturbation_90_domain_modality.json" - problem_ids_file.write_text("[]", encoding="utf-8") - - manifest_file = ( - tmp_path / "problem_ids_for_perturbation_90_domain_modality_manifest.json" - ) - manifest_file.write_text( - json.dumps( - { - "dataset": "physics", - "dataset_load_kwargs": { - "variant": "full", - "split": "eval", - }, - } - ), - encoding="utf-8", - ) - - assert batch_prepare_common.infer_dataset_load_kwargs( - "physics", problem_ids_file - ) == { - "variant": "full", - "split": "eval", - } - - -def test_load_filtered_dataset_preserves_problem_id_order(tmp_path, monkeypatch): - problem_ids_file = tmp_path / "problem_ids_for_perturbation_90_domain_modality.json" - problem_ids_file.write_text( - json.dumps(["mechanics/1_2", "atomic/1-1", "mechanics/1_2"]), - encoding="utf-8", - ) - - manifest_file = ( - tmp_path / "problem_ids_for_perturbation_90_domain_modality_manifest.json" - ) - manifest_file.write_text( - json.dumps( - { - "dataset": "physics", - "dataset_load_kwargs": { - "variant": "full", - "split": "eval", - }, - } - ), - encoding="utf-8", - ) - - called: dict[str, object] = {} - dataset = PhysicalDataset( - [ - PhysicsProblem(problem_id="atomic/1-1", question="Atomic question"), - PhysicsProblem(problem_id="mechanics/1_2", question="Mechanics question"), - ], - info={"name": "physics"}, - split="eval", - ) - - def fake_load(*, dataset_name, sample_size, auto_download, **kwargs): - called["dataset_name"] = dataset_name - called["sample_size"] = sample_size - called["auto_download"] = auto_download - called["kwargs"] = kwargs - return dataset - - monkeypatch.setattr( - batch_prepare_common.DatasetHub, - "load", - staticmethod(fake_load), - ) - - filtered = batch_prepare_common.load_filtered_dataset( - dataset_name="physics", - auto_download=False, - max_problems=None, - problem_ids_file=problem_ids_file, - ) - - assert called == { - "dataset_name": "physics", - "sample_size": None, - "auto_download": False, - "kwargs": {"variant": "full", "split": "eval"}, - } - assert [problem.problem_id for problem in filtered] == [ - "mechanics/1_2", - "atomic/1-1", - ] diff --git a/tests/uq/test_extract_answer_parsers.py b/tests/uq/test_extract_answer_parsers.py deleted file mode 100644 index f143b81..0000000 --- a/tests/uq/test_extract_answer_parsers.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Regression tests for malformed blocks.""" - -from uncertainty_quantification_physical_reasoning.scripts.script_physical_reasoning.extract_answer.extract_answer_after_think import ( - parse_answer_from_response as parse_after_think, -) -from uncertainty_quantification_physical_reasoning.scripts.script_physical_reasoning.extract_answer_before_think import ( - parse_answer_from_response as parse_before_think, -) - - -def test_parse_after_think_unterminated_answer_tag(): - response = "work\n\n" "\n" "final result" - - assert parse_after_think(response) == "final result" - - -def test_parse_before_think_unterminated_answer_tag(): - response = "\n" "final result\n" "reasoning" - - assert parse_before_think(response) == "final result" diff --git a/tests/uq/test_single_inference_with_answer_tags.py b/tests/uq/test_single_inference_with_answer_tags.py deleted file mode 100644 index ff3d075..0000000 --- a/tests/uq/test_single_inference_with_answer_tags.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Tests for single inference answer-tag persistence behavior.""" - -from __future__ import annotations - -import json - -from uncertainty_quantification_physical_reasoning.scripts.script_physical_reasoning.inferences import ( - single_inference_with_answer_tags as inference_script, -) - - -class _ExplodingClient: - provider = "dashscope" - - def __init__(self, message: str) -> None: - self._message = message - - def chat(self, **_kwargs): - raise RuntimeError(self._message) - - -class _StaticClient: - provider = "dashscope" - - def __init__(self, response: str) -> None: - self._response = response - - def chat(self, **_kwargs): - return self._response - - -def test_process_single_problem_persists_failure_result(tmp_path, monkeypatch): - monkeypatch.setattr( - inference_script, - "format_problem_question_text_for_batch", - lambda problem: problem.get("question"), - ) - success, _elapsed, problem_id, error_message = ( - inference_script.process_single_problem( - problem={"problem_id": "123", "question": "What is shown?"}, - index=1, - total_problems=1, - client=_ExplodingClient("provider timeout"), - output_dir=tmp_path, - dataset_name="seephys", - model_name="qwen3.6-plus", - ) - ) - - assert success is False - assert problem_id == "123" - assert "provider timeout" in (error_message or "") - - payload = json.loads((tmp_path / "problem_123.json").read_text(encoding="utf-8")) - assert payload["request_failed"] is True - assert payload["request_succeeded"] is False - assert payload["model_response"] == "" - assert payload["model_answer"] is None - assert payload["request_error_message"] == "Inference failed: provider timeout" - assert payload["visual_input_mode"] == "images" - - -def test_process_single_problem_marks_max_token_failure(tmp_path, monkeypatch): - monkeypatch.setattr( - inference_script, - "format_problem_question_text_for_batch", - lambda problem: problem.get("question"), - ) - success, _elapsed, problem_id, error_message = ( - inference_script.process_single_problem( - problem={"problem_id": "456", "question": "Solve it."}, - index=1, - total_problems=1, - client=_ExplodingClient("Request failed with finish_reason=length"), - output_dir=tmp_path, - dataset_name="seephys", - model_name="qwen3.6-plus", - ) - ) - - assert success is False - assert problem_id == "456" - assert "finish_reason=length" in (error_message or "") - - payload = json.loads((tmp_path / "problem_456.json").read_text(encoding="utf-8")) - assert payload["request_failed"] is True - assert payload["request_succeeded"] is False - assert payload["finish_reason"] == "MAX_TOKEN" - - -def test_process_single_problem_persists_empty_success(tmp_path, monkeypatch): - monkeypatch.setattr( - inference_script, - "format_problem_question_text_for_batch", - lambda problem: problem.get("question"), - ) - success, _elapsed, problem_id, error_message = ( - inference_script.process_single_problem( - problem={"problem_id": "789", "question": "Answer this."}, - index=1, - total_problems=1, - client=_StaticClient(""), - output_dir=tmp_path, - dataset_name="seephys", - model_name="qwen3.6-plus", - ) - ) - - assert success is True - assert problem_id == "789" - assert error_message is None - - payload = json.loads((tmp_path / "problem_789.json").read_text(encoding="utf-8")) - assert payload["request_failed"] is False - assert payload["request_succeeded"] is True - assert payload["model_response"] == "" - assert payload["model_answer"] is None