diff --git a/src/lab_bench_2/attachment_builder.py b/src/lab_bench_2/attachment_builder.py index d06f1b0..eff3acd 100644 --- a/src/lab_bench_2/attachment_builder.py +++ b/src/lab_bench_2/attachment_builder.py @@ -6,11 +6,6 @@ import hashlib from pathlib import Path -# MEDIA_TYPES is reused from EdisonScientific/labbench2 evals/utils.py. -# Its unusual entries (".json" / ".csv" -> "text/plain") are deliberate -# provider-compatibility workarounds: Vertex AI rejects application/json and -# Anthropic's Messages document API rejects text/csv. -from evals.utils import MEDIA_TYPES from inspect_ai.model import Content, ContentDocument, ContentImage IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg"} @@ -34,6 +29,12 @@ def build(files: list[Path], resize_oversized_images: bool = False) -> list[Cont def mime_type(file_path: Path) -> str: + # MEDIA_TYPES is reused from EdisonScientific/labbench2 evals/utils.py. + # Its unusual entries (".json" / ".csv" -> "text/plain") are deliberate + # provider-compatibility workarounds: Vertex AI rejects application/json and + # Anthropic's Messages document API rejects text/csv. + from evals.utils import MEDIA_TYPES + return MEDIA_TYPES.get(file_path.suffix.lower(), "application/octet-stream") diff --git a/src/lab_bench_2/dataset.py b/src/lab_bench_2/dataset.py index b6b393d..56fdc8b 100644 --- a/src/lab_bench_2/dataset.py +++ b/src/lab_bench_2/dataset.py @@ -13,9 +13,8 @@ import logging from collections.abc import Sequence from pathlib import Path -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast -from evals.models import LabBenchQuestion from inspect_ai.dataset import Dataset, MemoryDataset, Sample, hf_dataset from inspect_ai.model import ChatMessage, ChatMessageUser, Content, ContentText @@ -23,6 +22,12 @@ from lab_bench_2.prompt_composer import Mode from utils.sample_ids import create_stable_id +if TYPE_CHECKING: + # ``evals`` ships with the optional ``labbench2`` dependency; import it only + # for type checking so module load doesn't require the extra (which would + # break Inspect's entry-point task discovery). Runtime uses import it locally. + from evals.models import LabBenchQuestion + LAB_BENCH_2_DATASET_PATH = "EdisonScientific/labbench2" LAB_BENCH_2_DATASET_REVISION = "27d12d72af24e3f70db8a99df63e567366cbdb80" LAB_BENCH_2_DATASET_SPLIT = "train" @@ -40,6 +45,8 @@ def record_to_sample(record: dict[str, Any], mode: Mode = "inject") -> Sample: Precondition: the record's question supports ``mode``. The loader filters unsupported records via ``_question_supports_mode`` before calling this. """ + from evals.models import LabBenchQuestion + question = LabBenchQuestion.model_validate(record) metadata: dict[str, Any] = { @@ -102,6 +109,8 @@ def load_lab_bench_2_dataset( """ def to_samples(record: dict[str, Any]) -> list[Sample]: + from evals.models import LabBenchQuestion + question = LabBenchQuestion.model_validate(record) if not _question_supports_mode(question, mode): logger.info( diff --git a/src/lab_bench_2/file_downloader.py b/src/lab_bench_2/file_downloader.py index f5f506a..5a1b3cd 100644 --- a/src/lab_bench_2/file_downloader.py +++ b/src/lab_bench_2/file_downloader.py @@ -10,14 +10,17 @@ from pathlib import Path from typing import cast -from evals.utils import GCS_BUCKET, download_question_files - -def fetch(gcs_prefix: str, bucket_name: str = GCS_BUCKET) -> Path: +def fetch(gcs_prefix: str, bucket_name: str | None = None) -> Path: """Download files under ``gcs_prefix`` and return the local directory. - Raises ``RuntimeError`` if the prefix yields no files. + ``bucket_name`` defaults to the reference implementation's ``GCS_BUCKET`` + when omitted. Raises ``RuntimeError`` if the prefix yields no files. """ + from evals.utils import GCS_BUCKET, download_question_files + + if bucket_name is None: + bucket_name = GCS_BUCKET files_path = cast( Path, download_question_files(bucket_name=bucket_name, gcs_prefix=gcs_prefix), diff --git a/src/lab_bench_2/prompt_composer.py b/src/lab_bench_2/prompt_composer.py index 4e8dc4e..504eae8 100644 --- a/src/lab_bench_2/prompt_composer.py +++ b/src/lab_bench_2/prompt_composer.py @@ -5,8 +5,6 @@ from pathlib import Path from typing import Literal -from evals.utils import is_text_injectable_format - Mode = Literal["file", "inject", "retrieve"] # Copied verbatim from EdisonScientific/labbench2 @@ -44,6 +42,8 @@ def compose( def _injected_files_text(files: list[Path]) -> str: + from evals.utils import is_text_injectable_format + chunks = [ f"## {f.name}\n\n{f.read_text()}" for f in files if is_text_injectable_format(f) ] diff --git a/src/lab_bench_2/scorers.py b/src/lab_bench_2/scorers.py index 15d0e43..3bb0635 100644 --- a/src/lab_bench_2/scorers.py +++ b/src/lab_bench_2/scorers.py @@ -6,7 +6,12 @@ from pathlib import Path from typing import Any, cast -from inspect_ai.model import GenerateConfig, ResponseSchema, get_model +from inspect_ai.model import ( + GenerateConfig, + ModelOutput, + ResponseSchema, + get_model, +) from inspect_ai.scorer import ( CORRECT, INCORRECT, @@ -27,6 +32,10 @@ DEFAULT_GRADER_MODEL = "anthropic/claude-sonnet-4-5" GRADER_ROLE = "grader" +# Content moderation can block a grader response non-deterministically, so a +# blocked call might clear on a retry. Cap the attempts before giving up. +MAX_GRADER_ATTEMPTS = 3 + JUDGE_VERDICT_CORRECT = "correct" JUDGE_VERDICT_INCORRECT = "incorrect" JUDGE_VERDICT_UNSURE = "unsure" @@ -59,6 +68,20 @@ def _judge_score(prompt_template: str) -> Scorer: name="evaluation_result", json_schema=json_schema(EvaluationResult) ) + async def call_grader_with_retry(prompt: str) -> ModelOutput | None: + result: ModelOutput | None = None + grader = get_model(role=GRADER_ROLE, default=DEFAULT_GRADER_MODEL) + attempts = 0 + while attempts < MAX_GRADER_ATTEMPTS: + attempts += 1 + result = await grader.generate( + prompt, + config=GenerateConfig(temperature=0.0, response_schema=response_schema), + ) + if not _grader_refused(result): + break + return result + async def score(state: TaskState, target: Target) -> Score: answer = state.output.completion.strip() if not answer: @@ -66,17 +89,28 @@ async def score(state: TaskState, target: Target) -> Score: value=INCORRECT, answer="", explanation="No answer was produced." ) - grader = get_model(role=GRADER_ROLE, default=DEFAULT_GRADER_MODEL) - prompt = prompt_template.format( question=state.input_text, correct_answer=target.text, answer=answer, ) - result = await grader.generate( - prompt, - config=GenerateConfig(temperature=0.0, response_schema=response_schema), - ) + + result = await call_grader_with_retry(prompt) + + if result is None or _grader_refused(result): + stop_reason = getattr(result, "stop_reason", None) + return Score.unscored( + answer=answer, + explanation=( + f"Grader returned no gradeable response " + f"attempt(s) (stop_reason={stop_reason}); sample left unscored." + ), + metadata={ + "verdict": None, + "verdict_source": "refusal", + "grader_stop_reason": stop_reason, + }, + ) try: evaluation = EvaluationResult.model_validate_json(result.completion) @@ -99,6 +133,17 @@ async def score(state: TaskState, target: Target) -> Score: return score +def _grader_refused(result: ModelOutput) -> bool: + """Return True when the grader produced no gradeable response. + + A ``content_filter`` stop reason or an empty completion means moderation + blocked the grader (or it returned nothing), not that the answer is wrong. + """ + if result.stop_reason == "content_filter": + return True + return not (result.completion or "").strip() + + @scorer(metrics=[accuracy(), stderr()]) def semantic_judge_scorer() -> Scorer: """Grade an open-ended answer against the reference using a judge model.""" diff --git a/tests/lab_bench_2/test_file_downloader.py b/tests/lab_bench_2/test_file_downloader.py index 4a39996..1464727 100644 --- a/tests/lab_bench_2/test_file_downloader.py +++ b/tests/lab_bench_2/test_file_downloader.py @@ -36,9 +36,7 @@ def test_raises_when_no_files_downloaded( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: # given a download that returns an empty directory - monkeypatch.setattr( - file_downloader, "download_question_files", lambda **_: tmp_path - ) + monkeypatch.setattr("evals.utils.download_question_files", lambda **_: tmp_path) # when / then with pytest.raises(RuntimeError, match="none were downloaded"): @@ -49,9 +47,7 @@ def test_returns_directory_when_populated( ) -> None: # given a populated download (tmp_path / "ok.txt").write_text("ok") - monkeypatch.setattr( - file_downloader, "download_question_files", lambda **_: tmp_path - ) + monkeypatch.setattr("evals.utils.download_question_files", lambda **_: tmp_path) # when result = file_downloader.fetch("some/prefix") @@ -70,7 +66,7 @@ def stub(**kwargs: str) -> Path: captured.update(kwargs) return tmp_path - monkeypatch.setattr(file_downloader, "download_question_files", stub) + monkeypatch.setattr("evals.utils.download_question_files", stub) # when file_downloader.fetch("some/prefix", bucket_name="custom-bucket") diff --git a/tests/lab_bench_2/test_import_hygiene.py b/tests/lab_bench_2/test_import_hygiene.py new file mode 100644 index 0000000..eae5067 --- /dev/null +++ b/tests/lab_bench_2/test_import_hygiene.py @@ -0,0 +1,42 @@ +import subprocess +import sys +import textwrap + + +def test_package_imports_without_optional_labbench2_dependency() -> None: + # given a fresh interpreter where the optional ``labbench2`` extra (which + # provides the ``evals`` / ``labbench2`` modules) cannot be imported + script = textwrap.dedent( + """ + import sys + + class _Blocker: + def find_spec(self, name, path=None, target=None): + if name.split(".")[0] in {"evals", "labbench2"}: + raise ModuleNotFoundError(name) + return None + + sys.meta_path.insert(0, _Blocker()) + + # Importing the package is what Inspect does for entry-point task + # discovery; it must not pull in the optional dependency. + import lab_bench_2 + from lab_bench_2 import lab_bench_2 as task, SUPPORTED_TAGS + + assert callable(task) + assert "litqa3" in SUPPORTED_TAGS + assert "evals" not in sys.modules + assert "labbench2" not in sys.modules + """ + ) + + # when the package is imported in that interpreter + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + ) + + # then discovery succeeds without the optional dependency installed + assert result.returncode == 0, result.stderr diff --git a/tests/lab_bench_2/test_scorers.py b/tests/lab_bench_2/test_scorers.py index a027491..27201eb 100644 --- a/tests/lab_bench_2/test_scorers.py +++ b/tests/lab_bench_2/test_scorers.py @@ -1,3 +1,4 @@ +import math from pathlib import Path from types import SimpleNamespace from typing import Any @@ -122,12 +123,28 @@ def test_exact_match_judge_scorer_is_scorer() -> None: assert isinstance(exact_match_judge_scorer(), Scorer) -def _patch_grader(monkeypatch: pytest.MonkeyPatch, completion: str) -> None: +def _patch_grader( + monkeypatch: pytest.MonkeyPatch, + responses: str | list[tuple[str, str]], +) -> dict[str, int]: + """Patch the grader model with canned responses. + + ``responses`` is either a single completion string (one non-refused + response) or a list of ``(completion, stop_reason)`` specs returned across + successive ``generate`` calls to simulate retries. Returns a dict whose + ``calls`` entry tracks how many times the grader was invoked. + """ + specs = [(responses, "stop")] if isinstance(responses, str) else list(responses) + counter = {"calls": 0} + class _Grader: async def generate(self, prompt: str, **kwargs: Any) -> SimpleNamespace: - return SimpleNamespace(completion=completion) + counter["calls"] += 1 + completion, stop_reason = specs[min(counter["calls"] - 1, len(specs) - 1)] + return SimpleNamespace(completion=completion, stop_reason=stop_reason) monkeypatch.setattr(scorers, "get_model", lambda *args, **kwargs: _Grader()) + return counter class TestJudgeScorer: @@ -223,6 +240,87 @@ async def test_empty_answer_scores_incorrect( assert result.value == INCORRECT assert "No answer" in (result.explanation or "") + async def test_content_filter_then_success_retries_and_scores( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # given a grader blocked by the content filter, then a correct verdict + calls = _patch_grader( + monkeypatch, + [ + ("", "content_filter"), + ('{"rationale": "matches", "result": "correct"}', "stop"), + ], + ) + + # when + sut = semantic_judge_scorer() + result = await _score( + sut, _task_state("answer", {"tag": "litqa3"}), Target("ref") + ) + + # then the retry recovers the verdict after exactly two attempts + assert result.value == CORRECT + assert calls["calls"] == 2 + assert result.metadata == { + "verdict": "correct", + "verdict_source": "structured", + } + + async def test_persistent_content_filter_is_unscored( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # given every grader attempt is blocked by the content filter + calls = _patch_grader(monkeypatch, [("", "content_filter")]) + + # when + sut = semantic_judge_scorer() + result = await _score( + sut, _task_state("answer", {"tag": "litqa3"}), Target("ref") + ) + + # then the sample is left unscored after exhausting the retries + assert calls["calls"] == scorers.MAX_GRADER_ATTEMPTS + assert isinstance(result.value, float) and math.isnan(result.value) + assert result.metadata == { + "verdict": None, + "verdict_source": "refusal", + "grader_stop_reason": "content_filter", + } + assert "unscored" in (result.explanation or "") + + async def test_empty_grader_completion_is_treated_as_refusal( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # given a grader that returns an empty completion with no content filter + calls = _patch_grader(monkeypatch, [(" ", "stop")]) + + # when + sut = semantic_judge_scorer() + result = await _score( + sut, _task_state("answer", {"tag": "litqa3"}), Target("ref") + ) + + # then it is retried and left unscored, just like a content-filter block + assert calls["calls"] == scorers.MAX_GRADER_ATTEMPTS + assert isinstance(result.value, float) and math.isnan(result.value) + assert (result.metadata or {})["verdict_source"] == "refusal" + + async def test_first_try_success_calls_grader_once( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # given a grader that returns a verdict on the first attempt + calls = _patch_grader(monkeypatch, '{"rationale": "x", "result": "correct"}') + + # when + sut = semantic_judge_scorer() + result = await _score( + sut, _task_state("answer", {"tag": "litqa3"}), Target("ref") + ) + + # then no extra retries are issued + assert result.value == CORRECT + assert calls["calls"] == 1 + class TestCloningScorer: async def test_scores_correct_when_reward_passes(