From 86d5b1fca432e8f1ae10552f5e12f6e6e977a483 Mon Sep 17 00:00:00 2001 From: Patricia Pampanelli Date: Fri, 7 Aug 2026 12:19:21 -0700 Subject: [PATCH] degrade FileDetector format mismatch to unscored attempt instead of aborting the run Signed-off-by: Patricia Pampanelli --- garak/detectors/base.py | 14 ++++++-- tests/detectors/test_detectors_fileformats.py | 8 ++--- tests/harnesses/test_harnesses.py | 32 +++++++++++++++++++ 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/garak/detectors/base.py b/garak/detectors/base.py index 9c8736f2f..15b9c6d92 100644 --- a/garak/detectors/base.py +++ b/garak/detectors/base.py @@ -305,7 +305,11 @@ def detect(self, attempt: garak.attempt.Attempt) -> Iterable[float | None]: class FileDetector(Detector): - """Detector subclass for processing attempts whose outputs are filenames for checking""" + """Detector subclass for processing attempts whose outputs are filenames for checking + + Attempts whose ``notes["format"]`` does not match ``valid_format`` cannot be + scored; one ``None`` per output is returned so the run continues. + """ valid_format = "local filename" @@ -317,9 +321,13 @@ def detect(self, attempt: garak.attempt.Attempt) -> Iterable[float | None]: "format" not in attempt.notes or attempt.notes["format"] != self.valid_format ): - raise ValueError( - f"detectors.fileformats.{self.__class__.__name__} only processes outputs that are '{self.valid_format}'" + logging.warning( + "detectors.fileformats.%s only processes outputs that are '%s'; attempt not scored", + self.__class__.__name__, + self.valid_format, ) + yield from [None] * len(attempt.outputs) + return for local_filename in attempt.outputs: if not local_filename or not local_filename.text: diff --git a/tests/detectors/test_detectors_fileformats.py b/tests/detectors/test_detectors_fileformats.py index 06db0b3fa..2324c22fc 100644 --- a/tests/detectors/test_detectors_fileformats.py +++ b/tests/detectors/test_detectors_fileformats.py @@ -39,10 +39,10 @@ def test_fileispickled_invalid_format(): d = garak.detectors.fileformats.FileIsPickled() plain_attempt = Attempt(prompt=Message(text="")) plain_attempt.outputs = [Message(s) for s in ["a", "b", "c"]] - with pytest.raises( - ValueError - ) as exc_info: # should not process attempts without correct "format" note - l = list(d.detect(plain_attempt)) + l = list(d.detect(plain_attempt)) + assert l == [None] * len( + plain_attempt.outputs + ), "attempts without the correct 'format' note are not scored, one None per output" def test_fileispickled_valid_format(): diff --git a/tests/harnesses/test_harnesses.py b/tests/harnesses/test_harnesses.py index 79b71a2d0..cb5ea89e3 100644 --- a/tests/harnesses/test_harnesses.py +++ b/tests/harnesses/test_harnesses.py @@ -105,3 +105,35 @@ def set_description(desc, *a, **kw): "test.Blank" in desc and "always.Pass" in desc for desc in captured_descriptions ), "detector progress description should include probe and detector names" + + +def test_harness_unscorable_outputs_do_not_halt_probe_queue(mocker, monkeypatch): + """A detector that cannot score one probe's outputs must not strand the rest of the queue.""" + mocker.patch("garak.harnesses.base._initialize_runtime_services") + monkeypatch.setattr( + _config.buffmanager, + "buffs", + [garak.buffs.base.Buff()], + ) + + harness = garak.harnesses.base.Harness() + model = _plugins.load_plugin("generators.test.Blank") + probes = [ + _plugins.load_plugin("probes.test.Blank"), + _plugins.load_plugin("probes.test.Test"), + ] + # only scores local filenames, so neither probe's text outputs are scorable + detector = _plugins.load_plugin("detectors.fileformats.FileIsPickled") + evaluator = mocker.Mock() + + harness.run(model, probes, [detector], evaluator) + + evaluated_probes = { + attempt.probe_classname + for call in evaluator.evaluate.call_args_list + for attempt in call.args[0] + } + assert evaluated_probes == { + "test.Blank", + "test.Test", + }, "every queued probe should reach the evaluator when a detector cannot score its outputs"