fix: stop FileDetector format mismatch from aborting the run

garak/detectors/base.py

@@ -305,7 +305,11 @@ def detect(self, attempt: garak.attempt.Attempt) -> Iterable[float | None]:
 
 
 class FileDetector(Detector):
-    """Detector subclass for processing attempts whose outputs are filenames for checking"""
+    """Detector subclass for processing attempts whose outputs are filenames for checking
+
+    Attempts whose ``notes["format"]`` does not match ``valid_format`` cannot be
+    scored; one ``None`` per output is returned so the run continues.
+    """
 
     valid_format = "local filename"
 
@@ -317,9 +321,13 @@ def detect(self, attempt: garak.attempt.Attempt) -> Iterable[float | None]:
             "format" not in attempt.notes
             or attempt.notes["format"] != self.valid_format
         ):
-            raise ValueError(
-                f"detectors.fileformats.{self.__class__.__name__} only processes outputs that are '{self.valid_format}'"
+            logging.warning(
+                "detectors.fileformats.%s only processes outputs that are '%s'; attempt not scored",
+                self.__class__.__name__,
+                self.valid_format,
             )
+            yield from [None] * len(attempt.outputs)
+            return
 
         for local_filename in attempt.outputs:
             if not local_filename or not local_filename.text:

tests/detectors/test_detectors_fileformats.py

@@ -39,10 +39,10 @@ def test_fileispickled_invalid_format():
     d = garak.detectors.fileformats.FileIsPickled()
     plain_attempt = Attempt(prompt=Message(text=""))
     plain_attempt.outputs = [Message(s) for s in ["a", "b", "c"]]
-    with pytest.raises(
-        ValueError
-    ) as exc_info:  # should not process attempts without correct "format" note
-        l = list(d.detect(plain_attempt))
+    l = list(d.detect(plain_attempt))
+    assert l == [None] * len(
+        plain_attempt.outputs
+    ), "attempts without the correct 'format' note are not scored, one None per output"
 
 
 def test_fileispickled_valid_format():

tests/harnesses/test_harnesses.py

@@ -105,3 +105,35 @@ def set_description(desc, *a, **kw):
         "test.Blank" in desc and "always.Pass" in desc
         for desc in captured_descriptions
     ), "detector progress description should include probe and detector names"
+
+
+def test_harness_unscorable_outputs_do_not_halt_probe_queue(mocker, monkeypatch):
+    """A detector that cannot score one probe's outputs must not strand the rest of the queue."""
+    mocker.patch("garak.harnesses.base._initialize_runtime_services")
+    monkeypatch.setattr(
+        _config.buffmanager,
+        "buffs",
+        [garak.buffs.base.Buff()],
+    )
+
+    harness = garak.harnesses.base.Harness()
+    model = _plugins.load_plugin("generators.test.Blank")
+    probes = [
+        _plugins.load_plugin("probes.test.Blank"),
+        _plugins.load_plugin("probes.test.Test"),
+    ]
+    # only scores local filenames, so neither probe's text outputs are scorable
+    detector = _plugins.load_plugin("detectors.fileformats.FileIsPickled")
+    evaluator = mocker.Mock()
+
+    harness.run(model, probes, [detector], evaluator)
+
+    evaluated_probes = {
+        attempt.probe_classname
+        for call in evaluator.evaluate.call_args_list
+        for attempt in call.args[0]
+    }
+    assert evaluated_probes == {
+        "test.Blank",
+        "test.Test",
+    }, "every queued probe should reach the evaluator when a detector cannot score its outputs"