fix: report what a machine without the microcode cannot check

Two checks passed here and reported success on a runner that checked
nothing, which is the exact failure the doctor and the whole family's
skip-out-loud discipline exist to prevent.

The doctor's report of where each program waits read whatever images
happened to be on the machine, so its failure path never ran on a runner
with none. The throughput floor measured for real, so its report was
never checked where measuring is impossible. Both now take what they work
from, and the real measurement runs where the program is present and says
so where it is not.

A readme example that builds a part is skipped rather than counted as
broken, decided by this package's own why_not, which is the same sentence
its doctor prints.

Author: gufranco

README.md

@@ -25,7 +25,7 @@
   <a href="https://github.com/gufranco/snes-dsp-python/issues">Issues</a>
 </p>
 
-**6** parts across **5** microcodes · **1** processor underneath them all · **0** commands described by hand · **112** exchanges read out of **36** real cartridges compared, **0** failures · **830** tests · **100%** statement and branch coverage · **strict** types throughout · every image confirmed by **SHA-256** before a byte of it runs · no dependencies
+**6** parts across **5** microcodes · **1** processor underneath them all · **0** commands described by hand · **112** exchanges read out of **36** real cartridges compared, **0** failures · **847** tests · **100%** statement and branch coverage · **strict** types throughout · every image confirmed by **SHA-256** before a byte of it runs · no dependencies
 
 ```python
 from snesdsp import Chip

conformance/family.test.py

@@ -188,20 +188,6 @@ def store_attribute() -> str:  # pragma: no cover
     return found[0]
 
 
-def a_store_of_zeroes() -> Any:  # pragma: no cover
-    """A store the part will run through rather than halt in.
-
-    Built by the member rather than by name, for the same reason the attribute is
-    found rather than assumed. Left in scrambled memory a part reaches an
-    undocumented opcode within a few dozen instructions and stops, which is
-    correct behaviour and useless for testing a limit.
-    """
-    made = getattr(PACKAGE, "Memory", None)
-    if made is not None:
-        return made(image=bytes(0x10000))
-    return getattr(PACKAGE.Cpu(PACKAGE.DEFAULT_MODEL), store_attribute())
-
-
 def accounts_for_one_interrupt(node: Any) -> bool:  # pragma: no cover
     """Whether a record says anything about the part having one interrupt line.
 
@@ -233,11 +219,14 @@ def at_the_start(part: Any) -> None:  # pragma: no cover
 def a_running_part() -> Part:  # pragma: no cover
     """A part pointed at a field of no-operations, so a bound is what is tested.
 
-    Left in scrambled memory a part reaches an undocumented opcode within a few
-    dozen instructions and halts, which is correct behaviour and useless for
-    testing a limit.
+    `fill` is the one spelling across this family for a store holding one byte
+    everywhere. It exists for exactly this: left in scrambled memory a part
+    reaches an undocumented opcode within a few dozen instructions and stops,
+    which is correct behaviour and useless for testing a limit. Three of the four
+    clocked members did not have it and each needed a different keyword, so a
+    check written against any one of them reported the other three as broken.
     """
-    part = PACKAGE.Cpu(PACKAGE.DEFAULT_MODEL, a_store_of_zeroes())
+    part = PACKAGE.Cpu(PACKAGE.DEFAULT_MODEL, fill=0)
     at_the_start(part)
     checked: Part = part
     return checked
@@ -301,6 +290,21 @@ def test_and_the_record_accounts_for_what_lines_it_names(self) -> None:
         """
         self.assertTrue(accounts_for_one_interrupt(json.loads(RECORD.read_text())))
 
+    def test_a_record_naming_an_interrupt_as_a_key_is_read(self) -> None:
+        self.assertTrue(accounts_for_one_interrupt({"facts": {"interruptVector": 4}}))
+
+    def test_and_one_naming_it_only_in_a_sentence(self) -> None:
+        self.assertTrue(accounts_for_one_interrupt({"notStated": ["what the interrupt pin does"]}))
+
+    def test_and_one_naming_it_inside_a_list(self) -> None:
+        """The record shape that goes through the list branch rather than the dict one."""
+        self.assertTrue(accounts_for_one_interrupt([{"note": "one interrupt line"}]))
+
+    def test_and_a_record_that_never_mentions_one_is_reported(self) -> None:
+        self.assertFalse(
+            accounts_for_one_interrupt({"facts": {"window": [1, 2]}, "note": "a part"})
+        )
+
     def test_and_every_counter(self) -> None:
         part = a_part()
 

conformance/readme.test.py

@@ -9,20 +9,80 @@
 Each example runs in a fresh interpreter rather than in this one. An example that
 only works because a test already imported something is not an example a reader
 can paste, and running it here in-process would hide that.
+
+An example that cannot run at all here is reported as skipped rather than as
+broken, and only when the member itself says why. Two members model a part that
+runs a program their repository is not allowed to carry, so on a machine without
+one every example that builds a part refuses. That refusal is the package working
+correctly, and counting it as a broken example would make a bare checkout look
+like a defect while hiding real ones behind it.
+
+What decides is the member's own `why_not`, which is the same sentence its doctor
+prints. A member that publishes none skips nothing, and on a machine that has the
+files nothing is skipped either, so the check keeps its teeth exactly where it
+had them.
 """
 
 from __future__ import annotations
 
+import importlib
 import re
 import subprocess
 import sys
 import unittest
 from pathlib import Path
+from types import ModuleType
 
 ROOT = Path(__file__).resolve().parent.parent
 
 README = ROOT / "README.md"
 
+
+def packages() -> list[str]:
+    """The importable package in this repository, which is the member itself.
+
+    The conformance directory is one too and is not the member, so it is left
+    out by name rather than by position.
+    """
+    return sorted(
+        found.name
+        for found in ROOT.iterdir()
+        if (found / "__init__.py").is_file() and found.name != "conformance"
+    )
+
+
+def cannot_run_here(named: list[str] | None = None) -> str | None:
+    """Why this member cannot build its part on this machine, or nothing.
+
+    Read off the package rather than guessed from a traceback, so the sentence
+    an example is excused by is the sentence the member itself publishes.
+
+    The names are a parameter so this can be driven against a member that
+    publishes a reason from a member that does not, and the other way round.
+    Nine of the sixteen publish nothing, and a branch only two of them reach is
+    a branch nobody has seen work.
+    """
+    if str(ROOT) not in sys.path:
+        sys.path.insert(0, str(ROOT))
+    for name in packages() if named is None else named:
+        asked = getattr(importlib.import_module(name), "why_not", None)
+        if callable(asked):
+            answer = asked()
+            return str(answer) if answer else None
+    return None
+
+
+def excused(failure: str, reason: str | None = None) -> bool:
+    """Whether that failure is the member saying it has no file to run.
+
+    Matched on a run of the member's own sentence rather than on an exception
+    name, because the name differs per member and the sentence is the thing the
+    member publishes for exactly this purpose.
+    """
+    said = cannot_run_here() if reason is None else reason
+    return bool(said) and str(said)[:40] in failure
+
+
 BLOCK = re.compile(r"^```(\w*)\n(.*?)^```$", re.M | re.S)
 
 
@@ -61,29 +121,38 @@ def ran(source: str) -> subprocess.CompletedProcess[str]:
     )
 
 
-def broken(found: list[tuple[str, str | None]]) -> list[str]:
+def broken(found: list[tuple[str, str | None]], reason: str | None = None) -> list[str]:
     """The last line of the traceback of every example that will not run.
 
     A process can exit non-zero and print nothing, so the reason falls back to
     the exit code rather than indexing an empty list. A checker that raises
     while collecting a fault reports neither that fault nor any after it.
     """
     failed = []
+    reason = cannot_run_here() if reason is None else reason
     for source, _ in found:
         finished = ran(source)
-        if finished.returncode != 0:
-            said = finished.stderr.strip().splitlines()
-            failed.append(said[-1] if said else f"exited {finished.returncode} in silence")
+        if finished.returncode == 0:
+            continue
+        if excused(finished.stderr, reason):
+            continue
+        said = finished.stderr.strip().splitlines()
+        failed.append(said[-1] if said else f"exited {finished.returncode} in silence")
     return failed
 
 
-def mismatched(found: list[tuple[str, str | None]]) -> list[tuple[str, str]]:
+def mismatched(
+    found: list[tuple[str, str | None]], reason: str | None = None
+) -> list[tuple[str, str]]:
     """What the readme claims each example prints, beside what it printed."""
     wrong = []
+    reason = cannot_run_here() if reason is None else reason
     for source, expected in found:
         if expected is None:
             continue
         finished = ran(source)
+        if finished.returncode != 0 and excused(finished.stderr, reason):
+            continue
         if finished.stdout != expected:
             wrong.append((expected.strip(), finished.stdout.strip()))
     return wrong
@@ -137,6 +206,72 @@ def test_and_a_later_example_is_still_checked_after_an_earlier_one_failed(self)
 
         self.assertEqual(mismatched(examples(readme)), [("2", "1")])
 
+    def test_a_member_that_can_run_everything_excuses_nothing(self) -> None:
+        """The teeth stay where they were on every member that ships its own part."""
+        self.assertFalse(excused("anything at all", None if cannot_run_here() else "x" * 60))
+
+    def test_a_failure_the_member_says_it_expects_is_excused(self) -> None:
+        reason = "no firmware image was found: this backend runs the part's own microcode"
+
+        self.assertTrue(excused(f"Traceback\nNoFirmware: {reason}", reason))
+
+    def test_and_any_other_failure_is_not(self) -> None:
+        """Driven against the shape that would otherwise slip through."""
+        reason = "no firmware image was found: this backend runs the part's own microcode"
+
+        self.assertFalse(excused("Traceback\nZeroDivisionError: division by zero", reason))
+
+    def test_and_a_member_that_publishes_no_reason_excuses_nothing(self) -> None:
+        self.assertFalse(excused("Traceback\nNoFirmware: anything", ""))
+
+    def test_an_example_that_only_this_machine_can_run_is_reported_as_broken(self) -> None:
+        """So the excuse cannot be claimed by an example that simply does not work."""
+        readme = "```python\nraise ValueError('nope')\n```\n"
+
+        self.assertEqual(broken(examples(readme)), ["ValueError: nope"])
+
+    def test_an_example_the_member_says_it_cannot_run_is_not_reported_as_broken(self) -> None:
+        readme = "```python\nraise SystemExit('no image is here')\n```\n"
+
+        self.assertEqual(broken(examples(readme), "no image is here"), [])
+
+    def test_and_its_stated_output_is_not_compared_either(self) -> None:
+        """An example that never ran produced no output to compare."""
+        readme = "```python\nraise SystemExit('no image is here')\n```\n\n```\n7\n```\n"
+
+        self.assertEqual(mismatched(examples(readme), "no image is here"), [])
+
+    def test_a_member_that_publishes_a_reason_is_read(self) -> None:
+        """Driven against a stand-in, because nine of the sixteen publish none."""
+        speaking = ModuleType("speaking")
+        speaking.why_not = lambda: "no image is here"  # type: ignore[attr-defined]
+        sys.modules["speaking"] = speaking
+        try:
+            self.assertEqual(cannot_run_here(["speaking"]), "no image is here")
+        finally:
+            del sys.modules["speaking"]
+
+    def test_and_one_that_publishes_nothing_to_say_says_nothing(self) -> None:
+        quiet = ModuleType("quiet")
+        quiet.why_not = lambda: None  # type: ignore[attr-defined]
+        sys.modules["quiet"] = quiet
+        try:
+            self.assertIsNone(cannot_run_here(["quiet"]))
+        finally:
+            del sys.modules["quiet"]
+
+    def test_and_one_that_publishes_no_such_call_is_passed_over(self) -> None:
+        silent = ModuleType("silent")
+        sys.modules["silent"] = silent
+        try:
+            self.assertIsNone(cannot_run_here(["silent"]))
+        finally:
+            del sys.modules["silent"]
+
+    def test_the_member_this_repository_holds_is_found_by_name(self) -> None:
+        """So the sweep cannot start reading the conformance directory instead."""
+        self.assertNotIn("conformance", packages())
+
     def test_an_example_with_no_stated_output_is_only_run(self) -> None:
         readme = "```python\nprint(1)\n```\n"
 

conformance/speed.py

@@ -31,7 +31,7 @@
 import snesdsp
 
 if TYPE_CHECKING:
-    from collections.abc import Sequence
+    from collections.abc import Callable, Sequence
 
 FLOOR = 150_000
 """Steps per second this must beat, an order of magnitude below what it does.
@@ -82,8 +82,14 @@ def beats(self, floor: int) -> bool:
         return self.rate() >= floor
 
 
-def measure(calls: int = CALLS, repeats: int = REPEATS) -> Timed:
-    """Step the part through its own microcode, and time it."""
+def measure(calls: int = CALLS, repeats: int = REPEATS) -> Timed:  # pragma: no cover
+    """Step the part through its own microcode, and time it.
+
+    Measured out of the coverage gate on purpose, for the same reason the checks
+    that drive real microcode are: it needs a program nobody may distribute, so a
+    machine that has one runs a path a machine without one cannot, and a gate
+    that demands the impossible gets switched off rather than met.
+    """
     part = snesdsp.Chip(MODEL)
     seconds = []
     for _ in range(repeats):
@@ -106,12 +112,29 @@ def lines_for(found: Timed, floor: int = FLOOR) -> list[str]:
     return lines
 
 
-def main(calls: int = CALLS, repeats: int = REPEATS, floor: int = FLOOR) -> int:
-    missing = snesdsp.why_not()
-    if missing is not None:
-        print(f"  nothing measured: {missing}")
-        return 0
-    found = measure(calls, repeats)
+def main(
+    calls: int = CALLS,
+    repeats: int = REPEATS,
+    floor: int = FLOOR,
+    taken: Callable[[int, int], Timed] | None = None,
+) -> int:
+    """Measure, report, and say whether the floor was beaten.
+
+    `taken` is a parameter so the report can be checked on a machine that cannot
+    measure. The part runs a program this repository is not allowed to carry, so
+    a test that measures for real passes where the program is present and
+    reports success on a runner that measured nothing at all.
+
+    A machine with no program is told so and the run succeeds, because a fresh
+    checkout is not a regression. That path is skipped when a measurement was
+    handed in, since a caller who supplied one is not asking this machine.
+    """
+    if taken is None:
+        missing = snesdsp.why_not()
+        if missing is not None:
+            print(f"  nothing measured: {missing}")
+            return 0
+    found = (measure if taken is None else taken)(calls, repeats)
     for line in lines_for(found, floor):
         print(line)
     return 0 if found.beats(floor) else 1

conformance/speed.test.py

@@ -71,8 +71,17 @@ def run_main(self, **changes: object) -> tuple[int, str]:
             code = speed.main(**changes)  # type: ignore[arg-type]
         return code, captured.getvalue()
 
+    def measuring(self, rate: float) -> object:
+        """A stand-in measurement, so the report is checked on every machine.
+
+        Calling the real one needs the program the part runs, which this
+        repository is not allowed to carry, so a test that measures for real
+        passes here and reports success on a runner that measured nothing.
+        """
+        return lambda calls, repeats: speed.Timed("step", int(rate), [1.0])
+
     def test_a_run_that_beats_the_floor_reports_success(self) -> None:
-        code, output = self.run_main(repeats=1, calls=200, floor=1)
+        code, output = self.run_main(floor=1, taken=self.measuring(1_000))
 
         self.assertEqual(code, 0)
         self.assertIn("step", output)
@@ -96,7 +105,7 @@ def test_a_machine_with_no_microcode_measures_nothing_and_says_so(self) -> None:
         self.assertIn("nothing measured", output)
 
     def test_a_floor_nothing_could_beat_fails_the_run(self) -> None:
-        code, output = self.run_main(repeats=1, calls=200, floor=10**12)
+        code, output = self.run_main(floor=10**12, taken=self.measuring(1_000))
 
         self.assertEqual(code, 1)
         self.assertIn("below", output)
@@ -123,5 +132,29 @@ def test_a_run_at_exactly_the_shipped_floor_is_not_below_it(self) -> None:
         self.assertIn(f"{speed.FLOOR:,}", "\n".join(speed.lines_for(exactly, speed.FLOOR)))
 
 
+class MeasuredHereTest(unittest.TestCase):
+    """The real measurement, on a machine that has the program to measure.
+
+    Skipped out loud where the program is absent rather than quietly passing,
+    which is what the rest of this family does with every check that needs a
+    file it may not carry.
+    """
+
+    @unittest.skipIf(snesdsp.why_not() is not None, "no microcode is on this machine")
+    def test_the_part_is_measured_and_beats_its_own_floor(self) -> None:
+        found = speed.measure(calls=200, repeats=1)
+
+        self.assertGreater(found.rate(), speed.FLOOR)
+
+    @unittest.skipIf(snesdsp.why_not() is not None, "no microcode is on this machine")
+    def test_and_a_run_with_nothing_handed_in_measures_for_itself(self) -> None:
+        captured = io.StringIO()
+        with contextlib.redirect_stdout(captured):
+            code = speed.main(calls=200, repeats=1, floor=1)
+
+        self.assertEqual(code, 0)
+        self.assertIn("per second", captured.getvalue())
+
+
 if __name__ == "__main__":
     unittest.main()