test: make the job reader fail on a header it does not recognise

docs/testing.md

@@ -48,7 +48,7 @@ PHP branch support was verified locally on 2026-09-17 on PHP 8.2.33, 8.3.33, 8.4
 
 | Layer | Scope and counting |
 | --- | --- |
-| Python worker | 154 tests: lifecycle, validation and HTTP, process crashes, cancellation, timeouts, retention, allocation/exit races, the warm pool (that successive jobs share a child in which no live container or frame holds a reference to a finished job's input or result, after a success, a failure and real task payloads, with the detector checked against a planted needle; that a cancelled, timed-out or crashed child is replaced; that a child is recycled after its share of jobs; that the service token is absent from a child's environment and from its environment block when the service started it; that a message for an earlier job lands nowhere; that a child that never reports ready is replaced and that children dying at start are not restarted in a loop; and that a cancellation does not hold up reads), redaction rules including IPv6, lower-case IBANs and email addresses in any script and their speed on hostile input, NER aggregation, name joining, the second title-cased reading, the uncased model's reading of the whole text and the merge driven by stubs, the fetcher's pins and the model directories an ONNX worker checks for before it starts, batched cross-encoder scoring, the NER measurement's sample and scoring, the workflows' runners, action pins and checkout credentials, both Claude Code hooks (the post-edit rules, and the pre-call push guard through its own cases), progress bounds, and the model-name table every result reports from |
+| Python worker | 156 tests: lifecycle, validation and HTTP, process crashes, cancellation, timeouts, retention, allocation/exit races, the warm pool (that successive jobs share a child in which no live container or frame holds a reference to a finished job's input or result, after a success, a failure and real task payloads, with the detector checked against a planted needle; that a cancelled, timed-out or crashed child is replaced; that a child is recycled after its share of jobs; that the service token is absent from a child's environment and from its environment block when the service started it; that a message for an earlier job lands nowhere; that a child that never reports ready is replaced and that children dying at start are not restarted in a loop; and that a cancellation does not hold up reads), redaction rules including IPv6, lower-case IBANs and email addresses in any script and their speed on hostile input, NER aggregation, name joining, the second title-cased reading, the uncased model's reading of the whole text and the merge driven by stubs, the fetcher's pins and the model directories an ONNX worker checks for before it starts, batched cross-encoder scoring, the NER measurement's sample and scoring, the workflows' runners, action pins and checkout credentials, both Claude Code hooks (the post-edit rules, and the pre-call push guard through its own cases), progress bounds, and the model-name table every result reports from |
 | MCP server | 119 tests, offline with hash-pinned wheels: the protocol client against an in-process worker, every tool through the SDK's in-memory client with test tasks enabled, the entry point as a subprocess, result validators on synthetic data, the index builder (chunking, the format and each corruption it refuses, building against the worker, caching, its command line) `bridge_search` (the model gate, path confinement, reloads, rerank on and off), the reachability of every answer in the LLM evaluation, that a default search sees only part of its corpus and that its two-step question takes two steps, that the demo worker reaches none of the semantic evaluation's answers, that a tool waits out a full worker, before each of a search's jobs and within one budget, and says so when it stays full, the evaluation runner's command, isolation checks, scoring and per-backend launcher and index check without starting `claude`, a check that the server's model names match the worker's, that every path parameter says where it resolves and every position in a result says what it counts, and a check that every timing claim in a tool description is a measured median |
 | PHP contracts + HTTP | 134 checks per PHP version; this **includes** the 101 contract checks, not 134 + 101 |
 | FrankenPHP | 12 jobs submitted by six concurrent test threads through a reused PHP worker, a 512-document `top_k` request, embedding and redaction round trips, and invalid-input, missing-job and multibyte boundary checks |

tests/python/test_workflows.py

@@ -25,12 +25,17 @@
 
 
 def jobs(text: str) -> dict:
-    """Each job's name and its lines."""
+    """Each job's name and its lines.
+
+    A job's header is its key two spaces in, quoted or not, with nothing after the colon but a comment. A header
+    this does not recognise drops the lines of a first job, or adds a later job's lines to the job before it,
+    which is why the number of jobs read is checked against the `runs-on:` lines of the file.
+    """
     found, name = {}, None
     for line in text.split("\njobs:\n", 1)[1].splitlines():
-        header = re.match(r"^  ([A-Za-z0-9_-]+):\s*$", line)
+        header = re.match(r"""^  (['"]?)([A-Za-z0-9_-]+)\1:\s*(?:#.*)?$""", line)
         if header:
-            name = header.group(1)
+            name = header.group(2)
             found[name] = []
         elif name and line.startswith("    "):
             found[name].append(line.strip())
@@ -39,6 +44,11 @@ def jobs(text: str) -> dict:
     return found
 
 
+def runs_on_lines(text: str) -> int:
+    """How many `runs-on:` lines the file has, wherever they are: the number of jobs the reader should have found."""
+    return len(re.findall(r"^\s*runs-on:", text, re.MULTILINE))
+
+
 def steps(text: str) -> list:
     """Every step of every job, in file order, each the list of its lines with their indentation kept.
 
@@ -91,6 +101,46 @@ def test_the_workflows_are_found(self):
         for path in WORKFLOWS:
             self.assertTrue(jobs(path.read_text(encoding="utf-8")), path.name)
 
+    def test_the_job_reader_sees_every_job(self):
+        # Every job of these workflows runs on a runner, so it has exactly one `runs-on:` line: a job whose header
+        # the reader did not recognise shows up as a `runs-on:` line without a job, not as a job that was never
+        # checked. (A job that calls a reusable workflow has none; add it to the count if one appears.)
+        for path in WORKFLOWS:
+            text = path.read_text(encoding="utf-8")
+            with self.subTest(workflow=path.name):
+                self.assertEqual(len(jobs(text)), runs_on_lines(text))
+
+    def test_the_job_reader_on_the_headers_it_must_tell_apart(self):
+        text = """name: x
+on: push
+jobs:
+  php:  # the matrix
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo php
+  "mcp":
+    runs-on: ubuntu-latest
+  'gate':
+    needs: [php, mcp]
+    runs-on: ubuntu-latest
+  # a comment between jobs
+  "odd':
+    runs-on: ubuntu-latest
+  smoke: &anchored
+    runs-on: ubuntu-latest
+env:
+  runs-on: not-a-job
+"""
+        found = jobs(text)
+        # Mismatched quotes and an anchor are shapes the reader does not know; the count is what tells, since a
+        # job it misses leaves its `runs-on:` line in the file. The `env:` line after the jobs is counted too: a
+        # false failure at worst, never a job passed over.
+        self.assertEqual(list(found), ["php", "mcp", "gate"])
+        self.assertEqual(found["php"], ["runs-on: ubuntu-latest", "steps:", "- run: echo php"])
+        self.assertEqual(found["gate"][:2], ["needs: [php, mcp]", "runs-on: ubuntu-latest"])
+        self.assertEqual(runs_on_lines(text), 6)
+        self.assertNotEqual(len(found), runs_on_lines(text))
+
     def test_every_job_runs_on_github_hosted_ubuntu(self):
         for path in WORKFLOWS:
             for name, lines in jobs(path.read_text(encoding="utf-8")).items():