From a7ec4907fd3d1ef933a9ba1f39fecab008c27de5 Mon Sep 17 00:00:00 2001 From: Cristian Girlea Date: Thu, 15 Oct 2026 12:11:47 +0300 Subject: [PATCH] test: make the job reader fail on a header it does not recognise jobs() in test_workflows.py reads the workflows as text and dropped the lines of any job whose header it did not recognise, so a first job written ` php: # matrix` or ` "php":` was never checked by the per-job tests and nothing said so. The header now accepts a quoted key and a trailing comment, and a new test counts the `runs-on:` lines of each file against the jobs the reader found, so a header it still does not know fails the test instead of hiding the job. A fixture test covers the header shapes it must tell apart, two it does not know (mismatched quotes, an anchor), a top-level key after the jobs, and the count tripping on them. Seen failing with the old reader on tests.yml: a commented or a quoted first-job header left two jobs against three `runs-on:` lines, and an anchored header did the same; with the new reader the first two pass and the anchored one fails the count. docs/testing.md counts the two new tests. Closes #36 --- docs/testing.md | 2 +- tests/python/test_workflows.py | 56 ++++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 1ae6ef5..7aa0ffc 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -48,7 +48,7 @@ PHP branch support was verified locally on 2026-09-17 on PHP 8.2.33, 8.3.33, 8.4 | Layer | Scope and counting | | --- | --- | -| Python worker | 154 tests: lifecycle, validation and HTTP, process crashes, cancellation, timeouts, retention, allocation/exit races, the warm pool (that successive jobs share a child in which no live container or frame holds a reference to a finished job's input or result, after a success, a failure and real task payloads, with the detector checked against a planted needle; that a cancelled, timed-out or crashed child is replaced; that a child is recycled after its share of jobs; that the service token is absent from a child's environment and from its environment block when the service started it; that a message for an earlier job lands nowhere; that a child that never reports ready is replaced and that children dying at start are not restarted in a loop; and that a cancellation does not hold up reads), redaction rules including IPv6, lower-case IBANs and email addresses in any script and their speed on hostile input, NER aggregation, name joining, the second title-cased reading, the uncased model's reading of the whole text and the merge driven by stubs, the fetcher's pins and the model directories an ONNX worker checks for before it starts, batched cross-encoder scoring, the NER measurement's sample and scoring, the workflows' runners, action pins and checkout credentials, both Claude Code hooks (the post-edit rules, and the pre-call push guard through its own cases), progress bounds, and the model-name table every result reports from | +| Python worker | 156 tests: lifecycle, validation and HTTP, process crashes, cancellation, timeouts, retention, allocation/exit races, the warm pool (that successive jobs share a child in which no live container or frame holds a reference to a finished job's input or result, after a success, a failure and real task payloads, with the detector checked against a planted needle; that a cancelled, timed-out or crashed child is replaced; that a child is recycled after its share of jobs; that the service token is absent from a child's environment and from its environment block when the service started it; that a message for an earlier job lands nowhere; that a child that never reports ready is replaced and that children dying at start are not restarted in a loop; and that a cancellation does not hold up reads), redaction rules including IPv6, lower-case IBANs and email addresses in any script and their speed on hostile input, NER aggregation, name joining, the second title-cased reading, the uncased model's reading of the whole text and the merge driven by stubs, the fetcher's pins and the model directories an ONNX worker checks for before it starts, batched cross-encoder scoring, the NER measurement's sample and scoring, the workflows' runners, action pins and checkout credentials, both Claude Code hooks (the post-edit rules, and the pre-call push guard through its own cases), progress bounds, and the model-name table every result reports from | | MCP server | 119 tests, offline with hash-pinned wheels: the protocol client against an in-process worker, every tool through the SDK's in-memory client with test tasks enabled, the entry point as a subprocess, result validators on synthetic data, the index builder (chunking, the format and each corruption it refuses, building against the worker, caching, its command line) `bridge_search` (the model gate, path confinement, reloads, rerank on and off), the reachability of every answer in the LLM evaluation, that a default search sees only part of its corpus and that its two-step question takes two steps, that the demo worker reaches none of the semantic evaluation's answers, that a tool waits out a full worker, before each of a search's jobs and within one budget, and says so when it stays full, the evaluation runner's command, isolation checks, scoring and per-backend launcher and index check without starting `claude`, a check that the server's model names match the worker's, that every path parameter says where it resolves and every position in a result says what it counts, and a check that every timing claim in a tool description is a measured median | | PHP contracts + HTTP | 134 checks per PHP version; this **includes** the 101 contract checks, not 134 + 101 | | FrankenPHP | 12 jobs submitted by six concurrent test threads through a reused PHP worker, a 512-document `top_k` request, embedding and redaction round trips, and invalid-input, missing-job and multibyte boundary checks | diff --git a/tests/python/test_workflows.py b/tests/python/test_workflows.py index a7ab4ff..310c2f3 100644 --- a/tests/python/test_workflows.py +++ b/tests/python/test_workflows.py @@ -25,12 +25,17 @@ def jobs(text: str) -> dict: - """Each job's name and its lines.""" + """Each job's name and its lines. + + A job's header is its key two spaces in, quoted or not, with nothing after the colon but a comment. A header + this does not recognise drops the lines of a first job, or adds a later job's lines to the job before it, + which is why the number of jobs read is checked against the `runs-on:` lines of the file. + """ found, name = {}, None for line in text.split("\njobs:\n", 1)[1].splitlines(): - header = re.match(r"^ ([A-Za-z0-9_-]+):\s*$", line) + header = re.match(r"""^ (['"]?)([A-Za-z0-9_-]+)\1:\s*(?:#.*)?$""", line) if header: - name = header.group(1) + name = header.group(2) found[name] = [] elif name and line.startswith(" "): found[name].append(line.strip()) @@ -39,6 +44,11 @@ def jobs(text: str) -> dict: return found +def runs_on_lines(text: str) -> int: + """How many `runs-on:` lines the file has, wherever they are: the number of jobs the reader should have found.""" + return len(re.findall(r"^\s*runs-on:", text, re.MULTILINE)) + + def steps(text: str) -> list: """Every step of every job, in file order, each the list of its lines with their indentation kept. @@ -91,6 +101,46 @@ def test_the_workflows_are_found(self): for path in WORKFLOWS: self.assertTrue(jobs(path.read_text(encoding="utf-8")), path.name) + def test_the_job_reader_sees_every_job(self): + # Every job of these workflows runs on a runner, so it has exactly one `runs-on:` line: a job whose header + # the reader did not recognise shows up as a `runs-on:` line without a job, not as a job that was never + # checked. (A job that calls a reusable workflow has none; add it to the count if one appears.) + for path in WORKFLOWS: + text = path.read_text(encoding="utf-8") + with self.subTest(workflow=path.name): + self.assertEqual(len(jobs(text)), runs_on_lines(text)) + + def test_the_job_reader_on_the_headers_it_must_tell_apart(self): + text = """name: x +on: push +jobs: + php: # the matrix + runs-on: ubuntu-latest + steps: + - run: echo php + "mcp": + runs-on: ubuntu-latest + 'gate': + needs: [php, mcp] + runs-on: ubuntu-latest + # a comment between jobs + "odd': + runs-on: ubuntu-latest + smoke: &anchored + runs-on: ubuntu-latest +env: + runs-on: not-a-job +""" + found = jobs(text) + # Mismatched quotes and an anchor are shapes the reader does not know; the count is what tells, since a + # job it misses leaves its `runs-on:` line in the file. The `env:` line after the jobs is counted too: a + # false failure at worst, never a job passed over. + self.assertEqual(list(found), ["php", "mcp", "gate"]) + self.assertEqual(found["php"], ["runs-on: ubuntu-latest", "steps:", "- run: echo php"]) + self.assertEqual(found["gate"][:2], ["needs: [php, mcp]", "runs-on: ubuntu-latest"]) + self.assertEqual(runs_on_lines(text), 6) + self.assertNotEqual(len(found), runs_on_lines(text)) + def test_every_job_runs_on_github_hosted_ubuntu(self): for path in WORKFLOWS: for name, lines in jobs(path.read_text(encoding="utf-8")).items():