diff --git a/.github/actions/compose/action.yml b/.github/actions/compose/action.yml deleted file mode 100644 index a4be9fd..0000000 --- a/.github/actions/compose/action.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Docker Compose -description: >- - Ensure the Docker Compose v2 plugin. GitHub's runner images ship it; the lab's self-hosted runner image has only the - Docker CLI, so there a pinned release is installed for the runner user and checked against its published SHA-256. -runs: - using: composite - steps: - - name: Ensure the Docker Compose plugin - shell: bash - env: - COMPOSE_VERSION: v5.5.1 - COMPOSE_SHA256: db1889184726840f75c4f9c001048430d4f25b3be3cb084d3ddd762bc0aed576 - run: | - if ! docker compose version >/dev/null 2>&1; then - if [ "$(uname -m)" != x86_64 ]; then - echo "::error::Docker Compose is missing and only the x86_64 release is pinned; this runner is $(uname -m)" - exit 1 - fi - plugins="$HOME/.docker/cli-plugins" - mkdir -p "$plugins" - curl -fsSL --retry 3 -o "$plugins/docker-compose" \ - "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-linux-x86_64" - echo "$COMPOSE_SHA256 $plugins/docker-compose" | sha256sum -c - - chmod +x "$plugins/docker-compose" - fi - docker compose version diff --git a/.github/workflows/model-smoke.yml b/.github/workflows/model-smoke.yml index cadaec0..799865c 100644 --- a/.github/workflows/model-smoke.yml +++ b/.github/workflows/model-smoke.yml @@ -38,7 +38,6 @@ on: - tests/model_cases.py - docker/compose.yaml - .github/workflows/model-smoke.yml - - .github/actions/compose/** permissions: contents: read @@ -48,12 +47,10 @@ concurrency: group: model-smoke-${{ github.ref }} cancel-in-progress: true -# The lab's self-hosted runner scale set that the CI_RUNNER repository variable names, or GitHub's when it is unset. -# As written here, a fork's pull request goes to GitHub's disposable runners; a fork can edit this file, so the fork -# approval policy is what keeps its code off the lab's runner (docs/security.md). +# GitHub's hosted runners, as in tests.yml: the repository is public. jobs: model: - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.CI_RUNNER || 'ubuntu-latest' }} + runs-on: ubuntu-latest timeout-minutes: 35 env: BRIDGE_TOKEN: test-only-bridge-token-never-use-in-production @@ -61,7 +58,6 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: persist-credentials: false - - uses: ./.github/actions/compose - name: Prepare disposable cache run: mkdir -p .cache && chmod 777 .cache - name: Inspect acquisition and execution limits @@ -90,7 +86,7 @@ jobs: shell: bash env: BENCH_COMMIT: ${{ github.sha }} - BENCH_RUNNER: ${{ runner.environment == 'self-hosted' && format('self-hosted {0}', vars.CI_RUNNER) || 'GitHub-hosted ubuntu-latest' }} + BENCH_RUNNER: GitHub-hosted ubuntu-latest run: | docker compose -f docker/compose.yaml up -d worker docker compose -f docker/compose.yaml run --rm --no-deps -T model-bench | tee benchmark.md @@ -100,7 +96,7 @@ jobs: shell: bash env: BENCH_COMMIT: ${{ github.sha }} - BENCH_RUNNER: ${{ runner.environment == 'self-hosted' && format('self-hosted {0}', vars.CI_RUNNER) || 'GitHub-hosted ubuntu-latest' }} + BENCH_RUNNER: GitHub-hosted ubuntu-latest run: | docker compose -f docker/compose.yaml run --rm --no-deps -T model-ner | tee ner.md { echo "## NER precision and recall (informational)"; echo; cat ner.md; } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 088d972..b938f71 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,14 +9,12 @@ on: permissions: contents: read -# Every job runs on the lab's self-hosted runner scale set that the CI_RUNNER repository variable names, or on -# GitHub's when it is unset. As written here, a fork's pull request goes to GitHub's disposable runners, since the -# repository is public; a fork can edit this file in its pull request, so the fork approval policy is what keeps -# its code off the lab's runner (docs/security.md). +# The repository is public, so every job runs on GitHub's hosted runners, which GitHub does not meter for public +# repositories and which are disposable: a fork's pull request runs its own code on a runner that is thrown away. jobs: php: name: PHP ${{ matrix.php }} - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.CI_RUNNER || 'ubuntu-latest' }} + runs-on: ubuntu-latest timeout-minutes: 12 strategy: fail-fast: false @@ -38,7 +36,6 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: persist-credentials: false - - uses: ./.github/actions/compose - name: Inspect sandbox configuration run: docker compose -f docker/compose.yaml --profile test config - name: Unit and syntax tests @@ -61,7 +58,7 @@ jobs: mcp: name: MCP server - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.CI_RUNNER || 'ubuntu-latest' }} + runs-on: ubuntu-latest timeout-minutes: 12 env: BRIDGE_TOKEN: test-only-bridge-token-never-use-in-production @@ -69,7 +66,6 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: persist-credentials: false - - uses: ./.github/actions/compose - name: Prepare disposable cache run: mkdir -p .cache && chmod 777 .cache - name: Inspect sandbox configuration @@ -86,7 +82,7 @@ jobs: deterministic: if: always() needs: [php, mcp] - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.CI_RUNNER || 'ubuntu-latest' }} + runs-on: ubuntu-latest timeout-minutes: 1 steps: - name: Require every supported PHP version and the MCP server to pass diff --git a/README.md b/README.md index 16e6a0d..71593a8 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ docker compose -f docker/compose.yaml run --rm --no-deps php-tests sh -ec \ Coverage includes invalid contracts, duplicate/non-finite JSON, authentication, oversized bodies, queue deadlines, queued/running cancellation, hard process crashes, retention, malformed/oversized HTTP responses, redirect refusal, and reused-worker request isolation. Fault tasks are enabled only in the test service with `BRIDGE_TEST_TASKS=1`. -CI runs deterministic tests for pushes to main and for pull requests across PHP 8.2–8.5, treating PHP warnings and deprecations as test failures. Every job runs on the self-hosted runner scale set that the `CI_RUNNER` repository variable names (in the maintainer's lab, Actions Runner Controller with Docker-in-Docker), or on GitHub's runners when it is unset; the workflows send a fork's pull request to GitHub's disposable runners, because the repository is public and a self-hosted runner would execute the fork's code. A fork can edit the workflows in its own pull request, so with `CI_RUNNER` set, require approval for every outside contributor's workflow runs; see [the runner notes](docs/testing.md#ci-runners) and [security](docs/security.md). Where a runner lacks the Docker Compose plugin, a local action installs a pinned release checked by SHA-256. The stable `deterministic` check succeeds only if every PHP matrix job and the offline MCP server suite succeed. A separate workflow performs model acquisition and the real-model smoke tests on the default PHP 8.5 image: weekly, on pull requests that touch the model path, and on demand. It is not a required check, it does not use paid APIs or GPUs, and it also prints an informational latency table: its jobs must succeed, but its times never gate the run. +CI runs deterministic tests for pushes to main and for pull requests across PHP 8.2–8.5, treating PHP warnings and deprecations as test failures. Every job runs on GitHub's hosted runners (`ubuntu-latest`), because the repository is public: GitHub does not meter Actions minutes for public repositories, and a fork's pull request runs its own code on a disposable runner of GitHub's, not on a machine of the maintainer's; see [the runner notes](docs/testing.md#ci-runners) and [security](docs/security.md). The stable `deterministic` check succeeds only if every PHP matrix job and the offline MCP server suite succeed. A separate workflow performs model acquisition and the real-model smoke tests on the default PHP 8.5 image: weekly, on pull requests that touch the model path, and on demand. It is not a required check, it does not use paid APIs or GPUs, and it also prints an informational latency table: its jobs must succeed, but its times never gate the run. ## Important limits diff --git a/docs/security.md b/docs/security.md index b1bd9fa..8e8ab1f 100644 --- a/docs/security.md +++ b/docs/security.md @@ -7,7 +7,7 @@ This experimental service is designed for trusted application-to-service traffic - Only trusted task implementations are allowed. Python child processes inherit the service environment; they are not suitable for arbitrary user-provided scripts. - Container configuration uses a non-root user, read-only filesystem, dropped capabilities, no-new-privileges, CPU/memory/PID bounds, and an internal-only network for execution. It mounts only this checkout and the optional dependency/model cache, never a Docker socket or home directory. - Dependency acquisition is separate and internet-enabled. Execution installs hash-checked wheels from the local cache without internet access. The model downloader retrieves fixed files at a pinned revision and checks their hashes; it does not execute model-repository code. -- CI may run on a self-hosted runner, which executes a workflow's code with that runner's network access, and the lab's runs a privileged Docker-in-Docker daemon. The workflows as written send a fork's pull request to GitHub's disposable runners, but a fork can edit them in its own pull request and name the self-hosted runner, and GitHub runs a pull request's workflows as the pull request has them. The boundary is therefore the repository's fork pull request approval policy: with a self-hosted runner configured, require approval for all outside contributors, not only first-time ones, and review a fork's workflow changes before approving its run. The runner installs Docker Compose from a pinned release checked by SHA-256 when its image lacks it. +- CI runs on GitHub's hosted runners only, because the repository is public. GitHub runs a pull request's workflows as the pull request has them, so a fork's pull request executes its own code, workflow edits included; on a disposable runner of GitHub's that holds nothing of the maintainer's, that is the intended boundary. No workflow names a self-hosted runner, and `tests/python/test_workflows.py` fails if one does again; a fork can edit that test along with the workflows, so while any self-hosted runner is still registered for the repository, keep requiring approval for outside contributors' workflow runs, and unregister it so that there is none a pull request could route its code onto. The `actions/checkout` step is pinned by commit hash and does not persist credentials. - Model files, native libraries and runtime dependencies still carry supply-chain risk. Review upgrades, regenerate locks deliberately and rerun tests before changing pins. - Requests and results may contain sensitive documents. They live in RAM, may be present in process/core dumps, and are not encrypted by this service. Minimize retention and use appropriate host/storage policies. Embedding vectors are derived from their input text and can reveal much of it; store and transmit them with the same care as the text. - Redaction is assistive, not a compliance control. The rules find only the structured identifiers they recognise, and the optional named-entity models were trained on English news, so their recall on names from other domains or languages is lower and unmeasured here; review redacted output before treating it as free of personal data. The model files are third-party ONNX conversions of the original weights, pinned by revision and hash but not signed by the original author. diff --git a/docs/testing.md b/docs/testing.md index 557731b..a8dd894 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -36,9 +36,9 @@ PowerShell uses `$env:BRIDGE_PHP_IMAGE = '...'` instead of `export`. Set a dispo ### CI runners -CI runs on the self-hosted runner scale set that the `CI_RUNNER` repository variable names, and on GitHub's runners when it is unset. As written, the workflows send a fork's pull request to GitHub's disposable runners whatever the variable names, but a fork can edit them in its own pull request and name the lab's scale set, so the boundary that holds is the repository's fork pull request approval: with `CI_RUNNER` set, require approval for all outside contributors and read a fork's workflow changes before approving. Jobs that drive compose first ensure the Docker Compose plugin from a pinned, hash-checked release, since the lab's runner image has only the Docker CLI. `tests/python/test_workflows.py` holds that routing and that install in place, and the measurements below name the runner they ran on. +CI runs on GitHub's hosted runners, `ubuntu-latest` for every job, because the repository is public: GitHub does not meter Actions minutes for public repositories, and a fork's pull request then runs its own code, workflow edits included, on one of GitHub's disposable runners and nowhere else. No workflow names a self-hosted runner and no repository variable chooses one; keep requiring approval for outside contributors' workflow runs until no self-hosted runner is registered for the repository at all, since a fork's pull request can edit the workflows and the test alike. `tests/python/test_workflows.py` fails if a `runs-on` expression, a runner variable, a self-hosted label or a local action under `.github/actions` comes back. GitHub's runner images ship the Docker Compose plugin, so the jobs drive compose directly. The measurements below name the runner they ran on. -A self-hosted runner needs room for a job's containers in its Docker daemon. The model workflow keeps both model workers up, each allowed 1536 MiB with its wheels in tmpfs, beside FrankenPHP, the MCP server, the demo worker and one client, whose limits add up to about 5 GiB, so give the daemon 6 GiB; the lab's Docker-in-Docker template allows 1536 MiB. Every job on an ephemeral runner starts with an empty image store and pulls `python:3.12-bookworm` and the FrankenPHP images from Docker Hub, which limits unauthenticated pulls per IP; a pull-through registry mirror for the runners' daemons avoids that limit and the repeated downloads. While the runner is offline, same-repository pull requests and the weekly model run wait for it and the required check does not report; delete the `CI_RUNNER` variable to fall back to GitHub's runners. +Every job starts on a fresh runner with an empty image store and pulls `python:3.12-bookworm` and the FrankenPHP images from Docker Hub, which limits unauthenticated pulls per IP; GitHub's runners share address ranges, so a pull that fails with a rate-limit error is a rerun, not a regression. The model workflow keeps both model workers up, each allowed 1536 MiB with its wheels in tmpfs, beside FrankenPHP, the MCP server, the demo worker and one client, whose limits add up to about 5 GiB; a Docker daemon that reproduces that workflow locally needs about 6 GiB. CI checks the actual PHP minor before testing so a misconfigured image cannot silently test the default version four times. The stable required check `deterministic` depends on all four matrix jobs and on the offline MCP server job, and rejects any of them failed, cancelled or skipped. PHP contract/HTTP tests promote warnings and deprecations to exceptions. @@ -48,7 +48,7 @@ PHP branch support was verified locally on 2026-09-17 on PHP 8.2.33, 8.3.33, 8.4 | Layer | Scope and counting | | --- | --- | -| Python worker | 128 tests: lifecycle, validation and HTTP, process crashes, cancellation, timeouts, retention, allocation/exit races, redaction rules including IPv6, lower-case IBANs and email addresses in any script and their speed on hostile input, NER aggregation, name joining, the second title-cased reading, the uncased model's reading of the whole text and the merge driven by stubs, the fetcher's pins and the model directories an ONNX worker checks for before it starts, batched cross-encoder scoring, the NER measurement's sample and scoring, the workflows' runner routing and Compose install, progress bounds, and the model-name table every result reports from | +| Python worker | 127 tests: lifecycle, validation and HTTP, process crashes, cancellation, timeouts, retention, allocation/exit races, redaction rules including IPv6, lower-case IBANs and email addresses in any script and their speed on hostile input, NER aggregation, name joining, the second title-cased reading, the uncased model's reading of the whole text and the merge driven by stubs, the fetcher's pins and the model directories an ONNX worker checks for before it starts, batched cross-encoder scoring, the NER measurement's sample and scoring, the workflows' runners, progress bounds, and the model-name table every result reports from | | MCP server | 119 tests, offline with hash-pinned wheels: the protocol client against an in-process worker, every tool through the SDK's in-memory client with test tasks enabled, the entry point as a subprocess, result validators on synthetic data, the index builder (chunking, the format and each corruption it refuses, building against the worker, caching, its command line) `bridge_search` (the model gate, path confinement, reloads, rerank on and off), the reachability of every answer in the LLM evaluation, that a default search sees only part of its corpus and that its two-step question takes two steps, that the demo worker reaches none of the semantic evaluation's answers, that a tool waits out a full worker, before each of a search's jobs and within one budget, and says so when it stays full, the evaluation runner's command, isolation checks, scoring and per-backend launcher and index check without starting `claude`, a check that the server's model names match the worker's, that every path parameter says where it resolves and every position in a result says what it counts, and a check that every timing claim in a tool description is a measured median | | PHP contracts + HTTP | 134 checks per PHP version; this **includes** the 101 contract checks, not 134 + 101 | | FrankenPHP | 12 jobs submitted by six concurrent test threads through a reused PHP worker, a 512-document `top_k` request, embedding and redaction round trips, and invalid-input, missing-job and multibyte boundary checks | diff --git a/tests/python/test_workflows.py b/tests/python/test_workflows.py index 2ddc788..2ccb963 100644 --- a/tests/python/test_workflows.py +++ b/tests/python/test_workflows.py @@ -1,10 +1,9 @@ -"""The workflows run on the lab's self-hosted runner, as written here never for a fork's pull request, and every job -that drives Docker Compose gets the same pinned, hash-checked plugin wherever the runner lacks one. +"""The workflows run on GitHub's hosted runners, every job of them, with no self-hosted runner and no runner variable. -The repository is public, so a fork's pull request would run its own code on the runner it is routed to: these -workflows send it to GitHub's disposable runners. A fork can edit them in its pull request, though, so the boundary -that holds is the repository's fork approval policy, not this routing (docs/security.md). Read as text, since the -standard library has no YAML parser; the workflows keep their jobs two spaces in under `jobs:`. +The repository is public: GitHub does not meter Actions minutes for it, and a fork's pull request runs its own code +on a disposable runner of GitHub's, never on a machine of the maintainer's. Nothing here installs Docker Compose, +since GitHub's images ship the plugin. Read as text, since the standard library has no YAML parser; the workflows +keep their jobs two spaces in under `jobs:`. """ import re @@ -12,10 +11,8 @@ from pathlib import Path REPO = Path(__file__).resolve().parents[2] -WORKFLOWS = sorted((REPO / ".github" / "workflows").glob("*.yml")) -ACTION = REPO / ".github" / "actions" / "compose" / "action.yml" -RUNS_ON = ("runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == " - "github.repository) && vars.CI_RUNNER || 'ubuntu-latest' }}") +WORKFLOWS = sorted(path for ext in ("*.yml", "*.yaml") for path in (REPO / ".github" / "workflows").glob(ext)) +RUNS_ON = "runs-on: ubuntu-latest" def jobs(text: str) -> dict: @@ -39,51 +36,49 @@ def test_the_workflows_are_found(self): for path in WORKFLOWS: self.assertTrue(jobs(path.read_text(encoding="utf-8")), path.name) - def test_every_job_runs_on_the_lab_runner_unless_a_fork_asks(self): + def test_every_job_runs_on_github_hosted_ubuntu(self): for path in WORKFLOWS: for name, lines in jobs(path.read_text(encoding="utf-8")).items(): with self.subTest(workflow=path.name, job=name): self.assertEqual([line for line in lines if line.startswith("runs-on:")], [RUNS_ON]) - def test_every_job_that_runs_compose_ensures_the_plugin_after_checkout_and_before_compose(self): + def test_no_workflow_names_a_self_hosted_runner_or_a_runner_variable(self): + # A `runs-on` expression is how a self-hosted runner or a fallback to one would come back. + for path in WORKFLOWS: + text = path.read_text(encoding="utf-8") + with self.subTest(workflow=path.name): + self.assertNotIn("vars.CI_RUNNER", text) + self.assertNotIn("runner.environment", text) + self.assertNotRegex(text, r"(?i)self-hosted") + self.assertNotRegex(text, r"runs-on: *\$\{\{") + self.assertNotRegex(text, r"runs-on: *\[") + + def test_no_workflow_installs_compose_through_a_local_action(self): + # GitHub's runner images ship the Docker Compose plugin; the lab-only install action is gone with the lab. + self.assertFalse((REPO / ".github" / "actions").exists()) + for path in WORKFLOWS: + with self.subTest(workflow=path.name): + self.assertNotIn(".github/actions/", path.read_text(encoding="utf-8")) + + def test_every_job_that_runs_compose_checks_the_repository_out_first(self): for path in WORKFLOWS: for name, lines in jobs(path.read_text(encoding="utf-8")).items(): commands = [i for i, line in enumerate(lines) if "docker compose" in line and not line.startswith("#")] if not commands: continue with self.subTest(workflow=path.name, job=name): - ensure = lines.index("- uses: ./.github/actions/compose") checkout = next(i for i, line in enumerate(lines) if line.startswith("- uses: actions/checkout@")) - self.assertLess(checkout, ensure) - self.assertLess(ensure, commands[0]) - - def test_the_compose_plugin_is_a_pinned_release_checked_by_hash_and_installed_only_where_missing(self): - text = ACTION.read_text(encoding="utf-8") - self.assertRegex(text, r"COMPOSE_VERSION: v\d+\.\d+\.\d+\n") - self.assertRegex(text, r"COMPOSE_SHA256: [0-9a-f]{64}\n") - self.assertIn('| sha256sum -c -', text) - self.assertIn("if ! docker compose version", text) - self.assertIn("using: composite", text) + self.assertLess(checkout, commands[0]) def test_tests_run_for_pull_requests_and_for_pushes_to_main_only(self): - # A push to a pull request's branch would otherwise run every job twice on a one-runner lab. + # A push to a pull request's branch would otherwise run every job twice. text = (REPO / ".github" / "workflows" / "tests.yml").read_text(encoding="utf-8") self.assertIn("on:\n push:\n branches: [main]\n pull_request:\n", text) - def test_the_model_workflow_runs_when_the_compose_action_changes(self): - text = (REPO / ".github" / "workflows" / "model-smoke.yml").read_text(encoding="utf-8") - self.assertIn(" - .github/actions/compose/**\n", text) - - def test_a_runner_on_another_architecture_is_told_why_it_fails(self): - text = ACTION.read_text(encoding="utf-8") - self.assertIn("::error::", text) - self.assertNotIn('\n test "$(uname -m)" = x86_64\n', text) - def test_measurements_name_the_runner_they_ran_on(self): text = (REPO / ".github" / "workflows" / "model-smoke.yml").read_text(encoding="utf-8") - self.assertFalse("BENCH_RUNNER: GitHub-hosted ubuntu-latest" in text, "a measurement names a runner it may not use") - self.assertEqual(text.count("BENCH_RUNNER: ${{ runner.environment == 'self-hosted' && " - "format('self-hosted {0}', vars.CI_RUNNER) || 'GitHub-hosted ubuntu-latest' }}"), 2) + self.assertEqual(text.count("BENCH_RUNNER: GitHub-hosted ubuntu-latest\n"), 2) + self.assertEqual(text.count("BENCH_RUNNER:"), 2) if __name__ == "__main__":