From 601d6ce37677b218b4dbe14022411972ca9f92ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Mon, 12 Oct 2026 14:33:09 -0300 Subject: [PATCH 1/2] feat(tools): add catalog.py and the stage-A verifier Stdlib-only Python 3.9+ tooling for spec 001 (plan section 3), written from scratch: - catalog.py check/list: parse every benchmark_config.json, resolve Dockerfiles (default, - or the dockerfile override), require a port and a known *_url, normalize database, report test types per stage. - catalog.py changed/lint-docker: changed framework folders since a ref (ALL_TOOLING for tools/, tests/, .github/workflows/) and :latest or untagged FROM images in changed Dockerfiles. - catalog.py verify: docker build, run on a private network, wait for the port (120 s), run the stage-A verifiers, clean up; summarize merges shard summaries. - tools/verify: json, plaintext and common header rules (spec R5) behind one function, verify_variant(). - tests/: unittest suites with fixture catalogs, temporary git repos and a local stub HTTP server. Refs #1 Co-Authored-By: Claude Opus 5.5 (1M context) --- .gitignore | 2 + .../Go/broken-json/benchmark_config.json | 1 + .../Go/nodf/benchmark_config.json | 1 + .../no-urls/Go/nourls/benchmark_config.json | 1 + .../no-urls/Go/nourls/nourls.dockerfile | 1 + .../valid/Go/hello/benchmark_config.json | 29 + .../fixtures/valid/Go/hello/custom.dockerfile | 2 + .../valid/Go/hello/hello-alt.dockerfile | 2 + .../valid/Go/hello/hello-old.dockerfile | 2 + .../fixtures/valid/Go/hello/hello.dockerfile | 3 + .../valid/Python/dbonly/benchmark_config.json | 11 + .../valid/Python/dbonly/dbonly.dockerfile | 2 + tests/helpers.py | 99 +++ tests/test_changed_lint.py | 205 +++++ tests/test_check.py | 98 +++ tests/test_verify_cli.py | 173 +++++ tests/test_verify_rules.py | 162 ++++ tools/catalog.py | 724 ++++++++++++++++++ tools/verify/__init__.py | 124 +++ tools/verify/client.py | 49 ++ tools/verify/rules.py | 189 +++++ 21 files changed, 1880 insertions(+) create mode 100644 .gitignore create mode 100644 tests/fixtures/bad-json/Go/broken-json/benchmark_config.json create mode 100644 tests/fixtures/missing-dockerfile/Go/nodf/benchmark_config.json create mode 100644 tests/fixtures/no-urls/Go/nourls/benchmark_config.json create mode 100644 tests/fixtures/no-urls/Go/nourls/nourls.dockerfile create mode 100644 tests/fixtures/valid/Go/hello/benchmark_config.json create mode 100644 tests/fixtures/valid/Go/hello/custom.dockerfile create mode 100644 tests/fixtures/valid/Go/hello/hello-alt.dockerfile create mode 100644 tests/fixtures/valid/Go/hello/hello-old.dockerfile create mode 100644 tests/fixtures/valid/Go/hello/hello.dockerfile create mode 100644 tests/fixtures/valid/Python/dbonly/benchmark_config.json create mode 100644 tests/fixtures/valid/Python/dbonly/dbonly.dockerfile create mode 100644 tests/helpers.py create mode 100644 tests/test_changed_lint.py create mode 100644 tests/test_check.py create mode 100644 tests/test_verify_cli.py create mode 100644 tests/test_verify_rules.py create mode 100644 tools/catalog.py create mode 100644 tools/verify/__init__.py create mode 100644 tools/verify/client.py create mode 100644 tools/verify/rules.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..7a60b85e1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/tests/fixtures/bad-json/Go/broken-json/benchmark_config.json b/tests/fixtures/bad-json/Go/broken-json/benchmark_config.json new file mode 100644 index 000000000..9d0063ee2 --- /dev/null +++ b/tests/fixtures/bad-json/Go/broken-json/benchmark_config.json @@ -0,0 +1 @@ +{"framework": "broken-json", "tests": [ diff --git a/tests/fixtures/missing-dockerfile/Go/nodf/benchmark_config.json b/tests/fixtures/missing-dockerfile/Go/nodf/benchmark_config.json new file mode 100644 index 000000000..fa20de41d --- /dev/null +++ b/tests/fixtures/missing-dockerfile/Go/nodf/benchmark_config.json @@ -0,0 +1 @@ +{"framework": "nodf", "tests": [{"default": {"json_url": "/json", "port": 8080, "database": "None"}}]} diff --git a/tests/fixtures/no-urls/Go/nourls/benchmark_config.json b/tests/fixtures/no-urls/Go/nourls/benchmark_config.json new file mode 100644 index 000000000..927b2473e --- /dev/null +++ b/tests/fixtures/no-urls/Go/nourls/benchmark_config.json @@ -0,0 +1 @@ +{"framework": "nourls", "tests": [{"default": {"port": 8080, "database": "None"}}]} diff --git a/tests/fixtures/no-urls/Go/nourls/nourls.dockerfile b/tests/fixtures/no-urls/Go/nourls/nourls.dockerfile new file mode 100644 index 000000000..e039b6388 --- /dev/null +++ b/tests/fixtures/no-urls/Go/nourls/nourls.dockerfile @@ -0,0 +1 @@ +FROM golang:1.22 diff --git a/tests/fixtures/valid/Go/hello/benchmark_config.json b/tests/fixtures/valid/Go/hello/benchmark_config.json new file mode 100644 index 000000000..0743528b9 --- /dev/null +++ b/tests/fixtures/valid/Go/hello/benchmark_config.json @@ -0,0 +1,29 @@ +{ + "framework": "hello", + "tests": [{ + "default": { + "json_url": "/json", + "plaintext_url": "/plaintext", + "db_url": "/db", + "port": 8080, + "database": "Postgres" + }, + "alt": { + "json_url": "/json", + "port": "8081", + "database": "MySQL" + }, + "custom": { + "plaintext_url": "/plaintext", + "port": 8080, + "database": "None", + "dockerfile": "custom.dockerfile" + }, + "old": { + "json_url": "/json", + "port": 8080, + "database": "mongodb", + "tags": ["broken"] + } + }] +} diff --git a/tests/fixtures/valid/Go/hello/custom.dockerfile b/tests/fixtures/valid/Go/hello/custom.dockerfile new file mode 100644 index 000000000..beffaf769 --- /dev/null +++ b/tests/fixtures/valid/Go/hello/custom.dockerfile @@ -0,0 +1,2 @@ +FROM scratch +CMD ["/app"] diff --git a/tests/fixtures/valid/Go/hello/hello-alt.dockerfile b/tests/fixtures/valid/Go/hello/hello-alt.dockerfile new file mode 100644 index 000000000..1e30d3c7c --- /dev/null +++ b/tests/fixtures/valid/Go/hello/hello-alt.dockerfile @@ -0,0 +1,2 @@ +FROM golang:1.22 +CMD ["/app"] diff --git a/tests/fixtures/valid/Go/hello/hello-old.dockerfile b/tests/fixtures/valid/Go/hello/hello-old.dockerfile new file mode 100644 index 000000000..1e30d3c7c --- /dev/null +++ b/tests/fixtures/valid/Go/hello/hello-old.dockerfile @@ -0,0 +1,2 @@ +FROM golang:1.22 +CMD ["/app"] diff --git a/tests/fixtures/valid/Go/hello/hello.dockerfile b/tests/fixtures/valid/Go/hello/hello.dockerfile new file mode 100644 index 000000000..630208cde --- /dev/null +++ b/tests/fixtures/valid/Go/hello/hello.dockerfile @@ -0,0 +1,3 @@ +FROM golang:1.22 AS build +FROM build AS final +CMD ["/app"] diff --git a/tests/fixtures/valid/Python/dbonly/benchmark_config.json b/tests/fixtures/valid/Python/dbonly/benchmark_config.json new file mode 100644 index 000000000..934c85425 --- /dev/null +++ b/tests/fixtures/valid/Python/dbonly/benchmark_config.json @@ -0,0 +1,11 @@ +{ + "framework": "dbonly", + "tests": [{ + "default": { + "db_url": "/db", + "fortune_url": "/fortunes", + "port": 8080, + "database": "postgres" + } + }] +} diff --git a/tests/fixtures/valid/Python/dbonly/dbonly.dockerfile b/tests/fixtures/valid/Python/dbonly/dbonly.dockerfile new file mode 100644 index 000000000..44fb02f1b --- /dev/null +++ b/tests/fixtures/valid/Python/dbonly/dbonly.dockerfile @@ -0,0 +1,2 @@ +FROM python:3.12-slim +CMD ["python", "app.py"] diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 000000000..7b3758a36 --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,99 @@ +"""Shared test helpers: import path, fixtures and a stub HTTP server.""" + +from __future__ import annotations + +import contextlib +import email.utils +import io +import itertools +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +TESTS = Path(__file__).resolve().parent +REPO = TESTS.parent +FIXTURES = TESTS / "fixtures" +sys.path.insert(0, str(REPO / "tools")) + +import catalog # noqa: E402,F401 +import verify # noqa: E402,F401 + +_clock = itertools.count(1_700_000_000) + + +def fresh_date() -> str: + """A different, valid IMF-fixdate on every call (no sleeping needed).""" + return email.utils.formatdate(next(_clock), usegmt=True) + + +def response(body=b"", status=200, content_type="application/json", **overrides): + """Build a stub response. Header overrides: pass None to omit a header.""" + if isinstance(body, str): + body = body.encode() + headers = { + "Server": "stub", + "Date": fresh_date, + "Content-Type": content_type, + "Content-Length": str(len(body)), + } + for name, value in overrides.items(): + headers[name.replace("_", "-")] = value + return status, headers, body + + +class StubServer: + """Local HTTP server; ``routes`` maps path -> callable returning response().""" + + def __init__(self, routes): + self.routes = routes + stub = self + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self): # noqa: N802 + route = stub.routes.get(self.path) + if route is None: + status, headers, body = response(b"not found", 404, "text/plain") + else: + status, headers, body = route() + self.send_response_only(status) + for name, value in headers.items(): + if value is None: + continue + self.send_header(name, value() if callable(value) else value) + if headers.get("Content-Length") is None and headers.get("Transfer-Encoding") is None: + self.send_header("Connection", "close") + self.close_connection = True + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = "http://127.0.0.1:%d" % self.httpd.server_address[1] + self.thread = threading.Thread( + target=self.httpd.serve_forever, kwargs={"poll_interval": 0.02}, daemon=True) + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, *exc): + self.httpd.shutdown() + self.httpd.server_close() + + +@contextlib.contextmanager +def captured(): + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + yield out, err + + +def run_cli(*argv): + with captured() as (out, err): + code = catalog.main(list(argv)) + return code, out.getvalue(), err.getvalue() diff --git a/tests/test_changed_lint.py b/tests/test_changed_lint.py new file mode 100644 index 000000000..91bff507e --- /dev/null +++ b/tests/test_changed_lint.py @@ -0,0 +1,205 @@ +import json +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + +from helpers import catalog, run_cli + +# Throwaway repositories created by these tests are not signed and use a +# fixed test identity; they never leave the temporary directory. +GIT = ["git", "-c", "user.name=test", "-c", "user.email=test@example.invalid", + "-c", "commit.gpgsign=false", "-c", "init.defaultBranch=main"] + +CONFIG = '{"framework": "%s", "tests": [{"default": {"json_url": "/json", "port": 8080}}]}' + + +class TempRepo: + def __init__(self): + self.tmp = tempfile.TemporaryDirectory() + self.path = Path(self.tmp.name) + self.git("init", "-q") + + def git(self, *argv): + return subprocess.run(GIT + list(argv), cwd=str(self.path), check=True, + capture_output=True, text=True).stdout.strip() + + def write(self, rel, text): + p = self.path / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text) + + def framework(self, lang, name, dockerfile="FROM golang:1.22\n"): + self.write("%s/%s/benchmark_config.json" % (lang, name), CONFIG % name) + self.write("%s/%s/%s.dockerfile" % (lang, name, name), dockerfile) + + def commit(self, msg="c"): + self.git("add", "-A") + self.git("commit", "-q", "-m", msg) + return self.git("rev-parse", "HEAD") + + def close(self): + self.tmp.cleanup() + + +class ChangedTest(unittest.TestCase): + def setUp(self): + self.repo = TempRepo() + self.repo.framework("Go", "gin") + self.repo.framework("Go", "chi") + self.repo.framework("Python", "flask") + self.repo.write("README.md", "x\n") + self.base = self.repo.commit("base") + + def tearDown(self): + self.repo.close() + + def changed(self, *extra): + return run_cli("--root", str(self.repo.path), "changed", "--since", self.base, *extra) + + def test_framework_change_lists_only_that_folder(self): + self.repo.write("Go/gin/main.go", "package main\n") + self.repo.commit() + code, out, _ = self.changed() + self.assertEqual(code, 0) + self.assertEqual(out.split(), ["Go/gin"]) + + def test_docs_only_change_lists_nothing(self): + self.repo.write("README.md", "y\n") + self.repo.write("docs/specs/x.md", "y\n") + self.repo.commit() + code, out, _ = self.changed("--json") + self.assertEqual(json.loads(out), {"tooling": False, "folders": [], "matrix": []}) + + def test_tooling_change_prints_all_tooling(self): + for path in ("tools/catalog.py", "tests/test_x.py", ".github/workflows/ci.yml"): + with self.subTest(path=path): + repo = TempRepo() + try: + repo.framework("Go", "go-std") + base = repo.commit() + repo.write(path, "x\n") + repo.commit() + code, out, _ = run_cli("--root", str(repo.path), "changed", "--since", base) + self.assertEqual(out.split(), [catalog.ALL_TOOLING]) + _, out, _ = run_cli("--root", str(repo.path), "changed", "--since", base, + "--json") + data = json.loads(out) + self.assertTrue(data["tooling"]) + # Smoke sample entries that exist in this catalog are added. + self.assertEqual(data["matrix"], ["Go/go-std:default"]) + finally: + repo.close() + + def test_other_github_files_are_not_tooling(self): + self.repo.write(".github/CODEOWNERS", "* @x\n") + self.repo.commit() + _, out, _ = self.changed() + self.assertEqual(out.split(), []) + + def test_deleted_folder_is_not_listed(self): + self.repo.git("rm", "-q", "-r", "Go/chi") + self.repo.write("Python/flask/app.py", "print()\n") + self.repo.commit() + _, out, _ = self.changed() + self.assertEqual(out.split(), ["Python/flask"]) + + def test_root_in_subdirectory(self): + repo = TempRepo() + try: + repo.write("catalog/Go/gin/benchmark_config.json", CONFIG % "gin") + repo.write("catalog/Go/gin/gin.dockerfile", "FROM golang:1.22\n") + base = repo.commit() + repo.write("catalog/Go/gin/x.go", "x\n") + repo.commit() + code, out, _ = run_cli("--root", str(repo.path / "catalog"), "changed", "--since", base) + self.assertEqual(out.split(), ["Go/gin"]) + finally: + repo.close() + + +class LintDockerfileTest(unittest.TestCase): + def lint(self, text): + return catalog.lint_dockerfile(textwrap.dedent(text)) + + def test_latest_fails(self): + problems = self.lint("FROM ubuntu:latest\n") + self.assertEqual(len(problems), 1) + self.assertIn(":latest", problems[0][1]) + + def test_untagged_fails(self): + for image in ("ubuntu", "gcr.io/distroless/cc-debian12", "localhost:5000/img"): + with self.subTest(image=image): + self.assertEqual(len(self.lint("FROM %s\n" % image)), 1) + + def test_allowed_forms(self): + text = """\ + # FROM ignored:latest + ARG BASE=debian:12 + FROM --platform=linux/amd64 golang:1.22 AS build + FROM build AS test + FROM ${BASE} + FROM $BASE AS other + FROM scratch + FROM python@sha256:0123456789abcdef + FROM registry.example.com:5000/team/img:1.2.3 \\ + AS final + from Build + """ + self.assertEqual(self.lint(text), []) + + def test_alias_must_be_defined_before_use(self): + problems = self.lint("FROM base\nFROM debian:12 AS base\n") + self.assertEqual([p[0] for p in problems], [1]) + + +class LintDockerCommandTest(unittest.TestCase): + def setUp(self): + self.repo = TempRepo() + self.repo.framework("Go", "gin", "FROM golang:latest\n") # pre-existing, not linted + self.repo.framework("Go", "chi") + self.base = self.repo.commit() + + def tearDown(self): + self.repo.close() + + def lint(self): + return run_cli("--root", str(self.repo.path), "lint-docker", "--since", self.base) + + def test_unchanged_bad_dockerfile_is_ignored(self): + self.repo.write("Go/chi/main.go", "x\n") + self.repo.commit() + code, out, _ = self.lint() + self.assertEqual(code, 0) + self.assertIn("0 Dockerfile(s) checked", out) + + def test_changed_dockerfile_with_latest_fails(self): + self.repo.write("Go/chi/chi.dockerfile", "FROM golang:latest\n") + self.repo.commit() + code, out, _ = self.lint() + self.assertEqual(code, 1) + self.assertIn("Go/chi/chi.dockerfile:1: base image uses :latest", out) + + def test_added_untagged_dockerfile_fails(self): + self.repo.write("Go/chi/chi-alt.dockerfile", "FROM alpine\n") + self.repo.commit() + code, out, _ = self.lint() + self.assertEqual(code, 1) + self.assertIn("untagged base image 'alpine'", out) + + def test_changed_tagged_dockerfile_passes(self): + self.repo.write("Go/chi/chi.dockerfile", "FROM golang:1.23 AS b\nFROM scratch\n") + self.repo.commit() + code, _, _ = self.lint() + self.assertEqual(code, 0) + + def test_deleted_dockerfile_is_ignored(self): + self.repo.git("rm", "-q", "Go/gin/gin.dockerfile") + self.repo.commit() + code, _, _ = self.lint() + self.assertEqual(code, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_check.py b/tests/test_check.py new file mode 100644 index 000000000..bec76f64e --- /dev/null +++ b/tests/test_check.py @@ -0,0 +1,98 @@ +import json +import unittest + +from helpers import FIXTURES, catalog, run_cli + + +class LoadCatalogTest(unittest.TestCase): + def setUp(self): + self.frameworks = catalog.load_catalog(FIXTURES / "valid") + self.variants = {v.id: v for v in catalog.all_variants(self.frameworks)} + + def test_discovers_frameworks_and_variants(self): + self.assertEqual([fw.id for fw in self.frameworks], ["Go/hello", "Python/dbonly"]) + self.assertEqual( + sorted(self.variants), + ["Go/hello:alt", "Go/hello:custom", "Go/hello:default", "Go/hello:old", + "Python/dbonly:default"], + ) + + def test_dockerfile_resolution(self): + self.assertEqual(self.variants["Go/hello:default"].dockerfile, "hello.dockerfile") + self.assertEqual(self.variants["Go/hello:alt"].dockerfile, "hello-alt.dockerfile") + self.assertEqual(self.variants["Go/hello:custom"].dockerfile, "custom.dockerfile") + + def test_port_string_is_accepted(self): + self.assertEqual(self.variants["Go/hello:alt"].port, 8081) + + def test_database_normalized(self): + dbs = {k: v.database for k, v in self.variants.items()} + self.assertEqual(dbs["Go/hello:default"], "postgres") + self.assertEqual(dbs["Go/hello:alt"], "mysql") + self.assertEqual(dbs["Go/hello:custom"], "none") + self.assertEqual(dbs["Go/hello:old"], "mongodb") + + def test_broken_tag(self): + self.assertTrue(self.variants["Go/hello:old"].broken) + self.assertFalse(self.variants["Go/hello:default"].broken) + + def test_stage_a_types(self): + self.assertEqual(self.variants["Go/hello:default"].stage_types("A"), ["json", "plaintext"]) + self.assertEqual(self.variants["Python/dbonly:default"].stage_types("A"), []) + + +class CheckCommandTest(unittest.TestCase): + def test_valid_catalog_passes_with_counts(self): + code, out, _ = run_cli("--root", str(FIXTURES / "valid"), "check", "--json") + self.assertEqual(code, 0) + report = json.loads(out) + self.assertEqual(report["languages"], 2) + self.assertEqual(report["frameworks"], 2) + self.assertEqual(report["variants"], 5) + self.assertEqual(report["broken"], 1) + self.assertEqual(report["test_types"]["json"], {"stage": "A", "declared": 3, "non_broken": 2}) + self.assertEqual(report["test_types"]["db"]["declared"], 2) + self.assertEqual(report["errors"], []) + + def test_text_report(self): + code, out, _ = run_cli("--root", str(FIXTURES / "valid"), "check") + self.assertEqual(code, 0) + self.assertIn("frameworks: 2 variants: 5 broken: 1", out) + self.assertIn("stage B [declared, not verified]", out) + + def test_missing_dockerfile_fails(self): + code, out, _ = run_cli("--root", str(FIXTURES / "missing-dockerfile"), "check") + self.assertEqual(code, 1) + self.assertIn("Go/nodf:default: Dockerfile not found: nodf.dockerfile", out) + + def test_no_urls_fails(self): + code, out, _ = run_cli("--root", str(FIXTURES / "no-urls"), "check") + self.assertEqual(code, 1) + self.assertIn("Go/nourls:default: no known *_url declared", out) + + def test_bad_json_fails(self): + code, out, _ = run_cli("--root", str(FIXTURES / "bad-json"), "check") + self.assertEqual(code, 1) + self.assertIn("Go/broken-json: benchmark_config.json does not parse", out) + + +class ListCommandTest(unittest.TestCase): + def test_list_text(self): + code, out, _ = run_cli("--root", str(FIXTURES / "valid"), "list", "--lang", "go") + self.assertEqual(code, 0) + lines = out.splitlines() + self.assertIn("Go/hello:default\tjson,plaintext", lines) + self.assertIn("Go/hello:old\tjson\tbroken", lines) + self.assertFalse(any(line.startswith("Python/") for line in lines)) + + def test_list_json(self): + code, out, _ = run_cli("--root", str(FIXTURES / "valid"), "list", "--json") + self.assertEqual(code, 0) + rows = {r["id"]: r for r in json.loads(out)} + self.assertEqual(rows["Python/dbonly:default"]["stage_a"], []) + self.assertEqual(rows["Python/dbonly:default"]["test_types"], ["db", "fortune"]) + self.assertTrue(rows["Go/hello:old"]["broken"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_verify_cli.py b/tests/test_verify_cli.py new file mode 100644 index 000000000..038b9fcbf --- /dev/null +++ b/tests/test_verify_cli.py @@ -0,0 +1,173 @@ +"""`catalog.py verify` orchestration with a fake Docker and a stub server.""" + +import json +import re +import tempfile +import unittest +from pathlib import Path + +from helpers import FIXTURES, StubServer, captured, catalog, response, verify + + +class FakeDocker: + def __init__(self, base_url, build_results=None, running=True): + self.base_url_value = base_url + self.build_results = list(build_results or []) + self.is_running = running + self.calls = [] + + def build(self, context, dockerfile, tag, build_args): + self.calls.append(("build", dockerfile, tag, build_args["TFB_TEST_NAME"])) + ok = self.build_results.pop(0) if self.build_results else True + return ok, "" if ok else "network error" + + def network_create(self, name): + self.calls.append(("network_create", name)) + + def network_remove(self, name): + self.calls.append(("network_remove", name)) + + def run(self, image, name, network, port): + self.calls.append(("run", image, port)) + return name + + def base_url(self, name, port): + return self.base_url_value + + def running(self, name): + return self.is_running + + def logs(self, name, tail=50): + return "app log line" + + def remove(self, name): + self.calls.append(("remove", name)) + + def remove_image(self, tag): + self.calls.append(("rmi", tag)) + + def names(self, kind): + return [c for c in self.calls if c[0] == kind] + + +ROUTES = { + "/json": lambda: response('{"message":"Hello, World!"}'), + "/plaintext": lambda: response("Hello, World!", content_type="text/plain"), +} + + +class VerifyCommandTest(unittest.TestCase): + def run_verify(self, docker, *targets, extra=()): + argv = ["--root", str(FIXTURES / "valid"), "verify", *targets, "--date-delay", "0", + "--port-timeout", "2", *extra] + args = catalog.parser().parse_args(argv) + with captured() as (out, _err): + code = catalog.cmd_verify(args, docker=docker) + return code, out.getvalue() + + def test_framework_target_verifies_non_broken_variants(self): + with StubServer(ROUTES) as srv: + docker = FakeDocker(srv.url) + with tempfile.TemporaryDirectory() as tmp: + summary = Path(tmp) / "s.json" + code, out = self.run_verify(docker, "Go/hello", extra=("--summary", str(summary))) + data = json.loads(summary.read_text()) + self.assertEqual(code, 0, out) + self.assertIn("Go/hello:default json pass", out) + self.assertIn("Go/hello:default plaintext pass", out) + self.assertIn("Go/hello:default db declared-not-verified", out) + self.assertIn("Go/hello:old json skipped-broken", out) + built = [c[1] for c in docker.names("build")] + self.assertEqual(built, ["hello.dockerfile", "hello-alt.dockerfile", "custom.dockerfile"]) + self.assertEqual([c[3] for c in docker.names("build")], ["hello", "hello-alt", "hello-custom"]) + self.assertEqual(docker.names("run")[1][2], 8081) + # Every started container and network is cleaned up. + self.assertEqual(len(docker.names("remove")), 3) + self.assertEqual(len(docker.names("network_remove")), 3) + statuses = {v["id"]: {t: r["status"] for t, r in v["test_types"].items()} + for v in data["variants"]} + self.assertEqual(statuses["Go/hello:old"], {"json": "skipped-broken"}) + self.assertEqual(data["stage"], "A") + + def test_variant_target_and_db_only_variant_is_not_built(self): + docker = FakeDocker("http://127.0.0.1:9") + code, out = self.run_verify(docker, "Python/dbonly:default") + self.assertEqual(code, 0) + self.assertEqual(docker.names("build"), []) + self.assertIn("Python/dbonly:default db declared-not-verified", out) + self.assertIn("Python/dbonly:default fortune declared-not-verified", out) + + def test_build_failure_is_retried_once(self): + with StubServer(ROUTES) as srv: + docker = FakeDocker(srv.url, build_results=[False, True]) + code, out = self.run_verify(docker, "Go/hello:alt") + self.assertEqual(code, 0, out) + self.assertEqual(len(docker.names("build")), 2) + self.assertIn("retried", out) + + def test_build_failure_twice_fails_stage_a_types(self): + docker = FakeDocker("http://127.0.0.1:9", build_results=[False, False]) + code, out = self.run_verify(docker, "Go/hello:default") + self.assertEqual(code, 1) + self.assertIn("Go/hello:default json fail", out) + self.assertIn("Go/hello:default db declared-not-verified", out) + self.assertEqual(docker.names("run"), []) + + def test_container_exit_fails_and_cleans_up(self): + docker = FakeDocker("http://127.0.0.1:9", running=False) + code, out = self.run_verify(docker, "Go/hello:alt", extra=("--rmi",)) + self.assertEqual(code, 1) + self.assertIn("container exited", out) + self.assertIn("app log line", out) + self.assertEqual(len(docker.names("remove")), 1) + self.assertEqual(len(docker.names("network_remove")), 1) + self.assertEqual(len(docker.names("rmi")), 1) + + def test_port_timeout_fails(self): + docker = FakeDocker("http://127.0.0.1:9") + code, out = self.run_verify(docker, "Go/hello:alt") + self.assertEqual(code, 1) + self.assertIn("no HTTP response on port 8081", out) + + def test_failing_response_fails(self): + routes = dict(ROUTES, **{"/json": lambda: response('{"message":"nope"}')}) + with StubServer(routes) as srv: + code, out = self.run_verify(FakeDocker(srv.url), "Go/hello:default") + self.assertEqual(code, 1) + self.assertIn("Go/hello:default json fail", out) + self.assertIn("Go/hello:default plaintext pass", out) + + def test_unknown_target_fails(self): + code, _ = self.run_verify(FakeDocker("http://127.0.0.1:9"), "Go/nope") + self.assertEqual(code, 1) + + +class ImageTagTest(unittest.TestCase): + def test_tags_are_valid_docker_references(self): + ref = re.compile(r"^[a-z0-9]+(?:[._-][a-z0-9]+)*/[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*$") + for v in catalog.all_variants(catalog.load_catalog(FIXTURES / "valid")): + v.language, v.folder = "C++", "Hello_World.NET" + tag = catalog.image_tag(v) + self.assertRegex(tag, ref) + + +class SummarizeTest(unittest.TestCase): + def test_merges_shards(self): + with tempfile.TemporaryDirectory() as tmp: + a, b, out = Path(tmp) / "a.json", Path(tmp) / "b.json", Path(tmp) / "all.json" + a.write_text(json.dumps({"variants": [ + {"id": "Go/x:default", "test_types": {"json": {"status": "pass", "findings": []}}}]})) + b.write_text(json.dumps({"variants": [ + {"id": "C/y:default", "test_types": { + "json": {"status": "fail", "findings": []}, + "db": {"status": verify.NOT_VERIFIED, "findings": []}}}]})) + with captured() as (stdout, _): + catalog.main(["summarize", str(a), str(b), "--out", str(out)]) + merged = json.loads(out.read_text()) + self.assertEqual([v["id"] for v in merged["variants"]], ["C/y:default", "Go/x:default"]) + self.assertEqual(merged["counts"], {"fail": 1, "declared-not-verified": 1, "pass": 1}) + self.assertIn("| C/y:default | json | fail |", stdout.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_verify_rules.py b/tests/test_verify_rules.py new file mode 100644 index 000000000..81b77d8e6 --- /dev/null +++ b/tests/test_verify_rules.py @@ -0,0 +1,162 @@ +"""Stage A verifier rules against a local stub HTTP server (spec 001, R5).""" + +import unittest + +from helpers import StubServer, response, verify + +HELLO_JSON = '{"message":"Hello, World!"}' +URLS = {"json_url": "/json", "plaintext_url": "/plaintext"} + + +def ok_json(**kw): + return lambda: response(HELLO_JSON, **kw) + + +def ok_text(body="Hello, World!", **kw): + kw.setdefault("content_type", "text/plain") + return lambda: response(body, **kw) + + +class RulesTestCase(unittest.TestCase): + def run_variant(self, routes, urls=None): + with StubServer(routes) as srv: + return verify.verify_variant(srv.url, urls or URLS, "A", date_delay=0) + + def assertStatus(self, result, test_type, status, needle=None): + res = result[test_type] + self.assertEqual(res["status"], status, res["findings"]) + if needle: + self.assertTrue(any(needle in f["message"] for f in res["findings"]), + res["findings"]) + + +class PassTest(RulesTestCase): + def test_json_and_plaintext_pass(self): + r = self.run_variant({"/json": ok_json(), "/plaintext": ok_text()}) + self.assertStatus(r, "json", "pass") + self.assertStatus(r, "plaintext", "pass") + self.assertEqual(r["json"]["findings"], []) + + def test_charset_and_case_insensitive_values(self): + r = self.run_variant({ + "/json": lambda: response('{"Message": "hello, WORLD!"}', + content_type="application/json; charset=utf-8"), + "/plaintext": ok_text("HELLO, world!", content_type="text/plain;charset=UTF-8"), + }) + self.assertStatus(r, "json", "pass") + self.assertStatus(r, "plaintext", "pass") + + def test_chunked_transfer_encoding_is_a_size_header(self): + def chunked(): + status, headers, body = response("Hello, World!", content_type="text/plain", + Content_Length=None, Transfer_Encoding="chunked") + return status, headers, b"d\r\nHello, World!\r\n0\r\n\r\n" + r = self.run_variant({"/plaintext": chunked}, {"plaintext_url": "/plaintext"}) + self.assertStatus(r, "plaintext", "pass") + + +class WarnTest(RulesTestCase): + def test_json_extra_keys_warn(self): + r = self.run_variant({"/json": lambda: response('{"message":"Hello, World!","x":"y"}')}, + {"json_url": "/json"}) + self.assertStatus(r, "json", "warn", "Extra JSON keys") + + def test_plaintext_extra_bytes_warn(self): + r = self.run_variant({"/plaintext": ok_text("Hello, World!\n")}, + {"plaintext_url": "/plaintext"}) + self.assertStatus(r, "plaintext", "warn", "1 byte(s) more") + + def test_bad_date_format_warns(self): + dates = iter(["2024-01-01 00:00:%02d" % i for i in range(10)]) + r = self.run_variant({"/json": ok_json(Date=lambda: next(dates))}, {"json_url": "/json"}) + self.assertStatus(r, "json", "warn", "IMF-fixdate") + + +class FailTest(RulesTestCase): + def test_wrong_message_fails(self): + r = self.run_variant({"/json": lambda: response('{"message":"Goodbye"}')}, + {"json_url": "/json"}) + self.assertStatus(r, "json", "fail", "Expected message") + + def test_json_array_fails(self): + r = self.run_variant({"/json": lambda: response('[{"message":"Hello, World!"}]')}, + {"json_url": "/json"}) + self.assertStatus(r, "json", "fail", "not a JSON object") + + def test_missing_message_key_fails(self): + r = self.run_variant({"/json": lambda: response('{"msg":"Hello, World!"}')}, + {"json_url": "/json"}) + self.assertStatus(r, "json", "fail", "Missing required key") + + def test_invalid_json_fails(self): + r = self.run_variant({"/json": lambda: response("Hello, World!")}, {"json_url": "/json"}) + self.assertStatus(r, "json", "fail", "Invalid JSON") + + def test_empty_body_fails(self): + r = self.run_variant({"/json": lambda: response("")}, {"json_url": "/json"}) + self.assertStatus(r, "json", "fail", "Empty response body") + + def test_plaintext_without_hello_fails(self): + r = self.run_variant({"/plaintext": ok_text("Bye")}, {"plaintext_url": "/plaintext"}) + self.assertStatus(r, "plaintext", "fail", "does not contain") + + def test_wrong_status_fails(self): + r = self.run_variant({"/json": lambda: response(HELLO_JSON, status=201)}, + {"json_url": "/json"}) + self.assertStatus(r, "json", "fail", "HTTP status 201") + + def test_error_status_with_valid_body_fails(self): + r = self.run_variant({"/json": lambda: response(HELLO_JSON, status=500)}, + {"json_url": "/json"}) + self.assertStatus(r, "json", "fail", "HTTP status 500") + + def test_missing_required_headers_fail(self): + for header in ("Server", "Date", "Content-Type"): + with self.subTest(header=header): + r = self.run_variant({"/json": ok_json(**{header.replace("-", "_"): None})}, + {"json_url": "/json"}) + self.assertStatus(r, "json", "fail", "header missing: %s" % header) + + def test_missing_size_header_fails(self): + r = self.run_variant({"/json": ok_json(Content_Length=None)}, {"json_url": "/json"}) + self.assertStatus(r, "json", "fail", "Content-Length or Transfer-Encoding") + + def test_cached_date_fails(self): + r = self.run_variant({"/json": ok_json(Date="Mon, 01 Jan 2024 00:00:00 GMT")}, + {"json_url": "/json"}) + self.assertStatus(r, "json", "fail", "cached") + + def test_wrong_content_type_fails(self): + r = self.run_variant({ + "/json": ok_json(content_type="text/plain"), + "/plaintext": ok_text(content_type="text/html"), + }) + self.assertStatus(r, "json", "fail", "Content-Type 'text/plain'") + self.assertStatus(r, "plaintext", "fail", "Content-Type 'text/html'") + + def test_short_route_fails(self): + r = self.run_variant({"/j": ok_json(), "/plain": ok_text()}, + {"json_url": "/j", "plaintext_url": "/plain"}) + self.assertStatus(r, "json", "fail", "at least 5 characters") + self.assertStatus(r, "plaintext", "fail", "at least 10 characters") + + def test_no_server_fails(self): + r = verify.verify_variant("http://127.0.0.1:9", {"json_url": "/json"}, "A", date_delay=0) + self.assertStatus(r, "json", "fail", "No response") + + +class StageTest(RulesTestCase): + def test_later_stage_types_are_declared_not_verified(self): + urls = dict(URLS, db_url="/db", query_url="/queries?q=", cached_query_url="/cached?q=") + r = self.run_variant({"/json": ok_json(), "/plaintext": ok_text()}, urls) + self.assertEqual(list(r), ["json", "plaintext", "db", "query", "cached-query"]) + for t in ("db", "query", "cached-query"): + self.assertEqual(r[t]["status"], verify.NOT_VERIFIED) + + def test_unimplemented_stage_is_rejected(self): + with self.assertRaises(ValueError): + verify.verify_variant("http://127.0.0.1:9", URLS, "B") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/catalog.py b/tools/catalog.py new file mode 100644 index 000000000..77aa24c6b --- /dev/null +++ b/tools/catalog.py @@ -0,0 +1,724 @@ +#!/usr/bin/env python3 +"""Catalog tooling for BenchCommons/catalog-web-frameworks (spec 001, plan section 3). + +Stdlib only, Python 3.9+. Commands: + + check metadata check over the whole catalog + list [--lang L] [--json] list variants + changed --since REF [--json] changed framework folders (ALL_TOOLING for tooling changes) + lint-docker --since REF reject :latest / untagged FROM in changed Dockerfiles + verify TARGET [--stage A] build, run and verify variants (TARGET: Lang, Lang/fw or Lang/fw:key) + summarize FILE... merge verify summaries into one JSON and a Markdown report + +Every command takes --root to point at a catalog directory (default: repo root). +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import time +import uuid +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Sequence + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import verify as verifier # noqa: E402 + +REPO_ROOT = Path(__file__).resolve().parent.parent +CONFIG_NAME = "benchmark_config.json" +NON_CATALOG_DIRS = {"tools", "tests", "docs"} +TOOLING_PREFIXES = ("tools/", "tests/", ".github/workflows/") +ALL_TOOLING = "ALL_TOOLING" +# Fixed smoke sample verified when tooling changes (plan section 4): one +# variant per major language that needs no database at startup, so stage A +# can pass on hosted CI without a dataset. +SMOKE_SAMPLE = ( + "C/libreactor:default", + "Go/go-std:default", + "Java/javalin:default", + "JavaScript/express:default", + "Python/fastpysgi:default", + "Rust/axum:default", +) +DATABASES = {"postgres": "postgres", "postgresql": "postgres", "mysql": "mysql", + "mongodb": "mongodb", "mongo": "mongodb", "none": "none"} +PORT_WAIT_SECONDS = 120 + + +# -------------------------------------------------------------------------- +# Catalog model + + +@dataclass +class Variant: + language: str + folder: str # framework folder name + key: str + framework: str # "framework" field of the config + path: Path # framework folder + config: dict + dockerfile: Optional[str] = None + port: Optional[int] = None + urls: Dict[str, str] = field(default_factory=dict) + database: str = "none" + broken: bool = False + errors: List[str] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + + @property + def id(self) -> str: + return "%s/%s:%s" % (self.language, self.folder, self.key) + + @property + def test_name(self) -> str: + return self.framework if self.key == "default" else "%s-%s" % (self.framework, self.key) + + def stage_types(self, stage: str = "A") -> List[str]: + return list(verifier.types_for_stage(self.urls, stage)) + + def as_dict(self) -> dict: + return { + "id": self.id, + "language": self.language, + "framework": self.folder, + "key": self.key, + "dockerfile": self.dockerfile, + "port": self.port, + "database": self.database, + "broken": self.broken, + "test_types": list(verifier.declared_types(self.urls)), + "stage_a": self.stage_types("A"), + } + + +@dataclass +class Framework: + language: str + name: str + path: Path + variants: List[Variant] = field(default_factory=list) + errors: List[str] = field(default_factory=list) + + @property + def id(self) -> str: + return "%s/%s" % (self.language, self.name) + + +def framework_dirs(root: Path) -> List[Path]: + out = [] + for lang in sorted(p for p in root.iterdir() if p.is_dir()): + if lang.name.startswith(".") or lang.name in NON_CATALOG_DIRS: + continue + for fw in sorted(p for p in lang.iterdir() if p.is_dir()): + if (fw / CONFIG_NAME).is_file(): + out.append(fw) + return out + + +def normalize_database(value) -> Optional[str]: + if value is None: + return "none" + return DATABASES.get(str(value).strip().lower()) + + +def load_framework(path: Path) -> Framework: + fw = Framework(path.parent.name, path.name, path) + try: + with open(path / CONFIG_NAME, encoding="utf-8") as fh: + config = json.load(fh) + except (OSError, ValueError) as exc: + fw.errors.append("%s does not parse: %s" % (CONFIG_NAME, exc)) + return fw + if not isinstance(config, dict): + fw.errors.append("%s is not a JSON object" % CONFIG_NAME) + return fw + name = config.get("framework") + if not isinstance(name, str) or not name: + fw.errors.append("missing 'framework' name") + return fw + tests = config.get("tests") + if not isinstance(tests, list) or not tests or not all(isinstance(t, dict) for t in tests): + fw.errors.append("'tests' must be a non-empty list of objects") + return fw + seen = set() + for group in tests: + for key, test in group.items(): + if key in seen: + fw.errors.append("duplicate variant key '%s'" % key) + continue + seen.add(key) + fw.variants.append(load_variant(fw, name, key, test)) + return fw + + +def load_variant(fw: Framework, name: str, key: str, test) -> Variant: + v = Variant(fw.language, fw.name, key, name, fw.path, test if isinstance(test, dict) else {}) + if not isinstance(test, dict): + v.errors.append("variant is not an object") + return v + tags = test.get("tags") or [] + v.broken = isinstance(tags, list) and "broken" in tags + + port = test.get("port") + if isinstance(port, bool): + port = None + if isinstance(port, str) and port.strip().isdigit(): + port = int(port.strip()) + if isinstance(port, int) and 0 < port < 65536: + v.port = port + else: + v.errors.append("missing or invalid port: %r" % (test.get("port"),)) + + v.urls = {k: test[k] for k in verifier.TEST_TYPES if isinstance(test.get(k), str) and test[k]} + if not v.urls: + v.errors.append("no known *_url declared") + + dockerfile = test.get("dockerfile") or "%s.dockerfile" % v.test_name + v.dockerfile = dockerfile + if not (fw.path / dockerfile).is_file(): + v.errors.append("Dockerfile not found: %s" % dockerfile) + + db = normalize_database(test.get("database")) + if db is None: + v.warnings.append("unknown database %r" % test.get("database")) + v.database = str(test.get("database")) + else: + v.database = db + return v + + +def load_catalog(root: Path) -> List[Framework]: + return [load_framework(p) for p in framework_dirs(root)] + + +def all_variants(frameworks: Iterable[Framework]) -> List[Variant]: + return [v for fw in frameworks for v in fw.variants] + + +# -------------------------------------------------------------------------- +# check / list + + +def cmd_check(args) -> int: + root = Path(args.root) + frameworks = load_catalog(root) + variants = all_variants(frameworks) + errors: List[str] = [] + warnings: List[str] = [] + for fw in frameworks: + errors += ["%s: %s" % (fw.id, e) for e in fw.errors] + for v in variants: + errors += ["%s: %s" % (v.id, e) for e in v.errors] + warnings += ["%s: %s" % (v.id, w) for w in v.warnings] + + languages = {fw.language for fw in frameworks} + dockerfiles = sum(1 for fw in frameworks for _ in fw.path.glob("*.dockerfile")) + by_type = Counter() + by_type_ok = Counter() + for v in variants: + for name in verifier.declared_types(v.urls): + by_type[name] += 1 + if not v.broken: + by_type_ok[name] += 1 + databases = Counter(v.database for v in variants) + broken = sum(1 for v in variants if v.broken) + + report = { + "languages": len(languages), + "frameworks": len(frameworks), + "variants": len(variants), + "broken": broken, + "dockerfiles": dockerfiles, + "databases": dict(databases.most_common()), + "test_types": { + name: {"stage": stage, "declared": by_type[name], "non_broken": by_type_ok[name]} + for name, stage in verifier.TEST_TYPES.values() + }, + "errors": errors, + "warnings": warnings, + } + if args.json: + print(json.dumps(report, indent=2)) + else: + print("catalog: %s" % root.resolve()) + print("languages: %d frameworks: %d variants: %d broken: %d dockerfiles: %d" + % (len(languages), len(frameworks), len(variants), broken, dockerfiles)) + print("databases: " + ", ".join("%s %d" % kv for kv in databases.most_common())) + for stage in verifier.STAGES: + parts = [ + "%s %d (%d non-broken)" % (name, by_type[name], by_type_ok[name]) + for name, s in verifier.TEST_TYPES.values() if s == stage + ] + label = "verified" if stage in verifier.IMPLEMENTED_STAGES else "declared, not verified" + print("stage %s [%s]: %s" % (stage, label, ", ".join(parts))) + for w in warnings: + print("warning: " + w) + for e in errors: + print("error: " + e) + print("%d error(s), %d warning(s)" % (len(errors), len(warnings))) + return 1 if errors else 0 + + +def cmd_list(args) -> int: + variants = all_variants(load_catalog(Path(args.root))) + if args.lang: + variants = [v for v in variants if v.language.lower() == args.lang.lower()] + if args.json: + print(json.dumps([v.as_dict() for v in variants], indent=2)) + return 0 + for v in variants: + types = ",".join(v.stage_types("A")) or "-" + print("%s\t%s%s" % (v.id, types, "\tbroken" if v.broken else "")) + return 0 + + +# -------------------------------------------------------------------------- +# git helpers: changed / lint-docker + + +def git(root: Path, *argv: str) -> str: + return subprocess.run( + ["git", *argv], cwd=str(root), check=True, capture_output=True, text=True + ).stdout + + +def changed_paths(root: Path, since: str, diff_filter: Optional[str] = None): + """Return (toplevel, prefix, paths relative to the git toplevel).""" + top = Path(git(root, "rev-parse", "--show-toplevel").strip()).resolve() + prefix = root.resolve().relative_to(top).as_posix() + prefix = "" if prefix == "." else prefix + "/" + argv = ["diff", "--name-only", "--no-renames"] + if diff_filter: + argv.append("--diff-filter=" + diff_filter) + argv.append("%s...HEAD" % since) + paths = [p for p in git(root, *argv).splitlines() if p] + return top, prefix, paths + + +def changed_folders(root: Path, since: str): + top, prefix, paths = changed_paths(root, since) + tooling = any(p.startswith(TOOLING_PREFIXES) for p in paths) + folders = [] + for p in paths: + if not p.startswith(prefix): + continue + parts = p[len(prefix):].split("/") + if len(parts) < 3 or parts[0].startswith(".") or parts[0] in NON_CATALOG_DIRS: + continue + folder = "%s/%s" % (parts[0], parts[1]) + if folder not in folders and (root / folder / CONFIG_NAME).is_file(): + folders.append(folder) + return tooling, sorted(folders) + + +def cmd_changed(args) -> int: + root = Path(args.root) + tooling, folders = changed_folders(root, args.since) + matrix = list(folders) + if tooling: + for f in SMOKE_SAMPLE: + if f not in matrix and (root / f.split(":")[0] / CONFIG_NAME).is_file(): + matrix.append(f) + if args.json: + print(json.dumps({"tooling": tooling, "folders": folders, "matrix": matrix})) + return 0 + if tooling: + print(ALL_TOOLING) + for f in folders: + print(f) + return 0 + + +FROM_RE = re.compile(r"^\s*FROM\s+(.*)$", re.IGNORECASE) + + +def dockerfile_lines(text: str) -> List[tuple]: + """Logical lines (line number, text): comments dropped, continuations joined.""" + out = [] + buf, start = "", 0 + for n, raw in enumerate(text.splitlines(), 1): + if not buf and raw.lstrip().startswith("#"): + continue + if not buf: + start = n + line = raw.rstrip() + if line.endswith("\\"): + buf += line[:-1] + " " + continue + out.append((start, buf + line)) + buf = "" + if buf: + out.append((start, buf)) + return out + + +def lint_dockerfile(text: str) -> List[tuple]: + """Return (line, message) for each FROM using :latest or no tag.""" + problems = [] + aliases = set() + for n, line in dockerfile_lines(text): + m = FROM_RE.match(line) + if not m: + continue + tokens = [t for t in m.group(1).split() if not t.startswith("--")] + if not tokens: + problems.append((n, "FROM without an image")) + continue + image = tokens[0] + if len(tokens) >= 3 and tokens[1].lower() == "as": + aliases.add(tokens[2].lower()) + low = image.lower() + if low == "scratch" or low in aliases or "$" in image or "@" in image: + continue + last = image.rsplit("/", 1)[-1] + if ":" not in last: + problems.append((n, "untagged base image '%s'" % image)) + elif last.split(":", 1)[1] == "latest": + problems.append((n, "base image uses :latest '%s'" % image)) + return problems + + +def is_dockerfile(path: str) -> bool: + name = path.rsplit("/", 1)[-1] + return name.endswith(".dockerfile") or name == "Dockerfile" + + +def cmd_lint_docker(args) -> int: + root = Path(args.root) + if args.all: + files = sorted( + p for fw in framework_dirs(root) for p in fw.iterdir() + if p.is_file() and is_dockerfile(p.name) + ) + else: + top, _prefix, paths = changed_paths(root, args.since, diff_filter="d") + files = [top / p for p in paths if is_dockerfile(p)] + bad = 0 + for path in files: + for n, msg in lint_dockerfile(path.read_text(encoding="utf-8", errors="replace")): + bad += 1 + print("%s:%d: %s" % (os.path.relpath(path, root), n, msg)) + print("lint-docker: %d Dockerfile(s) checked, %d problem(s)" % (len(files), bad)) + return 1 if bad else 0 + + +# -------------------------------------------------------------------------- +# verify + + +class Docker: + """Thin wrapper over the docker CLI, replaceable in tests.""" + + def __init__(self, platform: Optional[str] = None, build_timeout: int = 1800): + self.platform = platform + self.build_timeout = build_timeout + + def _run(self, argv, timeout=None, check=False): + return subprocess.run( + ["docker", *argv], capture_output=True, text=True, timeout=timeout, check=check + ) + + def build(self, context: Path, dockerfile: str, tag: str, build_args: Dict[str, str]): + argv = ["build", "-f", str(context / dockerfile), "-t", tag] + if self.platform: + argv += ["--platform", self.platform] + for k, v in build_args.items(): + argv += ["--build-arg", "%s=%s" % (k, v)] + argv.append(str(context)) + try: + proc = self._run(argv, timeout=self.build_timeout) + except subprocess.TimeoutExpired: + return False, "docker build timed out after %ds" % self.build_timeout + return proc.returncode == 0, (proc.stdout + proc.stderr)[-4000:] + + def network_create(self, name: str): + self._run(["network", "create", name], check=True) + + def network_remove(self, name: str): + self._run(["network", "rm", name]) + + def run(self, image: str, name: str, network: str, port: int): + argv = ["run", "-d", "--name", name, "--network", network, + "--network-alias", "tfb-server", "--init", + "--ulimit", "nofile=65535:65535", + "--sysctl", "net.core.somaxconn=65535", + "-p", "127.0.0.1::%d" % port] + if self.platform: + argv += ["--platform", self.platform] + argv.append(image) + proc = self._run(argv) + if proc.returncode != 0: + raise RuntimeError("docker run failed: " + proc.stderr.strip()) + return name + + def base_url(self, name: str, port: int) -> Optional[str]: + """Host URL of the published port, or None while it is not mapped.""" + proc = self._run(["port", name, "%d/tcp" % port]) + lines = proc.stdout.split() + if proc.returncode != 0 or not lines: + return None + return "http://" + lines[0] + + def running(self, name: str) -> bool: + proc = self._run(["inspect", "-f", "{{.State.Running}}", name]) + return proc.stdout.strip() == "true" + + def logs(self, name: str, tail: int = 50) -> str: + proc = self._run(["logs", "--tail", str(tail), name]) + return proc.stdout + proc.stderr + + def remove(self, name: str): + self._run(["rm", "-f", name]) + + def remove_image(self, tag: str): + self._run(["rmi", "-f", tag]) + + +def wait_for_http(docker, name: str, port: int, route: str, timeout: float, + fetch=None): + """Wait until the server answers HTTP on its published port. + + Returns (base_url, None) when ready, else (None, reason). + """ + fetch = fetch or verifier.http_fetch + deadline = time.monotonic() + timeout + error = "port not published" + while True: + if not docker.running(name): + return None, "container exited before serving HTTP" + base_url = docker.base_url(name, port) + if base_url: + resp = fetch(base_url + route, "json") + if resp.status is not None: + return base_url, None + error = resp.error + if time.monotonic() >= deadline: + return None, "no HTTP response on port %d after %ds (%s)" % (port, timeout, error) + time.sleep(1) + + +def select_variants(frameworks: List[Framework], target: str) -> List[Variant]: + target = target.strip().strip("/") + key = None + if ":" in target: + target, key = target.split(":", 1) + parts = target.split("/") + out = [] + for fw in frameworks: + if fw.language != parts[0] or (len(parts) > 1 and fw.name != parts[1]): + continue + out += [v for v in fw.variants if key is None or v.key == key] + return out + + +def image_tag(v: Variant) -> str: + name = ("%s-%s-%s" % (v.language, v.folder, v.key)).lower() + return "cwf-verify/" + re.sub(r"[^a-z0-9_.-]+", "-", name).strip("-.") + + +def verify_one(v: Variant, docker, stage: str, args, fetch=None) -> dict: + record = {"id": v.id, "language": v.language, "framework": v.folder, "key": v.key, + "broken": v.broken, "test_types": {}, "notes": []} + declared = verifier.declared_types(v.urls) + if v.broken: + record["test_types"] = {t: {"status": verifier.SKIPPED_BROKEN, "findings": []} + for t in declared} + return record + wanted = verifier.types_for_stage(v.urls, stage) + if v.errors: + record["notes"] += v.errors + record["test_types"] = {t: {"status": verifier.FAIL if t in wanted else verifier.NOT_VERIFIED, + "findings": []} for t in declared} + return record + if not wanted: + record["test_types"] = {t: {"status": verifier.NOT_VERIFIED, "findings": []} + for t in declared} + return record + + def fail_all(reason: str): + record["notes"].append(reason) + record["test_types"] = { + t: {"status": verifier.FAIL if t in wanted else verifier.NOT_VERIFIED, + "findings": [{"level": "fail", "message": reason, "url": ""}] if t in wanted else []} + for t in declared} + return record + + tag = image_tag(v) + build_args = {"BENCHMARK_ENV": args.benchmark_env, "TFB_TEST_NAME": v.test_name, + "TFB_TEST_DATABASE": v.database} + ok, log = docker.build(v.path, v.dockerfile, tag, build_args) + if not ok: + # One retry for transient (network) build failures, reported in the summary. + record["notes"].append("docker build failed once; retried") + ok, log = docker.build(v.path, v.dockerfile, tag, build_args) + if not ok: + record["build_log_tail"] = log[-2000:] + return fail_all("docker build failed") + + suffix = uuid.uuid4().hex[:8] + network, name = "cwf-verify-%s" % suffix, "cwf-%s" % suffix + docker.network_create(network) + try: + try: + docker.run(tag, name, network, v.port) + except Exception as exc: # noqa: BLE001 + return fail_all(str(exc)) + first_route = next(iter(wanted.values())) + base_url, reason = wait_for_http(docker, name, v.port, first_route, + args.port_timeout, fetch) + if reason: + record["container_log_tail"] = docker.logs(name)[-2000:] + return fail_all(reason) + record["test_types"] = verifier.verify_variant( + base_url, v.urls, stage, fetch=fetch, date_delay=args.date_delay) + if any(r["status"] == verifier.FAIL for r in record["test_types"].values()): + record["container_log_tail"] = docker.logs(name)[-2000:] + return record + finally: + docker.remove(name) + docker.network_remove(network) + if args.rmi: + docker.remove_image(tag) + + +def print_record(record: dict, out=None): + out = out or sys.stdout + for name, res in record["test_types"].items(): + print("%s %s %s" % (record["id"], name, res["status"]), file=out) + for f in res["findings"]: + print(" %s: %s" % (f["level"], f["message"]), file=out) + for note in record.get("notes", []): + print(" note: %s" % note, file=out) + for key in ("build_log_tail", "container_log_tail"): + if record.get(key): + print(" --- %s ---" % key, file=out) + for line in record[key].splitlines()[-30:]: + print(" | " + line, file=out) + out.flush() + + +def record_failed(record: dict) -> bool: + return any(r["status"] == verifier.FAIL for r in record["test_types"].values()) + + +def cmd_verify(args, docker=None, fetch=None) -> int: + root = Path(args.root) + frameworks = load_catalog(root) + docker = docker or Docker(platform=args.platform, build_timeout=args.build_timeout) + records = [] + for target in args.targets: + variants = select_variants(frameworks, target) + if not variants: + print("error: no variant matches '%s'" % target, file=sys.stderr) + records.append({"id": target, "test_types": {}, "notes": ["no matching variant"], + "error": True}) + continue + for v in variants: + record = verify_one(v, docker, args.stage, args, fetch) + print_record(record) + records.append(record) + if args.summary: + Path(args.summary).write_text(json.dumps({"stage": args.stage, "variants": records}, + indent=2) + "\n") + failed = [r for r in records if r.get("error") or record_failed(r)] + counts = status_counts(records) + print("verify: %d variant(s); %s" % (len(records), format_counts(counts))) + return 1 if failed else 0 + + +def status_counts(records: Sequence[dict]) -> Counter: + return Counter(r["status"] for rec in records for r in rec["test_types"].values()) + + +def format_counts(counts: Counter) -> str: + order = [verifier.PASS, verifier.WARN, verifier.FAIL, verifier.SKIPPED_BROKEN, + verifier.NOT_VERIFIED] + return ", ".join("%s %d" % (k, counts.get(k, 0)) for k in order) + + +def cmd_summarize(args) -> int: + records = [] + for path in args.files: + records += json.loads(Path(path).read_text()).get("variants", []) + records.sort(key=lambda r: r["id"]) + counts = status_counts(records) + if args.out: + Path(args.out).write_text(json.dumps( + {"counts": dict(counts), "variants": records}, indent=2) + "\n") + lines = ["## Catalog verification (stage A)", "", + "%d variant(s): %s" % (len(records), format_counts(counts)), "", + "| Variant | Test type | Status |", "|---|---|---|"] + for rec in records: + for name, res in rec["test_types"].items(): + if res["status"] in (verifier.FAIL, verifier.WARN): + lines.append("| %s | %s | %s |" % (rec["id"], name, res["status"])) + print("\n".join(lines)) + return 0 + + +# -------------------------------------------------------------------------- +# CLI + + +def parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="catalog.py", description=__doc__.split("\n")[0]) + p.add_argument("--root", default=str(REPO_ROOT), + help="catalog directory (default: repo root)") + sub = p.add_subparsers(dest="command", required=True) + + s = sub.add_parser("check", help="metadata check over the whole catalog") + s.add_argument("--json", action="store_true") + s.set_defaults(func=cmd_check) + + s = sub.add_parser("list", help="list variants") + s.add_argument("--lang") + s.add_argument("--json", action="store_true") + s.set_defaults(func=cmd_list) + + s = sub.add_parser("changed", help="changed framework folders since REF") + s.add_argument("--since", required=True) + s.add_argument("--json", action="store_true", + help="print {tooling, folders, matrix}; matrix adds the smoke sample") + s.set_defaults(func=cmd_changed) + + s = sub.add_parser("lint-docker", help="reject :latest or untagged FROM images") + g = s.add_mutually_exclusive_group(required=True) + g.add_argument("--since") + g.add_argument("--all", action="store_true", help="lint every Dockerfile in the catalog") + s.set_defaults(func=cmd_lint_docker) + + s = sub.add_parser("verify", help="build, run and verify variants") + s.add_argument("targets", nargs="+", metavar="TARGET", + help="Language, Language/framework or Language/framework:key") + s.add_argument("--stage", default="A", choices=list(verifier.IMPLEMENTED_STAGES)) + s.add_argument("--platform", help="docker --platform, e.g. linux/amd64") + s.add_argument("--summary", help="write a JSON summary to this file") + s.add_argument("--rmi", action="store_true", help="remove each image after verifying") + s.add_argument("--build-timeout", type=int, default=1800) + s.add_argument("--port-timeout", type=int, default=PORT_WAIT_SECONDS) + s.add_argument("--date-delay", type=float, default=3.0, help=argparse.SUPPRESS) + s.add_argument("--benchmark-env", default="ci", help=argparse.SUPPRESS) + s.set_defaults(func=cmd_verify) + + s = sub.add_parser("summarize", help="merge verify summaries") + s.add_argument("files", nargs="+") + s.add_argument("--out", help="write the merged JSON here") + s.set_defaults(func=cmd_summarize) + return p + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = parser().parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/verify/__init__.py b/tools/verify/__init__.py new file mode 100644 index 000000000..552aaa38b --- /dev/null +++ b/tools/verify/__init__.py @@ -0,0 +1,124 @@ +"""Correctness verifiers for the web frameworks catalog (spec 001, R5). + +Callers use one function, :func:`verify_variant`. It takes the base URL of a +running implementation and the variant's ``*_url`` routes, runs the verifiers +for the requested stage and returns one result per declared test type. + +Stage A (json, plaintext) is implemented here. Test types of later stages are +reported as ``declared-not-verified``. +""" + +from __future__ import annotations + +from typing import Callable, Dict, Optional + +from .client import Response, fetch as http_fetch +from .rules import Finding, verify_json, verify_plaintext + +# url key in benchmark_config.json -> (test type, stage) +TEST_TYPES = { + "json_url": ("json", "A"), + "plaintext_url": ("plaintext", "A"), + "db_url": ("db", "B"), + "query_url": ("query", "B"), + "fortune_url": ("fortune", "B"), + "update_url": ("update", "B"), + "cached_query_url": ("cached-query", "C"), +} +STAGES = ("A", "B", "C") +IMPLEMENTED_STAGES = ("A",) + +PASS = "pass" +WARN = "warn" +FAIL = "fail" +NOT_VERIFIED = "declared-not-verified" +SKIPPED_BROKEN = "skipped-broken" + +_VERIFIERS = { + "json": verify_json, + "plaintext": verify_plaintext, +} + +Fetch = Callable[[str, str], Response] + + +def stage_of(test_type: str) -> str: + for name, stage in TEST_TYPES.values(): + if name == test_type: + return stage + raise KeyError(test_type) + + +def declared_types(urls: Dict[str, str]) -> Dict[str, str]: + """Map test type -> route for every known ``*_url`` key present.""" + out = {} + for key, (name, _stage) in TEST_TYPES.items(): + if urls.get(key): + out[name] = urls[key] + return out + + +def types_for_stage(urls: Dict[str, str], stage: str = "A") -> Dict[str, str]: + """Declared test types verified at ``stage`` (stages are cumulative).""" + limit = STAGES.index(stage) + return { + name: route + for name, route in declared_types(urls).items() + if STAGES.index(stage_of(name)) <= limit + } + + +def summarize(findings) -> str: + levels = {f.level for f in findings} + if FAIL in levels: + return FAIL + if WARN in levels: + return WARN + return PASS + + +def verify_variant( + base_url: str, + urls: Dict[str, str], + stage: str = "A", + fetch: Optional[Fetch] = None, + date_delay: float = 3.0, +) -> Dict[str, dict]: + """Verify one running variant. + + ``urls`` maps ``*_url`` keys to routes, as in ``benchmark_config.json``. + Returns ``{test_type: {"status": ..., "findings": [...]}}`` for every + declared test type, in ``TEST_TYPES`` order. + """ + if stage not in IMPLEMENTED_STAGES: + raise ValueError("stage %s is not implemented yet" % stage) + fetch = fetch or http_fetch + base_url = base_url.rstrip("/") + wanted = types_for_stage(urls, stage) + results: Dict[str, dict] = {} + for name, route in declared_types(urls).items(): + if name not in wanted: + results[name] = {"status": NOT_VERIFIED, "findings": []} + continue + findings = _VERIFIERS[name](base_url, route, fetch, date_delay) + results[name] = { + "status": summarize(findings), + "findings": [f.as_dict() for f in findings], + } + return results + + +__all__ = [ + "Finding", + "Response", + "TEST_TYPES", + "STAGES", + "PASS", + "WARN", + "FAIL", + "NOT_VERIFIED", + "SKIPPED_BROKEN", + "declared_types", + "types_for_stage", + "verify_variant", +] diff --git a/tools/verify/client.py b/tools/verify/client.py new file mode 100644 index 000000000..f285645e5 --- /dev/null +++ b/tools/verify/client.py @@ -0,0 +1,49 @@ +"""Minimal HTTP GET used by the verifiers (stdlib only).""" + +from __future__ import annotations + +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from typing import Dict, Optional + +ACCEPT = { + "json": "application/json,text/html;q=0.9,application/xhtml+xml;q=0.9," + "application/xml;q=0.8,*/*;q=0.7", + "plaintext": "text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9," + "application/xml;q=0.8,*/*;q=0.7", +} + + +@dataclass +class Response: + status: Optional[int] + headers: Dict[str, str] = field(default_factory=dict) # lower-cased names + body: Optional[bytes] = None + error: Optional[str] = None + + def header(self, name: str) -> Optional[str]: + return self.headers.get(name.lower()) + + +def fetch(url: str, accept: str = "json", timeout: float = 15.0) -> Response: + """GET ``url``. Never raises; transport errors go to ``Response.error``.""" + req = urllib.request.Request(url, headers={"Accept": ACCEPT.get(accept, accept)}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return Response(resp.status, _headers(resp.headers), resp.read()) + except urllib.error.HTTPError as exc: + body = exc.read() if exc.fp is not None else b"" + return Response(exc.code, _headers(exc.headers), body) + except Exception as exc: # connection refused, reset, timeout, bad status line + return Response(None, {}, None, "%s: %s" % (type(exc).__name__, exc)) + + +def _headers(msg) -> Dict[str, str]: + out: Dict[str, str] = {} + if msg is None: + return out + for name, value in msg.items(): + key = name.lower() + out[key] = value if key not in out else out[key] + ", " + value + return out diff --git a/tools/verify/rules.py b/tools/verify/rules.py new file mode 100644 index 000000000..4f7bb7c72 --- /dev/null +++ b/tools/verify/rules.py @@ -0,0 +1,189 @@ +"""Stage A rules: common response checks, json and plaintext (spec 001, R5). + +Clean-room implementation of the documented behaviour of the TFB verifier at +the pinned upstream commit. Each check yields findings at level ``pass``, +``warn`` or ``fail``; a test type fails if any finding fails. +""" + +from __future__ import annotations + +import json +import re +import time +from dataclasses import dataclass +from typing import Callable, List + +from .client import Response + +# Minimum route lengths: "/json" and "/plaintext". +MIN_ROUTE = {"json": 5, "plaintext": 10} + +CONTENT_TYPES = { + "json": re.compile(r"^application/json(; ?charset=(UTF|utf)-8)?$"), + "plaintext": re.compile(r"^text/plain(; ?charset=(UTF|utf)-8)?$"), + "html": re.compile(r"^text/html; ?charset=(UTF|utf)-8$"), +} + +# IMF-fixdate (RFC 7231 section 7.1.1.1). "UTC" is accepted like upstream. +DATE_RE = re.compile( + r"^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} " + r"(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} " + r"\d{2}:\d{2}:\d{2} (GMT|UTC)$" +) + +HELLO = "hello, world!" + + +@dataclass +class Finding: + level: str # "warn" or "fail" + message: str + url: str + + def as_dict(self) -> dict: + return {"level": self.level, "message": self.message, "url": self.url} + + +Fetch = Callable[[str, str], Response] + + +def _fail(msg, url): + return Finding("fail", msg, url) + + +def _warn(msg, url): + return Finding("warn", msg, url) + + +def check_body_present(resp: Response, url: str) -> List[Finding]: + if resp.status is None: + return [_fail("No response (%s)" % (resp.error or "unknown error"), url)] + if not resp.body: + return [_fail("Empty response body", url)] + return [] + + +def check_route(test_type: str, route: str, url: str) -> List[Finding]: + minimum = MIN_ROUTE[test_type] + if len(route) < minimum: + return [ + _fail( + "Route for %s must be at least %d characters, found '%s'" + % (test_type, minimum, route), + url, + ) + ] + return [] + + +def check_status(resp: Response, url: str, expected: int = 200) -> List[Finding]: + if resp.status != expected: + return [_fail("HTTP status %s, expected %d" % (resp.status, expected), url)] + return [] + + +def check_headers( + resp: Response, + url: str, + content_type: str, + refetch: Callable[[], Response], + date_delay: float, +) -> List[Finding]: + """Required headers, Date format, Date not cached, Content-Type.""" + out: List[Finding] = [] + for name in ("Server", "Date", "Content-Type"): + if resp.header(name) is None: + out.append(_fail("Required response header missing: %s" % name, url)) + if resp.header("Content-Length") is None and resp.header("Transfer-Encoding") is None: + out.append( + _fail( + "Required size header missing: include Content-Length or Transfer-Encoding", + url, + ) + ) + + date = resp.header("Date") + if date is not None and not DATE_RE.match(date): + out.append(_warn("Date header '%s' is not an RFC 7231 IMF-fixdate" % date, url)) + + if date_delay: + time.sleep(date_delay) + second = refetch() + if date is not None and second.header("Date") == date: + out.append( + _fail( + "Date header is cached: '%s' on two requests %gs apart" % (date, date_delay), + url, + ) + ) + + ctype = resp.header("Content-Type") + if ctype is not None and not CONTENT_TYPES[content_type].match(ctype): + out.append( + _fail( + "Content-Type '%s' does not match '%s'" + % (ctype, CONTENT_TYPES[content_type].pattern), + url, + ) + ) + return out + + +def check_hello_object(obj, url: str) -> List[Finding]: + if not isinstance(obj, dict) or not all(isinstance(v, str) for v in obj.values()): + return [_fail("Body is not a JSON object of string values", url)] + lowered = {k.lower(): v.lower() for k, v in obj.items()} + if "message" not in lowered: + return [_fail("Missing required key 'message'", url)] + if lowered["message"] != HELLO: + return [_fail("Expected message 'Hello, World!', got '%s'" % obj_message(obj), url)] + extra = [k for k in obj if k.lower() != "message"] + if extra: + return [_warn("Extra JSON keys, consider removing: %s" % ", ".join(extra), url)] + return [] + + +def obj_message(obj) -> str: + for k, v in obj.items(): + if k.lower() == "message": + return v + return "" + + +def verify_json(base_url: str, route: str, fetch: Fetch, date_delay: float = 3.0) -> List[Finding]: + url = base_url + route + resp = fetch(url, "json") + problems = check_body_present(resp, url) + parsed = None + if not problems: + try: + parsed = json.loads(resp.body) + except ValueError as exc: + problems.append(_fail("Invalid JSON: %s" % exc, url)) + problems += check_route("json", route, url) + if problems: + return problems + problems += check_hello_object(parsed, url) + problems += check_status(resp, url) + problems += check_headers(resp, url, "json", lambda: fetch(url, "json"), date_delay) + return problems + + +def verify_plaintext( + base_url: str, route: str, fetch: Fetch, date_delay: float = 3.0 +) -> List[Finding]: + url = base_url + route + resp = fetch(url, "plaintext") + problems = check_body_present(resp, url) + check_route("plaintext", route, url) + if problems: + return problems + body = resp.body.lower() + expected = HELLO.encode() + if expected not in body: + return [_fail("Body does not contain 'Hello, World!'", url)] + extra = len(body) - len(expected) + if extra > 0: + problems.append(_warn("Body has %d byte(s) more than 'Hello, World!'" % extra, url)) + problems += check_status(resp, url) + problems += check_headers(resp, url, "plaintext", lambda: fetch(url, "plaintext"), date_delay) + return problems From 1339090de4ba65402704442269d8bd6fd4ebf6db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20Bregu=C3=AAz?= Date: Mon, 12 Oct 2026 14:33:09 -0300 Subject: [PATCH 2/2] ci: add metadata, changed, verify and full jobs Implements plan section 4 next to the existing docs job: metadata check and unit tests on every push and PR, changed folders plus Dockerfile lint on PRs, a stage-A verify matrix (fail-fast off, smoke sample for tooling changes), and a weekly/dispatchable full run sharded by language with a merged JSON summary artifact. Refs #1 Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ci.yml | 151 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3e3ac51f..ab5d3a164 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,14 @@ on: push: branches: [main] pull_request: + schedule: + - cron: "17 3 * * 1" # weekly full stage-A verification (spec 001, R7.4) + workflow_dispatch: + inputs: + folders: + description: "Space-separated framework folders (e.g. 'Go/gin Rust/axum'); empty means the whole catalog" + type: string + default: "" permissions: contents: read @@ -15,3 +23,146 @@ jobs: # Implementation code (JSONC, BOMs, custom YAML tags) is checked by the catalog CI, not here. syntax-roots: "docs .github" link-globs: "'README.md' 'AGENTS.md' 'docs/**/*.md'" + + metadata: + name: Metadata check and tooling tests + if: github.event_name == 'push' || github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.9" # oldest supported version of tools/ + - name: Metadata check (whole catalog) + run: python3 tools/catalog.py check + - name: Unit tests + run: python3 -m unittest discover -s tests -v + + changed: + name: Changed folders and Dockerfile lint + if: github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.changed.outputs.matrix }} + any: ${{ steps.changed.outputs.any }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # merge base with the PR base + - name: Lint changed Dockerfiles + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: python3 tools/catalog.py lint-docker --since "$BASE_SHA" + - name: Changed framework folders + id: changed + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + python3 tools/catalog.py changed --since "$BASE_SHA" + json="$(python3 tools/catalog.py changed --since "$BASE_SHA" --json)" + echo "$json" + matrix="$(python3 -c 'import json,sys; print(json.dumps(json.loads(sys.argv[1])["matrix"]))' "$json")" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + if [ "$matrix" = "[]" ]; then echo "any=false"; else echo "any=true"; fi >> "$GITHUB_OUTPUT" + + verify: + name: Verify ${{ matrix.target }} (stage A) + needs: changed + if: needs.changed.outputs.any == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + target: ${{ fromJSON(needs.changed.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + - name: Stop preinstalled database services + run: sudo systemctl stop mysql postgresql || true + - name: Verify + env: + TARGET: ${{ matrix.target }} + run: python3 tools/catalog.py verify "$TARGET" --stage A --summary summary.json + - name: Job summary + if: always() + run: | + if [ -f summary.json ]; then + python3 tools/catalog.py summarize summary.json >> "$GITHUB_STEP_SUMMARY" + fi + + full-plan: + name: Plan full verification + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - name: Shards (one per language, or the requested folders) + id: plan + env: + FOLDERS: ${{ inputs.folders }} + run: | + matrix="$(python3 - <<'PY' + import json, os, subprocess + folders = os.environ.get("FOLDERS", "").split() + if not folders: + rows = json.loads(subprocess.check_output( + ["python3", "tools/catalog.py", "list", "--json"])) + folders = sorted({r["language"] for r in rows}) + print(json.dumps(folders)) + PY + )" + echo "$matrix" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + + full: + name: Full verify ${{ matrix.shard }} (stage A) + needs: full-plan + runs-on: ubuntu-24.04 + timeout-minutes: 360 + strategy: + fail-fast: false + max-parallel: 8 + matrix: + shard: ${{ fromJSON(needs.full-plan.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + - name: Stop preinstalled database services + run: sudo systemctl stop mysql postgresql || true + - name: Artifact name + id: name + env: + SHARD: ${{ matrix.shard }} + run: echo "name=summary-$(echo "$SHARD" | tr '/:' '--')" >> "$GITHUB_OUTPUT" + - name: Verify + env: + SHARD: ${{ matrix.shard }} + # --rmi keeps runner disk usage bounded across many builds. + run: python3 tools/catalog.py verify "$SHARD" --stage A --rmi --summary summary.json + - name: Upload shard summary + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.name.outputs.name }} + path: summary.json + if-no-files-found: warn + + full-summary: + name: Full verification summary + needs: full + if: always() && needs.full.result != 'skipped' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: summary-* + path: shards + - name: Merge shard summaries + run: | + python3 tools/catalog.py summarize shards/*/summary.json --out full-summary.json >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v4 + with: + name: full-summary + path: full-summary.json