feat: catalog tooling (check, changed, lint-docker, verify) and CI

.github/workflows/ci.yml

@@ -4,6 +4,14 @@ on:
   push:
     branches: [main]
   pull_request:
+  schedule:
+    - cron: "17 3 * * 1" # weekly full stage-A verification (spec 001, R7.4)
+  workflow_dispatch:
+    inputs:
+      folders:
+        description: "Space-separated framework folders (e.g. 'Go/gin Rust/axum'); empty means the whole catalog"
+        type: string
+        default: ""
 
 permissions:
   contents: read
@@ -15,3 +23,146 @@ jobs:
       # Implementation code (JSONC, BOMs, custom YAML tags) is checked by the catalog CI, not here.
       syntax-roots: "docs .github"
       link-globs: "'README.md' 'AGENTS.md' 'docs/**/*.md'"
+
+  metadata:
+    name: Metadata check and tooling tests
+    if: github.event_name == 'push' || github.event_name == 'pull_request'
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.9" # oldest supported version of tools/
+      - name: Metadata check (whole catalog)
+        run: python3 tools/catalog.py check
+      - name: Unit tests
+        run: python3 -m unittest discover -s tests -v
+
+  changed:
+    name: Changed folders and Dockerfile lint
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-24.04
+    outputs:
+      matrix: ${{ steps.changed.outputs.matrix }}
+      any: ${{ steps.changed.outputs.any }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # merge base with the PR base
+      - name: Lint changed Dockerfiles
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: python3 tools/catalog.py lint-docker --since "$BASE_SHA"
+      - name: Changed framework folders
+        id: changed
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          python3 tools/catalog.py changed --since "$BASE_SHA"
+          json="$(python3 tools/catalog.py changed --since "$BASE_SHA" --json)"
+          echo "$json"
+          matrix="$(python3 -c 'import json,sys; print(json.dumps(json.loads(sys.argv[1])["matrix"]))' "$json")"
+          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
+          if [ "$matrix" = "[]" ]; then echo "any=false"; else echo "any=true"; fi >> "$GITHUB_OUTPUT"
+
+  verify:
+    name: Verify ${{ matrix.target }} (stage A)
+    needs: changed
+    if: needs.changed.outputs.any == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        target: ${{ fromJSON(needs.changed.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Stop preinstalled database services
+        run: sudo systemctl stop mysql postgresql || true
+      - name: Verify
+        env:
+          TARGET: ${{ matrix.target }}
+        run: python3 tools/catalog.py verify "$TARGET" --stage A --summary summary.json
+      - name: Job summary
+        if: always()
+        run: |
+          if [ -f summary.json ]; then
+            python3 tools/catalog.py summarize summary.json >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+  full-plan:
+    name: Plan full verification
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-24.04
+    outputs:
+      matrix: ${{ steps.plan.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Shards (one per language, or the requested folders)
+        id: plan
+        env:
+          FOLDERS: ${{ inputs.folders }}
+        run: |
+          matrix="$(python3 - <<'PY'
+          import json, os, subprocess
+          folders = os.environ.get("FOLDERS", "").split()
+          if not folders:
+              rows = json.loads(subprocess.check_output(
+                  ["python3", "tools/catalog.py", "list", "--json"]))
+              folders = sorted({r["language"] for r in rows})
+          print(json.dumps(folders))
+          PY
+          )"
+          echo "$matrix"
+          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
+
+  full:
+    name: Full verify ${{ matrix.shard }} (stage A)
+    needs: full-plan
+    runs-on: ubuntu-24.04
+    timeout-minutes: 360
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix:
+        shard: ${{ fromJSON(needs.full-plan.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Stop preinstalled database services
+        run: sudo systemctl stop mysql postgresql || true
+      - name: Artifact name
+        id: name
+        env:
+          SHARD: ${{ matrix.shard }}
+        run: echo "name=summary-$(echo "$SHARD" | tr '/:' '--')" >> "$GITHUB_OUTPUT"
+      - name: Verify
+        env:
+          SHARD: ${{ matrix.shard }}
+        # --rmi keeps runner disk usage bounded across many builds.
+        run: python3 tools/catalog.py verify "$SHARD" --stage A --rmi --summary summary.json
+      - name: Upload shard summary
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.name.outputs.name }}
+          path: summary.json
+          if-no-files-found: warn
+
+  full-summary:
+    name: Full verification summary
+    needs: full
+    if: always() && needs.full.result != 'skipped'
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: summary-*
+          path: shards
+      - name: Merge shard summaries
+        run: |
+          python3 tools/catalog.py summarize shards/*/summary.json --out full-summary.json >> "$GITHUB_STEP_SUMMARY"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: full-summary
+          path: full-summary.json

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+*.pyc

tests/fixtures/bad-json/Go/broken-json/benchmark_config.json

@@ -0,0 +1 @@
+{"framework": "broken-json", "tests": [

tests/fixtures/missing-dockerfile/Go/nodf/benchmark_config.json

@@ -0,0 +1 @@
+{"framework": "nodf", "tests": [{"default": {"json_url": "/json", "port": 8080, "database": "None"}}]}

tests/fixtures/no-urls/Go/nourls/benchmark_config.json

@@ -0,0 +1 @@
+{"framework": "nourls", "tests": [{"default": {"port": 8080, "database": "None"}}]}

tests/fixtures/no-urls/Go/nourls/nourls.dockerfile

@@ -0,0 +1 @@
+FROM golang:1.22

tests/fixtures/valid/Go/hello/benchmark_config.json

@@ -0,0 +1,29 @@
+{
+  "framework": "hello",
+  "tests": [{
+    "default": {
+      "json_url": "/json",
+      "plaintext_url": "/plaintext",
+      "db_url": "/db",
+      "port": 8080,
+      "database": "Postgres"
+    },
+    "alt": {
+      "json_url": "/json",
+      "port": "8081",
+      "database": "MySQL"
+    },
+    "custom": {
+      "plaintext_url": "/plaintext",
+      "port": 8080,
+      "database": "None",
+      "dockerfile": "custom.dockerfile"
+    },
+    "old": {
+      "json_url": "/json",
+      "port": 8080,
+      "database": "mongodb",
+      "tags": ["broken"]
+    }
+  }]
+}

tests/fixtures/valid/Go/hello/custom.dockerfile

@@ -0,0 +1,2 @@
+FROM scratch
+CMD ["/app"]

tests/fixtures/valid/Go/hello/hello-alt.dockerfile

@@ -0,0 +1,2 @@
+FROM golang:1.22
+CMD ["/app"]

tests/fixtures/valid/Go/hello/hello-old.dockerfile

@@ -0,0 +1,2 @@
+FROM golang:1.22
+CMD ["/app"]

tests/fixtures/valid/Go/hello/hello.dockerfile

@@ -0,0 +1,3 @@
+FROM golang:1.22 AS build
+FROM build AS final
+CMD ["/app"]

tests/fixtures/valid/Python/dbonly/benchmark_config.json

@@ -0,0 +1,11 @@
+{
+  "framework": "dbonly",
+  "tests": [{
+    "default": {
+      "db_url": "/db",
+      "fortune_url": "/fortunes",
+      "port": 8080,
+      "database": "postgres"
+    }
+  }]
+}

tests/fixtures/valid/Python/dbonly/dbonly.dockerfile

@@ -0,0 +1,2 @@
+FROM python:3.12-slim
+CMD ["python", "app.py"]

tests/helpers.py

@@ -0,0 +1,99 @@
+"""Shared test helpers: import path, fixtures and a stub HTTP server."""
+
+from __future__ import annotations
+
+import contextlib
+import email.utils
+import io
+import itertools
+import sys
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+TESTS = Path(__file__).resolve().parent
+REPO = TESTS.parent
+FIXTURES = TESTS / "fixtures"
+sys.path.insert(0, str(REPO / "tools"))
+
+import catalog  # noqa: E402,F401
+import verify  # noqa: E402,F401
+
+_clock = itertools.count(1_700_000_000)
+
+
+def fresh_date() -> str:
+    """A different, valid IMF-fixdate on every call (no sleeping needed)."""
+    return email.utils.formatdate(next(_clock), usegmt=True)
+
+
+def response(body=b"", status=200, content_type="application/json", **overrides):
+    """Build a stub response. Header overrides: pass None to omit a header."""
+    if isinstance(body, str):
+        body = body.encode()
+    headers = {
+        "Server": "stub",
+        "Date": fresh_date,
+        "Content-Type": content_type,
+        "Content-Length": str(len(body)),
+    }
+    for name, value in overrides.items():
+        headers[name.replace("_", "-")] = value
+    return status, headers, body
+
+
+class StubServer:
+    """Local HTTP server; ``routes`` maps path -> callable returning response()."""
+
+    def __init__(self, routes):
+        self.routes = routes
+        stub = self
+
+        class Handler(BaseHTTPRequestHandler):
+            protocol_version = "HTTP/1.1"
+
+            def do_GET(self):  # noqa: N802
+                route = stub.routes.get(self.path)
+                if route is None:
+                    status, headers, body = response(b"not found", 404, "text/plain")
+                else:
+                    status, headers, body = route()
+                self.send_response_only(status)
+                for name, value in headers.items():
+                    if value is None:
+                        continue
+                    self.send_header(name, value() if callable(value) else value)
+                if headers.get("Content-Length") is None and headers.get("Transfer-Encoding") is None:
+                    self.send_header("Connection", "close")
+                    self.close_connection = True
+                self.end_headers()
+                self.wfile.write(body)
+
+            def log_message(self, *args):
+                pass
+
+        self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+        self.url = "http://127.0.0.1:%d" % self.httpd.server_address[1]
+        self.thread = threading.Thread(
+            target=self.httpd.serve_forever, kwargs={"poll_interval": 0.02}, daemon=True)
+
+    def __enter__(self):
+        self.thread.start()
+        return self
+
+    def __exit__(self, *exc):
+        self.httpd.shutdown()
+        self.httpd.server_close()
+
+
+@contextlib.contextmanager
+def captured():
+    out, err = io.StringIO(), io.StringIO()
+    with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
+        yield out, err
+
+
+def run_cli(*argv):
+    with captured() as (out, err):
+        code = catalog.main(list(argv))
+    return code, out.getvalue(), err.getvalue()