diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..ba42c07 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,36 @@ +# The tooling's own tests. Nothing here reads a release file: the release and +# amendment checks validate what is submitted. +name: checks + +on: + pull_request: + push: + branches: [main] + schedule: + # The examples job talks to real release hosts. It runs on its own clock, + # not in front of a pull request. + - cron: "37 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + tests: + name: tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - run: python3 -m unittest discover -s tools -t tools -v + + examples: + name: examples + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + # Re-derives the design repository's hand-stamped examples/ from the real + # release hosts and diffs. + - env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 tools/verify_examples.py --check-mirrors diff --git a/.github/workflows/watcher.yml b/.github/workflows/watcher.yml new file mode 100644 index 0000000..8ccc467 --- /dev/null +++ b/.github/workflows/watcher.yml @@ -0,0 +1,129 @@ +# Ask every listing's authority host for its releases and stamp every release +# that has no file under releases// yet (RFC 0033). +# +# The tick holds no state. What is stamped in the repository is the state, so a +# dropped run costs latency and not data, and a re-run stamps nothing twice. +name: watcher + +on: + schedule: + # Ten minutes is a tuning parameter and needs no RFC to change. GitHub's + # floor is five. The times sit off the hour, where runs are dropped worst. + - cron: "4,14,24,34,44,54 * * * *" + workflow_dispatch: + inputs: + listing: + description: "Only this listing id. Empty means every listing." + required: false + lookback_days: + description: "Ignore releases older than this many days. 0 scans the whole list." + required: false + default: "0" + dry_run: + description: "Derive everything, write and open nothing." + type: boolean + default: false + +permissions: + contents: read + +concurrency: + # Two ticks writing release files at once would race on the push, and the + # scan rule makes a queued tick as good as a parallel one. + group: watcher + cancel-in-progress: false + +jobs: + tick: + name: tick + runs-on: ubuntu-latest + steps: + # The one identity with access to both halves of the index, and the only + # actor the branch protection ruleset lets write here unattended. + - uses: actions/create-github-app-token@v2 + id: app-token + with: + app-id: ${{ vars.INDEX_APP_ID }} + private-key: ${{ secrets.INDEX_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: | + content-index-releases + content-index + + - uses: actions/checkout@v7 + with: + # Always the tip of main: an old checkout would decide "already + # stamped" against an outdated tree and then fail to push. + ref: main + token: ${{ steps.app-token.outputs.token }} + + # The authored documents the tick reads. A checkout costs one clone + # instead of one API request per listing. + - uses: actions/checkout@v7 + with: + repository: ${{ github.repository_owner }}/content-index + ref: main + path: .authored + token: ${{ steps.app-token.outputs.token }} + + # Derived cache, never state: the per-listing ETags, the failure counts + # and the issue numbers. Losing it costs one expensive tick. + - name: Restore the derived cache + uses: actions/cache/restore@v4 + with: + path: .watcher + key: watcher-${{ github.run_id }} + restore-keys: watcher- + + - name: Tick + env: + GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} + LISTING: ${{ inputs.listing }} + LOOKBACK_DAYS: ${{ inputs.lookback_days || '0' }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + set -euo pipefail + git config user.name "${{ steps.app-token.outputs.app-slug }}[bot]" + git config user.email "${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com" + + arguments=( + --authored .authored + --authored-repo "${{ github.repository_owner }}/content-index" + --cache .watcher/cache.json + --lookback-days "$LOOKBACK_DAYS" + # The workflow the sweep re-dispatches, and the check whose verdict + # it reads. Both belong to KSAModding/content-index#4, which has to + # accept a pull_request input and report under this check name. + --sweep-workflow checks.yml + --verdict-check validate + ) + if [ -n "$LISTING" ]; then + arguments+=(--listing "$LISTING") + fi + if [ "$DRY_RUN" = "true" ]; then + arguments+=(--dry-run) + fi + + python3 tools/watch.py "${arguments[@]}" + + - name: Push + if: ${{ success() && inputs.dry_run != true }} + run: | + set -euo pipefail + for attempt in 1 2 3; do + if git push; then + exit 0 + fi + # Something else landed on main, the hourly game versions job most + # likely. Release files never conflict with it. + git pull --rebase --autostash + done + echo "could not push after three attempts; the next tick rescans and re-stamps" + exit 1 + + - name: Save the derived cache + if: ${{ success() && inputs.dry_run != true }} + uses: actions/cache/save@v4 + with: + path: .watcher + key: watcher-${{ github.run_id }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea20022 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +# The authored repository, checked out beside this one by the watcher. +/.authored/ + +# The watcher's derived cache: ETags, failure counts, issue numbers. Rebuildable +# from the repository and the hosts, so it is never committed. +/.watcher/ + +__pycache__/ diff --git a/README.md b/README.md index cb12d05..a233b89 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,34 @@ It is seeded from `Content/Versions/`, the dated history every installed copy of That poll only ever sees the build that is current when it runs, so a build superseded within the hour can be missing from it. The copy on your own disk stays the complete source. +## The watcher + +`.github/workflows/watcher.yml` runs every ten minutes as the org App. +Each tick asks every listing's authority host for its releases and stamps every release that has no file under `releases//` yet, so a release published out of version order is stamped too. + +There is no queue. What is stamped here is the whole of the watcher's state, which is why a tick GitHub delays, drops or cancels costs latency and not data, and why a re-run stamps nothing twice. + +| Tool | What it does | +|---|---| +| `tools/stamp_release.py` | Authored document plus release archive in, release file out. The one place a release file is derived, shared with the release pull request checks so the two paths cannot disagree. Needs no token. | +| `tools/hosts.py` | The release hosts, GitHub and SpaceDock, behind one interface. GitHub is polled conditionally against a stored ETag, so an unchanged listing costs no rate limit at all. | +| `tools/watch.py` | One tick: scan, stamp, commit, append a mirror that only appeared later, keep one error issue per listing current on the authored repository, and sweep its open pull requests. | +| `tools/verify_examples.py` | Re-derives the design repository's hand-stamped `examples/` from their release hosts and diffs. | + +A release the watcher cannot stamp, a tag that does not parse or an archive whose install root is neither derivable nor authored, becomes one open issue per listing on the authored repository, kept current rather than reopened every tick. + +A version is stamped exactly once. A tag that reappears with different bytes is rejected and never overwritten, and both hashes are named in that issue. + +`download.mirrors` is the one field the watcher may append to after publish, and only after downloading the other host's archive and finding it byte-identical. + +An authored `game_max` naming a month that was still running at stamp time is stamped with no upper bound, and a later tick resolves and adds the bound once the month completes. + +To run a tick by hand, dispatch the workflow: `listing` narrows it to one id, and `dry_run` derives everything and writes nothing. Locally, against a checkout of the authored half: + +```text +python3 tools/watch.py --authored ../content-index --dry-run +``` + ## A published release is immutable Identity, the version, the download and the install data never change. diff --git a/tools/hosts.py b/tools/hosts.py new file mode 100644 index 0000000..78ebf26 --- /dev/null +++ b/tools/hosts.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""The release hosts the watcher polls, behind one interface. + +A host answers which releases exist and what the bytes of one of them are. The +authority from `[releases]` defines which releases exist; every other host is +only checked for a byte-identical archive, which is how `download.mirrors` gets +populated (RFC 0031, RFC 0033). + +The two failure kinds are reported differently and must not be confused: +HostError means this tick could not evaluate the host and the next one rescans, +StampError means the release itself is wrong and the author has to act. + +GitHub is polled conditionally against a stored ETag; SpaceDock serves no +validator, so a SpaceDock authority costs one request per tick. +""" + +import dataclasses +import json +import re +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timezone + +from stamp_release import StampError, normalize_version + +GITHUB_API = "https://api.github.com" +SPACEDOCK = "https://spacedock.info" + +USER_AGENT = "KSAModding-content-index-watcher" + +ARCHIVE_CONTENT_TYPES = frozenset( + {"application/zip", "application/x-zip-compressed", "application/octet-stream"} +) + +# 512 MiB. A mod archive is orders of magnitude smaller, and a runner that +# streams something enormous has already lost the tick for every other listing. +MAX_ARCHIVE_BYTES = 512 * 1024 * 1024 + +LINK_NEXT = re.compile(r'<([^>]+)>;\s*rel="next"') + + +class HostError(Exception): + """The host could not be evaluated this tick. Transient by assumption.""" + + +class OversizeError(HostError): + """The response blew the size limit. Permanent for an archive, so `_download` + turns it into a StampError.""" + + +@dataclasses.dataclass(frozen=True) +class HostRelease: + """One release as a host describes it, before anything is derived from it.""" + + host: str + tag: str + version: str | None + release_date: str | None + url: str | None + content_type: str = "application/zip" + size: int | None = None + prerelease: bool = False + changelog: str | None = None + asset_name: str | None = None + # The archives the host offered when none could be picked, so the error the + # author reads names them instead of claiming there was nothing there. + candidates: tuple = () + + def facts(self): + """The release facts the stamper takes.""" + return { + "tag": self.tag, + "release_date": self.release_date, + "url": self.url, + "content_type": self.content_type, + "prerelease": self.prerelease, + "changelog": self.changelog, + } + + +@dataclasses.dataclass(frozen=True) +class Response: + status: int + headers: dict + body: bytes + + +class Http: + """Plain urllib with the retry and rate limit behavior a tick needs.""" + + def __init__(self, token=None, timeout=60, retries=3, log=None): + self.token = token + self.timeout = timeout + self.retries = retries + self.log = log or (lambda message: None) + self.requests = 0 + + def get(self, url, accept=None, etag=None, api=False, limit=None): + """GET `url`, returning a Response. A 304 comes back with an empty body. + + Raises HostError for anything transient and urllib's HTTPError for a + status the caller has to interpret itself, such as 404. + """ + headers = {"User-Agent": USER_AGENT} + if accept: + headers["Accept"] = accept + if etag: + headers["If-None-Match"] = etag + if api and self.token: + headers["Authorization"] = f"Bearer {self.token}" + headers["X-GitHub-Api-Version"] = "2022-11-28" + + last = None + for attempt in range(self.retries): + request = urllib.request.Request(url, headers=headers, method="GET") + try: + self.requests += 1 + with urllib.request.urlopen(request, timeout=self.timeout) as answer: + return Response( + answer.status, dict(answer.headers), _read(answer, limit) + ) + except urllib.error.HTTPError as error: + if error.code == 304: + return Response(304, dict(error.headers), b"") + if error.code in (403, 429) and _rate_limited(error.headers): + raise HostError(f"{url}: rate limited by the host") from error + if error.code < 500 and error.code != 429: + raise + last = error + except (urllib.error.URLError, TimeoutError, OSError) as error: + last = error + + if attempt + 1 < self.retries: + time.sleep(2 ** attempt) + + raise HostError(f"{url}: {last}") + + +def _read(answer, limit): + limit = limit or MAX_ARCHIVE_BYTES + body = answer.read(limit + 1) + if len(body) > limit: + raise OversizeError(f"the response is larger than the {limit} byte limit") + return body + + +def _rate_limited(headers): + lower = {key.lower(): value for key, value in headers.items()} + return lower.get("x-ratelimit-remaining") == "0" or "retry-after" in lower + + +def _utc(timestamp): + """A host timestamp as the ISO 8601 UTC form a release file carries. + + A timestamp that does not parse yields None rather than passing the raw + string through: `release_date` is stamped exactly once, and the stamper + rejects a release without one, which scopes the failure to that release. + """ + if not timestamp: + return None + text = timestamp.strip().replace("Z", "+00:00") + try: + moment = datetime.fromisoformat(text) + except ValueError: + return None + if moment.tzinfo is None: + moment = moment.replace(tzinfo=timezone.utc) + return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _version_of(tag): + try: + return normalize_version(tag) + except StampError: + return None + + +def _parse_json(url, body): + """The body as JSON, or HostError: a 200 carrying HTML is a bad moment.""" + try: + return json.loads(body) + except json.JSONDecodeError as error: + raise HostError(f"{url}: the answer is not JSON, {error}") from error + + +def _on_host(base, value, what): + """`value` resolved against `base`, and only if it stayed on that host. + + `urljoin` returns an absolute or protocol-relative value unchanged, so a + host that answers `https://elsewhere/x.zip` would otherwise put a foreign + address into a stamped file. The index never trusts a fact it was handed, + and a URL is a fact like any other. + """ + if not value: + return None + resolved = urllib.parse.urljoin(base, str(value)) + wanted, got = urllib.parse.urlsplit(base), urllib.parse.urlsplit(resolved) + if (got.scheme, got.netloc) != (wanted.scheme, wanted.netloc): + raise StampError( + f"{what} '{value}' resolves to {resolved}, which is not on " + f"{wanted.netloc}" + ) + return resolved + + +class Host: + """A release host of one listing.""" + + kind = "" + + @property + def key(self): + """The cache key of this host's release list.""" + raise NotImplementedError + + def releases(self, etag=None): + """(releases, etag), or (None, etag) when the host answers 'unchanged'.""" + raise NotImplementedError + + def download(self, release): + """The archive's bytes, and the content type the host serves them as.""" + raise NotImplementedError + + +class GitHubHost(Host): + """A GitHub repository's releases. Polled conditionally, drafts ignored.""" + + kind = "github" + + def __init__(self, repository, http, listing_id=None, max_pages=5): + self.repository = str(repository).strip("/") + self.http = http + self.listing_id = listing_id or self.repository.split("/")[-1] + self.max_pages = max_pages + # True when the last scan hit max_pages with more pages left, so the + # caller can report the tail instead of silently never seeing it. + self.truncated = False + + @property + def key(self): + return f"github:{self.repository.lower()}" + + def releases(self, etag=None): + url = f"{GITHUB_API}/repos/{self.repository}/releases?per_page=100" + try: + first = self.http.get( + url, accept="application/vnd.github+json", etag=etag, api=True + ) + except urllib.error.HTTPError as error: + if error.code in (404, 451): + raise StampError( + f"the authority host has no repository '{self.repository}' the " + "watcher can read: it was renamed, made private, or removed" + ) from error + raise HostError(f"{url}: HTTP {error.code}") from error + + if first.status == 304: + return None, etag + + payloads = [_parse_json(url, first.body)] + following = LINK_NEXT.search(first.headers.get("Link", "") or "") + pages = 1 + while following and pages < self.max_pages: + try: + answer = self.http.get( + following.group(1), accept="application/vnd.github+json", api=True + ) + except urllib.error.HTTPError as error: + raise HostError(f"{following.group(1)}: HTTP {error.code}") from error + payloads.append(_parse_json(following.group(1), answer.body)) + following = LINK_NEXT.search(answer.headers.get("Link", "") or "") + pages += 1 + self.truncated = bool(following) + if following: + self.http.log( + f"{self.repository}: more than {self.max_pages * 100} releases, " + "the older ones are not scanned" + ) + + releases = [ + self._release(payload) + for page in payloads + for payload in page + if not payload.get("draft") + ] + return releases, first.headers.get("ETag") or etag + + def _release(self, payload): + tag = payload.get("tag_name") or "" + asset, candidates = self._asset(payload) + return HostRelease( + host=self.kind, + tag=tag, + version=_version_of(tag), + release_date=_utc(payload.get("published_at") or payload.get("created_at")), + url=asset.get("browser_download_url") if asset else None, + content_type=(asset or {}).get("content_type") or "application/zip", + size=(asset or {}).get("size"), + prerelease=bool(payload.get("prerelease")), + changelog=payload.get("html_url"), + asset_name=(asset or {}).get("name"), + candidates=() if asset else tuple(candidates), + ) + + def _asset(self, payload): + """The release's archive. + + One archive is the normal case. Where a release carries several, the one + named after the listing wins, because that is what every archive in the + index is named. Anything still ambiguous is reported to the author + rather than guessed at: picking the wrong asset would stamp a hash + clients then verify against the wrong file. + """ + uploaded = [ + asset + for asset in payload.get("assets") or [] + if asset.get("state") == "uploaded" + ] + assets = [ + asset for asset in uploaded if asset.get("name", "").lower().endswith(".zip") + ] or [ + asset + for asset in uploaded + if asset.get("content_type") in ("application/zip", "application/x-zip-compressed") + ] + names = [asset.get("name", "") for asset in assets] + if len(assets) <= 1: + return (assets[0] if assets else None), names + + identifier = self.listing_id.lower() + tag = (payload.get("tag_name") or "").lstrip("vV").lower() + for wanted in (f"{identifier}.zip", f"{identifier}-{tag}.zip", f"{identifier}_{tag}.zip"): + for asset in assets: + if asset["name"].lower() == wanted: + return asset, names + return None, names + + def download(self, release): + return _download(self.http, release) + + +class SpaceDockHost(Host): + """A SpaceDock mod's versions. + + SpaceDock serves no ETag on its API, so a SpaceDock host costs one request + per tick. It also has no draft or pre-release flag: every version it lists + is a published one. + """ + + kind = "spacedock" + + def __init__(self, mod_id, http): + try: + self.mod_id = int(mod_id) + except (TypeError, ValueError): + raise StampError( + f"'{mod_id}' is not a SpaceDock mod id, which is a number" + ) from None + self.http = http + + @property + def key(self): + return f"spacedock:{self.mod_id}" + + def releases(self, etag=None): + url = f"{SPACEDOCK}/api/mod/{self.mod_id}" + try: + answer = self.http.get(url, accept="application/json") + except urllib.error.HTTPError as error: + if error.code == 404: + raise StampError( + f"SpaceDock has no mod {self.mod_id}" + ) from error + raise HostError(f"{url}: HTTP {error.code}") from error + + payload = _parse_json(url, answer.body) + + page = payload.get("url") or f"/mod/{self.mod_id}" + changelog = _on_host(SPACEDOCK, page, "the mod page") + releases = [] + for version in payload.get("versions") or []: + tag = (version.get("friendly_version") or "").strip() + releases.append( + HostRelease( + host=self.kind, + tag=tag, + version=_version_of(tag), + release_date=_utc(version.get("created")), + url=_on_host(SPACEDOCK, version.get("download_path"), "the download path"), + content_type="application/zip", + prerelease=False, + changelog=changelog, + ) + ) + return releases, None + + def download(self, release): + return _download(self.http, release) + + +def _download(http, release): + if not release.url: + if release.candidates: + raise StampError( + f"the release carries {len(release.candidates)} archives and none of them " + f"is named after the listing ({', '.join(release.candidates)}), so there " + "is nothing to stamp: the watcher does not guess which archive a client " + "should verify against" + ) + raise StampError("the release carries no archive to download") + if release.size and release.size > MAX_ARCHIVE_BYTES: + raise StampError( + f"the archive is {release.size} bytes, above the " + f"{MAX_ARCHIVE_BYTES} byte limit" + ) + try: + answer = http.get(release.url, api=release.url.startswith(GITHUB_API)) + except OversizeError as error: + # Permanent, unlike the transient failures HostError stands for: the + # release stays too large next tick too, so the author hears about it + # instead of the watcher downloading and discarding it forever. + raise StampError(f"the archive at {release.url}: {error}") from error + except urllib.error.HTTPError as error: + # A gone archive is a fact about the release, reported to the author. + # Everything else is the host having a bad moment this tick. + if error.code in (404, 410, 451): + raise StampError( + f"the archive at {release.url} is gone (HTTP {error.code})" + ) from error + raise HostError(f"{release.url}: HTTP {error.code}") from error + served = (answer.headers.get("Content-Type") or "").split(";")[0].strip() + # What the host says the asset is beats what it happens to serve it as, and + # the stamper has the bytes to fall back on either way. + content_type = release.content_type or served + return answer.body, content_type + + +def build(releases_section, http, listing_id=None): + """The hosts of one listing, and its authority. + + Returns (authority, mirrors). With one host key that host is the authority; + with several, `authority` names which one, and the rest are mirror + candidates. No `[releases]` section at all means the listing does not enter + the index through the watcher, and this returns (None, []). + """ + section = releases_section or {} + named = {} + if section.get("github"): + named["github"] = GitHubHost(section["github"], http, listing_id) + if section.get("spacedock"): + named["spacedock"] = SpaceDockHost(section["spacedock"], http) + + if not named: + return None, [] + + if len(named) == 1: + (authority,) = named.values() + return authority, [] + + chosen = section.get("authority") + if chosen not in named: + raise StampError( + "[releases] names several hosts, so it needs an 'authority' key naming " + f"one of {', '.join(sorted(named))}" + ) + return named[chosen], [host for name, host in sorted(named.items()) if name != chosen] diff --git a/tools/stamp_release.py b/tools/stamp_release.py new file mode 100644 index 0000000..903d94c --- /dev/null +++ b/tools/stamp_release.py @@ -0,0 +1,711 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Authored document plus release archive in, release file out, per RFC 0031. + +The one place a release file is derived: the watcher stamps with it and the +release pull request checks re-derive with it, so the two paths cannot +disagree. It needs no token and no network, which is what makes it testable +against examples/ in the design repository. + +Anything wrong with the release itself raises StampError for the caller to +report. + +RFC 0031 defines the format, RFC 0035 the install descriptor, RFC 0017 the +version ordering the month bounds resolve against. +""" + +import argparse +import hashlib +import io +import json +import posixpath +import re +import sys +import tomllib +import zipfile +from datetime import datetime, timezone +from pathlib import Path + +SPEC_VERSION = 1 + +# SemVer 2.0.0, from semver.org, with the leading `v` a tag is allowed to carry. +SEMVER = re.compile( + r"^v?(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)" + r"(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)" + r"(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?" + r"(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" +) + +# A KSA production version as the game displays it (RFC 0017). Only the fourth +# component orders, and it is readable straight off the string, so a full bound +# needs no list lookup at all. +GAME_VERSION = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)\.(\d+)(?:-[^+]+)?(?:\+.*)?$") + +# A month bound, "the whole of that month" (RFC 0017). +GAME_MONTH = re.compile(r"^(\d{4})\.(\d{1,2})$") + +# Pre-release identifiers that mean "not a release at all", so a nightly does +# not look like one. Anything else pre-release is a testing build. +DEV_IDENTIFIERS = frozenset({"dev", "nightly", "snapshot", "canary", "ci", "git", "pre"}) + +MOD_TOML = "mod.toml" + +# The listing facts a release file freezes, in the order RFC 0031 lists them. +# `status` and `superseded_by` are deliberately not here: deprecation has to +# reach every release the moment it is declared, so a client reads it live. +LISTING_FIELDS = ("name", "authors", "abstract", "description", "license", "tags") + +DEPENDENCY_KINDS = ("required", "optional", "recommends", "suggests", "conflict") + +INSTALL_ANCHORS = ("mods", "user-data", "game-root", "standalone") + +# The id rules of RFC 0031: 1 to 64 ASCII characters, letters, digits, `-`, +# `_`, `.`, first and last a letter or digit. The id is a folder name on every +# platform and a path segment in this repository, so nothing else is safe. +ID_PATTERN = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9._-]{0,62}[A-Za-z0-9])?$") + +# Reserved case-insensitively against the id up to its first dot: the game +# ships `Content/Core`, and Windows treats dotted device names as devices. +RESERVED_IDS = frozenset( + {"core", "con", "prn", "aux", "nul"} + | {f"com{digit}" for digit in "123456789"} + | {f"lpt{digit}" for digit in "123456789"} +) + + +def valid_id(identifier): + """Whether the id satisfies the id rules of RFC 0031.""" + if not identifier or ID_PATTERN.match(identifier) is None: + return False + return identifier.split(".")[0].lower() not in RESERVED_IDS + + +class StampError(Exception): + """The release cannot be stamped, and the caller reports it. + + Not a transient failure: a host that is down or an archive that did not + download is the caller's business, and never reaches this module. + """ + + +def normalize_version(tag): + """The tag as SemVer 2.0.0, with a leading `v` stripped. + + Raises StampError when it does not parse, which is what rejects the release + at publish time with the error in front of the author. + """ + match = SEMVER.match((tag or "").strip()) + if match is None: + raise StampError(f"version '{tag}' does not parse as SemVer 2.0.0") + + version = "{major}.{minor}.{patch}".format(**match.groupdict()) + if match.group("prerelease"): + version += "-" + match.group("prerelease") + if match.group("build"): + version += "+" + match.group("build") + return version + + +def prerelease_identifiers(version): + """The dot-separated pre-release identifiers of a version, lowercased.""" + match = SEMVER.match(version) + part = match.group("prerelease") if match else None + return [identifier.lower() for identifier in part.split(".")] if part else [] + + +def release_status(version, host_prerelease): + """`stable`, `testing`, or `dev`, from the host's flag and the version. + + A pre-release identifier that names a development stream wins over the + host's flag, because a nightly marked as a normal release is still a + nightly. + """ + identifiers = prerelease_identifiers(version) + if DEV_IDENTIFIERS.intersection(identifiers): + return "dev" + if identifiers or host_prerelease: + return "testing" + return "stable" + + +def game_revision(version): + """The revision of a full game version string, the only component that orders.""" + match = GAME_VERSION.match(version.strip()) + return int(match.group(4)) if match else None + + +def month_of(version): + """The (year, month) a full game version string displays, or None.""" + match = GAME_VERSION.match(version.strip()) + return (int(match.group(1)), int(match.group(2))) if match else None + + +def month_is_over(year, month, now): + """Whether that calendar month has completed, as of `now` (UTC).""" + return (now.year, now.month) > (year, month) + + +def resolve_bound(bound, which, game_versions, now): + """Resolve an authored game bound to (display, revision). + + A full version string carries its own revision (RFC 0017); a month resolves + to its first revision as a lower bound and its last as an upper one. + + An upper bound naming a month still running has no last revision yet, so it + returns (None, None) and the release is stamped open. `Watcher.month_pass` + adds the bound once the month completes. + """ + bound = (bound or "").strip() + if not bound: + return None, None + + revision = game_revision(bound) + if revision is not None: + return bound, revision + + match = GAME_MONTH.match(bound) + if match is None: + raise StampError(f"{which} '{bound}' is neither a game version nor a month") + + year, month = int(match.group(1)), int(match.group(2)) + revisions = sorted( + game_revision(version) + for version in game_versions + if month_of(version) == (year, month) + ) + + if which == "game_max" and not month_is_over(year, month, now): + # Stamped open, and re-resolved once the month completes. + return None, None + + if not revisions: + raise StampError( + f"{which} '{bound}' names a month with no build in the game release list" + ) + + revision = revisions[0] if which == "game_min" else revisions[-1] + display = next( + version + for version in game_versions + if month_of(version) == (year, month) and game_revision(version) == revision + ) + return display, revision + + +def open_archive(archive): + """A ZipFile over the archive bytes. Anything unreadable is a StampError.""" + try: + return zipfile.ZipFile(io.BytesIO(archive)) + except zipfile.BadZipFile as error: + raise StampError(f"the archive is not a readable zip, {error}") from error + + +def top_level_directories(handle): + """The names of the directories at the root of the archive, in archive order.""" + seen = [] + for entry in handle.namelist(): + name = entry.replace("\\", "/") + head, _, rest = name.partition("/") + if not head or (not rest and not name.endswith("/")): + continue # A file sitting at the archive root. + if head not in seen: + seen.append(head) + return seen + + +def entries_under(handle, root): + """The file entries whose path lies under `root`, which may be the archive root.""" + prefix = f"{root}/" if root else "" + return [ + info + for info in handle.infolist() + if not info.is_dir() and info.filename.replace("\\", "/").startswith(prefix) + ] + + +def read_mod_toml(handle, root): + """The archive's own mod.toml, parsed, or None when it carries none. + + An archive without one, which is what a mod-loader archive looks like, is + not an error and contributes no code dependencies. + """ + name = f"{root}/{MOD_TOML}" if root else MOD_TOML + try: + raw = handle.read(name) + except KeyError: + return None + try: + return tomllib.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, tomllib.TOMLDecodeError) as error: + raise StampError(f"the archive's {name} is not valid TOML, {error}") from error + + +def derive_root(handle, listing_id, content_type): + """The install root derived from the archive's layout. + + For a mod, RFC 0031's rule: one top-level directory containing mod.toml, + its name matching the id, because that name is the identity the game will + see. For every other type the derived root is the archive root itself + (RFC 0035, rule 9), which this returns as "". + + Returns None when nothing is derivable, which needs an authored `root`. + """ + if content_type != "mod": + return "" + + directories = top_level_directories(handle) + with_manifest = [ + name for name in directories if f"{name}/{MOD_TOML}" in handle.namelist() + ] + + candidates = with_manifest or directories + if len(candidates) != 1: + return None + + name = candidates[0] + if name == listing_id: + return name + if name.lower() == listing_id.lower(): + raise StampError( + f"the archive's top-level directory is '{name}' and the id is " + f"'{listing_id}': the folder name is the identity the game sees, " + "so the casing has to match" + ) + raise StampError( + f"the archive's top-level directory is '{name}', which does not match " + f"the id '{listing_id}'" + ) + + +def relative_path(value, what): + """The value as the relative `/`-separated path RFC 0035 requires. + + Rule 1 makes an absolute path, a `~` path, or a path whose normalized form + escapes its anchor invalid, and rule 2 fixes the separator to `/`. The + stamped file carries these paths to every client, so the containment is a + validity rule here and not a recommendation, and the returned path is the + normalized form, so no client sees a `..` it would normalize its own way. + """ + text = str(value) + if not text or text.startswith(("/", "~")) or "\\" in text or re.match(r"^[A-Za-z]:", text): + raise StampError(f"{what} '{text}' is not a relative path with '/' separators") + depth = 0 + for part in text.split("/"): + if part in ("", "."): + continue + depth += -1 if part == ".." else 1 + if depth < 0: + raise StampError(f"{what} '{text}' escapes its anchor") + normalized = posixpath.normpath(text) + return "" if normalized == "." else normalized + + +def install_object(handle, listing_id, content_type, authored_install, provides=None): + """The release file's `install` object, or None when it says nothing. + + `root` is omitted when it is the archive root, and `target` and `path` are + absent where the type default applies. `[provides]` is never stamped, since + a stale copy would point a manager at the wrong directory, but its `launch` + is checked against this archive: RFC 0035 makes a missing one a per-release + rejection. + """ + authored_install = authored_install or {} + provides = provides or {} + authored_root = authored_install.get("root") + + if provides and content_type != "mod-loader": + raise StampError("[provides] is only permitted on a mod-loader (RFC 0035)") + + if authored_root is not None: + root, derived = relative_path(authored_root, "the authored install root"), False + if root and not entries_under(handle, root): + raise StampError(f"the authored install root '{root}' is not in the archive") + else: + root, derived = derive_root(handle, listing_id, content_type), True + if root is None: + raise StampError( + "the install root is neither derivable from the archive nor authored: " + "the standard layout is one top-level directory containing mod.toml, " + f"named '{listing_id}'" + ) + + install = {} + if root: + install["root"] = root + install["derived"] = derived + + target = authored_install.get("target") + path = authored_install.get("path") + if content_type == "mod" and (target is not None or path is not None): + # A mod's install location is not the author's to choose: the folder + # name is the identity the game sees, and RFC 0035 states the location + # as a default rather than an authorable field to keep it that way. + raise StampError("a mod cannot author an install target or path (RFC 0035)") + if content_type == "mod-loader" and authored_install and target is None: + raise StampError( + "a mod-loader [install] section needs a target: the type has no " + "default to fall back on (RFC 0035)" + ) + if target is not None: + if target not in INSTALL_ANCHORS: + raise StampError(f"the authored install target '{target}' is not an anchor") + install["target"] = target + if path is not None: + install["path"] = relative_path(path, "the authored install path") + + _check_provides(handle, root, target, provides) + + if set(install) == {"derived"}: + # Nothing but the fact that nobody authored anything, which the absence + # of the object already says. + return None + return install + + +def _check_provides(handle, root, target, provides): + """The release-time checks on a loader's [provides] section (RFC 0035). + + `launch` must exist in the archive, and rule 4 forbids a `standalone` + install without one. + """ + launch = provides.get("launch") + if launch is not None: + launch = relative_path(launch, "the provides launch path") + entry = f"{root}/{launch}" if root else launch + names = {name.replace("\\", "/") for name in handle.namelist()} + if entry not in names: + raise StampError( + f"the provides launch path '{launch}' is not in the release archive" + ) + if target == "standalone" and launch is None: + raise StampError( + "target = 'standalone' requires [provides] launch: a directory " + "nothing ever runs from is not an install (RFC 0035)" + ) + content_dir = provides.get("content-dir") + if content_dir is not None and content_dir not in INSTALL_ANCHORS: + raise StampError(f"the provides content-dir '{content_dir}' is not an anchor") + content_path = provides.get("content-path") + if content_path is not None: + relative_path(content_path, "the provides content-path") + + +def install_size(handle, root): + """The unpacked size of what gets installed, in bytes.""" + return sum(info.file_size for info in entries_under(handle, root)) + + +def derived_dependencies(mod_toml): + """The code dependencies the archive's own mod.toml declares. + + `[[StarMap.ModDependencies]]` is the only dependency data that exists in + the ecosystem, it is name-only, and `Optional` defaults to false, which is + the loader refusing to start the mod without it. + """ + blocks = ((mod_toml or {}).get("StarMap") or {}).get("ModDependencies") or [] + dependencies = [] + for block in blocks: + identifier = (block.get("ModId") or "").strip() + if not identifier: + raise StampError("a [[StarMap.ModDependencies]] block carries no ModId") + optional = bool(block.get("Optional", False)) + dependencies.append( + { + "id": identifier, + "kind": "optional" if optional else "required", + "source": "derived", + } + ) + return dependencies + + +def _authored_entry(entry): + """One authored dependency entry, validated, in release file shape.""" + kind = entry.get("kind") + if kind not in DEPENDENCY_KINDS: + raise StampError(f"dependency kind '{kind}' is not one of {', '.join(DEPENDENCY_KINDS)}") + + stamped = {} + if "any_of" in entry: + if entry.get("id"): + raise StampError("a dependency entry carries both id and any_of") + if kind not in ("required", "recommends"): + raise StampError(f"any_of is not valid with kind '{kind}'") + members = entry.get("any_of") or [] + if not members: + raise StampError("an any_of dependency entry names no members") + stamped["any_of"] = [ + { + key: member[key] + for key in ("id", "min", "max") + if member.get(key) is not None + } + for member in members + ] + for member in stamped["any_of"]: + if not member.get("id"): + raise StampError("an any_of member carries no id") + else: + identifier = (entry.get("id") or "").strip() + if not identifier: + raise StampError("a dependency entry carries no id") + stamped["id"] = identifier + + stamped["kind"] = kind + for key in ("min", "max"): + if entry.get(key) is not None: + stamped[key] = normalize_version(entry[key]) + stamped["source"] = "authored" + return stamped + + +def merge_dependencies(derived, authored): + """The merged dependency list of RFC 0031. + + Derived entries are ground truth, because the loader acts on them at + runtime, so nothing can suppress one. An authored entry replaces the derived + entry of the same id, and an authored `any_of` replaces the entry of every + member it names. Naming a member whose derived entry was not optional is an + error: the loader refuses to start without it, so the choice does not exist. + """ + merged = [dict(entry) for entry in derived] + by_id = {entry["id"].lower(): index for index, entry in enumerate(merged)} + replaced = set() + seen_authored = set() + + for entry in authored or []: + stamped = _authored_entry(entry) + names = ( + [member["id"] for member in stamped["any_of"]] + if "any_of" in stamped + else [stamped["id"]] + ) + + for name in names: + if name.lower() in seen_authored: + raise StampError( + f"the authored dependencies name '{name}' more than once" + ) + seen_authored.add(name.lower()) + + for name in names: + index = by_id.get(name.lower()) + if index is None: + continue + if "any_of" in stamped and merged[index]["kind"] != "optional": + raise StampError( + f"the any_of entry names '{name}', whose derived entry is required: " + "the loader refuses to start the mod without that dependency, so an " + "alternative set claims a choice that does not exist at runtime" + ) + replaced.add(index) + + merged.append(stamped) + + return [entry for index, entry in enumerate(merged) if index not in replaced] + + +def listing_snapshot(authored): + """The shared authored core as it stands now, frozen into this release.""" + snapshot = { + field: authored[field] + for field in LISTING_FIELDS + if authored.get(field) is not None + } + links = authored.get("links") + if links: + snapshot["links"] = dict(links) + return snapshot + + +def stamp(authored, release, archive, game_versions, mirrors=(), now=None): + """The release file for one release. + + `release` carries the facts the caller read off the host: + + tag the tag or version string the host names, required + release_date ISO 8601 UTC timestamp of the release, required + url direct download URL of the archive, required + content_type the archive format, defaults to application/zip + prerelease the host's pre-release flag, defaults to false + changelog URL of the release's changelog, optional + + Only the open month bound depends on `now`. + """ + now = now or datetime.now(timezone.utc) + + listing_id = (authored.get("id") or "").strip() + content_type = authored.get("type") + if not listing_id: + raise StampError("the authored document carries no id") + if not valid_id(listing_id): + raise StampError( + f"the id '{listing_id}' does not satisfy the id rules of RFC 0031" + ) + if content_type not in ("mod", "mod-loader"): + raise StampError(f"type '{content_type}' has no generated release file") + if authored.get("spec_version") != SPEC_VERSION: + raise StampError( + f"the authored document claims spec_version " + f"{authored.get('spec_version')!r}, and this stamper implements " + f"{SPEC_VERSION}" + ) + + version = normalize_version(release.get("tag")) + handle = open_archive(archive) + + install = install_object( + handle, listing_id, content_type, authored.get("install"), + provides=authored.get("provides"), + ) + root = (install or {}).get("root", "") + + mod_toml = read_mod_toml(handle, root) if content_type == "mod" else None + dependencies = merge_dependencies( + derived_dependencies(mod_toml), authored.get("dependencies") + ) + + compatibility = authored.get("compatibility") or {} + if not compatibility.get("game_min"): + raise StampError("the authored document states no game_min") + game_min, game_min_revision = resolve_bound( + compatibility.get("game_min"), "game_min", game_versions, now + ) + game_max, game_max_revision = resolve_bound( + compatibility.get("game_max"), "game_max", game_versions, now + ) + if game_max_revision is not None and game_max_revision < game_min_revision: + raise StampError( + f"game_max resolves to revision {game_max_revision}, below game_min's " + f"{game_min_revision}: the compatibility range is empty" + ) + + if not release.get("release_date"): + raise StampError("the host reports no release date") + if not release.get("url"): + raise StampError("the host reports no download URL for the archive") + + document = { + "spec_version": SPEC_VERSION, + "id": listing_id, + "type": content_type, + "version": version, + "version_scheme": "semver", + "release_status": release_status(version, release.get("prerelease", False)), + "release_date": release["release_date"], + "game_min": game_min, + "game_min_revision": game_min_revision, + } + if game_max is not None: + document["game_max"] = game_max + document["game_max_revision"] = game_max_revision + if compatibility.get("os"): + document["os"] = list(compatibility["os"]) + + # The archive parsed as a zip above, so that is what it is, whatever the + # host declared an asset uploaded as octet-stream to be. + content = release.get("content_type") + if content in (None, "", "application/octet-stream", "binary/octet-stream"): + content = "application/zip" + + download = { + "url": release["url"], + "sha256": hashlib.sha256(archive).hexdigest().upper(), + "size": len(archive), + "content_type": content, + } + if mirrors: + download["mirrors"] = list(mirrors) + document["download"] = download + + document["install_size"] = install_size(handle, root) + if install is not None: + document["install"] = install + + loader = authored.get("loader") + if loader and content_type == "mod": + stamped_loader = {"id": loader.get("id")} + if not stamped_loader["id"]: + raise StampError("the authored [loader] section names no id") + if not loader.get("min"): + raise StampError("the authored [loader] section states no min") + stamped_loader["min"] = normalize_version(loader["min"]) + if loader.get("max"): + stamped_loader["max"] = normalize_version(loader["max"]) + stamped_loader["source"] = "authored" + document["loader"] = stamped_loader + + document["dependencies"] = dependencies + if release.get("changelog"): + document["changelog"] = release["changelog"] + document["listing"] = listing_snapshot(authored) + + return document + + +def serialize(document): + """The release file's bytes, as every stamped file in the repository is written.""" + return json.dumps(document, indent=2, ensure_ascii=False) + "\n" + + +def main(argv=None): + parser = argparse.ArgumentParser( + description="Stamp one release file from an authored document and a release archive." + ) + parser.add_argument("--listing", required=True, type=Path, help="the authored TOML document") + parser.add_argument("--archive", required=True, type=Path, help="the release archive") + parser.add_argument( + "--release", + required=True, + type=Path, + help="JSON of the release facts read off the host: tag, release_date, url, " + "content_type, prerelease, changelog", + ) + parser.add_argument( + "--game-versions", type=Path, default=Path("game-versions.json"), + help="the game release list month bounds resolve against", + ) + parser.add_argument( + "--mirror", action="append", default=[], + help="a further URL serving byte-identical archive, verified by the caller", + ) + parser.add_argument("--out", type=Path, help="write here instead of to stdout") + arguments = parser.parse_args(argv) + + try: + with arguments.listing.open("rb") as handle: + authored = tomllib.load(handle) + release = json.loads(arguments.release.read_text(encoding="utf-8")) + game_versions = json.loads( + arguments.game_versions.read_text(encoding="utf-8") + )["versions"] + archive = arguments.archive.read_bytes() + except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError, + json.JSONDecodeError, KeyError, TypeError) as error: + print(f"cannot read the inputs: {error}", file=sys.stderr) + return 1 + + try: + document = stamp( + authored, + release, + archive, + game_versions, + mirrors=arguments.mirror, + ) + except StampError as error: + print(f"cannot stamp {arguments.listing}: {error}", file=sys.stderr) + return 1 + + rendered = serialize(document) + if arguments.out: + arguments.out.parent.mkdir(parents=True, exist_ok=True) + with arguments.out.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(rendered) + else: + sys.stdout.write(rendered) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/test_hosts.py b/tools/test_hosts.py new file mode 100644 index 0000000..4590271 --- /dev/null +++ b/tools/test_hosts.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Tests for the host adapters: no network, a fake Http answers or raises. + +The point of these is the HostError/StampError split on the failure paths: an +HTTPError that escapes either becomes an "unexpected" log line with no +author-facing issue, which is exactly what the watcher must not do. +""" + +import json +import unittest +import urllib.error + +from hosts import ( + MAX_ARCHIVE_BYTES, + GitHubHost, + HostError, + HostRelease, + OversizeError, + Response, + SpaceDockHost, + _download, + _utc, +) +from stamp_release import StampError + + +def http_error(code): + return urllib.error.HTTPError("https://x", code, "boom", {}, None) + + +class FakeHttp: + """Answers by URL prefix, or raises what the route holds.""" + + token = None + timeout = 1 + + def __init__(self, routes): + self.routes = routes + self.lines = [] + + def log(self, message): + self.lines.append(message) + + def get(self, url, accept=None, etag=None, api=False, limit=None): + for prefix, answer in self.routes.items(): + if url.startswith(prefix): + if isinstance(answer, Exception): + raise answer + return answer + raise AssertionError(f"unexpected URL {url}") + + +def release(url, candidates=(), size=None): + return HostRelease( + host="github", tag="v1.0.0", version="1.0.0", + release_date="2020-01-01T00:00:00Z", url=url, + candidates=tuple(candidates), size=size, + ) + + +class Downloads(unittest.TestCase): + def test_a_gone_archive_is_a_stamp_error(self): + # 404, 410 and 451 are facts about the release, reported to the author. + for code in (404, 410, 451): + http = FakeHttp({"https://github.com/": http_error(code)}) + with self.assertRaises(StampError, msg=code): + _download(http, release("https://github.com/o/r/releases/download/x.zip")) + + def test_any_other_http_error_is_a_host_error(self): + http = FakeHttp({"https://github.com/": http_error(403)}) + with self.assertRaises(HostError): + _download(http, release("https://github.com/o/r/releases/download/x.zip")) + + def test_an_oversized_archive_is_a_stamp_error_not_a_retry(self): + # Oversize is permanent: as a HostError the watcher would download and + # discard the archive every tick and no issue would ever open. + with self.assertRaises(StampError): + _download(FakeHttp({}), release("https://github.com/o/r/x.zip", + size=MAX_ARCHIVE_BYTES + 1)) + http = FakeHttp({"https://github.com/": OversizeError("too large")}) + with self.assertRaises(StampError): + _download(http, release("https://github.com/o/r/x.zip")) + + +class Pagination(unittest.TestCase): + def test_an_error_on_a_later_page_is_a_host_error(self): + # A 404 from page two must not unwind into the watcher's broad catch. + first = Response( + 200, + {"Link": '; rel="next"'}, + b"[]", + ) + http = FakeHttp({ + "https://api.github.com/repos/o/r/releases": first, + "https://api.github.com/page2": http_error(404), + }) + with self.assertRaises(HostError): + GitHubHost("o/r", http).releases() + + def test_a_missing_repository_stays_a_stamp_error(self): + http = FakeHttp({"https://api.github.com/repos/o/r/releases": http_error(404)}) + with self.assertRaises(StampError): + GitHubHost("o/r", http).releases() + + def test_a_page_that_is_not_json_is_a_host_error(self): + # A 200 carrying proxy HTML is the host having a bad moment, and it + # must not land in the watcher's broad catch as "unexpected". + http = FakeHttp({ + "https://api.github.com/repos/o/r/releases": Response(200, {}, b""), + }) + with self.assertRaises(HostError): + GitHubHost("o/r", http).releases() + + def test_a_scan_past_max_pages_says_so(self): + first = Response( + 200, + {"Link": '; rel="next"'}, + b"[]", + ) + http = FakeHttp({"https://api.github.com/repos/o/r/releases": first}) + host = GitHubHost("o/r", http, max_pages=1) + host.releases() + self.assertTrue(host.truncated) + + +class SpaceDock(unittest.TestCase): + def test_a_non_numeric_id_is_a_stamp_error(self): + # A ValueError would escape the HostError/StampError split entirely. + with self.assertRaises(StampError): + SpaceDockHost("abc", FakeHttp({})) + SpaceDockHost("4253", FakeHttp({})) + + def payload(self, download_path, page="/mod/1"): + body = json.dumps({ + "url": page, + "versions": [{"friendly_version": "1.0.0", "created": "2020-01-01T00:00:00Z", + "download_path": download_path}], + }).encode() + return FakeHttp({"https://spacedock.info/api/mod/1": Response(200, {}, body)}) + + def test_a_download_path_stays_on_the_host(self): + releases, _ = SpaceDockHost(1, self.payload("/mod/1/x/download/1.0.0")).releases() + self.assertEqual(releases[0].url, "https://spacedock.info/mod/1/x/download/1.0.0") + + def test_a_download_path_leaving_the_host_is_rejected(self): + # urljoin returns an absolute or protocol-relative value unchanged, so + # an unchecked join publishes a foreign address into a stamped file. + for escape in ("https://evil.example/x.zip", "//evil.example/x.zip", + "http://evil.example/x.zip"): + with self.assertRaises(StampError, msg=escape): + SpaceDockHost(1, self.payload(escape)).releases() + + def test_a_mod_page_leaving_the_host_is_rejected(self): + # changelog is published as a link and no checksum gates it. + with self.assertRaises(StampError): + SpaceDockHost(1, self.payload("/ok", page="https://evil.example/")).releases() + + +class Timestamps(unittest.TestCase): + def test_garbage_yields_none_rather_than_passing_through(self): + # release_date is stamped exactly once; the stamper rejects a release + # without one, which scopes the failure to that release. + self.assertIsNone(_utc("not a timestamp")) + self.assertIsNone(_utc(None)) + self.assertEqual(_utc("2020-01-01T00:00:00Z"), "2020-01-01T00:00:00Z") + self.assertEqual(_utc("2020-01-01T01:30:00+01:30"), "2020-01-01T00:00:00Z") + + +class AssetSelection(unittest.TestCase): + def payload(self, names, tag="0.4.6"): + return { + "tag_name": tag, + "published_at": "2020-01-01T00:00:00Z", + "html_url": "https://github.com/o/r/releases/tag/x", + "assets": [ + { + "state": "uploaded", + "name": name, + "browser_download_url": f"https://github.com/o/r/releases/download/{tag}/{name}", + "content_type": "application/zip", + "size": 1, + } + for name in names + ], + } + + def test_the_asset_named_after_the_listing_wins(self): + host = GitHubHost("o/r", FakeHttp({}), listing_id="StarMap") + chosen = host._release(self.payload(["StarMap-0.4.6.zip", "StarMapSource.zip"])) + self.assertEqual(chosen.asset_name, "StarMap-0.4.6.zip") + + def test_an_ambiguous_release_is_not_guessed_at(self): + host = GitHubHost("o/r", FakeHttp({}), listing_id="StarMap") + chosen = host._release(self.payload(["Launcher.zip", "Standalone.zip"])) + self.assertIsNone(chosen.url) + self.assertEqual(len(chosen.candidates), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/test_stamp_release.py b/tools/test_stamp_release.py new file mode 100644 index 0000000..41079c5 --- /dev/null +++ b/tools/test_stamp_release.py @@ -0,0 +1,494 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Tests for the stamper. No token, no network: `python3 -m unittest discover tools`. + +Every case here is one sentence from RFC 0031, RFC 0035 or RFC 0017 that the +stamper has to keep. The archives are built in memory, so the fixtures are the +tests. + +The other half of the stamper's test is the design repository's examples/, where +every value was produced by this procedure by hand: `tools/verify_examples.py` +re-derives all of them from their release hosts and diffs. +""" + +import io +import json +import unittest +import zipfile +from datetime import datetime, timezone + +from stamp_release import ( + StampError, + derived_dependencies, + merge_dependencies, + normalize_version, + relative_path, + release_status, + resolve_bound, + serialize, + stamp, + valid_id, +) + +GAME_VERSIONS = [ + "2026.7.5.4892", + "2026.7.6.4939", + "2026.7.9.5018", + "2026.8.3.5117", + "2026.8.5.5168", +] + +NOW = datetime(2026, 8, 10, 12, 0, tzinfo=timezone.utc) + +MOD_TOML = """\ +name = "AutoStage" +patches = [ "Patches/BurnControlPatch.xml" ] + +[StarMap] +EntryAssembly = "AutoStage" + +[[StarMap.ModDependencies]] +ModId = "KittenExtensions" +Optional = false + +[[StarMap.ModDependencies]] +ModId = "MeasureTools" +Optional = true +""" + +LISTING = { + "spec_version": 1, + "id": "AutoStage", + "type": "mod", + "name": "AutoStage", + "authors": ["Maxi"], + "abstract": "Automatic staging for Kitten Space Agency.", + "license": "MIT", + "tags": ["control"], + "status": "deprecated", + "superseded_by": "AutoStageNG", + "releases": {"github": "Maximilian-Nesslauer/KSA-AutoStage"}, + "links": {"forums": "https://forums.ahwoo.com/threads/autostage.891/"}, + "compatibility": {"game_min": "2026.8.3.5117"}, + "loader": {"id": "StarMap", "min": "0.4.5"}, +} + +RELEASE = { + "tag": "v0.4.3", + "release_date": "2026-08-05T17:48:57Z", + "url": "https://github.com/Maximilian-Nesslauer/KSA-AutoStage/releases/download/v0.4.3/AutoStage.zip", + "content_type": "application/zip", + "prerelease": False, + "changelog": "https://github.com/Maximilian-Nesslauer/KSA-AutoStage/releases/tag/v0.4.3", +} + + +def archive(files): + """A zip archive of {path: text}, as bytes.""" + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as handle: + for path, content in files.items(): + handle.writestr(path, content) + return buffer.getvalue() + + +def mod_archive(identifier="AutoStage", manifest=MOD_TOML): + return archive( + { + f"{identifier}/{identifier}.dll": "x" * 100, + f"{identifier}/mod.toml": manifest, + } + ) + + +class Versions(unittest.TestCase): + def test_a_leading_v_is_stripped(self): + self.assertEqual(normalize_version("v1.2.3"), "1.2.3") + self.assertEqual(normalize_version("1.2.3-rc.1+build.5"), "1.2.3-rc.1+build.5") + + def test_a_version_that_does_not_parse_is_rejected(self): + for tag in ("rc", "1.2", "2026.8.3.5117", "", None, "v1.02.3"): + with self.assertRaises(StampError): + normalize_version(tag) + + def test_a_nightly_does_not_look_like_a_release(self): + self.assertEqual(release_status("1.0.0", False), "stable") + self.assertEqual(release_status("1.0.0", True), "testing") + self.assertEqual(release_status("1.0.0-rc.1", False), "testing") + self.assertEqual(release_status("1.0.0-nightly.20260810", False), "dev") + + +class Bounds(unittest.TestCase): + def test_a_full_version_carries_its_own_revision(self): + self.assertEqual( + resolve_bound("2026.8.3.5117", "game_min", [], NOW), ("2026.8.3.5117", 5117) + ) + + def test_a_month_resolves_to_its_first_and_last_revision(self): + self.assertEqual( + resolve_bound("2026.7", "game_min", GAME_VERSIONS, NOW), ("2026.7.5.4892", 4892) + ) + self.assertEqual( + resolve_bound("2026.7", "game_max", GAME_VERSIONS, NOW), ("2026.7.9.5018", 5018) + ) + + def test_a_month_that_is_not_over_stamps_an_open_upper_bound(self): + self.assertEqual(resolve_bound("2026.8", "game_max", GAME_VERSIONS, NOW), (None, None)) + + def test_a_month_with_no_build_cannot_resolve(self): + with self.assertRaises(StampError): + resolve_bound("2026.9", "game_min", GAME_VERSIONS, NOW) + + def test_something_that_is_neither_is_an_error(self): + with self.assertRaises(StampError): + resolve_bound("recent", "game_min", GAME_VERSIONS, NOW) + + +class Dependencies(unittest.TestCase): + def derived(self): + return derived_dependencies({"StarMap": {"ModDependencies": [ + {"ModId": "KittenExtensions", "Optional": False}, + {"ModId": "MeasureTools", "Optional": True}, + ]}}) + + def test_optional_defaults_to_false(self): + derived = derived_dependencies( + {"StarMap": {"ModDependencies": [{"ModId": "KittenExtensions"}]}} + ) + self.assertEqual( + derived, [{"id": "KittenExtensions", "kind": "required", "source": "derived"}] + ) + + def test_an_archive_without_a_mod_toml_contributes_nothing(self): + self.assertEqual(derived_dependencies(None), []) + + def test_an_authored_entry_replaces_the_derived_entry_of_the_same_id(self): + merged = merge_dependencies( + self.derived(), + [{"id": "KittenExtensions", "kind": "required", "min": "0.4.0"}], + ) + self.assertEqual( + merged, + [ + {"id": "MeasureTools", "kind": "optional", "source": "derived"}, + { + "id": "KittenExtensions", + "kind": "required", + "min": "0.4.0", + "source": "authored", + }, + ], + ) + + def test_a_derived_entry_cannot_be_suppressed(self): + merged = merge_dependencies(self.derived(), [{"id": "Something", "kind": "suggests"}]) + self.assertEqual([entry["id"] for entry in merged], + ["KittenExtensions", "MeasureTools", "Something"]) + + def test_any_of_replaces_the_derived_entry_of_every_member_it_names(self): + merged = merge_dependencies( + self.derived(), + [{"kind": "recommends", "any_of": [{"id": "MeasureTools", "min": "1.0.0"}, + {"id": "DeltaVMap"}]}], + ) + self.assertEqual( + merged, + [ + {"id": "KittenExtensions", "kind": "required", "source": "derived"}, + { + "any_of": [{"id": "MeasureTools", "min": "1.0.0"}, {"id": "DeltaVMap"}], + "kind": "recommends", + "source": "authored", + }, + ], + ) + + def test_any_of_over_a_hard_derived_dependency_is_an_error(self): + with self.assertRaises(StampError): + merge_dependencies( + self.derived(), + [{"kind": "required", "any_of": [{"id": "KittenExtensions"}]}], + ) + + def test_an_unknown_kind_is_an_error(self): + with self.assertRaises(StampError): + merge_dependencies([], [{"id": "X", "kind": "needs"}]) + + +class Stamp(unittest.TestCase): + def stamp(self, listing=None, release=None, data=None, **kwargs): + return stamp( + listing or LISTING, + release or RELEASE, + data if data is not None else mod_archive(), + GAME_VERSIONS, + now=NOW, + **kwargs, + ) + + def test_the_worked_example(self): + document = self.stamp() + self.assertEqual(document["version"], "0.4.3") + self.assertEqual(document["version_scheme"], "semver") + self.assertEqual(document["release_status"], "stable") + self.assertEqual(document["game_min_revision"], 5117) + self.assertEqual(document["download"]["size"], len(mod_archive())) + self.assertEqual(len(document["download"]["sha256"]), 64) + self.assertEqual(document["download"]["sha256"], document["download"]["sha256"].upper()) + self.assertEqual(document["install"], {"root": "AutoStage", "derived": True}) + self.assertEqual(document["install_size"], 100 + len(MOD_TOML)) + self.assertEqual( + document["loader"], {"id": "StarMap", "min": "0.4.5", "source": "authored"} + ) + self.assertEqual( + [entry["id"] for entry in document["dependencies"]], + ["KittenExtensions", "MeasureTools"], + ) + + def test_the_listing_block_freezes_the_descriptive_facts_only(self): + listing = self.stamp()["listing"] + self.assertEqual(listing["name"], "AutoStage") + self.assertEqual(listing["links"]["forums"], LISTING["links"]["forums"]) + # Deprecation has to reach every release the moment it is declared, so + # a client reads it live from the authored file and never from here. + self.assertNotIn("status", listing) + self.assertNotIn("superseded_by", listing) + + def test_mirrors_are_only_there_when_the_caller_verified_one(self): + self.assertNotIn("mirrors", self.stamp()["download"]) + document = self.stamp(mirrors=["https://spacedock.info/x"]) + self.assertEqual(document["download"]["mirrors"], ["https://spacedock.info/x"]) + + def test_an_octet_stream_asset_is_still_a_zip(self): + release = dict(RELEASE, content_type="application/octet-stream") + self.assertEqual(self.stamp(release=release)["download"]["content_type"], "application/zip") + + def test_an_authored_month_max_that_is_over_lands_in_the_file(self): + listing = dict(LISTING, compatibility={"game_min": "2026.7", "game_max": "2026.7"}) + document = self.stamp(listing=listing) + self.assertEqual(document["game_min"], "2026.7.5.4892") + self.assertEqual(document["game_max_revision"], 5018) + + def test_an_authored_month_max_that_is_not_over_stamps_open(self): + listing = dict(LISTING, compatibility={"game_min": "2026.8.3.5117", "game_max": "2026.8"}) + self.assertNotIn("game_max", self.stamp(listing=listing)) + + def test_an_empty_compatibility_range_is_an_error(self): + # A max that resolves below the min stamps a range no game version can + # ever satisfy, so it is the author's mistake to fix, not a file to ship. + listing = dict( + LISTING, compatibility={"game_min": "2026.8.3.5117", "game_max": "2026.7"} + ) + with self.assertRaises(StampError): + self.stamp(listing=listing) + + def test_the_os_list_is_carried_when_authored(self): + listing = dict( + LISTING, compatibility={"game_min": "2026.8.3.5117", "os": ["windows"]} + ) + self.assertEqual(self.stamp(listing=listing)["os"], ["windows"]) + + def test_a_folder_name_that_is_not_the_id_is_an_error(self): + # The folder name is the identity the game will see (Mod.MakeUsing). + with self.assertRaises(StampError): + self.stamp(data=mod_archive("autostage")) + with self.assertRaises(StampError): + self.stamp(data=mod_archive("Whatever")) + + def test_an_unusual_layout_needs_an_authored_root(self): + data = archive({"build/AutoStage/mod.toml": MOD_TOML, "README.md": "x" * 9}) + with self.assertRaises(StampError): + self.stamp(data=data) + + listing = dict(LISTING, install={"root": "build/AutoStage"}) + document = self.stamp(listing=listing, data=data) + self.assertEqual(document["install"], {"root": "build/AutoStage", "derived": False}) + # Only what gets installed counts, so the README beside it does not. + self.assertEqual(document["install_size"], len(MOD_TOML)) + + def test_an_authored_root_that_is_not_in_the_archive_is_an_error(self): + with self.assertRaises(StampError): + self.stamp(listing=dict(LISTING, install={"root": "nowhere"})) + + def test_an_asset_only_mod_has_no_mod_toml_and_no_dependencies(self): + data = archive({"AutoStage/Parts/thing.json": "{}"}) + document = self.stamp(data=data) + self.assertEqual(document["install"], {"root": "AutoStage", "derived": True}) + self.assertEqual(document["dependencies"], []) + + def test_a_loader_installs_from_the_archive_root(self): + listing = { + "spec_version": 1, + "id": "StarMap", + "type": "mod-loader", + "name": "StarMap", + "authors": ["KlaasWhite"], + "abstract": "Mod loader that runs code mods.", + "license": "MIT", + "links": {"forums": "https://forums.ahwoo.com/threads/starmap-mod-loader.384/"}, + "compatibility": {"game_min": "2026.8.3.5117"}, + } + data = archive({"StarMap.exe": "x" * 10, "StarMap.dll": "y" * 20}) + document = stamp( + listing, dict(RELEASE, tag="0.4.6"), data, GAME_VERSIONS, now=NOW + ) + # Nothing to say about the install, so the object stays out (RFC 0035). + self.assertNotIn("install", document) + self.assertNotIn("loader", document) + self.assertEqual(document["install_size"], 30) + + # With an authored descriptor, the resolved destination is stamped. + # `standalone` needs a launch (RFC 0035, rule 4), and the launch has to + # exist in this release's archive (rule 3). + listing["install"] = {"target": "standalone"} + listing["provides"] = {"launch": "StarMap.exe"} + document = stamp( + listing, dict(RELEASE, tag="0.4.6"), data, GAME_VERSIONS, now=NOW + ) + self.assertEqual(document["install"], {"derived": True, "target": "standalone"}) + + def test_an_unknown_anchor_is_an_error(self): + with self.assertRaises(StampError): + self.stamp(listing=dict(LISTING, install={"target": "somewhere"})) + + def test_a_pack_has_no_generated_half(self): + with self.assertRaises(StampError): + self.stamp(listing=dict(LISTING, type="modpack")) + + def test_the_archive_has_to_be_a_zip(self): + with self.assertRaises(StampError): + self.stamp(data=b"not a zip at all") + + def test_game_min_is_required(self): + with self.assertRaises(StampError): + self.stamp(listing=dict(LISTING, compatibility={})) + + def test_the_file_is_written_the_way_every_stamped_file_is(self): + rendered = serialize(self.stamp()) + self.assertTrue(rendered.endswith("}\n")) + self.assertEqual(json.loads(rendered)["id"], "AutoStage") + self.assertIn('\n "version": "0.4.3",', rendered) + + def test_stamping_twice_gives_the_same_bytes(self): + self.assertEqual(serialize(self.stamp()), serialize(self.stamp())) + + def test_a_foreign_spec_version_is_refused(self): + # A document claiming a format this stamper does not implement must not + # be silently stamped as spec_version 1. + with self.assertRaises(StampError): + self.stamp(listing=dict(LISTING, spec_version=2)) + + def test_a_duplicate_authored_dependency_id_is_an_error(self): + listing = dict( + LISTING, + dependencies=[ + {"id": "KittenExtensions", "kind": "required", "min": "0.4.0"}, + {"id": "kittenextensions", "kind": "suggests"}, + ], + ) + with self.assertRaises(StampError): + self.stamp(listing=listing) + + +class Ids(unittest.TestCase): + def test_the_id_rules_of_rfc_0031(self): + for good in ("A", "AutoStage", "My.Mod-2_x", "a" * 64): + self.assertTrue(valid_id(good), good) + for bad in (None, "", ".Mod", "Mod.", "-Mod", "My Mod", "My/Mod", + "..", "../Evil", "a" * 65, "CON", "con.mod", "Core", "COM1"): + self.assertFalse(valid_id(bad), repr(bad)) + + def test_an_invalid_id_rejects_the_stamp(self): + listing = dict(LISTING, id="../AutoStage") + with self.assertRaises(StampError): + stamp(listing, RELEASE, mod_archive(), GAME_VERSIONS, now=NOW) + + +class Rfc0035(unittest.TestCase): + """The release-time rules of the install descriptor.""" + + def loader_listing(self, install=None, provides=None): + listing = { + "spec_version": 1, + "id": "StarMap", + "type": "mod-loader", + "name": "StarMap", + "authors": ["KlaasWhite"], + "abstract": "Mod loader that runs code mods.", + "license": "MIT", + "links": {"forums": "https://forums.ahwoo.com/threads/starmap-mod-loader.384/"}, + "compatibility": {"game_min": "2026.8.3.5117"}, + } + if install is not None: + listing["install"] = install + if provides is not None: + listing["provides"] = provides + return listing + + def loader_archive(self): + return archive({"StarMap.exe": "x" * 10, "StarMap.dll": "y" * 20}) + + def stamp_loader(self, install=None, provides=None): + return stamp( + self.loader_listing(install, provides), + dict(RELEASE, tag="0.4.6"), + self.loader_archive(), + GAME_VERSIONS, + now=NOW, + ) + + def test_paths_must_be_relative_and_contained(self): + # Rule 1: absolute, home-relative, and escaping paths are invalid. + for bad in ("/etc/x", "~x/y", "..", "../x", "a/../../x", "C:/x", "a\\b"): + with self.assertRaises(StampError, msg=bad): + relative_path(bad, "test") + # A `..` that stays inside its anchor is contained, and the published + # value is the normalized form, so no client normalizes its own way. + self.assertEqual(relative_path("a/../b", "test"), "b") + self.assertEqual(relative_path("./tools", "test"), "tools") + self.assertEqual(relative_path("build/AutoStage", "test"), "build/AutoStage") + + def test_an_escaping_install_path_rejects_the_release(self): + with self.assertRaises(StampError): + self.stamp_loader( + install={"target": "standalone", "path": "../outside"}, + provides={"launch": "StarMap.exe"}, + ) + + def test_a_mod_cannot_author_a_target_or_path(self): + # The folder name is the identity the game sees, so a mod's location is + # a default, not an authorable field. + for install in ({"target": "game-root"}, {"path": "sub"}): + listing = dict(LISTING, install=install) + with self.assertRaises(StampError, msg=install): + stamp(listing, RELEASE, mod_archive(), GAME_VERSIONS, now=NOW) + + def test_a_loader_install_section_needs_a_target(self): + # The type has no default to fall back on. + with self.assertRaises(StampError): + self.stamp_loader(install={"path": "somewhere"}) + + def test_provides_is_loader_only(self): + listing = dict(LISTING, provides={"launch": "AutoStage.dll"}) + with self.assertRaises(StampError): + stamp(listing, RELEASE, mod_archive(), GAME_VERSIONS, now=NOW) + + def test_a_launch_absent_from_the_archive_rejects_the_release(self): + with self.assertRaises(StampError): + self.stamp_loader(provides={"launch": "Missing.exe"}) + + def test_a_launch_present_in_the_archive_passes(self): + document = self.stamp_loader(provides={"launch": "StarMap.exe"}) + self.assertNotIn("install", document) + + def test_standalone_requires_a_launch(self): + with self.assertRaises(StampError): + self.stamp_loader(install={"target": "standalone"}) + + def test_an_unknown_content_dir_is_an_error(self): + with self.assertRaises(StampError): + self.stamp_loader(provides={"launch": "StarMap.exe", "content-dir": "somewhere"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/test_watch.py b/tools/test_watch.py new file mode 100644 index 0000000..e1a53f7 --- /dev/null +++ b/tools/test_watch.py @@ -0,0 +1,808 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Tests for the tick's own logic: no token, no network, no git. + +The API is a stub that records what the watcher would send, which is what makes +"one open issue per listing" and the sweep's decisions testable at all. +""" + +import hashlib +import json +import tempfile +import unittest +import urllib.error +from datetime import datetime, timezone +from pathlib import Path + +from hosts import HostRelease +from watch import Cache, Issues, Sweep, Watcher, parse_arguments + +GAME_VERSIONS = {"spec_version": 1, "versions": ["2026.8.3.5117"]} + + +def http_error(code): + return urllib.error.HTTPError("https://x", code, "boom", {}, None) + + +class StubApi: + """Answers reads from a routing table and records every write.""" + + repository = "KSAModding/content-index" + + def __init__(self, routes=None): + self.routes = routes or {} + self.sent = [] + + def get(self, path, **query): + for prefix, answer in self.routes.items(): + if path.startswith(prefix): + return answer(query) if callable(answer) else answer + return None + + def get_paged(self, path, key=None, max_pages=10, **query): + answer = self.get(path, **query) + if key: + return list((answer or {}).get(key) or []) + return list(answer or []) + + def send(self, method, path, payload): + self.sent.append((method, path, payload)) + if method == "POST" and path == "/issues": + return {"number": 42} + return {} + + def writes(self, method=None, path=None, contains=None): + return [ + entry + for entry in self.sent + if (method is None or entry[0] == method) + and (path is None or entry[1] == path) + and (contains is None or contains in entry[1]) + ] + + +class RecorderIssues: + """Records what the watcher would report, for tick-level tests.""" + + def __init__(self, listings=None, degraded=False): + self.reported = [] + self.resolved = [] + self.listings = listings or {} + self.degraded = degraded + + def report(self, listing_id, errors, cache): + self.reported.append((listing_id, list(errors))) + + def resolve(self, listing_id, cache, reason=None): + self.resolved.append(listing_id) + + def resolve_if(self, listing_id, signature, cache): + self.resolved.append((listing_id, signature)) + + def open_listings(self): + return self.listings + + +def cache(): + return Cache(None) + + +class OneIssuePerListing(unittest.TestCase): + def test_the_first_failure_opens_one_issue(self): + api = StubApi({"/issues": []}) + Issues(api, "watcher", log=lambda _: None).report( + "AutoStage", ["`0.4.3`: the archive is not a readable zip"], cache() + ) + opened = api.writes("POST", path="/issues") + self.assertEqual(len(opened), 1) + self.assertIn("watcher:listing=AutoStage", opened[0][2]["body"]) + self.assertIn("not a readable zip", opened[0][2]["body"]) + + def test_the_same_failure_again_edits_and_does_not_comment(self): + errors = ["`0.4.3`: the archive is not a readable zip"] + store = cache() + api = StubApi({"/issues": []}) + issues = Issues(api, "watcher", log=lambda _: None) + issues.report("AutoStage", errors, store) + + body = api.writes("POST", path="/issues")[0][2]["body"] + second = StubApi({"/issues/42": {"number": 42, "state": "open", "body": body}, + "/issues": []}) + Issues(second, "watcher", log=lambda _: None).report("AutoStage", errors, store) + + self.assertEqual(len(second.writes("POST", path="/issues")), 0) + self.assertEqual(len(second.writes("PATCH", path="/issues/42")), 1) + self.assertEqual(len(second.writes("POST", path="/issues/42/comments")), 0) + + def test_a_different_failure_comments_on_the_same_issue(self): + store = cache() + api = StubApi({"/issues": []}) + issues = Issues(api, "watcher", log=lambda _: None) + issues.report("AutoStage", ["one thing"], store) + body = api.writes("POST", path="/issues")[0][2]["body"] + + second = StubApi({"/issues/42": {"number": 42, "state": "open", "body": body}, + "/issues": []}) + Issues(second, "watcher", log=lambda _: None).report( + "AutoStage", ["something else entirely"], store + ) + self.assertEqual(len(second.writes("POST", path="/issues")), 0) + self.assertEqual(len(second.writes("POST", path="/issues/42/comments")), 1) + + def test_an_issue_is_found_by_its_marker_when_the_cache_is_gone(self): + body = "\nold" + api = StubApi({"/issues": [{"number": 7, "state": "open", "body": body}]}) + Issues(api, "watcher", log=lambda _: None).report("AutoStage", ["x"], cache()) + self.assertEqual(len(api.writes("POST", path="/issues")), 0) + self.assertEqual(len(api.writes("PATCH", path="/issues/7")), 1) + + def test_a_clean_tick_closes_the_issue(self): + body = "\nold" + api = StubApi({"/issues": [{"number": 7, "state": "open", "body": body}]}) + Issues(api, "watcher", log=lambda _: None).resolve("AutoStage", cache()) + self.assertEqual(api.writes("PATCH", path="/issues/7")[0][2], {"state": "closed"}) + + def test_a_pull_request_is_not_an_issue(self): + api = StubApi( + { + "/issues": [ + { + "number": 7, + "state": "open", + "body": "", + "pull_request": {}, + } + ] + } + ) + Issues(api, "watcher", log=lambda _: None).report("AutoStage", ["x"], cache()) + self.assertEqual(len(api.writes("POST", path="/issues")), 1) + + +def sweep_api(runs=(), suites=0, statuses=(), check_runs=(), comments=()): + return StubApi( + { + "/pulls": [{"number": 3, "head": {"sha": "abc"}}], + "/actions/runs": {"workflow_runs": list(runs)}, + "/commits/abc/check-suites": {"total_count": suites}, + "/commits/abc/status": {"statuses": list(statuses)}, + "/commits/abc/check-runs": {"check_runs": list(check_runs)}, + "/issues/3/comments": list(comments), + } + ) + + +class TheSweep(unittest.TestCase): + def sweep(self, api, store=None): + options = parse_arguments([]) + Sweep(api, store or cache(), options, log=lambda _: None).run() + return api + + def test_a_head_commit_with_no_check_suite_is_re_dispatched(self): + api = self.sweep(sweep_api(suites=0)) + dispatched = api.writes("POST", contains="/dispatches") + self.assertEqual(len(dispatched), 1) + self.assertEqual(dispatched[0][2]["inputs"], {"pull_request": "3"}) + + def test_a_run_that_could_not_evaluate_is_re_dispatched(self): + api = self.sweep( + sweep_api( + suites=1, + runs=[{"status": "completed", "conclusion": "success"}], + statuses=[{"context": "validate", "state": "error"}], + ) + ) + self.assertEqual(len(api.writes("POST", contains="/dispatches")), 1) + + def test_a_pass_is_left_alone(self): + api = self.sweep( + sweep_api( + suites=1, + runs=[{"status": "completed", "conclusion": "success"}], + statuses=[{"context": "validate", "state": "success"}], + ) + ) + self.assertEqual(api.sent, []) + + def test_a_reject_is_a_verdict_and_waits_for_the_author(self): + api = self.sweep( + sweep_api(suites=1, statuses=[{"context": "validate", "state": "failure"}]) + ) + self.assertEqual(api.sent, []) + + def test_a_check_run_conclusion_counts_as_the_verdict(self): + api = self.sweep( + sweep_api(suites=1, check_runs=[{"name": "validate", "conclusion": "neutral"}]) + ) + self.assertEqual(len(api.writes("POST", contains="/dispatches")), 1) + + def test_a_run_still_going_is_left_alone(self): + api = self.sweep(sweep_api(suites=1, runs=[{"status": "in_progress"}])) + self.assertEqual(api.sent, []) + + def test_waiting_for_approval_pings_a_steward_instead(self): + api = self.sweep(sweep_api(runs=[{"status": "waiting"}])) + self.assertEqual(len(api.writes("POST", contains="/dispatches")), 0) + comments = api.writes("POST", path="/issues/3/comments") + self.assertEqual(len(comments), 1) + self.assertIn("stewards", comments[0][2]["body"]) + + def test_a_steward_is_pinged_once_per_head_commit(self): + store = cache() + self.sweep(sweep_api(runs=[{"status": "waiting"}]), store) + again = self.sweep(sweep_api(runs=[{"status": "waiting"}]), store) + self.assertEqual(again.sent, []) + + def test_an_existing_ping_is_not_repeated_after_the_cache_is_gone(self): + api = self.sweep( + sweep_api( + runs=[{"status": "waiting"}], + comments=[{"body": "\nalready said"}], + ) + ) + self.assertEqual(api.sent, []) + + def test_a_dispatch_is_not_repeated_within_the_cooldown(self): + store = cache() + self.sweep(sweep_api(suites=0), store) + again = self.sweep(sweep_api(suites=0), store) + self.assertEqual(again.sent, []) + + def test_dispatches_give_up_after_the_attempt_limit(self): + store = cache() + store.section("sweep", "abc").update({"attempts": 3}) + api = self.sweep(sweep_api(suites=0), store) + self.assertEqual(api.sent, []) + + +class WatcherCase(unittest.TestCase): + def watcher(self, folder, argv=(), versions=None): + (folder / "game-versions.json").write_text( + json.dumps({"spec_version": 1, "versions": versions or GAME_VERSIONS["versions"]}) + ) + (folder / ".authored" / "listings").mkdir(parents=True, exist_ok=True) + options = parse_arguments( + [ + "--authored", str(folder / ".authored"), + "--releases", str(folder / "releases"), + "--game-versions", str(folder / "game-versions.json"), + "--cache", str(folder / "cache.json"), + *argv, + ] + ) + # Structural, not conventional: no test reaches the network even when + # the ambient environment carries a token. + options.token = None + watcher = Watcher(options) + watcher.log = lambda message: None + return watcher + + +class TheRepositoryIsTheState(WatcherCase): + + def test_stamped_versions_are_read_from_the_repository(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + releases = folder / "releases" / "AutoStage" + releases.mkdir(parents=True) + (releases / "0.4.3.json").write_text("{}") + (releases / "0.4.4.json").write_text("{}") + watcher = self.watcher(folder) + self.assertEqual( + sorted(watcher.stamped_versions("AutoStage")), ["0.4.3", "0.4.4"] + ) + self.assertEqual(watcher.stamped_versions("Unknown"), {}) + + def test_the_watcher_writes_release_files_and_nothing_else(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder, ["--no-commit"]) + with self.assertRaises(RuntimeError): + watcher.write(folder / "game-versions.json", "{}", "no") + with self.assertRaises(RuntimeError): + watcher.write(folder / ".authored" / "listings" / "X.toml", "", "no") + watcher.write(folder / "releases" / "X" / "1.0.0.json", "{}\n", "yes") + self.assertTrue((folder / "releases" / "X" / "1.0.0.json").is_file()) + + def test_a_delisted_listing_is_left_alone(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder) + listings = folder / ".authored" / "listings" + for identifier in ("Kept", "Gone"): + (listings / f"{identifier}.toml").write_text(f'id = "{identifier}"\n') + (folder / ".authored" / "index-status.toml").write_text( + '[[entries]]\nid = "Gone"\nstate = "delisted"\n' + '[[entries]]\nid = "Kept"\nstate = "disputed"\n' + ) + self.assertEqual([path.stem for path in watcher.listings()], ["Kept"]) + + def test_a_single_listing_can_be_dispatched(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder, ["--listing", "Wanted"]) + listings = folder / ".authored" / "listings" + for identifier in ("Wanted", "Other"): + (listings / f"{identifier}.toml").write_text(f'id = "{identifier}"\n') + self.assertEqual([path.stem for path in watcher.listings()], ["Wanted"]) + + def test_a_path_that_escapes_releases_is_refused(self): + # Path.parents is lexical, so the guard has to resolve: a `..` inside + # the path keeps `releases` in the parents while the file lands outside. + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder, ["--no-commit"]) + with self.assertRaises(RuntimeError): + watcher.write(folder / "releases" / ".." / "evil.json", "{}", "no") + with self.assertRaises(RuntimeError): + watcher.write(folder / "releases" / "X" / ".." / ".." / "evil.json", "{}", "no") + + def test_an_unreadable_index_status_fails_closed(self): + # Failing open would stamp releases a steward delisted. + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder) + (folder / ".authored" / "listings" / "A.toml").write_text('id = "A"\n') + (folder / ".authored" / "index-status.toml").write_text("= broken\n") + self.assertEqual(watcher.listings(), []) + + +class AnIssueOutlivesNothing(WatcherCase): + """An issue about a listing the watcher stopped scanning has to close.""" + + def tick(self, folder, documents, argv=(), **kwargs): + watcher = self.watcher(folder, ["--no-sweep", "--no-commit", *argv]) + watcher.issues = RecorderIssues(**kwargs) + for identifier in documents: + (folder / ".authored" / "listings" / f"{identifier}.toml").write_text( + f'id = "{identifier}"\n' + ) + watcher.tick() + return watcher + + def test_a_delisted_listing_stops_holding_its_issue_open(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + (folder / ".authored").mkdir(parents=True, exist_ok=True) + (folder / ".authored" / "index-status.toml").write_text( + '[[entries]]\nid = "Gone"\nstate = "delisted"\n' + ) + watcher = self.tick( + folder, ("Kept", "Gone"), + listings={"Gone": {"number": 7}, "Kept": {"number": 8}}, + ) + # Gone is never scanned, so only this pass can close it. Kept names + # no host either, which the case below covers. + self.assertIn("Gone", watcher.issues.resolved) + + def test_a_listing_that_drops_its_releases_section_closes_its_issue(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.tick(folder, ("Kept",), listings={}) + # Kept has no [releases] section, so nothing about a host is left + # to report and its issue closes. + self.assertEqual(watcher.issues.resolved, ["Kept"]) + + def test_a_narrow_dispatch_closes_nothing_it_did_not_look_at(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.tick( + folder, ("Wanted", "Other"), argv=("--listing", "Wanted"), + listings={"Other": {"number": 7}}, + ) + self.assertNotIn("Other", watcher.issues.resolved) + + def test_a_failed_issue_read_closes_nothing(self): + # Every id looks orphaned when the list could not be read. + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.tick( + folder, ("Kept",), listings={"Gone": {"number": 7}}, degraded=True + ) + self.assertNotIn("Gone", watcher.issues.resolved) + + +class TheTickSurvivesOneListing(WatcherCase): + def test_a_malformed_listing_does_not_abort_the_tick(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder, ["--no-sweep", "--no-commit"]) + watcher.issues = RecorderIssues() + listings = folder / ".authored" / "listings" + (listings / "Bad.toml").write_text("= broken\n") + (listings / "Good.toml").write_text('id = "Good"\n') + + watcher.tick() + + reported = [listing for listing, _ in watcher.issues.reported] + self.assertIn("Bad", reported) + # The other listing was still scanned, and the cache still saved. + self.assertNotIn("Good", watcher.failed) + self.assertTrue((folder / "cache.json").is_file()) + + def test_an_invalid_id_is_reported_and_never_a_path(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder, ["--no-sweep", "--no-commit"]) + watcher.issues = RecorderIssues() + (folder / ".authored" / "listings" / "Evil.toml").write_text('id = "../Evil"\n') + + watcher.tick() + + self.assertEqual(len(watcher.issues.reported), 1) + self.assertIn("id rules", watcher.issues.reported[0][1][0]) + + def test_a_stem_that_does_not_match_the_id_is_reported(self): + # The delisting and --listing filters match the stem, so a mismatch + # would let a delisted id keep being stamped under another file name. + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder, ["--no-sweep", "--no-commit"]) + watcher.issues = RecorderIssues() + (folder / ".authored" / "listings" / "A.toml").write_text('id = "B"\n') + + watcher.tick() + + self.assertEqual(len(watcher.issues.reported), 1) + self.assertIn("must match", watcher.issues.reported[0][1][0]) + + def test_a_dry_run_leaves_no_cache_behind(self): + # An ETag a dry run stored would make the next real tick take the 304 + # path over releases it never stamped. + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder, ["--no-sweep", "--dry-run"]) + watcher.tick() + self.assertFalse((folder / "cache.json").exists()) + + +class TheMonthPass(WatcherCase): + def release_file(self, folder, listing_id, version, document): + path = folder / "releases" / listing_id + path.mkdir(parents=True, exist_ok=True) + (path / f"{version}.json").write_text(json.dumps(document, indent=2) + "\n") + return path / f"{version}.json" + + def test_a_completed_month_is_resolved_onto_open_stamps(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher( + folder, ["--no-commit"], + versions=["2020.1.1.100", "2020.1.2.150", "2020.2.1.200"], + ) + path = self.release_file(folder, "M", "1.0.0", { + "id": "M", "version": "1.0.0", + "game_min": "2020.1.1.100", "game_min_revision": 100, + "download": {"url": "u"}, + }) + errors = [] + watcher.month_pass( + "M", {"id": "M", "compatibility": {"game_min": "2020.1.1.100", "game_max": "2020.1"}}, errors + ) + document = json.loads(path.read_text()) + self.assertEqual(errors, []) + self.assertEqual(document["game_max"], "2020.1.2.150") + self.assertEqual(document["game_max_revision"], 150) + # Inserted where a fresh stamp puts it, not appended at the end. + self.assertEqual( + list(document), + ["id", "version", "game_min", "game_min_revision", + "game_max", "game_max_revision", "download"], + ) + + def test_the_pass_is_add_only(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher( + folder, ["--no-commit"], versions=["2020.1.1.100", "2020.1.2.150"] + ) + path = self.release_file(folder, "M", "1.0.0", { + "id": "M", "version": "1.0.0", + "game_min": "2020.1.1.100", "game_min_revision": 100, + "game_max": "2020.1.1.100", "game_max_revision": 100, + }) + before = path.read_text() + watcher.month_pass( + "M", {"id": "M", "compatibility": {"game_min": "2020.1.1.100", "game_max": "2020.1"}}, [] + ) + self.assertEqual(path.read_text(), before) + + def test_a_running_month_stays_open(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder, ["--no-commit"], versions=["2020.1.1.100"]) + path = self.release_file(folder, "M", "1.0.0", { + "id": "M", "version": "1.0.0", + "game_min": "2020.1.1.100", "game_min_revision": 100, + }) + before = path.read_text() + future = f"{datetime.now(timezone.utc).year + 1}.1" + errors = [] + watcher.month_pass( + "M", {"id": "M", "compatibility": {"game_min": "2020.1.1.100", "game_max": future}}, errors + ) + self.assertEqual(path.read_text(), before) + self.assertEqual(errors, []) + + def test_a_month_below_the_stamped_min_is_reported_not_applied(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher( + folder, ["--no-commit"], versions=["2020.1.1.100", "2020.1.2.150"] + ) + path = self.release_file(folder, "M", "2.0.0", { + "id": "M", "version": "2.0.0", + "game_min": "2020.3.1.999", "game_min_revision": 999, + }) + before = path.read_text() + errors = [] + watcher.month_pass( + "M", {"id": "M", "compatibility": {"game_min": "2020.1.1.100", "game_max": "2020.1"}}, errors + ) + self.assertEqual(path.read_text(), before) + self.assertEqual(len(errors), 1) + + +class FakeAuthority: + """Serves fixed bytes, and counts how often a download happened.""" + + def __init__(self, payload=b"bytes"): + self.payload = payload + self.downloads = 0 + + def download(self, release): + self.downloads += 1 + return self.payload, "application/zip" + + +def release(version, url, size=None, date="2020-01-01T00:00:00Z"): + return HostRelease( + host="github", tag=f"v{version}", version=version, + release_date=date, url=url, size=size, + ) + + +class SwapsAndMirrors(WatcherCase): + def test_the_same_bytes_at_a_new_url_become_a_mirror(self): + # download.url is immutable, so the proven-identical new address lands + # in download.mirrors instead of rewriting anything. + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder, ["--no-commit"]) + digest = hashlib.sha256(b"bytes").hexdigest().upper() + path = folder / "releases" / "S" + path.mkdir(parents=True) + file = path / "1.0.0.json" + file.write_text(json.dumps({ + "id": "S", "version": "1.0.0", + "download": {"url": "old", "sha256": digest, "size": 5}, + })) + authority = FakeAuthority() + errors = [] + + settled = watcher.check_for_a_swap( + "S", authority, release("1.0.0", "new", size=999), file, errors + ) + + self.assertTrue(settled) + self.assertEqual(errors, []) + self.assertEqual(authority.downloads, 1) + document = json.loads(file.read_text()) + self.assertEqual(document["download"]["url"], "old") + self.assertEqual(document["download"]["mirrors"], ["new"]) + + # The verdict is cached: the next tick neither downloads again nor + # appends a duplicate. + settled = watcher.check_for_a_swap( + "S", authority, release("1.0.0", "new", size=999), file, errors + ) + self.assertTrue(settled) + self.assertEqual(authority.downloads, 1) + self.assertEqual(json.loads(file.read_text())["download"]["mirrors"], ["new"]) + + def test_different_bytes_are_still_a_reported_swap(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder, ["--no-commit"]) + path = folder / "releases" / "S" + path.mkdir(parents=True) + file = path / "1.0.0.json" + file.write_text(json.dumps({ + "id": "S", "version": "1.0.0", + "download": {"url": "old", "sha256": "AA", "size": 5}, + })) + errors = [] + watcher.check_for_a_swap( + "S", FakeAuthority(b"other"), release("1.0.0", "old", size=999), file, errors + ) + self.assertEqual(len(errors), 1) + self.assertIn("stamped exactly", errors[0]) + + def test_a_corrupt_stamped_file_is_reported_not_thrown(self): + # The repository is the state, so a stamped file that does not parse + # has to reach a human instead of the broad catch. + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder, ["--no-commit"]) + path = folder / "releases" / "S" + path.mkdir(parents=True) + file = path / "1.0.0.json" + file.write_text("not json at all") + errors = [] + settled = watcher.check_for_a_swap( + "S", FakeAuthority(), release("1.0.0", "old"), file, errors + ) + self.assertTrue(settled) + self.assertEqual(len(errors), 1) + self.assertIn("releases/S/1.0.0.json", errors[0]) + + def test_mirrors_for_respects_the_budget(self): + # A fresh-stamp burst must not multiply mirror downloads unbounded. + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder, ["--no-commit", "--mirror-budget", "0"]) + + class CountingHost: + key = "spacedock:1" + + def __init__(self): + self.downloads = 0 + + def releases(self, etag=None): + return [release("1.0.0", "mirror-url")], None + + def download(self, entry): + self.downloads += 1 + return b"bytes", "application/zip" + + host = CountingHost() + found = watcher.mirrors_for([host], release("1.0.0", "x"), b"bytes") + self.assertEqual(found, []) + self.assertEqual(host.downloads, 0) + + +class TheLookback(WatcherCase): + def test_skipped_releases_leave_the_tick_unsettled(self): + # A settled tick stores the ETag, and a later backfill dispatch with a + # wider window would then take the 304 path over the skipped releases. + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder, ["--lookback-days", "5", "--no-commit"]) + errors = [] + settled = watcher.stamp_pass( + "L", {}, None, [], [release("1.0.0", "http://x")], errors + ) + self.assertFalse(settled) + self.assertEqual(errors, []) + + +class Unreachable(WatcherCase): + def test_the_threshold_reports_and_keeps_reporting(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder) + watcher.issues = RecorderIssues() + state = {} + for _ in range(5): + watcher.unreachable("U", state, "down") + self.assertEqual(watcher.issues.reported, []) + watcher.unreachable("U", state, "down") + self.assertEqual(len(watcher.issues.reported), 1) + # Past the threshold the issue is kept current, >= not ==. + watcher.unreachable("U", state, "down") + self.assertEqual(len(watcher.issues.reported), 2) + self.assertIn("unreachable_signature", state) + + def test_a_recovery_resolves_by_signature(self): + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + watcher = self.watcher(folder) + watcher.issues = RecorderIssues() + state = {"unreachable": 7, "unreachable_signature": "abc123"} + watcher.recover("U", state) + self.assertEqual(watcher.issues.resolved, [("U", "abc123")]) + self.assertEqual(state["unreachable"], 0) + self.assertNotIn("unreachable_signature", state) + + +class LabelRefusingApi(StubApi): + def __init__(self, code): + super().__init__({"/issues": []}) + self.code = code + + def send(self, method, path, payload): + if method == "POST" and path == "/issues" and "labels" in payload: + raise http_error(self.code) + return super().send(method, path, payload) + + +class IssueRobustness(unittest.TestCase): + def test_only_a_422_falls_back_to_a_label_free_create(self): + api = LabelRefusingApi(422) + Issues(api, "watcher", log=lambda _: None).report("A", ["x"], Cache(None)) + created = api.writes("POST", path="/issues") + self.assertEqual(len(created), 1) + self.assertNotIn("labels", created[0][2]) + + def test_other_errors_do_not_retry_blind(self): + api = LabelRefusingApi(500) + Issues(api, "watcher", log=lambda _: None).report("A", ["x"], Cache(None)) + self.assertEqual(api.sent, []) + + def test_a_degraded_issue_list_never_creates(self): + # A blind create past a failed listing is how duplicates are born. + def failing(query): + raise http_error(500) + + api = StubApi({"/issues": failing}) + Issues(api, "watcher", log=lambda _: None).report("A", ["x"], Cache(None)) + self.assertEqual(api.sent, []) + + def test_resolve_if_matches_the_signature(self): + signature = Issues.signature_of(["down"]) + body = (f"\n" + f"\nold") + issue = {"number": 7, "state": "open", "body": body} + api = StubApi({"/issues/7": issue, "/issues": [issue]}) + Issues(api, "watcher", log=lambda _: None).resolve_if("A", signature, Cache(None)) + self.assertEqual(api.writes("PATCH", path="/issues/7")[0][2], {"state": "closed"}) + + def test_resolve_if_leaves_a_different_failure_alone(self): + body = ("\n" + "\nold") + api = StubApi({"/issues": [{"number": 7, "state": "open", "body": body}]}) + Issues(api, "watcher", log=lambda _: None).resolve_if("A", "0000", Cache(None)) + self.assertEqual(api.sent, []) + + +class DispatchRefusingApi(StubApi): + def send(self, method, path, payload): + if "/dispatches" in path: + raise http_error(404) + return super().send(method, path, payload) + + +class SweepRefusals(unittest.TestCase): + def sweep(self, api, store): + options = parse_arguments([]) + Sweep(api, store, options, log=lambda _: None).run() + return api + + def routes(self): + return { + "/pulls": [{"number": 3, "head": {"sha": "abc"}}], + "/actions/runs": {"workflow_runs": []}, + "/commits/abc/check-suites": {"total_count": 0}, + "/commits/abc/status": {"statuses": []}, + "/commits/abc/check-runs": {"check_runs": []}, + "/issues/3/comments": [], + } + + def test_a_refused_dispatch_is_retried_on_a_clock_not_never(self): + store = Cache(None) + api = DispatchRefusingApi(self.routes()) + self.sweep(api, store) + state = store.section("sweep", "abc") + self.assertIn("refused", state) + + # Within the refusal window nothing is tried again. + again = DispatchRefusingApi(self.routes()) + self.sweep(again, store) + self.assertEqual(again.writes("POST", contains="/dispatches"), []) + + # Once the window has passed, the pull request recovers. + state["refused"] = "2020-01-01T00:00:00Z" + recovered = StubApi(self.routes()) + self.sweep(recovered, store) + self.assertEqual(len(recovered.writes("POST", contains="/dispatches")), 1) + + def test_stale_sweep_state_is_pruned(self): + store = Cache(None) + store.section("sweep", "zzz")["attempts"] = 1 + self.sweep(StubApi(self.routes()), store) + self.assertNotIn("zzz", store.data["sweep"]) + self.assertIn("abc", store.data["sweep"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/verify_examples.py b/tools/verify_examples.py new file mode 100644 index 0000000..9e728ba --- /dev/null +++ b/tools/verify_examples.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Re-derive the design repository's examples and diff them against the stamper. + +examples/ in content-manager-design was stamped by hand from the same archives, +so it is an expectation this repository did not write itself. + + python3 tools/verify_examples.py + python3 tools/verify_examples.py --examples ../content-manager-design/examples + python3 tools/verify_examples.py --check-mirrors + +Needs the network, unlike the unit tests: the bytes have to come from the real +hosts, which exercises tools/hosts.py at the same time. +""" + +import argparse +import difflib +import json +import sys +import tomllib +import urllib.error +from datetime import datetime, timezone +from pathlib import Path + +import hosts +from stamp_release import StampError, serialize, stamp + +DESIGN = "KSAModding/content-manager-design" +RAW = "https://raw.githubusercontent.com/{repository}/main/{path}" +TREE = "https://api.github.com/repos/{repository}/git/trees/main?recursive=1" + + +def from_disk(folder): + listings, releases = {}, {} + for path in sorted((folder / "listings").glob("*.toml")): + listings[path.stem] = path.read_text(encoding="utf-8") + for path in sorted((folder / "releases").glob("*/*.json")): + releases[(path.parent.name, path.stem)] = path.read_text(encoding="utf-8") + return listings, releases + + +def from_github(http, repository): + answer = http.get(TREE.format(repository=repository), accept="application/vnd.github+json", api=True) + paths = [entry["path"] for entry in json.loads(answer.body)["tree"]] + + def fetch(path): + return http.get(RAW.format(repository=repository, path=path)).body.decode("utf-8") + + listings = { + Path(path).stem: fetch(path) + for path in paths + if path.startswith("examples/listings/") and path.endswith(".toml") + } + releases = { + (Path(path).parent.name, Path(path).stem): fetch(path) + for path in paths + if path.startswith("examples/releases/") and path.endswith(".json") + } + return listings, releases + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--examples", type=Path, + help="a local examples/ folder; without it the design repository is fetched", + ) + parser.add_argument("--repository", default=DESIGN) + parser.add_argument( + "--game-versions", type=Path, default=Path("game-versions.json"), + help="the game release list month bounds resolve against", + ) + parser.add_argument( + "--check-mirrors", action="store_true", + help="derive download.mirrors from the non-authority hosts instead of taking " + "the example's word for it. One more full download per mirror", + ) + arguments = parser.parse_args(argv) + + http = hosts.Http(token=_token()) + try: + listings, examples = ( + from_disk(arguments.examples) + if arguments.examples + else from_github(http, arguments.repository) + ) + except (hosts.HostError, urllib.error.HTTPError, OSError) as error: + print(f"could not read the examples: {error}", file=sys.stderr) + return 2 + + game_versions = json.loads( + arguments.game_versions.read_text(encoding="utf-8") + )["versions"] + + failures = 0 + for (listing_id, version), expected in sorted(examples.items()): + try: + rendered = _restamp( + http, listings[listing_id], version, game_versions, + json.loads(expected), arguments.check_mirrors, + ) + except (StampError, hosts.HostError, KeyError, urllib.error.HTTPError) as error: + print(f"{listing_id} {version}: {error}") + failures += 1 + continue + + if rendered == expected: + print(f"{listing_id} {version}: identical") + continue + + failures += 1 + print(f"{listing_id} {version}: differs from the hand-stamped example") + sys.stdout.writelines( + difflib.unified_diff( + expected.splitlines(True), rendered.splitlines(True), + "hand-stamped", "stamper", n=1, + ) + ) + + print(f"\n{len(examples)} example(s), {failures} failure(s)") + return 1 if failures else 0 + + +def _restamp(http, listing_toml, version, game_versions, expected, check_mirrors): + authored = tomllib.loads(listing_toml) + authority, mirror_hosts = hosts.build( + authored.get("releases"), http, authored.get("id") + ) + if authority is None: + raise StampError("the listing names no release host") + + releases, _ = authority.releases() + release = next((entry for entry in releases if entry.version == version), None) + if release is None: + raise StampError(f"{authority.key} does not offer {version}") + + archive, content_type = authority.download(release) + facts = release.facts() + facts["content_type"] = content_type + + mirrors = expected.get("download", {}).get("mirrors", []) + if check_mirrors: + mirrors = _mirrors(mirror_hosts, version, archive) + + # Wall clock on purpose: the original stamp time is unrecorded, and the + # watcher's month pass corrects a stamp once its game_max month completes, + # so re-deriving with the current time matches the corrected state an + # example is expected to hold. + document = stamp( + authored, facts, archive, game_versions, + mirrors=mirrors, now=datetime.now(timezone.utc), + ) + return serialize(document) + + +def _mirrors(mirror_hosts, version, archive): + import hashlib + + digest = hashlib.sha256(archive).hexdigest().upper() + found = [] + for host in mirror_hosts: + releases, _ = host.releases() + release = next((entry for entry in releases if entry.version == version), None) + if release is None: + continue + mirrored, _ = host.download(release) + if hashlib.sha256(mirrored).hexdigest().upper() == digest: + found.append(release.url) + return found + + +def _token(): + import os + + return os.environ.get("GITHUB_TOKEN") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/watch.py b/tools/watch.py new file mode 100644 index 0000000..d4773c0 --- /dev/null +++ b/tools/watch.py @@ -0,0 +1,1293 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""One tick of the watcher (RFC 0033). + +Scan every authored listing's authority host, stamp every release that has no +file under `releases//` yet, commit it, keep one error issue per listing +current on the authored repository, and sweep that repository's open pull +requests. + +What is stamped in the repository is the whole state, so a tick GitHub delays, +drops or cancels costs latency and not data, and a re-run stamps nothing twice. + +Per-release derivation is tools/stamp_release.py, the hosts are tools/hosts.py. +""" + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +import time +import tomllib +import traceback +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import hosts +from hosts import HostError +from stamp_release import ( + GAME_MONTH, + StampError, + month_is_over, + resolve_bound, + serialize, + stamp, + valid_id, +) + +GITHUB_API = "https://api.github.com" + +# 2: the host ETag entries moved to a per-listing key. The cache is derived, +# so a version bump just costs one expensive tick. +CACHE_VERSION = 2 + +# The marker that makes a listing's issue findable without a search, and the +# signature that decides whether a genuinely new error deserves a comment. +LISTING_MARKER = "" +SIGNATURE_MARKER = "" +WAITING_MARKER = "" + +# The same marker read back, to find which listing an open issue belongs to. +MARKED_LISTING = re.compile(r"") + +# A check run conclusion that is neither a pass nor a reject: the check could +# not run to a verdict, which never auto-merges and never auto-rejects, and is +# what the sweep re-dispatches. +COULD_NOT_EVALUATE = frozenset( + {"cancelled", "timed_out", "stale", "neutral", "skipped", "action_required"} +) +PENDING = frozenset({"queued", "in_progress", "waiting", "requested", "pending"}) + + +def now(): + return datetime.now(timezone.utc) + + +def iso(moment): + return moment.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def parse_iso(text): + try: + return datetime.fromisoformat((text or "").replace("Z", "+00:00")) + except (TypeError, ValueError): + return None + + +class Cache: + """Derived cache, never state. + + It holds the per-listing ETags, the consecutive-failure counts, the issue + numbers, and what the mirror and sweep passes already tried. Every entry is + rebuildable from the repository and the hosts, so losing the whole file + costs one expensive tick and nothing else. + """ + + def __init__(self, path, log=None): + self._log = log or (lambda message: None) + self.path = Path(path) if path else None + self.data = {"version": CACHE_VERSION} + if self.path and self.path.is_file(): + try: + loaded = json.loads(self.path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as error: + # Starting cold is fine, but a restore that fails every tick + # must not look exactly like one, so say it happened. + self._log(f"the derived cache could not be read and starts cold: {error}") + loaded = {} + if isinstance(loaded, dict) and loaded.get("version") == CACHE_VERSION: + self.data = loaded + for section in ("hosts", "listings", "mirrors", "swaps", "sweep"): + self.data.setdefault(section, {}) + + def section(self, name, key): + return self.data[name].setdefault(key, {}) + + def save(self): + """Best-effort, like every other cache operation: a full disk after the + stamping is done must not fail the tick and keep the push from running.""" + if not self.path: + return + try: + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("w", encoding="utf-8", newline="\n") as handle: + json.dump(self.data, handle, indent=1, sort_keys=True) + handle.write("\n") + except OSError as error: + self._log(f"the derived cache could not be written: {error}") + + +class Api: + """The GitHub API calls that write, so a dry run can be one flag.""" + + def __init__(self, http, repository, dry_run=False, log=print): + self.http = http + self.repository = repository + self.dry_run = dry_run + self.log = log + + def get(self, path, **query): + url = f"{GITHUB_API}/repos/{self.repository}{path}" + if query: + url += "?" + urllib.parse.urlencode(query) + answer = self.http.get(url, accept="application/vnd.github+json", api=True) + return json.loads(answer.body) if answer.body else None + + def get_paged(self, path, key=None, max_pages=10, **query): + """Every item across pages, because a one-page read that looks complete + is how a duplicate issue gets opened past 100 open ones.""" + items = [] + for page in range(1, max_pages + 1): + answer = self.get(path, per_page=100, page=page, **query) + batch = (answer or {}).get(key) if key else (answer or []) + batch = batch or [] + items.extend(batch) + if len(batch) < 100: + break + return items + + def send(self, method, path, payload): + url = f"{GITHUB_API}/repos/{self.repository}{path}" + if self.dry_run or not self.http.token: + self.log(f" would {method} {path} {json.dumps(payload)[:200]}") + return None + body = json.dumps(payload).encode("utf-8") + last = None + for attempt in range(3): + request = urllib.request.Request( + url, + data=body, + method=method, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.http.token}", + "Content-Type": "application/json", + "User-Agent": hosts.USER_AGENT, + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + try: + with urllib.request.urlopen(request, timeout=self.http.timeout) as answer: + raw = answer.read() + return json.loads(raw) if raw else None + except urllib.error.HTTPError as error: + # A 4xx is an answer. A 5xx, or a secondary rate limit on a + # write (a 403 with Retry-After), is the API having a bad + # moment, and one of those must not mark a listing failed for + # a reason that has nothing to do with the listing. + transient = error.code in (403, 429) and hosts._rate_limited(error.headers) + if error.code < 500 and not transient: + raise + last = error + except (urllib.error.URLError, TimeoutError, OSError) as error: + last = error + if attempt < 2: + time.sleep(2 ** attempt) + raise HostError(f"{method} {url}: {last}") + + +class Issues: + """One open issue per listing on the authored repository, kept current. + + A new tick never opens a second issue for a listing that already has one: + the body is rewritten to the current failure, and a comment is added only + when the failure itself changed, so a host that stays down is one issue and + no notifications. + """ + + def __init__(self, api, label, log=print): + self.api = api + self.label = label + self.log = log + self._open = {} + self._degraded = False + + @staticmethod + def signature_of(errors): + """The stable fingerprint of a failure, for edit-versus-comment decisions.""" + return hashlib.sha256("\n".join(sorted(errors)).encode()).hexdigest()[:16] + + def _all_open(self, labelled): + """The open issues of the authored repository, listed once per tick. + + The label narrows the list to the watcher's own issues, and the + unlabelled list is the fallback for the tick that opened an issue before + the label existed. A listing that fails marks the lookup degraded, so + `report` skips creating rather than duplicating an issue it could not + see. + """ + if labelled not in self._open: + query = {"labels": self.label} if labelled else {} + try: + issues = self.api.get_paged("/issues", state="open", **query) + except (urllib.error.HTTPError, HostError) as error: + self.log(f" could not list issues: {error}") + self._degraded = True + issues = [] + self._open[labelled] = [ + issue for issue in issues if "pull_request" not in issue + ] + return self._open[labelled] + + def find(self, listing_id, cache): + marker = LISTING_MARKER.format(id=listing_id) + remembered = cache.section("listings", listing_id).get("issue") + if remembered: + try: + issue = self.api.get(f"/issues/{remembered}") + except (urllib.error.HTTPError, HostError): + issue = None + if issue and issue.get("state") == "open" and marker in (issue.get("body") or ""): + return issue + for labelled in (True, False): + for issue in self._all_open(labelled): + if marker in (issue.get("body") or ""): + cache.section("listings", listing_id)["issue"] = issue["number"] + return issue + return None + + def report(self, listing_id, errors, cache): + """Keep the listing's issue current with `errors`. + + Never raises for an API-shaped failure: reporting is best-effort, and + several callers sit inside except clauses, where a raise would leave + the per-listing guard and take the rest of the tick with it. + """ + try: + self._report(listing_id, errors, cache) + except (urllib.error.HTTPError, HostError) as error: + self.log(f" could not keep the issue for {listing_id} current: {error}") + + def _report(self, listing_id, errors, cache): + signature = self.signature_of(errors) + body = self._body(listing_id, errors, signature) + title = f"{listing_id}: the watcher could not stamp a release" + issue = self.find(listing_id, cache) + + if issue is None: + if self._degraded: + self.log( + " not opening an issue: the issue list could not be read " + "this tick, and a blind create duplicates" + ) + return + try: + created = self.api.send( + "POST", "/issues", {"title": title, "body": body, "labels": [self.label]} + ) + except urllib.error.HTTPError as error: + if error.code != 422: + self.log(f" could not open an issue (HTTP {error.code})") + return + # A label the repository does not define is not worth losing the + # report over; the marker in the body is what the watcher finds + # the issue by anyway. + self.log(f" the '{self.label}' label was refused (HTTP 422)") + try: + created = self.api.send("POST", "/issues", {"title": title, "body": body}) + except urllib.error.HTTPError as retry_error: + self.log(f" could not open an issue (HTTP {retry_error.code})") + return + if created: + cache.section("listings", listing_id)["issue"] = created["number"] + self.log(f" opened {self.api.repository}#{created['number']}") + return + + number = issue["number"] + known = SIGNATURE_MARKER.format(signature=signature) in (issue.get("body") or "") + self.api.send("PATCH", f"/issues/{number}", {"title": title, "body": body}) + if not known: + self.api.send( + "POST", + f"/issues/{number}/comments", + {"body": "The watcher is now failing on something else:\n\n" + + "\n".join(f"- {error}" for error in errors)}, + ) + self.log(f" kept {self.api.repository}#{number} current") + + def open_listings(self): + """The listing id of every open issue the watcher owns, to its issue.""" + found = {} + for labelled in (True, False): + for issue in self._all_open(labelled): + match = MARKED_LISTING.search(issue.get("body") or "") + if match: + found.setdefault(match.group(1), issue) + return found + + @property + def degraded(self): + """Whether a read failed this tick, so the issue list is incomplete.""" + return self._degraded + + def resolve(self, listing_id, cache, reason=None): + """Close the listing's issue, because the tick evaluated it cleanly. + + Best-effort like `report`: a failure here leaves an issue open one tick + longer, which is not worth the rest of the tick. + """ + try: + self._resolve(listing_id, cache, reason) + except (urllib.error.HTTPError, HostError) as error: + self.log(f" could not close the issue for {listing_id}: {error}") + + def _resolve(self, listing_id, cache, reason=None): + issue = self.find(listing_id, cache) + if issue is None: + return + number = issue["number"] + self.api.send( + "POST", + f"/issues/{number}/comments", + {"body": reason + or "The watcher stamped this listing without an error, so this is done."}, + ) + self.api.send("PATCH", f"/issues/{number}", {"state": "closed"}) + cache.section("listings", listing_id).pop("issue", None) + self.log(f" closed {self.api.repository}#{number}") + + def resolve_if(self, listing_id, signature, cache): + """Close the listing's issue only when it reports exactly `signature`. + + The recovery from an unreachable host must not close an issue that + meanwhile reports something else, and the signature marker is what + tells the two apart. + """ + issue = self.find(listing_id, cache) + if issue is None: + return + if SIGNATURE_MARKER.format(signature=signature) in (issue.get("body") or ""): + self.resolve(listing_id, cache) + + def _body(self, listing_id, errors, signature): + return "\n".join( + [ + LISTING_MARKER.format(id=listing_id), + SIGNATURE_MARKER.format(signature=signature), + f"The watcher cannot stamp `{listing_id}`.", + "", + *[f"- {error}" for error in errors], + "", + "The watcher retries every tick and keeps this issue current rather than", + "opening a new one. It closes by itself once a tick evaluates the listing", + "without an error.", + "", + f"Last checked {iso(now())}.", + ] + ) + + +class Sweep: + """The event-driven half, swept once per tick. + + GitHub drops event-driven triggers with nothing to retry them, so every tick + re-dispatches the validation of an open pull request whose head commit has + no check suite or whose latest run ended in could-not-evaluate, and pings a + steward for a run waiting for approval, which a dispatch cannot release. + """ + + def __init__(self, api, cache, options, log=print): + self.api = api + self.cache = cache + self.options = options + self.log = log + + def run(self): + try: + pulls = self.api.get_paged("/pulls", state="open")[: self.options.sweep_limit] + except urllib.error.HTTPError as error: + self.log(f" could not list pull requests: HTTP {error.code}") + return + except HostError as error: + self.log(f" could not list pull requests: {error}") + return + + for pull in pulls: + try: + self._one(pull) + except (urllib.error.HTTPError, HostError) as error: + self.log(f" #{pull['number']}: {error}") + + # The sweep section would otherwise keep a key per head commit forever, + # in a file the workflow uploads and downloads every ten minutes. + open_shas = {pull["head"]["sha"] for pull in pulls} + section = self.cache.data["sweep"] + for sha in [key for key in section if key not in open_shas]: + del section[sha] + + def _one(self, pull): + number, sha = pull["number"], pull["head"]["sha"] + state = self.cache.section("sweep", sha) + + runs = (self.api.get("/actions/runs", head_sha=sha, per_page=50) or {}).get( + "workflow_runs", [] + ) + waiting = [ + run + for run in runs + if run.get("status") in ("waiting", "action_required") + or run.get("conclusion") == "action_required" + ] + if waiting: + self._ping(number, sha, state) + return + + suites = self.api.get(f"/commits/{sha}/check-suites") or {} + if any(run.get("status") in PENDING for run in runs): + return # Something is still running; a verdict is on its way. + + verdict = self._verdict(sha) + if suites.get("total_count") and verdict == "pass": + return + if verdict == "reject": + return # A reject is a verdict. It waits for the author, not for us. + + reason = ( + "no check suite" if not suites.get("total_count") else f"verdict {verdict}" + ) + self._dispatch(number, sha, state, reason) + + def _verdict(self, sha): + """`pass`, `reject`, `could-not-evaluate`, or `missing` for the required check.""" + wanted = self.options.verdict_check + status = self.api.get(f"/commits/{sha}/status") or {} + for entry in status.get("statuses") or []: + if entry.get("context") == wanted: + return { + "success": "pass", + "failure": "reject", + "error": "could-not-evaluate", + "pending": "missing", + }.get(entry.get("state"), "could-not-evaluate") + + checks = self.api.get(f"/commits/{sha}/check-runs") or {} + for entry in checks.get("check_runs") or []: + if entry.get("name") != wanted: + continue + conclusion = entry.get("conclusion") + if conclusion == "success": + return "pass" + if conclusion == "failure": + return "reject" + if conclusion in COULD_NOT_EVALUATE: + return "could-not-evaluate" + return "missing" + return "missing" + + def _dispatch(self, number, sha, state, reason): + attempts = state.get("attempts", 0) + last = parse_iso(state.get("last")) + refused = parse_iso(state.get("refused")) + if attempts >= self.options.sweep_attempts: + return + if last and now() - last < timedelta(minutes=self.options.sweep_cooldown): + return + if refused and now() - refused < timedelta(hours=self.options.sweep_refusal_hours): + return + + self.log(f" #{number}: re-dispatching validation ({reason})") + try: + self.api.send( + "POST", + f"/actions/workflows/{self.options.sweep_workflow}/dispatches", + {"ref": self.options.sweep_ref, "inputs": {"pull_request": str(number)}}, + ) + except urllib.error.HTTPError as error: + # A validation workflow that does not accept a dispatch yet is the + # authored repository's business, and never fails a tick. The + # refusal is retried on a clock rather than retired for good, so + # stuck pull requests recover the moment the workflow gains the + # input. + self.log( + f" #{number}: {self.options.sweep_workflow} did not accept the dispatch " + f"(HTTP {error.code}); the sweep needs it to take a pull_request input" + ) + state["refused"] = iso(now()) + return + state.pop("refused", None) + state["last"] = iso(now()) + state["attempts"] = attempts + 1 + + def _ping(self, number, sha, state): + if state.get("pinged"): + return + marker = WAITING_MARKER.format(sha=sha) + comments = self.api.get(f"/issues/{number}/comments", per_page=100) or [] + if any(marker in (comment.get("body") or "") for comment in comments): + state["pinged"] = True + return + self.log(f" #{number}: waiting for approval, pinging a steward") + self.api.send( + "POST", + f"/issues/{number}/comments", + { + "body": f"{marker}\n{self.options.steward_team} this run is sitting in " + "GitHub's waiting-for-approval state, which the watcher cannot release " + "with a dispatch. It needs a steward to approve the workflow run." + }, + ) + state["pinged"] = True + + +class Watcher: + def __init__(self, options): + self.options = options + self.releases_root = Path(options.releases) + self.authored_root = Path(options.authored) + self.cache = Cache(options.cache, log=self.log) + self.http = hosts.Http(token=options.token, log=self.log) + self.api = Api(self.http, options.authored_repo, options.dry_run, self.log) + self.issues = Issues(self.api, options.issue_label, self.log) + self.game_versions = json.loads( + Path(options.game_versions).read_text(encoding="utf-8") + )["versions"] + self.stamp_budget = options.stamp_budget + self.mirror_budget = options.mirror_budget + self.stamped = [] + self.mirrored = [] + self.failed = [] + self.lines = [] + self._mirror_lists = {} + + def log(self, message): + print(message, flush=True) + self.lines.append(str(message)) + + def folder(self, listing_id): + return self.releases_root / listing_id + + def stamped_versions(self, listing_id): + """The versions of a listing that have a file, which is the whole state.""" + folder = self.folder(listing_id) + if not folder.is_dir(): + return {} + return {path.stem: path for path in sorted(folder.glob("*.json"))} + + def read_release(self, path, errors): + """A stamped release file, or None with the corruption reported. + + The repository is the state, so a stamped file that does not parse is + exactly the corruption that has to reach a human, not the catch-all. + """ + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + errors.append( + f"the stamped file releases/{path.parent.name}/{path.name} " + f"is not readable JSON: {error}" + ) + return None + + def write(self, path, text, message): + """Write one release file and commit it. Refuses anything else. + + The branch protection bypass is scoped to an identity, not to a path, so + this is where the limit is enforced. Both sides are resolved: a `..` + keeps `releases` in `Path.parents` while the file lands outside it. + """ + path = Path(path) + try: + contained = path.resolve().is_relative_to(self.releases_root.resolve()) + except OSError: + contained = False + if not contained or path.resolve() == self.releases_root.resolve(): + raise RuntimeError(f"the watcher does not write {path}") + if self.options.dry_run: + self.log(f" would write {path} and commit '{message}'") + return + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(text) + self.commit(path, message) + + def commit(self, path, message): + """One commit per release file, scoped to that path and nothing else.""" + if self.options.no_commit: + return + subprocess.run(["git", "add", "--", str(path)], check=True) + unchanged = subprocess.run( + ["git", "diff", "--cached", "--quiet", "--", str(path)], check=False + ).returncode + if unchanged == 0: + return # The file already says this, so there is nothing to record. + subprocess.run( + ["git", "commit", "--quiet", "-m", message, "--", str(path)], check=True + ) + + def listings(self): + """The authored listing documents this tick looks at.""" + folder = self.authored_root / "listings" + if not folder.is_dir(): + self.log(f"{folder} does not exist, so there is nothing to watch") + return [] + + wanted = {name.lower() for name in self.options.listing or []} + delisted = self.delisted() + if delisted is None: + # Failing open would stamp releases a steward delisted, so an + # unreadable status file skips the whole tick's listings instead. + self.log("index-status.toml is unreadable, so no listing is scanned this tick") + return [] + chosen = [] + for path in sorted(folder.glob("*.toml")): + if wanted and path.stem.lower() not in wanted: + continue + if path.stem.lower() in delisted: + self.log(f"{path.stem}: delisted, so the watcher leaves it alone") + continue + chosen.append(path) + return chosen + + def delisted(self): + """The ids the index has delisted, or None when the file is unreadable. + + A delisted listing is out of the snapshot, so stamping further releases + for it would be the watcher arguing with a steward. + """ + path = self.authored_root / "index-status.toml" + if not path.is_file(): + return set() + try: + with path.open("rb") as handle: + document = tomllib.load(handle) + except (OSError, tomllib.TOMLDecodeError) as error: + self.log(f"could not read {path.name}: {error}") + return None + return { + (entry.get("id") or "").lower() + for entry in document.get("entries") or [] + if entry.get("state") == "delisted" + } + + def listing_problem(self, path, listing_id): + """Why this listing cannot be processed at all, or None. + + The id becomes a path segment under releases/, so the id rules are the + gate in front of every path this tick builds from it, and the file stem + is what the delisting and `--listing` filters match, so it has to name + the same listing. + """ + if not valid_id(listing_id): + return ( + f"the id '{listing_id}' does not satisfy the id rules of " + "RFC 0031, so the watcher does not use it" + ) + if path.stem.lower() != listing_id.lower(): + return ( + f"the file is named '{path.stem}.toml' but the document says " + f"id = '{listing_id}'; the file name and the id must match" + ) + return None + + def tick(self): + try: + paths = self.listings() + for path in paths: + listing_id = path.stem + try: + with path.open("rb") as handle: + authored = tomllib.load(handle) + listing_id = (authored.get("id") or path.stem).strip() + problem = self.listing_problem(path, listing_id) + if problem: + self.log(f"{listing_id}: {problem}") + self.failed.append(listing_id) + self.issues.report(listing_id, [problem], self.cache) + continue + self.log(f"{listing_id}:") + self.one_listing(listing_id, authored) + except tomllib.TOMLDecodeError as error: + # `report` never raises for API failures, which matters + # here: an exception inside an except clause would leave + # the loop past the sibling guard below. + message = f"{path.name} is not valid TOML: {error}" + self.log(f"{listing_id}: {message}") + self.failed.append(listing_id) + self.issues.report(listing_id, [message], self.cache) + except Exception as error: # noqa: BLE001 - one listing never fails the tick + self.log(f" unexpected: {error!r}") + self.log(traceback.format_exc()) + self.failed.append(listing_id) + self.issues.report( + listing_id, + [f"the watcher hit an internal error on this listing: {error!r}"], + self.cache, + ) + + self.close_orphans(paths) + + if not self.options.no_sweep: + self.log(f"sweeping {self.options.authored_repo}:") + Sweep(self.api, self.cache, self.options, self.log).run() + finally: + # A dry run must leave no trace: an ETag it stored would make the + # next real tick take the 304 path over releases it never stamped. + if not self.options.dry_run: + self.cache.save() + self.summarize() + return 0 + + def close_orphans(self, paths): + """Close the issue of a listing this tick no longer scans. + + A delisted or deleted listing never reaches `one_listing`, so nothing + else closes its issue. Closing is the destructive direction, so three + cases are skipped: a narrow dispatch, an empty listing set, which is + what an unreadable status file looks like, and a failed issue read. + """ + if self.options.listing or not paths or self.issues.degraded: + return + watched = {path.stem.lower() for path in paths} + for listing_id, issue in self.issues.open_listings().items(): + if listing_id.lower() in watched: + continue + self.log(f"{listing_id}: no longer scanned, closing #{issue['number']}") + self.issues.resolve( + listing_id, + self.cache, + reason="The watcher no longer scans this listing, so it has nothing " + "left to report here. A delisting, a deleted document, or a renamed " + "id gets here.", + ) + + def one_listing(self, listing_id, authored): + state = self.cache.section("listings", listing_id) + errors = [] + + self.month_pass(listing_id, authored, errors) + + try: + authority, mirrors = hosts.build( + authored.get("releases"), self.http, listing_id + ) + except StampError as error: + self.failed.append(listing_id) + self.issues.report(listing_id, errors + [str(error)], self.cache) + return + if authority is None: + self.log(" no [releases] section, so releases enter by pull request") + if errors: + self.failed.append(listing_id) + self.issues.report(listing_id, errors, self.cache) + else: + # The listing left the watcher's half. Anything it reports here + # is about a host it no longer names. + self.issues.resolve( + listing_id, + self.cache, + reason="This listing has no [releases] section any more, so the " + "watcher has nothing left to report here.", + ) + return + + # Keyed per listing, not per host: two listings naming the same + # repository must not blind each other through a shared ETag. + host_state = self.cache.section("hosts", f"{listing_id}/{authority.key}") + try: + releases, etag = authority.releases(host_state.get("etag")) + except HostError as error: + self.unreachable(listing_id, state, str(error)) + return + except StampError as error: + state["unreachable"] = 0 + state.pop("unreachable_signature", None) + self.failed.append(listing_id) + self.issues.report(listing_id, errors + [str(error)], self.cache) + return + + self.recover(listing_id, state) + settled = True + + if releases is None: + self.log(" unchanged since the last tick") + else: + self.log(f" {len(releases)} release(s) on {authority.key}") + if getattr(authority, "truncated", False): + errors.append( + "the host lists more releases than one scan covers, so the " + "oldest are not watched; raising the watcher's max_pages " + "needs a human" + ) + settled = self.stamp_pass( + listing_id, authored, authority, mirrors, releases, errors + ) + + self.mirror_pass(listing_id, mirrors, errors) + + if settled: + # The ETag stands for "every release behind this answer is either + # stamped or reported", so a tick that ran out of budget or could + # not reach the host always refetches, while a release that will + # never be stampable costs no request per tick. A payload that + # changes changes the ETag, so a fixed tag is picked up at once. + if etag: + host_state["etag"] = etag + host_state["checked"] = iso(now()) + else: + host_state.pop("etag", None) + + if errors: + self.failed.append(listing_id) + for error in errors: + self.log(f" reporting: {error}") + self.issues.report(listing_id, errors, self.cache) + elif releases is not None and settled: + self.issues.resolve(listing_id, self.cache) + + def stamp_pass(self, listing_id, authored, authority, mirror_hosts, releases, errors): + """Stamp every release with no file yet. + + Returns whether every release behind this answer is now settled, which + means stamped or reported. A rejected release is settled: nothing about + it will change until the host's answer does, and that changes the ETag. + A host that could not be reached and a budget that ran out are not. + """ + stamped = self.stamped_versions(listing_id) + cutoff = ( + now() - timedelta(days=self.options.lookback_days) + if self.options.lookback_days + else None + ) + settled = True + outside_lookback = 0 + + # Oldest first, so a budget that runs out leaves a monotone history and + # the next tick simply carries on. + ordered = sorted(releases, key=lambda release: (release.release_date or "", release.tag)) + for release in ordered: + if release.version is None: + errors.append( + f"the tag `{release.tag}` does not parse as a version, so the release " + "cannot be stamped; SemVer 2.0.0, with an optional leading `v`" + ) + continue + + if release.version in stamped: + if not self.check_for_a_swap( + listing_id, authority, release, stamped[release.version], errors + ): + settled = False + continue + + date = parse_iso(release.release_date) + if cutoff and date and date < cutoff: + # Skipped, not settled: the ETag must not claim these are done, + # or a later backfill dispatch takes the 304 path over them. + outside_lookback += 1 + settled = False + continue + + if self.stamp_budget <= 0: + self.log(" the stamp budget is spent, the next tick carries on") + return False + + try: + self.stamp_one(listing_id, authored, authority, mirror_hosts, release) + except HostError as error: + self.log(f" {release.version}: {error}") + settled = False + except StampError as error: + errors.append(f"`{release.version}`: {error}") + else: + self.stamp_budget -= 1 + + if outside_lookback: + self.log( + f" {outside_lookback} release(s) outside the lookback window, " + "left for a wider tick" + ) + return settled + + def stamp_one(self, listing_id, authored, authority, mirror_hosts, release): + archive, content_type = authority.download(release) + facts = release.facts() + facts["content_type"] = content_type + + mirrors = self.mirrors_for(mirror_hosts, release, archive) + document = stamp( + authored, facts, archive, self.game_versions, mirrors=mirrors, now=now() + ) + path = self.folder(listing_id) / f"{document['version']}.json" + self.write(path, serialize(document), f"Stamp {listing_id} {document['version']}") + self.log(f" stamped {document['version']} ({len(archive)} bytes)") + self.stamped.append(f"{listing_id} {document['version']}") + + def check_for_a_swap(self, listing_id, authority, release, path, errors): + """A stamped version is never overwritten, and a swap gets reported. + + The signal is the size and URL the release list already carries; a swap + keeping the byte count needs every archive re-downloaded per tick for an + answer `download.sha256` already gives the client. Two limits: SpaceDock + reports no size, so only the URL comparison remains there, and a deleted + asset reads as unchanged. + """ + document = self.read_release(path, errors) + if document is None: + return True # Reported; nothing changes here until a human acts. + download = document.get("download") or {} + stamped_digest = (download.get("sha256") or "").upper() + same_size = release.size is None or release.size == download.get("size") + same_url = (release.url or download.get("url")) == download.get("url") + if same_size and same_url: + return True + + # A rejection is permanent, so re-downloading the swapped archive every + # tick would spend the tick on an answer that is already known. The + # cache is derived: losing it costs one more download. + seen = self.cache.section("swaps", f"{listing_id}/{release.version}") + if seen.get("size") == release.size and seen.get("url") == release.url: + if seen.get("digest"): + errors.append( + self.swap_error(release.version, stamped_digest, seen["digest"]) + ) + return True + + try: + archive, _ = authority.download(release) + except HostError as error: + self.log(f" {release.version}: {error}") + return False + except StampError as error: + errors.append(f"`{release.version}`: {error}") + return True + + digest = hashlib.sha256(archive).hexdigest().upper() + seen.update({"size": release.size, "url": release.url, "checked": iso(now())}) + if digest == stamped_digest: + seen["digest"] = None + self.log(f" {release.version}: the same bytes at a new URL") + self.append_mirror(path, document, release.url) + return True + + seen["digest"] = digest + errors.append(self.swap_error(release.version, stamped_digest, digest)) + return True + + @staticmethod + def swap_error(version, stamped_digest, served_digest): + return ( + f"`{version}` is already stamped from `{stamped_digest}`, and the host now " + f"serves `{served_digest}` for the same tag. A version is stamped exactly " + "once and the file is never overwritten, so this release is rejected. The " + "way forward is a new version, or a yank of this one." + ) + + def month_pass(self, listing_id, authored, errors): + """Resolve an authored `game_max` month once that month is over. + + Adding the bound is the stamp correction RFC 0033 describes, not an + amendment: the file only becomes less permissive and nothing already + present is touched. It reads the authored document, so it needs no host + and runs for every listing. + """ + bound = ((authored.get("compatibility") or {}).get("game_max") or "").strip() + match = GAME_MONTH.match(bound) + if match is None: + return + if not month_is_over(int(match.group(1)), int(match.group(2)), now()): + return + + try: + display, revision = resolve_bound(bound, "game_max", self.game_versions, now()) + except StampError as error: + errors.append(f"game_max: {error}") + return + if display is None: + return + + for version, path in self.stamped_versions(listing_id).items(): + document = self.read_release(path, errors) + if document is None: + continue + if "game_max" in document or "game_min_revision" not in document: + continue + if revision < document["game_min_revision"]: + errors.append( + f"game_max `{bound}` resolves to revision {revision}, below " + f"`{version}`'s stamped game_min_revision " + f"{document['game_min_revision']}, so it is not applied" + ) + continue + updated = {} + for key, value in document.items(): + updated[key] = value + if key == "game_min_revision": + # Inserted where a fresh stamp would put it, so a corrected + # file and a fresh one have the same shape. + updated["game_max"] = display + updated["game_max_revision"] = revision + self.write( + path, + serialize(updated), + f"Resolve the game_max month for {listing_id} {version}", + ) + self.log(f" resolved game_max {display} onto {version}") + + def append_mirror(self, path, document, url): + """Record a further URL proven byte-identical, as a mirror. + + `download.url` is immutable, so a release whose authority now serves + the same bytes from a new address keeps its stamped URL and gains the + new one as a mirror, which RFC 0031 admits for any source whose bytes + match the sha256. + """ + download = document.get("download") or {} + if not url or url == download.get("url"): + return + mirrors = list(download.get("mirrors") or []) + if url in mirrors: + return + download["mirrors"] = mirrors + [url] + self.write( + path, + serialize(document), + f"Add a mirror for {document['id']} {document['version']}", + ) + self.mirrored.append(f"{document['id']} {document['version']}") + + def mirrors_for(self, mirror_hosts, release, archive): + """The non-authority hosts serving byte-identical bytes for this release. + + Shares the mirror budget with `mirror_pass`: verifying costs a full + download, and a fresh-stamp burst must not multiply that unbounded. A + mirror that did not fit the budget is appended by a later tick's pass. + """ + digest = hashlib.sha256(archive).hexdigest().upper() + found = [] + for host in mirror_hosts: + candidate = self.mirror_release(host, release.version) + if candidate is None: + continue + if self.mirror_budget <= 0: + break + self.mirror_budget -= 1 + url = self.verify_mirror(host, candidate, digest) + if url: + found.append(url) + return found + + def mirror_release(self, host, version): + """The mirror host's release for `version`, from one list per tick.""" + if host.key not in self._mirror_lists: + try: + listed, _ = host.releases() + except (HostError, StampError) as error: + self.log(f" {host.key}: {error}") + listed = [] + self._mirror_lists[host.key] = listed or [] + return next( + ( + release + for release in self._mirror_lists[host.key] + if release.version == version + ), + None, + ) + + def verify_mirror(self, host, release, digest): + """The mirror's URL when its bytes are identical, else None.""" + try: + archive, _ = host.download(release) + except (HostError, StampError) as error: + self.log(f" {host.key}: {error}") + return None + if hashlib.sha256(archive).hexdigest().upper() != digest: + self.log(f" {host.key} serves different bytes for {release.version}") + return None + return release.url + + def mirror_pass(self, listing_id, mirror_hosts, errors): + """Append a mirror that appeared after a release was stamped. + + The one field the watcher may append to after publish, watcher-only and + append-only. Verifying one costs a full download, so a tick works + through the least recently checked candidates within a budget, and the + rest wait for the next tick. + """ + if not mirror_hosts or self.mirror_budget <= 0: + return + + candidates = [] + for version, path in self.stamped_versions(listing_id).items(): + document = self.read_release(path, errors) + if document is None: + continue + known = set((document.get("download") or {}).get("mirrors") or []) + if len(known) >= len(mirror_hosts): + continue + key = f"{listing_id}/{version}" + checked = parse_iso(self.cache.section("mirrors", key).get("checked")) + candidates.append((checked or datetime.min.replace(tzinfo=timezone.utc), version, path)) + + for _, version, path in sorted(candidates, key=lambda entry: entry[0]): + if self.mirror_budget <= 0: + return + document = self.read_release(path, errors) + if document is None: + continue + download = document.get("download") + if not download: + continue + known = list(download.get("mirrors") or []) + found = [] + for host in mirror_hosts: + candidate = self.mirror_release(host, version) + if candidate is None or candidate.url in known: + continue + self.mirror_budget -= 1 + url = self.verify_mirror(host, candidate, (download.get("sha256") or "").upper()) + if url: + found.append(url) + self.cache.section("mirrors", f"{listing_id}/{version}")["checked"] = iso(now()) + if not found: + continue + download["mirrors"] = known + found + self.write( + path, + serialize(document), + f"Add a mirror for {listing_id} {version}", + ) + self.log(f" appended {len(found)} mirror(s) to {version}") + self.mirrored.append(f"{listing_id} {version}") + + def unreachable(self, listing_id, state, message): + """A host that could not be evaluated. Latency, not data. + + It reaches the author only after consecutive failed ticks, so a short + outage stays out of everyone's notifications. `>=` rather than `==` + keeps a lost cache from restarting the countdown silently, and the count + stays out of the text so an unchanged outage is not a new failure. + """ + count = state.get("unreachable", 0) + 1 + state["unreachable"] = count + self.log(f" could not be evaluated ({count} tick(s) in a row): {message}") + if count >= self.options.unreachable_ticks: + errors = [ + "the authority host has stayed unreachable across consecutive " + f"ticks: {message}" + ] + state["unreachable_signature"] = Issues.signature_of(errors) + self.failed.append(listing_id) + self.issues.report(listing_id, errors, self.cache) + + def recover(self, listing_id, state): + """The host answered again. Close the outage issue, and only that one. + + The signature guard keeps a recovery from closing an issue that + meanwhile reports something else about the listing. + """ + signature = state.pop("unreachable_signature", None) + state["unreachable"] = 0 + if signature: + self.issues.resolve_if(listing_id, signature, self.cache) + + def summarize(self): + summary = [ + "## Watcher", + "", + f"- stamped: {len(self.stamped)}", + f"- mirrors appended: {len(self.mirrored)}", + f"- listings with an error: {len(set(self.failed))}", + f"- host requests: {self.http.requests}", + ] + if self.stamped: + summary += ["", "### Stamped", ""] + [f"- `{name}`" for name in self.stamped] + if self.mirrored: + summary += ["", "### Mirrors", ""] + [f"- `{name}`" for name in self.mirrored] + if self.failed: + summary += ["", "### Reported", ""] + [ + f"- `{name}`" for name in sorted(set(self.failed)) + ] + + print("\n".join(summary[2:6])) + path = os.environ.get("GITHUB_STEP_SUMMARY") + if path: + with open(path, "a", encoding="utf-8") as handle: + handle.write("\n".join(summary) + "\n") + + +def parse_arguments(argv): + parser = argparse.ArgumentParser(description="One tick of the watcher.") + parser.add_argument( + "--authored", default=".authored", type=Path, + help="a checkout of the authored repository, which holds listings/", + ) + parser.add_argument( + "--authored-repo", default="KSAModding/content-index", + help="the authored repository, for issues and the sweep", + ) + parser.add_argument("--releases", default="releases", type=Path) + parser.add_argument("--game-versions", default="game-versions.json", type=Path) + parser.add_argument( + "--cache", default=".watcher/cache.json", type=Path, + help="the derived cache: ETags, failure counts, issue numbers. Never state", + ) + parser.add_argument( + "--listing", action="append", + help="only this listing, repeatable. For a manual dispatch", + ) + parser.add_argument( + "--lookback-days", type=int, default=0, + help="ignore releases older than this. 0 scans the host's whole list", + ) + parser.add_argument( + "--stamp-budget", type=int, default=20, + help="how many releases one tick stamps at most; the next tick carries on", + ) + parser.add_argument( + "--mirror-budget", type=int, default=4, + help="how many mirror candidates one tick verifies at most", + ) + parser.add_argument( + "--unreachable-ticks", type=int, default=6, + help="consecutive failed ticks before a host being down reaches the author", + ) + parser.add_argument("--issue-label", default="watcher") + parser.add_argument("--steward-team", default="@KSAModding/content-manager-stewards") + parser.add_argument("--sweep-workflow", default="checks.yml") + parser.add_argument("--sweep-ref", default="main") + parser.add_argument("--verdict-check", default="validate") + parser.add_argument("--sweep-limit", type=int, default=30) + parser.add_argument("--sweep-attempts", type=int, default=3) + parser.add_argument("--sweep-cooldown", type=int, default=30, help="minutes") + parser.add_argument( + "--sweep-refusal-hours", type=int, default=24, + help="hours before a dispatch the workflow refused is tried again", + ) + parser.add_argument("--no-sweep", action="store_true") + parser.add_argument("--no-commit", action="store_true") + parser.add_argument( + "--dry-run", action="store_true", + help="write nothing, commit nothing, and open nothing. Downloads still happen", + ) + parser.add_argument( + "--fail-on-error", action="store_true", + help="exit non-zero when a listing was reported. Off by default, because a " + "broken listing is an issue on the authored repository and not a red tick", + ) + options = parser.parse_args(argv) + options.token = os.environ.get("GITHUB_TOKEN") or os.environ.get("INDEX_TOKEN") + return options + + +def main(argv=None): + options = parse_arguments(argv) + if not options.token: + print( + "no token in GITHUB_TOKEN: the tick can read public hosts but cannot keep " + "issues current or sweep", + file=sys.stderr, + ) + watcher = Watcher(options) + watcher.tick() + return 1 if options.fail_on_error and watcher.failed else 0 + + +if __name__ == "__main__": + sys.exit(main())