diff --git a/.github/workflows/runner-bump.yml b/.github/workflows/runner-bump.yml new file mode 100644 index 0000000..b2d6da3 --- /dev/null +++ b/.github/workflows/runner-bump.yml @@ -0,0 +1,176 @@ +name: Runner Bump + +# Keeps ephemerd's embedded GitHub Actions runner current. GitHub deprecates +# old runner versions and then refuses their connections, which silently stalls +# every job on the affected pool (the macOS pool once ran ZERO jobs for a week on +# a stale runner), so this must not depend on someone remembering to bump it. +# +# Daily, it asks the GitHub release API for the newest non-draft, +# non-prerelease actions/runner release. If that is newer than RunnerVersion in +# mage/download/download.go, `go run ./mage/runnerbump` rewrites every pin site +# (RunnerVersion, the five runner entries of pinnedSHA256, and ARG +# RUNNER_VERSION in images/runner-ci-*/Dockerfile) with checksums taken from +# the release API's per-asset `digest` field, and this job opens a PR. It never +# merges: review and merge stay manual. +# +# Idempotency: one fixed branch, bot/actions-runner-bump, regenerated from main +# on every run. No new release -> nothing happens. Same release as the open PR +# -> nothing is pushed (so CI is not retriggered on the self-hosted fleet). A +# newer release -> the branch is force-pushed and the existing PR is retitled +# and re-described rather than a second one opened. If main reaches the latest +# release some other way (a manual bump), the stale bot PR is closed. The +# branch is machine-owned: do not push commits to it, they will be overwritten. +# +# Token: PRs opened with the default GITHUB_TOKEN do not trigger CI (GitHub's +# recursion guard), and the ephpm org also has "Allow GitHub Actions to create +# and approve pull requests" switched off, so the default token cannot open the +# PR at all. Configure a GitHub App and this job uses its installation token +# instead, which both opens the PR and lets ci.yml run on it: +# - repo or org variable BOT_APP_CLIENT_ID the App's Client ID +# - repo or org secret BOT_APP_PRIVATE_KEY a private key (.pem) of the App +# The App needs Contents: read & write and Pull requests: read & write, and must +# be installed on ephpm/ephemerd. Without those, the job falls back to +# GITHUB_TOKEN: it still pushes the branch, and fails with instructions if the +# PR cannot be opened. +# +# Runs on GitHub-hosted ubuntu-latest so it never takes a slot on the +# self-hosted fleet. + +on: + schedule: + # Daily, off the top of the hour. + - cron: "23 7 * * *" + workflow_dispatch: + inputs: + version: + description: "Exact actions/runner version to bump to (e.g. 2.338.0). Empty = latest stable release." + required: false + type: string + +permissions: + contents: write + pull-requests: write + +concurrency: + group: runner-bump + cancel-in-progress: false + +env: + BRANCH: bot/actions-runner-bump + +jobs: + bump: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Mint GitHub App token + id: app + if: vars.BOT_APP_CLIENT_ID != '' + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ vars.BOT_APP_CLIENT_ID }} + private-key: ${{ secrets.BOT_APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + token: ${{ steps.app.outputs.token || github.token }} + + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + # runnerbump is stdlib-only; restoring the repo's module cache would + # cost more than it saves. + cache: false + + - name: Bump the runner pins if a newer release exists + id: bump + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ inputs.version || '' }} + run: | + set -euo pipefail + go run ./mage/runnerbump -version "$VERSION" -body-out "$RUNNER_TEMP/pr-body.md" + + - name: Validate the edit + if: steps.bump.outputs.bump == 'true' + run: | + set -euo pipefail + unformatted=$(gofmt -l mage/) + if [ -n "$unformatted" ]; then + echo "::error::gofmt reports unformatted files after the bump: $unformatted" + exit 1 + fi + go build ./mage/... + git --no-pager diff --stat + + - name: Open or update the bump PR + if: steps.bump.outputs.bump == 'true' + env: + GH_TOKEN: ${{ steps.app.outputs.token || github.token }} + USING_APP: ${{ steps.app.outputs.token && 'true' || 'false' }} + CURRENT: ${{ steps.bump.outputs.current }} + LATEST: ${{ steps.bump.outputs.latest }} + run: | + set -euo pipefail + fail() { echo "::error::$*"; exit 1; } + + files=(mage/download/download.go images/runner-ci-linux/Dockerfile images/runner-ci-macos/Dockerfile images/runner-ci-windows/Dockerfile) + title="fix(runner): bump embedded actions runner to ${LATEST}" + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$BRANCH" + git add "${files[@]}" + git commit -q -m "$title" \ + -m "Bump the embedded actions/runner from ${CURRENT} to ${LATEST}: RunnerVersion and the five runner pinnedSHA256 entries in mage/download/download.go, and ARG RUNNER_VERSION in the CI runner images. Checksums are the release API's asset digests." \ + -m "Automated by .github/workflows/runner-bump.yml." \ + || fail "nothing to commit after the bump; runnerbump and this list of files are out of step" + + # Push only when the bumped files differ from what the bot branch + # already carries, so a rerun for the same release is a no-op and + # does not retrigger CI. + if git fetch -q --depth=1 origin "refs/heads/${BRANCH}" 2>/dev/null \ + && git diff --quiet FETCH_HEAD HEAD -- "${files[@]}"; then + echo "${BRANCH} already carries runner ${LATEST}; not pushing" + else + git push --force origin "HEAD:refs/heads/${BRANCH}" || fail "pushing ${BRANCH} failed" + echo "pushed ${BRANCH} (runner ${LATEST})" + fi + + pr=$(gh pr list --head "$BRANCH" --base main --state open --json number --jq '.[0].number // empty') + if [ -n "$pr" ]; then + gh pr edit "$pr" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md" \ + || fail "updating PR #${pr} failed" + echo "::notice::updated PR #${pr} for runner ${LATEST}" + exit 0 + fi + + if ! gh pr create --base main --head "$BRANCH" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md"; then + compare="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/compare/main...${BRANCH}?expand=1" + if [ "$USING_APP" != "true" ]; then + fail "could not open the PR with GITHUB_TOKEN. Set the BOT_APP_CLIENT_ID variable and BOT_APP_PRIVATE_KEY secret (see the header of runner-bump.yml), or enable 'Allow GitHub Actions to create and approve pull requests' in the repo's Actions settings. The branch is pushed; open it by hand: ${compare}" + fi + fail "opening the PR failed; the branch is pushed: ${compare}" + fi + echo "::notice::opened a PR for runner ${LATEST}" + + - name: Close a bot PR that main has caught up with + # Only on unpinned runs: a dispatch for an older explicit version must + # not close a PR for a newer one. + if: steps.bump.outputs.bump == 'false' && !inputs.version + env: + GH_TOKEN: ${{ steps.app.outputs.token || github.token }} + CURRENT: ${{ steps.bump.outputs.current }} + run: | + set -euo pipefail + pr=$(gh pr list --head "$BRANCH" --base main --state open --json number --jq '.[0].number // empty') + if [ -z "$pr" ]; then + echo "no open bot PR; nothing to do" + exit 0 + fi + gh pr close "$pr" --delete-branch \ + --comment "main is already on actions/runner ${CURRENT}, the latest stable release, so this bump is no longer needed." + echo "::notice::closed stale PR #${pr}" diff --git a/mage/download/download.go b/mage/download/download.go index c08f216..8234e3d 100644 --- a/mage/download/download.go +++ b/mage/download/download.go @@ -57,7 +57,10 @@ const ( // // Hashes are the upstream-published checksums for the versions pinned in the // const block above: -// - runner: release body "SHA" markers (github.com/actions/runner) +// - runner: the release API's per-asset "digest" (github.com/actions/runner), +// cross-checked against the release body "SHA" markers. Bumped +// automatically by .github/workflows/runner-bump.yml via mage/runnerbump; +// keep the runner pin sites in the shape runnerbump expects. // - cni: .tgz.sha256 // - containerd: .tar.gz.sha256sum // - runc: runc.sha256sum (per-arch runc. lines) diff --git a/mage/runnerbump/main.go b/mage/runnerbump/main.go new file mode 100644 index 0000000..21150f3 --- /dev/null +++ b/mage/runnerbump/main.go @@ -0,0 +1,475 @@ +// Command runnerbump bumps ephemerd's embedded GitHub Actions runner to the +// newest actions/runner release. +// +// It is the engine behind .github/workflows/runner-bump.yml, and is equally +// usable by hand: +// +// go run ./mage/runnerbump # bump to the latest stable release +// go run ./mage/runnerbump -version 2.338.0 # bump to a specific release +// go run ./mage/runnerbump -check # report only, edit nothing +// +// What a bump touches (the shape of ephpm/ephemerd#183): +// +// - mage/download/download.go: the RunnerVersion const, the +// "// GitHub Actions runner (actions/runner vX)" comment, and the five +// actions-runner-* archive entries of pinnedSHA256. +// - images/runner-ci-{linux,macos,windows}/Dockerfile: ARG RUNNER_VERSION. +// +// Checksums come from the release API's per-asset "digest" field (computed by +// GitHub over the uploaded bytes). An asset without a digest is downloaded and +// hashed instead. The release notes' "BEGIN SHA" markers are only ever used as +// a cross-check: a disagreement fails the bump, but the notes are never the +// sole source of a pin. Any of the five archives missing from the release +// fails the bump. +// +// A target that is not newer than the current RunnerVersion is a no-op, so the +// command is safe to run on a schedule. +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "go/format" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" +) + +// Paths, relative to -root, that a bump edits. +const downloadGo = "mage/download/download.go" + +var dockerfiles = []string{ + "images/runner-ci-linux/Dockerfile", + "images/runner-ci-macos/Dockerfile", + "images/runner-ci-windows/Dockerfile", +} + +// runnerPlatforms are the archives ephemerd pins, in pinnedSHA256 order. The +// name stem doubles as the release notes' "BEGIN SHA " marker. +var runnerPlatforms = []struct{ stem, ext string }{ + {"linux-x64", "tar.gz"}, + {"linux-arm64", "tar.gz"}, + {"win-x64", "zip"}, + {"osx-x64", "tar.gz"}, + {"osx-arm64", "tar.gz"}, +} + +func assetName(stem, ext, version string) string { + return fmt.Sprintf("actions-runner-%s-%s.%s", stem, version, ext) +} + +const repo = "actions/runner" + +// release is the subset of the GitHub release object runnerbump reads. +type release struct { + TagName string `json:"tag_name"` + HTMLURL string `json:"html_url"` + Body string `json:"body"` + Draft bool `json:"draft"` + Prerelease bool `json:"prerelease"` + Assets []asset `json:"assets"` +} + +type asset struct { + Name string `json:"name"` + Digest string `json:"digest"` + BrowserDownloadURL string `json:"browser_download_url"` +} + +// pin is one resolved archive checksum. +type pin struct { + Name string + SHA256 string + Source string // "release API digest" or "downloaded and hashed" +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "runnerbump:", err) + os.Exit(1) + } +} + +func run() error { + root := flag.String("root", ".", "repository root to edit") + target := flag.String("version", "", "runner version to bump to (default: latest stable actions/runner release)") + check := flag.Bool("check", false, "report whether a bump is available without editing anything") + releaseJSON := flag.String("release-json", "", "read the release object from this file instead of the GitHub API (testing)") + bodyOut := flag.String("body-out", "", "write a markdown PR body to this file when a bump happens") + flag.Parse() + + dlPath := filepath.Join(*root, downloadGo) + src, err := os.ReadFile(dlPath) + if err != nil { + return err + } + current, err := currentVersion(src) + if err != nil { + return err + } + + rel, err := loadRelease(*releaseJSON, *target) + if err != nil { + return err + } + if rel.Draft || rel.Prerelease { + return fmt.Errorf("release %s is a draft or prerelease; refusing to pin it", rel.TagName) + } + latest := strings.TrimPrefix(rel.TagName, "v") + if _, err := parseSemver(latest); err != nil { + return fmt.Errorf("release tag %q: %w", rel.TagName, err) + } + newer, err := semverLess(current, latest) + if err != nil { + return err + } + + fmt.Printf("current runner: %s\nrelease: %s (%s)\n", current, latest, rel.HTMLURL) + if !newer { + fmt.Println("up to date; nothing to do") + return writeOutputs(map[string]string{"bump": "false", "current": current, "latest": latest}) + } + if *check { + fmt.Println("bump available (-check: not editing)") + return writeOutputs(map[string]string{"bump": "false", "available": "true", "current": current, "latest": latest}) + } + + pins, err := resolvePins(rel, latest) + if err != nil { + return err + } + + out, err := rewriteDownloadGo(src, current, latest, pins) + if err != nil { + return fmt.Errorf("%s: %w", downloadGo, err) + } + if err := writeFileKeepMode(dlPath, out); err != nil { + return err + } + for _, df := range dockerfiles { + p := filepath.Join(*root, df) + b, err := os.ReadFile(p) + if err != nil { + return err + } + nb, err := rewriteDockerfile(b, latest) + if err != nil { + return fmt.Errorf("%s: %w", df, err) + } + if err := writeFileKeepMode(p, nb); err != nil { + return err + } + } + + fmt.Printf("bumped %s -> %s\n", current, latest) + for _, p := range pins { + fmt.Printf(" %-45s %s (%s)\n", p.Name, p.SHA256, p.Source) + } + if *bodyOut != "" { + if err := os.WriteFile(*bodyOut, []byte(prBody(current, latest, rel.HTMLURL, pins)), 0o644); err != nil { + return err + } + } + return writeOutputs(map[string]string{"bump": "true", "current": current, "latest": latest}) +} + +// --------------------------------------------------------------------------- +// Versions + +var runnerVersionRe = regexp.MustCompile(`(?m)^\s*RunnerVersion\s*=\s*"([^"]+)"`) + +func currentVersion(src []byte) (string, error) { + m := runnerVersionRe.FindAllSubmatch(src, -1) + if len(m) != 1 { + return "", fmt.Errorf("expected exactly one RunnerVersion const in %s, found %d", downloadGo, len(m)) + } + v := string(m[0][1]) + if _, err := parseSemver(v); err != nil { + return "", fmt.Errorf("RunnerVersion %q: %w", v, err) + } + return v, nil +} + +func parseSemver(v string) ([3]int, error) { + var out [3]int + parts := strings.Split(v, ".") + if len(parts) != 3 { + return out, fmt.Errorf("not a MAJOR.MINOR.PATCH version") + } + for i, p := range parts { + n, err := strconv.Atoi(p) + if err != nil || n < 0 || p == "" || (len(p) > 1 && p[0] == '0') { + return out, fmt.Errorf("not a MAJOR.MINOR.PATCH version") + } + out[i] = n + } + return out, nil +} + +// semverLess reports whether a < b. +func semverLess(a, b string) (bool, error) { + va, err := parseSemver(a) + if err != nil { + return false, fmt.Errorf("%q: %w", a, err) + } + vb, err := parseSemver(b) + if err != nil { + return false, fmt.Errorf("%q: %w", b, err) + } + for i := range va { + if va[i] != vb[i] { + return va[i] < vb[i], nil + } + } + return false, nil +} + +// --------------------------------------------------------------------------- +// Release lookup and checksums + +var httpClient = &http.Client{Timeout: 10 * time.Minute} + +func loadRelease(file, version string) (*release, error) { + var data []byte + var err error + if file != "" { + data, err = os.ReadFile(file) + } else { + // /releases/latest only ever returns a published, non-prerelease, + // non-draft release; run() re-checks both flags regardless. + path := "releases/latest" + if version != "" { + path = "releases/tags/v" + strings.TrimPrefix(version, "v") + } + data, err = apiGet("https://api.github.com/repos/" + repo + "/" + path) + } + if err != nil { + return nil, err + } + var rel release + if err := json.Unmarshal(data, &rel); err != nil { + return nil, fmt.Errorf("decoding release: %w", err) + } + if rel.TagName == "" { + return nil, errors.New("release has no tag_name") + } + return &rel, nil +} + +func apiGet(url string) ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + for _, k := range []string{"GH_TOKEN", "GITHUB_TOKEN"} { + if t := os.Getenv(k); t != "" { + req.Header.Set("Authorization", "Bearer "+t) + break + } + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GET %s: %s: %s", url, resp.Status, strings.TrimSpace(string(body))) + } + return body, nil +} + +var hex64 = regexp.MustCompile(`^[0-9a-f]{64}$`) + +func resolvePins(rel *release, version string) ([]pin, error) { + byName := make(map[string]asset, len(rel.Assets)) + for _, a := range rel.Assets { + byName[a.Name] = a + } + var pins []pin + var missing []string + for _, p := range runnerPlatforms { + name := assetName(p.stem, p.ext, version) + a, ok := byName[name] + if !ok { + missing = append(missing, name) + continue + } + sum, source, err := assetSHA256(a) + if err != nil { + return nil, fmt.Errorf("%s: %w", name, err) + } + if noted, ok := notesSHA256(rel.Body, p.stem); ok && noted != sum { + return nil, fmt.Errorf("%s: release-notes SHA %s disagrees with %s %s; refusing to pin", name, noted, source, sum) + } + pins = append(pins, pin{Name: name, SHA256: sum, Source: source}) + } + if len(missing) > 0 { + return nil, fmt.Errorf("release %s is missing required assets: %s", rel.TagName, strings.Join(missing, ", ")) + } + return pins, nil +} + +func assetSHA256(a asset) (sum, source string, err error) { + if d, ok := strings.CutPrefix(a.Digest, "sha256:"); ok { + d = strings.ToLower(d) + if !hex64.MatchString(d) { + return "", "", fmt.Errorf("malformed digest %q", a.Digest) + } + return d, "release API digest", nil + } + if a.Digest != "" { + return "", "", fmt.Errorf("unsupported digest algorithm %q", a.Digest) + } + if a.BrowserDownloadURL == "" { + return "", "", errors.New("no digest and no download URL") + } + resp, err := httpClient.Get(a.BrowserDownloadURL) + if err != nil { + return "", "", err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return "", "", fmt.Errorf("GET %s: %s", a.BrowserDownloadURL, resp.Status) + } + h := sha256.New() + if _, err := io.Copy(h, resp.Body); err != nil { + return "", "", fmt.Errorf("downloading %s: %w", a.BrowserDownloadURL, err) + } + return hex.EncodeToString(h.Sum(nil)), "downloaded and hashed", nil +} + +// notesSHA256 extracts the checksum actions/runner embeds in its release notes +// as HASH. +func notesSHA256(body, stem string) (string, bool) { + re := regexp.MustCompile(`\s*([0-9a-fA-F]{64})\s*" + strings.Repeat("0", 64) + "" + if _, err := resolvePins(rel, "2.338.0"); err == nil { + t.Fatal("want an error when the release notes disagree with the API digest") + } + rel.Body = "" + strings.Repeat("A", 64) + "" + if _, err := resolvePins(rel, "2.338.0"); err != nil { + t.Fatalf("matching (case-insensitive) notes SHA should pass: %v", err) + } +} + +func TestRewriteDockerfile(t *testing.T) { + src := "FROM x\n\nARG RUNNER_VERSION=2.333.1\nARG CNI_VERSION=1.6.2\n" + out, err := rewriteDockerfile([]byte(src), "2.338.0") + if err != nil { + t.Fatal(err) + } + if want := "FROM x\n\nARG RUNNER_VERSION=2.338.0\nARG CNI_VERSION=1.6.2\n"; string(out) != want { + t.Fatalf("got %q, want %q", out, want) + } + if _, err := rewriteDockerfile([]byte("FROM x\n"), "2.338.0"); err == nil { + t.Fatal("expected an error when the ARG is absent") + } +} + +func TestSemverLess(t *testing.T) { + for _, c := range []struct { + a, b string + want bool + }{ + {"2.337.0", "2.338.0", true}, + {"2.337.0", "2.337.0", false}, + {"2.338.0", "2.337.0", false}, + {"2.99.0", "2.100.0", true}, // numeric, not lexical + {"2.337.9", "3.0.0", true}, + } { + got, err := semverLess(c.a, c.b) + if err != nil || got != c.want { + t.Errorf("semverLess(%s, %s) = %v, %v; want %v", c.a, c.b, got, err, c.want) + } + } + if _, err := semverLess("2.337", "2.338.0"); err == nil { + t.Error("expected an error for a two-part version") + } +}