ingest.yml

name: Ingest and publish

# Dispatch runs are named after the package that cued them; every other
# event keeps GitHub's default title (the empty fallback string does that).
run-name: >-
  ${{ github.event_name == 'repository_dispatch'
      && format('Ingest {0} {1}', github.event.client_payload.repository, github.event.client_payload.tag)
      || '' }}

# Builds whatever the fleet's newest tags provide that the archive does not yet
# carry, includes it into the aptly pool, and publishes dists/ and pool/ to the
# R2 bucket behind https://apt.pkg.haus.
#
# Published pool files are immutable: the plan only ever adds missing versions.
# A run with nothing missing publishes nothing and touches nothing.
#
# State model, two branches, neither served:
#   `archive` -- the human-facing tree (listings, news, keyring) that Pages
#                serves; cloned into public/, committed back, deployed.
#   `aptly`   -- aptly's leveldb and its local pool; cloned into aptly-state/,
#                committed back. Off the archive branch because Pages serves
#                that branch whole and would double the site.
#
# Runs when a fleet repository reports a validated tag (repository_dispatch
# from action-debian-build's notify job), on manual dispatch, and on pushes
# that change the archive's own configuration.

on:
  repository_dispatch:
    types: [package-tagged]
  workflow_dispatch:
  push:
    branches: [master, main]
    paths:
      - repos.txt
      - scripts/**
      - .github/actions/install-aptly/**

permissions:
  contents: read

# Two ingests interleaving would race on the archive branch; one at a time,
# never cancelled mid-publish.
concurrency:
  group: ingest
  cancel-in-progress: false

jobs:
  plan:
    runs-on: ubuntu-24.04
    timeout-minutes: 15
    outputs:
      has_work: ${{ steps.plan.outputs.has_work }}
      matrix: ${{ steps.plan.outputs.matrix }}
    steps:
      - name: Checkout
        uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Check signing-key expiry
        env:
          ARCHIVE_SIGNING_KEY: ${{ secrets.ARCHIVE_SIGNING_KEY }}
        run: scripts/check-key-expiry.sh

      - name: Fetch aptly state
        env:
          REPO_URL: ${{ github.server_url }}/${{ github.repository }}
        # What the archive already carries is a database question, so the plan
        # job needs the database and nothing else. Distinguishes "the branch
        # does not exist" (first run: everything is missing) from "the clone
        # failed" (fail the run), which would otherwise plan a full rebuild.
        run: |
          if git ls-remote --exit-code "$REPO_URL" refs/heads/aptly >/dev/null; then
              git clone --branch aptly --single-branch --depth 1 "$REPO_URL" aptly-state
          else
              echo "FATAL: no aptly branch. It has to exist before the run can"
              echo "       commit to it -- commits are made through the API now,"
              echo "       which cannot create a branch."
              exit 1
          fi

      - name: Install aptly
        uses: ./.github/actions/install-aptly

      - name: Plan
        id: plan
        # The matrix fans out one leg per plan row (package x suite x arch),
        # so each leg's artifact can carry the real package filename.
        run: |
          scripts/ingest.sh plan > plan.tsv
          echo "--- plan ---"
          column -t -s "$(printf '\t')" plan.tsv || true

          if [ -s plan.tsv ]; then
              echo "has_work=true" >> "$GITHUB_OUTPUT"
          else
              echo "has_work=false" >> "$GITHUB_OUTPUT"
          fi

          jq -Rnc '[inputs | split("\t")
                    | {repo: .[0], pkg: .[2], suite: .[3], arch: .[4], version: .[5],
                       runner: (if .[4] == "arm64" then "ubuntu-24.04-arm" else "ubuntu-24.04" end)}]' \
              < plan.tsv > matrix.json
          echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT"

      - name: Upload plan
        uses: actions/upload-artifact@v7
        with:
          name: plan
          path: plan.tsv
          retention-days: 7

  build:
    name: build / ${{ matrix.pkg }} / ${{ matrix.suite }} / ${{ matrix.arch }}
    needs: plan
    if: needs.plan.outputs.has_work == 'true'
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.plan.outputs.matrix) }}
    steps:
      - name: Checkout
        uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Download plan
        uses: actions/download-artifact@v8
        with:
          name: plan

      - name: Build this leg's package
        env:
          SUITE: ${{ matrix.suite }}
          REPO: ${{ matrix.repo }}
        run: scripts/ingest.sh build plan.tsv "$SUITE" "$REPO"

      - name: Upload packages
        uses: actions/upload-artifact@v7
        with:
          # The real Debian filename: one leg builds exactly one plan row.
          name: ${{ matrix.pkg }}_${{ matrix.version }}_${{ matrix.arch }}.deb
          path: build/*
          if-no-files-found: error
          retention-days: 7

  publish:
    # Runs even when build was skipped: the pool index and keyring are
    # re-rendered idempotently, and the commit step no-ops when nothing
    # changed. Only a failed or cancelled upstream job stops a publish.
    needs: [plan, build]
    if: ${{ !cancelled() && needs.plan.result == 'success' && contains(fromJSON('["success", "skipped"]'), needs.build.result) }}
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    permissions:
      contents: write
      pages: write
      id-token: write
    environment:
      name: github-pages
      url: ${{ steps.deploy.outputs.page_url }}
    steps:
      - name: Checkout
        uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Fetch archive state
        env:
          REPO_URL: ${{ github.server_url }}/${{ github.repository }}
        # Distinguishes "the branch does not exist" (first run: start empty)
        # from "the clone failed" (fail the run): falling back to an empty tree
        # on a transient failure would publish an archive missing its pool.
        run: |
          if git ls-remote --exit-code "$REPO_URL" refs/heads/archive >/dev/null; then
              git clone --branch archive --single-branch --depth 1 "$REPO_URL" public
          else
              echo "FATAL: no archive branch. It has to exist before the run can"
              echo "       commit to it -- commits are made through the API now,"
              echo "       which cannot create a branch."
              exit 1
          fi

      - name: Fetch aptly state
        env:
          REPO_URL: ${{ github.server_url }}/${{ github.repository }}
        # aptly's leveldb, and only that. Its package pool is a second copy of
        # every published .deb and is deliberately thrown away each run: at the
        # root publish prefix aptly reads the pool only for packages whose
        # objects are missing from the bucket, which is none of them except the
        # ones this run just built.
        #
        # Kept off the archive branch because Pages serves that branch whole.
        run: |
          if git ls-remote --exit-code "$REPO_URL" refs/heads/aptly >/dev/null; then
              git clone --branch aptly --single-branch --depth 1 "$REPO_URL" aptly-state
          else
              echo "no aptly branch yet -- starting empty"
              mkdir -p aptly-state
          fi

      - name: Download built packages
        if: needs.build.result == 'success'
        uses: actions/download-artifact@v8
        with:
          pattern: '*.deb'
          path: build
          merge-multiple: true

      - name: Install aptly
        uses: ./.github/actions/install-aptly

      - name: Import the archive signing key
        env:
          ARCHIVE_SIGNING_KEY: ${{ secrets.ARCHIVE_SIGNING_KEY }}
        run: |
          printf '%s' "$ARCHIVE_SIGNING_KEY" | gpg --batch --import
          gpg --list-secret-keys

      - name: Include and publish to R2
        if: needs.build.result == 'success'
        env:
          R2_ACCESS_KEY_ID: ${{ secrets.PKGHAUS_APT_R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ secrets.PKGHAUS_APT_R2_SECRET_ACCESS_KEY }}
          R2_BUCKET: ${{ vars.R2_BUCKET }}
          R2_ENDPOINT: ${{ vars.R2_ENDPOINT }}
        run: scripts/ingest.sh include build

      # After the publish, which is the only moment new pool objects exist.
      # Skipped when the plan was empty: nothing was published, so nothing is
      # missing from the mirror.
      - name: Mirror the pool to the backup bucket
        if: needs.build.result == 'success'
        env:
          R2_ACCESS_KEY_ID: ${{ secrets.PKGHAUS_APT_R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ secrets.PKGHAUS_APT_R2_SECRET_ACCESS_KEY }}
          R2_BUCKET: ${{ vars.R2_BUCKET }}
          R2_ENDPOINT: ${{ vars.R2_ENDPOINT }}
          R2_BACKUP_BUCKET: ${{ vars.R2_BACKUP_BUCKET }}
        run: scripts/backup-pool.sh

      - name: Export the public keyring alongside the archive
        run: gpg --export > public/pkghaus-archive-keyring.gpg

      # After include (so the diff sees the new state), before render
      # (so the news page reflects this publish). Runs on empty plans
      # too: notices still merge, and an unchanged set emits nothing.
      - name: Update the news log
        run: scripts/news.sh

      - name: Render the pool index
        env:
          R2_ACCESS_KEY_ID: ${{ secrets.PKGHAUS_APT_R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ secrets.PKGHAUS_APT_R2_SECRET_ACCESS_KEY }}
          R2_BUCKET: ${{ vars.R2_BUCKET }}
          R2_ENDPOINT: ${{ vars.R2_ENDPOINT }}
        run: scripts/render-index.sh

      # Committed through the API rather than with git push, so GitHub signs
      # it. The author becomes github-actions[bot]; nothing here invents an
      # identity, and no signing key has to reach the runner.
      - name: Commit archive state
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: scripts/commit-branch.sh public archive "Ingest $(date -u '+%Y-%m-%d %H:%M:%S UTC')"

      - name: Commit aptly state
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          rm -f aptly-state/aptly.conf   # written per run, carries the R2 credentials
          rm -rf aptly-state/pool        # this run's .debs; the bucket has them now
          scripts/commit-branch.sh aptly-state aptly "Ingest $(date -u '+%Y-%m-%d %H:%M:%S UTC')"

      - name: Upload Pages artifact
        uses: actions/upload-pages-artifact@v5
        with:
          path: public
          # The action's tar excludes .[^/]* by default, which dropped
          # .well-known/security.txt from every deployment ever made: the
          # renderer wrote it, the branch carried it, and the published site
          # 404'd it.
          include-hidden-files: true

      - name: Deploy to Pages
        id: deploy
        uses: actions/deploy-pages@v5

      # Ordered after the deployment on purpose: purging first would let the
      # edge refill with the previous render for a full day.
      - name: Purge listing pages from the edge cache
        env:
          CLOUDFLARE_PURGE_TOKEN: ${{ secrets.PKGHAUS_APT_CLOUDFLARE_API_TOKEN }}
          CLOUDFLARE_ZONE_ID: ${{ vars.CLOUDFLARE_ZONE_ID }}
          R2_ACCESS_KEY_ID: ${{ secrets.PKGHAUS_APT_R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ secrets.PKGHAUS_APT_R2_SECRET_ACCESS_KEY }}
          R2_BUCKET: ${{ vars.R2_BUCKET }}
          R2_ENDPOINT: ${{ vars.R2_ENDPOINT }}
        run: scripts/purge-cache.sh