generate.yml

name: "Step 2, Generate the project from BaseVM"
on:
  schedule:
    - cron: '20 22 * * *'
  workflow_dispatch:
  push:
    branches:
      - 'main'
    paths:
      - 'conf/*'
      - 'hooks/*'
      - '.github/data/*'
      - '.github/workflows/generate.yml'

    
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: false

jobs:
  generate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
        with:
          token: ${{ secrets.VM_TOKEN }}
          fetch-depth: '2'
          # checkout fetches NO tags at any fetch-depth unless asked. Without
          # this, `git tag` below returns nothing, LATEST_TAG is empty and
          # ${LATEST_TAG:1} stamps package.json's version as "" -- which is
          # what every sibling shipped until 2026-08-19.
          fetch-tags: true

      - name: Preparing env variables
        run: |
          git switch main || (git fetch --all && git checkout -b main origin/main)
          . conf/default.release.conf
          echo "DEFAULT_RELEASE=$DEFAULT_RELEASE" >> $GITHUB_ENV
          echo "ALL_RELEASES=\"$(ls conf/ | grep -v default | sed 's/.conf//g' | tr '\n' ',' | sed "s/,\$//" | sed 's/,/", "/g')\"" >> $GITHUB_ENV
          LATEST_TAG="$(git tag --sort=-v:refname | head -n 1)"
          echo "LATEST_TAG=${LATEST_TAG}" >> $GITHUB_ENV
          echo "LATEST_VERSION_NUMBER=${LATEST_TAG:1}" >> $GITHUB_ENV

          oldHash="$(cat .github/workflows/test.yml | grep -i "uses: ${{ github.repository }}" | head -1 | cut -d @ -f 2)"
          echo "OLD_HASH=${oldHash}" >> $GITHUB_ENV

          git clone https://github.com/vmactions/base-vm.git

          # Collect base-vm commit messages since last generate
          BASEVM_HASH_FILE=".github/data/base-vm-hash.txt"
          BASEVM_NEW_HASH="$(git -C base-vm rev-parse HEAD)"
          if [ -f "$BASEVM_HASH_FILE" ]; then
            BASEVM_OLD_HASH="$(cat "$BASEVM_HASH_FILE")"
            BASEVM_LOG="$(git -C base-vm log --format="https://github.com/vmactions/base-vm/commit/%H %s" "${BASEVM_OLD_HASH}..${BASEVM_NEW_HASH}" 2>/dev/null || git -C base-vm log --format="https://github.com/vmactions/base-vm/commit/%H %s" -20)"
          else
            BASEVM_LOG="$(git -C base-vm log --format="https://github.com/vmactions/base-vm/commit/%H %s" -1)"
          fi
          echo "$BASEVM_NEW_HASH" > "$BASEVM_HASH_FILE"

          # Save log for later commit message
          echo "$BASEVM_LOG" > /tmp/basevm-log.txt

          mkdir -p .github/tpl
          cat base-vm/.github/tpl/README.tpl.md >.github/tpl/README.tpl.md

      - name: Bake sync methods from the builder release index
        run: |
          # Each conf/<release>[-<arch>].conf pins a BUILDER_VERSION. That
          # builder release publishes releases.json (gendata's index) as a
          # release asset, carrying per-release "sync" (VM_SYNC_METHODS,
          # comma separated, first = default) and "shutdown"
          # (VM_SHUTDOWN_CMD, the in-guest poweroff command used by
          # cache-after-prepare). Copy those into the local conf so the
          # action and test workflow read them with no run-time network call.
          #
          # The index is fetched as a RELEASE ASSET at the pinned version --
          # never a branch, never releases/latest -- so the values and the
          # image that ships in that release are a matched pair. One fetch
          # per distinct BUILDER_VERSION (cached below), not one per release.
          os="hurd"
          mkdir -p /tmp/relindex
          # jq -r hands back the UNESCAPED string, which then lands in a sed
          # replacement (where the delimiter, & and backslash are special) and
          # inside double quotes in the conf. Today every value is a plain
          # command, but escape/reject rather than trust that:
          #   sed_escape - backslash-escape the sed-special chars
          #   unsafe     - a value with a double quote or a control char would
          #                emit a conf that breaks when sourced; skip it loudly
          sed_escape() { printf '%s' "$1" | sed -e 's/[\\&|]/\\&/g'; }
          unsafe() { case "$1" in *'"'*) return 0 ;; esac; return 1; }
          for f in conf/*.conf; do
            base="$(basename "$f" .conf)"
            case "$base" in
              default.release) continue ;;
            esac
            bv="$( . "$f"; printf '%s' "$BUILDER_VERSION" )"
            if [ -z "$bv" ]; then
              echo "${base}: no BUILDER_VERSION, skip"
              continue
            fi
            url="https://github.com/anyvm-org/${os}-builder/releases/download/v${bv}/releases.json"
            idx="/tmp/relindex/${bv}.json"
            # Releases cut before the index asset existed (and old pins with
            # no tag at all, e.g. openbsd 7.2-7.6) 404 here. The step runs
            # under bash -e, so a bare command substitution would abort the
            # whole job: tolerate the failure and degrade to an empty index
            # -> both fields fall into their "leaving conf as-is" branches,
            # exactly the legacy behavior.
            if [ ! -f "$idx" ]; then
              curl -fsSL "$url" -o "$idx" 2>/dev/null || : > "$idx"
            fi
            # jq -e fails (non-zero) on null/absent, which `|| true` absorbs.
            methods="$(jq -re --arg t "$base" '.releases[] | select(.tag == $t) | .sync' "$idx" 2>/dev/null | tail -n 1 || true)"
            if [ -z "$methods" ]; then
              echo "${base}: no sync for tag ${base} in ${url}, leaving conf as-is"
            elif unsafe "$methods"; then
              echo "${base}: sync value contains a quote, refusing to write: ${methods}"
            else
              if grep -q '^VM_SYNC_METHODS=' "$f"; then
                sed -i "s|^VM_SYNC_METHODS=.*|VM_SYNC_METHODS=\"$(sed_escape "$methods")\"|" "$f"
              else
                printf '\nVM_SYNC_METHODS="%s"\n' "${methods}" >> "$f"
              fi
              echo "${base}: VM_SYNC_METHODS=\"${methods}\""
            fi
            shutcmd="$(jq -re --arg t "$base" '.releases[] | select(.tag == $t) | .shutdown' "$idx" 2>/dev/null | tail -n 1 || true)"
            if [ -z "$shutcmd" ]; then
              echo "${base}: no shutdown for tag ${base} in ${url}, leaving conf as-is"
            elif unsafe "$shutcmd"; then
              echo "${base}: shutdown value contains a quote, refusing to write: ${shutcmd}"
            else
              if grep -q '^VM_SHUTDOWN_CMD=' "$f"; then
                sed -i "s|^VM_SHUTDOWN_CMD=.*|VM_SHUTDOWN_CMD=\"$(sed_escape "$shutcmd")\"|" "$f"
              else
                printf '\nVM_SHUTDOWN_CMD="%s"\n' "${shutcmd}" >> "$f"
              fi
              echo "${base}: VM_SHUTDOWN_CMD=\"${shutcmd}\""
            fi
          done

      - name: Compute sync method map and missing-arch excludes
        run: |
          # Turn the per-conf VM_SYNC_METHODS into three kinds of value for the
          # renderer:
          #   VM_SYNC_METHODS_MAP    - {"<arch>":["method",...]} keyed by matrix.arch,
          #                            so test.yml gates each sync job with `if:`.
          #   VM_TEST_EXCLUDE_MISSING - matrix excludes for release/arch combos
          #                            that have no conf (so they never spawn).
          #   VM_TEST_EXCLUDE_<METHOD> - matrix excludes for combos whose arch
          #                            does not support that job's sync method
          #                            (so they never spawn either).
          . conf/default.release.conf
          def="$DEFAULT_RELEASE"

          # non-x86_64 arches the test matrix covers (keep in sync with test.tpl.yml)
          ARCHES="aarch64 riscv64 powerpc64 sparc64 ppc64le s390x i386 loongarch64 armv7"

          # methods (comma list) for an arch suffix, from the default release
          # conf; fall back to any release that has that arch.
          methods_for() {
            a="$1"
            if [ -n "$a" ]; then cf="conf/${def}-${a}.conf"; else cf="conf/${def}.conf"; fi
            if [ -n "$a" ] && [ ! -e "$cf" ]; then
              cf="$(ls conf/*-"$a".conf 2>/dev/null | head -n 1)"
            fi
            [ -n "$cf" ] && [ -e "$cf" ] && ( . "$cf"; printf '%s' "$VM_SYNC_METHODS" )
          }

          # methods for ONE release+arch, read from THAT release's own conf.
          # The per-method excludes below must use this rather than
          # methods_for(): sync capability is a property of the RELEASE, not
          # only of the arch. methods_for() always answers from the DEFAULT
          # release, which was fine while every release of an arch had the same
          # methods -- netbsd 11.0-microvm broke that assumption. It is x86_64
          # like 11.0 but drops nfs (its MICROVM kernel has no NFS client), so
          # the arch answer said "nfs supported", no exclude was emitted, and
          # the nfs leg ran and failed with "sync method 'nfs' is not supported
          # by netbsd 11.0-microvm" (netbsd-vm run 33459640344).
          # No conf for that release+arch -> empty -> unsupported, which keeps
          # the deliberate x86_64 behaviour below (a guest shipping no
          # conf/<release>.conf at all has its dead x86_64 leg dropped).
          methods_for_rel() {
            _r="$1"; _a="$2"
            if [ -n "$_a" ]; then _cf="conf/${_r}-${_a}.conf"; else _cf="conf/${_r}.conf"; fi
            [ -e "$_cf" ] && ( . "$_cf"; printf '%s' "$VM_SYNC_METHODS" )
          }

          # comma list -> JSON array: rsync,scp -> ["rsync","scp"] (empty -> [])
          to_json() {
            printf '['
            _first=1
            _old_ifs="$IFS"; IFS=','
            for _m in $1; do
              _m="$(printf '%s' "$_m" | tr -d ' ')"
              [ -z "$_m" ] && continue
              [ "$_first" = 1 ] || printf ','
              printf '"%s"' "$_m"
              _first=0
            done
            IFS="$_old_ifs"
            printf ']'
          }

          map="{\"x86_64\":$(to_json "$(methods_for '')")"
          for a in $ARCHES; do
            map="${map},\"${a}\":$(to_json "$(methods_for "$a")")"
          done
          printf '%s}' "$map" > .github/data/sync-map.json
          echo "sync-map: $(cat .github/data/sync-map.json)"

          : > .github/data/exclude-missing.txt
          for r in $(sed 's/[",]/ /g' conf/test.releases) ""; do
            rc="$r"; [ -z "$rc" ] && rc="$def"
            for a in $ARCHES; do
              if [ ! -e "conf/${rc}-${a}.conf" ]; then
                printf '          - release: "%s"\n            arch: %s\n' "$r" "$a" >> .github/data/exclude-missing.txt
              fi
            done
          done
          echo "exclude-missing:"; cat .github/data/exclude-missing.txt

          # Per-method excludes. A sync job's leg is pointless -- or worse,
          # RED -- when that release's VM_SYNC_METHODS does not list the job's
          # method: at best every step is gated off and the leg burns a runner
          # slot to produce a green job that did nothing, at worst the action
          # refuses the unsupported method and the leg fails.
          #
          # Keyed on RELEASE+arch via methods_for_rel(), while sync-map.json
          # above stays keyed on arch alone (it feeds ~20 step-level `if:`
          # expressions of the form MAP[arch]). The two therefore no longer
          # answer identically for a release that differs from its arch-mates.
          # That asymmetry is safe in the direction that occurs today -- a
          # release supporting FEWER methods gets its leg excluded here and so
          # never reaches the step gate. A release supporting MORE than the
          # default would be the reverse: the leg would run and every step
          # would skip, i.e. a silent do-nothing green. No such release exists
          # today; if one appears, make the map release-keyed too.
          #
          # Combos with no conf are skipped -- exclude-missing.txt already
          # covers them. x86_64 is the exception: it has no entry there (the
          # loop above walks $ARCHES, which is suffix arches only), so it is
          # checked unconditionally here, which also drops the dead x86_64 leg
          # on guests that ship no conf/<release>.conf at all (methods_for
          # returns empty for a missing conf -> supports nothing -> excluded).
          SYNC_JOB_METHODS="sshfs nfs scp rsync"
          for meth in $SYNC_JOB_METHODS; do
            : > ".github/data/exclude-${meth}.txt"
          done
          for r in $(sed 's/[",]/ /g' conf/test.releases) ""; do
            rc="$r"; [ -z "$rc" ] && rc="$def"
            for a in "" $ARCHES; do
              if [ -n "$a" ] && [ ! -e "conf/${rc}-${a}.conf" ]; then continue; fi
              have=",$(methods_for_rel "$rc" "$a" | tr -d ' '),"
              for meth in $SYNC_JOB_METHODS; do
                case "$have" in
                  *",${meth},"*) ;;
                  *) printf '          - release: "%s"\n            arch: "%s"\n' \
                       "$r" "$a" >> ".github/data/exclude-${meth}.txt" ;;
                esac
              done
            done
          done
          for meth in $SYNC_JOB_METHODS; do
            echo "exclude-${meth}:"; cat ".github/data/exclude-${meth}.txt"
          done

          # expose them all to template-render via the datafile (idempotent)
          df=".github/data/datafile.ini"
          grep -q '^VM_SYNC_METHODS_MAP=' "$df" || printf '\nVM_SYNC_METHODS_MAP=@.github/data/sync-map.json\n' >> "$df"
          grep -q '^VM_TEST_EXCLUDE_MISSING=' "$df" || printf '\nVM_TEST_EXCLUDE_MISSING=@.github/data/exclude-missing.txt\n' >> "$df"
          for meth in $SYNC_JOB_METHODS; do
            v="VM_TEST_EXCLUDE_$(printf '%s' "$meth" | tr 'a-z' 'A-Z')"
            grep -q "^${v}=" "$df" || printf '\n%s=@.github/data/exclude-%s.txt\n' "$v" "$meth" >> "$df"
          done

      - name: Generate files
        uses: anyvm-org/template-render@v0.0.4
        with:
          datafile: .github/data/datafile.ini
          files: |
            base-vm/.github/tpl/test.tpl.yml : .github/workflows/test.yml
            base-vm/.github/tpl/manual.tpl.yml : .github/workflows/manual.yml
            base-vm/.github/tpl/bump.tpl.yml : .github/workflows/bump.yml
            base-vm/.github/FUNDING.yml : .github/FUNDING.yml
            base-vm/.github/workflows/readme.yml : .github/workflows/readme.yml
            base-vm/.github/workflows/major.yml : .github/workflows/major.yml
            base-vm/package.json : package.json
            base-vm/index.js : index.js
            base-vm/action.yml : action.yml

      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version: '24'

      - name: Update node_modules
        run: |
          node -v
          [ -e "node_modules" ] && git rm -r node_modules
          npm install --save
          # npm install honors the committed package-lock.json, so vulnerable
          # transitive deps stay pinned forever. audit fix bumps only the
          # vulnerable ones, within the existing semver ranges (no --force).
          #
          # The advisories endpoint (registry.npmjs.org/-/npm/v1/security/
          # advisories/bulk) is a separate service from the package registry
          # and fails on its own: on 2026-09-04 it answered 503 / timed out
          # for 9+ hours while npm install kept working, and every Step 2 in
          # the fleet went red on it. npm install's built-in audit tolerates
          # that outage (npm/cli lib/utils/audit-error.js only throws when
          # the command is `audit`); a standalone `npm audit fix` exits 1
          # with "audit endpoint returned an error". An outage is no reason
          # to skip the regen -- the lock-pinned tree is what shipped before
          # audit fix existed -- so tolerate exactly that error and keep
          # every other audit fix failure (unfixable vulnerabilities) fatal.
          set -o pipefail
          if ! npm audit fix 2>&1 | tee /tmp/npm-audit-fix.log; then
            if grep -q 'audit endpoint returned an error' /tmp/npm-audit-fix.log; then
              echo "::warning::npm advisories endpoint unavailable, skipping audit fix this run"
            else
              exit 1
            fi
          fi
          git add node_modules

      - name: Check modifications
        run: |
          #if only hash id in test.yml changes, skip
          currentHash="$(cat .github/workflows/test.yml | grep -i "uses: ${{ github.repository }}" | head -1 | cut -d @ -f 2)"
          echo "Current hash: $currentHash"
          # On a brand-new repo with a single commit, HEAD~1 does not exist.
          # Under `bash -e` that aborts the step (exit 128), so fall back to
          # HEAD. --verify --quiet matters: a plain `git rev-parse HEAD~1`
          # echoes the literal string "HEAD~1" to stdout even when the rev is
          # missing, so the fallback would CONCATENATE ("HEAD~1\n<sha>") and
          # the newline then breaks the sed below with "unterminated s command"
          # (hit on nextbsd-vm's very first push).
          lastHash="$(git rev-parse --verify --quiet 'HEAD~1' 2>/dev/null || git rev-parse HEAD)"
          echo "Last hash: $lastHash"
          oldHash="$lastHash"
          echo "OLD_HASH: $OLD_HASH"
          echo "Old hash using last hash: $oldHash"
          git diff
          cp .github/workflows/test.yml test.yml.back
          sed -i "s/$currentHash/$oldHash/g" .github/workflows/test.yml
          if git diff --quiet; then
            echo "no changes"
          else
            #has changes:
            rm -f test.yml.back
            rm -rf base-vm
            #commit all changes and get new hash
            git config  user.email "ghactions@vmactions.org"
            git config  user.name "Generate.yml"
            git add .
            git commit -m "Generated from base-vm"
            newHash="$(git rev-parse main)"
            echo "New hash: $newHash"
            #modify test.yml
            sed -i "s/$oldHash/$newHash/g" .github/workflows/test.yml
          fi
      - name: Commit and push changes
        run: |
          git config user.email "ghactions@vmactions.org"
          git config user.name "Generate.yml"
          git add \
            .github/workflows/readme.yml \
            .github/workflows/major.yml \
            .github/workflows/manual.yml \
            .github/workflows/test.yml \
            .github/workflows/bump.yml \
            .github/tpl/README.tpl.md \
            .github/FUNDING.yml \
            .github/data/base-vm-hash.txt \
            .github/data/datafile.ini \
            .github/data/sync-map.json \
            .github/data/exclude-missing.txt \
            .github/data/exclude-sshfs.txt \
            .github/data/exclude-nfs.txt \
            .github/data/exclude-scp.txt \
            .github/data/exclude-rsync.txt \
            conf \
            package.json \
            package-lock.json \
            index.js \
            action.yml \
            node_modules
          if git diff --cached --quiet; then
            echo "Nothing to commit, skipping push"
          else
            printf "Update ${{ github.repository }} from base-vm\n\n%s\n" "$(cat /tmp/basevm-log.txt)" > /tmp/commit-msg.txt
            git commit -F /tmp/commit-msg.txt
            # test.yml pins the action to the "Generated from base-vm" commit (HEAD~1).
            # The git pull --rebase below can rewrite that commit when Step 1 / Update
            # Readme push to main at the same time, which would leave test.yml pointing
            # at a SHA that no longer exists ("unable to resolve action"). So pin the
            # hash AFTER each rebase and retry until the push lands cleanly.
            REPO="${{ github.repository }}"
            for attempt in 1 2 3 4 5; do
              if ! git pull --rebase --autostash; then
                # A concurrent generate run already pushed a regenerated tree;
                # the only conflict is the self-referencing pin in test.yml and
                # that run's result is equivalent. Defer to it instead of
                # letting the rebase conflict fail this step under `bash -e`.
                git rebase --abort || true
                git reset --hard origin/main
                echo "Concurrent generate detected; deferring to origin/main"
                break
              fi
              parent="$(git rev-parse HEAD~1)"
              if [ -n "$parent" ]; then
                sed -i "s#\(uses: ${REPO}@\)[0-9a-fA-F]\{7,40\}#\1${parent}#g" .github/workflows/test.yml
              fi
              if ! git diff --quiet -- .github/workflows/test.yml; then
                git add .github/workflows/test.yml
                git commit --amend --no-edit
              fi
              if git push; then
                echo "Pushed on attempt ${attempt}; test.yml pinned to ${parent}"
                break
              fi
              if [ "${attempt}" -eq 5 ]; then
                echo "Failed to push after ${attempt} attempts"
                exit 1
              fi
              echo "Push was contended, retrying (${attempt})..."
            done
          fi