Step 2, Generate the project from BaseVM #7
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Step 2, Generate the project from BaseVM" | |
| on: | |
| schedule: | |
| - cron: '20 22 * * *' | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - 'main' | |
| paths: | |
| - 'conf/*' | |
| - '.github/data/*' | |
| - '.github/workflows/generate.yml' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| generate: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| token: ${{ secrets.VM_TOKEN }} | |
| fetch-depth: '2' | |
| - name: Preparing env variables | |
| run: | | |
| git switch main || (git fetch --all && git checkout -b main origin/main) | |
| . conf/default.release.conf | |
| echo "DEFAULT_RELEASE=$DEFAULT_RELEASE" >> $GITHUB_ENV | |
| echo "ALL_RELEASES=\"$(ls conf/ | grep -v default | sed 's/.conf//g' | tr '\n' ',' | sed "s/,\$//" | sed 's/,/", "/g')\"" >> $GITHUB_ENV | |
| LATEST_TAG="$(git tag --sort=-v:refname | head -n 1)" | |
| echo "LATEST_TAG=${LATEST_TAG}" >> $GITHUB_ENV | |
| echo "LATEST_VERSION_NUMBER=${LATEST_TAG:1}" >> $GITHUB_ENV | |
| oldHash="$(cat .github/workflows/test.yml | grep -i "uses: ${{ github.repository }}" | head -1 | cut -d @ -f 2)" | |
| echo "OLD_HASH=${oldHash}" >> $GITHUB_ENV | |
| git clone https://github.com/vmactions/base-vm.git | |
| # Collect base-vm commit messages since last generate | |
| BASEVM_HASH_FILE=".github/data/base-vm-hash.txt" | |
| BASEVM_NEW_HASH="$(git -C base-vm rev-parse HEAD)" | |
| if [ -f "$BASEVM_HASH_FILE" ]; then | |
| BASEVM_OLD_HASH="$(cat "$BASEVM_HASH_FILE")" | |
| BASEVM_LOG="$(git -C base-vm log --format="https://github.com/vmactions/base-vm/commit/%H %s" "${BASEVM_OLD_HASH}..${BASEVM_NEW_HASH}" 2>/dev/null || git -C base-vm log --format="https://github.com/vmactions/base-vm/commit/%H %s" -20)" | |
| else | |
| BASEVM_LOG="$(git -C base-vm log --format="https://github.com/vmactions/base-vm/commit/%H %s" -1)" | |
| fi | |
| echo "$BASEVM_NEW_HASH" > "$BASEVM_HASH_FILE" | |
| # Save log for later commit message | |
| echo "$BASEVM_LOG" > /tmp/basevm-log.txt | |
| mkdir -p .github/tpl | |
| cat base-vm/.github/tpl/README.tpl.md >.github/tpl/README.tpl.md | |
| - name: Bake sync methods from builder confs | |
| run: | | |
| # Each conf/<release>[-<arch>].conf pins a BUILDER_VERSION. The | |
| # matching builder conf declares VM_SYNC_METHODS (comma separated; | |
| # first = default) and VM_SHUTDOWN_CMD (the in-guest poweroff command, | |
| # used by cache-after-prepare). Copy those values into the local conf | |
| # so the action and test workflow can read them without a network | |
| # call at run time. | |
| os="hurd" | |
| for f in conf/*.conf; do | |
| base="$(basename "$f" .conf)" | |
| case "$base" in | |
| default.release) continue ;; | |
| esac | |
| bv="$( . "$f"; printf '%s' "$BUILDER_VERSION" )" | |
| if [ -z "$bv" ]; then | |
| echo "${base}: no BUILDER_VERSION, skip" | |
| continue | |
| fi | |
| url="https://raw.githubusercontent.com/anyvm-org/${os}-builder/v${bv}/conf/${os}-${base}.conf" | |
| # Old builder pins (<2.0.0, e.g. openbsd 7.2-7.6) have no tag in | |
| # anyvm-org, so this curl 404s; the step runs under bash -e and a | |
| # bare command substitution would abort the whole job (exit 22). | |
| # Degrade to an empty confdata -> both fields fall into their | |
| # "leaving conf as-is" branches, same as the legacy behavior. | |
| confdata="$(curl -fsSL "$url" 2>/dev/null || true)" | |
| methods="$(printf '%s\n' "$confdata" | sed -n 's/^[[:space:]]*VM_SYNC_METHODS=//p' | tr -d '"' | tail -n 1)" | |
| if [ -z "$methods" ]; then | |
| echo "${base}: no VM_SYNC_METHODS in ${url}, leaving conf as-is" | |
| else | |
| if grep -q '^VM_SYNC_METHODS=' "$f"; then | |
| sed -i "s|^VM_SYNC_METHODS=.*|VM_SYNC_METHODS=\"${methods}\"|" "$f" | |
| else | |
| printf '\nVM_SYNC_METHODS="%s"\n' "${methods}" >> "$f" | |
| fi | |
| echo "${base}: VM_SYNC_METHODS=\"${methods}\"" | |
| fi | |
| shutcmd="$(printf '%s\n' "$confdata" | sed -n 's/^[[:space:]]*VM_SHUTDOWN_CMD=//p' | tr -d '"' | tail -n 1)" | |
| if [ -z "$shutcmd" ]; then | |
| echo "${base}: no VM_SHUTDOWN_CMD in ${url}, leaving conf as-is" | |
| else | |
| if grep -q '^VM_SHUTDOWN_CMD=' "$f"; then | |
| sed -i "s|^VM_SHUTDOWN_CMD=.*|VM_SHUTDOWN_CMD=\"${shutcmd}\"|" "$f" | |
| else | |
| printf '\nVM_SHUTDOWN_CMD="%s"\n' "${shutcmd}" >> "$f" | |
| fi | |
| echo "${base}: VM_SHUTDOWN_CMD=\"${shutcmd}\"" | |
| fi | |
| done | |
| - name: Compute sync method map and missing-arch excludes | |
| run: | | |
| # Turn the per-conf VM_SYNC_METHODS into two values for the renderer: | |
| # VM_SYNC_METHODS_MAP - {"<arch>":["method",...]} keyed by matrix.arch, | |
| # so test.yml gates each sync job with `if:`. | |
| # VM_TEST_EXCLUDE_MISSING - matrix excludes for release/arch combos | |
| # that have no conf (so they never spawn). | |
| . conf/default.release.conf | |
| def="$DEFAULT_RELEASE" | |
| # non-x86_64 arches the test matrix covers (keep in sync with test.tpl.yml) | |
| ARCHES="aarch64 riscv64 powerpc64 sparc64 ppc64le s390x i386" | |
| # methods (comma list) for an arch suffix, from the default release | |
| # conf; fall back to any release that has that arch. | |
| methods_for() { | |
| a="$1" | |
| if [ -n "$a" ]; then cf="conf/${def}-${a}.conf"; else cf="conf/${def}.conf"; fi | |
| if [ -n "$a" ] && [ ! -e "$cf" ]; then | |
| cf="$(ls conf/*-"$a".conf 2>/dev/null | head -n 1)" | |
| fi | |
| [ -n "$cf" ] && [ -e "$cf" ] && ( . "$cf"; printf '%s' "$VM_SYNC_METHODS" ) | |
| } | |
| # comma list -> JSON array: rsync,scp -> ["rsync","scp"] (empty -> []) | |
| to_json() { | |
| printf '[' | |
| _first=1 | |
| _old_ifs="$IFS"; IFS=',' | |
| for _m in $1; do | |
| _m="$(printf '%s' "$_m" | tr -d ' ')" | |
| [ -z "$_m" ] && continue | |
| [ "$_first" = 1 ] || printf ',' | |
| printf '"%s"' "$_m" | |
| _first=0 | |
| done | |
| IFS="$_old_ifs" | |
| printf ']' | |
| } | |
| map="{\"x86_64\":$(to_json "$(methods_for '')")" | |
| for a in $ARCHES; do | |
| map="${map},\"${a}\":$(to_json "$(methods_for "$a")")" | |
| done | |
| printf '%s}' "$map" > .github/data/sync-map.json | |
| echo "sync-map: $(cat .github/data/sync-map.json)" | |
| : > .github/data/exclude-missing.txt | |
| for r in $(sed 's/[",]/ /g' conf/test.releases) ""; do | |
| rc="$r"; [ -z "$rc" ] && rc="$def" | |
| for a in $ARCHES; do | |
| if [ ! -e "conf/${rc}-${a}.conf" ]; then | |
| printf ' - release: "%s"\n arch: %s\n' "$r" "$a" >> .github/data/exclude-missing.txt | |
| fi | |
| done | |
| done | |
| echo "exclude-missing:"; cat .github/data/exclude-missing.txt | |
| # expose both to template-render via the datafile (idempotent) | |
| df=".github/data/datafile.ini" | |
| grep -q '^VM_SYNC_METHODS_MAP=' "$df" || printf '\nVM_SYNC_METHODS_MAP=@.github/data/sync-map.json\n' >> "$df" | |
| grep -q '^VM_TEST_EXCLUDE_MISSING=' "$df" || printf '\nVM_TEST_EXCLUDE_MISSING=@.github/data/exclude-missing.txt\n' >> "$df" | |
| - name: Generate files | |
| uses: anyvm-org/template-render@v0.0.4 | |
| with: | |
| datafile: .github/data/datafile.ini | |
| files: | | |
| base-vm/.github/tpl/test.tpl.yml : .github/workflows/test.yml | |
| base-vm/.github/tpl/manual.tpl.yml : .github/workflows/manual.yml | |
| base-vm/.github/FUNDING.yml : .github/FUNDING.yml | |
| base-vm/.github/workflows/readme.yml : .github/workflows/readme.yml | |
| base-vm/.github/workflows/major.yml : .github/workflows/major.yml | |
| base-vm/package.json : package.json | |
| base-vm/index.js : index.js | |
| base-vm/action.yml : action.yml | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: '24' | |
| - name: Update node_modules | |
| run: | | |
| node -v | |
| [ -e "node_modules" ] && git rm -r node_modules | |
| npm install --save | |
| git add node_modules | |
| - name: Check modifications | |
| run: | | |
| #if only hash id in test.yml changes, skip | |
| currentHash="$(cat .github/workflows/test.yml | grep -i "uses: ${{ github.repository }}" | head -1 | cut -d @ -f 2)" | |
| echo "Current hash: $currentHash" | |
| # On a brand-new repo with a single commit, HEAD~1 does not exist. | |
| # Under `bash -e` that aborts the step (exit 128), so fall back to HEAD. | |
| lastHash="$(git rev-parse 'HEAD~1' 2>/dev/null || git rev-parse HEAD)" | |
| echo "Last hash: $lastHash" | |
| oldHash="$lastHash" | |
| echo "OLD_HASH: $OLD_HASH" | |
| echo "Old hash using last hash: $oldHash" | |
| git diff | |
| cp .github/workflows/test.yml test.yml.back | |
| sed -i "s/$currentHash/$oldHash/g" .github/workflows/test.yml | |
| if git diff --quiet; then | |
| echo "no changes" | |
| else | |
| #has changes: | |
| rm -f test.yml.back | |
| rm -rf base-vm | |
| #commit all changes and get new hash | |
| git config user.email "ghactions@vmactions.org" | |
| git config user.name "Generate.yml" | |
| git add . | |
| git commit -m "Generated from base-vm" | |
| newHash="$(git rev-parse main)" | |
| echo "New hash: $newHash" | |
| #modify test.yml | |
| sed -i "s/$oldHash/$newHash/g" .github/workflows/test.yml | |
| fi | |
| - name: Commit and push changes | |
| run: | | |
| git config user.email "ghactions@vmactions.org" | |
| git config user.name "Generate.yml" | |
| git add \ | |
| .github/workflows/readme.yml \ | |
| .github/workflows/major.yml \ | |
| .github/workflows/manual.yml \ | |
| .github/workflows/test.yml \ | |
| .github/tpl/README.tpl.md \ | |
| .github/FUNDING.yml \ | |
| .github/data/base-vm-hash.txt \ | |
| .github/data/datafile.ini \ | |
| .github/data/sync-map.json \ | |
| .github/data/exclude-missing.txt \ | |
| conf \ | |
| package.json \ | |
| package-lock.json \ | |
| index.js \ | |
| action.yml \ | |
| node_modules | |
| if git diff --cached --quiet; then | |
| echo "Nothing to commit, skipping push" | |
| else | |
| printf "Update ${{ github.repository }} from base-vm\n\n%s\n" "$(cat /tmp/basevm-log.txt)" > /tmp/commit-msg.txt | |
| git commit -F /tmp/commit-msg.txt | |
| # test.yml pins the action to the "Generated from base-vm" commit (HEAD~1). | |
| # The git pull --rebase below can rewrite that commit when Step 1 / Update | |
| # Readme push to main at the same time, which would leave test.yml pointing | |
| # at a SHA that no longer exists ("unable to resolve action"). So pin the | |
| # hash AFTER each rebase and retry until the push lands cleanly. | |
| REPO="${{ github.repository }}" | |
| for attempt in 1 2 3 4 5; do | |
| if ! git pull --rebase --autostash; then | |
| # A concurrent generate run already pushed a regenerated tree; | |
| # the only conflict is the self-referencing pin in test.yml and | |
| # that run's result is equivalent. Defer to it instead of | |
| # letting the rebase conflict fail this step under `bash -e`. | |
| git rebase --abort || true | |
| git reset --hard origin/main | |
| echo "Concurrent generate detected; deferring to origin/main" | |
| break | |
| fi | |
| parent="$(git rev-parse HEAD~1)" | |
| if [ -n "$parent" ]; then | |
| sed -i "s#\(uses: ${REPO}@\)[0-9a-fA-F]\{7,40\}#\1${parent}#g" .github/workflows/test.yml | |
| fi | |
| if ! git diff --quiet -- .github/workflows/test.yml; then | |
| git add .github/workflows/test.yml | |
| git commit --amend --no-edit | |
| fi | |
| if git push; then | |
| echo "Pushed on attempt ${attempt}; test.yml pinned to ${parent}" | |
| break | |
| fi | |
| if [ "${attempt}" -eq 5 ]; then | |
| echo "Failed to push after ${attempt} attempts" | |
| exit 1 | |
| fi | |
| echo "Push was contended, retrying (${attempt})..." | |
| done | |
| fi | |