From b304e2986750069894456187bb55d0b67991355f Mon Sep 17 00:00:00 2001 From: ryan-morris Date: Wed, 7 Oct 2026 15:11:50 -0500 Subject: [PATCH] fix(ci): retry artifact transfers and release uploads; queue releases instead of dropping them A release cell failed after its osx-x64 build succeeded: actions/upload-artifact died mid-transfer to blob storage (after 8 MiB, no error message), and the only recovery was a manual re-run of the whole build. Artifact transfer has no retry of its own, and the iOS and SHA256SUMS release uploads had none either. - Every build upload (artifact, win-x64 symbols) and every test/release download gets one retry after a 45s pause: attempt 1 is continue-on-error, attempt 2 is the verdict. Uploads retry with overwrite: true under the SAME name, since downstream jobs download by exact name. Inline steps, not a local composite action: the publish jobs have no checkout, and adding one to ~90 cells would be a new failure point of the same kind. - publish-ios upload and SHA256SUMS generation+upload get the same bounded 6-attempt retry the desktop publish upload already had (--clobber keeps both idempotent). - release concurrency: queue: max. With the default queue: single, a release that arrives while another is pending cancels and replaces it, so merging several release-triggering PRs during a long release silently dropped the middle ones -- and since each run selects only what its own push changed, nothing ever built them. max keeps up to 100 pending and runs them in order. (actionlint 1.7.12 does not know the key yet -- rhysd/actionlint#654 -- so it was verified on GitHub with a throwaway probe workflow.) - Comments that still said the release is public while assets upload are corrected: it is a draft until finalize, which gates on every upload and the manifest. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 37 ++++++++- .github/workflows/release.yml | 125 ++++++++++++++++++++++++------ .github/workflows/test-mobile.yml | 12 +++ .github/workflows/test.yml | 12 +++ 4 files changed, 161 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f3bc18d..7759af5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -323,7 +323,14 @@ jobs: # Tests run in the separate test.yml workflow (chained after this one), which # downloads the artifact uploaded below and runs the per-platform checks — # keeping build and test decoupled so tests can grow / re-run independently. - - uses: actions/upload-artifact@v7 + # Artifact transfer has no retry of its own: an upload that died mid-transfer to blob storage + # (no error message, after 8 MiB) failed a release cell whose build had already succeeded, and + # the only recovery was re-running the whole build. So one retry after a pause: attempt 1 + # tolerates failure; attempt 2 replaces whatever attempt 1 left (overwrite) and is the verdict. + # Same NAME on both -- release.yml and test.yml download by exact name. + - id: upload + continue-on-error: true + uses: actions/upload-artifact@v7 with: name: ${{ env.VARIANT }} path: artifacts/${{ matrix.rid }}/native @@ -334,18 +341,44 @@ jobs: # ios-sim slice) expired before publish could download them → "Artifact not found". Five # days gives comfortable margin over the longest observed run; they're throwaway either way. retention-days: 5 + - name: Wait before retrying the artifact upload + if: ${{ !cancelled() && steps.upload.outcome == 'failure' }} + run: sleep 45 + - name: Upload build artifact (retry) + if: ${{ !cancelled() && steps.upload.outcome == 'failure' }} + uses: actions/upload-artifact@v7 + with: + name: ${{ env.VARIANT }} + path: artifacts/${{ matrix.rid }}/native + if-no-files-found: error + retention-days: 5 + overwrite: true # Issue #20: unstripped win-x64 binaries (ffmpeg_g.exe, ffprobe_g.exe, versioned DLLs) staged # by 08_stage_artifacts.sh, for symbolizing a crash dump's module+offset frames. A separate # artifact, never part of the shipped one; release.yml downloads build artifacts by exact name, # so it cannot pick this up. Kept as long as GitHub allows, since it pairs with a release. - - uses: actions/upload-artifact@v7 + - id: upload_symbols if: matrix.rid == 'win-x64' + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: symbols-${{ env.VARIANT }} + path: artifacts/${{ matrix.rid }}/symbols + if-no-files-found: error + retention-days: 90 + - name: Wait before retrying the symbols upload + if: ${{ !cancelled() && steps.upload_symbols.outcome == 'failure' }} + run: sleep 45 + - name: Upload symbols (retry) + if: ${{ !cancelled() && steps.upload_symbols.outcome == 'failure' }} + uses: actions/upload-artifact@v7 with: name: symbols-${{ env.VARIANT }} path: artifacts/${{ matrix.rid }}/symbols if-no-files-found: error retention-days: 90 + overwrite: true - uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 816825e..4e0d516 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,6 +31,12 @@ on: concurrency: group: release cancel-in-progress: false # never cancel a release mid-flight + # Queue every pending release instead of keeping one. With the default (queue: single) a run that + # arrives while another is PENDING cancels and replaces it, so merging three release-triggering + # PRs during a long release silently dropped the middle one -- and its version selection + # (select-versions.sh diffs only its own push) was never built by anything. max keeps up to 100 + # pending and runs them in order. Not combinable with cancel-in-progress: true. + queue: max jobs: # Decide which version lines to (re)build this run, and compute each one's @@ -260,7 +266,22 @@ jobs: license: [lgpl, gpl] license_version: [3, 2] # → license token {license}v{series}: lgplv3/gplv3/lgplv2/gplv2 steps: - - uses: actions/download-artifact@v8 + # Artifact transfer has no retry of its own, and a storage blip here failed a whole release + # after every build had succeeded. One retry after a pause, inline rather than a local + # composite action: this job has no checkout, and adding one to ~90 publish cells would be a + # new failure point of the same kind. Attempt 1 tolerates failure; attempt 2 is the verdict. + - id: dl + continue-on-error: true + uses: actions/download-artifact@v8 + with: + name: ${{ matrix.build.version }}-${{ matrix.rid }}-${{ matrix.license }}v${{ matrix.license_version }} + path: artifacts/${{ matrix.rid }}-${{ matrix.license }}v${{ matrix.license_version }} + - name: Wait before retrying the artifact download + if: ${{ !cancelled() && steps.dl.outcome == 'failure' }} + run: sleep 45 + - name: Download build artifact (retry) + if: ${{ !cancelled() && steps.dl.outcome == 'failure' }} + uses: actions/download-artifact@v8 with: name: ${{ matrix.build.version }}-${{ matrix.rid }}-${{ matrix.license }}v${{ matrix.license_version }} path: artifacts/${{ matrix.rid }}-${{ matrix.license }}v${{ matrix.license_version }} @@ -417,11 +438,10 @@ jobs: run: | VERSION="${{ matrix.build.version }}" VARIANT="${{ matrix.rid }}-${{ matrix.license }}v${{ matrix.license_version }}" - # Asset upload is a mutable network operation against the API and had no retry, while every - # other network call in this repo now has one -- and this is the worst place to lack one: the - # release is already PUBLIC by the time these run, so a blip leaves a published release missing - # an artifact. --clobber makes the retry idempotent: a partially-uploaded asset is replaced, - # not duplicated. + # Asset upload is a mutable network operation against the API. The release is still a draft + # here (finalize publishes it), so a failure cannot expose an incomplete release -- but it + # does fail the whole release run over a blip. --clobber makes the retry idempotent: a + # partially-uploaded asset is replaced, not duplicated. upload_retry() { # local f="$1" try delay=4 wait for try in 1 2 3 4 5 6; do @@ -452,19 +472,56 @@ jobs: license: [lgpl, gpl] license_version: [3, 2] # ios v2 = LGPLv2.1/GPLv2 (App-Store-safe): no version3 steps: - - uses: actions/download-artifact@v8 + # Same one-retry pattern as publish (no checkout here either): four first attempts that + # tolerate failure, one shared pause if any failed, then a verdict attempt for each that did. + - id: dl_ios + continue-on-error: true + uses: actions/download-artifact@v8 + with: + name: ${{ matrix.build.version }}-ios-arm64-${{ matrix.license }}v${{ matrix.license_version }} + path: artifacts/ios-arm64 + - id: dl_sim + continue-on-error: true + uses: actions/download-artifact@v8 + with: + name: ${{ matrix.build.version }}-ios-sim-arm64-${{ matrix.license }}v${{ matrix.license_version }} + path: artifacts/ios-sim-arm64 + - id: dl_cat_arm + continue-on-error: true + uses: actions/download-artifact@v8 + with: + name: ${{ matrix.build.version }}-maccatalyst-arm64-${{ matrix.license }}v${{ matrix.license_version }} + path: artifacts/maccatalyst-arm64 + - id: dl_cat_x64 + continue-on-error: true + uses: actions/download-artifact@v8 + with: + name: ${{ matrix.build.version }}-maccatalyst-x64-${{ matrix.license }}v${{ matrix.license_version }} + path: artifacts/maccatalyst-x64 + - name: Wait before retrying artifact downloads + if: ${{ !cancelled() && (steps.dl_ios.outcome == 'failure' || steps.dl_sim.outcome == 'failure' || steps.dl_cat_arm.outcome == 'failure' || steps.dl_cat_x64.outcome == 'failure') }} + run: sleep 45 + - name: Download ios-arm64 (retry) + if: ${{ !cancelled() && steps.dl_ios.outcome == 'failure' }} + uses: actions/download-artifact@v8 with: name: ${{ matrix.build.version }}-ios-arm64-${{ matrix.license }}v${{ matrix.license_version }} path: artifacts/ios-arm64 - - uses: actions/download-artifact@v8 + - name: Download ios-sim-arm64 (retry) + if: ${{ !cancelled() && steps.dl_sim.outcome == 'failure' }} + uses: actions/download-artifact@v8 with: name: ${{ matrix.build.version }}-ios-sim-arm64-${{ matrix.license }}v${{ matrix.license_version }} path: artifacts/ios-sim-arm64 - - uses: actions/download-artifact@v8 + - name: Download maccatalyst-arm64 (retry) + if: ${{ !cancelled() && steps.dl_cat_arm.outcome == 'failure' }} + uses: actions/download-artifact@v8 with: name: ${{ matrix.build.version }}-maccatalyst-arm64-${{ matrix.license }}v${{ matrix.license_version }} path: artifacts/maccatalyst-arm64 - - uses: actions/download-artifact@v8 + - name: Download maccatalyst-x64 (retry) + if: ${{ !cancelled() && steps.dl_cat_x64.outcome == 'failure' }} + uses: actions/download-artifact@v8 with: name: ${{ matrix.build.version }}-maccatalyst-x64-${{ matrix.license }}v${{ matrix.license_version }} path: artifacts/maccatalyst-x64 @@ -543,10 +600,19 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - gh release upload "${{ matrix.build.tag }}" \ - "ffmpeg-${{ matrix.build.version }}-ios-${{ matrix.license }}v${{ matrix.license_version }}.tar.gz" \ - --clobber \ - --repo "${{ github.repository }}" + # Same bounded retry as the desktop publish upload; --clobber keeps it idempotent. + f="ffmpeg-${{ matrix.build.version }}-ios-${{ matrix.license }}v${{ matrix.license_version }}.tar.gz" + delay=4 + for try in 1 2 3 4 5 6; do + gh release upload "${{ matrix.build.tag }}" "${f}" --clobber \ + --repo "${{ github.repository }}" && exit 0 + [ "${try}" -eq 6 ] && break + wait=$(( delay + (RANDOM % 5) )) + echo " upload of ${f} failed (attempt ${try}/6) - retrying in ${wait}s..." >&2 + sleep "${wait}"; delay=$(( delay * 2 )) + done + echo "ERROR: gh release upload ${f} failed after 6 attempts" >&2 + exit 1 # Aggregate checksum manifest — one file covering every asset, so a downstream packager or # mirror verifies once instead of making an API call per asset. @@ -611,20 +677,33 @@ jobs: TAG: ${{ matrix.build.tag }} REPO: ${{ github.repository }} run: | - set -euo pipefail - bash scripts/ci/gen-manifest.sh "$TAG" "$REPO" > SHA256SUMS - echo "--- SHA256SUMS ($(wc -l < SHA256SUMS) entries) ---" - cat SHA256SUMS - gh release upload "$TAG" SHA256SUMS \ - --clobber --repo "$REPO" + set -uo pipefail + # Generation fetches every asset and the upload is an API write, so retry the pair: + # regeneration is repeatable (gen-manifest.sh excludes SHA256SUMS itself) and --clobber + # replaces a partial upload. + delay=4 + for try in 1 2 3 4 5 6; do + if bash scripts/ci/gen-manifest.sh "$TAG" "$REPO" > SHA256SUMS \ + && gh release upload "$TAG" SHA256SUMS --clobber --repo "$REPO"; then + echo "--- SHA256SUMS ($(wc -l < SHA256SUMS) entries) ---" + cat SHA256SUMS + exit 0 + fi + [ "${try}" -eq 6 ] && break + wait=$(( delay + (RANDOM % 5) )) + echo " SHA256SUMS generation/upload failed (attempt ${try}/6) - retrying in ${wait}s..." >&2 + sleep "${wait}"; delay=$(( delay * 2 )) + done + echo "ERROR: SHA256SUMS generation/upload failed after 6 attempts" >&2 + exit 1 - # This job runs after the release is public and nothing gates on it, so a silent failure - # would leave a release whose docs promise a SHA256SUMS that isn't there. Surface it. + # finalize gates on this job, so a failure leaves the release a DRAFT rather than a public + # release missing its checksums. Say so, and how to recover. - name: Report manifest failure if: failure() run: | echo "### SHA256SUMS generation FAILED for \`${{ matrix.build.tag }}\`" >> "$GITHUB_STEP_SUMMARY" - echo "The release is published but has no checksum manifest. Re-run this job, or generate it locally with \`bash scripts/ci/gen-manifest.sh \` and upload it." >> "$GITHUB_STEP_SUMMARY" + echo "The release stays a DRAFT (finalize gates on this job). Re-run the failed jobs, or generate it locally with \`bash scripts/ci/gen-manifest.sh \` and upload it." >> "$GITHUB_STEP_SUMMARY" # The release is created as a DRAFT by `release` and becomes visible only here -- after every # asset (publish, publish-ios) and the SHA256SUMS manifest are attached. Before this job existed diff --git a/.github/workflows/test-mobile.yml b/.github/workflows/test-mobile.yml index fb323a0..39ab94e 100644 --- a/.github/workflows/test-mobile.yml +++ b/.github/workflows/test-mobile.yml @@ -118,7 +118,19 @@ jobs: # is shared with the calling workflow, so every cell builds exactly one tree. ref: ${{ inputs.ref || github.sha }} + # One retry after a pause: artifact transfer has none of its own (see build.yml's upload). - name: Download build artifact + id: dl + continue-on-error: true + uses: actions/download-artifact@v8 + with: + name: ${{ env.VARIANT }} + path: artifacts/${{ matrix.rid }}/native + - name: Wait before retrying the artifact download + if: ${{ !cancelled() && steps.dl.outcome == 'failure' }} + run: sleep 45 + - name: Download build artifact (retry) + if: ${{ !cancelled() && steps.dl.outcome == 'failure' }} uses: actions/download-artifact@v8 with: name: ${{ env.VARIANT }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0a72c42..36e2d1c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -131,7 +131,19 @@ jobs: # is shared with the calling workflow, so every cell builds exactly one tree. ref: ${{ inputs.ref || github.sha }} + # One retry after a pause: artifact transfer has none of its own (see build.yml's upload). - name: Download build artifact + id: dl + continue-on-error: true + uses: actions/download-artifact@v8 + with: + name: ${{ env.VARIANT }} + path: artifacts/${{ matrix.rid }}/native + - name: Wait before retrying the artifact download + if: ${{ !cancelled() && steps.dl.outcome == 'failure' }} + run: sleep 45 + - name: Download build artifact (retry) + if: ${{ !cancelled() && steps.dl.outcome == 'failure' }} uses: actions/download-artifact@v8 with: name: ${{ env.VARIANT }}