nightly-e2e #65
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: nightly-e2e | |
| # Standalone TTY/UX + CLI-contract regression suite. Runs the heavy PTY tests the | |
| # cdk-terrain per-PR CI deliberately skips, against published preview releases. | |
| on: | |
| schedule: | |
| - cron: "0 3 * * *" # 03:00 UTC nightly | |
| workflow_dispatch: | |
| inputs: | |
| cli_ids: | |
| description: 'Space-separated CLI_IDs to test (e.g. "cdktn-next cdktn-latest cdktf-prefork")' | |
| required: false | |
| default: "cdktn-next cdktn-latest" | |
| force: | |
| description: "Run even if the preview version is unchanged" | |
| type: boolean | |
| required: false | |
| default: true | |
| concurrency: | |
| group: nightly-e2e | |
| cancel-in-progress: false | |
| # Least privilege at the workflow level; record-state elevates to contents: write. | |
| permissions: | |
| contents: read | |
| jobs: | |
| # Cron diff-detection: skip the matrix when cdktn-cli@next hasn't moved. | |
| decide: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| should_run: ${{ steps.check.outputs.should_run }} | |
| next_version: ${{ steps.check.outputs.next_version }} | |
| cli_ids: ${{ steps.ids.outputs.cli_ids }} | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 | |
| with: | |
| node-version-file: .nvmrc | |
| - id: check | |
| env: | |
| FORCE: ${{ (github.event_name == 'workflow_dispatch' && inputs.force) && '1' || '0' }} | |
| run: node scripts/check-new-preview.mjs | |
| - id: ids | |
| # Pass the dispatch input via env (not direct ${{ }} interpolation into the | |
| # run script) so an operator-supplied value can't inject shell — zizmor | |
| # template-injection. Identical output for normal inputs. | |
| env: | |
| CLI_IDS: ${{ inputs.cli_ids || 'cdktn-next cdktn-latest' }} | |
| run: echo "cli_ids=$CLI_IDS" >> "$GITHUB_OUTPUT" | |
| e2e: | |
| needs: decide | |
| if: needs.decide.outputs.should_run == 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # node-pty 1.1 drives a real PTY on all three: Unix PTY on linux/macOS, | |
| # Windows ConPTY on win (Win10 1809+). conpty validates the inquirer/ | |
| # cli-spinners/cli-table3 rendering on Windows terminals too. | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| cli_id: [cdktn-next, cdktn-latest] # extend with cdktf-prefork for cross-fork UX deltas | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 | |
| with: | |
| node-version-file: .nvmrc | |
| - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 | |
| - run: pnpm install --frozen-lockfile | |
| # node-pty needs a real terraform on PATH for synth/deploy to do anything. | |
| - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 | |
| with: | |
| terraform_version: "1.7.5" | |
| terraform_wrapper: false | |
| - name: Provision CLI under test | |
| run: node scripts/provision.mjs ${{ matrix.cli_id }} | |
| - name: Run e2e | |
| env: | |
| CLI_ID: ${{ matrix.cli_id }} | |
| run: pnpm test | |
| - name: Step summary | |
| if: always() | |
| env: | |
| CLI_ID: ${{ matrix.cli_id }} | |
| run: node scripts/step-summary.mjs reports/ci-report.json | |
| - name: Upload test report | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: report-${{ matrix.os }}-${{ matrix.cli_id }} | |
| path: reports/ci-report.json | |
| if-no-files-found: warn | |
| - name: Upload failure screenshots | |
| if: failure() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: svg-${{ matrix.os }}-${{ matrix.cli_id }} | |
| path: artifacts/ | |
| if-no-files-found: ignore | |
| # Publish "last run results" to GitHub Pages via the Actions build mode (NOT a | |
| # legacy branch build). Each matrix leg gets its OWN self-contained report at | |
| # _site/<leg>/index.html (no cross-leg data aggregation — build-report.mjs is | |
| # single-run); a generated _site/index.html landing page links to each leg with a | |
| # pass/fail summary (build-index.mjs). Each nightly cron or manual re-trigger | |
| # overwrites the whole site, so Pages always reflects the latest run. | |
| # | |
| # One-time setup: repo Settings → Pages → Source = "GitHub Actions". Do NOT POST to | |
| # the pages/builds API (that forces a legacy Jekyll build of the branch root). | |
| pages: | |
| needs: [decide, e2e] | |
| if: always() && (needs.e2e.result == 'success' || needs.e2e.result == 'failure') | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pages: write # deploy to GitHub Pages | |
| id-token: write # OIDC token the deploy-pages action verifies | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deploy.outputs.page_url }} | |
| concurrency: | |
| group: pages-deploy | |
| cancel-in-progress: false | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 | |
| with: | |
| node-version-file: .nvmrc | |
| - name: Download this run's reports + screenshots | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh run download ${{ github.run_id }} --pattern 'report-*' --dir _dl || echo "no reports" | |
| gh run download ${{ github.run_id }} --pattern 'svg-*' --dir _dl || echo "no screenshots" | |
| - name: Build a self-contained report per leg + the landing index | |
| env: | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| mkdir -p _site | |
| # One self-contained report per matrix leg → _site/<leg>/index.html. | |
| # Each leg's report is built independently (no cross-leg data aggregation); | |
| # svg-<leg> exists only when that leg had failures (else screenshots omitted). | |
| for d in _dl/report-*/; do | |
| [ -d "$d" ] || continue | |
| leg=$(basename "$d"); leg=${leg#report-} | |
| rep="$d/ci-report.json" | |
| [ -f "$rep" ] || continue | |
| art="_dl/svg-$leg" | |
| [ -d "$art" ] || art="_dl/__no_artifacts__" | |
| echo "Building report for leg: $leg" | |
| node scripts/build-report.mjs --report "$rep" --artifacts "$art" --run-url "$RUN_URL" --out "_site/$leg/index.html" | |
| done | |
| # Landing page linking to each ./<leg>/ with a per-leg pass/fail summary. | |
| node scripts/build-index.mjs --reports-dir _dl --site _site --run-url "$RUN_URL" --out _site/index.html | |
| - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 | |
| with: | |
| path: _site | |
| - id: deploy | |
| uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 | |
| # Surface failures as a dedup'd GitHub issue (the run-once policy means the cron | |
| # does NOT re-run failures; flakes are re-triggered manually via workflow_dispatch | |
| # force=true). One issue per cli_id, aggregated across OS legs. Recovery auto-closes. | |
| report: | |
| needs: [decide, e2e] | |
| if: always() && (needs.e2e.result == 'success' || needs.e2e.result == 'failure') | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| issues: write | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 | |
| with: | |
| node-version-file: .nvmrc | |
| - name: Download per-OS reports | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: gh run download ${{ github.run_id }} --pattern 'report-*' --dir _reports || echo "no reports to download" | |
| - name: Open/update/close regression issues | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| for id in cdktn-next cdktn-latest; do | |
| files=() | |
| for f in _reports/report-*-"$id"/ci-report.json; do | |
| [ -e "$f" ] && files+=("$f") | |
| done | |
| if [ ${#files[@]} -gt 0 ]; then | |
| node scripts/report-issue.mjs --cli-id "$id" --run-url "$RUN_URL" "${files[@]}" | |
| else | |
| echo "no report for $id" | |
| fi | |
| done | |
| # Record the tested preview version so the next cron can diff against it. | |
| # Run-once policy: advance the marker whether tests PASSED or FAILED (so long as | |
| # the suite actually ran) — the cron must not auto-retry. Skipped/cancelled → skip. | |
| record-state: | |
| needs: [decide, e2e] | |
| if: always() && needs.decide.outputs.next_version != '' && (needs.e2e.result == 'success' || needs.e2e.result == 'failure') | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write # commits state/last-tested.json back to the repo | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: true # required: this job pushes the marker commit | |
| - name: Update last-tested marker | |
| run: | | |
| node -e ' | |
| const fs=require("fs");const p="state/last-tested.json"; | |
| const s=JSON.parse(fs.readFileSync(p,"utf8")); | |
| s["cdktn-cli@next"]=process.env.V; | |
| fs.writeFileSync(p, JSON.stringify(s,null,2)+"\n"); | |
| ' | |
| env: | |
| V: ${{ needs.decide.outputs.next_version }} | |
| - name: Commit | |
| # next_version via env (not ${{ }} in the run script) so it can't inject | |
| # shell through the commit message — zizmor template-injection. | |
| env: | |
| V: ${{ needs.decide.outputs.next_version }} | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add state/last-tested.json | |
| git commit -m "chore: record tested preview $V" || echo "no change" | |
| git push |