Charts: add summary columns to the HeatmapChart #160884
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - '**.md' | |
| - '**/changelog/**' | |
| repository_dispatch: | |
| types: [ 'e2e tests**' ] | |
| workflow_dispatch: | |
| inputs: | |
| wp_version: | |
| description: 'Single WordPress version to run against, e.g. 6.9 or 7.1-RC1. Defaults to MIN_WP_VERSION from .github/versions.sh.' | |
| required: false | |
| type: string | |
| concurrency: | |
| # wp_version keeps concurrent dispatches for different WordPress versions from cancelling each other. | |
| group: e2e-tests-${{ github.event_name }}-${{ github.ref }}-${{ case( github.event_name == 'repository_dispatch', github.event.action, '' ) }}-${{ inputs.wp_version }} | |
| cancel-in-progress: true | |
| permissions: | |
| # actions/checkout, actions/upload-artifact, actions/download-artifact | |
| contents: read | |
| env: | |
| FORCE_COLOR: 1 | |
| USE_CLOUDFLARE_TUNNEL: ${{ case( vars.E2E_TUNNEL_TYPE == 'cloudflare', '1', '' ) }} | |
| jobs: | |
| create-test-matrix: | |
| name: "Determine tests matrix" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 # 2025-11-20: The pnpm install may take a few minutes on cache miss. | |
| # Only run tests in the main repository | |
| if: github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name | |
| outputs: | |
| matrix: ${{ steps.evaluate.outputs.matrix }} | |
| build-matrix: ${{ steps.evaluate.outputs.build-matrix }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| # For pull requests, list-changed-projects.sh needs the merge base. | |
| - name: Deepen to merge base | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/deepen-to-merge-base | |
| - name: Setup tools | |
| uses: ./.github/actions/tool-setup | |
| with: | |
| php: false | |
| # Required for list-changed-projects.sh | |
| - name: Install monorepo | |
| run: | | |
| pnpm install | |
| - name: Create test plan | |
| id: evaluate | |
| env: | |
| DISPATCH_REPO: ${{ github.event.client_payload.repository }} | |
| REF_NAME: ${{ github.event.client_payload.ref_name }} | |
| REF_TYPE: ${{ github.event.client_payload.ref_type }} | |
| WP_VERSION: ${{ inputs.wp_version }} | |
| run: | | |
| MATRIX="$(node .github/files/e2e-tests/e2e-matrix.js)" | |
| echo "matrix: $MATRIX" | |
| echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" | |
| if [[ "$GITHUB_EVENT_NAME" == repository_dispatch ]]; then | |
| echo "No build needed for $GITHUB_EVENT_NAME" | |
| BUILD_MATRIX="[]" | |
| else | |
| BUILD_MATRIX=$(jq -c '[ .[] | { buildGroup: .buildGroup, path: .path } ] | unique' <<<"$MATRIX") | |
| echo "build matrix: $BUILD_MATRIX" | |
| BAD=$( jq -r '[ group_by( .buildGroup )[] | select( length > 1 ) | .[0].buildGroup ] | unique | join( "," )' <<<"$BUILD_MATRIX" ) | |
| if [[ -n "$BAD" ]]; then | |
| echo "::error::One or more build groups have multiple paths: $BAD" | |
| exit 1 | |
| fi | |
| fi | |
| echo "build-matrix=$BUILD_MATRIX" >> "$GITHUB_OUTPUT" | |
| build-projects: | |
| name: "E2E: Build ${{ matrix.buildGroup }}" | |
| runs-on: ubuntu-latest | |
| needs: create-test-matrix | |
| timeout-minutes: 25 # 2025-11-20: These can vary dramatically, but most builds are around 5 minutes. | |
| if: needs.create-test-matrix.outputs.build-matrix != '[]' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson( needs.create-test-matrix.outputs.build-matrix ) }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| - name: Setup tools | |
| uses: ./.github/actions/tool-setup | |
| - name: Install monorepo | |
| run: pnpm install | |
| - name: Build projects | |
| id: build-step | |
| env: | |
| COMPOSER_ROOT_VERSION: "dev-trunk" | |
| BUILD_DIR: ./build-output | |
| PROJECT_PATH: ${{ matrix.path }} | |
| BUILD_GROUP: ${{ matrix.buildGroup }} | |
| run: | | |
| find . -path ./.github -prune -o -type f -print | sort > /tmp/before.txt | |
| find . -path ./.github -prune -o -type l -print | sort > /tmp/beforel.txt | |
| echo "::group::Build plugin(s)" | |
| cd "$PROJECT_PATH" | |
| pnpm run build | |
| cd "$GITHUB_WORKSPACE" | |
| echo "::endgroup::" | |
| find . -path ./.github -prune -o -type f -print | sort > /tmp/after.txt | |
| find . -path ./.github -prune -o -type l -print | sort > /tmp/afterl.txt | |
| # We only want to save the files that were actually created or changed. | |
| { | |
| comm -13 /tmp/before.txt /tmp/after.txt | |
| git -c core.quotepath=off diff --name-only | |
| } | sed 's!^./!!' | sort -u > /tmp/changed.txt | |
| comm -13 /tmp/beforel.txt /tmp/afterl.txt | sed 's!^./!!' > /tmp/changedl.txt | |
| # There will probably be a bunch of duplicate vendor files. Try to deduplicate them. | |
| echo "::group::Deduplicating files" | |
| xargs -d '\n' sha256sum < /tmp/changed.txt | sort | perl -nwe ' | |
| BEGIN { $h = ""; $f = ""; } | |
| die "WTF? $_" unless /^([0-9a-f]{64}) (\S.*)$/; | |
| if ( $h ne $1 ) { | |
| $h=$1; $f=$2; | |
| } else { | |
| print "$2 → $f\n"; | |
| unlink( $2 ) or die "Failed to delete $2: $!\n"; | |
| link( $f, $2 ) or die "Failed to link $2 → $f: $!\n"; | |
| } | |
| ' | |
| echo "::endgroup::" | |
| echo "::group::Creating artifact" | |
| sed 's!^!./!' /tmp/changed.txt /tmp/changedl.txt | tar -cvvf "$BUILD_GROUP.tar.xz" --verbatim-files-from --files-from=- | |
| echo "::endgroup::" | |
| - name: Save ${{ matrix.buildGroup }} build cache | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| path: ${{ matrix.buildGroup }}.tar.xz | |
| retention-days: 1 | |
| # Already compressed. | |
| compression-level: 0 | |
| archive: false | |
| e2e-tests: | |
| name: "${{ matrix.project }} e2e tests (WP ${{ matrix.wpVersion }})" | |
| runs-on: ubuntu-latest | |
| needs: [ create-test-matrix, build-projects ] | |
| # The "always() && ! cancelled() && ! failure()" bit is needed to still run if the build was skipped. | |
| if: > | |
| always() && ! cancelled() && ! failure() && | |
| needs.create-test-matrix.result == 'success' && needs.create-test-matrix.outputs.matrix != '[]' | |
| timeout-minutes: 25 # 2025-11-20: These can vary dramatically, but most runs are 3-10 minutes. | |
| env: | |
| WP_DEBUG_PATH: '${{ github.workspace }}/tools/docker/wordpress/wp-content/debug.log' | |
| # Used in tests for enhanced reporting of global projects. The WordPress version is part of the | |
| # name so the two runs of a project are distinguishable in the report. | |
| PROJECT_NAME: '${{ matrix.project }} WP ${{ matrix.wpVersion }}' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson( needs.create-test-matrix.outputs.matrix ) }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| - name: Restore ${{ matrix.buildGroup }} build cache | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| if: needs.build-projects.result == 'success' | |
| with: | |
| name: ${{ matrix.buildGroup }}.tar.xz | |
| - name: Extract ${{ matrix.buildGroup }} build cache | |
| if: needs.build-projects.result == 'success' | |
| env: | |
| BUILD_GROUP: ${{ matrix.buildGroup }} | |
| run: | | |
| tar -xvvf "$BUILD_GROUP.tar.xz" . | |
| - name: Setup tools | |
| uses: ./.github/actions/tool-setup | |
| - name: Install cloudflared | |
| if: env.USE_CLOUDFLARE_TUNNEL | |
| run: | | |
| curl -L --output cloudflared.deb https://github.com/cloudflare/cloudflared/releases/download/2025.7.0/cloudflared-linux-amd64.deb | |
| echo "a65ceb838f1deb941f7ea9101ff54eaba04179135fd562456a59ea293479e113 cloudflared.deb" | sha256sum -c | |
| sudo dpkg -i cloudflared.deb | |
| - name: Install monorepo | |
| run: | | |
| pnpm install | |
| - name: Checkout jetpack-production | |
| if: github.event_name == 'repository_dispatch' && github.event.client_payload.repository != 'Automattic/jetpack-production' | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| repository: Automattic/jetpack-production | |
| path: build-output/build/Automattic/jetpack-production | |
| - name: Checkout mirror repo | |
| if: github.event_name == 'repository_dispatch' | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| repository: ${{ github.event.client_payload.repository }} | |
| ref: ${{ github.event.client_payload.ref_name }} | |
| path: build-output/build/${{ github.event.client_payload.repository }} | |
| - name: Prepare build | |
| env: | |
| COMPOSER_ROOT_VERSION: "dev-trunk" | |
| BUILD_DIR: ./build-output | |
| PROJECT_PATH: ${{ matrix.path }} | |
| BUILD_JOB_RESULT: ${{ needs.build-projects.result }} | |
| run: | | |
| # Only execute these steps if the event name is 'repository_dispatch' | |
| if [[ "$GITHUB_EVENT_NAME" == repository_dispatch ]]; then | |
| if [[ "$BUILD_JOB_RESULT" != 'skipped' ]]; then | |
| echo "::error::Build job ran but result is not being used!" | |
| exit 1 | |
| fi | |
| echo "::group::Update volume mapping" | |
| .github/files/e2e-tests/map-plugins-for-e2e-env.sh | |
| echo "::endgroup::" | |
| else | |
| echo "Using artifact from build-projects job" | |
| fi | |
| - name: Test environment set-up | |
| working-directory: ${{ matrix.path }} | |
| env: | |
| CONFIG_KEY: ${{ secrets.E2E_CONFIG_KEY }} | |
| PROJECT_NAME: ${{ matrix.project }} WP ${{ matrix.wpVersion }} | |
| HOST_CWD: ${{ github.workspace }} | |
| WP_VERSION: ${{ matrix.wpVersion }} | |
| run: | | |
| echo "::group::Decrypt config" | |
| pnpm run config:decrypt | |
| echo "::endgroup::" | |
| # Ensure that wp debug.log exists and the web process can write to it | |
| mkdir -p "$(dirname "$WP_DEBUG_PATH")" | |
| touch "$WP_DEBUG_PATH" | |
| chmod 0666 "$WP_DEBUG_PATH" | |
| echo "::group::Start docker environment" | |
| pnpm run env:up | |
| echo "::endgroup::" | |
| echo "::group::Create tunnel" | |
| nohup pnpm run tunnel:up "$HOME/tunnel.log" | |
| echo "::endgroup::" | |
| - name: Run ${{ matrix.project }} tests | |
| working-directory: ${{ matrix.path }} | |
| env: | |
| JSON_ARGS: ${{ toJSON(matrix.testArgs) }} | |
| run: | | |
| mapfile -t TEST_ARGS < <(jq -r '.[]' <<<"$JSON_ARGS") | |
| pnpm run test:run "${TEST_ARGS[@]}" | |
| - name: Write report metadata | |
| if: ${{ always() }} | |
| env: | |
| SUITE: ${{ matrix.suite }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| OUTPUT_PATH: ${{ matrix.path }} | |
| run: .github/files/e2e-tests/report-metadata.sh | |
| - name: Test environment tear-down | |
| if: ${{ always() }} | |
| working-directory: ${{ matrix.path }} | |
| continue-on-error: true | |
| run: | | |
| # Copy debug.log to output folder, if it exists | |
| [[ ! -f "$WP_DEBUG_PATH" ]] || { mkdir -p "output/logs" && cp "$WP_DEBUG_PATH" "output/logs/php_errors.log"; } | |
| pnpm run tunnel:down | |
| echo "::group::Tunnel logs" | |
| cat "$HOME/tunnel.log" | |
| echo "::endgroup::" | |
| # Update permissions to workaround https://github.com/actions/cache/issues/753 | |
| sudo chown -R runner:docker "$GITHUB_WORKSPACE/tools/docker" | |
| - name: Upload test artifacts | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| # The WordPress version is part of the name: without it the two runs of a project would | |
| # race for the same artifact and upload-artifact would fail on the duplicate. | |
| name: test-output-${{ matrix.project }}-wp-${{ matrix.wpVersion }} | |
| path: ${{ matrix.path }}/output | |
| include-hidden-files: true | |
| test-report: | |
| name: "Test report" | |
| runs-on: ubuntu-latest | |
| if: ${{ always() && needs.e2e-tests.result != 'cancelled' && github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name }} | |
| needs: [e2e-tests] | |
| steps: | |
| - name: Set report context | |
| id: report-context | |
| env: | |
| DISPATCH_REF_NAME: ${{ github.event.client_payload.ref_name }} | |
| DISPATCH_REPO: ${{ github.event.client_payload.repository }} | |
| PR_TITLE: ${{ github.event.pull_request.title }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | |
| REPOSITORY=$GITHUB_REPOSITORY | |
| if [ "$GITHUB_EVENT_NAME" == pull_request ]; then | |
| BRANCH=$GITHUB_HEAD_REF | |
| elif [ "$GITHUB_EVENT_NAME" == repository_dispatch ]; then | |
| BRANCH=$DISPATCH_REF_NAME | |
| REPOSITORY=$DISPATCH_REPO | |
| else | |
| BRANCH=${GITHUB_REF:11} | |
| fi | |
| echo "EVENT_TYPE=e2e run $GITHUB_RUN_ID at $GITHUB_EVENT_NAME on $GITHUB_REPOSITORY" >> "$GITHUB_OUTPUT" | |
| echo "CLIENT_PAYLOAD=$( jq -nrc --arg repository "$REPOSITORY" --arg branch "$BRANCH" --arg pr_title "$PR_TITLE" --arg pr_number "$PR_NUMBER" --arg run_id "$GITHUB_RUN_ID" '{ repository: $repository, branch: $branch, pr_title: $pr_title, pr_number: $pr_number, run_id: $run_id }' )" >> "$GITHUB_OUTPUT" | |
| - name: Trigger test report workflow | |
| uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 | |
| with: | |
| token: ${{ secrets.E2E_TEST_REPORTS_TOKEN }} | |
| repository: automattic/jetpack-e2e-reports | |
| event-type: ${{ steps.report-context.outputs.event_type }} | |
| client-payload: ${{ steps.report-context.outputs.client_payload }} | |
| - name: Display link to test report in Action Summary | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 | |
| with: | |
| script: | | |
| await core.summary | |
| .addHeading( 'E2E Test Report' ) | |
| .addLink( | |
| `E2E Test Report`, | |
| `https://automattic.github.io/jetpack-e2e-reports/${ context.issue.number }/report/` | |
| ) | |
| .addBreak() | |
| .addRaw('Please note that it can take a few minutes after the e2e tests checks are complete for the report to be available.', true) | |
| .write() | |
| slack-notification: | |
| name: "Slack notification" | |
| runs-on: ubuntu-latest | |
| permissions: | |
| # ./projects/github-actions/test-results-to-slack | |
| actions: read | |
| # actions/checkout, actions/download-artifact | |
| contents: read | |
| # Only mirror-repo dispatches notify. PR feedback is already in the GitHub Checks UI, and a manual | |
| # dispatch has a human watching it who doesn't need the channel told. | |
| if: ${{ always() && needs.e2e-tests.result != 'cancelled' && github.event_name == 'repository_dispatch' }} | |
| needs: [e2e-tests] | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| - name: Set up tools | |
| uses: ./.github/actions/tool-setup | |
| - name: Build action | |
| env: | |
| COMPOSER_ROOT_VERSION: dev-trunk | |
| run: | | |
| pnpm install | |
| composer install | |
| composer build-development | |
| working-directory: ./projects/github-actions/test-results-to-slack | |
| - name: Download test artifacts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| # Without this we'd also pull the build-cache tarballs, which this job has no use for. | |
| pattern: test-output-* | |
| path: test-output | |
| - name: Decrypt config | |
| env: | |
| CONFIG_KEY: ${{ secrets.E2E_CONFIG_KEY }} | |
| run: ./.github/files/e2e-tests/encryption.sh decrypt | |
| - name: Tag release candidate runs | |
| if: github.event_name == 'repository_dispatch' && github.event.client_payload.ref_type == 'tag' | |
| env: | |
| VERSION: ${{ github.event.client_payload.ref_name }} | |
| run: echo "NOTIFICATION_SUITE=rc-${VERSION}" >> $GITHUB_ENV | |
| - name: "Send notification" | |
| uses: ./projects/github-actions/test-results-to-slack | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| slack_token: ${{ secrets.SLACK_TOKEN }} | |
| slack_channel: ${{ vars.SLACK_E2E_CHANNEL }} | |
| slack_icon_emoji: ":jetpack:" | |
| suite_name: ${{ env.NOTIFICATION_SUITE }} | |
| playwright_report_path: test-output/**/summary.json | |
| playwright_output_dir: test-output/**/results | |
| rules_configuration_path: .github/files/e2e-tests/notification-rules.json | |