Fail CI retry chains on the job that actually failed (#59570)

Summary:

The E2E and Fantom retry chains reported their final failure through a separate `report` job. The `test` job always succeeded (its test step used `continue-on-error: true`), and `report` read its `status` output and exited 1 on the last attempt. This caused three problems:

- The red check was a five-second status job, not the job that ran the tests, so finding the real failure meant opening a green job and looking for the step with a warning icon.
- "Re-run failed jobs" re-ran only `report`. It read the previous attempt's cached `status` output and failed again without running any tests.
- The Debug and Release matrix legs wrote the same `status` output, and GitHub keeps only the last value written to a given output name. If one flavor failed and the other passed and finished later, `status` came out empty, so no retry ran and the failure was dropped silently.

This change:

- Removes the `report` jobs. The test step runs as two copies of the same step, gated on `inputs.fail-on-error`: earlier attempts use `continue-on-error: true`, so they stay green and record the failure in the `status` output, and the final attempt runs without it, so the test step itself fails the job. The two copies share their `with:` block through a YAML anchor. (A single step with `continue-on-error: ${{ !inputs.fail-on-error }}` doesn't work: GitHub evaluated it as true on the final attempt, so the failure stayed hidden. Step `if:` conditions read `inputs` correctly.)
- Has each matrix leg write a flavor-specific output (`debug_status` / `release_status`). GitHub combines outputs with different names across matrix legs. The reusable workflow's `status` output is `debug_status || release_status`, or `jobs.test.outputs.status` for the single-job Fantom workflow.
- Also triggers the retry jobs in `test-all.yml` on `needs.<attempt>.result == 'failure'`. The iOS `report` jobs counted a failed `build` or `test` job as a failure, and this keeps that behavior for every workflow.

The flavor is lowercased with `tr` rather than `${FLAVOR,,}` because the macOS runners' default `bash` may be 3.2. A flavor with no matching output fails the step with an error instead of writing an output nothing reads; a red `test` job still triggers the retry through `needs.<attempt>.result`.

Changelog: [Internal]

Differential Revision: D128043004

Author: GijsWeterings

.github/workflows/e2e-android-rntester.yml

@@ -14,14 +14,17 @@ on:
         default: 0
     outputs:
       status:
-        description: 'The result of the E2E tests (success or failure)'
-        value: ${{ jobs.report.outputs.status }}
+        description: "'failure' if the E2E tests failed, empty otherwise"
+        value: ${{ jobs.test.outputs.debug_status || jobs.test.outputs.release_status }}
 
 jobs:
   test:
     runs-on: 4-core-ubuntu
+    # Matrix legs write differently named outputs so they are combined
+    # instead of the last leg to finish overwriting the others.
     outputs:
-      status: ${{ steps.report-status.outputs.status }}
+      debug_status: ${{ steps.report-status.outputs.debug_status }}
+      release_status: ${{ steps.report-status.outputs.release_status }}
     strategy:
       fail-fast: false
       matrix:
@@ -59,11 +62,11 @@ jobs:
           echo "should-run=$SHOULD_RUN" >> "$GITHUB_OUTPUT"
       - name: Run E2E Tests
         id: run-tests
-        if: steps.test-state.outputs.should-run == 'true'
+        if: ${{ steps.test-state.outputs.should-run == 'true' && !inputs.fail-on-error }}
         continue-on-error: true
         uses: ./.github/actions/maestro-android
         timeout-minutes: 90
-        with:
+        with: &test-inputs
           app-path: ./packages/rn-tester/android/app/build/outputs/apk/${{ matrix.flavor }}/app-arm64-v8a-${{ matrix.flavor }}.apk
           app-id: com.facebook.react.uiapp
           maestro-flow: ./packages/rn-tester/.maestro
@@ -75,6 +78,14 @@ jobs:
           # Running the debug ARM64 APK through native translation can delay
           # React commits long enough for Maestro to miss taps and assertions.
           exclude-tags: ${{ matrix.flavor == 'debug' && 'android-release-only' || '' }}
+      - name: Run E2E Tests
+        # Same step without continue-on-error, so the final attempt fails
+        # here. A step's continue-on-error expression doesn't see inputs.
+        id: run-tests-final
+        if: ${{ steps.test-state.outputs.should-run == 'true' && inputs.fail-on-error }}
+        uses: ./.github/actions/maestro-android
+        timeout-minutes: 90
+        with: *test-inputs
       - name: Store per-flow test state
         if: always()
         uses: ./.github/actions/upload-artifact
@@ -85,23 +96,15 @@ jobs:
           path: /tmp/maestro-android-state/results.json
       - name: Report status
         id: report-status
-        if: ${{ always() && steps.run-tests.outcome == 'failure' }}
-        run: echo "status=failure" >> $GITHUB_OUTPUT
-
-  report:
-    needs: test
-    if: always()
-    runs-on: ubuntu-latest
-    outputs:
-      status: ${{ steps.check.outputs.status }}
-    steps:
-      - id: check
+        if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }}
+        env:
+          FLAVOR: ${{ matrix.flavor }}
         run: |
-          if [[ "${{ needs.test.outputs.status }}" == "failure" ]]; then
-            echo "status=failure" >> $GITHUB_OUTPUT
-          else
-            echo "status=success" >> $GITHUB_OUTPUT
-          fi
-      - name: Fail if needed
-        if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }}
-        run: exit 1
+          flavor=$(echo "$FLAVOR" | tr "[:upper:]" "[:lower:]")
+          case "$flavor" in
+            debug | release) echo "${flavor}_status=failure" >> "$GITHUB_OUTPUT" ;;
+            *)
+              echo "::error::No job output for flavor '$FLAVOR'. Add a ${flavor}_status output to this job and to the workflow status output."
+              exit 1
+              ;;
+          esac

.github/workflows/e2e-android-templateapp.yml

@@ -14,14 +14,17 @@ on:
         default: 0
     outputs:
       status:
-        description: 'The result of the E2E tests (success or failure)'
-        value: ${{ jobs.report.outputs.status }}
+        description: "'failure' if the E2E tests failed, empty otherwise"
+        value: ${{ jobs.test.outputs.debug_status || jobs.test.outputs.release_status }}
 
 jobs:
   test:
     runs-on: 4-core-ubuntu
+    # Matrix legs write differently named outputs so they are combined
+    # instead of the last leg to finish overwriting the others.
     outputs:
-      status: ${{ steps.report-status.outputs.status }}
+      debug_status: ${{ steps.report-status.outputs.debug_status }}
+      release_status: ${{ steps.report-status.outputs.release_status }}
     strategy:
       fail-fast: false
       matrix:
@@ -81,11 +84,11 @@ jobs:
           echo "should-run=$SHOULD_RUN" >> "$GITHUB_OUTPUT"
       - name: Run E2E Tests
         id: run-tests
-        if: steps.test-state.outputs.should-run == 'true'
+        if: ${{ steps.test-state.outputs.should-run == 'true' && !inputs.fail-on-error }}
         continue-on-error: true
         uses: ./.github/actions/maestro-android
         timeout-minutes: 90
-        with:
+        with: &test-inputs
           app-path: /tmp/android-templateapp/app-${{ matrix.flavor }}.apk
           app-id: com.rntestproject
           maestro-flow: ./scripts/e2e/.maestro/
@@ -94,6 +97,14 @@ jobs:
           emulator-api-level: '35'
           emulator-target: google_apis
           emulator-arch: x86_64
+      - name: Run E2E Tests
+        # Same step without continue-on-error, so the final attempt fails
+        # here. A step's continue-on-error expression doesn't see inputs.
+        id: run-tests-final
+        if: ${{ steps.test-state.outputs.should-run == 'true' && inputs.fail-on-error }}
+        uses: ./.github/actions/maestro-android
+        timeout-minutes: 90
+        with: *test-inputs
       - name: Store per-flow test state
         if: always()
         uses: ./.github/actions/upload-artifact
@@ -104,23 +115,15 @@ jobs:
           path: /tmp/maestro-android-state/results.json
       - name: Report status
         id: report-status
-        if: ${{ always() && steps.run-tests.outcome == 'failure' }}
-        run: echo "status=failure" >> $GITHUB_OUTPUT
-
-  report:
-    needs: test
-    if: always()
-    runs-on: ubuntu-latest
-    outputs:
-      status: ${{ steps.check.outputs.status }}
-    steps:
-      - id: check
+        if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }}
+        env:
+          FLAVOR: ${{ matrix.flavor }}
         run: |
-          if [[ "${{ needs.test.outputs.status }}" == "failure" ]]; then
-            echo "status=failure" >> $GITHUB_OUTPUT
-          else
-            echo "status=success" >> $GITHUB_OUTPUT
-          fi
-      - name: Fail if needed
-        if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }}
-        run: exit 1
+          flavor=$(echo "$FLAVOR" | tr "[:upper:]" "[:lower:]")
+          case "$flavor" in
+            debug | release) echo "${flavor}_status=failure" >> "$GITHUB_OUTPUT" ;;
+            *)
+              echo "::error::No job output for flavor '$FLAVOR'. Add a ${flavor}_status output to this job and to the workflow status output."
+              exit 1
+              ;;
+          esac

.github/workflows/e2e-ios-rntester.yml

@@ -11,14 +11,17 @@ on:
         default: false
     outputs:
       status:
-        description: 'The result of the E2E tests (success or failure)'
-        value: ${{ jobs.report.outputs.status }}
+        description: "'failure' if the E2E tests failed, empty otherwise"
+        value: ${{ jobs.test.outputs.debug_status || jobs.test.outputs.release_status }}
 
 jobs:
   test:
     runs-on: macos-26-large
+    # Matrix legs write differently named outputs so they are combined
+    # instead of the last leg to finish overwriting the others.
     outputs:
-      status: ${{ steps.report-status.outputs.status }}
+      debug_status: ${{ steps.report-status.outputs.debug_status }}
+      release_status: ${{ steps.report-status.outputs.release_status }}
     strategy:
       fail-fast: false
       matrix:
@@ -37,35 +40,34 @@ jobs:
         run: ls -lR /tmp/RNTesterBuild
       - name: Run E2E Tests
         id: run-tests
+        if: ${{ !inputs.fail-on-error }}
         continue-on-error: true
         uses: ./.github/actions/maestro-ios
-        with:
+        with: &test-inputs
           app-path: '/tmp/RNTesterBuild/RNTester.app'
           app-id: com.meta.RNTester.localDevelopment
           maestro-flow: ./packages/rn-tester/.maestro/
           flavor: ${{ matrix.flavor }}
           device-model: iPhone-17-Pro
           device-os: iOS-26-2
+      - name: Run E2E Tests
+        # Same step without continue-on-error, so the final attempt fails
+        # here. A step's continue-on-error expression doesn't see inputs.
+        id: run-tests-final
+        if: ${{ inputs.fail-on-error }}
+        uses: ./.github/actions/maestro-ios
+        with: *test-inputs
       - name: Report status
         id: report-status
-        if: ${{ always() && steps.run-tests.outcome == 'failure' }}
-        run: echo "status=failure" >> $GITHUB_OUTPUT
-
-  report:
-    needs: test
-    if: always()
-    runs-on: ubuntu-latest
-    outputs:
-      status: ${{ steps.check.outputs.status }}
-    steps:
-      - id: check
+        if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }}
+        env:
+          FLAVOR: ${{ matrix.flavor }}
         run: |
-          if [[ "${{ needs.test.result }}" != "success" ||
-                "${{ needs.test.outputs.status }}" == "failure" ]]; then
-            echo "status=failure" >> $GITHUB_OUTPUT
-          else
-            echo "status=success" >> $GITHUB_OUTPUT
-          fi
-      - name: Fail if needed
-        if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }}
-        run: exit 1
+          flavor=$(echo "$FLAVOR" | tr "[:upper:]" "[:lower:]")
+          case "$flavor" in
+            debug | release) echo "${flavor}_status=failure" >> "$GITHUB_OUTPUT" ;;
+            *)
+              echo "::error::No job output for flavor '$FLAVOR'. Add a ${flavor}_status output to this job and to the workflow status output."
+              exit 1
+              ;;
+          esac

.github/workflows/e2e-ios-templateapp.yml

@@ -11,8 +11,8 @@ on:
         default: false
     outputs:
       status:
-        description: 'The result of the E2E tests (success or failure)'
-        value: ${{ jobs.report.outputs.status }}
+        description: "'failure' if the E2E tests failed, empty otherwise"
+        value: ${{ jobs.test.outputs.debug_status || jobs.test.outputs.release_status }}
 
 jobs:
   build:
@@ -100,8 +100,11 @@ jobs:
   test:
     needs: build
     runs-on: macos-26-large
+    # Matrix legs write differently named outputs so they are combined
+    # instead of the last leg to finish overwriting the others.
     outputs:
-      status: ${{ steps.report-status.outputs.status }}
+      debug_status: ${{ steps.report-status.outputs.debug_status }}
+      release_status: ${{ steps.report-status.outputs.release_status }}
     strategy:
       fail-fast: false
       matrix:
@@ -149,35 +152,33 @@ jobs:
           node ./scripts/e2e/init-project-e2e.js --projectName RNTestProject --currentBranch $BRANCH --directory /tmp/RNTestProject --pathToLocalReactNative $REACT_NATIVE_PKG
       - name: Run E2E Tests
         id: run-tests
+        if: ${{ !inputs.fail-on-error }}
         continue-on-error: true
         uses: ./.github/actions/maestro-ios
-        with:
+        with: &test-inputs
           app-path: '/tmp/RNTestProjectBuild/RNTestProject.app'
           app-id: org.reactjs.native.example.RNTestProject
           maestro-flow: ./scripts/e2e/.maestro/
           flavor: ${{ matrix.flavor }}
           working-directory: /tmp/RNTestProject
+      - name: Run E2E Tests
+        # Same step without continue-on-error, so the final attempt fails
+        # here. A step's continue-on-error expression doesn't see inputs.
+        id: run-tests-final
+        if: ${{ inputs.fail-on-error }}
+        uses: ./.github/actions/maestro-ios
+        with: *test-inputs
       - name: Report status
         id: report-status
-        if: ${{ always() && steps.run-tests.outcome == 'failure' }}
-        run: echo "status=failure" >> $GITHUB_OUTPUT
-
-  report:
-    needs: [build, test]
-    if: always()
-    runs-on: ubuntu-latest
-    outputs:
-      status: ${{ steps.check.outputs.status }}
-    steps:
-      - id: check
+        if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }}
+        env:
+          FLAVOR: ${{ matrix.flavor }}
         run: |
-          if [[ "${{ needs.build.result }}" != "success" ||
-                "${{ needs.test.result }}" != "success" ||
-                "${{ needs.test.outputs.status }}" == "failure" ]]; then
-            echo "status=failure" >> $GITHUB_OUTPUT
-          else
-            echo "status=success" >> $GITHUB_OUTPUT
-          fi
-      - name: Fail if needed
-        if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }}
-        run: exit 1
+          flavor=$(echo "$FLAVOR" | tr "[:upper:]" "[:lower:]")
+          case "$flavor" in
+            debug | release) echo "${flavor}_status=failure" >> "$GITHUB_OUTPUT" ;;
+            *)
+              echo "::error::No job output for flavor '$FLAVOR'. Add a ${flavor}_status output to this job and to the workflow status output."
+              exit 1
+              ;;
+          esac

.github/workflows/fantom-tests.yml

@@ -11,8 +11,8 @@ on:
         default: false
     outputs:
       status:
-        description: 'The result of the Fantom tests (success or failure)'
-        value: ${{ jobs.report.outputs.status }}
+        description: "'failure' if the Fantom tests failed, empty otherwise"
+        value: ${{ jobs.test.outputs.status }}
 
 jobs:
   test:
@@ -28,27 +28,16 @@ jobs:
         uses: actions/checkout@v6
       - name: Run Fantom Tests
         id: run-tests
+        if: ${{ !inputs.fail-on-error }}
         continue-on-error: true
         uses: ./.github/actions/run-fantom-tests
+      - name: Run Fantom Tests
+        # Same step without continue-on-error, so the final attempt fails
+        # here. A step's continue-on-error expression doesn't see inputs.
+        id: run-tests-final
+        if: ${{ inputs.fail-on-error }}
+        uses: ./.github/actions/run-fantom-tests
       - name: Report status
         id: report-status
-        if: ${{ always() && steps.run-tests.outcome == 'failure' }}
+        if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }}
         run: echo "status=failure" >> $GITHUB_OUTPUT
-
-  report:
-    needs: test
-    if: always()
-    runs-on: ubuntu-latest
-    outputs:
-      status: ${{ steps.check.outputs.status }}
-    steps:
-      - id: check
-        run: |
-          if [[ "${{ needs.test.outputs.status }}" == "failure" ]]; then
-            echo "status=failure" >> $GITHUB_OUTPUT
-          else
-            echo "status=success" >> $GITHUB_OUTPUT
-          fi
-      - name: Fail if needed
-        if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }}
-        run: exit 1