Merge pull request #79 from loadingalias/fix/s390x-release-evidence

ci: regenerate exact-tag s390x release evidence

Author: loadingalias

.github/workflows/_rust-job.yaml

@@ -9,6 +9,10 @@ on:
       runner:
         required: true
         type: string
+      checkout_ref:
+        required: false
+        type: string
+        default: ""
       timeout_minutes:
         required: false
         type: number
@@ -119,6 +123,7 @@ jobs:
         with:
           fetch-depth: 0
           persist-credentials: false
+          ref: ${{ inputs.checkout_ref || github.sha }}
 
       - name: Setup
         uses: ./.github/actions/setup
@@ -132,6 +137,7 @@ jobs:
       - name: Run
         shell: bash
         env:
+          CARGO_TARGET_S390X_UNKNOWN_LINUX_GNU_RUSTFLAGS: ${{ inputs.target == 's390x-unknown-linux-gnu' && '-C target-feature=+vector' || '' }}
           RSCRYPTO_CI_OPERATION: ${{ inputs.operation }}
           RSCRYPTO_CI_RUNNER: ${{ inputs.runner }}
           RSCRYPTO_CI_TEST_MODE: ${{ inputs.test_mode }}

.github/workflows/ct.yaml

@@ -38,6 +38,11 @@ on:
         required: false
         type: number
         default: 90
+      release_tag:
+        description: "Existing release tag whose s390x CT lane must be regenerated."
+        required: false
+        type: string
+        default: ""
   workflow_dispatch:
     inputs:
       platforms:
@@ -79,6 +84,11 @@ on:
         required: false
         type: number
         default: 90
+      release_tag:
+        description: "Existing release tag whose s390x CT lane must be regenerated."
+        required: false
+        type: string
+        default: ""
 
 env:
   RUST_BACKTRACE: 1
@@ -94,15 +104,54 @@ jobs:
     name: Resolve CT Matrix
     runs-on: ubuntu-latest
     outputs:
+      checkout_ref: ${{ steps.source.outputs.checkout_ref }}
       has_targets: ${{ steps.plan.outputs.has_targets }}
       matrix: ${{ steps.plan.outputs.matrix }}
     steps:
       - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
-          fetch-depth: 1
+          fetch-depth: 0
           persist-credentials: false
 
+      - name: Resolve CT source
+        id: source
+        shell: bash
+        env:
+          ARTIFACT_RETENTION_DAYS: ${{ inputs.artifact_retention_days }}
+          BINSEC_TIMEOUT: ${{ inputs.binsec_timeout }}
+          DUDECT_FILTER: ${{ inputs.dudect_filter }}
+          DUDECT_GATE: ${{ inputs.dudect_gate }}
+          DUDECT_TIMEOUT: ${{ inputs.dudect_timeout }}
+          EVENT_NAME: ${{ github.event_name }}
+          PLATFORMS: ${{ inputs.platforms }}
+          RELEASE_TAG: ${{ inputs.release_tag }}
+          UPLOAD_RAW_ARTIFACTS: ${{ inputs.upload_raw_artifacts }}
+        run: |
+          set -euo pipefail
+          if [[ -z "$RELEASE_TAG" ]]; then
+            echo "checkout_ref=" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          if [[ "$EVENT_NAME" != "workflow_dispatch" || "$GITHUB_REF" != "refs/heads/main" ]]; then
+            echo "release CT recovery must run from protected main" >&2
+            exit 1
+          fi
+          if [[ ! "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.-]+)?$ ]]; then
+            echo "release CT recovery tag is invalid: $RELEASE_TAG" >&2
+            exit 1
+          fi
+          git rev-parse -q --verify "${RELEASE_TAG}^{tag}" >/dev/null \
+            || { echo "release CT recovery requires an annotated tag" >&2; exit 1; }
+          if [[ "$PLATFORMS" != "ibm-s390x" || "$DUDECT_GATE" != "required" \
+            || -n "$DUDECT_FILTER" || "$UPLOAD_RAW_ARTIFACTS" != "true" \
+            || "$ARTIFACT_RETENTION_DAYS" != "90" || "$DUDECT_TIMEOUT" != "1800" \
+            || "$BINSEC_TIMEOUT" != "900" ]]; then
+            echo "release CT recovery must run the complete raw ibm-s390x lane with release timeouts and 90-day retention" >&2
+            exit 1
+          fi
+          echo "checkout_ref=$RELEASE_TAG" >> "$GITHUB_OUTPUT"
+
       - name: Build CT Matrix
         id: plan
         shell: bash
@@ -121,6 +170,7 @@ jobs:
         include: ${{ fromJSON(needs.plan.outputs.matrix) }}
     uses: ./.github/workflows/_rust-job.yaml
     with:
+      checkout_ref: ${{ needs.plan.outputs.checkout_ref }}
       runner: ${{ matrix.runner }}
       timeout_minutes: ${{ matrix.timeout_minutes }}
       cache_key: ct-${{ matrix.artifact_suffix }}

.github/workflows/release.yaml

@@ -10,6 +10,10 @@ on:
         description: Existing signed release tag to recover
         required: true
         type: string
+      s390x_ct_run:
+        description: Optional reviewed exact-tag s390x CT recovery run ID
+        required: false
+        type: string
 
 env:
   RUST_BACKTRACE: 1
@@ -46,6 +50,8 @@ jobs:
       weekly_commit: ${{ steps.evidence.outputs.weekly_commit }}
       weekly_version: ${{ steps.evidence.outputs.weekly_version }}
       weekly_evidence_mode: ${{ steps.evidence.outputs.weekly_evidence_mode }}
+      s390x_ct_run_id: ${{ steps.s390x_ct.outputs.s390x_run_id }}
+      s390x_ct_run_url: ${{ steps.s390x_ct.outputs.s390x_run_url }}
     steps:
       - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
@@ -101,15 +107,6 @@ jobs:
           RELEASE_COMMIT: ${{ steps.identity.outputs.release_commit }}
         run: scripts/ci/release-evidence-check.sh --commit "$RELEASE_COMMIT"
 
-      - name: Setup
-        uses: ./.github/actions/setup
-        with:
-          cache-key: release-preflight
-          tools-mode: release
-          toolchain-components: ""
-          enable-magic-cache: false
-          enable-rust-cache: true
-
       - name: Checkout reviewed recovery tooling
         if: github.event_name == 'workflow_dispatch'
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
@@ -119,6 +116,27 @@ jobs:
           persist-credentials: false
           ref: ${{ github.sha }}
 
+      - name: Verify s390x CT recovery evidence
+        if: github.event_name == 'workflow_dispatch' && inputs.s390x_ct_run != ''
+        id: s390x_ct
+        env:
+          GH_TOKEN: ${{ github.token }}
+          S390X_CT_RUN: ${{ inputs.s390x_ct_run }}
+          WORKFLOW_COMMIT: ${{ github.sha }}
+        run: |
+          target/release-automation/scripts/ci/release-ct-recovery-check.sh \
+            --run-id "$S390X_CT_RUN" \
+            --workflow-commit "$WORKFLOW_COMMIT"
+
+      - name: Setup
+        uses: ./.github/actions/setup
+        with:
+          cache-key: release-preflight
+          tools-mode: release
+          toolchain-components: ""
+          enable-magic-cache: false
+          enable-rust-cache: true
+
       - name: Install recovery SemVer checker
         if: github.event_name == 'workflow_dispatch'
         run: |
@@ -250,6 +268,19 @@ jobs:
           github-token: ${{ github.token }}
           run-id: ${{ needs.preflight.outputs.weekly_run_id }}
 
+      - name: Remove replaced s390x CT evidence
+        if: needs.preflight.outputs.s390x_ct_run_id != ''
+        run: rm -rf ct-release-artifacts/ct-raw-ibm-s390x
+
+      - name: Download recovered s390x CT evidence
+        if: needs.preflight.outputs.s390x_ct_run_id != ''
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          name: ct-raw-ibm-s390x
+          path: ct-release-artifacts/ct-raw-ibm-s390x
+          github-token: ${{ github.token }}
+          run-id: ${{ needs.preflight.outputs.s390x_ct_run_id }}
+
       - name: Download RISC-V CT evidence
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
         with:
@@ -371,6 +402,7 @@ jobs:
           CT_EVIDENCE_MODE: ${{ needs.preflight.outputs.weekly_evidence_mode }}
           CT_EVIDENCE_WEEKLY_RUN: ${{ needs.preflight.outputs.weekly_run_url }}
           CT_EVIDENCE_RISCV_RUN: ${{ needs.preflight.outputs.riscv_run_url }}
+          CT_EVIDENCE_S390X_RUN: ${{ needs.preflight.outputs.s390x_ct_run_url }}
           REPOSITORY_CONTROLS_NAME: ${{ steps.repository_controls.outputs.evidence_name }}
           REPOSITORY_CONTROLS_PATH: ${{ steps.repository_controls.outputs.evidence_path }}
           REPOSITORY_CONTROLS_SHA256: ${{ steps.repository_controls.outputs.evidence_sha256 }}
@@ -383,6 +415,10 @@ jobs:
         run: |
           set -euo pipefail
           notes_file="$(mktemp)"
+          s390x_recovery_note=""
+          if [[ -n "$CT_EVIDENCE_S390X_RUN" ]]; then
+            s390x_recovery_note="s390x recovery evidence run: $CT_EVIDENCE_S390X_RUN"
+          fi
           cat > "$notes_file" <<EOF
           Release tag: \`${RELEASE_TAG}\`
           Release commit: \`${RELEASE_COMMIT}\`
@@ -401,6 +437,7 @@ jobs:
           Evidence commit: \`${CT_EVIDENCE_COMMIT}\` (\`${CT_EVIDENCE_MODE}\`)
           Weekly evidence run: ${CT_EVIDENCE_WEEKLY_RUN}
           RISC-V evidence run: ${CT_EVIDENCE_RISCV_RUN}
+          ${s390x_recovery_note}
 
           Repository controls: \`${REPOSITORY_CONTROLS_NAME}\`
           SHA-256: \`${REPOSITORY_CONTROLS_SHA256}\`

docs/release.md

@@ -212,6 +212,29 @@ reviewed recovery path from `main`:
 gh workflow run release.yaml --ref main -f tag=vX.Y.Z
 ```
 
+If release packaging rejects only the s390x CT artifact, regenerate that lane
+against the existing tag before dispatching recovery:
+
+```bash
+gh workflow run ct.yaml --ref main \
+  -f platforms=ibm-s390x \
+  -f dudect_gate=required \
+  -f upload_raw_artifacts=true \
+  -f artifact_retention_days=90 \
+  -f release_tag=vX.Y.Z
+
+gh workflow run release.yaml --ref main \
+  -f tag=vX.Y.Z \
+  -f s390x_ct_run=RUN_ID
+```
+
+Run both dispatches from the same reviewed `main` commit. The CT recovery is
+limited to the complete native s390x lane and checks out the immutable tag;
+the release preflight rejects any replacement run from another workflow,
+branch, repository, or commit. The normal release evidence packager then
+validates the replacement artifact's tag commit, crate version, cases, hashes,
+and target provenance before publication.
+
 Recovery checks out the existing annotated tag, verifies its allowed signature,
 and binds the package, evidence, release manifest, and release notes to the
 tag's commit rather than the newer workflow commit. Confirm that Preflight

justfile

@@ -235,6 +235,7 @@ check-actions:
     @scripts/ci/check-worktree-test.sh
     @scripts/ci/pre-push-test.sh
     @scripts/ci/release-evidence-check-test.sh
+    @scripts/ci/release-ct-recovery-check-test.sh
     @scripts/ci/repository-controls-evidence-test.sh
     @scripts/ci/release-identity-test.sh
     @scripts/ci/publish-immutable-release-test.sh

scripts/README.md

@@ -42,6 +42,8 @@ the recipes reported by `just --list`.
 | `ci/pre-push-test.sh`          | `just check-actions` |
 | `ci/release-evidence-check.sh` | `just release-tag`, `release.yaml`, `ci/release-evidence-check-test.sh` |
 | `ci/release-evidence-check-test.sh` | `just check-actions` |
+| `ci/release-ct-recovery-check.sh` | `release.yaml`, `ci/release-ct-recovery-check-test.sh` |
+| `ci/release-ct-recovery-check-test.sh` | `just check-actions` |
 | `ci/repository-controls-evidence.sh` | `just release-tag`, `release.yaml`, `ci/repository-controls-evidence-test.sh` |
 | `ci/repository-controls-evidence-test.sh` | `just check-actions` |
 | `ci/package-release-source.sh` | `release.yaml`, `ci/release-identity-test.sh` |

scripts/ci/check-ci-ownership-test.sh

@@ -54,6 +54,36 @@ sed -i.bak 's#refs/heads/main#refs/heads/recovery#' \
 rm -f "$unprotected_recovery/.github/workflows/release.yaml.bak"
 expect_failure "$unprotected_recovery" "release recovery accepts unprotected workflow code"
 
+missing_ct_recovery_tag="$TMP_ROOT/missing-ct-recovery-tag"
+make_fixture "$missing_ct_recovery_tag"
+yq eval 'del(.on.workflow_dispatch.inputs.release_tag)' -i \
+  "$missing_ct_recovery_tag/.github/workflows/ct.yaml"
+expect_failure "$missing_ct_recovery_tag" "CT recovery has no immutable tag identity"
+
+unprotected_ct_recovery="$TMP_ROOT/unprotected-ct-recovery"
+make_fixture "$unprotected_ct_recovery"
+yq eval '(.jobs.plan.steps[] | select(.name == "Resolve CT source") | .run) |= sub("refs/heads/main"; "refs/heads/recovery")' -i \
+  "$unprotected_ct_recovery/.github/workflows/ct.yaml"
+expect_failure "$unprotected_ct_recovery" "CT recovery accepts unprotected workflow code"
+
+mutable_ct_recovery_checkout="$TMP_ROOT/mutable-ct-recovery-checkout"
+make_fixture "$mutable_ct_recovery_checkout"
+yq eval '(.jobs.ct.with.checkout_ref) = "${{ github.sha }}"' -i \
+  "$mutable_ct_recovery_checkout/.github/workflows/ct.yaml"
+expect_failure "$mutable_ct_recovery_checkout" "CT recovery ignores the immutable tag source"
+
+missing_s390x_vector_environment="$TMP_ROOT/missing-s390x-vector-environment"
+make_fixture "$missing_s390x_vector_environment"
+yq eval 'del(.jobs.run.steps[] | select(.name == "Run") | .env.CARGO_TARGET_S390X_UNKNOWN_LINUX_GNU_RUSTFLAGS)' -i \
+  "$missing_s390x_vector_environment/.github/workflows/_rust-job.yaml"
+expect_failure "$missing_s390x_vector_environment" "s390x sibling CT processes lose the vector target environment"
+
+unvalidated_s390x_recovery="$TMP_ROOT/unvalidated-s390x-recovery"
+make_fixture "$unvalidated_s390x_recovery"
+yq eval '(.jobs.preflight.steps[] | select(.name == "Verify s390x CT recovery evidence") | .run) = "true"' -i \
+  "$unvalidated_s390x_recovery/.github/workflows/release.yaml"
+expect_failure "$unvalidated_s390x_recovery" "release accepts unvalidated replacement s390x evidence"
+
 mutable_publish_checkout="$TMP_ROOT/mutable-publish-checkout"
 make_fixture "$mutable_publish_checkout"
 yq eval '(.jobs.publish.steps[] | select(.name == "Checkout") | .with.ref) = "${{ github.ref }}"' -i \

scripts/ci/check-ci-ownership.sh

@@ -48,6 +48,8 @@ CI_CHECK="$ROOT/scripts/ci/ci-check.sh"
 RUN_RUST_JOB="$ROOT/scripts/ci/run-rust-job.sh"
 RELEASE_PREFLIGHT="$ROOT/scripts/ci/release-preflight.sh"
 RELEASE_EVIDENCE="$ROOT/scripts/ci/release-evidence-check.sh"
+RELEASE_CT_RECOVERY="$ROOT/scripts/ci/release-ct-recovery-check.sh"
+RELEASE_CT_RECOVERY_TEST="$ROOT/scripts/ci/release-ct-recovery-check-test.sh"
 RELEASE_SOURCE="$ROOT/scripts/ci/package-release-source.sh"
 RELEASE_MANIFEST="$ROOT/scripts/ci/write-release-manifest.sh"
 RELEASE_IDENTITY_TEST="$ROOT/scripts/ci/release-identity-test.sh"
@@ -146,6 +148,8 @@ require_file "$CI_CHECK"
 require_file "$RUN_RUST_JOB"
 require_file "$RELEASE_PREFLIGHT"
 require_file "$RELEASE_EVIDENCE"
+require_file "$RELEASE_CT_RECOVERY"
+require_file "$RELEASE_CT_RECOVERY_TEST"
 require_file "$RELEASE_SOURCE"
 require_file "$RELEASE_MANIFEST"
 require_file "$RELEASE_IDENTITY_TEST"
@@ -234,6 +238,16 @@ done < <(
   || fail "the reusable Rust job operation must be required"
 [[ $(yq eval '.on.workflow_call.inputs.operation.type' "$RUST_JOB") == "string" ]] \
   || fail "the reusable Rust job operation must be typed as a string"
+[[ $(yq eval '.on.workflow_call.inputs.checkout_ref.type' "$RUST_JOB") == "string" ]] \
+  || fail "the reusable Rust job checkout ref must be typed as a string"
+# shellcheck disable=SC2016 # GitHub expression is an intentional literal workflow contract.
+[[ $(yq eval '.jobs.run.steps[] | select(.name == "Checkout") | .with.ref' "$RUST_JOB") \
+  == '${{ inputs.checkout_ref || github.sha }}' ]] \
+  || fail "the reusable Rust job must bind an explicit source ref before execution"
+# shellcheck disable=SC2016 # GitHub expression is an intentional literal workflow contract.
+[[ $(yq eval '.jobs.run.steps[] | select(.name == "Run") | .env.CARGO_TARGET_S390X_UNKNOWN_LINUX_GNU_RUSTFLAGS' "$RUST_JOB") \
+  == '${{ inputs.target == '\''s390x-unknown-linux-gnu'\'' && '\''-C target-feature=+vector'\'' || '\'''\'' }}' ]] \
+  || fail "s390x CT jobs must share one explicit vector target environment"
 [[ $(yq eval '[.jobs.run.steps[] | select(has("run"))] | length' "$RUST_JOB") -eq 1 ]] \
   || fail "the reusable Rust job must expose one fixed command step"
 [[ $(yq eval '.jobs.run.steps[] | select(has("run")) | .run' "$RUST_JOB") == "scripts/ci/run-rust-job.sh" ]] \
@@ -413,6 +427,8 @@ grep -Eq 'HOST_ARGS\+=\(--feature-matrix\)' "$CHECK_ALL" \
   || fail "release recovery must require an explicit existing tag"
 [[ $(yq eval '.on.workflow_dispatch.inputs.tag.type' "$RELEASE") == "string" ]] \
   || fail "release recovery tag input must be a string"
+[[ $(yq eval '.on.workflow_dispatch.inputs.s390x_ct_run.type' "$RELEASE") == "string" ]] \
+  || fail "release recovery s390x CT run input must be a string"
 # shellcheck disable=SC2016 # GitHub expressions are intentional literal workflow contracts.
 [[ $(yq eval '.jobs.preflight.steps[] | select(.name == "Checkout") | .with.ref' "$RELEASE") \
   == '${{ github.event_name == '\''workflow_dispatch'\'' && inputs.tag || github.ref }}' ]] \
@@ -427,6 +443,11 @@ grep -Fq 'refs/heads/main' <<<"$identity_step" \
 recovery_tool_step=$(yq eval '.jobs.preflight.steps[] | select(.name == "Install recovery SemVer checker") | .run' "$RELEASE")
 grep -Fq 'install-tools.sh" semver' <<<"$recovery_tool_step" \
   || fail "release recovery must use the authenticated SemVer tool installer"
+ct_recovery_step=$(yq eval '.jobs.preflight.steps[] | select(.name == "Verify s390x CT recovery evidence") | .run' "$RELEASE")
+grep -Fq 'release-ct-recovery-check.sh' <<<"$ct_recovery_step" \
+  || fail "release recovery must validate replacement s390x CT evidence"
+grep -Fq -- '--workflow-commit "$WORKFLOW_COMMIT"' <<<"$ct_recovery_step" \
+  || fail "replacement s390x CT evidence must come from the reviewed workflow commit"
 # shellcheck disable=SC2016 # `$crate` is an intentional literal in the release-preflight contract regex.
 [[ $(count_matches 'cargo semver-checks --package "\$crate" --all-features' "$RELEASE_PREFLIGHT") -eq 1 ]] \
   || fail "tag preflight must have exactly one final-version SemVer owner"
@@ -626,6 +647,28 @@ grep -Fq 'run-id: ${{ needs.preflight.outputs.weekly_run_id }}' "$RELEASE" \
   || fail "release must consume non-RISC-V CT artifacts from the validated Weekly run"
 grep -Fq 'run-id: ${{ needs.preflight.outputs.riscv_run_id }}' "$RELEASE" \
   || fail "release must consume RISC-V CT artifacts from the validated RISC-V run"
+grep -Fq 'run-id: ${{ needs.preflight.outputs.s390x_ct_run_id }}' "$RELEASE" \
+  || fail "release must consume recovered s390x CT artifacts from the validated recovery run"
+[[ $(yq eval '.on.workflow_dispatch.inputs.release_tag.type' "$CT") == "string" ]] \
+  || fail "CT recovery release tag input must be a string"
+ct_source_step=$(yq eval '.jobs.plan.steps[] | select(.name == "Resolve CT source") | .run' "$CT")
+grep -Fq 'refs/heads/main' <<<"$ct_source_step" \
+  || fail "release CT recovery must reject workflow code outside protected main"
+grep -Fq 'checkout_ref=$RELEASE_TAG' <<<"$ct_source_step" \
+  || fail "release CT recovery must bind execution to the immutable tag"
+grep -Fq 'PLATFORMS" != "ibm-s390x' <<<"$ct_source_step" \
+  || fail "release CT recovery must be limited to the s390x lane"
+grep -Fq 'DUDECT_TIMEOUT" != "1800"' <<<"$ct_source_step" \
+  || fail "release CT recovery must preserve the release DudeCT timeout"
+grep -Fq 'BINSEC_TIMEOUT" != "900"' <<<"$ct_source_step" \
+  || fail "release CT recovery must preserve the release BINSEC timeout"
+grep -Fq 'UPLOAD_RAW_ARTIFACTS" != "true"' <<<"$ct_source_step" \
+  || fail "release CT recovery must retain raw evidence"
+grep -Fq 'ARTIFACT_RETENTION_DAYS" != "90"' <<<"$ct_source_step" \
+  || fail "release CT recovery must retain evidence for the release lifetime"
+# shellcheck disable=SC2016 # GitHub expression is an intentional literal workflow contract.
+[[ $(yq eval '.jobs.ct.with.checkout_ref' "$CT") == '${{ needs.plan.outputs.checkout_ref }}' ]] \
+  || fail "release CT recovery must execute the resolved immutable tag source"
 if grep -Eq 'uses: ./\.github/workflows/(ct|rsa)\.yaml' "$RELEASE"; then
   fail "tag workflow must promote exact-commit evidence instead of rerunning CT or RSA"
 fi