diff --git a/.github/workflows/check-dars.yml b/.github/workflows/check-dars.yml index bc6a75e9..069f340b 100644 --- a/.github/workflows/check-dars.yml +++ b/.github/workflows/check-dars.yml @@ -9,23 +9,24 @@ on: jobs: check-dars: runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read steps: - name: Checkout repository - uses: actions/checkout@v7 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: lfs: true + persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: '20' - - name: Upgrade npm - run: npm install -g npm@11.10.0 - - name: Install npm dependencies - run: npm install + run: npm install --ignore-scripts --no-audit --no-fund - name: Verify DAR integrity run: npm run verify-dars diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39e908b5..944ce32d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ jobs: daml-build: runs-on: ubuntu-latest permissions: - contents: write + contents: read steps: - name: Checkout repository @@ -15,8 +15,7 @@ jobs: with: submodules: false lfs: true - # PAT_TOKEN (repo secret): checkout + later push use same creds so autofix commits trigger workflows. - token: ${{ secrets.PAT_TOKEN }} + persist-credentials: false - name: Setup Git LFS run: | @@ -43,14 +42,17 @@ jobs: - name: Check pinned dependencies run: npx --yes --package @hardlydifficult/ci-scripts@1.0.85 check-pinned-deps + - name: Test DAR version policy + run: npm run test:dar-version-policy + - name: Initialize submodules run: git submodule update --init --recursive - name: Run ESLint - run: npm run lint:fix + run: npm run lint - name: Run Prettier - run: npm run format:fix + run: npm run format - name: Restore Daml SDK cache uses: actions/cache@v5 @@ -81,30 +83,11 @@ jobs: - name: Test DAML run: npm run test - - name: Check for changes - id: check-changes + - name: Require committed generated artifacts run: | - if git diff --exit-code --quiet && git diff --cached --exit-code --quiet; then - echo "No changes detected from build" - echo "has_changes=false" >> $GITHUB_OUTPUT - else - echo "Changes detected from build" - echo "=== DIFF OF CHANGES ===" + if [ -n "$(git status --porcelain)" ]; then + echo "::error::Build or code generation changed the committed tree. Regenerate and commit the results locally." + git status --short git diff - echo "=== END DIFF ===" - echo "has_changes=true" >> $GITHUB_OUTPUT + exit 1 fi - - - name: Commit and push auto-fixes - if: steps.check-changes.outputs.has_changes == 'true' && github.event_name == 'push' - env: - PAT_TOKEN: ${{ secrets.PAT_TOKEN }} - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add . - git commit -m "ci: auto-fix lint and format changes" - git remote set-url origin "https://x-access-token:${PAT_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git push origin "HEAD:${GITHUB_REF_NAME}" - - # Same pattern as @hardlydifficult/typescript: one green run after auto-fix (tests already ran above on this tree). diff --git a/.github/workflows/dar-version-policy.yml b/.github/workflows/dar-version-policy.yml new file mode 100644 index 00000000..e54468cc --- /dev/null +++ b/.github/workflows/dar-version-policy.yml @@ -0,0 +1,210 @@ +name: DAR Version Policy + +on: + pull_request_target: + workflow_dispatch: + +concurrency: + group: dar-version-policy-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + dar-version-policy: + name: dar-version-policy + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + actions: read + contents: read + + steps: + # pull_request_target runs this workflow and every executable below from the trusted target commit. + - name: Checkout trusted target code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + # A stacked PR's base can itself be untrusted. Always execute the repository default branch. + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 + lfs: true + submodules: false + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: '20' + + - name: Install trusted dependencies + run: npm install --ignore-scripts --no-audit --no-fund + + - name: Restore Daml SDK cache + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ~/.dpm + key: dpm-sdk-${{ runner.os }}-${{ hashFiles('**/daml.yaml', 'scripts/install-dpm-sdks.sh') }} + restore-keys: | + dpm-sdk-${{ runner.os }}- + + - name: Install trusted DAML SDK + run: scripts/install-dpm-sdks.sh + + - name: Materialize pull request as data only + if: github.event_name == 'pull_request_target' + id: candidate + env: + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + git fetch --no-tags origin "refs/pull/${PR_NUMBER}/head" + if [ "$(git rev-parse FETCH_HEAD)" != "$PR_HEAD_SHA" ]; then + echo "::error::Fetched PR head does not match the event SHA." + exit 1 + fi + + lfs_include=$(npx tsx scripts/validate-pr-dar-lfs.ts --commit "$PR_HEAD_SHA") + if [ -n "$lfs_include" ]; then + git lfs fetch origin "$PR_HEAD_SHA" --include="$lfs_include" --exclude='' + fi + candidate_root="$RUNNER_TEMP/pr-candidate" + GIT_LFS_SKIP_SMUDGE=1 git -c core.hooksPath=/dev/null worktree add --detach "$candidate_root" "$PR_HEAD_SHA" + dar_paths=() + if [ -n "$lfs_include" ]; then + IFS=',' read -r -a dar_paths <<< "$lfs_include" + fi + npx tsx scripts/validate-candidate-dar-paths.ts --root "$candidate_root" -- "${dar_paths[@]}" + if [ "${#dar_paths[@]}" -gt 0 ]; then + git -C "$candidate_root" lfs checkout "${dar_paths[@]}" + fi + npx tsx scripts/validate-candidate-dar-paths.ts --root "$candidate_root" -- "${dar_paths[@]}" + echo "candidate_root=$candidate_root" >> "$GITHUB_OUTPUT" + + - name: Detect whether Mainnet attestation is required + if: github.event_name == 'pull_request_target' + id: mainnet_requirement + env: + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + npx tsx scripts/mainnet-marker-attestation.ts needs-attestation \ + --repository-root "$GITHUB_WORKSPACE" \ + --base-head HEAD \ + --candidate-head "$PR_HEAD_SHA" \ + > "$RUNNER_TEMP/mainnet-marker-requirement.json" + required=$(jq -er '.required' "$RUNNER_TEMP/mainnet-marker-requirement.json") + echo "required=$required" >> "$GITHUB_OUTPUT" + + - name: Verify immutable Mainnet marker attestation + if: >- + steps.mainnet_requirement.outputs.required == 'true' && + github.event.pull_request.head.repo.full_name == github.repository && + startsWith(github.event.pull_request.head.ref, 'automation/ocp-release-') + id: mainnet_attestation + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_TOKEN: ${{ github.token }} + PR_HEAD_BRANCH: ${{ github.event.pull_request.head.ref }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + attestation_dir="$RUNNER_TEMP/mainnet-marker-attestation" + mkdir -m 700 "$attestation_dir" + + npx tsx scripts/mainnet-marker-attestation.ts parse-branch \ + --branch "$PR_HEAD_BRANCH" > "$attestation_dir/branch.json" + run_id=$(jq -er '.runId' "$attestation_dir/branch.json") + run_attempt=$(jq -er '.runAttempt' "$attestation_dir/branch.json") + package=$(jq -er '.package' "$attestation_dir/branch.json") + + gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}" \ + > "$attestation_dir/run.json" + # The marker branch is pushed before this fallible artifact upload so deployment evidence is never stranded. + # A synchronize check can therefore start a few seconds before upload-artifact finishes; retry only that + # bounded publication window, then fail closed on a missing, expired, or ambiguous artifact. + selection_ready=false + for attempt in {1..12}; do + gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/artifacts?per_page=100" \ + > "$attestation_dir/artifacts.json" + if npx tsx scripts/mainnet-marker-attestation.ts inspect \ + --repository "$GITHUB_REPOSITORY" \ + --default-branch "$DEFAULT_BRANCH" \ + --workflow-path ".github/workflows/release.yml" \ + --branch "$PR_HEAD_BRANCH" \ + --workflow-run-json "$attestation_dir/run.json" \ + --artifacts-json "$attestation_dir/artifacts.json" \ + > "$attestation_dir/selection.json" 2> "$attestation_dir/selection-error.txt" + then + selection_ready=true + break + fi + if (( attempt < 12 )); then + sleep 5 + fi + done + if [ "$selection_ready" != true ]; then + cat "$attestation_dir/selection-error.txt" >&2 + exit 1 + fi + artifact_id=$(jq -er '.artifactId' "$attestation_dir/selection.json") + + gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}/zip" \ + > "$attestation_dir/artifact.zip" + archive_bytes=$(stat -c '%s' "$attestation_dir/artifact.zip") + if (( archive_bytes < 1 || archive_bytes > 1048576 )); then + echo "::error::Mainnet attestation archive has an unsafe size." + exit 1 + fi + mapfile -t archive_entries < <(unzip -Z1 "$attestation_dir/artifact.zip") + if (( ${#archive_entries[@]} != 1 )) || [[ "${archive_entries[0]}" != "mainnet-marker-attestation.json" ]]; then + echo "::error::Mainnet attestation archive must contain only mainnet-marker-attestation.json." + exit 1 + fi + unzip -p "$attestation_dir/artifact.zip" mainnet-marker-attestation.json \ + > "$attestation_dir/mainnet-marker-attestation.json" + artifact_bytes=$(stat -c '%s' "$attestation_dir/mainnet-marker-attestation.json") + if (( artifact_bytes < 1 || artifact_bytes > 65536 )); then + echo "::error::Mainnet attestation JSON has an unsafe size." + exit 1 + fi + + npx tsx scripts/mainnet-marker-attestation.ts verify \ + --repository-root "$GITHUB_WORKSPACE" \ + --repository "$GITHUB_REPOSITORY" \ + --default-branch "$DEFAULT_BRANCH" \ + --workflow-path ".github/workflows/release.yml" \ + --branch "$PR_HEAD_BRANCH" \ + --candidate-head "$PR_HEAD_SHA" \ + --trusted-default-head "$(git rev-parse HEAD)" \ + --workflow-run-json "$attestation_dir/run.json" \ + --artifact-json "$attestation_dir/mainnet-marker-attestation.json" + + attested_lock_key=$(jq -er '.lockKey' "$attestation_dir/mainnet-marker-attestation.json") + echo "Verified run ${run_id}, attempt ${run_attempt}, package ${package}." + echo "verified=true" >> "$GITHUB_OUTPUT" + echo "lock_key=$attested_lock_key" >> "$GITHUB_OUTPUT" + + - name: Validate candidate against live DevNet + env: + CANDIDATE_ROOT: ${{ steps.candidate.outputs.candidate_root }} + CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET }} + CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET }} + MAINNET_ATTESTED_LOCK_KEY: ${{ steps.mainnet_attestation.outputs.lock_key }} + run: | + set -euo pipefail + if [ "$GITHUB_EVENT_NAME" = "pull_request_target" ]; then + args=(--base HEAD --candidate-root "$CANDIDATE_ROOT") + if [[ -n "$MAINNET_ATTESTED_LOCK_KEY" ]]; then + args+=(--authorized-mainnet-marker-lock-key "$MAINNET_ATTESTED_LOCK_KEY") + fi + npm run check:dar-version-policy -- "${args[@]}" + else + npm run check:dar-version-policy -- --base HEAD --audit-all + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 26430053..100f4be5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,9 @@ on: type: string concurrency: - group: package-release-${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || github.ref_name }} + # One mutable DevNet candidate exists per package line. This repository currently has one release package, so all + # releases share a lock and cannot race between DevNet preflight and marker persistence. + group: package-release-${{ github.repository }} cancel-in-progress: false jobs: @@ -20,34 +22,46 @@ jobs: name: Build, upload, and publish release runs-on: ubuntu-latest permissions: - # Required to push generated release artifacts back to main. + # Required to push generated release artifacts to a protected draft-PR branch. contents: write + pull-requests: write # Required if package publication is later routed through GitHub Packages. packages: write # Required for npm trusted publishing/provenance support. id-token: write env: RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || github.ref_name }} - CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET }} - CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET }} - CANTON_MAINNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_MAINNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET }} - CANTON_MAINNET_5N_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_MAINNET_5N_LEDGER_JSON_API_CLIENT_SECRET }} + RELEASE_ARTIFACT_BRANCH: automation/ocp-release-${{ github.run_id }}-${{ github.run_attempt }} steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: ref: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || github.ref }} fetch-depth: 0 - lfs: true - submodules: recursive + lfs: false + submodules: false persist-credentials: false + # Do not install dependencies or execute repository scripts until this immutable release ref is proven to be main. + - name: Validate release tag is current main + run: | + git remote set-url origin "https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git" + git fetch origin main + if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then + echo "::error::Release tag $RELEASE_TAG is not the current origin/main commit; refusing to run release side effects." + exit 1 + fi + git remote set-url origin "https://github.com/${{ github.repository }}.git" + - name: Setup Git LFS run: | git lfs install git lfs pull + - name: Initialize submodules + run: git submodule update --init --recursive + - name: Setup Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: @@ -68,11 +82,17 @@ jobs: - name: Validate release secrets env: + CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET }} + CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET }} + CANTON_MAINNET_5N_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_MAINNET_5N_LEDGER_JSON_API_CLIENT_SECRET }} + CANTON_MAINNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_MAINNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET }} NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + PAT_TOKEN: ${{ secrets.PAT_TOKEN }} run: | missing=() for name in \ NODE_AUTH_TOKEN \ + PAT_TOKEN \ CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET \ CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET \ CANTON_MAINNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET \ @@ -103,7 +123,7 @@ jobs: NODE - name: Restore Daml SDK cache - uses: actions/cache@v5 + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 with: path: ~/.dpm key: dpm-sdk-${{ runner.os }}-${{ hashFiles('**/daml.yaml', 'scripts/install-dpm-sdks.sh') }} @@ -122,19 +142,9 @@ jobs: exit 1 fi - - name: Validate release tag is current main - run: | - git remote set-url origin "https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git" - git fetch origin main - if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then - echo "::error::Release tag $RELEASE_TAG is not the current origin/main commit; refusing to run release side effects." - exit 1 - fi - - name: Build and validate package run: | npm run build - npm run backup-dar -- --package "${{ steps.release_tag.outputs.package_key }}" --version "${{ steps.release_tag.outputs.version }}" npm run lint npm run format npm run lint:daml @@ -144,13 +154,171 @@ jobs: npm run test - name: Upload DAR to devnet + env: + CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET }} + CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET }} run: npm run upload-dar -- --package "${{ steps.release_tag.outputs.package_key }}" --network devnet - - name: Upload DAR to mainnet - run: npm run upload-dar -- --package "${{ steps.release_tag.outputs.package_key }}" --network mainnet + - name: Persist DevNet deployment marker + id: devnet_marker + env: + GH_TOKEN: ${{ secrets.PAT_TOKEN }} + run: | + public_origin="https://github.com/${{ github.repository }}.git" + trap 'git remote set-url origin "$public_origin"' EXIT + git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git" + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + release_sha=$(git rev-parse HEAD) + echo "release_sha=$release_sha" >> "$GITHUB_OUTPUT" + git switch -c "$RELEASE_ARTIFACT_BRANCH" + git add dars/dars.lock + + marker_commit_created=false + if git diff --cached --quiet; then + echo "DevNet marker was already present on the validated release commit." + else + git commit -m "chore: record DevNet DAR deployment for ${RELEASE_TAG}" + marker_commit_created=true + fi + + # Persist the upload result before consulting mutable branch state. Even if main advanced while the upload + # ran, this unique branch remains rooted at the already validated release SHA and cannot overwrite main. + git push --no-verify origin "HEAD:${RELEASE_ARTIFACT_BRANCH}" + + if [ "$marker_commit_created" = true ]; then + gh pr create \ + --draft \ + --base main \ + --head "$RELEASE_ARTIFACT_BRANCH" \ + --title "Record release artifacts for ${RELEASE_TAG}" \ + --body "DevNet upload succeeded and its dars.lock marker is durable on this branch. The release workflow is still running; keep this PR in draft." + fi + + # Never leave the PAT embedded in repository config for later package scripts. + git remote set-url origin "$public_origin" + trap - EXIT + unset GH_TOKEN + + git fetch origin main + main_advanced=false + if [ "$release_sha" != "$(git rev-parse origin/main)" ]; then + main_advanced=true + fi + + # Verification may stop later release side effects, but it must not strand a successful upload without a + # durable marker branch and draft PR. + npm run verify-dars + + if [ "$main_advanced" = true ]; then + echo "::error::Main advanced after the release SHA was validated. The DevNet marker is durable on ${RELEASE_ARTIFACT_BRANCH}; refusing later release side effects." + exit 1 + fi + + - name: Revalidate base and upload DAR to Mainnet + env: + CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET }} + CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET }} + CANTON_MAINNET_5N_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_MAINNET_5N_LEDGER_JSON_API_CLIENT_SECRET }} + CANTON_MAINNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_MAINNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET }} + RELEASE_SHA: ${{ steps.devnet_marker.outputs.release_sha }} + run: | + git fetch origin main + if ! git merge-base --is-ancestor "$RELEASE_SHA" origin/main || [ "$RELEASE_SHA" != "$(git rev-parse origin/main)" ]; then + echo "::error::Main changed after DevNet upload. The durable DevNet marker PR remains available; refusing irreversible Mainnet side effects." + exit 1 + fi + echo "Main still equals the validated release SHA. Mainnet never selects a version: upload-dar now independently requires this exact candidate identity to be preferred by both DevNet providers." + npm run upload-dar -- --package "${{ steps.release_tag.outputs.package_key }}" --network mainnet + + - name: Persist Mainnet deployment marker + id: mainnet_marker + env: + GH_TOKEN: ${{ secrets.PAT_TOKEN }} + run: | + set -euo pipefail + public_origin="https://github.com/${{ github.repository }}.git" + trap 'git remote set-url origin "$public_origin"' EXIT + git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git" + git add dars/dars.lock + echo "marker_created=false" >> "$GITHUB_OUTPUT" + if git diff --cached --quiet; then + echo "Mainnet marker was already present." + else + git commit -m "chore: record Mainnet DAR deployment for ${RELEASE_TAG}" + git push --no-verify origin "HEAD:${RELEASE_ARTIFACT_BRANCH}" + echo "marker_commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "marker_created=true" >> "$GITHUB_OUTPUT" + fi + + # Scrub authenticated remote state before any later repository script runs. + git remote set-url origin "$public_origin" + trap - EXIT + unset GH_TOKEN + + - name: Create immutable Mainnet marker attestation + if: ${{ steps.mainnet_marker.outputs.marker_created == 'true' }} + env: + MARKER_COMMIT: ${{ steps.mainnet_marker.outputs.marker_commit }} + PACKAGE_KEY: ${{ steps.release_tag.outputs.package_key }} + run: | + set -euo pipefail + npx tsx scripts/mainnet-marker-attestation.ts create \ + --repository-root "$GITHUB_WORKSPACE" \ + --repository "$GITHUB_REPOSITORY" \ + --default-branch main \ + --workflow-path ".github/workflows/release.yml" \ + --run-id "$GITHUB_RUN_ID" \ + --run-attempt "$GITHUB_RUN_ATTEMPT" \ + --package "$PACKAGE_KEY" \ + --marker-commit "$MARKER_COMMIT" \ + --output "$RUNNER_TEMP/mainnet-marker-attestation.json" + + - name: Upload immutable Mainnet marker attestation + if: ${{ steps.mainnet_marker.outputs.marker_created == 'true' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: dar-mainnet-marker-attestation-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.release_tag.outputs.package_key }} + path: ${{ runner.temp }}/mainnet-marker-attestation.json + if-no-files-found: error + retention-days: 90 + compression-level: 0 + + - name: Update durable release marker pull request + if: ${{ steps.mainnet_marker.outputs.marker_created == 'true' }} + env: + GH_TOKEN: ${{ secrets.PAT_TOKEN }} + run: | + set -euo pipefail + pr_url=$(gh pr list --head "$RELEASE_ARTIFACT_BRANCH" --state open --json url --jq '.[0].url') + if [ -z "$pr_url" ]; then + gh pr create \ + --draft \ + --base main \ + --head "$RELEASE_ARTIFACT_BRANCH" \ + --title "Record release artifacts for ${RELEASE_TAG}" \ + --body "DevNet and Mainnet uploads succeeded and both dars.lock markers are durable on this branch. The release workflow is still running; keep this PR in draft." + else + pr_number=$(gh pr view "$pr_url" --json number --jq '.number') + gh api \ + --method PATCH \ + "repos/${GITHUB_REPOSITORY}/pulls/${pr_number}" \ + -f body="DevNet and Mainnet uploads succeeded and both dars.lock markers are durable on this branch. The release workflow is still running; keep this PR in draft." \ + >/dev/null + fi + + - name: Verify durable Mainnet marker backup + run: | + # Keep verification after marker persistence for the same durability reason as the DevNet marker. + npm run verify-dars - name: Detect factory state id: factory + env: + CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET }} + CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET }} + CANTON_MAINNET_5N_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_MAINNET_5N_LEDGER_JSON_API_CLIENT_SECRET }} + CANTON_MAINNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_MAINNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET }} run: | npx tsx scripts/detect-factory-need.ts --package "${{ steps.release_tag.outputs.package_key }}" | tee factory-need.json node <<'NODE' @@ -162,6 +330,11 @@ jobs: - name: Create missing factories if: steps.factory.outputs.needs_factory == 'true' + env: + CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET }} + CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET }} + CANTON_MAINNET_5N_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_MAINNET_5N_LEDGER_JSON_API_CLIENT_SECRET }} + CANTON_MAINNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET: ${{ secrets.CANTON_MAINNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET }} run: | while IFS= read -r network; do if [ -n "$network" ]; then @@ -181,11 +354,15 @@ jobs: run: npm publish - name: Commit generated release artifacts + env: + GH_TOKEN: ${{ secrets.PAT_TOKEN }} run: | - git remote set-url origin "https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git" + public_origin="https://github.com/${{ github.repository }}.git" + trap 'git remote set-url origin "$public_origin"' EXIT + git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git" git fetch origin main - if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then - echo "::error::Release tag $RELEASE_TAG is not the current origin/main commit; refusing to push generated artifacts." + if [ "$(git merge-base HEAD origin/main)" != "$(git rev-parse origin/main)" ]; then + echo "::error::Main advanced during the release; leaving the durable draft artifact PR for reconciliation." exit 1 fi @@ -209,5 +386,26 @@ jobs: exit 0 fi - git commit -m "chore: record release artifacts for ${RELEASE_TAG} [skip ci]" - git push origin HEAD:main + git commit -m "chore: record release artifacts for ${RELEASE_TAG}" + git push --no-verify origin "HEAD:${RELEASE_ARTIFACT_BRANCH}" + + pr_url=$(gh pr list --head "$RELEASE_ARTIFACT_BRANCH" --state open --json url --jq '.[0].url') + if [ -z "$pr_url" ]; then + gh pr create \ + --draft \ + --base main \ + --head "$RELEASE_ARTIFACT_BRANCH" \ + --title "Record release artifacts for ${RELEASE_TAG}" \ + --body "DevNet and Mainnet uploads succeeded. Generated release artifacts and deployment markers are ready for validation; keep this PR in draft until checks and automated reviews complete." + else + pr_number=$(gh pr view "$pr_url" --json number --jq '.number') + gh api \ + --method PATCH \ + "repos/${GITHUB_REPOSITORY}/pulls/${pr_number}" \ + -f body="DevNet and Mainnet uploads succeeded. Generated release artifacts and deployment markers are ready for validation; keep this PR in draft until checks and automated reviews complete." \ + >/dev/null + fi + + git remote set-url origin "$public_origin" + trap - EXIT + unset GH_TOKEN diff --git a/OpenCapTable-v34/daml.yaml b/OpenCapTable-v34/daml.yaml index 2403aff1..6d6d6797 100644 --- a/OpenCapTable-v34/daml.yaml +++ b/OpenCapTable-v34/daml.yaml @@ -1,7 +1,7 @@ sdk-version: 3.4.10 name: OpenCapTable-v34 source: daml -version: 0.0.3 +version: 0.0.2 build-options: - -Wno-crypto-text-is-alpha - --typecheck-upgrades=no diff --git a/Test/daml.yaml b/Test/daml.yaml index 151e762a..cdf173c0 100644 --- a/Test/daml.yaml +++ b/Test/daml.yaml @@ -7,6 +7,6 @@ dependencies: - daml-stdlib - daml-script data-dependencies: - - ../OpenCapTable-v34/.daml/dist/OpenCapTable-v34-0.0.3.dar + - ../OpenCapTable-v34/.daml/dist/OpenCapTable-v34-0.0.2.dar build-options: - -Wno-template-interface-depends-on-daml-script diff --git a/dars/OpenCapTable-v34/0.0.1/OpenCapTable-v34-devnet-9f94e5a0084b8fd45e3a53a9f8d896dd2cc584bf1ceff90b6b8bf036b336b565.dar b/dars/OpenCapTable-v34/0.0.1/OpenCapTable-v34-devnet-9f94e5a0084b8fd45e3a53a9f8d896dd2cc584bf1ceff90b6b8bf036b336b565.dar new file mode 100644 index 00000000..29faf3ca --- /dev/null +++ b/dars/OpenCapTable-v34/0.0.1/OpenCapTable-v34-devnet-9f94e5a0084b8fd45e3a53a9f8d896dd2cc584bf1ceff90b6b8bf036b336b565.dar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85a53aa0cec145e92a453ad59f03e5cd03d8ba400ce8bb5759bf0f55edbf5056 +size 2393294 diff --git a/dars/OpenCapTable-v34/0.0.2/OpenCapTable-v34.dar b/dars/OpenCapTable-v34/0.0.2/OpenCapTable-v34.dar index 68074446..9821b4b1 100644 --- a/dars/OpenCapTable-v34/0.0.2/OpenCapTable-v34.dar +++ b/dars/OpenCapTable-v34/0.0.2/OpenCapTable-v34.dar @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2ce462b8afcb1bcdbe351c56b0a47de478477f3c655da00e41cb397f63e997d6 -size 2446238 +oid sha256:524b71370b42d579bd3b204c2ca05023da73c1bd4396efe9aa1aabf438576143 +size 2471277 diff --git a/dars/OpenCapTable-v34/0.0.3/OpenCapTable-v34.dar b/dars/OpenCapTable-v34/0.0.3/OpenCapTable-v34.dar deleted file mode 100644 index 12aae785..00000000 --- a/dars/OpenCapTable-v34/0.0.3/OpenCapTable-v34.dar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a19932418db9ccf3e48cee95d672f471700dc7d07f5dc0cd27ddf4a976db3a5d -size 2471163 diff --git a/dars/dars.lock b/dars/dars.lock index e155fb58..8c4124c5 100644 --- a/dars/dars.lock +++ b/dars/dars.lock @@ -1,6 +1,16 @@ { "version": 1, "packages": { + "OpenCapTable-v34/0.0.1/OpenCapTable-v34-devnet-9f94e5a0084b8fd45e3a53a9f8d896dd2cc584bf1ceff90b6b8bf036b336b565.dar": { + "sha256": "85a53aa0cec145e92a453ad59f03e5cd03d8ba400ce8bb5759bf0f55edbf5056", + "size": 2393294, + "sdkVersion": "3.4.10", + "uploadedAt": "2026-04-06T15:37:10.126Z", + "networks": [ + "devnet", + "mainnet" + ] + }, "OpenCapTable-v34/0.0.1/OpenCapTable-v34.dar": { "sha256": "d14f72adf08c64ad652f22aac932e9d267a9ae55d353eb6e7a20e5ec42ee6a79", "size": 2441273, @@ -12,17 +22,10 @@ ] }, "OpenCapTable-v34/0.0.2/OpenCapTable-v34.dar": { - "sha256": "2ce462b8afcb1bcdbe351c56b0a47de478477f3c655da00e41cb397f63e997d6", - "size": 2446238, - "sdkVersion": "3.4.10", - "uploadedAt": "2026-07-07T21:50:59.634Z", - "networks": [] - }, - "OpenCapTable-v34/0.0.3/OpenCapTable-v34.dar": { - "sha256": "a19932418db9ccf3e48cee95d672f471700dc7d07f5dc0cd27ddf4a976db3a5d", - "size": 2471163, + "sha256": "524b71370b42d579bd3b204c2ca05023da73c1bd4396efe9aa1aabf438576143", + "size": 2471277, "sdkVersion": "3.4.10", - "uploadedAt": "2026-07-13T15:33:09.214Z", + "uploadedAt": "2026-07-13T20:08:09.641Z", "networks": [] } } diff --git a/package.json b/package.json index 6051514c..b17d2e52 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "build": "tsx scripts/codegen/generate-captable.ts && PATH=\"$HOME/.dpm/bin:$PATH\" dpm build --all", "build:npm-runtime-lib": "tsc -p tsconfig.npm-runtime.json", "build:ts": "tsc", + "check:dar-version-policy": "tsx scripts/check-dar-version-policy.ts", "check:schema-gaps": "tsx scripts/schema-gap-checker/check-schema-gaps.ts", "check-upgrade-compat": "tsx scripts/check-upgrade-compatibility.ts", "clean": "for pkg in OpenCapTable-v34 Test; do (cd $pkg && PATH=\"$HOME/.dpm/bin:$PATH\" dpm clean); done", @@ -79,6 +80,7 @@ "prereplay-ocf:localnet": "tsx scripts/link-local-ocp-bindings.ts", "replay-ocf:localnet": "tsx scripts/replay-ocf-database-on-localnet.ts", "test": "cd Test && PATH=\"$HOME/.dpm/bin:$PATH\" dpm test --show-coverage --color --coverage-ignore-choice splice-amulet:.*", + "test:dar-version-policy": "tsx --test scripts/dar-version-policy.test.ts scripts/dar-marker-policy.test.ts scripts/dar-archive-policy.test.ts scripts/mainnet-marker-attestation.test.ts", "test:imports": "tsx scripts/test-imports.ts", "test:replay-ocf": "tsx scripts/replay-ocf-database-on-localnet.test.ts", "update-version": "tsx scripts/update-generated-package.ts", diff --git a/scripts/backup-dar.ts b/scripts/backup-dar.ts index f29a2092..12214538 100644 --- a/scripts/backup-dar.ts +++ b/scripts/backup-dar.ts @@ -1,23 +1,26 @@ #!/usr/bin/env node /** - * Backup a DAR file after mainnet upload. Copies from .daml/dist/ to dars/{package}/{version}/ and updates dars.lock. + * Synchronize a built DAR into dars/{package}/{version}/ and update dars.lock. * - * **Retention:** When bumping a package version, add the new backup with this script but **do not remove** prior - * `dars///` trees that are already in the repo—keep historical DARs for audit, re-upload, and - * debugging. See the repo wiki: https://github.com/Fairmint/open-captable-protocol-daml/wiki + * Undeployed versions are mutable PR candidates. Live DevNet is the version authority, any recorded or exact-live DAR + * is immutable, and superseded undeployed backups are pruned only after integrity and live-ID checks. * - * Usage: tsx scripts/backup-dar.ts --package --version [--network ] + * Usage: tsx scripts/backup-dar.ts --package --version */ import * as fs from 'fs'; import * as path from 'path'; import * as yaml from 'yaml'; -import { computeSha256, getDarsDir, loadDarsLock, saveDarsLock, type DarsLockEntry } from './dar-utils'; -import { getAllPackages, getPackage, parseNetworkArg, parsePackageArg, parseVersionArg } from './packages'; +import { applyBackupTransaction } from './dar-backup-transaction'; +import { computeSha256, getDarsDir, loadDarsLock, type DarsLockEntry } from './dar-utils'; +import { candidateDevnetNetworks, decideBackupMutation, planBackupRetention } from './dar-version-policy'; +import { inspectPackageBackups, validateDevnetDarCandidate } from './devnet-dar-policy'; +import { queryDevnetPackagePreferences } from './devnet-package-versions'; +import { getAllPackages, getPackage, parsePackageArg, parseVersionArg } from './packages'; function printUsage(errorMessage?: string): never { if (errorMessage) console.error(`❌ ${errorMessage}\n`); - console.error('Usage: tsx scripts/backup-dar.ts --package --version [--network ]'); + console.error('Usage: tsx scripts/backup-dar.ts --package --version '); console.error('\nPackages:'); for (const pkg of getAllPackages()) { console.error(` ${pkg.name}`); @@ -38,15 +41,24 @@ function getSdkVersion(sourceDir: string): string { } } -function main() { +function frozenVersionError(packageName: string, version: string): Error { + return new Error( + `${packageName} ${version} is immutable because its exact DAR is recorded on a network. ` + + `Run npm run upgrade-package -- --package ${packageName} --type minor to select the live DevNet candidate version.` + ); +} + +async function main(): Promise { const packageArg = parsePackageArg(); const version = parseVersionArg(); - const network = parseNetworkArg(); if (!packageArg || !version) printUsage('Missing required arguments'); const pkg = getPackage(packageArg); if (!pkg) printUsage(`Unknown package: ${packageArg}`); + if (version !== pkg.version) { + printUsage(`Requested version ${version} does not match ${pkg.name}/daml.yaml (${pkg.version})`); + } const rootDir = path.join(__dirname, '..'); const darsDir = getDarsDir(); @@ -65,67 +77,91 @@ function main() { } const lock = loadDarsLock(); + const sourceHash = computeSha256(sourcePath); + const existed = Object.prototype.hasOwnProperty.call(lock.packages, lockKey); + const existingEntry: DarsLockEntry | undefined = existed ? lock.packages[lockKey] : undefined; - // Already backed up? - if (lockKey in lock.packages) { - console.log(`ℹ️ Already backed up: ${lockKey}`); - + // Verify an existing backup before deciding whether it can be replaced. + if (existingEntry) { if (!fs.existsSync(destPath)) { console.error(`❌ Lock entry exists but file missing: ${destPath}`); process.exit(1); } // Verify hash - const hash = computeSha256(destPath); - if (hash !== lock.packages[lockKey].sha256) { + const storedHash = computeSha256(destPath); + if (storedHash !== existingEntry.sha256) { console.error(`❌ Hash mismatch! File may have been modified.`); process.exit(1); } + } - // Add network if specified - if (network && !lock.packages[lockKey].networks.includes(network)) { - lock.packages[lockKey].networks.push(network); - lock.packages[lockKey].networks.sort(); - saveDarsLock(lock); - console.log(`✅ Added network: ${network}`); - } + // Query and validate even when the backup bytes already match. This turns unrecorded exact-live backups into durable + // DevNet history and prevents a later branch from pruning or replacing them. + const preferences = await queryDevnetPackagePreferences(pkg.name); + const inspectedBackups = inspectPackageBackups(rootDir, lock, pkg.name); + const validation = validateDevnetDarCandidate({ + repositoryRoot: rootDir, + lock, + packageName: pkg.name, + packageVersion: version, + candidateDarPath: sourcePath, + preferences, + }); + console.log( + `✅ Live DevNet preflight passed for ${validation.candidatePackageId} (${validation.compatibilityBaselines.length} baseline(s))` + ); + + const retentionPlan = planBackupRetention(inspectedBackups, lockKey, preferences); + const inspectedCurrent = inspectedBackups.find((backup) => backup.lockKey === lockKey); + const currentIsExactLive = Boolean( + inspectedCurrent && + candidateDevnetNetworks(preferences, inspectedCurrent.packageVersion, inspectedCurrent.packageId).length > 0 + ); + const effectiveExistingEntry = + existingEntry && currentIsExactLive + ? { ...existingEntry, networks: [...new Set([...existingEntry.networks, 'devnet'])] } + : existingEntry; + + let mutation: ReturnType; + try { + mutation = decideBackupMutation(effectiveExistingEntry, sourceHash); + } catch { + throw frozenVersionError(pkg.name, version); + } + + let candidateWrite; + if (mutation !== 'no-op') { + const sourceStats = fs.statSync(sourcePath); + candidateWrite = { + lockKey, + sourcePath, + destPath, + replaceExisting: mutation === 'replace', + entry: { + sha256: sourceHash, + size: sourceStats.size, + sdkVersion: getSdkVersion(pkg.sourceDir), + uploadedAt: new Date().toISOString(), + networks: candidateDevnetNetworks(preferences, version, validation.candidatePackageId), + }, + }; + } + + applyBackupTransaction({ lock, retentionPlan, darsDir, candidateWrite }); + for (const key of retentionPlan.freezeKeys) console.log(`🔒 Recorded exact live DevNet backup: ${key}`); + for (const key of retentionPlan.pruneKeys) console.log(`🧹 Pruned superseded undeployed backup: ${key}`); + if (mutation === 'no-op') { + console.log(`ℹ️ Backup already matches current build: ${lockKey}`); return; } - // Create backup - fs.mkdirSync(destDir, { recursive: true }); - fs.copyFileSync(sourcePath, destPath); - - const hash = computeSha256(destPath); - const stats = fs.statSync(destPath); - - lock.packages[lockKey] = { - sha256: hash, - size: stats.size, - sdkVersion: getSdkVersion(pkg.sourceDir), - uploadedAt: new Date().toISOString(), - networks: network ? [network] : [], - }; - - // Sort for consistent ordering - const sorted: Record = {}; - Object.keys(lock.packages) - .sort() - .forEach((k) => { - sorted[k] = lock.packages[k]; - }); - lock.packages = sorted; - - saveDarsLock(lock); - - console.log(`✅ Backed up: ${lockKey}`); - console.log(` SHA256: ${hash}`); - console.log(` Size: ${stats.size} bytes`); + console.log(`${existed ? '✅ Synchronized' : '✅ Backed up'}: ${lockKey}`); + console.log(` SHA256: ${sourceHash}`); + console.log(` Size: ${candidateWrite?.entry.size ?? 0} bytes`); } -try { - main(); -} catch (err) { - console.error('❌ Error:', err); +void main().catch((err: unknown) => { + console.error('❌ Error:', err instanceof Error ? err.message : String(err)); process.exit(1); -} +}); diff --git a/scripts/candidate-path-safety.ts b/scripts/candidate-path-safety.ts new file mode 100644 index 00000000..38fdfe77 --- /dev/null +++ b/scripts/candidate-path-safety.ts @@ -0,0 +1,34 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +export const MAX_CANDIDATE_LOCK_BYTES = 5 * 1024 * 1024; +export const MAX_CANDIDATE_METADATA_BYTES = 1024 * 1024; + +/** Verify an untrusted candidate file is regular, bounded, and still inside its root after resolving parent symlinks. */ +export function assertContainedRegularFile( + rootDir: string, + filePath: string, + label: string, + maxBytes: number +): fs.Stats { + const resolvedRoot = path.resolve(rootDir); + const resolvedPath = path.resolve(filePath); + if (!resolvedPath.startsWith(`${resolvedRoot}${path.sep}`)) { + throw new Error(`${label} escapes candidate root: ${filePath}`); + } + + const stats = fs.lstatSync(resolvedPath); + if (!stats.isFile()) { + throw new Error(`${label} is not a regular file: ${filePath}`); + } + if (stats.size > maxBytes) { + throw new Error(`${label} exceeds ${maxBytes} bytes: ${filePath}`); + } + + const realRoot = fs.realpathSync(resolvedRoot); + const realPath = fs.realpathSync(resolvedPath); + if (!realPath.startsWith(`${realRoot}${path.sep}`)) { + throw new Error(`${label} escapes candidate root through a symlink: ${filePath}`); + } + return stats; +} diff --git a/scripts/check-dar-version-policy.ts b/scripts/check-dar-version-policy.ts new file mode 100644 index 00000000..a754f7f5 --- /dev/null +++ b/scripts/check-dar-version-policy.ts @@ -0,0 +1,473 @@ +#!/usr/bin/env node +/** Trusted DevNet policy check. Candidate files are data; this script must come from the target/default branch. */ + +import { execFileSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as yaml from 'yaml'; +import { + assertContainedRegularFile, + MAX_CANDIDATE_LOCK_BYTES, + MAX_CANDIDATE_METADATA_BYTES, +} from './candidate-path-safety'; +import type { DarsLock } from './dar-utils'; +import { getDarLockKey } from './dar-utils'; +import { + assertDarsLockSchema, + assertDevnetMarkerIdentity, + classifyCandidateOnlyBackups, + getExactLiveProviderNames, + getNetworkMarkerAdditions, +} from './dar-marker-policy'; +import { inspectDarPackageId } from './dar-package-id'; +import { assertDevnetPreferencesConsistent } from './dar-version-policy'; +import { validateDevnetDarCandidate, verifyLockedDar } from './devnet-dar-policy'; +import { queryDevnetPackagePreferences, type DevnetPackagePreference } from './devnet-package-versions'; +import { LEDGER_SCRIPT_PROVIDERS } from './providers'; + +const ROOT_DIR = path.join(__dirname, '..'); +const DEVNET_SECRET_NAMES = [ + 'CANTON_DEVNET_INTELLECT_LEDGER_JSON_API_CLIENT_SECRET', + 'CANTON_DEVNET_5N_LEDGER_JSON_API_CLIENT_SECRET', +] as const; +const MAX_HISTORY_LOCK_COMMITS = 5000; +const GIT_READ_TIMEOUT_MS = 60_000; + +interface DamlMetadata { + name: string; + version: string; +} + +interface Args { + authorizedMainnetMarkerLockKey?: string; + auditAll: boolean; + baseRef: string; + candidateRoot: string; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function parseArgs(): Args { + const args = process.argv.slice(2); + const valueAfter = (flag: string): string | undefined => { + const index = args.indexOf(flag); + return index >= 0 ? args[index + 1] : undefined; + }; + return { + authorizedMainnetMarkerLockKey: valueAfter('--authorized-mainnet-marker-lock-key'), + auditAll: args.includes('--audit-all'), + baseRef: valueAfter('--base') ?? 'origin/main', + candidateRoot: path.resolve(valueAfter('--candidate-root') ?? ROOT_DIR), + }; +} + +function readCandidateFile(candidateRoot: string, relativePath: string): string { + const resolvedRoot = path.resolve(candidateRoot); + const resolvedPath = path.resolve(resolvedRoot, relativePath); + assertContainedRegularFile(resolvedRoot, resolvedPath, `Candidate file ${relativePath}`, MAX_CANDIDATE_LOCK_BYTES); + return fs.readFileSync(resolvedPath, 'utf8'); +} + +function loadCandidateLock(candidateRoot: string): DarsLock { + const parsed = JSON.parse(readCandidateFile(candidateRoot, 'dars/dars.lock')) as unknown; + if (!isRecord(parsed) || typeof parsed.version !== 'number' || !isRecord(parsed.packages)) { + throw new Error('Candidate dars/dars.lock has an invalid structure.'); + } + const packages: DarsLock['packages'] = {}; + for (const [lockKey, value] of Object.entries(parsed.packages)) { + if ( + !isRecord(value) || + typeof value.sha256 !== 'string' || + typeof value.size !== 'number' || + typeof value.sdkVersion !== 'string' || + typeof value.uploadedAt !== 'string' || + !Array.isArray(value.networks) || + !value.networks.every((network) => typeof network === 'string') + ) { + throw new Error(`Candidate dars.lock entry ${lockKey} has an invalid structure.`); + } + packages[lockKey] = { + sha256: value.sha256, + size: value.size, + sdkVersion: value.sdkVersion, + uploadedAt: value.uploadedAt, + networks: value.networks, + }; + } + const lock = { version: parsed.version, packages }; + assertDarsLockSchema(lock, 'Candidate dars.lock'); + return lock; +} + +function gitShow(baseRef: string, filePath: string): string | null { + try { + return execFileSync('git', ['show', `${baseRef}:${filePath}`], { + cwd: ROOT_DIR, + encoding: 'utf8', + maxBuffer: MAX_CANDIDATE_LOCK_BYTES + 1024, + timeout: GIT_READ_TIMEOUT_MS, + }); + } catch { + return null; + } +} + +function loadHistoricalLocks(baseRef: string): DarsLock[] { + const rawCommits = execFileSync('git', ['log', '--format=%H', baseRef, '--', 'dars/dars.lock'], { + cwd: ROOT_DIR, + encoding: 'utf8', + maxBuffer: 1024 * 1024, + timeout: GIT_READ_TIMEOUT_MS, + }); + const commits = rawCommits.split('\n').filter(Boolean); + if (commits.length > MAX_HISTORY_LOCK_COMMITS) { + throw new Error(`Refusing to inspect more than ${MAX_HISTORY_LOCK_COMMITS} historical dars.lock revisions.`); + } + return commits.map((commit, index) => { + const text = gitShow(commit, 'dars/dars.lock'); + if (!text) throw new Error(`Unable to read historical dars.lock at ${commit}.`); + const lock = JSON.parse(text) as unknown; + assertDarsLockSchema(lock, `Historical dars.lock ${index + 1} (${commit})`); + return lock; + }); +} + +function findCandidatePackages(candidateRoot: string): Array<{ sourceDir: string; metadata: DamlMetadata }> { + const packages: Array<{ sourceDir: string; metadata: DamlMetadata }> = []; + for (const entry of fs.readdirSync(candidateRoot, { withFileTypes: true })) { + if (!entry.isDirectory() || entry.name.startsWith('.') || entry.name === 'Test') continue; + const metadataPath = path.join(candidateRoot, entry.name, 'daml.yaml'); + if (!fs.existsSync(metadataPath)) continue; + assertContainedRegularFile(candidateRoot, metadataPath, 'Candidate DAML metadata', MAX_CANDIDATE_METADATA_BYTES); + const metadata = yaml.parse(fs.readFileSync(metadataPath, 'utf8'), { + maxAliasCount: 50, + schema: 'core', + }) as Partial; + if ( + typeof metadata.name !== 'string' || + !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(metadata.name) || + typeof metadata.version !== 'string' || + !/^\d+\.\d+\.\d+$/.test(metadata.version) + ) { + throw new Error(`Invalid package metadata in ${entry.name}/daml.yaml.`); + } + packages.push({ sourceDir: entry.name, metadata: metadata as DamlMetadata }); + } + if (packages.length > 50) throw new Error('Refusing candidate with more than 50 top-level DAML packages.'); + return packages; +} + +function formatLivePreferences(preferences: Awaited>): string { + return LEDGER_SCRIPT_PROVIDERS.map((provider) => { + const preference = preferences.find((entry) => entry.provider === provider); + return preference ? `${provider}=${preference.packageVersion} (${preference.packageId})` : `${provider}=absent`; + }).join(', '); +} + +function removeDevnetSecrets(): Map { + const saved = new Map(); + for (const name of DEVNET_SECRET_NAMES) { + const value = process.env[name]; + if (value !== undefined) saved.set(name, value); + delete process.env[name]; + } + return saved; +} + +function restoreDevnetSecrets(saved: Map): void { + for (const [name, value] of saved) process.env[name] = value; +} + +function lockEntryWithoutNetworks(entry: DarsLock['packages'][string]): Omit { + return { + sha256: entry.sha256, + size: entry.size, + sdkVersion: entry.sdkVersion, + uploadedAt: entry.uploadedAt, + }; +} + +async function validateBaseLockRetention( + baseLock: DarsLock, + candidateLock: DarsLock, + candidateRoot: string, + currentCandidateKeys: ReadonlySet, + liveBaselineKeys: ReadonlySet, + getPreferences: (packageName: string) => Promise +): Promise { + let recordedRetained = 0; + let removalsProvenSafe = 0; + + for (const [lockKey, baseEntry] of Object.entries(baseLock.packages)) { + const parts = lockKey.split('/'); + if (parts.length !== 3 || !parts[0] || !parts[1]) { + throw new Error(`Invalid base DAR lock key: ${lockKey}`); + } + const [packageName, packageVersion] = parts; + const candidateEntry = Object.prototype.hasOwnProperty.call(candidateLock.packages, lockKey) + ? candidateLock.packages[lockKey] + : undefined; + + if (baseEntry.networks.length > 0) { + if (!candidateEntry) throw new Error(`Recorded DAR history cannot be removed: ${lockKey}`); + if ( + JSON.stringify(lockEntryWithoutNetworks(candidateEntry)) !== JSON.stringify(lockEntryWithoutNetworks(baseEntry)) + ) { + throw new Error(`Recorded DAR metadata must remain identical: ${lockKey}`); + } + const missingMarkers = baseEntry.networks.filter((network) => !candidateEntry.networks.includes(network)); + if (missingMarkers.length > 0) { + throw new Error(`Recorded DAR ${lockKey} removed network marker(s): ${missingMarkers.join(', ')}`); + } + verifyLockedDar(candidateRoot, candidateLock, lockKey); + recordedRetained++; + continue; + } + + if (candidateEntry) { + const metadataMatches = + JSON.stringify(lockEntryWithoutNetworks(candidateEntry)) === + JSON.stringify(lockEntryWithoutNetworks(baseEntry)); + if (!metadataMatches) { + if (!currentCandidateKeys.has(lockKey) && !liveBaselineKeys.has(lockKey)) { + throw new Error( + `Historical unrecorded DAR ${lockKey} cannot be replaced in place; remove it through the exact live-ID ` + + 'safety check and add the new current candidate separately.' + ); + } + // The active mutable candidate is integrity-checked and fully live-validated by the package pass below. + continue; + } + verifyLockedDar(candidateRoot, candidateLock, lockKey); + continue; + } + + // The base/default-branch DAR is trusted input. Inspect its exact ID only after verifying the base lock hash. + const baseDarPath = verifyLockedDar(ROOT_DIR, baseLock, lockKey); + const basePackageId = inspectDarPackageId(baseDarPath, packageName, packageVersion); + const preferences = await getPreferences(packageName); + assertDevnetPreferencesConsistent(preferences); + const exactLive = preferences.some( + (preference) => preference.packageId === basePackageId && preference.packageVersion === packageVersion + ); + if (exactLive) { + throw new Error(`Cannot remove ${lockKey}: its exact package ID ${basePackageId} is preferred on DevNet.`); + } + removalsProvenSafe++; + console.log(`🧹 ${lockKey}: removed undeployed backup proven absent from both DevNet providers`); + } + + console.log( + `🔒 Lock retention: ${recordedRetained} recorded backup(s) retained; ${removalsProvenSafe} undeployed removal(s) proven safe\n` + ); +} + +async function main(): Promise { + const { authorizedMainnetMarkerLockKey, auditAll, baseRef, candidateRoot } = parseArgs(); + if (!fs.lstatSync(candidateRoot).isDirectory()) { + throw new Error(`Candidate root is not a directory: ${candidateRoot}`); + } + + const baseLockText = gitShow(baseRef, 'dars/dars.lock'); + if (!baseLockText) { + throw new Error(`Unable to read dars/dars.lock from ${baseRef}. Fetch the base branch first.`); + } + const baseLock = JSON.parse(baseLockText) as DarsLock; + assertDarsLockSchema(baseLock, 'Trusted dars.lock'); + const candidateLock = loadCandidateLock(candidateRoot); + const packages = findCandidatePackages(candidateRoot); + const currentCandidateKeys = new Set( + packages.map(({ metadata }) => getDarLockKey(metadata.name, metadata.version, metadata.name)) + ); + const currentPackageNames = new Set(packages.map(({ metadata }) => metadata.name)); + const preferenceCache = new Map>(); + const getPreferences = async (packageName: string): Promise => { + let pending = preferenceCache.get(packageName); + if (!pending) { + pending = queryDevnetPackagePreferences(packageName); + preferenceCache.set(packageName, pending); + } + return pending; + }; + const liveBaselineKeys = new Set(); + for (const [lockKey, candidateEntry] of Object.entries(candidateLock.packages)) { + if (candidateEntry.networks.length > 0 || currentCandidateKeys.has(lockKey)) continue; + const baseEntry = Object.prototype.hasOwnProperty.call(baseLock.packages, lockKey) + ? baseLock.packages[lockKey] + : undefined; + if ( + baseEntry && + JSON.stringify(lockEntryWithoutNetworks(candidateEntry)) === JSON.stringify(lockEntryWithoutNetworks(baseEntry)) + ) { + continue; + } + + const [packageName, packageVersion] = lockKey.split('/'); + if (!currentPackageNames.has(packageName)) continue; + const darPath = verifyLockedDar(candidateRoot, candidateLock, lockKey); + const savedSecrets = removeDevnetSecrets(); + let packageId: string; + try { + packageId = inspectDarPackageId(darPath, packageName, packageVersion); + } finally { + restoreDevnetSecrets(savedSecrets); + } + const preferences = await getPreferences(packageName); + assertDevnetPreferencesConsistent(preferences); + const exactProviders = getExactLiveProviderNames({ packageId, packageName, packageVersion }, preferences); + if (exactProviders.length > 0) { + liveBaselineKeys.add(lockKey); + console.log( + `🛟 ${lockKey}: preserving exact live baseline ${packageId} from ${exactProviders.join(', ')} DevNet` + ); + } + } + const hasCandidateOnlyRecordedEntry = Object.entries(candidateLock.packages).some( + ([lockKey, entry]) => !Object.prototype.hasOwnProperty.call(baseLock.packages, lockKey) && entry.networks.length > 0 + ); + const candidateOnly = classifyCandidateOnlyBackups( + baseLock, + candidateLock, + currentCandidateKeys, + hasCandidateOnlyRecordedEntry ? loadHistoricalLocks(baseRef) : [], + liveBaselineKeys + ); + const baselineRecoveryPackageNames = new Set([ + ...[...liveBaselineKeys].map((lockKey) => lockKey.split('/')[0]), + ...candidateOnly.candidateOnlyKeys.map((lockKey) => lockKey.split('/')[0]), + ]); + const markerAdditions = getNetworkMarkerAdditions(baseLock, candidateLock, { + authorizedMainnetMarkerLockKey, + allowSameCandidateMainnet: true, + currentCandidateKeys, + liveBaselineKeys, + restoredRecordedKeys: new Set(candidateOnly.restoredRecordedKeys), + }); + let checked = 0; + let failures = 0; + + console.log( + `🔎 ${auditAll ? 'Auditing all DARs' : 'Checking changed DARs'} against live DevNet (candidate: ${candidateRoot})...\n` + ); + + for (const lockKey of candidateOnly.candidateOnlyKeys) { + const [packageName, packageVersion] = lockKey.split('/'); + const darPath = verifyLockedDar(candidateRoot, candidateLock, lockKey); + const savedSecrets = removeDevnetSecrets(); + let packageId: string; + try { + packageId = inspectDarPackageId(darPath, packageName, packageVersion); + } finally { + restoreDevnetSecrets(savedSecrets); + } + const classification = candidateOnly.restoredRecordedKeys.includes(lockKey) + ? 'restored recorded history' + : liveBaselineKeys.has(lockKey) + ? 'exact live partial-upload baseline' + : 'current mutable candidate'; + console.log(`✅ ${lockKey}: ${classification} verified (${packageId})`); + } + if (candidateOnly.candidateOnlyKeys.length > 0) console.log(); + + await validateBaseLockRetention( + baseLock, + candidateLock, + candidateRoot, + currentCandidateKeys, + liveBaselineKeys, + getPreferences + ); + + for (const addition of markerAdditions) { + if (addition.network === 'mainnet') { + console.log(`✅ ${addition.lockKey}: mainnet marker has the required DevNet evidence`); + continue; + } + const preferences = await getPreferences(addition.packageName); + const darPath = verifyLockedDar(candidateRoot, candidateLock, addition.lockKey); + const savedSecrets = removeDevnetSecrets(); + let packageId: string; + try { + packageId = inspectDarPackageId(darPath, addition.packageName, addition.packageVersion); + } finally { + restoreDevnetSecrets(savedSecrets); + } + assertDevnetMarkerIdentity( + addition, + { packageId, packageName: addition.packageName, packageVersion: addition.packageVersion }, + preferences, + LEDGER_SCRIPT_PROVIDERS + ); + console.log(`✅ ${addition.lockKey}: devnet marker matches every configured provider`); + } + if (markerAdditions.length > 0) console.log(); + + for (const { sourceDir, metadata } of packages) { + const candidateKey = getDarLockKey(metadata.name, metadata.version, metadata.name); + const candidateEntry = Object.prototype.hasOwnProperty.call(candidateLock.packages, candidateKey) + ? candidateLock.packages[candidateKey] + : undefined; + const baseMetadataText = gitShow(baseRef, `${sourceDir}/daml.yaml`); + const baseMetadata = baseMetadataText ? (yaml.parse(baseMetadataText) as DamlMetadata) : null; + let changed = true; + if (baseMetadata) { + const baseKey = getDarLockKey(baseMetadata.name, baseMetadata.version, baseMetadata.name); + const baseEntry = Object.prototype.hasOwnProperty.call(baseLock.packages, baseKey) + ? baseLock.packages[baseKey] + : undefined; + changed = + baseMetadata.name !== metadata.name || + baseMetadata.version !== metadata.version || + JSON.stringify(baseEntry ? lockEntryWithoutNetworks(baseEntry) : null) !== + JSON.stringify(candidateEntry ? lockEntryWithoutNetworks(candidateEntry) : null); + } + if (baselineRecoveryPackageNames.has(metadata.name)) changed = true; + + if (!auditAll && !changed) { + console.log(`⏭️ ${metadata.name} ${metadata.version}: candidate bytes unchanged from ${baseRef}`); + continue; + } + + checked++; + console.log(`📦 ${metadata.name} ${metadata.version}`); + try { + if (!candidateEntry) throw new Error(`Missing candidate dars.lock entry ${candidateKey}.`); + const candidateDarPath = path.join(candidateRoot, 'dars', candidateKey); + const preferences = await getPreferences(metadata.name); + console.log(` DevNet: ${formatLivePreferences(preferences)}`); + // Do not expose credentials to dpm while it parses untrusted candidate DAR data. + const savedSecrets = removeDevnetSecrets(); + let result: ReturnType; + try { + result = validateDevnetDarCandidate({ + repositoryRoot: candidateRoot, + lock: candidateLock, + packageName: metadata.name, + packageVersion: metadata.version, + candidateDarPath, + expectedCandidateSha256: candidateEntry.sha256, + preferences, + }); + } finally { + restoreDevnetSecrets(savedSecrets); + } + console.log(` Candidate package ID: ${result.candidatePackageId}`); + console.log( + `✅ Version ${result.expectedVersion}; checked ${result.compatibilityBaselines.length} unique live baseline(s)\n` + ); + } catch (error) { + failures++; + console.error(`❌ ${error instanceof Error ? error.message : String(error)}\n`); + } + } + + console.log(`📊 ${checked} candidate(s) checked, ${failures} failure(s)`); + if (failures > 0) process.exit(1); +} + +void main().catch((error: unknown) => { + console.error(`❌ ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); +}); diff --git a/scripts/check-upgrade-compatibility.ts b/scripts/check-upgrade-compatibility.ts index eb5c0a67..34e524e5 100644 --- a/scripts/check-upgrade-compatibility.ts +++ b/scripts/check-upgrade-compatibility.ts @@ -5,15 +5,12 @@ * against the most recent backup for each package. Fails CI if: * * - Breaking changes are introduced without a major version bump - * - Compatible changes are made without bumping the minor version + * - A built DAR does not match its mutable candidate backup * * Usage: npx tsx scripts/check-upgrade-compatibility.ts * - * Why can “comments only” still fail? The check compares two concrete DAR builds with the same package name and - * version. Any source change can yield a different LF package (package id / archive bytes). The validator may then - * reject the pair as not a valid upgrade. For non-breaking edits, bump the package patch in daml.yaml (see `npm run - * upgrade-package` with `--type minor` for unversioned folders like CouponMinter) so CI compares backup v0.0.1 against - * v0.0.2 instead of v0.0.1 against a different v0.0.1 build. + * Any source change can yield a different LF package id. Before the current version reaches a network, refresh that + * version's candidate backup in place. Live DevNet policy checks select the version separately. * * Every deployable package must have its **current** `daml.yaml` version recorded under `dars/` (lock entry + file on * disk) with a SHA256 matching the committed backup. After building, run `npx tsx scripts/backup-dar.ts --package @@ -23,11 +20,13 @@ * name/version with different package ids, which `upgrade-check` rejects.) */ -import { execSync } from 'child_process'; +import { execFileSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; +import { assertDarArchiveSafe, DAML_UPGRADE_CHECK_TIMEOUT_MS } from './dar-archive-policy'; import { computeSha256, getDarLockKey, getDarsDir, loadDarsLock } from './dar-utils'; +import { compareSemver, isDeployed } from './dar-version-policy'; const ROOT_DIR = path.join(__dirname, '..'); @@ -49,19 +48,7 @@ function parsePackageName(name: string): { baseName: string; majorVersion: numbe return { baseName: name, majorVersion: null }; } -/** Semver compare: negative if a < b, zero if equal, positive if a > b. */ -function compareSemver(a: string, b: string): number { - const pa = a.split('.').map((x) => parseInt(x, 10) || 0); - const pb = b.split('.').map((x) => parseInt(x, 10) || 0); - for (let i = 0; i < 3; i++) { - const da = pa[i] ?? 0; - const db = pb[i] ?? 0; - if (da !== db) return da - db; - } - return 0; -} - -/** Get all backed-up packages from dars.lock, grouped by exact package name (file must exist on disk). */ +/** Get DevNet-marked packages from dars.lock, grouped by exact package name (file must exist on disk). */ function getBackedUpPackages(): Map> { const lock = loadDarsLock(); const darsDir = getDarsDir(); @@ -69,7 +56,8 @@ function getBackedUpPackages(): Map>(); - for (const [lockKey, _entry] of Object.entries(lock.packages)) { + for (const [lockKey, entry] of Object.entries(lock.packages)) { + if (!entry.networks.includes('devnet')) continue; // lockKey format: "OpenCapTable-v34/0.0.1/OpenCapTable-v34.dar" const parts = lockKey.split('/'); if (parts.length !== 3) continue; @@ -159,19 +147,21 @@ function reportUpgradeFailure(packageName: string, baseName: string, output: str for (const line of lines.slice(-(maxLines / 2))) indent(line); } console.error(''); - console.error( - ` If this was a non-breaking change, bump the patch in daml.yaml (e.g. \`npm run upgrade-package -- --package ${baseName} --type minor\`).` - ); + console.error(' Fix the source or dependency change if this was expected to be backwards compatible.'); console.error(' To introduce breaking changes, bump the major version:'); console.error(` npm run upgrade-package -- --package ${baseName} --type major\n`); } /** Run dpm upgrade-check and return success/failure. */ function runUpgradeCheck(oldDar: string, newDar: string): { success: boolean; output: string } { + assertDarArchiveSafe(oldDar); + assertDarArchiveSafe(newDar); try { - const output = execSync(`dpm upgrade-check --both "${oldDar}" "${newDar}"`, { + const output = execFileSync('dpm', ['upgrade-check', '--both', oldDar, newDar], { encoding: 'utf8', + killSignal: 'SIGKILL', stdio: ['pipe', 'pipe', 'pipe'], + timeout: DAML_UPGRADE_CHECK_TIMEOUT_MS, env: { ...process.env, PATH: `${process.env.HOME}/.dpm/bin:${process.env.PATH}` }, }); return { success: true, output }; @@ -252,8 +242,15 @@ function main(): void { console.error(` Expected (dars.lock): ${lockEntry.sha256}`); console.error(` Actual (build): ${builtHash}`); console.error(''); - console.error(' The tree under dars/ must be the exact DAR for this daml.yaml version.'); - console.error(' After `npm run build`, run backup-dar for this package and commit.\n'); + if (isDeployed(lockEntry)) { + console.error(' This exact backup is recorded on a network and is immutable.'); + console.error(` Run npm run upgrade-package -- --package ${currentPackageName} --type minor.\n`); + } else { + console.error(' This version is still an undeployed PR candidate; keep the version.'); + console.error( + ` Refresh it with npm run backup-dar -- --package ${currentPackageName} --version ${currentDar.version}, then commit dars/.\n` + ); + } hasFailures = true; checkedCount++; continue; @@ -287,7 +284,9 @@ function main(): void { ); } } else { - console.log(`✅ ${currentPackageName}: No older backed-up version to upgrade-check (first release in dars/)\n`); + console.log( + `✅ ${currentPackageName}: No older deployed version to upgrade-check (first release in this line)\n` + ); } checkedCount++; @@ -298,7 +297,7 @@ function main(): void { if (hasFailures) { console.error('\n❌ Upgrade compatibility check failed!'); - console.error(' Fix the issues above or bump the major version for breaking changes.'); + console.error(' Fix the issues above; use a major package line only for breaking changes.'); process.exit(1); } diff --git a/scripts/dar-archive-policy.test.ts b/scripts/dar-archive-policy.test.ts new file mode 100644 index 00000000..bb58e5cb --- /dev/null +++ b/scripts/dar-archive-policy.test.ts @@ -0,0 +1,222 @@ +import assert from 'node:assert/strict'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { describe, it } from 'node:test'; +import { deflateRawSync } from 'node:zlib'; +import { assertDarArchiveSafe, MAX_DAR_ARCHIVE_ENTRIES, MAX_DAR_ENTRY_UNCOMPRESSED_BYTES } from './dar-archive-policy'; + +interface SyntheticEntry { + advertisedCrc32?: number; + centralExtra?: Buffer; + compressedSize?: number; + data?: Buffer; + externalAttributes?: number; + flags?: number; + localExtra?: Buffer; + method?: number; + name: string; + uncompressedSize?: number; +} + +const CRC32_TABLE = new Uint32Array(256); +for (let value = 0; value < CRC32_TABLE.length; value++) { + let remainder = value; + for (let bit = 0; bit < 8; bit++) { + remainder = (remainder & 1) !== 0 ? 0xedb88320 ^ (remainder >>> 1) : remainder >>> 1; + } + CRC32_TABLE[value] = remainder >>> 0; +} + +function crc32(bytes: Buffer): number { + let checksum = 0xffffffff; + for (const byte of bytes) checksum = CRC32_TABLE[(checksum ^ byte) & 0xff] ^ (checksum >>> 8); + return (checksum ^ 0xffffffff) >>> 0; +} + +function makeZip(entries: SyntheticEntry[]): Buffer { + const localParts: Buffer[] = []; + const centralParts: Buffer[] = []; + let localOffset = 0; + + for (const entry of entries) { + const data = entry.data ?? Buffer.from('x'); + const rawName = Buffer.from(entry.name, 'ascii'); + const flags = entry.flags ?? 0; + const method = entry.method ?? 0; + const compressedData = method === 8 ? deflateRawSync(data, { level: 1 }) : data; + const compressedSize = entry.compressedSize ?? compressedData.length; + const uncompressedSize = entry.uncompressedSize ?? data.length; + const advertisedCrc32 = entry.advertisedCrc32 ?? crc32(data); + const localExtra = entry.localExtra ?? Buffer.alloc(0); + const centralExtra = entry.centralExtra ?? Buffer.alloc(0); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); + local.writeUInt16LE(flags, 6); + local.writeUInt16LE(method, 8); + local.writeUInt32LE(advertisedCrc32, 14); + local.writeUInt32LE(compressedSize, 18); + local.writeUInt32LE(uncompressedSize, 22); + local.writeUInt16LE(rawName.length, 26); + local.writeUInt16LE(localExtra.length, 28); + localParts.push(local, rawName, localExtra, compressedData); + + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE((3 << 8) | 20, 4); + central.writeUInt16LE(20, 6); + central.writeUInt16LE(flags, 8); + central.writeUInt16LE(method, 10); + central.writeUInt32LE(advertisedCrc32, 16); + central.writeUInt32LE(compressedSize, 20); + central.writeUInt32LE(uncompressedSize, 24); + central.writeUInt16LE(rawName.length, 28); + central.writeUInt16LE(centralExtra.length, 30); + central.writeUInt32LE(entry.externalAttributes ?? (0x81a0 << 16) >>> 0, 38); + central.writeUInt32LE(localOffset, 42); + centralParts.push(central, rawName, centralExtra); + localOffset += local.length + rawName.length + localExtra.length + compressedData.length; + } + + const centralDirectory = Buffer.concat(centralParts); + const eocd = Buffer.alloc(22); + eocd.writeUInt32LE(0x06054b50, 0); + eocd.writeUInt16LE(entries.length, 8); + eocd.writeUInt16LE(entries.length, 10); + eocd.writeUInt32LE(centralDirectory.length, 12); + eocd.writeUInt32LE(localOffset, 16); + return Buffer.concat([...localParts, centralDirectory, eocd]); +} + +function withArchive(bytes: Buffer, callback: (darPath: string) => void): void { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dar-archive-policy-')); + try { + const darPath = path.join(root, 'candidate.dar'); + fs.writeFileSync(darPath, bytes); + callback(darPath); + } finally { + fs.rmSync(root, { force: true, recursive: true }); + } +} + +function zip64Extra(payloadBytes = 8): Buffer { + const extra = Buffer.alloc(4 + payloadBytes); + extra.writeUInt16LE(0x0001, 0); + extra.writeUInt16LE(payloadBytes, 2); + return extra; +} + +void describe('hostile DAR archive policy', () => { + void it('accepts a bounded ordinary ZIP and a real repository DAR when materialized', () => { + withArchive(makeZip([{ name: 'META-INF/MANIFEST.MF' }]), (darPath) => { + assert.deepEqual(assertDarArchiveSafe(darPath), { entryCount: 1, totalUncompressedBytes: 1 }); + }); + + const repositoryDar = path.join(__dirname, '..', 'dars', 'OpenCapTable-v34', '0.0.1', 'OpenCapTable-v34.dar'); + if (fs.existsSync(repositoryDar)) { + const result = assertDarArchiveSafe(repositoryDar); + assert.ok(result.entryCount > 0); + assert.ok(result.totalUncompressedBytes > 0); + } + }); + + void it('rejects malformed EOCD and local bounds', () => { + withArchive(Buffer.from('not a zip'), (darPath) => { + assert.throws(() => assertDarArchiveSafe(darPath), /EOCD|too small/); + }); + const malformed = makeZip([{ name: 'safe.dalf' }]); + const eocdOffset = malformed.length - 22; + const centralOffset = malformed.readUInt32LE(eocdOffset + 16); + malformed.writeUInt32LE(centralOffset, centralOffset + 42); + withArchive(malformed, (darPath) => { + assert.throws(() => assertDarArchiveSafe(darPath), /local header.*outside archive bounds/); + }); + }); + + void it('rejects traversal, encrypted entries, and Unix symlinks', () => { + withArchive(makeZip([{ name: '../escape.dalf' }]), (darPath) => { + assert.throws(() => assertDarArchiveSafe(darPath), /traversal path/); + }); + withArchive(makeZip([{ flags: 0x0001, name: 'encrypted.dalf' }]), (darPath) => { + assert.throws(() => assertDarArchiveSafe(darPath), /encrypted entry/); + }); + withArchive(makeZip([{ externalAttributes: (0xa000 << 16) >>> 0, name: 'symlink.dalf' }]), (darPath) => { + assert.throws(() => assertDarArchiveSafe(darPath), /symlink entry/); + }); + }); + + void it('rejects ZIP64 metadata while permitting only DAMLC no-op size echoes', () => { + withArchive(makeZip([{ centralExtra: zip64Extra(), name: 'zip64.dalf' }]), (darPath) => { + assert.throws(() => assertDarArchiveSafe(darPath), /ZIP64 is not permitted/); + }); + + const localEcho = zip64Extra(16); + localEcho.writeBigUInt64LE(1n, 4); + localEcho.writeBigUInt64LE(1n, 12); + withArchive( + makeZip([{ centralExtra: zip64Extra(0), localExtra: localEcho, name: 'damlc-compatible.dalf' }]), + (darPath) => assert.doesNotThrow(() => assertDarArchiveSafe(darPath)) + ); + }); + + void it('bounds entry count plus per-entry and aggregate uncompressed sizes', () => { + const tooMany = Array.from({ length: MAX_DAR_ARCHIVE_ENTRIES + 1 }, (_, index) => ({ name: `e${index}` })); + withArchive(makeZip(tooMany), (darPath) => { + assert.throws(() => assertDarArchiveSafe(darPath), /entry count.*exceeds allowed range/); + }); + withArchive( + makeZip([ + { + data: Buffer.from('x'), + method: 8, + name: 'large.dalf', + uncompressedSize: MAX_DAR_ENTRY_UNCOMPRESSED_BYTES + 1, + }, + ]), + (darPath) => assert.throws(() => assertDarArchiveSafe(darPath), /declares.*uncompressed bytes/) + ); + withArchive( + makeZip( + Array.from({ length: 5 }, (_, index) => ({ + data: Buffer.from('x'), + method: 8, + name: `aggregate-${index}.dalf`, + uncompressedSize: MAX_DAR_ENTRY_UNCOMPRESSED_BYTES, + })) + ), + (darPath) => assert.throws(() => assertDarArchiveSafe(darPath), /aggregate uncompressed size exceeds/) + ); + }); + + void it('bounds actual inflation even when a deflated entry advertises a tiny size', () => { + withArchive( + makeZip([ + { + data: Buffer.alloc(MAX_DAR_ENTRY_UNCOMPRESSED_BYTES + 1), + method: 8, + name: 'forged-small-size.dalf', + uncompressedSize: 1, + }, + ]), + (darPath) => assert.throws(() => assertDarArchiveSafe(darPath), /actual expansion exceeds/) + ); + }); + + void it('verifies actual uncompressed size and CRC-32', () => { + withArchive( + makeZip([ + { + data: Buffer.from('larger than advertised'), + method: 8, + name: 'wrong-size.dalf', + uncompressedSize: 1, + }, + ]), + (darPath) => assert.throws(() => assertDarArchiveSafe(darPath), /actual uncompressed size/) + ); + withArchive(makeZip([{ advertisedCrc32: 0, data: Buffer.from('nonempty'), name: 'wrong-crc.dalf' }]), (darPath) => + assert.throws(() => assertDarArchiveSafe(darPath), /CRC-32 mismatch/) + ); + }); +}); diff --git a/scripts/dar-archive-policy.ts b/scripts/dar-archive-policy.ts new file mode 100644 index 00000000..906774ae --- /dev/null +++ b/scripts/dar-archive-policy.ts @@ -0,0 +1,391 @@ +import * as fs from 'fs'; +import { inflateRawSync } from 'node:zlib'; + +const EOCD_SIGNATURE = 0x06054b50; +const CENTRAL_FILE_SIGNATURE = 0x02014b50; +const LOCAL_FILE_SIGNATURE = 0x04034b50; +const ZIP64_EXTRA_FIELD_ID = 0x0001; +const ZIP64_UINT16 = 0xffff; +const ZIP64_UINT32 = 0xffffffff; +const UNIX_FILE_TYPE_MASK = 0xf000; +const UNIX_SYMLINK_TYPE = 0xa000; + +export const MAX_DAR_ARCHIVE_BYTES = 100 * 1024 * 1024; +export const MAX_DAR_ARCHIVE_ENTRIES = 4096; +export const MAX_DAR_ENTRY_UNCOMPRESSED_BYTES = 64 * 1024 * 1024; +export const MAX_DAR_TOTAL_UNCOMPRESSED_BYTES = 256 * 1024 * 1024; +export const MAX_DAR_ENTRY_NAME_BYTES = 4096; +export const DAML_INSPECT_TIMEOUT_MS = 60_000; +export const DAML_UPGRADE_CHECK_TIMEOUT_MS = 5 * 60_000; + +const MAX_EOCD_SEARCH_BYTES = 22 + 0xffff; +const MAX_CENTRAL_DIRECTORY_BYTES = 8 * 1024 * 1024; +const ALLOWED_GENERAL_PURPOSE_FLAGS = 0x0008 | 0x0800; +const ALLOWED_COMPRESSION_METHODS = new Set([0, 8]); +const INFLATE_CHUNK_BYTES = 64 * 1024; + +const CRC32_TABLE = new Uint32Array(256); +for (let value = 0; value < CRC32_TABLE.length; value++) { + let remainder = value; + for (let bit = 0; bit < 8; bit++) { + remainder = (remainder & 1) !== 0 ? 0xedb88320 ^ (remainder >>> 1) : remainder >>> 1; + } + CRC32_TABLE[value] = remainder >>> 0; +} + +export interface DarArchiveSummary { + entryCount: number; + totalUncompressedBytes: number; +} + +interface ArchiveInterval { + end: number; + name: string; + start: number; +} + +interface InflateRawInfo { + buffer: Buffer; + engine: { bytesWritten: number }; +} + +function crc32(bytes: Buffer): number { + let checksum = 0xffffffff; + for (const byte of bytes) checksum = CRC32_TABLE[(checksum ^ byte) & 0xff] ^ (checksum >>> 8); + return (checksum ^ 0xffffffff) >>> 0; +} + +function inflateRawBounded(name: string, compressed: Buffer, maximumBytes: number): Buffer { + try { + // Node's native inflater processes the stream incrementally and enforces maxOutputLength while producing output, + // so a forged central-directory size cannot make it allocate beyond the remaining entry/aggregate budget. + const result = inflateRawSync(compressed, { + chunkSize: INFLATE_CHUNK_BYTES, + info: true, + maxOutputLength: Math.max(1, maximumBytes), + }) as unknown as InflateRawInfo; + if (result.engine.bytesWritten !== compressed.length) { + throw new Error(`Unsafe DAR ZIP: deflated entry has trailing compressed bytes: ${name}`); + } + if (result.buffer.length > maximumBytes) { + throw new Error(`Unsafe DAR ZIP: actual expansion exceeds the remaining bounded size: ${name}`); + } + return result.buffer; + } catch (error) { + if (error instanceof Error && error.message.startsWith('Unsafe DAR ZIP:')) throw error; + if ((error as NodeJS.ErrnoException).code === 'ERR_BUFFER_TOO_LARGE') { + throw new Error(`Unsafe DAR ZIP: actual expansion exceeds the remaining bounded size: ${name}`); + } + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`Malformed DAR ZIP: invalid deflate stream for ${name}: ${detail}`); + } +} + +function readExactly(fd: number, length: number, position: number, label: string): Buffer { + const buffer = Buffer.alloc(length); + const bytesRead = fs.readSync(fd, buffer, 0, length, position); + if (bytesRead !== length) throw new Error(`Malformed DAR ZIP: truncated ${label}`); + return buffer; +} + +function assertRange(start: number, length: number, limit: number, label: string): void { + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(length) || + start < 0 || + length < 0 || + start + length > limit + ) { + throw new Error(`Malformed DAR ZIP: ${label} is outside archive bounds`); + } +} + +function assertNoZip64Extra( + extra: Buffer, + label: string, + localSizeEcho?: { compressedSize: number; uncompressedSize: number } +): void { + let cursor = 0; + while (cursor < extra.length) { + if (cursor + 4 > extra.length) throw new Error(`Malformed DAR ZIP: truncated ${label} extra field`); + const fieldId = extra.readUInt16LE(cursor); + const fieldSize = extra.readUInt16LE(cursor + 2); + cursor += 4; + if (cursor + fieldSize > extra.length) throw new Error(`Malformed DAR ZIP: truncated ${label} extra value`); + if (fieldId === ZIP64_EXTRA_FIELD_ID) { + // DAMLC emits two redundant ZIP64 encodings on otherwise ordinary ZIP32 archives: an empty central field, and a + // 16-byte local field that exactly repeats the authoritative 32-bit uncompressed/compressed sizes. Permit only + // those no-op encodings; any sentinel, extra payload, or mismatch is genuine/ambiguous ZIP64 and is rejected. + const isEmptyPlaceholder = fieldSize === 0; + const isExactLocalSizeEcho = + localSizeEcho !== undefined && + fieldSize === 16 && + extra.readBigUInt64LE(cursor) === BigInt(localSizeEcho.uncompressedSize) && + extra.readBigUInt64LE(cursor + 8) === BigInt(localSizeEcho.compressedSize); + if (!isEmptyPlaceholder && !isExactLocalSizeEcho) { + throw new Error(`Unsafe DAR ZIP: ZIP64 is not permitted (${label})`); + } + } + cursor += fieldSize; + } +} + +function decodeAndValidateEntryName(rawName: Buffer): string { + if (rawName.length === 0 || rawName.length > MAX_DAR_ENTRY_NAME_BYTES) { + throw new Error(`Unsafe DAR ZIP: entry name must be 1-${MAX_DAR_ENTRY_NAME_BYTES} bytes`); + } + for (const byte of rawName) { + if (byte < 0x20 || byte > 0x7e) { + throw new Error('Unsafe DAR ZIP: entry names must contain printable ASCII only'); + } + } + const name = rawName.toString('ascii'); + if (name.startsWith('/') || name.startsWith('\\') || /^[A-Za-z]:/.test(name) || name.includes('\\')) { + throw new Error(`Unsafe DAR ZIP path: ${name}`); + } + const segments = name.split('/'); + if (segments.some((segment) => segment === '' || segment === '.' || segment === '..')) { + throw new Error(`Unsafe DAR ZIP traversal path: ${name}`); + } + return name; +} + +function findEocd(fd: number, archiveSize: number): { offset: number; record: Buffer } { + if (archiveSize < 22) throw new Error('Malformed DAR ZIP: archive is too small for EOCD'); + const tailLength = Math.min(archiveSize, MAX_EOCD_SEARCH_BYTES); + const tailOffset = archiveSize - tailLength; + const tail = readExactly(fd, tailLength, tailOffset, 'EOCD search window'); + for (let index = tail.length - 22; index >= 0; index--) { + if (tail.readUInt32LE(index) !== EOCD_SIGNATURE) continue; + const commentLength = tail.readUInt16LE(index + 20); + if (index + 22 + commentLength !== tail.length) continue; + return { offset: tailOffset + index, record: tail.subarray(index, index + 22) }; + } + throw new Error('Malformed DAR ZIP: EOCD record not found'); +} + +function assertSafeFlags(flags: number, name: string): void { + if ((flags & ~ALLOWED_GENERAL_PURPOSE_FLAGS) !== 0) { + if ((flags & (0x0001 | 0x0040 | 0x2000)) !== 0) { + throw new Error(`Unsafe DAR ZIP: encrypted entry is not permitted: ${name}`); + } + throw new Error(`Unsafe DAR ZIP: unsupported general-purpose flags 0x${flags.toString(16)}: ${name}`); + } +} + +/** + * Validate a DAR as a conservative, bounded ZIP before any external DAML tool parses it. Deflated entries are inflated + * under hard actual-output limits, then checked against their declared sizes and CRC-32 values. + */ +export function assertDarArchiveSafe(darPath: string): DarArchiveSummary { + const stats = fs.lstatSync(darPath); + if (!stats.isFile()) throw new Error(`DAR must be a regular file: ${darPath}`); + if (stats.size > MAX_DAR_ARCHIVE_BYTES) { + throw new Error(`Unsafe DAR ZIP: archive exceeds ${MAX_DAR_ARCHIVE_BYTES} bytes: ${darPath}`); + } + + const fd = fs.openSync(darPath, 'r'); + try { + const { offset: eocdOffset, record: eocd } = findEocd(fd, stats.size); + const diskNumber = eocd.readUInt16LE(4); + const centralDirectoryDisk = eocd.readUInt16LE(6); + const entriesOnDisk = eocd.readUInt16LE(8); + const totalEntries = eocd.readUInt16LE(10); + const centralDirectorySize = eocd.readUInt32LE(12); + const centralDirectoryOffset = eocd.readUInt32LE(16); + + if ( + diskNumber !== 0 || + centralDirectoryDisk !== 0 || + entriesOnDisk !== totalEntries || + entriesOnDisk === ZIP64_UINT16 || + totalEntries === ZIP64_UINT16 || + centralDirectorySize === ZIP64_UINT32 || + centralDirectoryOffset === ZIP64_UINT32 + ) { + throw new Error('Unsafe DAR ZIP: multidisk and ZIP64 archives are not permitted'); + } + if (totalEntries === 0 || totalEntries > MAX_DAR_ARCHIVE_ENTRIES) { + throw new Error(`Unsafe DAR ZIP: entry count ${totalEntries} exceeds allowed range 1-${MAX_DAR_ARCHIVE_ENTRIES}`); + } + if (centralDirectorySize > MAX_CENTRAL_DIRECTORY_BYTES) { + throw new Error(`Unsafe DAR ZIP: central directory exceeds ${MAX_CENTRAL_DIRECTORY_BYTES} bytes`); + } + assertRange(centralDirectoryOffset, centralDirectorySize, eocdOffset, 'central directory'); + if (centralDirectoryOffset + centralDirectorySize !== eocdOffset) { + throw new Error('Malformed DAR ZIP: central directory does not end at EOCD'); + } + + const central = readExactly(fd, centralDirectorySize, centralDirectoryOffset, 'central directory'); + let declaredCursor = 0; + let declaredTotalUncompressedBytes = 0; + for (let entryIndex = 0; entryIndex < totalEntries; entryIndex++) { + if (declaredCursor + 46 > central.length || central.readUInt32LE(declaredCursor) !== CENTRAL_FILE_SIGNATURE) { + throw new Error(`Malformed DAR ZIP: invalid central header at entry ${entryIndex + 1}`); + } + const uncompressedSize = central.readUInt32LE(declaredCursor + 24); + if (uncompressedSize > MAX_DAR_ENTRY_UNCOMPRESSED_BYTES) { + throw new Error( + `Unsafe DAR ZIP: entry ${entryIndex + 1} declares ${uncompressedSize} uncompressed bytes; maximum is ${MAX_DAR_ENTRY_UNCOMPRESSED_BYTES}` + ); + } + declaredTotalUncompressedBytes += uncompressedSize; + if (declaredTotalUncompressedBytes > MAX_DAR_TOTAL_UNCOMPRESSED_BYTES) { + throw new Error( + `Unsafe DAR ZIP: aggregate uncompressed size exceeds ${MAX_DAR_TOTAL_UNCOMPRESSED_BYTES} bytes` + ); + } + const headerLength = + 46 + + central.readUInt16LE(declaredCursor + 28) + + central.readUInt16LE(declaredCursor + 30) + + central.readUInt16LE(declaredCursor + 32); + if (declaredCursor + headerLength > central.length) { + throw new Error(`Malformed DAR ZIP: central entry ${entryIndex + 1} exceeds directory bounds`); + } + declaredCursor += headerLength; + } + if (declaredCursor !== central.length) { + throw new Error('Malformed DAR ZIP: unparsed central-directory bytes remain'); + } + + const names = new Set(); + const localOffsets = new Set(); + const intervals: ArchiveInterval[] = []; + let cursor = 0; + let actualTotalUncompressedBytes = 0; + + for (let entryIndex = 0; entryIndex < totalEntries; entryIndex++) { + if (cursor + 46 > central.length || central.readUInt32LE(cursor) !== CENTRAL_FILE_SIGNATURE) { + throw new Error(`Malformed DAR ZIP: invalid central header at entry ${entryIndex + 1}`); + } + const versionNeeded = central.readUInt16LE(cursor + 6); + const flags = central.readUInt16LE(cursor + 8); + const method = central.readUInt16LE(cursor + 10); + const expectedCrc32 = central.readUInt32LE(cursor + 16); + const compressedSize = central.readUInt32LE(cursor + 20); + const uncompressedSize = central.readUInt32LE(cursor + 24); + const nameLength = central.readUInt16LE(cursor + 28); + const extraLength = central.readUInt16LE(cursor + 30); + const commentLength = central.readUInt16LE(cursor + 32); + const startDisk = central.readUInt16LE(cursor + 34); + const externalAttributes = central.readUInt32LE(cursor + 38); + const localOffset = central.readUInt32LE(cursor + 42); + const headerLength = 46 + nameLength + extraLength + commentLength; + if (cursor + headerLength > central.length) { + throw new Error(`Malformed DAR ZIP: central entry ${entryIndex + 1} exceeds directory bounds`); + } + if ( + (versionNeeded & 0xff) >= 45 || + compressedSize === ZIP64_UINT32 || + uncompressedSize === ZIP64_UINT32 || + localOffset === ZIP64_UINT32 || + startDisk === ZIP64_UINT16 + ) { + throw new Error(`Unsafe DAR ZIP: ZIP64 metadata is not permitted at entry ${entryIndex + 1}`); + } + + const rawName = central.subarray(cursor + 46, cursor + 46 + nameLength); + const name = decodeAndValidateEntryName(rawName); + assertSafeFlags(flags, name); + if (!ALLOWED_COMPRESSION_METHODS.has(method)) { + throw new Error(`Unsafe DAR ZIP: unsupported compression method ${method}: ${name}`); + } + if (method === 0 && compressedSize !== uncompressedSize) { + throw new Error(`Malformed DAR ZIP: stored entry size mismatch: ${name}`); + } + const extra = central.subarray(cursor + 46 + nameLength, cursor + 46 + nameLength + extraLength); + assertNoZip64Extra(extra, `central entry ${name}`); + const unixMode = externalAttributes >>> 16; + if ((unixMode & UNIX_FILE_TYPE_MASK) === UNIX_SYMLINK_TYPE) { + throw new Error(`Unsafe DAR ZIP: symlink entry is not permitted: ${name}`); + } + if (names.has(name)) throw new Error(`Unsafe DAR ZIP: duplicate entry path: ${name}`); + if (localOffsets.has(localOffset)) + throw new Error(`Unsafe DAR ZIP: duplicate local header offset: ${localOffset}`); + names.add(name); + localOffsets.add(localOffset); + assertRange(localOffset, 30, centralDirectoryOffset, `local header for ${name}`); + const local = readExactly(fd, 30, localOffset, `local header for ${name}`); + if (local.readUInt32LE(0) !== LOCAL_FILE_SIGNATURE) { + throw new Error(`Malformed DAR ZIP: invalid local header signature: ${name}`); + } + const localVersionNeeded = local.readUInt16LE(4); + const localFlags = local.readUInt16LE(6); + const localMethod = local.readUInt16LE(8); + const localCrc32 = local.readUInt32LE(14); + const localCompressedSize = local.readUInt32LE(18); + const localUncompressedSize = local.readUInt32LE(22); + const localNameLength = local.readUInt16LE(26); + const localExtraLength = local.readUInt16LE(28); + if ( + (localVersionNeeded & 0xff) >= 45 || + localCompressedSize === ZIP64_UINT32 || + localUncompressedSize === ZIP64_UINT32 + ) { + throw new Error(`Unsafe DAR ZIP: ZIP64 local metadata is not permitted: ${name}`); + } + if (localFlags !== flags || localMethod !== method) { + throw new Error(`Malformed DAR ZIP: local/central metadata mismatch: ${name}`); + } + const localVariableLength = localNameLength + localExtraLength; + assertRange(localOffset + 30, localVariableLength, centralDirectoryOffset, `local metadata for ${name}`); + const localVariable = readExactly(fd, localVariableLength, localOffset + 30, `local metadata for ${name}`); + const localName = localVariable.subarray(0, localNameLength); + if (!localName.equals(rawName)) throw new Error(`Malformed DAR ZIP: local/central filename mismatch: ${name}`); + assertNoZip64Extra(localVariable.subarray(localNameLength), `local entry ${name}`, { + compressedSize, + uncompressedSize, + }); + if ((flags & 0x0008) === 0) { + if ( + localCrc32 !== expectedCrc32 || + localCompressedSize !== compressedSize || + localUncompressedSize !== uncompressedSize + ) { + throw new Error(`Malformed DAR ZIP: local/central CRC or size mismatch: ${name}`); + } + } else if ( + (localCrc32 !== 0 && localCrc32 !== expectedCrc32) || + (localCompressedSize !== 0 && localCompressedSize !== compressedSize) || + (localUncompressedSize !== 0 && localUncompressedSize !== uncompressedSize) + ) { + throw new Error(`Malformed DAR ZIP: data-descriptor metadata mismatch: ${name}`); + } + const dataStart = localOffset + 30 + localVariableLength; + assertRange(dataStart, compressedSize, centralDirectoryOffset, `compressed data for ${name}`); + const compressed = readExactly(fd, compressedSize, dataStart, `compressed data for ${name}`); + const remainingAggregateBytes = MAX_DAR_TOTAL_UNCOMPRESSED_BYTES - actualTotalUncompressedBytes; + const maximumActualBytes = Math.min(MAX_DAR_ENTRY_UNCOMPRESSED_BYTES, remainingAggregateBytes); + const expanded = method === 0 ? compressed : inflateRawBounded(name, compressed, maximumActualBytes); + if (expanded.length > maximumActualBytes) { + throw new Error(`Unsafe DAR ZIP: actual expansion exceeds the remaining bounded size: ${name}`); + } + if (expanded.length !== uncompressedSize) { + throw new Error( + `Malformed DAR ZIP: actual uncompressed size ${expanded.length} disagrees with declared size ${uncompressedSize}: ${name}` + ); + } + if (crc32(expanded) !== expectedCrc32) { + throw new Error(`Malformed DAR ZIP: CRC-32 mismatch: ${name}`); + } + actualTotalUncompressedBytes += expanded.length; + intervals.push({ end: dataStart + compressedSize, name, start: localOffset }); + cursor += headerLength; + } + + if (cursor !== central.length) throw new Error('Malformed DAR ZIP: unparsed central-directory bytes remain'); + intervals.sort((left, right) => left.start - right.start); + for (let index = 1; index < intervals.length; index++) { + const previous = intervals[index - 1]; + const current = intervals[index]; + if (current.start < previous.end) { + throw new Error(`Malformed DAR ZIP: local entries overlap: ${previous.name} and ${current.name}`); + } + } + + return { entryCount: totalEntries, totalUncompressedBytes: actualTotalUncompressedBytes }; + } finally { + fs.closeSync(fd); + } +} diff --git a/scripts/dar-backup-transaction.ts b/scripts/dar-backup-transaction.ts new file mode 100644 index 00000000..fe2d26a9 --- /dev/null +++ b/scripts/dar-backup-transaction.ts @@ -0,0 +1,237 @@ +import * as crypto from 'crypto'; +import * as fs from 'fs'; +import * as path from 'path'; +import { computeSha256, saveDarsLock, type DarsLock, type DarsLockEntry } from './dar-utils'; +import type { BackupRetentionPlan } from './dar-version-policy'; + +export interface CandidateBackupWrite { + lockKey: string; + sourcePath: string; + destPath: string; + entry: DarsLockEntry; + replaceExisting: boolean; +} + +interface BackupTransactionOptions { + lock: DarsLock; + retentionPlan: BackupRetentionPlan; + darsDir: string; + candidateWrite?: CandidateBackupWrite; + /** Injectable only so transaction rollback can be tested without touching the repository lock. */ + saveLock?: (lock: DarsLock) => void; +} + +interface StagedFile { + originalPath: string; + stagedPath: string; +} + +function cloneLock(lock: DarsLock): DarsLock { + return { + version: lock.version, + packages: Object.fromEntries( + Object.entries(lock.packages).map(([key, entry]) => [key, { ...entry, networks: [...entry.networks] }]) + ), + }; +} + +function sortLockPackages(lock: DarsLock): void { + const sorted: Record = {}; + for (const key of Object.keys(lock.packages).sort()) sorted[key] = lock.packages[key]; + lock.packages = sorted; +} + +function resolveLockPath(darsDir: string, lockKey: string): string { + const darsRoot = path.resolve(darsDir); + const filePath = path.resolve(darsRoot, lockKey); + if (!filePath.startsWith(`${darsRoot}${path.sep}`)) { + throw new Error(`Unsafe DAR lock key: ${lockKey}`); + } + return filePath; +} + +function removeEmptyParentDirectories(filePath: string, darsDir: string): void { + const darsRoot = path.resolve(darsDir); + let directory = path.dirname(filePath); + while (directory.startsWith(`${darsRoot}${path.sep}`) && directory !== darsRoot) { + try { + fs.rmdirSync(directory); + directory = path.dirname(directory); + } catch { + return; + } + } +} + +function assertCandidateWrite(candidateWrite: CandidateBackupWrite, darsDir: string, lock: DarsLock): void { + const expectedDestPath = resolveLockPath(darsDir, candidateWrite.lockKey); + if (path.resolve(candidateWrite.destPath) !== expectedDestPath) { + throw new Error(`Candidate destination does not match its lock key: ${candidateWrite.lockKey}`); + } + const sourceStats = fs.lstatSync(candidateWrite.sourcePath); + if (!sourceStats.isFile()) { + throw new Error(`Refusing non-regular candidate DAR: ${candidateWrite.sourcePath}`); + } + if (sourceStats.size !== candidateWrite.entry.size) { + throw new Error( + `Candidate DAR has size ${sourceStats.size}; transaction entry requires ${candidateWrite.entry.size}.` + ); + } + const sourceHash = computeSha256(candidateWrite.sourcePath); + if (sourceHash !== candidateWrite.entry.sha256) { + throw new Error( + `Candidate DAR has SHA256 ${sourceHash}; transaction entry requires ${candidateWrite.entry.sha256}.` + ); + } + + const hasEntry = Object.prototype.hasOwnProperty.call(lock.packages, candidateWrite.lockKey); + const hasFile = fs.existsSync(candidateWrite.destPath); + if (candidateWrite.replaceExisting) { + if (!hasEntry || !hasFile) { + throw new Error(`Cannot replace missing candidate backup: ${candidateWrite.lockKey}`); + } + } else if (hasEntry || hasFile) { + throw new Error(`Cannot create candidate backup over existing state: ${candidateWrite.lockKey}`); + } +} + +function assertFileMatchesEntry(filePath: string, entry: DarsLockEntry): void { + const stats = fs.lstatSync(filePath); + if (!stats.isFile() || stats.size !== entry.size || computeSha256(filePath) !== entry.sha256) { + throw new Error(`Staged candidate DAR does not match its transaction entry: ${filePath}`); + } +} + +/** + * Apply candidate replacement, live-marker updates, pruning, and the lock update as one rollback-capable operation. The + * lock is saved only after every new filesystem state has been installed. If that save fails, all DAR paths are + * restored before the error escapes. + */ +export function applyBackupTransaction(options: BackupTransactionOptions): void { + const { lock, retentionPlan, darsDir, candidateWrite, saveLock = saveDarsLock } = options; + if (retentionPlan.freezeKeys.length === 0 && retentionPlan.pruneKeys.length === 0 && candidateWrite === undefined) { + return; + } + + if (candidateWrite) assertCandidateWrite(candidateWrite, darsDir, lock); + if (candidateWrite && retentionPlan.pruneKeys.includes(candidateWrite.lockKey)) { + throw new Error(`Cannot prune and write the same candidate backup: ${candidateWrite.lockKey}`); + } + + for (const key of retentionPlan.freezeKeys) { + if (!Object.prototype.hasOwnProperty.call(lock.packages, key)) { + throw new Error(`Cannot freeze missing lock entry: ${key}`); + } + } + for (const key of retentionPlan.pruneKeys) { + if (!Object.prototype.hasOwnProperty.call(lock.packages, key)) { + throw new Error(`Cannot prune missing lock entry: ${key}`); + } + const prunePath = resolveLockPath(darsDir, key); + if (!fs.existsSync(prunePath)) throw new Error(`Cannot prune missing DAR backup: ${key}`); + } + + const nextLock = cloneLock(lock); + for (const key of retentionPlan.freezeKeys) { + const { networks } = nextLock.packages[key]; + if (!networks.includes('devnet')) networks.push('devnet'); + networks.sort(); + } + for (const key of retentionPlan.pruneKeys) delete nextLock.packages[key]; + if (candidateWrite) nextLock.packages[candidateWrite.lockKey] = candidateWrite.entry; + sortLockPackages(nextLock); + + const transactionId = `${process.pid}-${Date.now()}-${crypto.randomUUID()}`; + const stagedPrunes: StagedFile[] = []; + let stagedCandidatePath: string | undefined; + let previousCandidatePath: string | undefined; + let candidateInstalled = false; + + try { + if (candidateWrite) { + fs.mkdirSync(path.dirname(candidateWrite.destPath), { recursive: true }); + stagedCandidatePath = `${candidateWrite.destPath}.candidate-${transactionId}`; + fs.copyFileSync(candidateWrite.sourcePath, stagedCandidatePath, fs.constants.COPYFILE_EXCL); + assertFileMatchesEntry(stagedCandidatePath, candidateWrite.entry); + } + + for (const key of retentionPlan.pruneKeys) { + const originalPath = resolveLockPath(darsDir, key); + const stagedPath = `${originalPath}.prune-${transactionId}`; + fs.renameSync(originalPath, stagedPath); + stagedPrunes.push({ originalPath, stagedPath }); + } + + if (candidateWrite && stagedCandidatePath) { + if (candidateWrite.replaceExisting) { + previousCandidatePath = `${candidateWrite.destPath}.previous-${transactionId}`; + fs.renameSync(candidateWrite.destPath, previousCandidatePath); + } + fs.renameSync(stagedCandidatePath, candidateWrite.destPath); + stagedCandidatePath = undefined; + candidateInstalled = true; + } + + saveLock(nextLock); + lock.version = nextLock.version; + lock.packages = nextLock.packages; + } catch (error) { + const rollbackErrors: string[] = []; + if (candidateInstalled && candidateWrite) { + try { + fs.unlinkSync(candidateWrite.destPath); + } catch (rollbackError) { + rollbackErrors.push(`remove candidate: ${String(rollbackError)}`); + } + } + if (previousCandidatePath && candidateWrite && fs.existsSync(previousCandidatePath)) { + try { + fs.renameSync(previousCandidatePath, candidateWrite.destPath); + } catch (rollbackError) { + rollbackErrors.push(`restore candidate: ${String(rollbackError)}`); + } + } + for (const { originalPath, stagedPath } of [...stagedPrunes].reverse()) { + if (!fs.existsSync(stagedPath)) continue; + try { + fs.renameSync(stagedPath, originalPath); + } catch (rollbackError) { + rollbackErrors.push(`restore ${originalPath}: ${String(rollbackError)}`); + } + } + if (stagedCandidatePath && fs.existsSync(stagedCandidatePath)) { + try { + fs.unlinkSync(stagedCandidatePath); + } catch (rollbackError) { + rollbackErrors.push(`remove staged candidate: ${String(rollbackError)}`); + } + } + if (candidateWrite) removeEmptyParentDirectories(candidateWrite.destPath, darsDir); + + if (rollbackErrors.length > 0) { + throw new Error( + `${error instanceof Error ? error.message : String(error)} Rollback also failed: ${rollbackErrors.join('; ')}` + ); + } + throw error; + } + + const cleanupFiles = [ + ...stagedPrunes.map(({ originalPath, stagedPath }) => ({ originalPath, stagedPath })), + ...(previousCandidatePath && candidateWrite + ? [{ originalPath: candidateWrite.destPath, stagedPath: previousCandidatePath }] + : []), + ]; + for (const { originalPath, stagedPath } of cleanupFiles) { + try { + fs.unlinkSync(stagedPath); + removeEmptyParentDirectories(originalPath, darsDir); + } catch (error) { + console.warn( + `Retention transaction committed, but staged file cleanup failed for ${stagedPath}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } +} diff --git a/scripts/dar-candidate-path-policy.ts b/scripts/dar-candidate-path-policy.ts new file mode 100644 index 00000000..1fde38cd --- /dev/null +++ b/scripts/dar-candidate-path-policy.ts @@ -0,0 +1,46 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { assertSafeDarTreePath } from './dar-lfs-policy'; + +function assertInsideRoot(rootRealPath: string, filePath: string, label: string): void { + const realPath = fs.realpathSync(filePath); + if (!realPath.startsWith(`${rootRealPath}${path.sep}`)) { + throw new Error(`${label} resolves outside the candidate root: ${filePath}`); + } +} + +function assertDirectory(rootRealPath: string, directory: string): void { + const stats = fs.lstatSync(directory); + if (stats.isSymbolicLink() || !stats.isDirectory()) { + throw new Error(`Candidate DAR parent must be a non-symlink directory: ${directory}`); + } + assertInsideRoot(rootRealPath, directory, 'Candidate DAR parent'); +} + +/** Verify candidate DAR paths and every parent are real, non-symlink paths confined to the detached worktree. */ +export function assertCandidateDarPaths(candidateRoot: string, darPaths: string[]): void { + const rootStats = fs.lstatSync(candidateRoot); + if (rootStats.isSymbolicLink() || !rootStats.isDirectory()) { + throw new Error(`Candidate root must be a non-symlink directory: ${candidateRoot}`); + } + const rootRealPath = fs.realpathSync(candidateRoot); + const darsDirectory = path.join(candidateRoot, 'dars'); + assertDirectory(rootRealPath, darsDirectory); + + for (const darPath of darPaths) { + assertSafeDarTreePath(darPath); + const components = darPath.split('/'); + let currentPath = candidateRoot; + for (const component of components.slice(0, -1)) { + currentPath = path.join(currentPath, component); + assertDirectory(rootRealPath, currentPath); + } + + const absoluteDarPath = path.join(candidateRoot, ...components); + const stats = fs.lstatSync(absoluteDarPath); + if (stats.isSymbolicLink() || !stats.isFile()) { + throw new Error(`Candidate DAR must be a non-symlink regular file: ${darPath}`); + } + assertInsideRoot(rootRealPath, absoluteDarPath, 'Candidate DAR'); + } +} diff --git a/scripts/dar-lfs-policy.ts b/scripts/dar-lfs-policy.ts new file mode 100644 index 00000000..c3d8f776 --- /dev/null +++ b/scripts/dar-lfs-policy.ts @@ -0,0 +1,58 @@ +const MAX_DAR_COUNT = 250; +const MAX_DAR_SIZE_BYTES = 100n * 1024n * 1024n; +const MAX_TOTAL_DAR_BYTES = 1024n * 1024n * 1024n; +const MAX_POINTER_BLOB_BYTES = 512; + +const SAFE_DAR_PATH = + /^dars\/[A-Za-z0-9][A-Za-z0-9._-]*\/(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\/[A-Za-z0-9][A-Za-z0-9._-]*\.dar$/; +const CANONICAL_LFS_POINTER = + /^version https:\/\/git-lfs\.github\.com\/spec\/v1\noid sha256:([0-9a-f]{64})\nsize (0|[1-9]\d*)\n$/; + +export interface DarTreeBlob { + blob: string; + mode: string; + path: string; + type: string; +} + +export interface ValidatedDarLfsPointer { + oid: string; + path: string; + size: number; +} + +export function assertSafeDarTreePath(darPath: string): void { + if (!SAFE_DAR_PATH.test(darPath)) throw new Error(`Unsafe DAR tree path: ${darPath}`); +} + +/** Validate one PR-tree DAR as a canonical, bounded Git LFS pointer before any LFS object download. */ +export function validateDarLfsPointer(entry: DarTreeBlob): ValidatedDarLfsPointer { + assertSafeDarTreePath(entry.path); + if (entry.mode !== '100644' || entry.type !== 'blob') { + throw new Error(`DAR tree path must be a non-executable regular blob: ${entry.path}`); + } + if (Buffer.byteLength(entry.blob, 'utf8') > MAX_POINTER_BLOB_BYTES) { + throw new Error(`DAR pointer blob is larger than ${MAX_POINTER_BLOB_BYTES} bytes: ${entry.path}`); + } + const match = CANONICAL_LFS_POINTER.exec(entry.blob); + if (!match) throw new Error(`DAR is not a canonical Git LFS pointer: ${entry.path}`); + + const size = BigInt(match[2]); + if (size > MAX_DAR_SIZE_BYTES) { + throw new Error(`DAR LFS object exceeds 100 MiB: ${entry.path} declares ${size} bytes`); + } + return { oid: match[1], path: entry.path, size: Number(size) }; +} + +/** Apply count and aggregate-size limits to every validated DAR path in the PR tree. */ +export function validateDarLfsTree(entries: DarTreeBlob[]): ValidatedDarLfsPointer[] { + if (entries.length > MAX_DAR_COUNT) { + throw new Error(`PR tree contains ${entries.length} DARs; maximum is ${MAX_DAR_COUNT}`); + } + const pointers = entries.map(validateDarLfsPointer); + const totalSize = pointers.reduce((total, pointer) => total + BigInt(pointer.size), 0n); + if (totalSize > MAX_TOTAL_DAR_BYTES) { + throw new Error(`PR tree DARs declare ${totalSize} bytes total; maximum is 1 GiB`); + } + return pointers; +} diff --git a/scripts/dar-marker-policy.test.ts b/scripts/dar-marker-policy.test.ts new file mode 100644 index 00000000..a858fb97 --- /dev/null +++ b/scripts/dar-marker-policy.test.ts @@ -0,0 +1,327 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { describe, it } from 'node:test'; +import { + assertContainedRegularFile, + MAX_CANDIDATE_LOCK_BYTES, + MAX_CANDIDATE_METADATA_BYTES, +} from './candidate-path-safety'; +import type { DarsLock, DarsLockEntry } from './dar-utils'; +import { + assertDarsLockSchema, + assertDevnetMarkerIdentity, + classifyCandidateOnlyBackups, + getExactLiveProviderNames, + getNetworkMarkerAdditions, +} from './dar-marker-policy'; +import { verifyLockedDar } from './devnet-dar-policy'; +import type { DevnetPackagePreference } from './devnet-package-versions'; + +const LOCK_KEY = 'Example-v01/1.2.3/Example-v01.dar'; + +function entry(networks: string[] = [], sha256 = 'a'.repeat(64)): DarsLockEntry { + return { networks, sdkVersion: '3.5.1', sha256, size: 12, uploadedAt: '2026-07-13T00:00:00.000Z' }; +} + +function lock(value?: DarsLockEntry): DarsLock { + return { version: 1, packages: value ? { [LOCK_KEY]: value } : {} }; +} + +function preference(provider: 'intellect' | '5n', packageId = 'exact'): DevnetPackagePreference { + return { + packageId, + packageName: 'Example-v01', + packageVersion: '1.2.3', + provider, + }; +} + +void describe('DAR network marker policy', () => { + void it('rejects unknown and duplicate markers', () => { + assert.throws(() => getNetworkMarkerAdditions(lock(), lock(entry(['staging']))), /unknown network marker.*staging/); + assert.throws( + () => getNetworkMarkerAdditions(lock(), lock(entry(['devnet', 'devnet']))), + /duplicate network marker.*devnet/ + ); + }); + + void it('rejects removal of a recorded marker', () => { + assert.throws( + () => getNetworkMarkerAdditions(lock(entry(['devnet'])), lock(entry([]))), + /recorded network marker.*removed.*devnet/ + ); + }); + + void it('keeps recorded metadata immutable', () => { + assert.throws( + () => getNetworkMarkerAdditions(lock(entry(['devnet'])), lock(entry(['devnet'], 'b'.repeat(64)))), + /recorded DAR metadata sha256 cannot change/ + ); + }); + + void it('rejects historical in-place replacement but permits the active mutable candidate', () => { + const base = lock(entry()); + const candidate = lock(entry([], 'b'.repeat(64))); + assert.throws( + () => getNetworkMarkerAdditions(base, candidate), + /historical unrecorded DAR metadata cannot be replaced in place/ + ); + assert.doesNotThrow(() => + getNetworkMarkerAdditions(base, candidate, { currentCandidateKeys: new Set([LOCK_KEY]) }) + ); + assert.doesNotThrow(() => getNetworkMarkerAdditions(base, candidate, { liveBaselineKeys: new Set([LOCK_KEY]) })); + }); + + void it('rejects malformed candidate lock entry fields', () => { + assert.throws( + () => + assertDarsLockSchema( + { version: 1, packages: { [LOCK_KEY]: { ...entry(), networks: 'devnet' } } }, + 'Candidate dars.lock' + ), + /networks must be an array of strings/ + ); + assert.throws( + () => assertDarsLockSchema(lock(entry([], 'ABC')), 'Candidate dars.lock'), + /sha256 must be 64 lowercase hexadecimal characters/ + ); + }); + + void it('returns a new devnet marker for exact-live verification', () => { + assert.deepEqual(getNetworkMarkerAdditions(lock(entry()), lock(entry(['devnet']))), [ + { + darName: 'Example-v01', + lockKey: LOCK_KEY, + network: 'devnet', + packageName: 'Example-v01', + packageVersion: '1.2.3', + }, + ]); + }); + + void it('allows mainnet with trusted-base DevNet or a same-candidate DevNet proof', () => { + assert.throws( + () => getNetworkMarkerAdditions(lock(entry(['devnet'])), lock(entry(['devnet', 'mainnet']))), + /new mainnet marker requires explicit trusted workflow provenance/ + ); + assert.deepEqual( + getNetworkMarkerAdditions(lock(entry(['devnet'])), lock(entry(['devnet', 'mainnet'])), { + authorizedMainnetMarkerLockKey: LOCK_KEY, + }), + [ + { + darName: 'Example-v01', + lockKey: LOCK_KEY, + network: 'mainnet', + packageName: 'Example-v01', + packageVersion: '1.2.3', + }, + ] + ); + assert.throws( + () => + getNetworkMarkerAdditions(lock(entry()), lock(entry(['mainnet'])), { + authorizedMainnetMarkerLockKey: LOCK_KEY, + allowSameCandidateMainnet: true, + }), + /mainnet requires a trusted-base devnet marker or a devnet marker proven in this candidate/ + ); + assert.deepEqual( + getNetworkMarkerAdditions(lock(entry()), lock(entry(['mainnet', 'devnet'])), { + authorizedMainnetMarkerLockKey: LOCK_KEY, + allowSameCandidateMainnet: true, + }), + [ + { + darName: 'Example-v01', + lockKey: LOCK_KEY, + network: 'devnet', + packageName: 'Example-v01', + packageVersion: '1.2.3', + }, + { + darName: 'Example-v01', + lockKey: LOCK_KEY, + network: 'mainnet', + packageName: 'Example-v01', + packageVersion: '1.2.3', + }, + ] + ); + }); + + void it('scopes Mainnet provenance to exactly the attested lock key', () => { + const otherLockKey = 'Other-v01/1.2.3/Other-v01.dar'; + const base: DarsLock = { + version: 1, + packages: { [LOCK_KEY]: entry(['devnet']), [otherLockKey]: entry(['devnet'], 'b'.repeat(64)) }, + }; + const candidate: DarsLock = { + version: 1, + packages: { + [LOCK_KEY]: entry(['devnet', 'mainnet']), + [otherLockKey]: entry(['devnet', 'mainnet'], 'b'.repeat(64)), + }, + }; + assert.throws( + () => + getNetworkMarkerAdditions(base, candidate, { + authorizedMainnetMarkerLockKey: LOCK_KEY, + }), + new RegExp(`verified attestation authorizes only ${LOCK_KEY.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}`) + ); + assert.throws( + () => + getNetworkMarkerAdditions(base, base, { + authorizedMainnetMarkerLockKey: LOCK_KEY, + }), + /must authorize exactly that one marker addition/ + ); + }); + + void it('rejects locked byte replacement and parent-directory symlink escape', () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ocp-marker-retention-')); + try { + const expectedBytes = Buffer.from('expected'); + const expectedEntry: DarsLockEntry = { + ...entry(), + sha256: createHash('sha256').update(expectedBytes).digest('hex'), + size: expectedBytes.length, + }; + const tamperedRoot = path.join(tempRoot, 'tampered'); + const tamperedDar = path.join(tamperedRoot, 'dars', LOCK_KEY); + fs.mkdirSync(path.dirname(tamperedDar), { recursive: true }); + fs.writeFileSync(tamperedDar, 'tampered'); + assert.throws(() => verifyLockedDar(tamperedRoot, lock(expectedEntry), LOCK_KEY), /SHA256.*requires/); + + const symlinkRoot = path.join(tempRoot, 'symlink'); + const outsideRoot = path.join(tempRoot, 'outside'); + fs.mkdirSync(path.join(symlinkRoot, 'dars'), { recursive: true }); + const outsideDar = path.join(outsideRoot, LOCK_KEY); + fs.mkdirSync(path.dirname(outsideDar), { recursive: true }); + fs.writeFileSync(outsideDar, expectedBytes); + fs.symlinkSync(path.join(outsideRoot, 'Example-v01'), path.join(symlinkRoot, 'dars', 'Example-v01')); + assert.throws( + () => verifyLockedDar(symlinkRoot, lock(expectedEntry), LOCK_KEY), + /escapes candidate root through a symlink/ + ); + } finally { + fs.rmSync(tempRoot, { force: true, recursive: true }); + } + }); + + void it('rejects oversized candidate metadata and lock data before parsing', () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ocp-marker-bounds-')); + try { + const metadataPath = path.join(tempRoot, 'Example-v01', 'daml.yaml'); + const lockPath = path.join(tempRoot, 'dars', 'dars.lock'); + fs.mkdirSync(path.dirname(metadataPath), { recursive: true }); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync(metadataPath, Buffer.alloc(MAX_CANDIDATE_METADATA_BYTES + 1)); + fs.writeFileSync(lockPath, Buffer.alloc(MAX_CANDIDATE_LOCK_BYTES + 1)); + assert.throws( + () => assertContainedRegularFile(tempRoot, metadataPath, 'DAML metadata', MAX_CANDIDATE_METADATA_BYTES), + /DAML metadata exceeds/ + ); + assert.throws( + () => assertContainedRegularFile(tempRoot, lockPath, 'dars.lock', MAX_CANDIDATE_LOCK_BYTES), + /dars\.lock exceeds/ + ); + } finally { + fs.rmSync(tempRoot, { force: true, recursive: true }); + } + }); + + void it('requires the exact locked identity on every configured DevNet provider', () => { + const addition = { lockKey: LOCK_KEY, packageName: 'Example-v01', packageVersion: '1.2.3' }; + const identity = { packageId: 'exact', packageName: 'Example-v01', packageVersion: '1.2.3' }; + assert.doesNotThrow(() => + assertDevnetMarkerIdentity(addition, identity, [preference('intellect'), preference('5n')], ['intellect', '5n']) + ); + assert.throws( + () => assertDevnetMarkerIdentity(addition, identity, [preference('intellect')], ['intellect', '5n']), + /preferred by every configured DevNet provider.*5n=absent/ + ); + assert.throws( + () => + assertDevnetMarkerIdentity( + addition, + identity, + [preference('intellect'), preference('5n', 'different')], + ['intellect', '5n'] + ), + /preferred by every configured DevNet provider.*5n=1\.2\.3\/different/ + ); + }); + + void it('allows only the canonical current candidate as candidate-only unrecorded history', () => { + const candidate = lock(entry()); + assert.throws( + () => classifyCandidateOnlyBackups(lock(), candidate, new Set(), []), + /candidate-only unrecorded DAR must be the canonical current candidate/ + ); + assert.deepEqual(classifyCandidateOnlyBackups(lock(), candidate, new Set([LOCK_KEY]), []), { + candidateOnlyKeys: [LOCK_KEY], + currentCandidateKeys: [LOCK_KEY], + restoredRecordedKeys: [], + }); + }); + + void it('preserves non-current B only when one split provider proves B is the exact live baseline', () => { + const liveIdentity = { packageId: 'partial-live', packageName: 'Example-v01', packageVersion: '1.2.3' }; + const splitPreferences: DevnetPackagePreference[] = [ + { ...liveIdentity, provider: 'intellect' }, + { + packageId: 'older-live', + packageName: 'Example-v01', + packageVersion: '1.2.2', + provider: '5n', + }, + ]; + assert.deepEqual(getExactLiveProviderNames(liveIdentity, splitPreferences), ['intellect']); + assert.deepEqual( + getExactLiveProviderNames({ ...liveIdentity, packageId: 'stale-base-candidate' }, splitPreferences), + [] + ); + + const candidate = lock(entry()); + assert.throws( + () => classifyCandidateOnlyBackups(lock(), candidate, new Set(), []), + /candidate-only unrecorded DAR must be the canonical current candidate/ + ); + assert.deepEqual(classifyCandidateOnlyBackups(lock(), candidate, new Set(), [], new Set([LOCK_KEY])), { + candidateOnlyKeys: [LOCK_KEY], + currentCandidateKeys: [], + restoredRecordedKeys: [], + }); + }); + + void it('restores a recorded alias only with an exact immutable identity from trusted history', () => { + const historicalEntry = entry(['devnet', 'mainnet']); + const restoredKey = 'Example-v01/1.2.3/restored-live.dar'; + const candidate: DarsLock = { version: 1, packages: { [restoredKey]: entry(['mainnet', 'devnet']) } }; + const historical = lock(historicalEntry); + const classification = classifyCandidateOnlyBackups(lock(), candidate, new Set(), [historical]); + assert.deepEqual(classification, { + candidateOnlyKeys: [restoredKey], + currentCandidateKeys: [], + restoredRecordedKeys: [restoredKey], + }); + assert.deepEqual( + getNetworkMarkerAdditions(lock(), candidate, { restoredRecordedKeys: new Set([restoredKey]) }), + [] + ); + + const forged: DarsLock = { + version: 1, + packages: { [restoredKey]: entry(['devnet', 'mainnet'], 'b'.repeat(64)) }, + }; + assert.throws( + () => classifyCandidateOnlyBackups(lock(), forged, new Set(), [historical]), + /no exact immutable identity in trusted default-branch history/ + ); + }); +}); diff --git a/scripts/dar-marker-policy.ts b/scripts/dar-marker-policy.ts new file mode 100644 index 00000000..32281429 --- /dev/null +++ b/scripts/dar-marker-policy.ts @@ -0,0 +1,314 @@ +import type { DarsLock, DarsLockEntry } from './dar-utils'; +import type { DevnetPackagePreference } from './devnet-package-versions'; + +const ALLOWED_NETWORKS = new Set(['devnet', 'mainnet']); +const IMMUTABLE_RECORDED_FIELDS = ['sha256', 'size', 'sdkVersion', 'uploadedAt'] as const; +const MAX_DAR_BYTES = 100 * 1024 * 1024; +const MAX_LOCK_ENTRIES = 10_000; + +export interface DarMarkerIdentity { + packageId: string; + packageName: string; + packageVersion: string; +} + +export interface NetworkMarkerAddition { + darName: string; + lockKey: string; + network: 'devnet' | 'mainnet'; + packageName: string; + packageVersion: string; +} + +export interface NetworkMarkerPolicyOptions { + authorizedMainnetMarkerLockKey?: string; + allowSameCandidateMainnet?: boolean; + currentCandidateKeys?: ReadonlySet; + liveBaselineKeys?: ReadonlySet; + restoredRecordedKeys?: ReadonlySet; +} + +export interface CandidateOnlyBackupClassification { + candidateOnlyKeys: string[]; + currentCandidateKeys: string[]; + restoredRecordedKeys: string[]; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** Validate every untrusted lock entry before any field is used for retention or path selection. */ +export function assertDarsLockSchema(lock: unknown, label: string): asserts lock is DarsLock { + if (!isRecord(lock) || !Number.isSafeInteger(lock.version) || lock.version !== 1 || !isRecord(lock.packages)) { + throw new Error(`${label}: expected version 1 and a packages object`); + } + if (Object.keys(lock.packages).length > MAX_LOCK_ENTRIES) { + throw new Error(`${label}: refusing more than ${MAX_LOCK_ENTRIES} DAR entries`); + } + + for (const [lockKey, rawEntry] of Object.entries(lock.packages)) { + parseLockKey(lockKey); + if (!isRecord(rawEntry)) { + throw new Error(`${label} ${lockKey}: entry must be an object`); + } + const { networks, sdkVersion, sha256, size, uploadedAt } = rawEntry as Partial; + if (typeof sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(sha256)) { + throw new Error(`${label} ${lockKey}: sha256 must be 64 lowercase hexadecimal characters`); + } + if (typeof size !== 'number' || !Number.isSafeInteger(size) || size < 0 || size > MAX_DAR_BYTES) { + throw new Error(`${label} ${lockKey}: size must be an integer between 0 and ${MAX_DAR_BYTES}`); + } + if (typeof sdkVersion !== 'string' || sdkVersion.length === 0 || sdkVersion.length > 100) { + throw new Error(`${label} ${lockKey}: sdkVersion must be a non-empty string of at most 100 characters`); + } + if (typeof uploadedAt !== 'string' || !Number.isFinite(Date.parse(uploadedAt))) { + throw new Error(`${label} ${lockKey}: uploadedAt must be a valid timestamp`); + } + if (!Array.isArray(networks) || !networks.every((network) => typeof network === 'string')) { + throw new Error(`${label} ${lockKey}: networks must be an array of strings`); + } + const unknown = networks.filter((network) => !ALLOWED_NETWORKS.has(network)); + if (unknown.length > 0) { + throw new Error(`${label} ${lockKey}: unknown network marker(s): ${[...new Set(unknown)].join(', ')}`); + } + const duplicates = networks.filter((network, index) => networks.indexOf(network) !== index); + if (duplicates.length > 0) { + throw new Error(`${label} ${lockKey}: duplicate network marker(s): ${[...new Set(duplicates)].join(', ')}`); + } + } +} + +function parseLockKey(lockKey: string): Omit { + const parts = lockKey.split('/'); + if (parts.length !== 3 || !parts[2]?.endsWith('.dar')) { + throw new Error(`Invalid DAR lock key: ${lockKey}`); + } + const [packageName, packageVersion, darFile] = parts; + const darName = darFile.slice(0, -'.dar'.length); + const safeName = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + if (!safeName.test(packageName) || !safeName.test(darName) || !/^\d+\.\d+\.\d+$/.test(packageVersion)) { + throw new Error(`Unsafe DAR lock key: ${lockKey}`); + } + return { darName, packageName, packageVersion }; +} + +function recordedIdentityMatches(left: DarsLockEntry, right: DarsLockEntry): boolean { + return ( + IMMUTABLE_RECORDED_FIELDS.every((field) => left[field] === right[field]) && + [...left.networks].sort().join('\0') === [...right.networks].sort().join('\0') + ); +} + +/** Identify providers which prove an unmarked backup is an exact live compatibility baseline. */ +export function getExactLiveProviderNames( + identity: DarMarkerIdentity, + preferences: DevnetPackagePreference[] +): string[] { + return preferences + .filter( + (preference) => + preference.packageName === identity.packageName && + preference.packageVersion === identity.packageVersion && + preference.packageId === identity.packageId + ) + .map((preference) => preference.provider); +} + +/** + * Classify entries newly introduced relative to the trusted base. An unrecorded entry may only be the canonical active + * candidate for a discovered top-level package. A recorded alias is accepted only when trusted default-branch history + * contains the same package line and exact immutable lock identity; restoring it is not a new marker addition. + */ +export function classifyCandidateOnlyBackups( + base: DarsLock, + candidate: DarsLock, + currentCandidateKeys: ReadonlySet, + historicalLocks: DarsLock[], + liveBaselineKeys: ReadonlySet = new Set() +): CandidateOnlyBackupClassification { + assertDarsLockSchema(base, 'Trusted dars.lock'); + assertDarsLockSchema(candidate, 'Candidate dars.lock'); + historicalLocks.forEach((lock, index) => assertDarsLockSchema(lock, `Historical dars.lock ${index + 1}`)); + + const candidateOnlyKeys: string[] = []; + const activeKeys: string[] = []; + const restoredKeys: string[] = []; + for (const [lockKey, entry] of Object.entries(candidate.packages)) { + if (Object.prototype.hasOwnProperty.call(base.packages, lockKey)) continue; + candidateOnlyKeys.push(lockKey); + if (entry.networks.length === 0) { + if (!currentCandidateKeys.has(lockKey) && !liveBaselineKeys.has(lockKey)) { + throw new Error( + `${lockKey}: candidate-only unrecorded DAR must be the canonical current candidate for its package line` + ); + } + if (currentCandidateKeys.has(lockKey)) activeKeys.push(lockKey); + continue; + } + + const coordinates = parseLockKey(lockKey); + const hasHistoricalProof = historicalLocks.some((historicalLock) => + Object.entries(historicalLock.packages).some(([historicalKey, historicalEntry]) => { + const historicalCoordinates = parseLockKey(historicalKey); + return ( + historicalCoordinates.packageName === coordinates.packageName && + historicalCoordinates.packageVersion === coordinates.packageVersion && + recordedIdentityMatches(historicalEntry, entry) + ); + }) + ); + if (!hasHistoricalProof) { + throw new Error( + `${lockKey}: candidate-only recorded DAR has no exact immutable identity in trusted default-branch history` + ); + } + restoredKeys.push(lockKey); + } + + return { + candidateOnlyKeys: candidateOnlyKeys.sort(), + currentCandidateKeys: activeKeys.sort(), + restoredRecordedKeys: restoredKeys.sort(), + }; +} + +/** + * Validate the only permitted lock-marker transitions. Recorded metadata and markers are immutable; a PR may add + * `devnet` after a live exact-ID proof. `mainnet` normally requires that trusted marker; callers with a serialized + * artifact-PR release flow may explicitly permit a same-candidate `devnet` addition proven by the same check. + */ +export function getNetworkMarkerAdditions( + base: DarsLock, + candidate: DarsLock, + options: NetworkMarkerPolicyOptions = {} +): NetworkMarkerAddition[] { + assertDarsLockSchema(base, 'Trusted dars.lock'); + assertDarsLockSchema(candidate, 'Candidate dars.lock'); + const currentCandidateKeys = options.currentCandidateKeys ?? new Set(); + const liveBaselineKeys = options.liveBaselineKeys ?? new Set(); + const restoredRecordedKeys = options.restoredRecordedKeys ?? new Set(); + if (options.authorizedMainnetMarkerLockKey) parseLockKey(options.authorizedMainnetMarkerLockKey); + + for (const [lockKey, baseEntry] of Object.entries(base.packages)) { + const candidateEntry = Object.prototype.hasOwnProperty.call(candidate.packages, lockKey) + ? candidate.packages[lockKey] + : undefined; + if (!candidateEntry) { + if (baseEntry.networks.length > 0) { + throw new Error(`${lockKey}: recorded DAR history cannot be removed`); + } + continue; + } + + const removed = baseEntry.networks.filter((network) => !candidateEntry.networks.includes(network)); + if (removed.length > 0) { + throw new Error(`${lockKey}: recorded network marker(s) removed: ${removed.join(', ')}`); + } + const changedFields = IMMUTABLE_RECORDED_FIELDS.filter((field) => candidateEntry[field] !== baseEntry[field]); + if (changedFields.length > 0) { + if (baseEntry.networks.length > 0) { + for (const field of changedFields) { + throw new Error(`${lockKey}: recorded DAR metadata ${field} cannot change`); + } + } + if (!currentCandidateKeys.has(lockKey) && !liveBaselineKeys.has(lockKey)) { + throw new Error( + `${lockKey}: historical unrecorded DAR metadata cannot be replaced in place; remove it through the live-ID ` + + 'safety check and add the new current candidate separately' + ); + } + } + } + + const additions: NetworkMarkerAddition[] = []; + for (const [lockKey, candidateEntry] of Object.entries(candidate.packages)) { + if (restoredRecordedKeys.has(lockKey)) continue; + const baseEntry = Object.prototype.hasOwnProperty.call(base.packages, lockKey) ? base.packages[lockKey] : undefined; + const added = candidateEntry.networks.filter((network) => !baseEntry?.networks.includes(network)); + if (added.length === 0) continue; + const coordinates = parseLockKey(lockKey); + + for (const network of added) { + if (network !== 'devnet' && network !== 'mainnet') { + throw new Error(`${lockKey}: unknown network marker: ${network}`); + } + if (network === 'mainnet' && lockKey !== options.authorizedMainnetMarkerLockKey) { + const scope = options.authorizedMainnetMarkerLockKey + ? `; the verified attestation authorizes only ${options.authorizedMainnetMarkerLockKey}` + : ''; + throw new Error(`${lockKey}: new mainnet marker requires explicit trusted workflow provenance${scope}`); + } + if ( + network === 'mainnet' && + !baseEntry?.networks.includes('devnet') && + (!options.allowSameCandidateMainnet || !candidateEntry.networks.includes('devnet')) + ) { + const sameCandidateEvidence = options.allowSameCandidateMainnet + ? ' or a devnet marker proven in this candidate' + : ''; + throw new Error(`${lockKey}: mainnet requires a trusted-base devnet marker${sameCandidateEvidence}`); + } + additions.push({ ...coordinates, lockKey, network }); + } + } + const mainnetAdditions = additions.filter((addition) => addition.network === 'mainnet'); + if ( + options.authorizedMainnetMarkerLockKey && + (mainnetAdditions.length !== 1 || mainnetAdditions[0]?.lockKey !== options.authorizedMainnetMarkerLockKey) + ) { + throw new Error( + `Verified Mainnet attestation for ${options.authorizedMainnetMarkerLockKey} must authorize exactly that one marker addition` + ); + } + return additions.sort((left, right) => { + if (left.network === right.network) return 0; + return left.network === 'devnet' ? -1 : 1; + }); +} + +/** Require one exact preferred identity from every configured DevNet provider before accepting a new marker. */ +export function assertDevnetMarkerIdentity( + addition: Pick, + identity: DarMarkerIdentity, + preferences: DevnetPackagePreference[], + expectedProviders: readonly string[] +): void { + const expected = new Set(expectedProviders); + const byProvider = new Map(); + for (const preference of preferences) { + if (!expected.has(preference.provider)) { + throw new Error(`${addition.lockKey}: unexpected DevNet provider response: ${preference.provider}`); + } + if (byProvider.has(preference.provider)) { + throw new Error(`${addition.lockKey}: duplicate DevNet provider response: ${preference.provider}`); + } + byProvider.set(preference.provider, preference); + } + + const mismatches = expectedProviders.filter((provider) => { + const preference = byProvider.get(provider); + if (!preference) return true; + return ( + preference.packageName !== identity.packageName || + preference.packageVersion !== identity.packageVersion || + preference.packageId !== identity.packageId + ); + }); + if ( + identity.packageName !== addition.packageName || + identity.packageVersion !== addition.packageVersion || + mismatches.length > 0 + ) { + const state = expectedProviders + .map((provider) => { + const preference = byProvider.get(provider); + return preference ? `${provider}=${preference.packageVersion}/${preference.packageId}` : `${provider}=absent`; + }) + .join(', '); + throw new Error( + `${addition.lockKey}: devnet marker requires the exact locked DAR ${identity.packageVersion}/${identity.packageId} ` + + `to be preferred by every configured DevNet provider; current state: ${state}` + ); + } +} diff --git a/scripts/dar-package-id.ts b/scripts/dar-package-id.ts new file mode 100644 index 00000000..f87db22d --- /dev/null +++ b/scripts/dar-package-id.ts @@ -0,0 +1,78 @@ +import { execFileSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import { assertDarArchiveSafe, DAML_INSPECT_TIMEOUT_MS } from './dar-archive-policy'; + +function isObject(value: unknown): value is Record { + return value !== null && typeof value === 'object'; +} + +function readStringField(value: Record, keys: string[]): string | undefined { + for (const key of keys) { + const candidate = value[key]; + if (typeof candidate === 'string' && candidate.trim()) return candidate; + } + return undefined; +} + +function packageMetadataMatches(value: Record, packageName: string, version: string): boolean { + const name = readStringField(value, ['name', 'packageName', 'package_name']); + const packageVersion = readStringField(value, ['version', 'packageVersion', 'package_version']); + return name === packageName && (!packageVersion || packageVersion === version); +} + +function findPackageIdInInspectJson(value: unknown, packageName: string, version: string): string | null { + if (Array.isArray(value)) { + for (const item of value) { + const found = findPackageIdInInspectJson(item, packageName, version); + if (found) return found; + } + return null; + } + if (!isObject(value)) return null; + + const { packages } = value; + if (isObject(packages)) { + const mainPackageId = readStringField(value, ['main_package_id', 'mainPackageId']); + if (mainPackageId) { + const mainPackage = packages[mainPackageId]; + if (isObject(mainPackage) && packageMetadataMatches(mainPackage, packageName, version)) return mainPackageId; + } + for (const [packageId, metadata] of Object.entries(packages)) { + if (isObject(metadata) && packageMetadataMatches(metadata, packageName, version)) return packageId; + } + } + + const packageId = readStringField(value, ['packageId', 'package_id']); + if (packageId && packageMetadataMatches(value, packageName, version)) return packageId; + + for (const child of Object.values(value)) { + const found = findPackageIdInInspectJson(child, packageName, version); + if (found) return found; + } + return null; +} + +function resolveDpmExecutable(): string { + const homeDpm = process.env.HOME ? path.join(process.env.HOME, '.dpm', 'bin', 'dpm') : ''; + return homeDpm && fs.existsSync(homeDpm) ? homeDpm : 'dpm'; +} + +export function inspectDarPackageId(darPath: string, packageName: string, version: string): string { + assertDarArchiveSafe(darPath); + let raw: string; + try { + raw = execFileSync(resolveDpmExecutable(), ['damlc', 'inspect-dar', darPath, '--json'], { + encoding: 'utf8', + killSignal: 'SIGKILL', + maxBuffer: 20 * 1024 * 1024, + timeout: DAML_INSPECT_TIMEOUT_MS, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Unable to inspect ${packageName} ${version} DAR package id: ${message}`); + } + const packageId = findPackageIdInInspectJson(JSON.parse(raw) as unknown, packageName, version); + if (!packageId) throw new Error(`Could not find ${packageName} ${version} in ${darPath}`); + return packageId; +} diff --git a/scripts/dar-version-policy.test.ts b/scripts/dar-version-policy.test.ts new file mode 100644 index 00000000..ad418e49 --- /dev/null +++ b/scripts/dar-version-policy.test.ts @@ -0,0 +1,535 @@ +import assert from 'node:assert/strict'; +import * as crypto from 'node:crypto'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { describe, it } from 'node:test'; +import { applyBackupTransaction } from './dar-backup-transaction'; +import { assertCandidateDarPaths } from './dar-candidate-path-policy'; +import { validateDarLfsPointer, validateDarLfsTree } from './dar-lfs-policy'; +import type { DarsLock, DarsLockEntry } from './dar-utils'; +import { findBaselineDar } from './devnet-dar-policy'; +import { + assertDevnetPreferencesConsistent, + assertDevnetMarkerForMainnet, + candidateDevnetNetworks, + decideBackupMutation, + decideDevnetCandidateVersion, + incrementPatch, + nextDevnetCandidateVersion, + planBackupRetention, + resolveMajorPackageLineTarget, + uniqueDevnetBaselines, +} from './dar-version-policy'; + +const entry = (sha256: string, networks: string[] = []): DarsLockEntry => ({ + sha256, + size: 1, + sdkVersion: '3.4.10', + uploadedAt: 'now', + networks, +}); + +void describe('DevNet-anchored candidate versions', () => { + void it('starts an absent package at 0.0.1', () => { + assert.equal(nextDevnetCandidateVersion([]), '0.0.1'); + assert.deepEqual(decideDevnetCandidateVersion([], 'candidate'), { + expectedVersion: '0.0.1', + highestDevnetVersion: null, + reason: 'first-devnet-candidate', + }); + }); + + void it('keeps a candidate that is the exact preferred package on both providers', () => { + const preferences = [ + { packageId: 'same', packageVersion: '0.0.3' }, + { packageId: 'same', packageVersion: '0.0.3' }, + ]; + assert.deepEqual(decideDevnetCandidateVersion(preferences, 'same'), { + expectedVersion: '0.0.3', + highestDevnetVersion: '0.0.3', + reason: 'matches-devnet', + }); + }); + + void it('advances and remains unmarked when only one provider reports the exact candidate', () => { + const partialPreferences = [{ packageId: 'same', packageVersion: '0.0.3' }]; + assert.deepEqual(decideDevnetCandidateVersion(partialPreferences, 'same'), { + expectedVersion: '0.0.4', + highestDevnetVersion: '0.0.3', + reason: 'advance-after-devnet', + }); + assert.deepEqual(candidateDevnetNetworks(partialPreferences, '0.0.3', 'same'), []); + }); + + void it('requires one patch when candidate bytes differ from DevNet', () => { + const preferences = [ + { packageId: 'live', packageVersion: '0.0.3' }, + { packageId: 'live', packageVersion: '0.0.3' }, + ]; + assert.equal(decideDevnetCandidateVersion(preferences, 'candidate').expectedVersion, '0.0.4'); + }); + + void it('uses one patch beyond the higher provider during version divergence', () => { + const preferences = [ + { packageId: 'older', packageVersion: '0.0.2' }, + { packageId: 'newer', packageVersion: '0.0.5' }, + ]; + assert.equal(decideDevnetCandidateVersion(preferences, 'newer').expectedVersion, '0.0.6'); + }); + + void it('handles patch rollover without changing the major or minor components', () => { + assert.equal(incrementPatch('1.2.99'), '1.2.100'); + }); + + void it('rejects the same version resolving to different package IDs', () => { + assert.throws( + () => + assertDevnetPreferencesConsistent([ + { packageId: 'one', packageVersion: '0.0.3' }, + { packageId: 'two', packageVersion: '0.0.3' }, + ]), + /different package IDs/ + ); + }); + + void it('rejects one package ID reported with different versions', () => { + assert.throws( + () => + assertDevnetPreferencesConsistent([ + { packageId: 'same', packageVersion: '0.0.2' }, + { packageId: 'same', packageVersion: '0.0.3' }, + ]), + /different versions/ + ); + }); + + void it('keeps every distinct live compatibility baseline exactly once', () => { + const baselines = uniqueDevnetBaselines([ + { packageId: 'older', packageVersion: '0.0.2' }, + { packageId: 'newer', packageVersion: '0.0.3' }, + { packageId: 'newer', packageVersion: '0.0.3' }, + ]); + assert.deepEqual(baselines, [ + { packageId: 'older', packageVersion: '0.0.2' }, + { packageId: 'newer', packageVersion: '0.0.3' }, + ]); + }); + + void it('marks newly written exact-live bytes as DevNet history', () => { + const preferences = [ + { packageId: 'live', packageVersion: '0.0.3' }, + { packageId: 'live', packageVersion: '0.0.3' }, + ]; + assert.deepEqual(candidateDevnetNetworks(preferences, '0.0.3', 'live'), ['devnet']); + assert.deepEqual(candidateDevnetNetworks(preferences, '0.0.4', 'candidate'), []); + }); + + void it('queries the target package line when selecting a major upgrade version', async () => { + const queried: string[] = []; + const target = await resolveMajorPackageLineTarget('OpenCapTable', 34, (packageName) => { + queried.push(packageName); + return [ + { packageId: 'live-v35', packageVersion: '0.0.3' }, + { packageId: 'live-v35', packageVersion: '0.0.3' }, + ]; + }); + assert.deepEqual(queried, ['OpenCapTable-v35']); + assert.deepEqual(target, { + candidateVersion: '0.0.4', + majorVersion: 'v35', + packageName: 'OpenCapTable-v35', + }); + + const absent = await resolveMajorPackageLineTarget('OpenCapTable', 35, () => []); + assert.equal(absent.candidateVersion, '0.0.1'); + }); +}); + +void describe('pre-download DAR LFS policy', () => { + const pointer = (size: number) => + `version https://git-lfs.github.com/spec/v1\noid sha256:${'a'.repeat(64)}\nsize ${size}\n`; + const entryFor = (darPath: string, size = 1024) => ({ + blob: pointer(size), + mode: '100644', + path: darPath, + type: 'blob', + }); + + void it('accepts canonical pointers only at safe package/version paths', () => { + assert.deepEqual(validateDarLfsPointer(entryFor('dars/Example-v01/0.0.2/live-alias.dar')), { + oid: 'a'.repeat(64), + path: 'dars/Example-v01/0.0.2/live-alias.dar', + size: 1024, + }); + assert.throws(() => validateDarLfsPointer(entryFor('dars/Example-v01/../escape.dar')), /Unsafe DAR tree path/); + assert.throws( + () => validateDarLfsPointer({ ...entryFor('dars/Example-v01/0.0.2/alias.dar'), mode: '120000' }), + /non-executable regular blob/ + ); + assert.throws( + () => + validateDarLfsPointer({ + ...entryFor('dars/Example-v01/0.0.2/alias.dar'), + blob: 'not an lfs pointer\n', + }), + /canonical Git LFS pointer/ + ); + }); + + void it('bounds individual, aggregate, and count-based downloads', () => { + assert.throws( + () => validateDarLfsPointer(entryFor('dars/Example-v01/0.0.2/large.dar', 100 * 1024 * 1024 + 1)), + /exceeds 100 MiB/ + ); + assert.throws( + () => + validateDarLfsTree( + Array.from({ length: 11 }, (_, index) => + entryFor(`dars/Example-v01/0.0.2/part-${index}.dar`, 100 * 1024 * 1024) + ) + ), + /maximum is 1 GiB/ + ); + assert.throws( + () => + validateDarLfsTree( + Array.from({ length: 251 }, (_, index) => entryFor(`dars/Example-v01/0.0.2/part-${index}.dar`)) + ), + /maximum is 250/ + ); + }); + + void it('rejects symlinked DAR parents and files after detached checkout', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dar-candidate-paths-')); + const outside = fs.mkdtempSync(path.join(os.tmpdir(), 'dar-candidate-outside-')); + const darPath = 'dars/Example-v01/0.0.2/Example-v01.dar'; + try { + const versionDirectory = path.join(root, 'dars', 'Example-v01', '0.0.2'); + fs.mkdirSync(versionDirectory, { recursive: true }); + fs.writeFileSync(path.join(versionDirectory, 'Example-v01.dar'), 'pointer'); + assert.doesNotThrow(() => assertCandidateDarPaths(root, [darPath])); + + fs.rmSync(path.join(versionDirectory, 'Example-v01.dar')); + fs.writeFileSync(path.join(outside, 'outside.dar'), 'outside'); + fs.symlinkSync(path.join(outside, 'outside.dar'), path.join(versionDirectory, 'Example-v01.dar')); + assert.throws(() => assertCandidateDarPaths(root, [darPath]), /non-symlink regular file/); + + fs.rmSync(versionDirectory, { recursive: true, force: true }); + fs.symlinkSync(outside, versionDirectory); + assert.throws(() => assertCandidateDarPaths(root, [darPath]), /non-symlink directory/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + fs.rmSync(outside, { recursive: true, force: true }); + } + }); +}); + +void describe('Mainnet promotion marker', () => { + const lockKey = 'Example-v01/0.0.2/Example-v01.dar'; + + void it('requires the exact candidate lock entry to record DevNet first', () => { + assert.throws(() => assertDevnetMarkerForMainnet(undefined, lockKey), /committed devnet marker/); + assert.throws(() => assertDevnetMarkerForMainnet(entry('candidate', ['mainnet']), lockKey), /devnet marker/); + assert.doesNotThrow(() => assertDevnetMarkerForMainnet(entry('candidate', ['devnet']), lockKey)); + }); +}); + +void describe('exact DevNet baseline lookup', () => { + const preference = { + packageId: 'live-id', + packageName: 'Example-v01', + packageVersion: '0.0.1', + provider: 'intellect' as const, + }; + + void it('finds an exact live DAR under a distinct alias filename', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dar-baseline-alias-')); + try { + const aliasKey = 'Example-v01/0.0.1/live-deployment.dar'; + const aliasPath = path.join(root, 'dars', aliasKey); + fs.mkdirSync(path.dirname(aliasPath), { recursive: true }); + fs.writeFileSync(aliasPath, 'live'); + const hash = crypto.createHash('sha256').update('live').digest('hex'); + const lock: DarsLock = { + version: 1, + packages: { [aliasKey]: { ...entry(hash), size: 4 } }, + }; + + assert.equal( + findBaselineDar(root, lock, preference, 'candidate-id', 'candidate.dar', () => 'live-id'), + aliasPath + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + void it('fails closed when the exact live baseline is missing or corrupt', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dar-baseline-failure-')); + try { + assert.throws( + () => findBaselineDar(root, { version: 1, packages: {} }, preference, 'candidate-id', 'candidate.dar'), + /exact DevNet baseline.*is missing/ + ); + + const corruptKey = 'Example-v01/0.0.1/live-deployment.dar'; + const corruptPath = path.join(root, 'dars', corruptKey); + fs.mkdirSync(path.dirname(corruptPath), { recursive: true }); + fs.writeFileSync(corruptPath, 'live'); + const corruptLock: DarsLock = { + version: 1, + packages: { [corruptKey]: { ...entry('incorrect-hash'), size: 4 } }, + }; + assert.throws(() => findBaselineDar(root, corruptLock, preference, 'candidate-id', 'candidate.dar'), /SHA256/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); + +void describe('backup transaction', () => { + void it('creates an exact-live candidate with a durable DevNet marker', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dar-backup-exact-live-')); + try { + const darsDir = path.join(root, 'dars'); + const currentKey = 'Example-v01/0.0.3/Example-v01.dar'; + const currentPath = path.join(darsDir, currentKey); + const sourcePath = path.join(root, 'candidate.dar'); + fs.writeFileSync(sourcePath, 'live'); + const hash = crypto.createHash('sha256').update('live').digest('hex'); + const lock: DarsLock = { version: 1, packages: {} }; + const preferences = [ + { packageId: 'live-id', packageVersion: '0.0.3' }, + { packageId: 'live-id', packageVersion: '0.0.3' }, + ]; + + applyBackupTransaction({ + lock, + darsDir, + retentionPlan: { freezeKeys: [], pruneKeys: [] }, + candidateWrite: { + lockKey: currentKey, + sourcePath, + destPath: currentPath, + replaceExisting: false, + entry: { + ...entry(hash, candidateDevnetNetworks(preferences, '0.0.3', 'live-id')), + size: 4, + }, + }, + saveLock: () => undefined, + }); + + assert.equal(fs.readFileSync(currentPath, 'utf8'), 'live'); + assert.deepEqual(lock.packages[currentKey].networks, ['devnet']); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + void it('commits candidate replacement, retention, and the lock in one save', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dar-backup-transaction-')); + try { + const darsDir = path.join(root, 'dars'); + const currentKey = 'Example-v01/0.0.3/Example-v01.dar'; + const staleKey = 'Example-v01/0.0.2/Example-v01.dar'; + const currentPath = path.join(darsDir, currentKey); + const stalePath = path.join(darsDir, staleKey); + const sourcePath = path.join(root, 'candidate.dar'); + fs.mkdirSync(path.dirname(currentPath), { recursive: true }); + fs.mkdirSync(path.dirname(stalePath), { recursive: true }); + fs.writeFileSync(currentPath, 'old'); + fs.writeFileSync(stalePath, 'stale'); + fs.writeFileSync(sourcePath, 'new'); + + const hash = (value: string) => crypto.createHash('sha256').update(value).digest('hex'); + const lock: DarsLock = { + version: 1, + packages: { + [currentKey]: { ...entry(hash('old')), size: 3 }, + [staleKey]: { ...entry(hash('stale')), size: 5 }, + }, + }; + let savedLock: DarsLock | undefined; + let saveCount = 0; + + applyBackupTransaction({ + lock, + darsDir, + retentionPlan: { freezeKeys: [], pruneKeys: [staleKey] }, + candidateWrite: { + lockKey: currentKey, + sourcePath, + destPath: currentPath, + replaceExisting: true, + entry: { ...entry(hash('new'), ['devnet']), size: 3 }, + }, + saveLock: (nextLock) => { + saveCount += 1; + savedLock = JSON.parse(JSON.stringify(nextLock)) as DarsLock; + }, + }); + + assert.equal(saveCount, 1); + assert.equal(fs.readFileSync(currentPath, 'utf8'), 'new'); + assert.equal(fs.existsSync(stalePath), false); + assert.deepEqual(savedLock, lock); + assert.deepEqual(lock.packages[currentKey].networks, ['devnet']); + assert.equal(Object.prototype.hasOwnProperty.call(lock.packages, staleKey), false); + assert.deepEqual(fs.readdirSync(path.dirname(currentPath)), ['Example-v01.dar']); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + void it('restores candidate bytes, pruned backups, and the in-memory lock when saving fails', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dar-backup-rollback-')); + try { + const darsDir = path.join(root, 'dars'); + const currentKey = 'Example-v01/0.0.3/Example-v01.dar'; + const staleKey = 'Example-v01/0.0.2/Example-v01.dar'; + const currentPath = path.join(darsDir, currentKey); + const stalePath = path.join(darsDir, staleKey); + const sourcePath = path.join(root, 'candidate.dar'); + fs.mkdirSync(path.dirname(currentPath), { recursive: true }); + fs.mkdirSync(path.dirname(stalePath), { recursive: true }); + fs.writeFileSync(currentPath, 'old'); + fs.writeFileSync(stalePath, 'stale'); + fs.writeFileSync(sourcePath, 'new'); + + const hash = (value: string) => crypto.createHash('sha256').update(value).digest('hex'); + const lock: DarsLock = { + version: 1, + packages: { + [currentKey]: { ...entry(hash('old')), size: 3 }, + [staleKey]: { ...entry(hash('stale')), size: 5 }, + }, + }; + const originalLock = JSON.parse(JSON.stringify(lock)) as DarsLock; + + assert.throws( + () => + applyBackupTransaction({ + lock, + darsDir, + retentionPlan: { freezeKeys: [], pruneKeys: [staleKey] }, + candidateWrite: { + lockKey: currentKey, + sourcePath, + destPath: currentPath, + replaceExisting: true, + entry: { ...entry(hash('new')), size: 3 }, + }, + saveLock: () => { + throw new Error('simulated lock save failure'); + }, + }), + /simulated lock save failure/ + ); + + assert.equal(fs.readFileSync(currentPath, 'utf8'), 'old'); + assert.equal(fs.readFileSync(stalePath, 'utf8'), 'stale'); + assert.deepEqual(lock, originalLock); + assert.deepEqual(fs.readdirSync(path.dirname(currentPath)), ['Example-v01.dar']); + assert.deepEqual(fs.readdirSync(path.dirname(stalePath)), ['Example-v01.dar']); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); + +void describe('candidate backup mutation', () => { + void it('creates a missing backup and reuses matching bytes', () => { + assert.equal(decideBackupMutation(undefined, 'new'), 'create'); + assert.equal(decideBackupMutation(entry('same'), 'same'), 'no-op'); + }); + + void it('replaces only an unrecorded candidate', () => { + assert.equal(decideBackupMutation(entry('old'), 'new'), 'replace'); + assert.throws(() => decideBackupMutation(entry('old', ['devnet']), 'new'), /immutable/); + assert.throws(() => decideBackupMutation(entry('old', ['mainnet']), 'new'), /immutable/); + }); + + void it('prunes only superseded undeployed backups and freezes exact live backups', () => { + const plan = planBackupRetention( + [ + { + lockKey: 'Example-v01/0.0.4/Example-v01.dar', + packageId: 'current', + packageVersion: '0.0.4', + entry: entry('current'), + }, + { + lockKey: 'Example-v01/0.0.3/Example-v01.dar', + packageId: 'live', + packageVersion: '0.0.3', + entry: entry('live'), + }, + { + lockKey: 'Example-v01/0.0.2/Example-v01.dar', + packageId: 'recorded', + packageVersion: '0.0.2', + entry: entry('recorded', ['mainnet']), + }, + { + lockKey: 'Example-v01/0.0.1/Example-v01.dar', + packageId: 'stale', + packageVersion: '0.0.1', + entry: entry('stale'), + }, + ], + 'Example-v01/0.0.4/Example-v01.dar', + [ + { packageId: 'live', packageVersion: '0.0.3' }, + { packageId: 'live', packageVersion: '0.0.3' }, + ] + ); + + assert.deepEqual(plan, { + freezeKeys: ['Example-v01/0.0.3/Example-v01.dar'], + pruneKeys: ['Example-v01/0.0.1/Example-v01.dar'], + }); + }); + + void it('never prunes the current candidate and freezes it when it is exact-live', () => { + const currentKey = 'Example-v01/0.0.3/Example-v01.dar'; + assert.deepEqual( + planBackupRetention( + [ + { + lockKey: currentKey, + packageId: 'live', + packageVersion: '0.0.3', + entry: entry('live'), + }, + ], + currentKey, + [ + { packageId: 'live', packageVersion: '0.0.3' }, + { packageId: 'live', packageVersion: '0.0.3' }, + ] + ), + { freezeKeys: [currentKey], pruneKeys: [] } + ); + }); + + void it('retains but does not mark a DAR preferred by only one provider', () => { + const partialKey = 'Example-v01/0.0.3/Example-v01.dar'; + assert.deepEqual( + planBackupRetention( + [ + { + lockKey: partialKey, + packageId: 'partial-live', + packageVersion: '0.0.3', + entry: entry('partial-live'), + }, + ], + 'Example-v01/0.0.4/Example-v01.dar', + [{ packageId: 'partial-live', packageVersion: '0.0.3' }] + ), + { freezeKeys: [], pruneKeys: [] } + ); + }); +}); diff --git a/scripts/dar-version-policy.ts b/scripts/dar-version-policy.ts new file mode 100644 index 00000000..9ecf7820 --- /dev/null +++ b/scripts/dar-version-policy.ts @@ -0,0 +1,237 @@ +import type { DarsLockEntry } from './dar-utils'; +import { LEDGER_SCRIPT_PROVIDERS } from './providers'; + +export interface PackagePreferenceLike { + packageId: string; + packageVersion: string; +} + +export interface DevnetCandidateDecision { + expectedVersion: string; + highestDevnetVersion: string | null; + reason: 'first-devnet-candidate' | 'matches-devnet' | 'advance-after-devnet'; +} + +export type BackupMutation = 'create' | 'no-op' | 'replace'; + +export interface InspectedBackupLike { + lockKey: string; + packageId: string; + packageVersion: string; + entry: DarsLockEntry; +} + +export interface BackupRetentionPlan { + freezeKeys: string[]; + pruneKeys: string[]; +} + +export interface MajorPackageLineTarget { + candidateVersion: string; + majorVersion: string; + packageName: string; +} + +function parseSemver(version: string): [number, number, number] { + const match = version.match(/^(\d+)\.(\d+)\.(\d+)$/); + if (!match) { + throw new Error(`Invalid semantic version: ${version}`); + } + return [Number(match[1]), Number(match[2]), Number(match[3])]; +} + +/** Compare strict x.y.z semantic versions. */ +export function compareSemver(a: string, b: string): number { + const left = parseSemver(a); + const right = parseSemver(b); + for (let index = 0; index < left.length; index++) { + if (left[index] !== right[index]) { + return left[index] - right[index]; + } + } + return 0; +} + +/** Increment only the patch component used for compatible DAML releases. */ +export function incrementPatch(version: string): string { + const [major, minor, patch] = parseSemver(version); + return `${major}.${minor}.${patch + 1}`; +} + +/** Any recorded network upload makes that exact DAR immutable. */ +export function isDeployed(entry: DarsLockEntry | undefined): boolean { + return Boolean(entry && entry.networks.length > 0); +} + +/** Decide whether candidate bytes may change before the live policy preflight runs. */ +export function decideBackupMutation(entry: DarsLockEntry | undefined, sourceHash: string): BackupMutation { + if (!entry) return 'create'; + if (entry.sha256 === sourceHash) return 'no-op'; + if (isDeployed(entry)) { + throw new Error('A network-recorded DAR backup is immutable.'); + } + return 'replace'; +} + +export function maxVersion(versions: string[]): string | null { + return versions.reduce( + (latest, version) => (!latest || compareSemver(version, latest) > 0 ? version : latest), + null + ); +} + +/** The only version for a new DAR which is not already the exact preferred DevNet package. */ +export function nextDevnetCandidateVersion(preferences: PackagePreferenceLike[]): string { + const highest = maxVersion(preferences.map((preference) => preference.packageVersion)); + return highest ? incrementPatch(highest) : '0.0.1'; +} + +/** Select a new major package line from that target line's own live DevNet state. */ +export async function resolveMajorPackageLineTarget( + baseName: string, + currentMajorVersion: number, + queryPreferences: (packageName: string) => PackagePreferenceLike[] | Promise +): Promise { + const majorVersion = `v${(currentMajorVersion + 1).toString().padStart(2, '0')}`; + const packageName = `${baseName}-${majorVersion}`; + const preferences = await queryPreferences(packageName); + assertDevnetPreferencesConsistent(preferences); + return { + candidateVersion: nextDevnetCandidateVersion(preferences), + majorVersion, + packageName, + }; +} + +/** + * Decide the version for candidate bytes from the two live DevNet preferences. + * + * A candidate may retain the live version only when every configured provider returns a preference that resolves to its + * exact package ID and those preferences agree on one version. A partial provider result is not an exact-live + * candidate: it advances one patch beyond the highest result that was returned. Any different candidate, including a + * provider-version divergence, must use exactly one patch after the highest live DevNet version. + */ +export function decideDevnetCandidateVersion( + preferences: PackagePreferenceLike[], + candidatePackageId: string +): DevnetCandidateDecision { + if (preferences.length === 0) { + return { + expectedVersion: '0.0.1', + highestDevnetVersion: null, + reason: 'first-devnet-candidate', + }; + } + + const highestDevnetVersion = maxVersion(preferences.map((preference) => preference.packageVersion)); + if (!highestDevnetVersion) { + throw new Error('Unable to determine the highest DevNet package version.'); + } + + const liveVersions = new Set(preferences.map((preference) => preference.packageVersion)); + const hasEveryProviderPreference = preferences.length === LEDGER_SCRIPT_PROVIDERS.length; + const matchesEveryPreference = preferences.every((preference) => preference.packageId === candidatePackageId); + if (hasEveryProviderPreference && liveVersions.size === 1 && matchesEveryPreference) { + return { + expectedVersion: highestDevnetVersion, + highestDevnetVersion, + reason: 'matches-devnet', + }; + } + + return { + expectedVersion: incrementPatch(highestDevnetVersion), + highestDevnetVersion, + reason: 'advance-after-devnet', + }; +} + +/** Reject impossible live states before using them as version or compatibility authority. */ +export function assertDevnetPreferencesConsistent(preferences: PackagePreferenceLike[]): void { + const packageIdsByVersion = new Map>(); + const versionsByPackageId = new Map>(); + + for (const preference of preferences) { + const ids = packageIdsByVersion.get(preference.packageVersion) ?? new Set(); + ids.add(preference.packageId); + packageIdsByVersion.set(preference.packageVersion, ids); + + const versions = versionsByPackageId.get(preference.packageId) ?? new Set(); + versions.add(preference.packageVersion); + versionsByPackageId.set(preference.packageId, versions); + } + + for (const [version, packageIds] of packageIdsByVersion) { + if (packageIds.size > 1) { + throw new Error( + `DevNet providers resolve version ${version} to different package IDs: ${[...packageIds].join(', ')}` + ); + } + } + for (const [packageId, versions] of versionsByPackageId) { + if (versions.size > 1) { + throw new Error( + `DevNet providers report package ID ${packageId} with different versions: ${[...versions].join(', ')}` + ); + } + } +} + +/** One compatibility baseline per distinct live package ID/version, even when both providers agree. */ +export function uniqueDevnetBaselines(preferences: T[]): T[] { + return [ + ...new Map( + preferences.map((preference) => [`${preference.packageVersion}:${preference.packageId}`, preference]) + ).values(), + ]; +} + +/** Network markers for candidate bytes that are already an exact preferred DevNet package. */ +export function candidateDevnetNetworks( + preferences: PackagePreferenceLike[], + packageVersion: string, + packageId: string +): string[] { + return preferences.length === LEDGER_SCRIPT_PROVIDERS.length && + preferences.every( + (preference) => preference.packageVersion === packageVersion && preference.packageId === packageId + ) + ? ['devnet'] + : []; +} + +/** A Mainnet promotion is permitted only after this exact lock entry records its DevNet deployment. */ +export function assertDevnetMarkerForMainnet(entry: DarsLockEntry | undefined, lockKey: string): void { + if (!entry?.networks.includes('devnet')) { + throw new Error(`Mainnet upload requires a committed devnet marker for ${lockKey}.`); + } +} + +/** + * Plan retention only after callers have integrity-checked and inspected every same-package backup. The current + * candidate, every recorded backup, and every exact live DevNet package are always retained. + */ +export function planBackupRetention( + backups: InspectedBackupLike[], + currentLockKey: string, + preferences: PackagePreferenceLike[] +): BackupRetentionPlan { + assertDevnetPreferencesConsistent(preferences); + const livePairs = new Set(preferences.map((preference) => `${preference.packageVersion}:${preference.packageId}`)); + const freezeKeys: string[] = []; + const pruneKeys: string[] = []; + + for (const backup of backups) { + const isExactLive = livePairs.has(`${backup.packageVersion}:${backup.packageId}`); + const isExactOnEveryProvider = + candidateDevnetNetworks(preferences, backup.packageVersion, backup.packageId).length > 0; + if (isExactOnEveryProvider && !backup.entry.networks.includes('devnet')) { + freezeKeys.push(backup.lockKey); + } + if (backup.lockKey !== currentLockKey && !isDeployed(backup.entry) && !isExactLive) { + pruneKeys.push(backup.lockKey); + } + } + + return { freezeKeys: freezeKeys.sort(), pruneKeys: pruneKeys.sort() }; +} diff --git a/scripts/detect-factory-need.ts b/scripts/detect-factory-need.ts index 79e201f7..e37ce401 100644 --- a/scripts/detect-factory-need.ts +++ b/scripts/detect-factory-need.ts @@ -5,9 +5,9 @@ * Usage: tsx scripts/detect-factory-need.ts --package ocp */ -import { execFileSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; +import { inspectDarPackageId } from './dar-package-id'; import { getBackedUpDarPath, getFreshDarPath } from './dar-utils'; import { getPackage, parsePackageArg } from './packages'; import type { ContractNetwork } from './types'; @@ -46,104 +46,6 @@ function readJsonFile(filePath: string): T | null { return JSON.parse(fs.readFileSync(filePath, 'utf8')) as T; } -function isObject(value: unknown): value is Record { - return value !== null && typeof value === 'object'; -} - -function readStringField(value: Record, keys: string[]): string | undefined { - for (const key of keys) { - const candidate = value[key]; - if (typeof candidate === 'string' && candidate.trim().length > 0) { - return candidate; - } - } - return undefined; -} - -function packageMetadataMatches(value: Record, packageName: string, version: string): boolean { - const name = readStringField(value, ['name', 'packageName', 'package_name']); - const packageVersion = readStringField(value, ['version', 'packageVersion', 'package_version']); - return name === packageName && (!packageVersion || packageVersion === version); -} - -function findPackageIdInPackageMap( - value: Record, - packageName: string, - version: string -): string | null { - const { packages } = value; - if (!isObject(packages)) { - return null; - } - - const mainPackageId = readStringField(value, ['main_package_id', 'mainPackageId']); - if (mainPackageId) { - const mainPackage = packages[mainPackageId]; - if (isObject(mainPackage) && packageMetadataMatches(mainPackage, packageName, version)) { - return mainPackageId; - } - } - - for (const [packageId, metadata] of Object.entries(packages)) { - if (isObject(metadata) && packageMetadataMatches(metadata, packageName, version)) { - return packageId; - } - } - - return null; -} - -function findPackageIdInInspectJson(value: unknown, packageName: string, version: string): string | null { - if (Array.isArray(value)) { - for (const item of value) { - const found = findPackageIdInInspectJson(item, packageName, version); - if (found) return found; - } - return null; - } - - if (!isObject(value)) { - return null; - } - - const packageMapMatch = findPackageIdInPackageMap(value, packageName, version); - if (packageMapMatch) { - return packageMapMatch; - } - - const packageId = readStringField(value, ['packageId', 'package_id']); - if (packageId && packageMetadataMatches(value, packageName, version)) { - return packageId; - } - - for (const child of Object.values(value)) { - const found = findPackageIdInInspectJson(child, packageName, version); - if (found) return found; - } - return null; -} - -function inspectDarPackageId(darPath: string, packageName: string, version: string): string { - let raw: string; - try { - raw = execFileSync('dpm', ['damlc', 'inspect-dar', darPath, '--json'], { - cwd: ROOT_DIR, - encoding: 'utf8', - maxBuffer: 20 * 1024 * 1024, - }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - throw new Error(`Unable to inspect DAR package id with dpm: ${message}`); - } - - const parsed = JSON.parse(raw) as unknown; - const packageId = findPackageIdInInspectJson(parsed, packageName, version); - if (!packageId) { - throw new Error(`Could not find package id for ${packageName} ${version} in ${darPath}`); - } - return packageId; -} - function getDarPath(packageName: string, version: string, darName: string): string { return ( getBackedUpDarPath(packageName, version, darName) ?? diff --git a/scripts/devnet-dar-policy.ts b/scripts/devnet-dar-policy.ts new file mode 100644 index 00000000..c7d46e58 --- /dev/null +++ b/scripts/devnet-dar-policy.ts @@ -0,0 +1,214 @@ +import { execFileSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import { assertContainedRegularFile } from './candidate-path-safety'; +import { assertDarArchiveSafe, DAML_UPGRADE_CHECK_TIMEOUT_MS } from './dar-archive-policy'; +import type { DarsLock } from './dar-utils'; +import { computeSha256 } from './dar-utils'; +import { inspectDarPackageId } from './dar-package-id'; +import { + assertDevnetPreferencesConsistent, + decideDevnetCandidateVersion, + uniqueDevnetBaselines, +} from './dar-version-policy'; +import type { DevnetPackagePreference } from './devnet-package-versions'; + +const MAX_DAR_SIZE_BYTES = 100 * 1024 * 1024; + +export interface DevnetDarValidationResult { + candidatePackageId: string; + expectedVersion: string; + highestDevnetVersion: string | null; + compatibilityBaselines: Array<{ + packageId: string; + packageVersion: string; + darPath: string; + }>; +} + +export interface InspectedDarBackup { + lockKey: string; + darPath: string; + packageId: string; + packageVersion: string; + entry: DarsLock['packages'][string]; +} + +interface ValidateDevnetDarOptions { + repositoryRoot: string; + lock: DarsLock; + packageName: string; + packageVersion: string; + candidateDarPath: string; + expectedCandidateSha256?: string; + preferences: DevnetPackagePreference[]; + /** Mainnet uploads require the exact candidate to be preferred by both DevNet providers. */ + requireExactOnProviderCount?: number; +} + +function assertRegularDar(repositoryRoot: string, filePath: string): fs.Stats { + return assertContainedRegularFile(repositoryRoot, filePath, 'DAR', MAX_DAR_SIZE_BYTES); +} + +function resolveLockDarPath(repositoryRoot: string, lockKey: string): string { + const darsRoot = path.resolve(repositoryRoot, 'dars'); + const darPath = path.resolve(darsRoot, lockKey); + if (!darPath.startsWith(`${darsRoot}${path.sep}`)) { + throw new Error(`Unsafe DAR lock key: ${lockKey}`); + } + return darPath; +} + +export function verifyLockedDar(repositoryRoot: string, lock: DarsLock, lockKey: string): string { + if (!Object.prototype.hasOwnProperty.call(lock.packages, lockKey)) { + throw new Error(`Missing dars.lock entry: ${lockKey}`); + } + const entry = lock.packages[lockKey]; + const darPath = resolveLockDarPath(repositoryRoot, lockKey); + if (!fs.existsSync(darPath)) throw new Error(`Missing DAR backup: ${lockKey}`); + const actualSize = assertRegularDar(repositoryRoot, darPath).size; + if (actualSize !== entry.size) { + throw new Error(`DAR backup ${lockKey} has size ${actualSize}; dars.lock requires ${entry.size}`); + } + const actualHash = computeSha256(darPath); + if (actualHash !== entry.sha256) { + throw new Error(`DAR backup ${lockKey} has SHA256 ${actualHash}; dars.lock requires ${entry.sha256}`); + } + return darPath; +} + +/** Integrity-check and identify every backup for an exact package name before retention mutates anything. */ +export function inspectPackageBackups( + repositoryRoot: string, + lock: DarsLock, + packageName: string +): InspectedDarBackup[] { + const backups: InspectedDarBackup[] = []; + for (const [lockKey, entry] of Object.entries(lock.packages)) { + const parts = lockKey.split('/'); + if (parts[0] !== packageName) continue; + if (parts.length !== 3 || !parts[1] || !parts[2]) { + throw new Error(`Invalid DAR lock key for ${packageName}: ${lockKey}`); + } + const packageVersion = parts[1]; + const darPath = verifyLockedDar(repositoryRoot, lock, lockKey); + const packageId = inspectDarPackageId(darPath, packageName, packageVersion); + backups.push({ lockKey, darPath, packageId, packageVersion, entry }); + } + return backups; +} + +export function findBaselineDar( + repositoryRoot: string, + lock: DarsLock, + preference: DevnetPackagePreference, + candidatePackageId: string, + candidateDarPath: string, + inspectPackageId: typeof inspectDarPackageId = inspectDarPackageId +): string { + if (preference.packageId === candidatePackageId) return candidateDarPath; + + const candidateEntries = Object.keys(lock.packages).filter((lockKey) => { + const [packageName, packageVersion] = lockKey.split('/'); + return packageName === preference.packageName && packageVersion === preference.packageVersion; + }); + + for (const lockKey of candidateEntries) { + const darPath = verifyLockedDar(repositoryRoot, lock, lockKey); + const packageId = inspectPackageId(darPath, preference.packageName, preference.packageVersion); + if (packageId === preference.packageId) return darPath; + } + + throw new Error( + `The exact DevNet baseline ${preference.packageName} ${preference.packageVersion} (${preference.packageId}) is missing from dars/. Restore that deployed DAR before changing the package.` + ); +} + +function runUpgradeCheck(oldDar: string, newDar: string, label: string): void { + assertDarArchiveSafe(oldDar); + assertDarArchiveSafe(newDar); + try { + execFileSync('dpm', ['upgrade-check', '--both', oldDar, newDar], { + encoding: 'utf8', + killSignal: 'SIGKILL', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: DAML_UPGRADE_CHECK_TIMEOUT_MS, + env: { ...process.env, PATH: `${process.env.HOME}/.dpm/bin:${process.env.PATH}` }, + }); + } catch (error) { + const failure = error as { stdout?: string; stderr?: string; message?: string }; + const output = `${failure.stdout ?? ''}${failure.stderr ?? ''}`.trim(); + throw new Error( + `dpm upgrade-check --both failed from ${label}.${output ? `\n${output}` : ` ${failure.message ?? ''}`}` + ); + } +} + +/** Validate one candidate against the exact preferred package on every DevNet provider. */ +export function validateDevnetDarCandidate(options: ValidateDevnetDarOptions): DevnetDarValidationResult { + const { + repositoryRoot, + lock, + packageName, + packageVersion, + candidateDarPath, + expectedCandidateSha256, + preferences, + requireExactOnProviderCount, + } = options; + + assertRegularDar(repositoryRoot, candidateDarPath); + if (expectedCandidateSha256) { + const actualHash = computeSha256(candidateDarPath); + if (actualHash !== expectedCandidateSha256) { + throw new Error(`Candidate DAR has SHA256 ${actualHash}; dars.lock requires ${expectedCandidateSha256}.`); + } + } + assertDevnetPreferencesConsistent(preferences); + const candidatePackageId = inspectDarPackageId(candidateDarPath, packageName, packageVersion); + const decision = decideDevnetCandidateVersion(preferences, candidatePackageId); + + if (packageVersion !== decision.expectedVersion) { + const liveDescription = + preferences.length === 0 + ? 'absent on both DevNet providers' + : preferences + .map((preference) => `${preference.provider}=${preference.packageVersion} (${preference.packageId})`) + .join(', '); + throw new Error( + `${packageName} candidate ${packageVersion} (${candidatePackageId}) must use ${decision.expectedVersion}; DevNet is ${liveDescription}.` + ); + } + + if (requireExactOnProviderCount !== undefined) { + const exactPreferences = preferences.filter((preference) => preference.packageId === candidatePackageId); + if (exactPreferences.length !== requireExactOnProviderCount) { + throw new Error( + `${packageName} ${packageVersion} must be the exact preferred package on all ${requireExactOnProviderCount} DevNet providers before a Mainnet upload.` + ); + } + } + + const uniquePreferences = uniqueDevnetBaselines(preferences); + const compatibilityBaselines = uniquePreferences.map((preference) => ({ + packageId: preference.packageId, + packageVersion: preference.packageVersion, + darPath: findBaselineDar(repositoryRoot, lock, preference, candidatePackageId, candidateDarPath), + })); + + for (const baseline of compatibilityBaselines) { + if (baseline.packageId === candidatePackageId) continue; + runUpgradeCheck( + baseline.darPath, + candidateDarPath, + `${packageName} ${baseline.packageVersion} (${baseline.packageId})` + ); + } + + return { + candidatePackageId, + expectedVersion: decision.expectedVersion, + highestDevnetVersion: decision.highestDevnetVersion, + compatibilityBaselines, + }; +} diff --git a/scripts/devnet-package-versions.ts b/scripts/devnet-package-versions.ts new file mode 100644 index 00000000..20d6abd1 --- /dev/null +++ b/scripts/devnet-package-versions.ts @@ -0,0 +1,57 @@ +import type { ProviderType } from '@fairmint/canton-node-sdk'; +import { LEDGER_SCRIPT_PROVIDERS } from './providers'; +import { createLedgerJsonApiClient } from './utils'; + +export interface DevnetPackagePreference { + packageId: string; + packageName: string; + packageVersion: string; + provider: ProviderType; +} + +function errorText(error: unknown): string { + if (error instanceof Error) return `${error.name}: ${error.message}`; + try { + return JSON.stringify(error); + } catch { + return String(error); + } +} + +function isPackageNameNotFound(error: unknown): boolean { + return /PACKAGE_NAMES?_NOT_FOUND|PackageNamesNotFound|package names?.*not found/i.test(errorText(error)); +} + +async function queryProvider( + packageName: string, + provider: (typeof LEDGER_SCRIPT_PROVIDERS)[number] +): Promise { + const client = createLedgerJsonApiClient('devnet', provider); + try { + const response = await client.interactiveSubmissionGetPreferredPackageVersion({ + packageName, + parties: [client.getPartyId()], + }); + const reference = response.packagePreference?.packageReference; + if (!reference) return null; + if (reference.packageName !== packageName) { + throw new Error( + `${provider} DevNet returned ${reference.packageName} while querying preferred package ${packageName}.` + ); + } + return { ...reference, provider }; + } catch (error) { + if (isPackageNameNotFound(error)) { + return null; + } + throw new Error(`Unable to query ${provider} DevNet package preference for ${packageName}: ${errorText(error)}`); + } +} + +/** Query both configured DevNet participants. Any failed query fails closed. */ +export async function queryDevnetPackagePreferences(packageName: string): Promise { + const results = await Promise.all( + LEDGER_SCRIPT_PROVIDERS.map(async (provider) => queryProvider(packageName, provider)) + ); + return results.filter((result): result is DevnetPackagePreference => result !== null); +} diff --git a/scripts/mainnet-marker-attestation.test.ts b/scripts/mainnet-marker-attestation.test.ts new file mode 100644 index 00000000..e8983488 --- /dev/null +++ b/scripts/mainnet-marker-attestation.test.ts @@ -0,0 +1,270 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { describe, it } from 'node:test'; +import type { DarsLock, DarsLockEntry } from './dar-utils'; +import { + createMainnetMarkerAttestation, + getMainnetAttestationArtifactName, + MAINNET_ATTESTATION_WORKFLOW, + parseOcpMainnetMarkerBranch, + requiresMainnetAttestation, + selectWorkflowArtifact, + validateWorkflowRun, + verifyMainnetMarkerAttestation, +} from './mainnet-marker-attestation'; +import { requirePackageConfig } from './packages'; + +const REPOSITORY = 'Fairmint/open-captable-protocol-daml'; +const PACKAGE = 'ocp'; +const RUN_ID = 123456; +const RUN_ATTEMPT = 2; +const BRANCH = `automation/ocp-release-${RUN_ID}-${RUN_ATTEMPT}`; +const PACKAGE_CONFIG = requirePackageConfig(PACKAGE); +const LOCK_KEY = `${PACKAGE_CONFIG.name}/${PACKAGE_CONFIG.version}/${PACKAGE_CONFIG.darName}.dar`; +const CONTEXT = { defaultBranch: 'main', repository: REPOSITORY, workflowPath: MAINNET_ATTESTATION_WORKFLOW }; + +function entry(networks: string[]): DarsLockEntry { + return { + networks, + sdkVersion: '3.5.1', + sha256: 'a'.repeat(64), + size: 42, + uploadedAt: '2026-07-13T12:00:00.000Z', + }; +} + +function git(repositoryRoot: string, args: string[]): string { + return execFileSync('git', args, { cwd: repositoryRoot, encoding: 'utf8' }).trim(); +} + +function writeLock(repositoryRoot: string, lock: DarsLock): void { + fs.mkdirSync(path.join(repositoryRoot, 'dars'), { recursive: true }); + fs.writeFileSync(path.join(repositoryRoot, 'dars', 'dars.lock'), `${JSON.stringify(lock, null, 2)}\n`); +} + +interface RepositoryFixture { + baseSha: string; + candidateSha: string; + markerSha: string; + repositoryRoot: string; +} + +function repositoryFixture(options: { extraFile?: boolean; multipleMarkers?: boolean } = {}): RepositoryFixture { + const repositoryRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ocp-mainnet-attestation-')); + git(repositoryRoot, ['init', '-b', 'main']); + git(repositoryRoot, ['config', 'user.name', 'Test']); + git(repositoryRoot, ['config', 'user.email', 'test@example.com']); + const otherKey = `EquityClearing-v02/0.0.2/EquityClearing-v02.dar`; + const baseLock: DarsLock = { + version: 1, + packages: { + [LOCK_KEY]: entry(['devnet']), + ...(options.multipleMarkers ? { [otherKey]: entry(['devnet']) } : {}), + }, + }; + writeLock(repositoryRoot, baseLock); + git(repositoryRoot, ['add', 'dars/dars.lock']); + git(repositoryRoot, ['commit', '-m', 'base']); + const baseSha = git(repositoryRoot, ['rev-parse', 'HEAD']); + + const markerLock: DarsLock = { + version: 1, + packages: { + [LOCK_KEY]: entry(['devnet', 'mainnet']), + ...(options.multipleMarkers ? { [otherKey]: entry(['devnet', 'mainnet']) } : {}), + }, + }; + writeLock(repositoryRoot, markerLock); + if (options.extraFile) fs.writeFileSync(path.join(repositoryRoot, 'forged.txt'), 'forged\n'); + git(repositoryRoot, ['add', '.']); + git(repositoryRoot, ['commit', '-m', 'marker']); + const markerSha = git(repositoryRoot, ['rev-parse', 'HEAD']); + return { baseSha, candidateSha: markerSha, markerSha, repositoryRoot }; +} + +function workflowRun(headSha: string): Record { + return { + event: 'workflow_dispatch', + head_branch: 'main', + head_repository: { full_name: REPOSITORY }, + head_sha: headSha, + id: RUN_ID, + path: MAINNET_ATTESTATION_WORKFLOW, + repository: { full_name: REPOSITORY }, + run_attempt: RUN_ATTEMPT, + status: 'in_progress', + }; +} + +void describe('Mainnet marker workflow attestation', () => { + void it('binds a trusted workflow run, immutable artifact, and exact one-marker commit', () => { + const fixture = repositoryFixture(); + try { + assert.equal(requiresMainnetAttestation(fixture.repositoryRoot, fixture.baseSha, fixture.markerSha), true); + assert.equal(requiresMainnetAttestation(fixture.repositoryRoot, fixture.baseSha, fixture.baseSha), false); + const artifact = createMainnetMarkerAttestation({ + markerCommitSha: fixture.markerSha, + package: PACKAGE, + repository: REPOSITORY, + repositoryRoot: fixture.repositoryRoot, + runAttempt: RUN_ATTEMPT, + runId: RUN_ID, + workflowPath: MAINNET_ATTESTATION_WORKFLOW, + }); + assert.deepEqual( + verifyMainnetMarkerAttestation({ + artifact, + branch: BRANCH, + candidateHead: fixture.candidateSha, + context: CONTEXT, + repositoryRoot: fixture.repositoryRoot, + trustedDefaultHead: fixture.baseSha, + workflowRun: workflowRun(fixture.baseSha), + }), + artifact + ); + } finally { + fs.rmSync(fixture.repositoryRoot, { force: true, recursive: true }); + } + }); + + void it('rejects forged automation branch coordinates and workflow provenance', () => { + assert.throws( + () => parseOcpMainnetMarkerBranch('automation/ocp-release-not-a-run-1'), + /must be automation\/ocp-release/ + ); + const parsed = parseOcpMainnetMarkerBranch(BRANCH); + assert.throws( + () => + validateWorkflowRun({ ...workflowRun('a'.repeat(40)), path: '.github/workflows/forged.yml' }, parsed, CONTEXT), + /workflow path/ + ); + assert.throws( + () => validateWorkflowRun({ ...workflowRun('a'.repeat(40)), head_branch: 'attacker' }, parsed, CONTEXT), + /trusted workflow ref/ + ); + assert.doesNotThrow(() => + validateWorkflowRun( + { + ...workflowRun('a'.repeat(40)), + event: 'push', + head_branch: 'OpenCapTable-v34-v0.0.2', + }, + parsed, + CONTEXT + ) + ); + }); + + void it('fails closed on missing, expired, and multiple exact artifacts', () => { + const parsed = parseOcpMainnetMarkerBranch(BRANCH); + const name = getMainnetAttestationArtifactName(parsed); + assert.deepEqual( + selectWorkflowArtifact( + { artifacts: [{ expired: false, id: 7, name, size_in_bytes: 512 }], total_count: 1 }, + parsed + ), + { artifactId: 7, artifactName: name } + ); + assert.throws(() => selectWorkflowArtifact({ artifacts: [], total_count: 0 }, parsed), /found 0/); + assert.throws( + () => + selectWorkflowArtifact( + { artifacts: [{ expired: true, id: 1, name, size_in_bytes: 512 }], total_count: 1 }, + parsed + ), + /expired/ + ); + assert.throws( + () => + selectWorkflowArtifact( + { + artifacts: [ + { expired: false, id: 1, name, size_in_bytes: 512 }, + { expired: false, id: 2, name, size_in_bytes: 512 }, + ], + total_count: 2, + }, + parsed + ), + /found 2/ + ); + assert.throws( + () => + selectWorkflowArtifact( + { artifacts: [{ expired: false, id: 1, name, size_in_bytes: 2 * 1024 * 1024 }], total_count: 1 }, + parsed + ), + /exceeds/ + ); + }); + + void it('rejects marker commits that change another path or add multiple markers', () => { + for (const options of [{ extraFile: true }, { multipleMarkers: true }]) { + const fixture = repositoryFixture(options); + try { + assert.throws( + () => + createMainnetMarkerAttestation({ + markerCommitSha: fixture.markerSha, + package: PACKAGE, + repository: REPOSITORY, + repositoryRoot: fixture.repositoryRoot, + runAttempt: RUN_ATTEMPT, + runId: RUN_ID, + workflowPath: MAINNET_ATTESTATION_WORKFLOW, + }), + options.extraFile ? /may change only dars\/dars\.lock/ : /exactly one mainnet marker/ + ); + } finally { + fs.rmSync(fixture.repositoryRoot, { force: true, recursive: true }); + } + } + }); + + void it('rejects an artifact whose package or immutable lock metadata was forged', () => { + const fixture = repositoryFixture(); + try { + const artifact = createMainnetMarkerAttestation({ + markerCommitSha: fixture.markerSha, + package: PACKAGE, + repository: REPOSITORY, + repositoryRoot: fixture.repositoryRoot, + runAttempt: RUN_ATTEMPT, + runId: RUN_ID, + workflowPath: MAINNET_ATTESTATION_WORKFLOW, + }); + assert.throws( + () => + verifyMainnetMarkerAttestation({ + artifact: { ...artifact, package: 'forged' }, + branch: BRANCH, + candidateHead: fixture.candidateSha, + context: CONTEXT, + repositoryRoot: fixture.repositoryRoot, + trustedDefaultHead: fixture.baseSha, + workflowRun: workflowRun(fixture.baseSha), + }), + /does not match its repository, workflow run, or automation branch/ + ); + assert.throws( + () => + verifyMainnetMarkerAttestation({ + artifact: { ...artifact, lockEntry: { ...artifact.lockEntry, sha256: 'b'.repeat(64) } }, + branch: BRANCH, + candidateHead: fixture.candidateSha, + context: CONTEXT, + repositoryRoot: fixture.repositoryRoot, + trustedDefaultHead: fixture.baseSha, + workflowRun: workflowRun(fixture.baseSha), + }), + /immutable sha256 does not match/ + ); + } finally { + fs.rmSync(fixture.repositoryRoot, { force: true, recursive: true }); + } + }); +}); diff --git a/scripts/mainnet-marker-attestation.ts b/scripts/mainnet-marker-attestation.ts new file mode 100644 index 00000000..9f83dcd7 --- /dev/null +++ b/scripts/mainnet-marker-attestation.ts @@ -0,0 +1,535 @@ +#!/usr/bin/env node +/** + * Create and verify immutable GitHub Actions attestations for Mainnet DAR markers. + * + * The trusted upload workflow writes an attestation only after its Mainnet upload and marker commit succeed. The + * pull_request_target policy then resolves that exact run artifact and verifies both GitHub provenance and the Git + * commit transition before allowing a Mainnet marker addition. + */ + +import { execFileSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; +import type { DarsLock, DarsLockEntry } from './dar-utils'; +import { assertDarsLockSchema } from './dar-marker-policy'; +import { requirePackageConfig } from './packages'; + +export const MAINNET_ATTESTATION_SCHEMA_VERSION = 1; +export const MAINNET_ATTESTATION_FILE = 'mainnet-marker-attestation.json'; +export const MAINNET_ATTESTATION_WORKFLOW = '.github/workflows/release.yml'; +const MAX_JSON_BYTES = 5 * 1024 * 1024; +const MAX_ATTESTATION_ARTIFACT_BYTES = 1024 * 1024; +const SHA_PATTERN = /^[0-9a-f]{40}$/; +const SAFE_PACKAGE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +const IMMUTABLE_FIELDS = ['sha256', 'size', 'sdkVersion', 'uploadedAt'] as const; + +type JsonRecord = Record; + +export interface ParsedMarkerBranch { + package: string; + runAttempt: number; + runId: number; +} + +export interface MainnetMarkerAttestation { + lockEntry: Pick; + lockKey: string; + markerCommitSha: string; + network: 'mainnet'; + package: string; + repository: string; + runAttempt: number; + runId: number; + schemaVersion: 1; + workflowPath: string; +} + +export interface WorkflowRunIdentity { + headSha: string; +} + +export interface ArtifactSelection { + artifactId: number; + artifactName: string; +} + +interface AttestationContext { + defaultBranch: string; + repository: string; + workflowPath: string; +} + +interface MarkerTransition { + entry: DarsLockEntry; + lockKey: string; + parentSha: string; +} + +function isRecord(value: unknown): value is JsonRecord { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function assertExactKeys(value: JsonRecord, expectedKeys: readonly string[], label: string): void { + const actual = Object.keys(value).sort(); + const expected = [...expectedKeys].sort(); + if (!isDeepStrictEqual(actual, expected)) { + throw new Error(`${label}: expected exactly ${expected.join(', ')}; received ${actual.join(', ')}`); + } +} + +function requireString(value: unknown, label: string): string { + if (typeof value !== 'string' || value.length === 0) throw new Error(`${label} must be a non-empty string`); + return value; +} + +function requirePositiveInteger(value: unknown, label: string): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 1) { + throw new Error(`${label} must be a positive safe integer`); + } + return value; +} + +function requireSha(value: unknown, label: string): string { + const sha = requireString(value, label); + if (!SHA_PATTERN.test(sha)) throw new Error(`${label} must be a lowercase 40-character Git SHA`); + return sha; +} + +function readBoundedJson(filePath: string, label: string): unknown { + const stats = fs.statSync(filePath); + if (!stats.isFile() || stats.size < 1 || stats.size > MAX_JSON_BYTES) { + throw new Error(`${label} must be a non-empty regular file no larger than ${MAX_JSON_BYTES} bytes`); + } + return JSON.parse(fs.readFileSync(filePath, 'utf8')) as unknown; +} + +function git(repositoryRoot: string, args: string[]): string { + return execFileSync('git', args, { cwd: repositoryRoot, encoding: 'utf8' }).trim(); +} + +function isAncestor(repositoryRoot: string, ancestor: string, descendant: string): boolean { + try { + execFileSync('git', ['merge-base', '--is-ancestor', ancestor, descendant], { + cwd: repositoryRoot, + stdio: 'ignore', + }); + return true; + } catch { + return false; + } +} + +function readLockAtCommit(repositoryRoot: string, commitSha: string): DarsLock { + const text = git(repositoryRoot, ['show', `${commitSha}:dars/dars.lock`]); + if (Buffer.byteLength(text, 'utf8') > MAX_JSON_BYTES) { + throw new Error(`dars.lock at ${commitSha} exceeds ${MAX_JSON_BYTES} bytes`); + } + const parsed = JSON.parse(text) as unknown; + assertDarsLockSchema(parsed, `dars.lock at ${commitSha}`); + return parsed; +} + +function immutableEntry(entry: DarsLockEntry): MainnetMarkerAttestation['lockEntry'] { + return { + sdkVersion: entry.sdkVersion, + sha256: entry.sha256, + size: entry.size, + uploadedAt: entry.uploadedAt, + }; +} + +function assertEntryMatches( + actual: DarsLockEntry, + expected: MainnetMarkerAttestation['lockEntry'], + label: string +): void { + for (const field of IMMUTABLE_FIELDS) { + if (actual[field] !== expected[field]) { + throw new Error(`${label}: immutable ${field} does not match the attestation`); + } + } +} + +function parseLockCoordinates(lockKey: string): { darName: string; packageName: string } { + const parts = lockKey.split('/'); + if (parts.length !== 3 || !parts[0] || !/^\d+\.\d+\.\d+$/.test(parts[1] ?? '')) { + throw new Error(`Invalid attested lock key: ${lockKey}`); + } + const file = parts[2]; + if (!file.endsWith('.dar')) throw new Error(`Invalid attested lock key: ${lockKey}`); + const darName = file.slice(0, -4); + if (!SAFE_PACKAGE_PATTERN.test(parts[0]) || !SAFE_PACKAGE_PATTERN.test(darName)) { + throw new Error(`Unsafe attested lock key: ${lockKey}`); + } + return { darName, packageName: parts[0] }; +} + +function assertPackageMatchesLock(packageKey: string, lockKey: string): void { + const packageConfig = requirePackageConfig(packageKey); + const { darName, packageName } = parseLockCoordinates(lockKey); + if (packageName !== packageConfig.name || darName !== packageConfig.darName) { + throw new Error( + `Attested package ${packageKey} resolves to ${packageConfig.name}/${packageConfig.darName}, not ${lockKey}` + ); + } +} + +export function parseOcpMainnetMarkerBranch(branch: string): ParsedMarkerBranch { + const match = /^automation\/ocp-release-([1-9][0-9]*)-([1-9][0-9]*)$/.exec(branch); + if (!match) { + throw new Error('Mainnet marker branch must be automation/ocp-release--'); + } + const runId = Number(match[1]); + const runAttempt = Number(match[2]); + if (!Number.isSafeInteger(runId) || !Number.isSafeInteger(runAttempt)) { + throw new Error('Mainnet marker branch contains an unsafe run id or attempt'); + } + return { package: 'ocp', runAttempt, runId }; +} + +export function getMainnetAttestationArtifactName(parsed: ParsedMarkerBranch): string { + return `dar-mainnet-marker-attestation-${parsed.runId}-${parsed.runAttempt}-${parsed.package}`; +} + +export function requiresMainnetAttestation(repositoryRoot: string, baseRef: string, candidateRef: string): boolean { + const safeRef = /^(?:HEAD|[0-9a-f]{40})$/; + if (!safeRef.test(baseRef) || !safeRef.test(candidateRef)) throw new Error('Unsafe Git ref for marker comparison'); + const baseSha = git(repositoryRoot, ['rev-parse', '--verify', `${baseRef}^{commit}`]); + const candidateSha = git(repositoryRoot, ['rev-parse', '--verify', `${candidateRef}^{commit}`]); + const baseLock = readLockAtCommit(repositoryRoot, baseSha); + const candidateLock = readLockAtCommit(repositoryRoot, candidateSha); + return Object.entries(candidateLock.packages).some(([lockKey, entry]) => { + const baseNetworks = Object.prototype.hasOwnProperty.call(baseLock.packages, lockKey) + ? baseLock.packages[lockKey].networks + : []; + return entry.networks.includes('mainnet') && !baseNetworks.includes('mainnet'); + }); +} + +export function validateWorkflowRun( + rawRun: unknown, + parsed: ParsedMarkerBranch, + context: AttestationContext +): WorkflowRunIdentity { + if (!isRecord(rawRun)) throw new Error('GitHub workflow run response must be an object'); + const repository = isRecord(rawRun.repository) ? rawRun.repository.full_name : undefined; + const headRepository = isRecord(rawRun.head_repository) ? rawRun.head_repository.full_name : undefined; + const failures: string[] = []; + if (rawRun.id !== parsed.runId) failures.push('run id'); + if (rawRun.run_attempt !== parsed.runAttempt) failures.push('run attempt'); + if (rawRun.event !== 'workflow_dispatch' && rawRun.event !== 'push') failures.push('workflow event'); + if (rawRun.path !== context.workflowPath) failures.push('workflow path'); + const releaseTagRef = + typeof rawRun.head_branch === 'string' && /^OpenCapTable-v[0-9]+-v[0-9]+\.[0-9]+\.[0-9]+$/.test(rawRun.head_branch); + const trustedWorkflowRef = + rawRun.event === 'workflow_dispatch' + ? rawRun.head_branch === context.defaultBranch + : rawRun.head_branch === context.defaultBranch || releaseTagRef; + if (!trustedWorkflowRef) failures.push('trusted workflow ref'); + if (typeof repository !== 'string' || repository.toLowerCase() !== context.repository.toLowerCase()) { + failures.push('repository'); + } + if (typeof headRepository !== 'string' || headRepository.toLowerCase() !== context.repository.toLowerCase()) { + failures.push('head repository'); + } + let headSha = ''; + try { + headSha = requireSha(rawRun.head_sha, 'Workflow run head SHA'); + } catch { + failures.push('head SHA'); + } + if (failures.length > 0) { + throw new Error(`GitHub run does not match trusted Mainnet upload provenance: ${failures.join(', ')}`); + } + return { headSha }; +} + +export function selectWorkflowArtifact(rawListing: unknown, parsed: ParsedMarkerBranch): ArtifactSelection { + if (!isRecord(rawListing) || !Array.isArray(rawListing.artifacts)) { + throw new Error('GitHub workflow artifact response must contain an artifacts array'); + } + if ( + typeof rawListing.total_count !== 'number' || + !Number.isSafeInteger(rawListing.total_count) || + rawListing.total_count < rawListing.artifacts.length + ) { + throw new Error('GitHub workflow artifact response has an invalid total_count'); + } + if (rawListing.total_count !== rawListing.artifacts.length) { + throw new Error('GitHub workflow artifact response is incomplete; refusing to select from a partial page'); + } + + const expectedName = getMainnetAttestationArtifactName(parsed); + const matches = rawListing.artifacts.filter((value) => isRecord(value) && value.name === expectedName); + if (matches.length !== 1) { + throw new Error( + `Expected exactly one immutable Mainnet attestation artifact named ${expectedName}; found ${matches.length}` + ); + } + const match = matches[0]; + if (match.expired !== false) throw new Error(`Mainnet attestation artifact ${expectedName} is expired`); + const artifactBytes = requirePositiveInteger(match.size_in_bytes, 'Mainnet attestation artifact size'); + if (artifactBytes > MAX_ATTESTATION_ARTIFACT_BYTES) { + throw new Error(`Mainnet attestation artifact ${expectedName} exceeds ${MAX_ATTESTATION_ARTIFACT_BYTES} bytes`); + } + return { + artifactId: requirePositiveInteger(match.id, 'Mainnet attestation artifact id'), + artifactName: expectedName, + }; +} + +export function parseMainnetMarkerAttestation(raw: unknown): MainnetMarkerAttestation { + if (!isRecord(raw)) throw new Error('Mainnet marker attestation must be an object'); + assertExactKeys( + raw, + [ + 'schemaVersion', + 'repository', + 'workflowPath', + 'runId', + 'runAttempt', + 'package', + 'network', + 'markerCommitSha', + 'lockKey', + 'lockEntry', + ], + 'Mainnet marker attestation' + ); + if (raw.schemaVersion !== MAINNET_ATTESTATION_SCHEMA_VERSION) { + throw new Error(`Unsupported Mainnet marker attestation schema: ${String(raw.schemaVersion)}`); + } + if (raw.network !== 'mainnet') throw new Error('Mainnet marker attestation network must be mainnet'); + if (!isRecord(raw.lockEntry)) throw new Error('Mainnet marker attestation lockEntry must be an object'); + assertExactKeys(raw.lockEntry, IMMUTABLE_FIELDS, 'Mainnet marker attestation lockEntry'); + const { lockEntry } = raw; + const sha256 = requireString(lockEntry.sha256, 'Attested DAR sha256'); + if (!/^[0-9a-f]{64}$/.test(sha256)) + throw new Error('Attested DAR sha256 must be 64 lowercase hexadecimal characters'); + const size = requirePositiveInteger(lockEntry.size, 'Attested DAR size'); + const sdkVersion = requireString(lockEntry.sdkVersion, 'Attested DAR SDK version'); + const uploadedAt = requireString(lockEntry.uploadedAt, 'Attested DAR uploadedAt'); + if (!Number.isFinite(Date.parse(uploadedAt))) throw new Error('Attested DAR uploadedAt must be a valid timestamp'); + + return { + lockEntry: { sdkVersion, sha256, size, uploadedAt }, + lockKey: requireString(raw.lockKey, 'Attested lock key'), + markerCommitSha: requireSha(raw.markerCommitSha, 'Attested marker commit'), + network: 'mainnet', + package: requireString(raw.package, 'Attested package'), + repository: requireString(raw.repository, 'Attested repository'), + runAttempt: requirePositiveInteger(raw.runAttempt, 'Attested run attempt'), + runId: requirePositiveInteger(raw.runId, 'Attested run id'), + schemaVersion: MAINNET_ATTESTATION_SCHEMA_VERSION, + workflowPath: requireString(raw.workflowPath, 'Attested workflow path'), + }; +} + +function inspectMarkerTransition(repositoryRoot: string, markerCommitSha: string): MarkerTransition { + const resolved = git(repositoryRoot, ['rev-parse', '--verify', `${markerCommitSha}^{commit}`]); + if (resolved !== markerCommitSha) throw new Error('Marker commit SHA did not resolve exactly'); + + const lineage = git(repositoryRoot, ['rev-list', '--parents', '-n', '1', markerCommitSha]).split(/\s+/); + if (lineage.length !== 2) throw new Error('Mainnet marker commit must have exactly one parent'); + const parentSha = lineage[1]; + const changedPaths = git(repositoryRoot, ['diff-tree', '--no-commit-id', '--name-only', '-r', markerCommitSha]) + .split('\n') + .filter(Boolean); + if (!isDeepStrictEqual(changedPaths, ['dars/dars.lock'])) { + throw new Error(`Mainnet marker commit may change only dars/dars.lock; changed: ${changedPaths.join(', ')}`); + } + + const parentLock = readLockAtCommit(repositoryRoot, parentSha); + const markerLock = readLockAtCommit(repositoryRoot, markerCommitSha); + if (parentLock.version !== markerLock.version) throw new Error('Mainnet marker commit changed dars.lock version'); + const parentKeys = Object.keys(parentLock.packages).sort(); + const markerKeys = Object.keys(markerLock.packages).sort(); + if (!isDeepStrictEqual(parentKeys, markerKeys)) throw new Error('Mainnet marker commit changed the DAR lock key set'); + + let addition: { entry: DarsLockEntry; lockKey: string } | null = null; + for (const lockKey of parentKeys) { + const parentEntry = parentLock.packages[lockKey]; + const markerEntry = markerLock.packages[lockKey]; + const immutableMatches = IMMUTABLE_FIELDS.every((field) => parentEntry[field] === markerEntry[field]); + if (!immutableMatches) throw new Error(`Mainnet marker commit changed immutable metadata for ${lockKey}`); + if (isDeepStrictEqual(parentEntry.networks, markerEntry.networks)) continue; + if ( + addition || + parentEntry.networks.includes('mainnet') || + !isDeepStrictEqual(markerEntry.networks, [...parentEntry.networks, 'mainnet']) + ) { + throw new Error('Mainnet marker commit must add exactly one mainnet marker and make no other lock changes'); + } + addition = { entry: markerEntry, lockKey }; + } + if (!addition) throw new Error('Mainnet marker commit did not add exactly one mainnet marker'); + return { ...addition, parentSha }; +} + +export function createMainnetMarkerAttestation(options: { + markerCommitSha: string; + package: string; + repository: string; + repositoryRoot: string; + runAttempt: number; + runId: number; + workflowPath: string; +}): MainnetMarkerAttestation { + const markerCommitSha = requireSha(options.markerCommitSha, 'Marker commit SHA'); + const transition = inspectMarkerTransition(options.repositoryRoot, markerCommitSha); + assertPackageMatchesLock(options.package, transition.lockKey); + return { + lockEntry: immutableEntry(transition.entry), + lockKey: transition.lockKey, + markerCommitSha, + network: 'mainnet', + package: options.package, + repository: options.repository, + runAttempt: requirePositiveInteger(options.runAttempt, 'Run attempt'), + runId: requirePositiveInteger(options.runId, 'Run id'), + schemaVersion: MAINNET_ATTESTATION_SCHEMA_VERSION, + workflowPath: options.workflowPath, + }; +} + +export function verifyMainnetMarkerAttestation(options: { + artifact: unknown; + branch: string; + candidateHead: string; + context: AttestationContext; + repositoryRoot: string; + trustedDefaultHead: string; + workflowRun: unknown; +}): MainnetMarkerAttestation { + const parsedBranch = parseOcpMainnetMarkerBranch(options.branch); + const run = validateWorkflowRun(options.workflowRun, parsedBranch, options.context); + const artifact = parseMainnetMarkerAttestation(options.artifact); + + if ( + artifact.repository !== options.context.repository || + artifact.workflowPath !== options.context.workflowPath || + artifact.runId !== parsedBranch.runId || + artifact.runAttempt !== parsedBranch.runAttempt || + artifact.package !== parsedBranch.package + ) { + throw new Error('Mainnet marker attestation does not match its repository, workflow run, or automation branch'); + } + assertPackageMatchesLock(artifact.package, artifact.lockKey); + const candidateHead = requireSha(options.candidateHead, 'Candidate head SHA'); + if (!isAncestor(options.repositoryRoot, artifact.markerCommitSha, candidateHead)) { + throw new Error('Attested Mainnet marker commit is not an ancestor of the candidate head'); + } + if (!isAncestor(options.repositoryRoot, run.headSha, artifact.markerCommitSha)) { + throw new Error('Trusted workflow run head SHA is not an ancestor of the Mainnet marker commit'); + } + const trustedDefaultHead = requireSha(options.trustedDefaultHead, 'Trusted default-branch head SHA'); + if (!isAncestor(options.repositoryRoot, run.headSha, trustedDefaultHead)) { + throw new Error('Workflow run head SHA is not part of the trusted default-branch history'); + } + + const transition = inspectMarkerTransition(options.repositoryRoot, artifact.markerCommitSha); + if (transition.lockKey !== artifact.lockKey) throw new Error('Attested lock key does not match the marker commit'); + assertEntryMatches(transition.entry, artifact.lockEntry, artifact.lockKey); + + const candidateLock = readLockAtCommit(options.repositoryRoot, candidateHead); + if (!Object.prototype.hasOwnProperty.call(candidateLock.packages, artifact.lockKey)) { + throw new Error('Candidate head does not retain the attested Mainnet marker'); + } + const candidateEntry = candidateLock.packages[artifact.lockKey]; + if (!candidateEntry.networks.includes('mainnet')) { + throw new Error('Candidate head does not retain the attested Mainnet marker'); + } + assertEntryMatches(candidateEntry, artifact.lockEntry, `Candidate ${artifact.lockKey}`); + return artifact; +} + +function getArg(args: string[], name: string): string { + const indexes = args.flatMap((value, index) => (value === name ? [index] : [])); + if (indexes.length !== 1 || !args[indexes[0] + 1]) throw new Error(`Expected exactly one ${name} argument`); + return args[indexes[0] + 1]; +} + +function parseIntegerArg(args: string[], name: string): number { + const value = Number(getArg(args, name)); + return requirePositiveInteger(value, name); +} + +function writeJson(filePath: string, value: unknown): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, { flag: 'wx', mode: 0o600 }); +} + +function main(): void { + const [command, ...args] = process.argv.slice(2); + if (command === 'parse-branch') { + process.stdout.write(`${JSON.stringify(parseOcpMainnetMarkerBranch(getArg(args, '--branch')))}\n`); + return; + } + if (command === 'needs-attestation') { + const required = requiresMainnetAttestation( + path.resolve(getArg(args, '--repository-root')), + getArg(args, '--base-head'), + getArg(args, '--candidate-head') + ); + process.stdout.write(`${JSON.stringify({ required })}\n`); + return; + } + + const context: AttestationContext = { + defaultBranch: getArg(args, '--default-branch'), + repository: getArg(args, '--repository'), + workflowPath: getArg(args, '--workflow-path'), + }; + if (command === 'inspect') { + const branch = getArg(args, '--branch'); + const parsed = parseOcpMainnetMarkerBranch(branch); + validateWorkflowRun(readBoundedJson(getArg(args, '--workflow-run-json'), 'Workflow run JSON'), parsed, context); + const selection = selectWorkflowArtifact( + readBoundedJson(getArg(args, '--artifacts-json'), 'Workflow artifacts JSON'), + parsed + ); + process.stdout.write(`${JSON.stringify(selection)}\n`); + return; + } + if (command === 'create') { + const attestation = createMainnetMarkerAttestation({ + markerCommitSha: getArg(args, '--marker-commit'), + package: getArg(args, '--package'), + repository: context.repository, + repositoryRoot: path.resolve(getArg(args, '--repository-root')), + runAttempt: parseIntegerArg(args, '--run-attempt'), + runId: parseIntegerArg(args, '--run-id'), + workflowPath: context.workflowPath, + }); + writeJson(path.resolve(getArg(args, '--output')), attestation); + return; + } + if (command === 'verify') { + const artifact = verifyMainnetMarkerAttestation({ + artifact: readBoundedJson(getArg(args, '--artifact-json'), 'Mainnet marker attestation'), + branch: getArg(args, '--branch'), + candidateHead: getArg(args, '--candidate-head'), + context, + repositoryRoot: path.resolve(getArg(args, '--repository-root')), + trustedDefaultHead: getArg(args, '--trusted-default-head'), + workflowRun: readBoundedJson(getArg(args, '--workflow-run-json'), 'Workflow run JSON'), + }); + process.stdout.write( + `Verified Mainnet marker attestation for ${artifact.package} (${artifact.lockKey}) at ${artifact.markerCommitSha}\n` + ); + return; + } + throw new Error( + 'Usage: mainnet-marker-attestation.ts [options]' + ); +} + +if (require.main === module) { + try { + main(); + } catch (error) { + console.error(`Mainnet marker attestation failed: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + } +} diff --git a/scripts/upgrade-package.ts b/scripts/upgrade-package.ts index 4b8556a5..fe573323 100755 --- a/scripts/upgrade-package.ts +++ b/scripts/upgrade-package.ts @@ -11,6 +11,12 @@ import * as fs from 'fs'; import * as path from 'path'; import * as yaml from 'yaml'; +import { + assertDevnetPreferencesConsistent, + nextDevnetCandidateVersion, + resolveMajorPackageLineTarget, +} from './dar-version-policy'; +import { queryDevnetPackagePreferences } from './devnet-package-versions'; interface UpgradeOptions { packageName: string; @@ -20,6 +26,7 @@ interface UpgradeOptions { interface PackageInfo { currentFolder: string; currentMajorVersion: string; + currentPackageName: string; currentFullVersion: string; newFolder?: string; newMajorVersion?: string; @@ -104,8 +111,9 @@ function readDamlYaml(folderPath: string): { name: string; version: string } { } /** Get package information for upgrade */ -function getPackageInfo(packageName: string, upgradeType: 'major' | 'minor'): PackageInfo { +async function getPackageInfo(packageName: string, upgradeType: 'major' | 'minor'): Promise { const currentFolder = findPackageFolder(packageName); + const { name: currentPackageName, version: currentFullVersion } = readDamlYaml(currentFolder); const match = currentFolder.match(/^(.+)-(v(\d+))$/); // Check if package has version suffix @@ -119,20 +127,14 @@ function getPackageInfo(packageName: string, upgradeType: 'major' | 'minor'): Pa ); } - const { version: currentFullVersion } = readDamlYaml(currentFolder); - - // Minor upgrade - increment patch version - const versionParts = currentFullVersion.split('.'); - if (versionParts.length !== 3) { - throw new Error(`Invalid version format: ${currentFullVersion}`); - } - versionParts[2] = (parseInt(versionParts[2], 10) + 1).toString(); - + const preferences = await queryDevnetPackagePreferences(currentPackageName); + assertDevnetPreferencesConsistent(preferences); return { currentFolder, currentMajorVersion: '', // No major version for packages without suffix + currentPackageName, currentFullVersion, - newFullVersion: versionParts.join('.'), + newFullVersion: nextDevnetCandidateVersion(preferences), }; } @@ -140,28 +142,23 @@ function getPackageInfo(packageName: string, upgradeType: 'major' | 'minor'): Pa const currentMajorVersion = match[2]; // e.g., "v07" const majorVersionNum = parseInt(match[3], 10); // e.g., 7 - const { version: currentFullVersion } = readDamlYaml(currentFolder); - const info: PackageInfo = { currentFolder, currentMajorVersion, + currentPackageName, currentFullVersion, newFullVersion: currentFullVersion, }; if (upgradeType === 'major') { - const newMajorVersionNum = majorVersionNum + 1; - info.newMajorVersion = `v${newMajorVersionNum.toString().padStart(2, '0')}`; - info.newFolder = `${baseName}-${info.newMajorVersion}`; - info.newFullVersion = '0.0.1'; + const target = await resolveMajorPackageLineTarget(baseName, majorVersionNum, queryDevnetPackagePreferences); + info.newMajorVersion = target.majorVersion; + info.newFolder = target.packageName; + info.newFullVersion = target.candidateVersion; } else { - // Minor upgrade - increment patch version - const versionParts = currentFullVersion.split('.'); - if (versionParts.length !== 3) { - throw new Error(`Invalid version format: ${currentFullVersion}`); - } - versionParts[2] = (parseInt(versionParts[2], 10) + 1).toString(); - info.newFullVersion = versionParts.join('.'); + const preferences = await queryDevnetPackagePreferences(currentPackageName); + assertDevnetPreferencesConsistent(preferences); + info.newFullVersion = nextDevnetCandidateVersion(preferences); } return info; @@ -274,12 +271,22 @@ function performMajorUpgrade(info: PackageInfo): void { /** Perform minor upgrade */ function performMinorUpgrade(info: PackageInfo): void { - console.log('\n🔄 Performing MINOR upgrade...\n'); + console.log('\n🔄 Selecting the compatible release version...\n'); console.log(`Package: ${info.currentFolder}`); console.log(`Current version: ${info.currentFullVersion}`); - console.log(`New version: ${info.newFullVersion}`); + console.log(`Working version: ${info.newFullVersion}`); console.log(); + if (info.newFullVersion === info.currentFullVersion) { + console.log('✅ Current version is already the one mutable candidate selected from live DevNet.'); + console.log('\nNext steps:'); + console.log('1. Build and test the package'); + console.log( + `2. Refresh the candidate: npm run backup-dar -- --package ${info.currentPackageName} --version ${info.currentFullVersion}` + ); + return; + } + // Step 1: Update daml.yaml updateDamlYaml(info.currentFolder, info.newFullVersion); console.log(); @@ -296,7 +303,7 @@ function performMinorUpgrade(info: PackageInfo): void { console.log(' (No references found in other files)'); } - console.log('\n✅ Minor upgrade completed successfully!'); + console.log('\n✅ Selected exactly one patch beyond the highest live DevNet preference!'); console.log(`\nNext steps:`); console.log(`1. Review changes: git diff`); console.log(`2. Build the package: cd ${info.currentFolder} && daml build`); @@ -305,10 +312,10 @@ function performMinorUpgrade(info: PackageInfo): void { } /** Main function */ -function main(): void { +async function main(): Promise { try { const options = parseArgs(); - const info = getPackageInfo(options.packageName, options.upgradeType); + const info = await getPackageInfo(options.packageName, options.upgradeType); if (options.upgradeType === 'major') { performMajorUpgrade(info); @@ -326,4 +333,4 @@ function main(): void { } } -main(); +void main(); diff --git a/scripts/upload-dar.ts b/scripts/upload-dar.ts index f5460681..4949a029 100644 --- a/scripts/upload-dar.ts +++ b/scripts/upload-dar.ts @@ -2,8 +2,7 @@ /** * Upload a DAR file to devnet or mainnet. * - * Requires the DAR to be backed up first. If not backed up, the script will automatically run the backup process before - * uploading. + * Requires the fresh build to exactly match the committed DAR backup. Upload never rewrites candidate bytes. * * **Backed-up DARs:** Upload uses the version recorded under `dars/` + `dars.lock`. Older versions remain in `dars/` on * purpose—see https://github.com/Fairmint/open-captable-protocol-daml/wiki @@ -16,39 +15,47 @@ * ` (with Canton's **ALLOW_VET_INCOMPATIBLE_UPGRADES** force flag) to vet the new package id. */ -import { execSync } from 'child_process'; import * as fs from 'fs'; -import { getFreshDarPath, isDarBackedUp, recordNetworkUpload, requireBackedUpDar } from './dar-utils'; -import { parseNetworkArg, parsePackageArg, printPackageUsage, requireNetwork, requirePackage } from './packages'; +import { + computeSha256, + getDarLockKey, + getFreshDarPath, + loadDarsLock, + recordNetworkUpload, + requireBackedUpDar, +} from './dar-utils'; +import { assertDevnetMarkerForMainnet } from './dar-version-policy'; +import { validateDevnetDarCandidate } from './devnet-dar-policy'; +import { queryDevnetPackagePreferences } from './devnet-package-versions'; +import { + type PackageConfig, + parseNetworkArg, + parsePackageArg, + printPackageUsage, + requireNetwork, + requirePackage, +} from './packages'; import { LEDGER_SCRIPT_PROVIDERS } from './providers'; import { createLedgerJsonApiClient } from './utils'; -/** Ensure the DAR is backed up before upload. If not backed up, automatically run the backup process. */ -function ensureDarBackedUp(packageName: string, version: string, darName: string): void { - if (isDarBackedUp(packageName, version, darName)) { - return; - } - - // Check if fresh DAR exists to backup - const freshPath = getFreshDarPath(packageName, version, darName); +function requireFreshDar(pkg: PackageConfig): string { + const freshPath = getFreshDarPath(pkg.sourceDir, pkg.version, pkg.darName); if (!freshPath) { console.error(`❌ No DAR found to backup`); - console.error(` Expected: ${packageName}/.daml/dist/${darName}-${version}.dar`); + console.error(` Expected: ${pkg.sourceDir}/.daml/dist/${pkg.darName}-${pkg.version}.dar`); console.error(` Run "npm run build" first to build the DAR.`); process.exit(1); } + return freshPath; +} - console.log(`📋 DAR not backed up yet, backing up first...\n`); - - try { - execSync(`npm run backup-dar -- --package ${packageName} --version ${version}`, { - stdio: 'inherit', - cwd: process.cwd(), - }); - console.log(''); - } catch { - console.error(`\n❌ Failed to backup DAR`); - process.exit(1); +function assertBuildMatchesBackup(freshPath: string, backupPath: string): void { + const freshHash = computeSha256(freshPath); + const backupHash = computeSha256(backupPath); + if (freshHash !== backupHash) { + throw new Error( + `Fresh build SHA256 ${freshHash} does not match committed backup ${backupHash}. Run backup-dar and commit the candidate before uploading.` + ); } } @@ -64,11 +71,31 @@ async function main() { console.log(`\n📦 Uploading ${pkg.name} v${pkg.version} to ${network}\n`); - // Ensure DAR is backed up first (auto-backup if needed) - ensureDarBackedUp(pkg.name, pkg.version, pkg.darName); - - // Now require the backed-up DAR (this verifies integrity) + const freshPath = requireFreshDar(pkg); const darPath = requireBackedUpDar(pkg.name, pkg.version, pkg.darName); + assertBuildMatchesBackup(freshPath, darPath); + const lock = loadDarsLock(); + + if (network === 'mainnet') { + const lockKey = getDarLockKey(pkg.name, pkg.version, pkg.darName); + const entry = Object.prototype.hasOwnProperty.call(lock.packages, lockKey) ? lock.packages[lockKey] : undefined; + assertDevnetMarkerForMainnet(entry, lockKey); + } + + // DevNet is the sole live version authority even for a Mainnet upload. Never query Mainnet package preferences. + const preferences = await queryDevnetPackagePreferences(pkg.name); + const validation = validateDevnetDarCandidate({ + repositoryRoot: process.cwd(), + lock, + packageName: pkg.name, + packageVersion: pkg.version, + candidateDarPath: darPath, + preferences, + requireExactOnProviderCount: network === 'mainnet' ? LEDGER_SCRIPT_PROVIDERS.length : undefined, + }); + console.log( + `✅ Live DevNet policy and ${validation.compatibilityBaselines.length} compatibility baseline(s) verified for ${validation.candidatePackageId}\n` + ); // Upload to each provider independently so one unhealthy participant (e.g. devnet Intellect with no synchronizer) // does not block the other. @@ -123,7 +150,10 @@ async function main() { } recordNetworkUpload(pkg.name, pkg.version, pkg.darName, network); - console.log(`\n🎉 Upload complete\n`); + console.log(`\n🎉 Upload complete; ${network} marker recorded in dars.lock\n`); } -void main(); +void main().catch((error: unknown) => { + console.error(`\n❌ ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); +}); diff --git a/scripts/validate-candidate-dar-paths.ts b/scripts/validate-candidate-dar-paths.ts new file mode 100644 index 00000000..bf55a258 --- /dev/null +++ b/scripts/validate-candidate-dar-paths.ts @@ -0,0 +1,21 @@ +#!/usr/bin/env node +/** Validate detached-worktree DAR paths immediately before and after explicit Git LFS materialization. */ + +import { assertCandidateDarPaths } from './dar-candidate-path-policy'; + +function main(): void { + const rootIndex = process.argv.indexOf('--root'); + const separatorIndex = process.argv.indexOf('--'); + const candidateRoot = rootIndex >= 0 ? process.argv[rootIndex + 1] : undefined; + if (!candidateRoot || separatorIndex < 0 || separatorIndex <= rootIndex) { + throw new Error('Usage: tsx scripts/validate-candidate-dar-paths.ts --root -- [dar-path ...]'); + } + assertCandidateDarPaths(candidateRoot, process.argv.slice(separatorIndex + 1)); +} + +try { + main(); +} catch (error) { + console.error(`Candidate DAR path validation failed: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); +} diff --git a/scripts/validate-pr-dar-lfs.ts b/scripts/validate-pr-dar-lfs.ts new file mode 100644 index 00000000..f6a1c1d1 --- /dev/null +++ b/scripts/validate-pr-dar-lfs.ts @@ -0,0 +1,62 @@ +#!/usr/bin/env node +/** Inspect a verified PR commit and emit only safe, bounded DAR paths for `git lfs fetch --include`. */ + +import { execFileSync } from 'child_process'; +import { validateDarLfsTree, type DarTreeBlob } from './dar-lfs-policy'; + +function parseCommitArg(): string { + const index = process.argv.indexOf('--commit'); + const commit = index >= 0 ? process.argv[index + 1] : undefined; + if (!commit || !/^[0-9a-f]{40,64}$/.test(commit)) { + throw new Error('Usage: tsx scripts/validate-pr-dar-lfs.ts --commit '); + } + return commit; +} + +function readDarTree(commit: string): DarTreeBlob[] { + const raw = execFileSync('git', ['ls-tree', '-r', '-z', '--full-tree', commit, '--', 'dars'], { + encoding: 'utf8', + maxBuffer: 4 * 1024 * 1024, + }); + const entries: DarTreeBlob[] = []; + for (const record of raw.split('\0')) { + if (!record) continue; + const separator = record.indexOf('\t'); + const metadata = separator >= 0 ? record.slice(0, separator) : ''; + const treePath = separator >= 0 ? record.slice(separator + 1) : ''; + const metadataParts = metadata.split(' '); + if (metadataParts.length !== 3 || metadataParts.some((part) => !part) || !treePath) { + throw new Error(`Unable to parse DAR tree entry from ${commit}`); + } + const [mode, type, oid] = metadataParts; + if (!treePath.endsWith('.dar')) continue; + if (entries.length >= 250) throw new Error('PR tree contains more than 250 DARs'); + + const blobSize = Number(execFileSync('git', ['cat-file', '-s', oid], { encoding: 'utf8', maxBuffer: 1024 }).trim()); + if (!Number.isSafeInteger(blobSize) || blobSize < 0 || blobSize > 512) { + throw new Error(`DAR pointer Git blob is not small and canonical: ${treePath}`); + } + const blob = execFileSync('git', ['cat-file', 'blob', oid], { + encoding: 'utf8', + maxBuffer: 1024, + }); + entries.push({ blob, mode, path: treePath, type }); + } + return entries; +} + +function main(): void { + const commit = parseCommitArg(); + const pointers = validateDarLfsTree(readDarTree(commit)); + process.stderr.write( + `Validated ${pointers.length} DAR LFS pointer(s), ${pointers.reduce((total, item) => total + item.size, 0)} bytes declared.\n` + ); + process.stdout.write(pointers.map(({ path }) => path).join(',')); +} + +try { + main(); +} catch (error) { + console.error(`DAR LFS validation failed: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); +} diff --git a/scripts/verify-canton-payments-splice-amulet.ts b/scripts/verify-canton-payments-splice-amulet.ts index 9a1bd53f..986e9865 100644 --- a/scripts/verify-canton-payments-splice-amulet.ts +++ b/scripts/verify-canton-payments-splice-amulet.ts @@ -11,6 +11,7 @@ import { execFileSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; +import { assertDarArchiveSafe, DAML_INSPECT_TIMEOUT_MS } from './dar-archive-policy'; const ROOT = path.join(__dirname, '..'); const CANTON_PAYMENTS_DIR = path.join(ROOT, 'CantonPayments'); @@ -40,10 +41,13 @@ function main(): void { process.exit(1); } + assertDarArchiveSafe(darPath); const raw = execFileSync('dpm', ['damlc', 'inspect-dar', path.relative(CANTON_PAYMENTS_DIR, darPath), '--json'], { cwd: CANTON_PAYMENTS_DIR, encoding: 'utf8', + killSignal: 'SIGKILL', maxBuffer: 32 * 1024 * 1024, + timeout: DAML_INSPECT_TIMEOUT_MS, }); const j = JSON.parse(raw) as { main_package_id: string;