diff --git a/.github/workflows/changelog-merged-proof.yml b/.github/workflows/changelog-merged-proof.yml new file mode 100644 index 0000000000..888b17d7a9 --- /dev/null +++ b/.github/workflows/changelog-merged-proof.yml @@ -0,0 +1,75 @@ +name: Merged changelog proof + +on: + push: + branches: [pylon] + +permissions: {} + +jobs: + merged-changelog-proof: + name: Check changelog fragment + runs-on: ubuntu-24.04 + permissions: + actions: read + checks: read + contents: read + pull-requests: read + steps: + - name: Prove the merged pull request head check + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const repository = `${owner}/${repo}`; + const mergeSha = context.sha; + if ( + repository !== "pylon-code/prime-agent" || context.eventName !== "push" || + context.ref !== "refs/heads/pylon" || !/^[0-9a-f]{40}$/.test(mergeSha) + ) throw new Error("Merged changelog proof requires the canonical exact pylon push."); + const associated = await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, { + owner, repo, commit_sha: mergeSha, per_page: 100, + }); + const pullRequests = []; + for (const candidate of associated) { + const pull = (await github.rest.pulls.get({ owner, repo, pull_number: candidate.number })).data; + if ( + pull.merged_at && pull.merge_commit_sha === mergeSha && pull.base.ref === "pylon" && + pull.base.repo.full_name === repository + ) pullRequests.push(pull); + } + if (pullRequests.length !== 1) throw new Error("Merge SHA does not resolve to exactly one merged pylon pull request."); + const pull = pullRequests[0]; + if ( + pull.head.repo?.full_name !== repository || !/^[0-9a-f]{40}$/.test(pull.head.sha) || + pull.head.sha === mergeSha + ) throw new Error("Merged pull request head is not an exact canonical pre-merge SHA."); + const checks = await github.paginate(github.rest.checks.listForRef, { + owner, repo, ref: pull.head.sha, filter: "latest", per_page: 100, + }); + const candidates = checks.filter((check) => + check.name === "Check changelog fragment" && check.head_sha === pull.head.sha && + check.app?.id === 15368 && check.status === "completed" && check.conclusion === "success" + ); + let proved = false; + for (const check of candidates) { + const suite = (await github.rest.checks.getSuite({ owner, repo, check_suite_id: check.check_suite.id })).data; + const runId = /^https:\/\/github\.com\/pylon-code\/prime-agent\/actions\/runs\/([0-9]+)(?:\/job\/[0-9]+)?$/.exec(check.details_url ?? "")?.[1]; + if (!runId) continue; + const run = (await github.rest.actions.getWorkflowRun({ owner, repo, run_id: Number(runId) })).data; + const workflow = (await github.rest.actions.getWorkflow({ owner, repo, workflow_id: run.workflow_id })).data; + if ( + suite.app?.id === 15368 && suite.head_sha === pull.head.sha && suite.status === "completed" && + suite.conclusion === "success" && run.check_suite_id === suite.id && run.event === "pull_request" && + run.status === "completed" && run.conclusion === "success" && run.head_sha === pull.head.sha && + run.head_branch === pull.head.ref && run.head_repository?.id === 1349002285 && + run.head_repository?.full_name === repository && run.repository?.id === 1349002285 && + run.repository?.full_name === repository && workflow.path === ".github/workflows/changelog-fragment.yml" && + run.pull_requests?.some((candidate) => candidate.number === pull.number) + ) { + proved = true; + break; + } + } + if (!proved) throw new Error("No successful GitHub Actions PR-head changelog check has the exact required provenance."); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14440edac4..7a7f0ca187 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -304,7 +304,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, macos-15, windows-2025] + os: [ubuntu-24.04, macos-15] env: PYLON_RELEASE_NODE: 22.23.2 PYLON_RELEASE_NPM: 11.10.1 diff --git a/.github/workflows/pylon-preview-release.yml b/.github/workflows/pylon-preview-release.yml new file mode 100644 index 0000000000..00ac153061 --- /dev/null +++ b/.github/workflows/pylon-preview-release.yml @@ -0,0 +1,1183 @@ +name: Pylon preview publication + +on: + push: + branches: [pylon] + +permissions: {} + +concurrency: + group: pylon-preview-${{ github.sha }} + cancel-in-progress: false + +env: + PYLON_RELEASE_NODE: 22.23.2 + PYLON_RELEASE_NPM: 11.10.1 + +jobs: + admission: + name: Preview source admission + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Require the canonical protected push + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + if ( + owner !== "pylon-code" || + repo !== "prime-agent" || + context.eventName !== "push" || + context.ref !== "refs/heads/pylon" || + !/^[0-9a-f]{40}$/.test(context.sha) + ) { + core.setFailed("Preview publication requires an exact canonical pylon push."); + return; + } + const pylon = await github.rest.git.getRef({ owner, repo, ref: "heads/pylon" }); + if (pylon.data.object.type !== "commit" || pylon.data.object.sha !== context.sha) { + core.setFailed("Preview publication event is stale relative to protected pylon."); + return; + } + const expectedPolicy = [ + { context: "Check changelog fragment", appId: 15368, workflowPath: ".github/workflows/changelog-merged-proof.yml" }, + { context: "build-check-test", appId: 15368, workflowPath: ".github/workflows/ci.yml" }, + ]; + const protection = await github.graphql( + `query($owner:String!,$repo:String!,$ref:String!){repository(owner:$owner,name:$repo){ref(qualifiedName:$ref){branchProtectionRule{requiresStatusChecks requiredStatusChecks{context app{databaseId}}}}}}`, + { ...context.repo, ref: "refs/heads/pylon" }, + ); + const rule = protection.repository?.ref?.branchProtectionRule; + const actualPolicy = Array.isArray(rule?.requiredStatusChecks) ? rule.requiredStatusChecks.map((requirement) => ({ + context: requirement.context, appId: requirement.app?.databaseId ?? null, + })).sort((left, right) => left.context < right.context ? -1 : left.context > right.context ? 1 : 0) : null; + if (!rule?.requiresStatusChecks || JSON.stringify(actualPolicy) !== JSON.stringify(expectedPolicy.map(({ context, appId }) => ({ context, appId })))) { + throw new Error("Protected pylon must require exactly the two app-bound publication checks."); + } + // Exact-SHA workflow proof is checked by the final publisher after the push checks can complete. + + pack: + name: Preview offline pack (${{ matrix.copy }}) + needs: admission + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + copy: [a, b] + steps: + - name: Checkout exact pushed source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.sha }} + persist-credentials: false + fetch-depth: 1 + + - name: Setup pinned Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ env.PYLON_RELEASE_NODE }} + + - name: Install pinned build inputs + run: | + sudo apt-get update + sudo apt-get install -y libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev + npm install --global "npm@${PYLON_RELEASE_NPM}" + test "$(node --version)" = "v${PYLON_RELEASE_NODE}" + test "$(npm --version)" = "${PYLON_RELEASE_NPM}" + npm ci + + - name: Test publication contract + run: | + npm run test:pylon-release + npm run test:pylon-publication + + - name: Build and pack without network + run: | + sudo env \ + "PATH=$PATH" \ + "HOME=$HOME" \ + "GIT_CONFIG_COUNT=1" \ + "GIT_CONFIG_KEY_0=safe.directory" \ + "GIT_CONFIG_VALUE_0=$GITHUB_WORKSPACE" \ + unshare --net -- npm run release:pylon:pack + + - name: Verify and prepare six exact subjects + run: | + npm run release:pylon:verify + npm run release:pylon:preview -- --publication-policy-revision 1 + npm run release:pylon:verify-preview + + - name: Upload isolated preview subjects + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pylon-preview-pack-${{ matrix.copy }} + path: .npm/pylon-release/artifacts + include-hidden-files: true + if-no-files-found: error + retention-days: 3 + + reproducibility: + name: Preview byte reproducibility + needs: pack + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: read + contents: read + steps: + - name: Verify first workflow artifact provenance + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const run = (await github.rest.actions.getWorkflowRun({ ...context.repo, run_id: context.runId })).data; + const workflow = (await github.rest.actions.getWorkflow({ ...context.repo, workflow_id: run.workflow_id })).data; + if ( + context.repo.owner !== "pylon-code" || context.repo.repo !== "prime-agent" || + run.id !== context.runId || run.run_number !== Number(process.env.GITHUB_RUN_NUMBER) || + String(run.run_attempt) !== process.env.GITHUB_RUN_ATTEMPT || run.repository?.id !== 1349002285 || + run.repository?.full_name !== "pylon-code/prime-agent" || run.head_repository?.id !== 1349002285 || + run.head_repository?.full_name !== "pylon-code/prime-agent" || run.event !== context.eventName || + run.head_sha !== context.sha || run.head_branch !== "pylon" || context.ref !== "refs/heads/pylon" || + workflow.path !== ".github/workflows/pylon-preview-release.yml" + ) throw new Error("Artifact workflow provenance is not canonical."); + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + ...context.repo, run_id: context.runId, per_page: 100, + }); + const matches = artifacts.filter((artifact) => artifact.name === "pylon-preview-pack-a"); + if (matches.length !== 1 || matches[0].expired || !/^sha256:[0-9a-f]{64}$/.test(matches[0].digest ?? "")) { + throw new Error("Artifact is ambiguous, expired, or lacks a SHA-256 transport digest."); + } + + - name: Verify second workflow artifact provenance + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const run = (await github.rest.actions.getWorkflowRun({ ...context.repo, run_id: context.runId })).data; + const workflow = (await github.rest.actions.getWorkflow({ ...context.repo, workflow_id: run.workflow_id })).data; + if ( + context.repo.owner !== "pylon-code" || context.repo.repo !== "prime-agent" || + run.id !== context.runId || run.run_number !== Number(process.env.GITHUB_RUN_NUMBER) || + String(run.run_attempt) !== process.env.GITHUB_RUN_ATTEMPT || run.repository?.id !== 1349002285 || + run.repository?.full_name !== "pylon-code/prime-agent" || run.head_repository?.id !== 1349002285 || + run.head_repository?.full_name !== "pylon-code/prime-agent" || run.event !== context.eventName || + run.head_sha !== context.sha || run.head_branch !== "pylon" || context.ref !== "refs/heads/pylon" || + workflow.path !== ".github/workflows/pylon-preview-release.yml" + ) throw new Error("Artifact workflow provenance is not canonical."); + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + ...context.repo, run_id: context.runId, per_page: 100, + }); + const matches = artifacts.filter((artifact) => artifact.name === "pylon-preview-pack-b"); + if (matches.length !== 1 || matches[0].expired || !/^sha256:[0-9a-f]{64}$/.test(matches[0].digest ?? "")) { + throw new Error("Artifact is ambiguous, expired, or lacks a SHA-256 transport digest."); + } + + - name: Download first offline pack + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: pylon-preview-pack-a + path: pack-a + + - name: Download second offline pack + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: pylon-preview-pack-b + path: pack-b + + - name: Require byte-identical six-subject outputs + run: diff --recursive --brief pack-a pack-b + + install: + name: Preview installed artifact (${{ matrix.os }}) + needs: [pack, reproducibility] + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + permissions: + actions: read + contents: read + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + steps: + - name: Checkout exact pushed source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.sha }} + persist-credentials: false + fetch-depth: 1 + + - name: Setup pinned Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ env.PYLON_RELEASE_NODE }} + + - name: Install pinned npm + run: npm install --global "npm@${{ env.PYLON_RELEASE_NPM }}" + + - name: Verify workflow artifact provenance + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const run = (await github.rest.actions.getWorkflowRun({ ...context.repo, run_id: context.runId })).data; + const workflow = (await github.rest.actions.getWorkflow({ ...context.repo, workflow_id: run.workflow_id })).data; + if ( + context.repo.owner !== "pylon-code" || context.repo.repo !== "prime-agent" || + run.id !== context.runId || run.run_number !== Number(process.env.GITHUB_RUN_NUMBER) || + String(run.run_attempt) !== process.env.GITHUB_RUN_ATTEMPT || run.repository?.id !== 1349002285 || + run.repository?.full_name !== "pylon-code/prime-agent" || run.head_repository?.id !== 1349002285 || + run.head_repository?.full_name !== "pylon-code/prime-agent" || run.event !== context.eventName || + run.head_sha !== context.sha || run.head_branch !== "pylon" || context.ref !== "refs/heads/pylon" || + workflow.path !== ".github/workflows/pylon-preview-release.yml" + ) throw new Error("Artifact workflow provenance is not canonical."); + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + ...context.repo, run_id: context.runId, per_page: 100, + }); + const matches = artifacts.filter((artifact) => artifact.name === "pylon-preview-pack-a"); + if (matches.length !== 1 || matches[0].expired || !/^sha256:[0-9a-f]{64}$/.test(matches[0].digest ?? "")) { + throw new Error("Artifact is ambiguous, expired, or lacks a SHA-256 transport digest."); + } + + - name: Download byte-identical preview subjects + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: pylon-preview-pack-a + path: .npm/pylon-release/artifacts + + - name: Verify and install the exact preview bytes + run: | + npm run release:pylon:verify-preview + node -e "require('node:fs').unlinkSync('.npm/pylon-release/artifacts/pylon-preview-channel-v1.json')" + npm run release:pylon:smoke + + stage-draft: + name: Stage exact preview draft + needs: [admission, pack, reproducibility, install, verify-attestation] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + environment: pylon-preview + permissions: + actions: read + contents: write + outputs: + draft_id: ${{ steps.stage.outputs.result }} + steps: + - name: Download approved preview subjects + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: pylon-preview-pack-a + path: publication + + - name: Validate exact preview tag identity before protected mutation + id: preview-tag + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + ARTIFACT_DIR: publication + with: + script: | + const fs = require("node:fs"); + const path = require("node:path"); + const crypto = require("node:crypto"); + if (`${context.repo.owner}/${context.repo.repo}` !== "pylon-code/prime-agent" || context.eventName !== "push" || context.ref !== "refs/heads/pylon") { + throw new Error("Preview tag planning requires the canonical pylon push."); + } + const releaseBytes = fs.readFileSync(path.join(process.env.ARTIFACT_DIR, "pylon-prime-agent-release-v1.json")); + const previewBytes = fs.readFileSync(path.join(process.env.ARTIFACT_DIR, "pylon-preview-channel-v1.json")); + const release = JSON.parse(releaseBytes); + const preview = JSON.parse(previewBytes); + const tag = `pylon-build-g${context.sha.slice(0, 12)}-r${release.build?.recipeRevision}`; + if ( + release.source?.commit !== context.sha || release.source?.tree !== preview.build?.source?.tree || + release.build?.id !== tag || preview.build?.tag !== tag || + preview.build?.releaseManifest?.sha256 !== crypto.createHash("sha256").update(releaseBytes).digest("hex") || + preview.publicationPolicyRevision !== 1 || preview.sequenceEpoch !== 1 || + preview.sequence !== Number(process.env.GITHUB_RUN_NUMBER) || preview.workflowRunId !== process.env.GITHUB_RUN_ID + ) throw new Error("Preview tag plan is not bound to the exact source and workflow sequence."); + core.setOutput("tag", tag); + + - name: Mint repository-scoped ruleset auditor token + id: ruleset-auditor + uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 + with: + app-id: ${{ vars.PYLON_RULESET_AUDITOR_APP_ID }} + private-key: ${{ secrets.PYLON_RULESET_AUDITOR_PRIVATE_KEY }} + owner: pylon-code + repositories: prime-agent + permission-administration: read + + - name: Require live pylon immediately before the preview tag ruleset audit + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const pylon = await github.rest.git.getRef({ ...context.repo, ref: "heads/pylon" }); + if (pylon.data.object.type !== "commit" || pylon.data.object.sha !== context.sha) { + throw new Error("Preview tag transaction became stale before its authoritative ruleset audit."); + } + + - name: Require authoritative publication tag ruleset before preview tag CAS + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + github-token: ${{ steps.ruleset-auditor.outputs.token }} + script: | + const response = await github.request("GET /repos/{owner}/{repo}/rulesets/{ruleset_id}", { + owner: "pylon-code", repo: "prime-agent", ruleset_id: 21950766, includes_parents: false, + headers: { accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" }, + }); + const restRuleset = response?.data; + const exactKeys = (value, keys) => value !== null && typeof value === "object" && !Array.isArray(value) && + JSON.stringify(Object.keys(value).sort()) === JSON.stringify([...keys].sort()); + const exactSortedStrings = (value, expected) => Array.isArray(value) && value.every((entry) => typeof entry === "string") && + JSON.stringify([...value].sort()) === JSON.stringify([...expected].sort()); + const expectedIncludes = ["refs/tags/pylon-build-*", "refs/tags/pylon-stable-*"]; + const restConditions = restRuleset?.conditions; + const restRefName = restConditions?.ref_name; + const restRules = restRuleset?.rules; + const restUpdateRules = Array.isArray(restRules) ? restRules.filter((rule) => rule?.type === "update") : []; + const restDeletionRules = Array.isArray(restRules) ? restRules.filter((rule) => rule?.type === "deletion") : []; + if ( + response?.status !== 200 || !restRuleset || restRuleset.id !== 21950766 || + restRuleset.node_id !== "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4" || restRuleset.name !== "Pylon immutable publication tags" || + restRuleset.source_type !== "Repository" || restRuleset.source !== "pylon-code/prime-agent" || + restRuleset.target !== "tag" || restRuleset.enforcement !== "active" || + Object.hasOwn(restRuleset, "bypass_actors") && (!Array.isArray(restRuleset.bypass_actors) || restRuleset.bypass_actors.length !== 0) || + Object.hasOwn(restRuleset, "current_user_can_bypass") && restRuleset.current_user_can_bypass !== "never" || + !exactKeys(restConditions, ["ref_name"]) || !exactKeys(restRefName, ["exclude", "include"]) || + !exactSortedStrings(restRefName.exclude, []) || !exactSortedStrings(restRefName.include, expectedIncludes) || + !Array.isArray(restRules) || restRules.length !== 2 || restUpdateRules.length !== 1 || restDeletionRules.length !== 1 || + !exactKeys(restUpdateRules[0], ["parameters", "type"]) || + !exactKeys(restUpdateRules[0]?.parameters, ["update_allows_fetch_and_merge"]) || + restUpdateRules[0]?.parameters?.update_allows_fetch_and_merge !== false || !exactKeys(restDeletionRules[0], ["type"]) + ) throw new Error("REST ruleset-auditor response differs from the exact active immutable tag ruleset."); + const query = `query PylonPublicationRulesetAudit($owner: String!, $repo: String!, $rulesetDatabaseId: Int!) { + repository(owner: $owner, name: $repo) { + id + databaseId + nameWithOwner + ruleset(databaseId: $rulesetDatabaseId, includeParents: false) { + id + databaseId + name + enforcement + target + source { + __typename + ... on Repository { + id + databaseId + nameWithOwner + } + } + bypassActors { totalCount } + conditions { + refName { include exclude } + organizationProperty { __typename } + repositoryId { __typename } + repositoryName { __typename } + repositoryProperty { __typename } + } + rules(first: 100) { + totalCount + nodes { + type + parameters { + __typename + ... on UpdateParameters { updateAllowsFetchAndMerge } + } + } + } + } + } + }`; + const authoritative = await github.graphql(query, { + owner: "pylon-code", repo: "prime-agent", rulesetDatabaseId: 21950766, + }); + const repository = authoritative?.repository; + const ruleset = repository?.ruleset; + const source = ruleset?.source; + const bypassActors = ruleset?.bypassActors; + const conditions = ruleset?.conditions; + const refName = conditions?.refName; + const rules = ruleset?.rules; + const nodes = rules?.nodes; + const updateRules = Array.isArray(nodes) ? nodes.filter((rule) => rule?.type === "UPDATE") : []; + const deletionRules = Array.isArray(nodes) ? nodes.filter((rule) => rule?.type === "DELETION") : []; + if ( + Object.hasOwn(authoritative ?? {}, "errors") || !repository || repository.id !== "R_kgDOUGgkLQ" || + repository.databaseId !== 1349002285 || repository.nameWithOwner !== "pylon-code/prime-agent" || + !ruleset || ruleset.id !== restRuleset.node_id || ruleset.id !== "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4" || + ruleset.databaseId !== 21950766 || ruleset.name !== "Pylon immutable publication tags" || + ruleset.enforcement !== "ACTIVE" || ruleset.target !== "TAG" || + !exactKeys(source, ["__typename", "databaseId", "id", "nameWithOwner"]) || source.__typename !== "Repository" || + source.id !== repository.id || source.databaseId !== repository.databaseId || source.nameWithOwner !== repository.nameWithOwner || + !exactKeys(bypassActors, ["totalCount"]) || !Number.isInteger(bypassActors.totalCount) || bypassActors.totalCount !== 0 || + !exactKeys(conditions, ["organizationProperty", "refName", "repositoryId", "repositoryName", "repositoryProperty"]) || + conditions.organizationProperty !== null || conditions.repositoryId !== null || conditions.repositoryName !== null || + conditions.repositoryProperty !== null || !exactKeys(refName, ["exclude", "include"]) || + !exactSortedStrings(refName.exclude, []) || !exactSortedStrings(refName.include, expectedIncludes) || + !exactKeys(rules, ["nodes", "totalCount"]) || !Number.isInteger(rules.totalCount) || rules.totalCount !== 2 || + !Array.isArray(nodes) || nodes.length !== 2 || nodes.some((node) => node === null) || + updateRules.length !== 1 || deletionRules.length !== 1 || !exactKeys(updateRules[0], ["parameters", "type"]) || + !exactKeys(updateRules[0].parameters, ["__typename", "updateAllowsFetchAndMerge"]) || + updateRules[0].parameters.__typename !== "UpdateParameters" || updateRules[0].parameters.updateAllowsFetchAndMerge !== false || + !exactKeys(deletionRules[0], ["parameters", "type"]) || deletionRules[0].parameters !== null + ) throw new Error("GraphQL ruleset-auditor response is null, partial, redacted, or differs from the exact non-bypassable target contract."); + - name: Create or refetch the exact protected preview tag + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + TAG: ${{ steps.preview-tag.outputs.tag }} + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const tag = process.env.TAG; + if (`${owner}/${repo}` !== "pylon-code/prime-agent" || context.eventName !== "push" || context.ref !== "refs/heads/pylon" || + !/^pylon-build-g[0-9a-f]{12}-r[1-9][0-9]*$/.test(tag ?? "")) { + throw new Error("Protected preview tag CAS identity is malformed."); + } + try { + await github.rest.git.createRef({ owner, repo, ref: `refs/tags/${tag}`, sha: context.sha }); + } catch (error) { + if (error.status !== 422) throw error; + } + const ref = await github.rest.git.getRef({ owner, repo, ref: `tags/${tag}` }); + if (ref.data.object.type !== "commit" || ref.data.object.sha !== context.sha) { + throw new Error("Protected preview tag is annotated or targets a different commit."); + } + + - name: Create or finish the exact durable draft + id: stage + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + ARTIFACT_DIR: publication + with: + script: | + const fs = require("node:fs"); + const path = require("node:path"); + const crypto = require("node:crypto"); + const owner = context.repo.owner; + const repo = context.repo.repo; + if (`${owner}/${repo}` !== "pylon-code/prime-agent" || context.eventName !== "push" || context.ref !== "refs/heads/pylon") { + throw new Error("Preview draft staging requires the canonical pylon push."); + } + const requireLivePylon = async () => { + const pylon = await github.rest.git.getRef({ owner, repo, ref: "heads/pylon" }); + if (pylon.data.object.type !== "commit" || pylon.data.object.sha !== context.sha) throw new Error("Preview draft staging is stale."); + }; + await requireLivePylon(); + const dir = process.env.ARTIFACT_DIR; + const names = fs.readdirSync(dir).sort(); + if (names.length !== 6 || names.some((name) => !fs.lstatSync(path.join(dir, name)).isFile())) { + throw new Error("Preview draft requires exactly six regular files."); + } + const releaseBytes = fs.readFileSync(path.join(dir, "pylon-prime-agent-release-v1.json")); + const previewBytes = fs.readFileSync(path.join(dir, "pylon-preview-channel-v1.json")); + const release = JSON.parse(releaseBytes); + const preview = JSON.parse(previewBytes); + const sha256 = (bytes) => crypto.createHash("sha256").update(bytes).digest("hex"); + const tag = `pylon-build-g${context.sha.slice(0, 12)}-r${release.build?.recipeRevision}`; + if ( + release.source?.commit !== context.sha || release.source?.tree !== preview.build?.source?.tree || + release.build?.id !== tag || preview.build?.tag !== tag || preview.build?.releaseManifest?.sha256 !== sha256(releaseBytes) || + preview.publicationPolicyRevision !== 1 || preview.sequenceEpoch !== 1 || + preview.sequence !== Number(process.env.GITHUB_RUN_NUMBER) || preview.workflowRunId !== process.env.GITHUB_RUN_ID + ) throw new Error("Preview draft manifests do not bind the exact source and workflow sequence."); + const expectedNames = [...release.assets.map((asset) => asset.file), "pylon-prime-agent-release-v1.json", "pylon-preview-channel-v1.json"].sort(); + if (names.join("\n") !== expectedNames.join("\n")) throw new Error("Preview draft file set differs."); + const assets = names.map((name) => { + const bytes = fs.readFileSync(path.join(dir, name)); + return { name, bytes, size: bytes.length, sha256: sha256(bytes) }; + }); + for (const receipt of release.assets) { + const actual = assets.find((asset) => asset.name === receipt.file); + if (!actual || actual.size !== receipt.size || actual.sha256 !== receipt.sha256) throw new Error(`Preview draft asset differs: ${receipt.file}`); + } + const name = `Pylon Prime preview ${tag}`; + const body = [ + "Pylon Prime preview publication.", "", `Tag: ${tag}`, `Source: ${context.sha}`, + `Tree: ${release.source.tree}`, `Recipe: r${release.build.recipeRevision}`, "", + "Verify the immutable release and artifact attestations before use.", + ].join("\n"); + const requireExactTag = async () => { + const ref = await github.rest.git.getRef({ owner, repo, ref: `tags/${tag}` }); + if (ref.data.object.type !== "commit" || ref.data.object.sha !== context.sha) { + throw new Error("Preview publication tag is annotated or targets a different commit."); + } + return ref.data; + }; + await requireExactTag(); + const releases = await github.paginate(github.rest.repos.listReleases, { owner, repo, per_page: 100 }); + const matching = releases.filter((candidate) => candidate.tag_name === tag); + if (matching.length > 1) throw new Error("Preview draft tag is ambiguous."); + let draft = matching[0]; + if (draft && !draft.draft) { + if ( + draft.immutable !== true || draft.tag_name !== tag || draft.name !== name || draft.body !== body || + draft.prerelease !== true || draft.target_commitish !== context.sha || draft.assets.length !== assets.length + ) throw new Error("Existing preview publication differs from this exact rerun."); + for (const expected of assets) { + const actual = draft.assets.find((asset) => asset.name === expected.name); + if (!actual || actual.size !== expected.size || actual.digest !== `sha256:${expected.sha256}`) { + throw new Error(`Existing preview publication differs: ${expected.name}`); + } + } + const tagRef = await github.rest.git.getRef({ owner, repo, ref: `tags/${tag}` }); + if (tagRef.data.object.type !== "commit" || tagRef.data.object.sha !== context.sha) throw new Error("Existing preview tag differs."); + return draft.id; + } + let createdDraft = false; + if (!draft) { + try { + // Final live-tip read immediately precedes the first release mutation. + await requireLivePylon(); + draft = (await github.rest.repos.createRelease({ + owner, repo, tag_name: tag, target_commitish: context.sha, name, body, + draft: true, prerelease: true, make_latest: "false", + })).data; + createdDraft = true; + } catch (error) { + if (error.status !== 422) throw error; + const raced = (await github.paginate(github.rest.repos.listReleases, { owner, repo, per_page: 100 })) + .filter((candidate) => candidate.tag_name === tag); + if (raced.length !== 1 || !raced[0].draft) { + throw new Error("Preview draft creation raced (422) without one exact recoverable draft."); + } + draft = raced[0]; + } + } + if ( + draft.tag_name !== tag || draft.name !== name || draft.body !== body || draft.target_commitish !== context.sha || + draft.prerelease !== true || draft.immutable === true + ) throw new Error("Existing preview draft identity differs."); + for (const actual of draft.assets) { + const expected = assets.find((asset) => asset.name === actual.name); + if (!expected || actual.size !== expected.size || actual.digest !== `sha256:${expected.sha256}`) { + throw new Error(`Existing preview draft asset differs: ${actual.name}`); + } + } + const present = new Set(draft.assets.map((asset) => asset.name)); + const missing = assets.filter((asset) => !present.has(asset.name)); + if (!createdDraft && missing.length > 0) { + // A resumed draft gets a fresh point-in-time authorization before its first mutation. + await requireLivePylon(); + } + for (const asset of missing) { + await github.request("POST /repos/{owner}/{repo}/releases/{release_id}/assets", { + owner, repo, release_id: draft.id, name: asset.name, data: asset.bytes, + headers: { "content-type": "application/octet-stream", "content-length": asset.size }, + }); + } + const staged = (await github.rest.repos.getRelease({ owner, repo, release_id: draft.id })).data; + if (staged.assets.length !== assets.length) throw new Error("Preview draft was not fully staged."); + for (const expected of assets) { + const actual = staged.assets.find((asset) => asset.name === expected.name); + if (!actual || actual.size !== expected.size || actual.digest !== `sha256:${expected.sha256}`) throw new Error(`Staged preview asset differs: ${expected.name}`); + } + return draft.id; + + attest: + name: Approve and attest six preview subjects + needs: [pack, reproducibility, install] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + environment: pylon-preview + permissions: + actions: read + id-token: write + attestations: write + steps: + - name: Download byte-identical preview subjects + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: pylon-preview-pack-a + path: .npm/pylon-release/artifacts + + - name: Validate exact subjects before signing + shell: bash + run: | + node <<'NODE' + const crypto = require("node:crypto"); + const fs = require("node:fs"); + const path = require("node:path"); + const dir = ".npm/pylon-release/artifacts"; + const canonical = (value) => { + if (value === null || ["string", "boolean"].includes(typeof value)) return value; + if (typeof value === "number" && Number.isFinite(value)) return value; + if (Array.isArray(value)) return value.map(canonical); + if (!value || typeof value !== "object" || Object.getPrototypeOf(value) !== Object.prototype) throw new Error("Unsupported manifest value."); + return Object.fromEntries(Object.keys(value).sort().map((key) => { + if (value[key] === undefined) throw new Error("Undefined manifest value."); + return [key, canonical(value[key])]; + })); + }; + const canonicalJson = (value) => `${JSON.stringify(canonical(value), null, 2)}\n`; + const sha256 = (bytes) => crypto.createHash("sha256").update(bytes).digest("hex"); + const releaseName = "pylon-prime-agent-release-v1.json"; + const previewName = "pylon-preview-channel-v1.json"; + const releaseBytes = fs.readFileSync(path.join(dir, releaseName)); + const previewBytes = fs.readFileSync(path.join(dir, previewName)); + const release = JSON.parse(releaseBytes); + const preview = JSON.parse(previewBytes); + if (previewBytes.toString("utf8") !== canonicalJson(preview)) throw new Error("Preview manifest is not canonical JSON."); + if ( + preview.sequenceEpoch !== 1 || preview.sequence !== Number(process.env.GITHUB_RUN_NUMBER) || + preview.workflowRunId !== process.env.GITHUB_RUN_ID + ) throw new Error("Preview manifest workflow sequence differs from this approved attestation run."); + if (preview.build?.releaseManifest?.file !== releaseName || preview.build.releaseManifest.sha256 !== sha256(releaseBytes)) { + throw new Error("Preview does not bind the exact build manifest."); + } + if (!Array.isArray(release.assets) || release.assets.length !== 4 || !Array.isArray(preview.assets) || preview.assets.length !== 4) { + throw new Error("Preview must describe exactly four archives."); + } + const expected = new Map([[releaseName, { size: releaseBytes.length, sha256: sha256(releaseBytes) }], [previewName, { size: previewBytes.length, sha256: sha256(previewBytes) }]]); + for (const asset of release.assets) { + if ( + !asset || Object.keys(asset).sort().join(",") !== "file,package,sha256,sha512,size" || + path.basename(asset.file) !== asset.file || !/^pylon-prime-agent(?:-(?:ai|core|tui))?-\d+\.\d+\.\d+\.tgz$/.test(asset.file) || + !Number.isSafeInteger(asset.size) || asset.size < 1 || !/^[0-9a-f]{64}$/.test(asset.sha256) + ) throw new Error("Build manifest contains an unsafe archive receipt."); + expected.set(asset.file, { size: asset.size, sha256: asset.sha256 }); + } + if (expected.size !== 6) throw new Error("Subject names are not unique."); + const previewAssets = new Map(preview.assets.map((asset) => [asset.file, asset])); + for (const asset of release.assets) { + const channel = previewAssets.get(asset.file); + if (!channel || channel.size !== asset.size || channel.sha256 !== asset.sha256 || channel.sha512 !== asset.sha512) { + throw new Error(`Preview receipt differs for ${asset.file}.`); + } + } + const names = fs.readdirSync(dir).sort(); + if (names.length !== 6 || names.join("\n") !== [...expected.keys()].sort().join("\n")) throw new Error("Attestation subject set has an extra or missing file."); + for (const name of names) { + const file = path.join(dir, name); + const stat = fs.lstatSync(file); + if (!stat.isFile()) throw new Error(`Attestation subject is not a regular file: ${name}`); + const bytes = fs.readFileSync(file); + const receipt = expected.get(name); + if (bytes.length !== receipt.size || sha256(bytes) !== receipt.sha256) throw new Error(`Attestation subject bytes differ: ${name}`); + } + NODE + + - name: Generate build provenance for exactly six subjects + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: .npm/pylon-release/artifacts/* + + verify-attestation: + name: Verify preview provenance + needs: [admission, pack, reproducibility, install, attest] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: read + contents: read + steps: + - name: Checkout protected verification policy + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.sha }} + persist-credentials: false + fetch-depth: 1 + + - name: Verify workflow artifact provenance + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const run = (await github.rest.actions.getWorkflowRun({ ...context.repo, run_id: context.runId })).data; + const workflow = (await github.rest.actions.getWorkflow({ ...context.repo, workflow_id: run.workflow_id })).data; + if ( + context.repo.owner !== "pylon-code" || context.repo.repo !== "prime-agent" || + run.id !== context.runId || run.run_number !== Number(process.env.GITHUB_RUN_NUMBER) || + String(run.run_attempt) !== process.env.GITHUB_RUN_ATTEMPT || run.repository?.id !== 1349002285 || + run.repository?.full_name !== "pylon-code/prime-agent" || run.head_repository?.id !== 1349002285 || + run.head_repository?.full_name !== "pylon-code/prime-agent" || run.event !== context.eventName || + run.head_sha !== context.sha || run.head_branch !== "pylon" || context.ref !== "refs/heads/pylon" || + workflow.path !== ".github/workflows/pylon-preview-release.yml" + ) throw new Error("Artifact workflow provenance is not canonical."); + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + ...context.repo, run_id: context.runId, per_page: 100, + }); + const matches = artifacts.filter((artifact) => artifact.name === "pylon-preview-pack-a"); + if (matches.length !== 1 || matches[0].expired || !/^sha256:[0-9a-f]{64}$/.test(matches[0].digest ?? "")) { + throw new Error("Artifact is ambiguous, expired, or lacks a SHA-256 transport digest."); + } + + - name: Download six attested subjects + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: pylon-preview-pack-a + path: publication + + - name: Verify exact signer, subjects, source, and Rekor evidence + env: + GH_TOKEN: ${{ github.token }} + run: | + source_tree="$(git rev-parse 'HEAD^{tree}')" + npm run release:pylon:verify-attestations -- \ + --artifact-dir publication \ + --source-sha "${{ github.sha }}" \ + --source-tree "$source_tree" + + publish: + name: Publish immutable preview + needs: [stage-draft, verify-attestation] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + environment: pylon-preview + permissions: + actions: read + checks: read + contents: write + steps: + - name: Verify workflow artifact provenance + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const run = (await github.rest.actions.getWorkflowRun({ ...context.repo, run_id: context.runId })).data; + const workflow = (await github.rest.actions.getWorkflow({ ...context.repo, workflow_id: run.workflow_id })).data; + if ( + context.repo.owner !== "pylon-code" || context.repo.repo !== "prime-agent" || + run.id !== context.runId || run.run_number !== Number(process.env.GITHUB_RUN_NUMBER) || + String(run.run_attempt) !== process.env.GITHUB_RUN_ATTEMPT || run.repository?.id !== 1349002285 || + run.repository?.full_name !== "pylon-code/prime-agent" || run.head_repository?.id !== 1349002285 || + run.head_repository?.full_name !== "pylon-code/prime-agent" || run.event !== context.eventName || + run.head_sha !== context.sha || run.head_branch !== "pylon" || context.ref !== "refs/heads/pylon" || + workflow.path !== ".github/workflows/pylon-preview-release.yml" + ) throw new Error("Artifact workflow provenance is not canonical."); + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + ...context.repo, run_id: context.runId, per_page: 100, + }); + const matches = artifacts.filter((artifact) => artifact.name === "pylon-preview-pack-a"); + if (matches.length !== 1 || matches[0].expired || !/^sha256:[0-9a-f]{64}$/.test(matches[0].digest ?? "")) { + throw new Error("Artifact is ambiguous, expired, or lacks a SHA-256 transport digest."); + } + + - name: Download attested preview subjects + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: pylon-preview-pack-a + path: publication + + - name: Verify exact checks and freeze the approved preview draft + id: finalize + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + ARTIFACT_DIR: publication + DRAFT_ID: ${{ needs.stage-draft.outputs.draft_id }} + with: + script: | + const fs = require("node:fs"); + const path = require("node:path"); + const crypto = require("node:crypto"); + const owner = context.repo.owner; + const repo = context.repo.repo; + const repository = `${owner}/${repo}`; + const refName = "heads/pylon"; + const sourceSha = context.sha; + if ( + repository !== "pylon-code/prime-agent" || + context.eventName !== "push" || + context.ref !== "refs/heads/pylon" || + !/^[0-9a-f]{40}$/.test(sourceSha) + ) { + throw new Error("Preview publisher requires the canonical exact pylon push."); + } + const requireLivePylon = async () => { + const livePylon = await github.rest.git.getRef({ owner, repo, ref: refName }); + if (livePylon.data.object.type !== "commit" || livePylon.data.object.sha !== sourceSha) { + throw new Error("Preview publication became stale while verification ran."); + } + }; + await requireLivePylon(); + const releaseBytes = fs.readFileSync(path.join(process.env.ARTIFACT_DIR, "pylon-prime-agent-release-v1.json")); + const previewBytes = fs.readFileSync(path.join(process.env.ARTIFACT_DIR, "pylon-preview-channel-v1.json")); + const releaseManifest = JSON.parse(releaseBytes); + const previewManifest = JSON.parse(previewBytes); + const sha256 = (bytes) => crypto.createHash("sha256").update(bytes).digest("hex"); + const expectedTag = `pylon-build-g${sourceSha.slice(0, 12)}-r${releaseManifest.build.recipeRevision}`; + if ( + releaseManifest.source.repository !== "https://github.com/pylon-code/prime-agent" || + releaseManifest.source.commit !== sourceSha || + releaseManifest.build.id !== expectedTag || + previewManifest.channel !== "preview" || + previewManifest.repository !== "https://github.com/pylon-code/prime-agent" || + previewManifest.build.tag !== expectedTag || + previewManifest.build.source.commit !== sourceSha || + previewManifest.build.source.tree !== releaseManifest.source.tree || + previewManifest.build.releaseManifest.sha256 !== sha256(releaseBytes) || + previewManifest.publicationPolicyRevision !== 1 || previewManifest.sequenceEpoch !== 1 || + previewManifest.sequence !== Number(process.env.GITHUB_RUN_NUMBER) || previewManifest.workflowRunId !== process.env.GITHUB_RUN_ID + ) { + throw new Error("Downloaded preview metadata is not bound to this exact push and workflow sequence."); + } + const commit = await github.rest.git.getCommit({ owner, repo, commit_sha: sourceSha }); + if (commit.data.tree.sha !== releaseManifest.source.tree) throw new Error("GitHub source tree differs from the build manifest."); + const protection = await github.graphql( + `query($owner:String!,$repo:String!,$ref:String!){repository(owner:$owner,name:$repo){ref(qualifiedName:$ref){branchProtectionRule{requiresStatusChecks requiredStatusChecks{context app{databaseId}}}}}}`, + { owner, repo, ref: "refs/heads/pylon" }, + ); + const rule = protection.repository?.ref?.branchProtectionRule; + const required = rule?.requiredStatusChecks; + const expectedPolicy = [ + { context: "Check changelog fragment", appId: 15368, workflowPath: ".github/workflows/changelog-merged-proof.yml" }, + { context: "build-check-test", appId: 15368, workflowPath: ".github/workflows/ci.yml" }, + ]; + const actualPolicy = Array.isArray(required) ? required.map((requirement) => ({ + context: requirement.context, appId: requirement.app?.databaseId ?? null, + })).sort((left, right) => left.context < right.context ? -1 : left.context > right.context ? 1 : 0) : null; + if (!rule?.requiresStatusChecks || JSON.stringify(actualPolicy) !== JSON.stringify(expectedPolicy.map(({ context, appId }) => ({ context, appId })))) { + throw new Error("Protected pylon must require exactly the two app-bound publication checks."); + } + const checks = await github.paginate(github.rest.checks.listForRef, { + owner, repo, ref: sourceSha, filter: "latest", per_page: 100, + }); + for (const requirement of expectedPolicy) { + const candidates = checks.filter((check) => + check.name === requirement.context && check.head_sha === sourceSha && check.app?.id === requirement.appId && + check.status === "completed" && check.conclusion === "success" + ); + let proved = false; + for (const check of candidates) { + const runId = /^https:\/\/github\.com\/pylon-code\/prime-agent\/actions\/runs\/([0-9]+)(?:\/job\/[0-9]+)?$/.exec(check.details_url ?? "")?.[1]; + if (!runId) continue; + const suite = (await github.rest.checks.getSuite({ owner, repo, check_suite_id: check.check_suite.id })).data; + const run = (await github.rest.actions.getWorkflowRun({ owner, repo, run_id: Number(runId) })).data; + const workflow = (await github.rest.actions.getWorkflow({ owner, repo, workflow_id: run.workflow_id })).data; + if ( + suite.app?.id === requirement.appId && suite.head_sha === sourceSha && suite.status === "completed" && suite.conclusion === "success" && + run.check_suite_id === suite.id && run.repository?.id === 1349002285 && run.repository?.full_name === repository && + run.head_repository?.id === 1349002285 && run.head_repository?.full_name === repository && run.event === "push" && + run.head_branch === "pylon" && run.head_sha === sourceSha && run.status === "completed" && run.conclusion === "success" && + workflow.path === requirement.workflowPath + ) { proved = true; break; } + } + if (!proved) throw new Error(`Required check ${requirement.context} lacks an exact canonical push workflow proof.`); + } + const files = fs.readdirSync(process.env.ARTIFACT_DIR).sort(); + const expectedFiles = [ + ...releaseManifest.assets.map((asset) => asset.file), + "pylon-preview-channel-v1.json", + "pylon-prime-agent-release-v1.json", + ].sort(); + if (JSON.stringify(files) !== JSON.stringify(expectedFiles) || files.length !== 6) { + throw new Error("Preview publisher received a subject set other than the exact six files."); + } + const assets = files.map((name) => { + const bytes = fs.readFileSync(path.join(process.env.ARTIFACT_DIR, name)); + return { name, bytes, size: bytes.byteLength, sha256: sha256(bytes) }; + }); + for (const asset of releaseManifest.assets) { + const actual = assets.find((candidate) => candidate.name === asset.file); + if (!actual || actual.size !== asset.size || actual.sha256 !== asset.sha256) { + throw new Error(`Preview subject differs from build manifest: ${asset.file}`); + } + } + const tag = expectedTag; + core.setOutput("tag", tag); + const name = `Pylon Prime preview ${tag}`; + const body = [ + "Pylon Prime preview publication.", "", `Tag: ${tag}`, `Source: ${sourceSha}`, + `Tree: ${releaseManifest.source.tree}`, `Recipe: r${releaseManifest.build.recipeRevision}`, "", + "Verify the immutable release and artifact attestations before use.", + ].join("\n"); + const requireExactTag = async () => { + const tagRef = await github.rest.git.getRef({ owner, repo, ref: `tags/${tag}` }); + if (tagRef.data.object.type !== "commit" || tagRef.data.object.sha !== sourceSha) { + throw new Error("Preview tag does not target the exact source commit."); + } + }; + const assertExact = async (release) => { + if ( + release.immutable !== true || release.draft !== false || release.tag_name !== tag || release.name !== name || + release.body !== body || release.prerelease !== true || release.target_commitish !== sourceSha || + release.assets.length !== assets.length + ) throw new Error("Existing preview release is mutable or has different metadata."); + for (const expected of assets) { + const actual = release.assets.find((candidate) => candidate.name === expected.name); + if (!actual || actual.size !== expected.size || actual.digest !== `sha256:${expected.sha256}`) { + throw new Error(`Existing preview asset differs: ${expected.name}`); + } + } + await requireExactTag(); + }; + let existing; + try { + existing = (await github.rest.repos.getReleaseByTag({ owner, repo, tag })).data; + } catch (error) { + if (error.status !== 404) throw error; + } + if (!existing) { + const matching = (await github.paginate(github.rest.repos.listReleases, { owner, repo, per_page: 100 })) + .filter((release) => release.tag_name === tag); + if (matching.length > 1) throw new Error("Preview tag resolves to multiple releases."); + existing = matching[0]; + } + if (existing && !existing.draft) { + if (String(existing.id) !== process.env.DRAFT_ID) throw new Error("Idempotent preview release id differs from approved staging."); + await assertExact(existing); + core.info(`Immutable preview ${tag} already contains identical bytes and metadata.`); + core.setOutput("release_id", ""); + return; + } + let draft = existing; + if (draft) { + if ( + draft.immutable === true || draft.tag_name !== tag || draft.name !== name || draft.body !== body || + draft.prerelease !== true || draft.target_commitish !== sourceSha + ) throw new Error("Partial preview draft identity differs; refusing to edit it."); + for (const actual of draft.assets) { + const expected = assets.find((candidate) => candidate.name === actual.name); + if (!expected || actual.size !== expected.size || actual.digest !== `sha256:${expected.sha256}`) { + throw new Error(`Partial preview draft contains a changed asset: ${actual.name}`); + } + } + } else { + throw new Error("Approved preview draft is missing; publisher will not recreate it after attestation."); + } + if (String(draft.id) !== process.env.DRAFT_ID) throw new Error("Approved preview draft id changed after attestation."); + if (draft.assets.length !== assets.length) throw new Error("Approved preview draft is not fully staged."); + for (const expected of assets) { + const actual = draft.assets.find((asset) => asset.name === expected.name); + if (!actual || actual.size !== expected.size || actual.digest !== `sha256:${expected.sha256}`) { + throw new Error(`Approved preview draft asset differs: ${expected.name}`); + } + } + core.setOutput("release_id", String(draft.id)); + + - name: Mint repository-scoped ruleset auditor token + id: ruleset-auditor + if: steps.finalize.outputs.release_id != '' + uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 + with: + app-id: ${{ vars.PYLON_RULESET_AUDITOR_APP_ID }} + private-key: ${{ secrets.PYLON_RULESET_AUDITOR_PRIVATE_KEY }} + owner: pylon-code + repositories: prime-agent + permission-administration: read + + - name: Require live pylon, exact tag, and exact draft before final ruleset audit + if: steps.finalize.outputs.release_id != '' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + ARTIFACT_DIR: publication + DRAFT_ID: ${{ steps.finalize.outputs.release_id }} + TAG: ${{ steps.finalize.outputs.tag }} + with: + script: | + const fs = require("node:fs"); + const path = require("node:path"); + const crypto = require("node:crypto"); + const pylon = await github.rest.git.getRef({ ...context.repo, ref: "heads/pylon" }); + if (pylon.data.object.type !== "commit" || pylon.data.object.sha !== context.sha) { + throw new Error("Preview publication became stale before its final authoritative ruleset audit."); + } + const tag = await github.rest.git.getRef({ ...context.repo, ref: `tags/${process.env.TAG}` }); + if (tag.data.object.type !== "commit" || tag.data.object.sha !== context.sha) { + throw new Error("Preview publication tag changed before its final authoritative ruleset audit."); + } + const release = JSON.parse(fs.readFileSync(path.join(process.env.ARTIFACT_DIR, "pylon-prime-agent-release-v1.json"))); + const body = [ + "Pylon Prime preview publication.", "", `Tag: ${process.env.TAG}`, `Source: ${context.sha}`, + `Tree: ${release.source?.tree}`, `Recipe: r${release.build?.recipeRevision}`, "", + "Verify the immutable release and artifact attestations before use.", + ].join("\n"); + const expected = fs.readdirSync(process.env.ARTIFACT_DIR).map((name) => { + const bytes = fs.readFileSync(path.join(process.env.ARTIFACT_DIR, name)); + return { name, size: bytes.length, digest: `sha256:${crypto.createHash("sha256").update(bytes).digest("hex")}` }; + }); + const draft = (await github.rest.repos.getRelease({ ...context.repo, release_id: Number(process.env.DRAFT_ID) })).data; + if ( + !draft.draft || draft.immutable === true || draft.tag_name !== process.env.TAG || + draft.name !== `Pylon Prime preview ${process.env.TAG}` || draft.body !== body || draft.prerelease !== true || + draft.target_commitish !== context.sha || draft.assets?.length !== expected.length + ) throw new Error("Preview draft changed before final authoritative ruleset audit."); + for (const asset of expected) { + const actual = draft.assets.find((candidate) => candidate.name === asset.name); + if (!actual || actual.size !== asset.size || actual.digest !== asset.digest) { + throw new Error(`Preview draft asset changed before final ruleset audit: ${asset.name}`); + } + } + + - name: Require authoritative publication tag ruleset before immutable preview publish + if: steps.finalize.outputs.release_id != '' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + github-token: ${{ steps.ruleset-auditor.outputs.token }} + script: | + const response = await github.request("GET /repos/{owner}/{repo}/rulesets/{ruleset_id}", { + owner: "pylon-code", repo: "prime-agent", ruleset_id: 21950766, includes_parents: false, + headers: { accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" }, + }); + const restRuleset = response?.data; + const exactKeys = (value, keys) => value !== null && typeof value === "object" && !Array.isArray(value) && + JSON.stringify(Object.keys(value).sort()) === JSON.stringify([...keys].sort()); + const exactSortedStrings = (value, expected) => Array.isArray(value) && value.every((entry) => typeof entry === "string") && + JSON.stringify([...value].sort()) === JSON.stringify([...expected].sort()); + const expectedIncludes = ["refs/tags/pylon-build-*", "refs/tags/pylon-stable-*"]; + const restConditions = restRuleset?.conditions; + const restRefName = restConditions?.ref_name; + const restRules = restRuleset?.rules; + const restUpdateRules = Array.isArray(restRules) ? restRules.filter((rule) => rule?.type === "update") : []; + const restDeletionRules = Array.isArray(restRules) ? restRules.filter((rule) => rule?.type === "deletion") : []; + if ( + response?.status !== 200 || !restRuleset || restRuleset.id !== 21950766 || + restRuleset.node_id !== "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4" || restRuleset.name !== "Pylon immutable publication tags" || + restRuleset.source_type !== "Repository" || restRuleset.source !== "pylon-code/prime-agent" || + restRuleset.target !== "tag" || restRuleset.enforcement !== "active" || + Object.hasOwn(restRuleset, "bypass_actors") && (!Array.isArray(restRuleset.bypass_actors) || restRuleset.bypass_actors.length !== 0) || + Object.hasOwn(restRuleset, "current_user_can_bypass") && restRuleset.current_user_can_bypass !== "never" || + !exactKeys(restConditions, ["ref_name"]) || !exactKeys(restRefName, ["exclude", "include"]) || + !exactSortedStrings(restRefName.exclude, []) || !exactSortedStrings(restRefName.include, expectedIncludes) || + !Array.isArray(restRules) || restRules.length !== 2 || restUpdateRules.length !== 1 || restDeletionRules.length !== 1 || + !exactKeys(restUpdateRules[0], ["parameters", "type"]) || + !exactKeys(restUpdateRules[0]?.parameters, ["update_allows_fetch_and_merge"]) || + restUpdateRules[0]?.parameters?.update_allows_fetch_and_merge !== false || !exactKeys(restDeletionRules[0], ["type"]) + ) throw new Error("REST ruleset-auditor response differs from the exact active immutable tag ruleset."); + const query = `query PylonPublicationRulesetAudit($owner: String!, $repo: String!, $rulesetDatabaseId: Int!) { + repository(owner: $owner, name: $repo) { + id + databaseId + nameWithOwner + ruleset(databaseId: $rulesetDatabaseId, includeParents: false) { + id + databaseId + name + enforcement + target + source { + __typename + ... on Repository { + id + databaseId + nameWithOwner + } + } + bypassActors { totalCount } + conditions { + refName { include exclude } + organizationProperty { __typename } + repositoryId { __typename } + repositoryName { __typename } + repositoryProperty { __typename } + } + rules(first: 100) { + totalCount + nodes { + type + parameters { + __typename + ... on UpdateParameters { updateAllowsFetchAndMerge } + } + } + } + } + } + }`; + const authoritative = await github.graphql(query, { + owner: "pylon-code", repo: "prime-agent", rulesetDatabaseId: 21950766, + }); + const repository = authoritative?.repository; + const ruleset = repository?.ruleset; + const source = ruleset?.source; + const bypassActors = ruleset?.bypassActors; + const conditions = ruleset?.conditions; + const refName = conditions?.refName; + const rules = ruleset?.rules; + const nodes = rules?.nodes; + const updateRules = Array.isArray(nodes) ? nodes.filter((rule) => rule?.type === "UPDATE") : []; + const deletionRules = Array.isArray(nodes) ? nodes.filter((rule) => rule?.type === "DELETION") : []; + if ( + Object.hasOwn(authoritative ?? {}, "errors") || !repository || repository.id !== "R_kgDOUGgkLQ" || + repository.databaseId !== 1349002285 || repository.nameWithOwner !== "pylon-code/prime-agent" || + !ruleset || ruleset.id !== restRuleset.node_id || ruleset.id !== "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4" || + ruleset.databaseId !== 21950766 || ruleset.name !== "Pylon immutable publication tags" || + ruleset.enforcement !== "ACTIVE" || ruleset.target !== "TAG" || + !exactKeys(source, ["__typename", "databaseId", "id", "nameWithOwner"]) || source.__typename !== "Repository" || + source.id !== repository.id || source.databaseId !== repository.databaseId || source.nameWithOwner !== repository.nameWithOwner || + !exactKeys(bypassActors, ["totalCount"]) || !Number.isInteger(bypassActors.totalCount) || bypassActors.totalCount !== 0 || + !exactKeys(conditions, ["organizationProperty", "refName", "repositoryId", "repositoryName", "repositoryProperty"]) || + conditions.organizationProperty !== null || conditions.repositoryId !== null || conditions.repositoryName !== null || + conditions.repositoryProperty !== null || !exactKeys(refName, ["exclude", "include"]) || + !exactSortedStrings(refName.exclude, []) || !exactSortedStrings(refName.include, expectedIncludes) || + !exactKeys(rules, ["nodes", "totalCount"]) || !Number.isInteger(rules.totalCount) || rules.totalCount !== 2 || + !Array.isArray(nodes) || nodes.length !== 2 || nodes.some((node) => node === null) || + updateRules.length !== 1 || deletionRules.length !== 1 || !exactKeys(updateRules[0], ["parameters", "type"]) || + !exactKeys(updateRules[0].parameters, ["__typename", "updateAllowsFetchAndMerge"]) || + updateRules[0].parameters.__typename !== "UpdateParameters" || updateRules[0].parameters.updateAllowsFetchAndMerge !== false || + !exactKeys(deletionRules[0], ["parameters", "type"]) || deletionRules[0].parameters !== null + ) throw new Error("GraphQL ruleset-auditor response is null, partial, redacted, or differs from the exact non-bypassable target contract."); + - name: Publish the exact approved preview draft + if: steps.finalize.outputs.release_id != '' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + ARTIFACT_DIR: publication + DRAFT_ID: ${{ steps.finalize.outputs.release_id }} + with: + script: | + const fs = require("node:fs"); + const path = require("node:path"); + const crypto = require("node:crypto"); + const owner = context.repo.owner; + const repo = context.repo.repo; + const releaseBytes = fs.readFileSync(path.join(process.env.ARTIFACT_DIR, "pylon-prime-agent-release-v1.json")); + const release = JSON.parse(releaseBytes); + const draftId = Number(process.env.DRAFT_ID); + const tag = `pylon-build-g${context.sha.slice(0, 12)}-r${release.build?.recipeRevision}`; + const name = `Pylon Prime preview ${tag}`; + const body = [ + "Pylon Prime preview publication.", "", `Tag: ${tag}`, `Source: ${context.sha}`, + `Tree: ${release.source?.tree}`, `Recipe: r${release.build?.recipeRevision}`, "", + "Verify the immutable release and artifact attestations before use.", + ].join("\n"); + const sha256 = (bytes) => crypto.createHash("sha256").update(bytes).digest("hex"); + const assets = fs.readdirSync(process.env.ARTIFACT_DIR).sort().map((assetName) => { + const bytes = fs.readFileSync(path.join(process.env.ARTIFACT_DIR, assetName)); + return { name: assetName, size: bytes.length, sha256: sha256(bytes) }; + }); + if ( + `${owner}/${repo}` !== "pylon-code/prime-agent" || context.eventName !== "push" || context.ref !== "refs/heads/pylon" || + !Number.isSafeInteger(draftId) || draftId < 1 || release.source?.commit !== context.sha || release.build?.id !== tag || assets.length !== 6 + ) throw new Error("Approved preview publication identity is malformed."); + await github.rest.repos.updateRelease({ owner, repo, release_id: draftId, draft: false }); + const published = (await github.rest.repos.getRelease({ owner, repo, release_id: draftId })).data; + if ( + published.immutable !== true || published.draft !== false || published.tag_name !== tag || published.name !== name || + published.body !== body || published.prerelease !== true || published.target_commitish !== context.sha || + published.assets.length !== assets.length + ) throw new Error("Published preview release differs from the exact approved draft."); + for (const expected of assets) { + const actual = published.assets.find((asset) => asset.name === expected.name); + if (!actual || actual.size !== expected.size || actual.digest !== `sha256:${expected.sha256}`) { + throw new Error(`Published preview asset differs: ${expected.name}`); + } + } + const tagRef = await github.rest.git.getRef({ owner, repo, ref: `tags/${tag}` }); + if (tagRef.data.object.type !== "commit" || tagRef.data.object.sha !== context.sha) { + throw new Error("Published preview tag differs from the exact source commit."); + } + + - name: Verify GitHub immutable-release attestation + env: + GH_TOKEN: ${{ github.token }} + run: | + tag="$(node -e "console.log(JSON.parse(require('node:fs').readFileSync('publication/pylon-preview-channel-v1.json')).build.tag)")" + verified=false + for attempt in 1 2 3 4 5 6; do + if gh release verify "$tag" --repo pylon-code/prime-agent; then + verified=true + break + fi + if [ "$attempt" = 6 ]; then + exit 1 + fi + sleep 10 + done + test "$verified" = true + for asset in publication/*; do + gh release verify-asset "$tag" "$asset" --repo pylon-code/prime-agent + done diff --git a/.github/workflows/pylon-stable-release.yml b/.github/workflows/pylon-stable-release.yml new file mode 100644 index 0000000000..07385129da --- /dev/null +++ b/.github/workflows/pylon-stable-release.yml @@ -0,0 +1,1553 @@ +name: Pylon stable promotion + +on: + workflow_dispatch: + inputs: + preview_tag: + description: Existing immutable preview tag to promote + required: true + type: string + resume_identity: + description: Reservation tag, stable draft tag, or numeric draft release id for explicit recovery + required: false + type: string + operation: + description: Promote, or withdraw one prior stable sequence while promoting this build + required: true + default: promote + type: choice + options: [promote, withdraw, resume-promote, resume-withdraw] + revoke_stable_tag: + description: Existing stable tag to append to the revocation list for withdrawal + required: false + type: string + reason: + description: Deterministic lowercase withdrawal reason code + required: false + default: withdrawn + type: string + +permissions: {} + +concurrency: + group: pylon-stable-publication + cancel-in-progress: false + +env: + PYLON_RELEASE_NODE: 22.23.2 + PYLON_RELEASE_NPM: 11.10.1 + +jobs: + admission: + name: Stable source admission + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: read + checks: read + contents: read + outputs: + source_sha: ${{ steps.admit.outputs.source_sha }} + source_tree: ${{ steps.admit.outputs.source_tree }} + preview_tag: ${{ steps.admit.outputs.preview_tag }} + mode: ${{ steps.admit.outputs.mode }} + draft_id: ${{ steps.admit.outputs.draft_id }} + reservation_tag: ${{ steps.admit.outputs.reservation_tag }} + policy_sha: ${{ steps.admit.outputs.policy_sha }} + policy_tree: ${{ steps.admit.outputs.policy_tree }} + reservation_present: ${{ steps.admit.outputs.reservation_present }} + steps: + - name: Require protected pylon and an exact verified preview source + id: admit + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + PREVIEW_TAG: ${{ inputs.preview_tag }} + OPERATION: ${{ inputs.operation }} + RESUME_IDENTITY: ${{ inputs.resume_identity }} + REVOKE_STABLE_TAG: ${{ inputs.revoke_stable_tag }} + REASON: ${{ inputs.reason }} + with: + script: | + const crypto = require("node:crypto"); + const owner = context.repo.owner; + const repo = context.repo.repo; + const repository = `${owner}/${repo}`; + const requestedPreview = process.env.PREVIEW_TAG; + const operation = process.env.OPERATION; + const resume = operation.startsWith("resume-"); + const originalOperation = resume ? operation.slice("resume-".length) : operation; + if ( + repository !== "pylon-code/prime-agent" || context.eventName !== "workflow_dispatch" || + context.ref !== "refs/heads/pylon" || !/^[0-9a-f]{40}$/.test(context.sha) || + !/^pylon-build-g[0-9a-f]{12}-r[1-9][0-9]*$/.test(requestedPreview) || + !["promote", "withdraw"].includes(originalOperation) + ) throw new Error("Stable promotion requires a canonical pylon dispatch and preview tag."); + if ( + (!resume && process.env.RESUME_IDENTITY) || (resume && !process.env.RESUME_IDENTITY) || + (originalOperation === "promote" && process.env.REVOKE_STABLE_TAG) || + (originalOperation === "withdraw" && !/^pylon-stable-[0-9]{6}-g[0-9a-f]{12}-r[1-9][0-9]*$/.test(process.env.REVOKE_STABLE_TAG)) || + (originalOperation === "withdraw" && !/^[a-z0-9][a-z0-9-]{2,63}$/.test(process.env.REASON)) + ) throw new Error("Stable operation or recovery inputs are malformed."); + const pylon = await github.rest.git.getRef({ owner, repo, ref: "heads/pylon" }); + if (pylon.data.object.type !== "commit" || pylon.data.object.sha !== context.sha) { + throw new Error("Stable dispatch is stale relative to protected pylon."); + } + const currentCommit = await github.rest.git.getCommit({ owner, repo, commit_sha: context.sha }); + let previewTag = requestedPreview; + let sourceSha; + let sourceTree; + let policySha = context.sha; + let policyTree = currentCommit.data.tree.sha; + let draftId = ""; + let reservationTag = ""; + let reservationPresent = "false"; + if (resume) { + const identity = process.env.RESUME_IDENTITY; + let reservation; + let draft; + if (/^pylon-stable-sequence-[0-9]{6}$/.test(identity)) { + reservationTag = identity; + reservationPresent = "true"; + const ref = await github.rest.git.getRef({ owner, repo, ref: `tags/${identity}` }); + if (ref.data.object.type !== "tag") throw new Error("Recovery reservation is not annotated."); + reservation = (await github.rest.git.getTag({ owner, repo, tag_sha: ref.data.object.sha })).data; + const fields = new Map(reservation.message.trimEnd().split("\n").slice(1).map((line) => { + const split = line.indexOf(": "); + return [line.slice(0, split), line.slice(split + 2)]; + })); + draftId = fields.get("Draft release") ?? ""; + if (!/^[0-9]+$/.test(draftId)) throw new Error("Reservation lacks its exact draft release id."); + draft = (await github.rest.repos.getRelease({ owner, repo, release_id: Number(draftId) })).data; + } else if (/^[0-9]+$/.test(identity)) { + draftId = identity; + draft = (await github.rest.repos.getRelease({ owner, repo, release_id: Number(identity) })).data; + } else if (/^pylon-stable-[0-9]{6}-g[0-9a-f]{12}-r[1-9][0-9]*$/.test(identity)) { + const releases = await github.paginate(github.rest.repos.listReleases, { owner, repo, per_page: 100 }); + const matches = releases.filter((release) => release.tag_name === identity && release.draft); + if (matches.length !== 1) throw new Error("Recovery draft tag is absent or ambiguous."); + draft = matches[0]; + draftId = String(draft.id); + } else throw new Error("Recovery identity must be a reservation, stable draft tag, or numeric release id."); + if (!draft.draft || draft.immutable === true || !Array.isArray(draft.assets) || draft.assets.length > 1) { + throw new Error("Recovery identity does not resolve to one exact unpublished stable draft."); + } + if (typeof draft.body !== "string" || Buffer.byteLength(draft.body, "utf8") > 80 * 1024) { + throw new Error("Recovery draft body is absent or exceeds the safe 80 KiB bound."); + } + const encoded = /^Manifest base64: ([A-Za-z0-9+/]+={0,2})$/m.exec(draft.body)?.[1]; + const encodedSize = /^Manifest bytes: ([1-9][0-9]*)$/m.exec(draft.body)?.[1]; + const encodedDigest = /^Manifest sha256: ([0-9a-f]{64})$/m.exec(draft.body)?.[1]; + if (!encoded || !encodedSize || !encodedDigest || (draft.body.match(/^Stable recovery manifest: base64-v1$/gm) ?? []).length !== 1) { + throw new Error("Recovery draft body lacks one exact stable manifest envelope."); + } + const bytes = Buffer.from(encoded, "base64"); + const digest = crypto.createHash("sha256").update(bytes).digest("hex"); + if (bytes.toString("base64") !== encoded || bytes.length !== Number(encodedSize) || bytes.length > 48 * 1024 || digest !== encodedDigest) { + throw new Error("Recovery draft body manifest is altered, truncated, or oversized."); + } + const manifest = JSON.parse(bytes); + const canonical = (value) => { + if (value === null || ["string", "boolean"].includes(typeof value)) return value; + if (typeof value === "number" && Number.isFinite(value)) return value; + if (Array.isArray(value)) return value.map(canonical); + if (!value || typeof value !== "object" || Object.getPrototypeOf(value) !== Object.prototype) throw new Error("Unsupported recovery manifest value."); + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])])); + }; + if (bytes.toString("utf8") !== `${JSON.stringify(canonical(manifest), null, 2)}\n`) throw new Error("Recovery manifest is not canonical JSON."); + previewTag = manifest.build?.previewTag; + sourceSha = manifest.build?.source?.commit; + sourceTree = manifest.build?.source?.tree; + policySha = manifest.promotion?.policyCommit; + policyTree = manifest.promotion?.policyTree; + const expectedBody = [ + "Pylon Prime stable publication.", "", `Tag: ${manifest.tag}`, `Source: ${sourceSha}`, `Tree: ${sourceTree}`, + `Policy: ${policySha}`, `Policy tree: ${policyTree}`, `Recipe: r${manifest.build?.recipeRevision}`, + "Stable recovery manifest: base64-v1", `Manifest bytes: ${bytes.length}`, `Manifest sha256: ${digest}`, + `Manifest base64: ${encoded}`, "", "Verify the immutable release and artifact attestations before use.", + ].join("\n"); + if ( + previewTag !== requestedPreview || manifest.promotion?.kind !== originalOperation || draft.body !== expectedBody || + draft.tag_name !== manifest.tag || draft.target_commitish !== policySha || + !/^[0-9a-f]{40}$/.test(sourceSha ?? "") || !/^[0-9a-f]{40}$/.test(sourceTree ?? "") || + !/^[0-9a-f]{40}$/.test(policySha ?? "") || !/^[0-9a-f]{40}$/.test(policyTree ?? "") + ) throw new Error("Recovery draft does not match the operator or exact manifest identity."); + const asset = draft.assets[0]; + if (asset) { + if (asset.name !== "pylon-stable-channel-v1.json") throw new Error("Recovery draft singleton asset has an unexpected name."); + const response = await github.request("GET /repos/{owner}/{repo}/releases/assets/{asset_id}", { + owner, repo, asset_id: asset.id, headers: { accept: "application/octet-stream" }, + }); + const downloaded = Buffer.from(response.data); + if (!downloaded.equals(bytes) || asset.size !== bytes.length || asset.digest !== `sha256:${digest}`) { + throw new Error("Recovery draft asset differs from the exact body-carried manifest."); + } + } + if (originalOperation === "withdraw") { + const revocation = manifest.promotion.revocation; + if (revocation?.stableTag !== process.env.REVOKE_STABLE_TAG || revocation?.reason !== process.env.REASON) { + throw new Error("Recovery withdrawal inputs do not match the approved draft."); + } + } + const policyCommit = await github.rest.git.getCommit({ owner, repo, commit_sha: policySha }); + if (policyCommit.data.tree.sha !== policyTree) throw new Error("Recovery policy tree changed."); + const policyReachability = await github.rest.repos.compareCommitsWithBasehead({ owner, repo, basehead: `${policySha}...${context.sha}` }); + if (!['ahead', 'identical'].includes(policyReachability.data.status) || policyReachability.data.merge_base_commit.sha !== policySha) { + throw new Error("Recovery policy is no longer an ancestor of live pylon."); + } + const computedReservation = `pylon-stable-sequence-${String(manifest.sequence).padStart(6, "0")}`; + if (reservation) { + const withdrawalLines = originalOperation === "withdraw" ? [ + `Withdraw stable tag: ${manifest.promotion.revocation.stableTag}`, + `Withdraw build tag: ${manifest.promotion.revocation.buildTag}`, + `Withdraw reason: ${manifest.promotion.revocation.reason}`, + ] : []; + const expected = [ + "Pylon stable sequence reservation", `Sequence: ${String(manifest.sequence).padStart(6, "0")}`, + `Policy: ${policySha}`, `Policy tree: ${policyTree}`, `Operation: ${originalOperation}`, ...withdrawalLines, + `Stable tag: ${manifest.tag}`, `Preview: ${previewTag}`, `Manifest: sha256:${digest}`, + `Draft release: ${draft.id}`, "", + ].join("\n"); + if ( + reservationTag !== computedReservation || reservation.tag !== reservationTag || reservation.message !== expected || + reservation.object.type !== "commit" || reservation.object.sha !== policySha + ) throw new Error("Recovery reservation does not bind the exact approved draft identity."); + } else { + reservationTag = computedReservation; + try { + await github.rest.git.getRef({ owner, repo, ref: `tags/${reservationTag}` }); + throw new Error("Draft-only recovery sequence already has a reservation."); + } catch (error) { + if (error.status !== 404) throw error; + } + } + } else { + const release = (await github.rest.repos.getReleaseByTag({ owner, repo, tag: previewTag })).data; + if (release.draft || release.prerelease !== true || release.immutable !== true || release.assets.length !== 6) { + throw new Error("Preview release is not an immutable six-subject prerelease."); + } + const tag = await github.rest.git.getRef({ owner, repo, ref: `tags/${previewTag}` }); + if (tag.data.object.type !== "commit" || !/^[0-9a-f]{40}$/.test(tag.data.object.sha)) { + throw new Error("Preview tag is not a lightweight exact-commit tag."); + } + sourceSha = tag.data.object.sha; + if (release.target_commitish !== sourceSha || previewTag !== `pylon-build-g${sourceSha.slice(0, 12)}-r${previewTag.split("-r").at(-1)}`) { + throw new Error("Preview release metadata is not bound to its exact tag target."); + } + const comparison = await github.rest.repos.compareCommitsWithBasehead({ owner, repo, basehead: `${sourceSha}...${context.sha}` }); + if (!['ahead', 'identical'].includes(comparison.data.status) || comparison.data.merge_base_commit.sha !== sourceSha) { + throw new Error("Preview source is not reachable from protected pylon."); + } + const sourceCommit = await github.rest.git.getCommit({ owner, repo, commit_sha: sourceSha }); + sourceTree = sourceCommit.data.tree.sha; + } + const protection = await github.graphql( + `query($owner:String!,$repo:String!,$ref:String!){repository(owner:$owner,name:$repo){ref(qualifiedName:$ref){branchProtectionRule{requiresStatusChecks requiredStatusChecks{context app{databaseId}}}}}}`, + { owner, repo, ref: "refs/heads/pylon" }, + ); + const rule = protection.repository?.ref?.branchProtectionRule; + const required = rule?.requiredStatusChecks; + const expectedPolicy = [ + { context: "Check changelog fragment", appId: 15368, workflowPath: ".github/workflows/changelog-merged-proof.yml" }, + { context: "build-check-test", appId: 15368, workflowPath: ".github/workflows/ci.yml" }, + ]; + const actualPolicy = Array.isArray(required) ? required.map((requirement) => ({ + context: requirement.context, appId: requirement.app?.databaseId ?? null, + })).sort((left, right) => left.context < right.context ? -1 : left.context > right.context ? 1 : 0) : null; + if (!rule?.requiresStatusChecks || JSON.stringify(actualPolicy) !== JSON.stringify(expectedPolicy.map(({ context, appId }) => ({ context, appId })))) { + throw new Error("Protected pylon must require exactly the two app-bound publication checks."); + } + const proveChecks = async (sha, label) => { + const checks = await github.paginate(github.rest.checks.listForRef, { owner, repo, ref: sha, filter: "latest", per_page: 100 }); + for (const requirement of expectedPolicy) { + const candidates = checks.filter((check) => check.name === requirement.context && check.head_sha === sha && check.app?.id === requirement.appId && check.status === "completed" && check.conclusion === "success"); + let proved = false; + for (const check of candidates) { + const runId = /^https:\/\/github\.com\/pylon-code\/prime-agent\/actions\/runs\/([0-9]+)(?:\/job\/[0-9]+)?$/.exec(check.details_url ?? "")?.[1]; + if (!runId) continue; + const suite = (await github.rest.checks.getSuite({ owner, repo, check_suite_id: check.check_suite.id })).data; + const run = (await github.rest.actions.getWorkflowRun({ owner, repo, run_id: Number(runId) })).data; + const workflow = (await github.rest.actions.getWorkflow({ owner, repo, workflow_id: run.workflow_id })).data; + if ( + suite.app?.id === requirement.appId && suite.head_sha === sha && suite.status === "completed" && suite.conclusion === "success" && + run.check_suite_id === suite.id && run.repository?.id === 1349002285 && run.repository?.full_name === repository && + run.head_repository?.id === 1349002285 && run.head_repository?.full_name === repository && run.event === "push" && + run.head_branch === "pylon" && run.head_sha === sha && run.status === "completed" && run.conclusion === "success" && + workflow.path === requirement.workflowPath + ) { proved = true; break; } + } + if (!proved) throw new Error(`Required check ${requirement.context} lacks canonical exact-SHA proof on ${label}.`); + } + }; + for (const [sha, label] of new Map([[sourceSha, "preview source"], [policySha, "reservation policy"], [context.sha, "fresh live pylon"]])) { + await proveChecks(sha, label); + } + core.setOutput("source_sha", sourceSha); + core.setOutput("source_tree", sourceTree); + core.setOutput("preview_tag", previewTag); + core.setOutput("mode", resume ? "resume" : "normal"); + core.setOutput("draft_id", draftId); + core.setOutput("reservation_tag", reservationTag); + core.setOutput("policy_sha", policySha); + core.setOutput("policy_tree", policyTree); + core.setOutput("reservation_present", reservationPresent); + + verify-preview: + name: Verify immutable preview provenance + needs: admission + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - name: Checkout protected verifier source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.sha }} + persist-credentials: false + fetch-depth: 1 + + - name: Download immutable preview assets + env: + GH_TOKEN: ${{ github.token }} + PREVIEW_TAG: ${{ needs.admission.outputs.preview_tag }} + run: | + mkdir -p .npm/pylon-stable/preview + gh release download "$PREVIEW_TAG" --repo pylon-code/prime-agent --dir .npm/pylon-stable/preview + + - name: Verify release, manifests, digests, signer, source, and Rekor inclusion + env: + GH_TOKEN: ${{ github.token }} + PREVIEW_TAG: ${{ needs.admission.outputs.preview_tag }} + SOURCE_SHA: ${{ needs.admission.outputs.source_sha }} + SOURCE_TREE: ${{ needs.admission.outputs.source_tree }} + run: | + gh release verify "$PREVIEW_TAG" --repo pylon-code/prime-agent + receipt="$(npm run --silent release:pylon:verify-preview -- --historical --artifact-dir .npm/pylon-stable/preview | tail -n 1)" + test "$(node -e 'const x=JSON.parse(process.argv[1]); console.log(x.source.commit)' "$receipt")" = "$SOURCE_SHA" + test "$(node -e 'const x=JSON.parse(process.argv[1]); console.log(x.source.tree)' "$receipt")" = "$SOURCE_TREE" + npm run release:pylon:verify-attestations -- \ + --historical \ + --artifact-dir .npm/pylon-stable/preview \ + --source-sha "$SOURCE_SHA" \ + --source-tree "$SOURCE_TREE" + + - name: Upload verified preview bytes + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: verified-stable-preview + path: .npm/pylon-stable/preview + if-no-files-found: error + retention-days: 3 + + install: + name: Stable candidate installed artifact (${{ matrix.os }}) + needs: [admission, verify-preview] + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + permissions: + actions: read + contents: read + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + steps: + - name: Checkout current protected install policy + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.sha }} + persist-credentials: false + fetch-depth: 1 + + - name: Setup pinned Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ env.PYLON_RELEASE_NODE }} + + - name: Install pinned npm + run: npm install --global "npm@${{ env.PYLON_RELEASE_NPM }}" + + - name: Verify workflow artifact provenance + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const run = (await github.rest.actions.getWorkflowRun({ ...context.repo, run_id: context.runId })).data; + const workflow = (await github.rest.actions.getWorkflow({ ...context.repo, workflow_id: run.workflow_id })).data; + if ( + context.repo.owner !== "pylon-code" || context.repo.repo !== "prime-agent" || run.repository?.id !== 1349002285 || + run.repository?.full_name !== "pylon-code/prime-agent" || run.head_repository?.id !== 1349002285 || + run.head_repository?.full_name !== "pylon-code/prime-agent" || run.event !== context.eventName || + run.head_sha !== context.sha || run.head_branch !== "pylon" || context.ref !== "refs/heads/pylon" || + workflow.path !== ".github/workflows/pylon-stable-release.yml" + ) throw new Error("Artifact workflow provenance is not canonical."); + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + ...context.repo, run_id: context.runId, per_page: 100, + }); + const matches = artifacts.filter((artifact) => artifact.name === "verified-stable-preview"); + if (matches.length !== 1 || matches[0].expired || !/^sha256:[0-9a-f]{64}$/.test(matches[0].digest ?? "")) { + throw new Error("Artifact is ambiguous, expired, or lacks a SHA-256 transport digest."); + } + + - name: Download verified preview bytes + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: verified-stable-preview + path: .npm/pylon-release/artifacts + + - name: Install the same exact preview bytes + run: | + npm run release:pylon:verify-preview -- --historical + npm run release:pylon:smoke -- --historical + + prepare: + name: Resolve exact stable transaction + needs: [admission, verify-preview, install] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + outputs: + publish: ${{ steps.prepare.outputs.publish }} + tag: ${{ steps.prepare.outputs.tag }} + source_sha: ${{ steps.prepare.outputs.source_sha }} + source_tree: ${{ steps.prepare.outputs.source_tree }} + sequence: ${{ steps.prepare.outputs.sequence }} + draft_id: ${{ steps.prepare.outputs.draft_id }} + reservation_tag: ${{ steps.prepare.outputs.reservation_tag }} + manifest_sha256: ${{ steps.prepare.outputs.manifest_sha256 }} + mode: ${{ needs.admission.outputs.mode }} + steps: + - name: Checkout protected promotion policy + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.sha }} + persist-credentials: false + fetch-depth: 1 + + - name: Verify workflow artifact provenance + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const run = (await github.rest.actions.getWorkflowRun({ ...context.repo, run_id: context.runId })).data; + const workflow = (await github.rest.actions.getWorkflow({ ...context.repo, workflow_id: run.workflow_id })).data; + if ( + context.repo.owner !== "pylon-code" || context.repo.repo !== "prime-agent" || run.repository?.id !== 1349002285 || + run.repository?.full_name !== "pylon-code/prime-agent" || run.head_repository?.id !== 1349002285 || + run.head_repository?.full_name !== "pylon-code/prime-agent" || run.event !== context.eventName || + run.head_sha !== context.sha || run.head_branch !== "pylon" || context.ref !== "refs/heads/pylon" || + workflow.path !== ".github/workflows/pylon-stable-release.yml" + ) throw new Error("Artifact workflow provenance is not canonical."); + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + ...context.repo, run_id: context.runId, per_page: 100, + }); + const matches = artifacts.filter((artifact) => artifact.name === "verified-stable-preview"); + if (matches.length !== 1 || matches[0].expired || !/^sha256:[0-9a-f]{64}$/.test(matches[0].digest ?? "")) { + throw new Error("Artifact is ambiguous, expired, or lacks a SHA-256 transport digest."); + } + + - name: Download verified preview bytes + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: verified-stable-preview + path: .npm/pylon-stable/preview + + - name: Build the next append-only stable state + id: prepare + env: + GH_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ github.token }} + OPERATION: ${{ inputs.operation }} + REVOKE_STABLE_TAG: ${{ inputs.revoke_stable_tag }} + REASON: ${{ inputs.reason }} + MODE: ${{ needs.admission.outputs.mode }} + DRAFT_ID: ${{ needs.admission.outputs.draft_id }} + RESERVATION_TAG: ${{ needs.admission.outputs.reservation_tag }} + run: | + policy_tree="$(git rev-parse 'HEAD^{tree}')" + if [ "$MODE" = resume ]; then + original_operation="${OPERATION#resume-}" + args=( + --draft-id "$DRAFT_ID" + --reservation-tag "$RESERVATION_TAG" + --preview-tag "${{ needs.admission.outputs.preview_tag }}" + --operation "$original_operation" + --out-dir .npm/pylon-stable/output + ) + if [ "$original_operation" = withdraw ]; then + args+=(--revoke-tag "$REVOKE_STABLE_TAG" --reason "$REASON") + fi + node scripts/recover-pylon-stable-manifest.mjs "${args[@]}" + else + args=( + --artifact-dir .npm/pylon-stable/preview + --out-dir .npm/pylon-stable/output + --operation "$OPERATION" + --policy-sha "${{ github.sha }}" + --policy-tree "$policy_tree" + --publication-policy-revision 1 + ) + if [ "$OPERATION" = withdraw ]; then + args+=(--revoke-tag "$REVOKE_STABLE_TAG" --reason "$REASON") + fi + node scripts/prepare-pylon-stable-manifest.mjs "${args[@]}" + fi + + - name: Upload new stable manifest + if: steps.prepare.outputs.publish == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pylon-stable-manifest + path: .npm/pylon-stable/output/pylon-stable-channel-v1.json + if-no-files-found: error + retention-days: 3 + + attest: + name: Approve and attest stable channel manifest + if: needs.prepare.outputs.publish == 'true' && needs.prepare.outputs.mode == 'normal' + needs: prepare + runs-on: ubuntu-24.04 + timeout-minutes: 5 + environment: pylon-stable + permissions: + actions: read + id-token: write + attestations: write + steps: + - name: Verify workflow artifact provenance + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const run = (await github.rest.actions.getWorkflowRun({ ...context.repo, run_id: context.runId })).data; + const workflow = (await github.rest.actions.getWorkflow({ ...context.repo, workflow_id: run.workflow_id })).data; + if ( + context.repo.owner !== "pylon-code" || context.repo.repo !== "prime-agent" || run.repository?.id !== 1349002285 || + run.repository?.full_name !== "pylon-code/prime-agent" || run.head_repository?.id !== 1349002285 || + run.head_repository?.full_name !== "pylon-code/prime-agent" || run.event !== context.eventName || + run.head_sha !== context.sha || run.head_branch !== "pylon" || context.ref !== "refs/heads/pylon" || + workflow.path !== ".github/workflows/pylon-stable-release.yml" + ) throw new Error("Artifact workflow provenance is not canonical."); + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + ...context.repo, run_id: context.runId, per_page: 100, + }); + const matches = artifacts.filter((artifact) => artifact.name === "pylon-stable-manifest"); + if (matches.length !== 1 || matches[0].expired || !/^sha256:[0-9a-f]{64}$/.test(matches[0].digest ?? "")) { + throw new Error("Artifact is ambiguous, expired, or lacks a SHA-256 transport digest."); + } + + - name: Download stable manifest + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: pylon-stable-manifest + path: publication + + - name: Validate exact stable subject before signing + shell: bash + run: | + node <<'NODE' + const fs = require("node:fs"); + const path = require("node:path"); + const dir = "publication"; + const name = "pylon-stable-channel-v1.json"; + const names = fs.readdirSync(dir); + if (names.length !== 1 || names[0] !== name) throw new Error("Stable attestation requires exactly one manifest."); + const file = path.join(dir, name); + if (!fs.lstatSync(file).isFile()) throw new Error("Stable attestation subject is not a regular file."); + const bytes = fs.readFileSync(file); + const manifest = JSON.parse(bytes); + const canonical = (value) => { + if (value === null || ["string", "boolean"].includes(typeof value)) return value; + if (typeof value === "number" && Number.isFinite(value)) return value; + if (Array.isArray(value)) return value.map(canonical); + if (!value || typeof value !== "object" || Object.getPrototypeOf(value) !== Object.prototype) throw new Error("Unsupported manifest value."); + return Object.fromEntries(Object.keys(value).sort().map((key) => { + if (value[key] === undefined) throw new Error("Undefined manifest value."); + return [key, canonical(value[key])]; + })); + }; + if (bytes.toString("utf8") !== `${JSON.stringify(canonical(manifest), null, 2)}\n`) throw new Error("Stable manifest is not canonical JSON."); + if ( + manifest.schemaVersion !== 1 || manifest.channel !== "stable" || + manifest.repository !== "https://github.com/pylon-code/prime-agent" || + manifest.promotion?.publicationPolicyRevision !== 1 || + !/^pylon-stable-[0-9]{6}-g[0-9a-f]{12}-r[1-9][0-9]*$/.test(manifest.tag) + ) throw new Error("Stable manifest identity is malformed."); + NODE + + - name: Generate stable manifest provenance + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: publication/pylon-stable-channel-v1.json + + verify-attestation: + name: Verify stable manifest provenance + if: needs.prepare.outputs.publish == 'true' && needs.prepare.outputs.mode == 'normal' + needs: [prepare, attest] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Checkout protected verification policy + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.sha }} + persist-credentials: false + fetch-depth: 1 + + - name: Download stable manifest + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: pylon-stable-manifest + path: publication + + - name: Verify exact stable signer and Rekor inclusion + env: + GH_TOKEN: ${{ github.token }} + run: | + policy_tree="$(git rev-parse 'HEAD^{tree}')" + npm run release:pylon:verify-stable-attestation -- \ + --manifest publication/pylon-stable-channel-v1.json \ + --promotion-sha "${{ github.sha }}" \ + --promotion-tree "$policy_tree" + + authorize-stable-resume: + name: Authorize exact stable recovery + if: needs.prepare.outputs.publish == 'true' && needs.prepare.outputs.mode == 'resume' + needs: [prepare, install] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + environment: pylon-stable + permissions: {} + steps: + - name: Record explicit recovery approval + run: echo "Approved exact Pylon stable recovery" + + stage-draft: + name: Stage exact approved stable draft + if: needs.prepare.outputs.publish == 'true' && needs.prepare.outputs.mode == 'normal' + needs: [prepare, verify-attestation] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: write + outputs: + draft_id: ${{ steps.stage.outputs.result }} + steps: + - name: Download the approved stable manifest + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: pylon-stable-manifest + path: publication + + - name: Create or finish the exact durable draft + id: stage + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + STABLE_MANIFEST: publication/pylon-stable-channel-v1.json + with: + script: | + const fs = require("node:fs"); + const crypto = require("node:crypto"); + const owner = context.repo.owner; + const repo = context.repo.repo; + if (`${owner}/${repo}` !== "pylon-code/prime-agent" || context.eventName !== "workflow_dispatch" || context.ref !== "refs/heads/pylon") { + throw new Error("Stable draft staging requires the canonical pylon dispatch."); + } + const stat = fs.lstatSync(process.env.STABLE_MANIFEST); + if (!stat.isFile()) throw new Error("Stable draft subject is not one regular file."); + const bytes = fs.readFileSync(process.env.STABLE_MANIFEST); + if (bytes.length < 1 || bytes.length > 48 * 1024) throw new Error("Stable manifest exceeds the safe 48 KiB recovery bound."); + const manifest = JSON.parse(bytes); + const canonical = (value) => { + if (value === null || ["string", "boolean"].includes(typeof value)) return value; + if (typeof value === "number" && Number.isFinite(value)) return value; + if (Array.isArray(value)) return value.map(canonical); + if (!value || typeof value !== "object" || Object.getPrototypeOf(value) !== Object.prototype) throw new Error("Unsupported stable manifest value."); + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])])); + }; + if (bytes.toString("utf8") !== `${JSON.stringify(canonical(manifest), null, 2)}\n`) throw new Error("Stable draft manifest is not canonical."); + const digest = crypto.createHash("sha256").update(bytes).digest("hex"); + const tag = manifest.tag; + const parsed = /^pylon-stable-([0-9]{6})-g([0-9a-f]{12})-r([1-9][0-9]*)$/.exec(tag ?? ""); + const preview = /^pylon-build-g([0-9a-f]{12})-r([1-9][0-9]*)$/.exec(manifest.build?.previewTag ?? ""); + if ( + !parsed || !preview || Number(parsed[1]) !== manifest.sequence || parsed[2] !== manifest.build.source?.commit?.slice(0, 12) || + Number(parsed[3]) !== manifest.build.recipeRevision || preview[1] !== parsed[2] || Number(preview[2]) !== manifest.build.recipeRevision || + manifest.build.previewTag !== manifest.build.id || ![1].includes(manifest.build.publicationPolicyRevision) || + manifest.promotion?.policyCommit !== context.sha || manifest.promotion?.publicationPolicyRevision !== 1 + ) throw new Error("Stable draft identity is malformed or not signed by this policy commit."); + const name = `Pylon Prime stable ${tag}`; + const encoded = bytes.toString("base64"); + const body = [ + "Pylon Prime stable publication.", "", `Tag: ${tag}`, `Source: ${manifest.build.source.commit}`, + `Tree: ${manifest.build.source.tree}`, `Policy: ${manifest.promotion.policyCommit}`, + `Policy tree: ${manifest.promotion.policyTree}`, `Recipe: r${manifest.build.recipeRevision}`, + "Stable recovery manifest: base64-v1", `Manifest bytes: ${bytes.length}`, `Manifest sha256: ${digest}`, + `Manifest base64: ${encoded}`, "", "Verify the immutable release and artifact attestations before use.", + ].join("\n"); + if (Buffer.byteLength(body, "utf8") > 80 * 1024) throw new Error("Stable recovery release body exceeds 80 KiB."); + const releases = await github.paginate(github.rest.repos.listReleases, { owner, repo, per_page: 100 }); + const matching = releases.filter((release) => release.tag_name === tag); + if (matching.length > 1) throw new Error("Stable draft tag is ambiguous."); + const sameSequence = releases.filter((release) => /^pylon-stable-[0-9]{6}-g/.test(release.tag_name ?? "") && Number(/^pylon-stable-([0-9]{6})-/.exec(release.tag_name)[1]) === manifest.sequence); + if (sameSequence.some((release) => release.tag_name !== tag)) throw new Error("Stable sequence already has a different release identity."); + let draft = matching[0]; + if (draft && !draft.draft) throw new Error("Stable release was already published before this transaction staged its draft."); + if (!draft) { + for (const ref of [`tags/${tag}`, `tags/pylon-stable-sequence-${String(manifest.sequence).padStart(6, "0")}`]) { + try { + await github.rest.git.getRef({ owner, repo, ref }); + throw new Error(`Stable draft cannot stage over existing ${ref}.`); + } catch (error) { + if (error.status !== 404) throw error; + } + } + try { + draft = (await github.rest.repos.createRelease({ + owner, repo, tag_name: tag, target_commitish: manifest.promotion.policyCommit, + name, body, draft: true, prerelease: false, make_latest: "false", + })).data; + } catch (error) { + if (error.status === 422) { + await github.paginate(github.rest.repos.listReleases, { owner, repo, per_page: 100 }); + throw new Error("Stable draft creation raced (422); refetched state and stopped."); + } + throw error; + } + } + const asset = draft.assets?.[0]; + if ( + draft.immutable === true || !draft.draft || draft.tag_name !== tag || draft.name !== name || draft.body !== body || + draft.prerelease !== false || draft.target_commitish !== manifest.promotion.policyCommit || draft.assets.length > 1 || + (asset && (asset.name !== "pylon-stable-channel-v1.json" || asset.size !== bytes.length || asset.digest !== `sha256:${digest}`)) + ) throw new Error("Existing stable draft identity or asset differs."); + if (!asset) { + await github.request("POST /repos/{owner}/{repo}/releases/{release_id}/assets", { + owner, repo, release_id: draft.id, name: "pylon-stable-channel-v1.json", data: bytes, + headers: { "content-type": "application/json", "content-length": bytes.length }, + }); + } + const staged = (await github.rest.repos.getRelease({ owner, repo, release_id: draft.id })).data; + if (!staged.draft || staged.assets?.length !== 1 || staged.assets[0].name !== "pylon-stable-channel-v1.json" || staged.assets[0].size !== bytes.length || staged.assets[0].digest !== `sha256:${digest}`) { + throw new Error("Stable draft was not fully uploaded with the exact manifest receipt."); + } + const downloaded = await github.request("GET /repos/{owner}/{repo}/releases/assets/{asset_id}", { + owner, repo, asset_id: staged.assets[0].id, headers: { accept: "application/octet-stream" }, + }); + if (crypto.createHash("sha256").update(Buffer.from(downloaded.data)).digest("hex") !== digest) { + throw new Error("Re-downloaded stable draft bytes differ after upload."); + } + return staged.id; + + publish: + name: Reserve and publish immutable stable sequence + if: >- + always() && needs.prepare.outputs.publish == 'true' && needs.prepare.result == 'success' && + needs.install.result == 'success' && + ((needs.prepare.outputs.mode == 'normal' && needs.verify-attestation.result == 'success' && needs.stage-draft.result == 'success') || + (needs.prepare.outputs.mode == 'resume' && needs.authorize-stable-resume.result == 'success')) + needs: [admission, install, prepare, attest, verify-attestation, stage-draft, authorize-stable-resume] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + environment: pylon-stable + permissions: + actions: read + checks: read + contents: write + steps: + - name: Re-download and validate the exact stable transaction + id: transaction + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + DRAFT_ID: ${{ needs.stage-draft.outputs.draft_id || needs.prepare.outputs.draft_id }} + MODE: ${{ needs.prepare.outputs.mode }} + POLICY_SHA: ${{ needs.admission.outputs.policy_sha }} + POLICY_TREE: ${{ needs.admission.outputs.policy_tree }} + PREVIEW_TAG: ${{ needs.admission.outputs.preview_tag }} + RESERVATION_PRESENT: ${{ needs.admission.outputs.reservation_present }} + OPERATION: ${{ inputs.operation }} + REVOKE_STABLE_TAG: ${{ inputs.revoke_stable_tag }} + REASON: ${{ inputs.reason }} + EXPECTED_MANIFEST_SHA256: ${{ needs.prepare.outputs.manifest_sha256 }} + with: + script: | + const crypto = require("node:crypto"); + const owner = context.repo.owner; + const repo = context.repo.repo; + const repository = `${owner}/${repo}`; + const draftId = Number(process.env.DRAFT_ID); + const mode = process.env.MODE; + const operation = process.env.OPERATION.replace(/^resume-/, ""); + if ( + repository !== "pylon-code/prime-agent" || context.eventName !== "workflow_dispatch" || context.ref !== "refs/heads/pylon" || + !Number.isSafeInteger(draftId) || draftId < 1 || !["normal", "resume"].includes(mode) || !["promote", "withdraw"].includes(operation) + ) throw new Error("Stable publisher requires one exact canonical transaction."); + const current = await github.rest.git.getRef({ owner, repo, ref: "heads/pylon" }); + if (current.data.object.type !== "commit" || current.data.object.sha !== context.sha) throw new Error("Stable publication run is stale relative to live pylon."); + let draft = (await github.rest.repos.getRelease({ owner, repo, release_id: draftId })).data; + if (!draft.draft || draft.immutable === true || !Array.isArray(draft.assets) || draft.assets.length > 1) { + throw new Error("Stable transaction no longer resolves to one exact unpublished draft."); + } + if (typeof draft.body !== "string" || Buffer.byteLength(draft.body, "utf8") > 80 * 1024) { + throw new Error("Stable recovery body is absent or exceeds 80 KiB."); + } + const encoded = /^Manifest base64: ([A-Za-z0-9+/]+={0,2})$/m.exec(draft.body)?.[1]; + const encodedSize = /^Manifest bytes: ([1-9][0-9]*)$/m.exec(draft.body)?.[1]; + const encodedDigest = /^Manifest sha256: ([0-9a-f]{64})$/m.exec(draft.body)?.[1]; + if (!encoded || !encodedSize || !encodedDigest || (draft.body.match(/^Stable recovery manifest: base64-v1$/gm) ?? []).length !== 1) { + throw new Error("Stable recovery body lacks one exact manifest envelope."); + } + const bytes = Buffer.from(encoded, "base64"); + const digest = crypto.createHash("sha256").update(bytes).digest("hex"); + if (digest !== process.env.EXPECTED_MANIFEST_SHA256) throw new Error("Stable draft bytes differ from the exact prepared or recovered manifest digest."); + if (bytes.toString("base64") !== encoded || bytes.length !== Number(encodedSize) || bytes.length > 48 * 1024 || digest !== encodedDigest) { + throw new Error("Stable recovery body manifest is altered, truncated, or oversized."); + } + const manifest = JSON.parse(bytes); + const canonical = (value) => { + if (value === null || ["string", "boolean"].includes(typeof value)) return value; + if (typeof value === "number" && Number.isFinite(value)) return value; + if (Array.isArray(value)) return value.map(canonical); + if (!value || typeof value !== "object" || Object.getPrototypeOf(value) !== Object.prototype) throw new Error("Unsupported stable manifest value."); + return Object.fromEntries(Object.keys(value).sort().map((key) => { + if (value[key] === undefined) throw new Error("Undefined stable manifest value."); + return [key, canonical(value[key])]; + })); + }; + if (bytes.toString("utf8") !== `${JSON.stringify(canonical(manifest), null, 2)}\n`) throw new Error("Stable draft bytes are not canonical JSON."); + const recoveryBody = [ + "Pylon Prime stable publication.", "", `Tag: ${manifest.tag}`, `Source: ${manifest.build?.source?.commit}`, + `Tree: ${manifest.build?.source?.tree}`, `Policy: ${manifest.promotion?.policyCommit}`, + `Policy tree: ${manifest.promotion?.policyTree}`, `Recipe: r${manifest.build?.recipeRevision}`, + "Stable recovery manifest: base64-v1", `Manifest bytes: ${bytes.length}`, `Manifest sha256: ${digest}`, + `Manifest base64: ${encoded}`, "", "Verify the immutable release and artifact attestations before use.", + ].join("\n"); + if (draft.body !== recoveryBody) throw new Error("Stable recovery body metadata differs from its exact manifest."); + let asset = draft.assets[0]; + if (asset && asset.name !== "pylon-stable-channel-v1.json") throw new Error("Stable draft singleton asset has an unexpected name."); + if (!asset) { + await github.request("POST /repos/{owner}/{repo}/releases/{release_id}/assets", { + owner, repo, release_id: draft.id, name: "pylon-stable-channel-v1.json", data: bytes, + headers: { "content-type": "application/json", "content-length": bytes.length }, + }); + draft = (await github.rest.repos.getRelease({ owner, repo, release_id: draft.id })).data; + if (!draft.draft || draft.assets?.length !== 1 || draft.body !== recoveryBody) throw new Error("Recovered stable singleton did not stage exactly once."); + asset = draft.assets[0]; + } + const downloaded = await github.request("GET /repos/{owner}/{repo}/releases/assets/{asset_id}", { + owner, repo, asset_id: asset.id, headers: { accept: "application/octet-stream" }, + }); + const downloadedBytes = Buffer.from(downloaded.data); + if (!downloadedBytes.equals(bytes) || asset.name !== "pylon-stable-channel-v1.json" || asset.size !== bytes.length || asset.digest !== `sha256:${digest}`) { + throw new Error("Stable draft singleton differs from the exact body-carried attested bytes."); + } + const stableMatch = /^pylon-stable-([0-9]{6})-g([0-9a-f]{12})-r([1-9][0-9]*)$/.exec(manifest.tag ?? ""); + const previewMatch = /^pylon-build-g([0-9a-f]{12})-r([1-9][0-9]*)$/.exec(manifest.build?.previewTag ?? ""); + if ( + manifest.schemaVersion !== 1 || manifest.channel !== "stable" || manifest.repository !== "https://github.com/pylon-code/prime-agent" || + !stableMatch || !previewMatch || Number(stableMatch[1]) !== manifest.sequence || stableMatch[2] !== manifest.build?.source?.commit?.slice(0, 12) || + Number(stableMatch[3]) !== manifest.build?.recipeRevision || previewMatch[1] !== stableMatch[2] || + Number(previewMatch[2]) !== manifest.build?.recipeRevision || manifest.build.previewTag !== manifest.build.id || + manifest.build.previewSequence?.sequenceEpoch !== 1 || !Number.isSafeInteger(manifest.build.previewSequence?.sequence) || + manifest.build.previewSequence.sequence < 1 || !/^[1-9][0-9]*$/.test(manifest.build.previewSequence?.workflowRunId ?? "") || + manifest.build.previewTag !== process.env.PREVIEW_TAG || ![1].includes(manifest.build.publicationPolicyRevision) || + manifest.promotion?.kind !== operation || manifest.promotion?.publicationPolicyRevision !== 1 || + manifest.promotion?.policyCommit !== process.env.POLICY_SHA || manifest.promotion?.policyTree !== process.env.POLICY_TREE + ) throw new Error("Stable manifest, preview recipe, operator request, or policy identity differs."); + if (operation === "withdraw") { + const revocation = manifest.promotion.revocation; + if ( + revocation?.stableTag !== process.env.REVOKE_STABLE_TAG || revocation?.reason !== process.env.REASON || + revocation?.buildTag !== manifest.revocations?.find((entry) => entry.stableTag === process.env.REVOKE_STABLE_TAG)?.buildTag || + revocation?.revokedBySequence !== manifest.sequence + ) throw new Error("Stable withdrawal differs from the original exact operator fields."); + } else if (process.env.REVOKE_STABLE_TAG) throw new Error("Stable promotion unexpectedly carries withdrawal input."); + if ( + manifest.history?.highWater !== manifest.sequence - 1 || + (manifest.sequence === 1 ? manifest.history.previous !== null : + !/^pylon-stable-[0-9]{6}-g/.test(manifest.history?.previous?.tag ?? "") || !/^[0-9a-f]{64}$/.test(manifest.history?.previous?.sha256 ?? "")) + ) throw new Error("Stable manifest high-water is malformed."); + const policyCommit = await github.rest.git.getCommit({ owner, repo, commit_sha: manifest.promotion.policyCommit }); + const sourceCommit = await github.rest.git.getCommit({ owner, repo, commit_sha: manifest.build.source.commit }); + if (policyCommit.data.tree.sha !== manifest.promotion.policyTree || sourceCommit.data.tree.sha !== manifest.build.source.tree) { + throw new Error("Stable policy or source tree differs from GitHub."); + } + for (const sha of [manifest.promotion.policyCommit, manifest.build.source.commit]) { + const comparison = await github.rest.repos.compareCommitsWithBasehead({ owner, repo, basehead: `${sha}...${context.sha}` }); + if (!["ahead", "identical"].includes(comparison.data.status) || comparison.data.merge_base_commit.sha !== sha) { + throw new Error("Stable policy or source is not an ancestor of current protected pylon."); + } + } + const previewRelease = (await github.rest.repos.getReleaseByTag({ owner, repo, tag: manifest.build.previewTag })).data; + const previewBody = [ + "Pylon Prime preview publication.", "", `Tag: ${manifest.build.previewTag}`, `Source: ${manifest.build.source.commit}`, + `Tree: ${manifest.build.source.tree}`, `Recipe: r${manifest.build.recipeRevision}`, "", + "Verify the immutable release and artifact attestations before use.", + ].join("\n"); + const previewAssets = new Map(manifest.build.assets.map((asset) => [asset.file, { size: asset.size, digest: `sha256:${asset.sha256}` }])); + previewAssets.set("pylon-prime-agent-release-v1.json", { digest: `sha256:${manifest.build.releaseManifest.sha256}` }); + previewAssets.set("pylon-preview-channel-v1.json", { digest: `sha256:${manifest.build.previewManifest.sha256}` }); + if ( + previewRelease.immutable !== true || previewRelease.draft || previewRelease.prerelease !== true || + previewRelease.tag_name !== manifest.build.previewTag || previewRelease.name !== `Pylon Prime preview ${manifest.build.previewTag}` || + previewRelease.body !== previewBody || previewRelease.target_commitish !== manifest.build.source.commit || previewRelease.assets.length !== previewAssets.size + ) throw new Error("Immutable preview identity changed before stable CAS."); + for (const asset of previewRelease.assets) { + const expected = previewAssets.get(asset.name); + if (!expected || asset.digest !== expected.digest || (expected.size !== undefined && asset.size !== expected.size)) throw new Error(`Immutable preview asset changed: ${asset.name}`); + previewAssets.delete(asset.name); + } + if (previewAssets.size !== 0) throw new Error("Immutable preview is missing an exact subject."); + const previewManifestAsset = previewRelease.assets.find((asset) => asset.name === "pylon-preview-channel-v1.json"); + const previewDownload = await github.request("GET /repos/{owner}/{repo}/releases/assets/{asset_id}", { + owner, repo, asset_id: previewManifestAsset.id, headers: { accept: "application/octet-stream" }, + }); + const previewBytes = Buffer.from(previewDownload.data); + const previewManifest = JSON.parse(previewBytes); + if ( + crypto.createHash("sha256").update(previewBytes).digest("hex") !== manifest.build.previewManifest.sha256 || + Object.keys(manifest.build.previewSequence).sort().join(",") !== "sequence,sequenceEpoch,workflowRunId" || + previewManifest.sequenceEpoch !== manifest.build.previewSequence.sequenceEpoch || + previewManifest.sequence !== manifest.build.previewSequence.sequence || + previewManifest.workflowRunId !== manifest.build.previewSequence.workflowRunId || + previewManifest.publicationPolicyRevision !== manifest.build.publicationPolicyRevision + ) throw new Error("Stable manifest does not copy the immutable preview sequence and policy identity."); + const previewRef = await github.rest.git.getRef({ owner, repo, ref: `tags/${manifest.build.previewTag}` }); + if (previewRef.data.object.type !== "commit" || previewRef.data.object.sha !== manifest.build.source.commit) throw new Error("Immutable preview tag changed before stable CAS."); + + const protection = await github.graphql( + `query($owner:String!,$repo:String!,$ref:String!){repository(owner:$owner,name:$repo){ref(qualifiedName:$ref){branchProtectionRule{requiresStatusChecks requiredStatusChecks{context app{databaseId}}}}}}`, + { owner, repo, ref: "refs/heads/pylon" }, + ); + const rule = protection.repository?.ref?.branchProtectionRule; + const required = rule?.requiredStatusChecks; + const expectedPolicy = [ + { context: "Check changelog fragment", appId: 15368, workflowPath: ".github/workflows/changelog-merged-proof.yml" }, + { context: "build-check-test", appId: 15368, workflowPath: ".github/workflows/ci.yml" }, + ]; + const actualPolicy = Array.isArray(required) ? required.map((requirement) => ({ + context: requirement.context, appId: requirement.app?.databaseId ?? null, + })).sort((left, right) => left.context < right.context ? -1 : left.context > right.context ? 1 : 0) : null; + if (!rule?.requiresStatusChecks || JSON.stringify(actualPolicy) !== JSON.stringify(expectedPolicy.map(({ context, appId }) => ({ context, appId })))) { + throw new Error("Protected pylon must require exactly the two app-bound publication checks."); + } + const proveChecks = async (sha, label) => { + const checks = await github.paginate(github.rest.checks.listForRef, { owner, repo, ref: sha, filter: "latest", per_page: 100 }); + for (const requirement of expectedPolicy) { + const candidates = checks.filter((check) => check.name === requirement.context && check.head_sha === sha && check.app?.id === requirement.appId && check.status === "completed" && check.conclusion === "success"); + let proved = false; + for (const check of candidates) { + const runId = /^https:\/\/github\.com\/pylon-code\/prime-agent\/actions\/runs\/([0-9]+)(?:\/job\/[0-9]+)?$/.exec(check.details_url ?? "")?.[1]; + if (!runId) continue; + const suite = (await github.rest.checks.getSuite({ owner, repo, check_suite_id: check.check_suite.id })).data; + const run = (await github.rest.actions.getWorkflowRun({ owner, repo, run_id: Number(runId) })).data; + const workflow = (await github.rest.actions.getWorkflow({ owner, repo, workflow_id: run.workflow_id })).data; + if ( + suite.app?.id === requirement.appId && suite.head_sha === sha && suite.status === "completed" && suite.conclusion === "success" && + run.check_suite_id === suite.id && run.repository?.id === 1349002285 && run.repository?.full_name === repository && + run.head_repository?.id === 1349002285 && run.head_repository?.full_name === repository && run.event === "push" && + run.head_branch === "pylon" && run.head_sha === sha && run.status === "completed" && run.conclusion === "success" && + workflow.path === requirement.workflowPath + ) { proved = true; break; } + } + if (!proved) throw new Error(`Required check ${requirement.context} lacks canonical exact-SHA proof on ${label}.`); + } + }; + for (const [sha, label] of new Map([ + [manifest.build.source.commit, "original preview source"], + [manifest.promotion.policyCommit, "original stable policy"], + [context.sha, "fresh current pylon"], + ])) await proveChecks(sha, label); + + const allReleases = await github.paginate(github.rest.repos.listReleases, { owner, repo, per_page: 100 }); + const published = allReleases.filter((release) => !release.draft && /^pylon-stable-[0-9]{6}-g[0-9a-f]{12}-r[1-9][0-9]*$/.test(release.tag_name ?? "")); + const publishedSequences = published.map((release) => Number(/^pylon-stable-([0-9]{6})-/.exec(release.tag_name)[1])).sort((a, b) => a - b); + if (publishedSequences.length !== manifest.sequence - 1 || publishedSequences.some((sequence, index) => sequence !== index + 1)) { + throw new Error("Published stable history is not the exact N-1 prefix."); + } + if (manifest.sequence > 1) { + const previous = published.find((release) => release.tag_name === manifest.history.previous.tag); + if ( + !previous || previous.immutable !== true || previous.assets?.length !== 1 || previous.assets[0].name !== "pylon-stable-channel-v1.json" || + previous.assets[0].digest !== `sha256:${manifest.history.previous.sha256}` + ) throw new Error("Stable manifest does not chain from the exact immutable N-1 release."); + } + const sameSequence = allReleases.filter((release) => /^pylon-stable-[0-9]{6}-g/.test(release.tag_name ?? "") && Number(/^pylon-stable-([0-9]{6})-/.exec(release.tag_name)[1]) === manifest.sequence); + if (sameSequence.length !== 1 || sameSequence[0].id !== draft.id) throw new Error("Stable sequence has a different or ambiguous draft/release identity."); + const name = `Pylon Prime stable ${manifest.tag}`; + const body = recoveryBody; + if ( + draft.tag_name !== manifest.tag || draft.name !== name || draft.body !== body || draft.prerelease !== false || + draft.target_commitish !== manifest.promotion.policyCommit + ) throw new Error("Stable draft metadata changed before CAS."); + let stableRef; + const requireStableRef = async () => { + stableRef = (await github.rest.git.getRef({ owner, repo, ref: `tags/${manifest.tag}` })).data; + if (stableRef.object.type !== "commit" || stableRef.object.sha !== manifest.promotion.policyCommit) { + throw new Error("Stable publication tag is annotated or targets a different commit."); + } + return stableRef; + }; + try { + await requireStableRef(); + if (mode !== "resume" || process.env.RESERVATION_PRESENT !== "true") { + throw new Error("Stable publication tag exists before its exact reserved recovery."); + } + } catch (error) { + if (error.status !== 404) throw error; + stableRef = undefined; + } + const reservationTag = `pylon-stable-sequence-${String(manifest.sequence).padStart(6, "0")}`; + const withdrawalLines = operation === "withdraw" ? [ + `Withdraw stable tag: ${manifest.promotion.revocation.stableTag}`, + `Withdraw build tag: ${manifest.promotion.revocation.buildTag}`, + `Withdraw reason: ${manifest.promotion.revocation.reason}`, + ] : []; + const reservationMessage = [ + "Pylon stable sequence reservation", `Sequence: ${String(manifest.sequence).padStart(6, "0")}`, + `Policy: ${manifest.promotion.policyCommit}`, `Policy tree: ${manifest.promotion.policyTree}`, + `Operation: ${operation}`, ...withdrawalLines, `Stable tag: ${manifest.tag}`, `Preview: ${manifest.build.previewTag}`, + `Manifest: sha256:${digest}`, `Draft release: ${draft.id}`, "", + ].join("\n"); + const reservationRefs = (await github.paginate(github.rest.git.listMatchingRefs, { owner, repo, ref: "tags/pylon-stable-sequence-", per_page: 100 })) + .filter((ref) => /^refs\/tags\/pylon-stable-sequence-[0-9]{6}$/.test(ref.ref)); + const expectedPriorReservations = Array.from({ length: manifest.sequence - 1 }, (_, index) => index + 1); + const priorReservations = reservationRefs.filter((ref) => Number(ref.ref.slice(-6)) < manifest.sequence).map((ref) => Number(ref.ref.slice(-6))).sort((a, b) => a - b); + if (JSON.stringify(priorReservations) !== JSON.stringify(expectedPriorReservations)) throw new Error("Stable reservation history is not the exact N-1 prefix."); + let reservation = reservationRefs.find((ref) => ref.ref === `refs/tags/${reservationTag}`); + const requireReservation = async () => { + if (!reservation || reservation.object.type !== "tag") throw new Error("Stable reservation is not one annotated tag."); + const annotation = (await github.rest.git.getTag({ owner, repo, tag_sha: reservation.object.sha })).data; + if ( + annotation.tag !== reservationTag || annotation.message !== reservationMessage || + annotation.object.type !== "commit" || annotation.object.sha !== manifest.promotion.policyCommit + ) throw new Error("Stable reservation belongs to a different exact transaction."); + return annotation; + }; + let reservationSha; + if (process.env.RESERVATION_PRESENT === "true") { + if (mode !== "resume") throw new Error("Only explicit recovery may consume an existing reservation."); + await requireReservation(); + reservationSha = reservation.object.sha; + } else { + if (reservation) throw new Error("A draft-only transaction found an unexpected sequence reservation; explicit reservation recovery is required."); + const annotated = (await github.rest.git.createTag({ + owner, repo, tag: reservationTag, message: reservationMessage, + object: manifest.promotion.policyCommit, type: "commit", + tagger: { + name: "github-actions[bot]", email: "41898282+github-actions[bot]@users.noreply.github.com", + date: new Date().toISOString(), + }, + })).data; + if (!/^[0-9a-f]{40}$/.test(annotated.sha ?? "")) throw new Error("Stable reservation annotation lacks an exact object id."); + reservationSha = annotated.sha; + } + core.setOutput("create_reservation", process.env.RESERVATION_PRESENT === "true" ? "false" : "true"); + core.setOutput("reservation_tag", reservationTag); + core.setOutput("reservation_sha", reservationSha); + core.setOutput("create_stable_tag", stableRef ? "false" : "true"); + core.setOutput("stable_tag", manifest.tag); + core.setOutput("stable_sha", manifest.promotion.policyCommit); + core.setOutput("draft_id", String(draft.id)); + + - name: Mint repository-scoped ruleset auditor token + id: ruleset-auditor + uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 + with: + app-id: ${{ vars.PYLON_RULESET_AUDITOR_APP_ID }} + private-key: ${{ secrets.PYLON_RULESET_AUDITOR_PRIVATE_KEY }} + owner: pylon-code + repositories: prime-agent + permission-administration: read + + - name: Require live pylon before the reservation ruleset audit + if: steps.transaction.outputs.create_reservation == 'true' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const pylon = await github.rest.git.getRef({ ...context.repo, ref: "heads/pylon" }); + if (pylon.data.object.type !== "commit" || pylon.data.object.sha !== context.sha) { + throw new Error("Stable transaction became stale before its reservation ruleset audit."); + } + + - name: Require authoritative publication tag ruleset before reservation CAS + if: steps.transaction.outputs.create_reservation == 'true' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + github-token: ${{ steps.ruleset-auditor.outputs.token }} + script: | + const response = await github.request("GET /repos/{owner}/{repo}/rulesets/{ruleset_id}", { + owner: "pylon-code", repo: "prime-agent", ruleset_id: 21950766, includes_parents: false, + headers: { accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" }, + }); + const restRuleset = response?.data; + const exactKeys = (value, keys) => value !== null && typeof value === "object" && !Array.isArray(value) && + JSON.stringify(Object.keys(value).sort()) === JSON.stringify([...keys].sort()); + const exactSortedStrings = (value, expected) => Array.isArray(value) && value.every((entry) => typeof entry === "string") && + JSON.stringify([...value].sort()) === JSON.stringify([...expected].sort()); + const expectedIncludes = ["refs/tags/pylon-build-*", "refs/tags/pylon-stable-*"]; + const restConditions = restRuleset?.conditions; + const restRefName = restConditions?.ref_name; + const restRules = restRuleset?.rules; + const restUpdateRules = Array.isArray(restRules) ? restRules.filter((rule) => rule?.type === "update") : []; + const restDeletionRules = Array.isArray(restRules) ? restRules.filter((rule) => rule?.type === "deletion") : []; + if ( + response?.status !== 200 || !restRuleset || restRuleset.id !== 21950766 || + restRuleset.node_id !== "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4" || restRuleset.name !== "Pylon immutable publication tags" || + restRuleset.source_type !== "Repository" || restRuleset.source !== "pylon-code/prime-agent" || + restRuleset.target !== "tag" || restRuleset.enforcement !== "active" || + Object.hasOwn(restRuleset, "bypass_actors") && (!Array.isArray(restRuleset.bypass_actors) || restRuleset.bypass_actors.length !== 0) || + Object.hasOwn(restRuleset, "current_user_can_bypass") && restRuleset.current_user_can_bypass !== "never" || + !exactKeys(restConditions, ["ref_name"]) || !exactKeys(restRefName, ["exclude", "include"]) || + !exactSortedStrings(restRefName.exclude, []) || !exactSortedStrings(restRefName.include, expectedIncludes) || + !Array.isArray(restRules) || restRules.length !== 2 || restUpdateRules.length !== 1 || restDeletionRules.length !== 1 || + !exactKeys(restUpdateRules[0], ["parameters", "type"]) || + !exactKeys(restUpdateRules[0]?.parameters, ["update_allows_fetch_and_merge"]) || + restUpdateRules[0]?.parameters?.update_allows_fetch_and_merge !== false || !exactKeys(restDeletionRules[0], ["type"]) + ) throw new Error("REST ruleset-auditor response differs from the exact active immutable tag ruleset."); + const query = `query PylonPublicationRulesetAudit($owner: String!, $repo: String!, $rulesetDatabaseId: Int!) { + repository(owner: $owner, name: $repo) { + id + databaseId + nameWithOwner + ruleset(databaseId: $rulesetDatabaseId, includeParents: false) { + id + databaseId + name + enforcement + target + source { + __typename + ... on Repository { + id + databaseId + nameWithOwner + } + } + bypassActors { totalCount } + conditions { + refName { include exclude } + organizationProperty { __typename } + repositoryId { __typename } + repositoryName { __typename } + repositoryProperty { __typename } + } + rules(first: 100) { + totalCount + nodes { + type + parameters { + __typename + ... on UpdateParameters { updateAllowsFetchAndMerge } + } + } + } + } + } + }`; + const authoritative = await github.graphql(query, { + owner: "pylon-code", repo: "prime-agent", rulesetDatabaseId: 21950766, + }); + const repository = authoritative?.repository; + const ruleset = repository?.ruleset; + const source = ruleset?.source; + const bypassActors = ruleset?.bypassActors; + const conditions = ruleset?.conditions; + const refName = conditions?.refName; + const rules = ruleset?.rules; + const nodes = rules?.nodes; + const updateRules = Array.isArray(nodes) ? nodes.filter((rule) => rule?.type === "UPDATE") : []; + const deletionRules = Array.isArray(nodes) ? nodes.filter((rule) => rule?.type === "DELETION") : []; + if ( + Object.hasOwn(authoritative ?? {}, "errors") || !repository || repository.id !== "R_kgDOUGgkLQ" || + repository.databaseId !== 1349002285 || repository.nameWithOwner !== "pylon-code/prime-agent" || + !ruleset || ruleset.id !== restRuleset.node_id || ruleset.id !== "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4" || + ruleset.databaseId !== 21950766 || ruleset.name !== "Pylon immutable publication tags" || + ruleset.enforcement !== "ACTIVE" || ruleset.target !== "TAG" || + !exactKeys(source, ["__typename", "databaseId", "id", "nameWithOwner"]) || source.__typename !== "Repository" || + source.id !== repository.id || source.databaseId !== repository.databaseId || source.nameWithOwner !== repository.nameWithOwner || + !exactKeys(bypassActors, ["totalCount"]) || !Number.isInteger(bypassActors.totalCount) || bypassActors.totalCount !== 0 || + !exactKeys(conditions, ["organizationProperty", "refName", "repositoryId", "repositoryName", "repositoryProperty"]) || + conditions.organizationProperty !== null || conditions.repositoryId !== null || conditions.repositoryName !== null || + conditions.repositoryProperty !== null || !exactKeys(refName, ["exclude", "include"]) || + !exactSortedStrings(refName.exclude, []) || !exactSortedStrings(refName.include, expectedIncludes) || + !exactKeys(rules, ["nodes", "totalCount"]) || !Number.isInteger(rules.totalCount) || rules.totalCount !== 2 || + !Array.isArray(nodes) || nodes.length !== 2 || nodes.some((node) => node === null) || + updateRules.length !== 1 || deletionRules.length !== 1 || !exactKeys(updateRules[0], ["parameters", "type"]) || + !exactKeys(updateRules[0].parameters, ["__typename", "updateAllowsFetchAndMerge"]) || + updateRules[0].parameters.__typename !== "UpdateParameters" || updateRules[0].parameters.updateAllowsFetchAndMerge !== false || + !exactKeys(deletionRules[0], ["parameters", "type"]) || deletionRules[0].parameters !== null + ) throw new Error("GraphQL ruleset-auditor response is null, partial, redacted, or differs from the exact non-bypassable target contract."); + - name: Create the exact protected stable reservation ref + if: steps.transaction.outputs.create_reservation == 'true' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + RESERVATION_TAG: ${{ steps.transaction.outputs.reservation_tag }} + RESERVATION_SHA: ${{ steps.transaction.outputs.reservation_sha }} + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const tag = process.env.RESERVATION_TAG; + const sha = process.env.RESERVATION_SHA; + if (`${owner}/${repo}` !== "pylon-code/prime-agent" || !/^pylon-stable-sequence-[0-9]{6}$/.test(tag ?? "") || !/^[0-9a-f]{40}$/.test(sha ?? "")) { + throw new Error("Stable reservation CAS identity is malformed."); + } + try { + await github.rest.git.createRef({ owner, repo, ref: `refs/tags/${tag}`, sha }); + } catch (error) { + if (error.status === 422) { + try { await github.rest.git.getRef({ owner, repo, ref: `tags/${tag}` }); } catch {} + throw new Error("Stable sequence reservation raced (422); refetched state and stopped without N+1, move, or delete."); + } + throw error; + } + + - name: Require the exact reservation and live pylon before final-tag ruleset audit + if: steps.transaction.outputs.create_stable_tag == 'true' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + RESERVATION_TAG: ${{ steps.transaction.outputs.reservation_tag }} + RESERVATION_SHA: ${{ steps.transaction.outputs.reservation_sha }} + with: + script: | + const reservation = await github.rest.git.getRef({ ...context.repo, ref: `tags/${process.env.RESERVATION_TAG}` }); + if (reservation.data.object.type !== "tag" || reservation.data.object.sha !== process.env.RESERVATION_SHA) { + throw new Error("Stable reservation changed before final-tag ruleset audit."); + } + const pylon = await github.rest.git.getRef({ ...context.repo, ref: "heads/pylon" }); + if (pylon.data.object.type !== "commit" || pylon.data.object.sha !== context.sha) { + throw new Error("Stable transaction became stale before final-tag ruleset audit."); + } + + - name: Require authoritative publication tag ruleset before final stable tag CAS + if: steps.transaction.outputs.create_stable_tag == 'true' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + github-token: ${{ steps.ruleset-auditor.outputs.token }} + script: | + const response = await github.request("GET /repos/{owner}/{repo}/rulesets/{ruleset_id}", { + owner: "pylon-code", repo: "prime-agent", ruleset_id: 21950766, includes_parents: false, + headers: { accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" }, + }); + const restRuleset = response?.data; + const exactKeys = (value, keys) => value !== null && typeof value === "object" && !Array.isArray(value) && + JSON.stringify(Object.keys(value).sort()) === JSON.stringify([...keys].sort()); + const exactSortedStrings = (value, expected) => Array.isArray(value) && value.every((entry) => typeof entry === "string") && + JSON.stringify([...value].sort()) === JSON.stringify([...expected].sort()); + const expectedIncludes = ["refs/tags/pylon-build-*", "refs/tags/pylon-stable-*"]; + const restConditions = restRuleset?.conditions; + const restRefName = restConditions?.ref_name; + const restRules = restRuleset?.rules; + const restUpdateRules = Array.isArray(restRules) ? restRules.filter((rule) => rule?.type === "update") : []; + const restDeletionRules = Array.isArray(restRules) ? restRules.filter((rule) => rule?.type === "deletion") : []; + if ( + response?.status !== 200 || !restRuleset || restRuleset.id !== 21950766 || + restRuleset.node_id !== "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4" || restRuleset.name !== "Pylon immutable publication tags" || + restRuleset.source_type !== "Repository" || restRuleset.source !== "pylon-code/prime-agent" || + restRuleset.target !== "tag" || restRuleset.enforcement !== "active" || + Object.hasOwn(restRuleset, "bypass_actors") && (!Array.isArray(restRuleset.bypass_actors) || restRuleset.bypass_actors.length !== 0) || + Object.hasOwn(restRuleset, "current_user_can_bypass") && restRuleset.current_user_can_bypass !== "never" || + !exactKeys(restConditions, ["ref_name"]) || !exactKeys(restRefName, ["exclude", "include"]) || + !exactSortedStrings(restRefName.exclude, []) || !exactSortedStrings(restRefName.include, expectedIncludes) || + !Array.isArray(restRules) || restRules.length !== 2 || restUpdateRules.length !== 1 || restDeletionRules.length !== 1 || + !exactKeys(restUpdateRules[0], ["parameters", "type"]) || + !exactKeys(restUpdateRules[0]?.parameters, ["update_allows_fetch_and_merge"]) || + restUpdateRules[0]?.parameters?.update_allows_fetch_and_merge !== false || !exactKeys(restDeletionRules[0], ["type"]) + ) throw new Error("REST ruleset-auditor response differs from the exact active immutable tag ruleset."); + const query = `query PylonPublicationRulesetAudit($owner: String!, $repo: String!, $rulesetDatabaseId: Int!) { + repository(owner: $owner, name: $repo) { + id + databaseId + nameWithOwner + ruleset(databaseId: $rulesetDatabaseId, includeParents: false) { + id + databaseId + name + enforcement + target + source { + __typename + ... on Repository { + id + databaseId + nameWithOwner + } + } + bypassActors { totalCount } + conditions { + refName { include exclude } + organizationProperty { __typename } + repositoryId { __typename } + repositoryName { __typename } + repositoryProperty { __typename } + } + rules(first: 100) { + totalCount + nodes { + type + parameters { + __typename + ... on UpdateParameters { updateAllowsFetchAndMerge } + } + } + } + } + } + }`; + const authoritative = await github.graphql(query, { + owner: "pylon-code", repo: "prime-agent", rulesetDatabaseId: 21950766, + }); + const repository = authoritative?.repository; + const ruleset = repository?.ruleset; + const source = ruleset?.source; + const bypassActors = ruleset?.bypassActors; + const conditions = ruleset?.conditions; + const refName = conditions?.refName; + const rules = ruleset?.rules; + const nodes = rules?.nodes; + const updateRules = Array.isArray(nodes) ? nodes.filter((rule) => rule?.type === "UPDATE") : []; + const deletionRules = Array.isArray(nodes) ? nodes.filter((rule) => rule?.type === "DELETION") : []; + if ( + Object.hasOwn(authoritative ?? {}, "errors") || !repository || repository.id !== "R_kgDOUGgkLQ" || + repository.databaseId !== 1349002285 || repository.nameWithOwner !== "pylon-code/prime-agent" || + !ruleset || ruleset.id !== restRuleset.node_id || ruleset.id !== "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4" || + ruleset.databaseId !== 21950766 || ruleset.name !== "Pylon immutable publication tags" || + ruleset.enforcement !== "ACTIVE" || ruleset.target !== "TAG" || + !exactKeys(source, ["__typename", "databaseId", "id", "nameWithOwner"]) || source.__typename !== "Repository" || + source.id !== repository.id || source.databaseId !== repository.databaseId || source.nameWithOwner !== repository.nameWithOwner || + !exactKeys(bypassActors, ["totalCount"]) || !Number.isInteger(bypassActors.totalCount) || bypassActors.totalCount !== 0 || + !exactKeys(conditions, ["organizationProperty", "refName", "repositoryId", "repositoryName", "repositoryProperty"]) || + conditions.organizationProperty !== null || conditions.repositoryId !== null || conditions.repositoryName !== null || + conditions.repositoryProperty !== null || !exactKeys(refName, ["exclude", "include"]) || + !exactSortedStrings(refName.exclude, []) || !exactSortedStrings(refName.include, expectedIncludes) || + !exactKeys(rules, ["nodes", "totalCount"]) || !Number.isInteger(rules.totalCount) || rules.totalCount !== 2 || + !Array.isArray(nodes) || nodes.length !== 2 || nodes.some((node) => node === null) || + updateRules.length !== 1 || deletionRules.length !== 1 || !exactKeys(updateRules[0], ["parameters", "type"]) || + !exactKeys(updateRules[0].parameters, ["__typename", "updateAllowsFetchAndMerge"]) || + updateRules[0].parameters.__typename !== "UpdateParameters" || updateRules[0].parameters.updateAllowsFetchAndMerge !== false || + !exactKeys(deletionRules[0], ["parameters", "type"]) || deletionRules[0].parameters !== null + ) throw new Error("GraphQL ruleset-auditor response is null, partial, redacted, or differs from the exact non-bypassable target contract."); + - name: Create or refetch the exact protected stable tag + if: steps.transaction.outputs.create_stable_tag == 'true' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + STABLE_TAG: ${{ steps.transaction.outputs.stable_tag }} + STABLE_SHA: ${{ steps.transaction.outputs.stable_sha }} + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const tag = process.env.STABLE_TAG; + const sha = process.env.STABLE_SHA; + if (`${owner}/${repo}` !== "pylon-code/prime-agent" || !/^pylon-stable-[0-9]{6}-g[0-9a-f]{12}-r[1-9][0-9]*$/.test(tag ?? "") || !/^[0-9a-f]{40}$/.test(sha ?? "")) { + throw new Error("Stable tag CAS identity is malformed."); + } + try { + await github.rest.git.createRef({ owner, repo, ref: `refs/tags/${tag}`, sha }); + } catch (error) { + if (error.status !== 422) throw error; + } + const stable = await github.rest.git.getRef({ owner, repo, ref: `tags/${tag}` }); + if (stable.data.object.type !== "commit" || stable.data.object.sha !== sha) { + throw new Error("Stable publication tag is annotated or targets a different commit."); + } + + - name: Require exact refs, draft, and live pylon before final publish ruleset audit + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + DRAFT_ID: ${{ steps.transaction.outputs.draft_id }} + EXPECTED_MANIFEST_SHA256: ${{ needs.prepare.outputs.manifest_sha256 }} + RESERVATION_TAG: ${{ steps.transaction.outputs.reservation_tag }} + RESERVATION_SHA: ${{ steps.transaction.outputs.reservation_sha }} + STABLE_TAG: ${{ steps.transaction.outputs.stable_tag }} + STABLE_SHA: ${{ steps.transaction.outputs.stable_sha }} + with: + script: | + const crypto = require("node:crypto"); + const pylon = await github.rest.git.getRef({ ...context.repo, ref: "heads/pylon" }); + if (pylon.data.object.type !== "commit" || pylon.data.object.sha !== context.sha) { + throw new Error("Stable transaction became stale before final publish ruleset audit."); + } + const reservation = await github.rest.git.getRef({ ...context.repo, ref: `tags/${process.env.RESERVATION_TAG}` }); + if (reservation.data.object.type !== "tag" || reservation.data.object.sha !== process.env.RESERVATION_SHA) { + throw new Error("Stable reservation changed before final publish ruleset audit."); + } + const stable = await github.rest.git.getRef({ ...context.repo, ref: `tags/${process.env.STABLE_TAG}` }); + if (stable.data.object.type !== "commit" || stable.data.object.sha !== process.env.STABLE_SHA) { + throw new Error("Stable tag changed before final publish ruleset audit."); + } + const draft = (await github.rest.repos.getRelease({ ...context.repo, release_id: Number(process.env.DRAFT_ID) })).data; + const encoded = /^Manifest base64: ([A-Za-z0-9+/]+={0,2})$/m.exec(draft.body ?? "")?.[1]; + const encodedSize = /^Manifest bytes: ([1-9][0-9]*)$/m.exec(draft.body ?? "")?.[1]; + const encodedDigest = /^Manifest sha256: ([0-9a-f]{64})$/m.exec(draft.body ?? "")?.[1]; + const bytes = encoded ? Buffer.from(encoded, "base64") : Buffer.alloc(0); + const digest = crypto.createHash("sha256").update(bytes).digest("hex"); + const asset = draft.assets?.[0]; + if ( + !draft.draft || draft.immutable === true || draft.tag_name !== process.env.STABLE_TAG || + draft.name !== `Pylon Prime stable ${process.env.STABLE_TAG}` || draft.prerelease !== false || + draft.target_commitish !== process.env.STABLE_SHA || bytes.toString("base64") !== encoded || + bytes.length !== Number(encodedSize) || digest !== encodedDigest || digest !== process.env.EXPECTED_MANIFEST_SHA256 || + draft.assets?.length !== 1 || asset.name !== "pylon-stable-channel-v1.json" || + asset.size !== bytes.length || asset.digest !== `sha256:${digest}` + ) throw new Error("Stable draft changed before final publish ruleset audit."); + + - name: Require authoritative publication tag ruleset before immutable stable publish + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + github-token: ${{ steps.ruleset-auditor.outputs.token }} + script: | + const response = await github.request("GET /repos/{owner}/{repo}/rulesets/{ruleset_id}", { + owner: "pylon-code", repo: "prime-agent", ruleset_id: 21950766, includes_parents: false, + headers: { accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" }, + }); + const restRuleset = response?.data; + const exactKeys = (value, keys) => value !== null && typeof value === "object" && !Array.isArray(value) && + JSON.stringify(Object.keys(value).sort()) === JSON.stringify([...keys].sort()); + const exactSortedStrings = (value, expected) => Array.isArray(value) && value.every((entry) => typeof entry === "string") && + JSON.stringify([...value].sort()) === JSON.stringify([...expected].sort()); + const expectedIncludes = ["refs/tags/pylon-build-*", "refs/tags/pylon-stable-*"]; + const restConditions = restRuleset?.conditions; + const restRefName = restConditions?.ref_name; + const restRules = restRuleset?.rules; + const restUpdateRules = Array.isArray(restRules) ? restRules.filter((rule) => rule?.type === "update") : []; + const restDeletionRules = Array.isArray(restRules) ? restRules.filter((rule) => rule?.type === "deletion") : []; + if ( + response?.status !== 200 || !restRuleset || restRuleset.id !== 21950766 || + restRuleset.node_id !== "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4" || restRuleset.name !== "Pylon immutable publication tags" || + restRuleset.source_type !== "Repository" || restRuleset.source !== "pylon-code/prime-agent" || + restRuleset.target !== "tag" || restRuleset.enforcement !== "active" || + Object.hasOwn(restRuleset, "bypass_actors") && (!Array.isArray(restRuleset.bypass_actors) || restRuleset.bypass_actors.length !== 0) || + Object.hasOwn(restRuleset, "current_user_can_bypass") && restRuleset.current_user_can_bypass !== "never" || + !exactKeys(restConditions, ["ref_name"]) || !exactKeys(restRefName, ["exclude", "include"]) || + !exactSortedStrings(restRefName.exclude, []) || !exactSortedStrings(restRefName.include, expectedIncludes) || + !Array.isArray(restRules) || restRules.length !== 2 || restUpdateRules.length !== 1 || restDeletionRules.length !== 1 || + !exactKeys(restUpdateRules[0], ["parameters", "type"]) || + !exactKeys(restUpdateRules[0]?.parameters, ["update_allows_fetch_and_merge"]) || + restUpdateRules[0]?.parameters?.update_allows_fetch_and_merge !== false || !exactKeys(restDeletionRules[0], ["type"]) + ) throw new Error("REST ruleset-auditor response differs from the exact active immutable tag ruleset."); + const query = `query PylonPublicationRulesetAudit($owner: String!, $repo: String!, $rulesetDatabaseId: Int!) { + repository(owner: $owner, name: $repo) { + id + databaseId + nameWithOwner + ruleset(databaseId: $rulesetDatabaseId, includeParents: false) { + id + databaseId + name + enforcement + target + source { + __typename + ... on Repository { + id + databaseId + nameWithOwner + } + } + bypassActors { totalCount } + conditions { + refName { include exclude } + organizationProperty { __typename } + repositoryId { __typename } + repositoryName { __typename } + repositoryProperty { __typename } + } + rules(first: 100) { + totalCount + nodes { + type + parameters { + __typename + ... on UpdateParameters { updateAllowsFetchAndMerge } + } + } + } + } + } + }`; + const authoritative = await github.graphql(query, { + owner: "pylon-code", repo: "prime-agent", rulesetDatabaseId: 21950766, + }); + const repository = authoritative?.repository; + const ruleset = repository?.ruleset; + const source = ruleset?.source; + const bypassActors = ruleset?.bypassActors; + const conditions = ruleset?.conditions; + const refName = conditions?.refName; + const rules = ruleset?.rules; + const nodes = rules?.nodes; + const updateRules = Array.isArray(nodes) ? nodes.filter((rule) => rule?.type === "UPDATE") : []; + const deletionRules = Array.isArray(nodes) ? nodes.filter((rule) => rule?.type === "DELETION") : []; + if ( + Object.hasOwn(authoritative ?? {}, "errors") || !repository || repository.id !== "R_kgDOUGgkLQ" || + repository.databaseId !== 1349002285 || repository.nameWithOwner !== "pylon-code/prime-agent" || + !ruleset || ruleset.id !== restRuleset.node_id || ruleset.id !== "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4" || + ruleset.databaseId !== 21950766 || ruleset.name !== "Pylon immutable publication tags" || + ruleset.enforcement !== "ACTIVE" || ruleset.target !== "TAG" || + !exactKeys(source, ["__typename", "databaseId", "id", "nameWithOwner"]) || source.__typename !== "Repository" || + source.id !== repository.id || source.databaseId !== repository.databaseId || source.nameWithOwner !== repository.nameWithOwner || + !exactKeys(bypassActors, ["totalCount"]) || !Number.isInteger(bypassActors.totalCount) || bypassActors.totalCount !== 0 || + !exactKeys(conditions, ["organizationProperty", "refName", "repositoryId", "repositoryName", "repositoryProperty"]) || + conditions.organizationProperty !== null || conditions.repositoryId !== null || conditions.repositoryName !== null || + conditions.repositoryProperty !== null || !exactKeys(refName, ["exclude", "include"]) || + !exactSortedStrings(refName.exclude, []) || !exactSortedStrings(refName.include, expectedIncludes) || + !exactKeys(rules, ["nodes", "totalCount"]) || !Number.isInteger(rules.totalCount) || rules.totalCount !== 2 || + !Array.isArray(nodes) || nodes.length !== 2 || nodes.some((node) => node === null) || + updateRules.length !== 1 || deletionRules.length !== 1 || !exactKeys(updateRules[0], ["parameters", "type"]) || + !exactKeys(updateRules[0].parameters, ["__typename", "updateAllowsFetchAndMerge"]) || + updateRules[0].parameters.__typename !== "UpdateParameters" || updateRules[0].parameters.updateAllowsFetchAndMerge !== false || + !exactKeys(deletionRules[0], ["parameters", "type"]) || deletionRules[0].parameters !== null + ) throw new Error("GraphQL ruleset-auditor response is null, partial, redacted, or differs from the exact non-bypassable target contract."); + - name: Publish only the exact protected stable draft + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + DRAFT_ID: ${{ steps.transaction.outputs.draft_id }} + RESERVATION_TAG: ${{ steps.transaction.outputs.reservation_tag }} + RESERVATION_SHA: ${{ steps.transaction.outputs.reservation_sha }} + STABLE_TAG: ${{ steps.transaction.outputs.stable_tag }} + STABLE_SHA: ${{ steps.transaction.outputs.stable_sha }} + with: + script: | + const crypto = require("node:crypto"); + const owner = context.repo.owner; + const repo = context.repo.repo; + const draftId = Number(process.env.DRAFT_ID); + if (`${owner}/${repo}` !== "pylon-code/prime-agent" || !Number.isSafeInteger(draftId) || draftId < 1) { + throw new Error("Stable immutable publication identity is malformed."); + } + await github.rest.repos.updateRelease({ owner, repo, release_id: draftId, draft: false }); + const immutable = (await github.rest.repos.getRelease({ owner, repo, release_id: draftId })).data; + const encoded = /^Manifest base64: ([A-Za-z0-9+/]+={0,2})$/m.exec(immutable.body ?? "")?.[1]; + const encodedSize = /^Manifest bytes: ([1-9][0-9]*)$/m.exec(immutable.body ?? "")?.[1]; + const encodedDigest = /^Manifest sha256: ([0-9a-f]{64})$/m.exec(immutable.body ?? "")?.[1]; + const bytes = encoded ? Buffer.from(encoded, "base64") : Buffer.alloc(0); + const digest = crypto.createHash("sha256").update(bytes).digest("hex"); + if (bytes.toString("base64") !== encoded || bytes.length !== Number(encodedSize) || digest !== encodedDigest) { + throw new Error("Published stable recovery envelope differs from its exact manifest bytes."); + } + const manifest = JSON.parse(bytes); + const name = `Pylon Prime stable ${manifest.tag}`; + if ( + immutable.immutable !== true || immutable.draft || immutable.tag_name !== process.env.STABLE_TAG || immutable.name !== name || + immutable.prerelease !== false || immutable.target_commitish !== process.env.STABLE_SHA || + manifest.tag !== process.env.STABLE_TAG || manifest.promotion?.policyCommit !== process.env.STABLE_SHA || + immutable.assets?.length !== 1 || immutable.assets[0].name !== "pylon-stable-channel-v1.json" || + immutable.assets[0].size !== bytes.length || immutable.assets[0].digest !== `sha256:${digest}` + ) throw new Error("Stable release immutable postconditions differ from the reserved transaction."); + const stable = await github.rest.git.getRef({ owner, repo, ref: `tags/${process.env.STABLE_TAG}` }); + if (stable.data.object.type !== "commit" || stable.data.object.sha !== process.env.STABLE_SHA) { + throw new Error("Published stable tag differs from the exact policy commit."); + } + const reservation = await github.rest.git.getRef({ owner, repo, ref: `tags/${process.env.RESERVATION_TAG}` }); + if (reservation.data.object.type !== "tag" || reservation.data.object.sha !== process.env.RESERVATION_SHA) { + throw new Error("Published stable reservation differs from its exact annotation."); + } + const releases = (await github.paginate(github.rest.repos.listReleases, { owner, repo, per_page: 100 })) + .filter((release) => /^pylon-stable-[0-9]{6}-g/.test(release.tag_name ?? "")); + if (releases.filter((release) => release.tag_name === manifest.tag).length !== 1) { + throw new Error("Stable publication did not retain one globally unique sequence."); + } diff --git a/.github/workflows/pylon-upstream-sync.yml b/.github/workflows/pylon-upstream-sync.yml index 5a1228555c..7ff7f40745 100644 --- a/.github/workflows/pylon-upstream-sync.yml +++ b/.github/workflows/pylon-upstream-sync.yml @@ -9,11 +9,7 @@ concurrency: group: pylon-upstream-sync cancel-in-progress: false -permissions: - actions: read - contents: write - issues: write - pull-requests: read +permissions: {} jobs: sync: @@ -21,6 +17,12 @@ jobs: if: github.repository == 'pylon-code/prime-agent' && github.ref == 'refs/heads/pylon' runs-on: ubuntu-latest timeout-minutes: 10 + environment: pylon-upstream-sync + permissions: + actions: read + contents: write + issues: write + pull-requests: read outputs: candidate_sha: ${{ steps.sync.outputs.candidate_sha }} pr_url: ${{ steps.sync.outputs.pr_url }} @@ -28,14 +30,24 @@ jobs: - name: Checkout Pylon product branch uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - ref: pylon + ref: ${{ github.sha }} fetch-depth: 0 - - name: Mirror Prime and prepare a review candidate + - name: Prove the approved exact live revision, then mirror Prime id: sync env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: scripts/pylon-sync-upstream.sh + run: | + set -euo pipefail + test "$GITHUB_REPOSITORY" = "pylon-code/prime-agent" + test "$GITHUB_EVENT_NAME" = "${{ github.event_name }}" + case "$GITHUB_EVENT_NAME" in schedule|workflow_dispatch) ;; *) exit 1 ;; esac + test "$GITHUB_REF" = "refs/heads/pylon" + test "$GITHUB_SHA" = "${{ github.sha }}" + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + test "$(git rev-parse --show-toplevel)" = "$GITHUB_WORKSPACE" + test "$(gh api repos/pylon-code/prime-agent/git/ref/heads/pylon --jq .object.sha)" = "$GITHUB_SHA" + exec scripts/pylon-sync-upstream.sh verify: name: Verify candidate ${{ needs.sync.outputs.candidate_sha }} with trusted fork CI diff --git a/.pylon/features.yaml b/.pylon/features.yaml index a3803a4767..13c6904b7a 100644 --- a/.pylon/features.yaml +++ b/.pylon/features.yaml @@ -384,3 +384,18 @@ decisions: revisit_when: - Pylon no longer maintains a Prime distribution or the upstream artifact recipe can emit the exact fork-owned deterministic contract without publication side effects. - The source, recipe, toolchain, manifest, internal URL, and digest invariants can be preserved without fork-specific release code. + + protected-pylon-publication: + area: distribution + state: candidate + owner: pylon-fork + decision: retain + pylon_refs: + - https://github.com/pylon-code/prime-agent/issues/29 + upstream_refs: + - https://github.com/PrimeIntellect-ai/prime-agent/pull/32 + fork_change: protected-preview-and-append-only-stable-publication-v1 + upstream_support: Prime's inherited publication path targets upstream R2 and npm channels and does not provide canonical Pylon-only exact-SHA admission, six immutable preview subjects, byte-preserving manual promotion, signed monotonic stable history, or append-only withdrawal. + revisit_when: + - Prime ships a repository-neutral immutable release and promotion primitive that preserves Pylon's exact source, workflow, attestation, stable-history, and withdrawal policy without upstream credentials or channel names. + - Pylon adopts another verifiable distribution transport that fully supersedes GitHub immutable Releases. diff --git a/.pylon/release-artifacts.md b/.pylon/release-artifacts.md index 33835b918d..69e33582ee 100644 --- a/.pylon/release-artifacts.md +++ b/.pylon/release-artifacts.md @@ -1,6 +1,6 @@ # Pylon Prime release artifacts -Pylon builds Prime Agent from the protected `pylon` branch without changing the installed package or command identity. This document defines the deterministic candidate artifact boundary. Publication, attestation, Pylon-side verification, and installation are separate changes. +Pylon builds Prime Agent from the protected `pylon` branch without changing the installed package or command identity. This document defines the deterministic candidate artifact boundary. Protected GitHub publication and attestation are defined in `docs/pylon-publication.md`; Pylon-side receipt verification and managed installation remain separate changes. ## Frozen recipe @@ -45,7 +45,7 @@ No channel, timestamp, actor, workflow run, branch, mutable URL, or feature flag Every internal dependency must resolve by matching package name to its exact URL below the immutable build release. Missing semver rewrites and cross-wired archives fail packing and verification. Archive SHA-512 values are repeated in `npm-shrinkwrap.json`. These shrinkwrap entries are auditable receipts; npm does not reliably enforce a dependency archive's nested shrinkwrap during installation. The Pylon verifier and installer must enforce manifest and attestation digests before installation. -The release manifest records the exact source, recipe, toolchain, lock digest, minimum Node version, package/command identity, sorted archive names, sizes, SHA-256, SHA-512, and external attestation subjects. It does not contain its own digest because that would be self-referential. +The release manifest records the exact source, recipe, toolchain, lock digest, minimum Node version, package/command identity, sorted archive names, sizes, SHA-256, SHA-512, and external attestation subjects. It does not contain its own digest because that would be self-referential. The separate signed preview-channel manifest adds the independent publication policy revision, epoch-1 workflow `run_number` ordering, and exact run id; the SLSA invocation supplies the rerun attempt without changing canonical preview bytes. Publication workflow edits bump that policy revision and do not require an artifact recipe bump unless the artifact recipe itself changes. ## Local verification @@ -61,7 +61,7 @@ npm run release:pylon:smoke The pack command cleans and rebuilds only package `dist` outputs, compiles committed model data, records their exact receipt, and refuses dirty source inputs. The smoke performs a local-asset consumer install with lifecycle scripts disabled. It imports the public `prime-agent` package specifier, checks the frozen SDK tokens, invokes the installed command through pinned npm, and proves the Pylon build refuses the stock R2 self-updater without invoking even a fenced fake npm command. -Trusted CI performs two isolated Ubuntu packs and compares all five files byte-for-byte. It then repeats temporary-prefix checks on Ubuntu, macOS, and Windows. POSIX additionally requires the manifest build ID in the daemon hello, proves exact post-attach capability negotiation and authoritative owned cleanup, and retains exact worker identities for bounded failure cleanup. Windows explicitly verifies in-process ACP compatibility mode while native named-pipe authentication remains unavailable. +Trusted CI performs two isolated Ubuntu packs and compares all five files byte-for-byte. It then repeats temporary-prefix runtime checks on Ubuntu Linux and macOS; Ubuntu covers the supported WSL2 path. These gates require the manifest build ID in the daemon hello, prove exact post-attach capability negotiation and authoritative owned cleanup, and retain exact worker identities for bounded failure cleanup. Native Windows runtime publication and its ACP fallback are deferred. Cross-platform archive paths and package metadata remain portable, but that portability is not a native Windows support claim. Matching CI packs are evidence for the pinned source, recipe, toolchain, and runner inputs. They are not a timeless or mathematically hermetic reproducibility claim. @@ -71,4 +71,6 @@ Matching CI packs are evidence for the pinned source, recipe, toolchain, and run Issue #28 creates no tag or GitHub Release and needs only `contents: read`. Its artifact jobs receive no repository secrets, and their Actions uploads are short-lived CI transport. They must not use npm publish, R2, `contents: write`, OIDC, or attestations. -Issue #29 owns protected preview publication, keyless attestations, stable promotion, rollback, and yanking. Pylon issues #193 and #194 own signed receipt verification and opt-in side-by-side install/update/rollback/switch-back. Until those land, the artifacts are build candidates, not a managed Pylon installation channel. +Issue #29 adds protected preview publication, six exact keyless attestations, manual byte-preserving stable promotion, and append-only withdrawal. Preview releases contain the four tarballs, this build manifest, and a monotonic workflow-run-bound `pylon-preview-channel-v1.json`; stable releases contain only a signed `pylon-stable-channel-v1.json` sequence record that copies the preview sequence identity and uses permanent N-only reservation refs for global sequence uniqueness. Exact formats, environment gates, promotion/withdrawal operations, and independent verification commands live in `docs/pylon-publication.md`. + +Pylon issues #193 and #194 own signed receipt verification and opt-in side-by-side install/update/rollback/switch-back. Until those land, published artifacts are verifiable release inputs, not a managed Pylon installation channel. diff --git a/.pylon/upstream-review.md b/.pylon/upstream-review.md index a1a295b84e..6c30eb4268 100644 --- a/.pylon/upstream-review.md +++ b/.pylon/upstream-review.md @@ -187,9 +187,20 @@ This ledger records Prime upstream evidence and the decision taken for each over - Pylon base: `pylon@70b65305c20a089278dfd6282b826d37648328df`, after merged snapshot catch-up PR #19. Upstream evidence remains fully audited through `PrimeIntellect-ai/prime-agent@a903d4b6768f484bd6d459b7b0aa7dee38e461e2`; this distribution-only candidate does not advance `reviewed_upstream_commit`. - Reviewed the inherited R2 packer, root/workspace build scripts, live model generator, bundle build identity, package metadata, lockfile, self-update feed, protected CI, and the deliberately removed upstream publication workflow. Restoring upstream publication would target the wrong branch, identities, channels, credentials, and trust boundary. - `deterministic-pylon-release-artifacts`: **redesign** as a fork-owned, channel-neutral recipe. Pin one source commit/tree, recipe revision, Node/npm pair, complete committed lock, immutable GitHub build-release URL pattern, four exact tarball names, and one exact manifest. Keep the installed root package and command named `prime-agent`; preserve scoped internal package names and rewrite only artifact names, repository provenance, internal URLs, and integrity metadata. -- Normal development retains live catalog generation. The release path compiles the committed generated model snapshot under an offline guard and injects an exact recipe-derived bundle identity. Two isolated Ubuntu packs must match byte-for-byte; temporary-prefix import/version/update-policy checks cover Ubuntu, macOS, and Windows. POSIX additionally proves the frozen root SDK tokens and post-attach negotiated capability; Windows remains an explicit ACP smoke because native named-pipe trust is not established. +- Normal development retains live catalog generation. The release path compiles the committed generated model snapshot under an offline guard and injects an exact recipe-derived bundle identity. Two isolated Ubuntu packs must match byte-for-byte; temporary-prefix import/version/update-policy checks cover Ubuntu Linux and macOS, with Ubuntu covering WSL2. Those gates also prove the frozen root SDK tokens and post-attach negotiated capability. Native Windows runtime support and its ACP fallback are deferred; generic archive portability is retained. - `pylonDistribution` is immutable local build metadata only. It disables the stock R2 updater and points explicit update attempts to the Pylon release host, but it never enables daemon behavior and is not signed-distribution proof. Issues #29, pylon#193, and pylon#194 separately own publication, attestations, verification, and managed install/update/rollback. - Published npm shrinkwrap entries are auditable graph receipts, not an install-time security boundary: npm does not reliably enforce a dependency tarball's nested shrinkwrap. Exact internal package URLs bind the graph to immutable build assets, and the later Pylon verifier/installer must enforce manifest and attestation digests before installation. - Publication is excluded. This candidate must not create tags/releases, upload to R2/npm, advance preview/stable channels, request write/OIDC/attestation permissions, or claim unsigned metadata is authenticated. - The pre-ledger source candidate `ebeae345614aafd74e0f4d57270d7282cc9d1499` passed a fresh pinned npm 11.10.1 install; eight release-contract tests; 46 config/version/package-update tests; `npm run check`; exact source/lock/manifest verification; two clean pinned macOS packs with all five files byte-identical; public package-specifier import; installed-command version; fenced updater refusal; exact daemon hello build identity; post-attach negotiation; owned cleanup; forced in-process ACP; and an injected post-create timeout that removed no state until every start-identified process was gone. Negative probes rejected dirty source, ignored copied inputs, stale `dist`, noncanonical SRI, missing internal rewrites, and cross-wired assets. Two independent adversarial reviews drove the source/input, Windows npm, updater, detached-process, and PID-reuse repairs. -- Validation remains candidate-bound. This ledger-only head needs renewed focused checks, and trusted exact-head CI must still prove isolated Ubuntu byte equality plus Ubuntu/macOS/Windows installed checks before merge. Publication and signed provenance remain excluded. +- Validation remains candidate-bound. This ledger-only head needs renewed focused checks, and trusted exact-head CI must still prove isolated Ubuntu byte equality plus Ubuntu Linux/macOS installed checks before merge. Publication and signed provenance remain excluded. + +## 2026-08-31 — protected Pylon publication candidate + +- Pylon base: exact merged artifact commit `pylon@63fb578aace412da02c999e383b7dde8c9a84f3a`. Upstream evidence remains audited through the ledger's recorded Prime commit; this distribution-governance work does not advance `reviewed_upstream_commit`. +- Reviewed Pylon issue #29 and comments, Prime PR #32 and its complete workflow/script surface, protected `pylon` branch checks, repository rules, GitHub immutable-release and attestation interfaces, the deterministic issue #28 recipe, and the deliberately removed inherited R2/npm publication path. +- `protected-pylon-publication`: **retain** a Pylon-owned design. Prime's channel and credential model cannot safely name or govern Pylon releases. Canonical pushes now build one immutable preview identity with epoch-1 workflow-run ordering, attest exactly four tarballs plus the build and preview manifests in the directly approved environment, and publish only after fresh exact-SHA protected checks, canonical signed workflow-run proof, Ubuntu Linux/macOS install gates (Ubuntu covers WSL2), and live-tip revalidation. +- Promotion is manual, serialized, and rebuild-free. It verifies the immutable preview and six exact SLSA/Rekor attestations, installs the same bytes on Ubuntu Linux/macOS (Ubuntu covers WSL2), signs one stable manifest, fully stages/re-hashes a GitHub draft, then uses one permanent N-only annotated ref as CAS before publishing. Explicit fresh-run recovery reuses the old exact draft/attestation after policy advances and never reprepares, reattests, skips, moves, or deletes. Stable tags are contiguous; every manifest binds its high-water sequence, exact prior digest, policy, preview run sequence/digests, and cumulative revocations. +- Withdrawal is a later signed sequence, never deletion or replacement. Exact existing immutable releases are idempotent replays. Changed collisions, partial-draft mismatches, `422` reservation races, stale workflow reruns, wrong repositories/refs/workflows/app ids/SHAs, check-status relabeling, artifact ambiguity/expiry, signer or subject changes, sequence gaps, and revocation removal all fail closed. +- Build/verify, attestation, and publication remain separate privilege domains. Publication writers do not checkout or execute repository/downloaded code. Normal attesters carry the one direct environment approval and OIDC/attestation writes; downstream draft/final jobs alone get contents write. Stable recovery uses a mutually exclusive zero-write direct approval and the old exact attestation. Actions and the reviewed attestation composite chain are full-SHA pinned. `pylon-preview` and `pylon-stable` use exact `pylon` custom-branch policies and explicit solo-maintainer approval. Active no-bypass tag ruleset `21950766` allows creation but prevents update/deletion of `pylon-build-*` and `pylon-stable-*` refs, including N-only sequence reservations. Immutable Releases remains enabled. +- Offline publication tests cover canonical bytes, closed current/historical recipes, rerun-stable preview run sequencing and consumer high-water, exact check/workflow-run proof, wrong signer/source/subject/invocation and missing-Rekor rejection, stable consumer rollback/equivocation, digest-chained history, append-only revocations, exact approval/content-writer graphs, action-chain pins, and publisher no-source-execution. The operator and independent-verification runbook is `docs/pylon-publication.md`. +- Revisit only if Prime provides a repository-neutral immutable publication primitive that fully preserves Pylon's protected-source, provenance, history, and withdrawal guarantees, or if Pylon deliberately replaces GitHub Releases with an equivalent verifiable transport. diff --git a/PYLON.md b/PYLON.md index c678af083a..f4f3c0537f 100644 --- a/PYLON.md +++ b/PYLON.md @@ -45,7 +45,7 @@ Only workflows explicitly reviewed and approved for Pylon may run from the defau Mirroring upstream must not publish packages, binaries, or beta releases. The `pylon` branch intentionally omits the inherited release workflow, and the fork must never expose upstream-known release secrets at repository scope. Do not restore that workflow during upstream merges. -The `pylon` branch owns a separate offline deterministic artifact recipe. It emits channel-neutral Pylon-named tarballs and an immutable source/toolchain/digest manifest while preserving the installed `prime-agent` package and command identity. CI artifacts are unsigned candidates, not releases, signed provenance, or permission to install. Protected GitHub publication and attestation, Pylon-side receipt verification, and managed install/update/rollback remain separate reviewed changes. Never restore upstream R2/npm publication or infer runtime capabilities from fork metadata. +The `pylon` branch owns a separate offline deterministic artifact recipe. It emits channel-neutral Pylon-named tarballs and an immutable source/toolchain/digest manifest while preserving the installed `prime-agent` package and command identity. CI artifacts are unsigned candidates, not releases, signed provenance, or permission to install. Protected publication creates immutable preview releases with exact keyless provenance, and manual promotion creates signed append-only stable-manifest sequences without rebuilding; see `docs/pylon-publication.md`. Pylon-side receipt verification and managed install/update/rollback remain separate reviewed changes. Never restore upstream R2/npm publication or infer runtime capabilities from fork metadata. ## Product integration principles diff --git a/docs/pylon-publication.md b/docs/pylon-publication.md new file mode 100644 index 0000000000..93efd86452 --- /dev/null +++ b/docs/pylon-publication.md @@ -0,0 +1,214 @@ +# Protected Pylon publication + +Pylon publishes Prime Agent in two steps. A protected `pylon` push can create one immutable preview. A maintainer can later promote those exact bytes to the append-only stable channel or publish a later withdrawal. Promotion never rebuilds or executes old repository source. + +## Administrative prerequisites + +Publication fails closed unless all of these controls exist: + +- the canonical repository is `pylon-code/prime-agent`, with immutable GitHub Releases enabled; +- `refs/heads/pylon` requires strict exact-SHA `build-check-test` and `Check changelog fragment` checks from GitHub Actions app `15368`; +- `pylon-preview` and `pylon-stable` use custom deployment branches with only `pylon`, require reviewer `rynfar` (user id `11325514`), set `prevent_self_review: false`, and set `can_admins_bypass: false`; +- `pylon-upstream-sync` has the same sole custom `pylon` branch, reviewer, `prevent_self_review: false`, and `can_admins_bypass: false` policy before the scheduled sync workflow is enabled; +- the stable workflow keeps `pylon-stable-publication` serialized with `cancel-in-progress: false`; +- active no-bypass repository ruleset `21950766`, **Pylon immutable publication tags**, targets exactly `refs/tags/pylon-build-*` and `refs/tags/pylon-stable-*` with no excludes, has GraphQL `bypassActors.totalCount: 0`, permits creation, and forbids every update and deletion; and +- repository action policy requires full commit-SHA pins. + +Before enabling any writer, read back all three environment protection-rule responses. Each must show `can_admins_bypass: false`, reviewer `rynfar`, `prevent_self_review: false`, and exactly one custom deployment branch named `pylon`. Treat a missing, extra, or different value as a publication blocker. + +The normal preview and stable attester jobs carry `pylon-preview` and `pylon-stable` directly. Approval therefore occurs before OIDC signing. Read-only verification follows. Every contents writer remains directly downstream of verified attestation or of the mutually exclusive approved recovery path. Every job that creates a protected publication ref or makes a release immutable also carries its channel environment directly. An explicit stable recovery creates no new attestation, so its zero-write `authorize-stable-resume` job carries `pylon-stable`, and the final publisher carries it again. The upstream-sync contents writer carries `pylon-upstream-sync` directly. + +Environment approval applies per deployment job, not once per workflow. A preview run can therefore ask for approval for attestation, preview-tag staging, and final immutable publication. A normal stable run can ask at attestation and final publication. A recovery run can ask at recovery authorization and final publication. Do not remove a later gate because an earlier job used the same environment. GitHub can group pending deployments in one approval screen, but operators must review every named job before approving it. + +Publication uses `GITHUB_TOKEN` for the existing minimum contents/checks/actions operations. It additionally uses one read-only GitHub App installation token only in dedicated inline ruleset-audit steps. Each audit performs a REST shape check and then a final authoritative GraphQL read with that same token. The App token never enters a contents mutation, checkout, downloaded artifact, shell, repository script, environment variable, or workflow output. The pinned mint action masks the token and revokes it in its post step. Do not set `skip-token-revoke`. Do not add npm, R2, PAT, or other repository secrets. The checkout-free publishers execute only frozen inline code; they never execute source from the repository or a downloaded artifact. Upstream sync does not use the auditor App. It checks out exactly `${{ github.sha }}` and, in the same shell that executes repository code, proves the canonical repository/event/ref, exact `HEAD`, workspace, and immediate live `pylon` SHA; a stale approved run stops before the sync script. + +### Ruleset-auditor GitHub App + +Create a dedicated GitHub App for publication ruleset readback: + +1. Grant only repository **Administration: read**. GitHub adds unavoidable **Metadata: read**. Grant no other permission and no write permission. +2. Install it on `pylon-code` for **Only select repositories**, with only repository id `1349002285`, `pylon-code/prime-agent`, selected. Do not install it organization-wide. The installation must not be suspended. +3. Generate a private key. Store the App id only as protected-environment variable `PYLON_RULESET_AUDITOR_APP_ID` and the PEM only as protected-environment secret `PYLON_RULESET_AUDITOR_PRIVATE_KEY` in both `pylon-preview` and `pylon-stable`. Do not create repository-level, organization-level, file-based, output-based, or job-environment fallbacks. +4. Keep the pinned `actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349` inputs closed to `owner: pylon-code`, `repositories: prime-agent`, and `permission-administration: read`. Its inspected v2 bundle calls `core.setSecret` before exposing the token and revokes it with `DELETE /installation/token` in the post action. Do not add `github-api-url`, another permission, or another input. + +An Administration-read App can receive a REST ruleset response that omits `bypass_actors` and `current_user_can_bypass`. That omission is expected redaction, not evidence of an empty bypass list. The combined validator still requires REST status 200; exact numeric and node ids; name, repository source and type; active tag target; exact include and exclude conditions; and exactly one update rule with `update_allows_fetch_and_merge: false` plus one deletion rule. It does not require the two REST bypass fields. If either field is present, only `bypass_actors: []` and `current_user_can_bypass: never` are safe; any other present value fails. + +The last authoritative read before each separate `GITHUB_TOKEN` mutation is the exact GraphQL `repository.ruleset(databaseId: 21950766, includeParents: false)` query with the same downscoped App token. It binds outer repository id `R_kgDOUGgkLQ`, database id `1349002285`, and `pylon-code/prime-agent`; ruleset node id `RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4`, database id, name, active tag target, and repository source; exact ref conditions with every other condition target null; `bypassActors.totalCount` as integer zero; and exactly the `UPDATE`/`UpdateParameters(updateAllowsFetchAndMerge: false)` and `DELETION`/null-parameters nodes. A GraphQL error, null or partial object, redacted bypass connection or count, nonzero count, unexpected id, condition, or rule blocks publication. Octokit turns GraphQL `errors` into a thrown audit failure. + +### Mandatory live App acceptance + +Before enabling either publication writer, and after any App key, installation, permission, repository selection, or ruleset change, run the maintainer-only acceptance CLI from a trusted checkout. This is mandatory. It never changes repository or ruleset state, and it never runs inside a publisher. It accepts a private-key **path**, reads bounded key bytes only to sign a bounded local JWT, never accepts the PEM value as an argument, and never prints the JWT, installation tokens, or key. Keep the key file outside the repository. Do not use `cat`, command substitution, `gh auth token`, a PAT, or a user token. + +Configure a different known public ruleset whose GraphQL bypass aggregate is known to be nonzero. The canary proves that this App token does not turn a visible nonzero aggregate into zero. The command below prints only a non-secret acceptance summary: + +```sh +export PYLON_RULESET_AUDITOR_APP_ID='' +export PYLON_RULESET_AUDITOR_PRIVATE_KEY_PATH='/secure/path/to/app-private-key.pem' +export PYLON_RULESET_AUDITOR_APP_SLUG='' +export PYLON_RULESET_CANARY_OWNER='' +export PYLON_RULESET_CANARY_REPO='' +export PYLON_RULESET_CANARY_RULESET_ID='' + +node scripts/accept-pylon-ruleset-auditor-app.mjs \ + --app-id "$PYLON_RULESET_AUDITOR_APP_ID" \ + --private-key-path "$PYLON_RULESET_AUDITOR_PRIVATE_KEY_PATH" \ + --app-slug "$PYLON_RULESET_AUDITOR_APP_SLUG" \ + --canary-owner "$PYLON_RULESET_CANARY_OWNER" \ + --canary-repo "$PYLON_RULESET_CANARY_REPO" \ + --canary-ruleset-id "$PYLON_RULESET_CANARY_RULESET_ID" + +unset PYLON_RULESET_AUDITOR_APP_ID PYLON_RULESET_AUDITOR_PRIVATE_KEY_PATH PYLON_RULESET_AUDITOR_APP_SLUG +unset PYLON_RULESET_CANARY_OWNER PYLON_RULESET_CANARY_REPO PYLON_RULESET_CANARY_RULESET_ID +``` + +The CLI verifies `GET /app` id, slug, and exact read-only permissions; the exact unsuspended selected-repository `pylon-code` installation; and the exact singleton installation repository. It mints and revokes a full-installation Administration-read token to paginate that singleton, then mints an exact `prime-agent` runtime token and inspects its returned repository and permission scope. With the runtime token it runs the combined target validator and then the nonzero canary. It revokes each minted token even when later validation fails. Any endpoint, scope, identity, pagination, revocation, GraphQL, target, or canary mismatch fails closed. In particular, a GraphQL null/error or canary count zero is not acceptance. + +A missing protected-environment variable, omitted secret, unavailable installation, token-mint failure, endpoint/auth failure, target or canary redaction, or token-revocation setup change blocks mutation. Do not run this acceptance CLI from a publisher. Publishers retain the pinned no-checkout, no-source-execution design. + +## Preview publication + +`.github/workflows/pylon-preview-release.yml` runs only for an exact canonical push to `refs/heads/pylon`. It uses Node `22.23.2` and npm `11.10.1`, packs twice with build networking disabled, compares all subjects byte for byte, and installs the first pack on Ubuntu Linux and macOS. Ubuntu is the supported gate for Linux and WSL2; native Windows publication support is deferred. + +The preview identity is: + +```text +pylon-build-g-r +``` + +Its immutable prerelease contains four tarballs plus: + +```text +pylon-prime-agent-release-v1.json +pylon-preview-channel-v1.json +``` + +The canonical preview manifest binds the full source commit/tree, artifact recipe, build-manifest digest, archive digests, exact preview signer policy, and this monotonic channel identity: + +```json +{ + "publicationPolicyRevision": 1, + "sequenceEpoch": 1, + "sequence": 123, + "workflowRunId": "33428882721" +} +``` + +`sequence` is the positive safe integer `github.run_number` for the one preview workflow. `workflowRunId` is its exact positive decimal run id. Failed runs create gaps, so consumers allow a higher non-adjacent sequence. A workflow sequence reset requires a new signed epoch/schema and consumer migration; it must never silently reuse epoch 1. Ordering never comes from a commit abbreviation, SemVer, a timestamp, the GitHub “latest” pointer, or a tag sort. + +`runAttempt` is deliberately not in manifest bytes. A rerun keeps the same run id, run number, manifest, and build-tag identity. The verified SLSA workflow/v1 predicate supplies the actual `/runs//attempts/` invocation. Verification requires its signed run id to equal `workflowRunId`, then reads that exact immutable attempt endpoint and its attempt-specific jobs. It proves the run number, repository id, workflow path/ref, push event, source SHA/branch, GitHub Actions check-suite app, and successful directly environment-gated attester job. The aggregate `/runs/` view is mutable across reruns and is not an attestation trust root; a later failed rerun cannot invalidate an earlier exact signed and published attempt. + +The approved attester signs exactly six subjects with pinned `actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8`, whose reviewed pinned chain delegates to `actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d`. A read-only job verifies the exact subject set, SLSA v1 workflow predicate, GitHub OIDC issuer, signer digest/ref, public Rekor entry, and run invocation. Only then can checkout-free contents jobs fully stage and publish the exact draft. + +The directly `pylon-preview`-gated staging job re-reads live `pylon`, then a dedicated App-authenticated step performs the exact combined REST/GraphQL audit immediately before the separate `GITHUB_TOKEN` preview-tag CAS step. The directly gated publisher repeats live branch/tag checks and a fresh combined audit immediately before the separate immutable-release update. GraphQL is the final authoritative read in each audit. A stale admission or earlier audit is irrelevant. GitHub does not offer an atomic transaction across branch reads, tag creation, and release publication. Each read and compare-and-set is a separate fail-closed point-in-time check; this design does not claim cross-resource atomicity. + +## Preview consumer high-water + +Download one preview into a new directory. The integrated verifier checks all bytes, six attestations, signer workflow at the signer digest, signed run invocation, Actions run number, and then atomically advances explicit consumer-local state: + +```sh +tag=pylon-build-g0123456789ab-r1 +mkdir publication +gh release download "$tag" --repo pylon-code/prime-agent --dir publication +GH_TOKEN="$(gh auth token)" npm run release:pylon:verify-preview-history -- \ + --historical \ + --artifact-dir publication \ + --state "$HOME/.local/state/pylon-prime/preview-high-water.json" \ + --initialize +``` + +Use `--initialize` only after manually inspecting the first full verified receipt. Omit it thereafter. The canonical JSON at `--state` remains the CLI-compatible projection. The adjacent private `.journal` directory is the concurrency authority. Its authenticated checkpoint names one current epoch, anchors the exact prior immutable tip, and carries that tip's bounded canonical state bytes. Within the epoch, base-digest transition links and one contiguous operation-slot namespace are immutable no-replace records. Each slot is bound to its exact checkpoint epoch and generation and carries either a random-token normal operation or a deterministic rotation operation. Normal-operation 10-second heartbeats yield to one permanent `released`, `retired`, or `commit` decision. `transaction.commitState(candidate)` only validates and privately stages one copied candidate for the current callback. It does not publish a commit terminal, a transition, or the projection while callback code is still running. Only after the callback returns successfully does the wrapper publish the immutable commit decision and finish its transitions, projection, and applied marker before the overall call returns. A throw, process exit, or stale-owner retirement after staging but before callback success leaves no state commit. The active heartbeat and unresolved shared operation slot keep every later normal operation and rotation out for the full callback. A stale 30-second claim is retired, and a complete post-callback commit is helpable after every crash point. Owned write temporaries live in the separate bounded `.owned-temporaries-v2` namespace, so authenticated logical-entry caps never make orphan cleanup unreachable. The verifier preserves live-writer fencing, rejects gaps, cycles, unreachable records, orphan markers, symlinks, unexpected entries, and excess record, temporary, depth, or byte work, and repairs a missing or stale JSON projection from the journal tip. + +`${state}.lock` is not the current journal namespace. For a fresh v2 journal, it is a permanent exact regular-file downgrade guard for clients that used `proper-lockfile`. Current tooling publishes that file by fsyncing a named owned temporary, hard-linking it no-replace, and fsyncing the parent. For migrated v1 authority, the original `${state}.lock` directory stays in place and contains an immutable `.pylon-consumer-v1-retired.json` marker. The marker binds the exact complete pre-marker authority digest and tip digest, and only that exact marker is excluded from the v1 authority digest. Its file, lock directory, and parent are fsynced before migration continues. The nonempty directory permanently blocks an old client's `rmdir` and subsequent atomic lock-directory `mkdir`. A directory without that exact marker is treated as a live or ambiguous legacy lease and fails closed. If no `${state}.transactions` authority exists, stop all old clients, confirm no owner remains, and remove that lease directory manually before retrying; current verification never enters or steals it. If the transaction namespace exists, preserve the directory and use the migration command below. + +Versions before the checkpoint journal used `${state}.transactions` plus claim, terminal, and applied records in a `${state}.lock` directory. Legacy detection does not depend on that transaction namespace alone. Before guard or journal initialization, current verification independently inspects `${state}.transactions`, `${state}.lock.v1-retired`, an in-place `${state}.lock` directory and retirement marker, and any existing v2 head whose `sourceAuthoritySha256` is non-genesis. Any one signal requires the complete exact legacy source. A missing or deleted companion namespace, malformed entry, wrong mode, or symlink fails closed. The presence of the transaction namespace is always prior authority; current verification refuses to seed or trust a v2 projection around it. After stopping every old client and confirming that every old claim is terminal, migrate once: + +```sh +npm run release:pylon:migrate-consumer-journal -- \ + --state "$HOME/.local/state/pylon-prime/preview-high-water.json" +``` + +This explicit quiescent command pins and bounds every v1 file, authenticates the complete transition chain and every relevant commit/help record, and accepts a projection only when it is the exact tip or an authenticated stale prefix. A commit is complete only when its exact applied marker and every decided transition are durable. An incomplete commit is recoverable only when `kill(pid, 0)` proves its recorded owner is gone with `ESRCH`; a live PID, PID reuse, `EPERM`, or any uncertain liveness blocks without retiring the authority. After all permitted dead-owner help, the command re-reads the source and atomically publishes the immutable retirement marker inside the existing `${state}.lock` directory. It never renames that directory and never creates or replaces `${state}.lock.v1-retired`. A prior `${state}.lock.v1-retired` layout from an interrupted local migration is accepted only as read-only source evidence and must use the exact regular downgrade guard. The deterministic v2 checkpoint binds the digest and tip of the complete old authority. Concurrent migrators re-read and join an exact marker or checkpoint that appeared after their initial read; conflicting marker, authority, directory, or checkpoint data fails closed. The command re-authenticates the full source immediately before marker publication, immediately before checkpoint publication, before projection repair, and before success. Every step is fsynced, deterministic, concurrently joinable, and retryable after a crash. Corrupt, active, missing, unreachable, extra, symlinked, over-limit, or permission-unsafe old authority fails closed. + +Rotate before an epoch reaches 3,800 transitions or 60,000 claims: + +```sh +npm run release:pylon:rotate-consumer-journal -- \ + --state "$HOME/.local/state/pylon-prime/preview-high-water.json" +``` + +Normal updates and rotation allocate from one immutable next-operation slot namespace. Every allocator scans and resolves the latest slot, rescans the same epoch and intent, and publishes only that exact next generation with no replacement; a lost publication loops from the new authority. No allocator may publish generation `N+1` while `N` is active or unresolved. A normal slot uses a random token and the configured finite normal-claim cap. A rotation slot is cap-exempt and carries the deterministic intent derived only from the exact current checkpoint and immutable tip, so rotators with different caller caps join the same generation, epoch id, directory, and checkpoint. Once a rotation wins its slot it is never released or retired: normal callers and later rotators help it through prior-writer quiescence, and no normal operation can cross it. If a normal operation wins the shared next slot first, rotation re-reads its committed tip and derives a new slot. Immediately before checkpoint linking and before success, rotation scans the complete bounded root set and rejects every competing same-epoch directory or checkpoint. Dead or already-retired normal-operation temporaries are removed; live prior temporaries keep rotation pending until they quiesce, while live helpers for the same deterministic rotation may join the same no-replace checkpoint link. Projection repair and commit help use bounded internal retries only for authenticated replacement or ctime races. Each retry re-walks the immutable journal tip before rereading or repairing the projection; malformed metadata, symlinks, and transaction digest corruption remain terminal errors. The current projection and high-water JSON schema do not change. After the new epoch is durable, a new fenced owner removes only the authenticated retired epoch and predecessor checkpoint, so active fencing data, directory entries, scan depth, and bytes remain bounded. Exact concurrent helpers treat a peer's already-removed retired epoch, predecessor checkpoint, or owned temporary as the same completed cleanup, re-scan an advanced authenticated head, and join the exact deterministic rotation instead of surfacing a transient path error. + +These pathname checks are not a portable `openat` security sandbox. The verifier rejects observed symlinks and non-directories, pins every read to a no-follow file descriptor where Node exposes it, bounds bytes before allocation, and re-stats after an exact read. Every operation requires a numeric current uid. Every relied-on state, guard, journal, temporary namespace, epoch, claim, marker, transition, and migration-authority entry must already be owned by that uid and have exact `0600` file or `0700` directory mode. Group/world-writable entries are rejected before parsing or use and are never chmod-and-trusted, because another process may retain a writable file descriptor. Newly created directories and files use exact `0700` and `0600`; their contents and directory entries are fsynced before success. For old private state with other modes, stop every process that may hold a descriptor, preserve an offline backup, correct the modes while fully quiescent, and retry. Tooling never performs that migration implicitly. The state parent remains a trusted user-owned local directory with no hostile mutation by the same OS user. Platforms without a numeric current uid fail closed. + +## Stable promotion + +Run **Actions → Pylon stable promotion → Run workflow** on `pylon` with `operation=promote`, an immutable `preview_tag`, and no recovery or withdrawal identity. + +Current policy can promote older artifacts only when `scripts/pylon-prime-supported-release-recipes-v1.json` closes two independent immutable identity sets. A `recipeRevision` entry contains only the build manifest schema and Node/npm/minimum-Node tuple. A `publicationPolicyRevision` entry contains the exact preview/stable workflow paths and SHA-256 of both workflow byte strings. Preview manifests bind the preview policy revision that signed them. Stable manifests preserve that preview policy revision beside the build recipe and record the current stable policy revision under `promotion`. + +The verifier selects preview workflow bytes by the preview manifest's policy revision and stable workflow bytes by `promotion.publicationPolicyRevision`. A future stable policy revision can therefore promote historical recipe/policy-r1 preview bytes without rewriting r1. Unknown, duplicate, nonpositive, or extra registry identities fail closed. A publication workflow edit requires a new immutable publication policy revision and reviewed digests; it does not by itself require an artifact recipe revision. Bump the recipe only when the artifact recipe identity changes. Never rewrite either historical entry. The Ubuntu Linux/macOS install uses current protected verifier source; it never checks out or executes the older source. The preview tag recipe must equal the build recipe copied into stable. + +Normal stable transaction order is strict: + +1. Download the immutable preview. Verify six exact bytes, its signed workflow/run sequence, public Rekor evidence, source/tree, old preview workflow policy, current ancestry, and original/current exact-SHA checks. +2. Install those same bytes on Ubuntu Linux and macOS. +3. Read and validate the complete immutable stable release/tag digest chain. Before extending a nonempty chain, verify the latest singleton stable manifest against the exact stable workflow/ref, signer policy commit/tree, SLSA v1, public Rekor, and the workflow's directly gated static policy at that signer digest. +4. Prepare one canonical next manifest. The directly `pylon-stable`-gated attester signs that singleton. A separate read-only job verifies it. +5. A checkout-free contents writer creates or resumes one exact draft. Creation durably places the exact canonical manifest bytes, byte count, and SHA-256 in the bounded release-body recovery envelope before asset upload. It uploads and re-downloads/re-hashes the singleton. +6. The final checkout-free publisher re-downloads the draft from GitHub Releases, not an old Actions artifact. For a zero-asset crash draft, it recovers only the exact body-carried attested bytes, uploads the missing singleton once, and re-downloads/re-hashes it before any CAS. It rechecks the live current tip/checks, old policy tree/ancestry/checks, immutable preview, recipe, N-1 history, operation fields, and draft id/digest. +7. The directly `pylon-stable`-gated publisher mints one repository-scoped auditor token. Immediately before creating annotated `pylon-stable-sequence-NNNNNN`, before creating or refetching the exact lightweight stable tag, and before making the draft immutable, a separate read-only step uses that same token for the exact REST validation followed by the authoritative GraphQL zero-bypass proof. Each following mutation step uses only `GITHUB_TOKEN`. Only after both exact refs exist does it make that draft immutable and check postconditions. + +The reservation annotation binds sequence, policy commit/tree, the exact promote/withdraw tuple and reason, stable and preview tags, stable-manifest SHA-256, and draft release id. A reservation `422` refetches and stops for explicit recovery. Final tag `422` handling refetches and accepts only the exact lightweight full-commit target; a wrong or annotated object fails before immutable publication. No path selects N+1, moves, deletes, or reuses a ref. The reservation freezes the approved old policy tuple if `pylon` advances later. Reservation CAS, final tag CAS, and release publication are ordered GitHub operations, not one atomic GitHub transaction. Each protected mutation has a new combined audit whose last authoritative read is GraphQL; earlier admission and the prior mutation's audit do not authorize it. + +Stable tags remain: + +```text +pylon-stable--g-r +``` + +The signed stable manifest copies the preview sequence epoch/number/run id, full preview identity/digests, artifact recipe and preview publication policy revision. Its `promotion` record adds the current stable publication policy revision with the policy commit/tree, exact previous stable tag/digest, high-water, operation, and cumulative sorted revocations. + +## Explicit stable recovery + +A crash can leave any of these exact recoverable states: + +- a zero-asset draft whose bounded body already carries the attested canonical manifest; +- a complete approved draft before CAS; +- the complete draft plus its permanent sequence reservation; or +- both the reservation and exact final lightweight tag before release publication. + +Start a fresh run on current `pylon` with `operation=resume-promote` or `resume-withdraw`, the original `preview_tag`, withdrawal fields, and `resume_identity` set to the numeric draft release id, stable draft tag, or exact reservation tag. The run recovers manifest bytes from the exact draft body and requires any existing singleton to match byte for byte. The canonical stable manifest is limited to 48 KiB and the complete encoded release body to 80 KiB. Altered, truncated, oversized, or ambiguous envelopes/assets fail. It never relies on an old Actions artifact. + +Recovery requires the exact operator tuple; exact body-carried bytes and any present draft asset; old stable attestation and exact workflow-byte approval policy; exact draft/annotation/digest; every signed N-1 history receipt; immutable old preview and preview attestation/run sequence; original source/policy checks; fresh current checks; old policy exact tree and ancestry; current Ubuntu Linux/macOS install; and one fresh `pylon-stable` approval. History reading excludes exactly the selected recovery draft id and rejects every other draft. Recovery uploads and re-hashes a missing singleton before reservation/final-tag CAS. It does not reprepare or reattest. + +Draft-only recovery can create the still-free N reservation. Reservation recovery can finalize only the exact already-reserved tuple. An unexpected reservation, draft, tag, asset, sequence, annotation, signer, or digest fails closed. + +## Withdrawal + +Use `operation=withdraw`, a preview for the new sequence, the exact prior `revoke_stable_tag`, and a lowercase reason code. This appends one signed revocation bound to the old stable/build tags. Repeat withdrawal fails. Never delete, replace, or retag withdrawn history. + +## Stable consumer high-water + +Verify each immutable release and stable attestation first. Then give the verifier every canonical stable manifest from sequence 1 through current and explicit local state: + +```sh +npm run release:pylon:verify-stable-history -- \ + --state "$HOME/.local/state/pylon-prime/stable-high-water.json" \ + --initialize \ + stable-history/pylon-stable-*/pylon-stable-channel-v1.json +``` + +Use `--initialize` once, then omit it. The CLI requires the complete contiguous canonical manifest chain, regular non-symlink inputs, and explicit local state. It parses and hashes all manifests before acquiring the same tokenized lock and immutable base-digest transaction journal described for preview state. The canonical JSON state path is a repairable projection of that journal tip. The CLI rejects malformed authoritative state, a lower valid prefix, and any rewrite at or below the witnessed sequence. It commits only a monotonic journal advance and repairs the projection before success. + +## Failure and incident handling + +- **Approval is absent:** configure the exact environment. Never remove or bypass `environment:`. +- **Required proof is missing:** fix branch protection/check provenance and create a new protected merge. Never synthesize status. +- **Draft differs:** stop. Do not delete an asset or rebuild/reprepare around it. Exact recovery accepts only the bounded body-carried manifest and an absent or identical singleton; every other difference fails. +- **Reservation race or `422`:** inspect the ref and owning run. Resume only the exact tuple. Never choose N+1, move, or delete. +- **Attestation/Rekor/workflow policy fails:** do not approve, promote, install, or advance consumer state. +- **Release is immutable:** workflows never delete it. A withdrawal is a later sequence. +- **Invalid tag squat:** publication stays blocked. Record an incident and export the active ruleset plus tag/release/Actions audit evidence. A repository administrator must make one reviewed temporary ruleset change that permits deleting only the named invalid ref, delete it by exact ref/object identity, and immediately restore/read back ruleset `21950766` with the original targets, no bypass actors, update/deletion blocks, and `current_user_can_bypass: never`. Never let publication automation perform this recovery. +- **Invalid immutable release:** preserve evidence first. GitHub may require an administrator to temporarily disable immutable releases before exact-id deletion. Delete only the proven invalid release, restore/read back immutable releases immediately, and link every API response in the incident. Never alter a valid published sequence. + +Run offline policy tests with `npm run test:pylon-publication` and App-acceptance unit tests with `npm run test:pylon-ruleset-auditor-app`. They cover exact current/historical workflow digests and registry closure, immutable signed attempt evidence, zero-asset crash recovery, deterministic stale recovery, active heartbeats, transaction crash convergence, path-boundary checks, exact required-check paths/apps, preview/stable tag squats and CAS order, withdrawal tuples, rollback state, approval DAGs, every contents writer, pinned actions, no source/download execution in publication writers, mocked App scopes and endpoints, REST and GraphQL redaction, the nonzero canary, and token revocation. diff --git a/package.json b/package.json index 49905b460b..ef0b66f742 100644 --- a/package.json +++ b/package.json @@ -32,8 +32,18 @@ "release:pylon:hydrate-lock": "node scripts/hydrate-pylon-release-lock.mjs", "release:pylon:pack": "node scripts/build-pylon-prime-agent-release.mjs --pack", "release:pylon:verify": "node scripts/verify-pylon-prime-agent-release.mjs", + "release:pylon:preview": "node scripts/prepare-pylon-preview-manifest.mjs", + "release:pylon:verify-preview": "node scripts/verify-pylon-preview-publication.mjs", + "release:pylon:verify-preview-history": "node scripts/verify-pylon-preview-history.mjs", + "release:pylon:verify-attestations": "node scripts/verify-pylon-publication-attestations.mjs", + "release:pylon:verify-stable-attestation": "node scripts/verify-pylon-stable-attestation.mjs", + "release:pylon:verify-stable-history": "node scripts/verify-pylon-stable-history.mjs", + "release:pylon:rotate-consumer-journal": "node scripts/rotate-pylon-consumer-journal.mjs", + "release:pylon:migrate-consumer-journal": "node scripts/migrate-pylon-consumer-journal.mjs", "release:pylon:smoke": "node scripts/smoke-pylon-prime-agent-release.mjs", "test:pylon-release": "node --test scripts/pylon-prime-agent-release.test.mjs", + "test:pylon-publication": "node --test scripts/pylon-publication.test.mjs", + "test:pylon-ruleset-auditor-app": "node --test scripts/pylon-ruleset-auditor-acceptance.test.mjs", "release:patch": "node scripts/release.mjs patch", "release:minor": "node scripts/release.mjs minor", "release:major": "node scripts/release.mjs major", diff --git a/scripts/accept-pylon-ruleset-auditor-app.mjs b/scripts/accept-pylon-ruleset-auditor-app.mjs new file mode 100755 index 0000000000..d78cb1e40f --- /dev/null +++ b/scripts/accept-pylon-ruleset-auditor-app.mjs @@ -0,0 +1,293 @@ +#!/usr/bin/env node + +import { createPrivateKey, sign } from "node:crypto"; +import { lstatSync, readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +import { + auditPublicationRuleset, + PYLON_PUBLICATION_RULESET_GRAPHQL_QUERY, + PYLON_REPOSITORY_ID, + PYLON_REPOSITORY_NAME, + PYLON_REPOSITORY_NAME_WITH_OWNER, + PYLON_REPOSITORY_OWNER, + PYLON_RULESET_BYPASS_CANARY_GRAPHQL_QUERY, + PYLON_RULESET_ID, + validateRulesetBypassCanary, +} from "./lib/pylon-ruleset-auditor.mjs"; + +const API_VERSION = "2022-11-28"; +const MAX_PRIVATE_KEY_BYTES = 64 * 1024; +const PYLON_ORGANIZATION_ID = 314_006_107; + +function base64Url(value) { + return Buffer.from(value).toString("base64url"); +} + +function exactObject(actual, expected) { + return actual !== null && typeof actual === "object" && !Array.isArray(actual) && + JSON.stringify(Object.entries(actual).sort()) === JSON.stringify(Object.entries(expected).sort()); +} + +function exactRepository(repository) { + return repository?.id === PYLON_REPOSITORY_ID && repository?.name === PYLON_REPOSITORY_NAME && + repository?.full_name === PYLON_REPOSITORY_NAME_WITH_OWNER && repository?.owner?.login === PYLON_REPOSITORY_OWNER; +} + +function assertAppPermissions(permissions, description) { + if (!exactObject(permissions, { administration: "read", metadata: "read" })) { + throw new Error(`${description} permissions are not exact Administration: read plus unavoidable Metadata: read.`); + } +} + +function assertTokenPermissions(permissions, description) { + if ( + !exactObject(permissions, { administration: "read" }) && + !exactObject(permissions, { administration: "read", metadata: "read" }) + ) throw new Error(`${description} response permissions exceed or omit the exact read-only scope.`); +} + +function readPrivateKey(privateKeyPath) { + if (typeof privateKeyPath !== "string" || !privateKeyPath) throw new Error("--private-key-path is required."); + const metadata = lstatSync(privateKeyPath); + if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size < 1 || metadata.size > MAX_PRIVATE_KEY_BYTES) { + throw new Error("The App private-key path must be one bounded regular file."); + } + return readFileSync(privateKeyPath); +} + +export function createAppJwt({ appId, privateKeyPath, now = Date.now() }) { + if (typeof appId !== "string" || !/^[1-9][0-9]*$/.test(appId)) throw new Error("--app-id must be a positive decimal GitHub App id."); + if (!Number.isSafeInteger(now) || now < 0) throw new Error("JWT clock is invalid."); + const seconds = Math.floor(now / 1000); + const encodedHeader = base64Url(JSON.stringify({ alg: "RS256", typ: "JWT" })); + const encodedPayload = base64Url(JSON.stringify({ iat: seconds - 60, exp: seconds + 540, iss: appId })); + const signingInput = `${encodedHeader}.${encodedPayload}`; + const privateKeyBytes = readPrivateKey(privateKeyPath); + try { + const key = createPrivateKey(privateKeyBytes); + return `${signingInput}.${sign("RSA-SHA256", Buffer.from(signingInput), key).toString("base64url")}`; + } finally { + privateKeyBytes.fill(0); + } +} + +async function defaultRequest({ method, path, token, body }) { + const response = await fetch(`https://api.github.com${path}`, { + method, + headers: { + accept: "application/vnd.github+json", + authorization: `Bearer ${token}`, + "content-type": "application/json", + "user-agent": "pylon-ruleset-auditor-acceptance", + "x-github-api-version": API_VERSION, + }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: AbortSignal.timeout(30_000), + }); + const bytes = Buffer.from(await response.arrayBuffer()); + if (bytes.length > 4 * 1024 * 1024) throw new Error("GitHub acceptance response exceeds the bounded body limit."); + let data = null; + if (bytes.length > 0) { + try { + data = JSON.parse(bytes); + } catch { + throw new Error(`GitHub acceptance endpoint returned non-JSON status ${response.status}.`); + } + } + return { status: response.status, data }; +} + +function requireStatus(response, status, description) { + if (response?.status !== status) throw new Error(`${description} returned status ${response?.status ?? "missing"}.`); + return response.data; +} + +async function graphql(request, token, query, variables, description) { + const response = await request({ method: "POST", path: "/graphql", token, body: { query, variables } }); + const envelope = requireStatus(response, 200, description); + if ( + envelope === null || typeof envelope !== "object" || Array.isArray(envelope) || + Object.hasOwn(envelope, "errors") || envelope.data === null || typeof envelope.data !== "object" || Array.isArray(envelope.data) + ) throw new Error(`${description} returned GraphQL errors or a null/partial envelope.`); + return envelope.data; +} + +function mintedToken(response, description) { + const data = requireStatus(response, 201, description); + if (typeof data?.token !== "string" || !data.token) throw new Error(`${description} returned no revocable token.`); + return { data, token: data.token }; +} + +function validateTokenResponse(data, description, now, requireRepositoryResponse) { + const expiration = Date.parse(data.expires_at); + const repositoriesAreExact = Array.isArray(data.repositories) && data.repositories.length === 1 && exactRepository(data.repositories[0]); + if ( + typeof data.expires_at !== "string" || !Number.isFinite(expiration) || expiration <= now || expiration > now + 65 * 60 * 1000 || + data.repository_selection !== "selected" || requireRepositoryResponse && !repositoriesAreExact || + !requireRepositoryResponse && data.repositories !== undefined && !repositoriesAreExact + ) throw new Error(`${description} response does not carry the exact selected prime-agent repository scope.`); + assertTokenPermissions(data.permissions, description); +} + +async function revokeToken(request, token) { + const response = await request({ method: "DELETE", path: "/installation/token", token }); + requireStatus(response, 204, "Installation-token revocation"); +} + +async function withRevokedToken(request, token, use) { + try { + return await use(token); + } finally { + await revokeToken(request, token); + } +} + +function validateCanary(canary) { + if ( + canary === null || typeof canary !== "object" || !/^[A-Za-z0-9_.-]+$/.test(canary.owner ?? "") || + !/^[A-Za-z0-9_.-]+$/.test(canary.repo ?? "") || !Number.isSafeInteger(canary.rulesetDatabaseId) || + canary.rulesetDatabaseId < 1 || + `${canary.owner}/${canary.repo}:${canary.rulesetDatabaseId}` === `${PYLON_REPOSITORY_NAME_WITH_OWNER}:${PYLON_RULESET_ID}` + ) throw new Error("A different known-public positive-bypass ruleset canary is required."); + return canary; +} + +export async function acceptRulesetAuditorApp({ + appId, + privateKeyPath, + appSlug, + canary, + request = defaultRequest, + now = Date.now(), +}) { + if (typeof appSlug !== "string" || !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(appSlug)) { + throw new Error("--app-slug must be the exact expected GitHub App slug."); + } + validateCanary(canary); + const jwt = createAppJwt({ appId, privateKeyPath, now }); + const app = requireStatus(await request({ method: "GET", path: "/app", token: jwt }), 200, "Authenticated App readback"); + if (String(app?.id) !== appId || app?.slug !== appSlug) throw new Error("Authenticated App id or slug differs from the expected identity."); + assertAppPermissions(app.permissions, "Authenticated App"); + + const installation = requireStatus( + await request({ method: "GET", path: `/orgs/${PYLON_REPOSITORY_OWNER}/installation`, token: jwt }), + 200, + "pylon-code App installation readback", + ); + if ( + !Number.isSafeInteger(installation?.id) || installation.id < 1 || String(installation.app_id) !== appId || + installation.app_slug !== appSlug || installation.account?.login !== PYLON_REPOSITORY_OWNER || + installation.account?.id !== PYLON_ORGANIZATION_ID || installation.account?.type !== "Organization" || + installation.target_id !== PYLON_ORGANIZATION_ID || installation.target_type !== "Organization" || + installation.repository_selection !== "selected" || installation.suspended_at !== null || installation.suspended_by !== null + ) throw new Error("pylon-code App installation identity, selection, or suspension state differs from the exact policy."); + assertAppPermissions(installation.permissions, "pylon-code App installation"); + + const tokenPath = `/app/installations/${installation.id}/access_tokens`; + const fullMint = mintedToken(await request({ + method: "POST", + path: tokenPath, + token: jwt, + body: { permissions: { administration: "read" } }, + }), "Full-installation token mint"); + await withRevokedToken(request, fullMint.token, async (token) => { + validateTokenResponse(fullMint.data, "Full-installation token mint", now, false); + const repositories = []; + let expectedTotal = null; + for (let page = 1; page <= 500; page += 1) { + const data = requireStatus(await request({ + method: "GET", + path: `/installation/repositories?per_page=100&page=${page}`, + token, + }), 200, "Installation repository pagination"); + if (!Number.isSafeInteger(data?.total_count) || data.total_count < 0 || !Array.isArray(data.repositories)) { + throw new Error("Installation repository page is malformed."); + } + expectedTotal ??= data.total_count; + if (data.total_count !== expectedTotal || data.repositories.length > 100) { + throw new Error("Installation repository pagination changed or exceeded its page bound."); + } + repositories.push(...data.repositories); + if (repositories.length >= expectedTotal) break; + if (data.repositories.length === 0) throw new Error("Installation repository pagination stopped before total_count."); + } + if (expectedTotal !== 1 || repositories.length !== 1 || !exactRepository(repositories[0])) { + throw new Error("App installation repository selection is not the exact pylon-code/prime-agent singleton."); + } + }); + + const runtimeMint = mintedToken(await request({ + method: "POST", + path: tokenPath, + token: jwt, + body: { repositories: [PYLON_REPOSITORY_NAME], permissions: { administration: "read" } }, + }), "Runtime token mint"); + await withRevokedToken(request, runtimeMint.token, async (token) => { + validateTokenResponse(runtimeMint.data, "Runtime token mint", now, true); + await auditPublicationRuleset({ + requestRest: () => request({ + method: "GET", + path: `/repos/${PYLON_REPOSITORY_NAME_WITH_OWNER}/rulesets/${PYLON_RULESET_ID}`, + token, + }), + requestGraphql: (query, variables) => graphql(request, token, query, variables, "Target ruleset GraphQL audit"), + }); + const canaryResponse = await graphql( + request, + token, + PYLON_RULESET_BYPASS_CANARY_GRAPHQL_QUERY, + canary, + "Bypass-count canary GraphQL audit", + ); + validateRulesetBypassCanary(canaryResponse, canary); + }); + + return { + appId, + appSlug, + installationId: installation.id, + repository: PYLON_REPOSITORY_NAME_WITH_OWNER, + rulesetId: PYLON_RULESET_ID, + accepted: true, + }; +} + +function parseArguments(argv) { + const values = {}; + const allowed = new Set(["app-id", "private-key-path", "app-slug", "canary-owner", "canary-repo", "canary-ruleset-id"]); + for (let index = 0; index < argv.length; index += 2) { + const option = argv[index]; + const value = argv[index + 1]; + if (!option?.startsWith("--") || !allowed.has(option.slice(2)) || value === undefined || value.startsWith("--")) { + throw new Error("Usage: accept-pylon-ruleset-auditor-app --app-id ID --private-key-path PATH --app-slug SLUG --canary-owner OWNER --canary-repo REPO --canary-ruleset-id ID"); + } + if (Object.hasOwn(values, option)) throw new Error(`Duplicate option: ${option}`); + values[option] = value; + } + if (values["--canary-ruleset-id"] === undefined || !/^[1-9][0-9]*$/.test(values["--canary-ruleset-id"])) { + throw new Error("--canary-ruleset-id must be a positive decimal ruleset id."); + } + return { + appId: values["--app-id"], + privateKeyPath: values["--private-key-path"], + appSlug: values["--app-slug"], + canary: { + owner: values["--canary-owner"], + repo: values["--canary-repo"], + rulesetDatabaseId: Number(values["--canary-ruleset-id"]), + }, + }; +} + +async function main() { + const result = await acceptRulesetAuditorApp(parseArguments(process.argv.slice(2))); + process.stdout.write(`${JSON.stringify(result)}\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main().catch((error) => { + process.stderr.write(`Ruleset-auditor App acceptance failed: ${error instanceof Error ? error.message : "unknown error"}\n`); + process.exitCode = 1; + }); +} diff --git a/scripts/lib/pylon-bounded-file.mjs b/scripts/lib/pylon-bounded-file.mjs new file mode 100644 index 0000000000..2932290b1a --- /dev/null +++ b/scripts/lib/pylon-bounded-file.mjs @@ -0,0 +1,136 @@ +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readSync, +} from "node:fs"; +import { lstat, open } from "node:fs/promises"; +export const PYLON_PUBLICATION_MANIFEST_MAX_BYTES = 64 * 1024; +export const PYLON_STABLE_HISTORY_MAX_MANIFESTS = 4096; +export const PYLON_STABLE_HISTORY_MAX_BYTES = 32 * 1024 * 1024; + + +function sameStat(left, right) { + return left.dev === right.dev && left.ino === right.ino && left.size === right.size && + left.mtimeMs === right.mtimeMs && left.ctimeMs === right.ctimeMs; +} + +export async function readBoundedRegularFile( + path, + { + maxBytes, + minBytes = 1, + description = "Input", + openFile = open, + lstatEntry = lstat, + validateHandle, + hooks, + } = {}, +) { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 1 || !Number.isSafeInteger(minBytes) || minBytes < 0 || minBytes > maxBytes) { + throw new Error("Bounded file limits are invalid."); + } + let pathEntry; + try { + pathEntry = await lstatEntry(path); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } + if (pathEntry.isSymbolicLink?.() || !pathEntry.isFile()) { + throw new Error(`${description} is not one regular non-symlink file.`); + } + let handle; + try { + handle = await openFile(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + } catch (error) { + if (error?.code === "ENOENT") return null; + if (["ELOOP", "EISDIR"].includes(error?.code)) { + throw new Error(`${description} is not one regular non-symlink file.`); + } + throw error; + } + try { + let before = await handle.stat(); + if (!before.isFile()) throw new Error(`${description} is not one regular non-symlink file.`); + if (validateHandle) before = await validateHandle(handle, before, description); + if (before.size < minBytes || before.size > maxBytes) throw new Error(`${description} exceeds its format byte limit or is malformed.`); + await hooks?.afterInitialStat?.({ path, handle, stat: before }); + const bytes = Buffer.alloc(before.size); + let offset = 0; + while (offset < bytes.length) { + const { bytesRead } = await handle.read(bytes, offset, bytes.length - offset, offset); + if (bytesRead === 0) throw new Error(`${description} changed while it was read.`); + offset += bytesRead; + } + const extra = Buffer.alloc(1); + const { bytesRead: extraBytes } = await handle.read(extra, 0, 1, bytes.length); + await hooks?.beforeFinalStat?.({ path, handle, bytes }); + const after = await handle.stat(); + if (extraBytes !== 0 || !sameStat(before, after)) throw new Error(`${description} changed while it was read.`); + return bytes; + } finally { + await handle.close(); + } +} + + +export function readBoundedRegularFileSync( + path, + { + maxBytes, + minBytes = 1, + description = "Input", + openFile = openSync, + lstatEntry = lstatSync, + statFile = fstatSync, + readFile = readSync, + closeFile = closeSync, + hooks, + } = {}, +) { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 1 || !Number.isSafeInteger(minBytes) || minBytes < 0 || minBytes > maxBytes) { + throw new Error("Bounded file limits are invalid."); + } + let pathEntry; + try { + pathEntry = lstatEntry(path); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } + if (pathEntry.isSymbolicLink?.() || !pathEntry.isFile()) { + throw new Error(`${description} is not one regular non-symlink file.`); + } + let descriptor; + try { + descriptor = openFile(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + } catch (error) { + if (error?.code === "ENOENT") return null; + if (["ELOOP", "EISDIR"].includes(error?.code)) throw new Error(`${description} is not one regular non-symlink file.`); + throw error; + } + try { + const before = statFile(descriptor); + if (!before.isFile()) throw new Error(`${description} is not one regular non-symlink file.`); + if (before.size < minBytes || before.size > maxBytes) throw new Error(`${description} exceeds its format byte limit or is malformed.`); + hooks?.afterInitialStat?.({ path, descriptor, stat: before }); + const bytes = Buffer.alloc(before.size); + let offset = 0; + while (offset < bytes.length) { + const bytesRead = readFile(descriptor, bytes, offset, bytes.length - offset, offset); + if (bytesRead === 0) throw new Error(`${description} changed while it was read.`); + offset += bytesRead; + } + const extra = Buffer.alloc(1); + const extraBytes = readFile(descriptor, extra, 0, 1, bytes.length); + hooks?.beforeFinalStat?.({ path, descriptor, bytes }); + const after = statFile(descriptor); + if (extraBytes !== 0 || !sameStat(before, after)) throw new Error(`${description} changed while it was read.`); + return bytes; + } finally { + closeFile(descriptor); + } +} diff --git a/scripts/lib/pylon-consumer-lock.mjs b/scripts/lib/pylon-consumer-lock.mjs new file mode 100644 index 0000000000..bc3c6620e7 --- /dev/null +++ b/scripts/lib/pylon-consumer-lock.mjs @@ -0,0 +1,2834 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { link, lstat, mkdir, open, readdir, rename, rm } from "node:fs/promises"; +import { basename, dirname, join, parse, relative, resolve, sep } from "node:path"; + +import { readBoundedRegularFile } from "./pylon-bounded-file.mjs"; + +export const PYLON_CONSUMER_LOCK_STALE_MS = 30_000; +export const PYLON_CONSUMER_LOCK_UPDATE_MS = 10_000; +export const PYLON_CONSUMER_ROTATE_CLAIM_TRIGGER = 60_000; +export const PYLON_CONSUMER_ROTATE_TRANSITION_TRIGGER = 3_800; +const LOCK_SCHEMA_VERSION = 2; +const LEGACY_LOCK_SCHEMA_VERSION = 1; +const TRANSACTION_SCHEMA_VERSION = 1; +const CHECKPOINT_SCHEMA_VERSION = 2; +const ROTATION_INTENT_SCHEMA_VERSION = 2; +const LEGACY_GUARD_SCHEMA_VERSION = 1; +const LEGACY_RETIREMENT_SCHEMA_VERSION = 1; +const GENESIS_DIGEST = "0".repeat(64); +const DEFAULT_STATE_MAX_BYTES = 1024 * 1024; +const MAX_STATE_BYTES = 16 * 1024 * 1024; +const DEFAULT_JOURNAL_MAX_BYTES = 64 * 1024 * 1024; +const MAX_JOURNAL_BYTES = 256 * 1024 * 1024; +const MAX_TRANSACTION_DEPTH = 4096; +const MAX_LOCK_GENERATIONS = 65_536; +const MAX_OPERATION_GENERATIONS = MAX_LOCK_GENERATIONS + 1; +const MAX_JOURNAL_ROOT_ENTRIES = 16; +const MAX_TEMPORARY_ENTRIES = 65_536; +const PROJECTION_RETRY_LIMIT = 32; +const TEMPORARY_DIRECTORY_NAME = ".owned-temporaries-v2"; +const LEGACY_RETIREMENT_MARKER_NAME = ".pylon-consumer-v1-retired.json"; +const uuidSource = "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"; +const uuidPattern = new RegExp(`^${uuidSource}$`); +const claimPattern = /^claim-([0-9]{16})\.json$/; +const transitionPattern = /^transition-([0-9a-f]{64})\.json$/; +const legacyTransitionPattern = /^([0-9a-f]{64})\.json$/; +const checkpointPattern = new RegExp(`^checkpoint-([0-9]{16})-(${uuidSource})\\.json$`); +const epochPattern = new RegExp(`^epoch-([0-9]{16})-(${uuidSource})$`); +const heartbeatPattern = new RegExp(`^heartbeat-([0-9]{16})-(${uuidSource})\\.json$`); +const terminalPattern = new RegExp(`^terminal-([0-9]{16})-(${uuidSource})\\.json$`); +const appliedPattern = new RegExp(`^applied-([0-9]{16})-(${uuidSource})\\.json$`); +const temporaryPattern = new RegExp( + `^\\.pylon-consumer-tmp-v1-p([1-9][0-9]*)-e(${uuidSource})-g([0-9]{16})-w(${uuidSource})-n([0-9a-f]{12})-k([a-z0-9-]{1,40})-t([0-9a-f]{64})\\.tmp$`, +); + +function exactKeys(value, keys) { + return value !== null && typeof value === "object" && !Array.isArray(value) && + Object.keys(value).sort().join(",") === [...keys].sort().join(","); +} + +function metadataBytes(value) { + return Buffer.from(`${JSON.stringify(value)}\n`); +} + +function digest(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function generationName(generation) { + if (!Number.isSafeInteger(generation) || generation < 0 || generation > 9_999_999_999_999_999) { + throw new Error("Consumer high-water lock generation is exhausted or malformed."); + } + return String(generation).padStart(16, "0"); +} + +function deterministicUuid(value) { + const hex = digest(Buffer.from(value)); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-4${hex.slice(13, 16)}-8${hex.slice(17, 20)}-${hex.slice(20, 32)}`; +} + +function checkpointName(checkpoint) { + return `checkpoint-${generationName(checkpoint.epoch)}-${checkpoint.epochId}.json`; +} + +function epochName(checkpoint) { + return `epoch-${generationName(checkpoint.epoch)}-${checkpoint.epochId}`; +} + +function claimPath(context, generation) { + return join(context.epochDirectory, `claim-${generationName(generation)}.json`); +} + +function heartbeatPath(context, claim) { + return join(context.epochDirectory, `heartbeat-${generationName(claim.generation)}-${claim.token}.json`); +} + +function terminalPath(context, claim) { + return join(context.epochDirectory, `terminal-${generationName(claim.generation)}-${claim.token}.json`); +} + +function appliedPath(context, claim) { + return join(context.epochDirectory, `applied-${generationName(claim.generation)}-${claim.token}.json`); +} + +function transitionPath(context, baseDigest) { + return join(context.epochDirectory, `transition-${baseDigest}.json`); +} + +function validateClaim(value, context, stateMaxBytes) { + if ( + !value || value.schemaVersion !== LOCK_SCHEMA_VERSION || !Number.isSafeInteger(value.generation) || + value.generation < 1 || value.generation > MAX_OPERATION_GENERATIONS || !uuidPattern.test(value.token ?? "") || + !["normal", "rotation"].includes(value.type) + ) throw new Error("Consumer high-water operation claim is malformed."); + if (value.type === "normal") { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "type", "ownerPid", "createdAtMs"]) || + !Number.isSafeInteger(value.ownerPid) || value.ownerPid < 1 || + !Number.isSafeInteger(value.createdAtMs) || value.createdAtMs < 0 + ) throw new Error("Consumer high-water normal operation claim is malformed."); + return value; + } + if (!exactKeys(value, ["schemaVersion", "generation", "token", "type", "intent"]) || !context) { + throw new Error("Consumer high-water rotation operation claim is malformed."); + } + const intent = validateRotationIntent(value.intent, context, stateMaxBytes); + if (value.token !== intent.checkpoint.epochId) { + throw new Error("Consumer high-water rotation operation claim differs from its deterministic intent."); + } + return value; +} + +function validateHeartbeat(value, claim) { + if ( + claim.type !== "normal" || + !exactKeys(value, ["schemaVersion", "generation", "token", "refreshedAtMs"]) || + value.schemaVersion !== LOCK_SCHEMA_VERSION || value.generation !== claim.generation || value.token !== claim.token || + !Number.isSafeInteger(value.refreshedAtMs) || value.refreshedAtMs < claim.createdAtMs + ) throw new Error("Consumer high-water lock heartbeat is malformed."); + return value; +} + +function transactionFor(baseDigest, candidateBytes) { + return { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + baseDigest, + candidateDigest: digest(candidateBytes), + candidateBase64: candidateBytes.toString("base64"), + }; +} + +function validateTransaction(value, expectedBaseDigest, stateMaxBytes) { + if ( + !exactKeys(value, ["schemaVersion", "baseDigest", "candidateDigest", "candidateBase64"]) || + value.schemaVersion !== TRANSACTION_SCHEMA_VERSION || value.baseDigest !== expectedBaseDigest || + !/^[0-9a-f]{64}$/.test(value.candidateDigest ?? "") || typeof value.candidateBase64 !== "string" || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.candidateBase64) + ) throw new Error("Consumer high-water transaction is malformed."); + const candidateBytes = Buffer.from(value.candidateBase64, "base64"); + if ( + candidateBytes.length < 1 || candidateBytes.length > stateMaxBytes || + candidateBytes.toString("base64") !== value.candidateBase64 || digest(candidateBytes) !== value.candidateDigest || + value.candidateDigest === value.baseDigest + ) throw new Error("Consumer high-water transaction payload is malformed."); + return { value, candidateBytes }; +} + +function validateCheckpoint(value, stateMaxBytes) { + if ( + !exactKeys(value, [ + "schemaVersion", "epoch", "epochId", "previousCheckpointSha256", "previousTipSha256", + "historySha256", "anchorDigest", "anchorBase64", "retiredEpochDirectory", "sourceAuthoritySha256", + "sourceAuthorityTipDigest", "sourceAuthorityTipBase64", + ]) || value.schemaVersion !== CHECKPOINT_SCHEMA_VERSION || !Number.isSafeInteger(value.epoch) || value.epoch < 1 || + !uuidPattern.test(value.epochId ?? "") || !/^[0-9a-f]{64}$/.test(value.previousCheckpointSha256 ?? "") || + !/^[0-9a-f]{64}$/.test(value.previousTipSha256 ?? "") || !/^[0-9a-f]{64}$/.test(value.historySha256 ?? "") || + !/^[0-9a-f]{64}$/.test(value.anchorDigest ?? "") || !/^[0-9a-f]{64}$/.test(value.sourceAuthoritySha256 ?? "") || + !/^[0-9a-f]{64}$/.test(value.sourceAuthorityTipDigest ?? "") || + !(value.retiredEpochDirectory === null || epochPattern.test(value.retiredEpochDirectory)) || + !(value.anchorBase64 === null || typeof value.anchorBase64 === "string") || + !(value.sourceAuthorityTipBase64 === null || typeof value.sourceAuthorityTipBase64 === "string") + ) throw new Error("Consumer high-water journal checkpoint is malformed."); + let anchorBytes = null; + if (value.anchorBase64 !== null) { + if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.anchorBase64)) { + throw new Error("Consumer high-water journal checkpoint is malformed."); + } + anchorBytes = Buffer.from(value.anchorBase64, "base64"); + if ( + anchorBytes.length < 1 || anchorBytes.length > stateMaxBytes || anchorBytes.toString("base64") !== value.anchorBase64 || + digest(anchorBytes) !== value.anchorDigest + ) throw new Error("Consumer high-water journal checkpoint anchor is malformed."); + } else if (value.anchorDigest !== GENESIS_DIGEST) { + throw new Error("Consumer high-water journal checkpoint anchor is malformed."); + } + if (value.sourceAuthorityTipBase64 === null) { + if (value.sourceAuthorityTipDigest !== GENESIS_DIGEST) { + throw new Error("Consumer high-water checkpoint source-authority tip is malformed."); + } + } else { + if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.sourceAuthorityTipBase64)) { + throw new Error("Consumer high-water checkpoint source-authority tip is malformed."); + } + const sourceTip = Buffer.from(value.sourceAuthorityTipBase64, "base64"); + if ( + sourceTip.length < 1 || sourceTip.length > stateMaxBytes || + sourceTip.toString("base64") !== value.sourceAuthorityTipBase64 || digest(sourceTip) !== value.sourceAuthorityTipDigest + ) throw new Error("Consumer high-water checkpoint source-authority tip is malformed."); + } + if (value.epoch === 1) { + if ( + value.previousCheckpointSha256 !== GENESIS_DIGEST || value.previousTipSha256 !== GENESIS_DIGEST || + value.retiredEpochDirectory !== null + ) throw new Error("Consumer high-water genesis checkpoint is malformed."); + } else if (value.retiredEpochDirectory === null || value.previousTipSha256 !== value.anchorDigest) { + throw new Error("Consumer high-water rotated checkpoint is malformed."); + } + return { value, anchorBytes }; +} + +function validateRotationIntent(value, context, stateMaxBytes) { + if ( + !exactKeys(value, ["schemaVersion", "epoch", "epochId", "checkpointSha256", "tipSha256", "checkpoint"]) || + value.schemaVersion !== ROTATION_INTENT_SCHEMA_VERSION || value.epoch !== context.checkpoint.epoch || + value.epochId !== context.checkpoint.epochId || value.checkpointSha256 !== context.checkpointDigest || + !/^[0-9a-f]{64}$/.test(value.tipSha256 ?? "") + ) throw new Error("Consumer high-water rotation intent is malformed."); + const checkpoint = validateCheckpoint(value.checkpoint, stateMaxBytes).value; + if ( + checkpoint.epoch !== context.checkpoint.epoch + 1 || + checkpoint.previousCheckpointSha256 !== context.checkpointDigest || + checkpoint.previousTipSha256 !== value.tipSha256 || checkpoint.anchorDigest !== value.tipSha256 || + checkpoint.retiredEpochDirectory !== basename(context.epochDirectory) || + checkpoint.sourceAuthoritySha256 !== context.checkpoint.sourceAuthoritySha256 || + checkpoint.sourceAuthorityTipDigest !== context.checkpoint.sourceAuthorityTipDigest || + checkpoint.sourceAuthorityTipBase64 !== context.checkpoint.sourceAuthorityTipBase64 || + checkpoint.historySha256 !== digest(Buffer.from( + `${context.checkpoint.historySha256}:${context.checkpointDigest}:${value.tipSha256}`, + )) + ) throw new Error("Consumer high-water rotation intent does not anchor the exact epoch and tip."); + return value; +} + +function validateTerminal(value, claim, stateMaxBytes) { + const common = ["schemaVersion", "generation", "token", "outcome"]; + if ( + claim.type !== "normal" || !value || value.schemaVersion !== LOCK_SCHEMA_VERSION || value.generation !== claim.generation || + value.token !== claim.token || !["released", "retired", "commit"].includes(value.outcome) + ) throw new Error("Consumer high-water lock terminal marker is malformed."); + if (["released", "retired"].includes(value.outcome)) { + if (!exactKeys(value, common)) throw new Error("Consumer high-water lock terminal marker is malformed."); + return value; + } + if ( + !exactKeys(value, [...common, "transactions"]) || !Array.isArray(value.transactions) || + value.transactions.length < 1 || value.transactions.length > 2 + ) throw new Error("Consumer high-water lock commit marker is malformed."); + let expectedBase = value.transactions[0]?.baseDigest; + if (!/^[0-9a-f]{64}$/.test(expectedBase ?? "")) throw new Error("Consumer high-water lock commit marker is malformed."); + for (const transaction of value.transactions) { + validateTransaction(transaction, expectedBase, stateMaxBytes); + expectedBase = transaction.candidateDigest; + } + return value; +} + +function validateApplied(value, claim, terminal) { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "terminalSha256"]) || + value.schemaVersion !== LOCK_SCHEMA_VERSION || value.generation !== claim.generation || value.token !== claim.token || + terminal?.outcome !== "commit" || value.terminalSha256 !== digest(metadataBytes(terminal)) + ) throw new Error("Consumer high-water lock applied marker is malformed."); + return value; +} + +function validateLegacyClaim(value) { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "ownerPid", "createdAtMs"]) || + value.schemaVersion !== LEGACY_LOCK_SCHEMA_VERSION || !Number.isSafeInteger(value.generation) || value.generation < 1 || + !uuidPattern.test(value.token ?? "") || !Number.isSafeInteger(value.ownerPid) || value.ownerPid < 1 || + !Number.isSafeInteger(value.createdAtMs) || value.createdAtMs < 0 + ) throw new Error("Legacy consumer high-water lock claim is malformed."); + return value; +} + +function validateLegacyHeartbeat(value, claim) { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "refreshedAtMs"]) || + value.schemaVersion !== LEGACY_LOCK_SCHEMA_VERSION || value.generation !== claim.generation || + value.token !== claim.token || !Number.isSafeInteger(value.refreshedAtMs) || value.refreshedAtMs < claim.createdAtMs + ) throw new Error("Legacy consumer high-water heartbeat is malformed."); + return value; +} + +function validateLegacyTerminal(value, claim, stateMaxBytes) { + const common = ["schemaVersion", "generation", "token", "outcome"]; + if ( + !value || value.schemaVersion !== LEGACY_LOCK_SCHEMA_VERSION || value.generation !== claim.generation || + value.token !== claim.token || !["released", "retired", "commit"].includes(value.outcome) + ) throw new Error("Legacy consumer high-water terminal marker is malformed."); + if (value.outcome !== "commit") { + if (!exactKeys(value, common)) throw new Error("Legacy consumer high-water terminal marker is malformed."); + return value; + } + if ( + !exactKeys(value, [...common, "transactions"]) || !Array.isArray(value.transactions) || + value.transactions.length < 1 || value.transactions.length > 2 + ) throw new Error("Legacy consumer high-water commit marker is malformed."); + let expectedBase = value.transactions[0]?.baseDigest; + if (!/^[0-9a-f]{64}$/.test(expectedBase ?? "")) throw new Error("Legacy consumer high-water commit marker is malformed."); + for (const transaction of value.transactions) { + validateTransaction(transaction, expectedBase, stateMaxBytes); + expectedBase = transaction.candidateDigest; + } + return value; +} + +function validateLegacyApplied(value, claim, terminal) { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "terminalSha256"]) || + value.schemaVersion !== LEGACY_LOCK_SCHEMA_VERSION || value.generation !== claim.generation || + value.token !== claim.token || terminal?.outcome !== "commit" || + value.terminalSha256 !== digest(metadataBytes(terminal)) + ) throw new Error("Legacy consumer high-water applied marker is malformed."); + return value; +} + +function legacyGuardFor(statePath) { + return { + schemaVersion: LEGACY_GUARD_SCHEMA_VERSION, + kind: "pylon-consumer-legacy-lock-guard", + statePathSha256: digest(Buffer.from(statePath)), + }; +} + +function legacyRetirementMarkerFor(statePath, legacy) { + return { + schemaVersion: LEGACY_RETIREMENT_SCHEMA_VERSION, + kind: "pylon-consumer-v1-retirement", + statePathSha256: digest(Buffer.from(statePath)), + authoritySha256: legacy.authoritySha256, + tipSha256: legacy.tipDigest, + }; +} + +function validateLegacyRetirementMarker(value, statePath) { + if ( + !exactKeys(value, ["schemaVersion", "kind", "statePathSha256", "authoritySha256", "tipSha256"]) || + value.schemaVersion !== LEGACY_RETIREMENT_SCHEMA_VERSION || value.kind !== "pylon-consumer-v1-retirement" || + value.statePathSha256 !== digest(Buffer.from(statePath)) || + !/^[0-9a-f]{64}$/.test(value.authoritySha256 ?? "") || !/^[0-9a-f]{64}$/.test(value.tipSha256 ?? "") + ) throw new Error("Legacy consumer high-water retirement marker is malformed."); + return value; +} + +async function secureHandle(handle, stat, description, type, options) { + if ((type === "file" && !stat.isFile()) || (type === "directory" && !stat.isDirectory())) { + throw new Error(`${description} must be one real ${type}.`); + } + if (stat.uid !== options.currentUid) throw new Error(`${description} must be owned by the current uid.`); + const requiredMode = type === "directory" ? 0o700 : 0o600; + if ((stat.mode & 0o7777) !== requiredMode) { + throw new Error(`${description} must already have exact ${requiredMode.toString(8)} permissions before use.`); + } + return stat; +} + +async function secureDirectory(path, description, options) { + let handle; + try { + handle = await options.openFile( + path, + constants.O_RDONLY | (constants.O_DIRECTORY ?? 0) | (constants.O_NOFOLLOW ?? 0), + ); + } catch (error) { + if (["ELOOP", "ENOTDIR"].includes(error?.code)) throw new Error(`${description} must be one real directory.`); + throw error; + } + try { + await secureHandle(handle, await handle.stat(), description, "directory", options); + } finally { + await handle.close(); + } +} + +export async function syncConsumerStateDirectory(path, { openDirectory = open } = {}) { + let handle; + try { + handle = await openDirectory(path, "r"); + await handle.sync(); + } catch (error) { + if (!["EINVAL", "EPERM", "EISDIR"].includes(error?.code)) throw error; + } finally { + if (handle !== undefined) await handle.close(); + } +} + +export async function ensureDurableConsumerStateDirectory( + directory, + { lstatEntry = lstat, makeDirectory = mkdir, syncDirectory = syncConsumerStateDirectory, create = true } = {}, +) { + const absolute = resolve(directory); + const root = parse(absolute).root; + let parent = root; + const rootEntry = await lstatEntry(root); + if (!rootEntry.isDirectory()) throw new Error("Consumer high-water state directory must be one canonical real directory."); + const remainder = relative(root, absolute); + for (const component of remainder ? remainder.split(sep) : []) { + const current = join(parent, component); + let entry; + try { + entry = await lstatEntry(current); + } catch (error) { + if (error?.code !== "ENOENT" || !create) throw error; + try { + await makeDirectory(current, { mode: 0o700 }); + } catch (mkdirError) { + if (mkdirError?.code !== "EEXIST") throw mkdirError; + } + entry = await lstatEntry(current); + } + if (!entry.isDirectory() || entry.isSymbolicLink?.()) { + throw new Error("Consumer high-water state directory must be one canonical real directory."); + } + await syncDirectory(parent); + parent = current; + } + return absolute; +} + +async function ensureDirectory(path, description, options) { + try { + await options.makeDirectory(path, { mode: 0o700 }); + } catch (error) { + if (error?.code !== "EEXIST") throw error; + } + const entry = await options.lstatEntry(path); + if (!entry.isDirectory() || entry.isSymbolicLink?.()) throw new Error(`${description} must be one real directory.`); + await secureDirectory(path, description, options); + await options.syncDirectory(path); + await options.syncDirectory(dirname(path)); +} + +async function readSecureFile(path, maxBytes, description, options, minBytes = 1, hooks) { + return readBoundedRegularFile(path, { + maxBytes, + minBytes, + description, + openFile: options.openFile, + lstatEntry: options.lstatEntry, + hooks, + validateHandle: (handle, stat) => secureHandle(handle, stat, description, "file", options), + }); +} + +async function readExactMetadata(path, maxBytes, validate, description, options, budget) { + const bytes = await readSecureFile(path, maxBytes, description, options); + if (bytes === null) return null; + if (budget) { + budget.bytes += bytes.length; + if (budget.bytes > options.maxJournalBytes) throw new Error("Consumer high-water journal exceeds its safe byte bound."); + } + let value; + try { + value = validate(JSON.parse(bytes)); + } catch (error) { + if (error instanceof SyntaxError) throw new Error(`${description} is malformed.`); + throw error; + } + if (!bytes.equals(metadataBytes(value))) throw new Error(`${description} is not canonical.`); + return value; +} + +function temporaryName(targetPath, kind, writer, context) { + if (!/^[a-z0-9-]{1,40}$/.test(kind)) throw new Error("Consumer high-water temporary kind is malformed."); + const attempt = randomUUID().replaceAll("-", "").slice(0, 12); + return `.pylon-consumer-tmp-v1-p${process.pid}-e${context.checkpoint.epochId}-g${generationName(writer.generation)}` + + `-w${writer.token}-n${attempt}-k${kind}-t${digest(Buffer.from(resolve(targetPath)))}.tmp`; +} + +async function inspectTemporary(path, options) { + const match = temporaryPattern.exec(basename(path)); + if (!match) throw new Error("Consumer high-water journal contains an unexpected hidden entry."); + let handle; + try { + handle = await options.openFile(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + } catch (error) { + if (error?.code === "ENOENT") return null; + if (["ELOOP", "EISDIR"].includes(error?.code)) { + throw new Error("Consumer high-water owned temporary is not one regular non-symlink file."); + } + throw error; + } + try { + const stat = await secureHandle( + handle, + await handle.stat(), + "Consumer high-water owned temporary", + "file", + options, + ); + if (stat.size > options.metadataMaxBytes) throw new Error("Consumer high-water owned temporary exceeds its safe byte bound."); + } finally { + await handle.close(); + } + const kind = match[6]; + const allowedKinds = new Set([ + "checkpoint", "projection", "transition", "claim", "initial-heartbeat", "heartbeat", + "terminal-released", "terminal-retired", "terminal-commit", "applied", "legacy-guard", + "legacy-retirement", + ]); + if (!allowedKinds.has(kind)) throw new Error("Consumer high-water owned temporary target metadata is malformed."); + return { + path, + pid: Number(match[1]), + epochId: match[2], + generation: Number(match[3]), + token: match[4], + attempt: match[5], + kind, + targetSha256: match[7], + }; +} + +async function revalidateAuthority(context, operation, options) { + await options.hooks?.beforePathOperation?.({ + operation, + statePath: context.statePath, + lockDirectory: context.journalDirectory, + transactionDirectory: context.epochDirectory, + }); + await ensureDurableConsumerStateDirectory(dirname(context.statePath), { + ...options.directoryOperations, + create: false, + }); + await secureDirectory(dirname(context.statePath), "Consumer high-water state directory", options); + await secureDirectory(context.journalDirectory, "Consumer high-water journal directory", options); + await secureDirectory(context.temporaryDirectory, "Consumer high-water temporary directory", options); + await secureDirectory(context.epochDirectory, "Consumer high-water epoch directory", options); + const entries = await options.readDirectory(context.journalDirectory); + if (entries.length > MAX_JOURNAL_ROOT_ENTRIES + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water journal root exceeds its safe allocation bound."); + } + const checkpoints = entries.map((name) => ({ name, match: checkpointPattern.exec(name) })).filter((entry) => entry.match); + if (checkpoints.length < 1 || checkpoints.length > 2) throw new Error("Consumer high-water journal checkpoint set is malformed."); + checkpoints.sort((left, right) => Number(left.match[1]) - Number(right.match[1])); + if (checkpoints.at(-1).name !== basename(context.checkpointPath)) { + throw new Error("Consumer high-water journal epoch changed and fenced a paused writer."); + } + const current = await readExactMetadata( + context.checkpointPath, + options.metadataMaxBytes, + (value) => validateCheckpoint(value, options.stateMaxBytes).value, + "Consumer high-water journal checkpoint", + options, + ); + if (digest(metadataBytes(current)) !== context.checkpointDigest) { + throw new Error("Consumer high-water journal checkpoint changed and fenced a paused writer."); + } +} + +async function publishImmutable({ + path, + bytes, + directory, + kind, + context, + writer, + options, + revalidate = true, + beforeLink, +}) { + if (revalidate) await revalidateAuthority(context, kind, options); + const temporary = join(context.temporaryDirectory, temporaryName(path, kind, writer, context)); + let handle; + let linked = false; + try { + handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); + await handle.chmod?.(0o600); + await handle.writeFile(bytes); + await handle.sync(); + await handle.close(); + handle = undefined; + await options.hooks?.afterFileSync?.({ kind, path, temporary }); + await beforeLink?.(); + if (revalidate) await revalidateAuthority(context, `${kind}-link`, options); + try { + await options.linkFile(temporary, path); + linked = true; + } catch (error) { + if (error?.code !== "EEXIST") throw error; + } + if (linked) await options.hooks?.afterMetadataLink?.({ kind, path }); + await options.syncDirectory(directory); + await options.hooks?.afterMetadataDirectorySync?.({ kind, path, linked }); + return linked; + } finally { + if (handle !== undefined) await handle.close(); + await options.removeFile(temporary, { force: true }); + await options.syncDirectory(context.temporaryDirectory); + } +} + +async function publishMetadata(path, value, kind, context, writer, options) { + const created = await publishImmutable({ + path, + bytes: metadataBytes(value), + directory: dirname(path), + kind, + context, + writer, + options, + }); + if (created) return { value, created: true }; + await revalidateAuthority(context, `${kind}-existing`, options); + const existing = await readExactMetadata( + path, + options.metadataMaxBytes, + (candidate) => candidate, + "Consumer high-water lock metadata", + options, + ); + return { value: existing, created: false }; +} + +function genesisCheckpoint(statePath) { + const epochId = deterministicUuid(`pylon-consumer-journal:${statePath}`); + return { + schemaVersion: CHECKPOINT_SCHEMA_VERSION, + epoch: 1, + epochId, + previousCheckpointSha256: GENESIS_DIGEST, + previousTipSha256: GENESIS_DIGEST, + historySha256: digest(Buffer.from(`pylon-consumer-history:${digest(Buffer.from(statePath))}`)), + anchorDigest: GENESIS_DIGEST, + anchorBase64: null, + retiredEpochDirectory: null, + sourceAuthoritySha256: GENESIS_DIGEST, + sourceAuthorityTipDigest: GENESIS_DIGEST, + sourceAuthorityTipBase64: null, + }; +} + +async function scanJournalRoot(statePath, journalDirectory, options, replacementRetries = PROJECTION_RETRY_LIMIT) { + await secureDirectory(journalDirectory, "Consumer high-water journal directory", options); + await options.syncDirectory(journalDirectory); + const names = await options.readDirectory(journalDirectory); + if (names.length > MAX_JOURNAL_ROOT_ENTRIES + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water journal root exceeds its safe allocation bound."); + } + const checkpointEntries = []; + const epochEntries = []; + const temporaries = []; + let temporaryDirectorySeen = false; + for (const name of names) { + const path = join(journalDirectory, name); + if (name === TEMPORARY_DIRECTORY_NAME) { + if (temporaryDirectorySeen) throw new Error("Consumer high-water temporary namespace is duplicated."); + temporaryDirectorySeen = true; + const entry = await options.lstatEntry(path); + if (!entry.isDirectory() || entry.isSymbolicLink?.()) { + throw new Error("Consumer high-water temporary namespace must be one real directory."); + } + await secureDirectory(path, "Consumer high-water temporary directory", options); + const temporaryNames = await options.readDirectory(path); + if (temporaryNames.length > MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water temporary namespace exceeds its safe allocation bound."); + } + for (const temporaryName of temporaryNames) { + const temporary = await inspectTemporary(join(path, temporaryName), options); + if (temporary) temporaries.push(temporary); + } + continue; + } + const checkpointMatch = checkpointPattern.exec(name); + if (checkpointMatch) { + const checkpoint = await readExactMetadata( + path, + options.metadataMaxBytes, + (value) => validateCheckpoint(value, options.stateMaxBytes).value, + "Consumer high-water journal checkpoint", + options, + ); + if (checkpoint === null) { + const epoch = Number(checkpointMatch[1]); + const hasLaterCheckpoint = names.some((candidate) => { + const match = checkpointPattern.exec(candidate); + return match && Number(match[1]) > epoch; + }); + if (hasLaterCheckpoint && replacementRetries > 0) { + return scanJournalRoot(statePath, journalDirectory, options, replacementRetries - 1); + } + throw new Error("Consumer high-water journal lost its current checkpoint during an authenticated scan."); + } + if (checkpointName(checkpoint) !== name || checkpoint.epoch !== Number(checkpointMatch[1])) { + throw new Error("Consumer high-water journal checkpoint name is malformed."); + } + checkpointEntries.push({ name, path, checkpoint, digest: digest(metadataBytes(checkpoint)) }); + continue; + } + const epochMatch = epochPattern.exec(name); + if (epochMatch) { + let entry; + try { + entry = await options.lstatEntry(path); + } catch (error) { + const epoch = Number(epochMatch[1]); + const hasLaterEpoch = names.some((candidate) => { + const match = epochPattern.exec(candidate); + return match && Number(match[1]) > epoch; + }); + if (error?.code === "ENOENT" && hasLaterEpoch && replacementRetries > 0) { + return scanJournalRoot(statePath, journalDirectory, options, replacementRetries - 1); + } + throw error; + } + if (!entry.isDirectory() || entry.isSymbolicLink?.()) throw new Error("Consumer high-water epoch entry must be one real directory."); + try { + await secureDirectory(path, "Consumer high-water epoch directory", options); + } catch (error) { + const epoch = Number(epochMatch[1]); + const hasLaterEpoch = names.some((candidate) => { + const match = epochPattern.exec(candidate); + return match && Number(match[1]) > epoch; + }); + if (error?.code === "ENOENT" && hasLaterEpoch && replacementRetries > 0) { + return scanJournalRoot(statePath, journalDirectory, options, replacementRetries - 1); + } + throw error; + } + epochEntries.push({ name, path, epoch: Number(epochMatch[1]), epochId: epochMatch[2] }); + continue; + } + if (name.startsWith(".")) { + const temporary = await inspectTemporary(path, options); + if (temporary?.kind !== "checkpoint") { + throw new Error("Consumer high-water journal root contains an unexpected owned temporary."); + } + if (temporary) temporaries.push(temporary); + continue; + } + throw new Error("Consumer high-water journal root contains an unexpected entry."); + } + if (!temporaryDirectorySeen) throw new Error("Consumer high-water journal lacks its exact temporary namespace."); + const authoritativeEntries = checkpointEntries.length + epochEntries.length + 1; + if (authoritativeEntries > MAX_JOURNAL_ROOT_ENTRIES) { + throw new Error("Consumer high-water journal root exceeds its safe entry bound."); + } + checkpointEntries.sort((left, right) => left.checkpoint.epoch - right.checkpoint.epoch); + epochEntries.sort((left, right) => left.epoch - right.epoch); + if (checkpointEntries.length > 2 || epochEntries.length > 2) { + throw new Error("Consumer high-water journal root contains unbounded checkpoint metadata."); + } + for (let index = 1; index < checkpointEntries.length; index += 1) { + if (checkpointEntries[index - 1].checkpoint.epoch === checkpointEntries[index].checkpoint.epoch) { + throw new Error("Consumer high-water journal contains competing checkpoints for one parent epoch."); + } + if (checkpointEntries[index - 1].checkpoint.epoch + 1 !== checkpointEntries[index].checkpoint.epoch) { + throw new Error("Consumer high-water journal checkpoints are not contiguous."); + } + } + for (let index = 1; index < epochEntries.length; index += 1) { + if (epochEntries[index - 1].epoch === epochEntries[index].epoch) { + throw new Error("Consumer high-water journal contains competing epoch directories for one parent epoch."); + } + } + const head = checkpointEntries.at(-1) ?? null; + if (head) { + const previous = checkpointEntries.at(-2); + if (previous && ( + head.checkpoint.previousCheckpointSha256 !== previous.digest || + head.checkpoint.retiredEpochDirectory !== epochName(previous.checkpoint) || + head.checkpoint.sourceAuthoritySha256 !== previous.checkpoint.sourceAuthoritySha256 || + head.checkpoint.sourceAuthorityTipDigest !== previous.checkpoint.sourceAuthorityTipDigest || + head.checkpoint.sourceAuthorityTipBase64 !== previous.checkpoint.sourceAuthorityTipBase64 || + head.checkpoint.historySha256 !== digest(Buffer.from( + `${previous.checkpoint.historySha256}:${previous.digest}:${head.checkpoint.anchorDigest}`, + )) + )) throw new Error("Consumer high-water journal checkpoint does not anchor its exact predecessor."); + } + const missingHeadEpoch = head ? !epochEntries.some((entry) => entry.name === epochName(head.checkpoint)) : false; + return { checkpointEntries, epochEntries, temporaries, head, missingHeadEpoch }; +} + +async function initializeJournal( + statePath, + journalDirectory, + options, + bootstrapCheckpoint = genesisCheckpoint(statePath), + beforeCheckpointLink, +) { + let scan = await scanJournalRoot(statePath, journalDirectory, options); + if (scan.head) { + if (!scan.missingHeadEpoch) return scan; + if ( + scan.head.checkpoint.epoch !== 1 || !metadataBytes(scan.head.checkpoint).equals(metadataBytes(bootstrapCheckpoint)) || + scan.epochEntries.length !== 0 + ) throw new Error("Consumer high-water journal checkpoint lacks its exact epoch directory."); + await ensureDirectory( + join(journalDirectory, epochName(scan.head.checkpoint)), + "Consumer high-water epoch directory", + options, + ); + return scanJournalRoot(statePath, journalDirectory, options); + } + if (scan.epochEntries.length > 0) throw new Error("Consumer high-water journal contains an orphan epoch directory."); + const checkpoint = bootstrapCheckpoint; + const bootstrap = { generation: 0, token: checkpoint.epochId }; + const bootstrapContext = { + statePath, + journalDirectory, + checkpoint, + checkpointPath: join(journalDirectory, checkpointName(checkpoint)), + checkpointDigest: digest(metadataBytes(checkpoint)), + epochDirectory: join(journalDirectory, epochName(checkpoint)), + temporaryDirectory: join(journalDirectory, TEMPORARY_DIRECTORY_NAME), + }; + await publishImmutable({ + path: bootstrapContext.checkpointPath, + bytes: metadataBytes(checkpoint), + directory: journalDirectory, + kind: "checkpoint", + context: bootstrapContext, + writer: bootstrap, + options, + revalidate: false, + beforeLink: beforeCheckpointLink, + }); + await ensureDirectory(bootstrapContext.epochDirectory, "Consumer high-water epoch directory", options); + scan = await scanJournalRoot(statePath, journalDirectory, options); + if (!scan.head) throw new Error("Consumer high-water journal initialization did not publish a checkpoint."); + return scan; +} + +function contextFromHead(statePath, guardPath, journalDirectory, head) { + return { + statePath, + guardPath, + journalDirectory, + checkpoint: head.checkpoint, + checkpointPath: head.path, + checkpointDigest: head.digest, + epochDirectory: join(journalDirectory, epochName(head.checkpoint)), + temporaryDirectory: join(journalDirectory, TEMPORARY_DIRECTORY_NAME), + }; +} + +async function readProjection(context, operation, options) { + await revalidateAuthority(context, operation, options); + const bytes = await readSecureFile( + context.statePath, + options.stateMaxBytes, + "Consumer high-water state", + options, + 0, + options.hooks?.projectionRead, + ); + if (bytes === null) return { exists: false, bytes: null, sha256: null, malformed: false }; + if (bytes.length < 1) return { exists: true, bytes: null, sha256: null, malformed: true }; + return { exists: true, bytes, sha256: digest(bytes), malformed: false }; +} + +async function walkTransactions(context, options) { + await revalidateAuthority(context, "walk-transactions", options); + await options.syncDirectory(context.epochDirectory); + const entries = await options.readDirectory(context.epochDirectory); + if (entries.length > options.maxJournalEntries + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water epoch exceeds its safe allocation bound."); + } + const named = new Map(); + for (const name of entries) { + const match = transitionPattern.exec(name); + if (match) { + if (named.has(match[1])) throw new Error("Consumer high-water journal contains a duplicate transition."); + named.set(match[1], name); + if (named.size > options.maxTransactionDepth) { + throw new Error("Consumer high-water transaction chain exceeds its safe entry bound."); + } + } + } + const visited = new Set(); + let tipDigest = context.checkpoint.anchorDigest; + let tipBytes = validateCheckpoint(context.checkpoint, options.stateMaxBytes).anchorBytes; + const budget = { bytes: 0 }; + for (let depth = 0; named.has(tipDigest); depth += 1) { + if (depth >= options.maxTransactionDepth || visited.has(tipDigest)) { + throw new Error("Consumer high-water transaction chain is cyclic or exceeds its safe bound."); + } + visited.add(tipDigest); + const path = transitionPath(context, tipDigest); + await revalidateAuthority(context, "read-transition", options); + const value = await readExactMetadata( + path, + options.metadataMaxBytes, + (candidate) => validateTransaction(candidate, tipDigest, options.stateMaxBytes).value, + "Consumer high-water transaction", + options, + budget, + ); + const validated = validateTransaction(value, tipDigest, options.stateMaxBytes); + tipDigest = value.candidateDigest; + tipBytes = validated.candidateBytes; + } + if (visited.size !== named.size) throw new Error("Consumer high-water transaction chain contains an unreachable transition."); + return { tipDigest, tipBytes, length: visited.size }; +} + +function isProjectionReplacementTransient(error) { + return error?.code === "ENOENT" || error?.message === "Consumer high-water state changed while it was read."; +} + +function isCommitHelperReplacementTransient(error) { + return isProjectionReplacementTransient(error) || [ + "Consumer high-water journal epoch changed and fenced a paused writer.", + "Consumer high-water journal checkpoint changed and fenced a paused writer.", + ].includes(error?.message); +} + +async function repairProjection(context, initialTip, options, writer = options.activeWriter) { + let tip = initialTip; + for (let attempt = 0; attempt < PROJECTION_RETRY_LIMIT; attempt += 1) { + if (tip.tipBytes === null) return tip; + try { + const projection = await readProjection(context, "projection-read", options); + if (projection.sha256 !== tip.tipDigest) { + await options.hooks?.beforeProjectionWrite?.({ tipDigest: tip.tipDigest }); + await revalidateAuthority(context, "projection-write", options); + const temporary = join(context.temporaryDirectory, temporaryName(context.statePath, "projection", writer, context)); + let handle; + try { + handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); + await handle.chmod?.(0o600); + await handle.writeFile(tip.tipBytes); + await handle.sync(); + await handle.close(); + handle = undefined; + await options.hooks?.afterProjectionFileSync?.({ tipDigest: tip.tipDigest, temporary }); + await revalidateAuthority(context, "projection-rename", options); + await options.renameFile(temporary, context.statePath); + await options.hooks?.afterProjectionRename?.({ tipDigest: tip.tipDigest }); + await options.syncDirectory(context.temporaryDirectory); + await options.syncDirectory(dirname(context.statePath)); + await options.hooks?.afterProjectionDirectorySync?.({ tipDigest: tip.tipDigest }); + } finally { + if (handle !== undefined) await handle.close(); + await options.removeFile(temporary, { force: true }); + await options.syncDirectory(context.temporaryDirectory); + } + } + } catch (error) { + if (!isProjectionReplacementTransient(error)) throw error; + await revalidateAuthority(context, "projection-retry-authentication", options); + tip = await walkTransactions(context, options); + continue; + } + const latest = await walkTransactions(context, options); + if (latest.tipDigest === tip.tipDigest) return latest; + tip = latest; + } + throw new Error("Consumer high-water projection could not catch up with its immutable transaction tip."); +} + +async function publishTransition(context, transaction, claim, options) { + validateTransaction(transaction, transaction.baseDigest, options.stateMaxBytes); + const path = transitionPath(context, transaction.baseDigest); + const result = await publishMetadata(path, transaction, "transition", context, claim, options); + const existing = validateTransaction(result.value, transaction.baseDigest, options.stateMaxBytes).value; + if (!metadataBytes(existing).equals(metadataBytes(transaction))) { + throw new Error("Consumer high-water transaction lost its immutable base-digest compare-and-set."); + } +} + +async function scanEpoch(context, options) { + await revalidateAuthority(context, "scan-claims", options); + await options.syncDirectory(context.epochDirectory); + const names = await options.readDirectory(context.epochDirectory); + if (names.length > options.maxJournalEntries + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water epoch exceeds its safe allocation bound."); + } + const claimNames = new Map(); + const heartbeatNames = new Map(); + const terminalNames = new Map(); + const appliedNames = new Map(); + const temporaries = []; + let authoritativeEntryCount = 0; + for (const name of names) { + let match; + if ((match = claimPattern.exec(name))) { + if (claimNames.has(Number(match[1]))) throw new Error("Consumer high-water lock contains a duplicate claim."); + claimNames.set(Number(match[1]), name); + authoritativeEntryCount += 1; + } else if ((match = heartbeatPattern.exec(name))) { + heartbeatNames.set(`${Number(match[1])}:${match[2]}`, name); + authoritativeEntryCount += 1; + } else if ((match = terminalPattern.exec(name))) { + terminalNames.set(`${Number(match[1])}:${match[2]}`, name); + authoritativeEntryCount += 1; + } else if ((match = appliedPattern.exec(name))) { + appliedNames.set(`${Number(match[1])}:${match[2]}`, name); + authoritativeEntryCount += 1; + } else if (transitionPattern.test(name)) { + // Validated by the transaction walk before any state decision. + authoritativeEntryCount += 1; + } else if (name.startsWith(".")) { + const temporary = await inspectTemporary(join(context.epochDirectory, name), options); + if (temporary && ["checkpoint", "projection", "legacy-guard"].includes(temporary.kind)) { + throw new Error("Consumer high-water epoch contains an unexpected owned temporary."); + } + if (temporary) temporaries.push(temporary); + } else { + throw new Error("Consumer high-water epoch contains a malformed or unexpected entry."); + } + } + if (authoritativeEntryCount > options.maxJournalEntries) { + throw new Error("Consumer high-water epoch exceeds its safe entry bound."); + } + const budget = { bytes: 0 }; + const claims = []; + const byKey = new Map(); + for (const [generation, name] of [...claimNames].sort((left, right) => left[0] - right[0])) { + const claim = await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateClaim(value, context, options.stateMaxBytes), + "Consumer high-water operation claim", + options, + budget, + ); + if (claim.generation !== generation || name !== `claim-${generationName(generation)}.json`) { + throw new Error("Consumer high-water lock claim name differs from its exact generation."); + } + claims.push(claim); + byKey.set(`${generation}:${claim.token}`, claim); + } + if (claims.length > MAX_OPERATION_GENERATIONS) throw new Error("Consumer high-water operation generation bound is exhausted."); + for (let index = 0; index < claims.length; index += 1) { + if (claims[index].generation !== index + 1) throw new Error("Consumer high-water lock generations are not contiguous."); + } + for (const [key, name] of heartbeatNames) { + const claim = byKey.get(key); + if (!claim) throw new Error("Consumer high-water epoch contains an orphan heartbeat entry."); + await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateHeartbeat(value, claim), + "Consumer high-water lock heartbeat", + options, + budget, + ); + } + const terminals = new Map(); + for (const [key, name] of terminalNames) { + const claim = byKey.get(key); + if (!claim) throw new Error("Consumer high-water epoch contains an orphan terminal entry."); + terminals.set(key, await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateTerminal(value, claim, options.stateMaxBytes), + "Consumer high-water lock terminal marker", + options, + budget, + )); + } + const appliedClaims = new Set(); + for (const [key, name] of appliedNames) { + const claim = byKey.get(key); + const terminal = terminals.get(key); + if (!claim || !terminal) throw new Error("Consumer high-water epoch contains an orphan applied entry."); + await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateApplied(value, claim, terminal), + "Consumer high-water lock applied marker", + options, + budget, + ); + appliedClaims.add(key); + } + for (const claim of claims.slice(0, -1)) { + const key = `${claim.generation}:${claim.token}`; + const terminal = terminals.get(key); + if (claim.type === "rotation" || !terminal || (terminal.outcome === "commit" && !appliedClaims.has(key))) { + throw new Error("Consumer high-water operation generations crossed an unresolved earlier slot."); + } + } + return { claims, terminals, temporaries }; +} + +async function readTerminal(context, claim, options) { + await revalidateAuthority(context, "read-terminal", options); + return readExactMetadata( + terminalPath(context, claim), + options.metadataMaxBytes, + (value) => validateTerminal(value, claim, options.stateMaxBytes), + "Consumer high-water lock terminal marker", + options, + ); +} + +async function readHeartbeat(context, claim, options) { + await revalidateAuthority(context, "read-heartbeat", options); + const heartbeat = await readExactMetadata( + heartbeatPath(context, claim), + options.metadataMaxBytes, + (value) => validateHeartbeat(value, claim), + "Consumer high-water lock heartbeat", + options, + ); + return heartbeat ?? { ...claim, refreshedAtMs: claim.createdAtMs }; +} + +async function publishTerminal(context, claim, wanted, options) { + const result = await publishMetadata( + terminalPath(context, claim), + wanted, + `terminal-${wanted.outcome}`, + context, + claim, + options, + ); + return validateTerminal(result.value, claim, options.stateMaxBytes); +} + +async function refreshHeartbeat(context, claim, options) { + if (await readTerminal(context, claim, options) !== null) return false; + const value = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + refreshedAtMs: options.now(), + }; + const path = heartbeatPath(context, claim); + await revalidateAuthority(context, "heartbeat", options); + const temporary = join(context.temporaryDirectory, temporaryName(path, "heartbeat", claim, context)); + let handle; + try { + handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); + await handle.chmod?.(0o600); + await handle.writeFile(metadataBytes(value)); + await handle.sync(); + await handle.close(); + handle = undefined; + if (await readTerminal(context, claim, options) !== null) return false; + await revalidateAuthority(context, "heartbeat-rename", options); + await options.renameFile(temporary, path); + await options.syncDirectory(context.temporaryDirectory); + await options.syncDirectory(context.epochDirectory); + return true; + } finally { + if (handle !== undefined) await handle.close(); + await options.removeFile(temporary, { force: true }); + await options.syncDirectory(context.temporaryDirectory); + } +} + +function defaultHeartbeatScheduler({ interval, beat }) { + let stopped = false; + let timer; + let pending = Promise.resolve(); + const arm = () => { + if (stopped) return; + timer = setTimeout(() => { + pending = beat().catch(() => false).finally(arm); + }, interval); + timer.unref?.(); + }; + arm(); + return async () => { + stopped = true; + clearTimeout(timer); + await pending; + }; +} + +async function publishApplied(context, claim, terminal, options) { + const value = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + terminalSha256: digest(metadataBytes(terminal)), + }; + const result = await publishMetadata(appliedPath(context, claim), value, "applied", context, claim, options); + validateApplied(result.value, claim, terminal); + await options.hooks?.afterApplied?.({ claim, terminal }); +} + +async function readApplied(context, claim, terminal, options) { + await revalidateAuthority(context, "read-applied", options); + return readExactMetadata( + appliedPath(context, claim), + options.metadataMaxBytes, + (value) => validateApplied(value, claim, terminal), + "Consumer high-water lock applied marker", + options, + ); +} + +async function finishCommitAtAuthenticatedDescendant(context, options) { + const scan = await scanJournalRoot(context.statePath, context.journalDirectory, options); + const head = scan.head; + if ( + !head || scan.missingHeadEpoch || head.checkpoint.epoch !== context.checkpoint.epoch + 1 || + head.checkpoint.previousCheckpointSha256 !== context.checkpointDigest || + head.checkpoint.retiredEpochDirectory !== basename(context.epochDirectory) + ) return false; + const descendant = contextFromHead(context.statePath, context.guardPath, context.journalDirectory, head); + const tip = await walkTransactions(descendant, options); + await repairProjection(descendant, tip, options, { + generation: 0, + token: descendant.checkpoint.epochId, + type: "rotation", + }); + return true; +} + +async function finishCommit(context, claim, terminal, options) { + for (let attempt = 0; attempt < PROJECTION_RETRY_LIMIT; attempt += 1) { + try { + await readApplied(context, claim, terminal, options); + for (const transaction of terminal.transactions) await publishTransition(context, transaction, claim, options); + const tip = await walkTransactions(context, options); + await repairProjection(context, tip, options, claim); + await publishApplied(context, claim, terminal, options); + return; + } catch (error) { + if (!isCommitHelperReplacementTransient(error)) throw error; + try { + await revalidateAuthority(context, "finish-commit-retry-authentication", options); + await walkTransactions(context, options); + } catch (authenticationError) { + if (!isCommitHelperReplacementTransient(authenticationError)) throw authenticationError; + if (await finishCommitAtAuthenticatedDescendant(context, options)) return; + throw authenticationError; + } + } + } + throw new Error("Consumer high-water commit helper could not converge after bounded projection replacement retries."); +} + +function rotationCheckpoint(context, tip) { + const epochId = deterministicUuid( + `pylon-consumer-rotation-v2:${context.checkpointDigest}:${tip.tipDigest}`, + ); + const checkpoint = { + schemaVersion: CHECKPOINT_SCHEMA_VERSION, + epoch: context.checkpoint.epoch + 1, + epochId, + previousCheckpointSha256: context.checkpointDigest, + previousTipSha256: tip.tipDigest, + historySha256: digest(Buffer.from( + `${context.checkpoint.historySha256}:${context.checkpointDigest}:${tip.tipDigest}`, + )), + anchorDigest: tip.tipDigest, + anchorBase64: tip.tipBytes === null ? null : tip.tipBytes.toString("base64"), + retiredEpochDirectory: basename(context.epochDirectory), + sourceAuthoritySha256: context.checkpoint.sourceAuthoritySha256, + sourceAuthorityTipDigest: context.checkpoint.sourceAuthorityTipDigest, + sourceAuthorityTipBase64: context.checkpoint.sourceAuthorityTipBase64, + }; + validateCheckpoint(checkpoint, Number.MAX_SAFE_INTEGER); + return checkpoint; +} + +function rotationIntentFor(context, tip) { + return { + schemaVersion: ROTATION_INTENT_SCHEMA_VERSION, + epoch: context.checkpoint.epoch, + epochId: context.checkpoint.epochId, + checkpointSha256: context.checkpointDigest, + tipSha256: tip.tipDigest, + checkpoint: rotationCheckpoint(context, tip), + }; +} + +function rotationClaimFor(context, generation, tip) { + const intent = rotationIntentFor(context, tip); + return { + schemaVersion: LOCK_SCHEMA_VERSION, + generation, + token: intent.checkpoint.epochId, + type: "rotation", + intent, + }; +} + +async function effectiveTip(context, options) { + const chain = await walkTransactions(context, options); + if (chain.tipBytes !== null) return chain; + const projection = await readProjection(context, "rotation-legacy-state-read", options); + if (projection.malformed) throw new Error("Consumer high-water rotation cannot authenticate its legacy projection anchor."); + if (projection.bytes === null) return chain; + return { tipDigest: digest(projection.bytes), tipBytes: projection.bytes, length: chain.length }; +} + +async function scanRotationPublicationSet(context, checkpoint, options, requirePublished) { + const scan = await scanJournalRoot(context.statePath, context.journalDirectory, options); + const nextCheckpointPath = join(context.journalDirectory, checkpointName(checkpoint)); + const nextEpochPath = join(context.journalDirectory, epochName(checkpoint)); + const allowedCheckpointPaths = new Set([context.checkpointPath, nextCheckpointPath]); + const allowedEpochPaths = new Set([context.epochDirectory, nextEpochPath]); + const currentCheckpoint = scan.checkpointEntries.find((entry) => entry.path === context.checkpointPath); + const currentEpoch = scan.epochEntries.find((entry) => entry.path === context.epochDirectory); + if ( + scan.checkpointEntries.some((entry) => !allowedCheckpointPaths.has(entry.path)) || + scan.epochEntries.some((entry) => !allowedEpochPaths.has(entry.path)) || + (currentCheckpoint && currentCheckpoint.digest !== context.checkpointDigest) || + !scan.epochEntries.some((entry) => entry.path === nextEpochPath) || + (!requirePublished && (!currentCheckpoint || !currentEpoch)) + ) throw new Error("Consumer high-water rotation found a competing root or epoch publication."); + const published = scan.checkpointEntries.find((entry) => entry.path === nextCheckpointPath); + if (published && !metadataBytes(published.checkpoint).equals(metadataBytes(checkpoint))) { + throw new Error("Consumer high-water rotation found a competing checkpoint for the same epoch."); + } + if (requirePublished && (!published || scan.head?.path !== nextCheckpointPath || scan.missingHeadEpoch)) { + throw new Error("Consumer high-water rotation checkpoint did not become the unique complete journal head."); + } + return scan; +} + +async function finishRotationCheckpoint(context, checkpoint, writer, options) { + validateCheckpoint(checkpoint, options.stateMaxBytes); + if ( + checkpoint.epoch !== context.checkpoint.epoch + 1 || + checkpoint.previousCheckpointSha256 !== context.checkpointDigest || + checkpoint.retiredEpochDirectory !== basename(context.epochDirectory) || + checkpoint.sourceAuthoritySha256 !== context.checkpoint.sourceAuthoritySha256 || + checkpoint.sourceAuthorityTipDigest !== context.checkpoint.sourceAuthorityTipDigest || + checkpoint.sourceAuthorityTipBase64 !== context.checkpoint.sourceAuthorityTipBase64 || + checkpoint.historySha256 !== digest(Buffer.from( + `${context.checkpoint.historySha256}:${context.checkpointDigest}:${checkpoint.anchorDigest}`, + )) + ) throw new Error("Consumer high-water rotation does not anchor the exact current epoch."); + const tip = await effectiveTip(context, options); + const anchorBytes = validateCheckpoint(checkpoint, options.stateMaxBytes).anchorBytes; + if ( + checkpoint.previousTipSha256 !== tip.tipDigest || checkpoint.anchorDigest !== tip.tipDigest || + (anchorBytes === null ? tip.tipBytes !== null : !anchorBytes.equals(tip.tipBytes)) + ) throw new Error("Consumer high-water rotation does not anchor the exact immutable tip."); + const nextEpoch = join(context.journalDirectory, epochName(checkpoint)); + await ensureDirectory(nextEpoch, "Consumer high-water epoch directory", options); + await options.hooks?.afterRotationEpochSync?.({ checkpoint: structuredClone(checkpoint), nextEpoch }); + await secureDirectory(nextEpoch, "Consumer high-water next epoch directory", options); + await options.syncDirectory(nextEpoch); + if ((await options.readDirectory(nextEpoch)).length !== 0) { + throw new Error("Consumer high-water rotation found a competing next-epoch directory for the same parent."); + } + const nextPath = join(context.journalDirectory, checkpointName(checkpoint)); + await publishImmutable({ + path: nextPath, + bytes: metadataBytes(checkpoint), + directory: context.journalDirectory, + kind: "checkpoint", + context, + writer, + options, + beforeLink: () => scanRotationPublicationSet(context, checkpoint, options, false), + }); + await options.hooks?.afterRotationCheckpoint?.({ checkpoint: structuredClone(checkpoint), nextPath }); + await scanRotationPublicationSet(context, checkpoint, options, true); +} + +async function resolveLatestOperation(context, claim, options) { + if (claim.type === "rotation") { + await helpRotationOperation(context, claim, options); + return "rotated"; + } + const terminal = await readTerminal(context, claim, options); + if (terminal?.outcome === "commit") { + await finishCommit(context, claim, terminal, options); + return "resolved"; + } + if (terminal !== null) return "resolved"; + const heartbeat = await readHeartbeat(context, claim, options); + if (options.now() - heartbeat.refreshedAtMs < options.stale) return "active"; + await options.hooks?.afterObserveStale?.({ claim, heartbeat }); + const retired = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + outcome: "retired", + }; + const decision = await publishTerminal(context, claim, retired, options); + await options.hooks?.afterRetire?.({ claim, decision }); + if (decision.outcome === "commit") await finishCommit(context, claim, decision, options); + return "resolved"; +} + +function operationIdentity(claim) { + return claim ? `${claim.generation}:${claim.token}:${claim.type}` : null; +} + +function sameOperationClaim(left, right) { + return left === null + ? right === null + : right !== null && operationIdentity(left) === operationIdentity(right) && metadataBytes(left).equals(metadataBytes(right)); +} + +async function resolveOperationFrontier(context, options) { + const initial = await scanEpoch(context, options); + const latest = initial.claims.at(-1) ?? null; + if (latest) { + const outcome = await resolveLatestOperation(context, latest, options); + if (outcome === "rotated") return { rotated: true }; + if (outcome === "active") return { active: true }; + } + const scan = await scanEpoch(context, options); + if (!sameOperationClaim(scan.claims.at(-1) ?? null, latest)) return { retry: true }; + return { scan, frontier: latest, rotated: false, active: false }; +} + +async function tryCreateNormalClaim(context, generation, options) { + const claim = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation, + token: randomUUID(), + type: "normal", + ownerPid: process.pid, + createdAtMs: options.now(), + }; + const result = await publishMetadata(claimPath(context, generation), claim, "claim", context, claim, options); + if (!result.created) return null; + validateClaim(result.value, context, options.stateMaxBytes); + const heartbeat = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation, + token: claim.token, + refreshedAtMs: claim.createdAtMs, + }; + await publishMetadata(heartbeatPath(context, claim), heartbeat, "initial-heartbeat", context, claim, options); + await options.hooks?.afterClaim?.({ claim }); + return claim; +} + +async function tryCreateRotationClaim(context, generation, tip, options) { + const claim = rotationClaimFor(context, generation, tip); + await options.hooks?.beforeRotationDecision?.({ intent: structuredClone(claim.intent), claim: structuredClone(claim) }); + const result = await publishMetadata(claimPath(context, generation), claim, "claim", context, claim, options); + if (!result.created) return null; + validateClaim(result.value, context, options.stateMaxBytes); + await options.hooks?.afterRotationIntent?.({ intent: structuredClone(claim.intent), claim: structuredClone(claim) }); + return claim; +} + +async function acquireNormalOperation(context, options) { + for (;;) { + const frontier = await resolveOperationFrontier(context, options); + if (frontier.rotated) return { rotated: true }; + if (frontier.active) throw new Error(`Consumer high-water state is actively locked: ${context.journalDirectory}`); + if (frontier.retry) continue; + const nextGeneration = (frontier.scan.claims.at(-1)?.generation ?? 0) + 1; + if (nextGeneration > options.maxLockGenerations) { + throw new Error("Consumer high-water claim epoch is exhausted; run the consumer journal rotation command."); + } + const confirmation = await scanEpoch(context, options); + if (!sameOperationClaim(confirmation.claims.at(-1) ?? null, frontier.frontier)) continue; + const claim = await tryCreateNormalClaim(context, nextGeneration, options); + if (!claim) continue; + const afterClaim = await scanEpoch(context, options); + if (!sameOperationClaim(afterClaim.claims.at(-1) ?? null, claim)) { + throw new Error("Consumer high-water normal operation did not remain the unique latest slot."); + } + return { claim, temporaries: afterClaim.temporaries, rotated: false }; + } +} + +function temporaryIsFenced(temporary, context, writer) { + if (temporary.epochId !== context.checkpoint.epochId) return true; + if (writer.generation === 0) { + return temporary.generation !== 0 || temporary.token !== writer.token; + } + if (temporary.generation === 0 || temporary.generation < writer.generation) return true; + return temporary.generation === writer.generation && temporary.token !== writer.token; +} + +function temporaryBelongsToRetiredClaim(temporary, context, epochAuthority) { + if (!epochAuthority || temporary.epochId !== context.checkpoint.epochId || temporary.generation === 0) return false; + const claim = epochAuthority.claims.find((candidate) => ( + candidate.generation === temporary.generation && candidate.token === temporary.token + )); + return claim !== undefined && epochAuthority.terminals.has(`${claim.generation}:${claim.token}`); +} + +function temporaryProcessIsAlive(temporary, options) { + try { + options.processKill(temporary.pid, 0); + return true; + } catch (error) { + if (error?.code === "ESRCH") return false; + if (error?.code === "EPERM") return true; + throw error; + } +} + +async function cleanupAuthority( + context, + writer, + rootScan, + epochTemporaries, + options, + requireQuiescent, + epochAuthority = null, + allowedNextEpoch = null, +) { + await revalidateAuthority(context, "cleanup", options); + const candidatesByPath = new Map( + [...rootScan.temporaries, ...epochTemporaries].map((temporary) => [temporary.path, temporary]), + ); + const parentNames = await options.readDirectory(dirname(context.statePath)); + const targetDigests = new Set([digest(Buffer.from(resolve(context.statePath))), digest(Buffer.from(resolve(context.guardPath)))]); + for (const name of parentNames) { + if (!name.startsWith(".pylon-consumer-tmp-v1-")) continue; + const temporary = await inspectTemporary(join(dirname(context.statePath), name), options); + if (!temporary || !targetDigests.has(temporary.targetSha256)) continue; + const expectedKind = temporary.targetSha256 === digest(Buffer.from(resolve(context.statePath))) + ? "projection" + : "legacy-guard"; + if (temporary.kind !== expectedKind) { + throw new Error("Consumer high-water state directory contains an unexpected owned temporary."); + } + candidatesByPath.set(temporary.path, temporary); + } + for (const temporary of candidatesByPath.values()) { + const fenced = temporaryIsFenced(temporary, context, writer); + if (!fenced && temporary.token !== writer.token) { + throw new Error("Consumer high-water journal contains a live or future owned temporary."); + } + if (!fenced) { + if (!requireQuiescent) continue; + if (temporaryProcessIsAlive(temporary, options)) { + if (writer.type === "rotation" && temporary.generation === writer.generation && temporary.token === writer.token) { + continue; + } + throw new Error("Consumer high-water journal rotation operation is pending until every prior owned temporary writer quiesces."); + } + await options.removeFile(temporary.path, { force: true }); + await options.syncDirectory(dirname(temporary.path)); + continue; + } + const retiredClaimTemporary = writer.generation === 0 && + temporaryBelongsToRetiredClaim(temporary, context, epochAuthority); + if (!retiredClaimTemporary && temporaryProcessIsAlive(temporary, options)) { + if (requireQuiescent) { + throw new Error("Consumer high-water journal rotation operation is pending until every prior owned temporary writer quiesces."); + } + continue; + } + await options.removeFile(temporary.path, { force: true }); + await options.syncDirectory(dirname(temporary.path)); + } + let retiredEpochDeferred = false; + for (const epoch of rootScan.epochEntries) { + if (epoch.name === basename(context.epochDirectory)) continue; + if (allowedNextEpoch !== null && epoch.name === allowedNextEpoch) continue; + if (epoch.name !== context.checkpoint.retiredEpochDirectory) { + throw new Error("Consumer high-water journal contains an orphan epoch directory."); + } + let retiredNames; + try { + retiredNames = await options.readDirectory(epoch.path); + } catch (error) { + if (error?.code === "ENOENT") continue; + throw error; + } + if (retiredNames.length > options.maxJournalEntries + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water retired epoch exceeds its safe allocation bound."); + } + const retiredTemporaries = []; + for (const name of retiredNames) { + const path = join(epoch.path, name); + let entry; + try { + entry = await options.lstatEntry(path); + } catch (error) { + if (error?.code === "ENOENT") continue; + throw error; + } + if (entry.isSymbolicLink?.() || (!entry.isFile() && !entry.isDirectory())) { + throw new Error("Consumer high-water retired epoch contains an unsafe entry."); + } + if (name.startsWith(".")) { + const temporary = await inspectTemporary(path, options); + if (temporary) retiredTemporaries.push(temporary); + } else if ( + !claimPattern.test(name) && !heartbeatPattern.test(name) && !terminalPattern.test(name) && + !appliedPattern.test(name) && !transitionPattern.test(name) + ) { + throw new Error("Consumer high-water retired epoch contains an unexpected entry."); + } else if (!entry.isFile()) { + throw new Error("Consumer high-water retired epoch metadata must be regular files."); + } + } + if (retiredTemporaries.some((temporary) => temporaryProcessIsAlive(temporary, options))) { + if (requireQuiescent) { + throw new Error("Consumer high-water journal rotation operation is pending until every retired temporary writer quiesces."); + } + retiredEpochDeferred = true; + continue; + } + await options.removeFile(epoch.path, { recursive: true, force: true }); + await options.syncDirectory(context.journalDirectory); + } + for (const entry of rootScan.checkpointEntries) { + if (entry.path === context.checkpointPath) continue; + if ( + entry.digest !== context.checkpoint.previousCheckpointSha256 || + epochName(entry.checkpoint) !== context.checkpoint.retiredEpochDirectory + ) throw new Error("Consumer high-water journal contains an orphan checkpoint entry."); + if (retiredEpochDeferred) continue; + await options.removeFile(entry.path, { force: true }); + await options.syncDirectory(context.journalDirectory); + } + const final = await scanJournalRoot(context.statePath, context.journalDirectory, options); + const allowedCheckpoints = retiredEpochDeferred ? 2 : 1; + const expectedEpochs = new Set([context.epochDirectory]); + if (retiredEpochDeferred) { + expectedEpochs.add(join(context.journalDirectory, context.checkpoint.retiredEpochDirectory)); + } + if ( + allowedNextEpoch !== null && + final.epochEntries.some((entry) => entry.name === allowedNextEpoch) + ) expectedEpochs.add(join(context.journalDirectory, allowedNextEpoch)); + if ( + final.checkpointEntries.length !== allowedCheckpoints || final.epochEntries.length !== expectedEpochs.size || + final.temporaries.some((temporary) => !temporaryProcessIsAlive(temporary, options)) || + final.head?.path !== context.checkpointPath || + final.epochEntries.some((entry) => !expectedEpochs.has(entry.path)) + ) throw new Error("Consumer high-water journal did not converge to one bounded current epoch."); +} + +async function helpRotationOperation(context, claim, options) { + if (claim.type !== "rotation") throw new Error("Consumer high-water rotation helper requires one rotation operation slot."); + const intent = validateRotationIntent(claim.intent, context, options.stateMaxBytes); + const completedBeforeHelp = await completedRotationResult(context, intent, options).catch(() => null); + if (completedBeforeHelp) return true; + try { + const scan = await scanEpoch(context, options); + const latest = scan.claims.at(-1); + if (operationIdentity(latest) !== operationIdentity(claim) || !metadataBytes(latest).equals(metadataBytes(claim))) { + throw new Error("Consumer high-water rotation operation is not the unique latest slot."); + } + const tip = await effectiveTip(context, options); + if (tip.tipDigest !== intent.tipSha256) { + throw new Error("Consumer high-water rotation operation no longer matches its exact authoritative tip."); + } + const rootScan = await scanJournalRoot(context.statePath, context.journalDirectory, options); + const nextEpochName = epochName(intent.checkpoint); + await cleanupAuthority( + context, + claim, + rootScan, + scan.temporaries, + options, + true, + scan, + nextEpochName, + ); + await finishRotationCheckpoint(context, intent.checkpoint, claim, options); + return true; + } catch (error) { + const completed = await completedRotationResult(context, intent, options).catch(() => null); + if (completed) return true; + throw error; + } +} + +async function inspectLegacyGuard(context, options) { + let entry; + try { + entry = await options.lstatEntry(context.guardPath); + } catch (error) { + if (error?.code === "ENOENT") return "absent"; + throw error; + } + if (entry.isDirectory() && !entry.isSymbolicLink?.()) { + if (await lstatOrNull(join(context.guardPath, LEGACY_RETIREMENT_MARKER_NAME), options) === null) { + throw new Error( + `Legacy consumer lock directory exists at ${context.guardPath}. Stop every legacy proper-lockfile client, ` + + "confirm that no owner remains, remove that directory manually, and retry.", + ); + } + await secureDirectory(context.guardPath, "Legacy consumer high-water lock directory", options); + const marker = await readExactMetadata( + join(context.guardPath, LEGACY_RETIREMENT_MARKER_NAME), + options.metadataMaxBytes, + (value) => validateLegacyRetirementMarker(value, context.statePath), + "Legacy consumer high-water retirement marker", + options, + ); + await options.syncDirectory(context.guardPath); + await options.syncDirectory(dirname(context.guardPath)); + return "retirement-marker"; + } + if (!entry.isFile() || entry.isSymbolicLink?.()) { + throw new Error("Legacy consumer lock guard is not one exact regular non-symlink file."); + } + const expected = legacyGuardFor(context.statePath); + const actual = await readExactMetadata( + context.guardPath, + options.metadataMaxBytes, + (value) => value, + "Legacy consumer lock guard", + options, + ); + if (!metadataBytes(actual).equals(metadataBytes(expected))) { + throw new Error("Legacy consumer lock guard differs from the exact durable handoff guard."); + } + await options.syncDirectory(dirname(context.guardPath)); + return "guard"; +} + +async function ensureLegacyGuard(context, claim, options) { + if (["guard", "retirement-marker"].includes(await inspectLegacyGuard(context, options))) return; + const expected = legacyGuardFor(context.statePath); + await publishImmutable({ + path: context.guardPath, + bytes: metadataBytes(expected), + directory: dirname(context.guardPath), + kind: "legacy-guard", + context, + writer: claim, + options, + }); + if (await inspectLegacyGuard(context, options) !== "guard") { + throw new Error("Legacy consumer lock handoff did not publish the exact durable guard."); + } +} + +function normalizeOptions({ + stale = PYLON_CONSUMER_LOCK_STALE_MS, + update = PYLON_CONSUMER_LOCK_UPDATE_MS, + stateMaxBytes = DEFAULT_STATE_MAX_BYTES, + maxTransactionDepth = MAX_TRANSACTION_DEPTH, + maxLockGenerations = MAX_LOCK_GENERATIONS, + maxJournalBytes = DEFAULT_JOURNAL_MAX_BYTES, + now = Date.now, + startHeartbeat = defaultHeartbeatScheduler, + hooks, + directoryOperations = {}, + lstatEntry = lstat, + makeDirectory = mkdir, + syncDirectory = syncConsumerStateDirectory, + openFile = open, + linkFile = link, + readDirectory = readdir, + renameFile = rename, + removeFile = rm, + processKill = process.kill.bind(process), + currentUid = typeof process.getuid === "function" ? process.getuid() : null, +} = {}) { + if ( + !Number.isSafeInteger(stale) || !Number.isSafeInteger(update) || update < 1 || stale <= update || + !Number.isSafeInteger(stateMaxBytes) || stateMaxBytes < 1 || stateMaxBytes > MAX_STATE_BYTES || + !Number.isSafeInteger(maxTransactionDepth) || maxTransactionDepth < 1 || maxTransactionDepth > MAX_TRANSACTION_DEPTH || + !Number.isSafeInteger(maxLockGenerations) || maxLockGenerations < 2 || maxLockGenerations > MAX_LOCK_GENERATIONS || + !Number.isSafeInteger(maxJournalBytes) || maxJournalBytes < stateMaxBytes || maxJournalBytes > MAX_JOURNAL_BYTES || + !Number.isSafeInteger(currentUid) || currentUid < 0 + ) throw new Error("Consumer high-water lock timing, state-size, journal, or transaction bound is invalid."); + return { + stale, + update, + stateMaxBytes, + maxTransactionDepth, + maxLockGenerations, + maxJournalBytes, + maxJournalEntries: MAX_OPERATION_GENERATIONS * 4 + MAX_TRANSACTION_DEPTH + 32, + metadataMaxBytes: stateMaxBytes * 3 + 8192, + now, + startHeartbeat, + hooks, + directoryOperations, + lstatEntry, + makeDirectory, + syncDirectory, + openFile, + linkFile, + readDirectory, + renameFile, + removeFile, + processKill, + currentUid, + activeWriter: null, + }; +} + +function normalizeRotationOptions(rawOptions) { + const options = normalizeOptions(rawOptions); + options.stateMaxBytes = MAX_STATE_BYTES; + options.maxTransactionDepth = MAX_TRANSACTION_DEPTH; + options.maxLockGenerations = MAX_LOCK_GENERATIONS; + options.maxJournalBytes = MAX_JOURNAL_BYTES; + options.maxJournalEntries = MAX_OPERATION_GENERATIONS * 4 + MAX_TRANSACTION_DEPTH + 32; + options.metadataMaxBytes = MAX_STATE_BYTES * 3 + 8192; + return options; +} + +async function lstatOrNull(path, options) { + try { + return await options.lstatEntry(path); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } +} + +function legacyTerminalFileName(claim) { + return `terminal-${generationName(claim.generation)}-${claim.token}.json`; +} + +function legacyHeartbeatFileName(claim) { + return `heartbeat-${generationName(claim.generation)}-${claim.token}.json`; +} + +function legacyAppliedFileName(claim) { + return `applied-${generationName(claim.generation)}-${claim.token}.json`; +} + +function authorityDigest(entries, tipDigest, tipBytes) { + const hash = createHash("sha256"); + hash.update("pylon-consumer-v1-authority\0"); + const sorted = [...entries].sort((left, right) => { + if (left[0] < right[0]) return -1; + if (left[0] > right[0]) return 1; + return 0; + }); + for (const [name, bytes] of sorted) { + const nameBytes = Buffer.from(name); + const header = Buffer.alloc(12); + header.writeUInt32BE(nameBytes.length, 0); + header.writeBigUInt64BE(BigInt(bytes.length), 4); + hash.update(header); + hash.update(nameBytes); + hash.update(bytes); + } + hash.update(Buffer.from(`tip:${tipDigest}:`)); + if (tipBytes !== null) hash.update(tipBytes); + return hash.digest("hex"); +} + +async function readLegacyAuthority(statePath, lockDirectory, transactionDirectory, options) { + await secureDirectory(lockDirectory, "Legacy consumer high-water lock directory", options); + await secureDirectory(transactionDirectory, "Legacy consumer high-water transaction directory", options); + await options.syncDirectory(lockDirectory); + await options.syncDirectory(transactionDirectory); + const transactionNames = await options.readDirectory(transactionDirectory); + if (transactionNames.length > options.maxTransactionDepth) { + throw new Error("Legacy consumer high-water transaction directory exceeds its safe entry bound."); + } + const budget = { bytes: 0 }; + const authorityEntries = []; + const actualTransactions = new Map(); + for (const name of transactionNames) { + const match = legacyTransitionPattern.exec(name); + if (!match || actualTransactions.has(match[1])) { + throw new Error("Legacy consumer high-water transaction directory contains a malformed or extra entry."); + } + const value = await readExactMetadata( + join(transactionDirectory, name), + options.metadataMaxBytes, + (candidate) => validateTransaction(candidate, match[1], options.stateMaxBytes).value, + "Legacy consumer high-water transaction", + options, + budget, + ); + actualTransactions.set(match[1], value); + authorityEntries.push([`transactions/${name}`, metadataBytes(value)]); + } + const lockNames = await options.readDirectory(lockDirectory); + if (lockNames.length > MAX_OPERATION_GENERATIONS * 4 + 1) { + throw new Error("Legacy consumer high-water lock directory exceeds its safe entry bound."); + } + const claimNames = new Map(); + const heartbeatNames = new Map(); + const terminalNames = new Map(); + const appliedNames = new Map(); + let retirementMarker = null; + for (const name of lockNames) { + let match; + if (name === LEGACY_RETIREMENT_MARKER_NAME) { + if (retirementMarker !== null) throw new Error("Legacy consumer high-water retirement marker is duplicated."); + retirementMarker = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + (value) => validateLegacyRetirementMarker(value, statePath), + "Legacy consumer high-water retirement marker", + options, + budget, + ); + continue; + } + if ((match = claimPattern.exec(name))) claimNames.set(Number(match[1]), name); + else if ((match = heartbeatPattern.exec(name))) heartbeatNames.set(`${Number(match[1])}:${match[2]}`, name); + else if ((match = terminalPattern.exec(name))) terminalNames.set(`${Number(match[1])}:${match[2]}`, name); + else if ((match = appliedPattern.exec(name))) appliedNames.set(`${Number(match[1])}:${match[2]}`, name); + else throw new Error("Legacy consumer high-water lock directory contains a malformed or extra entry."); + } + const claims = []; + const byKey = new Map(); + for (const [generation, name] of [...claimNames].sort((left, right) => left[0] - right[0])) { + const claim = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + validateLegacyClaim, + "Legacy consumer high-water lock claim", + options, + budget, + ); + if (claim.generation !== generation || name !== `claim-${generationName(generation)}.json`) { + throw new Error("Legacy consumer high-water claim name differs from its exact generation."); + } + claims.push(claim); + byKey.set(`${claim.generation}:${claim.token}`, claim); + authorityEntries.push([`lock/${name}`, metadataBytes(claim)]); + } + if (claims.length > options.maxLockGenerations) throw new Error("Legacy consumer high-water claim bound is exhausted."); + for (let index = 0; index < claims.length; index += 1) { + if (claims[index].generation !== index + 1) throw new Error("Legacy consumer high-water claims are not contiguous."); + } + for (const [key, name] of heartbeatNames) { + const claim = byKey.get(key); + if (!claim || name !== legacyHeartbeatFileName(claim)) { + throw new Error("Legacy consumer high-water lock contains an orphan heartbeat."); + } + const heartbeat = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + (value) => validateLegacyHeartbeat(value, claim), + "Legacy consumer high-water heartbeat", + options, + budget, + ); + authorityEntries.push([`lock/${name}`, metadataBytes(heartbeat)]); + } + const terminals = new Map(); + for (const [key, name] of terminalNames) { + const claim = byKey.get(key); + if (!claim || name !== legacyTerminalFileName(claim)) { + throw new Error("Legacy consumer high-water lock contains an orphan terminal marker."); + } + const terminal = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + (value) => validateLegacyTerminal(value, claim, options.stateMaxBytes), + "Legacy consumer high-water terminal marker", + options, + budget, + ); + terminals.set(key, terminal); + authorityEntries.push([`lock/${name}`, metadataBytes(terminal)]); + } + for (const claim of claims) { + const key = `${claim.generation}:${claim.token}`; + if (!heartbeatNames.has(key)) throw new Error("Legacy consumer high-water claim lacks its exact heartbeat."); + } + const appliedTerminals = new Set(); + for (const [key, name] of appliedNames) { + const claim = byKey.get(key); + const terminal = terminals.get(key); + if (!claim || !terminal || name !== legacyAppliedFileName(claim)) { + throw new Error("Legacy consumer high-water lock contains an orphan applied marker."); + } + const applied = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + (value) => validateLegacyApplied(value, claim, terminal), + "Legacy consumer high-water applied marker", + options, + budget, + ); + authorityEntries.push([`lock/${name}`, metadataBytes(applied)]); + appliedTerminals.add(key); + } + const decidedTransactions = new Map(); + const decidedDigests = new Set([GENESIS_DIGEST]); + let tipDigest = GENESIS_DIGEST; + let tipBytes = null; + let decidedLength = 0; + for (const claim of claims) { + const terminal = terminals.get(`${claim.generation}:${claim.token}`); + if (!terminal || terminal.outcome !== "commit") continue; + for (const transaction of terminal.transactions) { + if (transaction.baseDigest !== tipDigest) { + throw new Error("Legacy consumer high-water commit decisions do not form one exact authoritative chain."); + } + const prior = decidedTransactions.get(transaction.baseDigest); + if (prior && !metadataBytes(prior).equals(metadataBytes(transaction))) { + throw new Error("Legacy consumer high-water commit decisions equivocate at one base digest."); + } + decidedTransactions.set(transaction.baseDigest, transaction); + const validated = validateTransaction(transaction, tipDigest, options.stateMaxBytes); + tipDigest = transaction.candidateDigest; + tipBytes = validated.candidateBytes; + decidedDigests.add(tipDigest); + decidedLength += 1; + if (decidedLength > options.maxTransactionDepth) { + throw new Error("Legacy consumer high-water decisions exceed their safe transaction bound."); + } + } + } + let actualDigest = GENESIS_DIGEST; + let actualCount = 0; + while (actualTransactions.has(actualDigest)) { + const actual = actualTransactions.get(actualDigest); + const decided = decidedTransactions.get(actualDigest); + if (!decided || !metadataBytes(actual).equals(metadataBytes(decided))) { + throw new Error("Legacy consumer high-water transition lacks its exact immutable commit decision."); + } + actualDigest = actual.candidateDigest; + actualCount += 1; + if (actualCount > options.maxTransactionDepth) { + throw new Error("Legacy consumer high-water transition chain exceeds its safe bound."); + } + } + if (actualCount !== actualTransactions.size) { + throw new Error("Legacy consumer high-water transaction chain contains a corrupt, unreachable, or extra transition."); + } + for (const [baseDigest, actual] of actualTransactions) { + const decided = decidedTransactions.get(baseDigest); + if (!decided || !metadataBytes(actual).equals(metadataBytes(decided))) { + throw new Error("Legacy consumer high-water transition differs from its exact commit decision."); + } + } + for (const key of appliedTerminals) { + const terminal = terminals.get(key); + for (const transaction of terminal.transactions) { + const actual = actualTransactions.get(transaction.baseDigest); + if (!actual || !metadataBytes(actual).equals(metadataBytes(transaction))) { + throw new Error("Legacy consumer high-water applied marker is missing its completed transition."); + } + } + } + const recoveries = []; + for (const claim of claims) { + const key = `${claim.generation}:${claim.token}`; + const terminal = terminals.get(key); + if (!terminal) { + recoveries.push({ kind: "retire", claim }); + continue; + } + if (terminal.outcome === "commit" && !appliedTerminals.has(key)) { + recoveries.push({ + kind: "commit", + claim, + terminal, + missingTransactions: terminal.transactions.filter((transaction) => !actualTransactions.has(transaction.baseDigest)), + }); + } + } + const projection = await readSecureFile( + statePath, + options.stateMaxBytes, + "Legacy consumer high-water projection", + options, + 0, + ); + if (projection !== null && projection.length < 1) throw new Error("Legacy consumer high-water projection is malformed."); + if (decidedLength === 0 && projection !== null) { + tipBytes = projection; + tipDigest = digest(projection); + authorityEntries.push(["explicit-quiescent-projection", projection]); + } else if (projection !== null && !decidedDigests.has(digest(projection))) { + throw new Error("Legacy consumer high-water projection is not an authenticated prefix of its immutable authority."); + } + + if (budget.bytes > options.maxJournalBytes) { + throw new Error("Legacy consumer high-water authority exceeds its safe byte bound."); + } + const authoritySha256 = authorityDigest(authorityEntries, tipDigest, tipBytes); + if (retirementMarker !== null) { + const expectedMarker = legacyRetirementMarkerFor(statePath, { authoritySha256, tipDigest }); + if (!metadataBytes(retirementMarker).equals(metadataBytes(expectedMarker))) { + throw new Error("Legacy consumer high-water retirement marker conflicts with the exact pre-marker authority or tip."); + } + if (recoveries.length !== 0) { + throw new Error("Legacy consumer high-water retirement marker was published before its authority became quiescent."); + } + } + return { + tipDigest, + tipBytes, + length: decidedLength, + authoritySha256, + authorityEntries, + recoveries, + retirementMarker, + }; +} + +function migrationCheckpoint(statePath, legacy) { + const checkpoint = { + schemaVersion: CHECKPOINT_SCHEMA_VERSION, + epoch: 1, + epochId: deterministicUuid(`pylon-consumer-v1-migration:${statePath}:${legacy.authoritySha256}:${legacy.tipDigest}`), + previousCheckpointSha256: GENESIS_DIGEST, + previousTipSha256: GENESIS_DIGEST, + historySha256: digest(Buffer.from( + `pylon-consumer-history:${digest(Buffer.from(statePath))}:v1:${legacy.authoritySha256}:${legacy.tipDigest}`, + )), + anchorDigest: legacy.tipDigest, + anchorBase64: legacy.tipBytes === null ? null : legacy.tipBytes.toString("base64"), + retiredEpochDirectory: null, + sourceAuthoritySha256: legacy.authoritySha256, + sourceAuthorityTipDigest: legacy.tipDigest, + sourceAuthorityTipBase64: legacy.tipBytes === null ? null : legacy.tipBytes.toString("base64"), + }; + validateCheckpoint(checkpoint, Number.MAX_SAFE_INTEGER); + return checkpoint; +} + +function sameLegacyAuthority(left, right) { + return left.authoritySha256 === right.authoritySha256 && left.tipDigest === right.tipDigest && + (left.tipBytes === null ? right.tipBytes === null : right.tipBytes !== null && left.tipBytes.equals(right.tipBytes)); +} + +function legacyOwnerIsDefinitivelyDead(claim, options) { + try { + options.processKill(claim.ownerPid, 0); + return false; + } catch (error) { + if (error?.code === "ESRCH") return true; + return false; + } +} + +function requireRecoverableLegacyOwners(legacy, options) { + for (const recovery of legacy.recoveries) { + if (!legacyOwnerIsDefinitivelyDead(recovery.claim, options)) { + throw new Error( + "Legacy consumer high-water migration is blocked by a live or uncertain incomplete v1 commit owner.", + ); + } + } +} + +function recoveredLegacyAuthorityEntries(legacy) { + const entries = []; + for (const recovery of legacy.recoveries) { + if (recovery.kind === "retire") { + const terminal = { + schemaVersion: LEGACY_LOCK_SCHEMA_VERSION, + generation: recovery.claim.generation, + token: recovery.claim.token, + outcome: "retired", + }; + entries.push([`lock/${legacyTerminalFileName(recovery.claim)}`, metadataBytes(terminal)]); + continue; + } + for (const transaction of recovery.missingTransactions) { + entries.push([`transactions/${transaction.baseDigest}.json`, metadataBytes(transaction)]); + } + const applied = { + schemaVersion: LEGACY_LOCK_SCHEMA_VERSION, + generation: recovery.claim.generation, + token: recovery.claim.token, + terminalSha256: digest(metadataBytes(recovery.terminal)), + }; + entries.push([`lock/${legacyAppliedFileName(recovery.claim)}`, metadataBytes(applied)]); + } + return entries; +} + +function expectedRecoveredLegacyAuthoritySha256(legacy) { + return authorityDigest( + [...legacy.authorityEntries, ...recoveredLegacyAuthorityEntries(legacy)], + legacy.tipDigest, + legacy.tipBytes, + ); +} + +function legacyAuthorityIsExactRecoveryProgress(previous, current) { + if ( + previous.tipDigest !== current.tipDigest || + (previous.tipBytes === null + ? current.tipBytes !== null + : current.tipBytes === null || !previous.tipBytes.equals(current.tipBytes)) + ) return false; + const required = new Map(previous.authorityEntries); + const allowed = new Map(recoveredLegacyAuthorityEntries(previous)); + const actual = new Map(current.authorityEntries); + if (required.size !== previous.authorityEntries.length || actual.size !== current.authorityEntries.length) return false; + for (const [name, bytes] of required) { + if (!actual.get(name)?.equals(bytes)) return false; + } + for (const [name, bytes] of actual) { + if (required.has(name)) continue; + if (!allowed.get(name)?.equals(bytes)) return false; + } + return true; +} +async function publishExactLegacyMetadata(path, value, validate, description, directory, context, writer, kind, options) { + await publishImmutable({ + path, + bytes: metadataBytes(value), + directory, + kind, + context, + writer, + options, + revalidate: false, + }); + const actual = await readExactMetadata(path, options.metadataMaxBytes, validate, description, options); + if (!metadataBytes(actual).equals(metadataBytes(value))) { + throw new Error(`${description} lost its immutable exact-value publication.`); + } +} + +async function helpLegacyAuthority(retiredLockDirectory, transactionDirectory, legacy, context, options) { + for (const recovery of legacy.recoveries) { + await secureDirectory(retiredLockDirectory, "Legacy consumer high-water lock directory", options); + if (await lstatOrNull(join(retiredLockDirectory, LEGACY_RETIREMENT_MARKER_NAME), options) !== null) { + throw new Error("Legacy consumer authority recovery cannot cross its immutable retirement marker."); + } + if (recovery.kind === "retire") { + const terminal = { + schemaVersion: LEGACY_LOCK_SCHEMA_VERSION, + generation: recovery.claim.generation, + token: recovery.claim.token, + outcome: "retired", + }; + await publishExactLegacyMetadata( + join(retiredLockDirectory, legacyTerminalFileName(recovery.claim)), + terminal, + (value) => validateLegacyTerminal(value, recovery.claim, options.stateMaxBytes), + "Legacy consumer high-water recovered terminal marker", + retiredLockDirectory, + context, + recovery.claim, + "terminal-retired", + options, + ); + continue; + } + for (const transaction of recovery.missingTransactions) { + await publishExactLegacyMetadata( + join(transactionDirectory, `${transaction.baseDigest}.json`), + transaction, + (value) => validateTransaction(value, transaction.baseDigest, options.stateMaxBytes).value, + "Legacy consumer high-water recovered transition", + transactionDirectory, + context, + recovery.claim, + "transition", + options, + ); + } + const applied = { + schemaVersion: LEGACY_LOCK_SCHEMA_VERSION, + generation: recovery.claim.generation, + token: recovery.claim.token, + terminalSha256: digest(metadataBytes(recovery.terminal)), + }; + await publishExactLegacyMetadata( + join(retiredLockDirectory, legacyAppliedFileName(recovery.claim)), + applied, + (value) => validateLegacyApplied(value, recovery.claim, recovery.terminal), + "Legacy consumer high-water recovered applied marker", + retiredLockDirectory, + context, + recovery.claim, + "applied", + options, + ); + } + await options.syncDirectory(retiredLockDirectory); + await options.syncDirectory(transactionDirectory); +} + +async function legacyMigrationSource(statePath, options) { + const guardPath = `${statePath}.lock`; + const retiredLockDirectory = `${statePath}.lock.v1-retired`; + const guardEntry = await lstatOrNull(guardPath, options); + const retiredEntry = await lstatOrNull(retiredLockDirectory, options); + if (retiredEntry && (!retiredEntry.isDirectory() || retiredEntry.isSymbolicLink?.())) { + throw new Error("Prior retired v1 consumer lock authority must be one real directory and is never replaced."); + } + if (guardEntry?.isDirectory() && !guardEntry.isSymbolicLink?.() && retiredEntry) { + throw new Error("Live and retired v1 consumer lock authority both exist; migration fails closed."); + } + if (retiredEntry) { + if (guardEntry !== null && (!guardEntry.isFile() || guardEntry.isSymbolicLink?.())) { + throw new Error("Prior retired v1 consumer authority has an unsafe or ambiguous live lock path."); + } + return { guardPath, sourceLockDirectory: retiredLockDirectory, layout: "prior-retired", guardEntry }; + } + if (guardEntry?.isDirectory() && !guardEntry.isSymbolicLink?.()) { + return { guardPath, sourceLockDirectory: guardPath, layout: "in-place", guardEntry }; + } + throw new Error("Prior v1 consumer lock authority is absent, unsafe, or ambiguous."); +} + +async function publishLegacyRetirementMarker(source, legacy, context, options) { + if (legacy.retirementMarker !== null) return legacy; + if (legacy.recoveries.length !== 0) { + throw new Error("Legacy consumer high-water authority must be quiescent before retirement marker publication."); + } + const markerPath = join(source.sourceLockDirectory, LEGACY_RETIREMENT_MARKER_NAME); + const marker = legacyRetirementMarkerFor(context.statePath, legacy); + const authenticateBeforeMarkerLink = async () => { + const currentSource = await legacyMigrationSource(context.statePath, options); + if (currentSource.layout !== "in-place" || currentSource.sourceLockDirectory !== source.sourceLockDirectory) { + throw new Error("Legacy consumer high-water source changed before retirement marker publication."); + } + const current = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if (current.retirementMarker !== null) { + if (sameLegacyAuthority(legacy, current)) { + throw Object.assign(new Error("Concurrent migration already published the exact retirement marker."), { + code: "PYLON_EXACT_RETIREMENT_JOIN", + }); + } + throw new Error("Legacy consumer high-water authority changed before retirement marker publication."); + } + if (!sameLegacyAuthority(legacy, current) || current.recoveries.length !== 0) { + throw new Error("Legacy consumer high-water authority changed before retirement marker publication."); + } + requireRecoverableLegacyOwners(current, options); + }; + try { + await publishImmutable({ + path: markerPath, + bytes: metadataBytes(marker), + directory: source.sourceLockDirectory, + kind: "legacy-retirement", + context, + writer: { generation: 0, token: context.checkpoint.epochId, type: "rotation" }, + options, + revalidate: false, + beforeLink: authenticateBeforeMarkerLink, + }); + } catch (error) { + if (error?.code !== "PYLON_EXACT_RETIREMENT_JOIN") throw error; + const joined = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if (joined.retirementMarker === null || !sameLegacyAuthority(legacy, joined)) throw error; + } + await options.syncDirectory(source.sourceLockDirectory); + await options.syncDirectory(dirname(source.sourceLockDirectory)); + const guarded = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if (guarded.retirementMarker === null || !sameLegacyAuthority(legacy, guarded)) { + throw new Error("Legacy consumer high-water retirement marker does not authenticate its exact pre-marker authority."); + } + await options.hooks?.afterMigrationRetirementMarker?.({ markerPath, marker: structuredClone(marker) }); + await options.hooks?.afterMigrationGuard?.({ guardPath: source.guardPath, markerPath }); + return guarded; +} + +async function publishPriorLayoutGuard(source, legacy, context, options) { + if (source.guardEntry === null) { + await publishImmutable({ + path: source.guardPath, + bytes: metadataBytes(legacyGuardFor(context.statePath)), + directory: dirname(source.guardPath), + kind: "legacy-guard", + context, + writer: { generation: 0, token: context.checkpoint.epochId, type: "rotation" }, + options, + revalidate: false, + beforeLink: async () => { + const current = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if (!sameLegacyAuthority(legacy, current) || current.recoveries.length !== 0) { + throw new Error("Prior retired v1 authority changed before downgrade guard publication."); + } + }, + }); + } + if (await inspectLegacyGuard(context, options) !== "guard") { + throw new Error("Prior retired v1 authority lacks its exact permanent downgrade guard."); + } + await options.syncDirectory(dirname(source.guardPath)); + await options.hooks?.afterMigrationGuard?.({ guardPath: source.guardPath }); +} + +async function validateMigratedAuthority(context, options) { + if (context.checkpoint.epoch < 1 || context.checkpoint.sourceAuthoritySha256 === GENESIS_DIGEST) { + throw new Error("Prior v1 consumer authority exists but the v2 journal lacks an authenticated migration checkpoint."); + } + const source = await legacyMigrationSource(context.statePath, options); + const sourceTipBytes = context.checkpoint.sourceAuthorityTipBase64 === null + ? null + : Buffer.from(context.checkpoint.sourceAuthorityTipBase64, "base64"); + const legacy = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if ( + legacy.authoritySha256 !== context.checkpoint.sourceAuthoritySha256 || + legacy.tipDigest !== context.checkpoint.sourceAuthorityTipDigest || legacy.recoveries.length !== 0 || + (sourceTipBytes === null ? legacy.tipBytes !== null : !sourceTipBytes.equals(legacy.tipBytes)) + ) throw new Error("The v2 migration checkpoint does not authenticate the complete prior v1 authority and tip."); + const guardKind = await inspectLegacyGuard(context, options); + if ( + (source.layout === "in-place" && (guardKind !== "retirement-marker" || legacy.retirementMarker === null)) || + (source.layout === "prior-retired" && guardKind !== "guard") + ) throw new Error("Prior v1 consumer authority is not fenced by its exact permanent downgrade guard."); + return { source, legacy }; +} + +export async function migrateConsumerStateJournal(statePath, rawOptions = {}) { + if (typeof statePath !== "string" || !statePath) throw new Error("A consumer-local state path is required for v1 journal migration."); + const options = normalizeOptions(rawOptions); + const absoluteStatePath = resolve(statePath); + const directory = dirname(absoluteStatePath); + await ensureDurableConsumerStateDirectory(directory, options.directoryOperations); + await secureDirectory(directory, "Consumer high-water state directory", options); + const transactionDirectory = `${absoluteStatePath}.transactions`; + const transactionEntry = await lstatOrNull(transactionDirectory, options); + if (!transactionEntry) throw new Error("No prior v1 consumer transaction authority exists to migrate."); + if (!transactionEntry.isDirectory() || transactionEntry.isSymbolicLink?.()) { + throw new Error("Prior v1 consumer transaction authority must be one real directory."); + } + let source = await legacyMigrationSource(absoluteStatePath, options); + let legacy = await readLegacyAuthority(absoluteStatePath, source.sourceLockDirectory, transactionDirectory, options); + const initialCheckpoint = migrationCheckpoint(absoluteStatePath, legacy); + await options.hooks?.afterMigrationAuthorityRead?.({ + checkpoint: structuredClone(initialCheckpoint), + legacy: structuredClone(legacy), + }); + + const journalDirectory = `${absoluteStatePath}.journal`; + await ensureDirectory(journalDirectory, "Consumer high-water journal directory", options); + const temporaryDirectory = join(journalDirectory, TEMPORARY_DIRECTORY_NAME); + await ensureDirectory(temporaryDirectory, "Consumer high-water temporary directory", options); + + source = await legacyMigrationSource(absoluteStatePath, options); + const currentLegacy = await readLegacyAuthority(absoluteStatePath, source.sourceLockDirectory, transactionDirectory, options); + if (!sameLegacyAuthority(legacy, currentLegacy) && !legacyAuthorityIsExactRecoveryProgress(legacy, currentLegacy)) { + throw new Error("Concurrent v1 migration changed the exact authenticated legacy authority or tip."); + } + legacy = currentLegacy; + if (source.layout === "in-place" && legacy.retirementMarker === null) { + requireRecoverableLegacyOwners(legacy, options); + const expectedRecoveredAuthoritySha256 = expectedRecoveredLegacyAuthoritySha256(legacy); + const recoveryCheckpoint = migrationCheckpoint(absoluteStatePath, legacy); + try { + await helpLegacyAuthority(source.sourceLockDirectory, transactionDirectory, legacy, { + statePath: absoluteStatePath, + guardPath: source.guardPath, + journalDirectory, + checkpoint: recoveryCheckpoint, + checkpointPath: join(journalDirectory, checkpointName(recoveryCheckpoint)), + checkpointDigest: digest(metadataBytes(recoveryCheckpoint)), + epochDirectory: join(journalDirectory, epochName(recoveryCheckpoint)), + temporaryDirectory, + }, options); + } catch (error) { + const joined = await readLegacyAuthority(absoluteStatePath, source.sourceLockDirectory, transactionDirectory, options); + if (joined.retirementMarker === null) throw error; + legacy = joined; + } + if (legacy.retirementMarker === null) { + const recovered = await readLegacyAuthority(absoluteStatePath, source.sourceLockDirectory, transactionDirectory, options); + if ( + recovered.recoveries.length !== 0 || recovered.authoritySha256 !== expectedRecoveredAuthoritySha256 || + recovered.tipDigest !== legacy.tipDigest || + (legacy.tipBytes === null ? recovered.tipBytes !== null : recovered.tipBytes === null || !legacy.tipBytes.equals(recovered.tipBytes)) + ) throw new Error("V1 authority recovery did not produce only the exact authenticated dead-owner completion."); + legacy = recovered; + } + } + if (source.layout === "prior-retired" && legacy.recoveries.length !== 0) { + throw new Error("Interrupted prior-layout v1 migration authority is supported read-only and still requires recovery."); + } + if (legacy.recoveries.length !== 0) { + throw new Error("Legacy consumer high-water authority is not quiescent after recovery."); + } + + let checkpoint = migrationCheckpoint(absoluteStatePath, legacy); + let bootstrapContext = { + statePath: absoluteStatePath, + guardPath: source.guardPath, + journalDirectory, + checkpoint, + checkpointPath: join(journalDirectory, checkpointName(checkpoint)), + checkpointDigest: digest(metadataBytes(checkpoint)), + epochDirectory: join(journalDirectory, epochName(checkpoint)), + temporaryDirectory, + }; + if (source.layout === "in-place") { + legacy = await publishLegacyRetirementMarker(source, legacy, bootstrapContext, options); + } else { + await publishPriorLayoutGuard(source, legacy, bootstrapContext, options); + } + + const guardedLegacy = await readLegacyAuthority( + absoluteStatePath, + source.sourceLockDirectory, + transactionDirectory, + options, + ); + if (!sameLegacyAuthority(legacy, guardedLegacy) || guardedLegacy.recoveries.length !== 0) { + throw new Error("V1 authority mutated across its exact durable retirement handoff."); + } + checkpoint = migrationCheckpoint(absoluteStatePath, guardedLegacy); + bootstrapContext = { + ...bootstrapContext, + checkpoint, + checkpointPath: join(journalDirectory, checkpointName(checkpoint)), + checkpointDigest: digest(metadataBytes(checkpoint)), + epochDirectory: join(journalDirectory, epochName(checkpoint)), + }; + const authenticateBeforeCheckpointLink = async () => { + const currentSource = await legacyMigrationSource(absoluteStatePath, options); + if (currentSource.layout !== source.layout || currentSource.sourceLockDirectory !== source.sourceLockDirectory) { + throw new Error("V1 authority source changed immediately before migration checkpoint publication."); + } + const current = await readLegacyAuthority( + absoluteStatePath, + source.sourceLockDirectory, + transactionDirectory, + options, + ); + if (!sameLegacyAuthority(guardedLegacy, current) || current.recoveries.length !== 0) { + throw new Error("V1 authority mutated immediately before migration checkpoint publication."); + } + }; + const scan = await initializeJournal( + absoluteStatePath, + journalDirectory, + options, + checkpoint, + authenticateBeforeCheckpointLink, + ); + if (!scan.head || !metadataBytes(scan.head.checkpoint).equals(metadataBytes(checkpoint))) { + throw new Error("V1 migration encountered a different existing v2 journal checkpoint."); + } + const context = contextFromHead(absoluteStatePath, source.guardPath, journalDirectory, scan.head); + await validateMigratedAuthority(context, options); + await repairProjection(context, await walkTransactions(context, options), options, { + generation: 0, + token: checkpoint.epochId, + type: "rotation", + }); + await validateMigratedAuthority(context, options); + await options.hooks?.afterMigrationComplete?.({ checkpoint: structuredClone(checkpoint) }); + await validateMigratedAuthority(context, options); + return { epoch: 1, tipSha256: checkpoint.anchorDigest, sourceAuthoritySha256: checkpoint.sourceAuthoritySha256 }; +} + +async function prepareContext(statePath, options) { + const absoluteStatePath = resolve(statePath); + const directory = dirname(absoluteStatePath); + await ensureDurableConsumerStateDirectory(directory, options.directoryOperations); + await secureDirectory(directory, "Consumer high-water state directory", options); + const guardPath = `${absoluteStatePath}.lock`; + const retiredLockDirectory = `${absoluteStatePath}.lock.v1-retired`; + const journalDirectory = `${absoluteStatePath}.journal`; + const legacyTransactionDirectory = `${absoluteStatePath}.transactions`; + + // Detect every old-authority signal before creating a guard or a genesis journal. + const legacyEntry = await lstatOrNull(legacyTransactionDirectory, options); + const guardEntry = await lstatOrNull(guardPath, options); + const retiredEntry = await lstatOrNull(retiredLockDirectory, options); + const journalEntry = await lstatOrNull(journalDirectory, options); + if (legacyEntry && (!legacyEntry.isDirectory() || legacyEntry.isSymbolicLink?.())) { + throw new Error("Prior v1 consumer transaction authority must be one real directory."); + } + if (retiredEntry && (!retiredEntry.isDirectory() || retiredEntry.isSymbolicLink?.())) { + throw new Error("Prior retired v1 consumer lock authority must be one real directory and is never replaced."); + } + if (guardEntry && ( + guardEntry.isSymbolicLink?.() || (!guardEntry.isFile() && !guardEntry.isDirectory()) + )) throw new Error("Legacy consumer lock guard is not one exact regular non-symlink file."); + + let inPlaceMarkerEntry = null; + if (guardEntry?.isDirectory() && !guardEntry.isSymbolicLink?.()) { + const markerPath = join(guardPath, LEGACY_RETIREMENT_MARKER_NAME); + inPlaceMarkerEntry = await lstatOrNull(markerPath, options); + if (inPlaceMarkerEntry) { + if (!inPlaceMarkerEntry.isFile() || inPlaceMarkerEntry.isSymbolicLink?.()) { + throw new Error("Legacy consumer high-water retirement marker must be one real file."); + } + await readExactMetadata( + markerPath, + options.metadataMaxBytes, + (value) => validateLegacyRetirementMarker(value, absoluteStatePath), + "Legacy consumer high-water retirement marker", + options, + ); + } + } + + let scan = null; + if (journalEntry) { + if (!journalEntry.isDirectory() || journalEntry.isSymbolicLink?.()) { + throw new Error("Consumer high-water journal directory must be one real directory."); + } + await secureDirectory(journalDirectory, "Consumer high-water journal directory", options); + try { + scan = await scanJournalRoot(absoluteStatePath, journalDirectory, options); + } catch (error) { + if (error?.message !== "Consumer high-water journal lacks its exact temporary namespace.") throw error; + const names = await options.readDirectory(journalDirectory); + if (names.length === 1 && names[0] === TEMPORARY_DIRECTORY_NAME) { + scan = await scanJournalRoot(absoluteStatePath, journalDirectory, options); + } else if (names.length !== 0) { + throw error; + } + } + } + const hasInPlaceLegacyDirectory = guardEntry?.isDirectory() && !guardEntry.isSymbolicLink?.(); + const hasMigratedV2Head = scan?.head?.checkpoint.sourceAuthoritySha256 !== undefined && + scan.head.checkpoint.sourceAuthoritySha256 !== GENESIS_DIGEST; + const hasLegacySignal = legacyEntry !== null || retiredEntry !== null || hasInPlaceLegacyDirectory || hasMigratedV2Head; + if (hasLegacySignal) { + if (!legacyEntry) { + if (hasInPlaceLegacyDirectory && !inPlaceMarkerEntry && !retiredEntry && !hasMigratedV2Head) { + throw new Error( + `Legacy consumer lock directory exists at ${guardPath}. Stop every legacy proper-lockfile client, ` + + "confirm that no owner remains, remove that directory manually, and retry.", + ); + } + throw new Error("Prior v1 consumer authority is incomplete because its transaction namespace is missing."); + } + // This independently validates the selected live/in-place or prior-retired lock namespace. + await legacyMigrationSource(absoluteStatePath, options); + if (!journalEntry) { + throw new Error( + "Prior v1 consumer authority exists. Stop every old client and run the explicit quiescent consumer journal migration command.", + ); + } + if (!scan?.head || scan.missingHeadEpoch) { + throw new Error("Prior v1 authority has no complete authenticated v2 migration checkpoint."); + } + const context = contextFromHead(absoluteStatePath, guardPath, journalDirectory, scan.head); + await validateMigratedAuthority(context, options); + return { context, scan }; + } + + if (scan?.head) { + if (scan.missingHeadEpoch) scan = await initializeJournal(absoluteStatePath, journalDirectory, options); + return { context: contextFromHead(absoluteStatePath, guardPath, journalDirectory, scan.head), scan }; + } + await ensureDirectory(journalDirectory, "Consumer high-water journal directory", options); + await ensureDirectory(join(journalDirectory, TEMPORARY_DIRECTORY_NAME), "Consumer high-water temporary directory", options); + scan = await initializeJournal(absoluteStatePath, journalDirectory, options); + return { context: contextFromHead(absoluteStatePath, guardPath, journalDirectory, scan.head), scan }; +} + +async function runNormalLocked(statePath, action, rawOptions) { + const options = normalizeOptions(rawOptions); + for (;;) { + const prepared = await prepareContext(statePath, options); + const acquired = await acquireNormalOperation(prepared.context, options); + if (acquired.rotated) continue; + const { context } = prepared; + const { claim, temporaries } = acquired; + options.activeWriter = claim; + let terminal = null; + let heartbeatStopped = false; + const stopHeartbeat = options.startHeartbeat({ + interval: options.update, + beat: () => refreshHeartbeat(context, claim, options), + }); + const stopHeartbeatOnce = async () => { + if (heartbeatStopped) return; + heartbeatStopped = true; + await stopHeartbeat(); + }; + const release = async (cause) => { + if (terminal !== null) return; + const wanted = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + outcome: "released", + }; + terminal = await publishTerminal(context, claim, wanted, options); + if (terminal.outcome !== "released") { + throw new Error("Consumer high-water lock ownership was retired before release.", { cause }); + } + }; + try { + const rootScan = await scanJournalRoot(context.statePath, context.journalDirectory, options); + await cleanupAuthority(context, claim, rootScan, temporaries, options, false); + await ensureLegacyGuard(context, claim, options); + let chain = await walkTransactions(context, options); + let legacyBytes = null; + if (chain.tipBytes === null) { + const legacy = await readProjection(context, "legacy-state-read", options); + if (legacy.malformed) throw new Error("Consumer high-water state is malformed."); + legacyBytes = legacy.bytes; + } else { + chain = await repairProjection(context, chain, options); + } + const baseBytes = chain.tipBytes ?? legacyBytes; + const baseDigest = baseBytes === null ? GENESIS_DIGEST : digest(baseBytes); + let stagedCandidate = null; + let candidateWasStaged = false; + const commitTransactions = async (candidateBytes) => { + const transactions = []; + if (chain.tipBytes === null && legacyBytes !== null) { + transactions.push(transactionFor(GENESIS_DIGEST, legacyBytes)); + } + if (candidateBytes !== null && digest(candidateBytes) !== baseDigest) { + transactions.push(transactionFor(baseDigest, candidateBytes)); + } + if (transactions.length === 0) return false; + if (chain.length + transactions.length > options.maxTransactionDepth) { + throw new Error("Consumer high-water transaction epoch reached its safe bound; run the consumer journal rotation command."); + } + const wanted = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + outcome: "commit", + transactions, + }; + await options.hooks?.beforeCommitDecision?.({ claim, transactions }); + terminal = await publishTerminal(context, claim, wanted, options); + if (terminal.outcome !== "commit" || !metadataBytes(terminal).equals(metadataBytes(wanted))) { + throw new Error("Consumer high-water transaction lost ownership before its commit decision."); + } + await options.hooks?.afterCommitDecision?.({ claim, terminal }); + await finishCommit(context, claim, terminal, options); + return true; + }; + const transaction = Object.freeze({ + readStateBytes: () => baseBytes === null ? null : Buffer.from(baseBytes), + commitState: async (value) => { + if (terminal !== null || candidateWasStaged) { + throw new Error("Consumer high-water transaction already staged a candidate or has a terminal decision."); + } + const bytes = Buffer.isBuffer(value) ? Buffer.from(value) : Buffer.from(value); + if (bytes.length < 1 || bytes.length > options.stateMaxBytes) throw new Error("Consumer high-water state is malformed."); + stagedCandidate = bytes; + candidateWasStaged = true; + }, + }); + let result; + let actionError; + try { + result = await action(context.statePath, transaction); + } catch (error) { + actionError = error; + } + if (actionError === undefined && (candidateWasStaged || legacyBytes !== null)) { + await commitTransactions(candidateWasStaged ? stagedCandidate : null); + } + await stopHeartbeatOnce(); + await release(actionError); + if (actionError !== undefined) throw actionError; + return result; + } catch (error) { + await stopHeartbeatOnce(); + await release(error); + throw error; + } + } +} + +async function completedRotationResult(context, intent, options) { + const scan = await scanJournalRoot(context.statePath, context.journalDirectory, options); + if (scan.head && metadataBytes(scan.head.checkpoint).equals(metadataBytes(intent.checkpoint))) { + await scanRotationPublicationSet(context, intent.checkpoint, options, true); + return { epoch: intent.checkpoint.epoch, tipSha256: intent.checkpoint.anchorDigest }; + } + return null; +} + +async function recoverCompletedCurrentRotation(context, scan, options) { + if (context.checkpoint.epoch === 1 || scan.claims.length !== 0) return null; + const tip = await effectiveTip(context, options); + if (tip.length !== 0 || tip.tipDigest !== context.checkpoint.anchorDigest) return null; + const writer = { generation: 0, token: context.checkpoint.epochId, type: "rotation" }; + await repairProjection(context, tip, options, writer); + const rootScan = await scanJournalRoot(context.statePath, context.journalDirectory, options); + await cleanupAuthority(context, writer, rootScan, scan.temporaries, options, false, scan); + return { epoch: context.checkpoint.epoch, tipSha256: context.checkpoint.anchorDigest }; +} + +async function runRotation(statePath, rawOptions) { + const options = normalizeRotationOptions(rawOptions); + for (;;) { + const { context } = await prepareContext(statePath, options); + await ensureLegacyGuard(context, { generation: 0, token: context.checkpoint.epochId, type: "rotation" }, options); + const initialScan = await scanEpoch(context, options); + const completed = await recoverCompletedCurrentRotation(context, initialScan, options); + if (completed) return completed; + const frontier = await resolveOperationFrontier(context, options); + if (frontier.rotated) continue; + if (frontier.active) { + throw new Error("Consumer high-water state is actively locked; rotation will retry after the claim quiesces."); + } + if (frontier.retry) continue; + const nextGeneration = (frontier.scan.claims.at(-1)?.generation ?? 0) + 1; + if (nextGeneration > MAX_OPERATION_GENERATIONS) { + throw new Error("Consumer high-water operation epoch is exhausted and cannot publish its cap-exempt rotation slot."); + } + const tip = await effectiveTip(context, options); + const wanted = rotationClaimFor(context, nextGeneration, tip); + const confirmation = await scanEpoch(context, options); + if (!sameOperationClaim(confirmation.claims.at(-1) ?? null, frontier.frontier)) continue; + const confirmedTip = await effectiveTip(context, options); + const confirmed = rotationClaimFor(context, nextGeneration, confirmedTip); + if (!metadataBytes(confirmed).equals(metadataBytes(wanted))) continue; + let claim; + try { + claim = await tryCreateRotationClaim(context, nextGeneration, confirmedTip, options); + } catch (error) { + const completedResult = await completedRotationResult(context, confirmed.intent, options).catch(() => null); + if (completedResult) return completedResult; + throw error; + } + if (!claim) continue; + try { + await helpRotationOperation(context, claim, options); + } catch (error) { + const completedResult = await completedRotationResult(context, claim.intent, options).catch(() => null); + if (completedResult) return completedResult; + throw error; + } + await scanRotationPublicationSet(context, claim.intent.checkpoint, options, true); + return { epoch: claim.intent.checkpoint.epoch, tipSha256: claim.intent.checkpoint.anchorDigest }; + } +} + +export async function withConsumerStateLock(statePath, action, rawOptions = {}) { + if (typeof action !== "function") throw new Error("Consumer high-water lock action must be a function."); + return runNormalLocked(statePath, action, rawOptions); +} + +export async function rotateConsumerStateJournal(statePath, rawOptions = {}) { + if (typeof statePath !== "string" || !statePath) throw new Error("A consumer-local state path is required for journal rotation."); + return runRotation(statePath, rawOptions); +} diff --git a/scripts/lib/pylon-publication.mjs b/scripts/lib/pylon-publication.mjs new file mode 100644 index 0000000000..8649806907 --- /dev/null +++ b/scripts/lib/pylon-publication.mjs @@ -0,0 +1,812 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +import { + normalizeNpmVersion, + PYLON_RELEASE_MANIFEST, + PYLON_RELEASE_PACKAGES, + PYLON_RELEASE_RECIPE_REVISION, + PYLON_RELEASE_REPOSITORY, + releaseAssetFile, + releaseBuildId, + validateReleaseManifest, +} from "./pylon-release.mjs"; + +export const PYLON_PREVIEW_MANIFEST = "pylon-preview-channel-v1.json"; +export const PYLON_STABLE_MANIFEST = "pylon-stable-channel-v1.json"; +export const PYLON_PUBLICATION_SCHEMA_VERSION = 1; +export const PYLON_PUBLICATION_REPOSITORY = "pylon-code/prime-agent"; +export const PYLON_PUBLICATION_REF = "refs/heads/pylon"; +export const PYLON_PREVIEW_WORKFLOW = ".github/workflows/pylon-preview-release.yml"; +export const PYLON_STABLE_WORKFLOW = ".github/workflows/pylon-stable-release.yml"; +export const GITHUB_ACTIONS_APP_ID = 15368; +export const PYLON_REQUIRED_CHECKS = Object.freeze([ + Object.freeze({ context: "Check changelog fragment", appId: GITHUB_ACTIONS_APP_ID, workflowPath: ".github/workflows/changelog-merged-proof.yml" }), + Object.freeze({ context: "build-check-test", appId: GITHUB_ACTIONS_APP_ID, workflowPath: ".github/workflows/ci.yml" }), +]); + +function compactJsonSource(text) { + let result = ""; + let quoted = false; + let escaped = false; + for (const character of text) { + if (quoted) { + result += character; + if (escaped) escaped = false; + else if (character === "\\") escaped = true; + else if (character === '"') quoted = false; + } else if (character === '"') { + quoted = true; + result += character; + } else if (!/\s/.test(character)) result += character; + } + if (quoted || escaped) throw new Error("Pylon historical release recipe/publication policy registry has truncated JSON."); + return result; +} + +export function parseSupportedReleaseRecipeRegistry(text) { + if (typeof text !== "string") throw new Error("Pylon historical release recipe/publication policy registry must be JSON text."); + const registry = JSON.parse(text); + if (compactJsonSource(text) !== JSON.stringify(registry)) { + throw new Error("Pylon historical release recipe/publication policy registry has duplicate keys or noncanonical JSON tokens."); + } + const recipeKeys = [ + "recipeRevision", "manifestSchemaVersion", "nodeVersion", "npmVersion", "minimumNodeVersion", + ]; + const publicationPolicyKeys = [ + "publicationPolicyRevision", "previewWorkflowPath", "previewWorkflowSha256", "stableWorkflowPath", "stableWorkflowSha256", + ]; + if ( + !registry || Object.keys(registry).sort().join(",") !== "publicationPolicies,recipes,schemaVersion" || + registry.schemaVersion !== 1 || !Array.isArray(registry.recipes) || registry.recipes.length === 0 || + !Array.isArray(registry.publicationPolicies) || registry.publicationPolicies.length === 0 || + registry.recipes.some((recipe) => + !recipe || Object.keys(recipe).sort().join(",") !== recipeKeys.toSorted().join(",") || + !Number.isSafeInteger(recipe.recipeRevision) || recipe.recipeRevision < 1 || recipe.manifestSchemaVersion !== 1 || + ![recipe.nodeVersion, recipe.npmVersion, recipe.minimumNodeVersion].every((value) => /^\d+\.\d+\.\d+$/.test(value)) + ) || + registry.publicationPolicies.some((policy) => + !policy || Object.keys(policy).sort().join(",") !== publicationPolicyKeys.toSorted().join(",") || + !Number.isSafeInteger(policy.publicationPolicyRevision) || policy.publicationPolicyRevision < 1 || + policy.previewWorkflowPath !== PYLON_PREVIEW_WORKFLOW || policy.stableWorkflowPath !== PYLON_STABLE_WORKFLOW || + ![policy.previewWorkflowSha256, policy.stableWorkflowSha256].every((value) => /^[0-9a-f]{64}$/.test(value)) + ) || + new Set(registry.recipes.map((recipe) => recipe.recipeRevision)).size !== registry.recipes.length || + new Set(registry.publicationPolicies.map((policy) => policy.publicationPolicyRevision)).size !== registry.publicationPolicies.length + ) throw new Error("Pylon historical release recipe/publication policy registry is malformed."); + return registry; +} + +const supportedRecipeRegistry = parseSupportedReleaseRecipeRegistry( + readFileSync(fileURLToPath(new URL("../pylon-prime-supported-release-recipes-v1.json", import.meta.url)), "utf8"), +); +export const PYLON_SUPPORTED_RELEASE_RECIPES = Object.freeze( + supportedRecipeRegistry.recipes.map((recipe) => Object.freeze({ ...recipe })), +); +export const PYLON_SUPPORTED_PUBLICATION_POLICIES = Object.freeze( + supportedRecipeRegistry.publicationPolicies.map((policy) => Object.freeze({ ...policy })), +); + +const previewTagPattern = /^pylon-build-g([0-9a-f]{12})-r([1-9][0-9]*)$/; +const stableTagPattern = /^pylon-stable-([0-9]{6})-g([0-9a-f]{12})-r([1-9][0-9]*)$/; +const stableReservationTagPattern = /^pylon-stable-sequence-([0-9]{6})$/; + +function isPlainObject(value) { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function compareText(left, right) { + return left < right ? -1 : left > right ? 1 : 0; +} + +function canonicalValue(value) { + if (value === null || typeof value === "string" || typeof value === "boolean") return value; + if (typeof value === "number" && Number.isFinite(value)) return value; + if (Array.isArray(value)) return value.map(canonicalValue); + if (!isPlainObject(value)) throw new Error("Publication JSON contains an unsupported value."); + return Object.fromEntries( + Object.keys(value) + .sort(compareText) + .map((key) => { + if (value[key] === undefined) throw new Error("Publication JSON contains undefined."); + return [key, canonicalValue(value[key])]; + }), + ); +} + +export function canonicalJson(value) { + return `${JSON.stringify(canonicalValue(value), null, 2)}\n`; +} + +export function sha256Bytes(value) { + return createHash("sha256").update(value).digest("hex"); +} + +export function parsePreviewTag(tag) { + const match = previewTagPattern.exec(tag); + if (!match) throw new Error(`Invalid Pylon preview tag: ${String(tag)}`); + return { commit12: match[1], recipeRevision: Number.parseInt(match[2], 10) }; +} + +export function stableSequenceReservationTag(sequence) { + if (!Number.isSafeInteger(sequence) || sequence < 1 || sequence > 999_999) { + throw new Error("Stable reservation sequence must be an integer from 000001 through 999999."); + } + return `pylon-stable-sequence-${String(sequence).padStart(6, "0")}`; +} + +export function stableReservationMessage(manifest, digest, draftId) { + if (!/^[0-9a-f]{64}$/.test(digest ?? "") || !Number.isSafeInteger(Number(draftId)) || Number(draftId) < 1) { + throw new Error("Stable reservation needs an exact manifest digest and draft id."); + } + const withdrawal = manifest?.promotion?.kind === "withdraw" ? [ + `Withdraw stable tag: ${manifest.promotion.revocation?.stableTag}`, + `Withdraw build tag: ${manifest.promotion.revocation?.buildTag}`, + `Withdraw reason: ${manifest.promotion.revocation?.reason}`, + ] : []; + return [ + "Pylon stable sequence reservation", `Sequence: ${String(manifest?.sequence).padStart(6, "0")}`, + `Policy: ${manifest?.promotion?.policyCommit}`, `Policy tree: ${manifest?.promotion?.policyTree}`, + `Operation: ${manifest?.promotion?.kind}`, ...withdrawal, `Stable tag: ${manifest?.tag}`, + `Preview: ${manifest?.build?.previewTag}`, `Manifest: sha256:${digest}`, `Draft release: ${Number(draftId)}`, "", + ].join("\n"); +} + +export function parseStableSequenceReservationTag(tag) { + const match = stableReservationTagPattern.exec(tag); + if (!match || Number.parseInt(match[1], 10) < 1) throw new Error(`Invalid Pylon stable reservation tag: ${String(tag)}`); + return { sequence: Number.parseInt(match[1], 10) }; +} + +export function stableTag({ sequence, sourceCommit, recipeRevision }) { + if (!Number.isSafeInteger(sequence) || sequence < 1 || sequence > 999_999) { + throw new Error("Stable sequence must be an integer from 000001 through 999999."); + } + if (!/^[0-9a-f]{40}$/.test(sourceCommit)) throw new Error("Stable source must be a full lowercase Git SHA."); + if (!Number.isSafeInteger(recipeRevision) || recipeRevision < 1) throw new Error("Invalid stable recipe revision."); + return `pylon-stable-${String(sequence).padStart(6, "0")}-g${sourceCommit.slice(0, 12)}-r${recipeRevision}`; +} + +export function parseStableTag(tag) { + const match = stableTagPattern.exec(tag); + if (!match) throw new Error(`Invalid Pylon stable tag: ${String(tag)}`); + const sequence = Number.parseInt(match[1], 10); + if (sequence < 1) throw new Error(`Invalid Pylon stable sequence: ${tag}`); + return { + sequence, + commit12: match[2], + recipeRevision: Number.parseInt(match[3], 10), + }; +} + +function exactKeys(value, keys) { + return isPlainObject(value) && Object.keys(value).sort().join(",") === [...keys].sort().join(","); +} + +export function validatePublishedReleaseManifest(manifest, supportedRecipes = PYLON_SUPPORTED_RELEASE_RECIPES) { + if (!exactKeys(manifest, ["schemaVersion", "source", "build", "package", "assets", "attestationSubjects"])) { + throw new Error("Unsupported published Pylon release manifest."); + } + const { source, build, package: publicPackage, assets, attestationSubjects } = manifest; + const recipe = supportedRecipes.find((candidate) => candidate.recipeRevision === build?.recipeRevision); + if ( + !recipe || !exactKeys(recipe, [ + "recipeRevision", "manifestSchemaVersion", "nodeVersion", "npmVersion", "minimumNodeVersion", + ]) || + manifest.schemaVersion !== recipe.manifestSchemaVersion || + !exactKeys(source, ["repository", "commit", "tree"]) || source.repository !== PYLON_RELEASE_REPOSITORY || + !/^[0-9a-f]{40}$/.test(source.commit ?? "") || !/^[0-9a-f]{40}$/.test(source.tree ?? "") || + !exactKeys(build, ["id", "recipeRevision", "node", "npm", "lockfile", "assetBaseUrl"]) || + build.id !== `pylon-build-g${source.commit.slice(0, 12)}-r${recipe.recipeRevision}` || + build.node !== recipe.nodeVersion || build.npm !== recipe.npmVersion || + !exactKeys(build.lockfile, ["file", "sha256"]) || build.lockfile.file !== "package-lock.json" || + !/^[0-9a-f]{64}$/.test(build.lockfile.sha256 ?? "") || + build.assetBaseUrl !== `${PYLON_RELEASE_REPOSITORY}/releases/download/${build.id}` || + !exactKeys(publicPackage, ["name", "command", "version", "minimumNode"]) || + publicPackage.name !== "prime-agent" || publicPackage.command !== "prime-agent" || + normalizeNpmVersion(publicPackage.version) !== publicPackage.version || publicPackage.minimumNode !== recipe.minimumNodeVersion || + !Array.isArray(assets) || assets.length !== PYLON_RELEASE_PACKAGES.length || + !Array.isArray(attestationSubjects) || attestationSubjects.length !== assets.length + ) throw new Error("Malformed historical Pylon release manifest."); + const expectedAssets = new Map(PYLON_RELEASE_PACKAGES.map((entry) => [ + releaseAssetFile(entry.assetStem, publicPackage.version), entry.packageName, + ])); + const sortedFiles = assets.map((asset) => asset.file).toSorted(); + if (assets.some((asset, index) => asset.file !== sortedFiles[index])) { + throw new Error("Published Pylon release assets are not sorted."); + } + for (let index = 0; index < assets.length; index += 1) { + const asset = assets[index]; + const subject = attestationSubjects[index]; + if ( + !exactKeys(asset, ["package", "file", "size", "sha256", "sha512"]) || + asset.package !== expectedAssets.get(asset.file) || !Number.isSafeInteger(asset.size) || asset.size < 1 || + !/^[0-9a-f]{64}$/.test(asset.sha256 ?? "") || !/^[0-9a-f]{128}$/.test(asset.sha512 ?? "") || + !exactKeys(subject, ["name", "digest"]) || !exactKeys(subject.digest, ["sha256", "sha512"]) || + subject.name !== asset.file || subject.digest.sha256 !== asset.sha256 || subject.digest.sha512 !== asset.sha512 + ) throw new Error(`Malformed historical Pylon release asset: ${String(asset?.file)}`); + expectedAssets.delete(asset.file); + } + if (expectedAssets.size !== 0) throw new Error("Historical Pylon release asset set is incomplete."); + return manifest; +} + +function publicationAssets(releaseManifest) { + return releaseManifest.assets.map(({ file, size, sha256, sha512 }) => ({ file, size, sha256, sha512 })); +} + +function validatePreviewSequence({ sequenceEpoch, sequence, workflowRunId }) { + if ( + sequenceEpoch !== 1 || !Number.isSafeInteger(sequence) || sequence < 1 || + !isCanonicalPositiveDecimal(workflowRunId) + ) throw new Error("Preview sequence identity is malformed."); + return { sequenceEpoch, sequence, workflowRunId }; +} + +function isCanonicalPositiveDecimal(value) { + return typeof value === "string" && /^[1-9][0-9]*$/.test(value); +} + +function publicationPolicyFor(revision, supportedPublicationPolicies) { + if (!Number.isSafeInteger(revision) || revision < 1) { + throw new Error("Publication policy revision must be an exact positive integer."); + } + const policy = supportedPublicationPolicies.find((candidate) => candidate.publicationPolicyRevision === revision); + if (!policy) throw new Error(`Unsupported publication policy revision: ${revision}`); + return policy; +} + +function previewManifestFor(releaseManifest, releaseManifestBytes, invocation, supportedPublicationPolicies) { + if (!Buffer.isBuffer(releaseManifestBytes) || releaseManifestBytes.byteLength === 0) { + throw new Error("Build manifest bytes are required."); + } + const tag = releaseManifest.build.id; + const sequence = validatePreviewSequence(invocation); + publicationPolicyFor(invocation.publicationPolicyRevision, supportedPublicationPolicies); + return { + schemaVersion: PYLON_PUBLICATION_SCHEMA_VERSION, + channel: "preview", + repository: PYLON_RELEASE_REPOSITORY, + publicationPolicyRevision: invocation.publicationPolicyRevision, + ...sequence, + build: { + tag, + id: releaseManifest.build.id, + recipeRevision: releaseManifest.build.recipeRevision, + source: releaseManifest.source, + releaseManifest: { + file: PYLON_RELEASE_MANIFEST, + sha256: sha256Bytes(releaseManifestBytes), + }, + }, + assets: publicationAssets(releaseManifest), + }; +} + +export function createPreviewManifest( + releaseManifest, + releaseManifestBytes, + invocation, + { supportedPublicationPolicies = PYLON_SUPPORTED_PUBLICATION_POLICIES } = {}, +) { + validateReleaseManifest(releaseManifest); + if (releaseManifest.build.id !== releaseBuildId(releaseManifest.source.commit)) throw new Error("Current preview build id is malformed."); + return previewManifestFor(releaseManifest, releaseManifestBytes, invocation, supportedPublicationPolicies); +} + +export function validatePreviewManifest( + previewManifest, + releaseManifest, + releaseManifestBytes, + { + historical = false, + supportedRecipes = PYLON_SUPPORTED_RELEASE_RECIPES, + supportedPublicationPolicies = PYLON_SUPPORTED_PUBLICATION_POLICIES, + } = {}, +) { + if (historical) validatePublishedReleaseManifest(releaseManifest, supportedRecipes); + else validateReleaseManifest(releaseManifest); + const expected = previewManifestFor(releaseManifest, releaseManifestBytes, previewManifest, supportedPublicationPolicies); + if (canonicalJson(previewManifest) !== canonicalJson(expected)) { + throw new Error("Preview manifest does not match the exact deterministic build manifest."); + } + const parsedTag = parsePreviewTag(previewManifest.build.tag); + if ( + parsedTag.commit12 !== previewManifest.build.source.commit.slice(0, 12) || + parsedTag.recipeRevision !== previewManifest.build.recipeRevision + ) { + throw new Error("Preview tag is not bound to the full source and recipe."); + } + return previewManifest; +} + +function validateRevocation(value) { + if ( + !exactKeys(value, ["stableTag", "buildTag", "reason", "revokedBySequence"]) || + parseStableTag(value.stableTag).sequence >= value.revokedBySequence || + !previewTagPattern.test(value.buildTag) || + typeof value.reason !== "string" || + !/^[a-z0-9][a-z0-9-]{2,63}$/.test(value.reason) || + !Number.isSafeInteger(value.revokedBySequence) + ) { + throw new Error("Malformed stable revocation entry."); + } + return value; +} + +export function createStableManifest({ + previewManifest, + previewManifestBytes, + sequence, + previous = null, + revocations = [], + promotion, + supportedPublicationPolicies = PYLON_SUPPORTED_PUBLICATION_POLICIES, +}) { + if (canonicalJson(previewManifest) !== previewManifestBytes.toString("utf8")) { + throw new Error("Preview manifest is not canonical publication JSON."); + } + const previewTag = parsePreviewTag(previewManifest.build?.tag); + publicationPolicyFor(previewManifest.publicationPolicyRevision, supportedPublicationPolicies); + if ( + previewManifest.schemaVersion !== PYLON_PUBLICATION_SCHEMA_VERSION || + previewManifest.channel !== "preview" || + previewManifest.repository !== PYLON_RELEASE_REPOSITORY || + previewTag.commit12 !== previewManifest.build.source.commit.slice(0, 12) || + previewTag.recipeRevision !== previewManifest.build.recipeRevision + ) { + throw new Error("Malformed preview manifest for stable promotion."); + } + const tag = stableTag({ + sequence, + sourceCommit: previewManifest.build.source.commit, + recipeRevision: previewManifest.build.recipeRevision, + }); + const normalizedRevocations = revocations + .map((entry) => ({ ...validateRevocation(entry) })) + .sort((left, right) => parseStableTag(left.stableTag).sequence - parseStableTag(right.stableTag).sequence); + if (new Set(normalizedRevocations.map((entry) => entry.stableTag)).size !== normalizedRevocations.length) { + throw new Error("A stable tag cannot be revoked more than once."); + } + if (sequence === 1) { + if (previous !== null) throw new Error("The first stable sequence cannot claim previous publication state."); + } else if ( + !exactKeys(previous, ["tag", "sha256"]) || + parseStableTag(previous.tag).sequence !== sequence - 1 || + !/^[0-9a-f]{64}$/.test(previous.sha256) + ) { + throw new Error("Stable publication must bind the exact previous sequence digest."); + } + if ( + !promotion || !["promote", "withdraw"].includes(promotion.kind) || + !/^[0-9a-f]{40}$/.test(promotion.policyCommit ?? "") || !/^[0-9a-f]{40}$/.test(promotion.policyTree ?? "") + ) { + throw new Error("Stable promotion must bind its protected policy commit/tree and operation."); + } + publicationPolicyFor(promotion.publicationPolicyRevision, supportedPublicationPolicies); + const expectedPromotionKeys = promotion.kind === "promote" + ? ["kind", "policyCommit", "policyTree", "publicationPolicyRevision"] + : ["kind", "policyCommit", "policyTree", "publicationPolicyRevision", "revocation"]; + if (!exactKeys(promotion, expectedPromotionKeys)) throw new Error("Malformed stable promotion metadata."); + if (promotion.kind === "withdraw") { + const revocation = validateRevocation(promotion.revocation); + if ( + revocation.revokedBySequence !== sequence || + !normalizedRevocations.some((entry) => canonicalJson(entry) === canonicalJson(revocation)) + ) { + throw new Error("Withdrawal must append its exact revocation at the new sequence."); + } + } + return { + schemaVersion: PYLON_PUBLICATION_SCHEMA_VERSION, + channel: "stable", + repository: PYLON_RELEASE_REPOSITORY, + sequence, + tag, + history: { + highWater: sequence - 1, + previous, + }, + build: { + previewSequence: validatePreviewSequence(previewManifest), + previewTag: previewManifest.build.tag, + id: previewManifest.build.id, + recipeRevision: previewManifest.build.recipeRevision, + publicationPolicyRevision: previewManifest.publicationPolicyRevision, + source: previewManifest.build.source, + releaseManifest: previewManifest.build.releaseManifest, + previewManifest: { + file: PYLON_PREVIEW_MANIFEST, + sha256: sha256Bytes(previewManifestBytes), + }, + assets: previewManifest.assets, + }, + promotion, + revocations: normalizedRevocations, + }; +} + +export function validateStableManifest( + stableManifest, + supportedRecipes = PYLON_SUPPORTED_RELEASE_RECIPES, + supportedPublicationPolicies = PYLON_SUPPORTED_PUBLICATION_POLICIES, +) { + if ( + !exactKeys(stableManifest, [ + "schemaVersion", + "channel", + "repository", + "sequence", + "tag", + "history", + "build", + "promotion", + "revocations", + ]) || + stableManifest.schemaVersion !== PYLON_PUBLICATION_SCHEMA_VERSION || + stableManifest.channel !== "stable" || + stableManifest.repository !== PYLON_RELEASE_REPOSITORY || + !Array.isArray(stableManifest.revocations) + ) { + throw new Error("Malformed Pylon stable manifest."); + } + const build = stableManifest.build; + const recipe = supportedRecipes.find((candidate) => candidate.recipeRevision === build?.recipeRevision); + publicationPolicyFor(build?.publicationPolicyRevision, supportedPublicationPolicies); + publicationPolicyFor(stableManifest.promotion?.publicationPolicyRevision, supportedPublicationPolicies); + if ( + !recipe || + !exactKeys(build, [ + "previewSequence", "previewTag", "id", "recipeRevision", "publicationPolicyRevision", "source", + "releaseManifest", "previewManifest", "assets", + ]) || + canonicalJson(validatePreviewSequence(build.previewSequence ?? {})) !== canonicalJson(build.previewSequence) || + build.previewTag !== build.id || !Number.isSafeInteger(build.recipeRevision) || build.recipeRevision < 1 || + !exactKeys(build.source, ["repository", "commit", "tree"]) || + build.source.repository !== PYLON_RELEASE_REPOSITORY || + !/^[0-9a-f]{40}$/.test(build.source.commit ?? "") || !/^[0-9a-f]{40}$/.test(build.source.tree ?? "") || + !exactKeys(build.releaseManifest, ["file", "sha256"]) || build.releaseManifest.file !== PYLON_RELEASE_MANIFEST || + !/^[0-9a-f]{64}$/.test(build.releaseManifest.sha256 ?? "") || + !exactKeys(build.previewManifest, ["file", "sha256"]) || build.previewManifest.file !== PYLON_PREVIEW_MANIFEST || + !/^[0-9a-f]{64}$/.test(build.previewManifest.sha256 ?? "") || + !Array.isArray(build.assets) || build.assets.length !== 4 + ) throw new Error("Stable build receipt is not an exact closed Pylon preview schema."); + const safeAsset = /^pylon-prime-agent(?:-(ai|core|tui))?-([0-9]+\.[0-9]+\.[0-9]+)\.tgz$/; + const roles = []; + const versions = []; + for (const asset of build.assets) { + const assetMatch = safeAsset.exec(asset.file ?? ""); + if ( + !exactKeys(asset, ["file", "size", "sha256", "sha512"]) || !assetMatch || + !Number.isSafeInteger(asset.size) || asset.size < 1 || + !/^[0-9a-f]{64}$/.test(asset.sha256 ?? "") || !/^[0-9a-f]{128}$/.test(asset.sha512 ?? "") + ) throw new Error("Stable build contains an unsafe or malformed preview asset."); + roles.push(assetMatch[1] ?? "root"); + versions.push(assetMatch[2]); + } + if ( + roles.toSorted().join(",") !== "ai,core,root,tui" || new Set(versions).size !== 1 || + new Set(build.assets.map((asset) => asset.file)).size !== build.assets.length || + canonicalJson(build.assets.toSorted((left, right) => compareText(left.file, right.file))) !== canonicalJson(build.assets) + ) throw new Error("Stable preview assets must be unique and sorted by exact file name."); + const tag = parseStableTag(stableManifest.tag); + if ( + !exactKeys(stableManifest.history, ["highWater", "previous"]) || + stableManifest.history.highWater !== stableManifest.sequence - 1 || + (stableManifest.sequence === 1 + ? stableManifest.history.previous !== null + : !exactKeys(stableManifest.history.previous, ["tag", "sha256"]) || + parseStableTag(stableManifest.history.previous.tag).sequence !== stableManifest.sequence - 1 || + !/^[0-9a-f]{64}$/.test(stableManifest.history.previous.sha256)) || + tag.sequence !== stableManifest.sequence || + tag.commit12 !== stableManifest.build?.source?.commit?.slice(0, 12) || + tag.recipeRevision !== stableManifest.build?.recipeRevision || + stableManifest.build.previewTag !== stableManifest.build.id || + parsePreviewTag(stableManifest.build.previewTag).commit12 !== tag.commit12 || + parsePreviewTag(stableManifest.build.previewTag).recipeRevision !== stableManifest.build.recipeRevision + ) { + throw new Error("Stable tag does not match its exact preview build."); + } + for (const revocation of stableManifest.revocations) validateRevocation(revocation); + const sortedRevocations = stableManifest.revocations.toSorted( + (left, right) => parseStableTag(left.stableTag).sequence - parseStableTag(right.stableTag).sequence, + ); + if (canonicalJson(sortedRevocations) !== canonicalJson(stableManifest.revocations)) { + throw new Error("Stable revocations are not sorted by sequence."); + } + if (new Set(stableManifest.revocations.map((entry) => entry.stableTag)).size !== stableManifest.revocations.length) { + throw new Error("Stable revocation history contains a duplicate."); + } + if (stableManifest.promotion?.kind === "withdraw") { + if ( + !exactKeys(stableManifest.promotion, ["kind", "policyCommit", "policyTree", "publicationPolicyRevision", "revocation"]) || + !/^[0-9a-f]{40}$/.test(stableManifest.promotion.policyCommit) || + !/^[0-9a-f]{40}$/.test(stableManifest.promotion.policyTree) || + !stableManifest.revocations.some( + (entry) => canonicalJson(entry) === canonicalJson(stableManifest.promotion.revocation), + ) + ) { + throw new Error("Stable withdrawal does not append one exact revocation."); + } + } else if ( + !exactKeys(stableManifest.promotion, ["kind", "policyCommit", "policyTree", "publicationPolicyRevision"]) || + stableManifest.promotion.kind !== "promote" || + !/^[0-9a-f]{40}$/.test(stableManifest.promotion.policyCommit) || + !/^[0-9a-f]{40}$/.test(stableManifest.promotion.policyTree) + ) { + throw new Error("Malformed stable promotion record."); + } + return stableManifest; +} + +function containsHistory(previous, next) { + return previous.every((entry) => next.some((candidate) => canonicalJson(entry) === canonicalJson(candidate))); +} + +export function validateStableHistory( + manifests, + supportedRecipes = PYLON_SUPPORTED_RELEASE_RECIPES, + supportedPublicationPolicies = PYLON_SUPPORTED_PUBLICATION_POLICIES, +) { + const ordered = manifests.toSorted((left, right) => left.sequence - right.sequence); + let previous; + for (let index = 0; index < ordered.length; index += 1) { + const current = validateStableManifest(ordered[index], supportedRecipes, supportedPublicationPolicies); + if (current.sequence !== index + 1) throw new Error("Stable publication history has a skipped or duplicate sequence."); + if (previous) { + if ( + current.history.previous.tag !== previous.tag || + current.history.previous.sha256 !== sha256Bytes(Buffer.from(canonicalJson(previous))) + ) { + throw new Error("Stable sequence does not bind the exact previous manifest digest."); + } + if (!containsHistory(previous.revocations, current.revocations)) { + throw new Error("Stable revocation history is not append-only."); + } + const delta = current.revocations.length - previous.revocations.length; + if (delta < 0 || delta > 1 || (current.promotion.kind === "withdraw") !== (delta === 1)) { + throw new Error("Stable sequence changed revocations outside one declared withdrawal."); + } + if (delta === 1) { + const added = current.revocations.find((entry) => !previous.revocations.some((prior) => prior.stableTag === entry.stableTag)); + if (!added || added.revokedBySequence !== current.sequence || canonicalJson(added) !== canonicalJson(current.promotion.revocation)) { + throw new Error("Stable withdrawal was not introduced by its declared sequence."); + } + } + } + previous = current; + } + const byTag = new Map(ordered.map((manifest) => [manifest.tag, manifest])); + for (const manifest of ordered) { + for (const revocation of manifest.revocations) { + const revoked = byTag.get(revocation.stableTag); + if (!revoked || revoked.sequence >= manifest.sequence || revoked.build.previewTag !== revocation.buildTag) { + throw new Error("Stable revocation does not bind the exact previously published build."); + } + } + } + return ordered; +} + +export function nextStableSequence(manifests) { + return validateStableHistory(manifests).length + 1; +} + +export function assertCanonicalInvocation({ repository, ref, eventName, sha, expectedEvent }) { + if ( + repository !== PYLON_PUBLICATION_REPOSITORY || + ref !== PYLON_PUBLICATION_REF || + eventName !== expectedEvent || + !/^[0-9a-f]{40}$/.test(sha) + ) { + throw new Error("Publication requires the canonical repository and protected pylon ref."); + } +} + +export function validateRequiredChecks({ sourceSha, requiredChecks, checkRuns }) { + if (!/^[0-9a-f]{40}$/.test(sourceSha)) throw new Error("Required checks need an exact source SHA."); + if (!Array.isArray(requiredChecks) || !Array.isArray(checkRuns)) { + throw new Error("Protected pylon required-check policy is unavailable."); + } + const actualPolicy = requiredChecks + .map((required) => ({ context: required?.context, appId: required?.appId })) + .sort((left, right) => compareText(String(left.context), String(right.context))); + const expectedPolicy = PYLON_REQUIRED_CHECKS.map(({ context, appId }) => ({ context, appId })); + if (canonicalJson(actualPolicy) !== canonicalJson(expectedPolicy)) { + throw new Error("Protected pylon required-check policy differs from the exact two GitHub Actions checks."); + } + for (const required of PYLON_REQUIRED_CHECKS) { + const matches = checkRuns.filter( + (candidate) => candidate.name === required.context && candidate.head_sha === sourceSha && candidate.app?.id === required.appId && + candidate.status === "completed" && candidate.conclusion === "success" && candidate.workflowPath === required.workflowPath, + ); + if (matches.length === 0) { + throw new Error(`Required check ${required.context} is not green on ${sourceSha} from app ${required.appId}.`); + } + } + return true; +} + +export function requiredCheckWorkflowPath(context) { + const required = PYLON_REQUIRED_CHECKS.find((candidate) => candidate.context === context); + if (!required) throw new Error(`Unknown required check context: ${String(context)}`); + return required.workflowPath; +} + +export function validateMergedChangelogProof({ repository, ref, eventName, mergeSha, pullRequests, headChecks, workflowRuns }) { + assertCanonicalInvocation({ repository, ref, eventName, sha: mergeSha, expectedEvent: "push" }); + const matches = pullRequests.filter( + (pr) => + pr.merged_at && + pr.merge_commit_sha === mergeSha && + pr.base?.ref === "pylon" && + pr.base?.repo?.full_name === PYLON_PUBLICATION_REPOSITORY && + pr.head?.repo?.full_name === PYLON_PUBLICATION_REPOSITORY && + /^[0-9a-f]{40}$/.test(pr.head?.sha ?? ""), + ); + if (matches.length !== 1) throw new Error("Merge SHA does not resolve to exactly one merged pylon pull request."); + const pullRequest = matches[0]; + const checks = headChecks.filter( + (check) => + check.name === "Check changelog fragment" && + check.head_sha === pullRequest.head.sha && + check.app?.id === GITHUB_ACTIONS_APP_ID && + check.status === "completed" && + check.conclusion === "success", + ); + const valid = checks.filter((check) => { + const run = workflowRuns.find((candidate) => candidate.checkRunId === check.id); + return ( + run?.event === "pull_request" && + run.head_sha === pullRequest.head.sha && + run.path === ".github/workflows/changelog-fragment.yml" && + run.repository === PYLON_PUBLICATION_REPOSITORY && + run.headRepository === PYLON_PUBLICATION_REPOSITORY && + run.pullRequests?.includes(pullRequest.number) + ); + }); + if (valid.length < 1) throw new Error("Merged PR has no successful GitHub Actions changelog head check with exact provenance."); + return pullRequest; +} + +export function validateAttestationEvidence(evidence, expected) { + if ( + evidence.repository !== expected.repository || + evidence.workflow !== expected.workflow || + evidence.ref !== expected.ref || + evidence.sourceSha !== expected.sourceSha || + evidence.issuer !== "https://token.actions.githubusercontent.com" || + evidence.rekorIncluded !== true || + evidence.subjectName !== expected.subjectName || + evidence.subjectSha256 !== expected.subjectSha256 + ) { + throw new Error("Attestation evidence does not satisfy the exact Pylon publication policy."); + } + return true; +} + +export function validateWorkflowArtifactProvenance(actual, expected) { + if ( + actual.runId !== expected.runId || + actual.repositoryId !== 1_349_002_285 || + actual.workflowPath !== expected.workflowPath || + actual.event !== expected.event || + actual.ref !== PYLON_PUBLICATION_REF || + actual.headSha !== expected.headSha || + actual.conclusion !== "success" || + actual.checkSuiteHeadSha !== expected.headSha || + actual.checkSuiteConclusion !== "success" || + !Array.isArray(actual.artifacts) || + actual.artifacts.length !== 1 + ) { + throw new Error("Workflow artifact is not bound to the exact canonical run."); + } + const artifact = actual.artifacts[0]; + if ( + artifact.name !== expected.artifactName || + artifact.expired !== false || + !/^sha256:[0-9a-f]{64}$/.test(artifact.digest ?? "") + ) { + throw new Error("Workflow artifact transport identity is missing, expired, or ambiguous."); + } + return artifact; +} + +export function assertImmutableReleaseIdentity(actual, expected) { + if ( + actual.immutable !== true || + actual.draft !== false || + actual.tag_name !== expected.tag || + actual.name !== expected.name || + actual.body !== expected.body || + actual.prerelease !== expected.prerelease || + actual.target_commitish !== expected.sourceSha + ) { + throw new Error("Existing release metadata differs or is not immutable."); + } + const expectedAssets = new Map(expected.assets.map((asset) => [asset.name, asset])); + if (!Array.isArray(actual.assets) || actual.assets.length !== expectedAssets.size) { + throw new Error("Existing release asset set differs."); + } + for (const asset of actual.assets) { + const wanted = expectedAssets.get(asset.name); + if (!wanted || asset.size !== wanted.size || asset.digest !== `sha256:${wanted.sha256}`) { + throw new Error(`Existing release asset differs: ${String(asset.name)}`); + } + expectedAssets.delete(asset.name); + } + if (expectedAssets.size > 0) throw new Error("Existing release is missing assets."); + return true; +} + +export const PYLON_STABLE_MANIFEST_MAX_BYTES = 48 * 1024; +export const PYLON_STABLE_RELEASE_BODY_MAX_BYTES = 80 * 1024; + +export function publicationReleaseBody({ + channel, tag, source, tree, recipeRevision, policyCommit, policyTree, stableManifestBytes, +}) { + if (!["preview", "stable"].includes(channel)) throw new Error("Invalid publication channel."); + const policy = channel === "stable" + ? [`Policy: ${policyCommit}`, `Policy tree: ${policyTree}`] + : []; + if (channel === "stable" && (!/^[0-9a-f]{40}$/.test(policyCommit ?? "") || !/^[0-9a-f]{40}$/.test(policyTree ?? ""))) { + throw new Error("Stable release body needs the protected promotion policy commit and tree."); + } + let recovery = []; + if (channel === "stable") { + if (!Buffer.isBuffer(stableManifestBytes) || stableManifestBytes.length < 1 || stableManifestBytes.length > PYLON_STABLE_MANIFEST_MAX_BYTES) { + throw new Error(`Stable recovery manifest must be from 1 through ${PYLON_STABLE_MANIFEST_MAX_BYTES} bytes.`); + } + recovery = [ + "Stable recovery manifest: base64-v1", + `Manifest bytes: ${stableManifestBytes.length}`, + `Manifest sha256: ${sha256Bytes(stableManifestBytes)}`, + `Manifest base64: ${stableManifestBytes.toString("base64")}`, + ]; + } + const body = [ + `Pylon Prime ${channel} publication.`, + "", + `Tag: ${tag}`, + `Source: ${source}`, + `Tree: ${tree}`, + ...policy, + `Recipe: r${recipeRevision}`, + ...recovery, + "", + "Verify the immutable release and artifact attestations before use.", + ].join("\n"); + if (Buffer.byteLength(body, "utf8") > PYLON_STABLE_RELEASE_BODY_MAX_BYTES) { + throw new Error(`Stable recovery release body exceeds ${PYLON_STABLE_RELEASE_BODY_MAX_BYTES} bytes.`); + } + return body; +} + +export function stableManifestBytesFromReleaseBody(body) { + if (typeof body !== "string" || Buffer.byteLength(body, "utf8") > PYLON_STABLE_RELEASE_BODY_MAX_BYTES) { + throw new Error("Stable recovery release body is missing or exceeds its safe byte bound."); + } + const size = /^Manifest bytes: ([1-9][0-9]*)$/m.exec(body)?.[1]; + const digest = /^Manifest sha256: ([0-9a-f]{64})$/m.exec(body)?.[1]; + const encoded = /^Manifest base64: ([A-Za-z0-9+/]+={0,2})$/m.exec(body)?.[1]; + if (!size || !digest || !encoded || (body.match(/^Stable recovery manifest: base64-v1$/gm) ?? []).length !== 1) { + throw new Error("Stable recovery release body has no exact manifest envelope."); + } + const bytes = Buffer.from(encoded, "base64"); + if ( + bytes.length !== Number(size) || bytes.length > PYLON_STABLE_MANIFEST_MAX_BYTES || bytes.toString("base64") !== encoded || + sha256Bytes(bytes) !== digest + ) throw new Error("Stable recovery manifest envelope is truncated, altered, or oversized."); + const manifest = validateStableManifest(JSON.parse(bytes)); + if (bytes.toString("utf8") !== canonicalJson(manifest)) throw new Error("Stable recovery manifest is not canonical JSON."); + const expectedBody = publicationReleaseBody({ + channel: "stable", tag: manifest.tag, source: manifest.build.source.commit, tree: manifest.build.source.tree, + recipeRevision: manifest.build.recipeRevision, policyCommit: manifest.promotion.policyCommit, + policyTree: manifest.promotion.policyTree, stableManifestBytes: bytes, + }); + if (body !== expectedBody) throw new Error("Stable recovery release body metadata differs from its exact manifest."); + return { bytes, manifest, digest }; +} diff --git a/scripts/lib/pylon-ruleset-auditor.mjs b/scripts/lib/pylon-ruleset-auditor.mjs new file mode 100644 index 0000000000..c9a9d93b1c --- /dev/null +++ b/scripts/lib/pylon-ruleset-auditor.mjs @@ -0,0 +1,168 @@ +export const PYLON_RULESET_ID = 21_950_766; +export const PYLON_RULESET_NODE_ID = "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4"; +export const PYLON_REPOSITORY_ID = 1_349_002_285; +export const PYLON_REPOSITORY_NODE_ID = "R_kgDOUGgkLQ"; +export const PYLON_REPOSITORY_OWNER = "pylon-code"; +export const PYLON_REPOSITORY_NAME = "prime-agent"; +export const PYLON_REPOSITORY_NAME_WITH_OWNER = `${PYLON_REPOSITORY_OWNER}/${PYLON_REPOSITORY_NAME}`; +export const PYLON_RULESET_NAME = "Pylon immutable publication tags"; +export const PYLON_RULESET_REF_INCLUDES = ["refs/tags/pylon-build-*", "refs/tags/pylon-stable-*"]; + +export const PYLON_PUBLICATION_RULESET_GRAPHQL_QUERY = `query PylonPublicationRulesetAudit($owner: String!, $repo: String!, $rulesetDatabaseId: Int!) { + repository(owner: $owner, name: $repo) { + id + databaseId + nameWithOwner + ruleset(databaseId: $rulesetDatabaseId, includeParents: false) { + id + databaseId + name + enforcement + target + source { + __typename + ... on Repository { + id + databaseId + nameWithOwner + } + } + bypassActors { totalCount } + conditions { + refName { include exclude } + organizationProperty { __typename } + repositoryId { __typename } + repositoryName { __typename } + repositoryProperty { __typename } + } + rules(first: 100) { + totalCount + nodes { + type + parameters { + __typename + ... on UpdateParameters { updateAllowsFetchAndMerge } + } + } + } + } + } +}`; + +export const PYLON_PUBLICATION_RULESET_GRAPHQL_VARIABLES = { + owner: PYLON_REPOSITORY_OWNER, + repo: PYLON_REPOSITORY_NAME, + rulesetDatabaseId: PYLON_RULESET_ID, +}; + +function exactKeys(value, keys) { + return value !== null && typeof value === "object" && !Array.isArray(value) && + JSON.stringify(Object.keys(value).sort()) === JSON.stringify([...keys].sort()); +} + +function exactSortedStrings(value, expected) { + return Array.isArray(value) && value.every((entry) => typeof entry === "string") && + JSON.stringify([...value].sort()) === JSON.stringify([...expected].sort()); +} + +function invalidRest() { + throw new Error("REST publication ruleset response differs from the exact target contract."); +} + +function invalidGraphql() { + throw new Error("GraphQL publication ruleset response is null, partial, redacted, or differs from the exact target contract."); +} + +export function validatePublicationRulesetRestResponse(response) { + const ruleset = response?.data; + const conditions = ruleset?.conditions; + const refName = conditions?.ref_name; + const rules = ruleset?.rules; + const updateRules = Array.isArray(rules) ? rules.filter((rule) => rule?.type === "update") : []; + const deletionRules = Array.isArray(rules) ? rules.filter((rule) => rule?.type === "deletion") : []; + if ( + response?.status !== 200 || !ruleset || ruleset.id !== PYLON_RULESET_ID || ruleset.node_id !== PYLON_RULESET_NODE_ID || + ruleset.name !== PYLON_RULESET_NAME || ruleset.source_type !== "Repository" || + ruleset.source !== PYLON_REPOSITORY_NAME_WITH_OWNER || ruleset.target !== "tag" || ruleset.enforcement !== "active" || + Object.hasOwn(ruleset, "bypass_actors") && (!Array.isArray(ruleset.bypass_actors) || ruleset.bypass_actors.length !== 0) || + Object.hasOwn(ruleset, "current_user_can_bypass") && ruleset.current_user_can_bypass !== "never" || + !exactKeys(conditions, ["ref_name"]) || !exactKeys(refName, ["exclude", "include"]) || + !exactSortedStrings(refName.exclude, []) || !exactSortedStrings(refName.include, PYLON_RULESET_REF_INCLUDES) || + !Array.isArray(rules) || rules.length !== 2 || updateRules.length !== 1 || deletionRules.length !== 1 || + !exactKeys(updateRules[0], ["parameters", "type"]) || + !exactKeys(updateRules[0]?.parameters, ["update_allows_fetch_and_merge"]) || + updateRules[0]?.parameters?.update_allows_fetch_and_merge !== false || + !exactKeys(deletionRules[0], ["type"]) + ) invalidRest(); + return ruleset; +} + +export function validatePublicationRulesetGraphqlResponse(response, restRuleset) { + if (Object.hasOwn(response ?? {}, "errors")) invalidGraphql(); + const repository = response?.repository; + const ruleset = repository?.ruleset; + const source = ruleset?.source; + const bypassActors = ruleset?.bypassActors; + const conditions = ruleset?.conditions; + const refName = conditions?.refName; + const rules = ruleset?.rules; + const nodes = rules?.nodes; + const updateRules = Array.isArray(nodes) ? nodes.filter((rule) => rule?.type === "UPDATE") : []; + const deletionRules = Array.isArray(nodes) ? nodes.filter((rule) => rule?.type === "DELETION") : []; + if ( + !restRuleset || !repository || repository.id !== PYLON_REPOSITORY_NODE_ID || repository.databaseId !== PYLON_REPOSITORY_ID || + repository.nameWithOwner !== PYLON_REPOSITORY_NAME_WITH_OWNER || !ruleset || ruleset.id !== restRuleset.node_id || + ruleset.id !== PYLON_RULESET_NODE_ID || ruleset.databaseId !== PYLON_RULESET_ID || ruleset.name !== PYLON_RULESET_NAME || + ruleset.enforcement !== "ACTIVE" || ruleset.target !== "TAG" || !exactKeys(source, ["__typename", "databaseId", "id", "nameWithOwner"]) || + source.__typename !== "Repository" || source.id !== repository.id || source.databaseId !== repository.databaseId || + source.nameWithOwner !== repository.nameWithOwner || !exactKeys(bypassActors, ["totalCount"]) || + !Number.isInteger(bypassActors.totalCount) || bypassActors.totalCount !== 0 || + !exactKeys(conditions, ["organizationProperty", "refName", "repositoryId", "repositoryName", "repositoryProperty"]) || + conditions.organizationProperty !== null || conditions.repositoryId !== null || conditions.repositoryName !== null || + conditions.repositoryProperty !== null || !exactKeys(refName, ["exclude", "include"]) || + !exactSortedStrings(refName.exclude, []) || !exactSortedStrings(refName.include, PYLON_RULESET_REF_INCLUDES) || + !exactKeys(rules, ["nodes", "totalCount"]) || !Number.isInteger(rules.totalCount) || rules.totalCount !== 2 || + !Array.isArray(nodes) || nodes.length !== 2 || nodes.some((node) => node === null) || + updateRules.length !== 1 || deletionRules.length !== 1 || !exactKeys(updateRules[0], ["parameters", "type"]) || + !exactKeys(updateRules[0].parameters, ["__typename", "updateAllowsFetchAndMerge"]) || + updateRules[0].parameters.__typename !== "UpdateParameters" || updateRules[0].parameters.updateAllowsFetchAndMerge !== false || + !exactKeys(deletionRules[0], ["parameters", "type"]) || deletionRules[0].parameters !== null + ) invalidGraphql(); + return ruleset; +} + +export async function auditPublicationRuleset({ requestRest, requestGraphql }) { + if (typeof requestRest !== "function" || typeof requestGraphql !== "function") { + throw new Error("Publication ruleset audit needs exact REST and GraphQL request functions."); + } + const restRuleset = validatePublicationRulesetRestResponse(await requestRest()); + const graphqlResponse = await requestGraphql( + PYLON_PUBLICATION_RULESET_GRAPHQL_QUERY, + PYLON_PUBLICATION_RULESET_GRAPHQL_VARIABLES, + ); + return validatePublicationRulesetGraphqlResponse(graphqlResponse, restRuleset); +} + +export const PYLON_RULESET_BYPASS_CANARY_GRAPHQL_QUERY = `query PylonRulesetBypassCanary($owner: String!, $repo: String!, $rulesetDatabaseId: Int!) { + repository(owner: $owner, name: $repo) { + nameWithOwner + ruleset(databaseId: $rulesetDatabaseId, includeParents: false) { + databaseId + bypassActors { totalCount } + } + } +}`; + +export function validateRulesetBypassCanary(response, expected) { + if (Object.hasOwn(response ?? {}, "errors")) { + throw new Error("GraphQL bypass-count canary returned errors."); + } + const repository = response?.repository; + const ruleset = repository?.ruleset; + const totalCount = ruleset?.bypassActors?.totalCount; + if ( + repository?.nameWithOwner !== `${expected.owner}/${expected.repo}` || ruleset?.databaseId !== expected.rulesetDatabaseId || + !Number.isInteger(totalCount) || totalCount <= 0 + ) throw new Error("GraphQL bypass-count canary is null, redacted, zero, or differs from its configured identity."); + return totalCount; +} diff --git a/scripts/lib/pylon-workflow-policy.mjs b/scripts/lib/pylon-workflow-policy.mjs new file mode 100644 index 0000000000..25dd6e9151 --- /dev/null +++ b/scripts/lib/pylon-workflow-policy.mjs @@ -0,0 +1,403 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; + +import { + PYLON_PREVIEW_WORKFLOW, + PYLON_PUBLICATION_REPOSITORY, + PYLON_STABLE_WORKFLOW, + PYLON_SUPPORTED_PUBLICATION_POLICIES, +} from "./pylon-publication.mjs"; + +export const ATTEST_BUILD_PROVENANCE_ACTION = "actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8"; +// The pinned composite action above immutably delegates to this reviewed signer implementation. +export const ATTEST_ACTION_CHAIN = "actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d"; +export const CREATE_GITHUB_APP_TOKEN_ACTION = "actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349"; + +function jobNames(workflow) { + const lines = workflow.replaceAll("\r\n", "\n").split("\n"); + const jobs = lines.findIndex((line) => line === "jobs:"); + if (jobs === -1) throw new Error("Approved workflow has no top-level jobs mapping."); + return lines.slice(jobs + 1).flatMap((line) => /^ ([A-Za-z0-9_-]+):\s*$/.exec(line)?.[1] ?? []); +} + +function jobBlock(workflow, jobName) { + const lines = workflow.replaceAll("\r\n", "\n").split("\n"); + const jobs = lines.findIndex((line) => line === "jobs:"); + if (jobs === -1) throw new Error("Approved workflow has no top-level jobs mapping."); + const start = lines.findIndex((line, index) => index > jobs && line === ` ${jobName}:`); + if (start === -1) throw new Error(`Approved workflow lacks the exact ${jobName} job.`); + let end = lines.length; + for (let index = start + 1; index < lines.length; index += 1) { + if (/^ [A-Za-z0-9_-]+:\s*$/.test(lines[index])) { + end = index; + break; + } + } + return lines.slice(start, end).join("\n"); +} + +function stepBlocks(block) { + const matches = [...block.matchAll(/^ - name: (.+)$/gm)]; + return matches.map((match, index) => ({ + name: match[1], + block: block.slice(match.index, matches[index + 1]?.index ?? block.length), + })); +} + +function parseStepShape(step, description) { + const lines = step.block.split("\n"); + const root = {}; + const nested = {}; + const first = /^ - ([a-z-]+):\s*(.+)$/.exec(lines[0]); + if (!first) throw new Error(`${description} is not one closed named YAML step.`); + root[first[1]] = first[2]; + for (let index = 1; index < lines.length; index += 1) { + const line = lines[index]; + if (!line) continue; + const entry = /^ ([a-z-]+):(?:\s*(.*))?$/.exec(line); + if (!entry) continue; + if (Object.hasOwn(root, entry[1])) throw new Error(`${description} has a duplicate step key.`); + root[entry[1]] = entry[2] ?? ""; + if (entry[2]) continue; + const values = {}; + for (index += 1; index < lines.length; index += 1) { + const child = /^ ([a-z-]+):\s*(.*)$/.exec(lines[index]); + if (!child) { + index -= 1; + break; + } + if (Object.hasOwn(values, child[1])) throw new Error(`${description} has a duplicate ${entry[1]} key.`); + values[child[1]] = child[2]; + if (child[2] === "|") { + while (index + 1 < lines.length && (!lines[index + 1] || lines[index + 1].startsWith(" "))) index += 1; + } + } + nested[entry[1]] = values; + } + return { root, nested }; +} + +function assertExactRulesetAuditorMint(step, description, expectedIf) { + if (/[#&]|(?:^|\s)\*[^/]|<<:/m.test(step.block)) { + throw new Error(`${description} may not use YAML comments, anchors, aliases, or merge keys.`); + } + const parsed = parseStepShape(step, description); + const expectedRoot = { + name: "Mint repository-scoped ruleset auditor token", + id: "ruleset-auditor", + uses: CREATE_GITHUB_APP_TOKEN_ACTION, + with: "", + }; + if (expectedIf !== null) expectedRoot.if = expectedIf; + exactObject(parsed.root, expectedRoot, `${description} step mapping`); + exactObject(parsed.nested.with ?? {}, { + "app-id": "${{ vars.PYLON_RULESET_AUDITOR_APP_ID }}", + "private-key": "${{ secrets.PYLON_RULESET_AUDITOR_PRIVATE_KEY }}", + owner: "pylon-code", + repositories: "prime-agent", + "permission-administration": "read", + }, `${description} with mapping`); + if (Object.keys(parsed.nested).length !== 1) throw new Error(`${description} has an unexpected nested mapping.`); +} + +function scalar(block, name) { + const matches = [...block.matchAll(new RegExp(`^ ${name}:\\s*([^\\n]+)\\s*$`, "gm"))]; + if (matches.length !== 1) throw new Error(`Approved job needs one exact ${name} value.`); + return matches[0][1].trim(); +} + +function mapping(block, name) { + const lines = block.split("\n"); + const start = lines.findIndex((line) => line === ` ${name}:`); + if (start === -1) throw new Error(`Approved job needs an exact ${name} mapping.`); + const values = {}; + for (let index = start + 1; index < lines.length; index += 1) { + const line = lines[index]; + if (/^ \S/.test(line)) break; + const match = /^ ([a-z-]+):\s*(\S+)\s*$/.exec(line); + if (match) values[match[1]] = match[2]; + else if (line.trim()) throw new Error(`Approved ${name} mapping is not closed.`); + } + return values; +} + +function exactObject(actual, expected, description) { + if (JSON.stringify(Object.entries(actual).sort()) !== JSON.stringify(Object.entries(expected).sort())) { + throw new Error(`Approved workflow ${description} differs from the closed policy.`); + } +} + +function needs(block) { + const value = scalar(block, "needs"); + if (value.startsWith("[") && value.endsWith("]")) { + return value.slice(1, -1).split(",").map((entry) => entry.trim()).filter(Boolean); + } + return [value]; +} + +function assertNoDownloadedOrRepositoryExecution(block, description) { + if ( + /actions\/checkout@|actions\/setup-node@|\bnpm\s+(?:ci|install|run)\b|\bnode\s+scripts\/|\btar\s|\.tgz\b[^\n]*(?:exec|run)|node:child_process|\b(?:execFile|spawn|fork|eval)\s*\(|new Function|require\(["']\.|import\s*\(|\bchmod\b|(?:^|\s)\.\//im.test(block) + ) throw new Error(`${description} may not checkout or execute repository/downloaded code.`); +} + +function canonicalList(values) { + return JSON.stringify([...values].sort()); +} + +export function validateApprovedAttestationWorkflow(workflow, channel) { + if (typeof workflow !== "string" || !workflow.endsWith("\n") || workflow.includes("\r")) { + throw new Error("Approved workflow bytes must be normalized text."); + } + const policy = channel === "preview" + ? { + environment: "pylon-preview", + workflow: PYLON_PREVIEW_WORKFLOW, + subject: ".npm/pylon-release/artifacts/*", + attestNeeds: ["pack", "reproducibility", "install"], + publisher: "publish", + } + : channel === "stable" + ? { + environment: "pylon-stable", + workflow: PYLON_STABLE_WORKFLOW, + subject: "publication/pylon-stable-channel-v1.json", + attestNeeds: ["prepare"], + publisher: "stage-draft", + } + : null; + if (!policy) throw new Error("Unknown approved attestation channel."); + if (!/^permissions:\s*\{\}\s*$/m.test(workflow)) throw new Error("Approved publication workflow needs deny-by-default permissions."); + const attest = jobBlock(workflow, "attest"); + if (scalar(attest, "environment") !== policy.environment) throw new Error("Attester lacks its exact approval environment."); + exactObject(mapping(attest, "permissions"), { + actions: "read", + attestations: "write", + "id-token": "write", + }, "attester permissions"); + if (JSON.stringify(needs(attest).sort()) !== JSON.stringify(policy.attestNeeds.sort())) { + throw new Error("Attester dependency path differs from the approved graph."); + } + const attestUses = [...attest.matchAll(/^ uses:\s*([^\s#]+)(?:\s+#.*)?$/gm)].map((match) => match[1]); + if (attestUses.filter((value) => value === ATTEST_BUILD_PROVENANCE_ACTION).length !== 1) { + throw new Error("Attester must use the one exact pinned provenance action."); + } + for (const action of attestUses) { + const revision = action.split("@").at(-1); + if (!/^[0-9a-f]{40}$/.test(revision)) throw new Error("Attester action is not pinned to a full SHA."); + } + const allowedAttesterActions = new Set([ + "actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea", + "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c", + ATTEST_BUILD_PROVENANCE_ACTION, + ]); + if (attestUses.some((action) => !allowedAttesterActions.has(action))) throw new Error("Attester uses an action outside the exact closed allowlist."); + const expectedAttesterActions = channel === "preview" + ? ["actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c", ATTEST_BUILD_PROVENANCE_ACTION] + : [ + "actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea", + "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c", + ATTEST_BUILD_PROVENANCE_ACTION, + ]; + if (canonicalList(attestUses) !== canonicalList(expectedAttesterActions)) throw new Error("Attester action graph differs from the exact allowlist."); + if ( + !attest.includes(" - name: Validate exact subjects before signing\n") && + !attest.includes(" - name: Validate exact stable subject before signing\n") || + (attest.match(/ node <<'NODE'/g) ?? []).length !== 1 + ) throw new Error("Attester lacks its one exact inline subject validator."); + const subjects = [...attest.matchAll(/^ subject-path:\s*(.+?)\s*$/gm)].map((match) => match[1]); + if (subjects.length !== 1 || subjects[0] !== policy.subject) throw new Error("Attester subject path differs from the exact approved subject."); + assertNoDownloadedOrRepositoryExecution(attest, "Attester"); + + const verifier = jobBlock(workflow, "verify-attestation"); + if (!needs(verifier).includes("attest")) throw new Error("Read-only verifier is not downstream of the approved attester."); + const verifierPermissions = mapping(verifier, "permissions"); + if (Object.values(verifierPermissions).some((value) => value === "write")) throw new Error("Attestation verifier is not read-only."); + + const blocks = new Map(jobNames(workflow).map((name) => [name, jobBlock(workflow, name)])); + const writers = [...blocks].filter(([, block]) => /^ contents: write$/m.test(block)).map(([name]) => name).sort(); + if (canonicalList(writers) !== canonicalList(["publish", "stage-draft"])) { + throw new Error("Approved publication workflow has an extra or missing contents writer."); + } + const stage = blocks.get("stage-draft"); + const publisher = blocks.get("publish"); + const protectedMutationJobs = channel === "preview" ? new Set(["stage-draft", "publish"]) : new Set(["publish"]); + for (const [name, block] of [["stage-draft", stage], ["publish", publisher]]) { + if (mapping(block, "permissions").contents !== "write" || /id-token:\s*write|attestations:\s*write/.test(block)) { + throw new Error(`${name} does not isolate contents write from OIDC.`); + } + const environmentMatches = [...block.matchAll(/^ environment:\s*(\S+)\s*$/gm)].map((match) => match[1]); + if (protectedMutationJobs.has(name)) { + if (environmentMatches.length !== 1 || environmentMatches[0] !== policy.environment) { + throw new Error(`${name} lacks its direct protected mutation environment.`); + } + } else if (environmentMatches.length !== 0) { + throw new Error(`${name} unexpectedly carries a protected environment.`); + } + assertNoDownloadedOrRepositoryExecution(block, `${name} contents publisher`); + } + + const expectedAudits = channel === "preview" ? { "stage-draft": 1, publish: 1 } : { publish: 3 }; + const authoritativeValidators = new Set(); + const protectedMutations = []; + let mintCount = 0; + let auditCount = 0; + for (const [name, expectedAuditCount] of Object.entries(expectedAudits)) { + const block = blocks.get(name); + const steps = stepBlocks(block); + const mintSteps = steps.filter((step) => /^ uses: actions\/create-github-app-token@/m.test(step.block)); + if (mintSteps.length !== 1) throw new Error(`${name} needs one exact ruleset-auditor token mint.`); + const expectedMintIf = channel === "preview" && name === "publish" ? "steps.finalize.outputs.release_id != ''" : null; + assertExactRulesetAuditorMint(mintSteps[0], `${name} ruleset-auditor mint`, expectedMintIf); + mintCount += 1; + const audits = steps.filter((step) => step.block.includes("github-token: ${{ steps.ruleset-auditor.outputs.token }}")); + if (audits.length !== expectedAuditCount) throw new Error(`${name} lacks one fresh authoritative audit per protected mutation.`); + auditCount += audits.length; + for (const audit of audits) { + const parsedAudit = parseStepShape(audit, `${name} ruleset audit`); + const expectedAuditRoot = { + name: audit.name, + uses: "actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1", + with: "", + }; + if (Object.hasOwn(parsedAudit.root, "if")) expectedAuditRoot.if = parsedAudit.root.if; + exactObject(parsedAudit.root, expectedAuditRoot, `${name} ruleset audit step mapping`); + const githubMemberReferences = [...audit.block.matchAll(/\bgithub\.([A-Za-z]+)/g)].map((match) => match[1]).sort(); + if ( + parsedAudit.nested.with?.["github-token"] !== "${{ steps.ruleset-auditor.outputs.token }}" || + parsedAudit.nested.with?.script !== "|" || Object.keys(parsedAudit.nested.with).length !== 2 || + Object.keys(parsedAudit.nested).length !== 1 || + canonicalList(githubMemberReferences) !== canonicalList(["graphql", "request"]) || + /\b(?:arguments|console|context|core|fetch|globalThis|process|require)\b/.test(audit.block) || + !/GET \/repos\/\{owner\}\/\{repo\}\/rulesets\/\{ruleset_id\}/.test(audit.block) || + !/ruleset_id: 21950766, includes_parents: false/.test(audit.block) || + !/restRuleset\.node_id !== "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4"/.test(audit.block) || + !/Object\.hasOwn\(restRuleset, "bypass_actors"\)/.test(audit.block) || + !/Object\.hasOwn\(restRuleset, "current_user_can_bypass"\)/.test(audit.block) || + !/ruleset\(databaseId: \$rulesetDatabaseId, includeParents: false\)/.test(audit.block) || + !/const authoritative = await github\.graphql\(query/.test(audit.block) || + !/bypassActors\.totalCount !== 0/.test(audit.block) || + !/updateAllowsFetchAndMerge !== false/.test(audit.block) || + /github\.rest\.git\.createRef|github\.rest\.repos\.updateRelease|github\.rest\.git\.createTag|repos\.createRelease/.test(audit.block) + ) throw new Error(`${name} ruleset audit is not the exact read-only combined REST and GraphQL proof.`); + const script = audit.block.slice(audit.block.indexOf(" script: |")); + authoritativeValidators.add(script); + } + for (let index = 0; index < steps.length; index += 1) { + const mutation = steps[index]; + if (!/github\.rest\.git\.createRef|github\.rest\.repos\.updateRelease/.test(mutation.block)) continue; + protectedMutations.push(`${name}:${mutation.name}`); + const audit = steps[index - 1]; + if (!audit?.block.includes("github-token: ${{ steps.ruleset-auditor.outputs.token }}")) { + throw new Error(`${name} protected mutation lacks an adjacent fresh authoritative audit.`); + } + const auditIf = parseStepShape(audit, `${name} adjacent ruleset audit`).root.if ?? null; + const mutationIf = parseStepShape(mutation, `${name} protected mutation`).root.if ?? null; + if (auditIf !== mutationIf) throw new Error(`${name} protected mutation and adjacent audit conditions differ.`); + if (mutation.block.includes("steps.ruleset-auditor.outputs.token") || mutation.block.includes("PYLON_RULESET_AUDITOR")) { + throw new Error(`${name} passes the ruleset-auditor credential to a contents mutation.`); + } + } + } + const occurrenceCount = (value) => workflow.split(value).length - 1; + const allExpressions = [...workflow.matchAll(/\$\{\{[\s\S]*?\}\}/g)].map((match) => match[0]); + const sensitiveExpressions = allExpressions + .filter((expression) => /PYLON_RULESET_AUDITOR|ruleset-auditor/.test(expression)); + const expectedSensitiveExpressions = [ + ...Array.from({ length: mintCount }, () => "${{ vars.PYLON_RULESET_AUDITOR_APP_ID }}"), + ...Array.from({ length: mintCount }, () => "${{ secrets.PYLON_RULESET_AUDITOR_PRIVATE_KEY }}"), + ...Array.from({ length: auditCount }, () => "${{ steps.ruleset-auditor.outputs.token }}"), + ]; + if ( + canonicalList(sensitiveExpressions) !== canonicalList(expectedSensitiveExpressions) || + canonicalList(allExpressions.filter((expression) => /\bsecrets\b/.test(expression))) !== + canonicalList(Array.from({ length: mintCount }, () => "${{ secrets.PYLON_RULESET_AUDITOR_PRIVATE_KEY }}")) || + canonicalList(allExpressions.filter((expression) => /\bvars\b/.test(expression))) !== + canonicalList(Array.from({ length: mintCount }, () => "${{ vars.PYLON_RULESET_AUDITOR_APP_ID }}")) || + occurrenceCount(CREATE_GITHUB_APP_TOKEN_ACTION) !== mintCount || occurrenceCount("private-key:") !== mintCount || + occurrenceCount("id: ruleset-auditor") !== mintCount || occurrenceCount("PYLON_RULESET_AUDITOR_APP_ID") !== mintCount || + occurrenceCount("PYLON_RULESET_AUDITOR_PRIVATE_KEY") !== mintCount || + occurrenceCount("steps.ruleset-auditor.outputs.token") !== auditCount || occurrenceCount("github-token:") !== auditCount || + /\$\{\{[\s\S]*?(?:toJSON|toJson)\s*\(\s*(?:steps|secrets|vars)\b/.test(workflow) || + /\$\{\{\s*steps\s*\}\}/.test(workflow) || /\bsteps\s*\[/.test(workflow) + ) throw new Error("Ruleset-auditor credentials or token outputs escape the exact mint and audit roles."); + if (authoritativeValidators.size !== 1) throw new Error("Protected mutations do not share one frozen authoritative ruleset validator."); + const expectedMutations = channel === "preview" + ? ["publish:Publish the exact approved preview draft", "stage-draft:Create or refetch the exact protected preview tag"] + : [ + "publish:Create or refetch the exact protected stable tag", + "publish:Create the exact protected stable reservation ref", + "publish:Publish only the exact protected stable draft", + ]; + if (canonicalList(protectedMutations) !== canonicalList(expectedMutations)) { + throw new Error("Protected mutation inventory differs from the closed ruleset-auditor policy."); + } + const nonAuditRulesetReads = [...workflow.matchAll(/GET \/repos\/\{owner\}\/\{repo\}\/rulesets\/\{ruleset_id\}/g)].length - + Object.values(expectedAudits).reduce((total, count) => total + count, 0); + if (nonAuditRulesetReads !== 0) throw new Error("Normal GITHUB_TOKEN ruleset admission is incorrectly treated as authoritative."); + if (!needs(stage).includes("verify-attestation")) throw new Error("Draft staging is not directly downstream of verified approval evidence."); + if (!needs(publisher).includes("stage-draft") || !needs(publisher).includes("verify-attestation") && channel === "preview") { + throw new Error("Final publisher dependency path differs from the approved graph."); + } + if (channel === "stable") { + const recovery = blocks.get("authorize-stable-resume"); + if ( + !recovery || scalar(recovery, "environment") !== "pylon-stable" || !/^ permissions: \{\}$/m.test(recovery) || + !/mode == 'resume'/.test(recovery) || !needs(publisher).includes("authorize-stable-resume") || + !/mode == 'normal'/.test(attest) || !/needs\.verify-attestation\.result == 'success'/.test(publisher) || + !/needs\.authorize-stable-resume\.result == 'success'/.test(publisher) + ) throw new Error("Stable normal and recovery approvals are not exact mutually exclusive paths."); + } + return { workflow: policy.workflow, environment: policy.environment }; +} + +export function readWorkflowAtSignerDigest(workflowPath, signerDigest) { + if (![PYLON_PREVIEW_WORKFLOW, PYLON_STABLE_WORKFLOW].includes(workflowPath)) throw new Error("Unsupported attestation workflow path."); + if (!/^[0-9a-f]{40}$/.test(signerDigest)) throw new Error("Workflow signer digest must be a full lowercase Git SHA."); + const token = process.env.GH_TOKEN ?? process.env.GITHUB_TOKEN; + if (!token) throw new Error("GH_TOKEN or GITHUB_TOKEN is required to authorize the historical signer workflow."); + const result = spawnSync( + "gh", + ["api", `repos/${PYLON_PUBLICATION_REPOSITORY}/contents/${workflowPath}?ref=${signerDigest}`], + { encoding: "utf8", timeout: 120_000, maxBuffer: 16 * 1024 * 1024, env: { ...process.env, GH_TOKEN: token } }, + ); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(`Unable to read approved workflow at signer digest: ${result.stderr}`); + const response = JSON.parse(result.stdout.replace(/\u001b\[[0-9;]*m/g, "")); + if (response.type !== "file" || response.path !== workflowPath || response.encoding !== "base64" || typeof response.content !== "string") { + throw new Error("Historical signer workflow response is not one exact file."); + } + return Buffer.from(response.content.replaceAll("\n", ""), "base64"); +} + +export function validateApprovedWorkflowBytes( + workflowPath, + workflow, + channel, + publicationPolicyRevision, + supportedPublicationPolicies = PYLON_SUPPORTED_PUBLICATION_POLICIES, +) { + const workflowBytes = Buffer.isBuffer(workflow) ? workflow : Buffer.from(workflow, "utf8"); + const workflowText = workflowBytes.toString("utf8"); + if (!Buffer.from(workflowText, "utf8").equals(workflowBytes)) throw new Error("Signer workflow is not exact UTF-8 bytes."); + if (!Number.isSafeInteger(publicationPolicyRevision) || publicationPolicyRevision < 1) { + throw new Error("Workflow policy needs an exact positive publication policy revision."); + } + const policy = supportedPublicationPolicies.find( + (candidate) => candidate.publicationPolicyRevision === publicationPolicyRevision, + ); + if (!policy) throw new Error(`Unsupported publication policy revision: ${publicationPolicyRevision}`); + const expectedPath = channel === "preview" ? policy.previewWorkflowPath : channel === "stable" ? policy.stableWorkflowPath : ""; + const expectedDigest = channel === "preview" ? policy.previewWorkflowSha256 : channel === "stable" ? policy.stableWorkflowSha256 : ""; + if (workflowPath !== expectedPath || !expectedPath) throw new Error("Signer workflow path differs from the exact publication policy channel."); + const actualDigest = createHash("sha256").update(workflowBytes).digest("hex"); + if (actualDigest !== expectedDigest) { + throw new Error(`Signer workflow bytes differ from publication policy p${publicationPolicyRevision} for ${channel}.`); + } + return validateApprovedAttestationWorkflow(workflowText, channel); +} + +export function verifyApprovedWorkflowAtSignerDigest(workflowPath, signerDigest, channel, publicationPolicyRevision) { + const workflow = readWorkflowAtSignerDigest(workflowPath, signerDigest); + return validateApprovedWorkflowBytes(workflowPath, workflow, channel, publicationPolicyRevision); +} diff --git a/scripts/migrate-pylon-consumer-journal.mjs b/scripts/migrate-pylon-consumer-journal.mjs new file mode 100755 index 0000000000..f41413897a --- /dev/null +++ b/scripts/migrate-pylon-consumer-journal.mjs @@ -0,0 +1,24 @@ +#!/usr/bin/env node + +import { resolve } from "node:path"; + +import { migrateConsumerStateJournal } from "./lib/pylon-consumer-lock.mjs"; + +function parseArgs(args) { + if (args.length !== 2 || args[0] !== "--state" || !args[1] || args[1].startsWith("--")) { + throw new Error("Usage: migrate-pylon-consumer-journal --state "); + } + return resolve(args[1]); +} + +try { + const result = await migrateConsumerStateJournal(parseArgs(process.argv.slice(2))); + console.log(JSON.stringify({ + journalEpoch: result.epoch, + tipSha256: result.tipSha256, + sourceAuthoritySha256: result.sourceAuthoritySha256, + })); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} diff --git a/scripts/prepare-pylon-preview-manifest.mjs b/scripts/prepare-pylon-preview-manifest.mjs new file mode 100644 index 0000000000..3b6fcc28cd --- /dev/null +++ b/scripts/prepare-pylon-preview-manifest.mjs @@ -0,0 +1,53 @@ +#!/usr/bin/env node + +import { readFileSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { PYLON_RELEASE_MANIFEST } from "./lib/pylon-release.mjs"; +import { + canonicalJson, + createPreviewManifest, + PYLON_PREVIEW_MANIFEST, +} from "./lib/pylon-publication.mjs"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const defaultArtifacts = join(root, ".npm", "pylon-release", "artifacts"); + +function parseArgs(args) { + const values = new Map(); + for (let index = 0; index < args.length; index += 2) { + const name = args[index]; + const value = args[index + 1]; + if (!name?.startsWith("--") || value === undefined) throw new Error("Preview preparation arguments must be name/value pairs."); + values.set(name, value); + } + if ([...values.keys()].some((name) => !["--artifact-dir", "--publication-policy-revision"].includes(name))) { + throw new Error("Unknown preview preparation argument."); + } + const publicationPolicyRevision = Number(values.get("--publication-policy-revision")); + if (!Number.isSafeInteger(publicationPolicyRevision) || publicationPolicyRevision < 1) { + throw new Error("Preview preparation requires an exact positive --publication-policy-revision."); + } + return { + artifactsDir: resolve(root, values.get("--artifact-dir") ?? defaultArtifacts), + publicationPolicyRevision, + }; +} + +try { + const { artifactsDir, publicationPolicyRevision } = parseArgs(process.argv.slice(2)); + const releaseManifestBytes = readFileSync(join(artifactsDir, PYLON_RELEASE_MANIFEST)); + const releaseManifest = JSON.parse(releaseManifestBytes); + const previewManifest = createPreviewManifest(releaseManifest, releaseManifestBytes, { + sequenceEpoch: 1, + sequence: Number(process.env.GITHUB_RUN_NUMBER), + workflowRunId: process.env.GITHUB_RUN_ID ?? "", + publicationPolicyRevision, + }); + writeFileSync(join(artifactsDir, PYLON_PREVIEW_MANIFEST), canonicalJson(previewManifest)); + console.log(`Created ${join(artifactsDir, PYLON_PREVIEW_MANIFEST)}`); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} diff --git a/scripts/prepare-pylon-stable-manifest.mjs b/scripts/prepare-pylon-stable-manifest.mjs new file mode 100644 index 0000000000..69830cdd66 --- /dev/null +++ b/scripts/prepare-pylon-stable-manifest.mjs @@ -0,0 +1,283 @@ +#!/usr/bin/env node + +import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + assertImmutableReleaseIdentity, + canonicalJson, + createStableManifest, + nextStableSequence, + parseStableTag, + publicationReleaseBody, + PYLON_PREVIEW_MANIFEST, + PYLON_PUBLICATION_REPOSITORY, + PYLON_STABLE_MANIFEST, + sha256Bytes, + validateStableHistory, + validateStableManifest, +} from "./lib/pylon-publication.mjs"; +import { verifyPreviewPublication } from "./verify-pylon-preview-publication.mjs"; +import { verifyStableAttestation } from "./verify-pylon-stable-attestation.mjs"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +function parseArgs(args) { + const values = new Map(); + for (let index = 0; index < args.length; index += 2) { + const name = args[index]; + const value = args[index + 1]; + if (!name?.startsWith("--") || value === undefined) throw new Error("Stable preparation arguments must be name/value pairs."); + values.set(name, value); + } + const artifactDir = resolve(root, values.get("--artifact-dir") ?? ".npm/pylon-stable/preview"); + const outDir = resolve(root, values.get("--out-dir") ?? ".npm/pylon-stable/output"); + const operation = values.get("--operation") ?? "promote"; + const policySha = values.get("--policy-sha") ?? ""; + const policyTree = values.get("--policy-tree") ?? ""; + const publicationPolicyRevision = Number(values.get("--publication-policy-revision")); + const revokeTag = values.get("--revoke-tag") ?? ""; + const reason = values.get("--reason") ?? "withdrawn"; + if (!["promote", "withdraw"].includes(operation)) throw new Error("Stable operation must be promote or withdraw."); + if (!/^[0-9a-f]{40}$/.test(policySha)) throw new Error("Stable preparation requires an exact --policy-sha."); + if (!/^[0-9a-f]{40}$/.test(policyTree)) throw new Error("Stable preparation requires an exact --policy-tree."); + if (!Number.isSafeInteger(publicationPolicyRevision) || publicationPolicyRevision < 1) { + throw new Error("Stable preparation requires an exact positive --publication-policy-revision."); + } + if (operation === "promote" && (revokeTag || values.has("--reason"))) { + throw new Error("A normal promotion cannot carry withdrawal metadata."); + } + if (operation === "withdraw" && !revokeTag) throw new Error("Withdrawal requires --revoke-tag."); + return { artifactDir, outDir, operation, policySha, policyTree, publicationPolicyRevision, revokeTag, reason }; +} + +function apiHeaders(accept = "application/vnd.github+json") { + const token = process.env.GITHUB_TOKEN; + if (!token) throw new Error("GITHUB_TOKEN is required to read stable publication history."); + return { + Accept: accept, + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "pylon-prime-stable-publication", + }; +} + +async function api(path, options = {}) { + const response = await fetch(`https://api.github.com${path}`, { + method: options.method ?? "GET", + headers: apiHeaders(options.accept), + redirect: "follow", + signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) throw new Error(`GitHub API ${path} failed with ${response.status}.`); + return options.bytes ? Buffer.from(await response.arrayBuffer()) : response.json(); +} + +async function paginate(path) { + const values = []; + for (let page = 1; ; page += 1) { + const separator = path.includes("?") ? "&" : "?"; + const batch = await api(`${path}${separator}per_page=100&page=${page}`); + if (!Array.isArray(batch)) throw new Error(`GitHub API ${path} did not return a list.`); + values.push(...batch); + if (batch.length < 100) return values; + } +} + +async function stableTagNames() { + try { + const refs = await paginate(`/repos/${PYLON_PUBLICATION_REPOSITORY}/git/matching-refs/tags/pylon-stable-`); + return refs + .map((entry) => entry.ref.replace(/^refs\/tags\//, "")) + .filter((tag) => /^pylon-stable-[0-9]{6}-g[0-9a-f]{12}-r[1-9][0-9]*$/.test(tag)) + .sort(); + } catch (error) { + if (String(error).includes("failed with 409") || String(error).includes("failed with 404")) return []; + throw error; + } +} + +export function selectStableHistoryReleases(releases, { excludeDraftId = null } = {}) { + if (!Array.isArray(releases)) throw new Error("Stable release history response is not a list."); + const selected = []; + for (const release of releases.filter((candidate) => candidate.tag_name?.startsWith("pylon-stable-"))) { + parseStableTag(release.tag_name); + if (release.draft && excludeDraftId !== null && String(release.id) === String(excludeDraftId)) continue; + if (release.draft) throw new Error(`Stable release ${release.tag_name} is an unexpected draft outside exact recovery.`); + selected.push(release); + } + return selected; +} + +export async function readStableHistory({ verifyAllAttestations = false, excludeDraftId = null, excludeDraftTag = null } = {}) { + const allReleases = await paginate(`/repos/${PYLON_PUBLICATION_REPOSITORY}/releases`); + const excludedDraft = excludeDraftId === null ? null : allReleases.find((release) => + release.draft && String(release.id) === String(excludeDraftId) && release.tag_name?.startsWith("pylon-stable-"), + ); + const releases = selectStableHistoryReleases(allReleases, { excludeDraftId }); + const manifests = []; + const manifestBytes = new Map(); + for (const release of releases) { + if (release.immutable !== true || release.assets?.length !== 1) { + throw new Error(`Stable release ${release.tag_name} is draft, mutable, or has an unexpected asset set.`); + } + const asset = release.assets[0]; + if (asset.name !== PYLON_STABLE_MANIFEST || !asset.url) { + throw new Error(`Stable release ${release.tag_name} lacks its only channel manifest.`); + } + const bytes = await api(new URL(asset.url).pathname, { accept: "application/octet-stream", bytes: true }); + if (sha256Bytes(bytes) !== asset.digest?.replace(/^sha256:/, "")) { + throw new Error(`Stable release asset digest mismatch for ${release.tag_name}.`); + } + const manifest = validateStableManifest(JSON.parse(bytes)); + if (canonicalJson(manifest) !== bytes.toString("utf8") || manifest.tag !== release.tag_name) { + throw new Error(`Stable release ${release.tag_name} has noncanonical or mismatched metadata.`); + } + assertImmutableReleaseIdentity(release, { + tag: manifest.tag, + name: `Pylon Prime stable ${manifest.tag}`, + body: publicationReleaseBody({ + channel: "stable", + tag: manifest.tag, + source: manifest.build.source.commit, + tree: manifest.build.source.tree, + recipeRevision: manifest.build.recipeRevision, + policyCommit: manifest.promotion.policyCommit, + policyTree: manifest.promotion.policyTree, + stableManifestBytes: bytes, + }), + prerelease: false, + sourceSha: manifest.promotion.policyCommit, + assets: [{ name: PYLON_STABLE_MANIFEST, size: bytes.byteLength, sha256: sha256Bytes(bytes) }], + }); + manifests.push(manifest); + manifestBytes.set(manifest.tag, bytes); + } + const ordered = validateStableHistory(manifests); + if (excludeDraftTag !== null && excludeDraftTag !== excludedDraft?.tag_name && excludedDraft !== null) { + throw new Error("Selected recovery draft id and tag do not identify the same draft."); + } + const tags = (await stableTagNames()).filter((tag) => tag !== (excludeDraftTag ?? excludedDraft?.tag_name)); + const releaseTags = ordered.map((manifest) => manifest.tag).sort(); + if (canonicalJson(tags) !== canonicalJson(releaseTags)) { + throw new Error("Stable tags and immutable release history differ."); + } + const authorizationSet = verifyAllAttestations ? ordered : ordered.slice(-1); + if (authorizationSet.length > 0) { + const verificationDir = mkdtempSync(join(tmpdir(), "pylon-stable-attestation-")); + const verificationPath = join(verificationDir, PYLON_STABLE_MANIFEST); + try { + for (const manifest of authorizationSet) { + writeFileSync(verificationPath, manifestBytes.get(manifest.tag), { mode: 0o600 }); + verifyStableAttestation( + verificationPath, + manifest.promotion.policyCommit, + manifest.promotion.policyTree, + ); + } + } finally { + rmSync(verificationDir, { recursive: true, force: true }); + } + } + return ordered; +} + +export function isExactWithdrawalReplay(latest, { previewTag, revokeTag, reason }) { + const revocation = latest?.promotion?.revocation; + return latest?.build?.previewTag === previewTag && latest?.promotion?.kind === "withdraw" && + revocation?.stableTag === revokeTag && /^pylon-build-g[0-9a-f]{12}-r[1-9][0-9]*$/.test(revocation?.buildTag ?? "") && + revocation?.reason === reason && revocation?.revokedBySequence === latest.sequence && + latest.revocations?.some((entry) => canonicalJson(entry) === canonicalJson(revocation)); +} + +function writeOutputs(values) { + if (!process.env.GITHUB_OUTPUT) return; + for (const [name, value] of Object.entries(values)) appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + try { + const args = parseArgs(process.argv.slice(2)); + const verified = verifyPreviewPublication(args.artifactDir, { historical: true }); + const previewBytes = readFileSync(join(args.artifactDir, PYLON_PREVIEW_MANIFEST)); + const history = await readStableHistory(); + const latest = history.at(-1); + let publish = true; + let stableManifest; + if ( + args.operation === "promote" && + latest?.build.previewTag === verified.previewManifest.build.tag && + latest.promotion.kind === "promote" + ) { + publish = false; + stableManifest = latest; + } else if ( + args.operation === "withdraw" && + isExactWithdrawalReplay(latest, { + previewTag: verified.previewManifest.build.tag, + revokeTag: args.revokeTag, + reason: args.reason, + }) + ) { + publish = false; + stableManifest = latest; + } else { + const sequence = nextStableSequence(history); + const revocations = latest ? structuredClone(latest.revocations) : []; + let promotion = { + kind: "promote", + policyCommit: args.policySha, + policyTree: args.policyTree, + publicationPolicyRevision: args.publicationPolicyRevision, + }; + if (args.operation === "withdraw") { + const revoked = history.find((manifest) => manifest.tag === args.revokeTag); + if (!revoked) throw new Error("Withdrawal can revoke only an existing stable sequence."); + if (revocations.some((entry) => entry.stableTag === args.revokeTag)) { + throw new Error("Stable sequence is already withdrawn."); + } + const revocation = { + stableTag: revoked.tag, + buildTag: revoked.build.previewTag, + reason: args.reason, + revokedBySequence: sequence, + }; + revocations.push(revocation); + promotion = { + kind: "withdraw", + policyCommit: args.policySha, + policyTree: args.policyTree, + publicationPolicyRevision: args.publicationPolicyRevision, + revocation, + }; + } + stableManifest = createStableManifest({ + previewManifest: verified.previewManifest, + previewManifestBytes: previewBytes, + sequence, + previous: latest + ? { tag: latest.tag, sha256: sha256Bytes(Buffer.from(canonicalJson(latest))) } + : null, + revocations, + promotion, + }); + } + mkdirSync(args.outDir, { recursive: true }); + const outputBytes = canonicalJson(stableManifest); + writeFileSync(join(args.outDir, PYLON_STABLE_MANIFEST), outputBytes); + writeOutputs({ + publish: String(publish), + manifest_sha256: sha256Bytes(Buffer.from(outputBytes)), + tag: stableManifest.tag, + source_sha: stableManifest.build.source.commit, + source_tree: stableManifest.build.source.tree, + sequence: String(stableManifest.sequence), + }); + console.log(JSON.stringify({ publish, tag: stableManifest.tag, sequence: stableManifest.sequence })); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/pylon-prime-agent-release.test.mjs b/scripts/pylon-prime-agent-release.test.mjs index ecb7ee4953..e6cbabe38b 100644 --- a/scripts/pylon-prime-agent-release.test.mjs +++ b/scripts/pylon-prime-agent-release.test.mjs @@ -272,10 +272,8 @@ test("smokes the exact caller-owned session SDK contract", () => { assert.match(smokeSource, /disposeOwnedSession\(\{ timeoutMs: 15_000 \}\)/); }); -test("bounds hosted Windows artifact installation without weakening POSIX checks", () => { - assert.equal(releaseInstallTimeoutMs("win32"), 360_000); - assert.equal(releaseInstallTimeoutMs("linux"), 180_000); - assert.equal(releaseInstallTimeoutMs("darwin"), 180_000); +test("bounds supported Linux and macOS artifact installation", () => { + assert.equal(releaseInstallTimeoutMs(), 180_000); }); test("uses the pinned npm CLI path for cross-platform release subprocesses", () => { diff --git a/scripts/pylon-prime-supported-release-recipes-v1.json b/scripts/pylon-prime-supported-release-recipes-v1.json new file mode 100644 index 0000000000..8d73871e6e --- /dev/null +++ b/scripts/pylon-prime-supported-release-recipes-v1.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": 1, + "recipes": [ + { + "recipeRevision": 1, + "manifestSchemaVersion": 1, + "nodeVersion": "22.23.2", + "npmVersion": "11.10.1", + "minimumNodeVersion": "22.8.0" + } + ], + "publicationPolicies": [ + { + "publicationPolicyRevision": 1, + "previewWorkflowPath": ".github/workflows/pylon-preview-release.yml", + "previewWorkflowSha256": "e790a5da7063bd40fbd886e84945c3200291194fdbd5b002079349e45356a41d", + "stableWorkflowPath": ".github/workflows/pylon-stable-release.yml", + "stableWorkflowSha256": "dfcecdf6b58f143f9b7a543eadd124c190350ae29ac9eadccb907f1398b0958a" + } + ] +} diff --git a/scripts/pylon-publication.test.mjs b/scripts/pylon-publication.test.mjs new file mode 100644 index 0000000000..416a376a57 --- /dev/null +++ b/scripts/pylon-publication.test.mjs @@ -0,0 +1,3102 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { + chmodSync, + closeSync, + existsSync, + fsyncSync, + mkdirSync, + mkdtempSync, + openSync, + readFileSync, + readdirSync, + realpathSync, + renameSync, + rmSync, + statSync, + symlinkSync, + truncateSync, + writeFileSync, + writeSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { createRequire } from "node:module"; +import { test } from "node:test"; + +import { + createReleaseManifest, + PYLON_RELEASE_MANIFEST, + PYLON_RELEASE_NPM_VERSION, + PYLON_RELEASE_NODE_VERSION, +} from "./lib/pylon-release.mjs"; +import { + assertCanonicalInvocation, + assertImmutableReleaseIdentity, + canonicalJson, + createPreviewManifest, + createStableManifest, + GITHUB_ACTIONS_APP_ID, + nextStableSequence, + parsePreviewTag, + parseStableTag, + parseSupportedReleaseRecipeRegistry, + publicationReleaseBody, + PYLON_REQUIRED_CHECKS, + PYLON_PREVIEW_MANIFEST, + PYLON_PREVIEW_WORKFLOW, + PYLON_PUBLICATION_REF, + PYLON_PUBLICATION_REPOSITORY, + PYLON_STABLE_MANIFEST, + PYLON_STABLE_WORKFLOW, + sha256Bytes, + stableManifestBytesFromReleaseBody, + stableReservationMessage, + stableSequenceReservationTag, + parseStableSequenceReservationTag, + stableTag, + validateAttestationEvidence, + validateMergedChangelogProof, + validatePreviewManifest, + validatePublishedReleaseManifest, + validateRequiredChecks, + validateStableHistory, + validateStableManifest, + validateWorkflowArtifactProvenance, +} from "./lib/pylon-publication.mjs"; +import { + ATTEST_ACTION_CHAIN, + CREATE_GITHUB_APP_TOKEN_ACTION, + validateApprovedAttestationWorkflow, + validateApprovedWorkflowBytes, +} from "./lib/pylon-workflow-policy.mjs"; +import { + PYLON_PUBLICATION_RULESET_GRAPHQL_QUERY, + PYLON_PUBLICATION_RULESET_GRAPHQL_VARIABLES, +} from "./lib/pylon-ruleset-auditor.mjs"; +import { validatePreviewWorkflowRunEvidence, verifyGhAttestationResult } from "./verify-pylon-publication-attestations.mjs"; +import { recordPreviewHighWater } from "./verify-pylon-preview-history.mjs"; +import { verifyStableHistoryWithState } from "./verify-pylon-stable-history.mjs"; +import { verifyPreviewPublication } from "./verify-pylon-preview-publication.mjs"; +import { + ensureDurableConsumerStateDirectory, + migrateConsumerStateJournal, + rotateConsumerStateJournal, + withConsumerStateLock, +} from "./lib/pylon-consumer-lock.mjs"; +import { + PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + PYLON_STABLE_HISTORY_MAX_MANIFESTS, + readBoundedRegularFile, +} from "./lib/pylon-bounded-file.mjs"; +import { isExactWithdrawalReplay, selectStableHistoryReleases } from "./prepare-pylon-stable-manifest.mjs"; +import { recoverStableDraft } from "./recover-pylon-stable-manifest.mjs"; + +const root = resolve(import.meta.dirname, ".."); +const nodeRequire = createRequire(import.meta.url); +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; +const source = { + repository: "https://github.com/pylon-code/prime-agent", + commit: "0123456789abcdef0123456789abcdef01234567", + tree: "89abcdef0123456789abcdef0123456789abcdef", +}; +const version = "0.8.1"; +const invocation = { + sequenceEpoch: 1, + sequence: 17, + workflowRunId: "33428882721", + publicationPolicyRevision: 1, +}; + +function consumerJournal(statePath) { + const journal = `${statePath}.journal`; + const checkpoint = readdirSync(journal).find((name) => name.startsWith("checkpoint-")); + const epoch = readdirSync(journal).find((name) => name.startsWith("epoch-")); + if (!checkpoint || !epoch) throw new Error("consumer journal fixture is incomplete"); + return { journal, checkpoint: join(journal, checkpoint), epoch: join(journal, epoch) }; +} + +function transitionNames(statePath) { + return readdirSync(consumerJournal(statePath).epoch).filter((name) => name.startsWith("transition-")); +} + +function fakeReleaseManifest() { + return createReleaseManifest({ + source, + version, + toolchain: { node: PYLON_RELEASE_NODE_VERSION, npm: PYLON_RELEASE_NPM_VERSION }, + lockfileSha256: "a".repeat(64), + artifacts: [ + ["prime-agent", "pylon-prime-agent-0.8.1.tgz", "d"], + ["@earendil-works/pi-ai", "pylon-prime-agent-ai-0.8.1.tgz", "a"], + ["@earendil-works/pi-agent-core", "pylon-prime-agent-core-0.8.1.tgz", "b"], + ["@earendil-works/pi-tui", "pylon-prime-agent-tui-0.8.1.tgz", "c"], + ].map(([packageName, file, byte]) => { + const bytes = Buffer.from(byte); + return { + package: packageName, + file, + size: bytes.byteLength, + sha256: byte.repeat(64), + sha512: byte.repeat(128), + }; + }), + }); +} + +function manifests() { + const release = fakeReleaseManifest(); + const releaseBytes = Buffer.from(`${JSON.stringify(release, null, 2)}\n`); + const preview = createPreviewManifest(release, releaseBytes, invocation); + const previewBytes = Buffer.from(canonicalJson(preview)); + return { release, releaseBytes, preview, previewBytes }; +} + +function firstStable() { + const { preview, previewBytes } = manifests(); + return createStableManifest({ + previewManifest: preview, + previewManifestBytes: previewBytes, + sequence: 1, + previous: null, + promotion: { kind: "promote", policyCommit: source.commit, policyTree: source.tree, publicationPolicyRevision: 1 }, + }); +} + +function secondStable(previous = firstStable(), options = {}) { + const { preview, previewBytes } = manifests(); + const sequence = 2; + const revocation = { + stableTag: previous.tag, + buildTag: previous.build.previewTag, + reason: "security-withdrawal", + revokedBySequence: sequence, + }; + return createStableManifest({ + previewManifest: preview, + previewManifestBytes: previewBytes, + sequence, + previous: { tag: previous.tag, sha256: sha256Bytes(Buffer.from(canonicalJson(previous))) }, + revocations: options.withdraw ? [revocation] : [], + promotion: options.withdraw ? { kind: "withdraw", policyCommit: source.commit, policyTree: source.tree, publicationPolicyRevision: 1, revocation } : { kind: "promote", policyCommit: source.commit, policyTree: source.tree, publicationPolicyRevision: 1 }, + }); +} + +function slsaAttestationOutput(subject, expected, { runId = expected.workflowRunId ?? "445566", attempt = "2" } = {}) { + return JSON.stringify([{ verificationResult: { + statement: { + predicateType: "https://slsa.dev/provenance/v1", + subject: [{ name: subject.name, digest: { sha256: subject.sha256 } }], + predicate: { + buildDefinition: { + buildType: "https://actions.github.io/buildtypes/workflow/v1", + externalParameters: { workflow: { + repository: `https://github.com/${expected.repository}`, path: expected.workflow, ref: PYLON_PUBLICATION_REF, + } }, + internalParameters: { github: { + event_name: expected.event, repository_id: "1349002285", repository_owner_id: "11325514", runner_environment: "github-hosted", + } }, + resolvedDependencies: [{ + uri: `git+https://github.com/${expected.repository}@${PYLON_PUBLICATION_REF}`, + digest: { gitCommit: expected.sourceSha }, + }], + }, + runDetails: { + builder: { id: `https://github.com/${expected.repository}/${expected.workflow}@${PYLON_PUBLICATION_REF}` }, + metadata: { invocationId: `https://github.com/${expected.repository}/actions/runs/${runId}/attempts/${attempt}` }, + }, + }, + }, + verifiedTimestamps: [{ type: "Tlog", uri: "https://rekor.sigstore.dev" }], + } }]); +} + +function githubScriptForStep(workflowPath, stepName) { + const lines = readFileSync(join(root, workflowPath), "utf8").split("\n"); + const step = lines.findIndex((line) => line === ` - name: ${stepName}`); + if (step < 0) throw new Error(`Missing workflow step ${stepName}.`); + const marker = lines.findIndex((line, index) => index > step && line === " script: |"); + if (marker < 0) throw new Error(`Missing github-script body for ${stepName}.`); + const body = []; + for (let index = marker + 1; index < lines.length; index += 1) { + if (lines[index] && !lines[index].startsWith(" ")) break; + body.push(lines[index].slice(12)); + } + return body.join("\n"); +} + +function exactPublicationTagRuleset() { + return { + id: 21_950_766, + node_id: "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4", + name: "Pylon immutable publication tags", + target: "tag", + source_type: "Repository", + source: "pylon-code/prime-agent", + enforcement: "active", + bypass_actors: [], + current_user_can_bypass: "never", + conditions: { + ref_name: { + exclude: [], + include: ["refs/tags/pylon-build-*", "refs/tags/pylon-stable-*"], + }, + }, + rules: [ + { type: "update", parameters: { update_allows_fetch_and_merge: false } }, + { type: "deletion" }, + ], + }; +} + +function exactPublicationTagRulesetGraphql() { + return { + repository: { + id: "R_kgDOUGgkLQ", + databaseId: 1_349_002_285, + nameWithOwner: "pylon-code/prime-agent", + ruleset: { + id: "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4", + databaseId: 21_950_766, + name: "Pylon immutable publication tags", + enforcement: "ACTIVE", + target: "TAG", + source: { + __typename: "Repository", + id: "R_kgDOUGgkLQ", + databaseId: 1_349_002_285, + nameWithOwner: "pylon-code/prime-agent", + }, + bypassActors: { totalCount: 0 }, + conditions: { + refName: { include: ["refs/tags/pylon-build-*", "refs/tags/pylon-stable-*"], exclude: [] }, + organizationProperty: null, + repositoryId: null, + repositoryName: null, + repositoryProperty: null, + }, + rules: { + totalCount: 2, + nodes: [ + { type: "UPDATE", parameters: { __typename: "UpdateParameters", updateAllowsFetchAndMerge: false } }, + { type: "DELETION", parameters: null }, + ], + }, + }, + }, + }; +} + +async function inlinePublicationTagRulesetValidator(restResponses, graphqlResponses = [exactPublicationTagRulesetGraphql()]) { + const script = githubScriptForStep( + ".github/workflows/pylon-preview-release.yml", + "Require authoritative publication tag ruleset before preview tag CAS", + ); + const validate = new AsyncFunction("github", script); + let restRequest = 0; + let graphqlRequest = 0; + const order = []; + const github = { + request: async (route, parameters) => { + order.push("REST"); + assert.equal(route, "GET /repos/{owner}/{repo}/rulesets/{ruleset_id}"); + assert.deepEqual(parameters, { + owner: "pylon-code", repo: "prime-agent", ruleset_id: 21_950_766, includes_parents: false, + headers: { accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" }, + }); + const response = restResponses[Math.min(restRequest, restResponses.length - 1)]; + restRequest += 1; + if (response instanceof Error) throw response; + if (response && Object.hasOwn(response, "status") && Object.hasOwn(response, "data")) return response; + return { status: 200, data: response }; + }, + graphql: async (query, variables) => { + order.push("GraphQL"); + assert.equal(query, PYLON_PUBLICATION_RULESET_GRAPHQL_QUERY); + assert.deepEqual(variables, PYLON_PUBLICATION_RULESET_GRAPHQL_VARIABLES); + const response = graphqlResponses[Math.min(graphqlRequest, graphqlResponses.length - 1)]; + graphqlRequest += 1; + if (response instanceof Error) throw response; + return response; + }, + }; + return { + validate: async () => { + const result = await validate(github); + assert.equal(order.at(-1), "GraphQL", "GraphQL must be the last authoritative read before return"); + return result; + }, + requests: () => ({ rest: restRequest, graphql: graphqlRequest, order: [...order] }), + }; +} + +test("canonical publication JSON sorts every object key and rejects unsupported values", () => { + assert.equal(canonicalJson({ z: 1, a: { y: 2, b: 3 } }), '{\n "a": {\n "b": 3,\n "y": 2\n },\n "z": 1\n}\n'); + assert.throws(() => canonicalJson({ bad: undefined }), /undefined/); + assert.throws(() => canonicalJson({ bad: Number.NaN }), /unsupported/); +}); + +test("preview and stable tags use exact closed grammars", () => { + assert.deepEqual(parsePreviewTag("pylon-build-g0123456789ab-r1"), { commit12: "0123456789ab", recipeRevision: 1 }); + assert.equal( + stableTag({ sequence: 7, sourceCommit: source.commit, recipeRevision: 1 }), + "pylon-stable-000007-g0123456789ab-r1", + ); + assert.equal(parseStableTag("pylon-stable-000007-g0123456789ab-r1").sequence, 7); + assert.equal(stableSequenceReservationTag(7), "pylon-stable-sequence-000007"); + assert.equal(stableSequenceReservationTag(7), stableSequenceReservationTag(parseStableTag("pylon-stable-000007-gffffffffffff-r9").sequence)); + assert.equal(parseStableSequenceReservationTag("pylon-stable-sequence-000007").sequence, 7); + for (const invalid of ["v1", "main", "pylon-build-g012345-r1", "pylon-stable-7-g0123456789ab-r1"]) { + assert.throws(() => (invalid.startsWith("pylon-stable") ? parseStableTag(invalid) : parsePreviewTag(invalid))); + } + assert.throws(() => parseStableSequenceReservationTag("pylon-stable-sequence-000000")); + assert.throws(() => parseStableSequenceReservationTag("pylon-stable-sequence-000001-g0123456789ab")); +}); + +test("preview manifest binds the full source tree, build, recipe, and build-manifest digest", () => { + const { release, releaseBytes, preview } = manifests(); + assert.equal(validatePreviewManifest(preview, release, releaseBytes), preview); + assert.equal(canonicalJson(createPreviewManifest(release, releaseBytes, { ...invocation, runAttempt: "2" })), canonicalJson(preview), "rerun attempts keep identical bytes"); + for (const invalid of [ + { ...invocation, sequenceEpoch: 2 }, + { ...invocation, sequence: 0 }, + { ...invocation, workflowRunId: "01" }, + ]) assert.throws(() => createPreviewManifest(release, releaseBytes, invalid), /sequence identity/); + for (const invalid of [ + { ...invocation, publicationPolicyRevision: 0 }, + { ...invocation, publicationPolicyRevision: 2 }, + { sequenceEpoch: 1, sequence: 17, workflowRunId: "33428882721" }, + ]) assert.throws(() => createPreviewManifest(release, releaseBytes, invalid), /policy revision/); + for (const mutate of [ + (value) => (value.build.source.commit = "f".repeat(40)), + (value) => (value.build.source.tree = "f".repeat(40)), + (value) => (value.build.recipeRevision = 2), + (value) => (value.build.releaseManifest.sha256 = "f".repeat(64)), + ]) { + const changed = structuredClone(preview); + mutate(changed); + assert.throws(() => validatePreviewManifest(changed, release, releaseBytes), /does not match/); + } +}); + +test("current policy validates an older supported closed recipe without executing old source", () => { + const oldRecipe = { + recipeRevision: 7, + manifestSchemaVersion: 1, + nodeVersion: "20.19.1", + npmVersion: "10.8.2", + minimumNodeVersion: "20.12.0", + }; + const release = fakeReleaseManifest(); + release.build.id = `pylon-build-g${source.commit.slice(0, 12)}-r7`; + release.build.recipeRevision = 7; + release.build.node = oldRecipe.nodeVersion; + release.build.npm = oldRecipe.npmVersion; + release.build.assetBaseUrl = `https://github.com/pylon-code/prime-agent/releases/download/${release.build.id}`; + release.package.minimumNode = oldRecipe.minimumNodeVersion; + const releaseBytes = Buffer.from(canonicalJson(release)); + const preview = createPreviewManifest(fakeReleaseManifest(), manifests().releaseBytes, invocation); + preview.build.tag = release.build.id; + preview.build.id = release.build.id; + preview.build.recipeRevision = 7; + preview.build.releaseManifest.sha256 = sha256Bytes(releaseBytes); + assert.equal(validatePublishedReleaseManifest(release, [oldRecipe]), release); + assert.equal( + validatePreviewManifest(preview, release, releaseBytes, { historical: true, supportedRecipes: [oldRecipe] }), + preview, + ); + assert.throws(() => validatePublishedReleaseManifest({ ...release, extra: true }, [oldRecipe]), /Unsupported/); + assert.throws(() => validatePreviewManifest(preview, release, releaseBytes), /release manifest|recipe|build/i); + const stable = createStableManifest({ + previewManifest: preview, + previewManifestBytes: Buffer.from(canonicalJson(preview)), + sequence: 1, + promotion: { kind: "promote", policyCommit: source.commit, policyTree: source.tree, publicationPolicyRevision: 1 }, + }); + assert.equal(validateStableManifest(stable, [oldRecipe]), stable); + assert.throws(() => validateStableManifest(stable), /closed|malformed/i); +}); + +test("consumer preview high-water allows gaps but rejects rollback and same-sequence equivocation", async () => { + const fixture = realpathSync(mkdtempSync(join(tmpdir(), "pylon-preview-state-"))); + try { + const { preview, previewBytes } = manifests(); + const statePath = join(fixture, "consumer", "nested", "preview.json"); + await assert.rejects(() => recordPreviewHighWater(preview, previewBytes, { statePath }), /--initialize/); + assert.equal((await recordPreviewHighWater(preview, previewBytes, { statePath, initialize: true })).advanced, true); + assert.equal((await recordPreviewHighWater(preview, previewBytes, { statePath })).advanced, false); + const later = structuredClone(preview); + later.sequence += 3; + later.workflowRunId = String(Number(later.workflowRunId) + 3); + assert.equal((await recordPreviewHighWater(later, Buffer.from(canonicalJson(later)), { statePath })).state.highWater.sequence, later.sequence); + await assert.rejects(() => recordPreviewHighWater(preview, previewBytes, { statePath }), /older/); + const equivocation = structuredClone(later); + equivocation.build.releaseManifest.sha256 = "f".repeat(64); + await assert.rejects( + () => recordPreviewHighWater(equivocation, Buffer.from(canonicalJson(equivocation)), { statePath }), + /equivocates/, + ); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } +}); + +test("stable history is contiguous, previous-digest chained, high-water marked, sorted, and append-only", () => { + const first = firstStable(); + const second = secondStable(first, { withdraw: true }); + assert.deepEqual(first.build.previewSequence, { + sequenceEpoch: invocation.sequenceEpoch, + sequence: invocation.sequence, + workflowRunId: invocation.workflowRunId, + }); + assert.equal(first.build.publicationPolicyRevision, invocation.publicationPolicyRevision); + assert.equal(first.history.highWater, 0); + assert.equal(second.history.highWater, 1); + assert.equal(nextStableSequence([second, first]), 3); + assert.deepEqual(validateStableHistory([first, second]), [first, second]); + const wrongPrevious = structuredClone(second); + wrongPrevious.history.previous.sha256 = "f".repeat(64); + assert.throws(() => validateStableHistory([first, wrongPrevious]), /previous manifest digest/); + const gap = structuredClone(second); + gap.sequence = 3; + gap.tag = gap.tag.replace("000002", "000003"); + gap.history.highWater = 2; + assert.throws(() => validateStableHistory([first, gap])); + const third = structuredClone(secondStable(first, { withdraw: false })); + third.sequence = 3; + third.tag = third.tag.replace("000002", "000003"); + third.history = { highWater: 2, previous: { tag: second.tag, sha256: sha256Bytes(Buffer.from(canonicalJson(second))) } }; + assert.throws(() => validateStableHistory([first, second, third]), /append-only/); +}); + +test("consumer stable high-water requires explicit initialization, is idempotent, and advances atomically", async () => { + const fixture = realpathSync(mkdtempSync(join(tmpdir(), "pylon-stable-state-"))); + try { + const first = firstStable(); + const second = secondStable(first); + const firstPath = join(fixture, "first.json"); + const secondPath = join(fixture, "second.json"); + const statePath = join(fixture, "consumer", "nested", "stable.json"); + writeFileSync(firstPath, canonicalJson(first)); + writeFileSync(secondPath, canonicalJson(second)); + await assert.rejects(() => verifyStableHistoryWithState([firstPath], { statePath }), /--initialize/); + const initialized = await verifyStableHistoryWithState([firstPath], { statePath, initialize: true }); + assert.equal(initialized.advanced, true); + assert.equal(initialized.state.highWater.sequence, 1); + const witnessedBytes = readFileSync(statePath, "utf8"); + const repeated = await verifyStableHistoryWithState([firstPath], { statePath }); + assert.equal(repeated.advanced, false); + assert.equal(readFileSync(statePath, "utf8"), witnessedBytes); + const advanced = await verifyStableHistoryWithState([firstPath, secondPath], { statePath }); + assert.equal(advanced.advanced, true); + assert.equal(advanced.state.highWater.sequence, 2); + writeFileSync(statePath, witnessedBytes); + const repairedRollback = await verifyStableHistoryWithState([firstPath, secondPath], { statePath }); + assert.equal(repairedRollback.state.highWater.sequence, 2, "the immutable transaction tip outranks a rolled-back projection"); + assert.equal(JSON.parse(readFileSync(statePath, "utf8")).highWater.sequence, 2); + rmSync(statePath); + await verifyStableHistoryWithState([firstPath, secondPath], { statePath }); + assert.equal(JSON.parse(readFileSync(statePath, "utf8")).highWater.sequence, 2, "a deleted projection is repaired from the journal"); + writeFileSync(statePath, ""); + await verifyStableHistoryWithState([firstPath, secondPath], { statePath }); + assert.equal(JSON.parse(readFileSync(statePath, "utf8")).highWater.sequence, 2, "an empty projection is repaired from the journal"); + const legacyPath = join(fixture, "legacy.json"); + writeFileSync(legacyPath, canonicalJson(initialized.state), { mode: 0o600 }); + const migrated = await verifyStableHistoryWithState([firstPath], { statePath: legacyPath }); + assert.equal(migrated.advanced, false); + assert.equal(transitionNames(legacyPath).length, 1); + await assert.rejects(() => verifyStableHistoryWithState([firstPath, secondPath], { statePath, initialize: true }), /cannot reset/); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } +}); + +test("consumer stable high-water rejects rollback and a rewritten witnessed sequence", async () => { + const fixture = realpathSync(mkdtempSync(join(tmpdir(), "pylon-stable-state-"))); + try { + const first = firstStable(); + const second = secondStable(first); + const firstPath = join(fixture, "first.json"); + const secondPath = join(fixture, "second.json"); + const statePath = join(fixture, "stable.json"); + writeFileSync(firstPath, canonicalJson(first)); + writeFileSync(secondPath, canonicalJson(second)); + await verifyStableHistoryWithState([firstPath, secondPath], { statePath, initialize: true }); + await assert.rejects(() => verifyStableHistoryWithState([firstPath], { statePath }), /older than/); + const rewrittenFirst = structuredClone(first); + rewrittenFirst.promotion.policyTree = "f".repeat(40); + writeFileSync(firstPath, canonicalJson(rewrittenFirst)); + await assert.rejects(() => verifyStableHistoryWithState([firstPath], { statePath }), /older than|rewrites/); + const rewrittenSecond = createStableManifest({ + previewManifest: manifests().preview, + previewManifestBytes: manifests().previewBytes, + sequence: 2, + previous: { tag: rewrittenFirst.tag, sha256: sha256Bytes(Buffer.from(canonicalJson(rewrittenFirst))) }, + promotion: { kind: "promote", policyCommit: source.commit, policyTree: "e".repeat(40), publicationPolicyRevision: 1 }, + }); + writeFileSync(secondPath, canonicalJson(rewrittenSecond)); + await assert.rejects(() => verifyStableHistoryWithState([firstPath, secondPath], { statePath }), /rewrites/); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } +}); + +test("consumer stable high-water rejects malformed, noncanonical, symlinked, and locked local state", async () => { + const fixture = realpathSync(mkdtempSync(join(tmpdir(), "pylon-stable-state-"))); + try { + const manifestPath = join(fixture, "first.json"); + const statePath = join(fixture, "stable.json"); + writeFileSync(manifestPath, canonicalJson(firstStable())); + writeFileSync(statePath, "{}\n", { mode: 0o600 }); + await assert.rejects(() => verifyStableHistoryWithState([manifestPath], { statePath }), /malformed/); + writeFileSync(statePath, JSON.stringify({ + schemaVersion: 1, + repository: "https://github.com/pylon-code/prime-agent", + channel: "stable", + highWater: { sequence: 1, tag: firstStable().tag, sha256: sha256Bytes(Buffer.from(canonicalJson(firstStable()))) }, + })); + await assert.rejects(() => verifyStableHistoryWithState([manifestPath], { statePath }), /not canonical/); + rmSync(statePath); + symlinkSync(manifestPath, statePath); + await assert.rejects(() => verifyStableHistoryWithState([manifestPath], { statePath }), /regular.*file/); + rmSync(statePath); + const realDirectory = join(fixture, "real-state-directory"); + const linkedDirectory = join(fixture, "linked-state-directory"); + mkdirSync(realDirectory); + symlinkSync(realDirectory, linkedDirectory); + await assert.rejects( + () => verifyStableHistoryWithState([manifestPath], { statePath: join(linkedDirectory, "stable.json"), initialize: true }), + /canonical real directory/, + ); + const badLockState = join(fixture, "bad-lock.json"); + symlinkSync(realDirectory, `${badLockState}.lock`); + await assert.rejects( + () => verifyStableHistoryWithState([manifestPath], { statePath: badLockState, initialize: true }), + /legacy consumer lock guard.*regular non-symlink file/i, + ); + const badJournalState = join(fixture, "bad-journal.json"); + symlinkSync(realDirectory, `${badJournalState}.journal`); + await assert.rejects( + () => verifyStableHistoryWithState([manifestPath], { statePath: badJournalState, initialize: true }), + /journal directory.*real directory/, + ); + const badJournalEntryState = join(fixture, "bad-journal-entry.json"); + mkdirSync(`${badJournalEntryState}.journal`, { mode: 0o700 }); + writeFileSync(join(`${badJournalEntryState}.journal`, ".unexpected"), "bad\n"); + await assert.rejects( + () => verifyStableHistoryWithState([manifestPath], { statePath: badJournalEntryState, initialize: true }), + /unexpected hidden entry/, + ); + + const exactMetadataState = join(fixture, "exact-metadata.json"); + await verifyStableHistoryWithState([manifestPath], { statePath: exactMetadataState, initialize: true }); + const exactMetadata = consumerJournal(exactMetadataState); + assert.equal(statSync(`${exactMetadataState}.lock`).mode & 0o777, 0o600); + assert.equal(statSync(exactMetadataState).mode & 0o777, 0o600); + assert.equal(statSync(exactMetadata.journal).mode & 0o777, 0o700); + assert.equal(statSync(join(exactMetadata.journal, ".owned-temporaries-v2")).mode & 0o777, 0o700); + assert.equal(statSync(exactMetadata.epoch).mode & 0o777, 0o700); + chmodSync(exactMetadataState, 0o666); + chmodSync(`${exactMetadataState}.lock`, 0o666); + chmodSync(exactMetadata.journal, 0o777); + chmodSync(exactMetadata.checkpoint, 0o666); + chmodSync(exactMetadata.epoch, 0o777); + for (const name of readdirSync(exactMetadata.epoch)) chmodSync(join(exactMetadata.epoch, name), 0o666); + await assert.rejects( + () => verifyStableHistoryWithState([manifestPath], { statePath: exactMetadataState }), + /exact 700 permissions|exact 600 permissions/, + ); + chmodSync(exactMetadataState, 0o600); + chmodSync(`${exactMetadataState}.lock`, 0o600); + chmodSync(exactMetadata.journal, 0o700); + chmodSync(exactMetadata.checkpoint, 0o600); + chmodSync(exactMetadata.epoch, 0o700); + for (const name of readdirSync(exactMetadata.epoch)) chmodSync(join(exactMetadata.epoch, name), 0o600); + if (typeof process.getuid === "function") { + await assert.rejects( + () => withConsumerStateLock(exactMetadataState, async () => {}, { currentUid: process.getuid() + 1 }), + /owned by the current uid/, + ); + } + + const heldWritableFdState = join(fixture, "held-writable-fd.json"); + const highBytes = Buffer.from('{"value":"HIGH"}\n'); + const lowBytes = Buffer.from('{"value":"LOW!"}\n'); + await withConsumerStateLock(heldWritableFdState, async (_path, transaction) => { + await transaction.commitState(highBytes); + }); + const heldWritableFd = openSync(heldWritableFdState, "r+"); + chmodSync(heldWritableFdState, 0o666); + await assert.rejects( + () => withConsumerStateLock(heldWritableFdState, async () => {}), + /exact 600 permissions/, + ); + writeSync(heldWritableFd, lowBytes, 0, lowBytes.length, 0); + fsyncSync(heldWritableFd); + closeSync(heldWritableFd); + chmodSync(heldWritableFdState, 0o600); + await withConsumerStateLock(heldWritableFdState, async (_path, transaction) => { + assert.equal(transaction.readStateBytes().equals(highBytes), true, "a held writable fd never rolls back journal authority"); + }); + assert.equal(readFileSync(heldWritableFdState).equals(highBytes), true); + + const orphanState = join(fixture, "orphan-metadata.json"); + await verifyStableHistoryWithState([manifestPath], { statePath: orphanState, initialize: true }); + const orphanEpoch = consumerJournal(orphanState).epoch; + const orphanToken = randomUUID(); + for (const [kind, value, message] of [ + ["heartbeat", { schemaVersion: 2, generation: 1, token: orphanToken, refreshedAtMs: 1 }, /orphan heartbeat/], + ["terminal", { schemaVersion: 2, generation: 1, token: orphanToken, outcome: "released" }, /orphan terminal/], + ["applied", { schemaVersion: 2, generation: 1, token: orphanToken, terminalSha256: "0".repeat(64) }, /orphan applied/], + ]) { + const path = join(orphanEpoch, `${kind}-0000000000000001-${orphanToken}.json`); + writeFileSync(path, `${JSON.stringify(value)}\n`, { mode: 0o600 }); + await assert.rejects(() => verifyStableHistoryWithState([manifestPath], { statePath: orphanState }), message); + rmSync(path); + } + writeFileSync(join(orphanEpoch, "unexpected-extra"), "bad\n", { mode: 0o600 }); + await assert.rejects( + () => verifyStableHistoryWithState([manifestPath], { statePath: orphanState }), + /malformed or unexpected entry/, + ); + rmSync(join(orphanEpoch, "unexpected-extra")); + const checkpoint = JSON.parse(readFileSync(consumerJournal(orphanState).checkpoint)); + const symlinkTemporary = join( + orphanEpoch, + `.pylon-consumer-tmp-v1-p1-e${checkpoint.epochId}-g0000000000000001-w${randomUUID()}` + + `-n${"a".repeat(12)}-kclaim-t${"0".repeat(64)}.tmp`, + ); + symlinkSync(manifestPath, symlinkTemporary); + await assert.rejects( + () => verifyStableHistoryWithState([manifestPath], { statePath: orphanState }), + /owned temporary.*regular non-symlink file/, + ); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } +}); + +test("consumer stable high-water pins and bounds every manifest before parsing", async () => { + const fixture = mkdtempSync(join(tmpdir(), "pylon-stable-state-")); + try { + const target = join(fixture, "target.json"); + const manifestPath = join(fixture, "first.json"); + const statePath = join(fixture, "stable.json"); + writeFileSync(target, canonicalJson(firstStable())); + symlinkSync(target, manifestPath); + await assert.rejects(() => verifyStableHistoryWithState([manifestPath], { statePath, initialize: true }), /regular non-symlink file/); + rmSync(manifestPath); + writeFileSync(manifestPath, JSON.stringify(firstStable())); + await assert.rejects(() => verifyStableHistoryWithState([manifestPath], { statePath, initialize: true }), /not canonical/); + + writeFileSync(manifestPath, Buffer.alloc(PYLON_PUBLICATION_MANIFEST_MAX_BYTES + 1)); + await assert.rejects( + () => verifyStableHistoryWithState([manifestPath], { statePath, initialize: true }), + /format byte limit/, + ); + writeFileSync(manifestPath, canonicalJson(firstStable())); + await assert.rejects( + () => verifyStableHistoryWithState([manifestPath], { + statePath, + initialize: true, + fileOptions: { hooks: { afterInitialStat: ({ path }) => writeFileSync(path, "x", { flag: "a" }) } }, + }), + /changed while it was read/, + ); + writeFileSync(manifestPath, canonicalJson(firstStable())); + await assert.rejects( + () => verifyStableHistoryWithState([manifestPath], { + statePath, + initialize: true, + fileOptions: { hooks: { afterInitialStat: ({ path }) => truncateSync(path, 1) } }, + }), + /changed while it was read/, + ); + + writeFileSync(manifestPath, canonicalJson(firstStable())); + const moved = join(fixture, "pinned-original.json"); + let swapped = false; + await assert.rejects( + () => verifyStableHistoryWithState([manifestPath], { + statePath: join(fixture, "pinned-state.json"), + initialize: true, + fileOptions: { + hooks: { + afterInitialStat: ({ path }) => { + if (swapped) return; + swapped = true; + renameSync(path, moved); + writeFileSync(path, "replacement must not be read"); + }, + }, + }, + }), + /changed while it was read/, + ); + assert.equal(swapped, true, "the descriptor re-stat detects a pathname swap without reading the replacement"); + + const maximum = join(fixture, "maximum.bin"); + writeFileSync(maximum, Buffer.alloc(PYLON_PUBLICATION_MANIFEST_MAX_BYTES, 0x61)); + assert.equal( + (await readBoundedRegularFile(maximum, { + maxBytes: PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + description: "Maximum valid bounded input", + })).length, + PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + ); + await assert.rejects( + () => verifyStableHistoryWithState( + Array(PYLON_STABLE_HISTORY_MAX_MANIFESTS + 1).fill(maximum), + { statePath, initialize: true }, + ), + /manifest work bound/, + ); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } +}); + +test("stable manifest nested schema rejects extras, malformed identities, unsafe assets, duplicates, and ordering changes", () => { + const stable = firstStable(); + assert.equal(validateStableManifest(stable), stable); + for (const mutate of [ + (value) => (value.build.extra = true), + (value) => (value.build.source.extra = true), + (value) => (value.build.source.repository = "https://github.com/fork/prime-agent"), + (value) => (value.build.source.commit = "abc"), + (value) => (value.build.source.tree = "abc"), + (value) => (value.build.recipeRevision = 2), + (value) => delete value.build.publicationPolicyRevision, + (value) => (value.build.publicationPolicyRevision = 2), + (value) => (value.build.releaseManifest.file = "other.json"), + (value) => (value.build.previewManifest.file = "other.json"), + (value) => (value.build.assets[0].file = "../escape.tgz"), + (value) => (value.build.assets[0].file = "pylon-prime-agent-ai-9.9.9.tgz"), + (value) => (value.build.assets[0].size = 0), + (value) => value.build.assets.push(structuredClone(value.build.assets[0])), + (value) => value.build.assets.reverse(), + (value) => (value.promotion.policyTree = "abc"), + (value) => delete value.promotion.publicationPolicyRevision, + (value) => (value.promotion.publicationPolicyRevision = 2), + ]) { + const changed = structuredClone(stable); + mutate(changed); + assert.throws(() => validateStableManifest(changed)); + } +}); + +test("stable history binds a revocation build tag to the exact prior stable sequence", () => { + const first = firstStable(); + const second = secondStable(first, { withdraw: true }); + second.revocations[0].buildTag = "pylon-build-gffffffffffff-r1"; + second.promotion.revocation.buildTag = "pylon-build-gffffffffffff-r1"; + assert.throws(() => validateStableHistory([first, second]), /exact previously published build/); +}); + +test("exact-SHA required checks reject wrong app, source, context, and result", () => { + const requiredChecks = [ + { context: "build-check-test", appId: GITHUB_ACTIONS_APP_ID }, + { context: "Check changelog fragment", appId: GITHUB_ACTIONS_APP_ID }, + ]; + const checkRuns = requiredChecks.map(({ context }) => ({ + name: context, + workflowPath: PYLON_REQUIRED_CHECKS.find((required) => required.context === context).workflowPath, + head_sha: source.commit, + app: { id: GITHUB_ACTIONS_APP_ID }, + status: "completed", + conclusion: "success", + })); + assert.equal(validateRequiredChecks({ sourceSha: source.commit, requiredChecks, checkRuns }), true); + for (const changedPolicy of [ + requiredChecks.slice(0, 1), + [...requiredChecks, { context: "extra", appId: GITHUB_ACTIONS_APP_ID }], + requiredChecks.map((required, index) => index === 0 ? { ...required, appId: null } : required), + ]) assert.throws(() => validateRequiredChecks({ sourceSha: source.commit, requiredChecks: changedPolicy, checkRuns }), /policy differs/); + const wrongPath = structuredClone(checkRuns); + wrongPath[0].workflowPath = ".github/workflows/other.yml"; + assert.throws(() => validateRequiredChecks({ sourceSha: source.commit, requiredChecks, checkRuns: wrongPath }), /not green/); + for (const mutate of [ + (run) => (run.app.id = 1), + (run) => (run.head_sha = "f".repeat(40)), + (run) => (run.name = "other"), + (run) => (run.conclusion = "failure"), + ]) { + const changed = structuredClone(checkRuns); + mutate(changed[0]); + assert.throws(() => validateRequiredChecks({ sourceSha: source.commit, requiredChecks, checkRuns: changed }), /not green/); + } +}); + +test("merged changelog proof never relabels a PR-head check as merge-SHA evidence", () => { + const mergeSha = "e".repeat(40); + const headSha = "d".repeat(40); + const proof = { + repository: PYLON_PUBLICATION_REPOSITORY, + ref: PYLON_PUBLICATION_REF, + eventName: "push", + mergeSha, + pullRequests: [{ + number: 29, + merged_at: "2026-01-01T00:00:00Z", + merge_commit_sha: mergeSha, + base: { ref: "pylon", repo: { full_name: PYLON_PUBLICATION_REPOSITORY } }, + head: { sha: headSha, repo: { full_name: PYLON_PUBLICATION_REPOSITORY } }, + }], + headChecks: [{ + id: 7, + name: "Check changelog fragment", + head_sha: headSha, + app: { id: GITHUB_ACTIONS_APP_ID }, + status: "completed", + conclusion: "success", + }], + workflowRuns: [{ + checkRunId: 7, + event: "pull_request", + head_sha: headSha, + path: ".github/workflows/changelog-fragment.yml", + repository: PYLON_PUBLICATION_REPOSITORY, + headRepository: PYLON_PUBLICATION_REPOSITORY, + pullRequests: [29], + }], + }; + assert.equal(validateMergedChangelogProof(proof).number, 29); + for (const mutate of [ + (value) => (value.repository = "fork/prime-agent"), + (value) => (value.ref = "refs/heads/main"), + (value) => (value.pullRequests[0].head.repo.full_name = "fork/prime-agent"), + (value) => (value.headChecks[0].app.id = 1), + (value) => (value.headChecks[0].conclusion = "failure"), + (value) => (value.workflowRuns[0].event = "push"), + (value) => (value.workflowRuns[0].path = ".github/workflows/other.yml"), + (value) => (value.workflowRuns[0].head_sha = mergeSha), + ]) { + const changed = structuredClone(proof); + mutate(changed); + assert.throws(() => validateMergedChangelogProof(changed)); + } +}); + +test("canonical invocation rejects main, tags, PR events, forks, and malformed source", () => { + const good = { + repository: PYLON_PUBLICATION_REPOSITORY, + ref: PYLON_PUBLICATION_REF, + eventName: "push", + sha: source.commit, + expectedEvent: "push", + }; + assert.doesNotThrow(() => assertCanonicalInvocation(good)); + for (const [key, value] of [["repository", "fork/prime-agent"], ["ref", "refs/heads/main"], ["eventName", "pull_request"], ["sha", "abc"]]) { + assert.throws(() => assertCanonicalInvocation({ ...good, [key]: value })); + } +}); + +test("attestation policy rejects wrong repository, workflow, ref, source, issuer, Rekor proof, and subject", () => { + const expected = { + repository: PYLON_PUBLICATION_REPOSITORY, + workflow: PYLON_PREVIEW_WORKFLOW, + ref: PYLON_PUBLICATION_REF, + sourceSha: source.commit, + subjectName: "artifact.tgz", + subjectSha256: "a".repeat(64), + }; + const evidence = { ...expected, issuer: "https://token.actions.githubusercontent.com", rekorIncluded: true }; + assert.equal(validateAttestationEvidence(evidence, expected), true); + for (const [key, value] of [ + ["repository", "fork/prime-agent"], ["workflow", ".github/workflows/other.yml"], ["ref", "refs/heads/main"], + ["sourceSha", "f".repeat(40)], ["issuer", "https://issuer.invalid"], ["rekorIncluded", false], + ["subjectName", "other.tgz"], ["subjectSha256", "f".repeat(64)], + ]) assert.throws(() => validateAttestationEvidence({ ...evidence, [key]: value }, expected)); +}); + +test("gh verification result requires full workflow/v1 invocation, exact subjects, and Rekor", () => { + const subject = { name: "artifact.tgz", sha256: "a".repeat(64) }; + const expected = { + repository: PYLON_PUBLICATION_REPOSITORY, workflow: PYLON_STABLE_WORKFLOW, + event: "workflow_dispatch", sourceSha: source.commit, + }; + const output = slsaAttestationOutput(subject, expected); + assert.deepEqual(verifyGhAttestationResult(output, subject, expected), [{ runId: "445566", runAttempt: "2" }]); + assert.throws(() => verifyGhAttestationResult(output, subject), /invocation/); + const missingPredicate = JSON.parse(output); + delete missingPredicate[0].verificationResult.statement.predicate; + assert.throws(() => verifyGhAttestationResult(JSON.stringify(missingPredicate), subject, expected), /workflow run/); + assert.throws(() => verifyGhAttestationResult(output, { ...subject, sha256: "f".repeat(64) }, expected), /subject set/); + const noRekor = output.replace("Tlog", "TimestampAuthority"); + assert.throws(() => verifyGhAttestationResult(noRekor, subject, expected), /Rekor/); + const extra = JSON.parse(output); + extra[0].verificationResult.statement.subject.push({ name: "extra", digest: { sha256: "b".repeat(64) } }); + assert.throws(() => verifyGhAttestationResult(JSON.stringify(extra), subject, expected), /subject set/); +}); + +test("attestation invocation binds the signed run id and attempt while run number stays API-derived", () => { + const expected = { + repository: PYLON_PUBLICATION_REPOSITORY, + workflow: PYLON_PREVIEW_WORKFLOW, + event: "push", + sourceSha: source.commit, + workflowRunId: invocation.workflowRunId, + }; + const subject = { name: "artifact.tgz", sha256: "a".repeat(64) }; + const statement = { + predicateType: "https://slsa.dev/provenance/v1", + subject: [{ name: subject.name, digest: { sha256: subject.sha256 } }], + predicate: { + buildDefinition: { + buildType: "https://actions.github.io/buildtypes/workflow/v1", + externalParameters: { workflow: { + repository: `https://github.com/${expected.repository}`, + path: expected.workflow, + ref: PYLON_PUBLICATION_REF, + } }, + internalParameters: { github: { + event_name: "push", repository_id: "1349002285", repository_owner_id: "11325514", runner_environment: "github-hosted", + } }, + resolvedDependencies: [{ + uri: `git+https://github.com/${expected.repository}@${PYLON_PUBLICATION_REF}`, + digest: { gitCommit: source.commit }, + }], + }, + runDetails: { + builder: { id: `https://github.com/${expected.repository}/${expected.workflow}@${PYLON_PUBLICATION_REF}` }, + metadata: { invocationId: `https://github.com/${expected.repository}/actions/runs/${expected.workflowRunId}/attempts/2` }, + }, + }, + }; + const output = JSON.stringify([{ verificationResult: { + statement, + verifiedTimestamps: [{ type: "Tlog", uri: "https://rekor.sigstore.dev" }], + } }]); + assert.deepEqual(verifyGhAttestationResult(output, subject, expected), [{ runId: invocation.workflowRunId, runAttempt: "2" }]); + const wrongRun = structuredClone(statement); + wrongRun.predicate.runDetails.metadata.invocationId = `https://github.com/${expected.repository}/actions/runs/999/attempts/2`; + assert.throws(() => verifyGhAttestationResult(JSON.stringify([{ verificationResult: { + statement: wrongRun, + verifiedTimestamps: [{ type: "Tlog", uri: "https://rekor.sigstore.dev" }], + } }]), subject, expected), /workflow run/); +}); + +test("preview sequence rejects a workflow API run-id or run-number mismatch", () => { + const { preview } = manifests(); + const run = { + id: Number(invocation.workflowRunId), run_number: invocation.sequence, event: "push", head_branch: "pylon", head_sha: source.commit, + path: PYLON_PREVIEW_WORKFLOW, repository: { id: 1_349_002_285, full_name: PYLON_PUBLICATION_REPOSITORY }, + head_repository: { id: 1_349_002_285, full_name: PYLON_PUBLICATION_REPOSITORY }, + check_suite_id: 7, status: "in_progress", conclusion: null, + }; + run.run_attempt = 2; + const attempt = { + run, + suite: { id: 7, app: { id: GITHUB_ACTIONS_APP_ID }, head_sha: source.commit }, + jobs: [{ name: "Approve and attest six preview subjects", run_attempt: 2, status: "completed", conclusion: "success" }], + }; + const signed = [{ runId: invocation.workflowRunId, runAttempt: "2" }]; + assert.equal(validatePreviewWorkflowRunEvidence({ attempts: [attempt] }, preview, signed).workflowRunId, invocation.workflowRunId); + assert.throws(() => validatePreviewWorkflowRunEvidence({ attempts: [{ ...attempt, run: { ...run, id: 9 } }] }, preview, signed), /sequence/); + assert.throws(() => validatePreviewWorkflowRunEvidence({ attempts: [{ ...attempt, run: { ...run, run_number: 18 } }] }, preview, signed), /sequence/); +}); + +test("immutable release replay is idempotent only for identical metadata and bytes", () => { + const expected = { + tag: "pylon-build-g0123456789ab-r1", + name: "Pylon Prime preview pylon-build-g0123456789ab-r1", + body: publicationReleaseBody({ channel: "preview", tag: "pylon-build-g0123456789ab-r1", source: source.commit, tree: source.tree, recipeRevision: 1 }), + prerelease: true, + sourceSha: source.commit, + assets: [{ name: "artifact.tgz", size: 1, sha256: "a".repeat(64) }], + }; + const actual = { + immutable: true, + draft: false, + tag_name: expected.tag, + name: expected.name, + body: expected.body, + prerelease: true, + target_commitish: source.commit, + assets: [{ name: "artifact.tgz", size: 1, digest: `sha256:${"a".repeat(64)}` }], + }; + assert.equal(assertImmutableReleaseIdentity(actual, expected), true); + for (const mutate of [ + (value) => (value.immutable = false), + (value) => (value.body = "changed"), + (value) => (value.assets[0].digest = `sha256:${"f".repeat(64)}`), + ]) { + const changed = structuredClone(actual); + mutate(changed); + assert.throws(() => assertImmutableReleaseIdentity(changed, expected)); + } +}); + +test("workflow artifacts reject wrong run, repository, workflow, event, ref, source, result, ambiguity, expiry, and digest", () => { + const expected = { runId: 7, workflowPath: ".github/workflows/pylon-preview-release.yml", event: "push", headSha: source.commit, artifactName: "pack" }; + const actual = { + runId: 7, repositoryId: 1_349_002_285, workflowPath: expected.workflowPath, event: "push", ref: PYLON_PUBLICATION_REF, + headSha: source.commit, conclusion: "success", checkSuiteHeadSha: source.commit, checkSuiteConclusion: "success", + artifacts: [{ name: "pack", expired: false, digest: `sha256:${"a".repeat(64)}` }], + }; + assert.equal(validateWorkflowArtifactProvenance(actual, expected).name, "pack"); + for (const mutate of [ + (value) => (value.runId = 8), (value) => (value.repositoryId = 1), (value) => (value.workflowPath = "other"), + (value) => (value.event = "pull_request"), (value) => (value.ref = "refs/heads/main"), + (value) => (value.headSha = "f".repeat(40)), (value) => (value.conclusion = "failure"), + (value) => value.artifacts.push(structuredClone(value.artifacts[0])), (value) => (value.artifacts[0].expired = true), + (value) => (value.artifacts[0].digest = "missing"), + ]) { + const changed = structuredClone(actual); + mutate(changed); + assert.throws(() => validateWorkflowArtifactProvenance(changed, expected)); + } +}); + +test("standalone preview verification rejects tamper, extras, symlinks, and noncanonical channel bytes", () => { + const fixture = mkdtempSync(join(tmpdir(), "pylon-publication-")); + try { + const release = fakeReleaseManifest(); + for (const asset of release.assets) { + const byte = asset.sha256[0]; + writeFileSync(join(fixture, asset.file), byte); + asset.sha256 = sha256Bytes(Buffer.from(byte)); + asset.sha512 = createHash("sha512").update(byte).digest("hex"); + } + release.attestationSubjects = release.assets.map((asset) => ({ + name: asset.file, + digest: { sha256: asset.sha256, sha512: asset.sha512 }, + })); + const releaseBytes = Buffer.from(`${JSON.stringify(release, null, 2)}\n`); + writeFileSync(join(fixture, PYLON_RELEASE_MANIFEST), releaseBytes); + const preview = createPreviewManifest(release, releaseBytes, invocation); + writeFileSync(join(fixture, PYLON_PREVIEW_MANIFEST), canonicalJson(preview)); + assert.equal(verifyPreviewPublication(fixture).subjects.length, 6); + writeFileSync(join(fixture, "extra"), "bad"); + assert.throws(() => verifyPreviewPublication(fixture), /Unexpected/); + rmSync(join(fixture, "extra")); + writeFileSync(join(fixture, release.assets[0].file), "tamper"); + assert.throws(() => verifyPreviewPublication(fixture), /digest mismatch/); + writeFileSync(join(fixture, release.assets[0].file), release.assets[0].sha256[0]); + const target = join(fixture, release.assets[0].file); + rmSync(target); + symlinkSync(join(fixture, release.assets[1].file), target); + assert.throws(() => verifyPreviewPublication(fixture), /regular file/); + rmSync(target); + writeFileSync(target, release.assets[0].sha256[0]); + writeFileSync(join(fixture, PYLON_PREVIEW_MANIFEST), JSON.stringify(preview)); + assert.throws(() => verifyPreviewPublication(fixture), /not canonical/); + writeFileSync(join(fixture, PYLON_PREVIEW_MANIFEST), Buffer.alloc(PYLON_PUBLICATION_MANIFEST_MAX_BYTES + 1)); + assert.throws(() => verifyPreviewPublication(fixture), /format byte limit/); + rmSync(join(fixture, PYLON_PREVIEW_MANIFEST)); + symlinkSync(join(fixture, PYLON_RELEASE_MANIFEST), join(fixture, PYLON_PREVIEW_MANIFEST)); + assert.throws(() => verifyPreviewPublication(fixture), /regular non-symlink file/); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } +}); + +test("recipe and publication policy registries close independent immutable identities", () => { + const registryText = readFileSync(join(root, "scripts/pylon-prime-supported-release-recipes-v1.json"), "utf8"); + const registry = parseSupportedReleaseRecipeRegistry(registryText); + const policy = registry.publicationPolicies[0]; + const preview = readFileSync(join(root, policy.previewWorkflowPath), "utf8"); + const stable = readFileSync(join(root, policy.stableWorkflowPath), "utf8"); + assert.deepEqual(validateApprovedWorkflowBytes( + policy.previewWorkflowPath, + preview, + "preview", + policy.publicationPolicyRevision, + ), { workflow: policy.previewWorkflowPath, environment: "pylon-preview" }); + assert.deepEqual(validateApprovedWorkflowBytes( + policy.stableWorkflowPath, + stable, + "stable", + policy.publicationPolicyRevision, + ), { workflow: policy.stableWorkflowPath, environment: "pylon-stable" }); + for (const changed of [ + `${preview}\n rogue:\n permissions: write-all\n runs-on: ubuntu-latest\n steps:\n - run: echo arbitrary\n`, + `${stable}\n rogue-oidc:\n permissions:\n id-token: write\n attestations: write\n runs-on: ubuntu-latest\n steps:\n - run: echo sign\n`, + preview.replace("jobs:\n", "jobs:\n publish:\n permissions: write-all\n"), + preview.replace("permissions: {}", "permissions: {}\npermissions: write-all"), + ]) assert.throws(() => validateApprovedWorkflowBytes( + policy.previewWorkflowPath, + changed, + "preview", + policy.publicationPolicyRevision, + ), /bytes differ/); + assert.throws(() => validateApprovedWorkflowBytes(policy.previewWorkflowPath, preview, "preview", 999), /Unsupported/); + + const stableR2 = stable + .replace("--publication-policy-revision 1", "--publication-policy-revision 2") + .replaceAll("promotion?.publicationPolicyRevision !== 1", "promotion?.publicationPolicyRevision !== 2"); + assert.notEqual(stableR2, stable); + assert.match(stableR2, /--publication-policy-revision 2/); + assert.match(stableR2, /promotion\?\.publicationPolicyRevision !== 2/); + assert.match(stableR2, /!\[1\]\.includes\(manifest\.build\.publicationPolicyRevision\)/); + const policyR2 = { + ...policy, + publicationPolicyRevision: 2, + stableWorkflowSha256: sha256Bytes(Buffer.from(stableR2)), + }; + const policies = [policy, policyR2]; + assert.deepEqual(validateApprovedWorkflowBytes( + policyR2.stableWorkflowPath, + stableR2, + "stable", + 2, + policies, + ), { workflow: policyR2.stableWorkflowPath, environment: "pylon-stable" }); + assert.throws(() => validateApprovedWorkflowBytes(policy.stableWorkflowPath, stableR2, "stable", 1, policies), /bytes differ/); + const promotedByR2 = firstStable(); + promotedByR2.promotion.publicationPolicyRevision = 2; + assert.equal(validateStableManifest(promotedByR2, registry.recipes, policies), promotedByR2); + assert.equal(promotedByR2.build.recipeRevision, 1); + assert.equal(promotedByR2.build.publicationPolicyRevision, 1); + assert.equal(promotedByR2.promotion.publicationPolicyRevision, 2); + const stableVerifier = readFileSync(join(root, "scripts/verify-pylon-stable-attestation.mjs"), "utf8"); + assert.match(stableVerifier, /manifest\.promotion\.publicationPolicyRevision/); + assert.doesNotMatch(stableVerifier, /"stable",\s*manifest\.build\.recipeRevision/); + const previewVerifier = readFileSync(join(root, "scripts/verify-pylon-publication-attestations.mjs"), "utf8"); + assert.match(previewVerifier, /verified\.previewManifest\.publicationPolicyRevision/); + for (const mutate of [ + (value) => delete value.promotion.publicationPolicyRevision, + (value) => (value.promotion.publicationPolicyRevision = 3), + (value) => (value.promotion.publicationPolicyRevision = 0), + (value) => delete value.build.publicationPolicyRevision, + ]) { + const changed = structuredClone(promotedByR2); + mutate(changed); + assert.throws(() => validateStableManifest(changed, registry.recipes, policies), /policy revision/); + } + + for (const mutate of [ + (value) => (value.extra = true), + (value) => (value.recipes[0].extra = true), + (value) => value.recipes.push(structuredClone(value.recipes[0])), + (value) => (value.recipes[0].recipeRevision = 0), + (value) => (value.publicationPolicies[0].extra = true), + (value) => value.publicationPolicies.push(structuredClone(value.publicationPolicies[0])), + (value) => (value.publicationPolicies[0].publicationPolicyRevision = 0), + ]) { + const changed = structuredClone(registry); + mutate(changed); + assert.throws(() => parseSupportedReleaseRecipeRegistry(JSON.stringify(changed)), /malformed/); + } + assert.throws( + () => parseSupportedReleaseRecipeRegistry('{"schemaVersion":1,"schemaVersion":1,"recipes":[],"publicationPolicies":[]}'), + /duplicate keys/, + ); +}); + +test("consumer state locking, recovery, transaction fencing, durability, and path boundary are deterministic", async () => { + const deferred = () => { + let resolvePromise; + let rejectPromise; + const promise = new Promise((resolve, reject) => { + resolvePromise = resolve; + rejectPromise = reject; + }); + return { promise, resolve: resolvePromise, reject: rejectPromise }; + }; + const manualRuntime = (clock, hooks = {}) => { + const beats = []; + return { + stale: 20, + update: 10, + stateMaxBytes: 1024, + now: () => clock.value, + hooks, + processKill: (pid, signal) => { + if (pid === 999_999) throw Object.assign(new Error("dead fixture owner"), { code: "ESRCH" }); + return process.kill(pid, signal); + }, + startHeartbeat: ({ beat }) => { + beats.push(beat); + return async () => {}; + }, + beats, + }; + }; + const bytes = (value) => Buffer.from(`${JSON.stringify({ value })}\n`); + const metadata = (value) => Buffer.from(`${JSON.stringify(value)}\n`); + const writePrivate = (path, value) => { + writeFileSync(path, value, { mode: 0o600 }); + chmodSync(path, 0o600); + }; + const runConsumerChild = (statePath, candidate) => new Promise((resolveChild, rejectChild) => { + const source = ` + import { withConsumerStateLock } from ${JSON.stringify(pathToFileURL(resolve("scripts/lib/pylon-consumer-lock.mjs")).href)}; + const statePath = process.argv[1]; + const candidate = process.argv[2]; + try { + await withConsumerStateLock(statePath, async (_path, transaction) => { + await transaction.commitState(Buffer.from(JSON.stringify({ value: candidate }) + "\\n")); + }); + process.exitCode = 0; + } catch (error) { + if (/actively locked/.test(error.message)) process.exitCode = 2; + else { console.error(error.stack); process.exitCode = 1; } + } + `; + const child = spawn(process.execPath, ["--input-type=module", "--eval", source, statePath, candidate], { + cwd: resolve("."), + stdio: ["ignore", "pipe", "pipe"], + }); + let stderr = ""; + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk) => { stderr += chunk; }); + child.on("error", rejectChild); + child.on("close", (code) => { + if ([0, 2].includes(code)) resolveChild(code); + else rejectChild(new Error(`consumer child failed with ${code}: ${stderr}`)); + }); + }); + const runRotationChild = (statePath) => new Promise((resolveChild, rejectChild) => { + const source = ` + import { rotateConsumerStateJournal } from ${JSON.stringify(pathToFileURL(resolve("scripts/lib/pylon-consumer-lock.mjs")).href)}; + try { + const result = await rotateConsumerStateJournal(process.argv[1]); + process.stdout.write(JSON.stringify(result)); + } catch (error) { + console.error(error.stack); + process.exitCode = 1; + } + `; + const child = spawn(process.execPath, ["--input-type=module", "--eval", source, statePath], { + cwd: resolve("."), + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { stdout += chunk; }); + child.stderr.on("data", (chunk) => { stderr += chunk; }); + child.on("error", rejectChild); + child.on("close", (code) => { + if (code === 0) resolveChild(JSON.parse(stdout)); + else rejectChild(new Error(`rotation child failed with ${code}: ${stderr}`)); + }); + }); + const v1Fixture = ( + statePath, + { projection = "one", secondTransition = false, terminal = true, applied = false, ownerPid = 999_999 } = {}, + ) => { + const lockDirectory = `${statePath}.lock`; + const transactionDirectory = `${statePath}.transactions`; + mkdirSync(lockDirectory, { mode: 0o700 }); + mkdirSync(transactionDirectory, { mode: 0o700 }); + const token = "12345678-1234-4123-8123-123456789abc"; + const firstBytes = bytes("one"); + const secondBytes = bytes("two"); + const firstDigest = sha256Bytes(firstBytes); + const secondDigest = sha256Bytes(secondBytes); + const firstTransaction = { + schemaVersion: 1, + baseDigest: "0".repeat(64), + candidateDigest: firstDigest, + candidateBase64: firstBytes.toString("base64"), + }; + const secondTransaction = { + schemaVersion: 1, + baseDigest: firstDigest, + candidateDigest: secondDigest, + candidateBase64: secondBytes.toString("base64"), + }; + const claim = { schemaVersion: 1, generation: 1, token, ownerPid, createdAtMs: 1 }; + const heartbeat = { schemaVersion: 1, generation: 1, token, refreshedAtMs: 1 }; + const commit = { schemaVersion: 1, generation: 1, token, outcome: "commit", transactions: [firstTransaction, secondTransaction] }; + writePrivate(join(lockDirectory, "claim-0000000000000001.json"), metadata(claim)); + writePrivate(join(lockDirectory, `heartbeat-0000000000000001-${token}.json`), metadata(heartbeat)); + if (terminal) writePrivate(join(lockDirectory, `terminal-0000000000000001-${token}.json`), metadata(commit)); + if (applied) writePrivate(join(lockDirectory, `applied-0000000000000001-${token}.json`), metadata({ + schemaVersion: 1, + generation: 1, + token, + terminalSha256: sha256Bytes(metadata(commit)), + })); + writePrivate(join(transactionDirectory, `${"0".repeat(64)}.json`), metadata(firstTransaction)); + if (secondTransition) writePrivate(join(transactionDirectory, `${firstDigest}.json`), metadata(secondTransaction)); + if (projection !== null) writePrivate(statePath, projection === "one" ? firstBytes : secondBytes); + return { lockDirectory, transactionDirectory, firstBytes, secondBytes, firstDigest, secondDigest, token }; + }; + + const virtualRoot = resolve("/"); + const first = join(virtualRoot, "pylon-durable-state-test"); + const second = join(first, "nested"); + const existing = new Set([virtualRoot]); + const durabilityEvents = []; + const operations = { + lstatEntry: async (path) => { + if (!existing.has(path)) throw Object.assign(new Error("missing"), { code: "ENOENT" }); + return { isDirectory: () => true }; + }, + makeDirectory: async (path, options) => { + assert.deepEqual(options, { mode: 0o700 }); + durabilityEvents.push(`mkdir:${path}`); + existing.add(path); + }, + syncDirectory: async (path) => durabilityEvents.push(`sync:${path}`), + }; + await ensureDurableConsumerStateDirectory(second, operations); + assert.deepEqual(durabilityEvents, [ + `mkdir:${first}`, `sync:${virtualRoot}`, `mkdir:${second}`, `sync:${first}`, + ]); + durabilityEvents.length = 0; + await ensureDurableConsumerStateDirectory(second, operations); + assert.deepEqual(durabilityEvents, [`sync:${virtualRoot}`, `sync:${first}`]); + + const creatorReachedSync = deferred(); + const letCreatorSync = deferred(); + const concurrentExisting = new Set([virtualRoot]); + const creatorOperations = { + lstatEntry: async (path) => { + if (!concurrentExisting.has(path)) throw Object.assign(new Error("missing"), { code: "ENOENT" }); + return { isDirectory: () => true }; + }, + makeDirectory: async (path) => concurrentExisting.add(path), + syncDirectory: async (path) => { + if (path === virtualRoot) { + creatorReachedSync.resolve(); + await letCreatorSync.promise; + } + }, + }; + const creator = ensureDurableConsumerStateDirectory(first, creatorOperations); + await creatorReachedSync.promise; + let observerSynced = false; + await ensureDurableConsumerStateDirectory(first, { + ...creatorOperations, + syncDirectory: async (path) => { + assert.equal(path, virtualRoot); + observerSynced = true; + }, + }); + assert.equal(observerSynced, true, "an observer flushes the concurrent creator's ancestor entry before returning"); + letCreatorSync.resolve(); + await creator; + + const fixture = realpathSync(mkdtempSync(join(tmpdir(), "pylon-consumer-lock-"))); + try { + const properLockfile = nodeRequire("proper-lockfile"); + const legacyLivePath = join(fixture, "legacy-live.json"); + const releaseLegacy = await properLockfile.lock(legacyLivePath, { realpath: false, retries: 0 }); + await assert.rejects( + () => withConsumerStateLock(legacyLivePath, async () => {}, manualRuntime({ value: 1 })), + /Legacy consumer lock directory exists.*confirm that no owner remains.*manually/i, + ); + assert.equal(statSync(`${legacyLivePath}.lock`).isDirectory(), true); + await releaseLegacy(); + await withConsumerStateLock(legacyLivePath, async () => {}, manualRuntime({ value: 1 })); + assert.equal(statSync(`${legacyLivePath}.lock`).isFile(), true, "the handoff guard is a permanent regular file"); + await assert.rejects( + () => properLockfile.lock(legacyLivePath, { realpath: false, retries: 0 }), + /lock|directory|ENOTDIR|EEXIST/i, + ); + + const legacyRacePath = join(fixture, "legacy-race.json"); + const guardReady = deferred(); + const letGuardLink = deferred(); + const currentRacer = withConsumerStateLock(legacyRacePath, async () => {}, manualRuntime({ value: 1 }, { + afterFileSync: async ({ kind }) => { + if (kind !== "legacy-guard") return; + guardReady.resolve(); + await letGuardLink.promise; + }, + })); + await guardReady.promise; + const releaseRaceLegacy = await properLockfile.lock(legacyRacePath, { realpath: false, retries: 0 }); + letGuardLink.resolve(); + await assert.rejects(currentRacer, /Legacy consumer lock directory exists/); + await releaseRaceLegacy(); + await withConsumerStateLock(legacyRacePath, async () => {}, manualRuntime({ value: 100 })); + await assert.rejects( + () => properLockfile.lock(legacyRacePath, { realpath: false, retries: 0 }), + /lock|directory|ENOTDIR|EEXIST/i, + ); + + const ambiguousLegacyPath = join(fixture, "legacy-ambiguous.json"); + mkdirSync(`${ambiguousLegacyPath}.lock`, { mode: 0o700 }); + await assert.rejects( + () => withConsumerStateLock(ambiguousLegacyPath, async () => {}, manualRuntime({ value: 100 })), + /remove that directory manually/, + ); + assert.equal(statSync(`${ambiguousLegacyPath}.lock`).isDirectory(), true, "ambiguous legacy leases are never stolen"); + + const v1MigrationPath = join(fixture, "v1-migration.json"); + const migratedV1 = v1Fixture(v1MigrationPath); + await assert.rejects( + () => withConsumerStateLock(v1MigrationPath, async () => {}, manualRuntime({ value: 1 })), + /explicit quiescent consumer journal migration command/, + ); + assert.equal(existsSync(`${v1MigrationPath}.journal`), false, "v1 authority is detected before v2 is seeded"); + const migrationResult = await migrateConsumerStateJournal(v1MigrationPath, manualRuntime({ value: 2 })); + assert.equal(migrationResult.tipSha256, migratedV1.secondDigest); + assert.notEqual(migrationResult.sourceAuthoritySha256, "0".repeat(64)); + assert.equal(readFileSync(v1MigrationPath).equals(migratedV1.secondBytes), true, "the immutable v1 decision repairs a one-behind projection"); + assert.equal(statSync(`${v1MigrationPath}.lock`).isDirectory(), true); + assert.equal(existsSync(join(`${v1MigrationPath}.lock`, ".pylon-consumer-v1-retired.json")), true); + assert.equal(existsSync(`${v1MigrationPath}.lock.v1-retired`), false); + await assert.rejects( + () => properLockfile.lock(v1MigrationPath, { realpath: false, retries: 0, stale: 1 }), + /lock|directory|ENOTEMPTY|EEXIST/i, + ); + assert.equal(existsSync(join(migratedV1.transactionDirectory, `${migratedV1.firstDigest}.json`)), true); + assert.equal( + readdirSync(`${v1MigrationPath}.lock`).some((name) => name.startsWith("applied-")), + true, + "a definitively dead incomplete v1 commit is completed only behind the durable guard", + ); + await withConsumerStateLock(v1MigrationPath, async (_path, transaction) => { + assert.equal(transaction.readStateBytes().equals(migratedV1.secondBytes), true); + }, manualRuntime({ value: 3 })); + + for (const [hookName, wantedKind] of [ + ["afterMigrationAuthorityRead", null], + ["afterFileSync", "legacy-retirement"], + ["afterMetadataLink", "legacy-retirement"], + ["afterMetadataDirectorySync", "legacy-retirement"], + ["afterMigrationRetirementMarker", null], + ["afterMigrationGuard", null], + ["afterFileSync", "checkpoint"], + ["afterMetadataLink", "checkpoint"], + ["afterMetadataDirectorySync", "checkpoint"], + ["afterMigrationComplete", null], + ]) { + const migrationCrashPath = join(fixture, `v1-migration-crash-${hookName}-${wantedKind ?? "migration"}.json`); + const expected = v1Fixture(migrationCrashPath); + let armed = true; + await assert.rejects( + () => migrateConsumerStateJournal(migrationCrashPath, manualRuntime({ value: 10 }, { + [hookName]: async (event = {}) => { + if (!armed || (wantedKind !== null && event.kind !== wantedKind)) return; + armed = false; + throw new Error(`simulated v1 migration crash at ${hookName}`); + }, + })), + /simulated v1 migration crash/, + ); + const recovered = await migrateConsumerStateJournal(migrationCrashPath, manualRuntime({ value: 20 })); + assert.equal(recovered.tipSha256, expected.secondDigest); + assert.equal(readFileSync(migrationCrashPath).equals(expected.secondBytes), true); + } + + const concurrentMigrationPath = join(fixture, "v1-concurrent-migration.json"); + const concurrentExpected = v1Fixture(concurrentMigrationPath); + const bothRead = deferred(); + const releaseMigration = deferred(); + let migrationReaders = 0; + const concurrentMigrationOptions = manualRuntime({ value: 1 }, { + afterMigrationAuthorityRead: async () => { + migrationReaders += 1; + if (migrationReaders === 2) bothRead.resolve(); + await releaseMigration.promise; + }, + }); + const firstMigrator = migrateConsumerStateJournal(concurrentMigrationPath, concurrentMigrationOptions); + const secondMigrator = migrateConsumerStateJournal(concurrentMigrationPath, concurrentMigrationOptions); + await bothRead.promise; + releaseMigration.resolve(); + const concurrentMigrations = await Promise.all([firstMigrator, secondMigrator]); + assert.deepEqual(concurrentMigrations.map((result) => result.tipSha256), [concurrentExpected.secondDigest, concurrentExpected.secondDigest]); + + const markerRacePath = join(fixture, "v1-marker-race.json"); + const markerRaceExpected = v1Fixture(markerRacePath); + const bothMarkersSynced = deferred(); + const releaseMarkers = deferred(); + let markerWriters = 0; + const markerRaceOptions = manualRuntime({ value: 1 }, { + afterFileSync: async ({ kind }) => { + if (kind !== "legacy-retirement") return; + markerWriters += 1; + if (markerWriters === 2) bothMarkersSynced.resolve(); + await releaseMarkers.promise; + }, + }); + const markerMigrators = [ + migrateConsumerStateJournal(markerRacePath, markerRaceOptions), + migrateConsumerStateJournal(markerRacePath, markerRaceOptions), + ]; + await bothMarkersSynced.promise; + releaseMarkers.resolve(); + const markerRaceResults = await Promise.all(markerMigrators); + assert.deepEqual(markerRaceResults, [markerRaceResults[0], markerRaceResults[0]]); + assert.equal(markerRaceResults[0].tipSha256, markerRaceExpected.secondDigest); + + const priorLayoutPath = join(fixture, "v1-prior-retired-layout.json"); + const priorLayout = v1Fixture(priorLayoutPath, { secondTransition: true, applied: true }); + renameSync(priorLayout.lockDirectory, `${priorLayoutPath}.lock.v1-retired`); + const priorLayoutResult = await migrateConsumerStateJournal(priorLayoutPath, manualRuntime({ value: 1 })); + assert.equal(priorLayoutResult.tipSha256, priorLayout.secondDigest); + assert.equal(statSync(`${priorLayoutPath}.lock`).isFile(), true); + assert.equal(statSync(`${priorLayoutPath}.lock.v1-retired`).isDirectory(), true); + assert.equal(existsSync(join(`${priorLayoutPath}.lock.v1-retired`, ".pylon-consumer-v1-retired.json")), false); + + const retiredOnlyPath = join(fixture, "v1-retired-only.json"); + mkdirSync(`${retiredOnlyPath}.lock.v1-retired`, { mode: 0o700 }); + await assert.rejects( + () => withConsumerStateLock(retiredOnlyPath, async () => {}, manualRuntime({ value: 1 })), + /transaction namespace is missing/, + ); + assert.equal(existsSync(`${retiredOnlyPath}.journal`), false, "a retired-only legacy signal is detected before v2 initialization"); + + const deletedTransactionsPath = join(fixture, "v1-deleted-transactions.json"); + v1Fixture(deletedTransactionsPath, { secondTransition: true, applied: true }); + await migrateConsumerStateJournal(deletedTransactionsPath, manualRuntime({ value: 1 })); + rmSync(`${deletedTransactionsPath}.transactions`, { recursive: true }); + await assert.rejects( + () => withConsumerStateLock(deletedTransactionsPath, async () => {}, manualRuntime({ value: 2 })), + /transaction namespace is missing/, + ); + await assert.rejects( + () => rotateConsumerStateJournal(deletedTransactionsPath, manualRuntime({ value: 2 })), + /transaction namespace is missing/, + ); + + const malformedMarkerOnlyPath = join(fixture, "v1-malformed-marker-only.json"); + mkdirSync(`${malformedMarkerOnlyPath}.lock`, { mode: 0o700 }); + writePrivate(join(`${malformedMarkerOnlyPath}.lock`, ".pylon-consumer-v1-retired.json"), "{}\n"); + await assert.rejects( + () => withConsumerStateLock(malformedMarkerOnlyPath, async () => {}, manualRuntime({ value: 1 })), + /retirement marker.*malformed/i, + ); + assert.equal(existsSync(`${malformedMarkerOnlyPath}.journal`), false); + + const corruptMigrationPath = join(fixture, "v1-corrupt-migration.json"); + const corruptV1 = v1Fixture(corruptMigrationPath); + writePrivate(join(corruptV1.transactionDirectory, "extra.json"), "{}\n"); + await assert.rejects( + () => migrateConsumerStateJournal(corruptMigrationPath, manualRuntime({ value: 1 })), + /malformed or extra entry/, + ); + const activeMigrationPath = join(fixture, "v1-active-migration.json"); + const activeV1 = v1Fixture(activeMigrationPath, { terminal: false, ownerPid: process.pid }); + rmSync(join(activeV1.transactionDirectory, `${"0".repeat(64)}.json`)); + await assert.rejects( + () => migrateConsumerStateJournal(activeMigrationPath, manualRuntime({ value: 100 })), + /live or uncertain incomplete v1 commit owner/, + ); + const falseAppliedPath = join(fixture, "v1-false-applied.json"); + v1Fixture(falseAppliedPath, { applied: true }); + await assert.rejects( + () => migrateConsumerStateJournal(falseAppliedPath, manualRuntime({ value: 1 })), + /applied marker is missing its completed transition/, + ); + + const liveCommitPath = join(fixture, "v1-live-incomplete-commit.json"); + const liveCommit = v1Fixture(liveCommitPath, { ownerPid: process.pid }); + await assert.rejects( + () => migrateConsumerStateJournal(liveCommitPath, manualRuntime({ value: 1 }, { + afterMigrationAuthorityRead: async ({ legacy }) => { + const [missing] = legacy.recoveries[0].missingTransactions; + writePrivate(join(liveCommit.transactionDirectory, `${missing.baseDigest}.json`), metadata(missing)); + }, + })), + /live or uncertain incomplete v1 commit owner/, + ); + assert.equal(statSync(liveCommit.lockDirectory).isDirectory(), true); + assert.equal(existsSync(`${liveCommitPath}.lock.v1-retired`), false); + assert.equal(existsSync(join(liveCommit.transactionDirectory, `${liveCommit.firstDigest}.json`)), true); + + for (const [label, processKill] of [ + ["pid-reuse", () => {}], + ["eperm", () => { throw Object.assign(new Error("uncertain"), { code: "EPERM" }); }], + ]) { + const uncertainPath = join(fixture, `v1-${label}.json`); + v1Fixture(uncertainPath); + await assert.rejects( + () => migrateConsumerStateJournal(uncertainPath, { ...manualRuntime({ value: 1 }), processKill }), + /live or uncertain incomplete v1 commit owner/, + ); + assert.equal(statSync(`${uncertainPath}.lock`).isDirectory(), true); + assert.equal(existsSync(join(`${uncertainPath}.lock`, ".pylon-consumer-v1-retired.json")), false); + assert.equal(existsSync(`${uncertainPath}.lock.v1-retired`), false); + } + + const completeAppliedPath = join(fixture, "v1-complete-applied.json"); + const completeApplied = v1Fixture(completeAppliedPath, { + secondTransition: true, + applied: true, + ownerPid: process.pid, + }); + const completeAppliedResult = await migrateConsumerStateJournal(completeAppliedPath, { + ...manualRuntime({ value: 1 }), + processKill: () => { throw Object.assign(new Error("must not inspect a complete owner"), { code: "EPERM" }); }, + }); + assert.equal(completeAppliedResult.tipSha256, completeApplied.secondDigest); + + const markerReauthPath = join(fixture, "v1-marker-final-reauth.json"); + v1Fixture(markerReauthPath, { secondTransition: true, applied: true }); + let markerSourceMutated = false; + await assert.rejects( + () => migrateConsumerStateJournal(markerReauthPath, manualRuntime({ value: 1 }, { + afterFileSync: async ({ kind }) => { + if (kind !== "legacy-retirement" || markerSourceMutated) return; + markerSourceMutated = true; + const token = "abcdefab-cdef-4abc-8def-abcdefabcdef"; + const claim = { schemaVersion: 1, generation: 2, token, ownerPid: process.pid, createdAtMs: 2 }; + const heartbeat = { schemaVersion: 1, generation: 2, token, refreshedAtMs: 2 }; + writePrivate(join(`${markerReauthPath}.lock`, "claim-0000000000000002.json"), metadata(claim)); + writePrivate(join(`${markerReauthPath}.lock`, `heartbeat-0000000000000002-${token}.json`), metadata(heartbeat)); + }, + })), + /authority changed before retirement marker publication/, + ); + assert.equal(markerSourceMutated, true); + assert.equal(existsSync(join(`${markerReauthPath}.lock`, ".pylon-consumer-v1-retired.json")), false); + + const mutableBeforeSuccessPath = join(fixture, "v1-mutable-before-success.json"); + v1Fixture(mutableBeforeSuccessPath, { secondTransition: true, applied: true }); + let mutatedBeforeSuccess = false; + await assert.rejects( + () => migrateConsumerStateJournal(mutableBeforeSuccessPath, manualRuntime({ value: 1 }, { + beforeProjectionWrite: async () => { + if (mutatedBeforeSuccess) return; + mutatedBeforeSuccess = true; + const token = "87654321-4321-4321-8321-cba987654321"; + const claim = { schemaVersion: 1, generation: 2, token, ownerPid: 999_998, createdAtMs: 2 }; + const heartbeat = { schemaVersion: 1, generation: 2, token, refreshedAtMs: 2 }; + const terminal = { schemaVersion: 1, generation: 2, token, outcome: "released" }; + const retired = `${mutableBeforeSuccessPath}.lock`; + writePrivate(join(retired, "claim-0000000000000002.json"), metadata(claim)); + writePrivate(join(retired, `heartbeat-0000000000000002-${token}.json`), metadata(heartbeat)); + writePrivate(join(retired, `terminal-0000000000000002-${token}.json`), metadata(terminal)); + }, + })), + /does not authenticate the complete prior v1 authority and tip|retirement marker conflicts/, + ); + assert.equal(mutatedBeforeSuccess, true); + + const dualAuthorityPath = join(fixture, "v1-dual-authority.json"); + v1Fixture(dualAuthorityPath); + await assert.rejects( + () => migrateConsumerStateJournal(dualAuthorityPath, manualRuntime({ value: 1 }, { + afterMigrationRetirementMarker: async () => mkdirSync(`${dualAuthorityPath}.lock.v1-retired`, { mode: 0o700 }), + })), + /Live and retired v1 consumer lock authority both exist/, + ); + assert.equal(statSync(`${dualAuthorityPath}.lock`).isDirectory(), true); + assert.equal(statSync(`${dualAuthorityPath}.lock.v1-retired`).isDirectory(), true); + await assert.rejects( + () => migrateConsumerStateJournal(dualAuthorityPath, manualRuntime({ value: 2 })), + /Live and retired v1 consumer lock authority both exist/, + ); + assert.equal(statSync(`${dualAuthorityPath}.lock.v1-retired`).isDirectory(), true); + + for (const hookName of ["afterFileSync", "afterMetadataLink", "afterMetadataDirectorySync"]) { + const bootstrapCrashPath = join(fixture, `bootstrap-crash-${hookName}.json`); + const reached = deferred(); + const crash = deferred(); + const interrupted = withConsumerStateLock(bootstrapCrashPath, async () => {}, manualRuntime({ value: 1 }, { + [hookName]: async ({ kind }) => { + if (kind !== "checkpoint") return; + reached.resolve(); + await crash.promise; + }, + })); + await reached.promise; + await withConsumerStateLock(bootstrapCrashPath, async (_path, transaction) => { + await transaction.commitState(bytes("recovered-bootstrap")); + }, manualRuntime({ value: 100 })); + crash.reject(new Error(`simulated bootstrap crash at ${hookName}`)); + await assert.rejects(interrupted, /simulated bootstrap crash/); + assert.deepEqual(JSON.parse(readFileSync(bootstrapCrashPath, "utf8")), { value: "recovered-bootstrap" }); + } + + const activePath = join(fixture, "active.json"); + const activeClock = { value: 1 }; + const activeOptions = manualRuntime(activeClock); + const releaseOwner = deferred(); + const ownerEntered = deferred(); + const owner = withConsumerStateLock(activePath, async (_path, transaction) => { + ownerEntered.resolve(); + await releaseOwner.promise; + await transaction.commitState(bytes("owner")); + }, activeOptions); + await ownerEntered.promise; + activeClock.value = 16; + assert.equal(await activeOptions.beats[0](), true); + activeClock.value = 31; + assert.equal(await activeOptions.beats[0](), true); + activeClock.value = 45; + await assert.rejects( + () => withConsumerStateLock(activePath, async () => {}, manualRuntime(activeClock)), + /actively locked/, + ); + releaseOwner.resolve(); + await owner; + assert.deepEqual(JSON.parse(readFileSync(activePath, "utf8")), { value: "owner" }); + + const stagedPath = join(fixture, "staged-callback-exclusivity.json"); + const stagedReady = deferred(); + const returnStagedAction = deferred(); + const stagedOwner = withConsumerStateLock(stagedPath, async (_path, transaction) => { + await transaction.commitState(bytes("staged-owner")); + stagedReady.resolve(); + await returnStagedAction.promise; + }, manualRuntime({ value: 1 })); + await stagedReady.promise; + const stagedJournal = consumerJournal(stagedPath); + assert.equal( + readdirSync(stagedJournal.epoch) + .filter((name) => name.startsWith("terminal-")) + .map((name) => JSON.parse(readFileSync(join(stagedJournal.epoch, name)))) + .some((decision) => decision.outcome === "commit"), + false, + "commitState stages bytes but publishes no resolvable commit while the callback is active", + ); + await assert.rejects( + () => withConsumerStateLock(stagedPath, async () => {}, manualRuntime({ value: 1 })), + /actively locked/, + ); + await assert.rejects( + () => rotateConsumerStateJournal(stagedPath, manualRuntime({ value: 1 })), + /actively locked/, + ); + returnStagedAction.resolve(); + await stagedOwner; + assert.deepEqual(JSON.parse(readFileSync(stagedPath, "utf8")), { value: "staged-owner" }); + await withConsumerStateLock(stagedPath, async (_path, transaction) => { + assert.deepEqual(JSON.parse(transaction.readStateBytes()), { value: "staged-owner" }); + }, manualRuntime({ value: 2 })); + + const stagedThrowPath = join(fixture, "staged-then-throw.json"); + await assert.rejects( + () => withConsumerStateLock(stagedThrowPath, async (_path, transaction) => { + await transaction.commitState(bytes("must-not-commit")); + throw new Error("action failed after staging"); + }, manualRuntime({ value: 1 })), + /action failed after staging/, + ); + await withConsumerStateLock(stagedThrowPath, async (_path, transaction) => { + assert.equal(transaction.readStateBytes(), null); + }, manualRuntime({ value: 2 })); + assert.equal(transitionNames(stagedThrowPath).length, 0); + + const stagedCrashPath = join(fixture, "staged-then-crashed.json"); + const crashingSource = ` + import { withConsumerStateLock } from ${JSON.stringify(pathToFileURL(resolve("scripts/lib/pylon-consumer-lock.mjs")).href)}; + const hold = setInterval(() => {}, 1000); + await withConsumerStateLock(process.argv[1], async (_path, transaction) => { + await transaction.commitState(Buffer.from(JSON.stringify({ value: "crashed-stage" }) + "\\n")); + process.stdout.write("staged\\n"); + await new Promise(() => {}); + }, { stale: 20, update: 10, now: () => 1, startHeartbeat: () => async () => {} }); + clearInterval(hold); + `; + const crashingChild = spawn(process.execPath, ["--input-type=module", "--eval", crashingSource, stagedCrashPath], { + cwd: resolve("."), + stdio: ["ignore", "pipe", "pipe"], + }); + const crashingChildStaged = deferred(); + let crashingChildOutput = ""; + let crashingChildError = ""; + crashingChild.stdout.setEncoding("utf8"); + crashingChild.stderr.setEncoding("utf8"); + crashingChild.stdout.on("data", (chunk) => { + crashingChildOutput += chunk; + if (crashingChildOutput.includes("staged\n")) crashingChildStaged.resolve(); + }); + crashingChild.stderr.on("data", (chunk) => { crashingChildError += chunk; }); + const crashingChildClosed = new Promise((resolveChild, rejectChild) => { + crashingChild.on("error", rejectChild); + crashingChild.on("close", (code, signal) => resolveChild({ code, signal })); + }); + await Promise.race([ + crashingChildStaged.promise, + crashingChildClosed.then(({ code, signal }) => { + throw new Error(`staged crash child exited before staging (${code ?? signal}): ${crashingChildError}`); + }), + ]); + assert.equal(crashingChild.kill("SIGKILL"), true); + const crashed = await crashingChildClosed; + assert.equal(crashed.signal, "SIGKILL", crashingChildError); + await withConsumerStateLock(stagedCrashPath, async (_path, transaction) => { + assert.equal(transaction.readStateBytes(), null); + }, manualRuntime({ value: 100 })); + assert.equal(transitionNames(stagedCrashPath).length, 0); + + const stagedRetirePath = join(fixture, "staged-then-retired.json"); + const stagedRetireClock = { value: 1 }; + const stagedBeforeRetire = deferred(); + const returnRetiredAction = deferred(); + const retiredAfterStage = withConsumerStateLock(stagedRetirePath, async (_path, transaction) => { + await transaction.commitState(bytes("retired-stage")); + stagedBeforeRetire.resolve(); + await returnRetiredAction.promise; + }, manualRuntime(stagedRetireClock)); + await stagedBeforeRetire.promise; + stagedRetireClock.value = 100; + await withConsumerStateLock(stagedRetirePath, async (_path, transaction) => { + await transaction.commitState(bytes("retirement-winner")); + }, manualRuntime(stagedRetireClock)); + returnRetiredAction.resolve(); + await assert.rejects(retiredAfterStage, /lost ownership|retired/); + assert.deepEqual(JSON.parse(readFileSync(stagedRetirePath, "utf8")), { value: "retirement-winner" }); + assert.equal(transitionNames(stagedRetirePath).length, 1); + + for (const [label, candidates] of [ + ["identical", ["same", "same"]], + ["distinct", ["left", "right"]], + ]) { + const multiprocessPath = join(fixture, `multiprocess-${label}.json`); + const childResults = await Promise.all(candidates.map((candidate) => runConsumerChild(multiprocessPath, candidate))); + assert.equal(childResults.every((code) => [0, 2].includes(code)), true); + await withConsumerStateLock(multiprocessPath, async (_path, transaction) => { + await transaction.commitState(bytes(`joined-${label}`)); + }, manualRuntime({ value: 100 })); + const journal = consumerJournal(multiprocessPath); + const commitTerminals = readdirSync(journal.epoch) + .filter((name) => name.startsWith("terminal-")) + .map((name) => JSON.parse(readFileSync(join(journal.epoch, name)))) + .filter((terminal) => terminal.outcome === "commit"); + const commitBases = commitTerminals.flatMap((terminal) => terminal.transactions.map((transaction) => transaction.baseDigest)); + assert.equal(new Set(commitBases).size, commitBases.length, "multiprocess candidates never commit twice from one base"); + if (label === "identical") { + assert.equal(commitTerminals.filter((terminal) => terminal.transactions.some( + (transaction) => transaction.candidateDigest === sha256Bytes(bytes("same")), + )).length, 1); + } + } + + const racePath = join(fixture, "recoverers.json"); + const raceClock = { value: 1 }; + const oldRelease = deferred(); + const oldEntered = deferred(); + const oldOwner = withConsumerStateLock(racePath, async () => { + oldEntered.resolve(); + await oldRelease.promise; + }, manualRuntime(raceClock)); + const oldOwnerRejected = assert.rejects(oldOwner, /retired/); + await oldEntered.promise; + raceClock.value = 100; + const observedBoth = deferred(); + const releaseObserved = deferred(); + const retiredBoth = deferred(); + const releaseRetired = deferred(); + let observed = 0; + let retired = 0; + const recoveryHooks = { + afterObserveStale: async () => { + observed += 1; + if (observed === 2) observedBoth.resolve(); + await releaseObserved.promise; + }, + afterRetire: async () => { + retired += 1; + if (retired === 2) retiredBoth.resolve(); + await releaseRetired.promise; + }, + }; + let recoveryActions = 0; + const winnerRelease = deferred(); + const winnerEntered = deferred(); + const recover = () => withConsumerStateLock(racePath, async () => { + recoveryActions += 1; + winnerEntered.resolve(); + await winnerRelease.promise; + }, manualRuntime(raceClock, recoveryHooks)); + const recoveryRejected = deferred(); + const trackRecovery = (promise) => promise.catch((error) => { + recoveryRejected.resolve(error); + throw error; + }); + const firstRecoverer = trackRecovery(recover()); + const secondRecoverer = trackRecovery(recover()); + const recoveriesPromise = Promise.allSettled([firstRecoverer, secondRecoverer]); + await observedBoth.promise; + releaseObserved.resolve(); + await retiredBoth.promise; + releaseRetired.resolve(); + await winnerEntered.promise; + assert.match((await recoveryRejected.promise).message, /actively locked/); + winnerRelease.resolve(); + const recoveries = await recoveriesPromise; + assert.equal(recoveryActions, 1, "only one simultaneous stale recoverer enters the next generation"); + assert.deepEqual(recoveries.map((result) => result.status).sort(), ["fulfilled", "rejected"]); + assert.match(recoveries.find((result) => result.status === "rejected").reason.message, /actively locked/); + oldRelease.resolve(); + await oldOwnerRejected; + const raceClaims = readdirSync(consumerJournal(racePath).epoch).filter((name) => name.startsWith("claim-")); + assert.equal(raceClaims.length, 2); + + const fencedPath = join(fixture, "fenced.json"); + const fencedClock = { value: 1 }; + const oldDecisionReached = deferred(); + const letOldDecide = deferred(); + const retiredWriter = withConsumerStateLock(fencedPath, async (_path, transaction) => { + await transaction.commitState(bytes("retired")); + }, manualRuntime(fencedClock, { + beforeCommitDecision: async () => { + oldDecisionReached.resolve(); + await letOldDecide.promise; + }, + })); + await oldDecisionReached.promise; + fencedClock.value = 100; + await withConsumerStateLock(fencedPath, async (_path, transaction) => { + await transaction.commitState(bytes("winner")); + }, manualRuntime(fencedClock)); + letOldDecide.resolve(); + await assert.rejects(retiredWriter, /lost ownership/); + assert.deepEqual(JSON.parse(readFileSync(fencedPath, "utf8")), { value: "winner" }); + assert.equal( + transitionNames(fencedPath).length, + 1, + "a retired writer cannot publish a sibling transition from GENESIS", + ); + + const projectionPath = join(fixture, "projection.json"); + const projectionClock = { value: 1 }; + const staleProjectionReady = deferred(); + const letStaleProjectionRename = deferred(); + let blockedProjection = false; + const firstProjection = withConsumerStateLock(projectionPath, async (_path, transaction) => { + await transaction.commitState(bytes("one")); + }, manualRuntime(projectionClock, { + afterProjectionFileSync: async () => { + if (blockedProjection) return; + blockedProjection = true; + staleProjectionReady.resolve(); + await letStaleProjectionRename.promise; + }, + })); + await staleProjectionReady.promise; + await withConsumerStateLock(projectionPath, async (_path, transaction) => { + assert.deepEqual(JSON.parse(transaction.readStateBytes()), { value: "one" }); + await transaction.commitState(bytes("two")); + }, manualRuntime(projectionClock)); + await withConsumerStateLock(projectionPath, async (_path, transaction) => { + await transaction.commitState(bytes("three")); + }, manualRuntime(projectionClock)); + letStaleProjectionRename.resolve(); + await firstProjection; + assert.deepEqual(JSON.parse(readFileSync(projectionPath, "utf8")), { value: "three" }); + assert.equal(transitionNames(projectionPath).length, 3); + + const ctimeProjectionPath = join(fixture, "projection-ctime-retry.json"); + await withConsumerStateLock(ctimeProjectionPath, async (_path, transaction) => { + await transaction.commitState(bytes("ctime-base")); + }, manualRuntime({ value: 1 })); + let changedProjectionCtime = false; + const authenticatedProjectionOperations = []; + await withConsumerStateLock(ctimeProjectionPath, async (_path, transaction) => { + assert.deepEqual(JSON.parse(transaction.readStateBytes()), { value: "ctime-base" }); + }, manualRuntime({ value: 2 }, { + beforePathOperation: async ({ operation }) => authenticatedProjectionOperations.push(operation), + projectionRead: { + beforeFinalStat: async () => { + if (changedProjectionCtime) return; + changedProjectionCtime = true; + writePrivate(ctimeProjectionPath, bytes("ctime-base")); + }, + }, + })); + assert.equal(changedProjectionCtime, true); + assert.equal(authenticatedProjectionOperations.includes("projection-retry-authentication"), true); + assert.deepEqual(JSON.parse(readFileSync(ctimeProjectionPath, "utf8")), { value: "ctime-base" }); + + const projectionStressPath = join(fixture, "projection-helper-stress.json"); + const firstStressProjectionReady = deferred(); + const releaseFirstStressProjection = deferred(); + let heldFirstStressProjection = false; + const firstStressWriter = withConsumerStateLock(projectionStressPath, async (_path, transaction) => { + await transaction.commitState(Buffer.from(`${JSON.stringify({ count: 1 })}\n`)); + }, manualRuntime({ value: 1 }, { + afterProjectionFileSync: async () => { + if (heldFirstStressProjection) return; + heldFirstStressProjection = true; + firstStressProjectionReady.resolve(); + await releaseFirstStressProjection.promise; + }, + })); + await firstStressProjectionReady.promise; + for (let count = 2; count <= 8; count += 1) { + await withConsumerStateLock(projectionStressPath, async (_path, transaction) => { + assert.equal(JSON.parse(transaction.readStateBytes()).count, count - 1); + await transaction.commitState(Buffer.from(`${JSON.stringify({ count })}\n`)); + }, manualRuntime({ value: count })); + } + releaseFirstStressProjection.resolve(); + await firstStressWriter; + const projectionStressRotations = await Promise.all(Array.from( + { length: 3 }, + () => rotateConsumerStateJournal(projectionStressPath, manualRuntime({ value: 20 })), + )); + assert.deepEqual(projectionStressRotations, Array.from({ length: 3 }, () => projectionStressRotations[0])); + assert.equal(projectionStressRotations[0].epoch, 2); + assert.deepEqual(JSON.parse(readFileSync(projectionStressPath, "utf8")), { count: 8 }); + + const symlinkProjectionPath = join(fixture, "projection-symlink-corruption.json"); + await withConsumerStateLock(symlinkProjectionPath, async (_path, transaction) => { + await transaction.commitState(bytes("symlink-base")); + }, manualRuntime({ value: 1 })); + const symlinkProjectionTarget = join(fixture, "projection-symlink-target.json"); + writePrivate(symlinkProjectionTarget, bytes("symlink-base")); + rmSync(symlinkProjectionPath); + symlinkSync(symlinkProjectionTarget, symlinkProjectionPath); + await assert.rejects( + () => withConsumerStateLock(symlinkProjectionPath, async () => {}, manualRuntime({ value: 2 })), + /not one regular non-symlink file/, + ); + + writeFileSync(join(consumerJournal(projectionPath).epoch, `transition-${"f".repeat(64)}.json`), "{}\n"); + await assert.rejects( + () => withConsumerStateLock(projectionPath, async () => {}, manualRuntime(projectionClock)), + /unreachable transition/, + ); + + for (const crashPoint of [ + ["afterFileSync", "legacy-guard"], + ["afterMetadataLink", "legacy-guard"], + ["afterMetadataDirectorySync", "legacy-guard"], + ["afterFileSync", "claim"], + ["afterMetadataLink", "claim"], + ["afterMetadataDirectorySync", "claim"], + ["afterFileSync", "terminal-commit"], + ["afterMetadataLink", "terminal-commit"], + ["afterMetadataDirectorySync", "terminal-commit"], + ["afterCommitDecision", null], + ["afterFileSync", "transition"], + ["afterMetadataLink", "transition"], + ["afterMetadataDirectorySync", "transition"], + ["afterProjectionFileSync", null], + ["afterProjectionRename", null], + ["afterProjectionDirectorySync", null], + ["afterApplied", null], + ]) { + const [hookName, wantedKind] = crashPoint; + const crashPath = join(fixture, `crash-${hookName}-${wantedKind ?? "projection"}.json`); + const crashClock = { value: 1 }; + const reached = deferred(); + const crash = deferred(); + let armed = true; + const hooks = { + [hookName]: async (event = {}) => { + if (!armed || (wantedKind !== null && event.kind !== wantedKind)) return; + armed = false; + reached.resolve(); + await crash.promise; + }, + }; + const interrupted = withConsumerStateLock(crashPath, async (_path, transaction) => { + await transaction.commitState(bytes("interrupted")); + }, manualRuntime(crashClock, hooks)); + await reached.promise; + crashClock.value = 100; + await withConsumerStateLock(crashPath, async (_path, transaction) => { + await transaction.commitState(bytes("recovered")); + }, manualRuntime(crashClock)); + crash.reject(new Error(`simulated crash at ${hookName}`)); + await assert.rejects(interrupted, /simulated crash|retired/); + await withConsumerStateLock(crashPath, async () => {}, manualRuntime(crashClock)); + assert.deepEqual(JSON.parse(readFileSync(crashPath, "utf8")), { value: "recovered" }); + assert.equal( + readdirSync(consumerJournal(crashPath).epoch).some((name) => name.startsWith(".")), + false, + "owned crash temporaries converge after fencing", + ); + } + + const legacyRotationPath = join(fixture, "legacy-rotation.json"); + writeFileSync(legacyRotationPath, bytes("legacy-anchor"), { mode: 0o600 }); + assert.equal((await rotateConsumerStateJournal(legacyRotationPath, manualRuntime({ value: 1 }))).epoch, 2); + await withConsumerStateLock(legacyRotationPath, async (_path, transaction) => { + assert.deepEqual(JSON.parse(transaction.readStateBytes()), { value: "legacy-anchor" }); + }, manualRuntime({ value: 2 })); + + const rotationAuthorityPath = join(fixture, "rotation-authority.json"); + const rotationClock = { value: 1 }; + const pausedRotationOwner = deferred(); + const rotationOwnerEntered = deferred(); + const oldRotationOwner = withConsumerStateLock(rotationAuthorityPath, async (_path, transaction) => { + rotationOwnerEntered.resolve(); + await pausedRotationOwner.promise; + await transaction.commitState(bytes("stale-after-rotation")); + }, manualRuntime(rotationClock)); + await rotationOwnerEntered.promise; + await assert.rejects( + () => rotateConsumerStateJournal(rotationAuthorityPath, manualRuntime(rotationClock)), + /actively locked/, + ); + rotationClock.value = 100; + const rotatedAuthority = await rotateConsumerStateJournal(rotationAuthorityPath, manualRuntime(rotationClock)); + assert.equal(rotatedAuthority.epoch, 2); + pausedRotationOwner.resolve(); + await assert.rejects(oldRotationOwner, /retired|fenced|lost ownership/); + await withConsumerStateLock(rotationAuthorityPath, async (_path, transaction) => { + assert.equal(transaction.readStateBytes(), null); + await transaction.commitState(bytes("after-rotation")); + }, manualRuntime(rotationClock)); + assert.deepEqual(JSON.parse(readFileSync(rotationAuthorityPath, "utf8")), { value: "after-rotation" }); + assert.deepEqual( + readdirSync(`${rotationAuthorityPath}.journal`).filter((name) => !name.startsWith(".")), + readdirSync(`${rotationAuthorityPath}.journal`).filter((name) => !name.startsWith(".")).filter( + (name) => name.startsWith("checkpoint-") || name.startsWith("epoch-"), + ), + ); + assert.equal(readdirSync(`${rotationAuthorityPath}.journal`).filter((name) => name.startsWith("checkpoint-")).length, 1); + assert.equal(readdirSync(`${rotationAuthorityPath}.journal`).filter((name) => name.startsWith("epoch-")).length, 1); + + const depthRotationPath = join(fixture, "depth-rotation.json"); + const depthOptions = { ...manualRuntime({ value: 1 }), maxTransactionDepth: 1 }; + await withConsumerStateLock(depthRotationPath, async (_path, transaction) => { + await transaction.commitState(bytes("one")); + }, depthOptions); + await assert.rejects( + () => withConsumerStateLock(depthRotationPath, async (_path, transaction) => { + await transaction.commitState(bytes("blocked")); + }, { ...manualRuntime({ value: 2 }), maxTransactionDepth: 1 }), + /run the consumer journal rotation command/, + ); + await rotateConsumerStateJournal(depthRotationPath, { ...manualRuntime({ value: 3 }), maxTransactionDepth: 1 }); + await withConsumerStateLock(depthRotationPath, async (_path, transaction) => { + assert.deepEqual(JSON.parse(transaction.readStateBytes()), { value: "one" }); + await transaction.commitState(bytes("two")); + }, { ...manualRuntime({ value: 4 }), maxTransactionDepth: 1 }); + assert.deepEqual(JSON.parse(readFileSync(depthRotationPath, "utf8")), { value: "two" }); + + const normalWinsPath = join(fixture, "normal-wins-rotation-slot.json"); + await withConsumerStateLock(normalWinsPath, async (_path, transaction) => { + await transaction.commitState(bytes("base")); + }, manualRuntime({ value: 1 })); + const normalClaimLinked = deferred(); + const releaseNormalClaim = deferred(); + const normalWinner = withConsumerStateLock(normalWinsPath, async (_path, transaction) => { + await transaction.commitState(bytes("normal-winner")); + }, manualRuntime({ value: 2 }, { + afterMetadataLink: async ({ kind }) => { + if (kind !== "claim") return; + normalClaimLinked.resolve(); + await releaseNormalClaim.promise; + }, + })); + await normalClaimLinked.promise; + await assert.rejects( + () => rotateConsumerStateJournal(normalWinsPath, manualRuntime({ value: 2 })), + /actively locked/, + ); + releaseNormalClaim.resolve(); + await normalWinner; + let rederivedRotationClaim; + const normalWinsRotation = await rotateConsumerStateJournal(normalWinsPath, manualRuntime({ value: 3 }, { + beforeRotationDecision: async ({ claim }) => { rederivedRotationClaim = claim; }, + })); + assert.equal(rederivedRotationClaim.generation, 3); + assert.equal(rederivedRotationClaim.intent.tipSha256, sha256Bytes(bytes("normal-winner"))); + assert.equal(normalWinsRotation.tipSha256, sha256Bytes(bytes("normal-winner"))); + + const rotationWinsPath = join(fixture, "rotation-wins-normal-slot.json"); + await withConsumerStateLock(rotationWinsPath, async (_path, transaction) => { + await transaction.commitState(bytes("rotation-base")); + }, manualRuntime({ value: 1 })); + const rotationClaimLinked = deferred(); + const releaseRotationClaim = deferred(); + const rotationWinner = rotateConsumerStateJournal(rotationWinsPath, manualRuntime({ value: 2 }, { + afterMetadataLink: async ({ kind }) => { + if (kind !== "claim") return; + rotationClaimLinked.resolve(); + await releaseRotationClaim.promise; + }, + })); + await rotationClaimLinked.promise; + const pendingRotationJournal = consumerJournal(rotationWinsPath); + const pendingRotationClaim = readdirSync(pendingRotationJournal.epoch) + .filter((name) => name.startsWith("claim-")) + .map((name) => JSON.parse(readFileSync(join(pendingRotationJournal.epoch, name)))) + .at(-1); + assert.equal(pendingRotationClaim.type, "rotation"); + assert.equal(pendingRotationClaim.generation, 2); + let callbackAfterRotation = 0; + await withConsumerStateLock(rotationWinsPath, async (_path, transaction) => { + callbackAfterRotation += 1; + assert.deepEqual(JSON.parse(transaction.readStateBytes()), { value: "rotation-base" }); + }, manualRuntime({ value: 3 })); + assert.equal(callbackAfterRotation, 1); + releaseRotationClaim.resolve(); + assert.equal((await rotationWinner).epoch, 2); + + const claimRotationPath = join(fixture, "claim-rotation.json"); + for (let generation = 1; generation <= 3; generation += 1) { + await withConsumerStateLock( + claimRotationPath, + async () => {}, + { ...manualRuntime({ value: generation }), maxLockGenerations: 3 }, + ); + } + await assert.rejects( + () => withConsumerStateLock( + claimRotationPath, + async () => {}, + { ...manualRuntime({ value: 3 }), maxLockGenerations: 3 }, + ), + /claim epoch is exhausted.*rotation command/, + ); + assert.equal((await rotateConsumerStateJournal( + claimRotationPath, + { ...manualRuntime({ value: 4 }), maxLockGenerations: 3 }, + )).epoch, 2); + + const finalClaimCrashPath = join(fixture, "final-claim-crash.json"); + await withConsumerStateLock( + finalClaimCrashPath, + async () => {}, + { ...manualRuntime({ value: 1 }), maxLockGenerations: 2 }, + ); + await assert.rejects( + () => withConsumerStateLock(finalClaimCrashPath, async () => {}, { + ...manualRuntime({ value: 2 }, { afterClaim: async () => { throw new Error("simulated final claim crash"); } }), + maxLockGenerations: 2, + }), + /simulated final claim crash/, + ); + await assert.rejects( + () => withConsumerStateLock( + finalClaimCrashPath, + async () => {}, + { ...manualRuntime({ value: 100 }), maxLockGenerations: 2 }, + ), + /claim epoch is exhausted/, + ); + const finalCrashJournal = consumerJournal(finalClaimCrashPath); + const finalCrashCheckpoint = JSON.parse(readFileSync(finalCrashJournal.checkpoint)); + const finalCrashClaim = JSON.parse(readFileSync(join( + finalCrashJournal.epoch, + readdirSync(finalCrashJournal.epoch).find((name) => name === "claim-0000000000000002.json"), + ))); + const finalCrashTemporaryDirectory = join(finalCrashJournal.journal, ".owned-temporaries-v2"); + for (let index = 0; index < 17; index += 1) { + const temporaryName = `.pylon-consumer-tmp-v1-p999999-e${finalCrashCheckpoint.epochId}` + + `-g0000000000000002-w${finalCrashClaim.token}-n${index.toString(16).padStart(12, "0")}` + + `-ktransition-t${"c".repeat(64)}.tmp`; + writePrivate(join(finalCrashTemporaryDirectory, temporaryName), "dead final-claim temporary"); + } + let rotationClaimHooks = 0; + assert.equal((await rotateConsumerStateJournal(finalClaimCrashPath, { + ...manualRuntime({ value: 100 }, { afterClaim: async () => { rotationClaimHooks += 1; } }), + maxLockGenerations: 2, + })).epoch, 2); + assert.equal(rotationClaimHooks, 0, "rotation never consumes a normal claim, including the finite final claim"); + assert.deepEqual(readdirSync(finalCrashTemporaryDirectory), []); + await withConsumerStateLock(finalClaimCrashPath, async (_path, transaction) => { + await transaction.commitState(bytes("normal-after-final-crash-rotation")); + }, { ...manualRuntime({ value: 101 }), maxLockGenerations: 2 }); + + const temporaryFloodPath = join(fixture, "temporary-flood.json"); + await withConsumerStateLock(temporaryFloodPath, async () => {}, { + ...manualRuntime({ value: 1 }), + maxLockGenerations: 2, + }); + const floodJournal = consumerJournal(temporaryFloodPath); + const floodCheckpoint = JSON.parse(readFileSync(floodJournal.checkpoint)); + const floodClaimName = readdirSync(floodJournal.epoch).find((name) => name.startsWith("claim-")); + const floodClaim = JSON.parse(readFileSync(join(floodJournal.epoch, floodClaimName))); + const floodTemporaryDirectory = join(floodJournal.journal, ".owned-temporaries-v2"); + const temporaryFileName = ({ pid, generation, token, attempt, kind }) => + `.pylon-consumer-tmp-v1-p${pid}-e${floodCheckpoint.epochId}-g${String(generation).padStart(16, "0")}` + + `-w${token}-n${attempt.toString(16).padStart(12, "0")}-k${kind}-t${"a".repeat(64)}.tmp`; + for (let index = 0; index < 40; index += 1) { + writePrivate(join(floodTemporaryDirectory, temporaryFileName({ + pid: 999_999, + generation: index < 20 ? 0 : 1, + token: index < 20 ? "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" : floodClaim.token, + attempt: index, + kind: index < 20 ? "checkpoint" : "transition", + })), "crashed temporary"); + } + await withConsumerStateLock(temporaryFloodPath, async () => {}, { + ...manualRuntime({ value: 2 }), + maxLockGenerations: 2, + processKill: () => { throw Object.assign(new Error("dead"), { code: "ESRCH" }); }, + }); + assert.deepEqual(readdirSync(floodTemporaryDirectory), [], "temporary cleanup runs before logical journal caps"); + const symlinkTemporary = join(floodTemporaryDirectory, temporaryFileName({ + pid: 999_999, + generation: 0, + token: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + attempt: 99, + kind: "checkpoint", + })); + symlinkSync(temporaryFloodPath, symlinkTemporary); + await assert.rejects( + () => rotateConsumerStateJournal(temporaryFloodPath, { + ...manualRuntime({ value: 3 }), + maxLockGenerations: 2, + }), + /owned temporary.*regular non-symlink file/, + ); + rmSync(symlinkTemporary); + + const liveTemporaryPath = join(fixture, "live-rotation-temporary.json"); + await withConsumerStateLock(liveTemporaryPath, async () => {}, manualRuntime({ value: 1 })); + const liveJournal = consumerJournal(liveTemporaryPath); + const liveCheckpoint = JSON.parse(readFileSync(liveJournal.checkpoint)); + const liveTemporaryDirectory = join(liveJournal.journal, ".owned-temporaries-v2"); + const liveTemporaryName = `.pylon-consumer-tmp-v1-p${process.pid}-e${liveCheckpoint.epochId}` + + `-g0000000000000000-waaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa-n000000000001-kcheckpoint-t${"b".repeat(64)}.tmp`; + writePrivate(join(liveTemporaryDirectory, liveTemporaryName), "live temporary"); + await assert.rejects( + () => rotateConsumerStateJournal(liveTemporaryPath, manualRuntime({ value: 2 })), + /rotation operation is pending.*temporary writer quiesces/, + ); + assert.equal(readdirSync(liveJournal.epoch).filter((name) => name.startsWith("claim-")).some( + (name) => JSON.parse(readFileSync(join(liveJournal.epoch, name))).type === "rotation", + ), true); + await assert.rejects( + () => withConsumerStateLock(liveTemporaryPath, async () => {}, manualRuntime({ value: 3 })), + /rotation operation is pending.*temporary writer quiesces/, + ); + rmSync(join(liveTemporaryDirectory, liveTemporaryName)); + assert.equal((await rotateConsumerStateJournal(liveTemporaryPath, manualRuntime({ value: 4 }))).epoch, 2); + + const concurrentRotationPath = join(fixture, "concurrent-rotation.json"); + await withConsumerStateLock(concurrentRotationPath, async (_path, transaction) => { + await transaction.commitState(bytes("concurrent-anchor")); + }, manualRuntime({ value: 1 })); + const bothRotationsReady = deferred(); + const releaseRotations = deferred(); + let rotationEpochWriters = 0; + const concurrentRotationHooks = { + afterRotationEpochSync: async () => { + rotationEpochWriters += 1; + if (rotationEpochWriters === 2) bothRotationsReady.resolve(); + await releaseRotations.promise; + }, + }; + const firstRotation = rotateConsumerStateJournal(concurrentRotationPath, { + ...manualRuntime({ value: 2 }, concurrentRotationHooks), + maxLockGenerations: 2, + maxTransactionDepth: 1, + }); + const secondRotation = rotateConsumerStateJournal(concurrentRotationPath, { + ...manualRuntime({ value: 2 }, concurrentRotationHooks), + maxLockGenerations: 19, + maxTransactionDepth: 17, + }); + await bothRotationsReady.promise; + releaseRotations.resolve(); + const concurrentRotations = await Promise.all([firstRotation, secondRotation]); + assert.deepEqual(concurrentRotations, [concurrentRotations[0], concurrentRotations[0]]); + assert.equal(concurrentRotations[0].epoch, 2); + await withConsumerStateLock(concurrentRotationPath, async (_path, transaction) => { + assert.deepEqual(JSON.parse(transaction.readStateBytes()), { value: "concurrent-anchor" }); + await transaction.commitState(bytes("concurrent-after-rotation")); + }, { ...manualRuntime({ value: 3 }), maxLockGenerations: 2 }); + + const rotationWavePath = join(fixture, "concurrent-rotation-process-wave.json"); + await withConsumerStateLock(rotationWavePath, async (_path, transaction) => { + await transaction.commitState(bytes("wave-anchor")); + }, manualRuntime({ value: 1 })); + const rotationWave = await Promise.all(Array.from({ length: 12 }, () => runRotationChild(rotationWavePath))); + assert.equal(rotationWave.every((result) => result.epoch === 2), true); + assert.deepEqual(rotationWave, Array.from({ length: 12 }, () => rotationWave[0])); + assert.equal(readdirSync(`${rotationWavePath}.journal`).filter((name) => name.startsWith("checkpoint-")).length, 1); + assert.equal(readdirSync(`${rotationWavePath}.journal`).filter((name) => name.startsWith("epoch-")).length, 1); + + for (const competitorHook of ["afterRotationEpochSync", "afterMetadataLink"]) { + const competingRotationPath = join(fixture, `competing-rotation-${competitorHook}.json`); + await withConsumerStateLock(competingRotationPath, async (_path, transaction) => { + await transaction.commitState(bytes("competing-anchor")); + }, manualRuntime({ value: 1 })); + let checkpoint; + let injected = false; + const injectCompetitor = () => { + if (injected) return; + injected = true; + const competingEpochId = randomUUID(); + const competing = { ...checkpoint, epochId: competingEpochId }; + const root = `${competingRotationPath}.journal`; + mkdirSync(join(root, `epoch-${String(competing.epoch).padStart(16, "0")}-${competingEpochId}`), { mode: 0o700 }); + writePrivate( + join(root, `checkpoint-${String(competing.epoch).padStart(16, "0")}-${competingEpochId}.json`), + metadata(competing), + ); + }; + await assert.rejects( + () => rotateConsumerStateJournal(competingRotationPath, manualRuntime({ value: 2 }, { + beforeRotationDecision: async ({ claim }) => { checkpoint = claim.intent.checkpoint; }, + afterRotationEpochSync: async () => { + if (competitorHook === "afterRotationEpochSync") injectCompetitor(); + }, + afterMetadataLink: async ({ kind }) => { + if (competitorHook === "afterMetadataLink" && kind === "checkpoint") injectCompetitor(); + }, + })), + /competing|checkpoint set|unbounded checkpoint metadata|fenced a paused writer/, + ); + assert.equal(injected, true); + } + + for (const [hookName, wantedKind] of [ + ["afterMetadataLink", "checkpoint"], + ["afterMetadataDirectorySync", "checkpoint"], + ["afterRotationCheckpoint", null], + ]) { + const responseLossPath = join(fixture, `rotation-response-loss-${hookName}.json`); + await withConsumerStateLock(responseLossPath, async (_path, transaction) => { + await transaction.commitState(bytes("response-loss-anchor")); + }, manualRuntime({ value: 1 })); + let armed = true; + const firstResult = await rotateConsumerStateJournal(responseLossPath, { + ...manualRuntime({ value: 2 }, { + [hookName]: async (event = {}) => { + if (!armed || (wantedKind !== null && event.kind !== wantedKind)) return; + armed = false; + throw new Error(`simulated response loss at ${hookName}`); + }, + }), + maxLockGenerations: 2, + }); + const retriedResult = await rotateConsumerStateJournal(responseLossPath, { + ...manualRuntime({ value: 3 }), + maxLockGenerations: 23, + }); + assert.deepEqual(retriedResult, firstResult); + assert.equal(retriedResult.epoch, 2); + await withConsumerStateLock(responseLossPath, async (_path, transaction) => { + assert.deepEqual(JSON.parse(transaction.readStateBytes()), { value: "response-loss-anchor" }); + }, manualRuntime({ value: 4 })); + } + + for (const [hookName, wantedKind] of [ + ["beforeRotationDecision", null], + ["afterFileSync", "claim"], + ["afterMetadataLink", "claim"], + ["afterMetadataDirectorySync", "claim"], + ["afterRotationIntent", null], + ]) { + const intentCrashPath = join(fixture, `rotation-operation-crash-${hookName}.json`); + await withConsumerStateLock(intentCrashPath, async (_path, transaction) => { + await transaction.commitState(bytes("intent-anchor")); + }, manualRuntime({ value: 1 })); + let armed = true; + await assert.rejects( + () => rotateConsumerStateJournal(intentCrashPath, manualRuntime({ value: 2 }, { + [hookName]: async (event = {}) => { + if (!armed || (wantedKind !== null && event.kind !== wantedKind)) return; + armed = false; + throw new Error(`simulated rotation operation crash at ${hookName}`); + }, + })), + /simulated rotation operation crash/, + ); + assert.equal((await rotateConsumerStateJournal(intentCrashPath, manualRuntime({ value: 3 }))).epoch, 2); + } + + for (const [hookName, wantedKind] of [ + ["afterRotationEpochSync", null], + ["afterFileSync", "checkpoint"], + ["afterMetadataLink", "checkpoint"], + ["afterMetadataDirectorySync", "checkpoint"], + ["afterRotationCheckpoint", null], + ]) { + const rotationCrashPath = join(fixture, `rotation-crash-${hookName}.json`); + const rotationCrashClock = { value: 1 }; + await withConsumerStateLock(rotationCrashPath, async (_path, transaction) => { + await transaction.commitState(bytes("anchored")); + }, manualRuntime(rotationCrashClock)); + const reached = deferred(); + const resume = deferred(); + let armed = true; + const interrupted = rotateConsumerStateJournal(rotationCrashPath, manualRuntime(rotationCrashClock, { + [hookName]: async (event = {}) => { + if (!armed || (wantedKind !== null && event.kind !== wantedKind)) return; + armed = false; + reached.resolve(); + await resume.promise; + }, + })); + await reached.promise; + rotationCrashClock.value = 100; + await withConsumerStateLock(rotationCrashPath, async () => {}, manualRuntime(rotationCrashClock)); + resume.resolve(); + await interrupted; + await withConsumerStateLock(rotationCrashPath, async (_path, transaction) => { + assert.deepEqual(JSON.parse(transaction.readStateBytes()), { value: "anchored" }); + }, manualRuntime(rotationCrashClock)); + const rootEntries = readdirSync(`${rotationCrashPath}.journal`); + assert.equal(rootEntries.filter((name) => name.startsWith("checkpoint-")).length, 1); + assert.equal(rootEntries.filter((name) => name.startsWith("epoch-")).length, 1); + assert.deepEqual(rootEntries.filter((name) => name.startsWith(".")), [".owned-temporaries-v2"]); + } + + const swapRoot = join(fixture, "swap-root"); + const swapDirectory = join(swapRoot, "state"); + const movedDirectory = join(swapRoot, "state-moved"); + mkdirSync(swapDirectory, { recursive: true, mode: 0o700 }); + const swapPath = join(swapDirectory, "state.json"); + let swapped = false; + await assert.rejects( + () => withConsumerStateLock(swapPath, async () => {}, manualRuntime({ value: 1 }, { + beforePathOperation: async ({ operation }) => { + if (swapped || operation !== "scan-claims") return; + swapped = true; + renameSync(swapDirectory, movedDirectory); + symlinkSync(movedDirectory, swapDirectory); + }, + })), + /canonical real directory/, + ); + assert.equal(swapped, true, "the injected component swap reached the immediate path revalidation boundary"); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } +}); + +test("admission is non-authoritative and every protected mutation has a fresh combined App audit", async () => { + for (const [workflow, step] of [ + [".github/workflows/pylon-preview-release.yml", "Require the canonical protected push"], + [".github/workflows/pylon-preview-release.yml", "Verify exact checks and freeze the approved preview draft"], + [".github/workflows/pylon-stable-release.yml", "Require protected pylon and an exact verified preview source"], + [".github/workflows/pylon-stable-release.yml", "Re-download and validate the exact stable transaction"], + ]) { + const script = githubScriptForStep(workflow, step); + for (const value of [ + "Check changelog fragment", "build-check-test", "15368", + ".github/workflows/changelog-merged-proof.yml", ".github/workflows/ci.yml", + ]) assert.ok(script.includes(value), `${workflow}:${step} lacks ${value}`); + assert.match(script, /JSON\.stringify\(actualPolicy\) !== JSON\.stringify/); + assert.doesNotMatch(script, /rulesets\/\{ruleset_id\}|bypass_actors|current_user_can_bypass/, + "normal GITHUB_TOKEN admission must not claim authoritative ruleset visibility"); + } + + const authoritativeSteps = [ + [".github/workflows/pylon-preview-release.yml", "Require authoritative publication tag ruleset before preview tag CAS"], + [".github/workflows/pylon-preview-release.yml", "Require authoritative publication tag ruleset before immutable preview publish"], + [".github/workflows/pylon-stable-release.yml", "Require authoritative publication tag ruleset before reservation CAS"], + [".github/workflows/pylon-stable-release.yml", "Require authoritative publication tag ruleset before final stable tag CAS"], + [".github/workflows/pylon-stable-release.yml", "Require authoritative publication tag ruleset before immutable stable publish"], + ]; + const frozenValidators = new Set(authoritativeSteps.map(([workflow, step]) => githubScriptForStep(workflow, step))); + assert.equal(frozenValidators.size, 1, "every protected mutation must use the same frozen combined validator bytes"); + for (const script of frozenValidators) { + assert.match(script, /ruleset_id: 21950766, includes_parents: false/); + assert.match(script, /restRuleset\.node_id !== "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4"/); + assert.match(script, /Object\.hasOwn\(restRuleset, "bypass_actors"\)/); + assert.match(script, /Object\.hasOwn\(restRuleset, "current_user_can_bypass"\)/); + assert.match(script, /ruleset\(databaseId: \$rulesetDatabaseId, includeParents: false\)/); + assert.ok(script.indexOf("await github.request") < script.indexOf("await github.graphql")); + assert.doesNotMatch(script.slice(script.indexOf("await github.graphql") + 1), /await github\./, + "GraphQL must remain the last authoritative GitHub read"); + } + + const validRest = exactPublicationTagRuleset(); + const validGraphql = exactPublicationTagRulesetGraphql(); + await (await inlinePublicationTagRulesetValidator([validRest], [validGraphql])).validate(); + const restWithoutVisibilityFields = structuredClone(validRest); + delete restWithoutVisibilityFields.bypass_actors; + delete restWithoutVisibilityFields.current_user_can_bypass; + await (await inlinePublicationTagRulesetValidator([restWithoutVisibilityFields], [validGraphql])).validate(); + + const restMutations = [ + (value) => delete value.id, + (value) => (value.id = 1), + (value) => delete value.node_id, + (value) => (value.node_id = "RRS_wrong"), + (value) => delete value.name, + (value) => (value.name = "Other ruleset"), + (value) => delete value.source_type, + (value) => (value.source_type = "Organization"), + (value) => delete value.source, + (value) => (value.source = "fork/prime-agent"), + (value) => delete value.target, + (value) => (value.target = "branch"), + (value) => delete value.enforcement, + (value) => (value.enforcement = "disabled"), + (value) => (value.bypass_actors = [{ actor_type: "RepositoryRole", actor_id: 5 }]), + (value) => (value.bypass_actors = undefined), + (value) => (value.current_user_can_bypass = "always"), + (value) => (value.current_user_can_bypass = undefined), + (value) => delete value.conditions, + (value) => (value.conditions.extra = {}), + (value) => delete value.conditions.ref_name, + (value) => delete value.conditions.ref_name.exclude, + (value) => value.conditions.ref_name.exclude.push("refs/tags/pylon-stable-sequence-*"), + (value) => delete value.conditions.ref_name.include, + (value) => (value.conditions.ref_name.include[1] = "refs/tags/pylon-stable-[0-9]*"), + (value) => value.conditions.ref_name.include.pop(), + (value) => delete value.rules, + (value) => value.rules.pop(), + (value) => delete value.rules[0].type, + (value) => delete value.rules[0].parameters, + (value) => delete value.rules[0].parameters.update_allows_fetch_and_merge, + (value) => (value.rules[0].parameters.update_allows_fetch_and_merge = true), + (value) => (value.rules[0].parameters.extra = false), + (value) => delete value.rules[1].type, + (value) => (value.rules[1].extra = false), + (value) => value.rules.push({ type: "creation" }), + ]; + for (const mutate of restMutations) { + const changed = structuredClone(validRest); + mutate(changed); + const rejected = await inlinePublicationTagRulesetValidator([changed], [validGraphql]); + await assert.rejects(() => rejected.validate(), /REST ruleset-auditor response/); + } + for (const unavailable of [ + new Error("ruleset auth or endpoint unavailable"), + { status: 401, data: validRest }, + { status: 403, data: { message: "Resource not accessible by integration" } }, + ]) { + const rejected = await inlinePublicationTagRulesetValidator([unavailable], [validGraphql]); + await assert.rejects(() => rejected.validate(), /unavailable|REST ruleset-auditor response/); + } + + const graphqlMutations = [ + () => null, + () => ({ repository: null }), + (value) => (value.errors = [{ message: "redacted" }]), + (value) => delete value.repository.id, + (value) => (value.repository.id = "R_wrong"), + (value) => delete value.repository.databaseId, + (value) => (value.repository.databaseId = 1), + (value) => delete value.repository.nameWithOwner, + (value) => (value.repository.nameWithOwner = "fork/prime-agent"), + (value) => (value.repository.ruleset = null), + (value) => delete value.repository.ruleset.id, + (value) => (value.repository.ruleset.id = "RRS_wrong"), + (value) => delete value.repository.ruleset.databaseId, + (value) => (value.repository.ruleset.databaseId = 1), + (value) => (value.repository.ruleset.name = "Other ruleset"), + (value) => (value.repository.ruleset.enforcement = "DISABLED"), + (value) => (value.repository.ruleset.target = "BRANCH"), + (value) => (value.repository.ruleset.source = null), + (value) => (value.repository.ruleset.source.id = "R_wrong"), + (value) => (value.repository.ruleset.source.databaseId = 1), + (value) => (value.repository.ruleset.source.nameWithOwner = "fork/prime-agent"), + (value) => (value.repository.ruleset.source.__typename = "Organization"), + (value) => (value.repository.ruleset.bypassActors = null), + (value) => (value.repository.ruleset.bypassActors.totalCount = null), + (value) => (value.repository.ruleset.bypassActors.totalCount = "0"), + (value) => (value.repository.ruleset.bypassActors.totalCount = 1), + (value) => (value.repository.ruleset.bypassActors.extra = 0), + (value) => (value.repository.ruleset.conditions = null), + (value) => (value.repository.ruleset.conditions.organizationProperty = { __typename: "OrganizationPropertyConditionTarget" }), + (value) => (value.repository.ruleset.conditions.repositoryId = { __typename: "RepositoryIdConditionTarget" }), + (value) => (value.repository.ruleset.conditions.extra = null), + (value) => (value.repository.ruleset.conditions.refName = null), + (value) => value.repository.ruleset.conditions.refName.include.pop(), + (value) => value.repository.ruleset.conditions.refName.exclude.push("refs/tags/unsafe-*"), + (value) => (value.repository.ruleset.rules = null), + (value) => (value.repository.ruleset.rules.totalCount = null), + (value) => (value.repository.ruleset.rules.totalCount = 3), + (value) => (value.repository.ruleset.rules.nodes = null), + (value) => (value.repository.ruleset.rules.nodes[0] = null), + (value) => value.repository.ruleset.rules.nodes.push({ type: "CREATION", parameters: null }), + (value) => (value.repository.ruleset.rules.nodes[0].type = "DELETION"), + (value) => (value.repository.ruleset.rules.nodes[0].parameters = null), + (value) => (value.repository.ruleset.rules.nodes[0].parameters.__typename = "OtherParameters"), + (value) => (value.repository.ruleset.rules.nodes[0].parameters.updateAllowsFetchAndMerge = true), + (value) => (value.repository.ruleset.rules.nodes[0].parameters.extra = false), + (value) => (value.repository.ruleset.rules.nodes[1].parameters = {}), + (value) => (value.repository.ruleset.rules.nodes[1].extra = null), + ]; + for (const mutate of graphqlMutations) { + let changed = structuredClone(validGraphql); + const replacement = mutate(changed); + if (replacement !== undefined) changed = replacement; + const rejected = await inlinePublicationTagRulesetValidator([validRest], [changed]); + await assert.rejects(() => rejected.validate(), /GraphQL ruleset-auditor response|Cannot read/); + } + for (const unavailable of [new Error("GraphQL errors: forbidden"), new Error("GraphQL response was redacted")]) { + const rejected = await inlinePublicationTagRulesetValidator([validRest], [unavailable]); + await assert.rejects(() => rejected.validate(), /GraphQL/); + } + + const staleRest = structuredClone(validRest); + staleRest.enforcement = "disabled"; + const restPointInTime = await inlinePublicationTagRulesetValidator([validRest, staleRest], [validGraphql]); + await restPointInTime.validate(); + await assert.rejects(() => restPointInTime.validate(), /REST ruleset-auditor response/); + assert.deepEqual(restPointInTime.requests(), { rest: 2, graphql: 1, order: ["REST", "GraphQL", "REST"] }); + const staleGraphql = structuredClone(validGraphql); + staleGraphql.repository.ruleset.bypassActors.totalCount = 1; + const graphqlPointInTime = await inlinePublicationTagRulesetValidator([validRest], [validGraphql, staleGraphql]); + await graphqlPointInTime.validate(); + await assert.rejects(() => graphqlPointInTime.validate(), /GraphQL ruleset-auditor response/); + assert.deepEqual(graphqlPointInTime.requests(), { + rest: 2, graphql: 2, order: ["REST", "GraphQL", "REST", "GraphQL"], + }); + + const executeProtectedMutation = async ({ appId, privateKey, mint, audit, mutate }) => { + if (!appId || !privateKey) throw new Error("GitHub App credentials are required"); + const token = await mint({ appId, privateKey }); + if (!token) throw new Error("GitHub App token creation returned no token"); + await audit(token); + await mutate(); + }; + let protectedMutationCalls = 0; + const base = { + appId: "1234", + privateKey: "test-private-key", + audit: async () => {}, + mutate: async () => { protectedMutationCalls += 1; }, + }; + await assert.rejects(() => executeProtectedMutation({ ...base, privateKey: "", mint: async () => "token" }), /credentials/); + await assert.rejects(() => executeProtectedMutation({ + ...base, + mint: async () => { throw new Error("simulated create-github-app-token failure"); }, + }), /simulated create-github-app-token failure/); + await assert.rejects(() => executeProtectedMutation({ ...base, mint: async () => "" }), /returned no token/); + assert.equal(protectedMutationCalls, 0); +}); + + +test("stable recovery body durably carries bounded exact canonical manifest bytes", () => { + const manifest = firstStable(); + const bytes = Buffer.from(canonicalJson(manifest)); + const body = publicationReleaseBody({ + channel: "stable", tag: manifest.tag, source: manifest.build.source.commit, tree: manifest.build.source.tree, + recipeRevision: manifest.build.recipeRevision, policyCommit: manifest.promotion.policyCommit, + policyTree: manifest.promotion.policyTree, stableManifestBytes: bytes, + }); + const recovered = stableManifestBytesFromReleaseBody(body); + assert.equal(recovered.bytes.equals(bytes), true); + assert.equal(recovered.digest, sha256Bytes(bytes)); + assert.throws(() => stableManifestBytesFromReleaseBody(body.replace("Manifest bytes: ", "Manifest bytes: 9")), /truncated|metadata/); + assert.throws(() => stableManifestBytesFromReleaseBody(body.slice(0, -1)), /metadata/); + assert.throws(() => publicationReleaseBody({ + channel: "stable", tag: manifest.tag, source: manifest.build.source.commit, tree: manifest.build.source.tree, + recipeRevision: 1, policyCommit: source.commit, policyTree: source.tree, + stableManifestBytes: Buffer.alloc(48 * 1024 + 1), + }), /49152/); +}); + +test("stable zero-asset recovery reuses the body-carried attested bytes and excludes only its draft", async () => { + const fixture = mkdtempSync(join(tmpdir(), "pylon-stable-recovery-")); + try { + const manifest = firstStable(); + const bytes = Buffer.from(canonicalJson(manifest)); + const digest = sha256Bytes(bytes); + const release = { + id: 51, draft: true, immutable: false, tag_name: manifest.tag, + name: `Pylon Prime stable ${manifest.tag}`, prerelease: false, + target_commitish: manifest.promotion.policyCommit, assets: [], + body: publicationReleaseBody({ + channel: "stable", tag: manifest.tag, source: manifest.build.source.commit, tree: manifest.build.source.tree, + recipeRevision: manifest.build.recipeRevision, policyCommit: manifest.promotion.policyCommit, + policyTree: manifest.promotion.policyTree, stableManifestBytes: bytes, + }), + }; + let verified = false; + const notFound = Object.assign(new Error("not found"), { status: 404 }); + const recovered = await recoverStableDraft({ + draftId: 51, reservationTag: "", previewTag: manifest.build.previewTag, operation: "promote", + revokeTag: "", reason: "", outDir: fixture, + }, { + api: async (path) => path.endsWith("/releases/51") ? release : Promise.reject(notFound), + readHistory: async (options) => { + assert.deepEqual(options, { verifyAllAttestations: true, excludeDraftId: 51, excludeDraftTag: manifest.tag }); + return []; + }, + verifyAttestation: (path, policyCommit, policyTree) => { + assert.equal(readFileSync(path).equals(bytes), true); + assert.equal(policyCommit, source.commit); + assert.equal(policyTree, source.tree); + verified = true; + }, + }); + assert.equal(recovered.digest, digest); + assert.equal(verified, true); + assert.deepEqual(selectStableHistoryReleases([release], { excludeDraftId: 51 }), []); + assert.throws(() => selectStableHistoryReleases([release], { excludeDraftId: 52 }), /unexpected draft/); + const altered = structuredClone(release); + altered.body = altered.body.replace(digest, "f".repeat(64)); + await assert.rejects(() => recoverStableDraft({ + draftId: 51, reservationTag: "", previewTag: manifest.build.previewTag, operation: "promote", + revokeTag: "", reason: "", outDir: fixture, + }, { api: async () => altered }), /altered|truncated/); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } +}); + +test("withdrawal replay is a no-op only for the exact latest promotion tuple and reason", () => { + const first = firstStable(); + const latest = secondStable(first, { withdraw: true }); + const request = { previewTag: latest.build.previewTag, revokeTag: first.tag, reason: "security-withdrawal" }; + assert.equal(isExactWithdrawalReplay(latest, request), true); + for (const changed of [ + { ...request, reason: "other-reason" }, + { ...request, revokeTag: latest.tag }, + { ...request, previewTag: "pylon-build-gffffffffffff-r1" }, + ]) assert.equal(isExactWithdrawalReplay(latest, changed), false); + const laterPromotion = structuredClone(latest); + laterPromotion.promotion = { kind: "promote", policyCommit: source.commit, policyTree: source.tree, publicationPolicyRevision: 1 }; + assert.equal(isExactWithdrawalReplay(laterPromotion, request), false); + const message = stableReservationMessage(latest, sha256Bytes(Buffer.from(canonicalJson(latest))), 51); + for (const field of ["Withdraw stable tag", "Withdraw build tag", "Withdraw reason"]) assert.match(message, new RegExp(`^${field}:`, "m")); +}); + +test("signed preview attempt evidence ignores a later aggregate rerun and response-loss conclusion", () => { + const { preview } = manifests(); + const signed = [{ runId: invocation.workflowRunId, runAttempt: "1" }]; + const attempt = { + run: { + id: Number(invocation.workflowRunId), run_attempt: 1, run_number: invocation.sequence, + event: "push", head_branch: "pylon", head_sha: source.commit, path: PYLON_PREVIEW_WORKFLOW, + repository: { id: 1_349_002_285, full_name: PYLON_PUBLICATION_REPOSITORY }, + head_repository: { id: 1_349_002_285, full_name: PYLON_PUBLICATION_REPOSITORY }, + check_suite_id: 77, status: "completed", conclusion: "failure", + }, + suite: { id: 77, app: { id: GITHUB_ACTIONS_APP_ID }, head_sha: source.commit, conclusion: "failure" }, + jobs: [{ name: "Approve and attest six preview subjects", run_attempt: 1, status: "completed", conclusion: "success" }], + }; + assert.equal(validatePreviewWorkflowRunEvidence({ attempts: [attempt], aggregateRun: { run_attempt: 2, conclusion: "failure" } }, preview, signed).workflowRunId, invocation.workflowRunId); + const verifier = readFileSync(join(root, "scripts/verify-pylon-publication-attestations.mjs"), "utf8"); + assert.match(verifier, /actions\/runs\/\$\{runId\}\/attempts\/\$\{runAttempt\}/); + assert.doesNotMatch(verifier, /actions\/runs\/\$\{previewManifest\.workflowRunId\}`/); +}); + +test("stable stage survives a crash after createRelease by recovering its exact zero-asset body", async () => { + const fixture = mkdtempSync(join(tmpdir(), "pylon-stage-crash-")); + const oldManifest = process.env.STABLE_MANIFEST; + try { + const manifest = firstStable(); + const bytes = Buffer.from(canonicalJson(manifest)); + const manifestPath = join(fixture, PYLON_STABLE_MANIFEST); + writeFileSync(manifestPath, bytes); + process.env.STABLE_MANIFEST = manifestPath; + let draft; + let crash = true; + const missing = () => Promise.reject(Object.assign(new Error("missing"), { status: 404 })); + const listReleases = async () => {}; + const github = { + paginate: async (method) => method === listReleases ? (draft ? [draft] : []) : [], + rest: { + git: { getRef: missing }, + repos: { + listReleases, + createRelease: async (request) => { + draft = { id: 51, draft: true, immutable: false, tag_name: request.tag_name, name: request.name, + body: request.body, prerelease: false, target_commitish: request.target_commitish, assets: [] }; + if (crash) throw new Error("simulated crash after createRelease"); + return { data: draft }; + }, + getRelease: async () => ({ data: draft }), + }, + }, + request: async (route, request) => { + if (route.startsWith("POST ")) { + draft.assets = [{ id: 9, name: request.name, size: request.data.length, digest: `sha256:${sha256Bytes(request.data)}` }]; + return { data: draft.assets[0] }; + } + return { data: bytes }; + }, + }; + const context = { repo: { owner: "pylon-code", repo: "prime-agent" }, eventName: "workflow_dispatch", ref: PYLON_PUBLICATION_REF, sha: source.commit }; + const script = githubScriptForStep(".github/workflows/pylon-stable-release.yml", "Create or finish the exact durable draft"); + const execute = new AsyncFunction("github", "context", "core", "require", script); + await assert.rejects(() => execute(github, context, {}, nodeRequire), /simulated crash/); + assert.equal(draft.assets.length, 0); + assert.equal(stableManifestBytesFromReleaseBody(draft.body).bytes.equals(bytes), true); + crash = false; + assert.equal(await execute(github, context, {}, nodeRequire), 51); + assert.equal(draft.assets.length, 1); + assert.equal(draft.assets[0].digest, `sha256:${sha256Bytes(bytes)}`); + } finally { + if (oldManifest === undefined) delete process.env.STABLE_MANIFEST; + else process.env.STABLE_MANIFEST = oldManifest; + rmSync(fixture, { recursive: true, force: true }); + } +}); + +test("final tag CAS models reject squats and preserve reservation-tag-publish order", async () => { + const exactCas = async ({ read, create, expected }) => { + try { + const existing = await read(); + if (existing.type !== "commit" || existing.sha !== expected) throw new Error("unsafe tag"); + return existing; + } catch (error) { + if (error.status !== 404) throw error; + try { await create(); } catch (race) { if (race.status !== 422) throw race; } + const raced = await read(); + if (raced.type !== "commit" || raced.sha !== expected) throw new Error("unsafe tag"); + return raced; + } + }; + const missing = Object.assign(new Error("missing"), { status: 404 }); + const raced = Object.assign(new Error("race"), { status: 422 }); + assert.deepEqual(await exactCas({ read: async () => ({ type: "commit", sha: source.commit }), create: async () => {}, expected: source.commit }), { type: "commit", sha: source.commit }); + let reads = 0; + assert.deepEqual(await exactCas({ + read: async () => { reads += 1; if (reads === 1) throw missing; return { type: "commit", sha: source.commit }; }, + create: async () => { throw raced; }, expected: source.commit, + }), { type: "commit", sha: source.commit }); + for (const unsafe of [{ type: "tag", sha: source.commit }, { type: "commit", sha: "f".repeat(40) }]) { + await assert.rejects(() => exactCas({ read: async () => unsafe, create: async () => {}, expected: source.commit }), /unsafe tag/); + } + const preview = readFileSync(join(root, ".github/workflows/pylon-preview-release.yml"), "utf8"); + assert.ok(preview.indexOf("refs/tags/${tag}") < preview.indexOf("repos.createRelease")); + assert.ok(preview.lastIndexOf("await requireExactTag()") < preview.lastIndexOf("repos.updateRelease")); + const stable = readFileSync(join(root, ".github/workflows/pylon-stable-release.yml"), "utf8"); + const publish = stable.slice(stable.indexOf("name: Reserve and publish immutable stable sequence")); + const reservationCas = publish.indexOf("- name: Create the exact protected stable reservation ref"); + const stableTagCas = publish.indexOf("- name: Create or refetch the exact protected stable tag"); + const immutablePublish = publish.indexOf("- name: Publish only the exact protected stable draft"); + assert.ok(publish.indexOf('POST /repos/{owner}/{repo}/releases/{release_id}/assets') < reservationCas); + assert.ok(publish.indexOf("downloadedBytes.equals(bytes)") < reservationCas); + assert.ok(reservationCas >= 0 && reservationCas < stableTagCas); + assert.ok(stableTagCas < immutablePublish); + assert.ok(immutablePublish < publish.indexOf("repos.updateRelease")); +}); + +test("workflow static policy proves direct approvals and every contents-write graph", () => { + const workflowFiles = readdirSync(join(root, ".github/workflows")) + .filter((file) => /\.ya?ml$/.test(file)) + .map((file) => `.github/workflows/${file}`); + const workflows = new Map(workflowFiles.map((file) => [file, readFileSync(join(root, file), "utf8")])); + for (const [file, workflow] of workflows) { + assert.doesNotMatch(workflow, /^permissions:\n(?: [^\n]+\n)* contents: write$/m, `${file} grants top-level contents write`); + for (const match of workflow.matchAll(/^\s*uses:\s*[^\s@]+@([^\s#]+)/gm)) { + assert.match(match[1], /^[0-9a-f]{40}$/, `${file} contains an unpinned action`); + } + } + const blocks = (workflow) => { + const matches = [...workflow.matchAll(/^ ([A-Za-z0-9_-]+):\s*$/gm)]; + return new Map(matches.map((match, index) => [ + match[1], + workflow.slice(match.index, matches[index + 1]?.index ?? workflow.length), + ])); + }; + const needs = (block, job) => new RegExp(`^ needs:.*(?:\\[|, | )${job}(?:\\]|,|$)`, "m").test(block); + assert.equal(ATTEST_ACTION_CHAIN, "actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d"); + assert.equal(CREATE_GITHUB_APP_TOKEN_ACTION, "actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349"); + const preview = workflows.get(".github/workflows/pylon-preview-release.yml"); + const stable = workflows.get(".github/workflows/pylon-stable-release.yml"); + assert.deepEqual(validateApprovedAttestationWorkflow(preview, "preview"), { + workflow: ".github/workflows/pylon-preview-release.yml", + environment: "pylon-preview", + }); + assert.deepEqual(validateApprovedAttestationWorkflow(stable, "stable"), { + workflow: ".github/workflows/pylon-stable-release.yml", + environment: "pylon-stable", + }); + for (const changed of [ + preview.replace("environment: pylon-preview", "environment: other"), + preview.replace("id-token: write", "id-token: read"), + preview.replace("4d101475d8b20a2381f78447822ac1eab6504dd8", "f".repeat(40)), + preview.replace("subject-path: .npm/pylon-release/artifacts/*", "subject-path: publication/*"), + preview.replace("needs: [pack, reproducibility, install]", "needs: pack"), + preview.replace(" - name: Generate build provenance", " - run: node scripts/untrusted.mjs\n - name: Generate build provenance"), + preview.replace("private-key: ${{ secrets.PYLON_RULESET_AUDITOR_PRIVATE_KEY }}", "private-key: ''"), + preview.replace("private-key: ${{ secrets.PYLON_RULESET_AUDITOR_PRIVATE_KEY }}", "private-key: ${{ env.APP_KEY }}"), + preview.replace("permission-administration: read", "permission-administration: write"), + preview.replace("permission-administration: read", "permission-administration: read\n permission-contents: read"), + preview.replace("permission-administration: read", "permission-administration: read\n github-api-url: https://api.github.com"), + preview.replace("permission-administration: read", "permission-administration: read\n skip-token-revoke: true"), + preview.replace("id: ruleset-auditor", "id: ruleset-auditor\n id: decoy"), + preview.replace("id: ruleset-auditor", "id: ruleset-auditor\n continue-on-error: true"), + preview.replace("id: ruleset-auditor", "id: ruleset-auditor\n env:\n APP_KEY: ${{ secrets.PYLON_RULESET_AUDITOR_PRIVATE_KEY }}"), + preview.replace("id: ruleset-auditor", "id: ruleset-auditor\n run: echo ${{ steps.ruleset-auditor.outputs.token }}"), + preview.replace("id: ruleset-auditor", "id: ruleset-auditor\n outputs:\n token: ${{ steps.ruleset-auditor.outputs.token }}"), + preview.replace("id: ruleset-auditor", "id: ruleset-auditor # trusted"), + preview.replace(" with:\n app-id:", " with: &auditor\n app-id:"), + preview.replace(" with:\n app-id:", " with: *auditor\n app-id:"), + preview.replace(" owner: pylon-code", " owner: pylon-code\n owner: attacker"), + preview.replace(CREATE_GITHUB_APP_TOKEN_ACTION, `${CREATE_GITHUB_APP_TOKEN_ACTION} # mutable comment`), + preview.replace(CREATE_GITHUB_APP_TOKEN_ACTION, "actions/create-github-app-token@" + "f".repeat(40)), + preview.replace(" - name: Require live pylon immediately", ` # decoy ${CREATE_GITHUB_APP_TOKEN_ACTION}\n - name: Require live pylon immediately`), + preview.replace("github-token: ${{ steps.ruleset-auditor.outputs.token }}", "github-token: ${{ fromJSON(steps.ruleset-auditor.outputs.token) }}"), + preview.replace("github-token: ${{ steps.ruleset-auditor.outputs.token }}", "github-token: ${{ steps.ruleset-auditor.outputs.token }}\n token: ${{ steps.ruleset-auditor.outputs.token }}"), + preview.replace(" - name: Create or refetch the exact protected preview tag", " - name: Token exposure\n env:\n TOKEN: ${{ steps.ruleset-auditor.outputs.token }}\n run: node scripts/untrusted.mjs\n - name: Create or refetch the exact protected preview tag"), + preview.replace(" - name: Create or refetch the exact protected preview tag", " - name: Indirect step exposure\n env:\n STEPS: ${{ toJSON(steps) }}\n - name: Create or refetch the exact protected preview tag"), + preview.replace(" - name: Create or refetch the exact protected preview tag", " - name: Bracket step exposure\n env:\n TOKEN: ${{ steps['ruleset-' + 'auditor'].outputs.token }}\n - name: Create or refetch the exact protected preview tag"), + preview.replace("github-token: ${{ steps.ruleset-auditor.outputs.token }}", "github-token: ${{\n steps.ruleset-auditor.outputs.token\n }}"), + preview.replace(" - name: Require authoritative publication tag ruleset before preview tag CAS\n uses:", " - name: Require authoritative publication tag ruleset before preview tag CAS\n continue-on-error: true\n uses:"), + preview.replace(" - name: Require authoritative publication tag ruleset before preview tag CAS\n uses:", " - name: Require authoritative publication tag ruleset before preview tag CAS\n if: false\n uses:"), + preview.replace(" script: |\n const response = await github.request", " script: |\n core.setOutput('token', await github.auth());\n const response = await github.request"), + ]) assert.throws(() => validateApprovedAttestationWorkflow(changed, "preview")); + + const approvedWriters = new Set([ + ".github/workflows/pylon-preview-release.yml:stage-draft", + ".github/workflows/pylon-preview-release.yml:publish", + ".github/workflows/pylon-stable-release.yml:stage-draft", + ".github/workflows/pylon-stable-release.yml:publish", + ".github/workflows/pylon-upstream-sync.yml:sync", + ]); + const foundWriters = new Set(); + for (const [file, workflow] of workflows) { + for (const [name, block] of blocks(workflow)) { + if (!/^ contents: write$/m.test(block)) continue; + const identity = `${file}:${name}`; + foundWriters.add(identity); + assert.ok(approvedWriters.has(identity), `unapproved contents writer: ${identity}`); + if (identity !== ".github/workflows/pylon-upstream-sync.yml:sync") { + assert.doesNotMatch(block, /actions\/checkout|actions\/setup-node|npm (?:ci|run|install)|node scripts\/|\.tgz\b.*(?:exec|run)/); + } + assert.doesNotMatch(block, /id-token: write|attestations: write/); + const protectedMutationWriter = new Set([ + ".github/workflows/pylon-preview-release.yml:stage-draft", + ".github/workflows/pylon-preview-release.yml:publish", + ".github/workflows/pylon-stable-release.yml:publish", + ]).has(identity); + if (protectedMutationWriter) { + assert.match(block, new RegExp(`^ environment: pylon-${file.includes("preview") ? "preview" : "stable"}$`, "m")); + assert.match(block, new RegExp(CREATE_GITHUB_APP_TOKEN_ACTION.replaceAll("/", "\\/"))); + assert.match(block, /app-id: \$\{\{ vars\.PYLON_RULESET_AUDITOR_APP_ID \}\}/); + assert.match(block, /private-key: \$\{\{ secrets\.PYLON_RULESET_AUDITOR_PRIVATE_KEY \}\}/); + assert.match(block, /permission-administration: read/); + assert.doesNotMatch(block, /skip-token-revoke:|permission-contents:|continue-on-error:/); + } + } + } + assert.deepEqual(foundWriters, approvedWriters); + const previewJobs = blocks(preview); + assert.ok(needs(previewJobs.get("stage-draft"), "verify-attestation")); + assert.match(previewJobs.get("stage-draft"), /^ environment: pylon-preview$/m); + assert.ok(needs(previewJobs.get("publish"), "stage-draft")); + assert.match(previewJobs.get("publish"), /^ environment: pylon-preview$/m); + assert.ok(needs(previewJobs.get("publish"), "verify-attestation")); + const stableJobs = blocks(stable); + assert.ok(needs(stableJobs.get("stage-draft"), "verify-attestation")); + assert.ok(needs(stableJobs.get("publish"), "stage-draft")); + assert.ok(needs(stableJobs.get("publish"), "authorize-stable-resume")); + assert.match(stableJobs.get("publish"), /^ environment: pylon-stable$/m); + assert.match(stableJobs.get("authorize-stable-resume"), /environment: pylon-stable/); + assert.match(stableJobs.get("authorize-stable-resume"), /permissions: \{\}/); + assert.match(stableJobs.get("attest"), /if: .*mode == 'normal'/); + assert.match(stableJobs.get("authorize-stable-resume"), /if: .*mode == 'resume'/); + const upstream = blocks(workflows.get(".github/workflows/pylon-upstream-sync.yml")); + assert.match(upstream.get("sync"), /environment: pylon-upstream-sync/); + assert.doesNotMatch(workflows.get(".github/workflows/pylon-upstream-sync.yml"), /authorize-upstream-sync/); + assert.match(stable, /group: pylon-stable-publication[\s\S]*cancel-in-progress: false/); + assert.match(stable, /final publish ruleset audit/i); + assert.equal((stable.match(/github-token: \$\{\{ steps\.ruleset-auditor\.outputs\.token \}\}/g) ?? []).length, 3); + assert.equal((preview.match(/github-token: \$\{\{ steps\.ruleset-auditor\.outputs\.token \}\}/g) ?? []).length, 2); + assert.match(stable, /refetched state and stopped without N\+1, move, or delete/); + assert.match(stable, /Draft release: \$\{draft\.id\}/); + assert.match(stable, /Withdraw build tag:/); + assert.match(stable, /final stable tag CAS/); + assert.doesNotMatch(stable, /manifest\.sequence\s*\+\+|updateRef|deleteRef|deleteRelease|deleteReleaseAsset/); + const stepBlocks = (job) => { + const matches = [...job.matchAll(/^ - name: (.+)$/gm)]; + return matches.map((match, index) => ({ + name: match[1], + block: job.slice(match.index, matches[index + 1]?.index ?? job.length), + })); + }; + const mutationInventory = []; + for (const [workflow, jobs] of [["preview", previewJobs], ["stable", stableJobs]]) { + for (const [jobName, job] of jobs) { + const steps = stepBlocks(job); + for (let index = 0; index < steps.length; index += 1) { + if (!/github\.rest\.git\.createRef|github\.rest\.repos\.updateRelease/.test(steps[index].block)) continue; + mutationInventory.push(`${workflow}:${jobName}:${steps[index].name}`); + assert.match(steps[index - 1].block, /github-token: \$\{\{ steps\.ruleset-auditor\.outputs\.token \}\}/, + "each protected mutation needs an adjacent fresh App-authenticated audit"); + assert.doesNotMatch(steps[index].block, /ruleset-auditor\.outputs\.token|PYLON_RULESET_AUDITOR/, + "the App token must not enter a contents-write step"); + } + } + } + assert.deepEqual(mutationInventory.sort(), [ + "preview:publish:Publish the exact approved preview draft", + "preview:stage-draft:Create or refetch the exact protected preview tag", + "stable:publish:Create or refetch the exact protected stable tag", + "stable:publish:Create the exact protected stable reservation ref", + "stable:publish:Publish only the exact protected stable draft", + ].sort()); + const attestationVerifier = readFileSync(join(root, "scripts/verify-pylon-publication-attestations.mjs"), "utf8"); + for (const flag of ["--cert-identity", "--signer-digest", "--source-ref", "--source-digest", "--cert-oidc-issuer", "--predicate-type", "--deny-self-hosted-runners"]) { + assert.match(attestationVerifier, new RegExp(flag)); + } + assert.match(attestationVerifier, /verifyApprovedWorkflowAtSignerDigest/); +}); diff --git a/scripts/pylon-ruleset-auditor-acceptance.test.mjs b/scripts/pylon-ruleset-auditor-acceptance.test.mjs new file mode 100644 index 0000000000..c2b4caf44f --- /dev/null +++ b/scripts/pylon-ruleset-auditor-acceptance.test.mjs @@ -0,0 +1,321 @@ +import assert from "node:assert/strict"; +import { generateKeyPairSync, verify } from "node:crypto"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, test } from "node:test"; + +import { + acceptRulesetAuditorApp, + createAppJwt, +} from "./accept-pylon-ruleset-auditor-app.mjs"; +import { + PYLON_PUBLICATION_RULESET_GRAPHQL_QUERY, + PYLON_PUBLICATION_RULESET_GRAPHQL_VARIABLES, + PYLON_RULESET_BYPASS_CANARY_GRAPHQL_QUERY, +} from "./lib/pylon-ruleset-auditor.mjs"; + +const fixture = mkdtempSync(join(tmpdir(), "pylon-ruleset-auditor-acceptance-")); +const keyPath = join(fixture, "app.pem"); +const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); +writeFileSync(keyPath, privateKey.export({ type: "pkcs8", format: "pem" }), { mode: 0o600 }); +after(() => rmSync(fixture, { recursive: true, force: true })); + +const appId = "123456"; +const appSlug = "pylon-publication-ruleset-auditor"; +const canary = { owner: "public-canary", repo: "ruleset-canary", rulesetDatabaseId: 98_765 }; +const repository = { + id: 1_349_002_285, + name: "prime-agent", + full_name: "pylon-code/prime-agent", + owner: { login: "pylon-code" }, +}; + +function restRuleset() { + return { + id: 21_950_766, + node_id: "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4", + name: "Pylon immutable publication tags", + target: "tag", + source_type: "Repository", + source: "pylon-code/prime-agent", + enforcement: "active", + conditions: { + ref_name: { + exclude: [], + include: ["refs/tags/pylon-build-*", "refs/tags/pylon-stable-*"], + }, + }, + rules: [ + { type: "update", parameters: { update_allows_fetch_and_merge: false } }, + { type: "deletion" }, + ], + }; +} + +function targetGraphql() { + return { + repository: { + id: "R_kgDOUGgkLQ", + databaseId: 1_349_002_285, + nameWithOwner: "pylon-code/prime-agent", + ruleset: { + id: "RRS_lACqUmVwb3NpdG9yec5QaCQtzgFO8S4", + databaseId: 21_950_766, + name: "Pylon immutable publication tags", + enforcement: "ACTIVE", + target: "TAG", + source: { + __typename: "Repository", + id: "R_kgDOUGgkLQ", + databaseId: 1_349_002_285, + nameWithOwner: "pylon-code/prime-agent", + }, + bypassActors: { totalCount: 0 }, + conditions: { + refName: { include: ["refs/tags/pylon-build-*", "refs/tags/pylon-stable-*"], exclude: [] }, + organizationProperty: null, + repositoryId: null, + repositoryName: null, + repositoryProperty: null, + }, + rules: { + totalCount: 2, + nodes: [ + { type: "UPDATE", parameters: { __typename: "UpdateParameters", updateAllowsFetchAndMerge: false } }, + { type: "DELETION", parameters: null }, + ], + }, + }, + }, + }; +} + +function exactApp() { + return { id: Number(appId), slug: appSlug, permissions: { administration: "read", metadata: "read" } }; +} + +function exactInstallation() { + return { + id: 654321, + app_id: Number(appId), + app_slug: appSlug, + account: { login: "pylon-code", id: 314_006_107, type: "Organization" }, + target_id: 314_006_107, + target_type: "Organization", + repository_selection: "selected", + suspended_at: null, + suspended_by: null, + permissions: { administration: "read", metadata: "read" }, + }; +} + +function tokenResponse(token) { + return { + token, + expires_at: new Date(1_800_000_000_000 + 60 * 60 * 1000).toISOString(), + permissions: { administration: "read" }, + repository_selection: "selected", + repositories: [structuredClone(repository)], + }; +} + +function mockAcceptance(overrides = {}) { + const calls = []; + let mint = 0; + const request = async (call) => { + calls.push(structuredClone(call)); + const { method, path, token, body } = call; + if (method === "GET" && path === "/app") { + assert.match(token, /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/); + return { status: 200, data: structuredClone(overrides.app ?? exactApp()) }; + } + if (method === "GET" && path === "/orgs/pylon-code/installation") { + assert.match(token, /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/); + return { status: 200, data: structuredClone(overrides.installation ?? exactInstallation()) }; + } + if (method === "POST" && path === "/app/installations/654321/access_tokens") { + assert.match(token, /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/); + mint += 1; + if (mint === 1) { + assert.deepEqual(body, { permissions: { administration: "read" } }); + return { status: 201, data: structuredClone(overrides.fullToken ?? tokenResponse("full-token")) }; + } + assert.deepEqual(body, { repositories: ["prime-agent"], permissions: { administration: "read" } }); + return { status: 201, data: structuredClone(overrides.runtimeToken ?? tokenResponse("runtime-token")) }; + } + if (method === "GET" && path === "/installation/repositories?per_page=100&page=1") { + assert.equal(token, "full-token"); + return { + status: 200, + data: structuredClone(overrides.repositoryPage ?? { total_count: 1, repositories: [repository] }), + }; + } + if (method === "DELETE" && path === "/installation/token") { + assert.ok(["full-token", "runtime-token"].includes(token)); + const status = overrides.revokeStatus?.[token] ?? 204; + return { status, data: null }; + } + if (method === "GET" && path === "/repos/pylon-code/prime-agent/rulesets/21950766") { + assert.equal(token, "runtime-token"); + return { status: 200, data: structuredClone(overrides.rest ?? restRuleset()) }; + } + if (method === "POST" && path === "/graphql") { + assert.equal(token, "runtime-token"); + if (body.query === PYLON_PUBLICATION_RULESET_GRAPHQL_QUERY) { + assert.deepEqual(body.variables, PYLON_PUBLICATION_RULESET_GRAPHQL_VARIABLES); + return { status: 200, data: structuredClone(overrides.targetEnvelope ?? { data: targetGraphql() }) }; + } + assert.equal(body.query, PYLON_RULESET_BYPASS_CANARY_GRAPHQL_QUERY); + assert.deepEqual(body.variables, canary); + return { + status: 200, + data: structuredClone(overrides.canaryEnvelope ?? { + data: { + repository: { + nameWithOwner: `${canary.owner}/${canary.repo}`, + ruleset: { databaseId: canary.rulesetDatabaseId, bypassActors: { totalCount: 3 } }, + }, + }, + }), + }; + } + throw new Error(`Unexpected mocked request: ${method} ${path}`); + }; + return { request, calls }; +} + +async function acceptWithMock(mock) { + return acceptRulesetAuditorApp({ appId, privateKeyPath: keyPath, appSlug, canary, request: mock.request, now: 1_800_000_000_000 }); +} + +function revocations(mock) { + return mock.calls.filter((call) => call.method === "DELETE").map((call) => call.token); +} + +test("locally signs one bounded GitHub App JWT without exposing private-key bytes", () => { + const now = 1_800_000_000_000; + const jwt = createAppJwt({ appId, privateKeyPath: keyPath, now }); + const [header, payload, signature] = jwt.split("."); + assert.deepEqual(JSON.parse(Buffer.from(header, "base64url")), { alg: "RS256", typ: "JWT" }); + assert.deepEqual(JSON.parse(Buffer.from(payload, "base64url")), { + iat: Math.floor(now / 1000) - 60, + exp: Math.floor(now / 1000) + 540, + iss: appId, + }); + assert.equal(verify("RSA-SHA256", Buffer.from(`${header}.${payload}`), publicKey, Buffer.from(signature, "base64url")), true); + assert.ok(jwt.length < 4096); +}); + +test("live acceptance mocks every read-only endpoint, exact scope, canary, and revocation", async () => { + const mock = mockAcceptance(); + assert.deepEqual(await acceptWithMock(mock), { + appId, + appSlug, + installationId: 654321, + repository: "pylon-code/prime-agent", + rulesetId: 21_950_766, + accepted: true, + }); + assert.deepEqual(mock.calls.map(({ method, path }) => `${method} ${path}`), [ + "GET /app", + "GET /orgs/pylon-code/installation", + "POST /app/installations/654321/access_tokens", + "GET /installation/repositories?per_page=100&page=1", + "DELETE /installation/token", + "POST /app/installations/654321/access_tokens", + "GET /repos/pylon-code/prime-agent/rulesets/21950766", + "POST /graphql", + "POST /graphql", + "DELETE /installation/token", + ]); + assert.deepEqual(revocations(mock), ["full-token", "runtime-token"]); +}); + +test("live acceptance rejects App and installation identity, permission, selection, and suspension drift", async () => { + const cases = [ + { app: { ...exactApp(), id: 1 } }, + { app: { ...exactApp(), slug: "other-app" } }, + { app: { ...exactApp(), permissions: { administration: "write", metadata: "read" } } }, + { installation: { ...exactInstallation(), repository_selection: "all" } }, + { installation: { ...exactInstallation(), suspended_at: "2030-01-01T00:00:00Z" } }, + { installation: { ...exactInstallation(), permissions: { metadata: "read" } } }, + ]; + for (const drift of cases) { + const mock = mockAcceptance(drift); + await assert.rejects(() => acceptWithMock(mock)); + assert.deepEqual(revocations(mock), []); + } +}); + +test("live acceptance rejects token scope and exact repository-selection drift and still revokes minted tokens", async () => { + for (const fullToken of [ + { ...tokenResponse("full-token"), expires_at: "2030-01-01T00:00:00Z" }, + { ...tokenResponse("full-token"), permissions: { administration: "write" } }, + { ...tokenResponse("full-token"), repository_selection: "all" }, + { ...tokenResponse("full-token"), repositories: [{ ...repository, id: 1 }] }, + ]) { + const mock = mockAcceptance({ fullToken }); + await assert.rejects(() => acceptWithMock(mock)); + assert.deepEqual(revocations(mock), ["full-token"]); + } + const mockPage = mockAcceptance({ repositoryPage: { total_count: 2, repositories: [repository, { ...repository, id: 1 }] } }); + await assert.rejects(() => acceptWithMock(mockPage)); + assert.deepEqual(revocations(mockPage), ["full-token"]); + for (const runtimeToken of [ + { ...tokenResponse("runtime-token"), permissions: { administration: "read", contents: "read" } }, + { ...tokenResponse("runtime-token"), repositories: [{ ...repository, full_name: "pylon-code/other" }] }, + ]) { + const mock = mockAcceptance({ runtimeToken }); + await assert.rejects(() => acceptWithMock(mock)); + assert.deepEqual(revocations(mock), ["full-token", "runtime-token"]); + } +}); + +test("live acceptance rejects target GraphQL null, partial, errors, redaction, and nonzero bypass while revoking", async () => { + const envelopes = [ + { data: null }, + { data: { repository: null } }, + { errors: [{ message: "Resource not accessible by integration" }], data: targetGraphql() }, + { data: { ...targetGraphql(), repository: { ...targetGraphql().repository, ruleset: null } } }, + (() => { + const value = targetGraphql(); + value.repository.ruleset.bypassActors = null; + return { data: value }; + })(), + (() => { + const value = targetGraphql(); + value.repository.ruleset.bypassActors.totalCount = 1; + return { data: value }; + })(), + ]; + for (const targetEnvelope of envelopes) { + const mock = mockAcceptance({ targetEnvelope }); + await assert.rejects(() => acceptWithMock(mock)); + assert.deepEqual(revocations(mock), ["full-token", "runtime-token"]); + } +}); + +test("live acceptance rejects a null, errored, redacted, or zero bypass-count canary and revocation failure", async () => { + for (const canaryEnvelope of [ + { data: null }, + { errors: [{ message: "redacted" }] }, + { data: { repository: null } }, + { data: { repository: { nameWithOwner: `${canary.owner}/${canary.repo}`, ruleset: null } } }, + { + data: { + repository: { + nameWithOwner: `${canary.owner}/${canary.repo}`, + ruleset: { databaseId: canary.rulesetDatabaseId, bypassActors: { totalCount: 0 } }, + }, + }, + }, + ]) { + const mock = mockAcceptance({ canaryEnvelope }); + await assert.rejects(() => acceptWithMock(mock)); + assert.deepEqual(revocations(mock), ["full-token", "runtime-token"]); + } + const revokeFailure = mockAcceptance({ revokeStatus: { "runtime-token": 500 } }); + await assert.rejects(() => acceptWithMock(revokeFailure), /revocation/); + assert.deepEqual(revocations(revokeFailure), ["full-token", "runtime-token"]); +}); diff --git a/scripts/recover-pylon-stable-manifest.mjs b/scripts/recover-pylon-stable-manifest.mjs new file mode 100644 index 0000000000..e1dcd0e313 --- /dev/null +++ b/scripts/recover-pylon-stable-manifest.mjs @@ -0,0 +1,147 @@ +#!/usr/bin/env node + +import { appendFileSync, mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + canonicalJson, + publicationReleaseBody, + PYLON_PUBLICATION_REPOSITORY, + PYLON_STABLE_MANIFEST, + sha256Bytes, + stableManifestBytesFromReleaseBody, + stableReservationMessage, + validateStableHistory, + validateStableManifest, +} from "./lib/pylon-publication.mjs"; +import { readStableHistory } from "./prepare-pylon-stable-manifest.mjs"; +import { verifyStableAttestation } from "./verify-pylon-stable-attestation.mjs"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +function parseArgs(args) { + const values = new Map(); + for (let index = 0; index < args.length; index += 2) { + if (!args[index]?.startsWith("--") || args[index + 1] === undefined) throw new Error("Recovery arguments must be name/value pairs."); + values.set(args[index], args[index + 1]); + } + const draftId = values.get("--draft-id") ?? ""; + const reservationTag = values.get("--reservation-tag") ?? ""; + const previewTag = values.get("--preview-tag") ?? ""; + const operation = values.get("--operation") ?? ""; + if (!/^[0-9]+$/.test(draftId)) throw new Error("Recovery needs an exact numeric draft id."); + if (reservationTag && !/^pylon-stable-sequence-[0-9]{6}$/.test(reservationTag)) throw new Error("Recovery reservation tag is malformed."); + if (!/^pylon-build-g[0-9a-f]{12}-r[1-9][0-9]*$/.test(previewTag) || !["promote", "withdraw"].includes(operation)) { + throw new Error("Recovery preview or operation is malformed."); + } + return { + draftId: Number(draftId), reservationTag, previewTag, operation, + revokeTag: values.get("--revoke-tag") ?? "", reason: values.get("--reason") ?? "", + outDir: resolve(root, values.get("--out-dir") ?? ".npm/pylon-stable/output"), + }; +} + +function headers(accept = "application/vnd.github+json") { + const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN; + if (!token) throw new Error("GITHUB_TOKEN is required for stable recovery."); + return { Accept: accept, Authorization: `Bearer ${token}`, "X-GitHub-Api-Version": "2022-11-28", "User-Agent": "pylon-stable-recovery" }; +} + +async function api(path, { bytes = false, accept } = {}) { + const response = await fetch(`https://api.github.com${path}`, { + headers: headers(accept), redirect: "follow", signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) { + const error = new Error(`GitHub API ${path} failed with ${response.status}.`); + error.status = response.status; + throw error; + } + return bytes ? Buffer.from(await response.arrayBuffer()) : response.json(); +} + +function outputs(values) { + if (!process.env.GITHUB_OUTPUT) return; + for (const [key, value] of Object.entries(values)) appendFileSync(process.env.GITHUB_OUTPUT, `${key}=${value}\n`); +} + +export async function recoverStableDraft(args, dependencies = {}) { + const readApi = dependencies.api ?? api; + const readHistory = dependencies.readHistory ?? readStableHistory; + const verifyAttestation = dependencies.verifyAttestation ?? verifyStableAttestation; + const release = await readApi(`/repos/${PYLON_PUBLICATION_REPOSITORY}/releases/${args.draftId}`); + if (!release.draft || release.immutable === true || !Array.isArray(release.assets) || release.assets.length > 1) { + throw new Error("Recovery release is not one exact mutable stable draft."); + } + const recovered = stableManifestBytesFromReleaseBody(release.body); + const { bytes, digest, manifest } = recovered; + const asset = release.assets[0]; + if (asset) { + if (asset.name !== PYLON_STABLE_MANIFEST || !asset.url) throw new Error("Recovery draft has an unexpected singleton asset."); + const downloaded = await readApi(new URL(asset.url).pathname, { bytes: true, accept: "application/octet-stream" }); + if ( + !downloaded.equals(bytes) || asset.size !== bytes.length || asset.digest !== `sha256:${digest}` || + sha256Bytes(downloaded) !== digest + ) throw new Error("Recovery draft asset differs from the exact body-carried attested bytes."); + } + const name = `Pylon Prime stable ${manifest.tag}`; + const body = publicationReleaseBody({ + channel: "stable", tag: manifest.tag, source: manifest.build.source.commit, tree: manifest.build.source.tree, + recipeRevision: manifest.build.recipeRevision, policyCommit: manifest.promotion.policyCommit, policyTree: manifest.promotion.policyTree, + stableManifestBytes: bytes, + }); + if ( + release.tag_name !== manifest.tag || release.name !== name || release.body !== body || release.prerelease || + release.target_commitish !== manifest.promotion.policyCommit || manifest.build.previewTag !== args.previewTag || + manifest.promotion.kind !== args.operation + ) throw new Error("Recovery draft metadata, bytes, preview, or operation differs."); + if (args.operation === "withdraw") { + if (manifest.promotion.revocation?.stableTag !== args.revokeTag || manifest.promotion.revocation?.reason !== args.reason) { + throw new Error("Recovery withdrawal inputs differ from the approved draft."); + } + } else if (args.revokeTag || args.reason) throw new Error("Promote recovery cannot carry withdrawal inputs."); + const history = await readHistory({ verifyAllAttestations: true, excludeDraftId: release.id, excludeDraftTag: manifest.tag }); + const combined = validateStableHistory([...history, manifest]); + if (combined.length !== manifest.sequence || combined.at(-1).tag !== manifest.tag) { + throw new Error("Recovery draft is not the exact next signed-history sequence."); + } + const expectedReservation = `pylon-stable-sequence-${String(manifest.sequence).padStart(6, "0")}`; + if (args.reservationTag) { + if (args.reservationTag !== expectedReservation) throw new Error("Recovery reservation sequence differs from the draft."); + const ref = await readApi(`/repos/${PYLON_PUBLICATION_REPOSITORY}/git/ref/tags/${args.reservationTag}`); + if (ref.object?.type !== "tag") throw new Error("Recovery reservation is not annotated."); + const annotation = await readApi(`/repos/${PYLON_PUBLICATION_REPOSITORY}/git/tags/${ref.object.sha}`); + if ( + annotation.tag !== args.reservationTag || annotation.message !== stableReservationMessage(manifest, digest, release.id) || + annotation.object?.type !== "commit" || annotation.object.sha !== manifest.promotion.policyCommit + ) throw new Error("Recovery reservation differs from the exact approved draft."); + } else { + try { + await readApi(`/repos/${PYLON_PUBLICATION_REPOSITORY}/git/ref/tags/${expectedReservation}`); + throw new Error("Draft-only recovery cannot replace an existing reservation."); + } catch (error) { + if (error.status !== 404) throw error; + } + } + mkdirSync(args.outDir, { recursive: true }); + const manifestPath = join(args.outDir, PYLON_STABLE_MANIFEST); + writeFileSync(manifestPath, bytes, { mode: 0o600 }); + verifyAttestation(manifestPath, manifest.promotion.policyCommit, manifest.promotion.policyTree); + return { release, manifest, bytes, digest, reservationTag: expectedReservation }; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + try { + const args = parseArgs(process.argv.slice(2)); + const recovered = await recoverStableDraft(args); + outputs({ + publish: "true", manifest_sha256: recovered.digest, tag: recovered.manifest.tag, source_sha: recovered.manifest.build.source.commit, + source_tree: recovered.manifest.build.source.tree, sequence: String(recovered.manifest.sequence), + draft_id: String(recovered.release.id), reservation_tag: recovered.reservationTag, + }); + console.log(JSON.stringify({ recovered: recovered.manifest.tag, draftId: recovered.release.id })); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/rotate-pylon-consumer-journal.mjs b/scripts/rotate-pylon-consumer-journal.mjs new file mode 100644 index 0000000000..dc4d765fa8 --- /dev/null +++ b/scripts/rotate-pylon-consumer-journal.mjs @@ -0,0 +1,20 @@ +#!/usr/bin/env node + +import { resolve } from "node:path"; + +import { rotateConsumerStateJournal } from "./lib/pylon-consumer-lock.mjs"; + +function parseArgs(args) { + if (args.length !== 2 || args[0] !== "--state" || !args[1] || args[1].startsWith("--")) { + throw new Error("Usage: rotate-pylon-consumer-journal --state "); + } + return resolve(args[1]); +} + +try { + const result = await rotateConsumerStateJournal(parseArgs(process.argv.slice(2))); + console.log(JSON.stringify({ journalEpoch: result.epoch, tipSha256: result.tipSha256 })); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} diff --git a/scripts/smoke-pylon-prime-agent-release.mjs b/scripts/smoke-pylon-prime-agent-release.mjs index 303610f0c6..5f8506767f 100644 --- a/scripts/smoke-pylon-prime-agent-release.mjs +++ b/scripts/smoke-pylon-prime-agent-release.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node import { spawn, spawnSync } from "node:child_process"; -import { randomUUID } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; @@ -13,6 +12,7 @@ import { run, } from "./lib/pylon-release.mjs"; import { verifyPylonPrimeAgentRelease } from "./verify-pylon-prime-agent-release.mjs"; +import { verifyPreviewPublication } from "./verify-pylon-preview-publication.mjs"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const defaultArtifacts = join(root, ".npm", "pylon-release", "artifacts"); @@ -24,13 +24,18 @@ export const PYLON_RELEASE_EXPECTED_SDK_FEATURES = Object.freeze([ ]); function parseArgs(args) { - if (args.length === 0) return defaultArtifacts; - if (args.length === 2 && args[0] === "--artifact-dir") return resolve(root, args[1]); - throw new Error("Usage: node scripts/smoke-pylon-prime-agent-release.mjs [--artifact-dir path]"); + const historicalIndex = args.indexOf("--historical"); + const historical = historicalIndex !== -1; + const remaining = args.filter((_, index) => index !== historicalIndex); + if (remaining.length === 0) return { artifactsDir: defaultArtifacts, historical }; + if (remaining.length === 2 && remaining[0] === "--artifact-dir") { + return { artifactsDir: resolve(root, remaining[1]), historical }; + } + throw new Error("Usage: smoke-pylon-prime-agent-release [--historical] [--artifact-dir path]"); } -export function releaseInstallTimeoutMs(platform = process.platform) { - return platform === "win32" ? 360_000 : 180_000; +export function releaseInstallTimeoutMs() { + return 180_000; } function runCli(command, args, options = {}) { @@ -104,21 +109,6 @@ function isProcessAlive(pid) { function getProcessStartId(pid) { if (!Number.isSafeInteger(pid) || pid <= 1) return undefined; - if (process.platform === "win32") { - const result = runCli( - "powershell.exe", - [ - "-NoLogo", - "-NoProfile", - "-NonInteractive", - "-Command", - `([System.Diagnostics.Process]::GetProcessById(${pid})).StartTime.ToUniversalTime().Ticks`, - ], - { timeoutMs: 2_000, maxBuffer: 4096 }, - ); - const ticks = result.status === 0 && !result.error ? result.stdout.trim() : ""; - return /^\d+$/.test(ticks) ? `win:${ticks}` : undefined; - } try { const stat = readFileSync(`/proc/${pid}/stat`, "utf8"); const commandEnd = stat.lastIndexOf(")"); @@ -627,201 +617,6 @@ console.log(JSON.stringify({ connected })); } if (failure) throw failure; } -async function smokeWindowsAcp({ probesDir, fixture }) { - const runnerPath = join(probesDir, "acp-runner.mjs"); - writeFileSync( - runnerPath, - `import { main } from "prime-agent"; -await main(process.argv.slice(2), { extensionFactories: [() => {}] }); -`, - ); - const pipe = `\\\\.\\pipe\\pylon-prime-artifact-${process.pid}-${randomUUID()}`; - const child = spawn( - process.execPath, - [ - runnerPath, - "--mode", - "acp", - "--provider", - "artifact-faux", - "--model", - "artifact-faux", - "--no-session", - "--no-tools", - "--no-extensions", - "--no-skills", - "--no-prompt-templates", - "--no-themes", - "--no-context-files", - "--offline", - "--daemon-socket", - pipe, - ], - { cwd: fixture.projectDir, env: fixture.env, stdio: ["pipe", "pipe", "pipe"], shell: false, detached: false }, - ); - const output = diagnostics(child, 1024 * 1024); - let buffer = ""; - let response; - let resolveResponse; - let rejectResponse; - const responsePromise = new Promise((resolveValue, rejectValue) => { - resolveResponse = resolveValue; - rejectResponse = rejectValue; - }); - child.stdout.on("data", (chunk) => { - buffer += chunk.toString("utf8"); - if (Buffer.byteLength(buffer) > 1024 * 1024) { - rejectResponse(new Error("ACP stdout exceeded 1 MiB.")); - return; - } - let newline; - while ((newline = buffer.indexOf("\n")) >= 0) { - const line = buffer.slice(0, newline).trim(); - buffer = buffer.slice(newline + 1); - if (!line) continue; - try { - const frame = JSON.parse(line); - if (frame.id === 1) { - response = frame; - resolveResponse(frame); - } - } catch { - rejectResponse(new Error("ACP emitted non-JSON stdout.")); - } - } - }); - child.once("close", () => { - if (!response) rejectResponse(new Error("ACP exited before initialize response.")); - }); - const timer = setTimeout(() => rejectResponse(new Error("ACP initialize timed out.")), 45_000); - let failure; - try { - child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", id: 1, method: "initialize", params: { protocolVersion: 1, clientCapabilities: {} } })}\n`); - const frame = await responsePromise; - clearTimeout(timer); - if ( - frame.jsonrpc !== "2.0" || - frame.error !== undefined || - frame.result?.protocolVersion !== 1 || - frame.result?.agentInfo?.name !== "prime-agent" - ) { - throw new Error(`Unexpected ACP initialize response: ${JSON.stringify(frame)}`); - } - child.stdin.end(); - if (!(await waitForExit(child, 10_000))) throw new Error("ACP did not exit after input EOF."); - if (child.exitCode !== 0 || output.overflow || output.error) { - throw new Error(`Installed ACP fallback failed.\n${output.stderr || output.stdout}`); - } - } catch (error) { - failure = error; - } finally { - clearTimeout(timer); - try { - await terminateCapturedChild(child); - } catch (terminationError) { - failure = failure ?? terminationError; - } - } - - const cleanupReceiptPath = join(probesDir, "unexpected-daemon.json"); - const cleanupPath = join(probesDir, "acp-daemon-cleanup.mjs"); - writeFileSync( - cleanupPath, - `${staticSdkImport()}import { writeFileSync } from "node:fs"; -const [pipe, receiptPath] = process.argv.slice(2); -const client = new sdk.DaemonClient(pipe); -let connected = false; -try { - await client.connect(500); - await client.waitForHello(500); - connected = true; - writeFileSync(receiptPath, JSON.stringify({ - supervisorPid: client.hello?.supervisorPid, - supervisorProcessStartId: client.hello?.supervisorProcessStartId - })); - const shutdown = await client.request({ type: "shutdown", force: true }, 10_000); - if (!shutdown.success) throw new Error(shutdown.error); -} catch (error) { - if (connected) throw error; -} finally { - client.close(); -} -console.log(JSON.stringify({ connected })); -`, - ); - const beforeCleanup = recordedWorkerIdentities(fixture.agentDir); - try { - const cleanup = spawn(process.execPath, [cleanupPath, pipe, cleanupReceiptPath], { - cwd: probesDir, - env: fixture.env, - stdio: ["ignore", "pipe", "pipe"], - shell: false, - detached: false, - }); - const cleanupResult = await collectChild(cleanup, 15_000); - if (cleanupResult.status !== 0) { - throw new Error(`Unexpected ACP daemon cleanup failed.\n${cleanupResult.stderr || cleanupResult.stdout}`); - } - const cleanupReceipt = JSON.parse(cleanupResult.stdout.trim().split("\n").at(-1)); - if (cleanupReceipt.connected === true) { - failure = failure ?? new Error("ACP fallback unexpectedly launched a detached daemon."); - } - } catch (cleanupError) { - failure = failure ?? cleanupError; - } finally { - try { - if (existsSync(cleanupReceiptPath)) { - const supervisor = JSON.parse(readFileSync(cleanupReceiptPath, "utf8")); - if ( - Number.isSafeInteger(supervisor.supervisorPid) && - supervisor.supervisorPid > 1 && - typeof supervisor.supervisorProcessStartId === "string" - ) { - const identity = { - pid: supervisor.supervisorPid, - processStartId: supervisor.supervisorProcessStartId, - }; - const state = await waitForTrackedIdentityChange(identity, 10_000); - if (state === "current") await terminateTrackedIdentity(identity); - else if (state === "unknown") { - throw unsafeCleanupError( - `Could not prove the start identity of unexpected ACP daemon ${identity.pid}.`, - ); - } - } else if ( - Number.isSafeInteger(supervisor.supervisorPid) && - supervisor.supervisorPid > 1 && - isProcessAlive(supervisor.supervisorPid) - ) { - throw unsafeCleanupError( - `Unexpected ACP daemon ${supervisor.supervisorPid} lacked a start identity; refusing to signal it.`, - ); - } - } - const afterCleanup = recordedWorkerIdentities(fixture.agentDir); - const identities = new Map( - [...beforeCleanup.identities, ...afterCleanup.identities].map((identity) => [ - `${identity.pid}:${identity.processStartId}`, - identity, - ]), - ); - for (const identity of identities.values()) await terminateTrackedIdentity(identity); - for (const pid of [...beforeCleanup.unprovedPids, ...afterCleanup.unprovedPids]) { - if (isProcessAlive(pid)) { - throw unsafeCleanupError(`Worker ${pid} remained alive without a start identity; refusing to signal it.`); - } - } - } catch (terminationError) { - if (terminationError && typeof terminationError === "object" && terminationError.preserveTempRoot === true) { - if (failure) terminationError.message = `${failure.message}\nCleanup failure: ${terminationError.message}`; - failure = terminationError; - } else { - failure = failure ?? terminationError; - } - } - } - if (failure) throw failure; -} function createLocalAssetConsumer(prefix, artifactsDir, manifest) { const bySourceName = new Map( PYLON_RELEASE_PACKAGES.map((releasePackage) => [ @@ -838,8 +633,13 @@ function createLocalAssetConsumer(prefix, artifactsDir, manifest) { ); } -export async function smokePylonPrimeAgentRelease(artifactsDir) { - const manifest = verifyPylonPrimeAgentRelease(artifactsDir); +export async function smokePylonPrimeAgentRelease(artifactsDir, { historical = false } = {}) { + if (process.platform === "win32") { + throw new Error("Native Windows artifact runtime verification is deferred; use the supported WSL2/Linux path."); + } + const manifest = historical + ? verifyPreviewPublication(artifactsDir, { historical: true }).releaseManifest + : verifyPylonPrimeAgentRelease(artifactsDir); const tempRoot = mkdtempSync(join(tmpdir(), "pylon-prime-release-")); let removeTempRoot = true; try { @@ -898,11 +698,7 @@ export async function smokePylonPrimeAgentRelease(artifactsDir) { throw new Error("Pylon artifact did not block the stock self-updater with Pylon release guidance."); } - if (process.platform === "win32" || process.env.PYLON_RELEASE_SMOKE_TEST_FORCE_ACP === "1") { - await smokeWindowsAcp({ probesDir, fixture }); - } else { - await smokePosixDaemon({ tempRoot, probesDir, cliEntry, fixture, expectedBuildId: manifest.build.id }); - } + await smokePosixDaemon({ tempRoot, probesDir, cliEntry, fixture, expectedBuildId: manifest.build.id }); console.log(`Installed and verified ${manifest.build.id} on ${process.platform}.`); return manifest; } catch (error) { @@ -918,7 +714,8 @@ export async function smokePylonPrimeAgentRelease(artifactsDir) { if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { try { - await smokePylonPrimeAgentRelease(parseArgs(process.argv.slice(2))); + const args = parseArgs(process.argv.slice(2)); + await smokePylonPrimeAgentRelease(args.artifactsDir, { historical: args.historical }); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exit(1); diff --git a/scripts/verify-pylon-preview-history.mjs b/scripts/verify-pylon-preview-history.mjs new file mode 100644 index 0000000000..8156c23f48 --- /dev/null +++ b/scripts/verify-pylon-preview-history.mjs @@ -0,0 +1,140 @@ +#!/usr/bin/env node + +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + readBoundedRegularFile, +} from "./lib/pylon-bounded-file.mjs"; +import { + canonicalJson, + parsePreviewTag, + PYLON_PREVIEW_MANIFEST, + sha256Bytes, +} from "./lib/pylon-publication.mjs"; +import { PYLON_RELEASE_REPOSITORY } from "./lib/pylon-release.mjs"; +import { withConsumerStateLock } from "./lib/pylon-consumer-lock.mjs"; +import { verifyPreviewAttestations } from "./verify-pylon-publication-attestations.mjs"; + +const STATE_SCHEMA_VERSION = 1; +const STATE_MAX_BYTES = 4 * 1024; +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +function exactKeys(value, keys) { + return value !== null && typeof value === "object" && !Array.isArray(value) && + Object.keys(value).sort().join(",") === [...keys].sort().join(","); +} + +function validateState(state) { + if ( + !exactKeys(state, ["schemaVersion", "repository", "channel", "sequenceEpoch", "highWater"]) || + state.schemaVersion !== STATE_SCHEMA_VERSION || state.repository !== PYLON_RELEASE_REPOSITORY || + state.channel !== "preview" || state.sequenceEpoch !== 1 || + !exactKeys(state.highWater, ["sequence", "tag", "sha256", "workflowRunId"]) || + !Number.isSafeInteger(state.highWater.sequence) || state.highWater.sequence < 1 || + parsePreviewTag(state.highWater.tag).recipeRevision < 1 || !/^[0-9a-f]{64}$/.test(state.highWater.sha256 ?? "") || + !/^[1-9][0-9]*$/.test(state.highWater.workflowRunId ?? "") + ) throw new Error("Consumer preview high-water state is malformed."); + return state; +} + +function readState(bytes) { + if (!Buffer.isBuffer(bytes) || bytes.length < 1 || bytes.length > STATE_MAX_BYTES) { + throw new Error("Consumer preview high-water state is malformed."); + } + const state = validateState(JSON.parse(bytes)); + if (bytes.toString("utf8") !== canonicalJson(state)) throw new Error("Consumer preview high-water state is not canonical JSON."); + return state; +} + +export async function recordPreviewHighWater(previewManifest, previewBytes, { statePath, initialize = false }) { + if (typeof statePath !== "string" || !statePath) throw new Error("A consumer-local --state path is required."); + if (!Buffer.isBuffer(previewBytes) || previewBytes.toString("utf8") !== canonicalJson(previewManifest)) { + throw new Error("Preview high-water requires exact canonical verified manifest bytes."); + } + if ( + previewManifest.sequenceEpoch !== 1 || !Number.isSafeInteger(previewManifest.sequence) || previewManifest.sequence < 1 || + !/^[1-9][0-9]*$/.test(previewManifest.workflowRunId ?? "") + ) throw new Error("Verified preview has a malformed monotonic sequence identity."); + const path = resolve(statePath); + const highWater = { + sequence: previewManifest.sequence, + tag: previewManifest.build.tag, + sha256: sha256Bytes(previewBytes), + workflowRunId: previewManifest.workflowRunId, + }; + return withConsumerStateLock(path, async (_lockedPath, transaction) => { + const priorBytes = transaction.readStateBytes(); + if (priorBytes === null && !initialize) throw new Error("No consumer preview high-water exists. Verify the release, then use --initialize once."); + if (priorBytes !== null && initialize) throw new Error("Consumer preview high-water already exists; --initialize cannot reset it."); + const prior = priorBytes === null ? null : readState(priorBytes); + if (prior) { + if (prior.sequenceEpoch !== previewManifest.sequenceEpoch) throw new Error("Preview sequence epoch changed without a new signed state schema."); + if (highWater.sequence < prior.highWater.sequence) throw new Error("Verified preview is older than the consumer high-water sequence."); + if (highWater.sequence === prior.highWater.sequence) { + if (canonicalJson(highWater) !== canonicalJson(prior.highWater)) { + throw new Error("Verified preview equivocates at the consumer high-water sequence."); + } + return { state: prior, advanced: false }; + } + } + const state = { + schemaVersion: STATE_SCHEMA_VERSION, + repository: PYLON_RELEASE_REPOSITORY, + channel: "preview", + sequenceEpoch: previewManifest.sequenceEpoch, + highWater, + }; + await transaction.commitState(Buffer.from(canonicalJson(state))); + return { state, advanced: true }; + }, { stateMaxBytes: STATE_MAX_BYTES }); +} + +function parseArgs(args) { + const remaining = [...args]; + const flag = (name) => { + const index = remaining.indexOf(name); + if (index === -1) return false; + remaining.splice(index, 1); + return true; + }; + const initialize = flag("--initialize"); + const historical = flag("--historical"); + const value = (name, fallback) => { + const index = remaining.indexOf(name); + if (index === -1) return fallback; + const result = remaining[index + 1]; + if (!result || result.startsWith("--")) throw new Error(`Missing value for ${name}.`); + remaining.splice(index, 2); + return result; + }; + const statePath = value("--state", ""); + const artifactDir = resolve(root, value("--artifact-dir", ".npm/pylon-release/artifacts")); + if (remaining.length > 0 || !statePath) throw new Error("Usage: verify-pylon-preview-history --state [--initialize] [--historical] [--artifact-dir path]"); + return { statePath, artifactDir, initialize, historical }; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + try { + const args = parseArgs(process.argv.slice(2)); + const previewPath = join(args.artifactDir, PYLON_PREVIEW_MANIFEST); + const previewBytes = await readBoundedRegularFile(previewPath, { + maxBytes: PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + description: "Preview manifest", + }); + if (previewBytes === null) throw new Error("Preview manifest does not exist."); + const untrusted = JSON.parse(previewBytes); + const verified = verifyPreviewAttestations({ + artifactDir: args.artifactDir, + sourceSha: untrusted.build?.source?.commit ?? "", + sourceTree: untrusted.build?.source?.tree ?? "", + historical: args.historical, + }); + const result = await recordPreviewHighWater(verified.previewManifest, previewBytes, args); + console.log(JSON.stringify({ highWater: result.state.highWater, advanced: result.advanced })); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/verify-pylon-preview-publication.mjs b/scripts/verify-pylon-preview-publication.mjs new file mode 100644 index 0000000000..5b0c0741bd --- /dev/null +++ b/scripts/verify-pylon-preview-publication.mjs @@ -0,0 +1,107 @@ +#!/usr/bin/env node + +import { lstatSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + readBoundedRegularFileSync, +} from "./lib/pylon-bounded-file.mjs"; +import { + hashBytes, + PYLON_RELEASE_MANIFEST, + validateReleaseManifest, +} from "./lib/pylon-release.mjs"; +import { + canonicalJson, + PYLON_PREVIEW_MANIFEST, + sha256Bytes, + validatePreviewManifest, + validatePublishedReleaseManifest, +} from "./lib/pylon-publication.mjs"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const defaultArtifacts = join(root, ".npm", "pylon-release", "artifacts"); + +function parseArgs(args) { + const historicalIndex = args.indexOf("--historical"); + const historical = historicalIndex !== -1; + const remaining = args.filter((_, index) => index !== historicalIndex); + if (remaining.length === 0) return { artifactsDir: defaultArtifacts, historical }; + if (remaining.length === 2 && remaining[0] === "--artifact-dir") { + return { artifactsDir: resolve(root, remaining[1]), historical }; + } + throw new Error("Usage: node scripts/verify-pylon-preview-publication.mjs [--historical] [--artifact-dir path]"); +} + +export function verifyPreviewPublication(artifactsDir, { historical = false } = {}) { + const releaseBytes = readBoundedRegularFileSync(join(artifactsDir, PYLON_RELEASE_MANIFEST), { + maxBytes: PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + description: "Release manifest", + }); + if (releaseBytes === null) throw new Error("Release manifest does not exist."); + const releaseManifest = JSON.parse(releaseBytes); + if (historical) validatePublishedReleaseManifest(releaseManifest); + else validateReleaseManifest(releaseManifest); + const previewBytes = readBoundedRegularFileSync(join(artifactsDir, PYLON_PREVIEW_MANIFEST), { + maxBytes: PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + description: "Preview manifest", + }); + if (previewBytes === null) throw new Error("Preview manifest does not exist."); + const previewManifest = JSON.parse(previewBytes); + if (canonicalJson(previewManifest) !== previewBytes.toString("utf8")) { + throw new Error("Preview manifest is not canonical publication JSON."); + } + validatePreviewManifest(previewManifest, releaseManifest, releaseBytes, { historical }); + const expectedFiles = new Set([ + PYLON_RELEASE_MANIFEST, + PYLON_PREVIEW_MANIFEST, + ...releaseManifest.assets.map((asset) => asset.file), + ]); + for (const file of readdirSync(artifactsDir)) { + if (!expectedFiles.delete(file)) throw new Error(`Unexpected preview publication subject: ${file}`); + if (!lstatSync(join(artifactsDir, file)).isFile()) { + throw new Error(`Preview publication subject is not one regular file: ${file}`); + } + } + if (expectedFiles.size > 0) throw new Error(`Missing preview publication subject: ${[...expectedFiles].join(", ")}`); + for (const asset of releaseManifest.assets) { + const bytes = readFileSync(join(artifactsDir, asset.file)); + if ( + statSync(join(artifactsDir, asset.file)).size !== asset.size || + hashBytes(bytes, "sha256") !== asset.sha256 || + hashBytes(bytes, "sha512") !== asset.sha512 + ) { + throw new Error(`Preview artifact digest mismatch for ${asset.file}.`); + } + } + return { + releaseManifest, + previewManifest, + subjects: [ + ...releaseManifest.assets.map((asset) => ({ name: asset.file, sha256: asset.sha256 })), + { name: PYLON_RELEASE_MANIFEST, sha256: sha256Bytes(releaseBytes) }, + { name: PYLON_PREVIEW_MANIFEST, sha256: sha256Bytes(previewBytes) }, + ].sort((left, right) => left.name.localeCompare(right.name)), + }; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + try { + const args = parseArgs(process.argv.slice(2)); + const verified = verifyPreviewPublication(args.artifactsDir, { historical: args.historical }); + console.log( + JSON.stringify({ + tag: verified.previewManifest.build.tag, + source: verified.previewManifest.build.source, + recipeRevision: verified.previewManifest.build.recipeRevision, + publicationPolicyRevision: verified.previewManifest.publicationPolicyRevision, + subjects: verified.subjects, + }), + ); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/verify-pylon-publication-attestations.mjs b/scripts/verify-pylon-publication-attestations.mjs new file mode 100644 index 0000000000..92be1b76a4 --- /dev/null +++ b/scripts/verify-pylon-publication-attestations.mjs @@ -0,0 +1,250 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + canonicalJson, + PYLON_PREVIEW_WORKFLOW, + PYLON_PUBLICATION_REF, + PYLON_PUBLICATION_REPOSITORY, +} from "./lib/pylon-publication.mjs"; +import { verifyApprovedWorkflowAtSignerDigest } from "./lib/pylon-workflow-policy.mjs"; +import { verifyPreviewPublication } from "./verify-pylon-preview-publication.mjs"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +function parseArgs(args) { + const historicalIndex = args.indexOf("--historical"); + const historical = historicalIndex !== -1; + const remaining = args.filter((_, index) => index !== historicalIndex); + const values = new Map(); + for (let index = 0; index < remaining.length; index += 2) { + const name = remaining[index]; + const value = remaining[index + 1]; + if (!name?.startsWith("--") || value === undefined) throw new Error("Attestation verification arguments must be name/value pairs."); + values.set(name, value); + } + const artifactDir = resolve(root, values.get("--artifact-dir") ?? ".npm/pylon-stable/preview"); + const sourceSha = values.get("--source-sha") ?? ""; + const sourceTree = values.get("--source-tree") ?? ""; + if (!/^[0-9a-f]{40}$/.test(sourceSha)) throw new Error("--source-sha must be a full lowercase Git SHA."); + if (!/^[0-9a-f]{40}$/.test(sourceTree)) throw new Error("--source-tree must be a full lowercase Git tree SHA."); + return { artifactDir, sourceSha, sourceTree, historical }; +} + +export function verifyGhAttestationResult(output, expectedSubjects, expectedInvocation) { + const expected = Array.isArray(expectedSubjects) ? expectedSubjects : [expectedSubjects]; + if (expected.length === 0) throw new Error("Attestation policy needs at least one exact subject."); + if ( + !expectedInvocation || expectedInvocation.repository !== PYLON_PUBLICATION_REPOSITORY || + !["push", "workflow_dispatch"].includes(expectedInvocation.event) || + !new RegExp("^[0-9a-f]{40}$").test(expectedInvocation.sourceSha ?? "") || + !/^\.github\/workflows\/[A-Za-z0-9._-]+\.yml$/.test(expectedInvocation.workflow ?? "") || + (expectedInvocation.workflowRunId !== undefined && !/^[1-9][0-9]*$/.test(expectedInvocation.workflowRunId)) + ) throw new Error("Attestation policy needs one exact canonical workflow invocation."); + const expectedSet = expected + .map((subject) => ({ name: subject.name, digest: { sha256: subject.sha256 } })) + .sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0); + if (new Set(expectedSet.map((subject) => subject.name)).size !== expectedSet.length) { + throw new Error("Expected attestation subject set contains a duplicate name."); + } + const results = JSON.parse(output.replace(/\u001b\[[0-9;]*m/g, "")); + if (!Array.isArray(results) || results.length === 0) throw new Error("No verified attestation for the exact subject set."); + const verifiedAttempts = new Map(); + for (const entry of results) { + const verification = entry.verificationResult; + const statement = verification?.statement; + const subjects = statement?.subject; + if (statement?.predicateType !== "https://slsa.dev/provenance/v1" || !Array.isArray(subjects)) { + throw new Error("Attestation predicate is not exact SLSA provenance."); + } + const predicate = statement.predicate; + const definition = predicate?.buildDefinition; + const workflow = definition?.externalParameters?.workflow; + const github = definition?.internalParameters?.github; + const dependency = definition?.resolvedDependencies; + const invocationId = predicate?.runDetails?.metadata?.invocationId; + const invocation = new RegExp(`^https://github\\.com/${expectedInvocation.repository}/actions/runs/([1-9][0-9]*)/attempts/([1-9][0-9]*)$`).exec(invocationId ?? ""); + if ( + definition?.buildType !== "https://actions.github.io/buildtypes/workflow/v1" || + workflow?.repository !== `https://github.com/${expectedInvocation.repository}` || + workflow?.path !== expectedInvocation.workflow || workflow?.ref !== PYLON_PUBLICATION_REF || + github?.event_name !== expectedInvocation.event || String(github?.repository_id) !== "1349002285" || + github?.runner_environment !== "github-hosted" || !/^[1-9][0-9]*$/.test(String(github?.repository_owner_id ?? "")) || + !Array.isArray(dependency) || dependency.length !== 1 || + dependency[0]?.uri !== `git+https://github.com/${expectedInvocation.repository}@${PYLON_PUBLICATION_REF}` || + dependency[0]?.digest?.gitCommit !== expectedInvocation.sourceSha || + predicate?.runDetails?.builder?.id !== `https://github.com/${expectedInvocation.repository}/${expectedInvocation.workflow}@${PYLON_PUBLICATION_REF}` || + !invocation || (expectedInvocation.workflowRunId !== undefined && invocation[1] !== expectedInvocation.workflowRunId) + ) throw new Error("Attestation SLSA invocation does not bind the exact workflow run and source."); + verifiedAttempts.set(`${invocation[1]}:${invocation[2]}`, { runId: invocation[1], runAttempt: invocation[2] }); + const actualSet = subjects + .map((subject) => { + if ( + !subject || Object.keys(subject).sort().join(",") !== "digest,name" || + typeof subject.name !== "string" || + !subject.digest || Object.keys(subject.digest).join(",") !== "sha256" || + !/^[0-9a-f]{64}$/.test(subject.digest.sha256) + ) throw new Error("Attestation statement contains a malformed subject."); + return { name: subject.name, digest: { sha256: subject.digest.sha256 } }; + }) + .sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0); + if ( + new Set(actualSet.map((subject) => subject.name)).size !== actualSet.length || + canonicalJson(actualSet) !== canonicalJson(expectedSet) + ) throw new Error("Attestation statement subject set has an extra, missing, duplicate, or changed subject."); + const hasRekor = Array.isArray(verification?.verifiedTimestamps) && verification.verifiedTimestamps.some( + (timestamp) => timestamp.type === "Tlog" && /^https:\/\/rekor\.sigstore\.dev(?:\/|$)/.test(timestamp.uri ?? ""), + ); + if (!hasRekor) throw new Error("Attestation lacks Sigstore public-good Rekor evidence."); + } + return [...verifiedAttempts.values()].sort((left, right) => Number(left.runId) - Number(right.runId) || Number(left.runAttempt) - Number(right.runAttempt)); +} + +function verifySubject(path, subject, allSubjects, sourceSha, expectedInvocation) { + const result = spawnSync( + "gh", + [ + "attestation", + "verify", + path, + "--repo", + PYLON_PUBLICATION_REPOSITORY, + "--cert-identity", + `https://github.com/${PYLON_PUBLICATION_REPOSITORY}/${PYLON_PREVIEW_WORKFLOW}@${PYLON_PUBLICATION_REF}`, + "--signer-digest", + sourceSha, + "--source-ref", + PYLON_PUBLICATION_REF, + "--source-digest", + sourceSha, + "--cert-oidc-issuer", + "https://token.actions.githubusercontent.com", + "--predicate-type", + "https://slsa.dev/provenance/v1", + "--deny-self-hosted-runners", + "--limit", + "100", + "--format", + "json", + ], + { encoding: "utf8", timeout: 120_000, maxBuffer: 16 * 1024 * 1024 }, + ); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(`gh attestation verify failed for ${subject.name}: ${result.stderr}`); + return verifyGhAttestationResult(result.stdout, allSubjects, expectedInvocation); +} + +function ghJson(path) { + const token = process.env.GH_TOKEN ?? process.env.GITHUB_TOKEN; + if (!token) throw new Error("GH_TOKEN or GITHUB_TOKEN is required to verify workflow-run sequence evidence."); + const result = spawnSync("gh", ["api", path], { + encoding: "utf8", timeout: 120_000, maxBuffer: 16 * 1024 * 1024, + env: { ...process.env, GH_TOKEN: token }, + }); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(`Could not verify workflow-run evidence: ${result.stderr}`); + return JSON.parse(result.stdout.replace(/\u001b\[[0-9;]*m/g, "")); +} + +export function validatePreviewWorkflowRunEvidence({ attempts }, previewManifest, attestedAttempts) { + const expected = { + repository: PYLON_PUBLICATION_REPOSITORY, + workflow: PYLON_PREVIEW_WORKFLOW, + event: "push", + sourceSha: previewManifest.build.source.commit, + workflowRunId: previewManifest.workflowRunId, + }; + if (!Array.isArray(attestedAttempts) || attestedAttempts.length === 0 || !Array.isArray(attempts)) { + throw new Error("Preview attestation has no exact workflow attempt evidence."); + } + const keys = attestedAttempts.map(({ runId, runAttempt }) => `${runId}:${runAttempt}`).sort(); + if (new Set(keys).size !== keys.length || attempts.length !== keys.length) { + throw new Error("Preview workflow attempt evidence is duplicate, missing, or ambiguous."); + } + const encountered = new Set(); + for (const evidence of attempts) { + const { run, suite, jobs } = evidence; + const key = `${run?.id}:${run?.run_attempt}`; + encountered.add(key); + if ( + !keys.includes(key) || String(run.id) !== expected.workflowRunId || run.run_number !== previewManifest.sequence || + run.event !== expected.event || run.head_branch !== "pylon" || run.head_sha !== expected.sourceSha || + run.path !== expected.workflow || run.repository?.id !== 1_349_002_285 || run.repository?.full_name !== expected.repository || + run.head_repository?.id !== 1_349_002_285 || run.head_repository?.full_name !== expected.repository || + !Number.isSafeInteger(run.run_attempt) || run.run_attempt < 1 || !run.check_suite_id || + !["in_progress", "completed"].includes(run.status) + ) throw new Error("Preview sequence does not match the exact signed canonical workflow attempt."); + if (suite.app?.id !== 15368 || suite.head_sha !== expected.sourceSha || suite.id !== run.check_suite_id) { + throw new Error("Preview workflow attempt is not owned by the GitHub Actions app on the exact source."); + } + if (!Array.isArray(jobs) || jobs.some((job) => job.run_attempt !== run.run_attempt)) { + throw new Error("Preview workflow jobs are not from the exact signed attempt."); + } + const attesters = jobs.filter((job) => job.name === "Approve and attest six preview subjects"); + if (attesters.length !== 1 || attesters[0].status !== "completed" || attesters[0].conclusion !== "success") { + throw new Error("Preview workflow attempt lacks its one successful directly approved attester job."); + } + } + if (encountered.size !== keys.length || keys.some((key) => !encountered.has(key))) { + throw new Error("Preview workflow attempt evidence does not cover every exact signed attempt."); + } + return expected; +} + +export function verifyPreviewWorkflowRun(previewManifest, attestedAttempts) { + const attempts = attestedAttempts.map(({ runId, runAttempt }) => { + if (runId !== previewManifest.workflowRunId || !/^[1-9][0-9]*$/.test(runAttempt)) { + throw new Error("Signed preview attempt does not belong to the canonical manifest workflow run."); + } + const run = ghJson(`repos/${PYLON_PUBLICATION_REPOSITORY}/actions/runs/${runId}/attempts/${runAttempt}`); + const suite = ghJson(`repos/${PYLON_PUBLICATION_REPOSITORY}/check-suites/${run.check_suite_id}`); + const jobPage = ghJson(`repos/${PYLON_PUBLICATION_REPOSITORY}/actions/runs/${runId}/attempts/${runAttempt}/jobs?per_page=100`); + if (!Array.isArray(jobPage.jobs) || jobPage.total_count !== jobPage.jobs.length) { + throw new Error("Exact signed workflow attempt jobs are truncated or malformed."); + } + return { run, suite, jobs: jobPage.jobs }; + }); + return validatePreviewWorkflowRunEvidence({ attempts }, previewManifest, attestedAttempts); +} + +export function verifyPreviewAttestations({ artifactDir, sourceSha, sourceTree, historical = false }) { + const verified = verifyPreviewPublication(artifactDir, { historical }); + if (verified.previewManifest.build.source.commit !== sourceSha || verified.previewManifest.build.source.tree !== sourceTree) { + throw new Error("Requested attestation source commit/tree does not match the preview manifest."); + } + const invocation = { + repository: PYLON_PUBLICATION_REPOSITORY, + workflow: PYLON_PREVIEW_WORKFLOW, + event: "push", + sourceSha, + workflowRunId: verified.previewManifest.workflowRunId, + }; + verifyApprovedWorkflowAtSignerDigest( + PYLON_PREVIEW_WORKFLOW, + sourceSha, + "preview", + verified.previewManifest.publicationPolicyRevision, + ); + const attempts = new Map(); + for (const subject of verified.subjects) { + for (const attempt of verifySubject(join(artifactDir, subject.name), subject, verified.subjects, sourceSha, invocation)) { + attempts.set(`${attempt.runId}:${attempt.runAttempt}`, attempt); + } + } + verifyPreviewWorkflowRun(verified.previewManifest, [...attempts.values()]); + return verified; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + try { + const args = parseArgs(process.argv.slice(2)); + const verified = verifyPreviewAttestations(args); + console.log(`Verified ${verified.subjects.length} exact preview attestations for ${args.sourceSha}.`); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/verify-pylon-stable-attestation.mjs b/scripts/verify-pylon-stable-attestation.mjs new file mode 100644 index 0000000000..2038d6da2e --- /dev/null +++ b/scripts/verify-pylon-stable-attestation.mjs @@ -0,0 +1,89 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + readBoundedRegularFileSync, +} from "./lib/pylon-bounded-file.mjs"; +import { + canonicalJson, + PYLON_PUBLICATION_REF, + PYLON_PUBLICATION_REPOSITORY, + PYLON_STABLE_MANIFEST, + PYLON_STABLE_WORKFLOW, + sha256Bytes, + validateStableManifest, +} from "./lib/pylon-publication.mjs"; +import { verifyApprovedWorkflowAtSignerDigest } from "./lib/pylon-workflow-policy.mjs"; +import { verifyGhAttestationResult } from "./verify-pylon-publication-attestations.mjs"; + +function parseArgs(args) { + const values = new Map(); + for (let index = 0; index < args.length; index += 2) values.set(args[index], args[index + 1]); + const path = resolve(values.get("--manifest") ?? PYLON_STABLE_MANIFEST); + const sourceSha = values.get("--promotion-sha") ?? ""; + const sourceTree = values.get("--promotion-tree") ?? ""; + if (!/^[0-9a-f]{40}$/.test(sourceSha)) throw new Error("--promotion-sha must be a full lowercase Git SHA."); + if (!/^[0-9a-f]{40}$/.test(sourceTree)) throw new Error("--promotion-tree must be a full lowercase Git tree SHA."); + return { path, sourceSha, sourceTree }; +} + +export function verifyStableAttestation(path, sourceSha, sourceTree) { + const bytes = readBoundedRegularFileSync(path, { + maxBytes: PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + description: "Stable attestation manifest", + }); + if (bytes === null) throw new Error("Stable attestation manifest does not exist."); + const manifest = validateStableManifest(JSON.parse(bytes)); + if (canonicalJson(manifest) !== bytes.toString("utf8")) throw new Error("Stable manifest is not canonical publication JSON."); + if (manifest.promotion.policyCommit !== sourceSha || manifest.promotion.policyTree !== sourceTree) { + throw new Error("Promotion commit/tree does not match the signed stable policy identity."); + } + const subject = { name: PYLON_STABLE_MANIFEST, sha256: sha256Bytes(bytes) }; + verifyApprovedWorkflowAtSignerDigest( + PYLON_STABLE_WORKFLOW, + sourceSha, + "stable", + manifest.promotion.publicationPolicyRevision, + ); + const result = spawnSync( + "gh", + [ + "attestation", "verify", path, + "--repo", PYLON_PUBLICATION_REPOSITORY, + "--cert-identity", `https://github.com/${PYLON_PUBLICATION_REPOSITORY}/${PYLON_STABLE_WORKFLOW}@${PYLON_PUBLICATION_REF}`, + "--signer-digest", sourceSha, + "--source-ref", PYLON_PUBLICATION_REF, + "--source-digest", sourceSha, + "--cert-oidc-issuer", "https://token.actions.githubusercontent.com", + "--predicate-type", "https://slsa.dev/provenance/v1", + "--deny-self-hosted-runners", + "--limit", "100", + "--format", "json", + ], + { encoding: "utf8", timeout: 120_000, maxBuffer: 16 * 1024 * 1024 }, + ); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(`gh attestation verify failed for the stable manifest: ${result.stderr}`); + verifyGhAttestationResult(result.stdout, [subject], { + repository: PYLON_PUBLICATION_REPOSITORY, + workflow: PYLON_STABLE_WORKFLOW, + event: "workflow_dispatch", + sourceSha, + }); + return manifest; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + try { + const args = parseArgs(process.argv.slice(2)); + const manifest = verifyStableAttestation(args.path, args.sourceSha, args.sourceTree); + console.log(`Verified stable manifest provenance for ${manifest.tag}.`); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/verify-pylon-stable-history.mjs b/scripts/verify-pylon-stable-history.mjs new file mode 100644 index 0000000000..9c3f0fd982 --- /dev/null +++ b/scripts/verify-pylon-stable-history.mjs @@ -0,0 +1,162 @@ +#!/usr/bin/env node + +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + PYLON_STABLE_HISTORY_MAX_BYTES, + PYLON_STABLE_HISTORY_MAX_MANIFESTS, + readBoundedRegularFile, +} from "./lib/pylon-bounded-file.mjs"; +import { PYLON_RELEASE_REPOSITORY } from "./lib/pylon-release.mjs"; +import { withConsumerStateLock } from "./lib/pylon-consumer-lock.mjs"; +import { + canonicalJson, + parseStableTag, + sha256Bytes, + validateStableHistory, + validateStableManifest, +} from "./lib/pylon-publication.mjs"; + +const STATE_SCHEMA_VERSION = 1; +const STATE_MAX_BYTES = 4 * 1024; + +function exactKeys(value, keys) { + return ( + value !== null && + typeof value === "object" && + !Array.isArray(value) && + Object.keys(value).sort().join(",") === [...keys].sort().join(",") + ); +} + +function validateConsumerState(state) { + if ( + !exactKeys(state, ["schemaVersion", "repository", "channel", "highWater"]) || + state.schemaVersion !== STATE_SCHEMA_VERSION || + state.repository !== PYLON_RELEASE_REPOSITORY || + state.channel !== "stable" || + !exactKeys(state.highWater, ["sequence", "tag", "sha256"]) || + !Number.isSafeInteger(state.highWater.sequence) || + parseStableTag(state.highWater.tag).sequence !== state.highWater.sequence || + !/^[0-9a-f]{64}$/.test(state.highWater.sha256 ?? "") + ) { + throw new Error("Consumer stable high-water state is malformed."); + } + return state; +} + +function readCanonicalState(bytes) { + if (!Buffer.isBuffer(bytes) || bytes.length < 1 || bytes.length > STATE_MAX_BYTES) { + throw new Error("Consumer stable high-water state is malformed."); + } + const state = validateConsumerState(JSON.parse(bytes)); + if (bytes.toString("utf8") !== canonicalJson(state)) { + throw new Error("Consumer stable high-water state is not canonical JSON."); + } + return state; +} + +async function verifiedManifestFiles(paths, fileOptions = {}) { + if (!Array.isArray(paths) || paths.length === 0) throw new Error("Provide every stable manifest from sequence 1 through current high-water."); + if (paths.length > PYLON_STABLE_HISTORY_MAX_MANIFESTS) { + throw new Error(`Stable history exceeds its ${PYLON_STABLE_HISTORY_MAX_MANIFESTS}-manifest work bound.`); + } + let totalBytes = 0; + const manifests = []; + for (const input of paths) { + const path = resolve(input); + const bytes = await readBoundedRegularFile(path, { + maxBytes: PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + description: `Stable manifest ${path}`, + ...fileOptions, + }); + if (bytes === null) throw new Error(`Stable manifest does not exist: ${path}`); + totalBytes += bytes.length; + if (totalBytes > PYLON_STABLE_HISTORY_MAX_BYTES) { + throw new Error("Stable history exceeds its total manifest byte bound."); + } + const manifest = validateStableManifest(JSON.parse(bytes)); + if (bytes.toString("utf8") !== canonicalJson(manifest)) throw new Error(`Stable manifest is not canonical: ${path}`); + manifests.push(manifest); + } + return manifests; +} + +export async function verifyStableHistoryWithState(paths, { statePath, initialize = false, fileOptions = {} }) { + if (typeof statePath !== "string" || !statePath) throw new Error("A consumer-local --state path is required."); + const absoluteStatePath = resolve(statePath); + const history = validateStableHistory(await verifiedManifestFiles(paths, fileOptions)); + const witnessed = new Map(history.map((manifest) => [manifest.sequence, { + tag: manifest.tag, + sha256: sha256Bytes(Buffer.from(canonicalJson(manifest))), + }])); + const latest = history.at(-1); + const highWater = { + sequence: latest.sequence, + tag: latest.tag, + sha256: witnessed.get(latest.sequence).sha256, + }; + return withConsumerStateLock(absoluteStatePath, async (_lockedPath, transaction) => { + const priorBytes = transaction.readStateBytes(); + const stateExists = priorBytes !== null; + if (!stateExists && !initialize) { + throw new Error("No consumer high-water state exists. Inspect the full history, then use --initialize once to accept its witnessed high-water."); + } + if (stateExists && initialize) throw new Error("Consumer high-water state already exists; --initialize cannot reset it."); + const priorState = stateExists ? readCanonicalState(priorBytes) : null; + if (priorState) { + if (latest.sequence < priorState.highWater.sequence) { + throw new Error("Verified stable history is older than the persisted consumer high-water mark."); + } + const priorWitness = witnessed.get(priorState.highWater.sequence); + if ( + !priorWitness || + priorWitness.tag !== priorState.highWater.tag || + priorWitness.sha256 !== priorState.highWater.sha256 + ) { + throw new Error("Verified stable history rewrites the consumer's persisted high-water sequence."); + } + } + const state = { + schemaVersion: STATE_SCHEMA_VERSION, + repository: PYLON_RELEASE_REPOSITORY, + channel: "stable", + highWater, + }; + const advanced = !priorState || highWater.sequence > priorState.highWater.sequence; + if (advanced) await transaction.commitState(Buffer.from(canonicalJson(state))); + return { history, state: advanced ? state : priorState, advanced }; + }, { stateMaxBytes: STATE_MAX_BYTES }); +} + +function parseArgs(args) { + const remaining = [...args]; + const stateIndex = remaining.indexOf("--state"); + if (stateIndex === -1 || !remaining[stateIndex + 1] || remaining[stateIndex + 1].startsWith("--")) { + throw new Error("Usage: verify-pylon-stable-history --state [--initialize] "); + } + const statePath = remaining[stateIndex + 1]; + remaining.splice(stateIndex, 2); + const initializeIndex = remaining.indexOf("--initialize"); + const initialize = initializeIndex !== -1; + if (initialize) remaining.splice(initializeIndex, 1); + if (remaining.some((value) => value.startsWith("--"))) throw new Error("Unknown stable history verifier option."); + return { statePath, initialize, paths: remaining }; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + try { + const args = parseArgs(process.argv.slice(2)); + const verified = await verifyStableHistoryWithState(args.paths, args); + console.log(JSON.stringify({ + sequences: verified.history.length, + highWater: verified.state.highWater, + advanced: verified.advanced, + })); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +}