diff --git a/.github/workflows/docker-sandboxes-images.yml b/.github/workflows/docker-sandboxes-images.yml index a85253f..c532a98 100644 --- a/.github/workflows/docker-sandboxes-images.yml +++ b/.github/workflows/docker-sandboxes-images.yml @@ -2,8 +2,8 @@ name: Docker Sandboxes prebuilt images on: schedule: - - cron: '37 */6 * * *' - - cron: '7 1,7,13,19 * * *' + - cron: '37 23 */7 * *' + - cron: '57 23 */7 * *' push: branches: - main @@ -137,6 +137,7 @@ jobs: runner_amd64_digest: ${{ steps.runner.outputs.amd64_digest }} runner_arm64_url: ${{ steps.runner.outputs.arm64_url }} runner_arm64_digest: ${{ steps.runner.outputs.arm64_digest }} + catalog_manifest_digest: ${{ steps.catalog.outputs.catalog_manifest_digest }} noop: ${{ steps.noop.outputs.noop }} steps: - name: Check out source @@ -171,8 +172,8 @@ jobs: profile="${DISPATCH_PROFILE:-act}" if [[ "$GITHUB_EVENT_NAME" == schedule ]]; then case "$SCHEDULE_EXPRESSION" in - '37 */6 * * *') profile=act ;; - '7 1,7,13,19 * * *') profile=full ;; + '37 23 */7 * *') profile=full ;; + '57 23 */7 * *') profile=act ;; *) echo "unsupported schedule expression: $SCHEDULE_EXPRESSION" >&2; exit 1 ;; esac fi @@ -276,6 +277,7 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Fetch current signed catalog state or initialize an empty ledger + id: catalog shell: bash env: PROFILE: ${{ steps.select.outputs.profile }} @@ -294,6 +296,7 @@ jobs: if catalog_digest="$(oras resolve "$catalog_reference" 2>"$catalog_error")"; then [[ "$catalog_digest" =~ ^sha256:[0-9a-f]{64}$ ]] go run ./cmd/epar-prebuilt-publisher verify-catalog --repository "$PACKAGE_REPOSITORY" --profile act --reference "${PACKAGE_REPOSITORY}@${catalog_digest}" --ref refs/heads/main --allowed-events schedule,workflow_dispatch,push --output "$catalog_dir/catalog-state.json" + echo "catalog_manifest_digest=$catalog_digest" >> "$GITHUB_OUTPUT" else if ! grep -Eqi 'manifest unknown|not found|404' "$catalog_error"; then cat "$catalog_error" >&2 @@ -313,6 +316,7 @@ jobs: "transitions": [] } JSON + echo 'catalog_manifest_digest=' >> "$GITHUB_OUTPUT" fi jq -e . "$catalog_dir/catalog-state.json" >/dev/null @@ -321,9 +325,26 @@ jobs: shell: bash env: PROFILE: ${{ steps.select.outputs.profile }} + EXPECTED_CATALOG_MANIFEST: ${{ steps.catalog.outputs.catalog_manifest_digest }} run: | set -euo pipefail catalog="$RUNNER_TEMP/epar-catalog/catalog-state.json" + expected_catalog_manifest="${EXPECTED_CATALOG_MANIFEST:-}" + current_catalog_manifest='' + catalog_cas_error="$RUNNER_TEMP/epar-catalog/catalog-reconcile-cas.error" + if current_catalog_manifest="$(oras resolve "${PACKAGE_REPOSITORY}:catalog-v1" 2>"$catalog_cas_error")"; then + [[ "$current_catalog_manifest" =~ ^sha256:[0-9a-f]{64}$ ]] + else + if ! grep -Eqi 'manifest unknown|not found|404' "$catalog_cas_error"; then + cat "$catalog_cas_error" >&2 + exit 1 + fi + current_catalog_manifest='' + fi + [[ "$current_catalog_manifest" == "$expected_catalog_manifest" ]] || { + echo "catalog-v1 changed while preparing alias reconciliation (expected ${expected_catalog_manifest:-missing}, observed ${current_catalog_manifest:-missing})" >&2 + exit 1 + } alias_ref="${PACKAGE_REPOSITORY}:${PROFILE}-latest" observed='' alias_error="$RUNNER_TEMP/epar-catalog/alias-reconcile.error" @@ -338,6 +359,21 @@ jobs: if [[ "$(jq -r '.needsRepair' "$plan")" == true ]]; then target="$(jq -er '.targetDigest' "$plan")" [[ "$(oras resolve "${PACKAGE_REPOSITORY}@${target}")" == "$target" ]] + alias_cas_error="$RUNNER_TEMP/epar-catalog/alias-reconcile-cas.error" + current_alias_digest='' + if current_alias_digest="$(oras resolve "$alias_ref" 2>"$alias_cas_error")"; then + [[ "$current_alias_digest" =~ ^sha256:[0-9a-f]{64}$ ]] + else + if ! grep -Eqi 'manifest unknown|not found|404' "$alias_cas_error"; then + cat "$alias_cas_error" >&2 + exit 1 + fi + current_alias_digest='' + fi + [[ "$current_alias_digest" == "$observed" ]] || { + echo "${PROFILE}-latest changed while preparing alias reconciliation (expected ${observed:-missing}, observed ${current_alias_digest:-missing})" >&2 + exit 1 + } oras tag "${PACKAGE_REPOSITORY}@${target}" "${PROFILE}-latest" [[ "$(oras resolve "$alias_ref")" == "$target" ]] echo "Recovered interrupted ${PROFILE} alias promotion to ${target}." >> "$GITHUB_STEP_SUMMARY" @@ -362,18 +398,75 @@ jobs: run: | set -euo pipefail catalog="$RUNNER_TEMP/epar-catalog/catalog-state.json" + matching_entries="$RUNNER_TEMP/epar-catalog/matching-entries.json" + matching_entry="$RUNNER_TEMP/epar-catalog/matching-entry.json" noop=false if [[ "$FORCE_CANDIDATE" != true ]]; then - # A retained candidate is also a no-op: rebuilding it can produce a - # different index digest and make the immutable catalog reject the - # same complete tuple. Only revoked/critical entries are ignored. - jq -e --arg profile "$PROFILE" --arg source "$SOURCE_INDEX_DIGEST" \ + # Only a complete candidate or active entry can suppress a build. + # Superseded entries must not suppress a build because their package + # is no longer the catalog's current accepted result. + jq -c --arg profile "$PROFILE" --arg source "$SOURCE_INDEX_DIGEST" \ --arg sourceAmd64 "$SOURCE_AMD64_DIGEST" --arg sourceArm64 "$SOURCE_ARM64_DIGEST" \ --arg recipe "$RECIPE_DIGEST" --arg revision "$RECIPE_REVISION" --arg sourceLock "$SOURCE_LOCK_DIGEST" --arg tool "$TOOL_DIGEST" \ --arg runtime "$RUNTIME_CONTRACT" --arg schema "$TEMPLATE_SCHEMA" --arg runner "$RUNNER_VERSION" \ --arg amd64 "$RUNNER_AMD64_DIGEST" --arg arm64 "$RUNNER_ARM64_DIGEST" \ - '.entries[] as $entry | (([.transitions[]? | select(.packageIndexDigest == $entry.packageIndexDigest) | .toStatus] | last) // $entry.status) as $effectiveStatus | select(($effectiveStatus == "candidate" or $effectiveStatus == "active" or $effectiveStatus == "superseded") and $entry.profile == $profile and $entry.source.indexDigest == $source and $entry.source.platformDigests["linux/amd64"] == $sourceAmd64 and $entry.source.platformDigests["linux/arm64"] == $sourceArm64 and $entry.recipe.digest == $recipe and $entry.recipe.recipeRevision == $revision and $entry.recipe.sourceLockDigest == $sourceLock and $entry.recipe.toolDigest == $tool and $entry.recipe.runtimeContract == $runtime and ($entry.recipe.templateSchema | tostring) == $schema and $entry.runner.version == $runner and $entry.runner.assetDigests["linux/amd64"] == $amd64 and $entry.runner.assetDigests["linux/arm64"] == $arm64)' \ - "$catalog" >/dev/null && noop=true || true + '. as $catalog | [ + $catalog.entries[] as $entry + | (([$catalog.transitions[]? | select(.packageIndexDigest == $entry.packageIndexDigest) | .toStatus] | last) // $entry.status) as $effectiveStatus + | select( + ($effectiveStatus == "candidate" or $effectiveStatus == "active") + and $entry.profile == $profile + and $entry.source.indexDigest == $source + and $entry.source.platformDigests["linux/amd64"] == $sourceAmd64 + and $entry.source.platformDigests["linux/arm64"] == $sourceArm64 + and $entry.recipe.digest == $recipe + and $entry.recipe.recipeRevision == $revision + and $entry.recipe.sourceLockDigest == $sourceLock + and $entry.recipe.toolDigest == $tool + and $entry.recipe.runtimeContract == $runtime + and ($entry.recipe.templateSchema | tostring) == $schema + and $entry.runner.version == $runner + and $entry.runner.assetDigests["linux/amd64"] == $amd64 + and $entry.runner.assetDigests["linux/arm64"] == $arm64 + and $entry.gates.sourceResolved == true + and $entry.gates.sourceRechecked == true + and $entry.gates.buildSucceeded == true + and $entry.gates.platformsValidated == true + and $entry.gates.provenanceGenerated == true + and $entry.gates.sbomGenerated == true + and $entry.gates.attestationVerified == true + and ($effectiveStatus != "active" or $catalog.aliases[$profile].packageIndexDigest == $entry.packageIndexDigest) + ) + | $entry + ]' "$catalog" > "$matching_entries" + + match_count="$(jq 'length' "$matching_entries")" + case "$match_count" in + 0) + echo 'No complete matching catalog entry; the hosted build will proceed.' + ;; + 1) + jq -e '.[0]' "$matching_entries" > "$matching_entry" + package_reference="$(jq -er '.packageReference' "$matching_entry")" + [[ "$package_reference" == "${PACKAGE_REPOSITORY}@sha256:"* ]] + # Verify the immutable package and its signed evidence before + # suppressing work. This is metadata-only and does not pull + # image layers. A failed verification is fail-closed. + go run ./cmd/epar-prebuilt-publisher verify-package \ + --reference "$package_reference" \ + --entry "$matching_entry" \ + --repository "$PACKAGE_REPOSITORY" \ + --ref refs/heads/main \ + --allowed-events schedule,workflow_dispatch,push \ + > "$RUNNER_TEMP/epar-catalog/package-verification.json" + noop=true + echo "Verified immutable package metadata for $package_reference; skipping hosted builds." + ;; + *) + echo "Catalog contains $match_count complete matching entries; refusing an ambiguous no-op." >&2 + exit 1 + ;; + esac fi echo "noop=$noop" >> "$GITHUB_OUTPUT" @@ -1118,6 +1211,7 @@ jobs: PACKAGE_REF: ${{ needs.publish.outputs.package_ref }} SOURCE_RECHECKED: ${{ needs.publish.outputs.source_rechecked }} ALLOW_ALIAS: ${{ github.ref == 'refs/heads/main' }} + EXPECTED_CATALOG_MANIFEST: ${{ needs.resolve.outputs.catalog_manifest_digest }} run: | set -euo pipefail immutable_catalog_ref="${PACKAGE_REPOSITORY}:catalog-v1-pkg-${CATALOG_DIGEST#sha256:}" @@ -1135,6 +1229,11 @@ jobs: fi old_catalog_digest='' fi + expected_catalog_manifest="${EXPECTED_CATALOG_MANIFEST:-}" + if [[ "$old_catalog_digest" != "$expected_catalog_manifest" ]]; then + echo "catalog-v1 moved since resolve (expected ${expected_catalog_manifest:-missing}, observed ${old_catalog_digest:-missing}); refusing to publish stale catalog state" >&2 + exit 1 + fi plan_action="$(jq -r '.action' "$PLAN")" move_alias=false alias_ref="${PACKAGE_REPOSITORY}:${PROFILE}-latest" @@ -1162,26 +1261,36 @@ jobs: trap - EXIT set +e if [[ "$alias_moved" == true ]]; then - if [[ -n "$old_alias_digest" ]]; then + actual_alias_digest="$(oras resolve "$alias_ref" 2>/dev/null || true)" + if [[ "$actual_alias_digest" == "${PACKAGE_REF##*@}" && -n "$old_alias_digest" ]]; then oras tag "${PACKAGE_REPOSITORY}@${old_alias_digest}" "${PROFILE}-latest" >/dev/null [[ "$(oras resolve "$alias_ref")" == "$old_alias_digest" ]] || echo 'alias rollback readback failed' >&2 - else + elif [[ "$actual_alias_digest" == "${PACKAGE_REF##*@}" ]]; then echo 'alias rollback cannot remove a first-publication alias; catalog rollback remains authoritative' >&2 + else + echo 'alias rollback skipped because the alias changed after this publication' >&2 fi fi - if [[ "$catalog_moved" == true && -n "$old_catalog_digest" ]]; then - oras tag "${PACKAGE_REPOSITORY}@${old_catalog_digest}" catalog-v1 >/dev/null - [[ "$(oras resolve "$catalog_moving")" == "$old_catalog_digest" ]] || echo 'catalog rollback readback failed' >&2 - elif [[ "$catalog_moved" == true ]]; then - echo 'catalog rollback cannot remove a first-publication catalog pointer; it remains signed with no trusted alias' >&2 + if [[ "$catalog_moved" == true ]]; then + actual_catalog_digest="$(oras resolve "$catalog_moving" 2>/dev/null || true)" + if [[ "$actual_catalog_digest" == "$CATALOG_MANIFEST" && -n "$old_catalog_digest" ]]; then + oras tag "${PACKAGE_REPOSITORY}@${old_catalog_digest}" catalog-v1 >/dev/null + [[ "$(oras resolve "$catalog_moving")" == "$old_catalog_digest" ]] || echo 'catalog rollback readback failed' >&2 + elif [[ "$actual_catalog_digest" == "$CATALOG_MANIFEST" ]]; then + echo 'catalog rollback cannot remove a first-publication catalog pointer; it remains signed with no trusted alias' >&2 + else + echo 'catalog rollback skipped because the catalog pointer changed after this publication' >&2 + fi fi exit "$rc" } - if [[ "$move_alias" == true ]]; then + if [[ "$ALLOW_ALIAS" == true ]]; then trap rollback_pointers EXIT oras tag "${PACKAGE_REPOSITORY}:catalog-v1-pkg-${CATALOG_DIGEST#sha256:}" catalog-v1 catalog_moved=true [[ "$(oras resolve "$catalog_moving")" == "$CATALOG_MANIFEST" ]] + fi + if [[ "$move_alias" == true ]]; then alias_cas_error="$RUNNER_TEMP/epar-promotion/alias-cas-resolve.error" if actual_alias_digest="$(oras resolve "$alias_ref" 2>"$alias_cas_error")"; then [[ -z "$expected_alias" || "$actual_alias_digest" == "$expected_alias" ]] @@ -1197,6 +1306,10 @@ jobs: [[ "$(oras resolve "$alias_ref")" == "${PACKAGE_REF##*@}" ]] trap - EXIT echo 'Signed catalog was moved before the authorized source-only profile alias; pointer rollback was armed until both readbacks passed.' >> "$GITHUB_STEP_SUMMARY" + elif [[ "$ALLOW_ALIAS" == true ]]; then + trap - EXIT + echo "Signed immutable candidate catalog: ${immutable_catalog_ref}" >> "$GITHUB_STEP_SUMMARY" + echo 'catalog-v1 was moved; the profile alias was not moved; protected EPAR acceptance remains required.' >> "$GITHUB_STEP_SUMMARY" else echo "Signed immutable candidate catalog: ${immutable_catalog_ref}" >> "$GITHUB_STEP_SUMMARY" echo 'catalog-v1 and the profile alias were not moved; protected EPAR acceptance remains required.' >> "$GITHUB_STEP_SUMMARY" @@ -1211,6 +1324,8 @@ jobs: attestations: read runs-on: ubuntu-latest timeout-minutes: 30 + outputs: + catalog_manifest: ${{ steps.review.outputs.catalog_manifest }} steps: - name: Check out source uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 @@ -1227,6 +1342,7 @@ jobs: version: 1.3.3 - name: Verify candidate identities and prepare the reviewer checklist + id: review shell: bash env: PROFILE: ${{ inputs.profile }} @@ -1288,6 +1404,23 @@ jobs: package_arm64="$(jq -er '.platforms[] | select(.platform == "linux/arm64") | .packageManifestDigest' "$entry")" source_amd64="$(jq -er '.source.platformDigests["linux/amd64"]' "$entry")" source_arm64="$(jq -er '.source.platformDigests["linux/arm64"]' "$entry")" + catalog_moving="${PACKAGE_REPOSITORY}:catalog-v1" + catalog_pointer_error="$RUNNER_TEMP/promotion-review-catalog-pointer.error" + expected_catalog_manifest='' + if expected_catalog_manifest="$(oras resolve "$catalog_moving" 2>"$catalog_pointer_error")"; then + [[ "$expected_catalog_manifest" =~ ^sha256:[0-9a-f]{64}$ ]] + else + if ! grep -Eqi 'manifest unknown|not found|404' "$catalog_pointer_error"; then + cat "$catalog_pointer_error" >&2 + exit 1 + fi + expected_catalog_manifest='' + fi + [[ -n "$expected_catalog_manifest" && "$candidate_catalog_manifest" == "$expected_catalog_manifest" ]] || { + echo "candidate catalog ${candidate_catalog_manifest} is not the current catalog-v1 head (${expected_catalog_manifest:-missing}); refusing to promote a stale ledger snapshot" >&2 + exit 1 + } + echo "catalog_manifest=$expected_catalog_manifest" >> "$GITHUB_OUTPUT" { echo '## EPAR prebuilt promotion review' echo @@ -1300,6 +1433,7 @@ jobs: echo "| Package index | \`${PACKAGE_REPOSITORY}@${CANDIDATE_DIGEST}\` |" echo "| Candidate catalog | \`${CANDIDATE_CATALOG_REFERENCE}\` |" echo "| Catalog manifest | \`${candidate_catalog_manifest}\` |" + echo "| Current catalog-v1 manifest at review | \`${expected_catalog_manifest:-missing}\` |" echo "| Upstream source | \`${source_reference}\` |" echo "| Upstream index | \`${source_digest}\` |" echo "| Recipe | \`${recipe_digest}\` at \`${recipe_revision}\` |" @@ -1518,6 +1652,7 @@ jobs: PACKAGE_REPOSITORY: ${{ env.PACKAGE_REPOSITORY }} CANDIDATE_DIGEST: ${{ inputs.candidate_digest }} PLAN: ${{ steps.verify.outputs.plan }} + EXPECTED_CATALOG_MANIFEST: ${{ needs.prepare-promotion-review.outputs.catalog_manifest }} run: | set -euo pipefail [[ "$(oras resolve "$IMMUTABLE_REF")" == "$CATALOG_MANIFEST" ]] @@ -1534,6 +1669,11 @@ jobs: fi old_catalog_digest='' fi + expected_catalog_manifest="${EXPECTED_CATALOG_MANIFEST:-}" + if [[ "$old_catalog_digest" != "$expected_catalog_manifest" ]]; then + echo "catalog-v1 changed after reviewer preparation (expected ${expected_catalog_manifest:-missing}, observed ${old_catalog_digest:-missing}); refusing stale manual promotion" >&2 + exit 1 + fi alias_tag="${PROFILE}-latest" alias_ref="${PACKAGE_REPOSITORY}:${alias_tag}" expected_alias="$(jq -r '.expectedAliasDigest // empty' "$PLAN")" @@ -1557,22 +1697,45 @@ jobs: trap - EXIT set +e if [[ "$alias_moved" == true ]]; then - if [[ -n "$old_alias_digest" ]]; then + actual_alias_digest="$(oras resolve "$alias_ref" 2>/dev/null || true)" + if [[ "$actual_alias_digest" == "$CANDIDATE_DIGEST" && -n "$old_alias_digest" ]]; then oras tag "${PACKAGE_REPOSITORY}@${old_alias_digest}" "$alias_tag" >/dev/null [[ "$(oras resolve "$alias_ref")" == "$old_alias_digest" ]] || echo 'manual alias rollback readback failed' >&2 - else + elif [[ "$actual_alias_digest" == "$CANDIDATE_DIGEST" ]]; then echo 'manual alias rollback cannot remove a first-publication alias; catalog rollback remains authoritative' >&2 + else + echo 'manual alias rollback skipped because the alias changed after this promotion' >&2 fi fi - if [[ "$catalog_moved" == true && -n "$old_catalog_digest" ]]; then - oras tag "${PACKAGE_REPOSITORY}@${old_catalog_digest}" catalog-v1 >/dev/null - [[ "$(oras resolve "$catalog_moving")" == "$old_catalog_digest" ]] || echo 'manual catalog rollback readback failed' >&2 - elif [[ "$catalog_moved" == true ]]; then - echo 'manual catalog rollback cannot remove a first-publication catalog pointer; it remains signed with no trusted alias' >&2 + if [[ "$catalog_moved" == true ]]; then + actual_catalog_digest="$(oras resolve "$catalog_moving" 2>/dev/null || true)" + if [[ "$actual_catalog_digest" == "$CATALOG_MANIFEST" && -n "$old_catalog_digest" ]]; then + oras tag "${PACKAGE_REPOSITORY}@${old_catalog_digest}" catalog-v1 >/dev/null + [[ "$(oras resolve "$catalog_moving")" == "$old_catalog_digest" ]] || echo 'manual catalog rollback readback failed' >&2 + elif [[ "$actual_catalog_digest" == "$CATALOG_MANIFEST" ]]; then + echo 'manual catalog rollback cannot remove a first-publication catalog pointer; it remains signed with no trusted alias' >&2 + else + echo 'manual catalog rollback skipped because the catalog pointer changed after this promotion' >&2 + fi fi exit "$rc" } trap rollback_manual_pointers EXIT + manual_catalog_cas_error="$RUNNER_TEMP/manual-catalog-cas-resolve.error" + current_catalog_digest='' + if current_catalog_digest="$(oras resolve "$catalog_moving" 2>"$manual_catalog_cas_error")"; then + [[ "$current_catalog_digest" =~ ^sha256:[0-9a-f]{64}$ ]] + else + if ! grep -Eqi 'manifest unknown|not found|404' "$manual_catalog_cas_error"; then + cat "$manual_catalog_cas_error" >&2 + exit 1 + fi + current_catalog_digest='' + fi + [[ "$current_catalog_digest" == "$old_catalog_digest" ]] || { + echo "catalog-v1 changed while preparing manual promotion (expected ${old_catalog_digest:-missing}, observed ${current_catalog_digest:-missing})" >&2 + exit 1 + } oras tag "$IMMUTABLE_REF" catalog-v1 catalog_moved=true [[ "$(oras resolve "$catalog_moving")" == "$CATALOG_MANIFEST" ]] @@ -1602,4 +1765,4 @@ jobs: shell: bash run: | set -euo pipefail - echo 'Upstream source, recipe, runner, and locked-tool tuple is unchanged; no package or catalog mutation was attempted.' >> "$GITHUB_STEP_SUMMARY" + echo 'Upstream source, recipe, runner, and locked-tool tuple is unchanged; the matching immutable package and signed evidence were verified from the catalog, so no hosted build or package mutation was attempted.' >> "$GITHUB_STEP_SUMMARY" diff --git a/README.md b/README.md index 8939420..49b1923 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,14 @@ flowchart LR Remove --> Start ``` +## See it in action + +[![EPAR demo — disposable GitHub Actions runners with Docker Sandboxes](https://img.youtube.com/vi/5xDEFpf6iZc/maxresdefault.jpg)](https://www.youtube.com/watch?v=5xDEFpf6iZc) + +Watch EPAR take a GitHub Actions job from a warm runner, execute it inside a Docker Sandboxes microVM, retire the runner after the job, and bring a clean replacement back to ready state. + +[Watch the demo on YouTube →](https://www.youtube.com/watch?v=5xDEFpf6iZc) + ## Why EPAR - **Put spare compute to work** — run long-running E2E, integration, and Docker-heavy CI on machines you already operate. diff --git a/cmd/epar-prebuilt-publisher/workflow_contract_test.go b/cmd/epar-prebuilt-publisher/workflow_contract_test.go index eafe37d..3346cf3 100644 --- a/cmd/epar-prebuilt-publisher/workflow_contract_test.go +++ b/cmd/epar-prebuilt-publisher/workflow_contract_test.go @@ -23,6 +23,94 @@ func TestWorkflowRepairsCatalogFirstPromotionBeforeNoop(t *testing.T) { } } +func TestWorkflowVerifiesMatchingPackageBeforeNoop(t *testing.T) { + workflow := strings.ReplaceAll(readPublisherWorkflow(t), "\r\n", "\n") + noop := strings.Index(workflow, " - name: Determine whether the immutable tuple is already active\n") + build := strings.Index(workflow, "\n build:\n") + if noop < 0 || build <= noop { + t.Fatalf("cannot isolate metadata-first no-op step: noop=%d build=%d", noop, build) + } + noopStep := workflow[noop:build] + for _, required := range []string{ + `matching_entries="$RUNNER_TEMP/epar-catalog/matching-entries.json"`, + `matching_entry="$RUNNER_TEMP/epar-catalog/matching-entry.json"`, + `($effectiveStatus == "candidate" or $effectiveStatus == "active")`, + `$entry.gates.sourceRechecked == true`, + `$entry.gates.attestationVerified == true`, + `go run ./cmd/epar-prebuilt-publisher verify-package`, + `--reference "$package_reference"`, + `> "$RUNNER_TEMP/epar-catalog/package-verification.json"`, + `Catalog contains $match_count complete matching entries; refusing an ambiguous no-op.`, + } { + if !strings.Contains(noopStep, required) { + t.Fatalf("metadata-first no-op contract is missing %q", required) + } + } + if strings.Contains(noopStep, `$effectiveStatus == "superseded"`) { + t.Fatal("superseded catalog entries must not suppress a rebuild") + } + verify := strings.Index(noopStep, `go run ./cmd/epar-prebuilt-publisher verify-package`) + noOpAssignment := strings.Index(noopStep, "noop=true") + if verify < 0 || noOpAssignment < 0 || verify >= noOpAssignment { + t.Fatalf("immutable package verification must precede noop=true: verify=%d noop=%d", verify, noOpAssignment) + } +} + +func TestWorkflowGuardsCatalogPointerAndPublishesCandidateLedgerOnMain(t *testing.T) { + workflow := strings.ReplaceAll(readPublisherWorkflow(t), "\r\n", "\n") + promote := strings.Index(workflow, " - name: Verify signed catalog and move only authorized aliases\n") + manual := strings.Index(workflow, "\n prepare-promotion-review:\n") + if promote < 0 || manual <= promote { + t.Fatalf("cannot isolate automatic catalog publication step: promote=%d manual=%d", promote, manual) + } + promoteStep := workflow[promote:manual] + for _, required := range []string{ + `EXPECTED_CATALOG_MANIFEST: ${{ needs.resolve.outputs.catalog_manifest_digest }}`, + `old_catalog_digest" != "$expected_catalog_manifest"`, + `if [[ "$ALLOW_ALIAS" == true ]]; then`, + `catalog-v1 was moved; the profile alias was not moved`, + `catalog rollback skipped because the catalog pointer changed after this publication`, + } { + if !strings.Contains(promoteStep, required) { + t.Fatalf("catalog publication safety contract is missing %q", required) + } + } + reconcile := strings.Index(workflow, " - name: Reconcile an interrupted catalog-first alias promotion\n") + noop := strings.Index(workflow, " - name: Determine whether the immutable tuple is already active\n") + if reconcile < 0 || noop <= reconcile || !strings.Contains(workflow[reconcile:noop], `current_alias_digest" == "$observed"`) { + t.Fatal("alias reconciliation must compare the observed alias head again before repair") + } +} + +func TestWorkflowCarriesManualCatalogHeadThroughProtectedPromotion(t *testing.T) { + workflow := strings.ReplaceAll(readPublisherWorkflow(t), "\r\n", "\n") + prepare := strings.Index(workflow, " prepare-promotion-review:\n") + manual := strings.Index(workflow, "\n manual-promote:\n") + if prepare < 0 || manual <= prepare { + t.Fatalf("cannot isolate protected promotion preparation: prepare=%d manual=%d", prepare, manual) + } + prepareJob := workflow[prepare:manual] + manualJob := workflow[manual:] + for _, required := range []string{ + `catalog_manifest: ${{ steps.review.outputs.catalog_manifest }}`, + `echo "catalog_manifest=$expected_catalog_manifest" >> "$GITHUB_OUTPUT"`, + `Current catalog-v1 manifest at review`, + } { + if !strings.Contains(prepareJob, required) { + t.Fatalf("protected review must record the catalog head: missing %q", required) + } + } + for _, required := range []string{ + `EXPECTED_CATALOG_MANIFEST: ${{ needs.prepare-promotion-review.outputs.catalog_manifest }}`, + `catalog-v1 changed after reviewer preparation`, + `[[ "$current_catalog_digest" == "$old_catalog_digest" ]]`, + } { + if !strings.Contains(manualJob, required) { + t.Fatalf("protected promotion must guard the reviewed catalog head: missing %q", required) + } + } +} + func TestWorkflowForceCandidatePreservesVerifiedEvidence(t *testing.T) { workflow := readPublisherWorkflow(t) for _, required := range []string{ @@ -62,7 +150,7 @@ func TestWorkflowUsesHostedBuildsAndExternalEPARAcceptance(t *testing.T) { `runnerName:$amd64DockerHubRunner`, `runnerName:$arm64PlaywrightRunner`, `runnerName:$arm64DockerHubRunner`, - `catalog-v1 and the profile alias were not moved`, + `catalog-v1 was moved; the profile alias was not moved`, } { if !strings.Contains(workflow, required) { t.Fatalf("publisher candidate acceptance contract is missing %q", required) @@ -217,10 +305,10 @@ func TestWorkflowPreparesReviewSummaryBeforeProtectedPromotion(t *testing.T) { func TestWorkflowBuildsAndPromotesFullWithoutPersistentNativeRunners(t *testing.T) { workflow := readPublisherWorkflow(t) for _, required := range []string{ - `- cron: '37 */6 * * *'`, - `- cron: '7 1,7,13,19 * * *'`, - `'37 */6 * * *') profile=act`, - `'7 1,7,13,19 * * *') profile=full`, + `- cron: '37 23 */7 * *'`, + `- cron: '57 23 */7 * *'`, + `'37 23 */7 * *') profile=full`, + `'57 23 */7 * *') profile=act`, `act|full) ;;`, `if: needs.resolve.outputs.profile == 'full'`, `Full publication requires at least 40 GiB free`, diff --git a/cmd/ephemeral-action-runner/init.go b/cmd/ephemeral-action-runner/init.go index 44971ae..aff4ba4 100644 --- a/cmd/ephemeral-action-runner/init.go +++ b/cmd/ephemeral-action-runner/init.go @@ -264,7 +264,7 @@ func runInitWithOptions(opts initOptions) error { return err } } - fmt.Fprintln(opts.Out, "EPAR first-run setup") + fmt.Fprintln(opts.Out, "Welcome to EPAR first-run setup wizard.") fmt.Fprintln(opts.Out, "") fmt.Fprintln(opts.Out, "This creates an EPAR runner configuration.") fmt.Fprintln(opts.Out, "Before continuing, create a GitHub App with organization self-hosted runner read/write access.") @@ -2059,6 +2059,8 @@ dockerSandboxes: policyGeneration: %s networkBaseline: open architectureEmulation: %s + recoveryMode: exclusive-auto + recoveryQuiescenceSeconds: 60 stagingRoot: .local/cache/docker-sandboxes/staging cpus: 4 memory: 8GiB diff --git a/cmd/ephemeral-action-runner/init_test.go b/cmd/ephemeral-action-runner/init_test.go index 75346b4..b0e0385 100644 --- a/cmd/ephemeral-action-runner/init_test.go +++ b/cmd/ephemeral-action-runner/init_test.go @@ -1125,11 +1125,20 @@ func TestInitDockerSandboxesGeneratesDesiredImageConfigAndProvisionsTemplate(t * if got, want := cfg.DockerSandboxes.NetworkBaseline, config.DockerSandboxesNetworkBaselineOpen; got != want { t.Fatalf("dockerSandboxes.networkBaseline = %q, want %q", got, want) } - if got, want := cfg.DockerSandboxes.ArchitectureEmulation, config.DockerSandboxesArchitectureEmulationBestEffort; got != want { + if got, want := cfg.DockerSandboxes.ArchitectureEmulation, config.DockerSandboxesArchitectureEmulationNativeOnly; got != want { t.Fatalf("dockerSandboxes.architectureEmulation = %q, want wizard value %q", got, want) } - if !strings.Contains(string(configContent), "architectureEmulation: best-effort") { - t.Fatalf("wizard config omitted best-effort architecture emulation:\n%s", configContent) + if got, want := cfg.DockerSandboxes.RecoveryMode, config.DockerSandboxesRecoveryModeExclusiveAuto; got != want { + t.Fatalf("dockerSandboxes.recoveryMode = %q, want wizard value %q", got, want) + } + if got, want := cfg.DockerSandboxes.RecoveryQuiescenceSeconds, config.DockerSandboxesDefaultRecoveryQuiescenceSeconds; got != want { + t.Fatalf("dockerSandboxes.recoveryQuiescenceSeconds = %d, want wizard value %d", got, want) + } + if !strings.Contains(string(configContent), "architectureEmulation: native-only") { + t.Fatalf("wizard config omitted native-only architecture emulation:\n%s", configContent) + } + if !strings.Contains(string(configContent), "recoveryMode: exclusive-auto") || !strings.Contains(string(configContent), "recoveryQuiescenceSeconds: 60") { + t.Fatalf("wizard config omitted exclusive automatic recovery defaults:\n%s", configContent) } for key, values := range map[string]struct{ got, want string }{ "rootDisk": {cfg.DockerSandboxes.RootDisk, "auto"}, @@ -1144,11 +1153,14 @@ func TestInitDockerSandboxesGeneratesDesiredImageConfigAndProvisionsTemplate(t * t.Fatalf("init output omitted %q:\n%s", want, out.String()) } } - for _, want := range []string{"Architecture emulation: best-effort", "QEMU/binfmt will be attempted; unsupported hosts continue with verified native containers and a warning."} { + for _, want := range []string{"Architecture emulation: native-only", "QEMU/binfmt is disabled by default; Docker Sandboxes runs native containers only."} { if !strings.Contains(out.String(), want) { t.Fatalf("wizard review omitted %q:\n%s", want, out.String()) } } + if strings.Contains(out.String(), "QEMU/binfmt will be attempted") { + t.Fatalf("wizard review unexpectedly enabled the QEMU attempt by default:\n%s", out.String()) + } setupHints := " Choose how EPAR should provision the reusable runner artifact during startup.\n Docker Sandboxes profiles must include a private Docker daemon; specialized and custom tags are not admitted.\n Image catalog: https://github.com/catthehacker/docker_images#images-available\n\nRunner base image:" if !strings.Contains(out.String(), setupHints) { t.Fatalf("Docker Sandboxes setup hints were not grouped before the option list:\n%s", out.String()) @@ -1288,6 +1300,7 @@ func TestDockerSandboxesPrebuiltConfigRendersExplicitDistributionAndReference(t "distribution: prebuilt", "prebuiltReference: " + config.DockerSandboxesPrebuiltActReference, "sourceImage: ghcr.io/catthehacker/ubuntu:act-latest", + "architectureEmulation: best-effort", "updateFrequency: weekly", "updateTime: \"07:00\"", } { @@ -2082,7 +2095,7 @@ func TestInitPromotedDockerSandboxesDefaultsOnlyAfterPassingPreflight(t *testing if got, want := cfg.DockerSandboxes.PolicyGeneration, record.PolicyFingerprint; got != want { t.Fatalf("dockerSandboxes.policyGeneration = %q, want %q", got, want) } - if got, want := cfg.DockerSandboxes.ArchitectureEmulation, config.DockerSandboxesArchitectureEmulationBestEffort; got != want { + if got, want := cfg.DockerSandboxes.ArchitectureEmulation, config.DockerSandboxesArchitectureEmulationNativeOnly; got != want { t.Fatalf("dockerSandboxes.architectureEmulation = %q, want Windows wizard value %q", got, want) } for key, values := range map[string]struct { @@ -2100,7 +2113,7 @@ func TestInitPromotedDockerSandboxesDefaultsOnlyAfterPassingPreflight(t *testing if err != nil { t.Fatal(err) } - for _, required := range []string{"type: docker-sandboxes", "dockerSandboxes:", "epar-docker-sandboxes", "policyGeneration: " + record.PolicyFingerprint, "architectureEmulation: best-effort"} { + for _, required := range []string{"type: docker-sandboxes", "dockerSandboxes:", "epar-docker-sandboxes", "policyGeneration: " + record.PolicyFingerprint, "architectureEmulation: native-only", "recoveryMode: exclusive-auto", "recoveryQuiescenceSeconds: 60"} { if !strings.Contains(string(configText), required) { t.Fatalf("generated Docker Sandboxes config omitted %q:\n%s", required, configText) } @@ -2113,11 +2126,14 @@ func TestInitPromotedDockerSandboxesDefaultsOnlyAfterPassingPreflight(t *testing if !strings.Contains(out.String(), "PASS: the exact promoted platform") || !strings.Contains(out.String(), "Docker Sandboxes — recommended (default)") { t.Fatalf("init output did not explain the promoted default:\n%s", out.String()) } - for _, want := range []string{"Architecture emulation: best-effort", "QEMU/binfmt will be attempted; unsupported hosts continue with verified native containers and a warning."} { + for _, want := range []string{"Architecture emulation: native-only", "QEMU/binfmt is disabled by default; Docker Sandboxes runs native containers only."} { if !strings.Contains(out.String(), want) { t.Fatalf("Windows wizard review omitted %q:\n%s", want, out.String()) } } + if strings.Contains(out.String(), "QEMU/binfmt will be attempted") { + t.Fatalf("Windows wizard review unexpectedly enabled the QEMU attempt by default:\n%s", out.String()) + } } func TestPromotedDockerSandboxesPlatformUsesSharedHostGuestMapping(t *testing.T) { diff --git a/cmd/ephemeral-action-runner/init_wizard.go b/cmd/ephemeral-action-runner/init_wizard.go index 571aa9a..5423c03 100644 --- a/cmd/ephemeral-action-runner/init_wizard.go +++ b/cmd/ephemeral-action-runner/init_wizard.go @@ -497,6 +497,8 @@ func promptInitReview(out io.Writer, reader *bufio.Reader, draft initWizardDraft fmt.Fprintf(out, " Architecture emulation: %s\n", architectureEmulation) if architectureEmulation == config.DockerSandboxesArchitectureEmulationBestEffort { fmt.Fprintln(out, " QEMU/binfmt will be attempted; unsupported hosts continue with verified native containers and a warning.") + } else if architectureEmulation == config.DockerSandboxesArchitectureEmulationNativeOnly { + fmt.Fprintln(out, " QEMU/binfmt is disabled by default; Docker Sandboxes runs native containers only.") } } default: @@ -576,5 +578,5 @@ func renderInitWizardConfig(draft initWizardDraft) (string, error) { } func initDockerSandboxesArchitectureEmulation() string { - return config.DockerSandboxesArchitectureEmulationBestEffort + return config.DockerSandboxesArchitectureEmulationNativeOnly } diff --git a/configs/docker-sandboxes.example.yml b/configs/docker-sandboxes.example.yml index 0498b96..b611ebd 100644 --- a/configs/docker-sandboxes.example.yml +++ b/configs/docker-sandboxes.example.yml @@ -55,9 +55,12 @@ dockerSandboxes: # The wizard reads and records the exact active host-global policy fingerprint. policyGeneration: sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789 networkBaseline: open - # best-effort attempts bundled QEMU/binfmt, then continues with a warning when the sandbox supports native containers only. - # Use required only when every runner must execute foreign-architecture containers. - architectureEmulation: best-effort + # native-only disables QEMU/binfmt by default and verifies the native guest and private Docker architecture. + # Use best-effort or required only after validating the exact foreign-architecture workload. + architectureEmulation: native-only + # exclusive-auto permits one bounded stop-wait-start recovery per create-admission incident; inventory recovery keeps its bounded backoff retries, and observe never mutates sbx. + recoveryMode: exclusive-auto + recoveryQuiescenceSeconds: 60 additionalAllow: - api.github.com - '*.githubusercontent.com:443' diff --git a/docs/advanced/cross-architecture-containers.md b/docs/advanced/cross-architecture-containers.md index 4779b37..cac479c 100644 --- a/docs/advanced/cross-architecture-containers.md +++ b/docs/advanced/cross-architecture-containers.md @@ -54,7 +54,7 @@ The setup helper is privileged. Use it only in trusted workflows and treat its p | Docker Container | Run the setup action inside the disposable job before Docker or Compose uses a foreign image. No EPAR configuration key enables universal emulation. | | WSL | Run the setup action inside the WSL runner if its Linux Docker daemon needs a foreign image. An x64 WSL runner does not gain ARM64 execution merely by pulling an ARM64 image. | | Tart on Apple Silicon (retired) | The guest is ARM64. The optional Rosetta path is retained for existing configurations and is not equivalent to QEMU/binfmt. Use a distinct label and validate the exact image/workload. | -| Docker Sandboxes | Keep `provider.platform` native. The default `best-effort` mode tries the bundled QEMU/binfmt handlers and warns when only native execution is available. Set `required` only when foreign execution must be an admission requirement; `native-only` skips the attempt. | +| Docker Sandboxes | Keep `provider.platform` native. The default `native-only` mode skips QEMU/binfmt. Prefer a native or multi-platform image; `best-effort` and `required` are explicit opt-ins and are not a complete cross-architecture guarantee in `sbx` v0.39. | | GitHub-hosted Windows or macOS | These labels do not replace a Linux Docker daemon for container actions or service containers. Use a suitable Linux execution surface. | Keep architecture-specific jobs on a distinct `runs-on` label. Do not label an ARM64 runner as `ubuntu-latest`: GitHub's `ubuntu-latest` is a GitHub-managed environment, and x64 assumptions can fail on ARM64. @@ -65,7 +65,7 @@ For an amd64-only service on an ARM64 host, first try a published ARM64 or multi For an ARM64 image on an x64 Linux runner, follow the same process with `platforms: arm64` and `--platform linux/arm64`. Never treat a successful `docker pull` as the proof; run a container and check both the expected architecture output and the real workload. -Docker Sandboxes has no fixed advertised target matrix. `best-effort` and `required` install every handler available from the immutable host-platform `tonistiigi/binfmt` artifact and let QEMU handle an executable when a registered ELF signature matches. In `best-effort`, missing sandbox-kernel support produces a controller warning and foreign images fail normally at execution time while native jobs continue. `native-only` deliberately makes no foreign-execution claim. +Docker Sandboxes has no fixed advertised target matrix. The default `native-only` mode deliberately makes no foreign-execution claim. `best-effort` and `required` install every handler available from the immutable host-platform `tonistiigi/binfmt` artifact and let QEMU handle an executable when a registered ELF signature matches, but `sbx` v0.39 does not fully support this path. In `best-effort`, missing sandbox-kernel support produces a controller warning and foreign images fail normally at execution time while native jobs continue. For a job that requires another architecture, use a native target-architecture EPAR machine with Docker Sandboxes or a separate Docker Container configuration with trusted Docker-in-Docker QEMU support. ## References diff --git a/docs/advanced/docker-sandboxes-template.md b/docs/advanced/docker-sandboxes-template.md index 98de03c..c63c9c8 100644 --- a/docs/advanced/docker-sandboxes-template.md +++ b/docs/advanced/docker-sandboxes-template.md @@ -56,4 +56,4 @@ The direct build does not create a Docker staging image. Once the imported templ The source lock pins build tooling, Tini, helper inputs, and platform-specific inputs. The default policy checks mutable source and Actions runner selectors weekly at 07:00 local time; `./start image update` checks immediately. EPAR activates a new immutable template only after build, import, and exact readback succeed. -Docker Sandboxes native-platform operation is current on Linux, macOS, and Windows hosts based on completed cross-platform build, import, lifecycle, and cleanup testing. Architecture capability is separate: the wizard uses `best-effort`, which enables QEMU where sandbox-local `binfmt_misc` works and otherwise admits only the verified native platform with a warning. Run local admission and workload validation for the exact host and workload; this status does not claim independent certification. Any future certification record should bind the reviewed native-controller source/build, full template identity, cache ID, metadata/archive digests, architecture mode, and reviewed evidence. +Docker Sandboxes native-platform operation is current on Linux, macOS, and Windows hosts based on completed cross-platform build, import, lifecycle, and cleanup testing. Architecture capability is separate: the wizard uses `native-only`, which skips QEMU and admits only the verified native platform. Run local admission and workload validation for the exact host and workload; this status does not claim independent cross-architecture certification. Any future certification record should bind the reviewed native-controller source/build, full template identity, cache ID, metadata/archive digests, architecture mode, and reviewed evidence. diff --git a/docs/configuration.md b/docs/configuration.md index 97609c4..2275eca 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,7 +27,7 @@ External-outage supervision is an invocation policy, not YAML. Pass `--external- | Provider | Host and artifact model | Image defaults | Provider-only configuration | | --- | --- | --- | --- | -| `docker-sandboxes` | A Linux, macOS, or Windows host with healthy `sbx diagnose --output json` results builds and imports a native Linux runner template. EPAR attempts QEMU/binfmt by default and warns rather than blocking when the sandbox runtime supports native containers only. | `image.distribution: local-build` selects a Catthehacker source for the current build path. The preview `prebuilt` Act path acquires only EPAR's official verified reference and records the exact digest/attestation in local state. | `dockerSandboxes` is required; `provider.platform` is `linux/amd64` or `linux/arm64`; runner-group enforcement must be `enforce`; `architectureEmulation` selects `best-effort`, `required`, or `native-only`. | +| `docker-sandboxes` | A Linux, macOS, or Windows host with healthy `sbx diagnose --output json` results builds and imports a native Linux runner template. EPAR uses native-only architecture admission by default and does not attempt QEMU/binfmt unless the mode is explicitly changed. | `image.distribution: local-build` selects a Catthehacker source for the current build path. The preview `prebuilt` Act path acquires only EPAR's official verified reference and records the exact digest/attestation in local state. | `dockerSandboxes` is required; `provider.platform` is `linux/amd64` or `linux/arm64`; runner-group enforcement must be `enforce`; `architectureEmulation` selects `best-effort`, `required`, or `native-only`. | | `docker-container` | Compatibility provider: a Docker-compatible host creates an outer disposable runner with its own inner Docker daemon. | `docker-image`, `ghcr.io/catthehacker/ubuntu:full-latest`, output `epar-docker-container-catthehacker-ubuntu`. | Optional `provider.platform`; `docker` proxy and mirror settings apply to its private daemon. | | `wsl` | Compatibility provider: Windows WSL2 imports a Docker image or rootfs tar into disposable Linux distros. | Docker source defaults to Catthehacker full Ubuntu, x64, with output under `work/images/`. | `provider.installRoot` controls WSL storage. | | `tart` | Retired Apple Silicon Linux VM path retained for existing configurations and exact runtime/cleanup compatibility; it has no onboarding path. | `ghcr.io/cirruslabs/ubuntu:latest`, output `epar-ubuntu-24-arm64`. | `provider.network` and optional `provider.rosettaTag`. Validate the exact workload before relying on Rosetta. | @@ -178,7 +178,9 @@ If the complete subsection is absent, EPAR warns and uses the strict recommended | `image.sourceImage` | `ghcr.io/catthehacker/ubuntu:full-latest` | Required with Docker Sandboxes; must be the exact `full-latest` or `act-latest` profile. | Desired Catthehacker source selector; EPAR builds and imports the runnable template automatically. Specialized and custom tags remain available to Docker Container and WSL. | | `policyGeneration` | lowercase `sha256:<64-hex>`; no default | Required with Docker Sandboxes. | Recorded fingerprint of the host-global Balanced policy. | | `networkBaseline` | `open` or `balanced`; `open` | Docker Sandboxes. | `open` adds a sandbox-scoped public-egress rule while denying host aliases; it does not change the host-global policy. | -| `architectureEmulation` | `best-effort`, `required`, or `native-only`; `best-effort` | Docker Sandboxes. | `best-effort` attempts bundled QEMU handlers, then verifies the native guest and private Docker architecture and continues with a warning when QEMU/binfmt is unavailable. `required` fails creation unless QEMU handlers are active. `native-only` skips the QEMU attempt. | +| `architectureEmulation` | `best-effort`, `required`, or `native-only`; `native-only` | Docker Sandboxes. | `native-only` skips the QEMU attempt and verifies the native guest and private Docker architecture. `best-effort` attempts bundled QEMU handlers and continues with a warning when QEMU/binfmt is unavailable. `required` fails creation unless QEMU handlers are active. | +| `recoveryMode` | `exclusive-auto` or `observe`; `exclusive-auto` | Docker Sandboxes. | `exclusive-auto` is the default and permits EPAR one bounded stop-wait-start recovery attempt per create-admission incident with the known sandbox-container signature; bounded inventory control-plane failures retain their existing probe and backoff retries, but cannot restart the daemon while an admission incident remains open. `observe` leaves those failures unhandled, does not probe or mutate the daemon, and follows existing fail-closed startup or reconciliation behavior; use it only as an explicit maintenance opt-out. | +| `recoveryQuiescenceSeconds` | integer from `1` through `300`; `60` | Docker Sandboxes. | Time that the daemon must remain authoritatively stopped before EPAR starts it again. EPAR also requires repeated stable inventory after the restart. | | `additionalAllow` | unique hostname or `*.domain`, optional port; empty | Docker Sandboxes. | Adds sandbox-scoped allow resources. With `open`, it cannot re-allow EPAR's host-alias deny guardrails. | | `additionalDeny` | unique hostname or `*.domain`, optional port; empty | Docker Sandboxes. | Adds sandbox-scoped deny resources. A resource cannot be in both allow and deny lists. | | `stagingRoot` | canonical project-relative `.local/...` path; `.local/cache/docker-sandboxes/staging` | Docker Sandboxes. | Per-create disposable staging root; cannot be absolute, escape `.local`, or overlap `.local/bin` or `.local/state`. | diff --git a/docs/development/docker-sandboxes-prebuilt.md b/docs/development/docker-sandboxes-prebuilt.md index de7a835..417a82d 100644 --- a/docs/development/docker-sandboxes-prebuilt.md +++ b/docs/development/docker-sandboxes-prebuilt.md @@ -4,11 +4,13 @@ EPAR publishes its Docker Sandboxes template as an immutable multi-platform OCI The canonical source is `ghcr.io/catthehacker/ubuntu`. The public package is `ghcr.io/solutionforest/ephemeral-action-runner/docker-sandboxes-template`. Docker Hub is never used as a source fallback because its OCI identities may differ from GHCR even when the logical image content matches. -Act (`act-latest`) is the first accepted profile. Full (`full-latest`) uses the same publication, verification, and runtime contracts but remains candidate-only until its independent amd64 and arm64 acceptance cycle completes; its first protected promotion enables the Full stable policy atomically with the alias move. +Act (`act-latest`) and Full (`full-latest`) use the same publication, verification, and runtime contracts. The signed catalog records candidates before acceptance; runtime resolution follows only an active profile alias. ## Workflow triggers and hosted build gates -`.github/workflows/docker-sandboxes-images.yml` polls Act every six hours at minute 37 and Full on an offset six-hour schedule at minute 7, supports manual dispatch, and publishes for recipe-related pushes only on `main`. GitHub executes both cron expressions only from the default branch. Pull requests to `develop` or `main` that change a publisher, recipe, or committed-asset path run publisher, signed-evidence, and asset validation without logging in to GHCR, building a package, or pushing any manifest. This prevents a normal `develop` to `main` promotion from publishing the same source change twice. +`.github/workflows/docker-sandboxes-images.yml` checks the Catthehacker day-of-month cadence at 23:37 UTC for Full and 23:57 UTC for Act, approximately 6–12 hours after the upstream 12:00 UTC schedule, supports manual dispatch, and publishes for recipe-related pushes only on `main`. The `*/7` day-of-month expression mirrors the upstream calendar pattern rather than one fixed weekday. GitHub executes both cron expressions only from the default branch and may delay scheduled starts under load. Pull requests to `develop` or `main` that change a publisher, recipe, or committed-asset path run publisher, signed-evidence, and asset validation without logging in to GHCR, building a package, or pushing any manifest. This prevents a normal `develop` to `main` promotion from publishing the same source change twice. + +Before allocating hosted build runners, the resolve job reads the signed moving catalog and compares the complete source, recipe, runtime, schema, runner, and locked-tool tuple. A candidate or active entry must also have all hosted build and signed-evidence gates set, and an active entry must still be the selected profile alias. When exactly one entry matches, the job runs `verify-package` against that entry's immutable package reference; this checks registry metadata, signed referrers, and claims without pulling image layers. Only a successful verification produces a no-op. No match starts a hosted build, an ambiguous match fails closed, and a package or evidence verification failure never silently falls back to rebuilding. The amd64 build runs on GitHub-hosted `ubuntu-latest`; the arm64 build runs on GitHub-hosted `ubuntu-24.04-arm`. The workflow has no persistent self-hosted runner dependency and no `EPAR_PREBUILT_LIVE` switch. Full jobs first reclaim disposable hosted-runner tool caches, require at least 40 GiB free before allocating 8 GiB of swap, serialize BuildKit execution, and allow a three-hour timeout; failing that capacity gate leaves Full unpublished rather than silently changing its recipe or dropping a platform. Hosted jobs resolve the GHCR source descriptor, build from its immutable digest, inspect both runnable platform manifests by digest, assemble an index from those exact digests, require exactly two descriptors, run package smoke checks, generate and sign one index-level SLSA provenance statement and one index-level SPDX SBOM, verify their referrers, and publish an immutable signed candidate catalog. Platform builds disable BuildKit's additional per-platform SBOM/provenance indexes because EPAR's trust decision uses the separately signed index-level evidence and catalog. The index-level SPDX document records the package, recipe, and exact platform identities rather than reproducing BuildKit's more detailed component inventory; this deliberate narrower audit scope avoids four additional per-publication GHCR version rows without weakening the evidence enforced by EPAR. @@ -23,7 +25,7 @@ Every new package is first recorded as `candidate`. Before manual acceptance, pu - signed SLSA provenance and SPDX SBOM referrers; - an immutable catalog object and canonical tag such as `catalog-v1-pkg-<64 hex catalog digest>`. -Candidate publication does not move `catalog-v1` or the profile's `*-latest` alias. This permits first-catalog bootstrap: a candidate can be acquired through its exact signed immutable catalog even when no moving catalog exists yet. +On trusted `main`, candidate publication moves the signed `catalog-v1` ledger pointer but does not move the profile's `*-latest` alias. This makes a complete candidate discoverable to later metadata checks without activating it. Non-main candidate publication remains immutable-only, and first-catalog bootstrap still uses the exact signed immutable catalog. Catalog readers resolve an exact catalog manifest, validate its artifact/config/single-layer media contract, and fetch that layer by descriptor digest into a caller-chosen file. They never extract the publisher-supplied OCI layer title as a filesystem path. New catalogs are published from controlled relative filenames, while this descriptor path remains compatible with the initial catalogs that recorded absolute runner-temporary titles. @@ -121,7 +123,7 @@ The evidence input is one JSON object so the workflow remains below GitHub's ten {"amd64PlaywrightRunId":123,"amd64PlaywrightRunnerName":"","amd64DockerHubRunId":124,"amd64DockerHubRunnerName":"","amd64ReceiptSha256":"sha256:<64 hex>","arm64PlaywrightRunId":125,"arm64PlaywrightRunnerName":"","arm64DockerHubRunId":126,"arm64DockerHubRunnerName":"","arm64ReceiptSha256":"sha256:<64 hex>"} ``` -Before GitHub requests approval for the protected environment, an unprotected `Prepare protected promotion review` job verifies the signed candidate catalog and package evidence and writes a job summary containing the exact package, catalog, source, recipe, runtime, runner, platform, four acceptance-run links, runner names, and receipt hashes. The reviewer opens that completed job summary, follows the four authenticated private-repository links, completes its checklist, and only then approves the waiting `epar-prebuilt-promotion` deployment. The prepare job cannot approve a deployment or move a package tag. +Before GitHub requests approval for the protected environment, an unprotected `Prepare protected promotion review` job verifies the signed candidate catalog and package evidence and requires that the candidate catalog is the current `catalog-v1` head; this prevents an older ledger snapshot from overwriting newer candidates. It writes a job summary containing the exact package, catalog, source, recipe, runtime, runner, platform, four acceptance-run links, runner names, receipt hashes, and reviewed catalog head. The reviewer opens that completed job summary, follows the four authenticated private-repository links, completes its checklist, and only then approves the waiting `epar-prebuilt-promotion` deployment. The prepare job cannot approve a deployment or move a package tag. The `epar-prebuilt-promotion` environment must require an authorized reviewer. After approval, the protected job independently repeats the immutable catalog and package checks, rechecks the upstream source, appends two profile-bound platform acceptance records, requires exactly the two approved workflows per platform, and performs protected catalog compare-and-swap. It then signs and verifies the promoted catalog, moves `catalog-v1`, and moves the matching `act-latest` or `full-latest` alias last. Incomplete, failed, misrouted, single-platform, wrong-profile, wrong-workflow, alias-raced, or source-raced evidence cannot promote. diff --git a/docs/providers/docker-sandboxes.md b/docs/providers/docker-sandboxes.md index 543f07c..5039bc2 100644 --- a/docs/providers/docker-sandboxes.md +++ b/docs/providers/docker-sandboxes.md @@ -29,7 +29,7 @@ Choose Docker Sandboxes when its local checks pass and you want a microVM bounda ## Support Status -EPAR recommends this provider in the wizard by capability, not by an operating-system allowlist: Docker must work, `sbx diagnose --output json` must report at least one passing check and no failed checks, and the controller architecture must have a matching native guest template. After configuration is saved, ordinary startup additionally requires storage, template, and configured architecture-capability admission before any runner starts. The wizard uses best-effort QEMU on every host: sandbox runtimes with usable `binfmt_misc` enable the bundled handlers, while runtimes without it continue as verified native runners with a warning. Native lifecycle support does not certify foreign workloads. +EPAR recommends this provider in the wizard by capability, not by an operating-system allowlist: Docker must work, `sbx diagnose --output json` must report at least one passing check and no failed checks, and the controller architecture must have a matching native guest template. After configuration is saved, ordinary startup additionally requires storage, template, and configured architecture-capability admission before any runner starts. The wizard selects `native-only` on every host: it verifies the native guest and private Docker architecture without attempting bundled handlers. Native lifecycle support does not certify foreign workloads. ## Prerequisites @@ -71,7 +71,9 @@ image: dockerSandboxes: policyGeneration: sha256: networkBaseline: open - architectureEmulation: best-effort + architectureEmulation: native-only + recoveryMode: exclusive-auto + recoveryQuiescenceSeconds: 60 stagingRoot: .local/cache/docker-sandboxes/staging cpus: 4 memory: 8GiB @@ -92,9 +94,9 @@ The reusable template disables and masks Ubuntu's periodic apt units. Docker San ## Cross-Architecture Containers -`dockerSandboxes.architectureEmulation` is an explicit capability contract. The default `best-effort` mode copies the pinned `tonistiigi/binfmt:qemu-v10.2.3-68` installer and all static QEMU interpreters from the immutable template, then each newly created sandbox tries the equivalent of `binfmt --install all` as root inside its private VM. This does not run a privileged container on the host. When handlers become active, EPAR records QEMU capability normally. When the sandbox kernel reports that `binfmt_misc` is unavailable, EPAR verifies the configured native guest and private Docker architecture, emits a warning, and continues. +`dockerSandboxes.architectureEmulation` is an explicit capability contract. The default `native-only` mode skips the QEMU/binfmt attempt and verifies the configured native guest and private Docker architecture. The template still contains the pinned `tonistiigi/binfmt:qemu-v10.2.3-68` installer and static QEMU interpreters for explicit opt-in modes. With `best-effort`, each newly created sandbox tries the equivalent of `binfmt --install all` as root inside its private VM; this does not run a privileged container on the host. When handlers become active, EPAR records QEMU capability normally. When the sandbox kernel reports that `binfmt_misc` is unavailable, EPAR verifies the configured native guest and private Docker architecture, emits a warning, and continues. -`required` uses the same QEMU attempt but fails creation unless at least one bundled handler is active; select it only when foreign-architecture execution is a runner admission requirement. `native-only` skips the attempt and requires no EPAR-owned QEMU handler. Both native paths verify that the guest kernel and private Docker daemon match `provider.platform`. Configurations that omit the key default to `best-effort`, so ordinary native jobs are not blocked by a sandbox-runtime QEMU limitation. +`required` uses the same QEMU attempt but fails creation unless at least one bundled handler is active; select it only when foreign-architecture execution is a runner admission requirement. `native-only` skips the attempt and requires no EPAR-owned QEMU handler. Both native paths verify that the guest kernel and private Docker daemon match `provider.platform`. Configurations that omit the key default to `native-only`, so ordinary native jobs do not depend on a sandbox-runtime QEMU limitation. Native image processes remain native because foreign binfmt handlers match only their foreign ELF signatures. Docker manifest selection is unchanged: EPAR does not set `DOCKER_DEFAULT_PLATFORM`, QEMU cannot create a missing manifest, and a Compose service should use its `platform` property when a multi-platform tag must select a foreign variant deliberately. A single-architecture local image can run without a service override when its image metadata already identifies the foreign platform. Treat actual workload startup, health checks, networking, and performance as the compatibility proof. @@ -116,14 +118,14 @@ Each allocation receives an empty owner-restricted staging directory, but Action The listener identity is explicit and self-consistent: `agent` owns its home, XDG, runtime, and Docker configuration directories, and every workflow action and shell command inherits those exact paths. Template construction removes Docker credentials inherited from source-image user homes, and the registration path performs a second narrow scrub of `.docker/config.json` and `.dockercfg` across the actual passwd homes after sandbox boot while preserving `.docker/sandbox/locks`; verification rejects reusable artifacts that retain registry authentication or point identity-derived paths at another user. A workflow login can therefore write only to the disposable sandbox's Docker client configuration, and that file disappears with the sandbox. Registry authorization can still be changed by Docker Sandboxes' host-side credential proxy as described below. -Docker Sandboxes can automatically forward the host SSH agent when its shared daemon inherits `SSH_AUTH_SOCK`. That would expose a host credential capability to every sandbox created by that daemon, so EPAR rejects any guest containing `SSH_AUTH_SOCK`, `SSH_AUTH_SOCK_GATEWAY`, `SSH_AGENT_PID`, or `/run/ssh-agent.sock`. EPAR removes these variables whenever it launches Docker Sandboxes commands, so a stopped daemon that those commands auto-start is sanitized. EPAR cannot repair an already-running daemon that another shell or tool started with forwarding enabled. If creation reports the known `failed to run sandbox container` signature, EPAR preserves that original error and adds this SSH-daemon remediation hint; unrelated create failures do not receive it. EPAR never stops or restarts a running shared daemon automatically. Coordinate with every process using Docker Sandboxes on the host, then stop the daemon and restart it from a sanitized environment before retrying EPAR: +Docker Sandboxes can automatically forward the host SSH agent when its shared daemon inherits `SSH_AUTH_SOCK`. That would expose a host credential capability to every sandbox created by that daemon, so EPAR rejects any guest containing `SSH_AUTH_SOCK`, `SSH_AUTH_SOCK_GATEWAY`, `SSH_AGENT_PID`, or `/run/ssh-agent.sock`. EPAR removes these variables whenever it launches Docker Sandboxes commands, so a stopped daemon that those commands auto-start is sanitized. If creation reports the known `failed to run sandbox container` signature, EPAR preserves that original error and adds this SSH-daemon remediation hint; unrelated create failures do not receive it. In the default `recoveryMode: exclusive-auto`, the pool classifies that immediate create stderr as a bounded admission incident and may perform one stop-wait-start recovery using the existing host-global gates, then retry lifecycle reconciliation. It also recovers bounded inventory control-plane failures. `observe` never probes or mutates the daemon. Coordinate with every process using Docker Sandboxes on the host before any interruption; if the bounded automatic attempt has already been used or manual recovery is required, stop the daemon and restart it from a sanitized environment before retrying EPAR: ```sh sbx daemon stop env -u SSH_AUTH_SOCK -u SSH_AUTH_SOCK_GATEWAY -u SSH_AGENT_PID sbx daemon start --detach ``` -Do not relax the verification or merely delete the relay socket: the gateway setting is itself a forwarding capability and the daemon's inherited environment is authoritative for subsequently created sandboxes. A stopped daemon may be started explicitly from the sanitized environment above or auto-started by an EPAR-launched `sbx` command; EPAR does not mutate a running shared daemon to recover from this failure. +Do not relax the verification or merely delete the relay socket: the gateway setting is itself a forwarding capability and the daemon's inherited environment is authoritative for subsequently created sandboxes. A stopped daemon may be started explicitly from the sanitized environment above or auto-started by an EPAR-launched `sbx` command. EPAR limits automatic admission recovery to one daemon restart per incident, including controller reconciliation retries, and never invokes `sbx reset`, `sbx logout`, or wildcard deletion for this condition. ## Docker Hub Credentials and Transparent Egress diff --git a/docs/security.md b/docs/security.md index c442589..9e3879f 100644 --- a/docs/security.md +++ b/docs/security.md @@ -30,9 +30,9 @@ EPAR intentionally does not implement a Docker-socket provider. A runner that co Docker Container uses a privileged outer container with a private inner Docker daemon. That gives good cleanup and Docker resource separation for each job, but it is still trusted-job infrastructure because `--privileged` weakens container isolation. -Docker Sandboxes places the listener, guest filesystem, and private Docker daemon inside a dedicated microVM sandbox. It provides EPAR's strongest current host boundary and is the primary provider on Linux, macOS, and Windows when the supported-platform, Docker, and machine-readable `sbx` readiness checks pass. Startup then performs the remaining storage, template, policy-rule, native architecture, runtime, and registration admission checks and fails closed. QEMU is a workload compatibility capability rather than part of the isolation boundary: `best-effort` warns and continues when sandbox-local `binfmt_misc` is unavailable, `required` makes it an admission requirement, and `native-only` skips it. +Docker Sandboxes places the listener, guest filesystem, and private Docker daemon inside a dedicated microVM sandbox. It provides EPAR's strongest current host boundary and is the primary provider on Linux, macOS, and Windows when the supported-platform, Docker, and machine-readable `sbx` readiness checks pass. Startup then performs the remaining storage, template, policy-rule, native architecture, runtime, and registration admission checks and fails closed. QEMU is a workload compatibility capability rather than part of the isolation boundary and is disabled by default: `native-only` skips it, `best-effort` warns and continues when sandbox-local `binfmt_misc` is unavailable, and `required` makes it an admission requirement. -Docker Sandboxes may forward a host SSH agent when its shared daemon inherits `SSH_AUTH_SOCK`. EPAR strips SSH-agent variables from child commands, which sanitizes a stopped daemon auto-started through those commands, and rejects any sandbox exposing the socket, gateway, or agent PID. A `failed to run sandbox container` create error receives a conditional remediation hint that preserves the original error and explains how to coordinate a sanitized daemon restart. EPAR never stops or restarts a running shared daemon automatically, and operators must restart an already-running daemon with those variables unset rather than weakening the check. +Docker Sandboxes may forward a host SSH agent when its shared daemon inherits `SSH_AUTH_SOCK`. EPAR strips SSH-agent variables from child commands, which sanitizes a stopped daemon auto-started through those commands, and rejects any sandbox exposing the socket, gateway, or agent PID. A `failed to run sandbox container` create error receives a conditional remediation hint that preserves the original error and explains how to coordinate a sanitized daemon restart. In the default `exclusive-auto` mode, EPAR may make one bounded stop-wait-start recovery attempt for that exact create-stage signature, using the existing host-global recovery gates and sanitized detached start; it also recovers bounded inventory control-plane failures. `observe` never probes or mutates the daemon. Operators must coordinate with other daemon users and must not weaken the admission check or use `sbx reset`/`sbx logout` as recovery. Tart is a retired Apple Silicon macOS VM path retained for existing configurations. It provides a VM boundary, but workflows still control the guest and any secrets exposed to the job. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 7ffa6ad..8460b07 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -9,6 +9,7 @@ Start with the symptom that most closely matches the failure. Regardless of prov - [Windows no-Go startup prints an HTTP/2 named-pipe diagnostic](#windows-no-go-startup-prints-an-http2-named-pipe-diagnostic) - [A Docker workload fails with an architecture error](#a-docker-workload-fails-with-an-architecture-error) - [Docker Sandboxes is unavailable or its preflight fails](#docker-sandboxes-is-unavailable-or-its-preflight-fails) +- [Docker Sandboxes daemon health passes but inventory hangs](#docker-sandboxes-daemon-health-passes-but-inventory-hangs) - [Docker Sandboxes rejects template, policy, or capacity](#docker-sandboxes-rejects-template-policy-or-capacity) - [Docker Sandboxes creation fails after a runtime-helper prompt](#docker-sandboxes-creation-fails-after-a-runtime-helper-prompt) - [Docker Sandboxes rejects a staging workspace because SSH-agent forwarding is present](#docker-sandboxes-rejects-a-staging-workspace-because-ssh-agent-forwarding-is-present) @@ -130,7 +131,11 @@ docker compose config `no matching manifest` means the image does not publish the requested platform; emulation cannot create a missing manifest. A platform warning alone does not prove failure, and exit code `139` alone does not prove an architecture mismatch. Enabling emulation does not change normal manifest selection, so request the foreign image with `docker run --platform ...` or the affected Compose service's `platform:` property. EPAR does not inject `DOCKER_DEFAULT_PLATFORM`. -For Docker Sandboxes, keep `provider.platform` native. The default `dockerSandboxes.architectureEmulation: best-effort` attempts the template's pinned QEMU handlers inside the sandbox VM. If `binfmt_misc` is unavailable, EPAR verifies the guest and private Docker architectures, warns that foreign containers may fail, and continues with native jobs. Use `required` only when missing QEMU must reject the runner, or `native-only` to skip the attempt. Do not add a privileged installer to the workflow: it cannot repair a sandbox kernel that does not expose the filesystem. +For Docker Sandboxes, keep `provider.platform` native. QEMU/binfmt is disabled by default on Linux, macOS, and Windows with `dockerSandboxes.architectureEmulation: native-only`, which verifies the guest and private Docker architecture without attempting foreign execution. As of `sbx` v0.39, QEMU is not fully supported inside Docker Sandboxes, so a passing diagnostic, template build, or image pull is not proof that a foreign-architecture workload will run. Prefer building or publishing a multi-platform image instead of relying on emulation. + +If a job requires another architecture, such as X64 on ARM64 or ARM64 on X64, either run EPAR on a machine with the target architecture when using the Docker Sandboxes provider, or run a separate EPAR instance/configuration with `provider.type: docker-container`. Docker Container uses Docker-in-Docker for isolation and can support QEMU/binfmt inside its private daemon for trusted jobs. Select it in the setup wizard by choosing `C. Show compatibility providers`, then `Docker Container`. Keep the target-architecture pool on a distinct runner label and verify the actual workload rather than only pulling the image. + +The `best-effort` and `required` modes remain available as explicit configuration choices, but they do not make Docker Sandboxes QEMU fully supported on `sbx` v0.39. Do not add a privileged installer to a Docker Sandboxes workflow: it cannot repair a sandbox kernel that does not expose the required filesystem. ## Docker Sandboxes is unavailable or its preflight fails @@ -148,6 +153,22 @@ sbx diagnose --output json EPAR requires a controller architecture with an available Linux guest template and at least one diagnostic pass with zero failures. Diagnostic warnings and skipped checks remain visible but do not disable the provider. Review the failed item and its hint in the JSON output, fix the prerequisite, then choose `R` to refresh availability; do not manually force a provider selection or substitute a compatibility provider for a configured Docker Sandboxes pool. Use `C. Show compatibility providers` only when you deliberately intend to create or maintain a compatibility configuration. +## Docker Sandboxes daemon health passes but inventory hangs + +### Symptom + +`sbx daemon status --json` reports `running` and `sbx diagnose -o json` passes, but `sbx ls --json` hangs, returns an empty response slowly, or ends with a runtime-list cancellation. EPAR may repeatedly report `context canceled`, quarantine instances after the host-trust or inventory deadline, and keep the controller process alive with no ready capacity. + +### Diagnosis and remediation + +The daemon health and diagnostic commands do not prove that the `/sandbox` inventory path or a managed sandbox's inner Docker API is responsive. Run `sbx ls --json` only with an external operating-system timeout; record whether it returns valid JSON, its elapsed time, and whether stderr reports a cancelled runtime listing. Treat a slow or cancelled inventory response as provider state unknown even when daemon status and diagnostics pass. + +Preserve the controller output, daemon log, process tree, durable pool lifecycle state, and a before/after count of the daemon's managed Docker-socket descriptors. Do not repeatedly invoke unbounded `sbx ls`, start a second EPAR controller, use `sbx reset`, remove a sandbox by prefix, prune Docker state, or manually delete GitHub runners. EPAR intentionally quarantines uncertain resources and keeps them counted against `pool.instances` until exact ownership and cleanup can be verified. + +With the default `dockerSandboxes.recoveryMode: exclusive-auto`, EPAR keeps the controller alive, acquires a host-global recovery lock, and performs a cold stop, authoritative stopped-state confirmation, configured quiescence interval, sanitized detached start, and repeated inventory verification. During that sequence it lets already-running provider commands drain and gates new provider commands so another lifecycle operation cannot race the daemon restart. It backs off after failed attempts and preserves exact capacity while recovery is unresolved. The recovery path never invokes `sbx reset`, `sbx logout`, `sbx prune`, or wildcard deletion. Set `recoveryMode: observe` only when an operator deliberately needs to prevent daemon mutation during maintenance. + +If automatic recovery is disabled or the controller is not running, stop only the affected controller before a manual daemon interruption. Then run `sbx daemon stop`, wait until `sbx daemon status --json` reports `stopped`, and start with `env -u SSH_AUTH_SOCK -u SSH_AUTH_SOCK_GATEWAY -u SSH_AGENT_PID sbx daemon start --detach`. Before starting a new EPAR controller, require `sbx daemon status --json` to report `running`, `sbx diagnose -o json` to have no failed checks, and repeated bounded `sbx ls --json` calls to complete with valid JSON in a stable time. A passing diagnostic report without a responsive inventory is not a healthy Sandbox provider. + ## Docker Sandboxes rejects template, policy, or capacity ### Symptom @@ -186,14 +207,14 @@ Sandbox creation reaches `verify dedicated docker sandbox staging workspace` and Docker Sandboxes may forward the host SSH agent when its shared daemon inherits the host's agent environment. EPAR rejects the resulting sandbox because the forwarded socket or gateway could let a workflow use host SSH credentials. This is not evidence that the staging mount is missing or read-only, and deleting only `/run/ssh-agent.sock` is insufficient when the forwarding gateway remains configured. -EPAR never stops or restarts a running shared daemon automatically. Coordinate the interruption with every process using the shared Docker Sandboxes daemon, then stop it and restart it with all forwarding variables removed before retrying EPAR: +In the default `recoveryMode: exclusive-auto`, EPAR treats the known immediate create-stage signature as a bounded admission incident and may perform one stop-wait-start recovery using the existing host-global gates, then retry lifecycle reconciliation. The recovery is limited to one daemon restart per incident, including controller reconciliation retries; `recoveryMode: observe` never probes or mutates the daemon. Coordinate the interruption with every process using the shared Docker Sandboxes daemon. If the automatic attempt has already been used or manual recovery is required, stop it and restart it with all forwarding variables removed before retrying EPAR: ```sh sbx daemon stop env -u SSH_AUTH_SOCK -u SSH_AUTH_SOCK_GATEWAY -u SSH_AGENT_PID sbx daemon start --detach ``` -EPAR strips these variables from Docker Sandboxes commands it launches, so a stopped daemon auto-started through those commands is sanitized, but an already-running daemon retains the environment with which another shell or tool started it. A stopped daemon can also be started explicitly from the sanitized environment above; EPAR does not mutate a running shared daemon as a recovery action. Do not disable this admission check or forward an agent into a reusable runner template. If the failed creation predates the immutable-receipt fix, preserve its reported sandbox UUID and use exact provider cleanup; never delete a same-name resource by prefix alone. +EPAR strips these variables from Docker Sandboxes commands it launches, so a stopped daemon auto-started through those commands is sanitized, but an already-running daemon retains the environment with which another shell or tool started it. Do not disable this admission check, use `sbx reset` or `sbx logout`, or forward an agent into a reusable runner template. If the failed creation predates the immutable-receipt fix, preserve its reported sandbox UUID and use exact provider cleanup; never delete a same-name resource by prefix alone. ## Docker Hub login succeeds but a private pull is denied in Docker Sandboxes @@ -209,6 +230,8 @@ Inspect the host Docker Sandboxes daemon log for a message that the proxy is ove On a Windows controller with `image.hostTrustMode: overlay`, `/run/epar/egress-relay-active` must be a root-owned regular marker, `docker info` must report `http://127.0.0.1:3129` as its HTTPS proxy and an empty HTTP proxy, and `sbx policy log ` must contain a fresh allowed `transparent` record for the controller relay port, normally rendered as `localhost:`. A fresh Docker Hub `forward` record during activation is an admission failure. `NoProxy=*` is only the pre-activation bootstrap contract or the runtime contract for configurations that do not require the Windows relay; a Windows overlay job never falls back to that route. In either mode `/etc/docker/daemon.json` must be root-owned and non-symlinked. Stop the controller, let exact cleanup finish, rebuild, and use a newly created runner after any contract mismatch. +If host-trust relay activation fails, start with `work/logs/epar-last-error.log`. The error includes a fixed stage such as `private-dockerd-contract`, `registry-tls-proof`, or `guest-bridge-health`; use that stage to select the matching guest and Docker Sandboxes diagnostics. These stage messages are intentionally redacted and do not include the relay token or configuration payload. + If a transparent Docker Sandboxes connection presents `Norton Web/Mail Shield Untrusted Root` or another antivirus-generated untrusted issuer, do not add that issuer to EPAR's trust overlay. It is a synthetic error certificate indicating that the inspector rejected Docker Sandboxes' upstream path, not a missing ordinary inspection root. Current Windows overlay runners avoid that path by making the native controller host establish the public TCP connection. Workflow clients retain end-to-end TLS through the raw listener and validate the normal host-approved inspection chain with `/opt/epar/trust/ca-bundle.pem`. The Docker listener terminates daemon TLS with a root-only per-sandbox ephemeral authority and independently verifies the upstream host-approved chain, working around Docker Engine's stalled HelloRetryRequest without exposing registry credentials to the host relay. Failure of either listener, daemon restart/readback, Registry TLS proof, or fresh relay policy evidence blocks registration. Do not set `DOCKER_SANDBOXES_NO_PROXY` expecting it to disable credential injection. That host variable only excludes destinations from an optional upstream proxy used after traffic reaches the mandatory Sandbox proxy. Replacing `docker/login-action` with `docker login`, combining login and pull in one shell step, or changing `DOCKER_CONFIG` also leaves an old daemon's forward route unchanged. diff --git a/internal/config/config.go b/internal/config/config.go index 83080b1..9a29f18 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -183,25 +183,32 @@ type DockerConfig struct { // the exact imported template identity is stored in EPAR's local artifact // receipt rather than user configuration. type DockerSandboxesConfig struct { - PolicyGeneration string - NetworkBaseline string - ArchitectureEmulation string - AdditionalAllow []string - AdditionalDeny []string - StagingRoot string - CPUs int - Memory string - RootDisk string - DockerDisk string - MaxConcurrentCreates int + PolicyGeneration string + NetworkBaseline string + ArchitectureEmulation string + RecoveryMode string + RecoveryQuiescenceSeconds int + AdditionalAllow []string + AdditionalDeny []string + StagingRoot string + CPUs int + Memory string + RootDisk string + DockerDisk string + MaxConcurrentCreates int } const ( - DockerSandboxesNetworkBaselineOpen = "open" - DockerSandboxesNetworkBaselineBalanced = "balanced" - DockerSandboxesArchitectureEmulationBestEffort = "best-effort" - DockerSandboxesArchitectureEmulationRequired = "required" - DockerSandboxesArchitectureEmulationNativeOnly = "native-only" + DockerSandboxesNetworkBaselineOpen = "open" + DockerSandboxesNetworkBaselineBalanced = "balanced" + DockerSandboxesArchitectureEmulationBestEffort = "best-effort" + DockerSandboxesArchitectureEmulationRequired = "required" + DockerSandboxesArchitectureEmulationNativeOnly = "native-only" + DockerSandboxesRecoveryModeExclusiveAuto = "exclusive-auto" + DockerSandboxesRecoveryModeObserve = "observe" + DockerSandboxesDefaultRecoveryQuiescenceSeconds = 60 + DockerSandboxesMinimumRecoveryQuiescenceSeconds = 1 + DockerSandboxesMaximumRecoveryQuiescenceSeconds = 300 ) var dockerSandboxesOpenDefaultDenyResources = []string{ @@ -307,14 +314,16 @@ func Default() Config { }, }, DockerSandboxes: DockerSandboxesConfig{ - NetworkBaseline: DockerSandboxesNetworkBaselineOpen, - ArchitectureEmulation: DockerSandboxesArchitectureEmulationBestEffort, - StagingRoot: ".local/cache/docker-sandboxes/staging", - CPUs: 4, - Memory: "8GiB", - RootDisk: DockerSandboxesAutomaticRootDisk, - DockerDisk: DockerSandboxesDefaultDockerDisk, - MaxConcurrentCreates: 2, + NetworkBaseline: DockerSandboxesNetworkBaselineOpen, + ArchitectureEmulation: DockerSandboxesArchitectureEmulationNativeOnly, + RecoveryMode: DockerSandboxesRecoveryModeExclusiveAuto, + RecoveryQuiescenceSeconds: DockerSandboxesDefaultRecoveryQuiescenceSeconds, + StagingRoot: ".local/cache/docker-sandboxes/staging", + CPUs: 4, + Memory: "8GiB", + RootDisk: DockerSandboxesAutomaticRootDisk, + DockerDisk: DockerSandboxesDefaultDockerDisk, + MaxConcurrentCreates: 2, }, Timeouts: TimeoutConfig{ BootSeconds: 180, @@ -759,6 +768,14 @@ func apply(cfg *Config, section, key, value string) error { cfg.DockerSandboxes.NetworkBaseline = strings.ToLower(value) case "architectureEmulation": cfg.DockerSandboxes.ArchitectureEmulation = value + case "recoveryMode": + cfg.DockerSandboxes.RecoveryMode = strings.ToLower(value) + case "recoveryQuiescenceSeconds": + v, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("invalid dockerSandboxes.recoveryQuiescenceSeconds: %w", err) + } + cfg.DockerSandboxes.RecoveryQuiescenceSeconds = v case "additionalAllow", "additionalDeny": return setListValue(cfg, section, key, parseList(value)) case "stagingRoot": @@ -1369,6 +1386,12 @@ func ValidateDockerSandboxes(sandboxes DockerSandboxesConfig) error { if sandboxes.ArchitectureEmulation != DockerSandboxesArchitectureEmulationBestEffort && sandboxes.ArchitectureEmulation != DockerSandboxesArchitectureEmulationRequired && sandboxes.ArchitectureEmulation != DockerSandboxesArchitectureEmulationNativeOnly { return fmt.Errorf("unsupported dockerSandboxes.architectureEmulation %q; supported values are best-effort, required, and native-only", sandboxes.ArchitectureEmulation) } + if sandboxes.RecoveryMode != DockerSandboxesRecoveryModeExclusiveAuto && sandboxes.RecoveryMode != DockerSandboxesRecoveryModeObserve { + return fmt.Errorf("unsupported dockerSandboxes.recoveryMode %q; supported values are exclusive-auto and observe", sandboxes.RecoveryMode) + } + if sandboxes.RecoveryQuiescenceSeconds < DockerSandboxesMinimumRecoveryQuiescenceSeconds || sandboxes.RecoveryQuiescenceSeconds > DockerSandboxesMaximumRecoveryQuiescenceSeconds { + return fmt.Errorf("dockerSandboxes.recoveryQuiescenceSeconds must be between %d and %d", DockerSandboxesMinimumRecoveryQuiescenceSeconds, DockerSandboxesMaximumRecoveryQuiescenceSeconds) + } if err := validateDockerSandboxHostnameList("additionalAllow", sandboxes.AdditionalAllow); err != nil { return err } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index af038e0..de0ca93 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1588,6 +1588,8 @@ security: dockerSandboxes: policyGeneration: sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789 networkBaseline: balanced + recoveryMode: observe + recoveryQuiescenceSeconds: 90 additionalAllow: [api.github.com, '*.githubusercontent.com:443'] additionalDeny: - telemetry.example.invalid @@ -1619,9 +1621,15 @@ dockerSandboxes: if got, want := cfg.DockerSandboxes.Memory, "4GiB"; got != want { t.Fatalf("dockerSandboxes.memory = %q, want %q", got, want) } - if got, want := cfg.DockerSandboxes.ArchitectureEmulation, DockerSandboxesArchitectureEmulationBestEffort; got != want { + if got, want := cfg.DockerSandboxes.ArchitectureEmulation, DockerSandboxesArchitectureEmulationNativeOnly; got != want { t.Fatalf("dockerSandboxes.architectureEmulation = %q, want omitted configuration to default to %q", got, want) } + if got, want := cfg.DockerSandboxes.RecoveryMode, DockerSandboxesRecoveryModeObserve; got != want { + t.Fatalf("dockerSandboxes.recoveryMode = %q, want %q", got, want) + } + if got, want := cfg.DockerSandboxes.RecoveryQuiescenceSeconds, 90; got != want { + t.Fatalf("dockerSandboxes.recoveryQuiescenceSeconds = %d, want %d", got, want) + } if got, want := cfg.DockerSandboxes.AdditionalAllow, []string{"api.github.com", "*.githubusercontent.com:443"}; !slices.Equal(got, want) { t.Fatalf("dockerSandboxes.additionalAllow = %#v, want %#v", got, want) } @@ -1665,6 +1673,20 @@ func TestLoadDockerSandboxesArchitectureEmulation(t *testing.T) { } } +func TestDockerSandboxesRecoveryDefaults(t *testing.T) { + cfg := Default() + if got, want := cfg.DockerSandboxes.RecoveryMode, DockerSandboxesRecoveryModeExclusiveAuto; got != want { + t.Fatalf("dockerSandboxes.recoveryMode = %q, want default %q", got, want) + } + if got, want := cfg.DockerSandboxes.RecoveryQuiescenceSeconds, DockerSandboxesDefaultRecoveryQuiescenceSeconds; got != want { + t.Fatalf("dockerSandboxes.recoveryQuiescenceSeconds = %d, want default %d", got, want) + } + cfg.DockerSandboxes.PolicyGeneration = "sha256:" + strings.Repeat("a", 64) + if err := ValidateDockerSandboxes(cfg.DockerSandboxes); err != nil { + t.Fatalf("ValidateDockerSandboxes() rejected recovery defaults: %v", err) + } +} + func TestValidateDockerSandboxesRejectsInvalidArchitectureEmulation(t *testing.T) { for _, value := range []string{"", "disabled", "Required", "native"} { t.Run(value, func(t *testing.T) { @@ -1714,6 +1736,20 @@ func TestValidateDockerSandboxesRejectsInvalidPreviewConfiguration(t *testing.T) name: "network baseline is unsupported", mutate: func(cfg *Config) { cfg.DockerSandboxes.NetworkBaseline = "locked-down" }, }, + { + name: "recovery mode is unsupported", + mutate: func(cfg *Config) { cfg.DockerSandboxes.RecoveryMode = "shared-auto" }, + }, + { + name: "recovery quiescence is not positive", + mutate: func(cfg *Config) { cfg.DockerSandboxes.RecoveryQuiescenceSeconds = 0 }, + }, + { + name: "recovery quiescence exceeds bound", + mutate: func(cfg *Config) { + cfg.DockerSandboxes.RecoveryQuiescenceSeconds = DockerSandboxesMaximumRecoveryQuiescenceSeconds + 1 + }, + }, { name: "allowlist wildcard is unsafe", mutate: func(cfg *Config) { cfg.DockerSandboxes.AdditionalAllow = []string{"**.example.test"} }, diff --git a/internal/image/docker_sandboxes_test.go b/internal/image/docker_sandboxes_test.go index 451c80d..d8d2edd 100644 --- a/internal/image/docker_sandboxes_test.go +++ b/internal/image/docker_sandboxes_test.go @@ -504,6 +504,25 @@ func TestDockerSandboxesDockerDaemonBootstrapsThenUsesAuthenticatedHostTrustRela `((keys - ["proxies", "registry-mirrors"]) | length) == 0 and .proxies == {"https-proxy": $proxy, "no-proxy": $no_proxy}`, `daemon_backup="${config_dir}/docker-daemon.pre-relay.json"`, `rollback_daemon()`, + `relay_operation="activation"`, + `echo "EPAR host-trust relay: ${relay_operation} failed at ${failed_stage} (exit=${status})" >&2`, + `relay_stage="bootstrap"`, + `relay_stage="validate-daemon-config"`, + `relay_stage="commit"`, + `relay_stage="validate-guest-relay"`, + `relay_stage="install-relay-ca"`, + `relay_stage="write-relay-config"`, + `relay_stage="publish-relay-config"`, + `relay_stage="guest-bridge-health"`, + `relay_stage="recover-daemon-transaction"`, + `relay_stage="detect-private-dockerd"`, + `relay_stage="configure-private-dockerd"`, + `relay_stage="restart-private-dockerd"`, + `relay_stage="private-dockerd-contract"`, + `relay_stage="registry-tls-proof"`, + `relay_stage="publish-active-marker"`, + `relay_stage="rollback-daemon"`, + `relay_stage="rollback-relay-ca"`, `if [[ "${mode}" == "--commit" ]]`, `if [[ "${mode}" == "--rollback" ]]`, `mv -f "${daemon_config}.rollback.new" "${daemon_config}"`, @@ -519,6 +538,13 @@ func TestDockerSandboxesDockerDaemonBootstrapsThenUsesAuthenticatedHostTrustRela t.Fatalf("Docker Sandboxes runtime relay activation omitted %q", required) } } + failureDiagnostic := `echo "EPAR host-trust relay: ${relay_operation} failed at ${failed_stage} (exit=${status})" >&2` + capturedStageIndex := strings.Index(activation, `failed_stage="${relay_stage}"`) + failureDiagnosticIndex := strings.Index(activation, failureDiagnostic) + rollbackStageIndex := strings.Index(activation, `relay_stage="rollback-daemon"`) + if capturedStageIndex < 0 || failureDiagnosticIndex < capturedStageIndex || rollbackStageIndex < failureDiagnosticIndex { + t.Fatal("Docker Sandboxes relay failure diagnostic must capture the original stage before rollback") + } } func TestDockerSandboxesDisabledTrustPolicyIsExplicit(t *testing.T) { diff --git a/internal/pool/control_plane_recovery.go b/internal/pool/control_plane_recovery.go new file mode 100644 index 0000000..8c59228 --- /dev/null +++ b/internal/pool/control_plane_recovery.go @@ -0,0 +1,273 @@ +package pool + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/solutionforest/ephemeral-action-runner/internal/config" + "github.com/solutionforest/ephemeral-action-runner/internal/provider" +) + +const ( + providerRecoveryDaemonStop = 2 * time.Minute + providerRecoveryDaemonReadback = 30 * time.Second + providerRecoveryProbeTimeout = 45 * time.Second + providerRecoveryProbeCount = 3 + providerRecoveryProbeInterval = 5 * time.Second + providerRecoveryLockRetry = 10 * time.Second + providerRecoveryBackoffInitial = time.Minute + providerRecoveryBackoffMaximum = 30 * time.Minute + providerRecoverySafetyMargin = time.Minute +) + +// recoverProviderControlPlane performs the shared orchestration around a +// provider-owned recovery operation. The provider performs the exact daemon +// stop/start sequence; the pool owns policy, cross-process exclusion, retry +// backoff, and repeated post-recovery inventory verification. +// +// The handled result means the caller should retain its current lifecycle map +// and retry rather than clean up or terminate the pool. A provider that is not +// recovery-capable, an explicit observe mode, or a non-control-plane failure +// returns handled=false so existing fail-closed behavior remains unchanged. +func (m *Manager) recoverProviderControlPlane(ctx context.Context, cause error) (handled bool, err error) { + admissionFailure := errors.Is(cause, provider.ErrControlPlaneAdmissionFailure) + if (!admissionFailure && !errors.Is(cause, provider.ErrControlPlaneFailure)) || !m.dockerSandboxesExclusiveRecovery() { + return false, nil + } + recoverer, ok := m.providerLifecycle().(provider.ControlPlaneRecoverer) + if !ok { + return false, nil + } + if !m.providerRecoveryWindowReady() { + return true, nil + } + if permitted, admissionIncident := m.reserveProviderRecovery(admissionFailure); !permitted { + next := m.scheduleProviderRecovery(providerRecoveryBackoffMaximum) + if admissionFailure || admissionIncident { + m.warnf("Docker Sandboxes create-admission recovery was already attempted; preserving exact capacity and retrying after %s\n", time.Until(next).Round(time.Second)) + } else { + m.warnf("Docker Sandboxes inventory recovery is suppressed while a create-admission incident remains open; preserving exact capacity and retrying after %s\n", time.Until(next).Round(time.Second)) + } + return true, nil + } + + // The timeout may have been caused by a transient runtime stall that + // cleared before recovery began. Recheck first so an already-healthy daemon + // is never restarted unnecessarily. + if !admissionFailure { + if probeErr := m.probeProviderInventory(ctx); probeErr == nil { + m.resetProviderRecovery() + m.infof("Docker Sandboxes inventory recovered before daemon intervention; resuming pool reconciliation\n") + return true, nil + } else if ctx.Err() != nil { + return true, ctx.Err() + } + } + + attempt := m.beginProviderRecoveryAttempt() + recoveryCause := "inventory failure" + if admissionFailure { + recoveryCause = "create-admission failure" + } + m.warnf("Docker Sandboxes control-plane recovery attempt %d starting after %s: %v\n", attempt, recoveryCause, cause) + quiescence := time.Duration(m.Config.DockerSandboxes.RecoveryQuiescenceSeconds) * time.Second + if quiescence <= 0 { + quiescence = config.DockerSandboxesDefaultRecoveryQuiescenceSeconds * time.Second + } + recoveryCtx, cancel := context.WithTimeout(ctx, providerRecoveryBudgetFor(quiescence)) + defer cancel() + if recoveryErr := recoverer.RecoverControlPlane(recoveryCtx, provider.ControlPlaneRecoveryRequest{Quiescence: quiescence}); recoveryErr != nil { + if errors.Is(recoveryErr, provider.ErrControlPlaneRecoveryBusy) { + if admissionFailure { + m.cancelProviderAdmissionRecovery() + } + m.cancelProviderRecoveryAttempt() + next := m.scheduleProviderRecovery(providerRecoveryLockRetry) + m.warnf("Docker Sandboxes control-plane recovery is already running on this host; preserving exact capacity and retrying after %s\n", time.Until(next).Round(time.Second)) + return true, nil + } + if ctx.Err() != nil { + return true, ctx.Err() + } + attempt, next := m.recordProviderRecoveryFailure() + m.warnf("Docker Sandboxes control-plane recovery attempt %d failed; preserving exact capacity and retrying after %s: %v\n", attempt, time.Until(next).Round(time.Second), recoveryErr) + return true, nil + } + + if verifyErr := m.verifyProviderInventoryAfterRecovery(recoveryCtx); verifyErr != nil { + if ctx.Err() != nil { + return true, ctx.Err() + } + attempt, next := m.recordProviderRecoveryFailure() + m.warnf("Docker Sandboxes recovery attempt %d did not produce stable inventory; preserving exact capacity and retrying after %s: %v\n", attempt, time.Until(next).Round(time.Second), verifyErr) + return true, nil + } + + m.resetProviderRecovery() + m.infof("Docker Sandboxes control-plane recovery succeeded; stable inventory verified and pool reconciliation will resume\n") + return true, nil +} + +func (m *Manager) dockerSandboxesExclusiveRecovery() bool { + if strings.TrimSpace(strings.ToLower(m.Config.Provider.Type)) != "docker-sandboxes" { + return false + } + mode := strings.TrimSpace(strings.ToLower(m.Config.DockerSandboxes.RecoveryMode)) + if mode == "" { + mode = config.DockerSandboxesRecoveryModeExclusiveAuto + } + return mode == config.DockerSandboxesRecoveryModeExclusiveAuto +} + +func (m *Manager) probeProviderInventory(parent context.Context) error { + lifecycle := m.providerLifecycle() + if lifecycle == nil { + return errors.New("provider lifecycle is unavailable") + } + ctx, cancel := context.WithTimeout(parent, providerRecoveryProbeTimeout) + defer cancel() + _, err := lifecycle.Inventory(ctx) + return err +} + +func (m *Manager) verifyProviderInventoryAfterRecovery(parent context.Context) error { + for attempt := 1; attempt <= providerRecoveryProbeCount; attempt++ { + if err := m.probeProviderInventory(parent); err != nil { + return fmt.Errorf("post-recovery inventory probe %d/%d failed: %w", attempt, providerRecoveryProbeCount, err) + } + if attempt < providerRecoveryProbeCount { + if err := waitWithContext(parent, providerRecoveryProbeInterval); err != nil { + return fmt.Errorf("wait between post-recovery inventory probes: %w", err) + } + } + } + return nil +} + +func (m *Manager) waitForProviderRecoveryWindow(ctx context.Context) error { + m.providerRecoveryMu.Lock() + next := m.providerRecoveryNext + m.providerRecoveryMu.Unlock() + now := m.currentTime() + if next.IsZero() || !now.Before(next) { + return nil + } + return waitWithContext(ctx, next.Sub(now)) +} + +func (m *Manager) providerRecoveryWindowReady() bool { + m.providerRecoveryMu.Lock() + next := m.providerRecoveryNext + m.providerRecoveryMu.Unlock() + return next.IsZero() || !m.currentTime().Before(next) +} + +func providerRecoveryBudgetFor(quiescence time.Duration) time.Duration { + if quiescence <= 0 { + quiescence = config.DockerSandboxesDefaultRecoveryQuiescenceSeconds * time.Second + } + return quiescence + + (2 * providerRecoveryDaemonStop) + + (3 * providerRecoveryDaemonReadback) + + (time.Duration(providerRecoveryProbeCount) * providerRecoveryProbeTimeout) + + (time.Duration(providerRecoveryProbeCount-1) * providerRecoveryProbeInterval) + + providerRecoverySafetyMargin +} + +func (m *Manager) beginProviderRecoveryAttempt() int { + m.providerRecoveryMu.Lock() + defer m.providerRecoveryMu.Unlock() + m.providerRecoveryTries++ + m.providerRecoveryNext = time.Time{} + return m.providerRecoveryTries +} + +func (m *Manager) cancelProviderRecoveryAttempt() { + m.providerRecoveryMu.Lock() + defer m.providerRecoveryMu.Unlock() + if m.providerRecoveryTries > 0 { + m.providerRecoveryTries-- + } + if m.providerRecoveryTries == 0 { + m.providerRecoveryNext = time.Time{} + } +} + +func (m *Manager) recordProviderRecoveryFailure() (int, time.Time) { + m.providerRecoveryMu.Lock() + defer m.providerRecoveryMu.Unlock() + delay := providerRecoveryBackoffInitial + for i := 1; i < m.providerRecoveryTries; i++ { + if delay >= providerRecoveryBackoffMaximum/2 { + delay = providerRecoveryBackoffMaximum + break + } + delay *= 2 + } + if delay > providerRecoveryBackoffMaximum { + delay = providerRecoveryBackoffMaximum + } + m.providerRecoveryNext = m.currentTime().Add(delay) + return m.providerRecoveryTries, m.providerRecoveryNext +} + +func (m *Manager) scheduleProviderRecovery(delay time.Duration) time.Time { + m.providerRecoveryMu.Lock() + defer m.providerRecoveryMu.Unlock() + if delay <= 0 { + delay = providerRecoveryLockRetry + } + m.providerRecoveryNext = m.currentTime().Add(delay) + return m.providerRecoveryNext +} + +func (m *Manager) resetProviderRecovery() { + m.providerRecoveryMu.Lock() + m.providerRecoveryTries = 0 + m.providerRecoveryNext = time.Time{} + m.providerRecoveryMu.Unlock() +} + +// reserveProviderRecovery atomically reserves the next recovery opportunity. +// Once an admission recovery has been attempted, ordinary inventory failures +// cannot restart the same daemon incident until a provider create succeeds. +func (m *Manager) reserveProviderRecovery(admissionFailure bool) (permitted, admissionIncident bool) { + m.providerRecoveryMu.Lock() + defer m.providerRecoveryMu.Unlock() + if m.providerAdmissionRecoveryAttempted { + return false, true + } + if admissionFailure { + m.providerAdmissionRecoveryAttempted = true + } + return true, false +} + +func (m *Manager) cancelProviderAdmissionRecovery() { + m.providerRecoveryMu.Lock() + m.providerAdmissionRecoveryAttempted = false + m.providerRecoveryMu.Unlock() +} + +func (m *Manager) resetProviderAdmissionRecovery() { + m.providerRecoveryMu.Lock() + m.providerAdmissionRecoveryAttempted = false + m.providerRecoveryMu.Unlock() +} + +func waitWithContext(ctx context.Context, duration time.Duration) error { + if duration <= 0 { + return nil + } + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/internal/pool/control_plane_recovery_test.go b/internal/pool/control_plane_recovery_test.go new file mode 100644 index 0000000..1a0954b --- /dev/null +++ b/internal/pool/control_plane_recovery_test.go @@ -0,0 +1,239 @@ +package pool + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/solutionforest/ephemeral-action-runner/internal/config" + "github.com/solutionforest/ephemeral-action-runner/internal/provider" +) + +type controlPlaneRecoveryLifecycle struct { + provider.Lifecycle + + mu sync.Mutex + inventoryErrs []error + inventoryCalls int + recoverCalls int + request provider.ControlPlaneRecoveryRequest + recoverErr error +} + +func (lifecycle *controlPlaneRecoveryLifecycle) Inventory(context.Context) ([]provider.InventoryItem, error) { + lifecycle.mu.Lock() + defer lifecycle.mu.Unlock() + lifecycle.inventoryCalls++ + if len(lifecycle.inventoryErrs) == 0 { + return nil, nil + } + err := lifecycle.inventoryErrs[0] + lifecycle.inventoryErrs = lifecycle.inventoryErrs[1:] + return nil, err +} + +func (lifecycle *controlPlaneRecoveryLifecycle) RecoverControlPlane(_ context.Context, request provider.ControlPlaneRecoveryRequest) error { + lifecycle.mu.Lock() + defer lifecycle.mu.Unlock() + lifecycle.recoverCalls++ + lifecycle.request = request + return lifecycle.recoverErr +} + +func TestControlPlaneRecoveryDefaultsToExclusiveAuto(t *testing.T) { + t.Setenv("EPAR_STATE_HOME", t.TempDir()) + lifecycle := &controlPlaneRecoveryLifecycle{ + inventoryErrs: []error{provider.NewControlPlaneFailure("inventory Docker Sandboxes", errors.New("wedged")), nil, nil, nil}, + } + cfg := config.Default() + cfg.Provider.Type = "docker-sandboxes" + manager := Manager{Config: cfg, Lifecycle: lifecycle, ProjectRoot: t.TempDir()} + + handled, err := manager.recoverProviderControlPlane(context.Background(), provider.ErrControlPlaneFailure) + if err != nil || !handled { + t.Fatalf("recoverProviderControlPlane() = handled %t, error %v; want handled success", handled, err) + } + lifecycle.mu.Lock() + defer lifecycle.mu.Unlock() + if lifecycle.recoverCalls != 1 { + t.Fatalf("recovery calls = %d, want 1", lifecycle.recoverCalls) + } + if lifecycle.request.Quiescence != time.Duration(config.DockerSandboxesDefaultRecoveryQuiescenceSeconds)*time.Second { + t.Fatalf("recovery quiescence = %s, want %ds", lifecycle.request.Quiescence, config.DockerSandboxesDefaultRecoveryQuiescenceSeconds) + } + if lifecycle.inventoryCalls != 4 { + t.Fatalf("inventory calls = %d, want one recheck plus three stable probes", lifecycle.inventoryCalls) + } +} + +func TestControlPlaneAdmissionRecoveryBypassesHealthyInventoryProbe(t *testing.T) { + t.Setenv("EPAR_STATE_HOME", t.TempDir()) + lifecycle := &controlPlaneRecoveryLifecycle{} + cfg := config.Default() + cfg.Provider.Type = "docker-sandboxes" + manager := Manager{Config: cfg, Lifecycle: lifecycle} + + handled, err := manager.recoverProviderControlPlane(context.Background(), provider.ErrControlPlaneAdmissionFailure) + if err != nil || !handled { + t.Fatalf("recoverProviderControlPlane() = handled %t, error %v; want handled success", handled, err) + } + lifecycle.mu.Lock() + defer lifecycle.mu.Unlock() + if lifecycle.recoverCalls != 1 { + t.Fatalf("recovery calls = %d, want 1", lifecycle.recoverCalls) + } + if lifecycle.inventoryCalls != providerRecoveryProbeCount { + t.Fatalf("inventory calls = %d, want only %d post-recovery probes", lifecycle.inventoryCalls, providerRecoveryProbeCount) + } +} + +func TestControlPlaneAdmissionRecoveryObserveModeDoesNotMutate(t *testing.T) { + lifecycle := &controlPlaneRecoveryLifecycle{} + cfg := config.Default() + cfg.Provider.Type = "docker-sandboxes" + cfg.DockerSandboxes.RecoveryMode = config.DockerSandboxesRecoveryModeObserve + manager := Manager{Config: cfg, Lifecycle: lifecycle} + + handled, err := manager.recoverProviderControlPlane(context.Background(), provider.ErrControlPlaneAdmissionFailure) + if err != nil || handled { + t.Fatalf("observe admission recovery = handled %t, error %v; want no automatic handling", handled, err) + } + lifecycle.mu.Lock() + defer lifecycle.mu.Unlock() + if lifecycle.recoverCalls != 0 || lifecycle.inventoryCalls != 0 { + t.Fatalf("observe mode invoked recovery=%d inventory=%d; want both zero", lifecycle.recoverCalls, lifecycle.inventoryCalls) + } +} + +func TestControlPlaneAdmissionRecoveryRunsAtMostOnceUntilCreateSucceeds(t *testing.T) { + t.Setenv("EPAR_STATE_HOME", t.TempDir()) + lifecycle := &controlPlaneRecoveryLifecycle{} + cfg := config.Default() + cfg.Provider.Type = "docker-sandboxes" + manager := Manager{Config: cfg, Lifecycle: lifecycle} + + handled, err := manager.recoverProviderControlPlane(context.Background(), provider.ErrControlPlaneAdmissionFailure) + if err != nil || !handled { + t.Fatalf("first admission recovery = handled %t, error %v; want handled success", handled, err) + } + handled, err = manager.recoverProviderControlPlane(context.Background(), provider.ErrControlPlaneAdmissionFailure) + if err != nil || !handled { + t.Fatalf("second admission recovery = handled %t, error %v; want handled backoff", handled, err) + } + lifecycle.mu.Lock() + if lifecycle.recoverCalls != 1 { + t.Fatalf("recovery calls after repeated admission failure = %d, want 1", lifecycle.recoverCalls) + } + lifecycle.mu.Unlock() + + manager.resetProviderAdmissionRecovery() + manager.providerRecoveryNext = time.Time{} + handled, err = manager.recoverProviderControlPlane(context.Background(), provider.ErrControlPlaneAdmissionFailure) + if err != nil || !handled { + t.Fatalf("re-armed admission recovery = handled %t, error %v; want handled success", handled, err) + } + lifecycle.mu.Lock() + defer lifecycle.mu.Unlock() + if lifecycle.recoverCalls != 2 { + t.Fatalf("recovery calls after successful-create rearm = %d, want 2", lifecycle.recoverCalls) + } +} + +func TestControlPlaneAdmissionRecoveryBlocksInventoryRestartUntilCreateSucceeds(t *testing.T) { + t.Setenv("EPAR_STATE_HOME", t.TempDir()) + lifecycle := &controlPlaneRecoveryLifecycle{} + cfg := config.Default() + cfg.Provider.Type = "docker-sandboxes" + manager := Manager{Config: cfg, Lifecycle: lifecycle} + + if handled, err := manager.recoverProviderControlPlane(context.Background(), provider.ErrControlPlaneAdmissionFailure); err != nil || !handled { + t.Fatalf("admission recovery = handled %t, error %v; want handled success", handled, err) + } + manager.providerRecoveryNext = time.Time{} + if handled, err := manager.recoverProviderControlPlane(context.Background(), provider.ErrControlPlaneFailure); err != nil || !handled { + t.Fatalf("inventory recovery during admission incident = handled %t, error %v; want handled backoff", handled, err) + } + lifecycle.mu.Lock() + defer lifecycle.mu.Unlock() + if lifecycle.recoverCalls != 1 { + t.Fatalf("recovery calls after admission-to-inventory transition = %d, want 1", lifecycle.recoverCalls) + } +} + +func TestControlPlaneRecoveryObserveModeDoesNotRestart(t *testing.T) { + lifecycle := &controlPlaneRecoveryLifecycle{} + cfg := config.Default() + cfg.Provider.Type = "docker-sandboxes" + cfg.DockerSandboxes.RecoveryMode = config.DockerSandboxesRecoveryModeObserve + manager := Manager{Config: cfg, Lifecycle: lifecycle} + + handled, err := manager.recoverProviderControlPlane(context.Background(), provider.ErrControlPlaneFailure) + if err != nil || handled { + t.Fatalf("observe recovery = handled %t, error %v; want no automatic handling", handled, err) + } + lifecycle.mu.Lock() + defer lifecycle.mu.Unlock() + if lifecycle.recoverCalls != 0 || lifecycle.inventoryCalls != 0 { + t.Fatalf("observe mode invoked recovery=%d inventory=%d; want both zero", lifecycle.recoverCalls, lifecycle.inventoryCalls) + } +} + +func TestControlPlaneRecoveryBacksOffAfterFailedRecovery(t *testing.T) { + t.Setenv("EPAR_STATE_HOME", t.TempDir()) + lifecycle := &controlPlaneRecoveryLifecycle{ + inventoryErrs: []error{provider.NewControlPlaneFailure("inventory Docker Sandboxes", errors.New("wedged"))}, + recoverErr: errors.New("daemon stop could not confirm stopped"), + } + cfg := config.Default() + cfg.Provider.Type = "docker-sandboxes" + manager := Manager{Config: cfg, Lifecycle: lifecycle} + + handled, err := manager.recoverProviderControlPlane(context.Background(), provider.ErrControlPlaneFailure) + if err != nil || !handled { + t.Fatalf("recoverProviderControlPlane() = handled %t, error %v; want handled retry", handled, err) + } + if !manager.providerRecoveryNext.After(time.Now()) { + t.Fatalf("provider recovery next attempt = %s, want future backoff", manager.providerRecoveryNext) + } + if manager.providerRecoveryTries != 1 { + t.Fatalf("provider recovery tries = %d, want 1", manager.providerRecoveryTries) + } +} + +func TestControlPlaneRecoveryCooldownDoesNotBlockSupervisor(t *testing.T) { + now := time.Date(2026, 8, 20, 1, 0, 0, 0, time.UTC) + lifecycle := &controlPlaneRecoveryLifecycle{} + cfg := config.Default() + cfg.Provider.Type = "docker-sandboxes" + manager := Manager{Config: cfg, Lifecycle: lifecycle, now: func() time.Time { return now }} + manager.providerRecoveryNext = now.Add(time.Hour) + + started := time.Now() + handled, err := manager.recoverProviderControlPlane(context.Background(), provider.ErrControlPlaneFailure) + if err != nil || !handled { + t.Fatalf("cooldown recovery = handled %t, error %v; want handled without error", handled, err) + } + if elapsed := time.Since(started); elapsed > time.Second { + t.Fatalf("cooldown blocked supervisor for %s", elapsed) + } + lifecycle.mu.Lock() + defer lifecycle.mu.Unlock() + if lifecycle.inventoryCalls != 0 || lifecycle.recoverCalls != 0 { + t.Fatalf("cooldown invoked inventory=%d recovery=%d; want both zero", lifecycle.inventoryCalls, lifecycle.recoverCalls) + } +} + +func TestProviderRecoveryBudgetCoversMaximumQuiescence(t *testing.T) { + quiescence := 5 * time.Minute + want := quiescence + + (2 * providerRecoveryDaemonStop) + + (3 * providerRecoveryDaemonReadback) + + (time.Duration(providerRecoveryProbeCount) * providerRecoveryProbeTimeout) + + (time.Duration(providerRecoveryProbeCount-1) * providerRecoveryProbeInterval) + + providerRecoverySafetyMargin + if got := providerRecoveryBudgetFor(quiescence); got != want || got <= quiescence { + t.Fatalf("provider recovery budget = %s, want %s and greater than quiescence", got, want) + } +} diff --git a/internal/pool/host_trust.go b/internal/pool/host_trust.go index 36579bb..8385e67 100644 --- a/internal/pool/host_trust.go +++ b/internal/pool/host_trust.go @@ -510,27 +510,6 @@ func (m *Manager) reconcileHostTrustRunners(ctx context.Context, active map[stri if !instance.ProviderOwned || (instance.Phase != LifecycleReady && instance.Phase != LifecycleDraining) { continue } - if _, requiresTransport := m.providerLifecycle().(provider.HostTrustRuntimeActivator); requiresTransport { - providerInstance, providerErr := m.providerInstance(ctx, name) - if providerErr != nil { - m.warnf("[%s] host trust transport identity warning; lease not refreshed: %v\n", name, providerErr) - if fenceErr := m.fenceHostTrustRunnerRegistration(ctx, instance, providerErr); fenceErr != nil { - m.warnf("[%s] host trust registration fencing warning: %v\n", name, fenceErr) - } - instance.Phase = LifecycleQuarantined - active[name] = instance - continue - } - if err := m.activateProviderHostTrustRuntime(ctx, providerInstance); err != nil { - m.warnf("[%s] host trust transport refresh warning; lease not refreshed: %v\n", name, err) - if fenceErr := m.fenceHostTrustRunnerRegistration(ctx, instance, err); fenceErr != nil { - m.warnf("[%s] host trust registration fencing warning: %v\n", name, fenceErr) - } - instance.Phase = LifecycleQuarantined - active[name] = instance - continue - } - } if instance.HostTrustGeneration != current.Generation { // Revoke the old generation before any remote status query. This is // safe for an already-running job (its hook already ran) and closes @@ -559,6 +538,36 @@ func (m *Manager) reconcileHostTrustRunners(ctx context.Context, active map[stri delete(busyHandoff, name) continue } + // Relay activation is a destructive guest transaction: it removes the + // job-start marker before reconfiguring the private daemon. Verify the + // current transport read-only first; if it is unhealthy, fence this + // registration and let normal replacement activate a fresh candidate + // before GitHub can assign it work. + lifecycle := m.providerLifecycle() + _, requiresTransportActivation := lifecycle.(provider.HostTrustRuntimeActivator) + _, hasTransportVerifier := lifecycle.(provider.HostTrustRuntimeVerifier) + if requiresTransportActivation || hasTransportVerifier { + providerInstance, providerErr := m.providerInstance(ctx, name) + if providerErr != nil { + m.warnf("[%s] host trust transport identity warning; lease not refreshed: %v\n", name, providerErr) + if fenceErr := m.fenceHostTrustRunnerRegistration(ctx, instance, providerErr); fenceErr != nil { + m.warnf("[%s] host trust registration fencing warning: %v\n", name, fenceErr) + } + instance.Phase = LifecycleQuarantined + active[name] = instance + continue + } + if verifyErr := m.verifyProviderHostTrustRuntime(ctx, providerInstance); verifyErr != nil { + transportErr := fmt.Errorf("host trust transport verification failed: %w", verifyErr) + m.warnf("[%s] host trust transport verification warning; lease not refreshed: %v\n", name, transportErr) + if fenceErr := m.fenceHostTrustRunnerRegistration(ctx, instance, transportErr); fenceErr != nil { + m.warnf("[%s] host trust registration fencing warning: %v\n", name, fenceErr) + } + instance.Phase = LifecycleQuarantined + active[name] = instance + continue + } + } if runner.Busy { if busyHandoff[name] { continue diff --git a/internal/pool/host_trust_test.go b/internal/pool/host_trust_test.go index 7205577..49c1f63 100644 --- a/internal/pool/host_trust_test.go +++ b/internal/pool/host_trust_test.go @@ -370,9 +370,12 @@ func TestHostTrustReconciliationPreservesRunnerDuringGitHub503(t *testing.T) { } } -func TestHostTrustReconciliationFencesLeaseWhenTransportRefreshFails(t *testing.T) { +func TestHostTrustReconciliationFencesWhenTransportVerificationFails(t *testing.T) { fake := &fakeProvider{instances: []provider.Instance{{Name: "runner-1", ProviderID: "fake:runner-1", State: "running"}}} - activator := &activatingLifecycle{Lifecycle: provider.AdaptLegacy(fake, false), err: errors.New("relay unavailable")} + activator := &hostTrustVerifyingLifecycle{ + activatingLifecycle: &activatingLifecycle{Lifecycle: provider.AdaptLegacy(fake, false)}, + verifyHostTrustErr: errors.New("relay marker unavailable"), + } github := &fakeGitHub{runner: gh.Runner{Name: "runner-1", ID: 42, Status: "online"}, found: true} manager := Manager{ Config: config.Config{ @@ -390,11 +393,17 @@ func TestHostTrustReconciliationFencesLeaseWhenTransportRefreshFails(t *testing. if retired := manager.reconcileHostTrustRunners(context.Background(), active, current, make(map[string]bool)); retired != 0 { t.Fatalf("retired runners = %d, want fenced preservation", retired) } - if activator.calls != 1 { - t.Fatalf("activation calls = %d, want 1", activator.calls) + if activator.calls != 0 { + t.Fatalf("activation calls = %d, want zero while fencing the unhealthy registered runner", activator.calls) + } + if activator.verifyCalls != 1 { + t.Fatalf("runtime verification calls = %d, want one before fencing the unhealthy registered runner", activator.verifyCalls) } - if got := atomic.LoadInt32(&github.runnerByNameCalls); got != 1 { - t.Fatalf("GitHub status calls = %d, want one exact registration fence lookup after activation failure", got) + if activator.verifyHostTrustCalls != 1 { + t.Fatalf("provider host-trust verification calls = %d, want one before fencing the unhealthy registered runner", activator.verifyHostTrustCalls) + } + if got := atomic.LoadInt32(&github.runnerByNameCalls); got != 2 { + t.Fatalf("GitHub status calls = %d, want status lookup plus exact registration fence lookup", got) } if got := atomic.LoadInt32(&github.deleteCalls); got != 1 { t.Fatalf("GitHub registration fence calls = %d, want 1", got) @@ -407,6 +416,82 @@ func TestHostTrustReconciliationFencesLeaseWhenTransportRefreshFails(t *testing. } } +func TestHostTrustReconciliationDoesNotReactivateHealthyTransport(t *testing.T) { + fake := &fakeProvider{instances: []provider.Instance{{Name: "runner-1", ProviderID: "fake:runner-1", State: "running"}}} + activator := &hostTrustVerifyingLifecycle{activatingLifecycle: &activatingLifecycle{Lifecycle: provider.AdaptLegacy(fake, false)}} + github := &fakeGitHub{runner: gh.Runner{Name: "runner-1", ID: 42, Status: "online", Busy: false}, found: true} + manager := Manager{ + Config: config.Config{ + Provider: config.ProviderConfig{Type: "docker-sandboxes"}, + Image: config.ImageConfig{HostTrustMode: config.HostTrustModeOverlay, HostTrustScopes: []string{"system"}}, + }, + Provider: fake, + Lifecycle: activator, + GitHub: github, + } + current := hosttrust.Snapshot{Generation: "g1", HostOS: "windows", Scopes: []string{"system"}, Certificates: []hosttrust.Certificate{{Name: "root.crt", PEM: []byte("pem")}}, CollectedAt: time.Now().UTC()} + active := map[string]ProvisionedInstance{"runner-1": {Name: "runner-1", ProviderID: "fake:runner-1", RunnerID: 42, HostTrustGeneration: "g1", ProviderOwned: true, Phase: LifecycleReady}} + + if retired := manager.reconcileHostTrustRunners(context.Background(), active, current, make(map[string]bool)); retired != 0 { + t.Fatalf("retired runners = %d, want 0", retired) + } + if activator.calls != 0 { + t.Fatalf("host trust transport activations = %d, want zero for healthy current-generation transport", activator.calls) + } + if activator.verifyCalls != 1 { + t.Fatalf("runtime verification calls = %d, want one for healthy current-generation transport", activator.verifyCalls) + } + if activator.verifyHostTrustCalls != 1 { + t.Fatalf("provider host-trust verification calls = %d, want one for healthy current-generation transport", activator.verifyHostTrustCalls) + } + if got := len(hostTrustLeaseInputs(fake)); got != 1 { + t.Fatalf("host trust lease writes = %d, want one refresh without relay reactivation", got) + } + if got := fake.commandCount("configure-egress-relay.sh"); got != 0 { + t.Fatalf("relay activation commands = %d, want zero for healthy current-generation transport", got) + } +} + +func TestHostTrustReconciliationFencesBusyRunnerWhenTransportVerificationFails(t *testing.T) { + fake := &fakeProvider{instances: []provider.Instance{{Name: "runner-1", ProviderID: "fake:runner-1", State: "running"}}} + activator := &hostTrustVerifyingLifecycle{ + activatingLifecycle: &activatingLifecycle{Lifecycle: provider.AdaptLegacy(fake, false)}, + verifyHostTrustErr: errors.New("relay marker unavailable"), + } + github := &fakeGitHub{runner: gh.Runner{Name: "runner-1", ID: 42, Status: "online", Busy: true}, found: true} + manager := Manager{ + Config: config.Config{ + Provider: config.ProviderConfig{Type: "docker-sandboxes"}, + Image: config.ImageConfig{HostTrustMode: config.HostTrustModeOverlay, HostTrustScopes: []string{"system"}}, + }, + Provider: fake, + Lifecycle: activator, + GitHub: github, + } + current := hosttrust.Snapshot{Generation: "g1", HostOS: "windows", Scopes: []string{"system"}, Certificates: []hosttrust.Certificate{{Name: "root.crt", PEM: []byte("pem")}}, CollectedAt: time.Now().UTC()} + active := map[string]ProvisionedInstance{"runner-1": {Name: "runner-1", ProviderID: "fake:runner-1", RunnerID: 42, HostTrustGeneration: "g1", ProviderOwned: true, Phase: LifecycleReady}} + + manager.reconcileHostTrustRunners(context.Background(), active, current, make(map[string]bool)) + if got := active["runner-1"].Phase; got != LifecycleQuarantined { + t.Fatalf("runner phase = %s, want %s", got, LifecycleQuarantined) + } + if activator.calls != 0 { + t.Fatalf("host trust transport activations = %d, want zero for failed verification", activator.calls) + } + if activator.verifyCalls != 1 { + t.Fatalf("runtime verification calls = %d, want one before fencing the busy runner", activator.verifyCalls) + } + if activator.verifyHostTrustCalls != 1 { + t.Fatalf("provider host-trust verification calls = %d, want one before fencing the busy runner", activator.verifyHostTrustCalls) + } + if got := atomic.LoadInt32(&github.deleteCalls); got != 1 { + t.Fatalf("GitHub registration fence calls = %d, want 1", got) + } + if got := len(hostTrustLeaseInputs(fake)); got != 0 { + t.Fatalf("host trust lease writes = %d, want zero after failed transport verification", got) + } +} + func TestHostTrustReconciliationFencesRegistrationWhenIdleLeaseRefreshFails(t *testing.T) { fake := &fakeProvider{execErrs: []error{errors.New("lease transport unavailable"), nil}} github := &fakeGitHub{runner: gh.Runner{Name: "runner-1", ID: 42, Status: "online"}, found: true} diff --git a/internal/pool/manager.go b/internal/pool/manager.go index dac78a0..d87c595 100644 --- a/internal/pool/manager.go +++ b/internal/pool/manager.go @@ -51,16 +51,20 @@ type Manager struct { transcriptMu sync.Mutex transcripts map[string]*logging.Transcript - hostTrustResolver func(context.Context) (hosttrust.Snapshot, error) - buildTrustResolver func(context.Context) (hosttrust.Snapshot, error) - hostTrustImageEnsurer func(context.Context) error - hostTrustImageMu sync.Mutex - imageEnsureMu sync.Mutex - imageEnsured bool - now func() time.Time - randomFloat64 func() float64 - externalOutageMu sync.Mutex - externalOutage *externalOutageRuntime + hostTrustResolver func(context.Context) (hosttrust.Snapshot, error) + buildTrustResolver func(context.Context) (hosttrust.Snapshot, error) + hostTrustImageEnsurer func(context.Context) error + hostTrustImageMu sync.Mutex + imageEnsureMu sync.Mutex + imageEnsured bool + now func() time.Time + randomFloat64 func() float64 + externalOutageMu sync.Mutex + externalOutage *externalOutageRuntime + providerRecoveryMu sync.Mutex + providerRecoveryNext time.Time + providerRecoveryTries int + providerAdmissionRecoveryAttempted bool } func (m *Manager) ConfigureStorageAdmissionOverride(allow bool, command string) { @@ -287,28 +291,85 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { return m.cleanupPoolWithStatus("owned GitHub runner registrations and provider instances", m.cleanupWithFreshContext) } var active map[string]ProvisionedInstance - err = m.RunExternalOutageStage(ctx, "initial-pool-reconciliation", func(attemptCtx context.Context) error { - var reconcileErr error - active, reconcileErr = m.reconcilePhysicalPool(attemptCtx, active, opts.Register) - return reconcileErr - }) + for { + if waitErr := m.waitForProviderRecoveryWindow(ctx); waitErr != nil { + err = waitErr + break + } + err = m.RunExternalOutageStage(ctx, "initial-pool-reconciliation", func(attemptCtx context.Context) error { + var reconcileErr error + active, reconcileErr = m.reconcilePhysicalPool(attemptCtx, active, opts.Register) + return reconcileErr + }) + if err == nil { + break + } + handled, recoveryErr := m.recoverProviderControlPlane(ctx, err) + if !handled { + break + } + if recoveryErr != nil { + if ctx.Err() != nil { + return m.cleanupPoolWithStatus("owned GitHub runner registrations and provider instances", m.cleanupWithFreshContext) + } + m.warnf("Docker Sandboxes control-plane recovery supervisor warning; preserving exact capacity and retrying: %v\n", recoveryErr) + } + } if err != nil { return m.withOutageExhaustionCleanup(fmt.Errorf("initial pool reconciliation: %w", err), active, opts.KeepOnExit) } - err = m.RunExternalOutageStage(ctx, "initial-over-capacity-reconciliation", func(attemptCtx context.Context) error { - var reconcileErr error - active, reconcileErr = m.reduceOverCapacity(attemptCtx, active, opts.Instances, opts.Register) - return reconcileErr - }) + for { + if waitErr := m.waitForProviderRecoveryWindow(ctx); waitErr != nil { + err = waitErr + break + } + err = m.RunExternalOutageStage(ctx, "initial-over-capacity-reconciliation", func(attemptCtx context.Context) error { + var reconcileErr error + active, reconcileErr = m.reduceOverCapacity(attemptCtx, active, opts.Instances, opts.Register) + return reconcileErr + }) + if err == nil { + break + } + handled, recoveryErr := m.recoverProviderControlPlane(ctx, err) + if !handled { + break + } + if recoveryErr != nil { + if ctx.Err() != nil { + return m.cleanupPoolWithStatus("owned GitHub runner registrations and provider instances", m.cleanupWithFreshContext) + } + m.warnf("Docker Sandboxes control-plane recovery supervisor warning; preserving exact capacity and retrying: %v\n", recoveryErr) + } + } if err != nil { return m.withOutageExhaustionCleanup(fmt.Errorf("initial over-capacity reconciliation: %w", err), active, opts.KeepOnExit) } var poolTrustGeneration string - err = m.RunExternalOutageStage(ctx, "initial-host-trust-preparation", func(attemptCtx context.Context) error { - var prepareErr error - poolTrustGeneration, prepareErr = m.prepareExistingHostTrustRuntimes(attemptCtx, active, opts.Register) - return prepareErr - }) + for { + if waitErr := m.waitForProviderRecoveryWindow(ctx); waitErr != nil { + err = waitErr + break + } + err = m.RunExternalOutageStage(ctx, "initial-host-trust-preparation", func(attemptCtx context.Context) error { + var prepareErr error + poolTrustGeneration, prepareErr = m.prepareExistingHostTrustRuntimes(attemptCtx, active, opts.Register) + return prepareErr + }) + if err == nil { + break + } + handled, recoveryErr := m.recoverProviderControlPlane(ctx, err) + if !handled { + break + } + if recoveryErr != nil { + if ctx.Err() != nil { + return m.cleanupPoolWithStatus("owned GitHub runner registrations and provider instances", m.cleanupWithFreshContext) + } + m.warnf("Docker Sandboxes control-plane recovery supervisor warning; preserving exact capacity and retrying: %v\n", recoveryErr) + } + } if err != nil { return m.withOutageExhaustionCleanup(fmt.Errorf("initial host-trust runtime preparation: %w", err), active, opts.KeepOnExit) } @@ -322,6 +383,13 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { } leaseAdd, stopLeaseKeeper := m.startHostTrustLeaseKeeper(ctx) for len(active) < opts.Instances { + if waitErr := m.waitForProviderRecoveryWindow(ctx); waitErr != nil { + stopLeaseKeeper() + if ctx.Err() != nil { + return cleanup() + } + return m.cleanupAfterPoolFailure(waitErr, active, opts.KeepOnExit) + } var vm ProvisionedInstance err = m.RunExternalOutageStage(ctx, "initial-capacity-provisioning", func(attemptCtx context.Context) error { vm = ProvisionedInstance{} @@ -340,6 +408,17 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { return provisionErr }) if err != nil { + handled, recoveryErr := m.recoverProviderControlPlane(ctx, err) + if handled { + if recoveryErr != nil { + if ctx.Err() != nil { + stopLeaseKeeper() + return cleanup() + } + m.warnf("Docker Sandboxes control-plane recovery supervisor warning; preserving exact capacity and retrying: %v\n", recoveryErr) + } + continue + } stopLeaseKeeper() if ctx.Err() != nil { return cleanup() @@ -402,6 +481,7 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { return cleanup() case <-ticker.C: now := m.currentTime() + imageMaintenanceWaiting := false dependencyCooldown := retry.active(now) if m.externalOutageEnabled() { _, dependencyCooldown, err = m.externalOutageCooldown(now) @@ -439,20 +519,31 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { if imageMaintenancePending { remaining, drainErr := m.drainPoolForImageUpdate(ctx, active, imageMaintenanceIdleChecks) if drainErr != nil { - m.warnf("scheduled image maintenance drain warning; retrying without creating replacements: %v\n", drainErr) - continue - } - if remaining > 0 { - continue - } - m.infof("scheduled image maintenance drain complete; building and activating the verified replacement artifact\n") - if updateErr := m.ApplyPendingImageUpdate(ctx, now); updateErr != nil { - m.warnf("scheduled image update failed; restoring pool capacity with the previous verified generation: %v\n", updateErr) + handled, recoveryErr := m.recoverProviderControlPlane(ctx, drainErr) + if handled { + if recoveryErr != nil { + if ctx.Err() != nil { + return cleanup() + } + m.warnf("Docker Sandboxes control-plane recovery supervisor warning; preserving exact capacity and retrying: %v\n", recoveryErr) + } + imageMaintenanceWaiting = true + } else { + m.warnf("scheduled image maintenance drain warning; retrying without creating replacements: %v\n", drainErr) + imageMaintenanceWaiting = true + } + } else if remaining > 0 { + imageMaintenanceWaiting = true } else { - m.infof("scheduled image update activated; restoring pool capacity\n") + m.infof("scheduled image maintenance drain complete; building and activating the verified replacement artifact\n") + if updateErr := m.ApplyPendingImageUpdate(ctx, now); updateErr != nil { + m.warnf("scheduled image update failed; restoring pool capacity with the previous verified generation: %v\n", updateErr) + } else { + m.infof("scheduled image update activated; restoring pool capacity\n") + } + imageMaintenancePending = false + clear(imageMaintenanceIdleChecks) } - imageMaintenancePending = false - clear(imageMaintenanceIdleChecks) } trustRetired := 0 trustCapacityReady := true @@ -595,6 +686,16 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { cancelMaintenance() cancelAttempt() if reconcileErr != nil { + handled, recoveryErr := m.recoverProviderControlPlane(ctx, reconcileErr) + if handled { + if recoveryErr != nil { + if ctx.Err() != nil { + return cleanup() + } + m.warnf("Docker Sandboxes control-plane recovery supervisor warning; preserving exact capacity and retrying: %v\n", recoveryErr) + } + continue + } if ctx.Err() != nil { return cleanup() } @@ -625,6 +726,16 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { cancelMaintenance() cancelAttempt() if reconcileErr != nil { + handled, recoveryErr := m.recoverProviderControlPlane(ctx, reconcileErr) + if handled { + if recoveryErr != nil { + if ctx.Err() != nil { + return cleanup() + } + m.warnf("Docker Sandboxes control-plane recovery supervisor warning; preserving exact capacity and retrying: %v\n", recoveryErr) + } + continue + } if errors.Is(reconcileErr, context.DeadlineExceeded) && ctx.Err() == nil { m.warnf("over-capacity reconciliation exceeded the host-trust maintenance budget; preserving exact capacity and retrying after lease refresh: %v\n", reconcileErr) continue @@ -649,6 +760,9 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { return errors.Join(err, m.cleanupAfterTerminalFailure(active, opts.KeepOnExit)) } } + if imageMaintenanceWaiting { + continue + } replacementCapacity := len(active) needsTrustCapacity := false if m.hostTrustEnabled() && currentHostTrust.Generation != "" { @@ -677,6 +791,16 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { cancelMaintenance() cancelAttempt() if err != nil { + handled, recoveryErr := m.recoverProviderControlPlane(ctx, err) + if handled { + if recoveryErr != nil { + if ctx.Err() != nil { + return cleanup() + } + m.warnf("Docker Sandboxes control-plane recovery supervisor warning; preserving exact capacity and retrying: %v\n", recoveryErr) + } + break + } if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { m.warnf("[%s] replacement preallocation reconciliation exceeded the host-trust maintenance budget; retrying after lease refresh: %v\n", name, err) break @@ -710,6 +834,16 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { active[vm.Name] = vm } if err != nil { + handled, recoveryErr := m.recoverProviderControlPlane(ctx, err) + if handled { + if recoveryErr != nil { + if ctx.Err() != nil { + return cleanup() + } + m.warnf("Docker Sandboxes control-plane recovery supervisor warning; preserving exact capacity and retrying: %v\n", recoveryErr) + } + break + } if ctx.Err() != nil { return cleanup() } @@ -774,7 +908,7 @@ func (m *Manager) drainPoolForImageUpdate(ctx context.Context, active map[string } } m.infof("[%s] retiring idle runner for scheduled image maintenance\n", name) - if err := m.retireInstance(context.Background(), vm, "scheduled image and Actions runner update"); err != nil { + if err := m.retireInstance(ctx, vm, "scheduled image and Actions runner update"); err != nil { return len(active), err } delete(active, name) @@ -880,7 +1014,10 @@ func (m *Manager) reconcilePhysicalPool(ctx context.Context, known map[string]Pr reconciled[name] = vm return reconciled, fmt.Errorf("record GitHub runner absence for %s: %w", name, err) } - if err := m.deleteLocalInstance(context.Background(), vm); err != nil { + if err := m.deleteLocalInstance(ctx, vm); err != nil { + if errors.Is(err, provider.ErrControlPlaneFailure) { + return reconciled, err + } vm.Phase = LifecycleCleanupPending reconciled[name] = vm m.warnf("[%s] unregistered-instance cleanup pending: %v\n", name, err) @@ -923,7 +1060,10 @@ func (m *Manager) reconcilePhysicalPool(ctx context.Context, known map[string]Pr continue } if vm.Phase == LifecycleQuarantined { - if err := m.retireInstance(context.Background(), vm, "GitHub recovered but quarantined runner remained offline"); err != nil { + if err := m.retireInstance(ctx, vm, "GitHub recovered but quarantined runner remained offline"); err != nil { + if errors.Is(err, provider.ErrControlPlaneFailure) { + return reconciled, err + } vm.Phase = LifecycleCleanupPending reconciled[name] = vm m.warnf("[%s] recovered-offline retirement pending: %v\n", name, err) @@ -944,7 +1084,10 @@ func (m *Manager) reconcilePhysicalPool(ctx context.Context, known map[string]Pr reconciled[name] = vm continue } - if err := m.retireInstance(context.Background(), vm, "reconciliation found offline runner with inactive listener"); err != nil { + if err := m.retireInstance(ctx, vm, "reconciliation found offline runner with inactive listener"); err != nil { + if errors.Is(err, provider.ErrControlPlaneFailure) { + return reconciled, err + } vm.Phase = LifecycleCleanupPending reconciled[name] = vm m.warnf("[%s] inactive-instance cleanup pending: %v\n", name, err) @@ -961,7 +1104,7 @@ func (m *Manager) reconcilePhysicalPool(ctx context.Context, known map[string]Pr m.warnf("reconciliation: quarantined unowned GitHub runner %s id=%d; prefix-only resources are report-only\n", runner.Name, runner.ID) continue } - if err := m.deleteRemoteRunner(context.Background(), runner); err != nil { + if err := m.deleteRemoteRunner(ctx, runner); err != nil { return reconciled, err } m.infof("reconciliation: deleted stale GitHub runner %s id=%d\n", runner.Name, runner.ID) @@ -994,7 +1137,10 @@ func (m *Manager) reduceOverCapacity(ctx context.Context, active map[string]Prov continue } vm.RunnerID = runner.ID - if err := m.retireInstance(context.Background(), vm, "reconciling legacy physical inventory above pool.instances"); err != nil { + if err := m.retireInstance(ctx, vm, "reconciling legacy physical inventory above pool.instances"); err != nil { + if errors.Is(err, provider.ErrControlPlaneFailure) { + return active, err + } vm.Phase = LifecycleCleanupPending active[name] = vm continue @@ -1051,7 +1197,10 @@ func (m *Manager) reconcileLocalInventoryWithContext(ctx context.Context, known reconciled[local.Name] = vm continue } - if err := m.deleteLocalInstance(context.Background(), vm); err != nil { + if err := m.deleteLocalInstance(ctx, vm); err != nil { + if errors.Is(err, provider.ErrControlPlaneFailure) { + return reconciled, err + } vm.Phase = LifecycleCleanupPending reconciled[local.Name] = vm m.warnf("[%s] stopped-instance cleanup pending: %v\n", local.Name, err) @@ -1213,6 +1362,9 @@ func isTransientDependencyError(err error) bool { if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return false } + if errors.Is(err, provider.ErrControlPlaneAdmissionFailure) { + return false + } var httpErr *gh.HTTPError if errors.As(err, &httpErr) { return httpErr.StatusCode == http.StatusTooManyRequests || httpErr.StatusCode >= http.StatusInternalServerError @@ -1640,6 +1792,9 @@ func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register if createStageErr != nil { return vm, createStageErr } + if created.Name == name && created.ProviderID != "" { + m.resetProviderAdmissionRecovery() + } if created.Name != name || created.ProviderID == "" { return vm, fmt.Errorf("provider create returned no immutable identity for %q", name) } diff --git a/internal/pool/manager_test.go b/internal/pool/manager_test.go index 6ce84a3..c8a1584 100644 --- a/internal/pool/manager_test.go +++ b/internal/pool/manager_test.go @@ -478,6 +478,119 @@ func TestRunPoolReplacesCompletedRunnerAfterBusyProvisioning(t *testing.T) { } } +func TestRunPoolFailsClosedWhenReplacementHostTrustActivationFails(t *testing.T) { + const activationFailure = "activate provider host-trust runtime: execute in docker sandbox failed: exit status 1: EPAR host-trust relay: activation failed at private-dockerd-contract (exit=1)" + snapshot := hosttrust.Snapshot{ + Generation: "g1", + HostOS: "windows", + Scopes: []string{"system"}, + Certificates: []hosttrust.Certificate{{Name: "root.crt", PEM: []byte("pem")}}, + CollectedAt: time.Now().UTC(), + } + fake := &fakeProvider{ip: "127.0.0.1"} + marker, err := hostTrustMarkerJSON(snapshot) + if err != nil { + t.Fatal(err) + } + fake.execFunc = func(_ context.Context, _ string, command []string, _ provider.ExecOptions) (provider.ExecResult, error) { + commandText := strings.Join(command, " ") + if commandText == "cat "+hostTrustMarkerGuest { + return provider.ExecResult{Stdout: string(marker)}, nil + } + if strings.Contains(commandText, runnerProcessRunningSentinel) { + return provider.ExecResult{Stdout: runnerProcessRunningSentinel + "\n"}, nil + } + return provider.ExecResult{}, nil + } + github := &fakeGitHub{ + waitRunner: gh.Runner{Name: "epar-test-1", ID: 123, Status: "online", Busy: true}, + } + activator := &activatingLifecycle{Lifecycle: provider.AdaptLegacy(fake, false)} + activator.onActivate = func(provider.Instance) { + if activator.calls == 2 { + activator.err = errors.New(activationFailure) + } + } + manager := newRegisteredTestManager(t, fake, github) + manager.Config.Provider.Type = "docker-sandboxes" + manager.Config.Image.HostTrustMode = config.HostTrustModeOverlay + manager.Config.Image.HostTrustScopes = []string{"system"} + manager.AllowInsufficientStorage = true + manager.Lifecycle = activator + manager.hostTrustResolver = func(context.Context) (hosttrust.Snapshot, error) { return snapshot, nil } + manager.hostTrustImageEnsurer = func(context.Context) error { return nil } + state, err := poolstate.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + manager.LifecycleState = state + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + err = manager.RunPool(ctx, RunOptions{ + Instances: 1, + Register: true, + ReplaceCompleted: true, + MonitorInterval: 5 * time.Millisecond, + PoolLockHeld: true, + HostTrustLockHeld: true, + }) + if err == nil || !strings.Contains(err.Error(), "private-dockerd-contract") { + t.Fatalf("RunPool() error = %v, want terminal host-trust stage failure", err) + } + if errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("RunPool() timed out instead of returning the replacement failure: %v", err) + } + if isTransientDependencyError(err) { + t.Fatalf("replacement failure was classified transient: %v", err) + } + if activator.calls != 2 { + t.Fatalf("host-trust activation calls = %d, want initial activation and one failed replacement", activator.calls) + } + if got := atomic.LoadInt32(&fake.cloneCalls); got != 2 { + t.Fatalf("Clone calls = %d, want exactly two candidates and no retry storm", got) + } + if got := atomic.LoadInt32(&github.registrationCalls); got != 1 { + t.Fatalf("registration token calls = %d, want only the initial runner registered", got) + } + if got := atomic.LoadInt32(&fake.deleteCalls); got != 2 { + t.Fatalf("provider delete calls = %d, want retired runner and failed replacement", got) + } + fake.mu.Lock() + deletedNames := append([]string(nil), fake.deletedNames...) + remaining := append([]provider.Instance(nil), fake.instances...) + fake.mu.Unlock() + if len(remaining) != 0 { + t.Fatalf("provider inventory after terminal cleanup = %#v, want empty", remaining) + } + if len(activator.instances) < 2 { + t.Fatalf("activated instances = %#v, want the failed replacement candidate", activator.instances) + } + replacementName := activator.instances[1].Name + deletedReplacement := false + for _, deletedName := range deletedNames { + if deletedName == replacementName { + deletedReplacement = true + break + } + } + if !deletedReplacement { + t.Fatalf("deleted provider names = %v, want exact failed replacement %q", deletedNames, replacementName) + } + records, err := state.List(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(records) != 2 { + t.Fatalf("lifecycle records = %#v, want initial and replacement tombstones", records) + } + for _, record := range records { + if record.Phase != poolstate.PhaseTombstoned || !strings.HasPrefix(record.ProviderID, "fake:") { + t.Fatalf("lifecycle record = %#v, want exact tombstoned provider identity", record) + } + } +} + func TestRunPoolAddsCurrentTrustCapacityWhileOldGenerationDrains(t *testing.T) { fake := &fakeProvider{ip: "127.0.0.1"} github := &fakeGitHub{ @@ -547,7 +660,7 @@ func TestRunPoolAddsCurrentTrustCapacityWhileOldGenerationDrains(t *testing.T) { } } -func TestRunPoolRefreshesHostTrustTransportOnLeaseCadence(t *testing.T) { +func TestRunPoolVerifiesHostTrustTransportWithoutReactivationOnLeaseCadence(t *testing.T) { snapshot := hosttrust.Snapshot{Generation: "g1", HostOS: "windows", Scopes: []string{"system"}, Certificates: []hosttrust.Certificate{{Name: "root.crt", PEM: []byte("pem")}}, CollectedAt: time.Now().UTC()} marker, err := hostTrustMarkerJSON(snapshot) if err != nil { @@ -569,7 +682,14 @@ func TestRunPoolRefreshesHostTrustTransportOnLeaseCadence(t *testing.T) { waitRunner: gh.Runner{Name: "epar-test-1", ID: 123, Status: "online"}, } activation := make(chan struct{}, 4) - activator := &activatingLifecycle{Lifecycle: provider.AdaptLegacy(fake, false), onActivate: func(provider.Instance) { activation <- struct{}{} }} + verification := make(chan struct{}, 4) + activator := &hostTrustVerifyingLifecycle{ + activatingLifecycle: &activatingLifecycle{ + Lifecycle: provider.AdaptLegacy(fake, false), + onActivate: func(provider.Instance) { activation <- struct{}{} }, + }, + onVerifyHostTrust: func(provider.Instance) { verification <- struct{}{} }, + } manager := newRegisteredTestManager(t, fake, github) manager.Config.Image.HostTrustMode = config.HostTrustModeOverlay manager.Config.Image.HostTrustScopes = []string{"system"} @@ -582,18 +702,22 @@ func TestRunPoolRefreshesHostTrustTransportOnLeaseCadence(t *testing.T) { go func() { done <- manager.RunPool(ctx, RunOptions{Instances: 1, Register: true, KeepOnExit: true, ReplaceCompleted: true, MonitorInterval: 5 * time.Millisecond, HostTrustLockHeld: true, PoolLockHeld: true}) }() - for i := 0; i < 2; i++ { - select { - case <-activation: - case <-time.After(2 * time.Second): - cancel() - t.Fatalf("host trust activation %d did not occur", i+1) - } + select { + case <-activation: + case <-time.After(2 * time.Second): + cancel() + t.Fatal("initial host trust activation did not occur") + } + select { + case <-verification: + case <-time.After(2 * time.Second): + cancel() + t.Fatal("steady-state host trust verification did not occur") } select { case <-activation: cancel() - t.Fatal("host trust transport was refreshed again before the lease refresh cadence elapsed") + t.Fatal("host trust transport was reactivated during steady-state lease reconciliation") case <-time.After(25 * time.Millisecond): } cancel() @@ -1355,10 +1479,12 @@ func TestRunPoolControllerRestartFencesRebindsAndLeasesExistingRunner(t *testing type activatingLifecycle struct { provider.Lifecycle - calls int - err error - instances []provider.Instance - onActivate func(provider.Instance) + calls int + verifyCalls int + err error + verifyErr error + instances []provider.Instance + onActivate func(provider.Instance) } func (l *activatingLifecycle) ActivateHostTrustRuntime(_ context.Context, instance provider.Instance) error { @@ -1370,6 +1496,29 @@ func (l *activatingLifecycle) ActivateHostTrustRuntime(_ context.Context, instan return l.err } +func (l *activatingLifecycle) VerifyRuntime(ctx context.Context, instance provider.Instance) (provider.RuntimeInfo, error) { + l.verifyCalls++ + if l.verifyErr != nil { + return provider.RuntimeInfo{}, l.verifyErr + } + return l.Lifecycle.VerifyRuntime(ctx, instance) +} + +type hostTrustVerifyingLifecycle struct { + *activatingLifecycle + verifyHostTrustCalls int + verifyHostTrustErr error + onVerifyHostTrust func(provider.Instance) +} + +func (l *hostTrustVerifyingLifecycle) VerifyHostTrustRuntime(_ context.Context, instance provider.Instance) error { + l.verifyHostTrustCalls++ + if l.onVerifyHostTrust != nil { + l.onVerifyHostTrust(instance) + } + return l.verifyHostTrustErr +} + type partialCreateLifecycle struct { provider.Lifecycle create func() (provider.Instance, error) @@ -1592,6 +1741,43 @@ func TestLegacyOverCapacityInventoryBlocksAllocation(t *testing.T) { } } +func TestReconciliationCleanupUsesMaintenanceContext(t *testing.T) { + p := &fakeProvider{instances: []provider.Instance{{Name: "epar-test-stopped", State: "stopped"}}} + p.deleteFunc = func(ctx context.Context, _ string) error { + <-ctx.Done() + return ctx.Err() + } + manager := newRegisteredTestManager(t, p, nil) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + active, err := manager.reconcilePhysicalPool(ctx, nil, false) + if err != nil { + t.Fatal(err) + } + if got := active["epar-test-stopped"].Phase; got != LifecycleCleanupPending { + t.Fatalf("stopped instance phase = %q, want cleanup-pending", got) + } + if got := atomic.LoadInt32(&p.deleteCalls); got != 1 { + t.Fatalf("local delete calls = %d, want 1", got) + } +} + +func TestReconciliationPropagatesControlPlaneCleanupFailure(t *testing.T) { + const daemonFailure = "inventory control plane unavailable" + p := &fakeProvider{ + instances: []provider.Instance{{Name: "epar-test-stopped", ProviderID: "fake:epar-test-stopped", State: "stopped"}}, + deleteErr: provider.NewControlPlaneFailure("delete Docker Sandboxes instance", errors.New(daemonFailure)), + } + manager := newRegisteredTestManager(t, p, nil) + active, err := manager.reconcilePhysicalPool(context.Background(), nil, false) + if !errors.Is(err, provider.ErrControlPlaneFailure) { + t.Fatalf("reconcilePhysicalPool() error = %v, want control-plane failure", err) + } + if _, found := active["epar-test-stopped"]; found { + t.Fatalf("active inventory = %#v, want caller to retain its prior map on typed failure", active) + } +} + func TestLegacyIdleOverCapacityIsReducedToTarget(t *testing.T) { p := &fakeProvider{instances: []provider.Instance{{Name: "epar-test-existing-1", State: "running"}, {Name: "epar-test-existing-2", State: "running"}, {Name: "epar-test-existing-3", State: "running"}}} g := &fakeGitHub{ @@ -1616,6 +1802,33 @@ func TestLegacyIdleOverCapacityIsReducedToTarget(t *testing.T) { } } +func TestLegacyOverCapacityPropagatesControlPlaneCleanupFailure(t *testing.T) { + p := &fakeProvider{ + instances: []provider.Instance{ + {Name: "epar-test-existing-1", ProviderID: "fake:epar-test-existing-1", State: "running"}, + {Name: "epar-test-existing-2", ProviderID: "fake:epar-test-existing-2", State: "running"}, + }, + deleteErr: provider.NewControlPlaneFailure("delete Docker Sandboxes instance", errors.New("daemon inventory unavailable")), + } + g := &fakeGitHub{ + runner: gh.Runner{ID: 9, Status: "online"}, + found: true, + listRunners: []gh.Runner{ + {Name: "epar-test-existing-1", ID: 1, Status: "online"}, + {Name: "epar-test-existing-2", ID: 2, Status: "online"}, + }, + } + manager := newRegisteredTestManager(t, p, g) + active, err := manager.reconcilePhysicalPool(context.Background(), nil, true) + if err != nil { + t.Fatal(err) + } + _, err = manager.reduceOverCapacity(context.Background(), active, 1, true) + if !errors.Is(err, provider.ErrControlPlaneFailure) { + t.Fatalf("reduceOverCapacity() error = %v, want control-plane failure", err) + } +} + func TestQuarantinedRunnersAdoptOrRetireAfterGitHubRecovery(t *testing.T) { p := &fakeProvider{instances: []provider.Instance{{Name: "epar-test-healthy", State: "running"}, {Name: "epar-test-offline", State: "running"}}} g := &fakeGitHub{listRunners: []gh.Runner{{Name: "epar-test-healthy", ID: 1, Status: "online"}, {Name: "epar-test-offline", ID: 2, Status: "offline"}}} @@ -1758,6 +1971,7 @@ func TestTransientDependencyClassification(t *testing.T) { {name: "forbidden", err: &gh.HTTPError{StatusCode: http.StatusForbidden}, want: false}, {name: "guest opaque forbidden", err: errors.New("Response status code does not indicate success: 403 (Forbidden)"), want: false}, {name: "cancellation", err: context.Canceled, want: false}, + {name: "sandbox create admission", err: provider.NewControlPlaneAdmissionFailure("create Docker Sandboxes instance", errors.New("failed to run sandbox container")), want: false}, {name: "deterministic configuration", err: errors.New("runner labels are invalid"), want: false}, } for _, test := range tests { @@ -1824,22 +2038,24 @@ func newRegisteredTestManager(t *testing.T, provider provider.Provider, github G } type fakeProvider struct { - execErr error - execErrs []error - execFunc func(context.Context, string, []string, provider.ExecOptions) (provider.ExecResult, error) - ip string - cloneErr error - startErr error - ipErr error - deleteErr error - listErr error - mu sync.Mutex + execErr error + execErrs []error + execFunc func(context.Context, string, []string, provider.ExecOptions) (provider.ExecResult, error) + ip string + cloneErr error + startErr error + ipErr error + deleteErr error + listErr error + deleteFunc func(context.Context, string) error + mu sync.Mutex configureEnv map[string]string configureOptions provider.ExecOptions commands []string execOptions []provider.ExecOptions instances []provider.Instance + deletedNames []string cloneCalls int32 execCalls int32 @@ -1931,8 +2147,14 @@ func (p *fakeProvider) Stop(context.Context, string) error { return nil } -func (p *fakeProvider) Delete(_ context.Context, name string) error { +func (p *fakeProvider) Delete(ctx context.Context, name string) error { atomic.AddInt32(&p.deleteCalls, 1) + p.mu.Lock() + p.deletedNames = append(p.deletedNames, name) + p.mu.Unlock() + if p.deleteFunc != nil { + return p.deleteFunc(ctx, name) + } if p.deleteErr != nil { return p.deleteErr } diff --git a/internal/pool/provider_lifecycle.go b/internal/pool/provider_lifecycle.go index f449ee6..2ce7f69 100644 --- a/internal/pool/provider_lifecycle.go +++ b/internal/pool/provider_lifecycle.go @@ -76,6 +76,25 @@ func (m *Manager) verifyProviderRuntime(ctx context.Context, instance provider.I return nil } +// verifyProviderHostTrustRuntime preserves the common runtime check and adds +// an optional provider-specific read-only trust-transport check. Providers +// that only implement HostTrustRuntimeActivator therefore retain the common +// VerifyRuntime fallback, while providers with a transport that needs a +// stronger proof can implement HostTrustRuntimeVerifier. +func (m *Manager) verifyProviderHostTrustRuntime(ctx context.Context, instance provider.Instance) error { + if err := m.verifyProviderRuntime(ctx, instance); err != nil { + return err + } + verifier, ok := m.providerLifecycle().(provider.HostTrustRuntimeVerifier) + if !ok { + return nil + } + if err := verifier.VerifyHostTrustRuntime(ctx, instance); err != nil { + return fmt.Errorf("verify provider host-trust runtime: %w", err) + } + return nil +} + func (m *Manager) verifyProviderAdmission(ctx context.Context, instance provider.Instance) error { if verifier, ok := m.Lifecycle.(provider.AdmissionVerifier); ok { if err := verifier.VerifyAdmission(ctx); err != nil { diff --git a/internal/provider/control_plane_lock.go b/internal/provider/control_plane_lock.go new file mode 100644 index 0000000..8450260 --- /dev/null +++ b/internal/provider/control_plane_lock.go @@ -0,0 +1,87 @@ +package provider + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/solutionforest/ephemeral-action-runner/internal/filelock" + storagecatalog "github.com/solutionforest/ephemeral-action-runner/internal/storage/catalog" +) + +const ( + controlPlaneLockDirectory = "provider-control-plane-recovery" + controlPlaneLockName = "docker-sandboxes.lock" + controlPlaneLockRetry = 50 * time.Millisecond +) + +var ErrControlPlaneRecoveryBusy = errors.New("provider control-plane recovery is already active") + +type controlPlaneLockContextKey struct{} + +func WithControlPlaneLock(ctx context.Context) context.Context { + return context.WithValue(ctx, controlPlaneLockContextKey{}, true) +} + +func ControlPlaneLockHeld(ctx context.Context) bool { + held, _ := ctx.Value(controlPlaneLockContextKey{}).(bool) + return held +} + +func controlPlaneLockPath() (string, error) { + root, err := storagecatalog.DefaultRoot() + if err != nil { + return "", err + } + lockRoot := filepath.Join(root, controlPlaneLockDirectory) + if err := os.MkdirAll(lockRoot, 0o700); err != nil { + return "", err + } + return filepath.Join(lockRoot, controlPlaneLockName), nil +} + +func tryAcquireControlPlaneLock() (func(), error) { + path, err := controlPlaneLockPath() + if err != nil { + return nil, err + } + lock, err := filelock.Acquire(path) + if err != nil { + return nil, err + } + return func() { _ = lock.Close() }, nil +} + +func TryAcquireControlPlaneRecoveryLock() (func(), error) { + release, err := tryAcquireControlPlaneLock() + if errors.Is(err, filelock.ErrLocked) { + return nil, fmt.Errorf("%w: %v", ErrControlPlaneRecoveryBusy, err) + } + return release, err +} + +func acquireControlPlaneCommandLock(ctx context.Context) (func(), error) { + for { + release, err := tryAcquireControlPlaneLock() + if err == nil { + return release, nil + } + if !errors.Is(err, filelock.ErrLocked) { + return nil, fmt.Errorf("acquire Docker Sandboxes host control-plane lock: %w", err) + } + timer := time.NewTimer(controlPlaneLockRetry) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } +} + +func AcquireControlPlaneCommandLock(ctx context.Context) (func(), error) { + return acquireControlPlaneCommandLock(ctx) +} diff --git a/internal/provider/control_plane_lock_test.go b/internal/provider/control_plane_lock_test.go new file mode 100644 index 0000000..c2520ac --- /dev/null +++ b/internal/provider/control_plane_lock_test.go @@ -0,0 +1,27 @@ +package provider + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestControlPlaneLockExcludesConcurrentCommands(t *testing.T) { + t.Setenv("EPAR_STATE_HOME", t.TempDir()) + release, err := TryAcquireControlPlaneRecoveryLock() + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) + defer cancel() + if _, err := AcquireControlPlaneCommandLock(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("AcquireControlPlaneCommandLock() = %v, want context deadline", err) + } + release() + commandRelease, err := AcquireControlPlaneCommandLock(context.Background()) + if err != nil { + t.Fatal(err) + } + commandRelease() +} diff --git a/internal/provider/dockersandboxes/architecture_emulation.go b/internal/provider/dockersandboxes/architecture_emulation.go index e193f18..bfe967a 100644 --- a/internal/provider/dockersandboxes/architecture_emulation.go +++ b/internal/provider/dockersandboxes/architecture_emulation.go @@ -41,6 +41,7 @@ func (qemuBinfmtEnabler) Enable(ctx context.Context, sandboxProvider *Provider, args: []string{"exec", instance.Name, "--", "sudo", "-n", architectureEmulationHelper}, operation: "enable Docker Sandboxes architecture emulation", outputLimit: diagnosticOutputLimit, + timeout: providerReadbackTimeout, }) if err != nil { return architectureEmulationResult{}, err @@ -63,6 +64,7 @@ func (enabler nativeArchitectureEnabler) Enable(ctx context.Context, sandboxProv args: []string{"exec", instance.Name, "--", "sudo", "-n", nativeArchitectureHelper, enabler.platform}, operation: "verify Docker Sandboxes native architecture", outputLimit: diagnosticOutputLimit, + timeout: providerReadbackTimeout, }) if err != nil { return architectureEmulationResult{}, err diff --git a/internal/provider/dockersandboxes/egress_relay.go b/internal/provider/dockersandboxes/egress_relay.go index afff532..0fb775c 100644 --- a/internal/provider/dockersandboxes/egress_relay.go +++ b/internal/provider/dockersandboxes/egress_relay.go @@ -23,13 +23,14 @@ import ( ) const ( - relayProtocolPrefix = "EPAR1 " - relayHeaderLimit = 1024 - relayMaxConnections = 128 - relayDialTimeout = 15 * time.Second - relayHeaderTimeout = 10 * time.Second - relayIdleTimeout = 5 * time.Minute - guestRelayPort = 3129 + relayProtocolPrefix = "EPAR1 " + relayHeaderLimit = 1024 + relayMaxConnections = 128 + relayDialTimeout = 15 * time.Second + relayHeaderTimeout = 10 * time.Second + relayIdleTimeout = 5 * time.Minute + guestRelayPort = 3129 + guestRelayProbeTimeout = 8 * time.Second ) type egressRelay struct { @@ -47,6 +48,30 @@ type guestRelayConfiguration struct { Token string `json:"token"` } +type relayTokenBinding struct { + ProviderID string + Token string + Epoch uint64 + PolicyRules map[string]struct{} +} + +type relayBindingSnapshot struct { + Instance provider.Instance + Token string + Epoch uint64 + Relay *egressRelay + Port int + PolicyRules map[string]struct{} +} + +const hostTrustRelayVerificationScript = `set -euo pipefail +test -f /run/epar/egress-relay-active +test ! -L /run/epar/egress-relay-active +test "$(stat -c '%U:%G:%a' /run/epar/egress-relay-active)" = "root:root:444" +test "$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --connect-timeout 1 --max-time 2 http://127.0.0.1:3129/health)" = "204" +registry_status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --connect-timeout 2 --max-time 5 --proxy http://127.0.0.1:3129 --noproxy '' --cacert /usr/local/share/ca-certificates/epar/epar-egress-relay.crt https://registry-1.docker.io/v2/)" +test "${registry_status}" = "401"` + func (p *Provider) ActivateHostTrustRuntime(ctx context.Context, instance provider.Instance) (activationErr error) { if !p.hostTrustRelayEnabled { if p.logger != nil { @@ -54,6 +79,11 @@ func (p *Provider) ActivateHostTrustRuntime(ctx context.Context, instance provid } return nil } + if err := validateInstance(instance, true); err != nil { + return err + } + releaseInstanceOperation := p.lockInstanceOperation(instance.Name) + defer releaseInstanceOperation() if p.logger != nil { p.logger.Debug(fmt.Sprintf("Docker Sandboxes host-trust relay activation started on controller port %d", p.hostTrustRelayPort), "provider", "docker-sandboxes", "instance", instance.Name) } @@ -64,13 +94,14 @@ func (p *Provider) ActivateHostTrustRuntime(ctx context.Context, instance provid if !present { return fmt.Errorf("docker sandbox is missing") } - relay, token, err := p.ensureRelayToken(instance.Name) + binding, err := p.ensureRelayToken(instance) if err != nil { return fmt.Errorf("prepare Docker Sandboxes host-trust relay: %w", err) } + relay := binding.Relay + token := binding.Token configured := false guestActivationAttempted := false - probeStarted := time.Now().UTC() var addedPolicyRules []provider.NetworkPolicyRule defer func() { if !configured { @@ -90,12 +121,13 @@ func (p *Provider) ActivateHostTrustRuntime(ctx context.Context, instance provid activationErr = errors.Join(activationErr, fmt.Errorf("roll back exact Docker Sandboxes host-trust relay policy: %w", rollbackErr)) } } - p.releaseRelayToken(instance.Name) + p.releaseRelayToken(binding) } }() resource := net.JoinHostPort("host.docker.internal", strconv.Itoa(relay.port)) - addedPolicyRules, err = p.applyHostTrustRelayPolicy(ctx, instance, provider.NetworkPolicyRule{ + var policyRuleNames []string + addedPolicyRules, policyRuleNames, err = p.applyHostTrustRelayPolicy(ctx, instance, provider.NetworkPolicyRule{ Name: "epar-host-trust-relay", Decision: provider.NetworkPolicyAllow, Resources: []string{resource}, @@ -103,6 +135,10 @@ func (p *Provider) ActivateHostTrustRuntime(ctx context.Context, instance provid if err != nil { return fmt.Errorf("allow exact Docker Sandboxes host-trust relay endpoint: %w", err) } + binding, err = p.bindRelayPolicyRules(binding, policyRuleNames) + if err != nil { + return err + } if p.logger != nil { p.logger.Debug("Docker Sandboxes host-trust relay policy is active", "provider", "docker-sandboxes", "instance", instance.Name) } @@ -118,6 +154,7 @@ func (p *Provider) ActivateHostTrustRuntime(ctx context.Context, instance provid } payload = append(payload, '\n') guestActivationAttempted = true + probeStarted := hostTrustRelayPolicyProbeStart() if _, err := p.Exec(ctx, instance, provider.ShellCommand("sudo -n /opt/epar/configure-egress-relay.sh"), provider.ExecOptions{ Stdin: string(payload), SensitiveValues: []string{token}, @@ -125,10 +162,16 @@ func (p *Provider) ActivateHostTrustRuntime(ctx context.Context, instance provid }); err != nil { return fmt.Errorf("activate Docker Sandboxes host-trust relay: %w", err) } + if err := p.verifyRelayBinding(binding); err != nil { + return err + } if p.logger != nil { p.logger.Debug("Docker Sandboxes guest host-trust relay is active", "provider", "docker-sandboxes", "instance", instance.Name) } - if err := p.verifyHostTrustRelayPolicy(ctx, instance, relay.port, probeStarted); err != nil { + if err := p.verifyBoundHostTrustRelayPolicy(ctx, binding, probeStarted); err != nil { + return err + } + if err := p.verifyExactRelayInstance(ctx, binding); err != nil { return err } if p.logger != nil { @@ -144,6 +187,55 @@ func (p *Provider) ActivateHostTrustRuntime(ctx context.Context, instance provid return nil } +// VerifyHostTrustRuntime performs only read-only checks for the controller +// relay and the exact guest identity. It deliberately does not recreate relay +// credentials or invoke the destructive guest configuration transaction. +func (p *Provider) VerifyHostTrustRuntime(ctx context.Context, instance provider.Instance) error { + if !p.hostTrustRelayEnabled { + return nil + } + if err := validateInstance(instance, true); err != nil { + return err + } + releaseInstanceOperation := p.lockInstanceOperation(instance.Name) + defer releaseInstanceOperation() + present, err := p.assertIdentity(ctx, instance) + if err != nil { + return err + } + if !present { + return fmt.Errorf("docker sandbox is missing") + } + binding, err := p.currentRelayBinding(instance) + if err != nil { + return err + } + if len(binding.PolicyRules) == 0 { + return fmt.Errorf("Docker Sandboxes host-trust relay policy proof is not bound to the exact instance") + } + + probeStarted := hostTrustRelayPolicyProbeStart() + probeCtx, cancelProbe := context.WithTimeout(ctx, guestRelayProbeTimeout) + result, err := p.Exec(probeCtx, instance, provider.ShellCommand(hostTrustRelayVerificationScript), provider.ExecOptions{ + SensitiveValues: []string{binding.Token}, + SuppressTranscript: true, + }) + cancelProbe() + if err != nil { + return fmt.Errorf("verify Docker Sandboxes host-trust relay from the exact instance: %w", err) + } + if strings.TrimSpace(result.Stdout) != "" { + return fmt.Errorf("Docker Sandboxes host-trust relay verification returned unexpected output") + } + if err := p.verifyRelayBinding(binding); err != nil { + return err + } + if err := p.verifyBoundHostTrustRelayPolicy(ctx, binding, probeStarted); err != nil { + return err + } + return p.verifyExactRelayInstance(ctx, binding) +} + func (p *Provider) finalizeGuestRelay(ctx context.Context, instance provider.Instance, operation string) error { if operation != "--commit" && operation != "--rollback" { return fmt.Errorf("unsupported guest relay transaction operation") @@ -152,10 +244,10 @@ func (p *Provider) finalizeGuestRelay(ctx context.Context, instance provider.Ins return err } -func (p *Provider) applyHostTrustRelayPolicy(ctx context.Context, instance provider.Instance, rule provider.NetworkPolicyRule) ([]provider.NetworkPolicyRule, error) { +func (p *Provider) applyHostTrustRelayPolicy(ctx context.Context, instance provider.Instance, rule provider.NetworkPolicyRule) ([]provider.NetworkPolicyRule, []string, error) { before, err := p.ReadNetworkPolicy(ctx, instance) if err != nil { - return nil, err + return nil, nil, err } beforeIDs := make(map[string]struct{}, len(before)) for _, existing := range before { @@ -166,34 +258,109 @@ func (p *Provider) applyHostTrustRelayPolicy(ctx context.Context, instance provi after, readErr := p.ReadNetworkPolicy(readbackCtx, instance) cancel() if readErr != nil { - return nil, errors.Join(applyErr, fmt.Errorf("read back relay policy delta: %w", readErr)) + return nil, nil, errors.Join(applyErr, fmt.Errorf("read back relay policy delta: %w", readErr)) } added := make([]provider.NetworkPolicyRule, 0, 1) + policyRuleNames := make([]string, 0, 1) for _, candidate := range after { + if candidate.Active && candidate.Name != "" && candidate.Decision == provider.NetworkPolicyAllow && candidate.ResourceType == "network" && len(candidate.Resources) == 1 && candidate.Resources[0] == rule.Resources[0] && isSandboxPolicyTarget(candidate.Scope, candidate.AppliesTo, instance.Name) { + policyRuleNames = append(policyRuleNames, candidate.Name) + } if _, existed := beforeIDs[candidate.ID]; existed || candidate.Decision != provider.NetworkPolicyAllow || candidate.ResourceType != "network" || len(candidate.Resources) != 1 || candidate.Resources[0] != rule.Resources[0] || !isRemovableSandboxPolicyRule(candidate, instance.Name) { continue } added = append(added, candidate) } - return added, applyErr + if len(policyRuleNames) == 0 { + return added, nil, errors.Join(applyErr, fmt.Errorf("Docker Sandboxes policy readback did not identify the exact active relay allow rule")) + } + return added, policyRuleNames, applyErr +} + +func hostTrustRelayPolicyProbeStart() time.Time { + // Docker Sandboxes policy logs may serialize timestamps at whole-second + // precision. Truncation permits only the sub-second representation gap. + return time.Now().UTC().Truncate(time.Second) +} + +func (p *Provider) verifyBoundHostTrustRelayPolicy(ctx context.Context, binding relayBindingSnapshot, startedAt time.Time) error { + if binding.Port <= 0 || len(binding.PolicyRules) == 0 { + return fmt.Errorf("Docker Sandboxes host-trust relay policy proof is not bound to the exact instance") + } + result, err := p.run(ctx, commandRequest{ + args: []string{"policy", "log", binding.Instance.Name, "--json"}, + operation: "verify Docker Sandboxes host-trust relay route", + outputLimit: diagnosticOutputLimit, + timeout: providerReadbackTimeout, + }) + if err != nil { + return err + } + decoder := json.NewDecoder(strings.NewReader(result.Stdout)) + decoder.DisallowUnknownFields() + var document policyLogDocument + if err := decoder.Decode(&document); err != nil || requireJSONEOF(decoder) != nil { + return fmt.Errorf("Docker Sandboxes policy log returned an unsupported json schema") + } + relayHosts := map[string]struct{}{ + net.JoinHostPort("localhost", strconv.Itoa(binding.Port)): {}, + net.JoinHostPort("host.docker.internal", strconv.Itoa(binding.Port)): {}, + } + for _, record := range document.Blocked { + if record.VMName != binding.Instance.Name || record.LastSeen.Before(startedAt) { + continue + } + if _, exactRelay := relayHosts[record.Host]; exactRelay { + return fmt.Errorf("Docker Sandboxes blocked the exact EPAR host-trust relay endpoint") + } + } + foundTransparentRelay := false + for _, record := range document.Allowed { + if record.VMName != binding.Instance.Name || record.LastSeen.Before(startedAt) { + continue + } + if record.Host == "registry-1.docker.io:443" && record.ProxyType == "forward" { + return fmt.Errorf("Docker Sandboxes routed the relay registry proof through credential-bearing forward egress") + } + if _, exactRelay := relayHosts[record.Host]; !exactRelay { + continue + } + if record.ProxyType != "transparent" { + return fmt.Errorf("Docker Sandboxes host-trust relay used unexpected %q routing", record.ProxyType) + } + if _, expectedRule := binding.PolicyRules[record.Rule]; !expectedRule { + return fmt.Errorf("Docker Sandboxes host-trust relay matched unexpected policy rule %q", record.Rule) + } + foundTransparentRelay = true + } + if !foundTransparentRelay { + return fmt.Errorf("Docker Sandboxes policy log did not confirm fresh transparent routing for the exact EPAR host-trust relay endpoint and bound policy rule") + } + return nil } -func (p *Provider) ensureRelayToken(instanceName string) (*egressRelay, string, error) { +func (p *Provider) ensureRelayToken(instance provider.Instance) (relayBindingSnapshot, error) { + if err := validateInstance(instance, true); err != nil { + return relayBindingSnapshot{}, err + } p.relayMu.Lock() defer p.relayMu.Unlock() if p.relayTokens == nil { - p.relayTokens = make(map[string]string) + p.relayTokens = make(map[string]relayTokenBinding) } if p.relayConnections == nil { p.relayConnections = make(map[string]map[net.Conn]struct{}) } - if token := p.relayTokens[instanceName]; token != "" && p.relay != nil { - return p.relay, token, nil + if binding, exists := p.relayTokens[instance.Name]; exists && binding.ProviderID == instance.ProviderID && binding.Token != "" && binding.Epoch != 0 && p.relay != nil { + return relayBindingSnapshotLocked(instance, binding, p.relay), nil + } + if _, exists := p.relayTokens[instance.Name]; exists { + p.revokeRelayTokenLocked(instance.Name) } if p.relay == nil { listener, err := net.Listen("tcp4", net.JoinHostPort("0.0.0.0", strconv.Itoa(p.hostTrustRelayPort))) if err != nil { - return nil, "", err + return relayBindingSnapshot{}, err } port := listener.Addr().(*net.TCPAddr).Port p.relay = &egressRelay{ @@ -211,10 +378,84 @@ func (p *Provider) ensureRelayToken(instanceName string) (*egressRelay, string, p.relay.close() p.relay = nil } - return nil, "", err + return relayBindingSnapshot{}, err + } + p.relayEpoch++ + if p.relayEpoch == 0 { + p.relayEpoch++ + } + binding := relayTokenBinding{ProviderID: instance.ProviderID, Token: token, Epoch: p.relayEpoch} + p.relayTokens[instance.Name] = binding + return relayBindingSnapshotLocked(instance, binding, p.relay), nil +} + +func relayBindingSnapshotLocked(instance provider.Instance, binding relayTokenBinding, relay *egressRelay) relayBindingSnapshot { + policyRules := make(map[string]struct{}, len(binding.PolicyRules)) + for rule := range binding.PolicyRules { + policyRules[rule] = struct{}{} + } + port := 0 + if relay != nil { + port = relay.port + } + return relayBindingSnapshot{Instance: instance, Token: binding.Token, Epoch: binding.Epoch, Relay: relay, Port: port, PolicyRules: policyRules} +} + +func (p *Provider) currentRelayBinding(instance provider.Instance) (relayBindingSnapshot, error) { + p.relayMu.Lock() + defer p.relayMu.Unlock() + binding, exists := p.relayTokens[instance.Name] + if !exists || binding.ProviderID != instance.ProviderID || binding.Token == "" || binding.Epoch == 0 || p.relay == nil || p.relay.port <= 0 { + return relayBindingSnapshot{}, fmt.Errorf("Docker Sandboxes host-trust relay is not bound to the exact instance") + } + return relayBindingSnapshotLocked(instance, binding, p.relay), nil +} + +func (p *Provider) bindRelayPolicyRules(snapshot relayBindingSnapshot, ruleNames []string) (relayBindingSnapshot, error) { + policyRules := make(map[string]struct{}, len(ruleNames)) + for _, ruleName := range ruleNames { + if ruleName == "" { + return relayBindingSnapshot{}, fmt.Errorf("Docker Sandboxes host-trust relay policy readback omitted the matched rule identity") + } + policyRules[ruleName] = struct{}{} } - p.relayTokens[instanceName] = token - return p.relay, token, nil + if len(policyRules) == 0 { + return relayBindingSnapshot{}, fmt.Errorf("Docker Sandboxes host-trust relay policy proof is unavailable") + } + p.relayMu.Lock() + defer p.relayMu.Unlock() + binding, exists := p.relayTokens[snapshot.Instance.Name] + if !exists || !relayBindingMatchesSnapshot(binding, p.relay, snapshot) { + return relayBindingSnapshot{}, fmt.Errorf("Docker Sandboxes host-trust relay binding changed during activation") + } + binding.PolicyRules = policyRules + p.relayTokens[snapshot.Instance.Name] = binding + return relayBindingSnapshotLocked(snapshot.Instance, binding, p.relay), nil +} + +func (p *Provider) verifyRelayBinding(snapshot relayBindingSnapshot) error { + p.relayMu.Lock() + defer p.relayMu.Unlock() + binding, exists := p.relayTokens[snapshot.Instance.Name] + if !exists || !relayBindingMatchesSnapshot(binding, p.relay, snapshot) { + return fmt.Errorf("Docker Sandboxes host-trust relay binding changed during exact-instance verification") + } + return nil +} + +func relayBindingMatchesSnapshot(binding relayTokenBinding, relay *egressRelay, snapshot relayBindingSnapshot) bool { + return relay != nil && relay == snapshot.Relay && relay.port == snapshot.Port && binding.ProviderID == snapshot.Instance.ProviderID && binding.Token == snapshot.Token && binding.Epoch == snapshot.Epoch +} + +func (p *Provider) verifyExactRelayInstance(ctx context.Context, snapshot relayBindingSnapshot) error { + present, err := p.assertIdentity(ctx, snapshot.Instance) + if err != nil { + return err + } + if !present { + return fmt.Errorf("docker sandbox is missing") + } + return p.verifyRelayBinding(snapshot) } func stableRelayPort(identity string) int { @@ -231,7 +472,7 @@ func (p *Provider) newUniqueRelayTokenLocked() (string, error) { candidate := base64.RawURLEncoding.EncodeToString(tokenBytes) duplicate := false for _, existing := range p.relayTokens { - if subtle.ConstantTimeCompare([]byte(candidate), []byte(existing)) == 1 { + if subtle.ConstantTimeCompare([]byte(candidate), []byte(existing.Token)) == 1 { duplicate = true break } @@ -243,13 +484,34 @@ func (p *Provider) newUniqueRelayTokenLocked() (string, error) { return "", fmt.Errorf("could not allocate a unique relay credential") } -func (p *Provider) releaseRelayToken(instanceName string) { - if instanceName == "" { +func (p *Provider) releaseRelayToken(snapshot relayBindingSnapshot) { + if snapshot.Instance.Name == "" || snapshot.Instance.ProviderID == "" || snapshot.Epoch == 0 || snapshot.Token == "" { return } p.relayMu.Lock() defer p.relayMu.Unlock() - p.revokeRelayTokenLocked(instanceName) + binding, exists := p.relayTokens[snapshot.Instance.Name] + if !exists || !relayBindingMatchesSnapshot(binding, p.relay, snapshot) { + return + } + p.revokeRelayTokenLocked(snapshot.Instance.Name) + if len(p.relayTokens) == 0 && p.relay != nil { + p.relay.close() + p.relay = nil + } +} + +func (p *Provider) releaseRelayTokenForInstance(instance provider.Instance) { + if instance.Name == "" || instance.ProviderID == "" { + return + } + p.relayMu.Lock() + defer p.relayMu.Unlock() + binding, exists := p.relayTokens[instance.Name] + if !exists || binding.ProviderID != instance.ProviderID { + return + } + p.revokeRelayTokenLocked(instance.Name) if len(p.relayTokens) == 0 && p.relay != nil { p.relay.close() p.relay = nil @@ -262,12 +524,13 @@ func (p *Provider) reconcileRelayTokens(items []provider.InventoryItem) { if len(p.relayTokens) == 0 { return } - present := make(map[string]struct{}, len(items)) + present := make(map[string]string, len(items)) for _, item := range items { - present[item.Instance.Name] = struct{}{} + present[item.Instance.Name] = item.Instance.ProviderID } - for instanceName := range p.relayTokens { - if _, exists := present[instanceName]; !exists { + for instanceName, binding := range p.relayTokens { + providerID, exists := present[instanceName] + if !exists || providerID != binding.ProviderID { p.revokeRelayTokenLocked(instanceName) } } @@ -285,10 +548,11 @@ func (p *Provider) revokeRelayTokenLocked(instanceName string) { delete(p.relayConnections, instanceName) } -func (p *Provider) registerRelayConnection(instanceName string, connection net.Conn) bool { +func (p *Provider) registerRelayConnection(instanceName string, epoch uint64, connection net.Conn) bool { p.relayMu.Lock() defer p.relayMu.Unlock() - if p.relayTokens[instanceName] == "" { + binding, exists := p.relayTokens[instanceName] + if !exists || binding.Epoch != epoch { return false } connections := p.relayConnections[instanceName] @@ -352,11 +616,11 @@ func (relay *egressRelay) handle(connection net.Conn) { if len(parts) != 3 || parts[0] != strings.TrimSpace(relayProtocolPrefix) { return } - instanceName, authenticated := relay.authenticate(parts[1]) + instanceName, bindingEpoch, authenticated := relay.authenticate(parts[1]) if !authenticated { return } - if !relay.provider.registerRelayConnection(instanceName, connection) { + if !relay.provider.registerRelayConnection(instanceName, bindingEpoch, connection) { return } defer relay.provider.unregisterRelayConnection(instanceName, connection) @@ -420,15 +684,15 @@ func readRelayHeader(reader *bufio.Reader, limit int) (string, error) { } } -func (relay *egressRelay) authenticate(candidate string) (string, bool) { +func (relay *egressRelay) authenticate(candidate string) (string, uint64, bool) { relay.provider.relayMu.Lock() defer relay.provider.relayMu.Unlock() for instanceName, expected := range relay.provider.relayTokens { - if len(candidate) == len(expected) && subtle.ConstantTimeCompare([]byte(candidate), []byte(expected)) == 1 { - return instanceName, true + if len(candidate) == len(expected.Token) && subtle.ConstantTimeCompare([]byte(candidate), []byte(expected.Token)) == 1 { + return instanceName, expected.Epoch, true } } - return "", false + return "", 0, false } func resolvePublicTLSDestination(ctx context.Context, target string) (string, error) { @@ -489,3 +753,4 @@ func relayPublicAddress(address netip.Addr) bool { } var _ provider.HostTrustRuntimeActivator = (*Provider)(nil) +var _ provider.HostTrustRuntimeVerifier = (*Provider)(nil) diff --git a/internal/provider/dockersandboxes/egress_relay_test.go b/internal/provider/dockersandboxes/egress_relay_test.go index db64e61..82369da 100644 --- a/internal/provider/dockersandboxes/egress_relay_test.go +++ b/internal/provider/dockersandboxes/egress_relay_test.go @@ -17,6 +17,12 @@ import ( "github.com/solutionforest/ephemeral-action-runner/internal/provider" ) +const testRelayPolicyRule = "host relay" + +func relayTestInstance(name string) provider.Instance { + return provider.Instance{Name: name, ProviderID: "12345678-1234-1234-1234-123456789abc"} +} + func TestReadRelayHeaderDoesNotLimitTunnelPayload(t *testing.T) { payload := bytes.Repeat([]byte("tls-payload-"), relayHeaderLimit) reader := bufio.NewReader(bytes.NewReader(append([]byte("EPAR1 token target:443\n"), payload...))) @@ -38,11 +44,13 @@ func TestReadRelayHeaderDoesNotLimitTunnelPayload(t *testing.T) { func TestEgressRelayAuthenticatesHealthWithoutExposingToken(t *testing.T) { p := NewWithDryRun("sbx", false) - relay, token, err := p.ensureRelayToken("sandbox-one") + binding, err := p.ensureRelayToken(relayTestInstance("sandbox-one")) if err != nil { t.Fatal(err) } - defer p.releaseRelayToken("sandbox-one") + defer p.releaseRelayToken(binding) + relay := binding.Relay + token := binding.Token if len(token) != 43 { t.Fatalf("token length = %d, want 43", len(token)) } @@ -68,11 +76,12 @@ func TestEgressRelayAuthenticatesHealthWithoutExposingToken(t *testing.T) { func TestEgressRelayRejectsUnknownToken(t *testing.T) { p := NewWithDryRun("sbx", false) - relay, _, err := p.ensureRelayToken("sandbox-one") + binding, err := p.ensureRelayToken(relayTestInstance("sandbox-one")) if err != nil { t.Fatal(err) } - defer p.releaseRelayToken("sandbox-one") + defer p.releaseRelayToken(binding) + relay := binding.Relay connection, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", relay.port), 2*time.Second) if err != nil { t.Fatal(err) @@ -88,15 +97,16 @@ func TestEgressRelayRejectsUnknownToken(t *testing.T) { func TestRelayTokenRevocationClosesAuthenticatedConnections(t *testing.T) { p := NewWithDryRun("sbx", false) - if _, _, err := p.ensureRelayToken("sandbox-one"); err != nil { + binding, err := p.ensureRelayToken(relayTestInstance("sandbox-one")) + if err != nil { t.Fatal(err) } server, client := net.Pipe() defer client.Close() - if !p.registerRelayConnection("sandbox-one", server) { + if !p.registerRelayConnection("sandbox-one", binding.Epoch, server) { t.Fatal("registerRelayConnection() = false") } - p.releaseRelayToken("sandbox-one") + p.releaseRelayToken(binding) _ = client.SetWriteDeadline(time.Now().Add(time.Second)) if _, err := client.Write([]byte("probe")); err == nil { t.Fatal("revoked relay connection remained writable") @@ -123,6 +133,7 @@ func TestHostTrustRelayActivationFailureRollsBackExactAddedPolicy(t *testing.T) rulePresent := false removed := false rolledBack := false + var activationToken string p.runCommand = func(_ context.Context, request commandRequest) (provider.ExecResult, error) { args := strings.Join(request.args, " ") switch { @@ -146,7 +157,11 @@ func TestHostTrustRelayActivationFailureRollsBackExactAddedPolicy(t *testing.T) rolledBack = true return provider.ExecResult{}, nil case strings.HasPrefix(args, "exec -i "+testName+" -- bash -lc ") && strings.Contains(args, "/opt/epar/configure-egress-relay.sh"): - return provider.ExecResult{}, errors.New("guest activation failed") + if len(request.sensitiveValues) != 1 { + t.Fatalf("sensitive value count = %d, want one relay token", len(request.sensitiveValues)) + } + activationToken = request.sensitiveValues[0] + return provider.ExecResult{Stderr: "EPAR host-trust relay: activation failed at private-dockerd-contract (exit=1) token=" + activationToken}, errors.New("guest activation failed") default: t.Fatalf("unexpected command: %v", request.args) return provider.ExecResult{}, nil @@ -155,7 +170,16 @@ func TestHostTrustRelayActivationFailureRollsBackExactAddedPolicy(t *testing.T) err := p.ActivateHostTrustRuntime(context.Background(), testInstance) if err == nil || !strings.Contains(err.Error(), "guest activation failed") { - t.Fatalf("activation error = %v, want guest failure", err) + t.Fatalf("activation error did not preserve the guest failure") + } + if !strings.Contains(err.Error(), "activation failed at private-dockerd-contract") { + t.Fatalf("activation error did not preserve the fixed failure stage") + } + if activationToken == "" { + t.Fatalf("activation request did not contain a relay token") + } + if strings.Contains(err.Error(), activationToken) { + t.Fatalf("activation error contained the sensitive relay token") } if rulePresent || !removed || !rolledBack { t.Fatalf("rollback state = policy present %t policy removed %t guest rolled back %t", rulePresent, removed, rolledBack) @@ -205,7 +229,7 @@ func TestHostTrustRelayActivationCommitsOnlyAfterFreshPolicyProof(t *testing.T) if err := p.ActivateHostTrustRuntime(context.Background(), testInstance); err != nil { t.Fatal(err) } - defer p.releaseRelayToken(testName) + defer p.releaseRelayTokenForInstance(testInstance) if !committed || !rulePresent || len(p.relayTokens) != 1 || p.relay == nil { t.Fatalf("committed activation state = commit %t policy %t tokens %d relay %v", committed, rulePresent, len(p.relayTokens), p.relay) } @@ -226,6 +250,170 @@ func TestHostTrustRelayDebugDiagnosticsCanBeEnabled(t *testing.T) { } } +func TestHostTrustRelayVerificationIsReadOnlyAndExact(t *testing.T) { + p := NewWithDryRun("sbx", false) + p.ConfigureHostTrustRelay(true, "verify-test") + binding, err := p.ensureRelayToken(testInstance) + if err != nil { + t.Fatal(err) + } + binding, err = p.bindRelayPolicyRules(binding, []string{testRelayPolicyRule}) + if err != nil { + t.Fatal(err) + } + defer p.releaseRelayToken(binding) + relay := binding.Relay + token := binding.Token + guestProbeSeen := false + policyLogSeen := false + p.runCommand = func(_ context.Context, request commandRequest) (provider.ExecResult, error) { + args := strings.Join(request.args, " ") + switch { + case args == "ls --json": + return provider.ExecResult{Stdout: readyListJSON}, nil + case strings.HasPrefix(args, "exec -i "+testName+" -- bash -lc "): + guestProbeSeen = true + if strings.Contains(args, "configure-egress-relay.sh") { + t.Fatal("read-only relay verification invoked guest relay configuration") + } + if strings.Contains(args, token) { + t.Fatal("read-only relay verification placed the relay token in the guest command") + } + if len(request.sensitiveValues) != 1 || request.sensitiveValues[0] != token { + t.Fatal("read-only relay verification did not mark the relay token as sensitive") + } + return provider.ExecResult{}, nil + case args == "policy log "+testName+" --json": + policyLogSeen = true + lastSeen := time.Now().UTC() + entry := policyLogEntry(net.JoinHostPort("localhost", fmt.Sprint(relay.port)), testName, "transparent", lastSeen) + return provider.ExecResult{Stdout: fmt.Sprintf(`{"blocked_hosts":[],"allowed_hosts":[%s]}`, entry)}, nil + default: + t.Fatalf("unexpected read-only relay verification command: %v", request.args) + return provider.ExecResult{}, nil + } + } + + if err := p.VerifyHostTrustRuntime(context.Background(), testInstance); err != nil { + t.Fatal(err) + } + if !guestProbeSeen || !policyLogSeen { + t.Fatalf("read-only relay proof = guest probe %t policy log %t, want both", guestProbeSeen, policyLogSeen) + } + stored := p.relayTokens[testName] + if stored.ProviderID != testInstance.ProviderID || stored.Token != token || stored.Epoch != binding.Epoch { + t.Fatal("read-only relay verification changed the exact relay credential") + } +} + +func TestHostTrustRelayVerificationFailsClosedWithoutExactControllerBinding(t *testing.T) { + p := NewWithDryRun("sbx", false) + p.ConfigureHostTrustRelay(true, "missing-binding-test") + commands := 0 + p.runCommand = func(_ context.Context, request commandRequest) (provider.ExecResult, error) { + commands++ + if strings.Join(request.args, " ") != "ls --json" { + t.Fatalf("unexpected command before exact relay binding failure: %v", request.args) + } + return provider.ExecResult{Stdout: readyListJSON}, nil + } + + err := p.VerifyHostTrustRuntime(context.Background(), testInstance) + if err == nil || !strings.Contains(err.Error(), "not bound to the exact instance") { + t.Fatalf("verification error = %v, want exact relay binding failure", err) + } + if commands != 1 { + t.Fatalf("commands before exact relay binding failure = %d, want identity readback only", commands) + } +} + +func TestHostTrustRelayVerificationFailsWhenBindingRebindsDuringGuestProbe(t *testing.T) { + p := NewWithDryRun("sbx", false) + p.ConfigureHostTrustRelay(true, "rebind-test") + original, err := p.ensureRelayToken(testInstance) + if err != nil { + t.Fatal(err) + } + original, err = p.bindRelayPolicyRules(original, []string{testRelayPolicyRule}) + if err != nil { + t.Fatal(err) + } + replacement := testInstance + replacement.ProviderID = "87654321-4321-4321-4321-cba987654321" + var replacementBinding relayBindingSnapshot + p.runCommand = func(_ context.Context, request commandRequest) (provider.ExecResult, error) { + args := strings.Join(request.args, " ") + switch { + case args == "ls --json": + return provider.ExecResult{Stdout: readyListJSON}, nil + case strings.HasPrefix(args, "exec -i "+testName+" -- bash -lc "): + var bindErr error + replacementBinding, bindErr = p.ensureRelayToken(replacement) + if bindErr != nil { + t.Fatal(bindErr) + } + return provider.ExecResult{}, nil + default: + t.Fatalf("unexpected command after relay rebind: %v", request.args) + return provider.ExecResult{}, nil + } + } + + err = p.VerifyHostTrustRuntime(context.Background(), testInstance) + if err == nil || !strings.Contains(err.Error(), "binding changed") { + t.Fatalf("verification error = %v, want exact binding epoch failure", err) + } + defer p.releaseRelayToken(replacementBinding) + p.releaseRelayToken(original) + current, currentErr := p.currentRelayBinding(replacement) + if currentErr != nil { + t.Fatalf("stale verification cleanup revoked replacement binding: %v", currentErr) + } + if current.Epoch != replacementBinding.Epoch || current.Token != replacementBinding.Token { + t.Fatal("stale verification cleanup changed the replacement binding") + } +} + +func TestStaleStopDoesNotReleaseReplacementRelayBinding(t *testing.T) { + p := NewWithDryRun("sbx", false) + replacement, err := p.ensureRelayToken(testInstance) + if err != nil { + t.Fatal(err) + } + defer p.releaseRelayToken(replacement) + stale := testInstance + stale.ProviderID = "87654321-4321-4321-4321-cba987654321" + p.runCommand = func(_ context.Context, request commandRequest) (provider.ExecResult, error) { + if strings.Join(request.args, " ") != "ls --json" { + t.Fatalf("stale stop reached a mutating command: %v", request.args) + } + return provider.ExecResult{Stdout: readyListJSON}, nil + } + + err = p.Stop(context.Background(), stale) + if err == nil || !strings.Contains(err.Error(), "identity changed") { + t.Fatalf("stale stop error = %v, want exact identity mismatch", err) + } + current, currentErr := p.currentRelayBinding(testInstance) + if currentErr != nil { + t.Fatalf("stale stop revoked replacement binding: %v", currentErr) + } + if current.Epoch != replacement.Epoch || current.Token != replacement.Token { + t.Fatal("stale stop changed the replacement binding") + } +} + +func TestHostTrustRelayGuestProbeFitsMaintenanceBudget(t *testing.T) { + if guestRelayProbeTimeout >= 15*time.Second { + t.Fatalf("guest relay probe timeout = %s, want less than the complete maintenance budget", guestRelayProbeTimeout) + } + for _, forbidden := range []string{"--max-time 15", "--max-time 30"} { + if strings.Contains(hostTrustRelayVerificationScript, forbidden) { + t.Fatalf("guest relay verification retained over-budget curl timeout %q", forbidden) + } + } +} + func TestRelayPublicAddressRejectsNonPublicAndSpecialRanges(t *testing.T) { rejected := []string{ "127.0.0.1", "10.0.0.1", "172.16.0.1", "192.168.0.1", "169.254.1.1", @@ -266,7 +454,7 @@ func TestSelectPublicTLSDestinationPrefersIPv4WithIPv6Fallback(t *testing.T) { } func TestVerifyHostTrustRelayPolicyRequiresFreshTransparentExactPort(t *testing.T) { - started := time.Now().UTC().Truncate(time.Microsecond) + started := time.Now().UTC().Truncate(time.Second) instance := provider.Instance{Name: "sandbox-one", ProviderID: "12345678-1234-1234-1234-123456789abc"} for _, test := range []struct { name string @@ -275,20 +463,23 @@ func TestVerifyHostTrustRelayPolicyRequiresFreshTransparentExactPort(t *testing. wantErr string }{ {name: "accepted", allowed: policyLogEntry("localhost:43123", "sandbox-one", "transparent", started), blocked: ""}, - {name: "stale", allowed: policyLogEntry("localhost:43123", "sandbox-one", "transparent", started.Add(-time.Minute)), wantErr: "did not confirm"}, + {name: "stale", allowed: policyLogEntry("localhost:43123", "sandbox-one", "transparent", started.Add(-time.Nanosecond)), wantErr: "did not confirm"}, + {name: "wrong port", allowed: policyLogEntry("localhost:43124", "sandbox-one", "transparent", started), wantErr: "did not confirm"}, {name: "wrong route", allowed: policyLogEntry("localhost:43123", "sandbox-one", "forward", started), wantErr: "unexpected"}, + {name: "wrong rule", allowed: policyLogEntryWithRule("localhost:43123", "sandbox-one", "transparent", "other-rule", started), wantErr: "unexpected policy rule"}, {name: "blocked", allowed: policyLogEntry("localhost:43123", "sandbox-one", "transparent", started), blocked: policyLogEntry("localhost:43123", "sandbox-one", "transparent", started), wantErr: "blocked"}, {name: "credential forward", allowed: policyLogEntry("registry-1.docker.io:443", "sandbox-one", "forward", started) + "," + policyLogEntry("localhost:43123", "sandbox-one", "transparent", started), wantErr: "credential-bearing"}, } { t.Run(test.name, func(t *testing.T) { p := NewWithDryRun("sbx", false) + binding := relayBindingSnapshot{Instance: instance, Port: 43123, PolicyRules: map[string]struct{}{testRelayPolicyRule: {}}} p.runCommand = func(_ context.Context, request commandRequest) (provider.ExecResult, error) { if strings.Join(request.args, " ") != "policy log sandbox-one --json" { t.Fatalf("unexpected command: %v", request.args) } return provider.ExecResult{Stdout: fmt.Sprintf(`{"blocked_hosts":[%s],"allowed_hosts":[%s]}`, test.blocked, test.allowed)}, nil } - err := p.verifyHostTrustRelayPolicy(context.Background(), instance, 43123, started) + err := p.verifyBoundHostTrustRelayPolicy(context.Background(), binding, started) if test.wantErr == "" && err != nil { t.Fatal(err) } @@ -300,7 +491,11 @@ func TestVerifyHostTrustRelayPolicyRequiresFreshTransparentExactPort(t *testing. } func policyLogEntry(host, vmName, proxyType string, lastSeen time.Time) string { - return fmt.Sprintf(`{"host":%q,"vm_name":%q,"proxy_type":%q,"rule":"","last_seen":%q,"since":%q,"count_since":1}`, host, vmName, proxyType, lastSeen.Format(time.RFC3339Nano), lastSeen.Format(time.RFC3339Nano)) + return policyLogEntryWithRule(host, vmName, proxyType, testRelayPolicyRule, lastSeen) +} + +func policyLogEntryWithRule(host, vmName, proxyType, rule string, lastSeen time.Time) string { + return fmt.Sprintf(`{"host":%q,"vm_name":%q,"proxy_type":%q,"rule":%q,"last_seen":%q,"since":%q,"count_since":1}`, host, vmName, proxyType, rule, lastSeen.Format(time.RFC3339Nano), lastSeen.Format(time.RFC3339Nano)) } func TestValidatePolicyCommandRejectsBroadPolicyAccess(t *testing.T) { diff --git a/internal/provider/dockersandboxes/network_policy.go b/internal/provider/dockersandboxes/network_policy.go index 42f1aa5..537980f 100644 --- a/internal/provider/dockersandboxes/network_policy.go +++ b/internal/provider/dockersandboxes/network_policy.go @@ -58,6 +58,7 @@ func (p *Provider) verifyHostTrustRelayPolicy(ctx context.Context, instance prov args: []string{"policy", "log", instance.Name, "--json"}, operation: "verify Docker Sandboxes host-trust relay route", outputLimit: diagnosticOutputLimit, + timeout: providerReadbackTimeout, }) if err != nil { return err @@ -121,7 +122,7 @@ func (p *Provider) ApplyNetworkPolicy(ctx context.Context, instance provider.Ins return err } args := []string{"policy", string(rule.Decision), "network", "--sandbox", instance.Name, strings.Join(rule.Resources, ",")} - result, runErr := p.run(ctx, commandRequest{args: args, operation: "apply docker sandbox network policy"}) + result, runErr := p.run(ctx, commandRequest{args: args, operation: "apply docker sandbox network policy", timeout: providerCleanupTimeout}) if runErr != nil && !strings.Contains(strings.ToLower(result.Stdout+"\n"+result.Stderr), "already covered") { return runErr } @@ -157,6 +158,7 @@ func (p *Provider) ReadGlobalNetworkPolicy(ctx context.Context) ([]provider.Netw args: []string{"policy", "ls", "--include-inactive", "--json"}, operation: "read docker sandboxes global network policy", outputLimit: diagnosticOutputLimit, + timeout: providerReadbackTimeout, }) if err != nil { return nil, err @@ -168,6 +170,7 @@ func (p *Provider) readNetworkPolicyVerified(ctx context.Context, instance provi result, err := p.run(ctx, commandRequest{ args: []string{"policy", "ls", instance.Name, "--include-inactive", "--json"}, operation: "read docker sandbox network policy", + timeout: providerReadbackTimeout, }) if err != nil { return nil, err @@ -209,6 +212,7 @@ func (p *Provider) RemoveNetworkPolicy(ctx context.Context, instance provider.In result, runErr := p.run(ctx, commandRequest{ args: []string{"policy", "rm", "network", "--sandbox", instance.Name, "--id", rule.ID}, operation: "remove docker sandbox network policy", + timeout: providerCleanupTimeout, }) if runErr != nil && !isMissingPolicyRule(result.Stdout+"\n"+result.Stderr+"\n"+runErr.Error()) { return runErr diff --git a/internal/provider/dockersandboxes/process_group_test.go b/internal/provider/dockersandboxes/process_group_test.go new file mode 100644 index 0000000..2de8d19 --- /dev/null +++ b/internal/provider/dockersandboxes/process_group_test.go @@ -0,0 +1,87 @@ +//go:build !windows + +package dockersandboxes + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/solutionforest/ephemeral-action-runner/internal/provider" +) + +func TestRunRawCancellationKillsManagedProcessGroup(t *testing.T) { + p := New("sh") + ctx, cancel := context.WithCancel(context.Background()) + childPIDPath := filepath.Join(t.TempDir(), "child.pid") + command := fmt.Sprintf("sleep 30 & echo $! > %q; printf ready; wait", childPIDPath) + started := make(chan struct{}) + type rawResult struct { + result provider.ExecResult + err error + } + finished := make(chan rawResult, 1) + go func() { + result, err := p.runRaw(ctx, commandRequest{ + args: []string{"-c", command}, + operation: "managed process group test", + outputLimit: defaultOutputLimit, + stdout: &cancellationSignalWriter{started: started}, + }) + finished <- rawResult{result: result, err: err} + }() + select { + case <-started: + case <-time.After(5 * time.Second): + cancel() + t.Fatal("managed process group helper did not start") + } + var childPID int + childDeadline := time.Now().Add(5 * time.Second) + for childPID == 0 && time.Now().Before(childDeadline) { + if data, err := os.ReadFile(childPIDPath); err == nil { + childPID, err = strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil { + t.Fatal(err) + } + break + } + time.Sleep(10 * time.Millisecond) + } + if childPID == 0 { + cancel() + t.Fatal("managed process group child did not report its PID") + } + t.Cleanup(func() { _ = syscall.Kill(childPID, syscall.SIGKILL) }) + startedAt := time.Now() + cancel() + select { + case outcome := <-finished: + if !errors.Is(outcome.err, context.Canceled) { + t.Fatalf("managed process group cancellation error = %v, want context.Canceled", outcome.err) + } + if elapsed := time.Since(startedAt); elapsed > 2*time.Second { + t.Fatalf("managed process group cancellation took %s", elapsed) + } + case <-time.After(8 * time.Second): + t.Fatal("managed process group did not terminate after cancellation") + } + processGoneDeadline := time.Now().Add(2 * time.Second) + for { + err := syscall.Kill(childPID, 0) + if errors.Is(err, syscall.ESRCH) { + break + } + if time.Now().After(processGoneDeadline) { + t.Fatalf("managed process group child still exists after cancellation: %v", err) + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/internal/provider/dockersandboxes/process_other.go b/internal/provider/dockersandboxes/process_other.go index f4fcf03..a25a552 100644 --- a/internal/provider/dockersandboxes/process_other.go +++ b/internal/provider/dockersandboxes/process_other.go @@ -2,6 +2,26 @@ package dockersandboxes -import "os/exec" +import ( + "os/exec" + "syscall" +) -func isolateKeepaliveProcess(*exec.Cmd) {} +func isolateKeepaliveProcess(command *exec.Cmd) { + command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} +} + +func isolateManagedProcess(command *exec.Cmd) { + command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} +} + +func attachManagedProcess(*exec.Cmd, bool) (func(), error) { + return func() {}, nil +} + +func killManagedProcess(command *exec.Cmd) error { + if command.Process == nil { + return nil + } + return syscall.Kill(-command.Process.Pid, syscall.SIGKILL) +} diff --git a/internal/provider/dockersandboxes/process_windows.go b/internal/provider/dockersandboxes/process_windows.go index 93c9007..bbf670d 100644 --- a/internal/provider/dockersandboxes/process_windows.go +++ b/internal/provider/dockersandboxes/process_windows.go @@ -3,13 +3,220 @@ package dockersandboxes import ( + "errors" + "fmt" "os/exec" + "strconv" + "sync" "syscall" + "time" + "unsafe" + + "golang.org/x/sys/windows" ) func isolateKeepaliveProcess(command *exec.Cmd) { - command.SysProcAttr = &syscall.SysProcAttr{ - HideWindow: true, - NoInheritHandles: true, + // Leave handle inheritance enabled so os/exec can pass the command's + // stdin/stdout/stderr handles to the child. On Windows, os/exec supplies an + // explicit PROC_THREAD_ATTRIBUTE_HANDLE_LIST, so unrelated inheritable + // handles are still excluded. NoInheritHandles would suppress that list as + // well and break EPAR's output capture (and some child process startups). + command.SysProcAttr = &syscall.SysProcAttr{HideWindow: true, CreationFlags: windows.CREATE_SUSPENDED} +} + +func isolateManagedProcess(command *exec.Cmd) { + // Keep the standard-handle list that os/exec builds; see the keepalive + // process comment above. + command.SysProcAttr = &syscall.SysProcAttr{HideWindow: true, CreationFlags: windows.CREATE_SUSPENDED} +} + +var managedProcessJobs sync.Map + +func attachManagedProcess(command *exec.Cmd, preserveDescendantsOnSuccess bool) (func(), error) { + if command.Process == nil { + return nil, fmt.Errorf("managed process has not started") + } + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return nil, fmt.Errorf("create Windows process job: %w", err) + } + closeJob := func() { _ = windows.CloseHandle(job) } + if !preserveDescendantsOnSuccess { + var limits windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION + limits.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, uintptr(unsafe.Pointer(&limits)), uint32(unsafe.Sizeof(limits))); err != nil { + closeJob() + return nil, fmt.Errorf("configure Windows process job: %w", err) + } + } + process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(command.Process.Pid)) + if err != nil { + closeJob() + return nil, fmt.Errorf("open managed process for Windows job: %w", err) + } + assignErr := windows.AssignProcessToJobObject(job, process) + _ = windows.CloseHandle(process) + if assignErr != nil { + closeJob() + return nil, fmt.Errorf("assign managed process to Windows job: %w", assignErr) + } + if err := assignExistingDescendantsToJob(job, command.Process.Pid); err != nil { + _ = windows.TerminateJobObject(job, 1) + closeJob() + return nil, fmt.Errorf("assign pre-existing Docker Sandboxes descendants to Windows job: %w", err) + } + if managedProcessWasCreatedSuspended(command) { + if err := resumeManagedProcess(command); err != nil { + _ = windows.TerminateJobObject(job, 1) + closeJob() + return nil, fmt.Errorf("resume Docker Sandboxes process after containment: %w", err) + } + } + managedProcessJobs.Store(command.Process.Pid, job) + var once sync.Once + return func() { + once.Do(func() { + managedProcessJobs.Delete(command.Process.Pid) + closeJob() + }) + }, nil +} + +func managedProcessWasCreatedSuspended(command *exec.Cmd) bool { + return command.SysProcAttr != nil && command.SysProcAttr.CreationFlags&windows.CREATE_SUSPENDED != 0 +} + +func resumeManagedProcess(command *exec.Cmd) error { + threadID, err := mainThreadID(command.Process.Pid) + if err != nil { + return err + } + thread, err := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, threadID) + if err != nil { + return err + } + defer windows.CloseHandle(thread) + _, err = windows.ResumeThread(thread) + return err +} + +func mainThreadID(processID int) (uint32, error) { + snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0) + if err != nil { + return 0, err + } + defer windows.CloseHandle(snapshot) + var entry windows.ThreadEntry32 + entry.Size = uint32(unsafe.Sizeof(entry)) + if err := windows.Thread32First(snapshot, &entry); err != nil { + return 0, err + } + for { + if entry.OwnerProcessID == uint32(processID) { + return entry.ThreadID, nil + } + if err := windows.Thread32Next(snapshot, &entry); err != nil { + if errors.Is(err, windows.ERROR_NO_MORE_FILES) { + break + } + return 0, err + } + } + return 0, fmt.Errorf("main thread for process %d was not found", processID) +} + +func assignExistingDescendantsToJob(job windows.Handle, rootPID int) error { + assigned := map[uint32]bool{uint32(rootPID): true} + for pass := 0; pass < 3; pass++ { + descendants, err := descendantProcessIDs(rootPID) + if err != nil { + return err + } + for _, pid := range descendants { + if assigned[pid] { + continue + } + process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, pid) + if err != nil { + return fmt.Errorf("open descendant process %d: %w", pid, err) + } + assignErr := windows.AssignProcessToJobObject(job, process) + _ = windows.CloseHandle(process) + if assignErr != nil { + return fmt.Errorf("assign descendant process %d: %w", pid, assignErr) + } + assigned[pid] = true + } + if pass < 2 { + time.Sleep(time.Millisecond) + } + } + return nil +} + +func descendantProcessIDs(rootPID int) ([]uint32, error) { + snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPPROCESS, 0) + if err != nil { + return nil, err + } + defer windows.CloseHandle(snapshot) + children := make(map[uint32][]uint32) + var entry windows.ProcessEntry32 + entry.Size = uint32(unsafe.Sizeof(entry)) + if err := windows.Process32First(snapshot, &entry); err != nil { + return nil, err + } + for { + children[entry.ParentProcessID] = append(children[entry.ParentProcessID], entry.ProcessID) + if err := windows.Process32Next(snapshot, &entry); err != nil { + if errors.Is(err, windows.ERROR_NO_MORE_FILES) { + break + } + return nil, err + } + } + seen := map[uint32]bool{uint32(rootPID): true} + queue := []uint32{uint32(rootPID)} + var descendants []uint32 + for len(queue) > 0 { + parent := queue[0] + queue = queue[1:] + for _, child := range children[parent] { + if seen[child] { + continue + } + seen[child] = true + descendants = append(descendants, child) + queue = append(queue, child) + } + } + return descendants, nil +} + +func killManagedProcess(command *exec.Cmd) error { + if command.Process == nil { + return nil + } + if value, ok := managedProcessJobs.Load(command.Process.Pid); ok { + if err := windows.TerminateJobObject(value.(windows.Handle), 1); err == nil { + return nil + } + } + taskkill := exec.Command("taskkill", "/T", "/F", "/PID", strconv.Itoa(command.Process.Pid)) + if err := taskkill.Start(); err != nil { + return err + } + done := make(chan error, 1) + go func() { + done <- taskkill.Wait() + }() + timer := time.NewTimer(2 * time.Second) + defer timer.Stop() + select { + case err := <-done: + return err + case <-timer.C: + _ = taskkill.Process.Kill() + return fmt.Errorf("taskkill did not finish within 2s") } } diff --git a/internal/provider/dockersandboxes/process_windows_test.go b/internal/provider/dockersandboxes/process_windows_test.go new file mode 100644 index 0000000..92d26ec --- /dev/null +++ b/internal/provider/dockersandboxes/process_windows_test.go @@ -0,0 +1,259 @@ +//go:build windows + +package dockersandboxes + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "golang.org/x/sys/windows" +) + +const ( + managedProcessTreeHelperEnv = "EPAR_WINDOWS_MANAGED_PROCESS_TREE_HELPER" + managedProcessTreePIDFileEnv = "EPAR_WINDOWS_MANAGED_PROCESS_TREE_PID_FILE" + managedProcessTreeReadyFileEnv = "EPAR_WINDOWS_MANAGED_PROCESS_TREE_READY_FILE" + managedProcessTreeDetachedEnv = "EPAR_WINDOWS_MANAGED_PROCESS_TREE_DETACHED" + managedProcessTreeHelperMarker = "managed-process-ready" + managedProcessTreeDetachedMarker = "detached-process-ready" +) + +func TestKillManagedProcessTerminatesProcessTree(t *testing.T) { + pidFile := filepath.Join(t.TempDir(), "child.pid") + command := managedProcessTreeCommand(pidFile, false) + var stdout, stderr bytes.Buffer + command.Stdout = &stdout + command.Stderr = &stderr + if err := command.Start(); err != nil { + t.Fatal(err) + } + childPID := 0 + waited := false + waiting := false + defer func() { + if childPID <= 0 { + childPID = readChildPID(pidFile) + } + if childPID > 0 { + terminateProcess(childPID) + } + if !waited && !waiting { + _ = command.Process.Kill() + _ = command.Wait() + } + }() + childPID, err := waitForChildPID(pidFile) + if err != nil { + t.Fatal(err) + } + cleanup, err := attachManagedProcess(command, false) + if err != nil { + t.Fatalf("attachManagedProcess() = %v", err) + } + defer cleanup() + if running, err := processIsRunning(childPID); err != nil || !running { + t.Fatalf("captured child process was not running before termination: running=%t error=%v", running, err) + } + if err := killManagedProcess(command); err != nil { + t.Fatalf("killManagedProcess() = %v", err) + } + finished := make(chan error, 1) + waiting = true + go func() { finished <- command.Wait() }() + select { + case <-finished: + waiting = false + waited = true + case <-time.After(5 * time.Second): + _ = killManagedProcess(command) + select { + case <-finished: + waiting = false + waited = true + case <-time.After(5 * time.Second): + t.Fatal("managed process tree did not terminate after taskkill") + } + } + running, err := processIsRunning(childPID) + if err != nil { + t.Fatalf("verify child process termination: %v", err) + } + if running { + t.Fatalf("child process %d survived managed process termination", childPID) + } + if !strings.Contains(stdout.String(), "managed-process-ready") { + t.Fatalf("managed process stdout was not captured: stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestAttachManagedProcessPreservesDetachedDescendant(t *testing.T) { + pidFile := filepath.Join(t.TempDir(), "child.pid") + command := managedProcessTreeCommand(pidFile, true) + var stdout, stderr bytes.Buffer + command.Stdout = &stdout + command.Stderr = &stderr + isolateManagedProcess(command) + if err := command.Start(); err != nil { + t.Fatal(err) + } + childPID := 0 + waited := false + defer func() { + if childPID <= 0 { + childPID = readChildPID(pidFile) + } + if childPID > 0 { + terminateProcess(childPID) + } + if !waited { + _ = command.Process.Kill() + _ = command.Wait() + } + }() + cleanup, err := attachManagedProcess(command, true) + if err != nil { + t.Fatalf("attachManagedProcess(preserve) = %v", err) + } + childPID, err = waitForChildPID(pidFile) + if err != nil { + t.Fatal(err) + } + waitErr := command.Wait() + waited = true + if waitErr != nil { + cleanup() + t.Fatalf("detached launcher exited with error: %v", waitErr) + } + cleanup() + if !strings.Contains(stdout.String(), "detached-process-ready") { + t.Fatalf("managed process stdout was not captured: stdout=%q stderr=%q", stdout.String(), stderr.String()) + } + running, err := processIsRunning(childPID) + if err != nil { + t.Fatalf("verify detached child process: %v", err) + } + if !running { + t.Fatalf("detached child process %d was terminated when the containment job closed", childPID) + } +} + +func TestManagedProcessTreeHelper(t *testing.T) { + switch os.Getenv(managedProcessTreeHelperEnv) { + case "": + return + case "child": + fmt.Fprintln(os.Stdout, "managed-process-child-ready") + time.Sleep(30 * time.Second) + case "parent": + pidFile := os.Getenv(managedProcessTreePIDFileEnv) + readyFile := os.Getenv(managedProcessTreeReadyFileEnv) + if pidFile == "" { + t.Fatal("managed process tree parent PID file is missing") + } + if readyFile == "" { + t.Fatal("managed process tree parent ready file is missing") + } + child := exec.Command(os.Args[0], "-test.run=^TestManagedProcessTreeHelper$") + child.Env = append(os.Environ(), managedProcessTreeHelperEnv+"=child", managedProcessTreePIDFileEnv+"="+pidFile, managedProcessTreeReadyFileEnv+"="+readyFile) + if err := child.Start(); err != nil { + t.Fatalf("start managed process tree child: %v", err) + } + if err := os.WriteFile(pidFile, []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = child.Process.Kill() + _ = child.Wait() + t.Fatalf("publish managed process tree child PID: %v", err) + } + marker := managedProcessTreeHelperMarker + if os.Getenv(managedProcessTreeDetachedEnv) == "1" { + marker = managedProcessTreeDetachedMarker + } + if _, err := fmt.Fprintln(os.Stdout, marker); err != nil { + _ = child.Process.Kill() + _ = child.Wait() + t.Fatalf("publish managed process tree readiness marker: %v", err) + } + if err := os.WriteFile(readyFile, []byte("ready"), 0o600); err != nil { + _ = child.Process.Kill() + _ = child.Wait() + t.Fatalf("publish managed process tree parent readiness: %v", err) + } + if os.Getenv(managedProcessTreeDetachedEnv) == "1" { + return + } + if err := child.Wait(); err != nil { + t.Fatalf("wait for managed process tree child: %v", err) + } + default: + t.Fatalf("unexpected managed process tree helper mode %q", os.Getenv(managedProcessTreeHelperEnv)) + } +} + +func managedProcessTreeCommand(pidFile string, detached bool) *exec.Cmd { + command := exec.Command(os.Args[0], "-test.run=^TestManagedProcessTreeHelper$") + readyFile := pidFile + ".ready" + command.Env = append(os.Environ(), managedProcessTreeHelperEnv+"=parent", managedProcessTreePIDFileEnv+"="+pidFile, managedProcessTreeReadyFileEnv+"="+readyFile) + if detached { + command.Env = append(command.Env, managedProcessTreeDetachedEnv+"=1") + } + return command +} + +func waitForChildPID(pidFile string) (int, error) { + readyFile := pidFile + ".ready" + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(readyFile); err == nil { + if pid := readChildPID(pidFile); pid > 0 { + return pid, nil + } + } else if !os.IsNotExist(err) { + return 0, fmt.Errorf("check process tree readiness: %w", err) + } + time.Sleep(25 * time.Millisecond) + } + return 0, fmt.Errorf("child process did not publish its PID within 5s") +} + +func readChildPID(pidFile string) int { + data, err := os.ReadFile(pidFile) + if err != nil { + return 0 + } + pid, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil || pid <= 0 { + return 0 + } + return pid +} + +func processIsRunning(pid int) (bool, error) { + process, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err != nil { + if err == windows.ERROR_INVALID_PARAMETER { + return false, nil + } + return false, fmt.Errorf("open process %d: %w", pid, err) + } + defer windows.CloseHandle(process) + var exitCode uint32 + if err := windows.GetExitCodeProcess(process, &exitCode); err != nil { + return false, fmt.Errorf("get process %d exit code: %w", pid, err) + } + return exitCode == 259, nil +} + +func terminateProcess(pid int) { + process, err := os.FindProcess(pid) + if err != nil { + return + } + _ = process.Kill() + _ = process.Release() +} diff --git a/internal/provider/dockersandboxes/promotion/preflight.go b/internal/provider/dockersandboxes/promotion/preflight.go index 1eb62e8..070f80f 100644 --- a/internal/provider/dockersandboxes/promotion/preflight.go +++ b/internal/provider/dockersandboxes/promotion/preflight.go @@ -20,8 +20,10 @@ import ( ) const ( - DisableEnvironment = "EPAR_DISABLE_DOCKER_SANDBOXES" - preflightOutputLimit = 256 << 10 + DisableEnvironment = "EPAR_DISABLE_DOCKER_SANDBOXES" + preflightOutputLimit = 256 << 10 + preflightCommandTimeout = 30 * time.Second + preflightWaitDelay = 5 * time.Second ) var ( @@ -186,13 +188,44 @@ func runSBXCommand(ctx context.Context, args []string) ([]byte, error) { if args[0] == "tui" || args[0] == "reset" { return nil, fmt.Errorf("refusing to invoke forbidden sbx subcommand %q", args[0]) } - command := exec.CommandContext(ctx, "sbx", args...) + commandCtx, cancel := context.WithTimeout(ctx, preflightCommandTimeout) + defer cancel() + releaseHostLock, err := provider.AcquireControlPlaneCommandLock(commandCtx) + if err != nil { + return nil, err + } + defer releaseHostLock() + command := exec.CommandContext(commandCtx, "sbx", args...) + isolatePreflightProcess(command) + command.WaitDelay = preflightWaitDelay + defaultCancel := command.Cancel + command.Cancel = func() error { + if err := killPreflightProcess(command); err != nil { + return defaultCancel() + } + return nil + } command.Env = sandboxCommandEnvironment() stdout := &preflightBuffer{limit: preflightOutputLimit} stderr := &preflightBuffer{limit: preflightOutputLimit} command.Stdout = stdout command.Stderr = stderr - err := command.Run() + err = command.Start() + if err == nil { + cleanup, attachErr := attachPreflightProcess(command) + if attachErr != nil { + cancel() + killErr := killPreflightProcess(command) + waitErr := command.Wait() + err = errors.Join(fmt.Errorf("attach sbx preflight process containment: %w", attachErr), killErr, waitErr) + } else { + defer cleanup() + err = command.Wait() + } + } + if ctxErr := commandCtx.Err(); ctxErr != nil { + err = errors.Join(ctxErr, err) + } if stdout.overflow || stderr.overflow { err = errors.Join(err, errors.New("sbx preflight output limit exceeded")) } diff --git a/internal/provider/dockersandboxes/promotion/process_unix.go b/internal/provider/dockersandboxes/promotion/process_unix.go new file mode 100644 index 0000000..5638c02 --- /dev/null +++ b/internal/provider/dockersandboxes/promotion/process_unix.go @@ -0,0 +1,23 @@ +//go:build !windows + +package promotion + +import ( + "os/exec" + "syscall" +) + +func isolatePreflightProcess(command *exec.Cmd) { + command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} +} + +func attachPreflightProcess(*exec.Cmd) (func(), error) { + return func() {}, nil +} + +func killPreflightProcess(command *exec.Cmd) error { + if command.Process == nil { + return nil + } + return syscall.Kill(-command.Process.Pid, syscall.SIGKILL) +} diff --git a/internal/provider/dockersandboxes/promotion/process_unix_test.go b/internal/provider/dockersandboxes/promotion/process_unix_test.go new file mode 100644 index 0000000..c6ea205 --- /dev/null +++ b/internal/provider/dockersandboxes/promotion/process_unix_test.go @@ -0,0 +1,32 @@ +//go:build !windows + +package promotion + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +func TestRunSBXCommandBoundsProcessTreeOnContextCancellation(t *testing.T) { + dir := t.TempDir() + t.Setenv("EPAR_STATE_HOME", t.TempDir()) + helper := filepath.Join(dir, "sbx") + if err := os.WriteFile(helper, []byte("#!/bin/sh\n(sleep 30) &\nwait\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + started := time.Now() + _, err := runSBXCommand(ctx, []string{"version"}) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("runSBXCommand() error = %v, want context deadline exceeded", err) + } + if elapsed := time.Since(started); elapsed > 3*time.Second { + t.Fatalf("runSBXCommand() took %s after cancellation; process tree was not bounded", elapsed) + } +} diff --git a/internal/provider/dockersandboxes/promotion/process_windows.go b/internal/provider/dockersandboxes/promotion/process_windows.go new file mode 100644 index 0000000..ef43318 --- /dev/null +++ b/internal/provider/dockersandboxes/promotion/process_windows.go @@ -0,0 +1,212 @@ +//go:build windows + +package promotion + +import ( + "errors" + "fmt" + "os/exec" + "strconv" + "sync" + "syscall" + "time" + "unsafe" + + "golang.org/x/sys/windows" +) + +func isolatePreflightProcess(command *exec.Cmd) { + // Leave handle inheritance enabled so os/exec can pass the command's + // stdin/stdout/stderr handles to sbx. On Windows, os/exec supplies an + // explicit PROC_THREAD_ATTRIBUTE_HANDLE_LIST, so unrelated inheritable + // handles are still excluded. NoInheritHandles would suppress that list as + // well and make the preflight JSON output unavailable to EPAR. + command.SysProcAttr = &syscall.SysProcAttr{HideWindow: true, CreationFlags: windows.CREATE_SUSPENDED} +} + +var preflightProcessJobs sync.Map + +func attachPreflightProcess(command *exec.Cmd) (func(), error) { + if command.Process == nil { + return nil, fmt.Errorf("preflight process has not started") + } + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return nil, fmt.Errorf("create Windows preflight job: %w", err) + } + closeJob := func() { _ = windows.CloseHandle(job) } + var limits windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION + limits.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, uintptr(unsafe.Pointer(&limits)), uint32(unsafe.Sizeof(limits))); err != nil { + closeJob() + return nil, fmt.Errorf("configure Windows preflight job: %w", err) + } + process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(command.Process.Pid)) + if err != nil { + closeJob() + return nil, fmt.Errorf("open preflight process for Windows job: %w", err) + } + assignErr := windows.AssignProcessToJobObject(job, process) + _ = windows.CloseHandle(process) + if assignErr != nil { + closeJob() + return nil, fmt.Errorf("assign preflight process to Windows job: %w", assignErr) + } + if err := assignExistingPreflightDescendantsToJob(job, command.Process.Pid); err != nil { + _ = windows.TerminateJobObject(job, 1) + closeJob() + return nil, fmt.Errorf("assign pre-existing preflight descendants to Windows job: %w", err) + } + if preflightProcessWasCreatedSuspended(command) { + if err := resumePreflightProcess(command); err != nil { + _ = windows.TerminateJobObject(job, 1) + closeJob() + return nil, fmt.Errorf("resume sbx preflight process after containment: %w", err) + } + } + preflightProcessJobs.Store(command.Process.Pid, job) + var once sync.Once + return func() { + once.Do(func() { + preflightProcessJobs.Delete(command.Process.Pid) + closeJob() + }) + }, nil +} + +func preflightProcessWasCreatedSuspended(command *exec.Cmd) bool { + return command.SysProcAttr != nil && command.SysProcAttr.CreationFlags&windows.CREATE_SUSPENDED != 0 +} + +func resumePreflightProcess(command *exec.Cmd) error { + threadID, err := preflightMainThreadID(command.Process.Pid) + if err != nil { + return err + } + thread, err := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, threadID) + if err != nil { + return err + } + defer windows.CloseHandle(thread) + _, err = windows.ResumeThread(thread) + return err +} + +func preflightMainThreadID(processID int) (uint32, error) { + snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0) + if err != nil { + return 0, err + } + defer windows.CloseHandle(snapshot) + var entry windows.ThreadEntry32 + entry.Size = uint32(unsafe.Sizeof(entry)) + if err := windows.Thread32First(snapshot, &entry); err != nil { + return 0, err + } + for { + if entry.OwnerProcessID == uint32(processID) { + return entry.ThreadID, nil + } + if err := windows.Thread32Next(snapshot, &entry); err != nil { + if errors.Is(err, windows.ERROR_NO_MORE_FILES) { + break + } + return 0, err + } + } + return 0, fmt.Errorf("main thread for process %d was not found", processID) +} + +func assignExistingPreflightDescendantsToJob(job windows.Handle, rootPID int) error { + assigned := map[uint32]bool{uint32(rootPID): true} + for pass := 0; pass < 3; pass++ { + descendants, err := preflightDescendantProcessIDs(rootPID) + if err != nil { + return err + } + for _, pid := range descendants { + if assigned[pid] { + continue + } + process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, pid) + if err != nil { + return fmt.Errorf("open descendant process %d: %w", pid, err) + } + assignErr := windows.AssignProcessToJobObject(job, process) + _ = windows.CloseHandle(process) + if assignErr != nil { + return fmt.Errorf("assign descendant process %d: %w", pid, assignErr) + } + assigned[pid] = true + } + if pass < 2 { + time.Sleep(time.Millisecond) + } + } + return nil +} + +func preflightDescendantProcessIDs(rootPID int) ([]uint32, error) { + snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPPROCESS, 0) + if err != nil { + return nil, err + } + defer windows.CloseHandle(snapshot) + children := make(map[uint32][]uint32) + var entry windows.ProcessEntry32 + entry.Size = uint32(unsafe.Sizeof(entry)) + if err := windows.Process32First(snapshot, &entry); err != nil { + return nil, err + } + for { + children[entry.ParentProcessID] = append(children[entry.ParentProcessID], entry.ProcessID) + if err := windows.Process32Next(snapshot, &entry); err != nil { + if errors.Is(err, windows.ERROR_NO_MORE_FILES) { + break + } + return nil, err + } + } + seen := map[uint32]bool{uint32(rootPID): true} + queue := []uint32{uint32(rootPID)} + var descendants []uint32 + for len(queue) > 0 { + parent := queue[0] + queue = queue[1:] + for _, child := range children[parent] { + if seen[child] { + continue + } + seen[child] = true + descendants = append(descendants, child) + queue = append(queue, child) + } + } + return descendants, nil +} + +func killPreflightProcess(command *exec.Cmd) error { + if command.Process == nil { + return nil + } + if value, ok := preflightProcessJobs.Load(command.Process.Pid); ok { + if err := windows.TerminateJobObject(value.(windows.Handle), 1); err == nil { + return nil + } + } + taskkill := exec.Command("taskkill", "/T", "/F", "/PID", strconv.Itoa(command.Process.Pid)) + if err := taskkill.Start(); err != nil { + return err + } + done := make(chan error, 1) + go func() { done <- taskkill.Wait() }() + timer := time.NewTimer(2 * time.Second) + defer timer.Stop() + select { + case err := <-done: + return err + case <-timer.C: + _ = taskkill.Process.Kill() + return fmt.Errorf("taskkill did not finish within 2s") + } +} diff --git a/internal/provider/dockersandboxes/promotion/process_windows_test.go b/internal/provider/dockersandboxes/promotion/process_windows_test.go new file mode 100644 index 0000000..a227ad8 --- /dev/null +++ b/internal/provider/dockersandboxes/promotion/process_windows_test.go @@ -0,0 +1,243 @@ +//go:build windows + +package promotion + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "golang.org/x/sys/windows" +) + +const ( + preflightProcessTreeHelperEnv = "EPAR_WINDOWS_PREFLIGHT_PROCESS_TREE_HELPER" + preflightProcessTreePIDFileEnv = "EPAR_WINDOWS_PREFLIGHT_PROCESS_TREE_PID_FILE" + preflightProcessTreeReadyFileEnv = "EPAR_WINDOWS_PREFLIGHT_PROCESS_TREE_READY_FILE" + preflightProcessTreeDetachedEnv = "EPAR_WINDOWS_PREFLIGHT_PROCESS_TREE_DETACHED" + preflightProcessTreeHelperMarker = "preflight-process-ready" +) + +func TestKillPreflightProcessTerminatesProcessTree(t *testing.T) { + pidFile := filepath.Join(t.TempDir(), "child.pid") + command := preflightProcessTreeCommand(pidFile, false) + var stdout, stderr bytes.Buffer + command.Stdout = &stdout + command.Stderr = &stderr + if err := command.Start(); err != nil { + t.Fatal(err) + } + childPID := 0 + waited := false + waiting := false + defer func() { + if childPID <= 0 { + childPID = readChildPID(pidFile) + } + if childPID > 0 { + terminateProcess(childPID) + } + if !waited && !waiting { + _ = command.Process.Kill() + _ = command.Wait() + } + }() + childPID, err := waitForChildPID(pidFile) + if err != nil { + t.Fatal(err) + } + cleanup, err := attachPreflightProcess(command) + if err != nil { + t.Fatalf("attachPreflightProcess() = %v", err) + } + defer cleanup() + if running, err := processIsRunning(childPID); err != nil || !running { + t.Fatalf("captured child process was not running before termination: running=%t error=%v", running, err) + } + if err := killPreflightProcess(command); err != nil { + t.Fatalf("killPreflightProcess() = %v", err) + } + finished := make(chan error, 1) + waiting = true + go func() { finished <- command.Wait() }() + select { + case <-finished: + waiting = false + waited = true + case <-time.After(5 * time.Second): + _ = killPreflightProcess(command) + select { + case <-finished: + waiting = false + waited = true + case <-time.After(5 * time.Second): + t.Fatal("preflight process tree did not terminate") + } + } + running, err := processIsRunning(childPID) + if err != nil { + t.Fatalf("verify child process termination: %v", err) + } + if running { + t.Fatalf("child process %d survived preflight process termination", childPID) + } + if !strings.Contains(stdout.String(), "preflight-process-ready") { + t.Fatalf("preflight process stdout was not captured: stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestPreflightProcessCapturesStdout(t *testing.T) { + command := preflightProcessOutputCommand() + var stdout, stderr bytes.Buffer + command.Stdout = &stdout + command.Stderr = &stderr + isolatePreflightProcess(command) + if err := command.Start(); err != nil { + t.Fatal(err) + } + waited := false + defer func() { + if !waited { + _ = command.Process.Kill() + _ = command.Wait() + } + }() + cleanup, err := attachPreflightProcess(command) + if err != nil { + t.Fatalf("attachPreflightProcess() = %v", err) + } + waitErr := command.Wait() + waited = true + if waitErr != nil { + cleanup() + t.Fatalf("preflight launcher exited with error: %v", waitErr) + } + cleanup() + if !strings.Contains(stdout.String(), "preflight-process-ready") { + t.Fatalf("preflight process stdout was not captured: stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestPreflightProcessTreeHelper(t *testing.T) { + switch os.Getenv(preflightProcessTreeHelperEnv) { + case "": + return + case "output-only": + fmt.Fprintln(os.Stdout, preflightProcessTreeHelperMarker) + case "child": + fmt.Fprintln(os.Stdout, "preflight-process-child-ready") + time.Sleep(30 * time.Second) + case "parent": + pidFile := os.Getenv(preflightProcessTreePIDFileEnv) + readyFile := os.Getenv(preflightProcessTreeReadyFileEnv) + if pidFile == "" { + t.Fatal("preflight process tree parent PID file is missing") + } + if readyFile == "" { + t.Fatal("preflight process tree parent ready file is missing") + } + child := exec.Command(os.Args[0], "-test.run=^TestPreflightProcessTreeHelper$") + child.Env = append(os.Environ(), preflightProcessTreeHelperEnv+"=child", preflightProcessTreePIDFileEnv+"="+pidFile, preflightProcessTreeReadyFileEnv+"="+readyFile) + if err := child.Start(); err != nil { + t.Fatalf("start preflight process tree child: %v", err) + } + if err := os.WriteFile(pidFile, []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = child.Process.Kill() + _ = child.Wait() + t.Fatalf("publish preflight process tree child PID: %v", err) + } + if _, err := fmt.Fprintln(os.Stdout, preflightProcessTreeHelperMarker); err != nil { + _ = child.Process.Kill() + _ = child.Wait() + t.Fatalf("publish preflight process tree readiness marker: %v", err) + } + if err := os.WriteFile(readyFile, []byte("ready"), 0o600); err != nil { + _ = child.Process.Kill() + _ = child.Wait() + t.Fatalf("publish preflight process tree parent readiness: %v", err) + } + if os.Getenv(preflightProcessTreeDetachedEnv) == "1" { + return + } + if err := child.Wait(); err != nil { + t.Fatalf("wait for preflight process tree child: %v", err) + } + default: + t.Fatalf("unexpected preflight process tree helper mode %q", os.Getenv(preflightProcessTreeHelperEnv)) + } +} + +func preflightProcessTreeCommand(pidFile string, detached bool) *exec.Cmd { + command := exec.Command(os.Args[0], "-test.run=^TestPreflightProcessTreeHelper$") + readyFile := pidFile + ".ready" + command.Env = append(os.Environ(), preflightProcessTreeHelperEnv+"=parent", preflightProcessTreePIDFileEnv+"="+pidFile, preflightProcessTreeReadyFileEnv+"="+readyFile) + if detached { + command.Env = append(command.Env, preflightProcessTreeDetachedEnv+"=1") + } + return command +} + +func preflightProcessOutputCommand() *exec.Cmd { + command := exec.Command(os.Args[0], "-test.run=^TestPreflightProcessTreeHelper$") + command.Env = append(os.Environ(), preflightProcessTreeHelperEnv+"=output-only") + return command +} + +func waitForChildPID(pidFile string) (int, error) { + readyFile := pidFile + ".ready" + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(readyFile); err == nil { + if pid := readChildPID(pidFile); pid > 0 { + return pid, nil + } + } else if !os.IsNotExist(err) { + return 0, fmt.Errorf("check process tree readiness: %w", err) + } + time.Sleep(25 * time.Millisecond) + } + return 0, fmt.Errorf("child process did not publish its PID within 5s") +} + +func readChildPID(pidFile string) int { + data, err := os.ReadFile(pidFile) + if err != nil { + return 0 + } + pid, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil || pid <= 0 { + return 0 + } + return pid +} + +func processIsRunning(pid int) (bool, error) { + process, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err != nil { + if err == windows.ERROR_INVALID_PARAMETER { + return false, nil + } + return false, fmt.Errorf("open process %d: %w", pid, err) + } + defer windows.CloseHandle(process) + var exitCode uint32 + if err := windows.GetExitCodeProcess(process, &exitCode); err != nil { + return false, fmt.Errorf("get process %d exit code: %w", pid, err) + } + return exitCode == 259, nil +} + +func terminateProcess(pid int) { + process, err := os.FindProcess(pid) + if err != nil { + return + } + _ = process.Kill() + _ = process.Release() +} diff --git a/internal/provider/dockersandboxes/promotion/space_unix.go b/internal/provider/dockersandboxes/promotion/space_unix.go index f6847a7..fe1f511 100644 --- a/internal/provider/dockersandboxes/promotion/space_unix.go +++ b/internal/provider/dockersandboxes/promotion/space_unix.go @@ -3,6 +3,7 @@ package promotion import ( + "context" "fmt" "math" "os" @@ -35,7 +36,14 @@ func sandboxVirtualizationAvailable() error { } return file.Close() case "darwin": - output, err := exec.Command("/usr/sbin/sysctl", "-n", "kern.hv_support").Output() + ctx, cancel := context.WithTimeout(context.Background(), preflightCommandTimeout) + defer cancel() + command := exec.CommandContext(ctx, "/usr/sbin/sysctl", "-n", "kern.hv_support") + command.WaitDelay = preflightWaitDelay + output, err := command.Output() + if ctxErr := ctx.Err(); ctxErr != nil { + err = ctxErr + } if err != nil { return fmt.Errorf("query kern.hv_support: %w", err) } diff --git a/internal/provider/dockersandboxes/provider.go b/internal/provider/dockersandboxes/provider.go index 8801120..7c08fc9 100644 --- a/internal/provider/dockersandboxes/provider.go +++ b/internal/provider/dockersandboxes/provider.go @@ -12,6 +12,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -23,15 +24,20 @@ import ( ) const ( - defaultOutputLimit = 8 << 20 - diagnosticOutputLimit = 256 << 10 - commandWaitDelay = 5 * time.Second - keepaliveStartupDelay = 500 * time.Millisecond + defaultOutputLimit = 8 << 20 + diagnosticOutputLimit = 256 << 10 + commandWaitDelay = 5 * time.Second + keepaliveStartupDelay = 500 * time.Millisecond + providerReadbackTimeout = 30 * time.Second + providerCleanupTimeout = 2 * time.Minute + providerCreateTimeout = 10 * time.Minute + daemonStatePollInterval = 250 * time.Millisecond + maximumRecoveryQuiescence = 5 * time.Minute ) const sandboxContainerFailureSignature = "failed to run sandbox container" -const sandboxContainerFailureRemediation = "The shared Docker Sandboxes daemon may have inherited host SSH-agent forwarding. EPAR removes SSH-agent variables when its commands start a stopped daemon. EPAR will not stop or restart a running shared daemon. Coordinate with every process using that daemon before an interruption, then run `sbx daemon stop` followed by `env -u SSH_AUTH_SOCK -u SSH_AUTH_SOCK_GATEWAY -u SSH_AGENT_PID sbx daemon start --detach` and retry." +const sandboxContainerFailureRemediation = "The shared Docker Sandboxes daemon may have inherited host SSH-agent forwarding. EPAR removes SSH-agent variables when its commands start a stopped daemon. In recoveryMode=exclusive-auto, the pool may make one bounded stop-wait-start recovery attempt for this create-stage signature; recoveryMode=observe never mutates the daemon. Coordinate with every process using that daemon before an interruption, then run `sbx daemon stop` followed by `env -u SSH_AUTH_SOCK -u SSH_AUTH_SOCK_GATEWAY -u SSH_AGENT_PID sbx daemon start --detach` and retry." const directWorkspaceVerificationScript = `set -euo pipefail if test -n "${SSH_AUTH_SOCK:-}" || test -n "${SSH_AUTH_SOCK_GATEWAY:-}" || test -n "${SSH_AGENT_PID:-}" || test -e /run/ssh-agent.sock || test -L /run/ssh-agent.sock; then @@ -61,21 +67,27 @@ docker info --format '{{json .ServerVersion}}'` type Provider struct { Binary string - runCommand runCommandFunc - activationMu sync.RWMutex - admissionBlockReason string - activeMu sync.RWMutex - activeTemplate provider.TemplateArtifact - dryRun bool - architectureEmulation architectureEmulationEnabler - architectureLogged sync.Map - logger *slog.Logger - relayMu sync.Mutex - relay *egressRelay - relayTokens map[string]string - relayConnections map[string]map[net.Conn]struct{} - hostTrustRelayEnabled bool - hostTrustRelayPort int + runCommand runCommandFunc + wait func(context.Context, time.Duration) error + controlPlaneGate controlPlaneCommandGate + recoverySlotOnce sync.Once + recoverySlot chan struct{} + activationMu sync.RWMutex + admissionBlockReason string + activeMu sync.RWMutex + activeTemplate provider.TemplateArtifact + dryRun bool + architectureEmulation architectureEmulationEnabler + architectureLogged sync.Map + logger *slog.Logger + instanceOperationGates [64]sync.Mutex + relayMu sync.Mutex + relay *egressRelay + relayTokens map[string]relayTokenBinding + relayEpoch uint64 + relayConnections map[string]map[net.Conn]struct{} + hostTrustRelayEnabled bool + hostTrustRelayPort int } type instanceReceipt struct { @@ -113,6 +125,12 @@ type commandRequest struct { sensitiveValues []string operation string outputLimit int + // timeout bounds this provider CLI operation; zero preserves the caller's + // lifetime for long-running guest commands and the detached keepalive. + timeout time.Duration + // preserveDescendantsOnSuccess is reserved for the exact detached daemon + // start command. Transient commands retain kill-on-close containment. + preserveDescendantsOnSuccess bool } type runCommandFunc func(ctx context.Context, request commandRequest) (provider.ExecResult, error) @@ -122,7 +140,18 @@ func New(binary string) *Provider { } func NewWithDryRun(binary string, dryRun bool) *Provider { - return newWithArchitectureEmulation(binary, dryRun, qemuBinfmtEnabler{}) + return NewWithArchitectureMode(binary, dryRun, architectureEmulationNativeOnly, defaultNativePlatform()) +} + +func defaultNativePlatform() string { + switch runtime.GOARCH { + case "amd64": + return "linux/amd64" + case "arm64": + return "linux/arm64" + default: + return "" + } } func NewWithArchitectureMode(binary string, dryRun bool, mode, platform string) *Provider { @@ -142,7 +171,7 @@ func newWithArchitectureEmulation(binary string, dryRun bool, enabler architectu if binary == "" { binary = "sbx" } - return &Provider{Binary: binary, dryRun: dryRun, architectureEmulation: enabler, relayTokens: make(map[string]string), relayConnections: make(map[string]map[net.Conn]struct{})} + return &Provider{Binary: binary, wait: waitForContext, dryRun: dryRun, architectureEmulation: enabler, relayTokens: make(map[string]relayTokenBinding), relayConnections: make(map[string]map[net.Conn]struct{})} } // ConfigureHostTrustRelay enables the Windows-host trust transport used by @@ -162,18 +191,178 @@ func (p *Provider) SetLogger(logger *slog.Logger) { p.logger = logger } +// lockInstanceOperation serializes provider mutations and identity-sensitive +// verification for one sandbox name. The relay server never acquires these +// gates, so guest relay traffic remains independent from lifecycle ordering. +func (p *Provider) lockInstanceOperation(name string) func() { + var hash uint32 = 2166136261 + for index := 0; index < len(name); index++ { + hash ^= uint32(name[index]) + hash *= 16777619 + } + gate := &p.instanceOperationGates[hash%uint32(len(p.instanceOperationGates))] + gate.Lock() + return gate.Unlock +} + // StartDaemon asks Docker Sandboxes to start its host daemon in the // background. The command is intentionally exact so onboarding cannot invoke // other daemon mutations through this path. func (p *Provider) StartDaemon(ctx context.Context) error { _, err := p.run(ctx, commandRequest{ - args: []string{"daemon", "start", "--detach"}, - operation: "start docker sandboxes daemon", - outputLimit: diagnosticOutputLimit, + args: []string{"daemon", "start", "--detach"}, + operation: "start docker sandboxes daemon", + outputLimit: diagnosticOutputLimit, + timeout: providerReadbackTimeout, + preserveDescendantsOnSuccess: true, }) return err } +type daemonControlState string + +const ( + daemonControlStateRunning daemonControlState = "running" + daemonControlStateStopped daemonControlState = "stopped" +) + +// RecoverControlPlane performs the provider-owned mutation for an exclusive +// Docker Sandboxes host. It never starts the daemon unless stopped state was +// observed both after the cold stop and again after the quiescence interval. +func (p *Provider) RecoverControlPlane(ctx context.Context, request provider.ControlPlaneRecoveryRequest) (err error) { + defer func() { + if err == nil { + return + } + var failure *provider.ControlPlaneRecoveryFailure + if !errors.As(err, &failure) { + err = provider.NewControlPlaneRecoveryFailure("Docker Sandboxes control-plane recovery", err) + } + }() + if request.Quiescence <= 0 || request.Quiescence > maximumRecoveryQuiescence { + return fmt.Errorf("Docker Sandboxes recovery quiescence must be greater than zero and no more than %s", maximumRecoveryQuiescence) + } + if p.dryRun { + return fmt.Errorf("Docker Sandboxes control-plane recovery is unavailable in dry-run mode") + } + releaseRecoverySlot, err := p.acquireRecoverySlot(ctx) + if err != nil { + return err + } + defer releaseRecoverySlot() + releaseControlPlaneGate, err := p.controlPlaneGate.beginRecovery(ctx) + if err != nil { + return err + } + defer releaseControlPlaneGate() + var releaseHostLock func() + if p.runCommand == nil { + releaseHostLock, err = provider.TryAcquireControlPlaneRecoveryLock() + if err != nil { + return err + } + defer releaseHostLock() + } + recoveryCtx := provider.WithControlPlaneLock(withControlPlaneGate(ctx)) + + _, stopErr := p.run(recoveryCtx, commandRequest{ + args: []string{"daemon", "stop"}, + operation: "stop docker sandboxes daemon for control-plane recovery", + outputLimit: diagnosticOutputLimit, + timeout: providerCleanupTimeout, + }) + if stateErr := p.waitForDaemonState(recoveryCtx, daemonControlStateStopped, providerCleanupTimeout); stateErr != nil { + if stopErr != nil { + return errors.Join(stopErr, fmt.Errorf("confirm Docker Sandboxes daemon stopped: %w", stateErr)) + } + return fmt.Errorf("confirm Docker Sandboxes daemon stopped: %w", stateErr) + } + if stopErr != nil && p.logger != nil { + p.logger.Warn("Docker Sandboxes daemon stop returned an error but authoritative status confirmed stopped; continuing exclusive recovery", "provider", "docker-sandboxes") + } + + if err := p.wait(recoveryCtx, request.Quiescence); err != nil { + return fmt.Errorf("wait for Docker Sandboxes daemon quiescence: %w", err) + } + state, err := p.readDaemonControlState(recoveryCtx) + if err != nil { + return fmt.Errorf("refusing Docker Sandboxes daemon start because stopped state is unknown after quiescence: %w", err) + } + if state != daemonControlStateStopped { + return fmt.Errorf("refusing Docker Sandboxes daemon start after quiescence: state is %q, want %q", state, daemonControlStateStopped) + } + + startErr := p.StartDaemon(recoveryCtx) + stateErr := p.waitForDaemonState(recoveryCtx, daemonControlStateRunning, providerReadbackTimeout) + if stateErr != nil { + if startErr != nil { + return errors.Join(startErr, fmt.Errorf("confirm Docker Sandboxes daemon running: %w", stateErr)) + } + return fmt.Errorf("confirm Docker Sandboxes daemon running: %w", stateErr) + } + if startErr != nil && p.logger != nil { + p.logger.Warn("Docker Sandboxes daemon detached start returned an error but authoritative status confirmed running; recovery succeeded", "provider", "docker-sandboxes") + } + return nil +} + +func (p *Provider) waitForDaemonState(ctx context.Context, expected daemonControlState, timeout time.Duration) error { + confirmationCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + for { + state, err := p.readDaemonControlState(confirmationCtx) + if err != nil { + return err + } + if state == expected { + return nil + } + if err := p.wait(confirmationCtx, daemonStatePollInterval); err != nil { + return fmt.Errorf("Docker Sandboxes daemon remained %q while waiting for %q: %w", state, expected, err) + } + } +} + +func (p *Provider) readDaemonControlState(ctx context.Context) (daemonControlState, error) { + result, err := p.run(ctx, commandRequest{ + args: []string{"daemon", "status", "--json"}, + operation: "read docker sandboxes daemon control state", + outputLimit: diagnosticOutputLimit, + timeout: providerReadbackTimeout, + }) + if err != nil { + return "", err + } + return parseDaemonControlState([]byte(result.Stdout)) +} + +func parseDaemonControlState(data []byte) (daemonControlState, error) { + state, _, err := parseDaemonStatus(data) + if err != nil { + return "", err + } + if state != strings.TrimSpace(state) { + return "", fmt.Errorf("docker sandboxes daemon status returned a non-canonical state") + } + switch normalized := daemonControlState(strings.ToLower(state)); normalized { + case daemonControlStateRunning, daemonControlStateStopped: + return normalized, nil + default: + return "", fmt.Errorf("docker sandboxes daemon status returned unsupported state %q", state) + } +} + +func waitForContext(ctx context.Context, duration time.Duration) error { + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + // VerifyHostReadiness requires machine-readable Docker Sandboxes diagnostics // to contain at least one passing check and no failed checks. Warnings and // skipped checks do not make an otherwise healthy installation unavailable. @@ -192,6 +381,8 @@ func (p *Provider) VerifyHostReadiness(ctx context.Context) (HostReadiness, erro } func (p *Provider) Create(ctx context.Context, request provider.CreateRequest) (provider.Instance, error) { + releaseInstanceOperation := p.lockInstanceOperation(request.Name) + defer releaseInstanceOperation() if p.dryRun { return provider.Instance{}, fmt.Errorf("docker-sandboxes does not support dry-run instance creation because exact sandbox and template-cache readback is required") } @@ -232,9 +423,13 @@ func (p *Provider) Create(ctx context.Context, request provider.CreateRequest) ( return provider.Instance{}, fmt.Errorf("docker sandbox name is already allocated") } } - var ownedStaging staging.OwnedDirectory + var ( + stagingRoot *staging.Staging + ownedStaging staging.OwnedDirectory + ) if p.runCommand == nil { - stagingRoot, openErr := staging.Open(filepath.Dir(request.StagingPath)) + var openErr error + stagingRoot, openErr = staging.Open(filepath.Dir(request.StagingPath)) if openErr != nil { return provider.Instance{}, openErr } @@ -264,8 +459,18 @@ func (p *Provider) Create(ctx context.Context, request provider.CreateRequest) ( if request.DockerDisk != "" { environment["DOCKER_SANDBOXES_DOCKER_SIZE"] = request.DockerDisk } - if _, err := p.run(ctx, commandRequest{args: args, environment: environment, operation: "create docker sandbox"}); err != nil { - return provider.Instance{}, withSandboxContainerFailureRemediation(err) + result, createErr := p.run(ctx, commandRequest{args: args, environment: environment, operation: "create docker sandbox", timeout: providerCreateTimeout}) + if createErr != nil { + if stagingRoot != nil { + if cleanupErr := stagingRoot.RemoveEmptyOwned(request.Name, ownedStaging.Identity); cleanupErr != nil { + createErr = errors.Join(createErr, fmt.Errorf("remove failed Docker Sandboxes staging workspace: %w", cleanupErr)) + } + } + failure := withSandboxContainerFailureRemediation(createErr) + if hasSandboxCreateAdmissionSignature(result.Stderr) { + return provider.Instance{}, provider.NewControlPlaneAdmissionFailure("create Docker Sandboxes instance", failure) + } + return provider.Instance{}, failure } items, err = p.inventoryVerified(ctx) if err != nil { @@ -322,6 +527,13 @@ func withSandboxContainerFailureRemediation(err error) error { return fmt.Errorf("%w; %s", err, sandboxContainerFailureRemediation) } +// hasSandboxCreateAdmissionSignature deliberately inspects only the stderr +// captured from the immediate `sbx create` operation. Matching a wrapped +// controller error would turn unrelated failures into daemon-restart triggers. +func hasSandboxCreateAdmissionSignature(stderr string) bool { + return strings.Contains(strings.ToLower(stderr), sandboxContainerFailureSignature) +} + func (p *Provider) logArchitectureCapability(instanceName string, emulation architectureEmulationResult) { if p.logger == nil { return @@ -360,6 +572,7 @@ func (p *Provider) ImportTemplate(ctx context.Context, archivePath string) error args: []string{"template", "load", archivePath}, operation: "load exact Docker Sandboxes runner template", outputLimit: diagnosticOutputLimit, + timeout: providerCreateTimeout, }); err != nil { return err } @@ -484,6 +697,7 @@ func (p *Provider) RemoveTemplate(ctx context.Context, artifact provider.Templat args: []string{"template", "rm", artifact.CacheID}, operation: "remove exact Docker Sandboxes runner template", outputLimit: diagnosticOutputLimit, + timeout: providerCleanupTimeout, }); err != nil { return err } @@ -586,6 +800,7 @@ func (p *Provider) verifyInspection(ctx context.Context, instance provider.Insta args: []string{"inspect", "--json", instance.Name}, operation: "verify docker sandbox attached capabilities", outputLimit: diagnosticOutputLimit, + timeout: providerReadbackTimeout, }) if err != nil { return err @@ -658,6 +873,7 @@ func (p *Provider) verifyNoPublishedPorts(ctx context.Context, instance provider args: []string{"ports", instance.Name, "--json"}, operation: "verify docker sandbox has no published ports", outputLimit: diagnosticOutputLimit, + timeout: providerReadbackTimeout, }) if err != nil { return err @@ -677,6 +893,7 @@ func (p *Provider) verifyDirectWorkspace(ctx context.Context, instance provider. args: []string{"exec", "-i", instance.Name, "--", "bash", "-lc", directWorkspaceVerificationScript}, stdin: strings.NewReader(""), operation: "verify dedicated docker sandbox staging workspace", + timeout: providerReadbackTimeout, }) if err != nil { return err @@ -724,6 +941,16 @@ func (p *Provider) startKeepalive(ctx context.Context, name string, request comm if err := validateCommandRequest(request); err != nil { return nil, err } + releaseGate, err := p.controlPlaneGate.acquire(ctx) + if err != nil { + return nil, err + } + defer releaseGate() + releaseHostLock, err := provider.AcquireControlPlaneCommandLock(ctx) + if err != nil { + return nil, err + } + defer releaseHostLock() // The caller's context bounds startup only. The returned keepalive owns the // sandbox lifetime and must survive the provisioning-attempt context that the // pool cancels as soon as Start returns. @@ -739,9 +966,17 @@ func (p *Provider) startKeepalive(ctx context.Context, name string, request comm if err := command.Start(); err != nil { return nil, fmt.Errorf("%s failed: %w", request.operation, err) } + cleanup, attachErr := attachManagedProcess(command, false) + if attachErr != nil { + killErr := killManagedProcess(command) + waitErr := waitForManagedCommandExit(command, commandWaitDelay) + return nil, fmt.Errorf("%s failed to establish process containment: %w", request.operation, errors.Join(attachErr, killErr, waitErr)) + } finished := make(chan error, 1) go func() { - finished <- command.Wait() + err := command.Wait() + cleanup() + finished <- err }() timer := time.NewTimer(keepaliveStartupDelay) defer timer.Stop() @@ -756,16 +991,50 @@ func (p *Provider) startKeepalive(ctx context.Context, name string, request comm } return nil, fmt.Errorf("%s failed: %w", request.operation, err) case <-ctx.Done(): - if err := command.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) { - return nil, errors.Join(ctx.Err(), fmt.Errorf("stop keepalive after canceled startup: %w", err)) + killErr := killManagedProcess(command) + waitErr := waitForManagedProcessExit(finished, commandWaitDelay) + if waitErr != nil { + if killErr != nil && !errors.Is(killErr, os.ErrProcessDone) { + return nil, errors.Join(ctx.Err(), fmt.Errorf("stop keepalive after canceled startup: %w", killErr), waitErr) + } + return nil, errors.Join(ctx.Err(), waitErr) } - <-finished return nil, ctx.Err() case <-timer.C: return &provider.RunningProcess{Name: name, PID: command.Process.Pid}, nil } } +func waitForManagedCommandExit(command *exec.Cmd, timeout time.Duration) error { + finished := make(chan error, 1) + go func() { finished <- command.Wait() }() + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case err := <-finished: + return err + case <-timer.C: + _ = command.Process.Kill() + select { + case err := <-finished: + return errors.Join(fmt.Errorf("managed process did not exit within %s", timeout), err) + case <-time.After(timeout): + return fmt.Errorf("managed process did not exit after forced termination") + } + } +} + +func waitForManagedProcessExit(finished <-chan error, timeout time.Duration) error { + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case <-finished: + return nil + case <-timer.C: + return fmt.Errorf("managed process did not exit within %s", timeout) + } +} + func (p *Provider) VerifyRuntime(ctx context.Context, instance provider.Instance) (provider.RuntimeInfo, error) { present, err := p.assertIdentity(ctx, instance) if err != nil { @@ -778,6 +1047,7 @@ func (p *Provider) VerifyRuntime(ctx context.Context, instance provider.Instance args: []string{"exec", "-i", instance.Name, "--", "bash", "-lc", runtimeVerificationScript}, stdin: strings.NewReader(""), operation: "verify docker sandbox runtime", + timeout: providerReadbackTimeout, }) if err != nil { return provider.RuntimeInfo{}, err @@ -836,6 +1106,7 @@ func (p *Provider) Diagnostics(ctx context.Context, instance provider.Instance) args: []string{"daemon", "status", "--json"}, operation: "read docker sandbox daemon status", outputLimit: diagnosticOutputLimit, + timeout: providerReadbackTimeout, }) if err != nil { return provider.Diagnostics{}, err @@ -863,6 +1134,7 @@ func (p *Provider) readHostReadiness(ctx context.Context) (HostReadiness, error) args: []string{"diagnose", "--output", "json"}, operation: "diagnose docker sandboxes", outputLimit: diagnosticOutputLimit, + timeout: providerReadbackTimeout, }) if err != nil { return HostReadiness{}, err @@ -883,12 +1155,14 @@ func (p *Provider) Stop(ctx context.Context, instance provider.Instance) error { if err := validateInstance(instance, true); err != nil { return err } - defer p.releaseRelayToken(instance.Name) + releaseInstanceOperation := p.lockInstanceOperation(instance.Name) + defer releaseInstanceOperation() + defer p.releaseRelayTokenForInstance(instance) present, err := p.assertIdentity(ctx, instance) if err != nil || !present { return err } - result, err := p.run(ctx, commandRequest{args: []string{"stop", instance.Name}, operation: "stop docker sandbox"}) + result, err := p.run(ctx, commandRequest{args: []string{"stop", instance.Name}, operation: "stop docker sandbox", timeout: providerCleanupTimeout}) if err != nil && isMissingSandbox(result.Stdout+"\n"+result.Stderr+"\n"+err.Error()) { return nil } @@ -899,7 +1173,9 @@ func (p *Provider) Delete(ctx context.Context, instance provider.Instance) error if err := validateInstance(instance, true); err != nil { return err } - defer p.releaseRelayToken(instance.Name) + releaseInstanceOperation := p.lockInstanceOperation(instance.Name) + defer releaseInstanceOperation() + defer p.releaseRelayTokenForInstance(instance) present, err := p.assertIdentity(ctx, instance) if err != nil || !present { return err @@ -921,7 +1197,7 @@ func (p *Provider) Delete(ctx context.Context, instance provider.Instance) error return fmt.Errorf("refusing Docker Sandbox deletion without an exact staging ownership receipt") } } - result, err := p.run(ctx, commandRequest{args: []string{"rm", "--force", instance.Name}, operation: "delete docker sandbox"}) + result, err := p.run(ctx, commandRequest{args: []string{"rm", "--force", instance.Name}, operation: "delete docker sandbox", timeout: providerCleanupTimeout}) if err != nil && isMissingSandbox(result.Stdout+"\n"+result.Stderr+"\n"+err.Error()) { err = nil } @@ -950,9 +1226,9 @@ func (p *Provider) Inventory(ctx context.Context) ([]provider.InventoryItem, err func (p *Provider) inventoryVerified(ctx context.Context) ([]provider.InventoryItem, error) { for attempt := 1; attempt <= 2; attempt++ { - result, err := p.run(ctx, commandRequest{args: []string{"ls", "--json"}, operation: "inventory docker sandboxes"}) + result, err := p.run(ctx, commandRequest{args: []string{"ls", "--json"}, operation: "inventory docker sandboxes", timeout: providerReadbackTimeout}) if err != nil { - return nil, err + return nil, provider.NewControlPlaneFailure("inventory Docker Sandboxes", err) } items, parseErr := parseInventory([]byte(result.Stdout)) if parseErr == nil { @@ -960,7 +1236,7 @@ func (p *Provider) inventoryVerified(ctx context.Context) ([]provider.InventoryI return items, nil } if attempt == 2 { - return nil, parseErr + return nil, provider.NewControlPlaneFailure("inventory Docker Sandboxes", parseErr) } if p.logger != nil { p.logger.Debug("Docker Sandboxes inventory returned invalid machine-readable output; retrying once", "provider", "docker-sandboxes", "stdoutBytes", len(result.Stdout)) @@ -977,6 +1253,7 @@ func (p *Provider) CachedTemplates(ctx context.Context) ([]CachedTemplate, error args: []string{"template", "ls", "--json"}, operation: "read docker sandbox template cache", outputLimit: diagnosticOutputLimit, + timeout: providerReadbackTimeout, }) if err != nil { return nil, err @@ -998,7 +1275,7 @@ func (p *Provider) CachedTemplates(ctx context.Context) ([]CachedTemplate, error } func (p *Provider) verifyImportedTemplate(ctx context.Context, reference, cacheID string) error { - result, err := p.run(ctx, commandRequest{args: []string{"template", "ls", "--json"}, operation: "verify cached docker sandbox template"}) + result, err := p.run(ctx, commandRequest{args: []string{"template", "ls", "--json"}, operation: "verify cached docker sandbox template", timeout: providerReadbackTimeout}) if err != nil { return err } @@ -1069,10 +1346,19 @@ func (p *Provider) run(ctx context.Context, request commandRequest) (provider.Ex if err := validateCommandRequest(request); err != nil { return provider.ExecResult{}, err } + if !controlPlaneGateHeld(ctx) { + release, err := p.controlPlaneGate.acquire(ctx) + if err != nil { + return provider.ExecResult{}, err + } + defer release() + } + operationCtx, cancel := contextWithTimeout(ctx, request.timeout) + defer cancel() if request.outputLimit == 0 { request.outputLimit = defaultOutputLimit } - if err := ctx.Err(); err != nil { + if err := operationCtx.Err(); err != nil { return provider.ExecResult{}, err } bufferedStdout, bufferedStderr, flush := provider.BufferSensitiveSinks(request.sensitiveValues, request.stdout, request.stderr) @@ -1082,16 +1368,16 @@ func (p *Provider) run(ctx context.Context, request commandRequest) (provider.Ex var result provider.ExecResult var runErr error if p.runCommand != nil { - result, runErr = p.runCommand(ctx, request) + result, runErr = p.runCommand(operationCtx, request) } else { - result, runErr = p.runRaw(ctx, request) + result, runErr = p.runRaw(operationCtx, request) } if len(result.Stdout) > request.outputLimit || len(result.Stderr) > request.outputLimit { runErr = errors.Join(runErr, fmt.Errorf("%s exceeded the output limit", request.operation)) result.Stdout = truncate(result.Stdout, request.outputLimit) result.Stderr = truncate(result.Stderr, request.outputLimit) } - if ctxErr := ctx.Err(); ctxErr != nil { + if ctxErr := operationCtx.Err(); ctxErr != nil { runErr = errors.Join(ctxErr, runErr) } result, finishErr := provider.FinishSensitiveExecution(result, runErr, flush(), request.sensitiveValues) @@ -1107,13 +1393,148 @@ func (p *Provider) run(ctx context.Context, request commandRequest) (provider.Ex return result, finishErr } +func contextWithTimeout(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { + if timeout <= 0 { + return parent, func() {} + } + return context.WithTimeout(parent, timeout) +} + +type controlPlaneGateContextKey struct{} + +func withControlPlaneGate(ctx context.Context) context.Context { + return context.WithValue(ctx, controlPlaneGateContextKey{}, true) +} + +func controlPlaneGateHeld(ctx context.Context) bool { + held, _ := ctx.Value(controlPlaneGateContextKey{}).(bool) + return held +} + +// controlPlaneCommandGate lets in-flight provider commands drain before a +// daemon recovery begins, then prevents new commands from racing its stop, +// quiescence, and start sequence. +type controlPlaneCommandGate struct { + initOnce sync.Once + mu sync.Mutex + active int + pending bool + recovering bool + changed chan struct{} +} + +func (gate *controlPlaneCommandGate) initialize() { + gate.initOnce.Do(func() { + gate.changed = make(chan struct{}) + }) +} + +func (gate *controlPlaneCommandGate) signalLocked() { + close(gate.changed) + gate.changed = make(chan struct{}) +} + +func (gate *controlPlaneCommandGate) acquire(ctx context.Context) (func(), error) { + gate.initialize() + for { + gate.mu.Lock() + if !gate.pending && !gate.recovering { + gate.active++ + gate.mu.Unlock() + return func() { gate.releaseOperation() }, nil + } + changed := gate.changed + gate.mu.Unlock() + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-changed: + } + } +} + +func (gate *controlPlaneCommandGate) releaseOperation() { + gate.mu.Lock() + gate.active-- + if gate.active == 0 && gate.pending { + gate.signalLocked() + } + gate.mu.Unlock() +} + +func (gate *controlPlaneCommandGate) beginRecovery(ctx context.Context) (func(), error) { + gate.initialize() + for { + gate.mu.Lock() + if !gate.pending && !gate.recovering { + gate.pending = true + gate.signalLocked() + } + if gate.pending && gate.active == 0 && !gate.recovering { + gate.pending = false + gate.recovering = true + gate.mu.Unlock() + return func() { gate.endRecovery() }, nil + } + changed := gate.changed + gate.mu.Unlock() + select { + case <-ctx.Done(): + gate.cancelRecovery() + return nil, ctx.Err() + case <-changed: + } + } +} + +func (gate *controlPlaneCommandGate) cancelRecovery() { + gate.mu.Lock() + if gate.pending && !gate.recovering { + gate.pending = false + gate.signalLocked() + } + gate.mu.Unlock() +} + +func (gate *controlPlaneCommandGate) endRecovery() { + gate.mu.Lock() + gate.recovering = false + gate.signalLocked() + gate.mu.Unlock() +} + +func (p *Provider) acquireRecoverySlot(ctx context.Context) (func(), error) { + p.recoverySlotOnce.Do(func() { + p.recoverySlot = make(chan struct{}, 1) + }) + select { + case p.recoverySlot <- struct{}{}: + return func() { <-p.recoverySlot }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + func (p *Provider) runRaw(ctx context.Context, request commandRequest) (provider.ExecResult, error) { + var releaseHostLock func() + if !provider.ControlPlaneLockHeld(ctx) { + var err error + releaseHostLock, err = provider.AcquireControlPlaneCommandLock(ctx) + if err != nil { + return provider.ExecResult{}, err + } + defer releaseHostLock() + } cmd := exec.CommandContext(ctx, p.Binary, request.args...) + isolateManagedProcess(cmd) cmd.WaitDelay = commandWaitDelay var cancellationKilledProcess atomic.Bool defaultCancel := cmd.Cancel cmd.Cancel = func() error { - err := defaultCancel() + err := killManagedProcess(cmd) + if err != nil { + err = defaultCancel() + } if err == nil { cancellationKilledProcess.Store(true) } @@ -1125,7 +1546,23 @@ func (p *Provider) runRaw(ctx context.Context, request commandRequest) (provider stderr := &boundedBuffer{limit: request.outputLimit} cmd.Stdout = captureWriter(stdout, request.stdout) cmd.Stderr = captureWriter(stderr, request.stderr) - err := cmd.Run() + err := cmd.Start() + if err == nil { + cleanup, attachErr := attachManagedProcess(cmd, request.preserveDescendantsOnSuccess) + if attachErr != nil { + killErr := killManagedProcess(cmd) + waitErr := waitForManagedCommandExit(cmd, commandWaitDelay) + err = errors.Join(fmt.Errorf("attach Docker Sandboxes process containment: %w", attachErr), killErr, waitErr) + } else { + defer cleanup() + err = cmd.Wait() + if request.preserveDescendantsOnSuccess && err != nil { + if killErr := killManagedProcess(cmd); killErr != nil && !errors.Is(killErr, os.ErrProcessDone) { + err = errors.Join(err, fmt.Errorf("clean up failed detached Docker Sandboxes daemon start: %w", killErr)) + } + } + } + } if cancellationKilledProcess.Load() { if ctxErr := ctx.Err(); ctxErr != nil { err = ctxErr @@ -1161,8 +1598,13 @@ func validateCommandRequest(request commandRequest) error { return err } } - if request.args[0] == "daemon" && (len(request.args) != 3 || !((request.args[1] == "status" && request.args[2] == "--json") || (request.args[1] == "start" && request.args[2] == "--detach"))) { - return fmt.Errorf("only exact daemon status or detached-start operations are permitted") + if request.args[0] == "daemon" { + exactStop := len(request.args) == 2 && request.args[1] == "stop" + exactStatus := len(request.args) == 3 && request.args[1] == "status" && request.args[2] == "--json" + exactDetachedStart := len(request.args) == 3 && request.args[1] == "start" && request.args[2] == "--detach" + if !exactStop && !exactStatus && !exactDetachedStart { + return fmt.Errorf("only exact daemon status, cold-stop, or detached-start operations are permitted") + } } for _, arg := range request.args { if strings.ContainsRune(arg, 0) { @@ -1256,6 +1698,7 @@ func decodeStrictJSON(data []byte, destination any) error { } var _ provider.Lifecycle = (*Provider)(nil) +var _ provider.ControlPlaneRecoverer = (*Provider)(nil) var _ provider.AdmissionVerifier = (*Provider)(nil) var _ provider.InstanceAdmissionVerifier = (*Provider)(nil) var _ provider.PolicyManager = (*Provider)(nil) diff --git a/internal/provider/dockersandboxes/provider_test.go b/internal/provider/dockersandboxes/provider_test.go index 4554b62..9df6368 100644 --- a/internal/provider/dockersandboxes/provider_test.go +++ b/internal/provider/dockersandboxes/provider_test.go @@ -20,6 +20,7 @@ import ( "time" "github.com/solutionforest/ephemeral-action-runner/internal/provider" + "github.com/solutionforest/ephemeral-action-runner/internal/provider/dockersandboxes/staging" ) const ( @@ -96,6 +97,307 @@ func TestStartDaemonUsesExactDetachedCommand(t *testing.T) { done() } +func TestStartDaemonScrubsSSHAgentEnvironment(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test helper uses a POSIX shell script") + } + t.Setenv("SSH_AUTH_SOCK", "/host/agent.sock") + t.Setenv("SSH_AUTH_SOCK_GATEWAY", "gateway.example.test:3129") + t.Setenv("SSH_AGENT_PID", "4242") + helper := filepath.Join(t.TempDir(), "sbx-test-helper") + script := "#!/bin/sh\n" + + "test \"$#\" -eq 3 && test \"$1\" = daemon && test \"$2\" = start && test \"$3\" = --detach || exit 91\n" + + "test -z \"${SSH_AUTH_SOCK:-}\" && test -z \"${SSH_AUTH_SOCK_GATEWAY:-}\" && test -z \"${SSH_AGENT_PID:-}\" || exit 92\n" + if err := os.WriteFile(helper, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + if err := New(helper).StartDaemon(context.Background()); err != nil { + t.Fatalf("StartDaemon() did not use exact scrubbed detached start: %v", err) + } +} + +func TestRecoverControlPlaneUsesExactColdStopAndDetachedStart(t *testing.T) { + p, done := scriptedProvider(t, + commandStep{args: []string{"daemon", "stop"}}, + commandStep{args: []string{"daemon", "status", "--json"}, result: provider.ExecResult{Stdout: `{"status":"running"}`}}, + commandStep{args: []string{"daemon", "status", "--json"}, result: provider.ExecResult{Stdout: `{"status":"stopped"}`}}, + commandStep{args: []string{"daemon", "status", "--json"}, result: provider.ExecResult{Stdout: `{"status":"stopped"}`}}, + commandStep{args: []string{"daemon", "start", "--detach"}}, + commandStep{args: []string{"daemon", "status", "--json"}, result: provider.ExecResult{Stdout: `{"status":"running"}`}}, + ) + var waits []time.Duration + p.wait = func(_ context.Context, duration time.Duration) error { + waits = append(waits, duration) + return nil + } + if err := p.RecoverControlPlane(context.Background(), provider.ControlPlaneRecoveryRequest{Quiescence: time.Minute}); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(waits, []time.Duration{daemonStatePollInterval, time.Minute}) { + t.Fatalf("recovery waits = %#v, want stop-state poll followed by exact one-minute quiescence", waits) + } + done() +} + +func TestRecoverControlPlaneRejectsUnboundedQuiescenceBeforeCommands(t *testing.T) { + for _, duration := range []time.Duration{0, -time.Second, maximumRecoveryQuiescence + time.Nanosecond} { + p := New("sbx-test-double") + called := false + p.runCommand = func(context.Context, commandRequest) (provider.ExecResult, error) { + called = true + return provider.ExecResult{}, nil + } + if err := p.RecoverControlPlane(context.Background(), provider.ControlPlaneRecoveryRequest{Quiescence: duration}); err == nil { + t.Fatalf("RecoverControlPlane() accepted quiescence %s", duration) + } + if called { + t.Fatalf("invalid quiescence %s reached provider commands", duration) + } + } +} + +func TestRecoverControlPlaneDoesNotStartWhenStoppedStateIsUnknown(t *testing.T) { + p, done := scriptedProvider(t, + commandStep{args: []string{"daemon", "stop"}}, + commandStep{args: []string{"daemon", "status", "--json"}, result: provider.ExecResult{Stdout: `{"status":"wedged"}`}}, + ) + err := p.RecoverControlPlane(context.Background(), provider.ControlPlaneRecoveryRequest{Quiescence: time.Minute}) + if err == nil || errors.Unwrap(err) == nil || !strings.Contains(errors.Unwrap(err).Error(), "unsupported state") { + t.Fatalf("RecoverControlPlane() error = %v, want unknown stopped-state refusal", err) + } + done() +} + +func TestRecoverControlPlaneRechecksStoppedStateAfterQuiescence(t *testing.T) { + p, done := scriptedProvider(t, + commandStep{args: []string{"daemon", "stop"}}, + commandStep{args: []string{"daemon", "status", "--json"}, result: provider.ExecResult{Stdout: `{"status":"stopped"}`}}, + commandStep{args: []string{"daemon", "status", "--json"}, result: provider.ExecResult{Stdout: `{"status":"running"}`}}, + ) + p.wait = func(context.Context, time.Duration) error { return nil } + err := p.RecoverControlPlane(context.Background(), provider.ControlPlaneRecoveryRequest{Quiescence: time.Minute}) + if err == nil || errors.Unwrap(err) == nil || !strings.Contains(errors.Unwrap(err).Error(), `state is "running"`) { + t.Fatalf("RecoverControlPlane() error = %v, want post-quiescence stopped-state refusal", err) + } + done() +} + +func TestRecoverControlPlaneSanitizesCommandOutput(t *testing.T) { + const secret = "recovery-secret-from-daemon" + p, done := scriptedProvider(t, + commandStep{args: []string{"daemon", "stop"}, err: errors.New(secret)}, + commandStep{args: []string{"daemon", "status", "--json"}, result: provider.ExecResult{Stdout: `{"status":"wedged"}`}}, + ) + err := p.RecoverControlPlane(context.Background(), provider.ControlPlaneRecoveryRequest{Quiescence: time.Minute}) + if err == nil { + t.Fatal("RecoverControlPlane() error = nil, want recovery failure") + } + if !errors.Is(err, provider.ErrControlPlaneRecoveryFailure) { + t.Fatalf("RecoverControlPlane() error = %v, want recovery sentinel", err) + } + if strings.Contains(err.Error(), secret) { + t.Fatalf("RecoverControlPlane() leaked daemon output: %v", err) + } + done() +} + +func TestRecoverControlPlaneQuiescenceBlocksConcurrentProviderCommands(t *testing.T) { + p := New("sbx-test-double") + quiescenceStarted := make(chan struct{}) + releaseQuiescence := make(chan struct{}) + recoveryDone := make(chan error, 1) + var mu sync.Mutex + var calls [][]string + statusCalls := 0 + p.runCommand = func(_ context.Context, request commandRequest) (provider.ExecResult, error) { + mu.Lock() + calls = append(calls, append([]string(nil), request.args...)) + mu.Unlock() + switch { + case reflect.DeepEqual(request.args, []string{"daemon", "stop"}): + return provider.ExecResult{}, nil + case reflect.DeepEqual(request.args, []string{"daemon", "status", "--json"}): + statusCalls++ + if statusCalls < 3 { + return provider.ExecResult{Stdout: `{"status":"stopped"}`}, nil + } + return provider.ExecResult{Stdout: `{"status":"running"}`}, nil + case reflect.DeepEqual(request.args, []string{"daemon", "start", "--detach"}): + return provider.ExecResult{}, nil + default: + return provider.ExecResult{Stdout: readyListJSON}, nil + } + } + p.wait = func(waitCtx context.Context, _ time.Duration) error { + close(quiescenceStarted) + select { + case <-releaseQuiescence: + return nil + case <-waitCtx.Done(): + return waitCtx.Err() + } + } + go func() { + recoveryDone <- p.RecoverControlPlane(context.Background(), provider.ControlPlaneRecoveryRequest{Quiescence: time.Minute}) + }() + select { + case <-quiescenceStarted: + case <-time.After(2 * time.Second): + t.Fatal("recovery did not reach quiescence") + } + operationCtx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) + defer cancel() + if _, err := p.Inventory(operationCtx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("concurrent inventory error = %v, want context deadline while recovery owns quiescence", err) + } + mu.Lock() + callCount := len(calls) + mu.Unlock() + if callCount != 2 { + t.Fatalf("provider commands during quiescence = %d, want only stop and first status readback", callCount) + } + close(releaseQuiescence) + if err := <-recoveryDone; err != nil { + t.Fatalf("recovery failed after quiescence release: %v", err) + } +} + +func TestStartKeepaliveWaitsForControlPlaneRecovery(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test helper uses a POSIX shell script") + } + helper := filepath.Join(t.TempDir(), "sbx-test-helper") + if err := os.WriteFile(helper, []byte("#!/bin/sh\nsleep 1\n"), 0o755); err != nil { + t.Fatal(err) + } + p := New(helper) + quiescenceStarted := make(chan struct{}) + releaseQuiescence := make(chan struct{}) + recoveryDone := make(chan error, 1) + var waitOnce sync.Once + statusCalls := 0 + p.runCommand = func(_ context.Context, request commandRequest) (provider.ExecResult, error) { + switch { + case reflect.DeepEqual(request.args, []string{"daemon", "stop"}): + return provider.ExecResult{}, nil + case reflect.DeepEqual(request.args, []string{"daemon", "status", "--json"}): + statusCalls++ + if statusCalls < 3 { + return provider.ExecResult{Stdout: `{"status":"stopped"}`}, nil + } + return provider.ExecResult{Stdout: `{"status":"running"}`}, nil + case reflect.DeepEqual(request.args, []string{"daemon", "start", "--detach"}): + return provider.ExecResult{}, nil + default: + t.Fatalf("unexpected recovery command: %#v", request.args) + return provider.ExecResult{}, nil + } + } + p.wait = func(waitCtx context.Context, _ time.Duration) error { + waitOnce.Do(func() { close(quiescenceStarted) }) + select { + case <-releaseQuiescence: + return nil + case <-waitCtx.Done(): + return waitCtx.Err() + } + } + go func() { + recoveryDone <- p.RecoverControlPlane(context.Background(), provider.ControlPlaneRecoveryRequest{Quiescence: time.Minute}) + }() + select { + case <-quiescenceStarted: + case <-time.After(2 * time.Second): + t.Fatal("recovery did not reach quiescence") + } + keepaliveDone := make(chan error, 1) + go func() { + _, err := p.startKeepalive(context.Background(), testName, commandRequest{args: []string{"exec", "keepalive"}, operation: "test managed keepalive"}) + keepaliveDone <- err + }() + select { + case err := <-keepaliveDone: + t.Fatalf("keepalive launched during recovery quiescence: %v", err) + case <-time.After(100 * time.Millisecond): + } + close(releaseQuiescence) + if err := <-recoveryDone; err != nil { + t.Fatalf("recovery failed after quiescence release: %v", err) + } + if err := <-keepaliveDone; err != nil { + t.Fatalf("keepalive failed after recovery release: %v", err) + } +} + +func TestRecoverControlPlaneQuiescenceCancellationPreventsStart(t *testing.T) { + p, done := scriptedProvider(t, + commandStep{args: []string{"daemon", "stop"}}, + commandStep{args: []string{"daemon", "status", "--json"}, result: provider.ExecResult{Stdout: `{"status":"stopped"}`}}, + ) + ctx, cancel := context.WithCancel(context.Background()) + p.wait = func(waitCtx context.Context, _ time.Duration) error { + cancel() + <-waitCtx.Done() + return waitCtx.Err() + } + err := p.RecoverControlPlane(ctx, provider.ControlPlaneRecoveryRequest{Quiescence: time.Minute}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("RecoverControlPlane() error = %v, want context cancellation", err) + } + done() +} + +func TestRecoverControlPlaneStopTimeoutPreventsFurtherCommands(t *testing.T) { + p := New("sbx-test-double") + calls := 0 + p.runCommand = func(ctx context.Context, request commandRequest) (provider.ExecResult, error) { + calls++ + if !reflect.DeepEqual(request.args, []string{"daemon", "stop"}) { + t.Fatalf("command args = %#v, want exact daemon stop", request.args) + } + <-ctx.Done() + return provider.ExecResult{}, ctx.Err() + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + err := p.RecoverControlPlane(ctx, provider.ControlPlaneRecoveryRequest{Quiescence: time.Minute}) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("RecoverControlPlane() error = %v, want context deadline", err) + } + if calls != 1 { + t.Fatalf("commands after timed-out stop = %d, want only the stop command", calls) + } +} + +func TestParseDaemonControlStateFailsClosed(t *testing.T) { + for _, test := range []struct { + name string + fixture string + want daemonControlState + wantErr bool + }{ + {name: "running", fixture: `{"status":"running"}`, want: daemonControlStateRunning}, + {name: "stopped case insensitive", fixture: `{"status":"STOPPED"}`, want: daemonControlStateStopped}, + {name: "unknown", fixture: `{"status":"starting"}`, wantErr: true}, + {name: "non canonical", fixture: `{"status":" stopped "}`, wantErr: true}, + {name: "missing", fixture: `{}`, wantErr: true}, + {name: "trailing", fixture: `{"status":"running"} {}`, wantErr: true}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := parseDaemonControlState([]byte(test.fixture)) + if test.wantErr { + if err == nil { + t.Fatalf("parseDaemonControlState(%s) = %q, want error", test.fixture, got) + } + return + } + if err != nil || got != test.want { + t.Fatalf("parseDaemonControlState(%s) = %q, %v, want %q", test.fixture, got, err, test.want) + } + }) + } +} + func TestRemoveTemplateUsesExactCacheIDAndRefusesLiveSandboxes(t *testing.T) { artifact := provider.TemplateArtifact{ Reference: "docker.io/library/epar-template:one", @@ -316,10 +618,14 @@ func TestCreateKnownSandboxContainerFailureAddsSSHDaemonRemediation(t *testing.T if !errors.Is(err, cause) { t.Fatalf("Create error = %v, want original command error to remain wrapped", err) } + if !errors.Is(err, provider.ErrControlPlaneAdmissionFailure) { + t.Fatalf("Create error = %v, want typed control-plane admission failure", err) + } for _, expected := range []string{ sandboxContainerFailureSignature, "EPAR removes SSH-agent variables when its commands start a stopped daemon", - "EPAR will not stop or restart a running shared daemon", + "recoveryMode=exclusive-auto", + "recoveryMode=observe never mutates the daemon", "Coordinate with every process using that daemon", "sbx daemon stop", "env -u SSH_AUTH_SOCK -u SSH_AUTH_SOCK_GATEWAY -u SSH_AGENT_PID sbx daemon start --detach", @@ -331,6 +637,63 @@ func TestCreateKnownSandboxContainerFailureAddsSSHDaemonRemediation(t *testing.T done() } +func TestCreateAdmissionClassificationUsesImmediateCreateStderrOnly(t *testing.T) { + const signature = "500 Internal Server Error: failed to run sandbox container" + if !hasSandboxCreateAdmissionSignature(signature) { + t.Fatal("known create stderr signature was not classified") + } + if hasSandboxCreateAdmissionSignature("permission denied") { + t.Fatal("unrelated create stderr was classified") + } +} + +func TestCreateDoesNotClassifyWrappedSignatureWithoutCreateStderr(t *testing.T) { + const signature = "500 Internal Server Error: failed to run sandbox container" + p, done := scriptedProvider(t, + commandStep{args: []string{"diagnose", "--output", "json"}, result: provider.ExecResult{Stdout: healthyDiagnoseJSON}}, + commandStep{args: []string{"template", "ls", "--json"}, result: provider.ExecResult{Stdout: templateListJSON}}, + commandStep{args: []string{"ls", "--json"}, result: provider.ExecResult{Stdout: `{"sandboxes":[]}`}}, + commandStep{args: []string{"create", "--name", testName, "--cpus", "4", "--memory", "8g", "--template", testTemplate, "shell", testWorkspace}, environment: map[string]string{}, result: provider.ExecResult{Stderr: "permission denied"}, err: errors.New(signature)}, + ) + _, err := p.Create(context.Background(), validCreateRequest()) + if err == nil || errors.Is(err, provider.ErrControlPlaneAdmissionFailure) { + t.Fatalf("Create error = %v, want ordinary failure without admission classification", err) + } + done() +} + +func TestCreateRemovesEmptyStagingAfterImmediateCreateFailure(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test helper uses a POSIX shell script") + } + root := filepath.Join(t.TempDir(), "staging") + if _, err := staging.Open(root); err != nil { + t.Fatal(err) + } + helper := filepath.Join(t.TempDir(), "sbx-test-helper") + script := fmt.Sprintf(`#!/bin/sh +case "$1:$2" in +diagnose:--output) printf '%%s' '%s' ;; +template:ls) printf '%%s' '%s' ;; +ls:--json) printf '%%s' '{"sandboxes":[]}' ;; +create:*) printf '%%s\n' '500 Internal Server Error: failed to run sandbox container' >&2; exit 1 ;; +*) exit 91 ;; +esac +`, healthyDiagnoseJSON, templateListJSON) + if err := os.WriteFile(helper, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + p := New(helper) + request := validCreateRequest() + request.StagingPath = filepath.Join(root, testName) + if _, err := p.Create(context.Background(), request); err == nil || !errors.Is(err, provider.ErrControlPlaneAdmissionFailure) { + t.Fatalf("Create() error = %v, want typed admission failure", err) + } + if _, err := os.Stat(request.StagingPath); !os.IsNotExist(err) { + t.Fatalf("failed create staging path stat error = %v, want exact empty staging removal", err) + } +} + func TestCreateUnrelatedFailureDoesNotAddSSHDaemonRemediation(t *testing.T) { cause := errors.New("permission denied") p, done := scriptedProvider(t, @@ -401,6 +764,17 @@ func TestNewWithArchitectureModeMapsEveryValidatedMode(t *testing.T) { } } +func TestNewDefaultsToNativeOnly(t *testing.T) { + p := New("sbx") + enabler, ok := p.architectureEmulation.(nativeArchitectureEnabler) + if !ok { + t.Fatalf("default architecture enabler = %T, want nativeArchitectureEnabler", p.architectureEmulation) + } + if enabler.platform != defaultNativePlatform() { + t.Fatalf("default native platform = %q, want %q", enabler.platform, defaultNativePlatform()) + } +} + func TestExperimentalV2ReceiptRemainsReadableForCleanup(t *testing.T) { payload, err := json.Marshal(map[string]any{ "schemaVersion": 2, @@ -1054,23 +1428,95 @@ func TestInventoryFailsClosedAfterSecondInvalidMachineReadableResponse(t *testin commandStep{args: []string{"ls", "--json"}, result: provider.ExecResult{Stdout: "Starting sandboxd daemon..."}}, commandStep{args: []string{"ls", "--json"}, result: provider.ExecResult{Stdout: `{"items":[]}`}}, ) - if _, err := p.Inventory(context.Background()); err == nil || !strings.Contains(err.Error(), "unsupported json schema") { + _, err := p.Inventory(context.Background()) + if err == nil || !errors.Is(err, provider.ErrControlPlaneFailure) { t.Fatalf("persistent invalid inventory output was accepted: %v", err) } + if cause := errors.Unwrap(err); cause == nil || !strings.Contains(cause.Error(), "unsupported json schema") { + t.Fatalf("persistent invalid inventory cause = %v, want schema error", cause) + } done() } func TestInventoryDoesNotRetryCommandFailure(t *testing.T) { - expected := errors.New("sandboxd unavailable") + expected := errors.New("sandboxd unavailable with secret-token") p, done := scriptedProvider(t, commandStep{args: []string{"ls", "--json"}, err: expected}, ) - if _, err := p.Inventory(context.Background()); !errors.Is(err, expected) { + _, err := p.Inventory(context.Background()) + if !errors.Is(err, expected) { t.Fatalf("Inventory() error = %v, want command failure", err) } + if !errors.Is(err, provider.ErrControlPlaneFailure) { + t.Fatalf("Inventory() error = %v, want control-plane sentinel", err) + } + var failure *provider.ControlPlaneFailure + if !errors.As(err, &failure) { + t.Fatalf("Inventory() error type = %T, want *provider.ControlPlaneFailure", err) + } + if strings.Contains(err.Error(), "secret-token") { + t.Fatalf("Inventory() user-facing error leaked command detail: %v", err) + } + done() +} + +func TestInventoryWrapsPersistentMalformedOutputAsControlPlaneFailure(t *testing.T) { + p, done := scriptedProvider(t, + commandStep{args: []string{"ls", "--json"}, result: provider.ExecResult{Stdout: `{"sandboxes":`}}, + commandStep{args: []string{"ls", "--json"}, result: provider.ExecResult{Stdout: `{"sandboxes":`}}, + ) + _, err := p.Inventory(context.Background()) + if !errors.Is(err, provider.ErrControlPlaneFailure) { + t.Fatalf("Inventory() error = %v, want control-plane sentinel", err) + } + var failure *provider.ControlPlaneFailure + if !errors.As(err, &failure) { + t.Fatalf("Inventory() error type = %T, want *provider.ControlPlaneFailure", err) + } done() } +func TestInventoryUsesProviderOwnedReadbackDeadline(t *testing.T) { + p := New("sbx-test-double") + p.runCommand = func(ctx context.Context, request commandRequest) (provider.ExecResult, error) { + if !reflect.DeepEqual(request.args, []string{"ls", "--json"}) { + t.Fatalf("inventory args = %#v", request.args) + } + deadline, ok := ctx.Deadline() + if !ok { + t.Fatal("inventory command had no provider-owned deadline") + } + remaining := time.Until(deadline) + if remaining <= 0 || remaining > providerReadbackTimeout || remaining < providerReadbackTimeout-5*time.Second { + t.Fatalf("inventory deadline remaining = %s, want approximately %s", remaining, providerReadbackTimeout) + } + return provider.ExecResult{Stdout: readyListJSON}, nil + } + items, err := p.Inventory(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(items) != 1 || items[0].Instance.ProviderID != testID { + t.Fatalf("inventory = %#v, want exact fixture identity", items) + } +} + +func TestRunHonorsOperationTimeout(t *testing.T) { + p := New("sbx-test-double") + p.runCommand = func(ctx context.Context, _ commandRequest) (provider.ExecResult, error) { + <-ctx.Done() + return provider.ExecResult{}, ctx.Err() + } + _, err := p.run(context.Background(), commandRequest{ + args: []string{"ls", "--json"}, + operation: "bounded test command", + timeout: 10 * time.Millisecond, + }) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("bounded command error = %v, want context deadline exceeded", err) + } +} + func TestLifecycleCommandsUseExactIdentityAndArgv(t *testing.T) { t.Run("start", func(t *testing.T) { p, done := identityScript(t, commandStep{args: []string{"exec", "-i", testName, "--", "/bin/sleep", "infinity"}}) @@ -1194,7 +1640,7 @@ func TestKeepaliveSurvivesSuccessfulStartContextCancellation(t *testing.T) { t.Fatal(err) } cancel() - deadline := time.Now().Add(2 * time.Second) + deadline := time.Now().Add(5 * time.Second) for { if content, readErr := os.ReadFile(marker); readErr == nil { if string(content) != "survived" { @@ -1239,6 +1685,29 @@ func TestKeepaliveCancellationDuringStartupStopsProcess(t *testing.T) { } } +func TestKeepaliveCancellationDuringStartupStopsProcessTree(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test helper uses a POSIX shell script") + } + marker := filepath.Join(t.TempDir(), "unexpected") + helper := filepath.Join(t.TempDir(), "sbx-test-helper") + script := "#!/bin/sh\n(sleep 1; printf child-survived >\"$2.child\") &\nprintf '%s' \"$!\" >\"$2.pid\"\nwait\n" + if err := os.WriteFile(helper, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(50*time.Millisecond, cancel) + p := New(helper) + _, err := p.startKeepalive(ctx, testName, commandRequest{args: []string{"exec", marker}, operation: "test managed keepalive"}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context cancellation", err) + } + time.Sleep(1500 * time.Millisecond) + if _, statErr := os.Stat(marker + ".child"); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("canceled startup left a descendant running: %v", statErr) + } +} + func TestRunRawNormalizesCommandContextKillToCancellation(t *testing.T) { if os.Getenv("EPAR_DOCKER_SANDBOXES_RUN_RAW_HELPER") == "1" { _, _ = os.Stdout.WriteString("ready\n") @@ -1620,6 +2089,11 @@ func TestInjectionCorpusIsRejectedBeforeCommandExecution(t *testing.T) { } func TestCommandBoundaryRejectsInteractiveAndDestructiveGlobalCommands(t *testing.T) { + for _, arguments := range [][]string{{"daemon", "status", "--json"}, {"daemon", "stop"}, {"daemon", "start", "--detach"}} { + if err := validateCommandRequest(commandRequest{args: arguments, operation: "test exact daemon command"}); err != nil { + t.Fatalf("exact Docker Sandboxes daemon command was rejected: %q: %v", arguments, err) + } + } for _, command := range []string{"tui", "reset", "run", "kit", "secret", "login", "logout", "setup", "ssh", "cp", "completion", "help"} { err := validateCommandRequest(commandRequest{args: []string{command}, operation: "test forbidden command"}) if err == nil { @@ -1636,7 +2110,7 @@ func TestCommandBoundaryRejectsInteractiveAndDestructiveGlobalCommands(t *testin t.Fatalf("non-exact Docker Sandboxes published-port inspection was accepted: %q", arguments) } } - for _, arguments := range [][]string{{"daemon"}, {"daemon", "start"}, {"daemon", "start", "--foreground"}, {"daemon", "stop"}, {"daemon", "stop", "--detach"}, {"daemon", "restart"}, {"daemon", "restart", "--detach"}, {"daemon", "status"}, {"daemon", "status", "--debug"}} { + for _, arguments := range [][]string{{"daemon"}, {"daemon", "start"}, {"daemon", "start", "--foreground"}, {"daemon", "stop", "--detach"}, {"daemon", "stop", "extra"}, {"daemon", "restart"}, {"daemon", "restart", "--detach"}, {"daemon", "status"}, {"daemon", "status", "--debug"}} { if err := validateCommandRequest(commandRequest{args: arguments, operation: "test forbidden daemon command"}); err == nil { t.Fatalf("non-exact Docker Sandboxes daemon command was accepted: %q", arguments) } diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 8f309a7..d3c4460 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -13,7 +13,98 @@ import ( "github.com/solutionforest/ephemeral-action-runner/internal/storage" ) -var ErrTemplateNotFound = errors.New("imported provider template not found") +var ( + ErrTemplateNotFound = errors.New("imported provider template not found") + ErrControlPlaneFailure = errors.New("provider control plane failure") + ErrControlPlaneAdmissionFailure = errors.New("provider control plane admission failure") + ErrControlPlaneRecoveryFailure = errors.New("provider control plane recovery failed") +) + +// ControlPlaneFailure marks a failed provider control-plane command without +// exposing command output through the user-facing error string. The original +// cause remains available to errors.Is/errors.As for cancellation and +// diagnostic handling. +type ControlPlaneFailure struct { + Operation string + cause error +} + +func (failure *ControlPlaneFailure) Error() string { + if operation := strings.TrimSpace(failure.Operation); operation != "" { + return operation + ": " + ErrControlPlaneFailure.Error() + } + return ErrControlPlaneFailure.Error() +} + +func (failure *ControlPlaneFailure) Unwrap() error { return failure.cause } + +func (failure *ControlPlaneFailure) Is(target error) bool { + return target == ErrControlPlaneFailure +} + +// NewControlPlaneFailure constructs a typed control-plane failure. Callers +// must supply an operation label that contains no command output or secrets. +func NewControlPlaneFailure(operation string, cause error) error { + return &ControlPlaneFailure{Operation: operation, cause: cause} +} + +// ControlPlaneAdmissionFailure marks a provider admission failure that may be +// recoverable by restarting the provider control plane. Its Error method +// preserves the provider's existing safe diagnostic because admission errors +// already carry the user-facing remediation and redacted command detail. +type ControlPlaneAdmissionFailure struct { + Operation string + cause error +} + +func (failure *ControlPlaneAdmissionFailure) Error() string { + if failure.cause != nil { + return failure.cause.Error() + } + if operation := strings.TrimSpace(failure.Operation); operation != "" { + return operation + ": " + ErrControlPlaneAdmissionFailure.Error() + } + return ErrControlPlaneAdmissionFailure.Error() +} + +func (failure *ControlPlaneAdmissionFailure) Unwrap() error { return failure.cause } + +func (failure *ControlPlaneAdmissionFailure) Is(target error) bool { + return target == ErrControlPlaneAdmissionFailure +} + +// NewControlPlaneAdmissionFailure constructs a typed provider admission +// failure without changing the existing user-facing diagnostic. +func NewControlPlaneAdmissionFailure(operation string, cause error) error { + return &ControlPlaneAdmissionFailure{Operation: operation, cause: cause} +} + +// ControlPlaneRecoveryFailure marks a failed provider recovery command without +// exposing command output through the user-facing error string. The original +// cause remains available to errors.Is/errors.As for cancellation and tests. +type ControlPlaneRecoveryFailure struct { + Operation string + cause error +} + +func (failure *ControlPlaneRecoveryFailure) Error() string { + if operation := strings.TrimSpace(failure.Operation); operation != "" { + return operation + ": " + ErrControlPlaneRecoveryFailure.Error() + } + return ErrControlPlaneRecoveryFailure.Error() +} + +func (failure *ControlPlaneRecoveryFailure) Unwrap() error { return failure.cause } + +func (failure *ControlPlaneRecoveryFailure) Is(target error) bool { + return target == ErrControlPlaneRecoveryFailure +} + +// NewControlPlaneRecoveryFailure constructs a typed recovery failure. Callers +// must supply an operation label that contains no command output or secrets. +func NewControlPlaneRecoveryFailure(operation string, cause error) error { + return &ControlPlaneRecoveryFailure{Operation: operation, cause: cause} +} type Instance struct { Name string @@ -114,6 +205,19 @@ type Lifecycle interface { Inventory(ctx context.Context) ([]InventoryItem, error) } +// ControlPlaneRecoveryRequest defines the bounded cold-start interval for an +// optional provider control-plane recovery operation. +type ControlPlaneRecoveryRequest struct { + Quiescence time.Duration +} + +// ControlPlaneRecoverer is an optional provider capability. Shared lifecycle +// orchestration decides whether recovery is permitted; implementations must +// fail closed when the stopped state cannot be established authoritatively. +type ControlPlaneRecoverer interface { + RecoverControlPlane(ctx context.Context, request ControlPlaneRecoveryRequest) error +} + // ArtifactManager is an optional provider capability for runtimes whose // reusable artifact is not prepared by the shared OCI image pipeline. type ArtifactManager interface { @@ -216,14 +320,27 @@ type InstanceAdmissionVerifier interface { // HostTrustRuntimeActivator is an optional provider capability for runtimes // that need provider-specific transport work after the common host CA overlay -// has been installed. The pool invokes it before registration and again after -// any runtime trust refresh. Implementations must fail closed: returning nil -// means every provider-owned trust transport needed by the runtime is active -// and verified for the exact instance. +// has been installed. The pool invokes it before registration and when a +// fresh runtime must be prepared after replacement or controller restart. +// Steady-state reconciliation never invokes this mutating capability on an +// already registered runner. Providers that need a transport-specific +// steady-state check should also implement HostTrustRuntimeVerifier; otherwise +// the pool falls back to the common VerifyRuntime check. type HostTrustRuntimeActivator interface { ActivateHostTrustRuntime(ctx context.Context, instance Instance) error } +// HostTrustRuntimeVerifier is an optional provider capability for read-only +// verification of transport state needed by an already registered runtime. +// The pool invokes it during steady-state reconciliation after the common +// VerifyRuntime check. Implementations must fail closed and must not mutate +// network policy, restart a daemon, reconfigure the guest, or expose +// credentials; returning nil means the exact instance's provider-owned trust +// transport is active and verified. +type HostTrustRuntimeVerifier interface { + VerifyHostTrustRuntime(ctx context.Context, instance Instance) error +} + type NetworkPolicyDecision string const ( diff --git a/scripts/build-native-controller.ps1 b/scripts/build-native-controller.ps1 index b3052b0..23e1e2a 100644 --- a/scripts/build-native-controller.ps1 +++ b/scripts/build-native-controller.ps1 @@ -243,6 +243,11 @@ function Get-EparDockerImageID { function Resolve-EparGoToolchainImage { param([AllowEmptyString()][string] $PreviousDevImageID = '') $previousImageID = Get-EparDockerImageID -Reference $GoImage + if ([string]::IsNullOrWhiteSpace($previousImageID)) { + Write-Host "EPAR did not find Docker's Go build image $GoImage locally. Docker will download it now before building the controller. This may take a few minutes." + } else { + Write-Host "EPAR is checking Docker's Go build image $GoImage before building the controller. Docker may download an updated image if needed." + } Write-EparBootstrapAcquisitionJournal -Phase 'pulling-go-toolchain' -PreviousGoImageID $previousImageID -PreviousDevImageID $PreviousDevImageID docker pull $GoImage | Out-Host if ($LASTEXITCODE -ne 0) { throw "failed to resolve the current Go toolchain image $GoImage" } @@ -850,6 +855,17 @@ function Test-EparNativeControllerSlot { return [pscustomobject]@{ Exists = $true; Owned = $true; Valid = $true; Reason = ''; Receipt = $receipt } } +function Get-EparFriendlyNativeControllerRebuildReason { + param([Parameter(Mandatory = $true)][string] $Reason) + switch -Regex ($Reason) { + '^slot is missing$' { return 'the project-local controller is not installed yet' } + '^source digest differs' { return 'the project source code has changed' } + '^build identity differs' { return 'the cached controller was built with a different compiler or build environment' } + '^slot executable' { return 'the cached controller executable needs to be refreshed' } + default { return 'the cached project-local controller needs to be refreshed' } + } +} + function Write-EparNativeControllerReceipt { param([Parameter(Mandatory = $true)][string] $Directory, [Parameter(Mandatory = $true)][string] $SourceDigest, [Parameter(Mandatory = $true)][string] $BuildDigest, [Parameter(Mandatory = $true)][string] $Builder, [Parameter(Mandatory = $true)][string] $Toolchain) $binaryPath = Join-Path $Directory $NativeExecutable @@ -907,7 +923,8 @@ if ($UseOld) { $expectedBuildDigest = if ($toolchain) { Get-EparNativeBuildDigest -SourceDigest $sourceDigest -Builder $Backend -Toolchain $toolchain } else { '' } $currentState = Test-EparNativeControllerSlot -Directory $currentSlot -ExpectedSourceDigest $sourceDigest -ExpectedBuildDigest $expectedBuildDigest if (-not $currentState.Valid) { - Write-Warning "Native controller rebuild required: $($currentState.Reason)" + $friendlyReason = Get-EparFriendlyNativeControllerRebuildReason -Reason $currentState.Reason + Write-Host "EPAR is preparing its project-local controller because $friendlyReason. This may take a few minutes." $buildLock = Enter-EparStableNativeControllerBuildLock -Path $buildLockPath try { if ($Backend -eq 'docker') { diff --git a/scripts/build-native-controller.sh b/scripts/build-native-controller.sh index 3727b23..381ea1b 100755 --- a/scripts/build-native-controller.sh +++ b/scripts/build-native-controller.sh @@ -108,9 +108,24 @@ epar_docker_image_id() { printf '%s\n' "$image_id" } +epar_friendly_native_controller_rebuild_reason() { + case "$1" in + 'slot is missing') printf '%s' 'the project-local controller is not installed yet' ;; + 'source digest mismatch'*) printf '%s' 'the project source code has changed' ;; + 'build identity mismatch'*) printf '%s' 'the cached controller was built with a different compiler or build environment' ;; + 'controller executable is missing'*) printf '%s' 'the cached controller executable needs to be refreshed' ;; + *) printf '%s' 'the cached project-local controller needs to be refreshed' ;; + esac +} + epar_resolve_go_toolchain_image() { local previous_id resolved_id previous_id="$(epar_docker_image_id "$go_image")" + if [[ -z "$previous_id" ]]; then + printf '%s\n' "EPAR did not find Docker's Go build image ${go_image} locally. Docker will download it now before building the controller. This may take a few minutes." >&2 + else + printf '%s\n' "EPAR is checking Docker's Go build image ${go_image} before building the controller. Docker may download an updated image if needed." >&2 + fi epar_write_bootstrap_acquisition_journal pulling-go-toolchain "$previous_id" '' '' "$previous_dev_image_id" docker pull "$go_image" >&2 resolved_id="$(epar_docker_image_id "$go_image")" @@ -667,7 +682,12 @@ if [[ -n "$expected_build_digest" ]] && epar_validate_slot "$current_slot" "$tar epar_launch_slot "$current_slot" current "$@" exit $? fi -if [[ -e "$current_slot" || -L "$current_slot" ]]; then echo "Native controller rebuild required: ${receipt_error:-installed slot has no verifiable build identity}." >&2; else echo 'Native controller build required: no current project-local slot exists.' >&2; fi +if [[ -e "$current_slot" || -L "$current_slot" ]]; then + rebuild_reason="${receipt_error:-installed slot has no verifiable build identity}" +else + rebuild_reason='slot is missing' +fi +printf 'EPAR is preparing its project-local controller because %s. This may take a few minutes.\n' "$(epar_friendly_native_controller_rebuild_reason "$rebuild_reason")" >&2 epar_acquire_stable_build_lock if [[ "$native_backend" == local-go ]]; then diff --git a/scripts/test/native-controller-cache-retention.sh b/scripts/test/native-controller-cache-retention.sh index d704281..1b4f6a8 100644 --- a/scripts/test/native-controller-cache-retention.sh +++ b/scripts/test/native-controller-cache-retention.sh @@ -146,7 +146,7 @@ epar_prune_native_controller_cache "$policy_root" "$policy_current" [[ -d "${policy_root}/${policy_grace}" ]] || { echo "retention removed a grace-protected revision beyond the byte budget" >&2; exit 1; } builder_source="$(cat "$builder")" -for required in 'golang:latest' 'controller.receipt' 'schemaVersion=3' 'sourceDigest' 'buildDigest' 'binaryDigest' 'lease-native-' 'epar_write_bootstrap_acquisition_journal' 'epar_resolve_go_toolchain_image' 'previousDevImageID' 'previous_dev_image_id' 'epar-native-controller-build.log' 'epar_report_tls_failure' 'TLS verification was not disabled' 'epar_prepare_bootstrap_build_trust' '--network none' 'GO111MODULE=off' 'GOTOOLCHAIN=local' 'SSL_CERT_FILE=/run/epar-bootstrap-ca.pem' 'scripts/bootstrap-trust' ':/run/epar-bootstrap-ca.pem:ro'; do +for required in 'golang:latest' 'controller.receipt' 'schemaVersion=3' 'sourceDigest' 'buildDigest' 'binaryDigest' 'lease-native-' 'epar_write_bootstrap_acquisition_journal' 'epar_resolve_go_toolchain_image' 'epar_friendly_native_controller_rebuild_reason' 'previousDevImageID' 'previous_dev_image_id' 'epar-native-controller-build.log' 'epar_report_tls_failure' 'TLS verification was not disabled' 'epar_prepare_bootstrap_build_trust' 'EPAR is preparing its project-local controller because' 'Docker will download it now before building the controller' '--network none' 'GO111MODULE=off' 'GOTOOLCHAIN=local' 'SSL_CERT_FILE=/run/epar-bootstrap-ca.pem' 'scripts/bootstrap-trust' ':/run/epar-bootstrap-ca.pem:ro'; do [[ "$builder_source" == *"$required"* ]] || { echo "stable native-controller wrapper contract is missing: ${required}" >&2; exit 1; } done launch_source="$(sed -n '/^epar_launch_slot()/,/^}/p' "$builder")" @@ -260,7 +260,9 @@ native_smoke_env=( 'EPAR_CONTROLLER_HOST_OS=darwin' 'EPAR_HOST_TRUST_INIT_DEFERRED=1' ) -(cd "$native_smoke_project" && env "${native_smoke_env[@]}" scripts/build-native-controller.sh start) +native_first_output="$(cd "$native_smoke_project" && env "${native_smoke_env[@]}" scripts/build-native-controller.sh start 2>&1)" +[[ "$native_first_output" == *'EPAR is preparing its project-local controller because the project-local controller is not installed yet.'* ]] || { echo 'first Docker controller build did not explain the rebuild' >&2; exit 1; } +[[ "$native_first_output" == *"EPAR is checking Docker's Go build image golang:latest before building the controller."* ]] || { echo 'first Docker controller build did not explain the Docker Go environment' >&2; exit 1; } grep -Fxq 'runtime build=<> runner=<> os=<> deferred=<> args=' "${native_smoke_root}/native.log" grep -Fxq 'bootstrap' "${native_smoke_root}/helper.log" [[ "$(wc -l <"${native_smoke_root}/helper.log" | tr -d ' ')" == 1 ]] || { echo 'ordinary cached-native start unexpectedly used a runtime trust bridge' >&2; exit 1; } diff --git a/scripts/test/start-command-forwarding.sh b/scripts/test/start-command-forwarding.sh index f9a49f6..d24e04a 100644 --- a/scripts/test/start-command-forwarding.sh +++ b/scripts/test/start-command-forwarding.sh @@ -18,6 +18,8 @@ cp "$source_root/scripts/bootstrap-trust/main.go" "$project/scripts/bootstrap-tr cp "$source_root/go.mod" "$source_root/go.sum" "$project/" cp -R "$source_root/cmd" "$source_root/internal" "$project/" chmod +x "$project/start" "$project/scripts/build-native-controller.sh" "$project/scripts/run-with-docker.sh" +grep -Fq 'Go is not installed or runnable on this machine' "$project/start" || { echo 'start wrapper does not explain the no-Go Docker fallback' >&2; exit 1; } +grep -Fq 'if the project-local controller needs to be rebuilt' "$project/start" || { echo 'start wrapper does not explain that Docker is conditional on a rebuild' >&2; exit 1; } cat >"$fake_bin/go" <<'SCRIPT' #!/usr/bin/env bash diff --git a/scripts/test/windows-native-controller-contract.ps1 b/scripts/test/windows-native-controller-contract.ps1 index cebe7d9..0466649 100644 --- a/scripts/test/windows-native-controller-contract.ps1 +++ b/scripts/test/windows-native-controller-contract.ps1 @@ -8,8 +8,9 @@ if ([string]::IsNullOrWhiteSpace($ProjectRoot)) { $ProjectRoot = [System.IO.Path]::GetFullPath($ProjectRoot) $builderPath = Join-Path $ProjectRoot 'scripts\build-native-controller.ps1' $dockerPath = Join-Path $ProjectRoot 'scripts\run-with-docker.ps1' +$startPath = Join-Path $ProjectRoot 'start.ps1' -foreach ($path in @($builderPath, $dockerPath)) { +foreach ($path in @($builderPath, $dockerPath, $startPath)) { $tokens = $null $errors = $null [System.Management.Automation.Language.Parser]::ParseFile($path, [ref] $tokens, [ref] $errors) | Out-Null @@ -18,6 +19,7 @@ foreach ($path in @($builderPath, $dockerPath)) { $builder = Get-Content -Raw -LiteralPath $builderPath $docker = Get-Content -Raw -LiteralPath $dockerPath +$start = Get-Content -Raw -LiteralPath $startPath foreach ($required in @( 'schemaVersion=3', 'artifactKind=native-controller', @@ -40,10 +42,17 @@ foreach ($required in @( 'EPAR_CONTROLLER_SLOT', 'Remove-Item -LiteralPath $OldSlot -Recurse -Force', 'Move-Item -LiteralPath $CurrentSlot -Destination $OldSlot', - 'Move-Item -LiteralPath $Candidate -Destination $CurrentSlot' + 'Move-Item -LiteralPath $Candidate -Destination $CurrentSlot', + 'Get-EparFriendlyNativeControllerRebuildReason', + 'EPAR is preparing its project-local controller because', + 'Docker will download it now before building the controller' )) { if (-not $builder.Contains($required)) { throw "native-controller v3 contract is missing: $required" } } +if (-not $start.Contains('Go is not installed or runnable on this machine')) { throw 'Windows ./start must explain the no-Go Docker fallback before controller resolution' } +if (-not $start.Contains('if the project-local controller needs to be rebuilt')) { throw 'Windows ./start must explain that Docker is conditional on a rebuild' } +if ($builder.IndexOf('EPAR is preparing its project-local controller because') -ge $builder.IndexOf('$buildLock = Enter-EparStableNativeControllerBuildLock')) { throw 'rebuild explanation must precede build-lock acquisition' } +if ($builder.IndexOf('Docker will download it now before building the controller') -ge $builder.IndexOf('docker pull $GoImage')) { throw 'Docker image explanation must precede the Go toolchain pull' } if ($builder -match '(^|[^A-Za-z])go\s+run\s+\./cmd/ephemeral-action-runner') { throw 'native-controller builder must not execute the controller with go run' } if ($docker -match '(^|[^A-Za-z])go\s+run\s+\./cmd/ephemeral-action-runner') { throw 'Docker wrapper must not execute the controller with go run' } if (-not $docker.Contains('EPAR_LEGACY_CONTROLLER_IN_DOCKER=1 is no longer supported')) { throw 'legacy Docker controller mode must fail clearly' } @@ -124,6 +133,7 @@ exit /b %ERRORLEVEL% $first = Invoke-RuntimeBuilder if ($first.ExitCode -ne 0) { throw "first hermetic native build failed: $($first.Output); fake go calls: $((Get-Content -LiteralPath $fakeGoLog -ErrorAction SilentlyContinue) -join ' | ')" } + if ($first.Output -notmatch 'EPAR is preparing its project-local controller because') { throw "first hermetic native build did not explain the rebuild: $($first.Output)" } $currentSlot = Join-Path $temporary '.local\bin\windows-amd64' $oldSlot = Join-Path $temporary '.local\bin\windows-amd64-old' $firstReceipt = Read-RuntimeReceipt -Path (Join-Path $currentSlot 'controller.receipt') diff --git a/start b/start index b73367c..057e1cd 100755 --- a/start +++ b/start @@ -56,7 +56,11 @@ if ((use_old == 1)); then fi if [[ "${USE_DOCKER_RUN}" == "1" ]] || { [[ "${USE_DOCKER_RUN}" == "auto" ]] && ! go_usable "${GO_BIN}"; }; then - echo "Go not found or not runnable (or EPAR_USE_DOCKER_RUN=1); using the Docker compiler for the cached native controller..." >&2 + if ! go_usable "${GO_BIN}"; then + printf '%s\n' "Go is not installed or runnable on this machine. EPAR will use Docker's Go build environment if the project-local controller needs to be rebuilt." >&2 + else + printf '%s\n' "EPAR is configured to use Docker's Go build environment if the project-local controller needs to be rebuilt." >&2 + fi exec env EPAR_NATIVE_CONTROLLER_BACKEND=docker "${script_dir}/scripts/build-native-controller.sh" "${epar_args[@]}" fi diff --git a/start.ps1 b/start.ps1 index 5030b5f..b92e8f6 100644 --- a/start.ps1 +++ b/start.ps1 @@ -49,8 +49,12 @@ if ($UseDockerRun -eq '0' -and -not $goUsable) { throw "Go not found or not runnable: $GoBin`nInstall Go, set EPAR_GO_BIN, or set EPAR_USE_DOCKER_RUN=1 to use the Docker compiler." } $Backend = if ($UseDockerRun -eq '1' -or ($UseDockerRun -eq 'auto' -and -not $goUsable)) { 'docker' } else { 'local-go' } -if ($Backend -eq 'docker') { - Write-Warning 'Using the Docker toolchain when a validated project-local controller rebuild is required.' +if (-not $UseOld -and $Backend -eq 'docker') { + if (-not $goUsable) { + Write-Host "Go is not installed or runnable on this machine. EPAR will use Docker's Go build environment if the project-local controller needs to be rebuilt." + } else { + Write-Host "EPAR is configured to use Docker's Go build environment if the project-local controller needs to be rebuilt." + } } try { diff --git a/templates/docker-sandboxes/guest/configure-egress-relay.sh b/templates/docker-sandboxes/guest/configure-egress-relay.sh index fe0dd4b..4e34e12 100644 --- a/templates/docker-sandboxes/guest/configure-egress-relay.sh +++ b/templates/docker-sandboxes/guest/configure-egress-relay.sh @@ -21,6 +21,8 @@ daemon_mutated=false dockerd_restart_attempted=false started_dockerd_pid="" relay_ca_changed=false +relay_operation="activation" +relay_stage="bootstrap" cleanup_sensitive_staging() { rm -f "${config_path}.input" "${config_path}.tmp" "${config_path}.new" "${daemon_config}.tmp" "${daemon_config}.new" "${daemon_config}.rollback.new" } @@ -93,12 +95,16 @@ on_exit() { trap - EXIT set +e if [[ "${status}" != "0" ]]; then + failed_stage="${relay_stage}" + echo "EPAR host-trust relay: ${relay_operation} failed at ${failed_stage} (exit=${status})" >&2 + relay_stage="rollback-daemon" if ! rollback_daemon; then echo "EPAR host-trust relay: private Docker daemon rollback failed" >&2 status=1 else rm -f "${daemon_backup}" fi + relay_stage="rollback-relay-ca" if [[ "${relay_ca_changed}" == "true" ]] && ! remove_relay_ca_trust; then echo "EPAR host-trust relay: local TLS authority rollback failed" >&2 status=1 @@ -109,12 +115,14 @@ on_exit() { } trap on_exit EXIT +relay_stage="bootstrap" for command_name in cmp curl docker install jq pgrep python3 readlink stat update-ca-certificates; do command -v "${command_name}" >/dev/null 2>&1 || { echo "EPAR host-trust relay: required command ${command_name} is unavailable" >&2 exit 1 } done +relay_stage="validate-daemon-config" [[ -f "${daemon_config}" && ! -L "${daemon_config}" ]] [[ "$(stat -c '%U:%G:%a' "${daemon_config}")" == "root:root:644" ]] install -d -m 0755 -o root -g root "${config_dir}" @@ -123,11 +131,18 @@ if [[ "$#" -gt "1" || ( "${mode}" != "activate" && "${mode}" != "--commit" && "$ echo "EPAR host-trust relay: unsupported transaction operation" >&2 exit 1 fi +case "${mode}" in + activate) relay_operation="activation" ;; + --commit) relay_operation="commit" ;; + --rollback) relay_operation="rollback" ;; +esac if [[ "${mode}" == "--commit" ]]; then + relay_stage="commit" rm -f "${daemon_backup}" exit 0 fi if [[ "${mode}" == "--rollback" ]]; then + relay_stage="rollback-daemon" rm -f /run/epar/egress-relay-active "${config_path}" if [[ -f "${daemon_backup}" ]]; then if [[ -n "$(docker ps -aq)" ]]; then @@ -152,6 +167,7 @@ if [[ "$#" != "0" ]]; then echo "EPAR host-trust relay: activation does not accept arguments" >&2 exit 1 fi +relay_stage="validate-guest-relay" [[ -x /opt/epar/epar-egress-bridge ]] [[ -s /opt/epar/trust/ca-bundle.pem && ! -L /opt/epar/trust/ca-bundle.pem ]] [[ -s "${relay_ca_source}" && ! -L "${relay_ca_source}" ]] @@ -163,6 +179,7 @@ fi bridge_pid="$(cat /run/epar/egress-bridge.pid)" [[ "${bridge_pid}" =~ ^[1-9][0-9]*$ ]] [[ "$(readlink -f "/proc/${bridge_pid}/exe" 2>/dev/null || true)" == "/opt/epar/epar-egress-bridge" ]] +relay_stage="install-relay-ca" install -d -m 0755 -o root -g root "$(dirname "${relay_ca_trust}")" if [[ -e "${relay_ca_trust}" ]]; then [[ -f "${relay_ca_trust}" && ! -L "${relay_ca_trust}" ]] @@ -172,6 +189,7 @@ else update-ca-certificates >/dev/null relay_ca_changed=true fi +relay_stage="write-relay-config" rm -f /run/epar/egress-relay-active rm -f "${config_path}.input" "${config_path}.tmp" "${config_path}.new" install -m 0600 -o root -g root /dev/null "${config_path}.input" @@ -227,14 +245,17 @@ rm -f "${config_path}.input" install -m 0600 -o root -g root "${config_path}.tmp" "${config_path}.new" rm -f "${config_path}.tmp" mv -f "${config_path}.new" "${config_path}" +relay_stage="publish-relay-config" [[ "$(stat -c '%U:%G:%a' "${config_path}")" == "root:root:600" && ! -L "${config_path}" ]] +relay_stage="guest-bridge-health" health_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 15 "http://127.0.0.1:3129/health")" if [[ "${health_code}" != "204" ]]; then echo "EPAR host-trust relay: authenticated guest bridge health check failed" >&2 exit 1 fi +relay_stage="recover-daemon-transaction" if [[ -e "${daemon_backup}" ]]; then if [[ -n "$(docker ps -aq)" ]]; then echo "EPAR host-trust relay: unfinished daemon transaction cannot be recovered after containers exist" >&2 @@ -252,6 +273,7 @@ if [[ -e "${daemon_backup}" ]]; then rm -f "${daemon_backup}" fi +relay_stage="detect-private-dockerd" daemon_already_configured=false if docker info >/dev/null 2>&1 \ && [[ -z "$(docker info --format '{{.HTTPProxy}}')" ]] \ @@ -264,6 +286,7 @@ if docker info >/dev/null 2>&1 \ daemon_already_configured=true fi +relay_stage="configure-private-dockerd" if [[ "${daemon_already_configured}" != "true" ]]; then install -m 0600 -o root -g root "${daemon_config}" "${daemon_backup}" jq --arg proxy "${bridge_proxy}" --arg no_proxy "${daemon_no_proxy}" ' @@ -291,6 +314,7 @@ if [[ "${daemon_already_configured}" != "true" ]]; then echo "EPAR host-trust relay: refusing to restart dockerd after containers exist" >&2 exit 1 fi + relay_stage="restart-private-dockerd" dockerd_restart_attempted=true kill -TERM "${dockerd_pid}" for _ in $(seq 1 60); do @@ -310,6 +334,7 @@ if [[ "${daemon_already_configured}" != "true" ]]; then fi fi +relay_stage="private-dockerd-contract" mapfile -t dockerd_pids < <(pgrep -x dockerd || true) [[ "${#dockerd_pids[@]}" == "1" ]] [[ "$(readlink -f "/proc/${dockerd_pids[0]}/exe" 2>/dev/null || true)" == "/usr/bin/dockerd" ]] @@ -318,11 +343,13 @@ tr '\0' '\n' <"/proc/${dockerd_pids[0]}/environ" | grep -Fx 'GODEBUG=tlsmlkem=0, [[ "$(docker info --format '{{.HTTPSProxy}}')" == "${bridge_proxy}" ]] [[ "$(docker info --format '{{.NoProxy}}')" == "${daemon_no_proxy}" ]] +relay_stage="registry-tls-proof" registry_status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 30 --proxy "${bridge_proxy}" --noproxy '' --cacert "${relay_ca_trust}" https://registry-1.docker.io/v2/)" if [[ "${registry_status}" != "401" ]]; then echo "EPAR host-trust relay: registry TLS proof returned HTTP ${registry_status}" >&2 exit 1 fi +relay_stage="publish-active-marker" install -m 0444 -o root -g root /dev/null /run/epar/egress-relay-active echo "EPAR host-trust relay: authenticated host-trust transport is active" diff --git a/templates/docker-sandboxes/helpers.sha256 b/templates/docker-sandboxes/helpers.sha256 index dac1b2c..9e07bff 100644 --- a/templates/docker-sandboxes/helpers.sha256 +++ b/templates/docker-sandboxes/helpers.sha256 @@ -2,7 +2,7 @@ 3f1cee32d05ffad64004377f0276ef8aa46a2848b32bca74ac35efb3d71fd1bf ./check-runner.sh 4fe8e512539f97d00db3c2856f452f017c4de6a04832f1c1af86f1994862df59 ./collect-runner-diagnostics.sh b8e9a32759f1ab713af9165c445960528c736eb696ebaa060eaf38667cdee0bf ./collect-software-inventory.sh -ca4de65c63f3a7e226e406c2f246e26d12818edf9aaad5f1e37de38e8a131f90 ./configure-egress-relay.sh +5a44fab73820b530f1fcb4974ffad57c19e92616ad41d984e1fa1236c3ed9680 ./configure-egress-relay.sh 6257018b1373f0c3ec7b4b77d859b2114693b1173acb51e927f73296cf123fb5 ./configure-runner.sh 1fbc8c68c8d75f3982e23718ed3d5bd984c2afee21e26657b7a64a9f185a747f ./docker-daemon.json e912446ff3b08f1095f1d3f3935835030c62ba893e438ec88b5f02f07e8b4cb9 ./enable-architecture-emulation.sh