diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 124339a..ab9bd7c 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ { "name": "autodev", "description": "Autonomous development workflow skills for coding agents", - "version": "6.6.2", + "version": "6.6.3", "source": "./", "author": { "name": "Jon Langevin", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 0c9050b..95b7249 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "autodev", "description": "Autonomous development workflow skills for coding agents: design, review, planning, execution, monitoring, and retrospectives", - "version": "6.6.2", + "version": "6.6.3", "author": { "name": "Jon Langevin", "email": "jon@gocodealone.com" diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index fa2ab87..c8331ca 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "autodev", "displayName": "Autonomous Dev Kit", "description": "Autonomous development workflow skills for coding agents", - "version": "6.6.2", + "version": "6.6.3", "author": { "name": "Jon Langevin", "email": "jon@gocodealone.com" diff --git a/.github/workflows/resource-lifecycle.yml b/.github/workflows/resource-lifecycle.yml new file mode 100644 index 0000000..1cc2c40 --- /dev/null +++ b/.github/workflows/resource-lifecycle.yml @@ -0,0 +1,38 @@ +name: Resource Lifecycle +on: + push: + paths: + - 'scripts/job-resources.py' + - 'tests/job-resources.py' + - 'tests/resource-lifecycle-docs.sh' + - 'docs/resource-lifecycle.md' + - 'docs/README.codex.md' + - 'skills/executing-plans/**' + - 'skills/subagent-driven-development/implementer-prompt.md' + - 'skills/finishing-a-development-branch/**' + - '.github/workflows/resource-lifecycle.yml' + pull_request: + paths: + - 'scripts/job-resources.py' + - 'tests/job-resources.py' + - 'tests/resource-lifecycle-docs.sh' + - 'docs/resource-lifecycle.md' + - 'docs/README.codex.md' + - 'skills/executing-plans/**' + - 'skills/subagent-driven-development/implementer-prompt.md' + - 'skills/finishing-a-development-branch/**' + - '.github/workflows/resource-lifecycle.yml' + workflow_dispatch: +permissions: + contents: read +jobs: + resource-lifecycle: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Metadata safety and concurrent ownership + run: python3 tests/job-resources.py + - name: Guidance contracts + run: bash tests/resource-lifecycle-docs.sh + - name: Python syntax + run: python3 -m py_compile scripts/job-resources.py tests/job-resources.py diff --git a/.hermes/plugin.json b/.hermes/plugin.json index fa2ab87..c8331ca 100644 --- a/.hermes/plugin.json +++ b/.hermes/plugin.json @@ -2,7 +2,7 @@ "name": "autodev", "displayName": "Autonomous Dev Kit", "description": "Autonomous development workflow skills for coding agents", - "version": "6.6.2", + "version": "6.6.3", "author": { "name": "Jon Langevin", "email": "jon@gocodealone.com" diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 1552c7f..db5888e 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,5 +1,16 @@ # Autonomous Dev Kit Release Notes +## v6.6.3 — 2026-10-05 + +Build-cache reuse and explicit job-resource closeout. + +- Execution, implementer and finishing guidance now prefers native/stable compatible Go caches, preserves inherited GOCACHE and requires reasons/ownership for cold-cache isolation. Docker mounts and persistent database data require explicit classification. +- Optional POSIX `scripts/job-resources.py` records session/job resources, explicit success/failure closeout and conservative review hints. Stable locks, no-follow descriptor operations and atomic JSON writes protect cooperating concurrent writers; corrupt/unknown metadata is preserved. +- Active, failed, unknown, reusable, database, Docker-volume and worktree resources stay protected. Overlapping protected paths, links, worktree markers, changed/unreadable trees and inspection limits suppress candidate hints. The helper never runs resource commands or deletes resources. +- Added resource safety/concurrency regressions, guidance contracts and Ubuntu CI. Direct CLI and actual repeated Go-build smoke validate preservation/reuse; source checks do not establish host callback invocation. + +No existing cache migration, cleanup daemon, automatic deletion or new lifecycle hook. Adoption is explicit guidance/CLI use and cannot guarantee bounded disk usage. Install in a safe maintenance window: native upgrades can prune older plugin paths used by active sessions. + ## v6.6.2 — 2026-10-04 Session plan status and checkpoint acceptance fixes. diff --git a/decisions/0007-resource-closeout-is-nondeleting.md b/decisions/0007-resource-closeout-is-nondeleting.md new file mode 100644 index 0000000..db00db8 --- /dev/null +++ b/decisions/0007-resource-closeout-is-nondeleting.md @@ -0,0 +1,23 @@ +# ADR 0007: resource closeout records review hints without deleting + +**Status:** Accepted +**Date:** 2026-10-05 + +Repeated temporary build caches and anonymous volumes consume disk, but stale +appearance does not establish ownership or disposable database data. Active caches +and concurrent jobs require preservation. + +Use existing native/stable compatible cache reuse in workflow guidance, and an +optional project-local ownership ledger with explicit job closeout. The helper +writes metadata only. Default/unknown/active/failed/reusable resources stay +protected; eligible disposable directories become manual review hints. Docker +volumes, databases and worktrees stay protected. No timeout/PID ownership reclaim. + +Rejected: automatic cleanup/prune, global GOCACHE rewrite, a new cache allocator, +resource migration, or undocumented host-hook enforcement. Those approaches add +data-loss/concurrency risks or recreate per-job cache growth. + +Consequences: no guaranteed disk quota or automatic reclamation. Crashed jobs can +leave active metadata until their owner explicitly closes them. Live consumer and +data checks are required before any separately authorized removal. Source skills +and direct CLI tests do not prove Codex lifecycle hook invocation. diff --git a/docs/README.codex.md b/docs/README.codex.md index 2758376..3ccc894 100644 --- a/docs/README.codex.md +++ b/docs/README.codex.md @@ -95,6 +95,14 @@ The `using-autodev` skill is discovered automatically and enforces skill usage d ## Usage +For build/test resource ownership, follow [resource lifecycle guidance](resource-lifecycle.md). +Preserve inherited `GOCACHE`, reuse compatible caches and justify cold-cache +isolation. Identify Docker mounts and protect database data; record success/failure +closeout while preserving active, unknown, failed and reusable resources. Optional +`scripts/job-resources.py` is a POSIX metadata CLI, with manual records on other +hosts. It adds no Codex lifecycle hook or automatic deletion. Host callback +invocation is not established by these source/CLI checks. + Skills are discovered automatically. Codex activates them when: - You mention a skill by name (e.g., "use brainstorming") - The task matches a skill's description diff --git a/docs/plans/2026-10-05-resource-lifecycle-code-review.md b/docs/plans/2026-10-05-resource-lifecycle-code-review.md new file mode 100644 index 0000000..aa0dbdc --- /dev/null +++ b/docs/plans/2026-10-05-resource-lifecycle-code-review.md @@ -0,0 +1,27 @@ +# Resource lifecycle independent code review + +Final verdict SHIP-IT; no remaining findings. Independent review inspected +implementation HEAD `7d2e4d923f1642b7afc7b0d934672b15e0bff9a1`, tree +`56b6c2ecd41cf2d5f293fdf82be7e382a146cb39`; initial BLOCK findings resolved. + +| ID | Severity | Finding | Resolution | +|---|---|---|---| +| RCR-01 | Important | Concurrent first lock creation intermittently fails; early test assertion leaves owned children unreaped. | Exclusive-create/existing-open bounded protocol; preserve missing existing-ledger lock; reap every child before assertions. Thirty stress batches pass. | +| RCR-02 | Important | Duplicate JSON fields silently drop unknown owners. | Reject duplicate object keys; report/record/closeout preserve ambiguous bytes. | +| RCR-03 | Important | Candidate scan can inspect detached old directory after path replacement. | Final ancestor-link and descendant identity/timestamp validation; candidate/ancestor/descendant/new-entry regressions preserve data. | + +Reviewer inspected full spec/diff, safety attacks, static guidance/version checks, +and actual helper/Go smoke evidence. No resource command, cache rewrite, scope +expansion or manifest mismatch found. Integration proof remains direct CLI/Go; +optional adoption and host hook invocation are not established. + +Follow-up RCR-04 (Minor): timeout while reaping a child could leave other owned +fixture processes alive. Added finally kill/reap and a real timeout regression; +this affects only test fixtures. A shared five-second lock deadline also matches +documented acquisition limits. Independent 36-test run, eight mutation attacks and +96 fresh-ledger concurrent operations passed before this harness-only addition. +Final independent run passed all 37 helper tests, nine mutation/corruption attacks +and 96 concurrent operations. Requirements/trace/scope, runtime proof, failure/ +recovery, corruption/schema, ownership/identifiers, concurrency, containment/ +privacy, performance bounds, compatibility, CI, rollback/release/version skew +and user guidance were scanned. No open production or harness finding remains. diff --git a/docs/plans/2026-10-05-resource-lifecycle-design-review.md b/docs/plans/2026-10-05-resource-lifecycle-design-review.md new file mode 100644 index 0000000..d17ebc6 --- /dev/null +++ b/docs/plans/2026-10-05-resource-lifecycle-design-review.md @@ -0,0 +1,31 @@ +# Resource lifecycle design and plan review + +Independent read-only adversarial reviewer; initial design FAIL, plan FAIL, +structural alignment PASS. No Critical findings; all inherited design and plan +bug classes scanned. No source/config/session changes by reviewer. + +| ID | Phase | Severity | Finding | Author response | +|---|---|---|---|---| +| D1 | design | Important | Exact-resource protection omitted overlapping parents/children. | Suppress protected path overlaps; inspect ancestor/descendant worktree markers, reject ambiguous/budget-exceeded trees. | +| D2 | design | Important | Check/write races and replaceable lock inode could lose ownership. | Separate stable lock, no-follow descriptor-relative operations, parent/lock identity validation and explicit same-user race boundary. | +| D3 | design | Minor | Same cache path does not establish reuse. | Verify effective cache unchanged and second unchanged build avoids package compilation. | +| P1 | plan | Important | Overlap regressions unspecified. | Add protected/reusable/database parent/child and nested/ancestor worktree cases with preservation. | +| P2 | plan | Important | Generic races could miss critical interleavings. | Deterministic directory/lock swap barriers; competing writers across JSON replacements, stable lock and no lost owners. | +| P3 | plan | Minor | Verification commands/results underspecified. | Exact unittest/compile/doc/CLI lifecycle and real Go reuse assertions added. | +| P4 | plan | Minor | Unauthorized wording could imply authentication. | Cross-session attribution rejection; explicit non-authentication limitation. | + +Scan: project guidance, assumptions, precedent, artifact class, YAGNI, failure +modes, security/privacy, infrastructure, multi-component/declared proof, UI, +rollback, simpler alternative, user intent and runtime validity. Additional plan +scan: decomposition, verification class, auth chains, dependencies, rollback/ +integration wiring, matrix/UI routes, infrastructure, loader layout, schema, +identifiers and compile validity. Findings above cover all non-clean results. + +Alternative: retain optional guidance/ledger with conservative overlap and stable +descriptor lock contract. Adoption remains voluntary; no bounded-disk guarantee. +Forward/reverse trace R1→Task2, R2/R3/R4→Task1, R5→Tasks3/4 is structural PASS. + +Cycle 2: design PASS, plan PASS, alignment PASS; no open Important/Critical +findings. D4 (Minor) clarified ancestor inspection stops before the project root; +the author added the boundary and will test an ordinary Git project's candidate. +All D1-D4/P1-P4 findings are resolved or assigned to the existing regression task. diff --git a/docs/plans/2026-10-05-resource-lifecycle-design.md b/docs/plans/2026-10-05-resource-lifecycle-design.md new file mode 100644 index 0000000..7e58fec --- /dev/null +++ b/docs/plans/2026-10-05-resource-lifecycle-design.md @@ -0,0 +1,125 @@ +# Resource lifecycle prevention design + +Authorization: the user requested optimizations to reduce repeated disk exhaustion, +an independently reviewed release, and preservation of active caches/installations. +The separate disk cleanup work owns its approved deletions; this change performs none. + +## Guidance and evidence + +- `AGENTS.md`: generic multi-host workflows, isolated changes, repository contracts, + synchronized manifests and existing release process. +- Parent audit: 113 temporary Go caches (107.53 GiB); 238 new Docker volumes + (30.24 GiB), 236 anonymous, since October 3. Active caches and unique database + data require preservation. This is delegated audit evidence, not a fresh disk scan. +- [Go build cache documentation](https://pkg.go.dev/cmd/go#hdr-Build_and_test_caching): + native cache reuse supports concurrent Go commands and compiler/options changes; + changed C libraries require explicit rebuild handling. +- [Docker volume documentation](https://docs.docker.com/engine/storage/volumes/): + anonymous volumes persist and are not automatically reused between containers. +- [Issue #101](https://github.com/GoCodeAlone/autonomous-dev-kit/issues/101): + supported resource/runtime acceptance and explicit ownership remain follow-ups. + +## Bounded response + +R1: Existing execution/implementation/finishing guidance must prefer the effective +Go cache or a compatible stable project/toolchain cache. Preserve existing +`GOCACHE`; never change global Go settings. Require a concrete isolation reason +before a new cold cache, record its owner and closeout. Cache compatibility is not +database compatibility: identify every Docker mount and explicitly classify data. + +R2: An optional POSIX Python standard-library helper records resources, closes +the calling job, and reports cleanup review candidates. It never executes Go, +Docker, removal, process stopping, or resource creation. Only its project-local +metadata is written. Guidance is usable without the helper on other platforms. + +R3: Active, unknown, failed, reusable, database, Docker-volume and worktree +resources remain protected. A disposable directory can become a review candidate +only inside the project's `.autodev/tmp`, with a stated reason, existing directory, +no symlink/escape/worktree marker, and every recorded owner explicitly closed +successfully. Candidates cannot contain or lie within a protected recorded path. +Ancestor and descendant worktree markers, symlink descendants, unreadable trees, +or inspection beyond a bounded entry budget keep the directory protected. +Ancestor inspection stops before the enclosing project root, whose own Git +marker is expected; a positive normal-Git-project fixture verifies this boundary. +Output is a stale-able hint: live consumers and data intent must be +checked again before any separately authorized deletion. + +R4: Concurrent writers must not lose ownership records. Session/job identities +are explicit attribution, not authentication. A different session cannot close a +job; closed jobs cannot be silently reused. Shared resources merge toward more +protective retention and purpose. Corrupt/unknown metadata is never reset. Failed +commands/crashes leave active ownership until explicit closeout, not PID/age guesses. + +R5: Integrate meaningful helper and guidance regressions into CI, perform real CLI +and repeated Go-build smoke, review independently, and release v6.6.3 through the +existing tag/marketplace process. Preserve active native installation until the +existing maintenance condition is satisfied. No new hook declaration or claim of +Codex lifecycle invocation: this is explicit skill guidance and optional CLI use. + +## Interface and ownership model + +`scripts/job-resources.py --project record --session --job +--kind directory|docker-volume|worktree --resource +--purpose build-cache|test-fixture|database|unknown +--retention protected|reusable|disposable [--reason ]` + +`closeout --session --job --outcome success|failure` and `report` +return JSON. Record creates an active job on its first resource; no implicit owner +claim from timestamps, directory names, container state, or another session's log. +Default purpose and retention are unknown/protected. Disposable declarations for +external paths, databases, volumes or worktrees fail instead of weakening policy. + +Metadata lives at `.autodev/state/job-resources.json` with a schema version and +project identity. POSIX `flock` uses a separate stable lock file that the helper +never replaces; only the JSON ledger is atomically replaced. Lock acquisition has +a bounded timeout. Metadata operations use no-follow directory descriptors and +descriptor-relative file opens/replacements, with parent/lock identity checks +before reads and commits. Observed directory/lock swaps fail without following +replacement targets. Cooperating helper writers are serialized; a same-user actor +can still move an already-opened original directory after validation. Such +tampering is not an authentication boundary: writes remain in that original owned +inode, not a replacement symlink's target, and detected changes are reported. +Read-only empty reports create nothing. Candidate paths are +revalidated at report time. The helper records identifiers and concise reasons, +not full command lines, environment dumps or credentials. + +## Alternatives and limits + +- Chosen: reuse native cache behavior plus explicit resource metadata and safe + closeout guidance. See `decisions/0007-resource-closeout-is-nondeleting.md`. +- Rejected: cleanup daemon, global cache rewrites, automatic volume pruning, + ownership inferred from age/PIDs, or migration of existing caches. +- Rejected: a new custom cache allocator. Native Go already provides stable reuse; + another per-job allocator could reproduce the reported growth. +- Limits: guidance does not programmatically block arbitrary future allocations. + The ledger knows only declared owners and cannot prove a path has no live + consumers. Docker/database data always requires manual review. Abrupt job loss + preserves resources and may leave conservative active records. + +## Validation and rollback + +Tests cover same-resource concurrent sessions, cross-session closeout rejection, +active/unknown/failed owners, policy escalation, repeated jobs, missing/corrupt or +future schema, lock/write failures, path traversal and deterministic directory/ +lock swaps, nested/ancestor worktrees, protected parent/child overlaps, and byte +preservation of external/cache/database sentinels. Candidate +reporting must never remove or modify a resource. Test fixtures are owned temporary +directories; no real Docker volumes or other worktrees are touched. + +Runtime smoke uses the shipped CLI and two real Go builds of an owned tiny project +with an existing effective cache, recording the same cache as reusable. Record +effective GOCACHE before/after and assert the unchanged second `go build -x` +contains no package compilation, rather than treating a shared path as reuse proof. +No new cold Go cache for demonstration. Host hook dispatch remains unverified and unused. +Rollback is a successor patch/revert; leave resource metadata/resources intact. +No installation refresh, destructive data migration, or protection-setting change. + +## Backport 2026-10-05: review-discovered interleavings + +Native concurrent nonexclusive lock creation reproduced intermittent ENOENT. +Use exclusive first creation plus bounded existing-open retry; never recreate a +missing lock beside an existing ledger. Independent review also demonstrated +duplicate JSON fields dropping unknown ownership and candidate scans of detached +old inodes. Reject duplicate fields and revalidate ancestor links plus bounded +descendant identities/timestamps before returning a hint. Inspection also stops +at 64 directory levels. Existing Tasks 1/3 cover these regressions; manifest unchanged. diff --git a/docs/plans/2026-10-05-resource-lifecycle-plan-review.md b/docs/plans/2026-10-05-resource-lifecycle-plan-review.md new file mode 100644 index 0000000..40ab8d7 --- /dev/null +++ b/docs/plans/2026-10-05-resource-lifecycle-plan-review.md @@ -0,0 +1,9 @@ +# Resource lifecycle plan review + +Independent read-only review: Cycle 1 FAIL; Cycle 2 PASS, alignment PASS. +P1 overlap, P2 deterministic race/serialization, P3 precise verification and P4 +attribution terminology resolved as recorded in the design-review companion. +Every design requirement maps to an existing task, and every task is justified. +Structural scope validation passed: one PR, four tasks, unique task assignment. +All design/plan bug classes were scanned; no open Important/Critical findings. +Minor D4 is included in Task 1: positive candidate within a normal Git project. diff --git a/docs/plans/2026-10-05-resource-lifecycle-verification.md b/docs/plans/2026-10-05-resource-lifecycle-verification.md new file mode 100644 index 0000000..3c3e663 --- /dev/null +++ b/docs/plans/2026-10-05-resource-lifecycle-verification.md @@ -0,0 +1,23 @@ +# Resource lifecycle verification + +Task 1: tests written before helper; initial run failed 24/24 because helper was +absent. Implemented CLI/filesystem tests passed 37/37, including real concurrent +writers, protected overlaps, unchanged sentinel bytes, failure/lock timeout, +directory/lock swaps, corrupt metadata and no-follow/nonregular-file rejection. +Repeated first-writer stress passed 30 concurrent batches after reproducing a +native concurrent create/open failure. Duplicate-field and scan-swap attacks +found by independent review have explicit preservation regressions. + +Tasks 2/3: all 13 preexisting AGENTS/CI suites plus helper and guidance suites +passed. Python syntax and diff hygiene passed. Guidance contracts are static +evidence, not agent runtime compliance. New path-filtered Ubuntu CI uses Python3 +and Bash without new dependencies or runtime pins. + +Actual shipped-helper smoke: record → protected report → success closeout → +review candidate; sentinel SHA256 unchanged. Real Go1.27.1 darwin/arm64 builds of +an owned module preserved effective existing GOCACHE; unchanged second build +performed no package compilation or linking. No cold cache, Docker command, +global Go setting change, deletion helper, or host-hook claim. + +Installation acceptance remains open; preserve active native plugin paths until +natural completion of consumers or an explicitly authorized maintenance window. diff --git a/docs/plans/2026-10-05-resource-lifecycle.md b/docs/plans/2026-10-05-resource-lifecycle.md new file mode 100644 index 0000000..c43b0c5 --- /dev/null +++ b/docs/plans/2026-10-05-resource-lifecycle.md @@ -0,0 +1,101 @@ +# Resource Lifecycle Implementation Plan + +**Goal:** Reduce fresh build-cache allocation and make job resources attributable +and visible at closeout, without automatic deletion. +**Architecture:** Existing workflow guidance plus an optional POSIX metadata ledger. +**Tech Stack:** Python 3.9+ standard library, Bash contracts, Git/gh release tooling. +**Base branch:** main + +## Scope Manifest + +**PR Count:** 1 +**Tasks:** 4 +**Estimated Lines of Change:** ~1000 +**Out of scope:** +- Actual cleanup, Docker/Go command automation, cache allocation/migration, cleanup + daemon, global GOCACHE changes, security expansion, other worktrees/sessions, + general ownership supersession, and active plugin upgrade before maintenance. + +**PR Grouping:** + +| PR # | Title | Tasks | Branch | +|------|-------|-------|--------| +| 1 | Record resource ownership and prefer stable cache reuse | Task 1, Task 2, Task 3, Task 4 | fix/resource-lifecycle-guidance | + +**Status:** Locked 2026-10-05T05:42:00Z + +## Guidance and trace + +Design R1 → Task 2; R2/R3/R4 → Task 1; R5 → Tasks 3/4. One source PR; the +existing marketplace workflow may generate its normal version notification PR. +User-authorized admin merge is permitted only after exact-head checks and clean +independent review, with no unresolved concern. Active install remains held. + +### Task 1: Nondeleting resource ledger and regression suite + +Files: create `scripts/job-resources.py`, `tests/job-resources.py`. +1. Write meaningful unittest/CLI regressions first; run against absent helper and + capture the expected failure. Use real subprocess concurrency and owned fixtures. +2. Implement record/closeout/report, safe metadata containment/locking/atomic writes, + conservative ownership/retention merge, schema preservation, and review hints. +3. Verify active/unknown/failed protection, cross-session attribution rejection, + competing writers across ledger replacements with stable lock identity, + missing/corrupt/future state, failed writes/locks, path/symlink/worktree rejection, + deterministic directory/lock swaps, protected parent/child overlaps, nested and + ancestor worktrees, and unchanged sentinel bytes. Attribution is not authentication. + No Go/Docker/delete subprocesses in helper. +4. Run `python3 tests/job-resources.py` and + `python3 -m py_compile scripts/job-resources.py tests/job-resources.py`; + expect all tests/compile exit 0. Commit the bounded helper/test change. + +### Task 2: Cache reuse and explicit resource guidance + +Files: create `docs/resource-lifecycle.md`, `tests/resource-lifecycle-docs.sh`; +modify `skills/executing-plans/SKILL.md`, +`skills/subagent-driven-development/implementer-prompt.md`, +`skills/finishing-a-development-branch/SKILL.md`, `docs/README.codex.md`. +1. Add doc-contract checks for every entrypoint, native/stable cache reuse, + inherited GOCACHE preservation, cold-cache reason, explicit Docker mounts/data, + closeout candidates, failure/unknown protection, and optional CLI limits. +2. Write concise actionable guidance and real helper examples using placeholders. + Keep database data protected; avoid anonymous volumes in durable/repeated jobs. +3. Run `bash tests/resource-lifecycle-docs.sh`, `bash tests/skill-content-grep.sh`, + and `bash tests/skill-cross-refs.sh`; expect exit 0. Read a failure/parallel-job pressure + scenario. Text checks do not establish runtime agent compliance or host dispatch. +4. Commit guidance and document concrete behavioral limits. + +### Task 3: CI and runtime/source verification + +Files: create `.github/workflows/resource-lifecycle.yml`; evidence in owned task +verification directory; committed design/plan review and code-review reports. +1. Wire helper and doc tests into a path-filtered Ubuntu CI job using Python3/Bash. +2. Run every AGENTS contract and existing CI-specific hook/Zed/path check plus new + resource tests. Verify plan/hash, diff hygiene and syntax; compare inherited lint. +3. `python3 scripts/job-resources.py --help` must list record/closeout/report. + Run actual CLI record→active report→success closeout→candidate report, verifying + resource sentinel bytes survive. Run two unchanged `go build -x` commands in an + owned project with the existing effective GOCACHE; before/after cache values + must match and second output must contain no package compilation. No new cold + cache or Docker resource. Capture commands/exit statuses as runtime evidence. +4. Independent spec/code review, including failure/concurrency/containment attacks; + fix findings and recheck exact final head. Record qualified runtime evidence. + +### Task 4: Reviewed patch release and maintenance handoff + +Files: all four manifests via `scripts/bump-version.sh`; `RELEASE-NOTES.md`. +1. Prepare v6.6.3 notes/manifests, then run the complete final relevant checks. +2. Push one source PR; attach when app tools are available. Confirm exact-head CI, + independent review and unresolved threads before normal/authorized admin squash. +3. Wait for merge-commit checks and official tag workflow. Verify peeled tag equals + merge commit and reviewed tree; publish immutable latest GitHub Release using + the existing tag and checked-in notes. Verify both source archives/manifests. +4. Verify marketplace notification/version PR checks and exact manifest delta. +5. Report active installed version without refreshing it; preserve settings/old + paths. Safe native installation requires natural completion of old-cache users + or explicit maintenance authorization. Overall acceptance stays open until then. + +## Completion boundary + +Source/release checkpoints are distinct from installation acceptance. No partial +source implementation, fabricated hook proof, or cleanup action is authorized. +Resource candidates are review inputs, never permission to remove them. diff --git a/docs/plans/2026-10-05-resource-lifecycle.md.scope-lock b/docs/plans/2026-10-05-resource-lifecycle.md.scope-lock new file mode 100644 index 0000000..abd2561 --- /dev/null +++ b/docs/plans/2026-10-05-resource-lifecycle.md.scope-lock @@ -0,0 +1 @@ +f2e14dd29c70726a6875f3ec09c05d683d412cfbc0f102bc73df2bc1f2d3230c diff --git a/docs/resource-lifecycle.md b/docs/resource-lifecycle.md new file mode 100644 index 0000000..7256eb4 --- /dev/null +++ b/docs/resource-lifecycle.md @@ -0,0 +1,86 @@ +# Build caches and job resources + +Before disk-heavy work, identify the effective cache, every container mount and +the job owning each newly created resource. Reuse compatible resources instead of +allocating a fresh cache for every task, review or retry. A low-space condition +should defer new heavy work for scoped owner review; preserve running jobs and data. + +For Go, inspect `go env GOCACHE` and preserve inherited `GOCACHE`. Prefer the native +cache, or an existing stable project/toolchain cache when the project's constraints +require it. Go's native cache supports concurrent commands and compiler/options +changes. Changed C libraries require explicit rebuild handling. Do not run global +`go env -w GOCACHE=...`, default to a new `mktemp` cache per job, or empty another +job's cache. A cold cache needs a concrete isolation/reproduction reason, an owner +and an explicit closeout record. See [Go cache documentation](https://pkg.go.dev/cmd/go#hdr-Build_and_test_caching). + +For Docker, declare each mount's name/path, purpose, owner and persistence before +launch. Prefer stable named build caches where compatible; anonymous volumes +persist and are not automatically reused. Database/application volumes are +protected data, including anonymous PostgreSQL volumes with unknown provenance. +Do not infer disposability from names, age, absence of containers, or reference +count zero. Never use broad prune, Docker reset/restart or process termination as +routine resource closeout. See [Docker volume documentation](https://docs.docker.com/engine/storage/volumes/). + +At success **and failure**, record the job outcome and resource disposition in +the task's evidence. Preserve active, unknown, failed and reusable resources. +List exact cleanup review candidates with ownership and isolation reasons; +recheck live use and data intent before any separately authorized deletion. +Do not infer completed ownership from PID disappearance or elapsed time. +Worktrees/source, database data, host configuration and plugin caches stay protected. + +## Optional metadata helper + +`scripts/job-resources.py` requires Python 3.9+ and POSIX flock/no-follow directory +support. On other hosts, keep equivalent explicit records manually. This is +optional skill guidance and direct CLI use; it adds no lifecycle hook, daemon, +automatic host dispatch, disk quota, cache allocator, or deletion command. +Adoption cannot guarantee bounded disk usage. + +Run the helper from the actual installed plugin/checkout location. In these +examples, `ADK_ROOT` is that location and `PROJECT_ROOT` is the existing project +or worktree owning the resources. Repeated jobs use new job IDs; session/job IDs +are attribution, not authentication. Keep IDs/reasons concise; never record +credentials, full command lines or environment dumps. + +```bash +# Record an existing effective cache; this command does not run Go or create it. +python3 "$ADK_ROOT/scripts/job-resources.py" --project "$PROJECT_ROOT" record \ + --session "$SESSION_ID" --job "$JOB_ID" --kind directory \ + --resource "$EXISTING_CACHE" --purpose build-cache --retention reusable + +# Only an already-created, isolated directory strictly within .autodev/tmp can +# be declared disposable. Explain why this job required a fresh resource. +python3 "$ADK_ROOT/scripts/job-resources.py" --project "$PROJECT_ROOT" record \ + --session "$SESSION_ID" --job "$JOB_ID" --kind directory \ + --resource "$PROJECT_ROOT/.autodev/tmp/$JOB_ID" --purpose test-fixture \ + --retention disposable --reason 'cold-cache regression requires isolation' + +python3 "$ADK_ROOT/scripts/job-resources.py" --project "$PROJECT_ROOT" closeout \ + --session "$SESSION_ID" --job "$JOB_ID" --outcome success +# Use --outcome failure after a failed job. Abrupt loss leaves active ownership. +python3 "$ADK_ROOT/scripts/job-resources.py" --project "$PROJECT_ROOT" report +``` + +The helper writes only `.autodev/state/job-resources.json` and its stable lock/ +transaction metadata. Defaults are unknown/protected. Volume/worktree declarations +stay protected; disposable database/volume/worktree or external-path declarations +are rejected. Every declared owner must close successfully before a disposable +directory can become a `review-candidate`. Protected path overlaps, symlinks, +nested/ancestor worktrees (excluding the enclosing project's own Git marker), +unreadable/changed directories, more than 10,000 inspected entries or 64 directory +levels keep it +protected. Reports never remove or modify resource contents. + +Cooperating writers serialize on a stable lock; JSON updates are atomic. Unknown, +corrupt, mismatched or oversized (1 MiB) metadata fails without resetting it. +Operational errors return JSON on stderr with exit 2; argument syntax uses normal +CLI help/errors. Lock acquisition times out after five seconds. A failed commit +may have reached disk before a later durability/identity error; inspect the ledger +before retrying. Never remove the lock or repair metadata while another job uses it. +Same-user tampering is outside the attribution boundary: no-follow descriptors +avoid following replacement targets, but an opened original directory can still +be moved. Detected changes fail conservatively. No age/PID owner reclaim exists. + +Candidates are stale-able review hints, never deletion permission. The ledger knows +only declared owners and cannot prove absence of live consumers. Rollback leaves +metadata and resources intact; there is no required migration of existing caches. diff --git a/scripts/job-resources.py b/scripts/job-resources.py new file mode 100644 index 0000000..e381500 --- /dev/null +++ b/scripts/job-resources.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 +"""Optional resource attribution ledger. Writes metadata; never deletes resources.""" +import argparse +import errno +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import stat +import sys +import time + +try: + import fcntl +except ImportError: + fcntl = None + +MAX_ENTRIES = 10000 +LOCK_TIMEOUT = 5 +MAX_METADATA = 1024 * 1024 +KINDS = ("directory", "docker-volume", "worktree") +PURPOSES = ("build-cache", "test-fixture", "database", "unknown") +RETENTIONS = ("disposable", "reusable", "protected") +NOFOLLOW = getattr(os, "O_NOFOLLOW", 0) +DIRECTORY = getattr(os, "O_DIRECTORY", 0) + + +class Invalid(ValueError): + pass + + +def identity(info): + return info.st_dev, info.st_ino + + +def key(*parts): + return hashlib.sha256("\0".join(parts).encode()).hexdigest() + + +def text(value, limit=200): + return isinstance(value, str) and 0 < len(value) <= limit and not any( + ord(c) < 32 or ord(c) == 127 for c in value) + + +def beneath(path, parent): + return path != parent and parent in path.parents + + +def regular(fd): + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + raise Invalid("metadata must be a regular, unshared file") + + +class Ledger: + """Stable flock inode and descriptor-relative metadata transactions.""" + def __init__(self, project, write): + self.project, self.write = project, write + self.fds, self.links, self.lock = [], [], None + self.empty = False + + def __enter__(self): + if fcntl is None or not NOFOLLOW or not DIRECTORY: + raise Invalid("POSIX flock/no-follow directories required; use manual guidance") + try: + fd = os.open(self.project, os.O_RDONLY | DIRECTORY | NOFOLLOW) + self.fds.append(fd) + for name in (".autodev", "state"): + parent = fd + try: + fd = os.open(name, os.O_RDONLY | DIRECTORY | NOFOLLOW, dir_fd=parent) + except FileNotFoundError: + if not self.write: + self.empty = True + return self + try: + os.mkdir(name, 0o700, dir_fd=parent) + except FileExistsError: + pass # another cooperating first writer created it + fd = os.open(name, os.O_RDONLY | DIRECTORY | NOFOLLOW, dir_fd=parent) + self.fds.append(fd) + self.links.append((parent, name, fd)) + self.directory = fd + try: + os.stat("job-resources.json", dir_fd=fd, follow_symlinks=False) + existing_ledger = True + except FileNotFoundError: + existing_ledger = False + if not self.write: + try: + os.stat("job-resources.json", dir_fd=fd, follow_symlinks=False) + except FileNotFoundError: + self.empty = True + return self + deadline = time.monotonic() + LOCK_TIMEOUT + while True: + self.validate_parents() + try: + if self.write and not existing_ledger: + try: + # Exclusive creation avoids native concurrent O_CREAT races. + self.lock = os.open("job-resources.lock", os.O_RDWR | NOFOLLOW | + os.O_CREAT | os.O_EXCL, 0o600, dir_fd=fd) + except FileExistsError: + self.lock = os.open("job-resources.lock", os.O_RDWR | NOFOLLOW, dir_fd=fd) + else: + self.lock = os.open("job-resources.lock", os.O_RDWR | NOFOLLOW, dir_fd=fd) + break + except FileNotFoundError: + if not self.write or existing_ledger or time.monotonic() >= deadline: + raise Invalid("metadata lock unavailable; manual review required") + self.validate_parents() + time.sleep(0.01) + regular(self.lock) + while True: + try: + fcntl.flock(self.lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if time.monotonic() >= deadline: + raise Invalid("metadata lock timeout; ownership unchanged") + time.sleep(0.05) + self.validate() + return self + except Exception: + self.__exit__(None, None, None) + raise + + def __exit__(self, *_): + if self.lock is not None: + os.close(self.lock) # closing releases flock even after failed acquisition + self.lock = None + for fd in reversed(self.fds): + os.close(fd) + self.fds = [] + + def validate_parents(self): + if identity(os.stat(self.project, follow_symlinks=False)) != identity(os.fstat(self.fds[0])): + raise Invalid("project directory changed") + for parent, name, fd in self.links: + info = os.stat(name, dir_fd=parent, follow_symlinks=False) + if not stat.S_ISDIR(info.st_mode) or identity(info) != identity(os.fstat(fd)): + raise Invalid("metadata directory changed") + + def validate(self): + self.validate_parents() + info = os.stat("job-resources.lock", dir_fd=self.directory, follow_symlinks=False) + if identity(info) != identity(os.fstat(self.lock)) or not stat.S_ISREG(info.st_mode): + raise Invalid("metadata lock changed") + + def load(self): + empty = {"schema": 1, "project": str(self.project), "jobs": {}, "resources": {}} + if self.empty: + return empty + self.validate() + try: + fd = os.open("job-resources.json", os.O_RDONLY | os.O_NONBLOCK | NOFOLLOW, dir_fd=self.directory) + except FileNotFoundError: + if self.write: + return empty + raise Invalid("ledger disappeared") + with os.fdopen(fd, "rb") as stream: + regular(stream.fileno()) + raw = stream.read(MAX_METADATA + 1) + if len(raw) > MAX_METADATA: + raise Invalid("metadata size limit exceeded; manual review required") + def unique_fields(pairs): + result = {} + for name, value in pairs: + if name in result: + raise Invalid("duplicate metadata fields; metadata preserved") + result[name] = value + return result + data = json.loads(raw, object_pairs_hook=unique_fields) + validate_data(data, self.project) + return data + + def save(self, data): + validate_data(data, self.project) + raw = (json.dumps(data, sort_keys=True, indent=2) + "\n").encode() + if len(raw) > MAX_METADATA: + raise Invalid("metadata size limit exceeded; manual review required") + self.validate() + name = ".job-resources-" + secrets.token_hex(12) + ".tmp" + try: + fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | NOFOLLOW, + 0o600, dir_fd=self.directory) + with os.fdopen(fd, "wb") as stream: + stream.write(raw) + stream.flush() + os.fsync(stream.fileno()) + self.validate() + os.replace(name, "job-resources.json", src_dir_fd=self.directory, + dst_dir_fd=self.directory) + os.fsync(self.directory) + self.validate() + finally: + try: + os.unlink(name, dir_fd=self.directory) # only our unpublished metadata + except FileNotFoundError: + pass + + +def validate_data(data, project): + if not isinstance(data, dict) or type(data.get("schema")) is not int or data.get("schema") != 1 or data.get("project") != str(project): + raise Invalid("unknown schema or project identity; metadata preserved") + if set(data) != {"schema", "project", "jobs", "resources"}: + raise Invalid("unknown metadata fields; metadata preserved") + if not isinstance(data.get("jobs"), dict) or not isinstance(data.get("resources"), dict): + raise Invalid("invalid ledger shape; metadata preserved") + for ident, job in data["jobs"].items(): + if not isinstance(job, dict) or not text(job.get("session")) or not text(job.get("job")): + raise Invalid("invalid job attribution") + if set(job) != {"session", "job", "state"}: + raise Invalid("unknown job fields") + if ident != key(job["session"], job["job"]) or job.get("state") not in ("active", "success", "failure"): + raise Invalid("invalid job identity/state") + for ident, resource in data["resources"].items(): + if not isinstance(resource, dict) or resource.get("kind") not in KINDS: + raise Invalid("invalid resource kind") + if set(resource) != {"kind", "resource", "purposes", "retention", "reason", "owners"}: + raise Invalid("unknown resource fields") + name = resource.get("resource") + if not text(name, 4096) or ident != key(resource["kind"], name): + raise Invalid("invalid resource identity") + if resource["kind"] != "docker-volume" and (not Path(name).is_absolute() or str(Path(os.path.normpath(name))) != name): + raise Invalid("invalid recorded path") + if resource.get("retention") not in RETENTIONS or not isinstance(resource.get("purposes"), list): + raise Invalid("invalid resource policy") + if not resource["purposes"] or any(p not in PURPOSES for p in resource["purposes"]): + raise Invalid("invalid resource purpose") + if not isinstance(resource.get("owners"), list) or any(not text(o) for o in resource["owners"]): + raise Invalid("invalid owner list") + if not isinstance(resource.get("reason"), str) or (resource["reason"] and not text(resource["reason"])): + raise Invalid("invalid resource reason") + + +def inspect_directory(project, path): + """Never follow links; bound entry inspection; stop before project's own .git.""" + base = project / ".autodev/tmp" + if not beneath(path, base): + return "outside project temporary directory" + fds = [] + try: + fd = os.open(project, os.O_RDONLY | DIRECTORY | NOFOLLOW) + fds.append(fd) + for part in path.relative_to(project).parts: + fd = os.open(part, os.O_RDONLY | DIRECTORY | NOFOLLOW, dir_fd=fd) + fds.append(fd) + try: + os.stat(".git", dir_fd=fd, follow_symlinks=False) + return "worktree marker" + except FileNotFoundError: + pass + pending = [(os.dup(fd), ())] + inspected = [] + count = 0 + try: + while pending: + current, relative = pending.pop() + try: + info = os.fstat(current) + inspected.append((relative, (identity(info), info.st_mtime_ns, info.st_ctime_ns))) + with os.scandir(current) as entries: + for entry in entries: + count += 1 + if count > MAX_ENTRIES: + return "inspection budget exceeded" + if entry.name == ".git": + return "nested worktree marker" + info = entry.stat(follow_symlinks=False) + if info.st_dev != os.fstat(current).st_dev: + return "foreign filesystem" + if stat.S_ISLNK(info.st_mode): + return "symlink descendant" + if stat.S_ISDIR(info.st_mode): + if len(relative) >= 64: + return "inspection depth exceeded" + child = os.open(entry.name, os.O_RDONLY | DIRECTORY | NOFOLLOW, dir_fd=current) + if identity(os.fstat(child)) != identity(info): + os.close(child) + return "directory changed during inspection" + pending.append((child, relative + (entry.name,))) + finally: + os.close(current) + finally: + for remaining, _ in pending: + os.close(remaining) + # Inspecting a detached old inode cannot justify the current path. + if identity(os.stat(project, follow_symlinks=False)) != identity(os.fstat(fds[0])): + return "project changed during inspection" + for parent, part, child in zip(fds, path.relative_to(project).parts, fds[1:]): + info = os.stat(part, dir_fd=parent, follow_symlinks=False) + if not stat.S_ISDIR(info.st_mode) or identity(info) != identity(os.fstat(child)): + return "directory changed during inspection" + try: + os.stat(".git", dir_fd=child, follow_symlinks=False) + return "worktree marker" + except FileNotFoundError: + pass + for relative, expected in inspected: + current = os.dup(fd) + try: + for part in relative: + child = os.open(part, os.O_RDONLY | DIRECTORY | NOFOLLOW, dir_fd=current) + os.close(current) + current = child + info = os.fstat(current) + if (identity(info), info.st_mtime_ns, info.st_ctime_ns) != expected: + return "tree changed during inspection" + finally: + os.close(current) + return None + except OSError: + return "missing, unreadable or changed directory" + finally: + for fd in reversed(fds): + os.close(fd) + + +def report(data, project): + rows = [] + for resource in data["resources"].values(): + reasons = [] + if resource["kind"] != "directory": + reasons.append("volumes/worktrees require manual data review") + if resource["retention"] != "disposable": + reasons.append("protected or reusable retention") + if "unknown" in resource["purposes"] or "database" in resource["purposes"]: + reasons.append("unknown or database purpose") + if not resource["reason"]: + reasons.append("no disposal reason") + if not resource["owners"] or any(data["jobs"].get(o, {}).get("state") != "success" for o in resource["owners"]): + reasons.append("active, failed or unknown owner") + if not reasons: + issue = inspect_directory(project, Path(resource["resource"])) + if issue: + reasons.append(issue) + rows.append(dict(resource, status="protected" if reasons else "review-candidate", reasons=reasons)) + # Suppression propagates: a newly protected candidate can protect an overlap. + changed = True + while changed: + changed = False + protected = [Path(r["resource"]) for r in rows if r["kind"] != "docker-volume" and r["status"] == "protected"] + try: + changed_path = any(p.resolve() != p for p in protected) + except (OSError, RuntimeError): + changed_path = True + for row in rows: + if row["status"] != "review-candidate": + continue + path = Path(row["resource"]) + if changed_path or any(path == p or beneath(path, p) or beneath(p, path) for p in protected): + row["status"] = "protected" + row["reasons"].append("protected path changed" if changed_path else "overlaps a protected recorded path") + changed = True + return {"resources": sorted(rows, key=lambda r: (r["kind"], r["resource"])), + "notice": "Review hints only; declared owners cannot prove absence of live consumers. No deletion performed."} + + +def record(data, args, project): + if not text(args.session) or not text(args.job) or (args.reason and not text(args.reason)): + raise Invalid("session/job/reason must be concise printable identifiers") + if args.kind == "docker-volume": + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,199}", args.resource): + raise Invalid("invalid volume name") + name = args.resource + else: + path = Path(args.resource) + if not path.is_absolute(): + path = project / path + requested_root = Path(os.path.abspath(args.project)) + if path == requested_root or beneath(path, requested_root): + path = project / path.relative_to(requested_root) + lexical = Path(os.path.abspath(path)) + name = str(lexical.resolve()) + if args.retention == "disposable" and (lexical != Path(name) or inspect_directory(project, lexical)): + raise Invalid("disposable directory must be contained, readable and free of links/worktrees") + if args.retention == "disposable" and (args.kind != "directory" or args.purpose in ("database", "unknown") or not args.reason): + raise Invalid("disposable requires a directory, known nondatabase purpose and explicit reason") + owner = key(args.session, args.job) + if owner in data["jobs"] and data["jobs"][owner]["state"] != "active": + raise Invalid("closed job cannot be reused; choose a new job identifier") + data["jobs"][owner] = {"session": args.session, "job": args.job, "state": "active"} + ident = key(args.kind, name) + row = data["resources"].setdefault(ident, {"kind": args.kind, "resource": name, + "purposes": [], "retention": args.retention, "reason": args.reason, "owners": []}) + row["retention"] = RETENTIONS[max(RETENTIONS.index(row["retention"]), RETENTIONS.index(args.retention))] + if args.purpose not in row["purposes"]: + row["purposes"].append(args.purpose) + if owner not in row["owners"]: + row["owners"].append(owner) + if not row["reason"]: + row["reason"] = args.reason + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project", required=True, help="existing project root") + commands = parser.add_subparsers(dest="command", required=True) + add = commands.add_parser("record", help="attribute a resource to an active job") + close = commands.add_parser("closeout", help="explicitly close this session/job") + commands.add_parser("report", help="read conservative review hints; delete nothing") + for command in (add, close): + command.add_argument("--session", required=True) + command.add_argument("--job", required=True) + add.add_argument("--kind", choices=KINDS, required=True) + add.add_argument("--resource", required=True) + add.add_argument("--purpose", choices=PURPOSES, default="unknown") + add.add_argument("--retention", choices=RETENTIONS, default="protected") + add.add_argument("--reason", default="") + close.add_argument("--outcome", choices=("success", "failure"), required=True) + args = parser.parse_args(argv) + try: + project = Path(args.project).resolve(strict=True) + with Ledger(project, args.command == "record") as ledger: + data = ledger.load() + if args.command == "record": + record(data, args, project) + ledger.save(data) + elif args.command == "closeout": + owner = key(args.session, args.job) + job = data["jobs"].get(owner) + if job is None: + raise Invalid("unknown session/job; cannot close another attribution") + if job["state"] not in ("active", args.outcome): + raise Invalid("closed job outcome cannot change") + if job["state"] == "active": + job["state"] = args.outcome + ledger.save(data) + output = report(data, project) + print(json.dumps(output, sort_keys=True)) + return 0 + except (OSError, ValueError, TypeError, RuntimeError) as error: + print(json.dumps({"error": str(error), "notice": "Resources preserved; inspect metadata manually."}), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/executing-plans/SKILL.md b/skills/executing-plans/SKILL.md index 11c17da..1501711 100644 --- a/skills/executing-plans/SKILL.md +++ b/skills/executing-plans/SKILL.md @@ -44,6 +44,14 @@ For each task: 3. Run verifications as specified 4. Mark as completed +Before disk-heavy builds/tests, follow `docs/resource-lifecycle.md`: preserve +inherited `GOCACHE`, reuse a compatible native/stable project/toolchain cache, +and justify any cold-cache allocation. Identify Docker mounts and protect database +data. Record session/job ownership and closeout on success or failure; active, +unknown, failed and reusable resources remain protected. The optional +`scripts/job-resources.py` records metadata and review candidates, never deletes. +Defer new heavy work when space is insufficient; preserve running jobs. + ### Step 3: Report When batch complete: - Show what was implemented diff --git a/skills/finishing-a-development-branch/SKILL.md b/skills/finishing-a-development-branch/SKILL.md index 120aef5..f2d2777 100644 --- a/skills/finishing-a-development-branch/SKILL.md +++ b/skills/finishing-a-development-branch/SKILL.md @@ -96,6 +96,14 @@ Stop. Don't proceed to Step 2. **If tests pass:** Continue to Step 1b. +Before branch/resource closeout, follow `docs/resource-lifecycle.md`: record +success/failure and exact cleanup review candidates with session/job ownership. +Preserve inherited `GOCACHE` and reusable caches for later jobs. Active, unknown, +failed resources, database data and worktrees stay protected. Optional +`scripts/job-resources.py` reports metadata hints only; candidates require fresh +live-use/data review and separate deletion authorization. Do not broad-prune or +clean another session's resources as part of finishing this branch. + ### Step 1b: Runtime Launch Validation (conditional) **Trigger:** the diff includes any of: diff --git a/skills/subagent-driven-development/implementer-prompt.md b/skills/subagent-driven-development/implementer-prompt.md index ca83015..53a4085 100644 --- a/skills/subagent-driven-development/implementer-prompt.md +++ b/skills/subagent-driven-development/implementer-prompt.md @@ -21,6 +21,14 @@ Task tool (general-purpose): Work from: [directory] + Resource discipline: follow `docs/resource-lifecycle.md`. Preserve inherited + GOCACHE and reuse compatible native/stable project/toolchain caches. Require + an isolation reason before a new cold cache; record its session/job owner. + Identify Docker mounts and protect database data. Record success/failure + closeout and exact cleanup review candidates, preserving active, unknown, + failed and reusable resources. Optional `scripts/job-resources.py` writes + metadata only. Never prune/delete resources or terminate another job. + If you have questions, ask them before starting. Don't guess or make assumptions. When complete, DM spec-reviewer that the task is ready for the diff --git a/tests/job-resources.py b/tests/job-resources.py new file mode 100644 index 0000000..6fd3fca --- /dev/null +++ b/tests/job-resources.py @@ -0,0 +1,529 @@ +#!/usr/bin/env python3 +"""Real CLI and filesystem regressions; all resources belong to test fixtures.""" +import contextlib +import errno +import importlib.util +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +HELPER = Path(__file__).resolve().parents[1] / "scripts/job-resources.py" + + +class Resources(unittest.TestCase): + def setUp(self): + self.assertTrue(HELPER.is_file(), "resource helper must exist") + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.project = Path(self.temp.name).resolve() / "project" + self.project.mkdir() + self.resource = self.project / ".autodev/tmp/job" + self.resource.mkdir(parents=True) + self.sentinel = self.resource / "sentinel" + self.sentinel.write_bytes(b"unique data\x00keep") + self.state = self.project / ".autodev/state/job-resources.json" + + def args(self, *args): + return ["--project", str(self.project), *map(str, args)] + + def cli(self, *args, error=False): + result = subprocess.run([sys.executable, str(HELPER), *self.args(*args)], + capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 2 if error else 0, result.stderr) + output = json.loads(result.stderr if error else result.stdout) + self.assertIn("error" if error else "resources", output) + self.assertEqual(self.sentinel.read_bytes(), b"unique data\x00keep") + return output + + def record(self, session="s", job="j", path=None, **options): + args = ["record", "--session", session, "--job", job, + "--kind", options.pop("kind", "directory"), + "--resource", str(path or self.resource)] + for key, value in options.items(): + args += ["--" + key, value] + return self.cli(*args) + + def disposable(self, **kwargs): + return self.record(purpose="test-fixture", retention="disposable", + reason="isolated cold-cache test", **kwargs) + + def close(self, session="s", job="j", outcome="success", error=False): + return self.cli("closeout", "--session", session, "--job", job, + "--outcome", outcome, error=error) + + def candidates(self, output=None): + return [r for r in (output or self.cli("report"))["resources"] + if r["status"] == "review-candidate"] + + def module(self): + spec = importlib.util.spec_from_file_location("job_resources", HELPER) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def complete(self, processes, timeout=15): + try: + return [(p.communicate(timeout=timeout), p.returncode) for p in processes] + finally: + for process in processes: + if process.poll() is None: + process.kill() # only owned fixture children; never independent jobs + process.communicate(timeout=5) + + def test_harness_timeout_reaps_every_owned_child(self): + processes = [subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + for _ in range(2)] + with self.assertRaises(subprocess.TimeoutExpired): + self.complete(processes, timeout=0.05) + self.assertTrue(all(p.poll() is not None for p in processes)) + + def in_process(self, module, *args): + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = module.main(self.args(*args)) + self.assertEqual(code, 2, stdout.getvalue()) + self.assertIn("error", json.loads(stderr.getvalue())) + + def test_empty_report_does_not_create_metadata(self): + self.cli("report") + self.assertFalse((self.project / ".autodev/state").exists()) + + def test_active_and_unknown_are_protected(self): + self.record() + self.assertFalse(self.candidates()) + self.close() + self.assertFalse(self.candidates()) + + def test_positive_candidate_in_normal_git_project(self): + (self.project / ".git").mkdir() + self.disposable() + self.assertFalse(self.candidates()) + output = self.close() + self.assertEqual(len(self.candidates(output)), 1) + self.close() # identical closeout is idempotent + + def test_failed_closeout_stays_protected(self): + self.disposable() + self.close(outcome="failure") + self.assertFalse(self.candidates()) + before = self.state.read_bytes() + self.close(error=True) + self.assertEqual(self.state.read_bytes(), before) + + def test_cross_session_and_closed_job_rejected(self): + self.disposable() + self.close(session="other", error=True) + self.close() + before = self.state.read_bytes() + self.cli("record", "--session", "s", "--job", "j", "--kind", "directory", + "--resource", self.resource, error=True) + self.assertEqual(self.state.read_bytes(), before) + + def test_shared_resource_requires_every_owner_closeout(self): + self.disposable() + self.disposable(session="other") + self.close() + self.assertFalse(self.candidates()) + self.close(session="other") + self.assertEqual(len(self.candidates()), 1) + + def test_retention_and_purpose_cannot_be_downgraded(self): + self.record(purpose="database") + self.disposable(session="other") + self.close() + self.close(session="other") + self.assertFalse(self.candidates()) + + def test_external_reusable_cache_unchanged(self): + external = Path(self.temp.name) / "stable-cache" + external.mkdir() + sentinel = external / "data" + sentinel.write_bytes(b"warm-cache") + self.record(path=external, purpose="build-cache", retention="reusable") + self.close() + self.assertFalse(self.candidates()) + self.assertEqual(sentinel.read_bytes(), b"warm-cache") + + def test_invalid_disposable_declarations(self): + for kind, path, purpose in [("docker-volume", "db-volume", "test-fixture"), + ("worktree", self.resource, "test-fixture"), + ("directory", Path(self.temp.name), "test-fixture"), + ("directory", self.resource, "database")]: + with self.subTest(kind=kind, path=path, purpose=purpose): + self.cli("record", "--session", "s", "--job", kind, + "--kind", kind, "--resource", path, "--purpose", purpose, + "--retention", "disposable", "--reason", "test", error=True) + + def test_docker_and_worktree_remain_protected(self): + self.record(kind="docker-volume", path="test-volume") + self.record(job="w", kind="worktree") + self.close() + self.close(job="w") + self.assertFalse(self.candidates()) + + def test_reason_required_and_missing_directory_protected(self): + self.cli("record", "--session", "s", "--job", "j", "--kind", "directory", + "--resource", self.resource, "--purpose", "test-fixture", + "--retention", "disposable", error=True) + self.disposable() + self.close() + self.sentinel.unlink() + self.resource.rmdir() + self.sentinel = self.project / "preserved" + self.sentinel.write_bytes(b"unique data\x00keep") + self.assertFalse(self.candidates()) + + def test_symlink_and_traversal_rejected(self): + link = self.project / ".autodev/tmp/link" + link.symlink_to(Path(self.temp.name), target_is_directory=True) + for path in [link, self.resource / "../../../outside"]: + self.cli("record", "--session", "s", "--job", "j", "--kind", "directory", + "--resource", path, "--purpose", "test-fixture", + "--retention", "disposable", "--reason", "test", error=True) + + def test_report_revalidates_symlink_descendant(self): + self.disposable() + self.close() + (self.resource / "escape").symlink_to(Path(self.temp.name)) + self.assertFalse(self.candidates()) + + def test_nested_and_ancestor_worktrees(self): + self.disposable() + self.close() + nested = self.resource / "nested" + nested.mkdir() + (nested / ".git").write_text("gitdir: unrelated\n") + self.assertFalse(self.candidates()) + (nested / ".git").unlink() + (self.resource.parent / ".git").mkdir() + self.assertFalse(self.candidates()) + + def test_protected_overlap_both_directions(self): + self.disposable() + child = self.resource / "cache" + child.mkdir() + self.record(session="other", path=child, purpose="build-cache", retention="reusable") + self.close() + self.close(session="other") + self.assertFalse(self.candidates()) + self.record(session="parent", path=self.resource.parent, purpose="database") + self.close(session="parent") + self.assertFalse(self.candidates()) + + def test_unknown_owner_is_protected(self): + self.disposable() + self.close() + data = json.loads(self.state.read_text()) + next(iter(data["resources"].values()))["owners"].append("missing-owner") + self.state.write_text(json.dumps(data)) + self.assertFalse(self.candidates()) + + def test_protected_parent_suppresses_disposable_child(self): + self.record(session="parent", path=self.resource.parent, purpose="database") + self.disposable() + self.close() + self.close(session="parent") + self.assertFalse(self.candidates()) + + def test_fifo_metadata_rejected_without_hanging(self): + self.record() + self.state.unlink() + os.mkfifo(self.state) + self.cli("report", error=True) + + def test_changed_protected_path_preserves_candidates(self): + self.disposable() + self.close() + protected = self.project / "cache" + protected.mkdir() + self.record(session="cache", path=protected, purpose="build-cache", retention="reusable") + protected.rmdir() + protected.symlink_to(self.resource, target_is_directory=True) + self.assertFalse(self.candidates()) + + def test_corrupt_future_and_wrong_project_preserved(self): + self.record() + original = json.loads(self.state.read_text()) + future = dict(original, schema=999) + wrong = dict(original, project="/unknown") + duplicate = json.dumps(original).replace('"schema": 1', '"schema": 999, "schema": 1') + for text in ["{broken", json.dumps(future), json.dumps(wrong), "[]", duplicate]: + self.state.write_text(text) + self.cli("report", error=True) + self.assertEqual(self.state.read_text(), text) + + def test_duplicate_fields_preserved_for_all_commands(self): + self.record() + duplicate = self.state.read_text().replace('"owners": [', '"owners": ["unknown"], "owners": [') + for args in [("report",), ("record", "--session", "other", "--job", "j", "--kind", "directory", + "--resource", str(self.resource)), + ("closeout", "--session", "s", "--job", "j", "--outcome", "success")]: + self.state.write_text(duplicate) + self.cli(*args, error=True) + self.assertEqual(self.state.read_text(), duplicate) + + def test_missing_existing_lock_is_not_recreated(self): + self.record() + before = self.state.read_bytes() + self.state.with_suffix(".lock").unlink() + self.cli("record", "--session", "other", "--job", "j", "--kind", "directory", + "--resource", self.resource, error=True) + self.assertFalse(self.state.with_suffix(".lock").exists()) + self.assertEqual(self.state.read_bytes(), before) + + def test_candidate_scan_swaps_and_changes_are_protected(self): + for mode in ["candidate", "ancestor", "descendant", "new-entry"]: + with self.subTest(mode=mode): + # Each subcase uses its own declared resource, avoiding restored paths. + resource = self.resource / mode + resource.mkdir() + sentinel = resource / "unique" + sentinel.write_bytes(b"original") + nested = resource / "nested" + nested.mkdir() + self.disposable(job=mode, path=resource) + self.close(job=mode) + module = self.module() + original = module.os.scandir + target = nested if mode == "descendant" else resource + target_inode = target.stat().st_ino + triggered = [False] + + def barrier(fd): + if not triggered[0] and os.fstat(fd).st_ino == target_inode: + triggered[0] = True + if mode == "new-entry": + (resource / ".git").mkdir() + else: + moved = target if mode != "ancestor" else resource.parent + saved = moved.with_name(moved.name + "-saved") + moved.rename(saved) + moved.mkdir() + (moved / ".git").mkdir() + return original(fd) + + stdout = io.StringIO() + with mock.patch.object(module.os, "scandir", barrier), contextlib.redirect_stdout(stdout): + self.assertEqual(module.main(self.args("report")), 0) + self.assertTrue(triggered[0]) + row = next(r for r in json.loads(stdout.getvalue())["resources"] if r["resource"] == str(resource)) + self.assertEqual(row["status"], "protected") + if mode == "ancestor": + self.assertEqual((resource.parent.with_name(resource.parent.name + "-saved") / + mode / "unique").read_bytes(), b"original") + (resource.parent / ".git").rmdir() + resource.parent.rmdir() + resource.parent.with_name(resource.parent.name + "-saved").rename(resource.parent) + elif mode == "candidate": + self.assertEqual((resource.with_name(resource.name + "-saved") / "unique").read_bytes(), b"original") + else: + self.assertEqual(sentinel.read_bytes(), b"original") + + def test_metadata_symlinks_rejected_without_external_writes(self): + self.record() + external = Path(self.temp.name) / "external" + external.mkdir() + for name in ["job-resources.json", "job-resources.lock"]: + target = self.state.parent / name + backup = target.with_suffix(".saved") + target.rename(backup) + target.symlink_to(external / "unknown") + self.cli("report", error=True) + self.assertEqual(list(external.iterdir()), []) + target.unlink() + backup.rename(target) + + def test_project_swap_during_scan_is_protected(self): + self.disposable() + self.close() + module = self.module() + original = module.os.scandir + inode = self.resource.stat().st_ino + triggered = [False] + saved = self.project.with_name("original-project") + + def barrier(fd): + if not triggered[0] and os.fstat(fd).st_ino == inode: + triggered[0] = True + self.project.rename(saved) + self.resource.mkdir(parents=True) + (self.resource / ".git").mkdir() + (self.resource / "replacement-data").write_bytes(b"protected replacement") + return original(fd) + + stdout = io.StringIO() + with mock.patch.object(module.os, "scandir", barrier), contextlib.redirect_stdout(stdout): + self.assertEqual(module.main(self.args("report")), 0) + self.assertTrue(triggered[0]) + self.assertFalse(self.candidates(json.loads(stdout.getvalue()))) + self.assertEqual((saved / ".autodev/tmp/job/sentinel").read_bytes(), b"unique data\x00keep") + self.assertEqual((self.resource / "replacement-data").read_bytes(), b"protected replacement") + + def test_new_entry_after_enumeration_is_protected(self): + self.disposable() + self.close() + module = self.module() + original = module.os.scandir + inode = self.resource.stat().st_ino + + @contextlib.contextmanager + def barrier(fd): + with original(fd) as entries: + yield entries + if os.fstat(fd).st_ino == inode: + (self.resource / ".git").mkdir() + + stdout = io.StringIO() + with mock.patch.object(module.os, "scandir", barrier), contextlib.redirect_stdout(stdout): + self.assertEqual(module.main(self.args("report")), 0) + self.assertFalse(self.candidates(json.loads(stdout.getvalue()))) + self.assertEqual(self.sentinel.read_bytes(), b"unique data\x00keep") + + def test_metadata_directory_symlink_rejected(self): + self.record() + directory = self.state.parent + directory.rename(directory.with_name("saved")) + external = Path(self.temp.name) / "outside" + external.mkdir() + directory.symlink_to(external, target_is_directory=True) + self.cli("report", error=True) + self.assertEqual(list(external.iterdir()), []) + + def test_failed_atomic_write_preserves_ledger(self): + self.record() + before = self.state.read_bytes() + module = self.module() + with mock.patch.object(module.os, "replace", side_effect=OSError(errno.ENOSPC, "full")): + self.in_process(module, "closeout", "--session", "s", "--job", "j", "--outcome", "success") + self.assertEqual(self.state.read_bytes(), before) + self.assertEqual(sorted(p.name for p in self.state.parent.iterdir()), + ["job-resources.json", "job-resources.lock"]) + + def test_lock_failure_preserves_ledger(self): + self.record() + before = self.state.read_bytes() + module = self.module() + with mock.patch.object(module.fcntl, "flock", side_effect=OSError(errno.EIO, "lock")): + self.in_process(module, "closeout", "--session", "s", "--job", "j", "--outcome", "success") + self.assertEqual(self.state.read_bytes(), before) + + def test_lock_timeout_and_unsupported_platform_preserve_state(self): + self.record() + before = self.state.read_bytes() + module = self.module() + with mock.patch.object(module, "LOCK_TIMEOUT", 0), mock.patch.object( + module.fcntl, "flock", side_effect=BlockingIOError(errno.EAGAIN, "busy")): + self.in_process(module, "closeout", "--session", "s", "--job", "j", "--outcome", "success") + with mock.patch.object(module, "fcntl", None): + self.in_process(module, "closeout", "--session", "s", "--job", "j", "--outcome", "success") + self.assertEqual(self.state.read_bytes(), before) + + def test_unreadable_tree_is_protected(self): + self.disposable() + self.close() + module = self.module() + with mock.patch.object(module.os, "scandir", side_effect=PermissionError("unreadable")): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.assertEqual(module.main(self.args("report")), 0) + self.assertFalse(self.candidates(json.loads(stdout.getvalue()))) + + def test_real_project_path_alias_is_supported(self): + alias = Path(self.temp.name) / "project-alias" + alias.symlink_to(self.project, target_is_directory=True) + result = subprocess.run([sys.executable, str(HELPER), "--project", str(alias), + "record", "--session", "s", "--job", "j", "--kind", "directory", + "--resource", str(alias / ".autodev/tmp/job"), "--purpose", "test-fixture", + "--retention", "disposable", "--reason", "test"], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(self.candidates(self.close())), 1) + + def test_directory_and_lock_swap_detected_before_write(self): + for swap in ["directory", "lock"]: + with self.subTest(swap=swap): + self.record(session=swap) + before = self.state.read_bytes() + module = self.module() + original = module.Ledger.validate + calls = [0] + external = Path(self.temp.name) / ("outside-" + swap) + external.mkdir() + + def barrier(ledger): + calls[0] += 1 + if calls[0] == 3: + if swap == "directory": + self.state.parent.rename(self.state.parent.with_name("saved")) + self.state.parent.symlink_to(external, target_is_directory=True) + else: + lock = self.state.with_suffix(".lock") + lock.rename(lock.with_suffix(".saved")) + lock.touch() + return original(ledger) + + with mock.patch.object(module.Ledger, "validate", barrier): + self.in_process(module, "closeout", "--session", swap, "--job", "j", "--outcome", "success") + self.assertEqual(list(external.iterdir()), []) + if swap == "directory": + self.state.parent.unlink() + self.state.parent.with_name("saved").rename(self.state.parent) + else: + self.state.with_suffix(".lock").unlink() + self.state.with_suffix(".saved").rename(self.state.with_suffix(".lock")) + self.assertEqual(self.state.read_bytes(), before) + + def test_budget_exhaustion_is_protected(self): + self.disposable() + self.close() + module = self.module() + with mock.patch.object(module, "MAX_ENTRIES", 0): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.assertEqual(module.main(self.args("report")), 0) + self.assertFalse(self.candidates(json.loads(stdout.getvalue()))) + + def test_parallel_writers_preserve_owners_and_stable_lock(self): + self.disposable() + inode = self.state.with_suffix(".lock").stat().st_ino + processes = [] + for i in range(8): + processes.append(subprocess.Popen([sys.executable, str(HELPER), *self.args( + "record", "--session", "parallel-" + str(i), "--job", "job", + "--kind", "directory", "--resource", self.resource, + "--purpose", "test-fixture", "--retention", "disposable", "--reason", "test")], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)) + completed = self.complete(processes) + for (stdout, stderr), code in completed: + self.assertEqual(code, 0, stderr) + self.assertIn("resources", json.loads(stdout)) + data = json.loads(self.state.read_text()) + self.assertEqual(len(data["jobs"]), 9) + self.assertEqual(len(next(iter(data["resources"].values()))["owners"]), 9) + self.assertEqual(self.state.with_suffix(".lock").stat().st_ino, inode) + self.assertFalse(self.candidates()) + + def test_parallel_distinct_resources_preserved(self): + processes = [] + for i in range(4): + path = self.resource / str(i) + path.mkdir() + processes.append(subprocess.Popen([sys.executable, str(HELPER), *self.args( + "record", "--session", str(i), "--job", "job", "--kind", "directory", + "--resource", path)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)) + completed = self.complete(processes) + for (_, stderr), code in completed: + self.assertEqual(code, 0, stderr) + data = json.loads(self.state.read_text()) + self.assertEqual(len(data["jobs"]), 4) + self.assertEqual(len(data["resources"]), 4) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/resource-lifecycle-docs.sh b/tests/resource-lifecycle-docs.sh new file mode 100644 index 0000000..3add82e --- /dev/null +++ b/tests/resource-lifecycle-docs.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Static guidance contracts; these do not prove agent/host runtime compliance. +set -euo pipefail +cd "$(dirname "$0")/.." +python3 - <<'PY' +from pathlib import Path +entries = ["skills/executing-plans/SKILL.md", + "skills/subagent-driven-development/implementer-prompt.md", + "skills/finishing-a-development-branch/SKILL.md", "docs/README.codex.md"] +for entry in entries: + content = Path(entry).read_text() + for required in ["resource-lifecycle", "GOCACHE", "database", "active", + "unknown", "failed", "reusable", "job-resources.py"]: + assert required.lower() in content.lower(), (entry, required) +content = Path("docs/resource-lifecycle.md").read_text() +for required in ["go env GOCACHE", "go env -w", "cold cache", "anonymous volumes", + "--outcome failure", "attribution, not authentication", "review-candidate", + "never deletion permission", "no lifecycle hook", "10,000", "1 MiB"]: + assert required in content, required +print("PASS: resource lifecycle guidance contracts (static only)") +PY