diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e941e8..1028497 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,12 +67,12 @@ jobs: [ "$fail" -eq 0 ] && echo "OK: all examples stamped at $V" exit $fail - external-packs: - name: external packs (materialize + track) + packs: + name: packs (declare + mirror + track) runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - name: Remote pack lands in the tracked dir, and a bump is a readable diff + - name: A declared pack mirrors into the repo, and a bump is a readable diff shell: bash run: | set -euo pipefail @@ -98,16 +98,19 @@ jobs: project: name: ext-test sync_version: "$V" - external: - dir: "intelligence/external" + packs: + shared-intel: + url: file://$PACK + ref: v1.0.0 + mirror: "intelligence/external/shared-intel" sources: rules: - "intelligence/rules" - - "git+file://$PACK@v1.0.0#rules" + - "@shared-intel/rules" agents: - "intelligence/agents" skills: - - "git+file://$PACK@v1.0.0#skills" + - "@shared-intel/skills" targets: agents: { enabled: true, output: "AGENTS.md" } claude: { enabled: true, output: ".claude" } @@ -145,13 +148,36 @@ jobs: git -C "$PACK" add -A git -C "$PACK" -c user.email=ci@example.com -c user.name=ci commit -qm v11 git -C "$PACK" tag v1.1.0 - sed -i 's/@v1.0.0/@v1.1.0/g' intelligence/config.yaml + # The pin lives in ONE place — the pack declaration, not each source. + sed -i 's/ref: v1.0.0/ref: v1.1.0/' intelligence/config.yaml bash intelligence/sync/scripts/sync.sh >/dev/null git diff --stat -- intelligence/external git diff -- intelligence/external | grep -q '^+Be careful AND explicit\.$' + git diff -- intelligence/external | grep -q '^+ref=v1.1.0$' test ! -e intelligence/external/shared-intel/skills/shared-do-thing test -f intelligence/external/shared-intel/skills/shared-ship-it/SKILL.md + # A MOVED upstream (same pack, new url) refreshes the mirror. The + # mirror is declared per pack, so a stamped directory stays this + # pack's even after its url changes — freezing it at the old content + # while the generated output follows the new repo would break exactly + # the reviewability the mirror exists for, and do it silently. + git add -A && git -c user.email=ci@example.com -c user.name=ci commit -qm bumped + MOVED=$RUNNER_TEMP/moved-intel + git clone -q "$PACK" "$MOVED" + printf -- '---\nname: shared-baseline\n---\n\nFrom the moved repo.\n' > "$MOVED/rules/shared-baseline.md" + git -C "$MOVED" add -A + git -C "$MOVED" -c user.email=ci@example.com -c user.name=ci commit -qm moved + # Same pin, new home — the clone carries the tags, so move v1.1.0 onto + # the moved commit rather than adding a ref the config does not use. + git -C "$MOVED" tag -f v1.1.0 >/dev/null + sed -i "s|url: file://$PACK\$|url: file://$MOVED|" intelligence/config.yaml + out=$(bash intelligence/sync/scripts/sync.sh 2>&1) + case "$out" in *"not a pack's"*) echo "$out"; echo "a moved upstream must refresh, not freeze"; exit 1 ;; esac + grep -q "^url=file://$MOVED\$" intelligence/external/shared-intel/.pack + grep -q '^From the moved repo\.$' intelligence/external/shared-intel/rules/shared-baseline.md + git checkout -- intelligence/config.yaml + # A directory that is not ours is never cleared. rm -rf intelligence/external mkdir -p intelligence/external/shared-intel @@ -159,12 +185,29 @@ jobs: bash intelligence/sync/scripts/sync.sh >/dev/null test -f intelligence/external/shared-intel/PRECIOUS.md - # external.dir inside a configured source is refused, not obeyed. - sed -i 's|dir: "intelligence/external"|dir: "intelligence/rules/packs"|' intelligence/config.yaml + # A mirror inside a configured source is refused, not obeyed. + git checkout -- intelligence/config.yaml + sed -i 's|mirror: "intelligence/external/shared-intel"|mirror: "intelligence/rules/packs"|' intelligence/config.yaml rc=0; bash intelligence/sync/scripts/sync.sh >/dev/null 2>&1 || rc=$? - [ "$rc" -eq 1 ] || { echo "expected exit 1 for external.dir inside a source, got $rc"; exit 1; } + [ "$rc" -eq 1 ] || { echo "expected exit 1 for a mirror inside a source, got $rc"; exit 1; } + + # An undeclared pack reference FAILS the run — it must never degrade + # into a silently skipped source the way a missing local path does. + git checkout -- intelligence/config.yaml + sed -i 's|"@shared-intel/rules"|"@typo-pack/rules"|' intelligence/config.yaml + rc=0; err=$(bash intelligence/sync/scripts/sync.sh 2>&1 >/dev/null) || rc=$? + [ "$rc" -eq 1 ] || { echo "expected exit 1 for an undeclared pack, got $rc"; exit 1; } + case "$err" in *"typo-pack"*) ;; *) echo "$err"; echo "error must name the pack"; exit 1 ;; esac + + # A pack with no mirror keeps working and leaves NOTHING in the repo. + git checkout -- intelligence/config.yaml + sed -i '/mirror: "intelligence\/external\/shared-intel"/d' intelligence/config.yaml + rm -rf intelligence/external + bash intelligence/sync/scripts/sync.sh >/dev/null + test ! -e intelligence/external + test -f .claude/rules/shared-baseline.md - echo "OK: external packs materialize, track, refresh and refuse unsafe dirs" + echo "OK: packs declare once, mirror, refresh, and fail closed on a bad reference" smoke: name: smoke (sync against examples) @@ -539,6 +582,77 @@ jobs: s=$(bash intelligence/sync/scripts/sync.sh); echo "$s" | grep -q 'IS_STATUS=ok' test -f AGENTS.md + - name: 0.10.0 converts inline git+ sources into declared packs + shell: bash + run: | + set -euo pipefail + PROJ=$RUNNER_TEMP/packmig + mkdir -p "$PROJ/intelligence"/{rules,agents,skills} + cp -r intelligence/sync "$PROJ/intelligence/sync" + printf '# Always-on context\n' > "$PROJ/intelligence/rules/context.md" + # A 0.9.0-shaped config: one global external dir, the pack identity + # duplicated per section, and one UNPINNED spec — the case where ref + # is absent but a subpath is present. + cat > "$PROJ/intelligence/config.yaml" <<'EOF' + project: + name: packmig + sync_version: "0.9.0" + external: + dir: "intelligence/external" + sources: + rules: + - "intelligence/rules" + - "git+https://example.invalid/org/shared.git@v1.2.0#rules" + agents: + - "intelligence/agents" + skills: + - "intelligence/skills" + - 'git+https://example.invalid/org/shared.git@v1.2.0#skills' + - "git+https://example.invalid/org/other.git#skills" + targets: + agents: + enabled: true + output: "AGENTS.md" + header: | + # Packmig + EOF + git -C "$PROJ" init -q + git -C "$PROJ" add -A + git -C "$PROJ" -c user.email=ci@example.com -c user.name=ci commit -q -m init + + cd "$PROJ" + out=$(REPO_URL="file://$GITHUB_WORKSPACE" bash intelligence/sync/scripts/update.sh --yes) + echo "$out" + case "$out" in *"IS_STATUS=migrated"*) ;; *) echo "expected IS_STATUS=migrated"; exit 1 ;; esac + + # Post-conditions of the breaking change. + ! grep -q '^external:' intelligence/config.yaml + grep -q '^packs:' intelligence/config.yaml + ! grep -qE '^[[:space:]]*-[[:space:]]*["'\'']?git\+' intelligence/config.yaml + # One declaration per distinct repo, each carrying its own pin. + test "$(grep -c '^ url: ' intelligence/config.yaml)" -eq 2 + test "$(grep -c '^ ref: v1.2.0$' intelligence/config.yaml)" -eq 1 + # external.dir became a per-pack mirror, so vendored content keeps + # landing where it already is. + grep -q '^ mirror: "intelligence/external/shared"$' intelligence/config.yaml + # An unpinned spec keeps its subpath and gains NO ref — the field is + # optional, and a subpath read as a ref would clone a missing branch. + grep -q '"@other/skills"' intelligence/config.yaml + test "$(grep -c '^ ref: ' intelligence/config.yaml)" -eq 1 + # Both sections reference the same pack by name — the pin has one home. + test "$(grep -c '"@shared/' intelligence/config.yaml)" -eq 2 + + # Idempotent: the whole chain re-runs without touching the config. + cp intelligence/config.yaml "$RUNNER_TEMP/packmig.yaml" + out2=$(REPO_URL="file://$GITHUB_WORKSPACE" bash intelligence/sync/scripts/update.sh --yes) + case "$out2" in *"IS_STATUS=ok"*) ;; *) echo "expected IS_STATUS=ok on re-run"; exit 1 ;; esac + diff -u "$RUNNER_TEMP/packmig.yaml" intelligence/config.yaml + + # An unreachable pack degrades to a warning, exactly as an unreachable + # inline spec did — the local sources still sync. + s=$(bash intelligence/sync/scripts/sync.sh); echo "$s" | grep -q 'IS_STATUS=ok' + test -f AGENTS.md + - name: Multi-version chain walks the whole registry forward, idempotent shell: bash run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c53962..a677296 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,24 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and Update intelligence-sync: fetch the latest engine from https://github.com/ainova-systems/intelligence-sync and run its update flow to migrate this project to the newest version. Leave my rules, agents, and project skills untouched. If it fails, read the CHANGELOG "### Breaking" entries between my version and the latest, base your fix plan on them, make sure you are running the latest scripts, and retry; ask me only if it still fails. ``` +## [0.10.0] — 2026-07-28 + +A remote source is now declared once as a **pack** and referenced by name, so its url and its pin live in exactly one place. + +### Breaking + +- **`external: { dir: … }` is replaced by `packs:`; `migrate_to_0_10_0` rewrites the config.** 0.9.0 put the mirror location in one global block while the identity of each pack — url and ref — stayed duplicated inside every `sources.*` entry that used it. A pack spanning rules, agents and skills therefore carried its `url@ref` three times, and nothing detected the drift when only two of them were bumped: rules pinned at one commit and skills at another is a config that looks fine and reads wrong. `packs:` inverts it. Each pack is declared once (`url`, optional `ref`, optional `mirror`) and referenced as `@[/]`, so the pin has a single home and the mirror is a per-pack path rather than one global directory. The migration is automatic and comment-preserving: every inline `git+` spec becomes a declared pack plus an `@name` reference, `external.dir` becomes each pack's `mirror:` so vendored content keeps landing where it already is, and the `external:` block is dropped. **Post-condition: no `external:` key in `config.yaml`, and a `packs:` block declaring every remote previously reached inline.** Idempotent — a config with no `external:` key and no `git+` token is left untouched. + +### Added + +- **`packs:` — declared remote sources, referenced by name.** `packs..{url,ref,mirror}` sits at three levels, exactly like `targets:`, so it is read by the existing `get_nested_yaml_value` and adds no parser. `mirror:` is both the location and the switch: present, the pack is materialized there and committed; absent, it stays transient in the run cache, which is 0.9.0's default behaviour and needs no flag to express. Because the mirror is *declared*, the whole derive-a-name machinery 0.9.0 needed is gone — no basename extraction, no charset sanitizing, no `pack-` fallback, no collision suffix — and a pack name is now purely a reference handle that never becomes a path component. **An undeclared `@pack` reference fails the run** (exit 1, naming the pack and listing the declared ones), deliberately unlike a missing local path, which only warns: the config claims to know that name, so a typo must not silently drop a whole rule set. That check runs up front, in `validate_pack_refs`, because `resolve_source_dir` is always called inside `$( )` — an error raised there would exit the substitution subshell, not the sync. Inline `git+[@][#]` specs keep working as *anonymous* packs: no name, no mirror, always transient. CI job `packs` covers declaration, mirroring, idempotency, the refresh diff, the never-clear guard, the unsafe-mirror refusal, the undeclared-pack failure and the transient path, against a `file://` pack repo. + +### Fixed + +- **`get_nested_yaml_value` cut values at the last colon, not the first.** The value strip was a greedy `.*:[[:space:]]*`, which on `url: https://host/repo.git` matched through `https:` and yielded `//host/repo.git`. It is now anchored with `[^:]*:`, and an unquoted value additionally drops a trailing ` # comment` per YAML while a `#` inside quotes stays content. This was latent in 0.9.0 — nothing read a URL through this helper — but it also silently truncated any `models..` value containing a colon. +- **A pack could lose every subpath but the last when the run cache was unset.** `materialize_pack` recorded its "already cleared this run" claim only when `IS_REMOTE_CACHE` was exported, yet cleared the directory unconditionally. Since `resolve_source_dir` runs in a command substitution — a fresh subshell each call — a caller that had not exported the cache re-cleared the mirror on every entry, so a pack referenced for rules, agents and skills kept only the last. The claim now uses the same cache-root fallback the clone does. +- **Empty-array expansion under `set -u`.** The new migration iterates with `${#arr[@]}` bounds rather than `"${!arr[@]}"`, which bash 3.2 — the macOS default, and a supported target — treats as unbound when the array is empty. + ## [0.9.0] — 2026-07-28 Remote packs can now be materialized into a tracked directory, so an upstream bump is reviewable instead of invisible. diff --git a/README.md b/README.md index 5b4b3a6..78e2b83 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ Zero dependencies. Just bash + awk. Linux, macOS, Windows (Git Bash / WSL). ## Ready-made packs -[intelligence-dev-packs](https://github.com/ainova-systems/intelligence-dev-packs) is a set of shared AI-first engineering packs - git/PR/review discipline plus an optional spec-driven development lifecycle - that you consume straight through this engine as remote `git+` sources, instead of authoring every rule and skill from scratch. Extracted from production AI-coded systems. +[intelligence-dev-packs](https://github.com/ainova-systems/intelligence-dev-packs) is a set of shared AI-first engineering packs - git/PR/review discipline plus an optional spec-driven development lifecycle - that you consume straight through this engine as declared `packs:`, instead of authoring every rule and skill from scratch. Extracted from production AI-coded systems. ## How It Works @@ -152,7 +152,7 @@ Each `intelligence//` (e.g. `sync/`, future `domain/`) is self-contained - [go-api-with-opencode](examples/go-api-with-opencode/) -- Go API with opencode (subagents in `.opencode/agents/`, skills via `.agents/skills/`) - [dotnet-api-with-react-frontend](examples/dotnet-api-with-react-frontend/) -- .NET backend + React frontend - [platform-with-submodules](examples/platform-with-submodules/) -- Multi-component platform with git submodules -- [with-remote-skills](examples/with-remote-skills/) -- Pulls shared rules/skills from a remote git repo (`git+` sources), materialized into a tracked `external:` dir +- [with-remote-skills](examples/with-remote-skills/) -- Pulls shared rules/skills from a remote git repo declared under `packs:`, mirrored into a tracked directory ## Documentation diff --git a/docs/ADAPTERS.md b/docs/ADAPTERS.md index 0f29484..fa6cf90 100644 --- a/docs/ADAPTERS.md +++ b/docs/ADAPTERS.md @@ -61,8 +61,10 @@ Source `lib/common.sh` for these utilities: | `map_access_to_claude_tools(access)` | Tool string for access level | | `map_access_to_claude_disallowed(access)` | Disallowed tools string | | `read_yaml_list(config, section)` | Read list from `config.yaml` | -| `resolve_source_dir(repo_root, src)` | Map a source entry to a local dir — `"$repo_root/$src"`, or a shallow clone for a remote `git+` spec | -| `source_is_remote(src)` | True (0) if a source entry is a remote `git+` spec | +| `resolve_source_dir(repo_root, src)` | Map a source entry to a local dir — `"$repo_root/$src"` for a path, or a shallow clone for a pack reference (`@[/]`) or an inline `git+` spec | +| `source_is_local_path(src)` | True (0) if a source entry is a plain repo-relative path — i.e. neither of the two below. Use this, not a negated `source_is_remote`, whenever a token is about to be pattern-matched against a real directory | +| `source_is_pack(src)` | True (0) if a source entry references a pack declared under `packs:` (`@`) | +| `source_is_remote(src)` | True (0) if a source entry is an inline remote `git+` spec | | `get_target_field(config, target, field)` | Read a field from a target's config block | ### Transformation Patterns @@ -140,7 +142,10 @@ sync_to_myide() { while IFS= read -r src; do [ -z "$src" ] && continue # resolve_source_dir maps a source entry to a local dir: "$repo_root/$src" - # for a local path, or a shallow clone for a remote `git+` spec. + # for a local path, or a shallow clone for a pack reference (`@`) + # or an inline `git+` spec. A pack's url/ref/mirror are read from + # config.yaml, which it takes from $IS_CONFIG_FILE (exported by sync.sh) + # unless you pass the config as a third argument. local dir dir="$(resolve_source_dir "$repo_root" "$src")" [ -d "$dir" ] || continue diff --git a/docs/CONVENTIONS.md b/docs/CONVENTIONS.md index eb1545e..9bcc127 100644 --- a/docs/CONVENTIONS.md +++ b/docs/CONVENTIONS.md @@ -64,58 +64,84 @@ A custom adapter belongs in the umbrella's `adapters/`, never in the module's `s Rule filenames, agent names, and skill names all share the same **domain prefix** (`backend-`, `frontend-`, `devops-`, `core-`, `tests-`, project codename, or monorepo component name). Pick the domain once from repo structure and reuse it — do not invent new domains without clear need. -### Remote sources (git) +### Packs (remote sources) -A `sources.{rules,agents,skills}` entry is normally a **local path** relative to the repo root. It may instead be a **remote git spec**, which `sync` shallow-clones on the fly and treats exactly like a local source directory — so a team can keep shared intelligence in one repo and pull it into many projects: +A `sources.{rules,agents,skills}` entry is normally a **local path** relative to the repo root. It may instead reference a **pack** — a remote git repo that `sync` shallow-clones and treats exactly like a local source directory — so a team can keep shared intelligence in one repo and pull it into many projects. + +A pack is **declared once** under `packs:` and referenced by name from as many sections as need it: ```yaml +packs: + shared-intel: + url: https://github.com/org/shared-intel.git + ref: v1.2.0 # the pin — one place, not one per section + mirror: "intelligence/external/shared-intel" # optional; see below + sources: - skills: - - "intelligence/skills" # local - - "git+https://github.com/org/shared-intel.git@v1.2.0#skills" # remote, pinned tag rules: - - "intelligence/rules" - - "git+ssh://git@github.com/org/shared-intel.git@main#rules" # remote, branch + - "intelligence/rules" # local + - "@shared-intel/rules" # pack + skills: + - "@shared-intel/skills" # same pack, same clone, same pin ``` -Spec format: `git+[@][#]` +Declaring the pack is what keeps the url and the ref in one place. Repeating them per section is how the two drift, and nothing catches it: rules pinned at one commit and skills at another is a config that looks fine and reads wrong. + +Reference format: `@[/]` + +- `` — a key under `packs:`. It is a **reference handle, never a path component**, so it needs no sanitizing; it must simply contain no `/`. +- `/` — optional directory inside the pack repo holding the rules / agents / skills. Omit to use the repo root. The subpath is preserved in the mirror, so `@shared-intel/packs/core/rules` lands at `/packs/core/rules`. +- **An undeclared pack fails the run** (exit 1, naming the pack and listing the declared ones). This is deliberately unlike a missing local path, which only warns: the config claims to know that name, so a typo must not quietly drop a whole rule set. + +Pack fields: + +- `url:` — must carry an explicit scheme: `https://`, `http://`, `ssh://`, `git://`, or `file://`. Other transports (notably the command-executing `ext::` / `fd::`) are **rejected** with a warning and skipped. +- `ref:` — optional tag, branch, or commit SHA. Omit for the default branch. +- `mirror:` — optional; see [Mirroring a pack into the repo](#mirroring-a-pack-into-the-repo). + +#### Inline specs (`git+…`) + +A source entry may still carry the whole spec inline: `git+[@][#]`. This is an **anonymous pack** — it has no declared name and no mirror, so it is always transient and cannot be referenced from elsewhere. Declare the pack under `packs:` to pin it once or to commit it. -- `` — must carry an explicit scheme: `https://`, `http://`, `ssh://`, `git://`, or `file://`. Other transports (notably the command-executing `ext::` / `fd::`) are **rejected** with a warning and skipped. -- `@` — optional tag, branch, or commit SHA. Parsed as the segment after the last `@`, accepted as a ref only when it contains no `/` (so `ssh://git@host/...` userinfo is not mistaken for a ref). Branch names containing `/` (e.g. `feature/x`) can't be expressed this way — use a tag, SHA, or slashless branch (pinning is recommended regardless). -- `#` — optional directory inside the cloned repo holding the rules / agents / skills. Omit to use the repo root. +The inline `@` is parsed as the segment after the last `@`, accepted as a ref only when it contains no `/` (so `ssh://git@host/...` userinfo is not mistaken for a ref). Branch names containing `/` (e.g. `feature/x`) can't be expressed this way — use `packs:` with a plain `ref:`, which has no such limit. Behavior and trust: -- **Fresh every sync.** Each `sync` run clones into a run-scoped temp dir and removes it on exit, so branch refs always pick up the latest. Within one run the same `repo@ref` is cloned only once, even when several entries (different `#subpath`s) reference it. +- **Fresh every sync.** Each `sync` run clones into a run-scoped temp dir and removes it on exit, so branch refs always pick up the latest. Within one run the same `url@ref` is cloned only once, even when several entries (different subpaths) reference it. - **Reproducibility / supply chain.** A remote's content becomes rules, agents, and skills the LLM reads as project context. Pin to a tag or SHA so an upstream change can't silently alter behavior, and only reference repos you trust. - **Containment.** The clone can't be made to read outside itself: `..` in `#subpath` is refused, remote repos are checked out with `core.symlinks=false` (a hostile `skills -> /etc` link becomes an inert text file, not a path the copy step follows), and the resolved directory is verified to sit inside the clone. - **Private repos** rely on ambient credentials (an SSH agent or git credential helper). `sync` runs git with `GIT_TERMINAL_PROMPT=0`, so a missing credential fails fast with a warning instead of hanging; local sources still sync. - **Best-effort.** A clone failure (offline, bad URL, missing subpath) warns on stderr and skips that one source — the rest of the sync proceeds and still reports `IS_STATUS=ok`. -#### Tracking packs in the repo (`external:`) +#### Mirroring a pack into the repo -By default a remote pack exists only inside the run cache, so the only trace of an upstream change is a shifted diff in the *generated* output, mixed in with your own content. Add an `external:` block and each pack is additionally materialized into a tracked directory, which makes a version bump readable as an ordinary diff: +Without `mirror:` a pack exists only inside the run cache, so the only trace of an upstream change is a shifted diff in the *generated* output, mixed in with your own content. Give the pack a `mirror:` and it is additionally materialized there, which makes a version bump readable as an ordinary diff: ```yaml -external: - dir: "intelligence/external" # omit the block to keep packs transient +packs: + shared-intel: + url: https://github.com/org/shared-intel.git + ref: v1.2.0 + mirror: "intelligence/external/shared-intel" # omit to keep the pack transient ``` ``` intelligence/external/ -└── shared-intel/ # named after the repo - ├── .pack # url + ref + resolved SHA +└── shared-intel/ # the path you declared — nothing is derived + ├── .pack # url + ref + resolved SHA, written by sync ├── rules/ # only the subpaths your sources reference └── skills/ ``` -Commit that directory — being able to review `git diff` after bumping a pin is the entire point. +Commit that directory — being able to review `git diff` after bumping a pin is the entire point. `/external/` is the recommended location, but the value is a plain path, so packs can live wherever suits the repo, one per pack. -- **Only the referenced subpaths are copied**, so a pack's `README`, CI config and tests never enter your repo. A source with no `#subpath` copies the whole repo. +- **The directory is declared, never derived.** `mirror:` says exactly where the pack goes, so there is no name to sanitize and no collision to resolve. +- **`.pack` is output, not config.** sync writes it; nothing reads it as configuration and it is not meant to be hand-edited. The pin lives in `config.yaml`. +- **Only the referenced subpaths are copied**, so a pack's `README`, CI config and tests never enter your repo. A reference with no subpath copies the whole repo. - **`.git` is never copied.** A nested repository would be recorded as a gitlink, whose contents git does not track — precisely the state this avoids. -- **One directory per `repo@ref`,** named after the repo. The first source entry to touch a pack in a run clears it, so content left by a previous ref (or by a source entry you have since deleted) does not linger. -- **Never destructive.** A directory is only cleared when its `.pack` names the same repo. A same-named directory of your own — or a second pack whose repo basename collides — is left alone and the pack goes to a suffixed name instead. -- Pack files now have committed paths, so `AGENTS.md` and the Pi adapter link to them like any local source instead of naming them bare. +- **Cleared once per run.** The first entry to touch a pack clears its mirror, so content left by a previous ref (or by a source entry you have since deleted) does not linger; later entries only replace their own subpath. +- **Never destructive.** The `.pack` stamp marks the directory as sync's to manage: a non-empty directory *without* one is left alone (with a warning) rather than deleted, so a directory of your own at that path is safe. A stamped directory stays the pack's even after you edit its `url:` — a moved or renamed upstream refreshes the mirror, which is the whole point of committing it. To hand a mirror path back to the project, delete the directory. A mirror that resolves to the repo root, escapes the repo, or sits inside a configured `sources.*` directory is refused outright — `sync` exits 1 before anything is written. +- Mirrored files have committed paths, so `AGENTS.md` and the Pi adapter link to them like any local source instead of naming them bare. A transient pack has no such path and is still named bare. ## Agent Frontmatter diff --git a/examples/dotnet-api-with-react-frontend/config.yaml b/examples/dotnet-api-with-react-frontend/config.yaml index db3b84d..0557afe 100644 --- a/examples/dotnet-api-with-react-frontend/config.yaml +++ b/examples/dotnet-api-with-react-frontend/config.yaml @@ -6,7 +6,7 @@ project: name: my-platform -sync_version: "0.9.0" +sync_version: "0.10.0" sources: rules: diff --git a/examples/go-api-with-opencode/config.yaml b/examples/go-api-with-opencode/config.yaml index 40a5c82..4022fb6 100644 --- a/examples/go-api-with-opencode/config.yaml +++ b/examples/go-api-with-opencode/config.yaml @@ -5,7 +5,7 @@ project: name: my-api -sync_version: "0.9.0" +sync_version: "0.10.0" sources: rules: diff --git a/examples/go-api-with-pi-and-codex/config.yaml b/examples/go-api-with-pi-and-codex/config.yaml index fcf1cd3..5d28a8a 100644 --- a/examples/go-api-with-pi-and-codex/config.yaml +++ b/examples/go-api-with-pi-and-codex/config.yaml @@ -5,7 +5,7 @@ project: name: my-api -sync_version: "0.9.0" +sync_version: "0.10.0" sources: rules: diff --git a/examples/go-api/config.yaml b/examples/go-api/config.yaml index d3d0b7d..f01743f 100644 --- a/examples/go-api/config.yaml +++ b/examples/go-api/config.yaml @@ -4,7 +4,7 @@ project: name: my-api -sync_version: "0.9.0" +sync_version: "0.10.0" sources: rules: diff --git a/examples/platform-with-submodules/config.yaml b/examples/platform-with-submodules/config.yaml index 18c0ab2..7dd9160 100644 --- a/examples/platform-with-submodules/config.yaml +++ b/examples/platform-with-submodules/config.yaml @@ -6,7 +6,7 @@ project: name: my-platform -sync_version: "0.9.0" +sync_version: "0.10.0" sources: rules: diff --git a/examples/with-remote-skills/config.yaml b/examples/with-remote-skills/config.yaml index a30f089..30cc3de 100644 --- a/examples/with-remote-skills/config.yaml +++ b/examples/with-remote-skills/config.yaml @@ -1,36 +1,39 @@ # Example: pulling shared skills/rules from a remote git repo -# A `sources.*` entry may be a remote `git+[@ref][#subpath]` spec instead -# of a local path. sync shallow-clones it (fresh each run) and treats the -# resolved directory exactly like a local source. Pin to a tag or SHA so an +# A pack is DECLARED once under `packs:` and referenced from `sources.*` by +# name — `@[/]` — so the url and the pin live in exactly one +# place however many sections use them. Pin `ref:` to a tag or SHA so an # upstream change can't silently alter your agents — see docs/CONVENTIONS.md -# "Remote sources (git)". Paths/URLs here are EXAMPLES — use your own. +# "Packs (remote sources)". Paths/URLs here are EXAMPLES — use your own. project: name: my-app -sync_version: "0.9.0" +sync_version: "0.10.0" -# Materialize remote packs into a tracked directory and commit it, so bumping a -# pin shows up as a reviewable diff. Drop this block and packs stay transient — -# cloned to a temp dir and gone at the end of the run. -external: - dir: "intelligence/external" +packs: + # `mirror:` materializes the pack into a tracked directory and commits it, so + # bumping `ref:` reads as an ordinary diff. Drop the line and the pack stays + # transient — cloned to a temp dir and gone at the end of the run. + shared-intel: + url: https://github.com/org/shared-intel.git + ref: v1.2.0 + mirror: "intelligence/external/shared-intel" sources: rules: - "intelligence/rules" - "intelligence/sync/rules" - # Shared rules from a central repo, pinned to a tag: - - "git+https://github.com/org/shared-intel.git@v1.2.0#rules" + # Shared rules from a central repo, pinned by the pack above: + - "@shared-intel/rules" agents: - "intelligence/agents" - "intelligence/sync/agents" skills: - "intelligence/skills" - "intelligence/sync/skills" - # Shared skills from the same repo @ the same tag — cloned once per sync, - # reused for the rules entry above (same repo@ref): - - "git+https://github.com/org/shared-intel.git@v1.2.0#skills" + # Shared skills from the SAME pack — one clone per sync, reused for the + # rules entry above: + - "@shared-intel/skills" targets: agents: diff --git a/intelligence/sync/INIT.md b/intelligence/sync/INIT.md index 6775551..4ea3b66 100644 --- a/intelligence/sync/INIT.md +++ b/intelligence/sync/INIT.md @@ -232,7 +232,7 @@ project: # Managed by intelligence-sync — applied schema version. Do not hand-edit; # preserve on re-bootstrap. (Value = intelligence/sync/scripts/VERSION.) -sync_version: "0.9.0" +sync_version: "0.10.0" sources: rules: diff --git a/intelligence/sync/docs/ADAPTERS.md b/intelligence/sync/docs/ADAPTERS.md index 0f29484..fa6cf90 100644 --- a/intelligence/sync/docs/ADAPTERS.md +++ b/intelligence/sync/docs/ADAPTERS.md @@ -61,8 +61,10 @@ Source `lib/common.sh` for these utilities: | `map_access_to_claude_tools(access)` | Tool string for access level | | `map_access_to_claude_disallowed(access)` | Disallowed tools string | | `read_yaml_list(config, section)` | Read list from `config.yaml` | -| `resolve_source_dir(repo_root, src)` | Map a source entry to a local dir — `"$repo_root/$src"`, or a shallow clone for a remote `git+` spec | -| `source_is_remote(src)` | True (0) if a source entry is a remote `git+` spec | +| `resolve_source_dir(repo_root, src)` | Map a source entry to a local dir — `"$repo_root/$src"` for a path, or a shallow clone for a pack reference (`@[/]`) or an inline `git+` spec | +| `source_is_local_path(src)` | True (0) if a source entry is a plain repo-relative path — i.e. neither of the two below. Use this, not a negated `source_is_remote`, whenever a token is about to be pattern-matched against a real directory | +| `source_is_pack(src)` | True (0) if a source entry references a pack declared under `packs:` (`@`) | +| `source_is_remote(src)` | True (0) if a source entry is an inline remote `git+` spec | | `get_target_field(config, target, field)` | Read a field from a target's config block | ### Transformation Patterns @@ -140,7 +142,10 @@ sync_to_myide() { while IFS= read -r src; do [ -z "$src" ] && continue # resolve_source_dir maps a source entry to a local dir: "$repo_root/$src" - # for a local path, or a shallow clone for a remote `git+` spec. + # for a local path, or a shallow clone for a pack reference (`@`) + # or an inline `git+` spec. A pack's url/ref/mirror are read from + # config.yaml, which it takes from $IS_CONFIG_FILE (exported by sync.sh) + # unless you pass the config as a third argument. local dir dir="$(resolve_source_dir "$repo_root" "$src")" [ -d "$dir" ] || continue diff --git a/intelligence/sync/docs/CONVENTIONS.md b/intelligence/sync/docs/CONVENTIONS.md index eb1545e..9bcc127 100644 --- a/intelligence/sync/docs/CONVENTIONS.md +++ b/intelligence/sync/docs/CONVENTIONS.md @@ -64,58 +64,84 @@ A custom adapter belongs in the umbrella's `adapters/`, never in the module's `s Rule filenames, agent names, and skill names all share the same **domain prefix** (`backend-`, `frontend-`, `devops-`, `core-`, `tests-`, project codename, or monorepo component name). Pick the domain once from repo structure and reuse it — do not invent new domains without clear need. -### Remote sources (git) +### Packs (remote sources) -A `sources.{rules,agents,skills}` entry is normally a **local path** relative to the repo root. It may instead be a **remote git spec**, which `sync` shallow-clones on the fly and treats exactly like a local source directory — so a team can keep shared intelligence in one repo and pull it into many projects: +A `sources.{rules,agents,skills}` entry is normally a **local path** relative to the repo root. It may instead reference a **pack** — a remote git repo that `sync` shallow-clones and treats exactly like a local source directory — so a team can keep shared intelligence in one repo and pull it into many projects. + +A pack is **declared once** under `packs:` and referenced by name from as many sections as need it: ```yaml +packs: + shared-intel: + url: https://github.com/org/shared-intel.git + ref: v1.2.0 # the pin — one place, not one per section + mirror: "intelligence/external/shared-intel" # optional; see below + sources: - skills: - - "intelligence/skills" # local - - "git+https://github.com/org/shared-intel.git@v1.2.0#skills" # remote, pinned tag rules: - - "intelligence/rules" - - "git+ssh://git@github.com/org/shared-intel.git@main#rules" # remote, branch + - "intelligence/rules" # local + - "@shared-intel/rules" # pack + skills: + - "@shared-intel/skills" # same pack, same clone, same pin ``` -Spec format: `git+[@][#]` +Declaring the pack is what keeps the url and the ref in one place. Repeating them per section is how the two drift, and nothing catches it: rules pinned at one commit and skills at another is a config that looks fine and reads wrong. + +Reference format: `@[/]` + +- `` — a key under `packs:`. It is a **reference handle, never a path component**, so it needs no sanitizing; it must simply contain no `/`. +- `/` — optional directory inside the pack repo holding the rules / agents / skills. Omit to use the repo root. The subpath is preserved in the mirror, so `@shared-intel/packs/core/rules` lands at `/packs/core/rules`. +- **An undeclared pack fails the run** (exit 1, naming the pack and listing the declared ones). This is deliberately unlike a missing local path, which only warns: the config claims to know that name, so a typo must not quietly drop a whole rule set. + +Pack fields: + +- `url:` — must carry an explicit scheme: `https://`, `http://`, `ssh://`, `git://`, or `file://`. Other transports (notably the command-executing `ext::` / `fd::`) are **rejected** with a warning and skipped. +- `ref:` — optional tag, branch, or commit SHA. Omit for the default branch. +- `mirror:` — optional; see [Mirroring a pack into the repo](#mirroring-a-pack-into-the-repo). + +#### Inline specs (`git+…`) + +A source entry may still carry the whole spec inline: `git+[@][#]`. This is an **anonymous pack** — it has no declared name and no mirror, so it is always transient and cannot be referenced from elsewhere. Declare the pack under `packs:` to pin it once or to commit it. -- `` — must carry an explicit scheme: `https://`, `http://`, `ssh://`, `git://`, or `file://`. Other transports (notably the command-executing `ext::` / `fd::`) are **rejected** with a warning and skipped. -- `@` — optional tag, branch, or commit SHA. Parsed as the segment after the last `@`, accepted as a ref only when it contains no `/` (so `ssh://git@host/...` userinfo is not mistaken for a ref). Branch names containing `/` (e.g. `feature/x`) can't be expressed this way — use a tag, SHA, or slashless branch (pinning is recommended regardless). -- `#` — optional directory inside the cloned repo holding the rules / agents / skills. Omit to use the repo root. +The inline `@` is parsed as the segment after the last `@`, accepted as a ref only when it contains no `/` (so `ssh://git@host/...` userinfo is not mistaken for a ref). Branch names containing `/` (e.g. `feature/x`) can't be expressed this way — use `packs:` with a plain `ref:`, which has no such limit. Behavior and trust: -- **Fresh every sync.** Each `sync` run clones into a run-scoped temp dir and removes it on exit, so branch refs always pick up the latest. Within one run the same `repo@ref` is cloned only once, even when several entries (different `#subpath`s) reference it. +- **Fresh every sync.** Each `sync` run clones into a run-scoped temp dir and removes it on exit, so branch refs always pick up the latest. Within one run the same `url@ref` is cloned only once, even when several entries (different subpaths) reference it. - **Reproducibility / supply chain.** A remote's content becomes rules, agents, and skills the LLM reads as project context. Pin to a tag or SHA so an upstream change can't silently alter behavior, and only reference repos you trust. - **Containment.** The clone can't be made to read outside itself: `..` in `#subpath` is refused, remote repos are checked out with `core.symlinks=false` (a hostile `skills -> /etc` link becomes an inert text file, not a path the copy step follows), and the resolved directory is verified to sit inside the clone. - **Private repos** rely on ambient credentials (an SSH agent or git credential helper). `sync` runs git with `GIT_TERMINAL_PROMPT=0`, so a missing credential fails fast with a warning instead of hanging; local sources still sync. - **Best-effort.** A clone failure (offline, bad URL, missing subpath) warns on stderr and skips that one source — the rest of the sync proceeds and still reports `IS_STATUS=ok`. -#### Tracking packs in the repo (`external:`) +#### Mirroring a pack into the repo -By default a remote pack exists only inside the run cache, so the only trace of an upstream change is a shifted diff in the *generated* output, mixed in with your own content. Add an `external:` block and each pack is additionally materialized into a tracked directory, which makes a version bump readable as an ordinary diff: +Without `mirror:` a pack exists only inside the run cache, so the only trace of an upstream change is a shifted diff in the *generated* output, mixed in with your own content. Give the pack a `mirror:` and it is additionally materialized there, which makes a version bump readable as an ordinary diff: ```yaml -external: - dir: "intelligence/external" # omit the block to keep packs transient +packs: + shared-intel: + url: https://github.com/org/shared-intel.git + ref: v1.2.0 + mirror: "intelligence/external/shared-intel" # omit to keep the pack transient ``` ``` intelligence/external/ -└── shared-intel/ # named after the repo - ├── .pack # url + ref + resolved SHA +└── shared-intel/ # the path you declared — nothing is derived + ├── .pack # url + ref + resolved SHA, written by sync ├── rules/ # only the subpaths your sources reference └── skills/ ``` -Commit that directory — being able to review `git diff` after bumping a pin is the entire point. +Commit that directory — being able to review `git diff` after bumping a pin is the entire point. `/external/` is the recommended location, but the value is a plain path, so packs can live wherever suits the repo, one per pack. -- **Only the referenced subpaths are copied**, so a pack's `README`, CI config and tests never enter your repo. A source with no `#subpath` copies the whole repo. +- **The directory is declared, never derived.** `mirror:` says exactly where the pack goes, so there is no name to sanitize and no collision to resolve. +- **`.pack` is output, not config.** sync writes it; nothing reads it as configuration and it is not meant to be hand-edited. The pin lives in `config.yaml`. +- **Only the referenced subpaths are copied**, so a pack's `README`, CI config and tests never enter your repo. A reference with no subpath copies the whole repo. - **`.git` is never copied.** A nested repository would be recorded as a gitlink, whose contents git does not track — precisely the state this avoids. -- **One directory per `repo@ref`,** named after the repo. The first source entry to touch a pack in a run clears it, so content left by a previous ref (or by a source entry you have since deleted) does not linger. -- **Never destructive.** A directory is only cleared when its `.pack` names the same repo. A same-named directory of your own — or a second pack whose repo basename collides — is left alone and the pack goes to a suffixed name instead. -- Pack files now have committed paths, so `AGENTS.md` and the Pi adapter link to them like any local source instead of naming them bare. +- **Cleared once per run.** The first entry to touch a pack clears its mirror, so content left by a previous ref (or by a source entry you have since deleted) does not linger; later entries only replace their own subpath. +- **Never destructive.** The `.pack` stamp marks the directory as sync's to manage: a non-empty directory *without* one is left alone (with a warning) rather than deleted, so a directory of your own at that path is safe. A stamped directory stays the pack's even after you edit its `url:` — a moved or renamed upstream refreshes the mirror, which is the whole point of committing it. To hand a mirror path back to the project, delete the directory. A mirror that resolves to the repo root, escapes the repo, or sits inside a configured `sources.*` directory is refused outright — `sync` exits 1 before anything is written. +- Mirrored files have committed paths, so `AGENTS.md` and the Pi adapter link to them like any local source instead of naming them bare. A transient pack has no such path and is still named bare. ## Agent Frontmatter diff --git a/intelligence/sync/scripts/VERSION b/intelligence/sync/scripts/VERSION index ac39a10..78bc1ab 100644 --- a/intelligence/sync/scripts/VERSION +++ b/intelligence/sync/scripts/VERSION @@ -1 +1 @@ -0.9.0 +0.10.0 diff --git a/intelligence/sync/scripts/lib/common.sh b/intelligence/sync/scripts/lib/common.sh index 6595ff4..f4732dd 100644 --- a/intelligence/sync/scripts/lib/common.sh +++ b/intelligence/sync/scripts/lib/common.sh @@ -113,6 +113,25 @@ source_is_remote() { esac } +# True (0) if a source token references a pack declared under `packs:` +# (`@` or `@/`). +source_is_pack() { + case "$1" in + @?*) return 0 ;; + *) return 1 ;; + esac +} + +# True (0) if a source token is a plain repo-relative path — i.e. NOT a pack +# reference and NOT an inline remote spec. The inverse of "resolves through a +# clone", which is what every caller that pattern-matches a token against a +# real directory needs. +source_is_local_path() { + source_is_pack "$1" && return 1 + source_is_remote "$1" && return 1 + return 0 +} + # Map an absolute file path to a repo-root-relative path for use as a link # target inside a COMMITTED file (e.g. AGENTS.md). A file under $repo_root gets # its repo-relative path. A file resolved OUTSIDE $repo_root comes from a remote @@ -170,7 +189,14 @@ repo_rel_dir() { # `[ -d "$dir" ] || continue` guard then skips an unresolved source. # Usage: dir="$(resolve_source_dir "$repo_root" "$src")" resolve_source_dir() { - local repo_root="$1" token="$2" + local repo_root="$1" token="$2" config_file="${3:-${IS_CONFIG_FILE:-}}" + + # A declared pack (`@[/]`) keeps its url / ref / mirror in + # config.yaml, so the token itself carries nothing but the reference. + if source_is_pack "$token"; then + resolve_pack_source "$repo_root" "$config_file" "$token" + return 0 + fi if ! source_is_remote "$token"; then printf '%s' "$repo_root/$token" @@ -178,12 +204,68 @@ resolve_source_dir() { fi # --- parse: git+[@][#] --- + # An inline spec is an ANONYMOUS pack: it has no declared name and no + # mirror, so it is always transient. Declare it under `packs:` to commit it. local rest="${token#git+}" local subpath="" urlref="$rest" case "$rest" in *\#*) subpath="${rest#*#}"; urlref="${rest%%#*}" ;; esac + # ref = segment after the last `@` in the post-scheme part, only if it has + # no `/` (else it is userinfo such as `git@host`, not a ref). + local url="$urlref" ref="" after_scheme="${urlref#*://}" + case "$after_scheme" in + *@*) + local cand="${after_scheme##*@}" + case "$cand" in + */*|"") ;; # userinfo / empty -> no ref + *) ref="$cand"; url="${urlref%@$ref}" ;; + esac + ;; + esac + + fetch_remote_source "$token" "$url" "$ref" "$subpath" "" +} + +# Resolve `@[/]` against the `packs:` block and fetch it. +# An undeclared name is a HARD error: unlike a mistyped local path (which the +# caller's `[ -d ]` guard silently skips), a pack reference names something the +# config claims to know, so a typo must not quietly drop a whole rule set. +# Usage: resolve_pack_source "$repo_root" "$config_file" "@shared/rules" +resolve_pack_source() { + local repo_root="$1" config_file="$2" token="$3" + + local rest="${token#@}" name subpath="" + case "$rest" in + */*) name="${rest%%/*}"; subpath="${rest#*/}" ;; + *) name="$rest" ;; + esac + + local url ref mirror_rel mirror_abs="" + url="$(get_pack_field "$config_file" "$name" "url")" + # Defensive only: validate_pack_refs has already failed the run for an + # undeclared pack. It has to, because every caller invokes this inside `$( )` + # — an exit here would end the substitution subshell, not the sync. + if [ -z "$url" ]; then + echo " WARN: pack '$name' is not declared under 'packs:': $token" >&2 + return 0 + fi + ref="$(get_pack_field "$config_file" "$name" "ref")" + mirror_rel="$(get_pack_field "$config_file" "$name" "mirror")" + [ -n "$mirror_rel" ] && mirror_abs="$(resolve_mirror_dir "$repo_root" "$config_file" "$name" "$mirror_rel")" + + fetch_remote_source "$token" "$url" "$ref" "$subpath" "$mirror_abs" "$mirror_rel" +} + +# Shallow-clone @ into the run cache, echo "/", and — +# when is set — additionally materialize it there so the content is +# committed. is only used for messages. +# ALWAYS returns 0 (echoes nothing on failure), per resolve_source_dir's contract. +# Usage: fetch_remote_source [] +fetch_remote_source() { + local token="$1" url="$2" ref="$3" subpath="$4" mirror="$5" mirror_rel="${6:-}" + # Reject path traversal in the subpath: a remote spec must not be able to # escape the clone dir (e.g. `#../../etc`). Checked before any clone. case "/$subpath/" in @@ -196,7 +278,7 @@ resolve_source_dir() { # Scheme whitelist — reject everything but plain fetch transports. The # ext::/fd:: transports execute arbitrary commands on clone, so a malicious # or mistyped config must never reach `git clone` with them. - case "$urlref" in + case "$url" in https://*|http://*|ssh://*|git://*|file://*) ;; *) echo " WARN: remote source rejected (unsupported scheme): $token" >&2 @@ -204,19 +286,6 @@ resolve_source_dir() { ;; esac - # ref = segment after the last `@` in the post-scheme part, only if it has - # no `/` (else it is userinfo such as `git@host`, not a ref). - local url="$urlref" ref="" after_scheme="${urlref#*://}" - case "$after_scheme" in - *@*) - local cand="${after_scheme##*@}" - case "$cand" in - */*|"") ;; # userinfo / empty -> no ref - *) ref="$cand"; url="${urlref%@$ref}" ;; - esac - ;; - esac - if ! command -v git >/dev/null 2>&1; then echo " WARN: remote source needs git, which is not installed: $token" >&2 return 0 @@ -285,77 +354,73 @@ resolve_source_dir() { ;; esac - # No `external:` block -> the clone stays in the transient run cache and - # nothing lands in the repo (the original behaviour). - if [ -z "${IS_EXTERNAL_DIR:-}" ]; then + # No `mirror:` -> the clone stays in the transient run cache and nothing + # lands in the repo. This is every inline `git+` spec, and any pack that + # declares no mirror. + if [ -z "$mirror" ]; then printf '%s' "$out" return 0 fi - materialize_pack "$dest" "$out" "$url" "$ref" "$key" "$subpath" + materialize_pack "$dest" "$out" "$url" "$ref" "$subpath" "$mirror" "$mirror_rel" return 0 } -# Read one `key=value` line out of a pack stamp file. Echoes nothing when the -# file or key is absent. Strips a trailing CR so a stamp that went through a -# CRLF-normalizing checkout still parses. -pack_stamp_field() { - local file="$1" key="$2" line - [ -f "$file" ] || return 0 - while IFS= read -r line || [ -n "$line" ]; do - line="${line%$'\r'}" - case "$line" in - "$key="*) printf '%s' "${line#"$key="}"; return 0 ;; - esac - done < "$file" -} - -# Copy a resolved remote source out of the transient clone into the project's -# external dir, so pack content is committed and an upstream bump shows up in -# `git diff` instead of only in the generated output. Echoes the materialized -# directory; on any failure echoes the clone dir instead, so a broken external -# dir degrades to the transient behaviour rather than losing the source. +# Copy a resolved remote source out of the transient clone into the pack's +# declared `mirror:` directory, so pack content is committed and an upstream +# bump shows up in `git diff` instead of only in the generated output. Echoes +# the materialized directory; on any failure echoes the clone dir instead, so a +# broken mirror degrades to the transient behaviour rather than losing the +# source. +# +# The directory is DECLARED, never derived — `mirror:` says exactly where the +# pack lives, so there is no name to sanitize and no collision to resolve. # -# One directory per repo@ref, named after the repo. The FIRST token to touch a -# pack in a run wipes it (clearing content left by a previous ref or a source -# entry that has since been removed) and writes the `.pack` stamp; later tokens -# for the same repo@ref only add their subpath. The claim is recorded in the run -# cache, which is what makes "wipe once per run" work across separate calls. +# The FIRST token to touch a pack in a run wipes it (clearing content left by a +# previous ref, or by a source entry that has since been removed); later tokens +# for the same pack only replace their own subpath. The claim is recorded in the +# clone cache, which is what makes "wipe once per run" work across the separate +# subshells each resolve_source_dir call runs in. # -# The wipe is guarded by the stamp: a directory that exists WITHOUT a `.pack` -# naming this repo is never deleted — it belongs to the project, not to us. -# Usage: materialize_pack +# The wipe is guarded by the stamp: a NON-EMPTY directory with no `.pack` in it +# is never deleted — it belongs to the project, not to us. +# is the path as authored in config.yaml, used only in messages. +# Usage: materialize_pack materialize_pack() { - local clone="$1" src_dir="$2" url="$3" ref="$4" key="$5" subpath="$6" - - local claim="" pack_dir="" - if [ -n "${IS_REMOTE_CACHE:-}" ]; then - claim="$IS_REMOTE_CACHE/$key.packdir" - [ -f "$claim" ] && IFS= read -r pack_dir < "$claim" - fi - - if [ -z "$pack_dir" ]; then - # Pack name = repo basename, minus `.git`. Anything outside a safe - # filename charset becomes `-` so a hostile URL cannot steer the path. - local name="${url%/}" - name="${name##*/}" - name="${name%.git}" - name="$(printf '%s' "$name" | tr -c 'A-Za-z0-9._-' '-')" - # Leading `.` or `-` is refused outright: `.git` as a pack name would be - # actively destructive, and a leading dash reads as a flag downstream. - case "$name" in ""|.*|-*) name="pack-$key" ;; esac - - pack_dir="$IS_EXTERNAL_DIR/$name" - # Collision: the name is already taken by a DIFFERENT repo (or by a - # project-owned directory with no stamp). Fall back to a suffixed name - # rather than touching content that is not ours. - if [ -d "$pack_dir" ] && [ "$(pack_stamp_field "$pack_dir/.pack" url)" != "$url" ]; then - echo " WARN: external pack dir '$name' is taken — using '$name-$key'" >&2 - pack_dir="$IS_EXTERNAL_DIR/$name-$key" + local clone="$1" src_dir="$2" url="$3" ref="$4" subpath="$5" pack_dir="$6" mirror_rel="${7:-}" + + # The claim is keyed on the DIRECTORY, not on url@ref: it records "this run + # already cleared this path". Keying it on the clone would let two packs + # that share a url@ref but declare different mirrors claim each other's, + # leaving the second mirror unstamped and never pruned. + # + # Same cache-root fallback as the clone: the claim must exist even when the + # caller is not sync.sh, or every token would re-wipe the pack and only the + # last subpath would survive. That fallback root is NOT run-scoped, though, + # so the claim carries `$$` — stable across the command-substitution + # subshells of one run, different for the next. A claim left behind by an + # earlier run must never suppress this run's wipe: that would rebuild the + # mirror with no `.pack` in it and freeze it against the guard below. + local cache_root claim + cache_root="${IS_REMOTE_CACHE:-${TMPDIR:-/tmp}/intelligence-sync-remotes}" + mkdir -p "$cache_root" 2>/dev/null || true + claim="$cache_root/$$-$(printf '%s' "$pack_dir" | cksum | awk '{print $1 "-" $2}').packdir" + + if [ ! -f "$claim" ]; then + # Refuse to wipe a directory that is not ours. Ownership is the PRESENCE + # of the stamp, not the url inside it: a mirror is declared per pack, so + # a stamped directory is this pack's even after its `url:` is edited — + # a moved or renamed upstream must refresh the mirror, not freeze it at + # the old content while the generated output silently follows the new. + if [ -d "$pack_dir" ] && [ -n "$(find "$pack_dir" -mindepth 1 -maxdepth 1 2>/dev/null)" ] \ + && [ ! -f "$pack_dir/.pack" ]; then + echo " WARN: mirror '$pack_dir' holds content that is not a pack's (no .pack stamp) — skipping materialization" >&2 + printf '%s' "$src_dir" + return 0 fi rm -rf "$pack_dir" if ! mkdir -p "$pack_dir"; then - echo " WARN: cannot create external pack dir '$pack_dir' — using the run cache" >&2 + echo " WARN: cannot create mirror dir '$pack_dir' — using the run cache" >&2 printf '%s' "$src_dir" return 0 fi @@ -364,8 +429,8 @@ materialize_pack() { printf 'ref=%s\n' "${ref:-}" printf 'sha=%s\n' "$(git -C "$clone" rev-parse HEAD 2>/dev/null || echo unknown)" } > "$pack_dir/.pack" - [ -n "$claim" ] && printf '%s\n' "$pack_dir" > "$claim" - echo " external: $url${ref:+ @$ref} -> ${pack_dir#"$IS_EXTERNAL_DIR"/}" >&2 + printf '%s\n' "$pack_dir" > "$claim" + echo " pack: $url${ref:+ @$ref} -> ${mirror_rel:-$pack_dir}" >&2 fi # Only a subpath is cleared here — clearing the pack root would delete the @@ -702,7 +767,12 @@ get_model_default() { } # Read a nested key from config.yaml: section -> sub -> key. -# Used to resolve `models..` overrides. +# Resolves `models..` overrides and `packs..`. +# +# The value strip is anchored at the FIRST colon (`[^:]*:`), never a greedy +# `.*:` — a greedy match cuts at the LAST colon on the line, which turns +# `url: https://host/repo.git` into `//host/repo.git`. An unquoted value also +# drops a trailing ` # comment`, per YAML; inside quotes a `#` is content. get_nested_yaml_value() { local file="$1" local section="$2" @@ -716,8 +786,16 @@ get_nested_yaml_value() { in_section && in_sub && /^ [a-zA-Z]/ { in_sub=0 } in_section && in_sub && $0 ~ "^ " key ":" { val = $0 - sub(/.*:[[:space:]]*["\047]?/, "", val) - sub(/["\047]?[[:space:]]*$/, "", val) + sub(/^[[:space:]]*[^:]*:[[:space:]]*/, "", val) + if (val ~ /^"/ || val ~ /^\047/) { + q = substr(val, 1, 1) + val = substr(val, 2) + i = index(val, q) + if (i > 0) val = substr(val, 1, i - 1) + } else { + sub(/[[:space:]]+#.*$/, "", val) + sub(/[[:space:]]+$/, "", val) + } print val exit } @@ -948,8 +1026,10 @@ validate_output_path() { while IFS= read -r src; do [ -z "$src" ] && continue # Remote sources never resolve to a local output path — skip them - # so a `git+...` spec is not pattern-matched against the output dir. - source_is_remote "$src" && continue + # so a `git+...` spec or an `@pack` reference is not pattern-matched + # against the output dir. A mirrored pack is covered separately, + # below, by its declared `mirror:`. + source_is_local_path "$src" || continue src_rel="$(normalize_path "$repo_root/$src")" src_rel="${src_rel#"$repo_root"/}" case "$rel" in @@ -962,56 +1042,47 @@ validate_output_path() { done < <(read_yaml_list "$config_file" "$section") done - # Reject the external pack directory — materialized pack content is source, - # and it is committed, so an adapter cleanup aimed at it would delete work - # that is not regenerated until the next successful clone. - local ext_rel ext_canon - ext_rel="$(get_external_dir "$config_file")" - if [ -n "$ext_rel" ]; then - ext_canon="$(normalize_path "$repo_root/$ext_rel")" - ext_canon="${ext_canon#"$repo_root"/}" + # Reject any pack mirror — materialized pack content is source, and it is + # committed, so an adapter cleanup aimed at it would delete work that is not + # regenerated until the next successful clone. + local mirror_rel + while IFS= read -r mirror_rel; do + [ -z "$mirror_rel" ] && continue case "$rel" in - "$ext_canon"|"$ext_canon"/*) - echo "ERROR: targets.$adapter.output ('$rel') points into the external pack dir ('$ext_rel')." >&2 + "$mirror_rel"|"$mirror_rel"/*) + echo "ERROR: targets.$adapter.output ('$rel') points into a pack mirror ('$mirror_rel')." >&2 echo " The adapter would delete materialized pack content." >&2 exit 1 ;; esac - fi + done < <(list_pack_mirrors "$repo_root" "$config_file") } -# Resolve and validate `external.dir` into the global IS_EXTERNAL_DIR (absolute, -# exported by sync.sh). Empty when the project has no `external:` block. +# Resolve and validate one pack's `mirror:` into an absolute directory. # -# materialize_pack `rm -rf`s a directory under this path, so the same class of -# check that guards adapter outputs applies — with one deliberate difference: -# the external dir is ALLOWED inside the intelligence umbrella, since -# `/external` is the recommended place for it. +# materialize_pack `rm -rf`s this path, so the same class of check that guards +# adapter outputs applies — with one deliberate difference: a mirror is ALLOWED +# inside the intelligence umbrella, since `/external/` is the +# recommended place for it. # -# Exits 1 with a clear message on rejection. -# Usage: resolve_external_dir "$REPO_ROOT" "$CONFIG_FILE" -resolve_external_dir() { - local repo_root="$1" - local config_file="$2" - - IS_EXTERNAL_DIR="" - local rel - rel="$(get_external_dir "$config_file")" - [ -n "$rel" ] || return 0 +# Echoes the absolute path; exits 1 with a clear message on rejection. +# Usage: resolve_mirror_dir "$REPO_ROOT" "$CONFIG_FILE" +resolve_mirror_dir() { + local repo_root="$1" config_file="$2" name="$3" rel="$4" local canon canon="$(normalize_path "$repo_root/$rel")" case "$canon" in ""|"/"|"$repo_root") - echo "ERROR: external.dir resolves to repo root or empty path: '$rel'" >&2 + echo "ERROR: packs.$name.mirror resolves to repo root or empty path: '$rel'" >&2 exit 1 ;; esac case "$canon" in "$repo_root"/*) ;; *) - echo "ERROR: external.dir escapes the repository: '$rel' (resolves to '$canon')." >&2 + echo "ERROR: packs.$name.mirror escapes the repository: '$rel' (resolves to '$canon')." >&2 exit 1 ;; esac @@ -1023,12 +1094,12 @@ resolve_external_dir() { for section in rules agents skills; do while IFS= read -r src; do [ -z "$src" ] && continue - source_is_remote "$src" && continue + source_is_local_path "$src" || continue src_rel="$(normalize_path "$repo_root/$src")" src_rel="${src_rel#"$repo_root"/}" case "$canon_rel" in "$src_rel"|"$src_rel"/*) - echo "ERROR: external.dir ('$rel') is inside a configured source ('$src')." >&2 + echo "ERROR: packs.$name.mirror ('$rel') is inside a configured source ('$src')." >&2 echo " Materializing a pack there would overwrite authored content." >&2 exit 1 ;; @@ -1036,7 +1107,76 @@ resolve_external_dir() { done < <(read_yaml_list "$config_file" "$section") done - IS_EXTERNAL_DIR="$canon" + printf '%s' "$canon" +} + +# Fail the run on a `@` source that names a pack the config does not +# declare, and on a declared `mirror:` that is unsafe to `rm -rf`. +# +# This runs UP FRONT, before any adapter, because resolve_source_dir is always +# called inside `$( )`: an error raised down there would exit the substitution +# subshell only, and the caller's `[ -d "$dir" ] || continue` guard would turn a +# typo into a silently dropped rule set — the exact failure this feature exists +# to remove. A missing local path stays a warning; a bad pack reference does not, +# because the config claims to know that name. +# +# Exits 1 with a clear message on rejection. +# Usage: validate_pack_refs "$REPO_ROOT" "$CONFIG_FILE" +validate_pack_refs() { + local repo_root="$1" config_file="$2" + local section src name known bad=0 + + for section in rules agents skills; do + while IFS= read -r src; do + [ -z "$src" ] && continue + source_is_pack "$src" || continue + name="${src#@}" + name="${name%%/*}" + if [ -z "$(get_pack_field "$config_file" "$name" "url")" ]; then + echo "ERROR: sources.$section entry '$src' references pack '$name', which has no 'packs.$name.url' in $config_file." >&2 + bad=1 + fi + done < <(read_yaml_list "$config_file" "$section") + done + + if [ "$bad" -ne 0 ]; then + known="$(read_yaml_keys "$config_file" "packs" | tr '\n' ' ')" + echo " Declared packs: ${known:-}" >&2 + # `targets:` accepts the flow form, so a user reasonably writes + # `packs:\n shared: { url: … }` — which reads as zero declared packs and + # makes the message above point at a typo that is not there. + if grep -qE '^[[:space:]]+[A-Za-z0-9._-]+:[[:space:]]*\{' "$config_file"; then + echo " Note: a pack must be declared in block form — 'name:' on its own line," >&2 + echo " then indented 'url:' / 'ref:' / 'mirror:'. The '{ … }' form is not read here." >&2 + fi + exit 1 + fi + + # Validate every declared mirror once, before a single clone runs, so an + # unsafe path fails the run rather than being discovered mid-materialization. + local rel + while IFS= read -r name; do + [ -z "$name" ] && continue + rel="$(get_pack_field "$config_file" "$name" "mirror")" + [ -n "$rel" ] || continue + resolve_mirror_dir "$repo_root" "$config_file" "$name" "$rel" >/dev/null + done < <(read_yaml_keys "$config_file" "packs") +} + +# Every declared pack's `mirror:`, one repo-relative path per line (packs with +# no mirror contribute nothing). Used by the guards that must not mistake +# materialized pack content for either an adapter output or an unsynced source. +# Usage: readarray -t mirrors < <(list_pack_mirrors "$REPO_ROOT" "$CONFIG_FILE") +list_pack_mirrors() { + local repo_root="$1" config_file="$2" + local name rel canon + while IFS= read -r name; do + [ -z "$name" ] && continue + rel="$(get_pack_field "$config_file" "$name" "mirror")" + [ -n "$rel" ] || continue + canon="$(normalize_path "$repo_root/$rel")" + printf '%s\n' "${canon#"$repo_root"/}" + done < <(read_yaml_keys "$config_file" "packs") } # Warn about prompt directories not listed in sources. @@ -1049,13 +1189,14 @@ warn_unsynced() { local repo_root="$1" local config_file="$2" - # Collect all configured source paths (local only — remote git specs are - # not filesystem dirs and cannot collide with an unsynced local directory). + # Collect all configured source paths (local only — a remote spec and an + # `@pack` reference are not filesystem dirs and cannot collide with an + # unsynced local directory). local all_sources=() for section in rules agents skills; do while IFS= read -r src; do [ -z "$src" ] && continue - source_is_remote "$src" && continue + source_is_local_path "$src" || continue all_sources+=("$src") done < <(read_yaml_list "$config_file" "$section") done @@ -1071,11 +1212,12 @@ warn_unsynced() { ignores+=("$sub") done < <(read_yaml_list "$config_file" "submodules") # Materialized packs hold rules/ agents/ skills/ dirs that are reached - # through their `git+` source entry, never listed as local sources — so the - # scan below would flag every one of them as unsynced. - local ext_rel - ext_rel="$(get_external_dir "$config_file")" - [ -n "$ext_rel" ] && ignores+=("${ext_rel%/}") + # through their `@` source entry, never listed as local sources — so + # the scan below would flag every one of them as unsynced. + local mirror_rel + while IFS= read -r mirror_rel; do + [ -n "$mirror_rel" ] && ignores+=("${mirror_rel%/}") + done < <(list_pack_mirrors "$repo_root" "$config_file") # Derive the intelligence folder basename from config.yaml's location — # whatever the user named it (`intelligence`, `Intelligence`, `prompts`). @@ -1358,9 +1500,28 @@ get_project_name() { get_yaml_field "$1" "project" "name" } -# Repo-relative directory where remote packs are materialized (external.dir), -# or empty when the project has no `external:` block — the default, where a -# clone lives only in the transient run cache and nothing reaches the repo. -get_external_dir() { - get_yaml_field "$1" "external" "dir" +# One field of a declared pack: `packs..`. +# `mirror` empty means the pack is transient — cloned per run, never committed. +get_pack_field() { + get_nested_yaml_value "$1" "packs" "$2" "$3" +} + +# Immediate sub-keys of a top-level block, one per line (`packs:` → pack names). +# Block form only: a pack always spans several lines, so the inline `{...}` form +# that `get_target_field` accommodates has no use here. +# Usage: readarray -t names < <(read_yaml_keys "config.yaml" "packs") +read_yaml_keys() { + local file="$1" section="$2" + [ -f "$file" ] || return 0 + awk -v section="$section" ' + { sub(/\r$/, "") } + $0 ~ "^" section ":[[:space:]]*$" { in_sec = 1; next } + /^[A-Za-z]/ { in_sec = 0 } + in_sec && /^ [A-Za-z0-9][A-Za-z0-9._-]*:[[:space:]]*$/ { + k = $0 + sub(/^[[:space:]]+/, "", k) + sub(/:[[:space:]]*$/, "", k) + print k + } + ' "$file" } diff --git a/intelligence/sync/scripts/lib/migrations.sh b/intelligence/sync/scripts/lib/migrations.sh index 7142a02..52e06b2 100644 --- a/intelligence/sync/scripts/lib/migrations.sh +++ b/intelligence/sync/scripts/lib/migrations.sh @@ -27,7 +27,7 @@ # matching migrate_to_ — nothing here is rewritten or reordered. # Ordered (ascending) list of migration target versions. Append only. -MIGRATIONS=( "0_3_1" "0_7_0" ) +MIGRATIONS=( "0_3_1" "0_7_0" "0_10_0" ) # The applied-schema version is a managed key in config.yaml — NOT a dotfile, # NOT scripts/VERSION. config.yaml is what most future breaking changes will @@ -410,6 +410,267 @@ migrate_to_0_7_0() { echo " [migrate 0.7.0] done — sources.rules += $rules_entry, sources.agents += $agents_entry" } +# --- migrate_to_0_10_0 ------------------------------------------------------ + +# Split `git+[@][#]` into _MIG_URL / _MIG_REF / _MIG_SUBPATH. +# Same grammar resolve_source_dir parses; kept here so a migration never has to +# source the engine library it is migrating towards. +# +# Globals, never a delimited string on stdout: two of the three fields are +# optional, and `read` with a whitespace IFS collapses a run of delimiters into +# one — so an unpinned `git+#rules` would come back as ref=rules with no +# subpath, silently rewriting a whole source into a branch that does not exist. +_mig_split_git_token() { + local rest="${1#git+}" urlref after cand + _MIG_SUBPATH=""; _MIG_REF="" + case "$rest" in + *\#*) _MIG_SUBPATH="${rest#*#}"; urlref="${rest%%#*}" ;; + *) urlref="$rest" ;; + esac + _MIG_URL="$urlref" + after="${urlref#*://}" + case "$after" in + *@*) + cand="${after##*@}" + case "$cand" in + */*|"") ;; + *) _MIG_REF="$cand"; _MIG_URL="${urlref%@$cand}" ;; + esac + ;; + esac +} + +# Pack name from a repo URL: basename minus `.git`, restricted to a safe +# filename charset. This is the ONLY place a name is still derived, it runs +# once, and the result is written into config.yaml where a human can rename it. +_mig_pack_name_from_url() { + local name="${1%/}" + name="${name##*/}" + name="${name%.git}" + name="$(printf '%s' "$name" | tr -c 'A-Za-z0-9._-' '-')" + case "$name" in ""|.*|-*) name="pack" ;; esac + printf '%s' "$name" +} + +# 0.10.0 replaces the single `external: { dir: … }` block with a `packs:` block: +# a remote source is DECLARED once (url + ref + optional mirror) and referenced +# from `sources.*` by name (`@/`). That removes the duplicated +# `url@ref` an inline spec forced into every section, and makes the mirror +# directory declared rather than derived from the URL. +# +# This migration rewrites the config for the project: every inline `git+` spec +# becomes a declared pack plus an `@name` reference, and `external.dir` becomes +# each pack's `mirror:` so vendored content keeps landing where it already is. +# Inline `git+` specs remain legal afterwards — they are simply no longer the +# only way to reach a remote source, and they are always transient. +# +# Precondition is structural, and each key is a one-way marker: `external:` only +# ever existed BEFORE 0.10.0, `packs:` only ever exists after it. So an +# `external:` key ⇒ convert; otherwise a `packs:` key ⇒ applied ⇒ silent no-op; +# otherwise convert only if there is an inline spec to convert. That last clause +# is what keeps the migration idempotent once inline specs are legal again: the +# run that converts them writes `packs:`, and every later run stops at the marker +# instead of appending a SECOND top-level `packs:` key for each spec added since. +# Fail-closed: the rewrite is staged in a temp file and verified before it +# replaces config.yaml. +# +# migrate_to_0_10_0 [ — unused] +migrate_to_0_10_0() { + local umbrella="$1" + local config="$umbrella/config.yaml" + [ -f "$config" ] || return 0 + + local has_external=0 + grep -q '^external:[[:space:]]*$' "$config" && has_external=1 + local has_packs=0 + grep -q '^packs:[[:space:]]*$' "$config" && has_packs=1 + local has_inline=0 + # ERE, and anchored to a list entry: `\|` is a GNU BRE extension that BSD + # grep (the macOS default) reads as a literal, and matching `git+` anywhere + # would fire on the comment that documents the old spec format. + grep -qE '^[[:space:]]*-[[:space:]]*["'\'']?git\+' "$config" && has_inline=1 + if [ "$has_external" -eq 0 ] && { [ "$has_packs" -eq 1 ] || [ "$has_inline" -eq 0 ]; }; then + return 0 + fi + + echo " [migrate 0.10.0] converting remote sources to declared packs" + IS_MIGRATED=1 + + # The old external dir, if any — it becomes each pack's mirror parent. + local ext_dir="" + if [ "$has_external" -eq 1 ]; then + ext_dir="$(awk ' + { sub(/\r$/, "") } + /^external:[[:space:]]*$/ { in_ext = 1; next } + /^[A-Za-z]/ { in_ext = 0 } + in_ext && /^ dir:/ { + v = $0 + sub(/^[[:space:]]*[^:]*:[[:space:]]*/, "", v) + # Two subs, never one alternation: POSIX awk takes the LONGEST + # match at the leftmost position, so `^["]|["].*$` would match + # the whole quoted value at position 1 and erase it. + sub(/^["\047]/, "", v) + sub(/["\047][[:space:]]*$/, "", v) + sub(/[[:space:]]+$/, "", v) + print v; exit + } + ' "$config")" + ext_dir="${ext_dir%/}" + fi + + # Collect the distinct url@ref pairs across every section, in first-seen + # order, assigning each a unique name. + local names=() urls=() refs=() seen=() + local section token url ref subpath sig i found name base n + for section in rules agents skills; do + while IFS= read -r token; do + case "$token" in git+*) ;; *) continue ;; esac + _mig_split_git_token "$token" + url="$_MIG_URL"; ref="$_MIG_REF" + sig="$url@$ref" + # `${#arr[@]}` guards, never `"${!arr[@]}"` on a possibly-empty + # array: bash 3.2 (macOS default) treats that as unbound under + # `set -u`, which every script here runs with. + found=0 + i=0 + while [ "$i" -lt "${#seen[@]}" ]; do + [ "${seen[$i]}" = "$sig" ] && { found=1; break; } + i=$((i + 1)) + done + [ "$found" -eq 1 ] && continue + base="$(_mig_pack_name_from_url "$url")" + name="$base"; n=2 + while :; do + found=0 + i=0 + while [ "$i" -lt "${#names[@]}" ]; do + [ "${names[$i]}" = "$name" ] && { found=1; break; } + i=$((i + 1)) + done + [ "$found" -eq 0 ] && break + name="$base-$n"; n=$((n + 1)) + done + seen+=("$sig"); names+=("$name"); urls+=("$url"); refs+=("$ref") + done < <(_mig_read_sources "$config" "$section") + done + + # Build the packs: block. + local packs_block="" mirror + i=0 + while [ "$i" -lt "${#names[@]}" ]; do + packs_block="$packs_block ${names[$i]}:"$'\n' + packs_block="$packs_block url: ${urls[$i]}"$'\n' + [ -n "${refs[$i]}" ] && packs_block="$packs_block ref: ${refs[$i]}"$'\n' + if [ -n "$ext_dir" ]; then + mirror="$ext_dir/${names[$i]}" + packs_block="$packs_block mirror: \"$mirror\""$'\n' + fi + i=$((i + 1)) + done + + # Map every inline spec to its `@name[/subpath]` replacement. + local map="" + for section in rules agents skills; do + while IFS= read -r token; do + case "$token" in git+*) ;; *) continue ;; esac + _mig_split_git_token "$token" + url="$_MIG_URL"; ref="$_MIG_REF"; subpath="$_MIG_SUBPATH" + sig="$url@$ref" + i=0 + while [ "$i" -lt "${#seen[@]}" ]; do + if [ "${seen[$i]}" = "$sig" ]; then + map="$map$token"$'\t'"@${names[$i]}${subpath:+/$subpath}"$'\n' + break + fi + i=$((i + 1)) + done + done < <(_mig_read_sources "$config" "$section") + done + + # An `external:` block with no remote source at all declares nothing — emit + # no `packs:` key rather than a dangling empty one, and drop `external:`. + local emit_packs=1 + [ "${#names[@]}" -eq 0 ] && emit_packs=0 + + local tmp="$config.mig.tmp" mapfile_="$config.mig.map" + printf '%s' "$map" > "$mapfile_" + awk -v packs="$packs_block" -v mapfile="$mapfile_" -v emit="$emit_packs" ' + BEGIN { + while ((getline line < mapfile) > 0) { + sub(/\r$/, "", line) + t = index(line, "\t") + if (t > 0) repl[substr(line, 1, t - 1)] = substr(line, t + 1) + } + close(mapfile) + } + { sub(/\r$/, "") } + # Drop the whole external: block. + /^external:[[:space:]]*$/ { in_ext = 1; next } + in_ext && /^[[:space:]]/ { next } + in_ext { in_ext = 0 } + # Emit packs: immediately before sources:. + /^sources:[[:space:]]*$/ && emit == 1 && !done_packs { + printf "packs:\n%s\n", packs + done_packs = 1 + } + # Rewrite an inline spec in place, preserving indentation and quoting. + /^[[:space:]]*-[[:space:]]*["\047]?git\+/ { + val = $0 + sub(/^[[:space:]]*-[[:space:]]*/, "", val) + gsub(/^["\047]|["\047][[:space:]]*$/, "", val) + if (val in repl) { + indent = $0 + sub(/-.*$/, "", indent) + print indent "- \"" repl[val] "\"" + next + } + } + { print } + END { if (emit == 1 && !done_packs) printf "packs:\n%s\n", packs } + ' "$config" > "$tmp" + + # Verify the staged file before it replaces anything: `external:` gone, and + # EXACTLY the expected number of top-level `packs:` keys — a second one + # would be duplicate-key YAML that strict parsers reject. + local packs_keys + packs_keys="$(grep -c '^packs:[[:space:]]*$' "$tmp" || true)" + if [ "$packs_keys" -ne "$emit_packs" ] || grep -q '^external:[[:space:]]*$' "$tmp"; then + rm -f "$tmp" "$mapfile_" + is_status error "migrate_to_0_10_0 could not rewrite config.yaml" + echo " ERROR: failed to convert remote sources to packs in $config." >&2 + echo " Declare each remote under 'packs:' and reference it as '@/'." >&2 + return "$IS_RC_ERROR" + fi + + mv "$tmp" "$config" + rm -f "$mapfile_" + if [ "$emit_packs" -eq 0 ]; then + echo " [migrate 0.10.0] done — no remote source to declare, dropped the empty 'external:' block" + else + echo " [migrate 0.10.0] done — ${#names[@]} pack(s) declared${ext_dir:+, mirrored under $ext_dir}" + fi +} + +# Read one `sources.
` list, one raw value per line (quotes stripped). +# Local to migrations so the chain never depends on the engine library. +_mig_read_sources() { + local config="$1" section="$2" + awk -v section="$section" ' + { sub(/\r$/, "") } + /^sources:[[:space:]]*$/ { in_src = 1; next } + /^[A-Za-z]/ { in_src = 0; in_sec = 0 } + in_src && $0 ~ "^ " section ":[[:space:]]*$" { in_sec = 1; next } + in_src && /^ [A-Za-z]/ { in_sec = 0 } + in_sec && /^[[:space:]]*-[[:space:]]*/ { + v = $0 + sub(/^[[:space:]]*-[[:space:]]*/, "", v) + gsub(/^["\047]|["\047][[:space:]]*$/, "", v) + sub(/[[:space:]]+$/, "", v) + if (v != "") print v + } + ' "$config" +} + # run_migrations [] # The dispatcher of the breaking-change chain. Correctness rests on idempotent # structural preconditions, NOT on the stamp: every migrate_to_* self-detects diff --git a/intelligence/sync/scripts/sync.sh b/intelligence/sync/scripts/sync.sh index 9df607b..0919e42 100644 --- a/intelligence/sync/scripts/sync.sh +++ b/intelligence/sync/scripts/sync.sh @@ -130,22 +130,31 @@ if [ -z "$TARGET_FILTER" ]; then fi fi -# Remote sources (git+ specs in sources.*) are shallow-cloned on demand by -# resolve_source_dir. Give it a run-scoped cache dir so each spec is fetched at -# most once per sync and is removed on exit. Honors $TMPDIR (never hardcodes -# /tmp), mirroring update.sh. +# Remote sources (declared `packs:` reached via `@`, and inline `git+` +# specs in sources.*) are shallow-cloned on demand by resolve_source_dir. Give +# it a run-scoped cache dir so each spec is fetched at most once per sync and is +# removed on exit. Honors $TMPDIR (never hardcodes /tmp), mirroring update.sh. IS_REMOTE_CACHE="$(mktemp -d -t intelligence-sync-remotes-XXXXXX 2>/dev/null || mktemp -d)" export IS_REMOTE_CACHE trap 'rm -rf "$IS_REMOTE_CACHE"' EXIT INT TERM -# With an `external:` block the clone is additionally materialized into a -# tracked directory in the repo, so an upstream bump is visible in `git diff` -# rather than only in the generated output. Without it, packs stay transient. -resolve_external_dir "$REPO_ROOT" "$CONFIG_FILE" -export IS_EXTERNAL_DIR -if [ -n "$IS_EXTERNAL_DIR" ]; then - echo " External packs: ${IS_EXTERNAL_DIR#"$REPO_ROOT"/}" -fi +# A `@` token carries only the reference; its url / ref / mirror live in +# config.yaml. resolve_source_dir keeps its published two-argument contract so +# project-owned adapters written against it keep working, so the config path +# reaches it the same way the clone cache does — through the environment. +IS_CONFIG_FILE="$CONFIG_FILE" +export IS_CONFIG_FILE + +# Fail closed on an undeclared pack reference or an unsafe mirror, before any +# adapter runs — see validate_pack_refs for why this cannot live in the resolver. +validate_pack_refs "$REPO_ROOT" "$CONFIG_FILE" + +# A pack that declares `mirror:` is additionally materialized into a tracked +# directory in the repo, so an upstream bump is visible in `git diff` rather +# than only in the generated output. Without it, the pack stays transient. +while IFS= read -r mirror_rel; do + [ -n "$mirror_rel" ] && echo " Pack mirror: $mirror_rel" +done < <(list_pack_mirrors "$REPO_ROOT" "$CONFIG_FILE") # Lint frontmatter across all source files (rules, agents, skills). # Catches issues like unquoted colons that strict YAML consumers reject. diff --git a/intelligence/sync/skills/intelligence-review-skills/SKILL.md b/intelligence/sync/skills/intelligence-review-skills/SKILL.md index db01b80..a14b7dd 100644 --- a/intelligence/sync/skills/intelligence-review-skills/SKILL.md +++ b/intelligence/sync/skills/intelligence-review-skills/SKILL.md @@ -15,7 +15,7 @@ Name reflects the umbrella usage of "skills" for all AI artifacts (rules + agent 1. **Resolve the layout — never assume folder names.** The umbrella is the directory holding `config.yaml`; the engine module is the directory under it holding `scripts/sync.sh` and `scripts/VERSION` (conventionally `sync/`). Read authoring conventions from `/docs/CONVENTIONS.md` and the `intelligence-authoring` rule. -2. **Enumerate from `config.yaml`, not from a guessed path.** The artifacts are exactly the directories listed under `sources.rules`, `sources.agents` and `sources.skills` — there may be several groups (e.g. a shared one and a project one), they may be nested, and a `git+` entry is a remote pack. Take the list from the config; a literal `intelligence/rules/` is wrong in any project that named things differently. +2. **Enumerate from `config.yaml`, not from a guessed path.** The artifacts are exactly the directories listed under `sources.rules`, `sources.agents` and `sources.skills` — there may be several groups (e.g. a shared one and a project one), they may be nested, and an `@` (or inline `git+`) entry is a remote pack declared under `packs:`. Take the list from the config; a literal `intelligence/rules/` is wrong in any project that named things differently. 3. **Skip everything the engine owns.** Sources under `/` (`/rules`, `/agents`, `/skills/intelligence-*`) are upstream-owned: `update.sh` replaces them wholesale, so a local "fix" there is deleted at the next update. Never propose an edit to them. If one of them is genuinely wrong, or a generic check is missing from this skill, that is a **proposal to upstream** — say so in the report rather than patching locally.