diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 6bbe1dc..4f01a88 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -7,8 +7,8 @@ "plugins": [ { "name": "clify", - "description": "Generate A+ Node.js CLIs from API documentation. Copy a hand-crafted exemplar, mechanically substitute API-specific content, then verify with a deterministic validation gate.", - "version": "0.2.0", + "description": "Generate A+ Node.js CLIs from API documentation. Copies a hand-crafted exemplar (structurally inspired by google/agents-cli), mechanically substitutes API-specific content, and verifies via a deterministic validation gate.", + "version": "0.3.0", "source": "./" } ] diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 7232650..108cd73 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "clify", - "version": "0.2.0", - "description": "Generate A+ Node.js CLIs from API documentation. Copy a hand-crafted exemplar, mechanically substitute API-specific content, then verify with a deterministic validation gate.", + "version": "0.3.0", + "description": "Generate A+ Node.js CLIs from API documentation. Copies a hand-crafted exemplar (structurally inspired by google/agents-cli), mechanically substitutes API-specific content, and verifies via a deterministic validation gate.", "author": { "name": "codeyogi911", "url": "https://github.com/codeyogi911" @@ -11,9 +11,7 @@ "license": "MIT", "keywords": ["cli-generator", "api", "openapi", "codegen", "agent"], "skills": [ - { "name": "clify-scaffold", "source": "skills/clify-scaffold/SKILL.md" }, - { "name": "clify-validate", "source": "skills/clify-validate/SKILL.md" }, - { "name": "clify-sync", "source": "skills/clify-sync/SKILL.md" } + { "name": "clify", "source": "skills/clify/SKILL.md" } ], "capabilities": ["network", "codegen", "file-write"] } diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 11bd767..f3acd27 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,12 @@ jobs: - name: clify unit tests run: npm test - name: validate exemplar - run: node bin/clify.mjs validate examples/jsonplaceholder-cli + run: node bin/clify.mjs validate examples/exemplar-cli - name: exemplar tests - working-directory: examples/jsonplaceholder-cli + working-directory: examples/exemplar-cli run: npm test + - name: scaffold-init smoke (full round-trip) + run: | + tmp=$(mktemp -d) + node bin/clify.mjs scaffold-init demo-api --target "$tmp" + node bin/clify.mjs validate "$tmp/demo-api-cli" diff --git a/README.md b/README.md index d6c9025..61a991d 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

- A Claude Code plugin that generates Node.js CLIs from API documentation by copying a hand-crafted exemplar, mechanically substituting API-specific content, and verifying with a deterministic validation gate. Inspired by google/agents-cli. + A Claude Code plugin that generates Node.js CLIs from API documentation by copying a hand-crafted exemplar, mechanically substituting API-specific content, and verifying with a deterministic validation gate. Structurally inspired by google/agents-cli.

@@ -20,21 +20,21 @@ MCPs are heavy; CLIs are the natural fit for agents. clify produces CLIs that are: -- **Tested by default.** Every generated repo ships with smoke + integration tests and CI on Node 20 & 22. +- **Tested by default.** Every generated repo ships with smoke + integration + auth tests and CI on Node 20 & 22. - **Verifiable.** A deterministic validation gate (`clify validate`) checks 8 categories — manifest consistency, coverage bookkeeping, structural reachability, declared nuances, secrets, CI, and tests. -- **Reproducible.** The deterministic phases (copy + rename + validate) are pure JS — same pass/fail for any agent. +- **Reproducible.** The deterministic phases (copy + rename + validate + grade) are pure JS — same pass/fail for any agent. - **Honest.** Endpoints that get dropped require an explicit reason in `coverage.json`. No silent omissions. ## Architecture -clify is **a binary + skills**, on the same model as `google/agents-cli`: +clify is **a binary + one skill**: | Layer | What it does | What runs it | |---|---|---| -| **Skills** | Fetch docs, parse free-form HTML/Markdown, consult the user on tradeoffs, rewrite per-API content. | LLM (Claude/Codex) | -| **`bin/clify.mjs`** | `clify validate

` — full validation gate. `clify scaffold-init ` — copy exemplar + rename. `clify sync-check ` — re-fetch docs, hash diff. | Pure JS, no LLM | +| **Skill (`clify`)** | Fetch docs, parse free-form HTML/Markdown, consult the user on tradeoffs, rewrite per-API content. | LLM (Claude) | +| **`bin/clify.mjs`** | `validate` — run the gate. `scaffold-init` — copy exemplar + rename. `sync-check` — re-fetch docs, hash diff. `eval` — grade against an `evals/cases/` file. | Pure JS, no LLM | -Anything verifiable by code goes in the binary. Anything requiring judgment stays in skills. The binary is what makes verification reproducible across agent vendors. +Anything verifiable by code goes in the binary. Anything requiring judgment stays in the skill. The binary is what makes verification reproducible across agent vendors. The earlier `clify-validate` and `clify-sync` skills are gone — they were thin wrappers around binary verbs an agent can call directly without a SKILL.md to read first. ## Install @@ -44,13 +44,20 @@ In a Claude Code session: /plugin install codeyogi911/clify ``` -Or clone and link: +Then verify the install — `/skills` should list `clify` as available, and the binary should run: + +``` +clify --help +clify --version # 0.3.0 +``` + +Or clone and link for local development: ``` git clone https://github.com/codeyogi911/clify cd clify npm install -npm link # makes `clify` available as a CLI +npm link # makes `clify` available globally ``` ## Use @@ -58,10 +65,10 @@ npm link # makes `clify` available as a CLI Inside Claude Code: ``` -/clify-scaffold https://docs.example.com/api +/clify https://docs.example.com/api ``` -The skill walks the 13-step pipeline (fetch → parse → consult → **ask where to put it** → init → substitute → validate → simplify → report). The generated CLI is **its own project**, in its own directory, with its own `git init` — by default a sibling of your current directory, but the skill asks before creating files. Output is `/-cli/`, never nested inside the calling repo unless you explicitly ask for that. +The skill walks six phases (fetch & detect → parse & group → consult → init → substitute → validate, simplify, report). The generated CLI is **its own project**, in its own directory, with its own `git init` — by default a sibling of your current directory, but the skill asks before creating files. Output is `/-cli/`, never nested inside the calling repo unless you explicitly ask for that. You can also call the binary verbs directly: @@ -69,6 +76,7 @@ You can also call the binary verbs directly: clify validate ./my-api-cli # check a generated repo clify scaffold-init demo-api --target . # copy exemplar + rename only clify sync-check ./my-api-cli # detect upstream doc drift +clify eval zoho-inventory --repo ./zoho-inventory-cli clify --help ``` @@ -84,40 +92,55 @@ Every check in [`references/validation-gate.md`](references/validation-gate.md) - CI runs `npm test` on Node 20 + 22 - `npm test` exits 0 -The exemplar at [`examples/jsonplaceholder-cli/`](examples/jsonplaceholder-cli/) is the canonical A+ implementation. Run the gate against it: +The exemplar at [`examples/exemplar-cli/`](examples/exemplar-cli/) is the canonical A+ implementation — a fictional API designed to exercise every nuance the scaffolder must handle. Run the gate against it: ``` -node bin/clify.mjs validate examples/jsonplaceholder-cli +node bin/clify.mjs validate examples/exemplar-cli ``` +## Evals + +`evals/` holds graded test cases. Each case file describes a real API and the attributes the scaffolder must produce: + +``` +clify eval zoho-inventory --repo ./zoho-inventory-cli +``` + +The harness runs the validation gate plus deterministic graders (auth scheme, resource count, declared nuances, knowledge files). The LLM-as-judge structural similarity score is a placeholder — wired in a follow-up. See [`evals/README.md`](evals/README.md). + ## Testing ``` -npm test # clify unit tests + deliberate-break tests +npm test # clify unit + deliberate-break tests +npm run test:exemplar # the exemplar's own smoke + integration + auth tests npm run validate-exemplar # run the gate against the bundled exemplar ``` -The clify CI workflow (`.github/workflows/test.yml`) runs both, plus the exemplar's own test suite, on Node 20 and 22. +The clify CI workflow (`.github/workflows/test.yml`) runs both on Node 20 and 22. ## Repo layout ``` clify/ -├── bin/clify.mjs top-level binary (verbs) +├── bin/clify.mjs top-level binary (verbs) ├── lib/ -│ ├── validate.mjs validation gate impl -│ ├── scaffold-init.mjs file-copy + rename -│ └── sync-check.mjs hash-diff -├── skills/ -│ ├── clify-scaffold/SKILL.md the 13-step generation pipeline -│ ├── clify-validate/SKILL.md wraps `clify validate` -│ └── clify-sync/SKILL.md wraps `clify sync-check` + regeneration -├── examples/jsonplaceholder-cli/ canonical A+ exemplar +│ ├── validate.mjs validation gate impl +│ ├── scaffold-init.mjs file-copy + rename +│ └── sync-check.mjs hash-diff +├── skills/clify/SKILL.md the 6-phase generation pipeline (lean) +├── examples/ +│ ├── exemplar-cli/ canonical A+ stencil +│ └── legacy/jsonplaceholder-cli/ reference: simple-API single-skill shape +├── evals/ +│ ├── cases/ one .json per real API +│ ├── grade.mjs deterministic graders +│ └── run.mjs eval harness ├── references/ -│ ├── conventions.md contracts every generated CLI honors -│ └── validation-gate.md every check the gate enforces -├── test/clify.test.mjs unit + deliberate-break tests -└── .github/workflows/test.yml CI (Node 20, 22) +│ ├── conventions.md contracts every generated CLI honors +│ ├── validation-gate.md every check the gate enforces +│ └── scaffold-pipeline.md per-phase detail (loaded on demand) +├── test/clify.test.mjs unit + deliberate-break tests +└── .github/workflows/test.yml CI (Node 20, 22) ``` ## License diff --git a/bin/clify.mjs b/bin/clify.mjs index 0c66237..be7552a 100755 --- a/bin/clify.mjs +++ b/bin/clify.mjs @@ -1,33 +1,35 @@ #!/usr/bin/env node // clify — top-level CLI for the codegen plugin. -// Verbs that do deterministic work: validate, scaffold-init, sync-check. -// Verb that delegates to LLM: scaffold (informational, points to /clify-scaffold skill). +// Verbs that do deterministic work: validate, scaffold-init, sync-check, eval. +// Verb that delegates to LLM: scaffold (informational, points to the /clify skill). import { readFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { validate } from "../lib/validate.mjs"; import { scaffoldInit } from "../lib/scaffold-init.mjs"; import { syncCheck } from "../lib/sync-check.mjs"; +import { runEval, formatReport as formatEvalReport } from "../evals/run.mjs"; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(__dirname, ".."); const VERSION = JSON.parse(readFileSync(join(REPO_ROOT, "package.json"), "utf8")).version; const HELP = `clify ${VERSION} -Generate, validate, and sync agent-friendly CLIs from API documentation. +Generate, validate, sync, and grade agent-friendly CLIs from API documentation. Usage: clify validate Run the validation gate (deterministic, no LLM) clify scaffold-init Copy the exemplar to -cli/ and rename (use --target to choose parent directory) clify sync-check Re-fetch docs, hash, print diff summary - clify scaffold (LLM step) Use /clify-scaffold in Claude Code + clify eval [--repo ] Grade a candidate repo against an evals/cases/ file + clify scaffold (LLM step) Use /clify in Claude Code clify --version clify --help -The validate, scaffold-init, and sync-check verbs are pure JS and reproducible across -agent vendors. The scaffold verb is informational — full scaffold needs LLM judgment -and runs as the /clify-scaffold skill in Claude Code. +The validate, scaffold-init, sync-check, and eval verbs are pure JS and reproducible +across agent vendors. The scaffold verb is informational — full scaffold needs LLM +judgment and runs as the /clify skill in Claude Code. `; async function main() { @@ -48,6 +50,7 @@ async function main() { case "validate": return await runValidate(rest); case "scaffold-init": return runScaffoldInit(rest); case "sync-check": return await runSyncCheck(rest); + case "eval": return await runEvalVerb(rest); case "scaffold": return runScaffoldInfo(rest); default: process.stderr.write(`error: unknown verb '${verb}'\n\n${HELP}`); @@ -107,7 +110,7 @@ function runScaffoldInit(args) { try { const result = scaffoldInit({ apiName, target }); if (json) process.stdout.write(JSON.stringify(result, null, 2) + "\n"); - else process.stdout.write(`scaffolded ${result.apiName} → ${result.dir}\n (next: cd into the dir, git init if you want history, then run /clify-scaffold's substitution phase)\n`); + else process.stdout.write(`scaffolded ${result.apiName} → ${result.dir}\n (next: cd into the dir, git init if you want history, then run the /clify skill's substitution phase)\n`); } catch (err) { process.stderr.write(`error: ${err.message}\n`); process.exit(1); @@ -137,7 +140,20 @@ async function runSyncCheck(args) { function runScaffoldInfo(args) { const url = args.find((a) => !a.startsWith("-")) || ""; - process.stdout.write(`Full scaffolding needs LLM judgment (parsing free-form docs, deciding endpoint→action mapping).\nRun in Claude Code: /clify-scaffold ${url}\n\nThe scaffold skill calls 'clify scaffold-init' and 'clify validate' as deterministic\nsub-steps, but the parsing and substitution phases are LLM-driven.\n`); + process.stdout.write(`Full scaffolding needs LLM judgment (parsing free-form docs, deciding endpoint→action mapping).\nRun in Claude Code: /clify ${url}\n\nThe clify skill calls 'clify scaffold-init' and 'clify validate' as deterministic\nsub-steps, but the parsing and substitution phases are LLM-driven.\n`); +} + +async function runEvalVerb(args) { + const caseName = args.find((a) => !a.startsWith("-")); + if (!caseName) { process.stderr.write("Usage: clify eval [--repo ] [--skip-tests] [--json]\n"); process.exit(2); } + let repoDir; + for (let i = 0; i < args.length; i++) if (args[i] === "--repo" && args[i + 1]) repoDir = args[++i]; + const skipTests = args.includes("--skip-tests"); + const json = args.includes("--json"); + const report = await runEval({ caseName, repoDir, skipTests }); + if (json) process.stdout.write(JSON.stringify(report, null, 2) + "\n"); + else process.stdout.write(formatEvalReport(report)); + process.exit(report.ok ? 0 : 1); } main().catch((err) => { diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000..96071aa --- /dev/null +++ b/evals/README.md @@ -0,0 +1,74 @@ +# clify evals + +Eval harness for the clify scaffolder. Each case file describes a real API and a set of expected attributes; the harness grades a candidate generated repo against those expectations. + +## Layout + +``` +evals/ +├── cases/ one .json per real API +│ └── zoho-inventory.json +├── grade.mjs deterministic graders (file presence, .clify.json fields) +├── run.mjs harness entry — loads case, runs validate + graders +└── README.md this file +``` + +## Run a case + +The harness assumes the candidate repo already exists (you've run the scaffolder). Pass its path via `--repo`, or let it default to `./-cli/` relative to your cwd. + +``` +clify eval zoho-inventory # default candidate path +clify eval zoho-inventory --repo ../zoho-cli # explicit candidate +clify eval zoho-inventory --json # machine-readable +clify eval zoho-inventory --skip-tests # skip running candidate's npm test +``` + +Or directly: + +``` +node evals/run.mjs zoho-inventory --repo /path/to/zoho-inventory-cli +``` + +## What the harness checks + +1. **Validation gate** — runs `clify validate `. Every category must pass. The case file's `expected.must_pass_validation_gate: true` makes this a hard requirement. +2. **Deterministic graders** (`grade.mjs`): + - `auth_scheme` matches `expected.auth_scheme` (or one of `auth_scheme_fallbacks`). + - Resource count ≥ `expected.resources_min`. + - Every `expected.must_include_nuances` is declared in `.clify.json.nuances`. + - Every `expected.must_have_knowledge_files` exists under `knowledge/`. +3. **Structural similarity** to the exemplar — *placeholder*. Today this returns `null` (skipped). The plan is to score files-present, helper-signatures-match, and modular-skills-layout against the exemplar via an LLM-as-judge sibling session. Until that's wired, the deterministic graders carry the load. + +## Adding cases + +Create `evals/cases/.json`: + +```json +{ + "name": "", + "url": "", + "description": "Why this API is interesting as an eval target.", + "expected": { + "auth_scheme": "bearer", + "resources_min": 5, + "must_include_nuances": ["pagination"], + "must_have_knowledge_files": [], + "must_pass_validation_gate": true, + "structural_similarity_to_exemplar": 0.85 + } +} +``` + +## Why Zoho Inventory is the inaugural case + +It exercises every nuance the scaffolder is supposed to handle: + +- **OAuth refresh** — multiple scopes, refresh_token grant against `accounts.zoho.`. Forces the scaffolder to either model OAuth as `bearer` (with refresh in `login.mjs`) or extend the auth schemes. +- **Multi-region routing** — `.com`, `.eu`, `.in`, `.com.au`, `.ca` each have their own accounts host *and* API host. Forces the scaffolder to handle a `_REGION` env var feeding `BASE_URL` resolution. +- **Offset pagination** — `page_context` with `has_more_page`. Different from the cursor pattern the exemplar demonstrates; tests whether the scaffolder generalizes pagination. +- **Multipart uploads** — image attachments on items. +- **Composite resources** — composite items (assemblies that bundle other items). +- **Plan-tier rate limits** — different free/paid limits. + +If the scaffolder produces a passing CLI for Zoho Inventory from its docs URL alone, it has demonstrated the agents-cli quality bar. diff --git a/evals/cases/zoho-inventory.json b/evals/cases/zoho-inventory.json new file mode 100644 index 0000000..ac39322 --- /dev/null +++ b/evals/cases/zoho-inventory.json @@ -0,0 +1,19 @@ +{ + "name": "zoho-inventory", + "url": "https://www.zoho.com/inventory/api/v1/introduction/", + "description": "Zoho Inventory: a real-world test that exercises every nuance the scaffolder should handle — OAuth refresh, multi-region routing, offset pagination, multipart uploads, composite resources, plan-tier rate limits.", + "expected": { + "auth_scheme": "oauth2-refresh", + "auth_scheme_fallbacks": ["bearer"], + "resources_min": 8, + "must_include_nuances": ["pagination", "multi-region", "rate-limit"], + "must_have_knowledge_files": ["oauth-refresh", "multi-region", "composite-items"], + "must_pass_validation_gate": true, + "structural_similarity_to_exemplar": 0.85 + }, + "rubric": { + "structural_similarity": "Files: bin/-cli.mjs is a thin dispatcher; lib/{api,auth,output,config,args,help,env}.mjs are present; commands/.mjs files exist (one per resource); skills/-cli-{workflow,auth,resources,knowledge}/SKILL.md all exist; knowledge/ has the listed files; .clify.json declares the listed nuances; tests pass.", + "code_quality": "Helper signatures match the exemplar (apiRequest, output, errorOut, splitGlobal, toParseArgs, checkRequired, paginate). Auth swapping is a registry edit, not a code rewrite. Pagination is a library iterator. No duplicated payload-building across actions of the same resource.", + "knowledge_correctness": "knowledge/oauth-refresh.md describes the actual Zoho refresh flow (refresh_token grant against accounts.zoho.); knowledge/multi-region.md enumerates .com/.eu/.in/.com.au/.ca with their accounts vs api hosts; knowledge/composite-items.md notes the assembly-stock relationship." + } +} diff --git a/evals/grade.mjs b/evals/grade.mjs new file mode 100644 index 0000000..265ec2f --- /dev/null +++ b/evals/grade.mjs @@ -0,0 +1,79 @@ +// Deterministic graders for an eval case. The LLM-as-judge portion is +// a separate plug-in (currently a no-op placeholder) that scores +// structural_similarity_to_exemplar; everything below this line is +// purely file-system + JSON inspection. +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; + +export function gradeDeterministic(repoDir, expected) { + const findings = []; + const ok = (name, detail) => findings.push({ name, ok: true, detail }); + const ko = (name, detail) => findings.push({ name, ok: false, detail }); + + const clifyPath = join(repoDir, ".clify.json"); + if (!existsSync(clifyPath)) { + ko("clify.json present", clifyPath); + return summarize(findings); + } + const cfg = JSON.parse(readFileSync(clifyPath, "utf8")); + + // auth scheme + const allowedSchemes = [expected.auth_scheme, ...(expected.auth_scheme_fallbacks || [])]; + if (allowedSchemes.includes(cfg.auth?.scheme)) { + ok("auth scheme", `scheme=${cfg.auth.scheme}`); + } else { + ko("auth scheme", `expected one of ${JSON.stringify(allowedSchemes)}, got ${cfg.auth?.scheme}`); + } + + // resources count + const cov = readJson(join(repoDir, "coverage.json")); + if (!cov) { + ko("coverage.json present", "missing"); + } else { + const resources = new Set(cov.endpoints.filter((e) => e.included).map((e) => e.resource)); + if (resources.size >= expected.resources_min) ok("resources count", `${resources.size} ≥ ${expected.resources_min}`); + else ko("resources count", `${resources.size} < ${expected.resources_min}`); + } + + // declared nuances + const declaredNuances = new Set( + Object.entries(cfg.nuances || {}) + .filter(([_, v]) => v && (Array.isArray(v) ? v.length : true)) + .map(([k]) => k) + .map((k) => k === "multiPart" ? "multipart" : k) + .map((k) => k === "rateLimits" ? "rate-limit" : k) + ); + for (const n of expected.must_include_nuances || []) { + if (declaredNuances.has(n)) ok(`nuance: ${n}`, "declared"); + else ko(`nuance: ${n}`, `not in .clify.json.nuances (${[...declaredNuances].join(", ")})`); + } + + // knowledge files + const knowledgeDir = join(repoDir, "knowledge"); + const knowledgeFiles = existsSync(knowledgeDir) ? readdirSync(knowledgeDir).map((f) => f.replace(/\.md$/, "")) : []; + for (const k of expected.must_have_knowledge_files || []) { + if (knowledgeFiles.includes(k)) ok(`knowledge: ${k}`, "present"); + else ko(`knowledge: ${k}`, `expected knowledge/${k}.md, found: ${knowledgeFiles.join(", ") || "(none)"}`); + } + + return summarize(findings); +} + +// Placeholder for the LLM-as-judge step. Scoring structural similarity +// against the exemplar requires a sibling Claude session; that's out of +// scope for the deterministic harness. For now this returns null (skipped), +// and the caller treats null as "manual review pending". +export async function gradeStructuralSimilarity(_repoDir, _exemplarDir, _threshold) { + return null; +} + +function readJson(path) { + if (!existsSync(path)) return null; + try { return JSON.parse(readFileSync(path, "utf8")); } catch { return null; } +} + +function summarize(findings) { + const passed = findings.filter((f) => f.ok).length; + const failed = findings.filter((f) => !f.ok).length; + return { findings, passed, failed, ok: failed === 0 }; +} diff --git a/evals/run.mjs b/evals/run.mjs new file mode 100644 index 0000000..fec5aba --- /dev/null +++ b/evals/run.mjs @@ -0,0 +1,113 @@ +// Eval harness. Each case file describes (a) a docs URL the scaffolder +// should turn into a CLI, and (b) a set of expected attributes. +// +// What this harness does today: +// 1. Reads the case file. +// 2. Looks up the candidate repo (default: ./-cli/ relative to +// where the harness is invoked, or via --repo ). +// 3. Runs `clify validate` against it, then runs the deterministic graders +// from grade.mjs against the same repo. +// 4. Prints a report. +// +// What it does NOT do today: drive the scaffolder end-to-end (that requires a +// Claude Code session). Plug that in later by spawning `claude code` with +// the case URL and waiting for the resulting repo path. +import { readFileSync, existsSync } from "node:fs"; +import { join, resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { validate } from "../lib/validate.mjs"; +import { gradeDeterministic, gradeStructuralSimilarity } from "./grade.mjs"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const CASES_DIR = join(__dirname, "cases"); +const EXEMPLAR_DIR = resolve(__dirname, "../examples/exemplar-cli"); + +export async function runEval({ caseName, repoDir, skipTests = false }) { + const casePath = caseName.endsWith(".json") ? resolve(caseName) : join(CASES_DIR, `${caseName}.json`); + if (!existsSync(casePath)) throw new Error(`Case file not found: ${casePath}`); + const caseSpec = JSON.parse(readFileSync(casePath, "utf8")); + + const candidate = repoDir ? resolve(repoDir) : resolve(`./${caseSpec.name}-cli`); + if (!existsSync(candidate)) { + return { + case: caseSpec.name, + candidate, + ok: false, + stages: { validation: { ok: false, error: `Candidate repo not found at ${candidate}. Run the scaffolder first or pass --repo .` } }, + }; + } + + const validation = await validate(candidate, { skipTests }); + const deterministic = gradeDeterministic(candidate, caseSpec.expected || {}); + const structural = await gradeStructuralSimilarity(candidate, EXEMPLAR_DIR, caseSpec.expected?.structural_similarity_to_exemplar); + + const ok = (caseSpec.expected?.must_pass_validation_gate ? validation.ok : true) && deterministic.ok && (structural === null || structural.ok); + + return { + case: caseSpec.name, + candidate, + ok, + stages: { + validation: { ok: validation.ok, summary: validation.summary, failed: validation.results.filter((r) => !r.ok) }, + deterministic, + structural: structural ?? { skipped: true, reason: "LLM-as-judge not yet wired; deterministic graders only" }, + }, + }; +} + +export function listCases() { + const cases = []; + for (const file of (function* () { + const fs = require("node:fs"); + for (const f of fs.readdirSync(CASES_DIR)) if (f.endsWith(".json")) yield f; + })()) cases.push(file.replace(/\.json$/, "")); + return cases; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const args = process.argv.slice(2); + const caseName = args.find((a) => !a.startsWith("-")); + if (!caseName) { + process.stderr.write("Usage: node evals/run.mjs [--repo ] [--skip-tests] [--json]\n"); + process.exit(2); + } + let repoDir; + for (let i = 0; i < args.length; i++) if (args[i] === "--repo" && args[i + 1]) repoDir = args[++i]; + const skipTests = args.includes("--skip-tests"); + const json = args.includes("--json"); + runEval({ caseName, repoDir, skipTests }) + .then((report) => { + if (json) process.stdout.write(JSON.stringify(report, null, 2) + "\n"); + else process.stdout.write(formatReport(report)); + process.exit(report.ok ? 0 : 1); + }) + .catch((err) => { + process.stderr.write(`error: ${err.stack || err.message}\n`); + process.exit(2); + }); +} + +export function formatReport(r) { + const lines = [`eval: ${r.case}`, ` candidate: ${r.candidate}`, ` ok: ${r.ok}`]; + if (r.stages.validation) { + const v = r.stages.validation; + if (v.error) lines.push(` validation: SKIPPED — ${v.error}`); + else lines.push(` validation: ${v.ok ? "PASS" : "FAIL"} (${v.summary?.passed ?? "?"}/${v.summary?.total ?? "?"})`); + if (v.failed?.length) for (const f of v.failed) lines.push(` ✗ ${f.category}/${f.name}`); + } + if (r.stages.deterministic) { + const d = r.stages.deterministic; + lines.push(` deterministic: ${d.ok ? "PASS" : "FAIL"} (${d.passed}/${d.passed + d.failed})`); + for (const f of d.findings) { + const mark = f.ok ? "✓" : "✗"; + lines.push(` ${mark} ${f.name}${f.detail ? ` — ${f.detail}` : ""}`); + } + } + if (r.stages.structural) { + const s = r.stages.structural; + if (s.skipped) lines.push(` structural: SKIPPED — ${s.reason}`); + else lines.push(` structural: ${s.ok ? "PASS" : "FAIL"} (score ${s.score})`); + } + lines.push(""); + return lines.join("\n"); +} diff --git a/examples/exemplar-cli/.claude-plugin/marketplace.json b/examples/exemplar-cli/.claude-plugin/marketplace.json new file mode 100644 index 0000000..58a7385 --- /dev/null +++ b/examples/exemplar-cli/.claude-plugin/marketplace.json @@ -0,0 +1,7 @@ +{ + "name": "exemplar-cli", + "description": "Generic stencil for the clify scaffolder. Inspired structurally by google/agents-cli; hand-crafted, fully tested, and fictional.", + "version": "0.1.0", + "author": { "name": "clify" }, + "source": "./" +} diff --git a/examples/exemplar-cli/.claude-plugin/plugin.json b/examples/exemplar-cli/.claude-plugin/plugin.json new file mode 100644 index 0000000..82e56b1 --- /dev/null +++ b/examples/exemplar-cli/.claude-plugin/plugin.json @@ -0,0 +1,14 @@ +{ + "name": "exemplar-cli", + "version": "0.1.0", + "description": "Generic stencil for the clify scaffolder. Inspired structurally by google/agents-cli; hand-crafted, fully tested, and fictional.", + "author": { "name": "clify" }, + "license": "MIT", + "skills": [ + { "name": "exemplar-cli-workflow", "source": "skills/exemplar-cli-workflow/SKILL.md" }, + { "name": "exemplar-cli-auth", "source": "skills/exemplar-cli-auth/SKILL.md" }, + { "name": "exemplar-cli-resources", "source": "skills/exemplar-cli-resources/SKILL.md" }, + { "name": "exemplar-cli-knowledge", "source": "skills/exemplar-cli-knowledge/SKILL.md" } + ], + "capabilities": ["network"] +} diff --git a/examples/exemplar-cli/.clify.json b/examples/exemplar-cli/.clify.json new file mode 100644 index 0000000..3aca406 --- /dev/null +++ b/examples/exemplar-cli/.clify.json @@ -0,0 +1,30 @@ +{ + "apiName": "exemplar", + "docsUrl": "https://docs.exemplar.test/api/v1", + "crawledUrls": ["https://docs.exemplar.test/api/v1"], + "contentHash": "sha256:exemplar-static-not-from-live-fetch", + "generatedAt": "2026-04-26T00:00:00Z", + "clifyVersion": "0.3.0", + "nodeMinVersion": "20", + "auth": { + "envVar": "EXEMPLAR_API_KEY", + "scheme": "bearer", + "validationCommand": "items list" + }, + "defaults": [], + "nuances": { + "pagination": "cursor", + "rateLimits": true, + "authScopes": false, + "deprecated": [], + "idempotency": ["items.create", "orders.create"], + "multiPart": ["orders.upload"], + "conditional": ["items.update"], + "businessRules": 1 + }, + "coverage": { + "totalParsed": 11, + "totalIncluded": 11, + "totalDropped": 0 + } +} diff --git a/examples/exemplar-cli/.env.example b/examples/exemplar-cli/.env.example new file mode 100644 index 0000000..39cda8c --- /dev/null +++ b/examples/exemplar-cli/.env.example @@ -0,0 +1,13 @@ +# Exemplar CLI environment variables. +# This file is the placeholder shape every clify-generated CLI inherits. +# Copy to `.env` (gitignored) and fill in values. + +# @required +# @how-to-get https://docs.exemplar.test/dashboard/api-keys +# @format Bearer-style opaque token (40+ chars). Treat as a secret. +# EXEMPLAR_API_KEY=your_exemplar_api_key_here + +# @optional +# @validation-command items list +# Override the API base URL — used by integration tests to point at a mock server. +# EXEMPLAR_BASE_URL=http://127.0.0.1:3000 diff --git a/examples/jsonplaceholder-cli/.github/workflows/test.yml b/examples/exemplar-cli/.github/workflows/test.yml similarity index 100% rename from examples/jsonplaceholder-cli/.github/workflows/test.yml rename to examples/exemplar-cli/.github/workflows/test.yml diff --git a/examples/jsonplaceholder-cli/.gitignore b/examples/exemplar-cli/.gitignore similarity index 100% rename from examples/jsonplaceholder-cli/.gitignore rename to examples/exemplar-cli/.gitignore diff --git a/examples/exemplar-cli/AGENTS.md b/examples/exemplar-cli/AGENTS.md new file mode 100644 index 0000000..0e730b8 --- /dev/null +++ b/examples/exemplar-cli/AGENTS.md @@ -0,0 +1,32 @@ +# Agent instructions for exemplar-cli + +You are working with `exemplar-cli`, a thin CLI over the (fictional) Exemplar API. This is the **stencil** repo for the clify scaffolder — anything you do here should hold up as a pattern every generated CLI inherits. + +## Before you act + +1. Read every file in `knowledge/` — they capture API quirks and business rules. +2. Use `exemplar-cli --help`, `exemplar-cli --help`, and `exemplar-cli --help`. Help is generated from the same registry the dispatcher reads, so it never drifts. +3. Run `exemplar-cli login --status --json` if a command surprises you with `auth_missing`. + +## Conventions + +- All resource commands follow ` [flags]`. The `login` command is dispatched separately. +- Errors are JSON on stderr with exit code 1. `code` tells you whether to retry (`retryable: true`) and how long (`retryAfter` seconds). +- The CLI never retries. That decision lives in the agent loop — see `skills/exemplar-cli-workflow/SKILL.md`. +- Set `EXEMPLAR_BASE_URL` to point at a mock server during testing. + +## Idempotency + +`POST /items` and `POST /orders` accept `--idempotency-key`. Generate one per logical operation (UUIDv4) and reuse it on retries to avoid duplicate writes. See `knowledge/idempotency-keys.md`. + +## Pagination + +Every `list` action returns `{ items, nextCursor }`. Use `--all` to walk pages or `--cursor ` to step manually. See `knowledge/cursor-pagination.md`. + +## Testing + +``` +npm test +``` + +Runs smoke, integration, and auth tests against an in-repo mock server. CI runs the same on Node 20 and 22. No network required. diff --git a/examples/jsonplaceholder-cli/LICENSE b/examples/exemplar-cli/LICENSE similarity index 100% rename from examples/jsonplaceholder-cli/LICENSE rename to examples/exemplar-cli/LICENSE diff --git a/examples/exemplar-cli/README.md b/examples/exemplar-cli/README.md new file mode 100644 index 0000000..a8f7231 --- /dev/null +++ b/examples/exemplar-cli/README.md @@ -0,0 +1,66 @@ +# exemplar-cli + +A hand-crafted, fictional CLI used as the **stencil** for the [clify](https://github.com/codeyogi911/clify) scaffolder. + +`exemplar-cli` is structurally inspired by [google/agents-cli](https://github.com/google/agents-cli): + +- Hierarchical subcommands (` `). +- One file per resource under `commands/`. +- Shared HTTP, auth, output, and config layers under `lib/`. +- A first-class `login` command with `--status`. +- Modular skills under `skills/-/`. +- Fully tested against an in-repo mock server — no network needed for `npm test`. + +The "Exemplar API" is fictional. The clify scaffolder copies this whole tree, mechanically renames `exemplar` / `EXEMPLAR` / `Exemplar` to the target API's name, and an LLM substitutes the resource registry, knowledge files, and tests to match. + +## Layout + +``` +exemplar-cli/ +├── bin/exemplar-cli.mjs thin dispatcher +├── lib/ +│ ├── api.mjs apiRequest + cursor pagination +│ ├── auth.mjs pluggable auth (bearer) +│ ├── config.mjs ~/.config/exemplar-cli/credentials.json +│ ├── env.mjs .env loader +│ ├── args.mjs splitGlobal, parseArgs adapters +│ ├── help.mjs --help generators +│ └── output.mjs output, errorOut +├── commands/ +│ ├── items.mjs list/get/create/update/delete (+ idempotency, if-match) +│ ├── item-variants.mjs sub-resource of items +│ ├── orders.mjs list/get/create/upload (multipart) +│ └── login.mjs token persistence + --status +├── skills/ modular, four files +├── knowledge/ business rules + patterns +├── test/ +│ ├── _helpers.mjs spawn-CLI helper +│ ├── _mock-server.mjs zero-dep HTTP mock +│ ├── smoke.test.mjs structural tests +│ ├── integration.test.mjs mock-driven CRUD + pagination + multipart +│ └── auth.test.mjs bearer wiring + login --status +├── .clify.json metadata read by the validator +├── coverage.json every endpoint, included or dropped +├── .env.example EXEMPLAR_API_KEY + EXEMPLAR_BASE_URL +└── .github/workflows/test.yml Node 20 + 22 CI +``` + +## Use + +``` +EXEMPLAR_API_KEY=test exemplar-cli items list --all +exemplar-cli items create --name Widget --sku W-001 --price 9.99 --idempotency-key "$(uuidgen)" +exemplar-cli orders upload --id ord-1 --file ./receipt.pdf +``` + +## Test + +``` +npm test +``` + +Runs smoke (no network), integration (against `test/_mock-server.mjs`), and auth tests on the current Node version. CI runs the same on Node 20 and 22. + +## Why a fictional API + +Real APIs come with real quirks, real auth flows, and real onboarding requirements. A stencil tied to one real API would teach the scaffolder that API's idiosyncrasies as universal patterns. By staying fictional, this exemplar keeps the structural lessons (how resources are split into files, how auth is pluggable, how pagination is library-level) free of any API's specific shape. diff --git a/examples/exemplar-cli/bin/exemplar-cli.mjs b/examples/exemplar-cli/bin/exemplar-cli.mjs new file mode 100644 index 0000000..0549bf4 --- /dev/null +++ b/examples/exemplar-cli/bin/exemplar-cli.mjs @@ -0,0 +1,167 @@ +#!/usr/bin/env node +// exemplar-cli — Hand-crafted stencil for the clify scaffolder. +// +// The entry is intentionally thin. Resource definitions live under commands/, +// shared HTTP and auth machinery under lib/. The scaffolder copies this whole +// tree, mechanically renames `exemplar` → ``, and an LLM substitutes the +// resource registry to match the target API. +import { parseArgs } from "node:util"; +import { readFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { loadEnv } from "../lib/env.mjs"; +import { splitGlobal, hasHelp, toParseArgs, checkRequired } from "../lib/args.mjs"; +import { output, errorOut } from "../lib/output.mjs"; +import { apiRequest, paginate } from "../lib/api.mjs"; +import { showRootHelp, showResourceHelp, showActionHelp } from "../lib/help.mjs"; + +import items from "../commands/items.mjs"; +import itemVariants from "../commands/item-variants.mjs"; +import orders from "../commands/orders.mjs"; +import { loginFlags, runLogin } from "../commands/login.mjs"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(__dirname, ".."); + +loadEnv(REPO_ROOT); + +const pkg = JSON.parse(readFileSync(join(REPO_ROOT, "package.json"), "utf8")); +const VERSION = pkg.version; + +// Resource registry. Each command file default-exports +// { name, actions, buildPayload? }. The bin assembles the lookup table here. +const COMMANDS = [items, itemVariants, orders]; +const REGISTRY = Object.fromEntries(COMMANDS.map((c) => [c.name, c.actions])); +const PAYLOAD_BUILDERS = Object.fromEntries(COMMANDS.map((c) => [c.name, c.buildPayload || (() => ({}))])); + +function interpolatePath(template, values) { + let path = template; + for (const m of template.matchAll(/:([a-zA-Z_]+)/g)) { + const k = m[1]; + const v = values[k]; + if (v === undefined) errorOut("validation_error", `Missing path parameter --${k}`); + path = path.replace(`:${k}`, encodeURIComponent(String(v))); + } + return path; +} + +async function runResourceAction(resourceArg, actionArg, remaining, global, rest) { + const def = REGISTRY[resourceArg][actionArg]; + + if (hasHelp(rest)) { + process.stdout.write(showActionHelp(resourceArg, actionArg, REGISTRY)); + return; + } + + let parsed; + try { + parsed = parseArgs({ args: remaining, options: toParseArgs(def.flags), strict: true, allowPositionals: false }); + } catch (err) { + errorOut("validation_error", err.message); + } + + const missing = checkRequired(parsed.values, def.flags); + if (missing.length) errorOut("validation_error", `Missing required flag(s): ${missing.map((m) => `--${m}`).join(", ")}`); + + const path = interpolatePath(def.path, parsed.values); + const buildPayload = PAYLOAD_BUILDERS[resourceArg]; + + let body; + if (def.method !== "GET" && def.method !== "DELETE" && !parsed.values.file) { + if (parsed.values.body) { + try { body = JSON.parse(parsed.values.body); } + catch { errorOut("validation_error", "--body must be valid JSON"); } + } else { + body = buildPayload(parsed.values); + } + } + + // Cursor pagination via --all on list-like actions. + if (global.all && actionArg === "list") { + const collected = []; + const query = {}; + if (parsed.values.cursor) query.cursor = parsed.values.cursor; + if (parsed.values.limit) query.limit = parsed.values.limit; + if (parsed.values.status) query.status = parsed.values.status; + for await (const item of paginate({ method: def.method, path, query, version: VERSION, dryRun: !!global.dry_run, verbose: !!global.verbose })) { + collected.push(item); + } + output(collected, !!global.json); + return; + } + + // Build query for list-like actions (non-paginating path). + let query; + if (def.method === "GET" && actionArg !== "get") { + query = {}; + for (const [k, v] of Object.entries(parsed.values)) { + if (v !== undefined && k !== "id") query[k] = v; + } + } + + const result = await apiRequest({ + method: def.method, + path, + query, + body, + file: parsed.values.file, + idempotencyKey: parsed.values["idempotency-key"], + ifMatch: parsed.values["if-match"], + dryRun: !!global.dry_run, + verbose: !!global.verbose, + version: VERSION, + }); + + output(result, !!global.json); +} + +async function main() { + const argv = process.argv.slice(2); + const { global, rest } = splitGlobal(argv); + + if (global.version) { process.stdout.write(VERSION + "\n"); return; } + + const positional = rest.filter((a) => a !== "--help" && a !== "-h"); + if (positional.length === 0) { process.stdout.write(showRootHelp(VERSION, REGISTRY)); return; } + + // Login is dispatched ahead of resource lookup. + if (positional[0] === "login") { + if (hasHelp(rest)) { + let out = `exemplar-cli login\n\nFlags:\n`; + for (const [name, spec] of Object.entries(loginFlags)) { + out += ` --${name.padEnd(10)} ${spec.type.padEnd(8)} ${spec.description}\n`; + } + process.stdout.write(out); + return; + } + let parsed; + try { parsed = parseArgs({ args: positional.slice(1), options: toParseArgs(loginFlags), strict: true, allowPositionals: false }); } + catch (err) { errorOut("validation_error", err.message); } + await runLogin(parsed.values, !!global.json); + return; + } + + const [resourceArg, actionArg, ...remaining] = positional; + + if (!REGISTRY[resourceArg]) { + const available = Object.keys(REGISTRY).concat(["login"]).join(", "); + errorOut("validation_error", `Unknown resource: ${resourceArg}. Available: ${available}`); + } + + if (!actionArg) { + process.stdout.write(showResourceHelp(resourceArg, REGISTRY)); + return; + } + + if (!REGISTRY[resourceArg][actionArg]) { + const available = Object.keys(REGISTRY[resourceArg]).join(", "); + errorOut("validation_error", `Unknown action: ${actionArg} on ${resourceArg}. Available: ${available}`); + } + + await runResourceAction(resourceArg, actionArg, remaining, global, rest); +} + +main().catch((err) => { + errorOut("network_error", err.message || String(err)); +}); diff --git a/examples/exemplar-cli/commands/item-variants.mjs b/examples/exemplar-cli/commands/item-variants.mjs new file mode 100644 index 0000000..e4079e2 --- /dev/null +++ b/examples/exemplar-cli/commands/item-variants.mjs @@ -0,0 +1,42 @@ +// `item-variants` resource: a sub-resource of items, flattened to keep +// command nesting at two levels (per references/conventions.md). The parent +// id travels as `--itemId` and is interpolated into the path template. +const VARIANT_BODY_FLAGS = { + body: { type: "string", description: "Raw JSON body (overrides individual flags)" }, + sku: { type: "string", description: "Variant SKU" }, + size: { type: "string", description: "Variant size (e.g. S, M, L)" }, + color: { type: "string", description: "Variant color" }, +}; + +export default { + name: "item-variants", + actions: { + list: { + method: "GET", + path: "/items/:itemId/variants", + description: "List variants for an item.", + flags: { + itemId: { type: "string", required: true, description: "Parent item id" }, + cursor: { type: "string", description: "Pagination cursor" }, + }, + }, + create: { + method: "POST", + path: "/items/:itemId/variants", + description: "Create a variant under an item.", + flags: { + itemId: { type: "string", required: true, description: "Parent item id" }, + ...VARIANT_BODY_FLAGS, + }, + }, + }, + buildPayload(values) { + const out = {}; + for (const [k, v] of Object.entries(values)) { + if (v === undefined) continue; + if (k === "itemId" || k === "body" || k === "cursor") continue; + out[k] = v; + } + return out; + }, +}; diff --git a/examples/exemplar-cli/commands/items.mjs b/examples/exemplar-cli/commands/items.mjs new file mode 100644 index 0000000..eec32bd --- /dev/null +++ b/examples/exemplar-cli/commands/items.mjs @@ -0,0 +1,74 @@ +// `items` resource: list (cursor pagination), get, create (idempotency), update, delete. +// +// Each command file default-exports a registry entry consumed by bin/exemplar-cli.mjs: +// { name, actions, buildPayload?(values) } +// `actions[name]` shape: { method, path, description, flags }. +// The dispatcher reads `flags` to build parseArgs options, interpolates `:placeholder` +// tokens in `path` from those flags, and sends the body for non-GET/DELETE methods. +const COMMON_BODY_FLAGS = { + body: { type: "string", description: "Raw JSON body (overrides individual flags)" }, + name: { type: "string", description: "Item name" }, + sku: { type: "string", description: "Stock keeping unit identifier" }, + price: { type: "string", description: "Unit price as decimal string (e.g. \"19.99\")" }, +}; + +export default { + name: "items", + actions: { + list: { + method: "GET", + path: "/items", + description: "List items. Pass --all to auto-paginate.", + flags: { + cursor: { type: "string", description: "Pagination cursor from a previous response" }, + limit: { type: "string", description: "Page size (server caps at 100)" }, + }, + }, + get: { + method: "GET", + path: "/items/:id", + description: "Fetch a single item by id.", + flags: { + id: { type: "string", required: true, description: "Item id" }, + }, + }, + create: { + method: "POST", + path: "/items", + description: "Create an item. Pass --idempotency-key to make the request safe to retry.", + flags: { + ...COMMON_BODY_FLAGS, + "idempotency-key": { type: "string", description: "Idempotency-Key header (optional but recommended)" }, + }, + }, + update: { + method: "PATCH", + path: "/items/:id", + description: "Update an item.", + flags: { + id: { type: "string", required: true, description: "Item id" }, + ...COMMON_BODY_FLAGS, + "if-match": { type: "string", description: "ETag for optimistic concurrency (optional)" }, + }, + }, + delete: { + method: "DELETE", + path: "/items/:id", + description: "Delete an item.", + flags: { + id: { type: "string", required: true, description: "Item id" }, + }, + }, + }, + buildPayload(values) { + const out = {}; + for (const [k, v] of Object.entries(values)) { + if (v === undefined) continue; + if (k === "id" || k === "body" || k === "cursor" || k === "limit") continue; + if (k === "idempotency-key" || k === "if-match") continue; + if (k === "price") out.price = String(v); + else out[k] = v; + } + return out; + }, +}; diff --git a/examples/exemplar-cli/commands/login.mjs b/examples/exemplar-cli/commands/login.mjs new file mode 100644 index 0000000..7dea664 --- /dev/null +++ b/examples/exemplar-cli/commands/login.mjs @@ -0,0 +1,28 @@ +// `login` is the auth-management command, not a REST resource. It lives +// outside the resource registry and is dispatched directly by bin/exemplar-cli.mjs. +// +// Mirrors the agents-cli pattern: ` login [--token ]` stores a token at +// ~/.config/exemplar-cli/credentials.json. ` login --status` reports +// whether the env var or stored token (or neither) is providing auth. +import { saveCredentials, credentialsPath } from "../lib/config.mjs"; +import { authStatus } from "../lib/auth.mjs"; +import { output, errorOut } from "../lib/output.mjs"; + +export const loginFlags = { + token: { type: "string", description: "API token to persist (omit to read interactively)" }, + status: { type: "boolean", description: "Show current auth source without changing it" }, +}; + +export async function runLogin(values, jsonRequested) { + if (values.status) { + output(authStatus(), jsonRequested); + return; + } + let token = values.token; + if (!token) token = (process.env.EXEMPLAR_LOGIN_TOKEN || "").trim(); + if (!token) { + errorOut("validation_error", "Pass --token , set EXEMPLAR_LOGIN_TOKEN, or set EXEMPLAR_API_KEY in your environment."); + } + saveCredentials({ token, savedAt: new Date().toISOString() }); + output({ ok: true, path: credentialsPath() }, jsonRequested); +} diff --git a/examples/exemplar-cli/commands/orders.mjs b/examples/exemplar-cli/commands/orders.mjs new file mode 100644 index 0000000..16f22f5 --- /dev/null +++ b/examples/exemplar-cli/commands/orders.mjs @@ -0,0 +1,57 @@ +// `orders` resource: list, get, create (idempotency), upload (multipart). +const COMMON_BODY_FLAGS = { + body: { type: "string", description: "Raw JSON body (overrides individual flags)" }, + customerId: { type: "string", description: "Customer placing the order" }, + notes: { type: "string", description: "Free-form order notes" }, +}; + +export default { + name: "orders", + actions: { + list: { + method: "GET", + path: "/orders", + description: "List orders.", + flags: { + cursor: { type: "string", description: "Pagination cursor" }, + status: { type: "string", description: "Filter by status (pending|paid|cancelled)" }, + }, + }, + get: { + method: "GET", + path: "/orders/:id", + description: "Fetch a single order by id.", + flags: { + id: { type: "string", required: true, description: "Order id" }, + }, + }, + create: { + method: "POST", + path: "/orders", + description: "Create an order. Pass --idempotency-key to make the request safe to retry.", + flags: { + ...COMMON_BODY_FLAGS, + "idempotency-key": { type: "string", description: "Idempotency-Key header (optional but recommended)" }, + }, + }, + upload: { + method: "POST", + path: "/orders/:id/upload", + description: "Attach a file (receipt, packing slip) to an order via multipart/form-data.", + flags: { + id: { type: "string", required: true, description: "Order id" }, + file: { type: "string", required: true, description: "Path to the file to attach" }, + }, + }, + }, + buildPayload(values) { + const out = {}; + for (const [k, v] of Object.entries(values)) { + if (v === undefined) continue; + if (k === "id" || k === "body" || k === "cursor" || k === "status" || k === "file") continue; + if (k === "idempotency-key") continue; + out[k] = v; + } + return out; + }, +}; diff --git a/examples/exemplar-cli/coverage.json b/examples/exemplar-cli/coverage.json new file mode 100644 index 0000000..34baafb --- /dev/null +++ b/examples/exemplar-cli/coverage.json @@ -0,0 +1,19 @@ +{ + "parsedAt": "2026-04-26T00:00:00Z", + "totalParsed": 11, + "totalIncluded": 11, + "totalDropped": 0, + "endpoints": [ + { "method": "GET", "path": "/items", "resource": "items", "action": "list", "included": true }, + { "method": "GET", "path": "/items/:id", "resource": "items", "action": "get", "included": true }, + { "method": "POST", "path": "/items", "resource": "items", "action": "create", "included": true }, + { "method": "PATCH", "path": "/items/:id", "resource": "items", "action": "update", "included": true }, + { "method": "DELETE", "path": "/items/:id", "resource": "items", "action": "delete", "included": true }, + { "method": "GET", "path": "/items/:itemId/variants", "resource": "item-variants", "action": "list", "included": true }, + { "method": "POST", "path": "/items/:itemId/variants", "resource": "item-variants", "action": "create", "included": true }, + { "method": "GET", "path": "/orders", "resource": "orders", "action": "list", "included": true }, + { "method": "GET", "path": "/orders/:id", "resource": "orders", "action": "get", "included": true }, + { "method": "POST", "path": "/orders", "resource": "orders", "action": "create", "included": true }, + { "method": "POST", "path": "/orders/:id/upload", "resource": "orders", "action": "upload", "included": true } + ] +} diff --git a/examples/exemplar-cli/knowledge/composite-orders.md b/examples/exemplar-cli/knowledge/composite-orders.md new file mode 100644 index 0000000..24a9381 --- /dev/null +++ b/examples/exemplar-cli/knowledge/composite-orders.md @@ -0,0 +1,15 @@ +--- +type: business-rule +source: docs +extracted: 2026-04-26 +confidence: high +applies-to: ["orders.create"] +--- + +An order's `lineItems[].itemId` must reference an existing item *and* every line item's `variantId` (when present) must belong to that parent item. The server returns `409 conflict` if a line item references an item that has been deleted between when the client built the order and when it submits — there is no "soft-delete reuse" path. + +Practical consequences: + +- After deleting an item, do not retry an in-flight order that referenced it. Re-validate the cart. +- Composite orders (orders that bundle multiple items into a kit) are not yet supported via the public API; ask the user if the docs mention "composite items" — that's a Zoho-specific concept that does not apply here. +- The CLI's `--idempotency-key` does not protect against this conflict; idempotency only guards against duplicate POSTs of the same payload, not against state drift between client and server. diff --git a/examples/exemplar-cli/knowledge/cursor-pagination.md b/examples/exemplar-cli/knowledge/cursor-pagination.md new file mode 100644 index 0000000..62a1c6a --- /dev/null +++ b/examples/exemplar-cli/knowledge/cursor-pagination.md @@ -0,0 +1,19 @@ +--- +type: pattern +source: docs +extracted: 2026-04-26 +confidence: high +applies-to: ["items.list", "item-variants.list", "orders.list"] +--- + +Every `list` action paginates via opaque cursors. Response shape: + +```json +{ "items": [ ... ], "nextCursor": "" | null } +``` + +When `nextCursor` is null or absent, iteration is complete. Otherwise pass it back as `--cursor ` to fetch the next page. + +The CLI's `--all` flag automates this loop — it concatenates pages and emits the combined array. Use `--all` for small/medium datasets where you want the whole list in memory; for very large datasets, page manually with `--cursor` so you can checkpoint between batches. + +Cursors are not stable across schema migrations: a cursor obtained yesterday may return `400 validation_error` today. Treat cursors as ephemeral within a session. diff --git a/examples/exemplar-cli/knowledge/idempotency-keys.md b/examples/exemplar-cli/knowledge/idempotency-keys.md new file mode 100644 index 0000000..6d3869e --- /dev/null +++ b/examples/exemplar-cli/knowledge/idempotency-keys.md @@ -0,0 +1,19 @@ +--- +type: pattern +source: docs +extracted: 2026-04-26 +confidence: high +applies-to: ["items.create", "orders.create"] +--- + +The Exemplar API honors the `Idempotency-Key` header on `POST /items` and `POST /orders`. Pass it via `--idempotency-key `. + +Server semantics: + +- A repeated key with the *same* body returns the original response (status, headers, body) without re-processing. +- A repeated key with a *different* body returns `422 validation_error` with `code: "idempotency_key_reuse"`. The CLI maps this to its standard `validation_error`; check `details.code` to disambiguate. +- Keys expire after 24 hours of last use. + +Recommended values: a UUIDv4 per logical operation. Do not reuse keys across distinct operations even if their payloads are identical — that defeats the safety net for a future bug fix that changes payload shape. + +When the CLI encounters a `network_error` or `timeout` on a `create`, retrying the same command with the same `--idempotency-key` is safe. diff --git a/examples/exemplar-cli/knowledge/rate-limit.md b/examples/exemplar-cli/knowledge/rate-limit.md new file mode 100644 index 0000000..69e7c22 --- /dev/null +++ b/examples/exemplar-cli/knowledge/rate-limit.md @@ -0,0 +1,14 @@ +--- +type: pattern +source: docs +extracted: 2026-04-26 +confidence: high +--- + +The Exemplar API enforces a per-token rate limit of 60 requests/minute. When you hit the cap: + +- HTTP `429` with a `Retry-After` header (seconds). +- The CLI maps this to `code: "rate_limited", retryable: true, retryAfter: `. +- The CLI does **not** auto-retry. Callers should sleep for `retryAfter` seconds and re-issue. + +For batch workloads, prefer paginated reads (`--all` walks pages serially, which is below the cap) over parallelising the same endpoint. Bursting parallel requests across resources is fine — the cap is global per token, not per endpoint. diff --git a/examples/exemplar-cli/lib/api.mjs b/examples/exemplar-cli/lib/api.mjs new file mode 100644 index 0000000..8c5edac --- /dev/null +++ b/examples/exemplar-cli/lib/api.mjs @@ -0,0 +1,95 @@ +// HTTP layer: apiRequest + cursor-pagination iterator. +// +// `apiRequest` is the only place that talks to fetch(). It handles auth +// injection, multipart uploads, dry-run mode, verbose logging, and maps +// HTTP status to the structured error codes documented in conventions.md. +import { readFileSync, existsSync } from "node:fs"; +import { errorOut } from "./output.mjs"; +import { applyAuth } from "./auth.mjs"; + +export const BASE_URL = (process.env.EXEMPLAR_BASE_URL || "https://api.exemplar.test").replace(/\/$/, ""); + +export async function apiRequest({ method, path, query, body, headers = {}, dryRun, verbose, idempotencyKey, ifMatch, file, version = "0.0.0" }) { + const url = new URL(BASE_URL + path); + if (query) { + for (const [k, v] of Object.entries(query)) { + if (v !== undefined && v !== null && v !== "") url.searchParams.set(k, String(v)); + } + } + + const reqHeaders = { "user-agent": `exemplar-cli/${version}`, ...headers }; + const auth = applyAuth(reqHeaders); + if (!auth.ok) errorOut("auth_missing", `Set EXEMPLAR_API_KEY (or run 'exemplar-cli login') to authenticate.`); + + let reqBody; + if (file) { + if (!existsSync(file)) errorOut("validation_error", `File not found: ${file}`); + const fd = new FormData(); + const buf = readFileSync(file); + const blob = new Blob([buf]); + fd.append("file", blob, file.split("/").pop()); + reqBody = fd; + } else if (body !== undefined && method !== "GET" && method !== "DELETE") { + reqHeaders["content-type"] = "application/json"; + reqBody = JSON.stringify(body); + } + + if (idempotencyKey) reqHeaders["idempotency-key"] = idempotencyKey; + if (ifMatch) reqHeaders["if-match"] = ifMatch; + + if (dryRun) { + return { __dryRun: true, method, url: url.toString(), headers: reqHeaders, body: body ?? null }; + } + + let res; + try { + res = await fetch(url, { method, headers: reqHeaders, body: reqBody }); + } catch (err) { + if (err.name === "AbortError" || /timeout/i.test(err.message || "")) { + errorOut("timeout", `Request timed out: ${err.message}`, { retryable: true }); + } + errorOut("network_error", `Network error: ${err.message}`, { retryable: true }); + } + + const status = res.status; + const retryAfterHeader = res.headers.get("retry-after"); + const retryAfter = retryAfterHeader ? Number(retryAfterHeader) : undefined; + + let parsed; + const text = await res.text(); + try { parsed = text ? JSON.parse(text) : null; } catch { parsed = text; } + + if (verbose) { + const headerObj = {}; + res.headers.forEach((v, k) => { headerObj[k] = v; }); + process.stderr.write(JSON.stringify({ status, headers: headerObj }) + "\n"); + } + + if (status >= 200 && status < 300) return parsed; + + const baseMsg = parsed && typeof parsed === "object" && parsed.message ? parsed.message : `HTTP ${status}`; + if (status === 401) errorOut("auth_invalid", baseMsg); + if (status === 403) errorOut("forbidden", baseMsg); + if (status === 404) errorOut("not_found", baseMsg); + if (status === 409) errorOut("conflict", baseMsg); + if (status === 400 || status === 422) errorOut("validation_error", baseMsg, { details: parsed }); + if (status === 429) errorOut("rate_limited", `Rate limited.${retryAfter ? ` Retry after ${retryAfter}s.` : ""}`, { retryable: true, retryAfter }); + if (status >= 500) errorOut("server_error", baseMsg, { retryable: true, retryAfter }); + errorOut("network_error", baseMsg); +} + +// Cursor-pagination iterator. Server convention: list responses carry +// `{ items: [...], nextCursor: "..." | null }`. When `nextCursor` is null +// or absent, iteration stops. Used by list actions when --all is set. +export async function* paginate(opts) { + let cursor = opts.query?.cursor; + while (true) { + const query = { ...(opts.query || {}) }; + if (cursor) query.cursor = cursor; else delete query.cursor; + const res = await apiRequest({ ...opts, query }); + const page = Array.isArray(res) ? { items: res, nextCursor: null } : (res || { items: [], nextCursor: null }); + for (const item of (page.items || [])) yield item; + if (!page.nextCursor) return; + cursor = page.nextCursor; + } +} diff --git a/examples/exemplar-cli/lib/args.mjs b/examples/exemplar-cli/lib/args.mjs new file mode 100644 index 0000000..86479ca --- /dev/null +++ b/examples/exemplar-cli/lib/args.mjs @@ -0,0 +1,38 @@ +// Global flag splitter + per-action arg-spec helpers. +export const GLOBAL_FLAGS = new Set(["--json", "--dry-run", "--version", "-v", "--verbose", "--all"]); + +export function splitGlobal(argv) { + const global = {}; + const rest = []; + for (const arg of argv) { + if (GLOBAL_FLAGS.has(arg)) { + const key = arg.replace(/^--?/, "").replace(/-/g, "_"); + global[key] = true; + if (arg === "-v") global.version = true; + if (arg === "--dry-run") global.dry_run = true; + } else { + rest.push(arg); + } + } + return { global, rest }; +} + +export function hasHelp(args) { + return args.includes("--help") || args.includes("-h"); +} + +export function toParseArgs(flagSpec) { + const options = {}; + for (const [name, spec] of Object.entries(flagSpec)) { + options[name] = { type: spec.type === "boolean" ? "boolean" : "string" }; + } + return options; +} + +export function checkRequired(values, flagSpec) { + const missing = []; + for (const [name, spec] of Object.entries(flagSpec)) { + if (spec.required && (values[name] === undefined || values[name] === "")) missing.push(name); + } + return missing; +} diff --git a/examples/exemplar-cli/lib/auth.mjs b/examples/exemplar-cli/lib/auth.mjs new file mode 100644 index 0000000..a041ce6 --- /dev/null +++ b/examples/exemplar-cli/lib/auth.mjs @@ -0,0 +1,34 @@ +// Pluggable auth resolver. The exemplar uses bearer; the scaffolder swaps +// `scheme` (and the matching `applyAuth` branch) for the target API. +// +// Schemes the validation gate accepts: bearer | api-key-header | basic | none. +// Add a new branch here and a matching .clify.json `auth.scheme` to extend. +import { loadCredentials } from "./config.mjs"; + +const SCHEME = "bearer"; +const ENV_VAR = "EXEMPLAR_API_KEY"; + +export function resolveToken() { + if (process.env[ENV_VAR]) return process.env[ENV_VAR]; + const creds = loadCredentials(); + return creds?.token || null; +} + +export function applyAuth(headers) { + const token = resolveToken(); + if (!token) return { ok: false, reason: "auth_missing" }; + if (SCHEME === "bearer") { + headers["authorization"] = `Bearer ${token}`; + } else if (SCHEME === "api-key-header") { + headers["x-api-key"] = token; + } else if (SCHEME === "basic") { + headers["authorization"] = `Basic ${Buffer.from(token).toString("base64")}`; + } + return { ok: true }; +} + +export function authStatus() { + const fromEnv = !!process.env[ENV_VAR]; + const fromConfig = !!loadCredentials()?.token; + return { scheme: SCHEME, envVar: ENV_VAR, fromEnv, fromConfig, authenticated: fromEnv || fromConfig }; +} diff --git a/examples/exemplar-cli/lib/config.mjs b/examples/exemplar-cli/lib/config.mjs new file mode 100644 index 0000000..29112db --- /dev/null +++ b/examples/exemplar-cli/lib/config.mjs @@ -0,0 +1,22 @@ +// Project-scoped credential store at ~/.config/exemplar-cli/credentials.json. +// Used by the `login` command. Env var EXEMPLAR_API_KEY always wins when set. +import { readFileSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; + +const CONFIG_DIR = join(homedir(), ".config", "exemplar-cli"); +const CRED_PATH = join(CONFIG_DIR, "credentials.json"); + +export function loadCredentials() { + if (!existsSync(CRED_PATH)) return null; + try { return JSON.parse(readFileSync(CRED_PATH, "utf8")); } + catch { return null; } +} + +export function saveCredentials(creds) { + if (!existsSync(CONFIG_DIR)) mkdirSync(CONFIG_DIR, { recursive: true }); + writeFileSync(CRED_PATH, JSON.stringify(creds, null, 2)); + try { chmodSync(CRED_PATH, 0o600); } catch { /* non-POSIX */ } +} + +export function credentialsPath() { return CRED_PATH; } diff --git a/examples/exemplar-cli/lib/env.mjs b/examples/exemplar-cli/lib/env.mjs new file mode 100644 index 0000000..adec1cc --- /dev/null +++ b/examples/exemplar-cli/lib/env.mjs @@ -0,0 +1,21 @@ +// .env loader. Reads from REPO ROOT only. No dotenv dependency. Shell wins +// over .env. Strips surrounding quotes; skips blanks and `#` comments. +import { readFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +export function loadEnv(repoRoot) { + const path = join(repoRoot, ".env"); + if (!existsSync(path)) return; + for (const line of readFileSync(path, "utf8").split("\n")) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const eq = trimmed.indexOf("="); + if (eq === -1) continue; + const key = trimmed.slice(0, eq).trim(); + let val = trimmed.slice(eq + 1).trim(); + if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) { + val = val.slice(1, -1); + } + if (process.env[key] === undefined) process.env[key] = val; + } +} diff --git a/examples/exemplar-cli/lib/help.mjs b/examples/exemplar-cli/lib/help.mjs new file mode 100644 index 0000000..66984eb --- /dev/null +++ b/examples/exemplar-cli/lib/help.mjs @@ -0,0 +1,37 @@ +// Help text generators. Read the runtime resource registry so help is +// always in sync with whatever commands/ files are loaded. +const CLI = "exemplar-cli"; + +export function showRootHelp(version, registry) { + let out = `${CLI} ${version}\nCLI for the Exemplar API.\n\nUsage:\n ${CLI} [flags]\n ${CLI} login [--token ] [--status]\n\nGlobal flags:\n --json Force JSON output\n --dry-run Print request without sending\n --verbose Print response status & headers to stderr\n --all Auto-paginate list actions\n --version, -v Print version\n --help, -h Show this help\n\nResources:\n`; + for (const r of Object.keys(registry).sort()) { + out += ` ${r.padEnd(18)} ${Object.keys(registry[r]).join(", ")}\n`; + } + out += `\nCommands:\n login Store an API token for the current user.\n\nUse '${CLI} --help' for actions, or ' --help' for flags.\n`; + return out; +} + +export function showResourceHelp(resource, registry) { + const actions = registry[resource]; + let out = `${CLI} ${resource}\n\nActions:\n`; + for (const [name, def] of Object.entries(actions)) { + out += ` ${name.padEnd(10)} ${def.method} ${def.path}\n`; + } + out += `\nUse '${CLI} ${resource} --help' for flags.\n`; + return out; +} + +export function showActionHelp(resource, action, registry) { + const def = registry[resource][action]; + let out = `${CLI} ${resource} ${action}\n\n${def.method} ${def.path}\n\n`; + if (def.description) out += `${def.description}\n\n`; + out += `Flags:\n`; + const entries = Object.entries(def.flags); + if (entries.length === 0) out += ` (none)\n`; + for (const [name, spec] of entries) { + const req = spec.required ? "required" : "optional"; + const desc = spec.description || ""; + out += ` --${name.padEnd(20)} ${spec.type.padEnd(8)} ${req.padEnd(8)} ${desc}\n`; + } + return out; +} diff --git a/examples/exemplar-cli/lib/output.mjs b/examples/exemplar-cli/lib/output.mjs new file mode 100644 index 0000000..0068a08 --- /dev/null +++ b/examples/exemplar-cli/lib/output.mjs @@ -0,0 +1,31 @@ +// Centralized output + structured-error formatter. +// `output()` chooses JSON when --json is requested or when stdout is piped. +// `errorOut()` writes JSON to stderr (when piped or forced via env) and exits 1. +export const isPiped = !process.stdout.isTTY; + +export function output(data, jsonRequested) { + const wantJson = jsonRequested || isPiped; + if (wantJson) { + process.stdout.write(JSON.stringify(data, null, 2) + "\n"); + } else if (data === null || data === undefined) { + process.stdout.write("\n"); + } else if (typeof data === "string") { + process.stdout.write(data + "\n"); + } else if (Array.isArray(data) || typeof data === "object") { + process.stdout.write(JSON.stringify(data, null, 2) + "\n"); + } else { + process.stdout.write(String(data) + "\n"); + } +} + +export function errorOut(code, message, opts = {}) { + const obj = { type: "error", code, message, retryable: !!opts.retryable }; + if (opts.retryAfter !== undefined) obj.retryAfter = opts.retryAfter; + if (opts.details) obj.details = opts.details; + if (process.env.__EXEMPLAR_FORCE_JSON_ERR === "1" || isPiped) { + process.stderr.write(JSON.stringify(obj) + "\n"); + } else { + process.stderr.write(`error[${code}]: ${message}\n`); + } + process.exit(1); +} diff --git a/examples/exemplar-cli/package.json b/examples/exemplar-cli/package.json new file mode 100644 index 0000000..1db013f --- /dev/null +++ b/examples/exemplar-cli/package.json @@ -0,0 +1,10 @@ +{ + "name": "exemplar-cli", + "version": "0.1.0", + "description": "Generic stencil for the clify scaffolder. Inspired structurally by google/agents-cli; hand-crafted, fully tested, and fictional.", + "type": "module", + "bin": { "exemplar-cli": "./bin/exemplar-cli.mjs" }, + "engines": { "node": ">=20" }, + "scripts": { "test": "node --test test/*.test.mjs" }, + "license": "MIT" +} diff --git a/examples/exemplar-cli/skills/exemplar-cli-auth/SKILL.md b/examples/exemplar-cli/skills/exemplar-cli-auth/SKILL.md new file mode 100644 index 0000000..ca26477 --- /dev/null +++ b/examples/exemplar-cli/skills/exemplar-cli-auth/SKILL.md @@ -0,0 +1,49 @@ +--- +name: exemplar-cli-auth +description: Manage authentication for the Exemplar API via exemplar-cli login. Use when the user asks how to authenticate, sign in, store/refresh credentials, or troubleshoot 401/403 errors. +allowed-tools: + - Bash + - Read +--- + +# exemplar-cli-auth + +Manage credentials for `exemplar-cli`. The Exemplar API uses bearer-token auth. + +## Where the token lives + +`exemplar-cli` reads its token in this order: + +1. `EXEMPLAR_API_KEY` from the environment (or `.env`). +2. The `token` field of `~/.config/exemplar-cli/credentials.json`, written by `exemplar-cli login`. + +If neither is set, every command fails with `auth_missing`. + +## Login + +``` +exemplar-cli login --token +``` + +Stores the token at `~/.config/exemplar-cli/credentials.json` (mode 0600). + +``` +exemplar-cli login --status --json +``` + +Reports `{ scheme, envVar, fromEnv, fromConfig, authenticated }` so you can see which source is providing auth. + +## Errors and remediation + +| Code | HTTP | Likely cause | Action | +|---|---|---|---| +| `auth_missing` | — | No env var set, no stored credentials | Set `EXEMPLAR_API_KEY` or run `login` | +| `auth_invalid` | 401 | Token rejected by server | Reissue from the dashboard, run `login` again | +| `forbidden` | 403 | Token valid but lacks the required scope | Check the dashboard's scope settings | + +The CLI does not retry auth errors. If `auth_invalid` is intermittent, the token has likely been rotated and a new one needs to be obtained. + +## Anti-patterns + +- ❌ Don't paste tokens into source files. The validation gate scans for them. +- ❌ Don't share `~/.config/exemplar-cli/credentials.json` between machines — re-run `login` per machine. diff --git a/examples/exemplar-cli/skills/exemplar-cli-knowledge/SKILL.md b/examples/exemplar-cli/skills/exemplar-cli-knowledge/SKILL.md new file mode 100644 index 0000000..f00ade7 --- /dev/null +++ b/examples/exemplar-cli/skills/exemplar-cli-knowledge/SKILL.md @@ -0,0 +1,46 @@ +--- +name: exemplar-cli-knowledge +description: Capture and consult business rules and runtime quirks for the Exemplar API. Use when the user asks "what are the gotchas with this API?" or wants to record a finding that should persist across sessions. +allowed-tools: + - Read + - Write +--- + +# exemplar-cli-knowledge + +API knowledge for Exemplar lives under `knowledge/`. Each file has YAML frontmatter and a free-form body. + +## When to read + +Before any non-trivial workflow, read every file in `knowledge/`. The files are small by design — meant to be read in full, not searched. + +## When to write + +After a workflow surfaces a non-obvious finding (an undocumented constraint, a sequencing requirement, a plan-tier limit, an enum value not in the OpenAPI spec), append a new file. Filename: `knowledge/.md`. Frontmatter: + +```yaml +--- +type: gotcha | pattern | shortcut | quirk | business-rule +applies-to: ["items.create", "orders.upload"] # optional but useful +source: docs | runtime +confidence: high | medium | low +extracted: 2026-04-26 +--- +``` + +Body: prose. Cite the surface that produced the finding (response message, dashboard string, runtime error). Don't include API secrets. + +## Existing knowledge + +| File | Type | Applies to | +|---|---|---| +| `knowledge/idempotency-keys.md` | business-rule | `items.create`, `orders.create` | +| `knowledge/cursor-pagination.md` | pattern | every list action | +| `knowledge/rate-limit.md` | pattern | every request | +| `knowledge/composite-orders.md` | business-rule | `orders.create` | + +## Anti-patterns + +- ❌ Don't write nuance prose into `knowledge/` if the corresponding `.clify.json.nuances.*` field isn't set; the validation gate cross-references them. +- ❌ Don't duplicate help text — the CLI's `--help` is the source of truth for flag shapes. +- ❌ Don't keep stale entries. When the API changes (`clify sync-check` reports drift), prune knowledge that no longer applies. diff --git a/examples/exemplar-cli/skills/exemplar-cli-resources/SKILL.md b/examples/exemplar-cli/skills/exemplar-cli-resources/SKILL.md new file mode 100644 index 0000000..70fbd3f --- /dev/null +++ b/examples/exemplar-cli/skills/exemplar-cli-resources/SKILL.md @@ -0,0 +1,48 @@ +--- +name: exemplar-cli-resources +description: Quick reference for every Exemplar API resource × action exposed by exemplar-cli. Use when the user asks "what does the items API support?" or needs the precise flag set for a specific endpoint. +allowed-tools: + - Bash + - Read +--- + +# exemplar-cli-resources + +Resource × action quick reference. When in doubt, prefer `exemplar-cli --help` — the help text is generated from the same registry and never drifts. + +## items + +| Action | Method | Path | Required flags | Notable optional | +|---|---|---|---|---| +| list | GET | /items | — | `--cursor`, `--limit`, global `--all` | +| get | GET | /items/:id | `--id` | — | +| create | POST | /items | — | `--name`, `--sku`, `--price`, `--idempotency-key`, `--body` | +| update | PATCH | /items/:id | `--id` | `--name`, `--sku`, `--price`, `--if-match`, `--body` | +| delete | DELETE | /items/:id | `--id` | — | + +## item-variants (sub-resource of items) + +| Action | Method | Path | Required flags | Notable optional | +|---|---|---|---|---| +| list | GET | /items/:itemId/variants | `--itemId` | `--cursor` | +| create | POST | /items/:itemId/variants | `--itemId` | `--sku`, `--size`, `--color`, `--body` | + +## orders + +| Action | Method | Path | Required flags | Notable optional | +|---|---|---|---|---| +| list | GET | /orders | — | `--cursor`, `--status`, global `--all` | +| get | GET | /orders/:id | `--id` | — | +| create | POST | /orders | — | `--customerId`, `--notes`, `--idempotency-key`, `--body` | +| upload | POST | /orders/:id/upload | `--id`, `--file` | — | + +## Patterns shared across resources + +- **Cursor pagination** — `list` actions accept `--cursor`. Combined with `--all`, the CLI walks pages until `nextCursor` is null. +- **Idempotency** — `create` actions accept `--idempotency-key`. The server treats repeated keys as the same request. +- **Conditional update** — `items update` accepts `--if-match ` and returns `409 conflict` if the ETag doesn't match. +- **Multipart upload** — `orders upload` accepts `--file ` and posts `multipart/form-data`. + +## Raw JSON escape hatch + +Every `create`/`update` action accepts `--body ''` to send a raw JSON payload instead of individual flags. Useful when the spec gains a new field before the CLI is regenerated. diff --git a/examples/exemplar-cli/skills/exemplar-cli-workflow/SKILL.md b/examples/exemplar-cli/skills/exemplar-cli-workflow/SKILL.md new file mode 100644 index 0000000..0291ae5 --- /dev/null +++ b/examples/exemplar-cli/skills/exemplar-cli-workflow/SKILL.md @@ -0,0 +1,85 @@ +--- +name: exemplar-cli-workflow +description: Drive end-to-end workflows on the Exemplar API via exemplar-cli. Use when the user asks to list/get/create/update/delete items, item-variants, or orders, attach files to orders, or chain multiple Exemplar operations together. +allowed-tools: + - Bash + - Read + - Write +--- + +# exemplar-cli-workflow + +Wrap the (fictional) [Exemplar API](https://docs.exemplar.test/api/v1) via `exemplar-cli`. The Exemplar API is a generic items / item-variants / orders surface used as the clify scaffolder's stencil — it demonstrates the patterns every generated CLI inherits. + +**Before running any command, read every file in `knowledge/`.** + +## Triggers + +- User says `/exemplar-cli` or `/exemplar` +- User asks to manipulate `items`, `item-variants`, or `orders` on Exemplar +- User asks to attach a file to an order +- User asks the agent to demonstrate a pagination or idempotency workflow + +## Setup + +The CLI needs an API token. Either: + +- Set `EXEMPLAR_API_KEY` in the environment, or +- Run `exemplar-cli login --token ` to persist it at `~/.config/exemplar-cli/credentials.json`. + +Check status with `exemplar-cli login --status --json`. + +For tests, point at a mock server with `EXEMPLAR_BASE_URL=http://127.0.0.1:`. + +## Quick reference + +| Resource | Actions | Notes | +|---|---|---| +| `items` | list, get, create, update, delete | Cursor pagination on list (`--all`); `--idempotency-key` on create; `--if-match` on update | +| `item-variants` | list, create | Sub-resource of items; pass parent via `--itemId` | +| `orders` | list, get, create, upload | `--idempotency-key` on create; multipart upload via `--file` on upload | + +The login command is dispatched separately: `exemplar-cli login [--token ] [--status]`. + +Use `exemplar-cli --help` for per-action flags. + +## Global flags + +- `--json` — force JSON output (auto when piped) +- `--dry-run` — print request without sending +- `--verbose` — print response status & headers to stderr +- `--all` — auto-paginate list actions (cursor) +- `--version`, `-v` +- `--help`, `-h` + +## Knowledge system + +Read every file in `knowledge/` before issuing commands. Knowledge files capture API quirks, business rules, and patterns. After a non-trivial command sequence, append new findings as `knowledge/.md` with frontmatter `type:` set to `gotcha`, `pattern`, `shortcut`, `quirk`, or `business-rule`. + +## Common workflows + +### 1. Create an item with safe retries + +``` +exemplar-cli items create \ + --name "Widget" --sku "W-001" --price "19.99" \ + --idempotency-key "$(uuidgen)" +``` + +If the request fails with a network error, repeating the same command with the same idempotency key is safe — the server returns the original response. + +### 2. Walk a paginated list + +``` +exemplar-cli items list --all --json | jq '. | length' +``` + +The CLI iterates `nextCursor` until the server returns `null`. + +### 3. Attach a receipt to an order + +``` +exemplar-cli orders upload --id ord-42 --file ./receipt.pdf +``` + +The CLI sends `multipart/form-data` with a single `file` part. diff --git a/examples/exemplar-cli/test/_helpers.mjs b/examples/exemplar-cli/test/_helpers.mjs new file mode 100644 index 0000000..1194009 --- /dev/null +++ b/examples/exemplar-cli/test/_helpers.mjs @@ -0,0 +1,33 @@ +// Spawn the CLI in a child process and collect stdout/stderr. +// `runJson` adds --json and parses both streams as JSON when possible. +import { spawn } from "node:child_process"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +export const REPO_ROOT = resolve(__dirname, ".."); +export const CLI = resolve(REPO_ROOT, "bin/exemplar-cli.mjs"); + +export function run(args = [], { env = {}, timeoutMs = 10000 } = {}) { + return new Promise((resolveRun) => { + const cleanEnv = { ...process.env, ...env }; + delete cleanEnv.EXEMPLAR_API_KEY; + if (env.EXEMPLAR_API_KEY !== undefined) cleanEnv.EXEMPLAR_API_KEY = env.EXEMPLAR_API_KEY; + cleanEnv.__EXEMPLAR_FORCE_JSON_ERR = "1"; + const child = spawn(process.execPath, [CLI, ...args], { env: cleanEnv }); + let stdout = "", stderr = ""; + child.stdout.on("data", (d) => { stdout += d.toString(); }); + child.stderr.on("data", (d) => { stderr += d.toString(); }); + const t = setTimeout(() => child.kill("SIGKILL"), timeoutMs); + child.on("close", (code) => { clearTimeout(t); resolveRun({ stdout, stderr, exitCode: code ?? 0 }); }); + }); +} + +export async function runJson(args = [], opts = {}) { + const r = await run([...args, "--json"], opts); + let parsed = null; + if (r.stdout.trim()) { try { parsed = JSON.parse(r.stdout); } catch {} } + let parsedErr = null; + if (r.stderr.trim()) { try { parsedErr = JSON.parse(r.stderr.trim().split("\n").pop()); } catch {} } + return { ...r, json: parsed, errJson: parsedErr }; +} diff --git a/examples/jsonplaceholder-cli/test/_mock-server.mjs b/examples/exemplar-cli/test/_mock-server.mjs similarity index 100% rename from examples/jsonplaceholder-cli/test/_mock-server.mjs rename to examples/exemplar-cli/test/_mock-server.mjs diff --git a/examples/exemplar-cli/test/auth.test.mjs b/examples/exemplar-cli/test/auth.test.mjs new file mode 100644 index 0000000..eb3294e --- /dev/null +++ b/examples/exemplar-cli/test/auth.test.mjs @@ -0,0 +1,42 @@ +// Auth-specific paths: bearer token wiring, login --status reflection, +// 401 mapping. Splits out of integration.test.mjs to keep responsibilities +// per-file (mirrors agents-cli's per-feature test layout). +import test from "node:test"; +import assert from "node:assert/strict"; +import { mockApi } from "./_mock-server.mjs"; +import { run, runJson } from "./_helpers.mjs"; + +test("Authorization: Bearer header is sent when EXEMPLAR_API_KEY is set", async () => { + const server = await mockApi({ "GET /items": { status: 200, body: { items: [], nextCursor: null } } }); + try { + await runJson(["items", "list"], { env: { EXEMPLAR_API_KEY: "shh-secret-token", EXEMPLAR_BASE_URL: server.url } }); + assert.equal(server.requests[0].headers.authorization, "Bearer shh-secret-token"); + } finally { await server.close(); } +}); + +test("401 → auth_invalid", async () => { + const server = await mockApi({ "GET /items": { status: 401, body: { message: "bad token" } } }); + try { + const r = await runJson(["items", "list"], { env: { EXEMPLAR_API_KEY: "wrong", EXEMPLAR_BASE_URL: server.url } }); + assert.equal(r.exitCode, 1); + assert.equal(r.errJson.code, "auth_invalid"); + assert.equal(r.errJson.retryable, false); + } finally { await server.close(); } +}); + +test("403 → forbidden", async () => { + const server = await mockApi({ "GET /items": { status: 403, body: { message: "no scope" } } }); + try { + const r = await runJson(["items", "list"], { env: { EXEMPLAR_API_KEY: "scoped", EXEMPLAR_BASE_URL: server.url } }); + assert.equal(r.exitCode, 1); + assert.equal(r.errJson.code, "forbidden"); + } finally { await server.close(); } +}); + +test("login --status reports auth source", async () => { + const r = await runJson(["login", "--status"], { env: { EXEMPLAR_API_KEY: "live" } }); + assert.equal(r.exitCode, 0, r.stderr); + assert.equal(r.json.authenticated, true); + assert.equal(r.json.fromEnv, true); + assert.equal(r.json.scheme, "bearer"); +}); diff --git a/examples/exemplar-cli/test/integration.test.mjs b/examples/exemplar-cli/test/integration.test.mjs new file mode 100644 index 0000000..a36d4e8 --- /dev/null +++ b/examples/exemplar-cli/test/integration.test.mjs @@ -0,0 +1,220 @@ +// Integration tests against the bundled mock server. Exercises the full +// HTTP path: cursor pagination across two pages, idempotency-key headers, +// multipart upload, and the structured error map. +import test from "node:test"; +import assert from "node:assert/strict"; +import { writeFileSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { mockApi } from "./_mock-server.mjs"; +import { run, runJson } from "./_helpers.mjs"; + +const ENV = { EXEMPLAR_API_KEY: "test-token" }; + +async function withMock(routes, fn) { + const server = await mockApi(routes); + try { await fn(server); } finally { await server.close(); } +} + +// ---------- items resource ---------- + +test("items list returns array", async () => { + await withMock({ + "GET /items": { status: 200, body: { items: [{ id: "1" }, { id: "2" }], nextCursor: null } }, + }, async (server) => { + const r = await runJson(["items", "list"], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } }); + assert.equal(r.exitCode, 0, r.stderr); + assert.equal(r.json.items.length, 2); + assert.equal(server.requests[0].method, "GET"); + assert.equal(server.requests[0].path, "/items"); + }); +}); + +test("items list --all walks cursor pages and concatenates", async () => { + await withMock({ + "GET /items": (req) => { + const cursor = req.query.cursor; + if (!cursor) return { status: 200, body: { items: [{ id: "1" }, { id: "2" }], nextCursor: "page-2" } }; + if (cursor === "page-2") return { status: 200, body: { items: [{ id: "3" }], nextCursor: null } }; + return { status: 400, body: { message: "unknown cursor" } }; + }, + }, async (server) => { + const r = await runJson(["items", "list", "--all"], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } }); + assert.equal(r.exitCode, 0, r.stderr); + assert.equal(r.json.length, 3); + assert.deepEqual(r.json.map((i) => i.id), ["1", "2", "3"]); + assert.equal(server.requests.length, 2); + assert.equal(server.requests[1].query.cursor, "page-2"); + }); +}); + +test("items get interpolates id into path", async () => { + await withMock({ + "GET /items/:id": (_req, params) => ({ status: 200, body: { id: params.id, name: "Widget" } }), + }, async (server) => { + const r = await runJson(["items", "get", "--id", "42"], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } }); + assert.equal(r.exitCode, 0, r.stderr); + assert.equal(r.json.id, "42"); + assert.equal(server.requests[0].path, "/items/42"); + }); +}); + +test("items create sends body and idempotency-key header", async () => { + await withMock({ + "POST /items": (req) => ({ status: 201, body: { id: "new", ...req.body } }), + }, async (server) => { + const r = await runJson( + ["items", "create", "--name", "Widget", "--sku", "W-001", "--price", "9.99", "--idempotency-key", "abc-123"], + { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } } + ); + assert.equal(r.exitCode, 0, r.stderr); + assert.equal(r.json.id, "new"); + assert.equal(r.json.name, "Widget"); + assert.equal(server.requests[0].headers["idempotency-key"], "abc-123"); + assert.deepEqual(server.requests[0].body, { name: "Widget", sku: "W-001", price: "9.99" }); + }); +}); + +test("items update sends if-match header for optimistic concurrency", async () => { + await withMock({ + "PATCH /items/:id": (req, params) => ({ status: 200, body: { id: params.id, ...req.body } }), + }, async (server) => { + const r = await runJson( + ["items", "update", "--id", "7", "--name", "Updated", "--if-match", "etag-xyz"], + { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } } + ); + assert.equal(r.exitCode, 0, r.stderr); + assert.equal(server.requests[0].headers["if-match"], "etag-xyz"); + assert.equal(server.requests[0].body.name, "Updated"); + }); +}); + +test("items delete returns null body", async () => { + await withMock({ + "DELETE /items/:id": { status: 200, body: {} }, + }, async (server) => { + const r = await runJson(["items", "delete", "--id", "9"], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } }); + assert.equal(r.exitCode, 0, r.stderr); + assert.equal(server.requests[0].method, "DELETE"); + assert.equal(server.requests[0].path, "/items/9"); + }); +}); + +// ---------- item-variants sub-resource ---------- + +test("item-variants list interpolates parent id", async () => { + await withMock({ + "GET /items/:itemId/variants": (_req, params) => ({ status: 200, body: { items: [{ id: "v1", parent: params.itemId }], nextCursor: null } }), + }, async (server) => { + const r = await runJson(["item-variants", "list", "--itemId", "100"], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } }); + assert.equal(r.exitCode, 0, r.stderr); + assert.equal(server.requests[0].path, "/items/100/variants"); + assert.equal(r.json.items[0].parent, "100"); + }); +}); + +test("item-variants create sends body without itemId in payload", async () => { + await withMock({ + "POST /items/:itemId/variants": (req, params) => ({ status: 201, body: { id: "v2", ...req.body, parent: params.itemId } }), + }, async (server) => { + const r = await runJson( + ["item-variants", "create", "--itemId", "100", "--sku", "W-001-S", "--size", "S", "--color", "red"], + { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } } + ); + assert.equal(r.exitCode, 0, r.stderr); + assert.deepEqual(server.requests[0].body, { sku: "W-001-S", size: "S", color: "red" }); + assert.equal(server.requests[0].path, "/items/100/variants"); + }); +}); + +// ---------- orders + multipart ---------- + +test("orders list filters by status query", async () => { + await withMock({ + "GET /orders": (req) => ({ status: 200, body: { items: [{ id: "o1", status: req.query.status }], nextCursor: null } }), + }, async (server) => { + const r = await runJson(["orders", "list", "--status", "pending"], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } }); + assert.equal(r.exitCode, 0, r.stderr); + assert.equal(server.requests[0].query.status, "pending"); + }); +}); + +test("orders upload posts multipart/form-data with --file", async () => { + const tmp = mkdtempSync(join(tmpdir(), "exemplar-mp-")); + const filePath = join(tmp, "receipt.txt"); + writeFileSync(filePath, "RECEIPT"); + try { + await withMock({ + "POST /orders/:id/upload": (req, params) => ({ status: 200, body: { id: params.id, contentType: req.headers["content-type"], rawLen: req.raw.length } }), + }, async (server) => { + const r = await runJson(["orders", "upload", "--id", "ord-1", "--file", filePath], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } }); + assert.equal(r.exitCode, 0, r.stderr); + assert.match(r.json.contentType, /multipart\/form-data/); + assert.ok(r.json.rawLen > 0, "raw multipart body should be non-empty"); + }); + } finally { rmSync(tmp, { recursive: true, force: true }); } +}); + +// ---------- error paths ---------- + +test("404 → not_found", async () => { + await withMock({ "GET /items/:id": { status: 404, body: { message: "nope" } } }, async (server) => { + const r = await runJson(["items", "get", "--id", "999"], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } }); + assert.equal(r.exitCode, 1); + assert.equal(r.errJson.code, "not_found"); + assert.equal(r.errJson.retryable, false); + }); +}); + +test("422 → validation_error with details", async () => { + await withMock({ "POST /items": { status: 422, body: { message: "bad", errors: ["name required"] } } }, async (server) => { + const r = await runJson(["items", "create", "--name", ""], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } }); + assert.equal(r.exitCode, 1); + assert.equal(r.errJson.code, "validation_error"); + assert.ok(r.errJson.details); + }); +}); + +test("429 with Retry-After → rate_limited + retryAfter", async () => { + await withMock({ "GET /items": { status: 429, headers: { "retry-after": "30" }, body: { message: "slow down" } } }, async (server) => { + const r = await runJson(["items", "list"], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } }); + assert.equal(r.exitCode, 1); + assert.equal(r.errJson.code, "rate_limited"); + assert.equal(r.errJson.retryable, true); + assert.equal(r.errJson.retryAfter, 30); + }); +}); + +test("500 → server_error retryable", async () => { + await withMock({ "GET /items": { status: 500, body: { message: "boom" } } }, async (server) => { + const r = await runJson(["items", "list"], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } }); + assert.equal(r.exitCode, 1); + assert.equal(r.errJson.code, "server_error"); + assert.equal(r.errJson.retryable, true); + }); +}); + +test("network down → network_error", async () => { + const server = await mockApi({}); + const url = server.url; + await server.close(); + const r = await runJson(["items", "list"], { env: { ...ENV, EXEMPLAR_BASE_URL: url } }); + assert.equal(r.exitCode, 1); + assert.equal(r.errJson.code, "network_error"); + assert.equal(r.errJson.retryable, true); +}); + +test("BASE_URL override is honored", async () => { + await withMock({ "GET /items": { status: 200, body: { items: [], nextCursor: null } } }, async (server) => { + const r = await runJson(["items", "list"], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } }); + assert.equal(r.exitCode, 0); + assert.equal(server.requests.length, 1); + }); +}); + +test("user-agent header is sent", async () => { + await withMock({ "GET /items": { status: 200, body: { items: [], nextCursor: null } } }, async (server) => { + await runJson(["items", "list"], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } }); + assert.match(server.requests[0].headers["user-agent"], /exemplar-cli\//); + }); +}); diff --git a/examples/exemplar-cli/test/smoke.test.mjs b/examples/exemplar-cli/test/smoke.test.mjs new file mode 100644 index 0000000..ee09a57 --- /dev/null +++ b/examples/exemplar-cli/test/smoke.test.mjs @@ -0,0 +1,102 @@ +// Structural tests for exemplar-cli — no network, no auth, no surprises. +// Verifies CLI shape: --version, --help breadth, error semantics, --dry-run. +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { run, runJson, REPO_ROOT } from "./_helpers.mjs"; + +const RESOURCES = ["items", "item-variants", "orders"]; +const ITEMS_ACTIONS = ["list", "get", "create", "update", "delete"]; + +test("--version prints package.json version", async () => { + const pkg = JSON.parse(readFileSync(join(REPO_ROOT, "package.json"), "utf8")); + const r = await run(["--version"]); + assert.equal(r.exitCode, 0); + assert.equal(r.stdout.trim(), pkg.version); +}); + +test("--help lists all resources", async () => { + const r = await run(["--help"]); + assert.equal(r.exitCode, 0); + for (const res of RESOURCES) assert.match(r.stdout, new RegExp(`\\b${res}\\b`)); + assert.match(r.stdout, /\blogin\b/); +}); + +test(" --help lists all actions", async () => { + for (const a of ITEMS_ACTIONS) { + const r = await run(["items", "--help"]); + assert.equal(r.exitCode, 0, `items --help failed: ${r.stderr}`); + assert.match(r.stdout, new RegExp(`\\b${a}\\b`)); + } +}); + +test(" --help shows flag table", async () => { + const r = await run(["items", "create", "--help"]); + assert.equal(r.exitCode, 0); + assert.match(r.stdout, /Flags:/); + assert.match(r.stdout, /--name/); + assert.match(r.stdout, /--idempotency-key/); +}); + +test("unknown resource → validation_error", async () => { + const r = await runJson(["nope", "list"], { env: { EXEMPLAR_API_KEY: "test" } }); + assert.equal(r.exitCode, 1); + assert.equal(r.errJson.code, "validation_error"); +}); + +test("unknown action → validation_error listing available", async () => { + const r = await runJson(["items", "frobnicate"], { env: { EXEMPLAR_API_KEY: "test" } }); + assert.equal(r.exitCode, 1); + assert.equal(r.errJson.code, "validation_error"); + assert.match(r.errJson.message, /list|get|create|update|delete/); +}); + +test("required flag missing → validation_error (no crash)", async () => { + const r = await runJson(["items", "get"], { env: { EXEMPLAR_API_KEY: "test" } }); + assert.equal(r.exitCode, 1); + assert.equal(r.errJson.code, "validation_error"); + assert.match(r.errJson.message, /--id/); +}); + +test("auth missing → auth_missing", async () => { + const r = await runJson(["items", "list"]); + assert.equal(r.exitCode, 1); + assert.equal(r.errJson.code, "auth_missing"); +}); + +test("--dry-run does not make network requests", async () => { + const r = await runJson(["items", "list", "--dry-run"], { + env: { EXEMPLAR_API_KEY: "test", EXEMPLAR_BASE_URL: "http://127.0.0.1:1" }, + }); + assert.equal(r.exitCode, 0); + assert.equal(r.json.__dryRun, true); + assert.equal(r.json.method, "GET"); +}); + +test("source has no hardcoded secrets", () => { + const src = readFileSync(join(REPO_ROOT, "bin/exemplar-cli.mjs"), "utf8"); + const PATTERNS = [/sk_live_[A-Za-z0-9]{20,}/, /ghp_[A-Za-z0-9]{20,}/, /Bearer\s+[A-Za-z0-9_\-]{30,}/, /xoxb-[A-Za-z0-9-]{20,}/]; + for (const p of PATTERNS) assert.ok(!p.test(src), `secret pattern ${p} matched`); +}); + +test("every resource × action is reachable via --help", async () => { + const ACTIONS_BY_RESOURCE = { + items: ITEMS_ACTIONS, + "item-variants": ["list", "create"], + orders: ["list", "get", "create", "upload"], + }; + for (const [res, actions] of Object.entries(ACTIONS_BY_RESOURCE)) { + for (const a of actions) { + const r = await run([res, a, "--help"]); + assert.equal(r.exitCode, 0, `${res} ${a} --help failed: ${r.stderr}`); + } + } +}); + +test("login --help describes login flags", async () => { + const r = await run(["login", "--help"]); + assert.equal(r.exitCode, 0); + assert.match(r.stdout, /--token/); + assert.match(r.stdout, /--status/); +}); diff --git a/examples/legacy/README.md b/examples/legacy/README.md new file mode 100644 index 0000000..16d4405 --- /dev/null +++ b/examples/legacy/README.md @@ -0,0 +1,13 @@ +# Legacy exemplars + +These exemplars were used by earlier clify versions. They are no longer +the scaffolding stencil, but are kept as reference for the simple-API case +(no auth, no pagination, no business rules). + +| Exemplar | Last used | Status | +|---|---|---| +| `jsonplaceholder-cli/` | clify v0.2 | Reference only. Single-file bin, single skill. The current stencil is `examples/exemplar-cli/`, structurally inspired by `google/agents-cli`. | + +The `clify validate` gate still runs against legacy exemplars (so you can +verify the older shape if you maintain a generated CLI on it), but new +generation uses the current exemplar. diff --git a/examples/jsonplaceholder-cli/.claude-plugin/marketplace.json b/examples/legacy/jsonplaceholder-cli/.claude-plugin/marketplace.json similarity index 100% rename from examples/jsonplaceholder-cli/.claude-plugin/marketplace.json rename to examples/legacy/jsonplaceholder-cli/.claude-plugin/marketplace.json diff --git a/examples/jsonplaceholder-cli/.claude-plugin/plugin.json b/examples/legacy/jsonplaceholder-cli/.claude-plugin/plugin.json similarity index 100% rename from examples/jsonplaceholder-cli/.claude-plugin/plugin.json rename to examples/legacy/jsonplaceholder-cli/.claude-plugin/plugin.json diff --git a/examples/jsonplaceholder-cli/.clify.json b/examples/legacy/jsonplaceholder-cli/.clify.json similarity index 100% rename from examples/jsonplaceholder-cli/.clify.json rename to examples/legacy/jsonplaceholder-cli/.clify.json diff --git a/examples/jsonplaceholder-cli/.env.example b/examples/legacy/jsonplaceholder-cli/.env.example similarity index 100% rename from examples/jsonplaceholder-cli/.env.example rename to examples/legacy/jsonplaceholder-cli/.env.example diff --git a/examples/legacy/jsonplaceholder-cli/.github/workflows/test.yml b/examples/legacy/jsonplaceholder-cli/.github/workflows/test.yml new file mode 100644 index 0000000..9e61709 --- /dev/null +++ b/examples/legacy/jsonplaceholder-cli/.github/workflows/test.yml @@ -0,0 +1,16 @@ +name: test +on: + push: + pull_request: +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + node: ["20", "22"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - run: npm test diff --git a/examples/legacy/jsonplaceholder-cli/.gitignore b/examples/legacy/jsonplaceholder-cli/.gitignore new file mode 100644 index 0000000..5be869c --- /dev/null +++ b/examples/legacy/jsonplaceholder-cli/.gitignore @@ -0,0 +1,5 @@ +.env +node_modules/ +.docs-snapshot/ +*.log +.DS_Store diff --git a/examples/jsonplaceholder-cli/AGENTS.md b/examples/legacy/jsonplaceholder-cli/AGENTS.md similarity index 100% rename from examples/jsonplaceholder-cli/AGENTS.md rename to examples/legacy/jsonplaceholder-cli/AGENTS.md diff --git a/examples/legacy/jsonplaceholder-cli/LICENSE b/examples/legacy/jsonplaceholder-cli/LICENSE new file mode 100644 index 0000000..a2e3f24 --- /dev/null +++ b/examples/legacy/jsonplaceholder-cli/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 clify contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/examples/jsonplaceholder-cli/README.md b/examples/legacy/jsonplaceholder-cli/README.md similarity index 100% rename from examples/jsonplaceholder-cli/README.md rename to examples/legacy/jsonplaceholder-cli/README.md diff --git a/examples/jsonplaceholder-cli/bin/jsonplaceholder-cli.mjs b/examples/legacy/jsonplaceholder-cli/bin/jsonplaceholder-cli.mjs similarity index 100% rename from examples/jsonplaceholder-cli/bin/jsonplaceholder-cli.mjs rename to examples/legacy/jsonplaceholder-cli/bin/jsonplaceholder-cli.mjs diff --git a/examples/jsonplaceholder-cli/coverage.json b/examples/legacy/jsonplaceholder-cli/coverage.json similarity index 100% rename from examples/jsonplaceholder-cli/coverage.json rename to examples/legacy/jsonplaceholder-cli/coverage.json diff --git a/examples/jsonplaceholder-cli/knowledge/.gitkeep b/examples/legacy/jsonplaceholder-cli/knowledge/.gitkeep similarity index 100% rename from examples/jsonplaceholder-cli/knowledge/.gitkeep rename to examples/legacy/jsonplaceholder-cli/knowledge/.gitkeep diff --git a/examples/jsonplaceholder-cli/knowledge/writes-do-not-persist.md b/examples/legacy/jsonplaceholder-cli/knowledge/writes-do-not-persist.md similarity index 100% rename from examples/jsonplaceholder-cli/knowledge/writes-do-not-persist.md rename to examples/legacy/jsonplaceholder-cli/knowledge/writes-do-not-persist.md diff --git a/examples/jsonplaceholder-cli/package.json b/examples/legacy/jsonplaceholder-cli/package.json similarity index 100% rename from examples/jsonplaceholder-cli/package.json rename to examples/legacy/jsonplaceholder-cli/package.json diff --git a/examples/jsonplaceholder-cli/skills/jsonplaceholder/SKILL.md b/examples/legacy/jsonplaceholder-cli/skills/jsonplaceholder/SKILL.md similarity index 100% rename from examples/jsonplaceholder-cli/skills/jsonplaceholder/SKILL.md rename to examples/legacy/jsonplaceholder-cli/skills/jsonplaceholder/SKILL.md diff --git a/examples/jsonplaceholder-cli/skills/sync/SKILL.md b/examples/legacy/jsonplaceholder-cli/skills/sync/SKILL.md similarity index 100% rename from examples/jsonplaceholder-cli/skills/sync/SKILL.md rename to examples/legacy/jsonplaceholder-cli/skills/sync/SKILL.md diff --git a/examples/jsonplaceholder-cli/test/_helpers.mjs b/examples/legacy/jsonplaceholder-cli/test/_helpers.mjs similarity index 100% rename from examples/jsonplaceholder-cli/test/_helpers.mjs rename to examples/legacy/jsonplaceholder-cli/test/_helpers.mjs diff --git a/examples/legacy/jsonplaceholder-cli/test/_mock-server.mjs b/examples/legacy/jsonplaceholder-cli/test/_mock-server.mjs new file mode 100644 index 0000000..7a0ff28 --- /dev/null +++ b/examples/legacy/jsonplaceholder-cli/test/_mock-server.mjs @@ -0,0 +1,98 @@ +// Zero-dep HTTP mock for integration tests. +// Contract: see references/conventions.md "Mock Server Contract". +import { createServer } from "node:http"; + +export async function mockApi(routes) { + const requests = []; + + const server = createServer(async (req, res) => { + const chunks = []; + for await (const c of req) chunks.push(c); + const raw = Buffer.concat(chunks).toString("utf8"); + const ct = (req.headers["content-type"] || "").toLowerCase(); + let body = raw; + if (ct.includes("application/json") && raw) { + try { body = JSON.parse(raw); } catch { /* leave as string */ } + } + + const urlObj = new URL(req.url, "http://placeholder"); + const path = urlObj.pathname; + const captured = { + method: req.method, + path, + query: Object.fromEntries(urlObj.searchParams.entries()), + headers: { ...req.headers }, + body, + raw, + }; + requests.push(captured); + + const match = matchRoute(routes, req.method, path); + if (!match) { + res.writeHead(404, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: "no mock route", method: req.method, path })); + return; + } + + let result = match.handler; + if (typeof result === "function") { + result = await result(captured, match.params); + } + + const status = result.status ?? 200; + const headers = { "content-type": "application/json", ...(result.headers || {}) }; + res.writeHead(status, headers); + if (result.body === undefined || result.body === null) { + res.end(); + } else if (typeof result.body === "string") { + res.end(result.body); + } else { + res.end(JSON.stringify(result.body)); + } + }); + + await new Promise((resolveListen, rejectListen) => { + server.once("error", rejectListen); + server.listen(0, "127.0.0.1", () => { + server.removeListener("error", rejectListen); + resolveListen(); + }); + }); + + const addr = server.address(); + const url = `http://127.0.0.1:${addr.port}`; + + return { + url, + requests, + async close() { + server.closeAllConnections?.(); + await new Promise((r) => server.close(() => r())); + }, + }; +} + +function matchRoute(routes, method, path) { + for (const [key, handler] of Object.entries(routes)) { + const [routeMethod, routePath] = key.split(/\s+/); + if (routeMethod !== method) continue; + const params = matchPath(routePath, path); + if (params) return { handler, params }; + } + return null; +} + +function matchPath(template, actual) { + const tParts = template.split("/").filter(Boolean); + const aParts = actual.split("/").filter(Boolean); + if (tParts.length !== aParts.length) return null; + const params = {}; + for (let i = 0; i < tParts.length; i++) { + if (tParts[i].startsWith(":")) { + params[tParts[i].slice(1)] = decodeURIComponent(aParts[i]); + } else if (tParts[i] !== aParts[i]) { + return null; + } + } + return params; +} diff --git a/examples/jsonplaceholder-cli/test/integration.test.mjs b/examples/legacy/jsonplaceholder-cli/test/integration.test.mjs similarity index 100% rename from examples/jsonplaceholder-cli/test/integration.test.mjs rename to examples/legacy/jsonplaceholder-cli/test/integration.test.mjs diff --git a/examples/jsonplaceholder-cli/test/smoke.test.mjs b/examples/legacy/jsonplaceholder-cli/test/smoke.test.mjs similarity index 100% rename from examples/jsonplaceholder-cli/test/smoke.test.mjs rename to examples/legacy/jsonplaceholder-cli/test/smoke.test.mjs diff --git a/lib/scaffold-init.mjs b/lib/scaffold-init.mjs index 11438a2..adcc37d 100644 --- a/lib/scaffold-init.mjs +++ b/lib/scaffold-init.mjs @@ -6,11 +6,11 @@ import { fileURLToPath } from "node:url"; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(__dirname, ".."); -const EXEMPLAR_DIR = join(REPO_ROOT, "examples/jsonplaceholder-cli"); +const EXEMPLAR_DIR = join(REPO_ROOT, "examples/exemplar-cli"); -const FROM = "jsonplaceholder"; -const FROM_UPPER = "JSONPLACEHOLDER"; -const FROM_TITLE = "JSONPlaceholder"; +const FROM = "exemplar"; +const FROM_UPPER = "EXEMPLAR"; +const FROM_TITLE = "Exemplar"; const TEXT_EXTENSIONS = new Set(["mjs", "js", "json", "md", "yml", "yaml", "txt", "example", "gitkeep", "gitignore"]); diff --git a/lib/validate.mjs b/lib/validate.mjs index 317a136..b927f8a 100644 --- a/lib/validate.mjs +++ b/lib/validate.mjs @@ -77,6 +77,26 @@ function readText(path) { try { return readFileSync(path, "utf8"); } catch { return null; } } +// Concatenate every CLI source file the validator should consider when +// checking for env var lookups, helper imports, or wiring strings. +// Old layout: just `bin/-cli.mjs`. New layout adds `lib/*.mjs` and +// `commands/*.mjs`. Both are scanned together; either layout passes. +function readCliSources(repoDir, binRel) { + const parts = []; + const binPath = join(repoDir, binRel); + if (existsSync(binPath)) parts.push(readText(binPath) || ""); + for (const sub of ["lib", "commands"]) { + const subPath = join(repoDir, sub); + if (!existsSync(subPath)) continue; + for (const entry of readdirSync(subPath, { withFileTypes: true })) { + if (entry.isFile() && entry.name.endsWith(".mjs")) { + parts.push(readText(join(subPath, entry.name)) || ""); + } + } + } + return parts.join("\n"); +} + // ---------- 1. manifests ---------- async function checkManifests(ctx) { @@ -198,18 +218,18 @@ async function checkSourceConventions(ctx) { const binEntries = pkg.bin ? Object.values(pkg.bin) : []; if (binEntries.length === 0) return; const binPath = join(ctx.dir, binEntries[0]); - const src = readText(binPath); - if (!src) { ctx.results.push(fail("manifest", `bin file not found at ${binEntries[0]}`)); return; } + if (!existsSync(binPath)) { ctx.results.push(fail("manifest", `bin file not found at ${binEntries[0]}`)); return; } - // BASE_URL override + // BASE_URL override — checked across bin + lib/ + commands/ since machinery + // commonly lives in lib/api.mjs or similar. + const cliSrc = readCliSources(ctx.dir, binEntries[0]); const apiName = pkg.name.replace(/-cli$/, "").toUpperCase().replace(/-/g, "_"); const baseEnv = `${apiName}_BASE_URL`; - if (!src.includes(baseEnv)) ctx.results.push(fail("manifest", `CLI does not honor ${baseEnv} env override`)); + if (!cliSrc.includes(baseEnv)) ctx.results.push(fail("manifest", `CLI does not honor ${baseEnv} env override`)); else ctx.results.push(pass("manifest", "CLI honors BASE_URL override")); // package.json bin path resolves - if (existsSync(binPath)) ctx.results.push(pass("manifest", "package.json bin path resolves")); - else ctx.results.push(fail("manifest", "package.json bin path does not resolve", { binPath })); + ctx.results.push(pass("manifest", "package.json bin path resolves")); } // ---------- 4. secrets ---------- @@ -310,12 +330,19 @@ async function checkStructural(ctx) { if (resourceHelpFailures.length === 0) ctx.results.push(pass("structural", "every action appears in resource --help")); else ctx.results.push(fail("structural", "actions missing from resource --help", { failures: resourceHelpFailures })); - // SKILL.md mentions each resource + // Primary skill must mention every resource and the knowledge dir. + // Modular layout (preferred): skills/-workflow/SKILL.md. + // Legacy single-skill layout: skills//SKILL.md. if (pkg.bin) { const apiSlug = pkg.name.replace(/-cli$/, ""); - const skillPath = join(ctx.dir, `skills/${apiSlug}/SKILL.md`); - if (!existsSync(skillPath)) { - ctx.results.push(fail("structural", `skills/${apiSlug}/SKILL.md missing`)); + const modularPath = join(ctx.dir, `skills/${pkg.name}-workflow/SKILL.md`); + const legacyPath = join(ctx.dir, `skills/${apiSlug}/SKILL.md`); + let skillPath = null; + if (existsSync(modularPath)) skillPath = modularPath; + else if (existsSync(legacyPath)) skillPath = legacyPath; + + if (!skillPath) { + ctx.results.push(fail("structural", `primary SKILL.md missing (looked at skills/${pkg.name}-workflow/ and skills/${apiSlug}/)`)); } else { const skill = readText(skillPath); const missing = []; @@ -338,7 +365,9 @@ async function checkNuances(ctx) { const nuances = cfg.nuances || {}; const pkg = readJson(join(ctx.dir, "package.json")); const binEntries = pkg.bin ? Object.values(pkg.bin) : []; - const src = binEntries.length ? (readText(join(ctx.dir, binEntries[0])) || "") : ""; + // Search across bin + lib/ + commands/ — wiring like FormData and + // idempotency-key headers commonly lives in lib/api.mjs. + const src = binEntries.length ? readCliSources(ctx.dir, binEntries[0]) : ""; const integrationTest = readText(join(ctx.dir, "test/integration.test.mjs")) || ""; // pagination diff --git a/package.json b/package.json index bb5b676..ad373ae 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "clify", - "version": "0.2.0", - "description": "Generate A+ Node.js CLIs from API documentation. Copy a hand-crafted exemplar, mechanically substitute API-specific content, then verify with a deterministic validation gate.", + "version": "0.3.0", + "description": "Generate A+ Node.js CLIs from API documentation. Copies a hand-crafted exemplar (structurally inspired by google/agents-cli), mechanically substitutes API-specific content, and verifies via a deterministic validation gate.", "type": "module", "bin": { "clify": "./bin/clify.mjs" @@ -11,7 +11,9 @@ }, "scripts": { "test": "node --test test/*.test.mjs", - "validate-exemplar": "node bin/clify.mjs validate examples/jsonplaceholder-cli" + "test:exemplar": "cd examples/exemplar-cli && npm test", + "validate-exemplar": "node bin/clify.mjs validate examples/exemplar-cli", + "eval": "node bin/clify.mjs eval" }, "license": "MIT", "repository": { diff --git a/references/conventions.md b/references/conventions.md index d71d05a..0794158 100644 --- a/references/conventions.md +++ b/references/conventions.md @@ -2,7 +2,7 @@ Rules and contracts for generated CLI repos. Rigid contracts (error format, `.clify.json` shape, env var override) **must** be followed exactly. Flexible guidance (command structure, nesting) should be adapted to the API. -The hand-crafted exemplar at [`examples/jsonplaceholder-cli/`](../examples/jsonplaceholder-cli/) is the canonical implementation of every rule below. When a contract here is ambiguous, the exemplar wins. +The hand-crafted exemplar at [`examples/exemplar-cli/`](../examples/exemplar-cli/) is the canonical implementation of every rule below. When a contract here is ambiguous, the exemplar wins. The exemplar is structurally inspired by [`google/agents-cli`](https://github.com/google/agents-cli) — hierarchical subcommands, one file per resource under `commands/`, shared machinery under `lib/`, modular skills. --- @@ -49,7 +49,7 @@ Parsed before resource routing. Use a known-set filter — never `parseArgs({ st | `--verbose` | false | Include request/response headers | | `--all` | false | Auto-paginate | -Reference impl: [`examples/jsonplaceholder-cli/bin/jsonplaceholder-cli.mjs`](../examples/jsonplaceholder-cli/bin/jsonplaceholder-cli.mjs). +Reference impl: [`examples/exemplar-cli/bin/exemplar-cli.mjs`](../examples/exemplar-cli/bin/exemplar-cli.mjs). --- @@ -73,7 +73,7 @@ Reference impl: [`examples/jsonplaceholder-cli/bin/jsonplaceholder-cli.mjs`](../ **Every generated CLI honors `_BASE_URL` env var to redirect requests to a mock server.** Default is the real API base URL. Without this, integration tests cannot reach the mock; the validation gate fails generation that omits this. ```js -const BASE_URL = process.env.JSONPLACEHOLDER_BASE_URL || "https://jsonplaceholder.typicode.com"; +const BASE_URL = process.env.EXEMPLAR_BASE_URL || "https://api.exemplar.test"; ``` The env var name is `_BASE_URL` — same prefix used for the API key and other defaults. @@ -86,12 +86,12 @@ The env var name is `_BASE_URL` — same prefix used for t ```js const server = await mockApi({ - "GET /posts": { status: 200, body: [{ id: 1, title: "x" }] }, - "GET /posts/:id": (req, params) => ({ status: 200, body: { id: Number(params.id) } }), - "POST /posts": (req) => ({ status: 201, body: { id: 101, ...req.body } }), - "GET /rate": { status: 429, headers: { "retry-after": "30" } }, + "GET /items": { status: 200, body: { items: [{ id: "1" }], nextCursor: null } }, + "GET /items/:id": (req, params) => ({ status: 200, body: { id: params.id } }), + "POST /items": (req) => ({ status: 201, body: { id: "new", ...req.body } }), + "GET /orders": { status: 429, headers: { "retry-after": "30" } }, }); -process.env.JSONPLACEHOLDER_BASE_URL = server.url; // overrides default in CLI +process.env.EXEMPLAR_BASE_URL = server.url; // overrides default in CLI // ...run CLI, then: assert.equal(server.requests[0].headers.authorization, "Bearer test"); await server.close(); @@ -106,7 +106,7 @@ Determinism rules (must hold on Node 20 and 22): - `close()` returns a Promise that resolves only after all sockets are drained. - Routes match `METHOD /path` exactly; `:param` placeholders extract path params; no regex. -Reference impl: [`examples/jsonplaceholder-cli/test/_mock-server.mjs`](../examples/jsonplaceholder-cli/test/_mock-server.mjs). +Reference impl: [`examples/exemplar-cli/test/_mock-server.mjs`](../examples/exemplar-cli/test/_mock-server.mjs). --- @@ -148,32 +148,32 @@ Root metadata, written by the scaffold skill, read by the validator and sync too ```json { - "apiName": "jsonplaceholder", - "docsUrl": "https://jsonplaceholder.typicode.com", - "crawledUrls": ["https://jsonplaceholder.typicode.com/guide/"], + "apiName": "exemplar", + "docsUrl": "https://docs.exemplar.test/api/v1", + "crawledUrls": ["https://docs.exemplar.test/api/v1"], "contentHash": "sha256:abc...", "generatedAt": "2026-04-26T00:00:00Z", - "clifyVersion": "0.2.0", + "clifyVersion": "0.3.0", "nodeMinVersion": "20", "auth": { - "envVar": "JSONPLACEHOLDER_API_KEY", - "scheme": "none", - "validationCommand": "posts list" + "envVar": "EXEMPLAR_API_KEY", + "scheme": "bearer", + "validationCommand": "items list" }, "defaults": [], "nuances": { - "pagination": null, - "rateLimits": false, + "pagination": "cursor", + "rateLimits": true, "authScopes": false, "deprecated": [], - "idempotency": [], - "multiPart": [], - "conditional": [], - "businessRules": 0 + "idempotency": ["items.create", "orders.create"], + "multiPart": ["orders.upload"], + "conditional": ["items.update"], + "businessRules": 1 }, "coverage": { - "totalParsed": 6, - "totalIncluded": 6, + "totalParsed": 11, + "totalIncluded": 11, "totalDropped": 0 } } @@ -267,7 +267,7 @@ Setup lives in the generated SKILL.md — no CLI binary changes. The LLM follows | `@validation-command ` | CLI command exercising this credential | | `@detect-command ` | Lists possible values | -Reference: [`examples/jsonplaceholder-cli/.env.example`](../examples/jsonplaceholder-cli/.env.example). +Reference: [`examples/exemplar-cli/.env.example`](../examples/exemplar-cli/.env.example). ### Placeholder Detection @@ -305,15 +305,67 @@ Generated SKILL.md preamble must include: **"Before running any command, read ev ### Code Structure +The exemplar shape (preferred — every newly generated CLI inherits it): + +``` +bin/-cli.mjs thin dispatcher +lib/api.mjs apiRequest + cursor pagination iterator +lib/auth.mjs pluggable auth (bearer | api-key-header | basic | none) +lib/output.mjs output() + errorOut() +lib/config.mjs ~/.config/-cli/credentials.json store (used by login) +lib/env.mjs .env loader (zero-dep) +lib/args.mjs splitGlobal + parseArgs adapters +lib/help.mjs help-text generators (read the registry) +commands/.mjs one file per resource, default-exports { name, actions, buildPayload? } +commands/login.mjs token persistence + --status (only when scheme ≠ none) +test/smoke.test.mjs smoke tests +test/integration.test.mjs mock-server-driven integration tests +test/auth.test.mjs bearer/api-key wiring + login --status +test/_mock-server.mjs mock-server helper +test/_helpers.mjs run/runJson child-process harness +``` + +Legacy single-file shape (still passes the gate, but new generation uses the split above): + ``` -bin/-cli.mjs single-file CLI, all resources -test/smoke.test.mjs smoke tests -test/integration.test.mjs mock-server-driven integration tests -test/_mock-server.mjs mock-server helper +bin/-cli.mjs everything in one file ``` Resource handlers are plain objects (not classes). One `apiRequest()` handles auth, dry-run, verbose, error mapping. Version read from package.json. +### Hierarchical subcommands + +Every command is one of: + +- `-cli [flags]` — the default shape +- `-cli login [--token ] [--status]` — auth management (when scheme ≠ none) + +Resources, actions, and the registry that maps them to method/path/flags all live under `commands/.mjs`. The bin file imports each, builds a single `REGISTRY` object, and dispatches. + +### Pluggable auth + +`lib/auth.mjs` exports `applyAuth(headers)`. The function: + +1. Reads `process.env._API_KEY` (or stored credential from `lib/config.mjs`). +2. Branches on `SCHEME` — one of `bearer | api-key-header | basic | none`. +3. Mutates the `headers` map with the right header. +4. Returns `{ ok, reason }` so `apiRequest` can fail fast with `auth_missing`. + +Adding a new scheme is a registry edit — branch on `SCHEME`, set the header, done. Don't fork `apiRequest`. + +### Modular skills + +Generated repos ship four skills under `skills/-cli-/SKILL.md`: + +| Skill | Purpose | +|---|---| +| `-cli-workflow` | End-to-end workflows. Mentions every resource and the `knowledge/` dir. **The validation gate looks for this file.** | +| `-cli-auth` | Auth setup, login, troubleshooting 401/403 | +| `-cli-resources` | Resource × action × flag quick reference | +| `-cli-knowledge` | How to write and consume `knowledge/*.md` | + +The legacy single-skill layout (`skills//SKILL.md`) still passes the validation gate as a fallback, but new generation uses the modular layout. + ### Three-level help `--help` → resources & actions. ` --help` → actions. ` --help` → per-action flags with required/optional + descriptions. @@ -342,7 +394,7 @@ Verify CLI structure, NOT API responses. Pass with no `.env` present. `node:test`. Helpers `run(...args)`, `runJson(...args)`. Strip API key from env in helper. 5s timeout. -Reference: [`examples/jsonplaceholder-cli/test/smoke.test.mjs`](../examples/jsonplaceholder-cli/test/smoke.test.mjs). +Reference: [`examples/exemplar-cli/test/smoke.test.mjs`](../examples/exemplar-cli/test/smoke.test.mjs). --- @@ -359,7 +411,7 @@ For each resource, exercise every declared action against `_mock-server.mjs`: - Auth path (when scheme ≠ none): missing key → `auth_missing`; bad key → `auth_invalid` - Network down: dial unreachable port → `network_error` -Reference: [`examples/jsonplaceholder-cli/test/integration.test.mjs`](../examples/jsonplaceholder-cli/test/integration.test.mjs). +Reference: [`examples/exemplar-cli/test/integration.test.mjs`](../examples/exemplar-cli/test/integration.test.mjs). --- @@ -371,7 +423,7 @@ Reference: [`examples/jsonplaceholder-cli/test/integration.test.mjs`](../example - Matrix: Node 20, 22 - Step: `npm test` (runs both smoke and integration) -Reference: [`examples/jsonplaceholder-cli/.github/workflows/test.yml`](../examples/jsonplaceholder-cli/.github/workflows/test.yml). +Reference: [`examples/exemplar-cli/.github/workflows/test.yml`](../examples/exemplar-cli/.github/workflows/test.yml). --- @@ -380,13 +432,19 @@ Reference: [`examples/jsonplaceholder-cli/.github/workflows/test.yml`](../exampl ``` -cli/ ├── bin/-cli.mjs +├── lib/ # api, auth, output, config, env, args, help +├── commands/ # one file per resource + login ├── skills/ -│ ├── /SKILL.md -│ └── sync/SKILL.md -├── knowledge/ # initially has .gitkeep + any business-rules extracted +│ ├── -cli-workflow/SKILL.md # primary; validate looks here +│ ├── -cli-auth/SKILL.md +│ ├── -cli-resources/SKILL.md +│ └── -cli-knowledge/SKILL.md +├── knowledge/ # business-rules + patterns extracted from docs ├── test/ │ ├── smoke.test.mjs │ ├── integration.test.mjs +│ ├── auth.test.mjs +│ ├── _helpers.mjs │ └── _mock-server.mjs ├── .claude-plugin/ │ ├── plugin.json @@ -437,8 +495,10 @@ No dependencies. No devDependencies. "author": { "name": "" }, "license": "MIT", "skills": [ - { "name": "", "source": "skills//SKILL.md" }, - { "name": "sync", "source": "skills/sync/SKILL.md" } + { "name": "-cli-workflow", "source": "skills/-cli-workflow/SKILL.md" }, + { "name": "-cli-auth", "source": "skills/-cli-auth/SKILL.md" }, + { "name": "-cli-resources", "source": "skills/-cli-resources/SKILL.md" }, + { "name": "-cli-knowledge", "source": "skills/-cli-knowledge/SKILL.md" } ], "capabilities": ["network"] } diff --git a/references/scaffold-pipeline.md b/references/scaffold-pipeline.md new file mode 100644 index 0000000..b35d78a --- /dev/null +++ b/references/scaffold-pipeline.md @@ -0,0 +1,277 @@ +# Scaffold pipeline — per-phase detail + +The `clify` skill's six phases are summarized in `skills/clify/SKILL.md`. This file is the in-depth reference for each phase, covering edge cases and the file-by-file substitution checklist. Read it when the lean SKILL.md is too terse for the situation at hand — you don't need to load it on every scaffolding run. + +--- + +## Phase 1: Fetch & detect + +### OpenAPI / Swagger fast path + +If the URL serves JSON or YAML with a top-level `openapi:` or `swagger:` key, parse it directly. Fields you care about: + +- `paths..` → method, path, operation id +- `parameters` → required path/query params (per-method or path-level) +- `requestBody.content..schema` → body shape; `multipart/form-data` triggers the multipart nuance +- `responses.*.headers["Retry-After"]` / `X-RateLimit-*` → rate-limit nuance signal +- `deprecated: true` → deprecated nuance +- `securitySchemes` → auth scheme detection (`http: bearer` → `bearer`, `apiKey: header` → `api-key-header`, `http: basic` → `basic`) + +### HTML / Markdown crawl + +If the URL serves HTML or Markdown, use `WebFetch` and: + +1. Extract anchor URLs that look like API doc pages — strip nav, marketing, changelog, examples-only pages. +2. Normalize URLs (drop fragments, lowercase host, strip trailing slash). +3. Fetch up to depth 2 with 1s rate limit. +4. Stop early if the dedupe set converges (no new URLs at depth N). + +If a fetched body is empty or shorter than ~200 bytes, the page is JS-rendered. Ask the user via `AskUserQuestion`: + +> "This page renders client-side and I can't extract endpoints from it. Do you have an OpenAPI/Swagger URL, or should I try a different docs URL?" + +--- + +## Phase 2: Parse & group + +### Verb mapping + +| HTTP | Action | Notes | +|---|---|---| +| GET (collection) | `list` | Returns array; honor pagination headers/body | +| GET (single, has `:id` in path) | `get` | Requires `--id` flag | +| POST | `create` | Returns the created resource (or 201 with location header) | +| PUT/PATCH | `update` | Requires `--id` flag | +| DELETE | `delete` | Requires `--id` flag | + +Anything else: use the API's own verb (`approve`, `cancel`, `merge-upstream`). + +### Resource grouping + +Group by the first non-version path segment. `/v1/items/:id` → `items`. `/api/v2/customers/:id/orders` → `customers` with `orders` as a sub-resource (flatten to `customer-orders`). + +### Nesting cap + +Two levels max — ` ` or ` `. Anything deeper gets flattened into a hyphenated resource name. If a single resource has more than ~10 actions, split it into sub-resources rather than packing them all into one command file. + +### Nuance detection cheat sheet + +| Signal | Nuance | Artifact | +|---|---|---| +| `cursor` / `next_page_token` / `page` / `offset` / `Link: rel=next` in responses | pagination=cursor\|page\|offset\|link-header | Multi-page integration test | +| `Idempotency-Key` header documented | idempotency=[....] | `--idempotency-key` flag wired; integration test asserts header | +| `multipart/form-data` content type | multiPart=[....] | `--file` flag wired; integration test posts a fixture | +| `deprecated: true` or "deprecated" prose | deprecated=[....] | `knowledge/deprecated-.md` OR coverage drop with reason `deprecated-in-docs` | +| `X-RateLimit-*` headers, "rate limit" prose | rateLimits=true | `knowledge/rate-limit.md` (soft warn) | +| OAuth scopes documented | authScopes=true | `knowledge/auth-scopes.md` (soft warn) | +| `If-Match` / `ETag` documented | conditional=[....] | `--if-match` flag (soft warn) | + +--- + +## Phase 3: Consult + +### Resource grouping + +Surface ambiguous groupings via `AskUserQuestion`: + +> "I parsed N endpoints into K resources. Two are ambiguous: should `/v1/users/:id/preferences` be a sub-resource of `users` (flattened to `user-preferences`) or its own top-level resource `preferences`? My default is `user-preferences`." + +### Drop list + +If you found endpoints that won't fit (webhook subscription POSTs, streaming SSE endpoints, deeply-nested actions, beta-flagged endpoints), surface the list: + +> "I'm going to drop these N endpoints. Reasons: [...]. Confirm or call out any I should keep?" + +Allowed drop reasons (validation gate enforces): `user-excluded-step-7`, `deprecated-in-docs`, `beta-flagged`, `internal-only`, `nesting-depth-cap`, `webhook-not-cli-shaped`, `streaming-not-cli-shaped`. + +### Repo location + +The generated CLI is a **separate project**, not nested inside the cwd. Default placement: `/-cli/`. Ask: + +> "Where should I put `-cli/`? Default is `` (sibling of your current directory). I'll also `git init` the new repo unless you say otherwise." + +Options: +- **Default** — parent of cwd. Right answer for almost every case. +- **Custom path** — user supplies absolute or relative. +- **Here (nested)** — only when explicit; warn that this couples the new CLI's git history to the parent. + +--- + +## Phase 4: Init from exemplar + +``` +clify scaffold-init --target +``` + +What this does: + +1. Copies `examples/exemplar-cli/` to `/-cli/` (refuses if dir exists). +2. Renames every file/dir containing `exemplar` to use `` instead. +3. In every text file, rewrites `exemplar` → ``, `EXEMPLAR` → ``, `Exemplar` → `` in that order. + +After scaffold-init: + +``` +cd /-cli && git init -q && git add -A && git commit -q -m "Initial scaffold from clify exemplar" +``` + +The initial commit makes Phase 5's substitutions a clean diff the user can review. + +--- + +## Phase 5: Substitute — file-by-file checklist + +### `commands/.mjs` + +Replace the exemplar's three command files (`items.mjs`, `item-variants.mjs`, `orders.mjs`, `login.mjs`) with one file per parsed resource. Each default-exports: + +```js +export default { + name: "", + actions: { + : { method, path, description, flags: { /* ... */ } }, + // ... + }, + buildPayload(values) { /* return JSON body */ }, +}; +``` + +Keep `login.mjs` if the API has authentication; delete it if `auth.scheme === "none"`. + +Flag rules: + +- Path params (`:id`, `:itemId`) become required string flags with the same name. +- Query params become optional flags. +- Body fields become individual flags. Common rule: keep a `--body ` raw escape hatch. +- Idempotent endpoints get `--idempotency-key` (a string flag). +- Conditional endpoints get `--if-match` (a string flag). +- Multipart upload endpoints take `--file ` (required). + +### `bin/-cli.mjs` + +Edit only: +- The `import` lines for each `commands/.mjs` file. +- The `COMMANDS = [...]` array. + +Leave `loadEnv`, `splitGlobal`, `runResourceAction`, `interpolatePath`, `main` exactly as in the exemplar. + +### `lib/auth.mjs` + +Edit `SCHEME` and `ENV_VAR` constants. Add a branch in `applyAuth` if your scheme isn't already there. The validation gate accepts: `bearer | api-key-header | basic | none`. For OAuth-style flows that need refresh, model them as `bearer` — the refresh logic lives in `login.mjs`. + +### `lib/api.mjs` + +Generally unchanged. Edit the `BASE_URL` default if the API doesn't have a single canonical URL (e.g., region-routed APIs may need a `_REGION` env var that selects the base URL — add it next to `BASE_URL`). + +### `.clify.json` + +```json +{ + "apiName": "", + "docsUrl": "", + "crawledUrls": [""], + "contentHash": "sha256:", + "generatedAt": "", + "clifyVersion": "", + "auth": { "envVar": "_API_KEY", "scheme": "", "validationCommand": " " }, + "nuances": { /* every detected nuance */ }, + "coverage": { "totalParsed": N, "totalIncluded": M, "totalDropped": K } +} +``` + +### `.env.example` + +For `scheme !== none`, include: + +``` +# @required +# @how-to-get +# @format +_API_KEY=your__api_key_here + +# @optional +# @validation-command +# _BASE_URL=http://127.0.0.1:3000 +``` + +### `skills/-cli-workflow/SKILL.md` + +Rewrite Triggers, Quick Reference table, Common Workflows. Preserve the YAML frontmatter shape (name, description, allowed-tools). The skill must mention every resource and the `knowledge/` directory. + +### `skills/-cli-{auth,resources,knowledge}/SKILL.md` + +Light edit. Auth: update token storage notes. Resources: regenerate the resource × action table from the parsed registry. Knowledge: list the knowledge files extracted from docs. + +### `test/integration.test.mjs` + +For each resource, exercise every action against the mock server. Fixture data should match the API's response shape. Add nuance tests: + +- Pagination → multi-page test using a function handler that toggles on `req.query.cursor`. +- Idempotency → assert `server.requests[0].headers["idempotency-key"]` is sent. +- Multipart → use a temp file and assert `content-type` matches `/multipart\/form-data/`. + +### `coverage.json` + +One entry per parsed endpoint. Schema: + +```json +{ "method": "GET", "path": "/items", "resource": "items", "action": "list", "included": true } +{ "method": "POST", "path": "/items/bulk", "resource": null, "action": null, "included": false, "dropped": true, "reason": "user-excluded-step-7" } +``` + +### `knowledge/.md` + +For each business rule, gotcha, or quirk extracted from prose. Frontmatter: + +```yaml +--- +type: business-rule | gotcha | pattern | shortcut | quirk +applies-to: [".", ...] +source: docs | runtime +confidence: high | medium | low +extracted: +--- +``` + +### `README.md` and `AGENTS.md` + +Re-author resource lists, common workflows, error handling. Keep the structural sections (Layout, Use, Test). + +--- + +## Phase 6: Validate, simplify, report + +### Failure remediation + +- **manifest** — fix the named field; cross-manifest mismatches usually mean `package.json` was edited but `plugin.json` wasn't. +- **coverage** — every `included: false` needs `dropped: true` + a valid `reason`. +- **structural** — a resource or action is in coverage but not in `--help`, or vice versa. Check the registry vs the help-text generator. +- **nuances** — declared a hard-fail nuance but didn't add the artifact. Either add it or set the nuance to empty. +- **secrets** — a real key pattern leaked. Remove it. +- **ci** — `.github/workflows/test.yml` missing or doesn't run `npm test`. +- **tests** — exemplar smoke/integration tests fail. Read `stderrTail` and fix the source. + +Up to 3 attempts. After that, surface the remaining failures to the user verbatim. Never claim done with failures. + +### Simplify pass + +After the gate passes, run `/simplify` over the files you edited. Look for: + +- Duplicated payload-building logic across `create`/`update` actions of the same resource → factor into `buildPayload`. +- Per-resource flag specs that share 80%+ fields → factor into a shared object spread. +- Help text that re-states what `flags[].description` already says → drop. + +Don't break a contract from `conventions.md` to win clarity. The contracts win. + +### Report + +End-of-scaffold summary should include: + +- API name + version (if known) +- Resources × actions: how many, listed +- Dropped endpoints: how many, with reasons +- Declared nuances: which fields are non-empty in `.clify.json.nuances` +- Knowledge files written: list +- Validation gate result: pass/fail with categories +- Path to generated repo +- Next step: `cd && npm test` diff --git a/skills/clify-scaffold/SKILL.md b/skills/clify-scaffold/SKILL.md deleted file mode 100644 index abd551f..0000000 --- a/skills/clify-scaffold/SKILL.md +++ /dev/null @@ -1,150 +0,0 @@ ---- -name: clify-scaffold -description: Generate an A+ Node.js CLI repo from an API documentation URL. Use when the user says "/clify-scaffold ", "/clify ", "generate a CLI for this API", or provides API docs and asks for an agent-friendly wrapper. Works by copying the bundled JSONPlaceholder exemplar and mechanically substituting API-specific content, then running the validation gate. -allowed-tools: - - Read - - Write - - Edit - - Bash - - Glob - - Grep - - WebFetch - - WebSearch - - AskUserQuestion ---- - -# clify-scaffold - -Generate a self-updating CLI repo from an API documentation URL. Output is installable as a Claude Code plugin, Codex agent, or standalone Node CLI. - -The strategy is **copy exemplar + mechanically substitute**, not "write from scratch." Quality comes from the exemplar (which is hand-crafted, tested, validated) — your job is the smaller, more verifiable transformation of API-specific content. - -## Triggers - -- `/clify-scaffold `, `/clify ` -- "Generate a CLI for this API" -- User provides an API docs URL and asks for a wrapper - -## Read these first - -1. [`references/conventions.md`](../../references/conventions.md) — every contract you must honor. -2. [`references/validation-gate.md`](../../references/validation-gate.md) — every check the gate enforces. The gate is the source of truth for "done." -3. [`examples/jsonplaceholder-cli/`](../../examples/jsonplaceholder-cli/) — the canonical exemplar. Section ordering, helper signatures, and code structure must match it. - -## Pipeline (13 steps) - -### 1. Fetch the docs - -`WebFetch ` for the user-provided URL. - -### 2. Detect format - -- **OpenAPI/Swagger** (JSON or YAML with `openapi` or `swagger` key) → parse directly. Skip crawling. -- **HTML / Markdown** → proceed to step 3. - -### 3. Crawl (HTML/Markdown only) - -Identify links to more API doc pages (skip nav, marketing, changelog). Fetch up to depth 2, dedupe by normalized URL, rate-limit at 1s between fetches. If the page is JS-rendered with empty content, ask the user for an OpenAPI URL instead. - -### 4. Parse endpoints - -Extract method, path, params, request/response shapes. Group by first path segment as resource. Apply standard verb mapping from `conventions.md` (CRUD); use the API's own verb for non-CRUD endpoints. - -### 5. Detect nuances - -Scan spec + prose for the nuances table in [`references/validation-gate.md`](../../references/validation-gate.md#nuances). Populate the `.clify.json` `nuances` map. **Hard-fail nuances** (pagination, idempotency, multipart, deprecated) require corresponding artifacts; **soft-warn nuances** are advisory. - -### 6. Extract business knowledge - -Second prose pass for non-endpoint rules (units, formats, defaults, sequencing, plan limits, identifier formats). Each rule queues a `knowledge/.md` file with `type: business-rule` frontmatter. See `examples/jsonplaceholder-cli/knowledge/writes-do-not-persist.md` for the shape. - -### 7. Consult and recommend - -Present findings to the user with opinionated recommendations. Use `AskUserQuestion` for unresolved tradeoffs (resource grouping, ambiguous verbs, what to drop). Record dropped endpoints with one of the allowed reasons (`user-excluded-step-7`, `deprecated-in-docs`, etc.) — never silently drop. - -### 8. Decide where the new repo lives, then init from exemplar - -The generated CLI is its own project — a separate directory, not nested inside whatever repo the user happened to invoke `/clify-scaffold` from. Before running the binary verb, **ask the user with `AskUserQuestion`**: - -> "Where should I put `-cli/`? Default is `` (sibling of your current directory). I'll also `git init` the new repo unless you say otherwise." - -Options to offer: -- **Default** — parent of cwd (`..`), so the layout is `parent//` next to `parent/-cli/`. Right answer for almost every case, including when the user runs the skill from inside the clify repo itself. -- **Custom path** — user supplies an absolute or relative directory. -- **Here (nested)** — only when the user is explicit; warn that this couples the new CLI's git history to the parent. - -Then run the binary verb with the chosen target: - -``` -clify scaffold-init --target -``` - -This is the **deterministic copy + rename phase** — never do it by hand. Copies `examples/jsonplaceholder-cli/` to `/-cli/` and rewrites `jsonplaceholder` → ``, `JSONPLACEHOLDER` → ``, `JSONPlaceholder` → `` everywhere. - -Print the resolved absolute path to the user so they can confirm before substantive content lands. - -After the copy, `git init` the new repo (skip if the user opted out): - -``` -cd /-cli && git init -q && git add -A && git commit -q -m "Initial scaffold from clify exemplar" -``` - -The initial commit makes step 9's substitutions a clean diff the user can review. - -### 9. Substitute API-specific content - -Edit only what changes per API; preserve section ordering and helper signatures from the exemplar. - -- `bin/-cli.mjs`: replace the resource registry with the parsed endpoints. Keep `loadEnv`, `apiRequest`, `output`, `errorOut`, `splitGlobal`, `toParseArgs`, `checkRequired`, `showRootHelp`, `showResourceHelp`, `showActionHelp`, `main` exactly as in the exemplar. Body builders (`buildPayload`, `bodyFlagsFor`) get rewritten per API. -- Auth: set `auth.scheme` and add the auth header in `apiRequest` (`Authorization: Bearer ...`, `X-API-Key: ...`, etc.). For `scheme: none`, leave the helper untouched. -- `test/integration.test.mjs`: rewrite per-resource fixtures to match target API shapes; add nuance tests as required by §5. -- `skills//SKILL.md`: rewrite Triggers, Quick Reference table, Common Workflows, Error Handling notes. Leave structure intact. -- `.env.example`: set the auth env var with `@required`/`@how-to-get` annotations (skip for `scheme: none`). -- `.clify.json`: fill `auth`, `defaults`, `nuances`, `coverage`, `contentHash` from the parsed spec. -- `README.md`, `AGENTS.md`: regenerate examples that mention specific resources. - -### 10. Write coverage + knowledge artifacts - -Emit `coverage.json` with every parsed endpoint marked `included: true` or `dropped: true` + reason. Write knowledge files queued in step 6. - -### 11. Validate (binary verb) - -``` -clify validate ./-cli -``` - -All eight categories must pass. Per-failure remediation: - -- **manifest** — fix the named field; cross-manifest mismatches usually mean `package.json` was edited but `plugin.json` wasn't. -- **coverage** — every `included: false` needs `dropped: true` + a valid `reason`. -- **structural** — a resource or action you generated isn't reachable; check the registry vs the help-text generator. -- **nuances** — declared a hard-fail nuance but didn't add the artifact. Either add it or set the nuance to empty. -- **secrets** — a real key pattern leaked; remove it. -- **ci** — `.github/workflows/test.yml` missing or doesn't run `npm test`. -- **tests** — exemplar smoke/integration tests fail because of a typo in the renamed registry. Read the test output and fix the source. - -Up to **3 fix-and-retry attempts**. On still-failing, surface the failed check names to the user and stop — do not declare done with failures. - -### 12. Simplify - -Run `/simplify` over the changed files. Remove dead code, collapse duplicated patterns, but never break a contract from `conventions.md`. - -### 13. Report - -Summarize to the user: API name, resources × actions count, dropped endpoints with reasons, declared nuances, knowledge files written, validation gate result. Include the path to the generated repo and `cd && npm test` as the next step. - -## Anti-patterns - -- ❌ Don't write the CLI from scratch — copy the exemplar via `scaffold-init`. -- ❌ Don't skip validation — the gate is the contract. -- ❌ Don't mark a generated repo "done" while validation is failing. -- ❌ Don't silently drop endpoints — every drop needs a reason in `coverage.json`. -- ❌ Don't write nuance prose into `knowledge/` if the corresponding `.clify.json.nuances.*` field isn't set; the gate cross-references them. -- ❌ Don't put auth tokens in source. The gate scans for them and will flag real-shaped tokens. - -## Edge cases - -- **API has no auth** → `auth.scheme: "none"`, leave `auth.envVar` set to `_API_KEY` for shape consistency, skip the `@required` annotation in `.env.example`. The gate accepts this. -- **API uses cookie auth** → not supported in v0.2; ask the user for a session-token alternative or stop with a clear message. -- **API only documented as a curl tutorial** (no spec, no structured docs) → ask the user for an OpenAPI URL or to confirm the resources by hand. -- **Resource has 12 actions** → flatten with sub-resources or hyphenated action names; do not exceed the two-level nesting cap. diff --git a/skills/clify-sync/SKILL.md b/skills/clify-sync/SKILL.md deleted file mode 100644 index 47c0090..0000000 --- a/skills/clify-sync/SKILL.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -name: clify-sync -description: Re-fetch the docs for a generated clify CLI, detect upstream drift via content hash, and regenerate the resource registry, integration tests, and SKILL.md if the docs have changed. Preserves knowledge/, .env, and immutable .clify.json fields. Use when the user says "/clify-sync ", "sync this CLI", or asks if a generated CLI is up to date. -allowed-tools: - - Bash - - Read - - Write - - Edit - - WebFetch ---- - -# clify-sync - -Re-fetch upstream docs and detect drift for a generated clify CLI. If the content hash has changed, regenerate the parts that derive from docs (CLI source, integration tests, SKILL.md sections, coverage). Preserve user-managed state (`knowledge/`, `.env`, `.clify.json` apiName + docsUrl). - -## Triggers - -- `/clify-sync ` -- "Sync this CLI" / "Is this CLI up to date with the docs?" - -## Workflow - -### 1. Read state - -``` -cat /.clify.json -``` - -Note `apiName`, `docsUrl`, `crawledUrls`, `contentHash`, `clifyVersion`. - -### 2. Hash check (binary verb) - -``` -clify sync-check --json -``` - -Re-fetches every URL in `crawledUrls`, recomputes the SHA-256, and reports `{ changed: bool, oldHash, newHash, fetched[] }`. This is **advisory only** — it does not regenerate. - -If `changed: false`, exit with "no upstream changes since last generation". - -### 3. Re-parse + re-detect - -If `changed: true`: -- Re-fetch the docs (the `sync-check` already fetched once; re-use the bodies if convenient). -- Re-parse endpoints (step 4 of the scaffold pipeline). -- Re-detect nuances (step 5). -- Re-extract business knowledge (step 6) — but **do not** overwrite existing `knowledge/*.md` files; queue suggested additions. - -### 4. Regenerate docs-derived files - -- `bin/-cli.mjs` — resource registry only (preserve helpers). -- `test/integration.test.mjs` — fixtures + nuance tests. -- `skills//SKILL.md` — Triggers, Quick Reference, Common Workflows. -- `coverage.json` — full rewrite. -- `.clify.json` — update `contentHash`, `generatedAt`, `nuances`, `coverage`. Keep `apiName`, `docsUrl`, `auth`. - -**Do NOT touch:** `knowledge/`, `.env`, `package.json` `name`/`version` (user controls bumping), `LICENSE`, `README.md` (unless the user opts in). - -### 5. Validate - -``` -clify validate -``` - -Same loop as the scaffold skill: up to 3 fix attempts. - -### 6. Knowledge review - -For every file in `knowledge/`, check if `applies-to:` references resources/actions still in the new registry. If not, flag for the user (don't auto-delete). - -### 7. Report - -Summarize: -- old vs new hash -- endpoints added / removed / unchanged -- knowledge files flagged for review -- nuances changes -- validation gate result - -## Anti-patterns - -- ❌ Don't overwrite `knowledge/`, `.env`, or user-edited README content. -- ❌ Don't auto-bump `package.json` version — the user owns that. -- ❌ Don't run regeneration without first checking if the hash actually changed. -- ❌ Don't claim sync is "complete" if the validation gate fails on the regenerated repo. diff --git a/skills/clify-validate/SKILL.md b/skills/clify-validate/SKILL.md deleted file mode 100644 index 84ab208..0000000 --- a/skills/clify-validate/SKILL.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -name: clify-validate -description: Run the clify validation gate against a generated CLI repo and report failures by category. Use when the user says "/clify-validate", "validate this repo", or after generation/modification of a clify-shaped CLI to confirm it still meets A+ standards. -allowed-tools: - - Bash - - Read ---- - -# clify-validate - -Run the validation gate (eight categories: manifest, coverage, structural, nuances, secrets, ci, tests, plus advisory warnings) against a generated CLI repo. Report failures grouped by category. Never claim "valid" while any check is failing. - -The gate is implemented in [`lib/validate.mjs`](../../lib/validate.mjs) and documented in [`references/validation-gate.md`](../../references/validation-gate.md). - -## Triggers - -- `/clify-validate ` -- "Validate this CLI" -- After editing a clify-shaped repo, before committing - -## Workflow - -1. Run the gate with structured output: - - ``` - clify validate --json - ``` - -2. Parse the JSON report: - - `summary.failed > 0` → at least one check failed - - `warnings[]` → soft-warn nuances, never fail the gate - -3. Group failures by `results[].category`. Quote the failure `name` and any `error` / `missing` / `failures` / `issues` / `offenders` field. - -4. Fix the underlying issue. Common patterns: - - | Category | Likely cause | - |---|---| - | manifest | `package.json` ↔ `plugin.json` ↔ `marketplace.json` desync, or `bin` path renamed without updating the manifest. | - | coverage | An endpoint was dropped without `dropped: true` + a valid `reason`. | - | structural | A resource or action exists in the registry but isn't in `--help` output (or vice versa). | - | nuances | A nuance was declared in `.clify.json` but the artifact (test, knowledge file, CLI flag) is missing. | - | secrets | A real-shaped API key is hardcoded in a source file. | - | ci | `.github/workflows/test.yml` missing or doesn't invoke `npm test`. | - | tests | The repo's `npm test` script failed — read `stderrTail`. | - -5. Re-run `clify validate `. Loop until 0 failures, capped at 3 attempts. If still failing, surface the failed check names verbatim and stop. - -## Anti-patterns - -- ❌ Don't claim a repo passes when warnings exist but you didn't read them — the user may want to act on warnings. -- ❌ Don't bypass a check by editing `lib/validate.mjs`. The check is there for a reason; fix the input. -- ❌ Don't run `clify validate` on a non-generated repo and expect it to pass. - -## Quick reference - -``` -clify validate # human-readable -clify validate --json # machine-parseable -clify validate --skip-tests # manifests only, fast -``` - -Exit code: 0 on full pass, 1 on any failure. diff --git a/skills/clify/SKILL.md b/skills/clify/SKILL.md new file mode 100644 index 0000000..5a9ebd7 --- /dev/null +++ b/skills/clify/SKILL.md @@ -0,0 +1,105 @@ +--- +name: clify +description: Generate an A+ Node.js CLI repo from an API documentation URL. Triggers include "/clify ", "generate a CLI for this API", or providing API docs and asking for an agent-friendly wrapper. Works by copying the bundled exemplar (`examples/exemplar-cli/`, structurally inspired by google/agents-cli) and mechanically substituting API-specific content, then running the validation gate. +allowed-tools: + - Read + - Write + - Edit + - Bash + - Glob + - Grep + - WebFetch + - WebSearch + - AskUserQuestion +--- + +# clify + +Generate a self-updating CLI repo from an API documentation URL. Output is installable as a Claude Code plugin or standalone Node CLI. + +The strategy is **copy exemplar + mechanically substitute** — quality comes from the exemplar (which is hand-crafted, tested, and validated by the gate). Your job is the smaller, more verifiable transformation of API-specific content. + +## Triggers + +- `/clify `, `/clify-scaffold ` +- "Generate a CLI for this API" / "Wrap this API as an agent CLI" +- User provides an API docs URL and asks for a wrapper + +## Read these first + +1. [`references/conventions.md`](../../references/conventions.md) — every contract you must honor. +2. [`references/validation-gate.md`](../../references/validation-gate.md) — what the gate enforces. The gate is the source of truth for "done." +3. [`references/scaffold-pipeline.md`](../../references/scaffold-pipeline.md) — the per-phase detail expanding the six steps below. +4. [`examples/exemplar-cli/`](../../examples/exemplar-cli/) — the canonical stencil. Section ordering, helper signatures, code structure, and skills layout must match it. + +## Six-phase pipeline + +### 1. Fetch & detect + +`WebFetch` the URL. If the body is OpenAPI/Swagger (JSON or YAML with `openapi:` or `swagger:` key), parse directly — fast path. Otherwise treat as HTML/Markdown and crawl up to depth 2 with `WebFetch`, deduping normalized URLs and rate-limiting at 1s. If the page is JS-rendered with empty content, ask for an OpenAPI URL instead via `AskUserQuestion`. + +### 2. Parse & group + +Extract method, path, params, request/response shapes. Group endpoints by first path segment as a resource. Apply CRUD verb mapping from `conventions.md`; keep the API's own verbs for non-CRUD endpoints. Cap nesting at two levels — flatten sub-resources into hyphenated top-level resource names (`item-variants`, not `items variants`). + +Detect nuances inline: pagination (cursor / page / offset / link-header), idempotency keys, multipart uploads, deprecated endpoints, rate limits, auth scopes, conditional requests, business rules. Each detected nuance maps to a `.clify.json.nuances.*` field plus an artifact (test, knowledge file, CLI flag). + +### 3. Consult + +Use `AskUserQuestion` for unresolved tradeoffs — resource grouping, ambiguous verbs, what to drop. Also ask **where** the new repo should live (default: parent of cwd, sibling of the current directory). Record dropped endpoints with one of the allowed reasons in `references/validation-gate.md` — never silently drop. + +### 4. Init from exemplar + +``` +clify scaffold-init --target +``` + +This is the **deterministic copy + rename phase** — never do it by hand. It copies `examples/exemplar-cli/` to `/-cli/` and rewrites `exemplar` → ``, `EXEMPLAR` → ``, `Exemplar` → ``. + +Then `git init` the new repo and commit the unmodified scaffold so the next phase's edits show up as a clean diff. + +### 5. Substitute + +Edit only what changes per API. Preserve helper signatures (`apiRequest`, `output`, `errorOut`, `splitGlobal`, `toParseArgs`, `checkRequired`, help generators) verbatim from the exemplar. Per-file substitutions: + +- `commands/.mjs` — replace the exemplar's items/orders/item-variants with the parsed resources. One file per resource. Each default-exports `{ name, actions, buildPayload? }`. +- `bin/-cli.mjs` — update the imports and `COMMANDS` array to reflect the new resource set; everything else stays. +- `lib/auth.mjs` — set `SCHEME` and `ENV_VAR`. Branch in `applyAuth` to the right header shape. +- `.clify.json` — fill `auth`, `defaults`, `nuances`, `coverage`, `contentHash` from the parsed spec. +- `.env.example` — set `@required` / `@how-to-get` annotations on the auth var (skip for `scheme: none`). +- `skills/-cli-workflow/SKILL.md` — Triggers, Quick Reference, Common Workflows. Keep the modular layout intact (workflow, auth, resources, knowledge skills). +- `test/integration.test.mjs` — rewrite per-resource fixtures. Add nuance tests as required (multi-page for pagination, idempotency-key header assertion, FormData for multipart). +- `coverage.json` + `knowledge/.md` files for any business rules surfaced in step 2. +- `README.md` and `AGENTS.md` — re-author the resource list and workflows. + +See `references/scaffold-pipeline.md` for the file-by-file checklist. + +### 6. Validate, simplify, report + +``` +clify validate ./-cli +``` + +All eight categories must pass. Up to **3 fix-and-retry attempts**. If still failing, surface the failed check names verbatim and stop — never declare done with failures. Then run `/simplify` over the changed files (collapse duplicated patterns, remove dead code) and report a summary: API name, resources × actions count, dropped endpoints with reasons, declared nuances, knowledge files, gate result, and the path to the generated repo. + +--- + +## Anti-patterns + +- Don't write the CLI from scratch — copy the exemplar via `scaffold-init`. +- Don't skip validation — the gate is the contract. +- Don't mark a generated repo "done" while validation is failing. +- Don't silently drop endpoints — every drop needs a reason in `coverage.json`. +- Don't write nuance prose into `knowledge/` if `.clify.json.nuances.*` isn't set; the gate cross-references them. +- Don't put auth tokens in source. The gate scans for them and will flag real-shaped tokens. + +## Edge cases + +- **API has no auth** → `auth.scheme: "none"`, leave `auth.envVar` set to `_API_KEY` for shape consistency, skip the `@required` annotation. The gate accepts this. +- **API uses cookie auth** → not supported; ask for a session-token alternative or stop with a clear message. +- **API only documented as a curl tutorial** → ask for an OpenAPI URL or to confirm the resource set by hand. +- **Resource has 12 actions** → flatten with sub-resources or hyphenated action names; do not exceed the two-level nesting cap. + +## Validate-only / sync-only + +If you only need to check an existing repo, run `clify validate ` directly — no skill needed. If you need to detect upstream doc drift, run `clify sync-check ` directly. Both are deterministic binary verbs that don't require this skill. diff --git a/test/clify.test.mjs b/test/clify.test.mjs index a7b6b99..5e023a1 100644 --- a/test/clify.test.mjs +++ b/test/clify.test.mjs @@ -10,11 +10,11 @@ import { syncCheck } from "../lib/sync-check.mjs"; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(__dirname, ".."); -const EXEMPLAR = join(REPO_ROOT, "examples/jsonplaceholder-cli"); +const EXEMPLAR = join(REPO_ROOT, "examples/exemplar-cli"); function freshCopy() { const dir = mkdtempSync(join(tmpdir(), "clify-test-")); - const dest = join(dir, "jsonplaceholder-cli"); + const dest = join(dir, "exemplar-cli"); cpSync(EXEMPLAR, dest, { recursive: true }); return { root: dir, repo: dest }; } @@ -29,7 +29,7 @@ test("validate: clean exemplar passes (skip live tests for speed)", async () => // ---------- substitute helper ---------- test("substitute: handles upper/title/lower in correct order", () => { - const text = "JSONPLACEHOLDER_API_KEY for JSONPlaceholder via jsonplaceholder-cli"; + const text = "EXEMPLAR_API_KEY for Exemplar via exemplar-cli"; const out = substitute(text, "demo-api", "DEMO_API", "Demo Api"); assert.equal(out, "DEMO_API_API_KEY for Demo Api via demo-api-cli"); }); @@ -43,10 +43,17 @@ test("scaffoldInit: produces a working renamed copy", () => { assert.match(result.dir, /movie-db-cli$/); const pkg = JSON.parse(readFileSync(join(result.dir, "package.json"), "utf8")); assert.equal(pkg.name, "movie-db-cli"); + // bin file is renamed and rewritten const cli = readFileSync(join(result.dir, "bin/movie-db-cli.mjs"), "utf8"); - assert.match(cli, /MOVIE_DB_BASE_URL/); assert.match(cli, /movie-db-cli/); - assert.ok(!cli.includes("jsonplaceholder")); + assert.ok(!cli.includes("exemplar"), "renamed bin should not contain literal 'exemplar'"); + // env var lookups in lib/ are substituted + const auth = readFileSync(join(result.dir, "lib/auth.mjs"), "utf8"); + assert.match(auth, /MOVIE_DB_API_KEY/); + const api = readFileSync(join(result.dir, "lib/api.mjs"), "utf8"); + assert.match(api, /MOVIE_DB_BASE_URL/); + // modular skills directory is renamed too + assert.ok(existsSync(join(result.dir, "skills/movie-db-cli-workflow/SKILL.md"))); } finally { rmSync(tmp, { recursive: true, force: true }); } }); @@ -68,14 +75,17 @@ test("scaffoldInit: refuses to overwrite", () => { test("validate: missing help-text resource → structural fail", async () => { const { root, repo } = freshCopy(); try { - // strip 'photos' from the resource registry but leave it in coverage.json - const cliPath = join(repo, "bin/jsonplaceholder-cli.mjs"); - const src = readFileSync(cliPath, "utf8"); - const broken = src.replace(/photos:\s*crudResource\("photos"\),\s*\n/, ""); - writeFileSync(cliPath, broken); + // Strip 'orders' from the resource registry, but leave it in coverage.json. + // The bin imports `orders` as a default export, so removing the import + + // its registry entry cleanly removes the resource from --help. + const binPath = join(repo, "bin/exemplar-cli.mjs"); + const src = readFileSync(binPath, "utf8"); + let broken = src.replace(/import orders from "\.\.\/commands\/orders\.mjs";\n/, ""); + broken = broken.replace(/, orders\]/, "]"); + writeFileSync(binPath, broken); const r = await validate(repo, { skipTests: true }); const failures = r.results.filter((x) => !x.ok && x.category === "structural"); - assert.ok(failures.length >= 1, "expected structural failure when resource missing from --help"); + assert.ok(failures.length >= 1, `expected structural failure when resource missing from --help; got results=${JSON.stringify(r.results.filter((x) => !x.ok), null, 2)}`); } finally { rmSync(root, { recursive: true, force: true }); } }); @@ -107,7 +117,7 @@ test("validate: silent drop in coverage.json → coverage fail", async () => { test("validate: hardcoded secret → secrets fail", async () => { const { root, repo } = freshCopy(); try { - const path = join(repo, "bin/jsonplaceholder-cli.mjs"); + const path = join(repo, "bin/exemplar-cli.mjs"); const src = readFileSync(path, "utf8"); // Build the secret at runtime so this source file itself doesn't contain a // matchable Stripe-pattern literal (would trigger GitHub secret scanning). @@ -132,17 +142,20 @@ test("validate: nuance declared without artifact → nuances fail", async () => try { const cpath = join(repo, ".clify.json"); const cfg = JSON.parse(readFileSync(cpath, "utf8")); - cfg.nuances.idempotency = ["posts.create"]; + // The exemplar already declares idempotency; flip multiPart to a non-existent + // action and the nuance check should fail because no integration test posts a file + // for that action. Easier: declare deprecated without an artifact. + cfg.nuances.deprecated = ["items.archive"]; writeFileSync(cpath, JSON.stringify(cfg, null, 2)); const r = await validate(repo, { skipTests: true }); - assert.ok(r.results.some((x) => !x.ok && x.category === "nuances" && /idempotency/.test(x.name))); + assert.ok(r.results.some((x) => !x.ok && x.category === "nuances")); } finally { rmSync(root, { recursive: true, force: true }); } }); test("validate: SKILL.md missing knowledge/ preamble → structural fail", async () => { const { root, repo } = freshCopy(); try { - const path = join(repo, "skills/jsonplaceholder/SKILL.md"); + const path = join(repo, "skills/exemplar-cli-workflow/SKILL.md"); const src = readFileSync(path, "utf8").replace(/knowledge\//g, "kb/"); writeFileSync(path, src); const r = await validate(repo, { skipTests: true }); @@ -153,8 +166,9 @@ test("validate: SKILL.md missing knowledge/ preamble → structural fail", async test("validate: missing BASE_URL override → manifest fail", async () => { const { root, repo } = freshCopy(); try { - const path = join(repo, "bin/jsonplaceholder-cli.mjs"); - const src = readFileSync(path, "utf8").replace(/JSONPLACEHOLDER_BASE_URL/g, "JSONPLACEHOLDER_URL"); + // BASE_URL lookup lives in lib/api.mjs in the new layout. + const path = join(repo, "lib/api.mjs"); + const src = readFileSync(path, "utf8").replace(/EXEMPLAR_BASE_URL/g, "EXEMPLAR_URL"); writeFileSync(path, src); const r = await validate(repo, { skipTests: true }); assert.ok(r.results.some((x) => !x.ok && /BASE_URL/.test(x.name)));