From 6f26409634b49ed6d186820d8364f843fa6b4047 Mon Sep 17 00:00:00 2001 From: jayavibhavnk Date: Wed, 11 Mar 2026 14:56:29 -0700 Subject: [PATCH 1/8] added weave-test --- README.md | 39 +++++- package-lock.json | 4 +- package.json | 2 +- src/index.ts | 227 +++++++++++++++++++++++++++++-- src/testing/autonomous.ts | 148 ++++++++++++++++++++ src/testing/discovery.ts | 147 ++++++++++++++++++++ src/testing/orchestrator.ts | 162 ++++++++++++++++++++++ src/testing/report.ts | 63 +++++++++ src/testing/runner.ts | 63 +++++++++ src/testing/types.ts | 60 ++++++++ tests/testing/autonomous.test.ts | 45 ++++++ tests/testing/discovery.test.ts | 57 ++++++++ tests/testing/runner.test.ts | 58 ++++++++ 13 files changed, 1057 insertions(+), 18 deletions(-) create mode 100644 src/testing/autonomous.ts create mode 100644 src/testing/discovery.ts create mode 100644 src/testing/orchestrator.ts create mode 100644 src/testing/report.ts create mode 100644 src/testing/runner.ts create mode 100644 src/testing/types.ts create mode 100644 tests/testing/autonomous.test.ts create mode 100644 tests/testing/discovery.test.ts create mode 100644 tests/testing/runner.test.ts diff --git a/README.md b/README.md index 02dd875..225d6ca 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # weave -**Graph-native memory CLI for AI agents.** Chat with AI that actually remembers — across sessions, across agents. +**Testing-native memory CLI for AI agents.** Run multi-step test workflows with persistent memory across sessions and specialized QA agents. + +> CLI binary name: `weave-test` (replace any older `weave` examples with `weave-test`). ``` ◈ weave v0.1.0 @@ -9,14 +11,15 @@ ## What is Weave? -Weave is a terminal CLI agent (like Claude Code) with a twist: **persistent, graph-structured memory**. Every conversation is remembered. Memories are connected by semantic similarity, temporal sequence, and shared entities. Your agents build knowledge over time — they never cold-start. +Weave is a terminal CLI agent (like Claude Code) focused on **software testing and quality improvement**. It combines **persistent graph memory** with **multi-agent QA roles** so each run gets smarter over time. -Built on the MemWeave memory architecture: +Built on the MemWeave memory architecture plus a test orchestration layer: - **Multi-layer memory graph** — semantic, temporal, causal, and entity edges - **Tiered memory** — working → short-term → long-term → archival with automatic promotion and decay -- **Multi-agent** — spawn multiple agents with different personas that share a memory fabric +- **Multi-agent** — testing personas (orchestrator, edge-case hunter, report analyst) share a memory fabric - **Hybrid retrieval** — vector similarity + graph traversal for smarter recall - **Local-first** — works with just hash-based embeddings + SQLite, no cloud required for memory +- **Testing pipeline** — command discovery (`lint`, `typecheck`, `test`, `integration`, `e2e`, `build`) with clear run reports ## Install @@ -43,8 +46,11 @@ weave init # Set your API key (or use Codex auth — see below) weave config set apiKey sk-your-openai-key -# Start chatting (memories persist automatically) -weave chat +# Initialize testing agents +weave test init + +# Run the testing pipeline on current project +weave test run # Or use Anthropic weave config set provider anthropic @@ -65,6 +71,27 @@ weave chat ## Commands +### Testing (Primary) + +```bash +weave test init # create test-focused agents +weave test run # discover and run tests in current dir +weave test plan # preview discovered + autonomous plan +weave test run --dir ../my-app # run against another project +weave test run --workspace release # keep separate test memory per workspace +weave test run --provider anthropic # use another model provider +weave test run --model gpt-4o-mini # choose specific model for insights +weave test run --max-auto 5 # add up to 5 autonomous expansions +weave test run --no-autonomous # run only discovered commands +``` + +The testing workflow: +- discovers test commands from project scripts/runtime, +- optionally proposes additional safe commands using autonomous planning (`--max-auto > 0`), +- runs them as a multi-step pipeline, +- analyzes failures and edge-case gaps, +- persists run intelligence to memory for future sessions. + ### Chat ```bash diff --git a/package-lock.json b/package-lock.json index 4e46360..3c2b59b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "memweave", - "version": "0.3.0", + "version": "0.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "memweave", - "version": "0.3.0", + "version": "0.4.0", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "^0.39.0", diff --git a/package.json b/package.json index 51247b9..591c920 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Graph-native memory CLI for AI agents — persistent, multi-agent, cross-session context", "type": "module", "bin": { - "weave": "./dist/index.js" + "weave-test": "./dist/index.js" }, "main": "./dist/index.js", "files": [ diff --git a/src/index.ts b/src/index.ts index d16b720..64bd941 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,8 +17,8 @@ const VERSION = "0.4.0"; const program = new Command(); program - .name("weave") - .description("Graph-native memory CLI for AI agents") + .name("weave-test") + .description("Testing-native multi-agent CLI with persistent memory") .version(VERSION, "-v, --version"); // ── weave chat ───────────────────────────────────────────── @@ -40,13 +40,13 @@ program console.log(""); console.log(errorLine("No API key found. Set one with:")); console.log( - `\n ${t.accent("weave config set apiKey")} ${t.dim("")}` + `\n ${t.accent("weave-test config set apiKey")} ${t.dim("")}` ); console.log( ` ${t.dim("or set")} ${t.accent("OPENAI_API_KEY")} ${t.dim("/")} ${t.accent("ANTHROPIC_API_KEY")} ${t.dim("env var")}` ); console.log( - ` ${t.dim("For OpenAI only: run")} ${t.accent("codex login --api-key ")} ${t.dim("then weave will use ~/.codex/auth.json")}\n` + ` ${t.dim("For OpenAI only: run")} ${t.accent("codex login --api-key ")} ${t.dim("then weave-test will use ~/.codex/auth.json")}\n` ); process.exit(1); } @@ -130,7 +130,7 @@ agentCmd const agents = fabric.listAgents(); if (agents.length === 0) { console.log( - `\n ${t.dim("No agents yet. Create one with:")} ${t.accent("weave agent spawn ")}\n` + `\n ${t.dim("No agents yet. Create one with:")} ${t.accent("weave-test agent spawn ")}\n` ); } else { const rows = agents.map((a) => { @@ -346,6 +346,215 @@ memoryCmd console.log(` ${t.success(icons.check)} Pruned: ${result.pruned}\n`); }); +// ── weave test ───────────────────────────────────────────── +const testCmd = program.command("test").description("Testing-focused multi-agent workflow"); + +testCmd + .command("init") + .description("Create default testing agents") + .option("-w, --workspace ", "Workspace to use", "default") + .action(async (options) => { + const config = loadConfig(); + const workspacePath = getWorkspacePath(options.workspace); + + const { MemoryFabric } = await import("./core/fabric.js"); + const fabric = await MemoryFabric.create({ ...config, workspacePath }); + + const defaults = [ + { + id: "test-orchestrator", + name: "test-orchestrator", + role: "QA Orchestrator", + description: "Coordinates multi-step test execution and prioritizes fixes by risk.", + }, + { + id: "edge-case-hunter", + name: "edge-case-hunter", + role: "Edge Case Hunter", + description: "Finds negative, boundary, race-condition, and regression edge cases.", + }, + { + id: "report-analyst", + name: "report-analyst", + role: "Test Report Analyst", + description: "Turns noisy test output into actionable summaries and next steps.", + }, + ]; + + for (const persona of defaults) { + fabric.getOrCreateAgent(persona.id, persona); + } + fabric.save(); + fabric.close(); + console.log(successLine("Testing agents initialized.")); + }); + +testCmd + .command("plan") + .description("Show discovered and autonomous testing plan without executing") + .option("-d, --dir ", "Target project directory", process.cwd()) + .option("-m, --model ", "Model to use for autonomous planning") + .option("-p, --provider ", "LLM provider (openai|anthropic|ollama|lmstudio)") + .option("--max-auto ", "Maximum autonomous test commands to add (0 disables)", "0") + .option("--no-autonomous", "Disable autonomous test expansion") + .action(async (options) => { + const config = loadConfig(); + const providerName = options.provider || config.provider; + const model = options.model || config.model; + const apiKey = resolveApiKey({ ...config, provider: providerName }); + const isLocal = providerName === "ollama" || providerName === "lmstudio"; + const targetDir = String(options.dir); + const maxAutonomous = Math.max(0, parseInt(String(options.maxAuto), 10) || 0); + const autonomousEnabled = Boolean(options.autonomous); + + const { discoverTestingPlan } = await import("./testing/discovery.js"); + const { generateAutonomousPlan, toAutonomousCommands } = await import("./testing/autonomous.js"); + const { createProvider } = await import("./llm/provider.js"); + const plan = discoverTestingPlan(targetDir); + + if (plan.commands.length === 0) { + console.log(errorLine("No test commands discovered for this project.")); + process.exit(1); + } + + const llm = apiKey || isLocal + ? createProvider(providerName, apiKey || "ollama", model, config.baseURL) + : null; + const autonomousItems = + autonomousEnabled && maxAutonomous > 0 + ? await generateAutonomousPlan(llm, model, plan, maxAutonomous) + : []; + const autonomousCommands = toAutonomousCommands(autonomousItems, plan.commands); + + console.log(`\n ${t.brandBold("Testing plan")}`); + console.log(` ${t.muted("─".repeat(50))}`); + for (const cmd of plan.commands) { + console.log(` ${t.muted(icons.arrow)} ${cmd.label} ${t.dim(`(${cmd.command})`)}`); + } + if (autonomousCommands.length > 0) { + console.log(` ${t.brandBold("Autonomous additions")}`); + for (const cmd of autonomousCommands) { + console.log(` ${t.muted(icons.arrow)} ${cmd.label} ${t.dim(`(${cmd.command})`)}`); + } + } + console.log(""); + }); + +testCmd + .command("run") + .description("Run discovered + autonomous tests, analyze results, and persist findings") + .option("-w, --workspace ", "Workspace to use", "default") + .option("-d, --dir ", "Target project directory", process.cwd()) + .option("-m, --model ", "Model to use for testing insights") + .option("-p, --provider ", "LLM provider (openai|anthropic|ollama|lmstudio)") + .option("-t, --timeout ", "Per-command timeout in milliseconds", "120000") + .option("--max-auto ", "Maximum autonomous test commands to add (0 disables)", "0") + .option("--no-autonomous", "Disable autonomous test expansion") + .action(async (options) => { + const config = loadConfig(); + const providerName = options.provider || config.provider; + const model = options.model || config.model; + const apiKey = resolveApiKey({ ...config, provider: providerName }); + const isLocal = providerName === "ollama" || providerName === "lmstudio"; + const workspacePath = getWorkspacePath(options.workspace); + + const { MemoryFabric } = await import("./core/fabric.js"); + const { discoverTestingPlan } = await import("./testing/discovery.js"); + const { runTestingCommands } = await import("./testing/runner.js"); + const { + generateAutonomousPlan, + toAutonomousCommands, + persistAutonomousPlan, + } = await import("./testing/autonomous.js"); + const { + buildTestingInsights, + makeTestingReport, + persistRunToMemory, + } = await import("./testing/orchestrator.js"); + const { renderTestingReport } = await import("./testing/report.js"); + const { createProvider } = await import("./llm/provider.js"); + + const fabric = await MemoryFabric.create({ + ...config, + provider: providerName, + model, + apiKey, + workspacePath, + }); + + const orchestrator = fabric.getOrCreateAgent("test-orchestrator", { + name: "test-orchestrator", + role: "QA Orchestrator", + description: "Coordinates multi-step test execution and prioritizes fixes by risk.", + model, + provider: providerName, + }); + const edgeHunter = fabric.getOrCreateAgent("edge-case-hunter", { + name: "edge-case-hunter", + role: "Edge Case Hunter", + description: "Finds negative, boundary, race-condition, and regression edge cases.", + model, + provider: providerName, + }); + const reporter = fabric.getOrCreateAgent("report-analyst", { + name: "report-analyst", + role: "Test Report Analyst", + description: "Turns noisy test output into actionable summaries and next steps.", + model, + provider: providerName, + }); + + const targetDir = String(options.dir); + const timeoutMs = parseInt(String(options.timeout), 10); + const maxAutonomous = Math.max(0, parseInt(String(options.maxAuto), 10) || 0); + const autonomousEnabled = Boolean(options.autonomous); + const plan = discoverTestingPlan(targetDir); + + if (plan.commands.length === 0) { + console.log(errorLine("No test commands discovered for this project.")); + console.log(` ${t.dim("Use scripts like test/lint/typecheck/build in package.json, then rerun.")}`); + fabric.close(); + process.exit(1); + } + + console.log(`\n ${t.brandBold("Running testing pipeline")}`); + console.log(` ${t.muted("─".repeat(40))}`); + for (const cmd of plan.commands) { + console.log(` ${t.muted(icons.arrow)} ${cmd.label} ${t.dim(`(${cmd.command})`)}`); + } + const llm = apiKey || isLocal + ? createProvider(providerName, apiKey || "ollama", model, config.baseURL) + : null; + + const autonomousItems = + autonomousEnabled && maxAutonomous > 0 + ? await generateAutonomousPlan(llm, model, plan, maxAutonomous) + : []; + const autonomousCommands = toAutonomousCommands(autonomousItems, plan.commands); + if (autonomousCommands.length > 0) { + console.log(` ${t.brandBold("Autonomous expansions")}`); + for (const cmd of autonomousCommands) { + console.log(` ${t.muted(icons.arrow)} ${cmd.label} ${t.dim(`(${cmd.command})`)}`); + } + await persistAutonomousPlan(orchestrator, edgeHunter, autonomousItems); + } + console.log(""); + + const results = runTestingCommands( + [...plan.commands, ...autonomousCommands], + targetDir, + timeoutMs + ); + const insights = await buildTestingInsights(llm, model, plan, results); + const report = makeTestingReport(targetDir, plan, results, insights); + + await persistRunToMemory(orchestrator, edgeHunter, reporter, report); + fabric.save(); + fabric.close(); + + console.log(renderTestingReport(report)); + }); + // ── weave config ─────────────────────────────────────────── const configCmd = program.command("config").description("Manage configuration"); @@ -512,20 +721,20 @@ program // ── weave init ───────────────────────────────────────────── program .command("init") - .description("Initialize weave in the current directory") + .description("Initialize weave-test in the current directory") .action(() => { ensureConfigDir(); console.log(banner(VERSION)); console.log(successLine("Weave initialized!")); console.log(`\n ${t.dim("Get started:")}`); console.log( - ` ${t.accent("weave config set apiKey")} ${t.dim("")} ${t.muted("# set your API key")}` + ` ${t.accent("weave-test config set apiKey")} ${t.dim("")} ${t.muted("# set your API key")}` ); console.log( - ` ${t.accent("weave chat")} ${t.muted("# start chatting")}` + ` ${t.accent("weave-test test init")} ${t.muted("# bootstrap testing agents")}` ); console.log( - ` ${t.accent("weave agent spawn researcher")} ${t.muted("# create agents")}` + ` ${t.accent("weave-test test run")} ${t.muted("# run testing workflow")}` ); console.log(""); }); diff --git a/src/testing/autonomous.ts b/src/testing/autonomous.ts new file mode 100644 index 0000000..b10f7f8 --- /dev/null +++ b/src/testing/autonomous.ts @@ -0,0 +1,148 @@ +import type { LLMProvider } from "../llm/provider.js"; +import type { AgentMemory } from "../core/agent.js"; +import { MemoryType } from "../core/types.js"; +import type { AutonomousPlanItem, TestCommand, TestCommandKind, TestingPlan } from "./types.js"; + +const SAFE_COMMAND_PATTERNS: RegExp[] = [ + /^npm run (test|lint|build|typecheck|check-types|integration|e2e)(\s|$)/i, + /^pnpm run (test|lint|build|typecheck|check-types|integration|e2e)(\s|$)/i, + /^yarn (test|lint|build|typecheck|check-types|integration|e2e)(\s|$)/i, + /^python -m pytest(\s|$)/i, + /^go test(\s|$)/i, + /^cargo test(\s|$)/i, +]; + +function isSafeCommand(command: string): boolean { + const trimmed = command.trim(); + if (!trimmed) return false; + if (trimmed.includes("&&") || trimmed.includes(";") || trimmed.includes("|")) return false; + if (/\b(--watch|-w|watch)\b/i.test(trimmed)) return false; + if (/--\s+[^\-\s][^\s]*/.test(trimmed)) return false; + if (/\s+[^\s]+\.(js|ts|tsx|jsx)\b/i.test(trimmed)) return false; + return SAFE_COMMAND_PATTERNS.some((pattern) => pattern.test(trimmed)); +} + +function isDerivedFromDiscoveredCommand(command: string, discovered: TestCommand[]): boolean { + const trimmed = command.trim(); + for (const base of discovered.map((d) => d.command.trim())) { + if (trimmed === base) return true; + if (trimmed.startsWith(base + " ")) return true; + } + return false; +} + +function normalizeKind(kind?: string): TestCommandKind { + const k = (kind || "").toLowerCase(); + if (k === "lint" || k === "typecheck" || k === "unit" || k === "integration" || k === "e2e" || k === "build") { + return k; + } + return "custom"; +} + +function parsePlanItems(text: string): AutonomousPlanItem[] { + const first = text.indexOf("["); + const last = text.lastIndexOf("]"); + if (first === -1 || last === -1 || last <= first) return []; + try { + const parsed = JSON.parse(text.slice(first, last + 1)) as unknown[]; + if (!Array.isArray(parsed)) return []; + return parsed + .map((item): AutonomousPlanItem | null => { + if (!item || typeof item !== "object") return null; + const obj = item as Record; + if (typeof obj.command !== "string" || typeof obj.label !== "string") return null; + return { + command: obj.command, + label: obj.label, + rationale: typeof obj.rationale === "string" ? obj.rationale : "Autonomous test expansion", + kind: normalizeKind(typeof obj.kind === "string" ? obj.kind : undefined), + }; + }) + .filter((x): x is AutonomousPlanItem => Boolean(x)); + } catch { + return []; + } +} + +export async function generateAutonomousPlan( + provider: LLMProvider | null, + model: string, + plan: TestingPlan, + maxItems: number +): Promise { + if (!provider) return []; + const discovered = plan.commands.map((c) => `${c.label} => ${c.command}`).join("\n"); + const prompt = [ + "You are an autonomous QA planner.", + `Project runtime=${plan.runtime}, packageManager=${plan.packageManager}.`, + "Given discovered commands, propose extra test commands to improve confidence and edge-case coverage.", + "Output JSON array only. Max items: " + maxItems + ".", + "Each item shape: {\"label\": string, \"command\": string, \"rationale\": string, \"kind\": \"unit|integration|e2e|lint|typecheck|build|custom\"}.", + "Only produce commands from this safe family: npm/pnpm/yarn test-like scripts, pytest, go test, cargo test.", + "Commands must be derived from discovered commands by appending flags/args only (for example: `npm run test -- --coverage`).", + "Do not use shell chaining, pipes, or destructive commands.", + "", + "Discovered commands:", + discovered || "(none)", + ].join("\n"); + + try { + const response = await provider.chat( + [ + { role: "system", content: "Return JSON only." }, + { role: "user", content: prompt }, + ], + model + ); + return parsePlanItems(response) + .filter((item) => isSafeCommand(item.command)) + .filter((item) => isDerivedFromDiscoveredCommand(item.command, plan.commands)) + .slice(0, maxItems); + } catch { + return []; + } +} + +export function toAutonomousCommands(items: AutonomousPlanItem[], existing: TestCommand[]): TestCommand[] { + const seen = new Set(existing.map((c) => c.command.trim())); + const out: TestCommand[] = []; + let idx = 0; + for (const item of items) { + const key = item.command.trim(); + if (!key || seen.has(key) || !isSafeCommand(key)) continue; + if (!isDerivedFromDiscoveredCommand(key, existing)) continue; + seen.add(key); + out.push({ + id: `auto-${idx++}`, + kind: item.kind || "custom", + label: item.label, + command: item.command, + required: false, + source: "autonomous", + rationale: item.rationale, + }); + } + return out; +} + +export async function persistAutonomousPlan( + planner: AgentMemory, + edgeHunter: AgentMemory, + items: AutonomousPlanItem[] +): Promise { + if (items.length === 0) return; + const summary = items + .map((i) => `${i.label}: ${i.command}`) + .slice(0, 12) + .join(" | "); + await planner.add(`Autonomous test plan generated: ${summary}`, { + importance: 0.78, + type: MemoryType.PROCEDURAL, + }); + for (const item of items.slice(0, 12)) { + await edgeHunter.add(`Edge-case execution idea: ${item.label} -> ${item.rationale}`, { + importance: 0.72, + type: MemoryType.SEMANTIC, + }); + } +} diff --git a/src/testing/discovery.ts b/src/testing/discovery.ts new file mode 100644 index 0000000..471380e --- /dev/null +++ b/src/testing/discovery.ts @@ -0,0 +1,147 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import type { TestCommand, TestingPlan } from "./types.js"; + +interface PackageJson { + scripts?: Record; +} + +function detectPackageManager(cwd: string): TestingPlan["packageManager"] { + if (fs.existsSync(path.join(cwd, "pnpm-lock.yaml"))) return "pnpm"; + if (fs.existsSync(path.join(cwd, "yarn.lock"))) return "yarn"; + if (fs.existsSync(path.join(cwd, "package-lock.json"))) return "npm"; + return "unknown"; +} + +function scriptCommand(pm: TestingPlan["packageManager"], script: string): string { + if (pm === "pnpm") return `pnpm run ${script}`; + if (pm === "yarn") return `yarn ${script}`; + return `npm run ${script}`; +} + +function addScriptIfPresent( + commands: TestCommand[], + scripts: Record, + packageManager: TestingPlan["packageManager"], + kind: TestCommand["kind"], + candidates: string[], + label: string, + required: boolean +): void { + const script = candidates.find((s) => s in scripts); + if (!script) return; + commands.push({ + id: `${kind}-${script}`, + kind, + label: `${label} (${script})`, + command: scriptCommand(packageManager, script), + required, + source: "discovery", + }); +} + +export function discoverTestingPlan(cwd: string): TestingPlan { + const packageManager = detectPackageManager(cwd); + const notes: string[] = []; + const commands: TestCommand[] = []; + const packagePath = path.join(cwd, "package.json"); + + if (fs.existsSync(packagePath)) { + const pkg = JSON.parse(fs.readFileSync(packagePath, "utf-8")) as PackageJson; + const scripts = pkg.scripts || {}; + notes.push("Detected Node.js project via package.json."); + + addScriptIfPresent(commands, scripts, packageManager, "lint", ["lint"], "Lint", false); + addScriptIfPresent( + commands, + scripts, + packageManager, + "typecheck", + ["typecheck", "check-types", "tsc"], + "Typecheck", + false + ); + addScriptIfPresent( + commands, + scripts, + packageManager, + "unit", + ["test", "test:unit"], + "Unit tests", + true + ); + addScriptIfPresent( + commands, + scripts, + packageManager, + "integration", + ["test:integration", "integration"], + "Integration tests", + false + ); + addScriptIfPresent( + commands, + scripts, + packageManager, + "e2e", + ["test:e2e", "e2e"], + "E2E tests", + false + ); + addScriptIfPresent(commands, scripts, packageManager, "build", ["build"], "Build", false); + + return { + runtime: "node", + packageManager, + commands, + notes, + }; + } + + if (fs.existsSync(path.join(cwd, "pyproject.toml")) || fs.existsSync(path.join(cwd, "requirements.txt"))) { + notes.push("Detected Python project."); + commands.push({ + id: "unit-pytest", + kind: "unit", + label: "Unit tests (pytest)", + command: "python -m pytest -q", + required: true, + source: "discovery", + }); + return { runtime: "python", packageManager: "unknown", commands, notes }; + } + + if (fs.existsSync(path.join(cwd, "go.mod"))) { + notes.push("Detected Go project."); + commands.push({ + id: "unit-go-test", + kind: "unit", + label: "Unit tests (go test)", + command: "go test ./...", + required: true, + source: "discovery", + }); + return { runtime: "go", packageManager: "unknown", commands, notes }; + } + + if (fs.existsSync(path.join(cwd, "Cargo.toml"))) { + notes.push("Detected Rust project."); + commands.push({ + id: "unit-cargo-test", + kind: "unit", + label: "Unit tests (cargo test)", + command: "cargo test", + required: true, + source: "discovery", + }); + return { runtime: "rust", packageManager: "unknown", commands, notes }; + } + + notes.push("No known project runtime detected."); + return { + runtime: "unknown", + packageManager: "unknown", + commands: [], + notes, + }; +} diff --git a/src/testing/orchestrator.ts b/src/testing/orchestrator.ts new file mode 100644 index 0000000..21f105b --- /dev/null +++ b/src/testing/orchestrator.ts @@ -0,0 +1,162 @@ +import type { AgentMemory } from "../core/agent.js"; +import { MemoryType } from "../core/types.js"; +import type { LLMProvider } from "../llm/provider.js"; +import type { + CommandExecutionResult, + TestingInsights, + TestingPlan, + TestingRunReport, +} from "./types.js"; + +function fallbackInsights(results: CommandExecutionResult[]): TestingInsights { + const failures = results.filter((r) => !r.passed); + const summary = + failures.length === 0 + ? "All executed checks passed. Continue expanding edge-case and integration coverage." + : `${failures.length} check(s) failed. Prioritize fixing failing checks before new feature work.`; + + const score = Math.max(0, 100 - failures.length * 25); + return { + summary, + qualityScore: score, + edgeCases: [ + "Invalid or empty user input paths and identifiers", + "Concurrent writes/race conditions under repeated requests", + "Boundary limits (very large payloads and deeply nested objects)", + ], + gaps: failures.length > 0 ? failures.map((f) => `${f.command.label} is failing`) : [], + nextSteps: [ + "Add regression tests for every failure fixed in this run", + "Add property-based tests for parser/validation logic", + "Run smoke tests against production-like environment variables", + ], + }; +} + +function safeJsonParse(text: string): Partial | null { + const first = text.indexOf("{"); + const last = text.lastIndexOf("}"); + if (first === -1 || last === -1 || last <= first) return null; + try { + return JSON.parse(text.slice(first, last + 1)) as Partial; + } catch { + return null; + } +} + +export async function buildTestingInsights( + provider: LLMProvider | null, + model: string, + plan: TestingPlan, + results: CommandExecutionResult[] +): Promise { + if (!provider) return fallbackInsights(results); + + const failures = results + .filter((r) => !r.passed) + .map((r) => ({ + command: r.command.command, + label: r.command.label, + output: r.output.slice(0, 2000), + })); + + const prompt = [ + "You are a senior test orchestrator agent.", + "Given the test execution data, return strict JSON with this shape:", + '{ "summary": string, "qualityScore": number, "edgeCases": string[], "gaps": string[], "nextSteps": string[] }', + "qualityScore must be between 0 and 100.", + "", + `Runtime: ${plan.runtime}; package manager: ${plan.packageManager}`, + `Executed commands: ${results.length}`, + `Passed: ${results.filter((r) => r.passed).length}`, + `Failed: ${failures.length}`, + "", + `Failures JSON: ${JSON.stringify(failures)}`, + ].join("\n"); + + try { + const response = await provider.chat( + [ + { role: "system", content: "Return JSON only. No markdown." }, + { role: "user", content: prompt }, + ], + model + ); + const parsed = safeJsonParse(response); + if (!parsed) return fallbackInsights(results); + return { + summary: + typeof parsed.summary === "string" + ? parsed.summary + : fallbackInsights(results).summary, + qualityScore: + typeof parsed.qualityScore === "number" + ? Math.min(100, Math.max(0, Math.round(parsed.qualityScore))) + : fallbackInsights(results).qualityScore, + edgeCases: Array.isArray(parsed.edgeCases) + ? parsed.edgeCases.filter((x): x is string => typeof x === "string") + : fallbackInsights(results).edgeCases, + gaps: Array.isArray(parsed.gaps) + ? parsed.gaps.filter((x): x is string => typeof x === "string") + : fallbackInsights(results).gaps, + nextSteps: Array.isArray(parsed.nextSteps) + ? parsed.nextSteps.filter((x): x is string => typeof x === "string") + : fallbackInsights(results).nextSteps, + }; + } catch { + return fallbackInsights(results); + } +} + +export function makeTestingReport( + projectPath: string, + plan: TestingPlan, + results: CommandExecutionResult[], + insights: TestingInsights +): TestingRunReport { + return { + projectPath, + createdAt: Date.now(), + plan, + results, + insights, + }; +} + +export async function persistRunToMemory( + orchestrator: AgentMemory, + edgeHunter: AgentMemory, + reporter: AgentMemory, + report: TestingRunReport +): Promise { + const resultSummary = report.results + .map((r) => `${r.command.kind}:${r.passed ? "pass" : "fail"}:${r.command.command}`) + .join(" | "); + + await orchestrator.add( + `Testing run summary (${new Date(report.createdAt).toISOString()}): ${resultSummary}`, + { + importance: 0.75, + type: MemoryType.EPISODIC, + metadata: { qualityScore: report.insights.qualityScore }, + } + ); + + for (const edgeCase of report.insights.edgeCases.slice(0, 10)) { + await edgeHunter.add(`Edge case to test: ${edgeCase}`, { + importance: 0.7, + type: MemoryType.PROCEDURAL, + }); + } + + await reporter.add( + `Quality score=${report.insights.qualityScore}. Summary: ${report.insights.summary}`, + { + importance: 0.8, + type: MemoryType.SEMANTIC, + metadata: { + failedCommands: report.results.filter((r) => !r.passed).map((r) => r.command.command), + }, + } + ); +} diff --git a/src/testing/report.ts b/src/testing/report.ts new file mode 100644 index 0000000..0305035 --- /dev/null +++ b/src/testing/report.ts @@ -0,0 +1,63 @@ +import { icons, t } from "../ui/theme.js"; +import type { TestingRunReport } from "./types.js"; + +function ms(msValue: number): string { + if (msValue < 1000) return `${msValue}ms`; + return `${(msValue / 1000).toFixed(1)}s`; +} + +export function renderTestingReport(report: TestingRunReport): string { + const total = report.results.length; + const passed = report.results.filter((r) => r.passed).length; + const failed = total - passed; + + const lines: string[] = []; + lines.push(""); + lines.push(` ${t.brandBold("Testing Run Report")}`); + lines.push(` ${t.muted("─".repeat(60))}`); + lines.push(` ${t.label("Project")} ${report.projectPath}`); + lines.push(` ${t.label("Runtime")} ${report.plan.runtime}`); + lines.push(` ${t.label("Commands")} ${total} (${passed} passed, ${failed} failed)`); + lines.push(` ${t.label("Quality score")} ${t.bold(String(report.insights.qualityScore))}/100`); + lines.push(` ${t.muted("─".repeat(60))}`); + + for (const result of report.results) { + const mark = result.passed ? t.success(icons.check) : t.error(icons.cross); + const color = result.passed ? t.success : t.error; + lines.push( + ` ${mark} ${color(result.command.label)} ${t.dim(`(${result.command.command})`)} ${t.muted(ms(result.durationMs))}` + ); + if (result.command.source === "autonomous" && result.command.rationale) { + lines.push(` ${t.dim(`auto rationale: ${result.command.rationale}`)}`); + } + const headline = result.output.split("\n")[0]?.trim(); + if (headline) lines.push(` ${t.dim(headline.substring(0, 120))}`); + } + + lines.push(` ${t.muted("─".repeat(60))}`); + lines.push(` ${t.label("Summary")} ${report.insights.summary}`); + + if (report.insights.edgeCases.length > 0) { + lines.push(` ${t.label("Edge cases")}`); + for (const item of report.insights.edgeCases.slice(0, 6)) { + lines.push(` ${icons.arrowRight} ${item}`); + } + } + + if (report.insights.gaps.length > 0) { + lines.push(` ${t.label("Coverage gaps")}`); + for (const item of report.insights.gaps.slice(0, 6)) { + lines.push(` ${icons.arrowRight} ${item}`); + } + } + + if (report.insights.nextSteps.length > 0) { + lines.push(` ${t.label("Recommended next steps")}`); + for (const item of report.insights.nextSteps.slice(0, 6)) { + lines.push(` ${icons.arrowRight} ${item}`); + } + } + + lines.push(""); + return lines.join("\n"); +} diff --git a/src/testing/runner.ts b/src/testing/runner.ts new file mode 100644 index 0000000..044b35e --- /dev/null +++ b/src/testing/runner.ts @@ -0,0 +1,63 @@ +import { execSync } from "node:child_process"; +import type { CommandExecutionResult, TestCommand } from "./types.js"; + +function parseCounts(output: string): { passedCount?: number; failedCount?: number } { + const passedMatch = output.match(/(\d+)\s+passed/i); + const failedMatch = output.match(/(\d+)\s+failed/i); + return { + passedCount: passedMatch ? parseInt(passedMatch[1], 10) : undefined, + failedCount: failedMatch ? parseInt(failedMatch[1], 10) : undefined, + }; +} + +function runOne(command: TestCommand, cwd: string, timeoutMs: number): CommandExecutionResult { + const startedAt = Date.now(); + let output = ""; + let exitCode = 0; + + try { + const stdout = execSync(command.command, { + cwd, + timeout: timeoutMs, + maxBuffer: 8 * 1024 * 1024, + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + }); + output = stdout || "(no output)"; + } catch (err: unknown) { + const e = err as { stdout?: string; stderr?: string; status?: number; message?: string }; + output = [e.stdout, e.stderr, e.message].filter(Boolean).join("\n").trim() || "Command failed"; + exitCode = e.status ?? 1; + } + + const endedAt = Date.now(); + const durationMs = endedAt - startedAt; + const counts = parseCounts(output); + + return { + command, + startedAt, + endedAt, + durationMs, + exitCode, + passed: exitCode === 0, + output: output.length > 10000 ? output.slice(0, 10000) + "\n... (truncated)" : output, + ...counts, + }; +} + +export function runTestingCommands( + commands: TestCommand[], + cwd: string, + timeoutMs = 120000 +): CommandExecutionResult[] { + const results: CommandExecutionResult[] = []; + for (const command of commands) { + const result = runOne(command, cwd, timeoutMs); + results.push(result); + if (!result.passed && command.required) { + break; + } + } + return results; +} diff --git a/src/testing/types.ts b/src/testing/types.ts new file mode 100644 index 0000000..70f7ee3 --- /dev/null +++ b/src/testing/types.ts @@ -0,0 +1,60 @@ +export type TestCommandKind = + | "lint" + | "typecheck" + | "unit" + | "integration" + | "e2e" + | "build" + | "custom"; + +export interface TestCommand { + id: string; + kind: TestCommandKind; + label: string; + command: string; + required: boolean; + source?: "discovery" | "autonomous"; + rationale?: string; +} + +export interface CommandExecutionResult { + command: TestCommand; + startedAt: number; + endedAt: number; + exitCode: number; + output: string; + passed: boolean; + durationMs: number; + passedCount?: number; + failedCount?: number; +} + +export interface TestingPlan { + runtime: "node" | "python" | "go" | "rust" | "unknown"; + packageManager: "npm" | "pnpm" | "yarn" | "unknown"; + commands: TestCommand[]; + notes: string[]; +} + +export interface TestingInsights { + summary: string; + qualityScore: number; + edgeCases: string[]; + gaps: string[]; + nextSteps: string[]; +} + +export interface TestingRunReport { + projectPath: string; + createdAt: number; + plan: TestingPlan; + results: CommandExecutionResult[]; + insights: TestingInsights; +} + +export interface AutonomousPlanItem { + label: string; + command: string; + rationale: string; + kind?: TestCommandKind; +} diff --git a/tests/testing/autonomous.test.ts b/tests/testing/autonomous.test.ts new file mode 100644 index 0000000..c515f46 --- /dev/null +++ b/tests/testing/autonomous.test.ts @@ -0,0 +1,45 @@ +import { describe, it, expect } from "vitest"; +import { toAutonomousCommands } from "../../src/testing/autonomous.js"; +import type { TestCommand } from "../../src/testing/types.js"; + +describe("autonomous planning", () => { + it("filters duplicates and unsafe commands", () => { + const existing: TestCommand[] = [ + { + id: "base-1", + kind: "unit", + label: "Unit tests", + command: "npm run test", + required: true, + }, + ]; + + const out = toAutonomousCommands( + [ + { + label: "dup", + command: "npm run test", + rationale: "duplicate", + kind: "unit", + }, + { + label: "safe", + command: "npm run test -- --coverage", + rationale: "expand confidence with coverage signal", + kind: "unit", + }, + { + label: "unsafe", + command: "rm -rf /", + rationale: "nope", + kind: "custom", + }, + ], + existing + ); + + expect(out).toHaveLength(1); + expect(out[0].command).toBe("npm run test -- --coverage"); + expect(out[0].source).toBe("autonomous"); + }); +}); diff --git a/tests/testing/discovery.test.ts b/tests/testing/discovery.test.ts new file mode 100644 index 0000000..5b606e7 --- /dev/null +++ b/tests/testing/discovery.test.ts @@ -0,0 +1,57 @@ +import { describe, it, expect } from "vitest"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { discoverTestingPlan } from "../../src/testing/discovery.js"; + +function withTempDir(fn: (dir: string) => void): void { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "weave-test-discovery-")); + try { + fn(dir); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +describe("discoverTestingPlan", () => { + it("discovers node scripts in sensible order", () => { + withTempDir((dir) => { + fs.writeFileSync( + path.join(dir, "package.json"), + JSON.stringify( + { + scripts: { + lint: "eslint .", + typecheck: "tsc --noEmit", + test: "vitest run", + "test:e2e": "playwright test", + }, + }, + null, + 2 + ) + ); + fs.writeFileSync(path.join(dir, "package-lock.json"), "{}"); + + const plan = discoverTestingPlan(dir); + expect(plan.runtime).toBe("node"); + expect(plan.packageManager).toBe("npm"); + expect(plan.commands.map((c) => c.kind)).toEqual([ + "lint", + "typecheck", + "unit", + "e2e", + ]); + expect(plan.commands[2].command).toBe("npm run test"); + }); + }); + + it("detects python fallback", () => { + withTempDir((dir) => { + fs.writeFileSync(path.join(dir, "pyproject.toml"), "[project]\nname='x'\n"); + const plan = discoverTestingPlan(dir); + expect(plan.runtime).toBe("python"); + expect(plan.commands[0].command).toContain("pytest"); + }); + }); +}); diff --git a/tests/testing/runner.test.ts b/tests/testing/runner.test.ts new file mode 100644 index 0000000..5ed5a30 --- /dev/null +++ b/tests/testing/runner.test.ts @@ -0,0 +1,58 @@ +import { describe, it, expect } from "vitest"; +import { runTestingCommands } from "../../src/testing/runner.js"; + +describe("runTestingCommands", () => { + it("runs commands and records pass/fail", () => { + const results = runTestingCommands( + [ + { + id: "ok", + kind: "custom", + label: "ok", + command: "node -e \"console.log('1 passed')\"", + required: true, + }, + { + id: "bad", + kind: "custom", + label: "bad", + command: "node -e \"process.exit(1)\"", + required: false, + }, + ], + process.cwd(), + 10_000 + ); + + expect(results).toHaveLength(2); + expect(results[0].passed).toBe(true); + expect(results[0].passedCount).toBe(1); + expect(results[1].passed).toBe(false); + }); + + it("stops on required failure", () => { + const results = runTestingCommands( + [ + { + id: "required-fail", + kind: "custom", + label: "required fail", + command: "node -e \"process.exit(1)\"", + required: true, + }, + { + id: "never", + kind: "custom", + label: "never", + command: "node -e \"console.log('skip')\"", + required: false, + }, + ], + process.cwd(), + 10_000 + ); + + expect(results).toHaveLength(1); + expect(results[0].passed).toBe(false); + }); +}); From 1524132ebefe97dd938f4f3fe259a0b438904eb5 Mon Sep 17 00:00:00 2001 From: jayavibhavnk Date: Wed, 11 Mar 2026 16:49:04 -0700 Subject: [PATCH 2/8] added automations and schedulers --- README.md | 29 ++ src/core/storage.ts | 152 ++++++++ src/index.ts | 484 ++++++++++++++++++------- src/testing/automation-cli.ts | 109 ++++++ src/testing/automation-store.ts | 246 +++++++++++++ src/testing/automation-types.ts | 49 +++ src/testing/run-workflow.ts | 153 ++++++++ src/testing/scheduler.ts | 116 ++++++ tests/testing/automation-cli.test.ts | 27 ++ tests/testing/automation-store.test.ts | 59 +++ tests/testing/scheduler.test.ts | 81 +++++ 11 files changed, 1382 insertions(+), 123 deletions(-) create mode 100644 src/testing/automation-cli.ts create mode 100644 src/testing/automation-store.ts create mode 100644 src/testing/automation-types.ts create mode 100644 src/testing/run-workflow.ts create mode 100644 src/testing/scheduler.ts create mode 100644 tests/testing/automation-cli.test.ts create mode 100644 tests/testing/automation-store.test.ts create mode 100644 tests/testing/scheduler.test.ts diff --git a/README.md b/README.md index 225d6ca..8ede8f4 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,12 @@ weave test run --provider anthropic # use another model provider weave test run --model gpt-4o-mini # choose specific model for insights weave test run --max-auto 5 # add up to 5 autonomous expansions weave test run --no-autonomous # run only discovered commands + +weave-test automation create --name "nightly smoke" --dir . --every 1d +weave-test automation remind "in 45 minutes" --name "rerun tests" --dir . +weave-test automation list +weave-test automation run +weave-test automation daemon # keep scheduler running locally ``` The testing workflow: @@ -92,6 +98,29 @@ The testing workflow: - analyzes failures and edge-case gaps, - persists run intelligence to memory for future sessions. +### Automations + +`weave-test` now supports durable test automations: + +- `automation create` for recurring schedules via `--every` or `--cron` +- `automation remind` for one-time reminders/check-backs +- `automation loop` as a Claude-style recurring shortcut +- `automation daemon` to keep due automations running locally + +Examples: + +```bash +weave-test automation create --name "daily regression" --dir . --every 1d +weave-test automation create --name "weekday smoke" --dir . --cron "0 9 * * 1-5" +weave-test automation remind "in 2 hours" --name "rerun failed checks" --dir . +weave-test automation loop 30m --name "poll health" --dir . --target testPlan +weave-test automation list +weave-test automation pause +weave-test automation resume +weave-test automation run +weave-test automation daemon --poll-ms 10000 +``` + ### Chat ```bash diff --git a/src/core/storage.ts b/src/core/storage.ts index 8f5cf87..41d1102 100644 --- a/src/core/storage.ts +++ b/src/core/storage.ts @@ -4,6 +4,10 @@ import type { AgentPersona, AgentState, } from "./types.js"; +import type { + AutomationRecord, + AutomationRunRecord, +} from "../testing/automation-types.js"; export class Storage { private db: any; @@ -59,6 +63,36 @@ export class Storage { CREATE INDEX IF NOT EXISTS idx_nodes_tier ON nodes(tier); CREATE INDEX IF NOT EXISTS idx_edges_source ON edges(source_id); CREATE INDEX IF NOT EXISTS idx_edges_target ON edges(target_id); + + CREATE TABLE IF NOT EXISTS automations ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + enabled INTEGER NOT NULL, + trigger_json TEXT NOT NULL, + target_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + last_run_at INTEGER, + next_run_at INTEGER, + failure_count INTEGER NOT NULL, + max_failures INTEGER NOT NULL, + last_status TEXT, + last_summary TEXT + ); + + CREATE TABLE IF NOT EXISTS automation_runs ( + id TEXT PRIMARY KEY, + automation_id TEXT NOT NULL, + started_at INTEGER NOT NULL, + ended_at INTEGER, + status TEXT NOT NULL, + summary TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_automations_enabled_next + ON automations(enabled, next_run_at); + CREATE INDEX IF NOT EXISTS idx_automation_runs_automation + ON automation_runs(automation_id, started_at); `); } @@ -185,6 +219,80 @@ export class Storage { this.db.prepare("UPDATE agents SET last_active = ? WHERE id = ?").run(Date.now(), id); } + saveAutomation(record: AutomationRecord): void { + this.db + .prepare( + `INSERT OR REPLACE INTO automations + (id, name, enabled, trigger_json, target_json, created_at, updated_at, + last_run_at, next_run_at, failure_count, max_failures, last_status, last_summary) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .run( + record.id, + record.name, + record.enabled ? 1 : 0, + JSON.stringify(record.trigger), + JSON.stringify(record.target), + record.createdAt, + record.updatedAt, + record.lastRunAt ?? null, + record.nextRunAt ?? null, + record.failureCount, + record.maxFailures, + record.lastStatus ?? null, + record.lastSummary ?? null + ); + } + + loadAutomations(): AutomationRecord[] { + const rows = this.db + .prepare("SELECT * FROM automations ORDER BY created_at ASC") + .all() as Record[]; + return rows.map(rowToAutomation); + } + + loadAutomation(id: string): AutomationRecord | null { + const row = this.db + .prepare("SELECT * FROM automations WHERE id = ?") + .get(id) as Record | undefined; + return row ? rowToAutomation(row) : null; + } + + deleteAutomation(id: string): void { + this.db.prepare("DELETE FROM automation_runs WHERE automation_id = ?").run(id); + this.db.prepare("DELETE FROM automations WHERE id = ?").run(id); + } + + saveAutomationRun(run: AutomationRunRecord): void { + this.db + .prepare( + `INSERT OR REPLACE INTO automation_runs + (id, automation_id, started_at, ended_at, status, summary) + VALUES (?, ?, ?, ?, ?, ?)` + ) + .run( + run.id, + run.automationId, + run.startedAt, + run.endedAt ?? null, + run.status, + run.summary + ); + } + + loadAutomationRuns(automationId?: string): AutomationRunRecord[] { + const rows = automationId + ? (this.db + .prepare( + "SELECT * FROM automation_runs WHERE automation_id = ? ORDER BY started_at DESC" + ) + .all(automationId) as Record[]) + : (this.db + .prepare("SELECT * FROM automation_runs ORDER BY started_at DESC") + .all() as Record[]); + return rows.map(rowToAutomationRun); + } + saveAll(nodes: MemoryNode[], edges: MemoryEdge[]): void { const txn = this.db.transaction(() => { for (const node of nodes) this.saveNode(node); @@ -226,3 +334,47 @@ function rowToEdge(row: Record): MemoryEdge { createdAt: row.created_at as number, }; } + +function rowToAutomation(row: Record): AutomationRecord { + return { + id: row.id as string, + name: row.name as string, + enabled: Boolean(row.enabled), + trigger: JSON.parse(row.trigger_json as string), + target: JSON.parse(row.target_json as string), + createdAt: row.created_at as number, + updatedAt: row.updated_at as number, + lastRunAt: + row.last_run_at === null || row.last_run_at === undefined + ? undefined + : (row.last_run_at as number), + nextRunAt: + row.next_run_at === null || row.next_run_at === undefined + ? undefined + : (row.next_run_at as number), + failureCount: row.failure_count as number, + maxFailures: row.max_failures as number, + lastStatus: + row.last_status === null || row.last_status === undefined + ? undefined + : (row.last_status as AutomationRecord["lastStatus"]), + lastSummary: + row.last_summary === null || row.last_summary === undefined + ? undefined + : (row.last_summary as string), + }; +} + +function rowToAutomationRun(row: Record): AutomationRunRecord { + return { + id: row.id as string, + automationId: row.automation_id as string, + startedAt: row.started_at as number, + endedAt: + row.ended_at === null || row.ended_at === undefined + ? undefined + : (row.ended_at as number), + status: row.status as AutomationRunRecord["status"], + summary: row.summary as string, + }; +} diff --git a/src/index.ts b/src/index.ts index 64bd941..ad17c4a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -398,46 +398,36 @@ testCmd .option("--max-auto ", "Maximum autonomous test commands to add (0 disables)", "0") .option("--no-autonomous", "Disable autonomous test expansion") .action(async (options) => { - const config = loadConfig(); - const providerName = options.provider || config.provider; - const model = options.model || config.model; - const apiKey = resolveApiKey({ ...config, provider: providerName }); - const isLocal = providerName === "ollama" || providerName === "lmstudio"; - const targetDir = String(options.dir); - const maxAutonomous = Math.max(0, parseInt(String(options.maxAuto), 10) || 0); - const autonomousEnabled = Boolean(options.autonomous); - - const { discoverTestingPlan } = await import("./testing/discovery.js"); - const { generateAutonomousPlan, toAutonomousCommands } = await import("./testing/autonomous.js"); - const { createProvider } = await import("./llm/provider.js"); - const plan = discoverTestingPlan(targetDir); - - if (plan.commands.length === 0) { - console.log(errorLine("No test commands discovered for this project.")); - process.exit(1); - } + try { + const { planTestWorkflow } = await import("./testing/run-workflow.js"); + const targetDir = String(options.dir); + const maxAutonomous = Math.max(0, parseInt(String(options.maxAuto), 10) || 0); + const autonomousEnabled = Boolean(options.autonomous); + const plan = await planTestWorkflow({ + workspace: "default", + dir: targetDir, + model: options.model, + provider: options.provider, + maxAuto: maxAutonomous, + autonomous: autonomousEnabled, + }); - const llm = apiKey || isLocal - ? createProvider(providerName, apiKey || "ollama", model, config.baseURL) - : null; - const autonomousItems = - autonomousEnabled && maxAutonomous > 0 - ? await generateAutonomousPlan(llm, model, plan, maxAutonomous) - : []; - const autonomousCommands = toAutonomousCommands(autonomousItems, plan.commands); - - console.log(`\n ${t.brandBold("Testing plan")}`); - console.log(` ${t.muted("─".repeat(50))}`); - for (const cmd of plan.commands) { - console.log(` ${t.muted(icons.arrow)} ${cmd.label} ${t.dim(`(${cmd.command})`)}`); - } - if (autonomousCommands.length > 0) { - console.log(` ${t.brandBold("Autonomous additions")}`); - for (const cmd of autonomousCommands) { + console.log(`\n ${t.brandBold("Testing plan")}`); + console.log(` ${t.muted("─".repeat(50))}`); + for (const cmd of plan.discoveredCommands) { console.log(` ${t.muted(icons.arrow)} ${cmd.label} ${t.dim(`(${cmd.command})`)}`); } + if (plan.autonomousCommands.length > 0) { + console.log(` ${t.brandBold("Autonomous additions")}`); + for (const cmd of plan.autonomousCommands) { + console.log(` ${t.muted(icons.arrow)} ${cmd.label} ${t.dim(`(${cmd.command})`)}`); + } + } + console.log(""); + } catch (err) { + console.log(errorLine(err instanceof Error ? err.message : String(err))); + process.exit(1); } - console.log(""); }); testCmd @@ -451,108 +441,356 @@ testCmd .option("--max-auto ", "Maximum autonomous test commands to add (0 disables)", "0") .option("--no-autonomous", "Disable autonomous test expansion") .action(async (options) => { - const config = loadConfig(); - const providerName = options.provider || config.provider; - const model = options.model || config.model; - const apiKey = resolveApiKey({ ...config, provider: providerName }); - const isLocal = providerName === "ollama" || providerName === "lmstudio"; - const workspacePath = getWorkspacePath(options.workspace); + try { + const { executeTestWorkflow } = await import("./testing/run-workflow.js"); + const targetDir = String(options.dir); + const timeoutMs = parseInt(String(options.timeout), 10); + const maxAutonomous = Math.max(0, parseInt(String(options.maxAuto), 10) || 0); + const autonomousEnabled = Boolean(options.autonomous); + + const result = await executeTestWorkflow({ + workspace: options.workspace, + dir: targetDir, + model: options.model, + provider: options.provider, + timeoutMs, + maxAuto: maxAutonomous, + autonomous: autonomousEnabled, + }); - const { MemoryFabric } = await import("./core/fabric.js"); - const { discoverTestingPlan } = await import("./testing/discovery.js"); - const { runTestingCommands } = await import("./testing/runner.js"); - const { - generateAutonomousPlan, - toAutonomousCommands, - persistAutonomousPlan, - } = await import("./testing/autonomous.js"); - const { - buildTestingInsights, - makeTestingReport, - persistRunToMemory, - } = await import("./testing/orchestrator.js"); - const { renderTestingReport } = await import("./testing/report.js"); - const { createProvider } = await import("./llm/provider.js"); + console.log(`\n ${t.brandBold("Running testing pipeline")}`); + console.log(` ${t.muted("─".repeat(40))}`); + for (const cmd of result.discoveredCommands) { + console.log(` ${t.muted(icons.arrow)} ${cmd.label} ${t.dim(`(${cmd.command})`)}`); + } + if (result.autonomousCommands.length > 0) { + console.log(` ${t.brandBold("Autonomous expansions")}`); + for (const cmd of result.autonomousCommands) { + console.log(` ${t.muted(icons.arrow)} ${cmd.label} ${t.dim(`(${cmd.command})`)}`); + } + } + console.log(""); + console.log(result.renderedReport); + } catch (err) { + console.log(errorLine(err instanceof Error ? err.message : String(err))); + process.exit(1); + } + }); - const fabric = await MemoryFabric.create({ - ...config, - provider: providerName, - model, - apiKey, - workspacePath, - }); +// ── weave-test automation ───────────────────────────────── +const automationCmd = program + .command("automation") + .description("Durable automations for recurring testing workflows"); + +automationCmd + .command("create") + .description("Create a durable automation") + .requiredOption("-n, --name ", "Automation name") + .requiredOption("-d, --dir ", "Target project directory") + .option("-w, --workspace ", "Workspace to store and run in", "default") + .option("--target ", "Automation target (testRun|testPlan)", "testRun") + .option("--every ", "Recurring interval like 15m, 2h, or 1d") + .option("--cron ", "5-field cron expression") + .option("--at