diff --git a/README.md b/README.md index 770671a..784cb1d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ ResiliReplay injects deterministic MCP failures, verifies bounded recovery, and executable regression tests. ```console -npx --yes resilireplay@latest mcp demo +npx --yes resilireplay@0.7.0 mcp demo ``` ```text @@ -27,10 +27,13 @@ Evidence: sha256:4d2479b98453732d6011c699c061d023353c8d5cf4159bdf6c5b096961f94c0 The demo is local, deterministic, credential-free, and cleaned up after it runs. It needs Node.js 22 or 24—no repository checkout, configuration, account, API key, paid model, or remote MCP server. +This quickstart pins the released `0.7.0` artifact for reproducibility. Later `@latest` examples are +convenient for following current documentation; pin an exact version in CI and retained evidence. [Product site](https://aliengineering-byte.github.io/resilireplay/) · [npm package](https://www.npmjs.com/package/resilireplay) · -[real MCP CI example](examples/mcp-reliability-ci/README.md) +[real MCP CI example](examples/mcp-reliability-ci/README.md) · +[machine-readable capabilities](aeb-capabilities.json) ## Ten-second before and after diff --git a/aeb-capabilities.json b/aeb-capabilities.json new file mode 100644 index 0000000..2eaa62a --- /dev/null +++ b/aeb-capabilities.json @@ -0,0 +1,134 @@ +{ + "$schema": "./schemas/aeb-capabilities-v1.schema.json", + "schemaVersion": "aeb.capabilities/v1", + "repository": "aliengineering-byte/resilireplay", + "currentVersion": "0.7.0", + "maturity": "released", + "problemStatement": "Test how an MCP tool behaves under a reviewed deterministic fault, verify bounded recovery, and preserve executable regression evidence.", + "capabilities": [ + { + "id": "local-mcp-recovery-demo", + "summary": "Run a credential-free local MCP failure, bounded retry, duplicate-effect check, and generated regression.", + "evidenceClass": "directly-tested" + }, + { + "id": "reviewed-mcp-tool-test", + "summary": "Dry-run and execute one allowlisted MCP tool against an exact approved plan digest.", + "evidenceClass": "directly-tested" + }, + { + "id": "deterministic-campaign", + "summary": "Run bounded fault campaigns against sanitized traces or reviewed MCP targets and emit a versioned campaign result.", + "evidenceClass": "directly-tested" + }, + { + "id": "executable-regression-generation", + "summary": "Convert supported failure evidence into a minimized replay fixture and executable Node.js regression.", + "evidenceClass": "directly-tested" + } + ], + "supportedInputs": [ + { + "name": "MCP Inspector configuration", + "formats": ["JSON"], + "boundary": "One reviewed server and one allowlisted tool; remote targets require explicit ownership confirmation." + }, + { + "name": "ResiliReplay campaign", + "formats": ["YAML", "JSON"], + "boundary": "Schema version 1.0 with finite concurrency, retry, scenario, and total-time budgets." + }, + { + "name": "Sanitized trace", + "formats": ["JSONL"], + "boundary": "Versioned ResiliReplay trace events; raw credentials and unrestricted tool bodies are not supported evidence." + } + ], + "generatedOutputs": [ + { + "name": "MCP test evidence", + "formats": ["JSON", "JSONL"], + "boundary": "Metadata and hashes replace tool arguments and result bodies before persistence." + }, + { + "name": "Executable regression bundle", + "formats": ["JavaScript", "JSONL", "YAML", "JSON"], + "boundary": "Generated only for supported causal failures and executed before a successful result is reported." + }, + { + "name": "Campaign run", + "formats": ["JSON"], + "boundary": "Versioned campaign-run.json with an integrity hash and explicit complete, invalid, cancelled, or incomplete status." + } + ], + "interfaces": ["CLI", "GitHub Action", "local stdio MCP server"], + "installationMethods": [ + { + "kind": "npx", + "command": "npx --yes resilireplay@0.7.0 --version" + } + ], + "canonicalExamples": [ + { + "id": "local-mcp-demo", + "command": "npx --yes resilireplay@0.7.0 mcp demo --json", + "success": "JSON result is PASS, recoveryAttempts is 1, duplicateEffects is 0, regressionExecuted and cleanupComplete are true." + } + ], + "evidenceTypes": [ + "MCP failure and recovery evidence", + "campaign-run.json", + "executable regression bundle", + "SARIF, JUnit, HTML, JSON, and Markdown reports" + ], + "supportedPlatforms": [ + { + "os": "Ubuntu", + "runtimes": ["Node.js 22", "Node.js 24"], + "evidence": ".github/workflows/ci.yml" + }, + { + "os": "Windows", + "runtimes": ["Node.js 22", "Node.js 24"], + "evidence": ".github/workflows/ci.yml" + } + ], + "knownLimitations": [ + "ResiliReplay is not a sandbox, security certification, or authorization layer.", + "Retry safety remains the caller's responsibility for stateful, side-effecting, or unknown tools.", + "The bundled demo proves a local deterministic fixture, not every MCP server, transport, or production environment.", + "Secondary agent-runtime support has different evidence levels; named integrations do not imply vendor endorsement." + ], + "relatedAebRepositories": [ + { + "repository": "aliengineering-byte/resilireplay-action-smoke", + "relationship": "Independent downstream CI verifies a released ResiliReplay GitHub Action.", + "required": false + }, + { + "repository": "aliengineering-byte/gaugemesh", + "relationship": "GaugeMesh can invoke ResiliReplay across a process boundary; neither package is a runtime dependency of the other.", + "required": false + } + ], + "feedbackLocations": [ + { + "purpose": "bug or reproducibility failure", + "url": "https://github.com/aliengineering-byte/resilireplay/issues/new?template=bug_report.yml" + }, + { + "purpose": "MCP compatibility or integration request", + "url": "https://github.com/aliengineering-byte/resilireplay/issues/new?template=compatibility.yml" + }, + { + "purpose": "successful field test or adoption evidence", + "url": "https://github.com/aliengineering-byte/resilireplay/issues/new?template=field_test.yml" + }, + { + "purpose": "feature request or documentation confusion", + "url": "https://github.com/aliengineering-byte/resilireplay/issues/new?template=feature_request.yml" + } + ], + "documentation": "https://aliengineering-byte.github.io/resilireplay/", + "securityPolicy": "https://github.com/aliengineering-byte/resilireplay/security/policy" +} diff --git a/docs/CAPABILITIES.md b/docs/CAPABILITIES.md new file mode 100644 index 0000000..afc7497 --- /dev/null +++ b/docs/CAPABILITIES.md @@ -0,0 +1,24 @@ +# Machine-readable capabilities + +[`aeb-capabilities.json`](../aeb-capabilities.json) is the repository's machine-discovery record. It +describes only released, evidence-backed ResiliReplay 0.7.0 behavior and links every platform claim +to the CI definition that exercises it. It is not an MCP standard, an AEB-wide evidence envelope, +or a substitute for the runtime schemas emitted by the CLI. + +The current contract is `aeb.capabilities/v1` and is defined by the strict JSON Schema at +[`schemas/aeb-capabilities-v1.schema.json`](../schemas/aeb-capabilities-v1.schema.json). Consumers +must inspect `schemaVersion` before validation. The v1 schema rejects unknown properties and unknown +schema revisions; a future incompatible revision will use a new schema identifier and file rather +than silently changing the meaning of v1. Consumers may continue to validate a pinned v1 document, +but must not interpret a future revision using v1 assumptions. + +The root manifest is the valid fixture. The test suite also carries an intentionally malformed +fixture with an undeclared field and verifies that both it and a future schema revision fail closed: + +```console +pnpm exec vitest run tests/capabilities-manifest.test.ts +``` + +`currentVersion` must match the root package version. Capability entries use the public claim classes +defined by the project: `directly-tested`, `measured-benchmark`, `external-primary-source`, +`design-intent`, or `hypothesis`. The current record advertises only directly tested behavior. diff --git a/schemas/aeb-capabilities-v1.schema.json b/schemas/aeb-capabilities-v1.schema.json new file mode 100644 index 0000000..abead15 --- /dev/null +++ b/schemas/aeb-capabilities-v1.schema.json @@ -0,0 +1,185 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:aeb:resilireplay:schema:aeb-capabilities:v1", + "title": "ResiliReplay capability manifest v1", + "description": "Strict machine-discovery contract for the released ResiliReplay repository. It is not an ecosystem evidence-exchange format.", + "type": "object", + "additionalProperties": false, + "required": [ + "$schema", + "schemaVersion", + "repository", + "currentVersion", + "maturity", + "problemStatement", + "capabilities", + "supportedInputs", + "generatedOutputs", + "interfaces", + "installationMethods", + "canonicalExamples", + "evidenceTypes", + "supportedPlatforms", + "knownLimitations", + "relatedAebRepositories", + "feedbackLocations", + "documentation", + "securityPolicy" + ], + "properties": { + "$schema": { "const": "./schemas/aeb-capabilities-v1.schema.json" }, + "schemaVersion": { "const": "aeb.capabilities/v1" }, + "repository": { "const": "aliengineering-byte/resilireplay" }, + "currentVersion": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" }, + "maturity": { "enum": ["released", "experimental", "research"] }, + "problemStatement": { "type": "string", "minLength": 20, "maxLength": 300 }, + "capabilities": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/capability" } + }, + "supportedInputs": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/dataBoundary" } + }, + "generatedOutputs": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/dataBoundary" } + }, + "interfaces": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "minLength": 2, "maxLength": 80 } + }, + "installationMethods": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/installation" } + }, + "canonicalExamples": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/example" } + }, + "evidenceTypes": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "minLength": 3, "maxLength": 120 } + }, + "supportedPlatforms": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/platform" } + }, + "knownLimitations": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "minLength": 10, "maxLength": 300 } + }, + "relatedAebRepositories": { + "type": "array", + "items": { "$ref": "#/$defs/relation" } + }, + "feedbackLocations": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/feedback" } + }, + "documentation": { "type": "string", "pattern": "^https://[^\\s]+$" }, + "securityPolicy": { "type": "string", "pattern": "^https://[^\\s]+$" } + }, + "$defs": { + "capability": { + "type": "object", + "additionalProperties": false, + "required": ["id", "summary", "evidenceClass"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" }, + "summary": { "type": "string", "minLength": 20, "maxLength": 240 }, + "evidenceClass": { + "enum": [ + "directly-tested", + "measured-benchmark", + "external-primary-source", + "design-intent", + "hypothesis" + ] + } + } + }, + "dataBoundary": { + "type": "object", + "additionalProperties": false, + "required": ["name", "formats", "boundary"], + "properties": { + "name": { "type": "string", "minLength": 3, "maxLength": 100 }, + "formats": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "minLength": 2, "maxLength": 40 } + }, + "boundary": { "type": "string", "minLength": 10, "maxLength": 300 } + } + }, + "installation": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "command"], + "properties": { + "kind": { "type": "string", "minLength": 2, "maxLength": 40 }, + "command": { "type": "string", "minLength": 5, "maxLength": 240 } + } + }, + "example": { + "type": "object", + "additionalProperties": false, + "required": ["id", "command", "success"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" }, + "command": { "type": "string", "minLength": 5, "maxLength": 240 }, + "success": { "type": "string", "minLength": 10, "maxLength": 300 } + } + }, + "platform": { + "type": "object", + "additionalProperties": false, + "required": ["os", "runtimes", "evidence"], + "properties": { + "os": { "type": "string", "minLength": 2, "maxLength": 80 }, + "runtimes": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "minLength": 2, "maxLength": 80 } + }, + "evidence": { "type": "string", "minLength": 3, "maxLength": 180 } + } + }, + "relation": { + "type": "object", + "additionalProperties": false, + "required": ["repository", "relationship", "required"], + "properties": { + "repository": { "type": "string", "pattern": "^aliengineering-byte/[a-z0-9-]+$" }, + "relationship": { "type": "string", "minLength": 10, "maxLength": 300 }, + "required": { "type": "boolean" } + } + }, + "feedback": { + "type": "object", + "additionalProperties": false, + "required": ["purpose", "url"], + "properties": { + "purpose": { "type": "string", "minLength": 3, "maxLength": 120 }, + "url": { "type": "string", "pattern": "^https://[^\\s]+$" } + } + } + } +} diff --git a/scripts/check-site.mjs b/scripts/check-site.mjs index d4f1cb9..44d606c 100644 --- a/scripts/check-site.mjs +++ b/scripts/check-site.mjs @@ -27,13 +27,13 @@ invariant( !/(google-analytics|googletagmanager|segment\.com|posthog|mixpanel)/iu.test(html), "Analytics are not allowed", ); -for (const [name, content] of [ - ["README", readme], - ["npm README", packageReadme], - ["landing page", html], +for (const [name, content, demoCommand] of [ + ["README", readme, "resilireplay@0.7.0 mcp demo"], + ["npm README", packageReadme, "resilireplay@latest mcp demo"], + ["landing page", html, "resilireplay@latest mcp demo"], ]) { invariant( - content.includes("resilireplay@latest mcp demo"), + content.includes(demoCommand), `${name} must include the MCP-first public demo command`, ); invariant( @@ -41,11 +41,15 @@ for (const [name, content] of [ `${name} must include the reviewed real-server command`, ); invariant( - content.indexOf("resilireplay@latest mcp demo") < - content.indexOf("resilireplay@latest mcp test"), + content.indexOf(demoCommand) < content.indexOf("resilireplay@latest mcp test"), `${name} must present mcp demo before other public commands`, ); } +invariant( + readme.includes("quickstart pins the released `0.7.0` artifact") && + readme.includes("Later `@latest` examples are"), + "README must distinguish the reproducible release pin from @latest convenience commands", +); invariant( adoptGuide.includes("resilireplay@0.4.0 adopt"), "Historical v0.4 adoption guide lost its pinned command", diff --git a/tests/capabilities-manifest.test.ts b/tests/capabilities-manifest.test.ts new file mode 100644 index 0000000..15b4d8b --- /dev/null +++ b/tests/capabilities-manifest.test.ts @@ -0,0 +1,51 @@ +import { readFile } from "node:fs/promises"; +import { Ajv2020, type AnySchema } from "ajv/dist/2020.js"; +import { describe, expect, it } from "vitest"; + +async function readJson(path: string): Promise { + return JSON.parse(await readFile(path, "utf8")); +} + +describe("AEB capability manifest", () => { + it("validates the released repository manifest against its strict v1 schema", async () => { + const [schema, manifest, packageManifest] = await Promise.all([ + readJson("schemas/aeb-capabilities-v1.schema.json"), + readJson("aeb-capabilities.json"), + readJson("package.json"), + ]); + const ajv = new Ajv2020({ allErrors: true, strict: true }); + const validate = ajv.compile(schema as AnySchema); + + expect(validate(manifest), JSON.stringify(validate.errors)).toBe(true); + expect((schema as { $id: string }).$id).toBe("urn:aeb:resilireplay:schema:aeb-capabilities:v1"); + expect(manifest).toMatchObject({ + schemaVersion: "aeb.capabilities/v1", + repository: "aliengineering-byte/resilireplay", + currentVersion: (packageManifest as { version: string }).version, + }); + }); + + it("rejects undeclared fields and unknown schema revisions", async () => { + const [schema, manifest, invalid] = await Promise.all([ + readJson("schemas/aeb-capabilities-v1.schema.json"), + readJson("aeb-capabilities.json"), + readJson("tests/fixtures/aeb-capabilities/invalid-unknown-field.json"), + ]); + const ajv = new Ajv2020({ allErrors: true, strict: true }); + const validate = ajv.compile(schema as AnySchema); + + expect(validate(invalid)).toBe(false); + expect(validate.errors).toEqual( + expect.arrayContaining([expect.objectContaining({ keyword: "additionalProperties" })]), + ); + + const future = { + ...(manifest as Record), + schemaVersion: "aeb.capabilities/v2", + }; + expect(validate(future)).toBe(false); + expect(validate.errors).toEqual( + expect.arrayContaining([expect.objectContaining({ instancePath: "/schemaVersion" })]), + ); + }); +}); diff --git a/tests/fixtures/aeb-capabilities/invalid-unknown-field.json b/tests/fixtures/aeb-capabilities/invalid-unknown-field.json new file mode 100644 index 0000000..c6c6740 --- /dev/null +++ b/tests/fixtures/aeb-capabilities/invalid-unknown-field.json @@ -0,0 +1,40 @@ +{ + "$schema": "./schemas/aeb-capabilities-v1.schema.json", + "schemaVersion": "aeb.capabilities/v1", + "repository": "aliengineering-byte/resilireplay", + "currentVersion": "0.7.0", + "maturity": "released", + "problemStatement": "A malformed discovery document must not acquire undeclared capabilities.", + "capabilities": [ + { + "id": "unverified-capability", + "summary": "This unsupported behavior must be rejected rather than advertised as a capability.", + "evidenceClass": "hypothesis" + } + ], + "supportedInputs": [ + { "name": "fixture", "formats": ["JSON"], "boundary": "Validation-only malformed fixture." } + ], + "generatedOutputs": [ + { "name": "fixture", "formats": ["JSON"], "boundary": "Validation-only malformed fixture." } + ], + "interfaces": ["fixture"], + "installationMethods": [{ "kind": "fixture", "command": "do not execute" }], + "canonicalExamples": [ + { + "id": "fixture", + "command": "do not execute", + "success": "This fixture must fail validation." + } + ], + "evidenceTypes": ["validation fixture"], + "supportedPlatforms": [{ "os": "fixture", "runtimes": ["fixture"], "evidence": "fixture" }], + "knownLimitations": ["This document is intentionally invalid."], + "relatedAebRepositories": [], + "feedbackLocations": [ + { "purpose": "fixture", "url": "https://github.com/aliengineering-byte/resilireplay/issues" } + ], + "documentation": "https://aliengineering-byte.github.io/resilireplay/", + "securityPolicy": "https://github.com/aliengineering-byte/resilireplay/security/policy", + "undeclared": true +}