Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ ResiliReplay injects deterministic MCP failures, verifies bounded recovery, and
executable regression tests.

```console
npx --yes resilireplay@latest mcp demo
npx --yes resilireplay@0.7.0 mcp demo
```

```text
Expand All @@ -27,10 +27,13 @@ Evidence: sha256:4d2479b98453732d6011c699c061d023353c8d5cf4159bdf6c5b096961f94c0

The demo is local, deterministic, credential-free, and cleaned up after it runs. It needs Node.js
22 or 24—no repository checkout, configuration, account, API key, paid model, or remote MCP server.
This quickstart pins the released `0.7.0` artifact for reproducibility. Later `@latest` examples are
convenient for following current documentation; pin an exact version in CI and retained evidence.

[Product site](https://aliengineering-byte.github.io/resilireplay/) ·
[npm package](https://www.npmjs.com/package/resilireplay) ·
[real MCP CI example](examples/mcp-reliability-ci/README.md)
[real MCP CI example](examples/mcp-reliability-ci/README.md) ·
[machine-readable capabilities](aeb-capabilities.json)

## Ten-second before and after

Expand Down
134 changes: 134 additions & 0 deletions aeb-capabilities.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
{
"$schema": "./schemas/aeb-capabilities-v1.schema.json",
"schemaVersion": "aeb.capabilities/v1",
"repository": "aliengineering-byte/resilireplay",
"currentVersion": "0.7.0",
"maturity": "released",
"problemStatement": "Test how an MCP tool behaves under a reviewed deterministic fault, verify bounded recovery, and preserve executable regression evidence.",
"capabilities": [
{
"id": "local-mcp-recovery-demo",
"summary": "Run a credential-free local MCP failure, bounded retry, duplicate-effect check, and generated regression.",
"evidenceClass": "directly-tested"
},
{
"id": "reviewed-mcp-tool-test",
"summary": "Dry-run and execute one allowlisted MCP tool against an exact approved plan digest.",
"evidenceClass": "directly-tested"
},
{
"id": "deterministic-campaign",
"summary": "Run bounded fault campaigns against sanitized traces or reviewed MCP targets and emit a versioned campaign result.",
"evidenceClass": "directly-tested"
},
{
"id": "executable-regression-generation",
"summary": "Convert supported failure evidence into a minimized replay fixture and executable Node.js regression.",
"evidenceClass": "directly-tested"
}
],
"supportedInputs": [
{
"name": "MCP Inspector configuration",
"formats": ["JSON"],
"boundary": "One reviewed server and one allowlisted tool; remote targets require explicit ownership confirmation."
},
{
"name": "ResiliReplay campaign",
"formats": ["YAML", "JSON"],
"boundary": "Schema version 1.0 with finite concurrency, retry, scenario, and total-time budgets."
},
{
"name": "Sanitized trace",
"formats": ["JSONL"],
"boundary": "Versioned ResiliReplay trace events; raw credentials and unrestricted tool bodies are not supported evidence."
}
],
"generatedOutputs": [
{
"name": "MCP test evidence",
"formats": ["JSON", "JSONL"],
"boundary": "Metadata and hashes replace tool arguments and result bodies before persistence."
},
{
"name": "Executable regression bundle",
"formats": ["JavaScript", "JSONL", "YAML", "JSON"],
"boundary": "Generated only for supported causal failures and executed before a successful result is reported."
},
{
"name": "Campaign run",
"formats": ["JSON"],
"boundary": "Versioned campaign-run.json with an integrity hash and explicit complete, invalid, cancelled, or incomplete status."
}
],
"interfaces": ["CLI", "GitHub Action", "local stdio MCP server"],
"installationMethods": [
{
"kind": "npx",
"command": "npx --yes resilireplay@0.7.0 --version"
}
],
"canonicalExamples": [
{
"id": "local-mcp-demo",
"command": "npx --yes resilireplay@0.7.0 mcp demo --json",
"success": "JSON result is PASS, recoveryAttempts is 1, duplicateEffects is 0, regressionExecuted and cleanupComplete are true."
}
],
"evidenceTypes": [
"MCP failure and recovery evidence",
"campaign-run.json",
"executable regression bundle",
"SARIF, JUnit, HTML, JSON, and Markdown reports"
],
"supportedPlatforms": [
{
"os": "Ubuntu",
"runtimes": ["Node.js 22", "Node.js 24"],
"evidence": ".github/workflows/ci.yml"
},
{
"os": "Windows",
"runtimes": ["Node.js 22", "Node.js 24"],
"evidence": ".github/workflows/ci.yml"
}
],
"knownLimitations": [
"ResiliReplay is not a sandbox, security certification, or authorization layer.",
"Retry safety remains the caller's responsibility for stateful, side-effecting, or unknown tools.",
"The bundled demo proves a local deterministic fixture, not every MCP server, transport, or production environment.",
"Secondary agent-runtime support has different evidence levels; named integrations do not imply vendor endorsement."
],
"relatedAebRepositories": [
{
"repository": "aliengineering-byte/resilireplay-action-smoke",
"relationship": "Independent downstream CI verifies a released ResiliReplay GitHub Action.",
"required": false
},
{
"repository": "aliengineering-byte/gaugemesh",
"relationship": "GaugeMesh can invoke ResiliReplay across a process boundary; neither package is a runtime dependency of the other.",
"required": false
}
],
"feedbackLocations": [
{
"purpose": "bug or reproducibility failure",
"url": "https://github.com/aliengineering-byte/resilireplay/issues/new?template=bug_report.yml"
},
{
"purpose": "MCP compatibility or integration request",
"url": "https://github.com/aliengineering-byte/resilireplay/issues/new?template=compatibility.yml"
},
{
"purpose": "successful field test or adoption evidence",
"url": "https://github.com/aliengineering-byte/resilireplay/issues/new?template=field_test.yml"
},
{
"purpose": "feature request or documentation confusion",
"url": "https://github.com/aliengineering-byte/resilireplay/issues/new?template=feature_request.yml"
}
],
"documentation": "https://aliengineering-byte.github.io/resilireplay/",
"securityPolicy": "https://github.com/aliengineering-byte/resilireplay/security/policy"
}
24 changes: 24 additions & 0 deletions docs/CAPABILITIES.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Machine-readable capabilities

[`aeb-capabilities.json`](../aeb-capabilities.json) is the repository's machine-discovery record. It
describes only released, evidence-backed ResiliReplay 0.7.0 behavior and links every platform claim
to the CI definition that exercises it. It is not an MCP standard, an AEB-wide evidence envelope,
or a substitute for the runtime schemas emitted by the CLI.

The current contract is `aeb.capabilities/v1` and is defined by the strict JSON Schema at
[`schemas/aeb-capabilities-v1.schema.json`](../schemas/aeb-capabilities-v1.schema.json). Consumers
must inspect `schemaVersion` before validation. The v1 schema rejects unknown properties and unknown
schema revisions; a future incompatible revision will use a new schema identifier and file rather
than silently changing the meaning of v1. Consumers may continue to validate a pinned v1 document,
but must not interpret a future revision using v1 assumptions.

The root manifest is the valid fixture. The test suite also carries an intentionally malformed
fixture with an undeclared field and verifies that both it and a future schema revision fail closed:

```console
pnpm exec vitest run tests/capabilities-manifest.test.ts
```

`currentVersion` must match the root package version. Capability entries use the public claim classes
defined by the project: `directly-tested`, `measured-benchmark`, `external-primary-source`,
`design-intent`, or `hypothesis`. The current record advertises only directly tested behavior.
185 changes: 185 additions & 0 deletions schemas/aeb-capabilities-v1.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "urn:aeb:resilireplay:schema:aeb-capabilities:v1",
"title": "ResiliReplay capability manifest v1",
"description": "Strict machine-discovery contract for the released ResiliReplay repository. It is not an ecosystem evidence-exchange format.",
"type": "object",
"additionalProperties": false,
"required": [
"$schema",
"schemaVersion",
"repository",
"currentVersion",
"maturity",
"problemStatement",
"capabilities",
"supportedInputs",
"generatedOutputs",
"interfaces",
"installationMethods",
"canonicalExamples",
"evidenceTypes",
"supportedPlatforms",
"knownLimitations",
"relatedAebRepositories",
"feedbackLocations",
"documentation",
"securityPolicy"
],
"properties": {
"$schema": { "const": "./schemas/aeb-capabilities-v1.schema.json" },
"schemaVersion": { "const": "aeb.capabilities/v1" },
"repository": { "const": "aliengineering-byte/resilireplay" },
"currentVersion": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" },
"maturity": { "enum": ["released", "experimental", "research"] },
"problemStatement": { "type": "string", "minLength": 20, "maxLength": 300 },
"capabilities": {
"type": "array",
"minItems": 1,
"uniqueItems": true,
"items": { "$ref": "#/$defs/capability" }
},
"supportedInputs": {
"type": "array",
"minItems": 1,
"items": { "$ref": "#/$defs/dataBoundary" }
},
"generatedOutputs": {
"type": "array",
"minItems": 1,
"items": { "$ref": "#/$defs/dataBoundary" }
},
"interfaces": {
"type": "array",
"minItems": 1,
"uniqueItems": true,
"items": { "type": "string", "minLength": 2, "maxLength": 80 }
},
"installationMethods": {
"type": "array",
"minItems": 1,
"items": { "$ref": "#/$defs/installation" }
},
"canonicalExamples": {
"type": "array",
"minItems": 1,
"items": { "$ref": "#/$defs/example" }
},
"evidenceTypes": {
"type": "array",
"minItems": 1,
"uniqueItems": true,
"items": { "type": "string", "minLength": 3, "maxLength": 120 }
},
"supportedPlatforms": {
"type": "array",
"minItems": 1,
"items": { "$ref": "#/$defs/platform" }
},
"knownLimitations": {
"type": "array",
"minItems": 1,
"uniqueItems": true,
"items": { "type": "string", "minLength": 10, "maxLength": 300 }
},
"relatedAebRepositories": {
"type": "array",
"items": { "$ref": "#/$defs/relation" }
},
"feedbackLocations": {
"type": "array",
"minItems": 1,
"items": { "$ref": "#/$defs/feedback" }
},
"documentation": { "type": "string", "pattern": "^https://[^\\s]+$" },
"securityPolicy": { "type": "string", "pattern": "^https://[^\\s]+$" }
},
"$defs": {
"capability": {
"type": "object",
"additionalProperties": false,
"required": ["id", "summary", "evidenceClass"],
"properties": {
"id": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" },
"summary": { "type": "string", "minLength": 20, "maxLength": 240 },
"evidenceClass": {
"enum": [
"directly-tested",
"measured-benchmark",
"external-primary-source",
"design-intent",
"hypothesis"
]
}
}
},
"dataBoundary": {
"type": "object",
"additionalProperties": false,
"required": ["name", "formats", "boundary"],
"properties": {
"name": { "type": "string", "minLength": 3, "maxLength": 100 },
"formats": {
"type": "array",
"minItems": 1,
"uniqueItems": true,
"items": { "type": "string", "minLength": 2, "maxLength": 40 }
},
"boundary": { "type": "string", "minLength": 10, "maxLength": 300 }
}
},
"installation": {
"type": "object",
"additionalProperties": false,
"required": ["kind", "command"],
"properties": {
"kind": { "type": "string", "minLength": 2, "maxLength": 40 },
"command": { "type": "string", "minLength": 5, "maxLength": 240 }
}
},
"example": {
"type": "object",
"additionalProperties": false,
"required": ["id", "command", "success"],
"properties": {
"id": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" },
"command": { "type": "string", "minLength": 5, "maxLength": 240 },
"success": { "type": "string", "minLength": 10, "maxLength": 300 }
}
},
"platform": {
"type": "object",
"additionalProperties": false,
"required": ["os", "runtimes", "evidence"],
"properties": {
"os": { "type": "string", "minLength": 2, "maxLength": 80 },
"runtimes": {
"type": "array",
"minItems": 1,
"uniqueItems": true,
"items": { "type": "string", "minLength": 2, "maxLength": 80 }
},
"evidence": { "type": "string", "minLength": 3, "maxLength": 180 }
}
},
"relation": {
"type": "object",
"additionalProperties": false,
"required": ["repository", "relationship", "required"],
"properties": {
"repository": { "type": "string", "pattern": "^aliengineering-byte/[a-z0-9-]+$" },
"relationship": { "type": "string", "minLength": 10, "maxLength": 300 },
"required": { "type": "boolean" }
}
},
"feedback": {
"type": "object",
"additionalProperties": false,
"required": ["purpose", "url"],
"properties": {
"purpose": { "type": "string", "minLength": 3, "maxLength": 120 },
"url": { "type": "string", "pattern": "^https://[^\\s]+$" }
}
}
}
}
Loading