From 4360fcdd9a3a109e688b799d53152ea6c48f8759 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 15 Jul 2026 01:54:14 +0000 Subject: [PATCH] =?UTF-8?q?Add=20HyperVault=20integration:=20cloud=20mind?= =?UTF-8?q?=20=E2=87=84=20sovereign=20chain=20+=20archive?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements PLAN_HYPERVAULT_INTEGRATION.md — a first-class bridge between the HyperVault cloud mind (hypervault.store) and AgentVault's ICP/Arweave stack across three tiers: hot (cloud), warm (canister), cold (Arweave). New src/hypervault/ module (exported as agentvault/hypervault): - client.ts: typed REST client over undici with retry/backoff and streamed NDJSON export/import. - auth.ts: key resolution (flag -> env -> vault) and hypervault.json state that stores only a keyRef, never the key. - snapshot.ts: agentvault-hypervault-snapshot-v1 bundle — per-entry SHA-256, Merkle root, ed25519 manifest signature, AES-256-GCM encryption via the audited CanisterEncryption. Private artifacts/conversations always encrypted. - index/: pure-TS weighted FTS index, cosine VectorIndex, RRF hybrid recall with graceful FTS-only fallback. - memory/knowledge/wiki-store adapters over the backbone interfaces. - mind-sync.ts: topological, idempotent mind-DAG replay onto a memory_repo canister with on-chain archive receipts. - pipeline.ts: connect/pull/push/snapshot/archive/verify/restore/status/ recall/bootstrap flows shared by the CLI and MCP server. - mcp-server.ts: native stdio JSON-RPC MCP server (no new dependency) exposing hypervault_* pipeline tools + the existing wiki_* tools. CLI: new `agentvault hypervault` command group, `agentvault mcp serve` subcommand, and `agentvault init --hypervault`. Security (audit C-1): vetkeys.decryptJSON now validates the GCM auth tag before returning plaintext; adds an encryptJSON counterpart and refuses payloads with no usable tag. EncryptedData gains an optional tag field. Tests: tests/hypervault/** (client, snapshot round-trip/tamper/encryption, indices/recall, mind-sync topo+idempotency, auth no-plaintext-key) and tests/mcp-server/** (protocol conformance). Full suite green (1554 tests); typecheck and lint clean. Docs: docs/guides/hypervault.md, docs/architecture/hypervault-integration.md, README HyperVault section, CHANGELOG. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01FbdRFUf2aN6kGSWhkvP5NX --- CHANGELOG.md | 36 + README.md | 21 + cli/commands/hypervault.ts | 544 +++++++++++ cli/commands/init.ts | 18 + cli/commands/mcp.ts | 22 + cli/index.ts | 6 + docs/architecture/hypervault-integration.md | 110 +++ docs/guides/hypervault.md | 109 +++ package.json | 4 + src/backup/thoughtform-bundle.ts | 28 + src/hypervault/auth.ts | 142 +++ src/hypervault/client.ts | 439 +++++++++ src/hypervault/index.ts | 82 ++ src/hypervault/index/builder.ts | 122 +++ src/hypervault/index/fts-index.ts | 128 +++ src/hypervault/index/recall.ts | 69 ++ src/hypervault/index/vector-index.ts | 120 +++ src/hypervault/knowledge-store.ts | 119 +++ src/hypervault/mcp-server.ts | 374 ++++++++ src/hypervault/memory-store.ts | 158 ++++ src/hypervault/mind-sync.ts | 252 +++++ src/hypervault/pipeline.ts | 993 ++++++++++++++++++++ src/hypervault/snapshot.ts | 633 +++++++++++++ src/hypervault/types.ts | 179 ++++ src/hypervault/wiki-store.ts | 208 ++++ src/security/types.ts | 7 + src/security/vetkeys.ts | 95 +- tests/hypervault/auth.test.ts | 60 ++ tests/hypervault/client.test.ts | 85 ++ tests/hypervault/fixtures.ts | 109 +++ tests/hypervault/index.test.ts | 82 ++ tests/hypervault/mind-sync.test.ts | 106 +++ tests/hypervault/snapshot.test.ts | 108 +++ tests/mcp-server/protocol.test.ts | 75 ++ tests/security/vetkeys-decrypt-json.test.ts | 39 + 35 files changed, 5673 insertions(+), 9 deletions(-) create mode 100644 cli/commands/hypervault.ts create mode 100644 docs/architecture/hypervault-integration.md create mode 100644 docs/guides/hypervault.md create mode 100644 src/hypervault/auth.ts create mode 100644 src/hypervault/client.ts create mode 100644 src/hypervault/index.ts create mode 100644 src/hypervault/index/builder.ts create mode 100644 src/hypervault/index/fts-index.ts create mode 100644 src/hypervault/index/recall.ts create mode 100644 src/hypervault/index/vector-index.ts create mode 100644 src/hypervault/knowledge-store.ts create mode 100644 src/hypervault/mcp-server.ts create mode 100644 src/hypervault/memory-store.ts create mode 100644 src/hypervault/mind-sync.ts create mode 100644 src/hypervault/pipeline.ts create mode 100644 src/hypervault/snapshot.ts create mode 100644 src/hypervault/types.ts create mode 100644 src/hypervault/wiki-store.ts create mode 100644 tests/hypervault/auth.test.ts create mode 100644 tests/hypervault/client.test.ts create mode 100644 tests/hypervault/fixtures.ts create mode 100644 tests/hypervault/index.test.ts create mode 100644 tests/hypervault/mind-sync.test.ts create mode 100644 tests/hypervault/snapshot.test.ts create mode 100644 tests/mcp-server/protocol.test.ts create mode 100644 tests/security/vetkeys-decrypt-json.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 38ae632..1c93d43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,42 @@ All notable changes to AgentVault will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [Unreleased] - HyperVault integration + +### Added +- **HyperVault ⇄ AgentVault bridge (`src/hypervault/`, `agentvault hypervault`).** + Makes the HyperVault cloud mind a first-class citizen across three tiers — + hot (cloud), warm (ICP canister), cold (Arweave): + - `HyperVaultClient` — typed REST client over undici with retry/backoff and + streamed NDJSON export (`GET /api/export`), plus import and archive-receipt + endpoints. + - `agentvault-hypervault-snapshot-v1` bundle format — reuses the + thoughtform-bundle envelope with per-entry SHA-256, a Merkle root, and an + ed25519 manifest signature. Encryption via the audited `CanisterEncryption` + (AES-256-GCM); private artifacts and conversations are always encrypted. + - Local indices — pure-TS weighted FTS index and a cosine `VectorIndex` over + exported embeddings, with reciprocal-rank hybrid recall and graceful + FTS-only fallback. + - Backbone/wiki adapters — `HyperVaultMemoryStore`, `HyperVaultKnowledgeStore`, + `HyperVaultWikiStore` run the existing interfaces against the cloud mind. + - On-chain mind mirror — topological, idempotent DAG replay onto a + `memory_repo` canister, with on-chain archive receipts. + - CLI: `hypervault connect · status · bootstrap · pull · push · snapshot · + archive · verify · restore · reindex · recall`; `init --hypervault`. + - Native MCP server — `agentvault mcp serve` (stdio JSON-RPC, no new + dependency) exposing the `hypervault_*` pipeline tools and the existing + `wiki_*` tools. + - New package export subpath `agentvault/hypervault`. + +### Security +- **C-1 (CRITICAL):** `vetkeys.decryptJSON` now validates the AES-256-GCM / + ChaCha20-Poly1305 authentication tag before returning plaintext (previously + `setAuthTag` was never called, so tampered ciphertext decrypted silently). + `EncryptedData` gains an optional `tag` field, a matching `encryptJSON` helper + is added, and payloads with no usable tag are refused rather than decrypted + unauthenticated. Legacy combined-layout payloads (tag appended to ciphertext) + are still supported. + ## [1.0.4] - 2026-05-24 - Security & build hygiene refresh ### Security diff --git a/README.md b/README.md index 320e7f7..18c7ff3 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,27 @@ agentvault show --canister-id agentvault fetch --canister-id ``` +## HyperVault + +[HyperVault](https://github.com/johnnyclem/hypervault) is the living cloud mind +(`hypervault.store`); AgentVault is the sovereign body. The `hypervault` command +group bridges them across three tiers — hot (cloud), warm (ICP canister), cold +(Arweave) — so a cloud account becomes reconstructible from chain alone. + +```bash +# Bootstrap a HyperVault-backed agent — memories, mind history, and indices included +npx agentvault@latest hypervault bootstrap my-agent --key hv_... + +# hypervault.store → fully secure, archived, blockchain-backed AgentVault +npx agentvault@latest hypervault archive --all --encrypt --network ic --arweave + +# Resurrect anywhere from a single Arweave transaction +npx agentvault@latest hypervault restore ar:// --to local +``` + +Both one-liners are also MCP tools via the native `agentvault mcp serve` server. +See [`docs/guides/hypervault.md`](docs/guides/hypervault.md) for the full guide. + ## CLI Commands ### Core Commands diff --git a/cli/commands/hypervault.ts b/cli/commands/hypervault.ts new file mode 100644 index 0000000..ed3ce8f --- /dev/null +++ b/cli/commands/hypervault.ts @@ -0,0 +1,544 @@ +/** + * `agentvault hypervault` — the HyperVault ⇄ AgentVault bridge + * + * Three tiers of one mind from a single command group: + * connect · status · bootstrap · pull · push · snapshot · archive · + * verify · restore · reindex · recall + * + * Key handling follows the AGENTS.md wallet-secret policy: `hv_` keys flow + * only through env / secrets vault / prompt. `--key` is accepted for the + * one-liner UX but is immediately vaulted and a warning is printed. + */ + +import { Command } from 'commander'; +import chalk from 'chalk'; +import ora from 'ora'; +import inquirer from 'inquirer'; +import { + archiveHyperVault, + bootstrapHyperVault, + clientFromProject, + connectHyperVault, + pullHyperVault, + pushHyperVault, + recallLocal, + reindexHyperVault, + restoreHyperVault, + snapshotHyperVault, + statusHyperVault, + verifySnapshotFile, + projectAgentId, +} from '../../src/hypervault/pipeline.js'; +import { createMemoryRepoActor } from '../../src/canister/memory-repo-actor.js'; + +const hypervaultCmd = new Command('hypervault'); + +hypervaultCmd + .alias('hv') + .description('Bridge HyperVault (cloud mind) with AgentVault (sovereign chain + archive)') + .action(() => { + console.log(chalk.yellow('Specify a subcommand:')); + console.log( + chalk.gray(` + ${chalk.cyan('agentvault hypervault connect')} Validate & vault your API key + ${chalk.cyan('agentvault hypervault status')} The three-tier picture + ${chalk.cyan('agentvault hypervault bootstrap my-agent')} Cloud account → running agent + ${chalk.cyan('agentvault hypervault pull')} Incremental sync down + ${chalk.cyan('agentvault hypervault push --dry-run')} Push local edits as mind commits + ${chalk.cyan('agentvault hypervault snapshot -o bundle')} Full export → bundle on disk + ${chalk.cyan('agentvault hypervault archive --all --encrypt --network ic --arweave')} + ${chalk.cyan('agentvault hypervault verify ')} Verify a bundle / tx / commit + ${chalk.cyan('agentvault hypervault restore ar:// --to local')} + ${chalk.cyan('agentvault hypervault recall ""')} Offline hybrid recall +`), + ); + }); + +// --------------------------------------------------------------------------- +// connect +// --------------------------------------------------------------------------- + +hypervaultCmd + .command('connect') + .description('Validate the HyperVault API key, store it in the secrets vault, write hypervault.json') + .option('--key ', 'API key (discouraged — prefer HYPERVAULT_API_KEY or the vault)') + .option('--api-url ', 'HyperVault API base URL') + .action(async (options: { key?: string; apiUrl?: string }) => { + let key = options.key; + if (!key && !process.env.HYPERVAULT_API_KEY) { + const agentId = projectAgentId(process.cwd()); + // Only prompt when the vault can't supply one; connectHyperVault will + // still try the vault first if we pass no key. + const probe = await connectHyperVault({ apiUrl: options.apiUrl, agentId }).catch(() => null); + if (!probe) { + const answer = await inquirer.prompt<{ key: string }>([ + { + type: 'password', + name: 'key', + message: 'Enter your HyperVault API key (hv_...):', + validate: (v: string) => (v.trim().length > 0 ? true : 'Key is required'), + }, + ]); + key = answer.key.trim(); + } + } + + const spinner = ora('Validating HyperVault key...').start(); + try { + const result = await connectHyperVault({ key, apiUrl: options.apiUrl }); + if (!result.valid) { + spinner.fail(chalk.red('HyperVault key was rejected. Create one at your hypervault.store dashboard.')); + process.exit(1); + } + spinner.succeed(chalk.green('HyperVault key validated and connected')); + if (result.vaulted) { + console.log(chalk.gray(` Key stored in secrets vault${result.keyRef ? ` (${result.keyRef})` : ''}`)); + } + if (result.userIdHint) console.log(chalk.gray(` Account: ${result.userIdHint}`)); + if (result.warning) console.log(chalk.yellow(` ⚠ ${result.warning}`)); + } catch (error) { + spinner.fail(chalk.red(errMsg(error))); + process.exit(1); + } + }); + +// --------------------------------------------------------------------------- +// status +// --------------------------------------------------------------------------- + +hypervaultCmd + .command('status') + .description('Show the whole three-tier picture: cloud, local, canister, Arweave') + .option('--api-url ', 'HyperVault API base URL') + .option('--no-remote', 'Skip the cloud check (local/canister only)') + .action(async (options: { apiUrl?: string; remote?: boolean }) => { + const spinner = ora('Gathering HyperVault status...').start(); + try { + const client = options.remote === false ? undefined : await clientFromProject({ apiUrl: options.apiUrl }).catch(() => undefined); + const status = await statusHyperVault({ client }); + spinner.stop(); + + console.log(chalk.bold('\nHyperVault status\n')); + if (!status.configured) { + console.log(chalk.yellow(' Not connected. Run `agentvault hypervault connect`.')); + return; + } + console.log(chalk.cyan(' Hot (cloud):')); + if (status.cloud) { + console.log(` ${status.keyValid ? chalk.green('✓') : chalk.red('✗')} ${status.apiUrl}`); + console.log(chalk.gray(` memories ${status.cloud.memories} · artifacts ${status.cloud.artifacts} · branches ${status.cloud.branches}`)); + } else { + console.log(chalk.gray(` ${status.apiUrl ?? 'unknown'} (${status.keyValid === false ? 'key invalid' : 'not checked'})`)); + } + + console.log(chalk.cyan(' Warm (local + canister):')); + console.log( + chalk.gray( + ` snapshot ${status.local.snapshotPresent ? '✓' : '—'} · working-tree ${status.local.memoriesInWorkingTree} · FTS ${status.local.ftsIndexed} · vectors ${status.local.vectorsIndexed}`, + ), + ); + if (status.canister) { + console.log(chalk.gray(` canister ${status.canister.id}${status.canister.currentBranch ? ` @${status.canister.currentBranch}` : ''}${status.canister.totalCommits ? ` (${status.canister.totalCommits} commits)` : ''}`)); + } + if (status.local.lastSync) console.log(chalk.gray(` last sync ${status.local.lastSync}`)); + + console.log(chalk.cyan(' Cold (Arweave):')); + console.log(chalk.gray(` ${status.arweave?.lastTx ? `✓ ${status.arweave.lastTx}` : '— never archived'}`)); + console.log(); + } catch (error) { + spinner.fail(chalk.red(errMsg(error))); + process.exit(1); + } + }); + +// --------------------------------------------------------------------------- +// bootstrap +// --------------------------------------------------------------------------- + +hypervaultCmd + .command('bootstrap') + .description('One-liner: HyperVault account → running agent (scaffold + connect + pull + indices + MCP)') + .argument('', 'Project directory to create') + .option('--key ', 'API key (discouraged — prefer HYPERVAULT_API_KEY or the vault)') + .option('--api-url ', 'HyperVault API base URL') + .option('--branch ', 'Mind branch to pull') + .option('--no-artifacts', 'Skip artifact content') + .option('--no-index', 'Skip building local indices') + .option('--soul ', 'Memory slug to use as the agent soul') + .action(async (project: string, options: { key?: string; apiUrl?: string; branch?: string; artifacts?: boolean; index?: boolean; soul?: string }) => { + console.log(chalk.bold('\n🧠 HyperVault bootstrap\n')); + const spinner = ora('Starting...').start(); + try { + const result = await bootstrapHyperVault({ + project, + key: options.key, + apiUrl: options.apiUrl, + branch: options.branch, + includeArtifacts: options.artifacts, + buildIndex: options.index, + soulSlug: options.soul, + onStep: (step, detail) => { + spinner.text = `${step}${detail ? `: ${detail}` : ''}`; + }, + }); + if (!result.connected) { + spinner.fail(chalk.red(result.warning ?? 'Bootstrap failed to connect')); + process.exit(1); + } + spinner.succeed(chalk.green(`Agent bootstrapped at ${result.projectPath}`)); + if (result.pull) { + console.log(chalk.gray(` Pulled ${result.pull.totalRecords} records · ${result.pull.memoriesWritten} memories · indices ${result.pull.indicesBuilt ? 'built' : 'skipped'}`)); + } + console.log(chalk.gray(` MCP config: ${result.mcpConfigPath}`)); + if (result.soulDetected) console.log(chalk.gray(' Soul detected → soul.md written')); + if (result.warning) console.log(chalk.yellow(` ⚠ ${result.warning}`)); + console.log(chalk.cyan('\n Next: ') + chalk.bold(`cd ${project} && agentvault hypervault status`)); + } catch (error) { + spinner.fail(chalk.red(errMsg(error))); + process.exit(1); + } + }); + +// --------------------------------------------------------------------------- +// pull +// --------------------------------------------------------------------------- + +hypervaultCmd + .command('pull') + .description('Incremental export → local snapshot, working tree, and indices') + .option('--api-url ', 'HyperVault API base URL') + .option('--branch ', 'Mind branch to pull') + .option('--since ', 'Export cursor (default: derived from last sync)') + .option('--no-artifacts', 'Skip artifact content') + .option('--no-index', 'Skip building local indices') + .action(async (options: { apiUrl?: string; branch?: string; since?: string; artifacts?: boolean; index?: boolean }) => { + const spinner = ora('Pulling from HyperVault...').start(); + try { + const client = await clientFromProject({ apiUrl: options.apiUrl }); + const result = await pullHyperVault({ + client, + branch: options.branch, + since: options.since, + includeArtifacts: options.artifacts, + buildIndex: options.index, + }); + spinner.succeed(chalk.green(`Pulled ${result.recordsPulled} records (${result.totalRecords} total)`)); + console.log(chalk.gray(` ${result.memoriesWritten} memories written · indices ${result.indicesBuilt ? 'rebuilt' : 'skipped'}`)); + console.log(chalk.gray(` Snapshot: ${result.snapshotFile}`)); + } catch (error) { + spinner.fail(chalk.red(errMsg(error))); + process.exit(1); + } + }); + +// --------------------------------------------------------------------------- +// push +// --------------------------------------------------------------------------- + +hypervaultCmd + .command('push') + .description('Push locally created/edited memories up as provenance-stamped mind commits') + .option('--api-url ', 'HyperVault API base URL') + .option('--dry-run', 'Print the diff-as-mind-commits without writing') + .action(async (options: { apiUrl?: string; dryRun?: boolean }) => { + const spinner = ora('Diffing local working tree...').start(); + try { + const client = await clientFromProject({ apiUrl: options.apiUrl }); + const result = await pushHyperVault({ client, dryRun: options.dryRun }); + spinner.stop(); + if (result.changes.length === 0) { + console.log(chalk.gray('Nothing to push — local working tree matches the last snapshot.')); + return; + } + for (const change of result.changes) { + const marker = change.kind === 'create' ? chalk.green('+ create') : chalk.yellow('~ update'); + console.log(` ${marker} ${change.title}`); + } + if (result.dryRun) { + console.log(chalk.gray(`\n${result.changes.length} change(s) would be pushed (dry run).`)); + } else { + console.log(chalk.green(`\n✓ Pushed ${result.pushed} change(s) as mind commits.`)); + } + } catch (error) { + spinner.fail(chalk.red(errMsg(error))); + process.exit(1); + } + }); + +// --------------------------------------------------------------------------- +// snapshot +// --------------------------------------------------------------------------- + +hypervaultCmd + .command('snapshot') + .description('Full export → agentvault-hypervault-snapshot-v1 bundle on disk') + .option('--api-url ', 'HyperVault API base URL') + .option('-o, --output ', 'Output bundle path') + .option('--encrypt', 'Encrypt entries (prompts for a passphrase unless HYPERVAULT_SNAPSHOT_PASSPHRASE is set)') + .option('--include-conversations', 'Include conversations/messages (always encrypted)') + .action(async (options: { apiUrl?: string; output?: string; encrypt?: boolean; includeConversations?: boolean }) => { + try { + const passphrase = await maybePassphrase(options.encrypt || options.includeConversations); + const spinner = ora('Exporting and bundling...').start(); + const client = await clientFromProject({ apiUrl: options.apiUrl }); + const result = await snapshotHyperVault({ + client, + outputPath: options.output, + passphrase, + includeConversations: options.includeConversations, + }); + spinner.succeed(chalk.green(`Snapshot written: ${result.path}`)); + console.log(chalk.gray(` ${formatBytes(result.sizeBytes)} · ${summarizeCounts(result.rowCounts)}`)); + } catch (error) { + console.error(chalk.red(errMsg(error))); + process.exit(1); + } + }); + +// --------------------------------------------------------------------------- +// archive +// --------------------------------------------------------------------------- + +hypervaultCmd + .command('archive') + .description('Sovereign archive: snapshot → encrypt → canister replay → Arweave → receipts → verify') + .option('--api-url ', 'HyperVault API base URL') + .option('--all', 'Archive the full account (default; use --since for incremental)') + .option('--encrypt', 'Encrypt the bundle (AES-256-GCM, passphrase-wrapped)') + .option('--network ', 'ICP network: local | ic') + .option('--canister-id ', 'memory_repo canister id (skip the warm tier if omitted)') + .option('--arweave', 'Upload the cold copy to Arweave') + .option('--arweave-jwk ', 'Path to the Arweave wallet JWK') + .option('--since ', 'Incremental archive from this cursor') + .option('-y, --yes', 'Skip confirmation prompts') + .action(async (options: { + apiUrl?: string; + all?: boolean; + encrypt?: boolean; + network?: string; + canisterId?: string; + arweave?: boolean; + arweaveJwk?: string; + since?: string; + yes?: boolean; + }) => { + console.log(chalk.bold('\n🗄 HyperVault sovereign archive\n')); + try { + if (!options.encrypt && !options.yes) { + const { proceed } = await inquirer.prompt<{ proceed: boolean }>([ + { type: 'confirm', name: 'proceed', message: chalk.red('Archiving WITHOUT --encrypt stores memories in plaintext. Continue?'), default: false }, + ]); + if (!proceed) { + console.log(chalk.yellow('Aborted. Re-run with --encrypt.')); + return; + } + } + + const passphrase = await maybePassphrase(options.encrypt); + const client = await clientFromProject({ apiUrl: options.apiUrl }); + + // Warm tier: canister actor + let actor; + if (options.canisterId) { + const { createAnonymousAgent } = await import('../../src/canister/memory-repo-actor.js'); + const host = options.network === 'ic' ? 'https://ic0.app' : undefined; + actor = createMemoryRepoActor(options.canisterId, createAnonymousAgent(host)); + } + + // Cold tier: Arweave JWK + let arweaveJwk: Record | undefined; + if (options.arweave || options.arweaveJwk) { + arweaveJwk = await loadArweaveJwk(options.arweaveJwk); + } + + const spinner = ora('Archiving...').start(); + const result = await archiveHyperVault({ + client, + passphrase, + actor, + canisterId: options.canisterId, + arweaveJwk, + since: options.since, + onStep: (step, detail) => { + spinner.text = `${step}${detail ? `: ${detail}` : ''}`; + }, + }); + spinner.stop(); + + console.log(chalk.green('✔ Archive pipeline complete')); + console.log(chalk.gray(` Bundle: ${result.snapshotFile}`)); + console.log(chalk.gray(` Rows: ${summarizeCounts(result.rowCounts)}`)); + if (result.mindSync) { + console.log(chalk.gray(` Canister: ${result.mindSync.commitsReplayed} replayed, ${result.mindSync.commitsSkipped} already synced, ${result.mindSync.thoughtformsStored} thoughtforms`)); + } + if (result.arweaveTx) { + console.log(chalk.gray(` Arweave: ${result.arweaveTx}`)); + console.log(chalk.gray(` Receipts: ${result.receiptOnChain ? 'on-chain ✓' : '—'}${result.receiptPosted ? ' cloud ✓' : ''}`)); + console.log(chalk.gray(` Verified: ${result.verified ? '✓' : '✗'}`)); + console.log(chalk.bold('\n✔ Archived. Resurrect anywhere with:')); + console.log(chalk.cyan(` npx agentvault@latest hypervault restore ar://${result.arweaveTx} --to local`)); + } + if (result.errors.length > 0) { + console.log(chalk.yellow(`\n ${result.errors.length} warning(s):`)); + for (const err of result.errors) console.log(chalk.yellow(` - ${err}`)); + } + } catch (error) { + console.error(chalk.red(errMsg(error))); + process.exit(1); + } + }); + +// --------------------------------------------------------------------------- +// verify +// --------------------------------------------------------------------------- + +hypervaultCmd + .command('verify') + .description('Verify a snapshot bundle: manifest hash, ed25519 signature, Merkle root, per-entry checksums') + .argument('', 'Snapshot bundle file path') + .option('--passphrase ', 'Passphrase for encrypted bundles (prefer the env var)') + .action(async (ref: string, options: { passphrase?: string }) => { + const spinner = ora('Verifying...').start(); + try { + const passphrase = options.passphrase ?? process.env.HYPERVAULT_SNAPSHOT_PASSPHRASE; + const result = await verifySnapshotFile(ref, { passphrase }); + if (result.valid) { + spinner.succeed(chalk.green('Bundle verified — signature, checksums, and Merkle root all pass')); + } else { + spinner.fail(chalk.red('Verification FAILED')); + console.log(chalk.gray(` signature ${result.signatureValid ? '✓' : '✗'} · checksums ${result.checksumsValid ? '✓' : '✗'} · merkle ${result.merkleRootValid ? '✓' : '✗'}`)); + for (const err of result.errors) console.log(chalk.yellow(` - ${err}`)); + process.exit(1); + } + } catch (error) { + spinner.fail(chalk.red(errMsg(error))); + process.exit(1); + } + }); + +// --------------------------------------------------------------------------- +// restore +// --------------------------------------------------------------------------- + +hypervaultCmd + .command('restore') + .description('Chain → anywhere: restore a snapshot (ar:// or file) to a local project or fresh hypervault account') + .argument('', 'ar:// or a snapshot file path') + .option('--to ', 'Restore target: local | hypervault', 'local') + .option('--key ', 'Destination HyperVault key (for --to hypervault)') + .option('--api-url ', 'HyperVault API base URL') + .option('--passphrase ', 'Passphrase for encrypted bundles (prefer the env var)') + .action(async (ref: string, options: { to?: string; key?: string; apiUrl?: string; passphrase?: string }) => { + const to = options.to === 'hypervault' ? 'hypervault' : 'local'; + const spinner = ora(`Restoring to ${to}...`).start(); + try { + const passphrase = options.passphrase ?? process.env.HYPERVAULT_SNAPSHOT_PASSPHRASE; + const client = to === 'hypervault' ? await clientFromProject({ key: options.key, apiUrl: options.apiUrl }) : undefined; + const result = await restoreHyperVault({ ref, to, passphrase, client }); + spinner.succeed(chalk.green(`Restored ${result.records} records (verify ${result.verify.valid ? '✓' : 'partial'})`)); + if (result.memoriesWritten !== undefined) console.log(chalk.gray(` ${result.memoriesWritten} memories written to .agentvault/memories/`)); + if (result.importedToHypervault !== undefined) console.log(chalk.gray(` ${result.importedToHypervault} records imported to hypervault`)); + } catch (error) { + spinner.fail(chalk.red(errMsg(error))); + process.exit(1); + } + }); + +// --------------------------------------------------------------------------- +// reindex +// --------------------------------------------------------------------------- + +hypervaultCmd + .command('reindex') + .description('Rebuild local vector/FTS indices from the pulled snapshot') + .option('--passphrase ', 'Passphrase for encrypted snapshots (prefer the env var)') + .action(async (options: { passphrase?: string }) => { + const spinner = ora('Rebuilding indices...').start(); + try { + const passphrase = options.passphrase ?? process.env.HYPERVAULT_SNAPSHOT_PASSPHRASE; + const result = await reindexHyperVault({ passphrase }); + spinner.succeed(chalk.green(`Reindexed ${result.memoriesIndexed} memories (${result.vectorsIndexed} vectors)`)); + } catch (error) { + spinner.fail(chalk.red(errMsg(error))); + process.exit(1); + } + }); + +// --------------------------------------------------------------------------- +// recall +// --------------------------------------------------------------------------- + +hypervaultCmd + .command('recall') + .description('Offline hybrid (lexical + semantic) recall over the local indices') + .argument('', 'Search query') + .option('-n, --limit ', 'Number of results', (v) => parseInt(v, 10), 10) + .action(async (query: string, options: { limit?: number }) => { + try { + const results = await recallLocal(query, { limit: options.limit }); + if (results.length === 0) { + console.log(chalk.gray('No matches.')); + return; + } + console.log(chalk.bold(`\n${results.length} result(s) for "${query}":\n`)); + for (const r of results) { + console.log(` ${chalk.cyan(r.score.toFixed(3))} ${chalk.bold(r.memory.title || r.memory.id)} ${chalk.gray(`[${r.matchedBy.join('+')}]`)}`); + const preview = r.memory.content.replace(/\s+/g, ' ').slice(0, 120); + console.log(chalk.gray(` ${preview}${r.memory.content.length > 120 ? '…' : ''}`)); + } + console.log(); + } catch (error) { + console.error(chalk.red(errMsg(error))); + process.exit(1); + } + }); + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +function errMsg(error: unknown): string { + return error instanceof Error ? error.message : 'Unknown error'; +} + +async function maybePassphrase(encrypt?: boolean): Promise { + if (!encrypt) return undefined; + const fromEnv = process.env.HYPERVAULT_SNAPSHOT_PASSPHRASE; + if (fromEnv) return fromEnv; + const { passphrase } = await inquirer.prompt<{ passphrase: string }>([ + { + type: 'password', + name: 'passphrase', + message: 'Passphrase to encrypt the bundle:', + validate: (v: string) => (v.length >= 8 ? true : 'Use at least 8 characters'), + }, + ]); + return passphrase; +} + +async function loadArweaveJwk(jwkPath?: string): Promise> { + const fs = await import('node:fs'); + const resolved = jwkPath ?? process.env.ARWEAVE_JWK_PATH; + if (!resolved) { + throw new Error('Arweave upload requires --arweave-jwk or the ARWEAVE_JWK_PATH env var'); + } + if (!fs.existsSync(resolved)) { + throw new Error(`Arweave JWK not found: ${resolved}`); + } + return JSON.parse(fs.readFileSync(resolved, 'utf-8')) as Record; +} + +function summarizeCounts(counts: Record): string { + return Object.entries(counts) + .filter(([, n]) => n > 0) + .map(([table, n]) => `${table} ${n}`) + .join(' · '); +} + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +export { hypervaultCmd }; diff --git a/cli/commands/init.ts b/cli/commands/init.ts index 87f1c0d..ad4cd0e 100644 --- a/cli/commands/init.ts +++ b/cli/commands/init.ts @@ -27,6 +27,7 @@ export interface InitOptions { yes?: boolean; verbose?: boolean; v?: boolean; + hypervault?: boolean; } export interface InitAnswers { @@ -312,6 +313,7 @@ export function initCommand(): Command { .option('-y, --yes', 'skip prompts and use defaults') .option('-v, --verbose', 'display detailed configuration information') .option('--vv', 'extra verbose mode for debugging') + .option('--hypervault', 'after scaffolding, connect to a HyperVault account') .action(async (project: string, options: InitOptions) => { console.log(chalk.bold('\n🔐 AgentVault Project Initialization\n')); @@ -342,6 +344,22 @@ export function initCommand(): Command { } await executeInit(answers, options, project); + + if (options.hypervault) { + console.log(); + console.log(chalk.cyan('Connecting to HyperVault...')); + try { + const { connectHyperVault } = await import('../../src/hypervault/pipeline.js'); + const result = await connectHyperVault({ agentId: answers.name, projectRoot: path.resolve(project) }); + if (result.valid) { + console.log(chalk.green(' ✓ HyperVault connected.'), chalk.gray('Run `agentvault hypervault pull` to sync your mind.')); + } else { + console.log(chalk.yellow(' HyperVault key not found or rejected. Run `agentvault hypervault connect` to finish.')); + } + } catch { + console.log(chalk.yellow(' Run `agentvault hypervault connect` to link a HyperVault account.')); + } + } }); return command; diff --git a/cli/commands/mcp.ts b/cli/commands/mcp.ts index 0aa62ee..6165896 100644 --- a/cli/commands/mcp.ts +++ b/cli/commands/mcp.ts @@ -256,4 +256,26 @@ mcpCmd } }); +mcpCmd + .command('serve') + .description('Run the native AgentVault MCP server (hypervault + wiki tools) over stdio') + .option('--transport ', 'Transport: stdio (default)', 'stdio') + .option('--project ', 'Project root to operate in', process.cwd()) + .option('--api-url ', 'HyperVault API base URL') + .action(async (options: { transport?: string; project?: string; apiUrl?: string }) => { + // On stdio, stdout is the JSON-RPC channel — keep it clean (no chalk banners). + if ((options.transport ?? 'stdio') !== 'stdio') { + console.error(chalk.red(`Unsupported transport: ${options.transport}. Only "stdio" is available.`)); + process.exit(1); + } + const { serveMcp } = await import('../../src/hypervault/mcp-server.js'); + try { + await serveMcp({ projectRoot: options.project ?? process.cwd(), apiUrl: options.apiUrl }); + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error'; + console.error(chalk.red(`MCP server error: ${message}`)); + process.exit(1); + } + }); + export { mcpCmd }; diff --git a/cli/index.ts b/cli/index.ts index cf953d2..b01cdd7 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -84,6 +84,9 @@ import { wikiCmd } from './commands/wiki.js'; // Merge command — local bundle <-> on-chain thoughtform merging import { mergeCommand } from './merge.js'; +// HyperVault — cloud mind ⇄ sovereign chain + archive bridge +import { hypervaultCmd } from './commands/hypervault.js'; + export function createProgram(): Command { const program = new Command(); @@ -170,6 +173,9 @@ export function createProgram(): Command { // Merge command — local bundle <-> on-chain thoughtform merging program.addCommand(mergeCommand()); + // HyperVault — cloud mind ⇄ sovereign chain + archive bridge + program.addCommand(hypervaultCmd); + return program; } diff --git a/docs/architecture/hypervault-integration.md b/docs/architecture/hypervault-integration.md new file mode 100644 index 0000000..e640919 --- /dev/null +++ b/docs/architecture/hypervault-integration.md @@ -0,0 +1,110 @@ +# HyperVault Integration Architecture + +This document describes how the `src/hypervault/` module bridges the HyperVault cloud mind with the AgentVault sovereign stack, and specifies the snapshot bundle format (normative). + +## Three tiers of one mind + +``` + ┌────────────────────────────────────────────┐ + │ hypervault.store │ + │ Supabase Postgres: memories · mind DAG · │ + │ artifacts · connections · pgvector index │ + │ REST API (X-HyperVault-Key) + GET /api/export + └───────▲──────────────────────▲──────────────┘ + │ │ + hypervault-mcp (19 tools) │ HTTPS (typed client) + │ │ +┌───────────────┴──────────────────────┴───────────────────┐ +│ AgentVault CLI / SDK │ +│ src/hypervault/ │ +│ ├── client.ts typed REST over undici │ +│ ├── auth.ts key resolution (env/vault/prompt) │ +│ ├── snapshot.ts export → snapshot-v1 bundle │ +│ ├── index/ local FTS + vector indices │ +│ ├── memory/knowledge/wiki-store.ts backbone adapters │ +│ ├── mind-sync.ts mind DAG → memory_repo canister │ +│ ├── pipeline.ts connect/pull/push/archive/restore │ +│ └── mcp-server.ts native MCP server (agentvault mcp serve) +└──────────┬───────────────────────────────────┬────────────┘ + │ dfx / @dfinity/agent │ arweave upload + ▼ ▼ +┌───────────────────────────┐ ┌──────────────────────────┐ +│ ICP canister (warm) │◀────▶│ Arweave (cold) │ +│ memory_repo: mind DAG │recpt │ signed snapshot bundle │ +│ mirror + ThoughtForms │ txid │ ed25519 manifest, State-│ +│ + archive receipts │ │ Hash tags │ +└───────────────────────────┘ └──────────────────────────┘ +``` + +- **Hot (HyperVault):** the live, queryable mind. Agents write through MCP. +- **Warm (ICP canister):** the sovereign mirror. `memory_repo` holds the mind DAG; survives a HyperVault outage. +- **Cold (Arweave):** the permanent archive. A whole account is reconstructible from one tx id plus keys. + +The chain mirror is **derived**, never authored directly (v1): HyperVault stays the write master. + +## Module map + +| File | Responsibility | +|---|---| +| `types.ts` | Zod-validated mirrors of the hypervault schema + export wire format + local state. | +| `client.ts` | `HyperVaultClient` — typed REST over undici, retry/backoff, NDJSON export streaming. | +| `auth.ts` | Key resolution chain (flag → env → vault) and the `hypervault.json` state file (keyRef only). | +| `snapshot.ts` | Build/verify the snapshot bundle; encryption via `CanisterEncryption`; restore to records. | +| `index/` | `FtsIndex` (pure-TS weighted inverted index), `VectorIndex` (cosine), `builder.ts`, `recall.ts` (RRF fusion). | +| `memory-store.ts`, `knowledge-store.ts`, `wiki-store.ts` | Backbone/wiki interface adapters over the cloud mind. | +| `mind-sync.ts` | Topological, idempotent DAG replay onto `memory_repo`; archive receipts. | +| `pipeline.ts` | The flows behind the CLI/MCP: connect, pull, push, snapshot, archive, verify, restore, status, recall, bootstrap. | +| `mcp-server.ts` | Native stdio JSON-RPC MCP server exposing `hypervault_*` + `wiki_*` tools. | + +## Snapshot bundle format — `agentvault-hypervault-snapshot-v1` (normative) + +A snapshot reuses the proven thoughtform-bundle envelope: `gzip(JSON({ format, createdAt, manifest, signature, publicKey, entries }))`, where `entries` maps a logical path to base64 content. + +### Entries + +``` +manifest (in the envelope, not an entry) — HvExportManifest-derived metadata +memories.ndjson — live memories (embeddings stripped into the sidecar) +mind/commits.ndjson — memory_commits (full DAG incl. merge parents, author provenance) +mind/revisions.ndjson — memory_revisions (full snapshots per change) +mind/branches.json — memory_branches +mind/links.ndjson — memory_links + memory_link_changes +artifacts/.html — artifact content, content-addressed by content_hash +artifacts/index.ndjson — artifact metadata (slug, title, tags, visibility, entry pointer) +connections.ndjson — artifact graph edges + memory_artifact_links +embeddings.bin — packed float32 vectors (row-major) +embeddings.idx.json — { dims, model, ids[] } sidecar for embeddings.bin +conversations.ndjson — optional (--include-conversations), always encrypted +``` + +### Manifest fields + +- `version`, `format`, `createdAt`, `schemaVersion`, `cursor`, `branchHeads`, `rowCounts`. +- `checksums` — SHA-256 (hex) of each entry's **plaintext** bytes. +- `merkleRoot` — Merkle root over all plaintext entries (`src/backup/merkle.ts` construction: sorted leaves, odd nodes promoted). +- `encryptedEntries` — entry paths stored encrypted. +- `encryption` — `{ mode, algorithm, salt, iterations }` when any entry is encrypted. +- `derived` — rebuild-only artifacts (never part of the integrity surface). + +### Integrity chain + +content SHA-256 per entry → Merkle root in manifest → ed25519 signature over the canonical (alphabetically-keyed) manifest, using `~/.agentvault/arweave-signing.key`. This mirrors `ArweaveArchiver.verifyBundle` semantics, so verification is uniform across the archive stack. + +### Encryption + +Entries are encrypted with `CanisterEncryption` (AES-256-GCM, validated auth tags — deliberately **not** the legacy `vetkeys.decryptJSON` path, whose C-1 auth-tag gap is fixed separately). The data key is derived from a passphrase via PBKDF2 (210k iterations, per-bundle random salt). Private artifacts and conversations are encrypted even when a snapshot-wide passphrase is not otherwise requested. + +### Indices are derived, not archived + +Local FTS/vector indices are always rebuildable from the snapshot, so they are listed as `derived` and are never part of the archived bundle's integrity surface. Cross-model embedding mixing is refused: the recorded `embedding_model` guards query-time compatibility. + +## On-chain mirror mapping + +| HyperVault | `memory_repo` canister | +|---|---| +| `memory_branches.name` | `createBranch` / `switchBranch` | +| `memory_commits` | `commit(message, diff, tags)` with the hypervault UUID, author provenance, and merge-parent recorded in `tags` (`hv:commit:`, `hv:parent:…`, `hv:author:…`) | +| `memory_revisions` per commit | the commit `diff` payload (JSON of revision ops) | +| memory content at head | `storeThoughtForm` entries, content-addressed by hash | + +Replay is topological (parents before children) and idempotent (commits whose UUID tag is already on-chain are skipped). Payloads over the on-chain chunk limit become content-hash pointers; large artifact blobs live on Arweave, and the canister stores `{ content_hash, arweave_tx }` pointers. diff --git a/docs/guides/hypervault.md b/docs/guides/hypervault.md new file mode 100644 index 0000000..3cdf48a --- /dev/null +++ b/docs/guides/hypervault.md @@ -0,0 +1,109 @@ +# HyperVault ⇄ AgentVault + +[HyperVault](https://github.com/johnnyclem/hypervault) is the **living, cloud-hosted mind**: per-user memories with auto-tagging and a knowledge graph, git-versioned mind history ("Git for a Mind"), pgvector semantic search, and saved artifacts, all behind `hypervault.store`. + +AgentVault is the **sovereign, permanent body**: ICP canisters with stable on-chain state, a git-style on-chain memory repo (ThoughtForms), ed25519-signed Arweave archival, and threshold encryption. + +The `agentvault hypervault` command group bridges them across three tiers of the same mind: + +- **Hot (HyperVault / Supabase):** the live, queryable, multi-device mind. +- **Warm (ICP canister):** the sovereign mirror — the mind DAG replayed on-chain, survives a HyperVault outage. +- **Cold (Arweave):** the permanent archive — signed, content-addressed, verifiable bundles. A whole account is reconstructible from a single transaction id plus keys. + +Every layer can rebuild the layer above it, and every write down the stack carries an integrity receipt back up. + +## The two one-liners + +```bash +# 1) Bootstrap a HyperVault-backed agent — memories, mind history, and indices included +npx agentvault@latest hypervault bootstrap my-agent --key hv_... + +# 2) hypervault.store → fully secure, archived, blockchain-backed AgentVault +npx agentvault@latest hypervault archive --all --encrypt --network ic --arweave +``` + +Both are also exposed as MCP tools (`hypervault_bootstrap`, `hypervault_archive`) via the native `agentvault mcp serve` server, so any MCP-capable agent can invoke the full pipeline itself. + +## Getting your key + +Create an API key (`hv_...`) in your hypervault.store dashboard. + +Keys flow **only** through the environment, the secrets vault, or an interactive prompt — never as a persisted CLI flag. `--key` is accepted for the one-liner UX, but it is immediately stored in the secrets vault and a warning suggests the env var next time. Preferred: + +```bash +export HYPERVAULT_API_KEY=hv_... +agentvault hypervault connect +``` + +`connect` validates the key, stores it in the secrets vault (`hypervault_api_key`), and writes `.agentvault/hypervault.json` — which records only a **keyRef** (a vault pointer), never the key itself. + +## Commands + +| Command | What it does | +|---|---| +| `hypervault connect` | Validate the key, vault it, write `hypervault.json`. | +| `hypervault status` | The whole three-tier picture in one screen. | +| `hypervault bootstrap ` | Scaffold + connect + pull + build indices + wire MCP. | +| `hypervault pull` | Incremental export → local snapshot, working tree (`.agentvault/memories/`), and indices. | +| `hypervault push [--dry-run]` | Push locally edited memories up as provenance-stamped mind commits. | +| `hypervault snapshot [-o file] [--encrypt]` | Full export → `agentvault-hypervault-snapshot-v1` bundle on disk. | +| `hypervault archive` | The sovereign archive pipeline (see below). | +| `hypervault verify ` | Verify a bundle: manifest hash, ed25519 signature, Merkle root, per-entry checksums. | +| `hypervault restore --to ` | Rebuild a project or a fresh cloud account from a bundle. | +| `hypervault reindex` | Rebuild local vector/FTS indices from the snapshot. | +| `hypervault recall ""` | Offline hybrid (lexical + semantic) recall over local indices. | + +## Flagship flow: bootstrap + +```bash +npx agentvault@latest hypervault bootstrap my-agent --key hv_XXXX +``` + +1. **Scaffold** — project dir, `agent.json` (with a `hypervault` block), `index.js`, `.agentvault/`. +2. **Connect** — validate the key, vault it, write `hypervault.json` with a `keyRef`. +3. **Pull** — stream `/api/export` into `.agentvault/memories/` (one markdown file per memory) and a snapshot bundle. +4. **Build indices** — FTS always; vectors from exported embeddings when an embedding provider is configured. +5. **Wire MCP** — emit `.mcp.json` registering both `agentvault mcp serve` and the upstream `hypervault-mcp`. +6. **Soul detection** — a memory tagged `soul` (or `--soul `) is written to `soul.md`, activating the existing memory-repo soul path. + +## Flagship flow: sovereign archive + +```bash +npx agentvault@latest hypervault archive --all --encrypt --network ic --arweave +``` + +The pipeline prints a checklist as it runs: + +1. **Export** — full (or `--since` incremental) NDJSON stream from `hypervault.store`. +2. **Bundle** — build the snapshot bundle; compute per-entry SHA-256 and a Merkle root. +3. **Encrypt** — AES-256-GCM per entry (via the audited `CanisterEncryption`), passphrase-wrapped. +4. **Canister commit** — replay the mind DAG onto a `memory_repo` canister (topological, idempotent). +5. **Arweave upload** — signed bundle with `App-Name`, `Bundle-Format`, `HyperVault-User`, `State-Hash` tags. +6. **Receipts** — an on-chain `archive-receipt:` commit, plus an optional POST to hypervault. +7. **Verify** — re-fetch from Arweave and check the whole integrity chain. + +It closes with the resurrection line: + +``` +✔ Archived. Resurrect anywhere with: + npx agentvault@latest hypervault restore ar:// --to local +``` + +## Resurrection + +```bash +# chain → local working agent (memories + indices rebuilt) +npx agentvault@latest hypervault restore ar:// --to local + +# chain → fresh hypervault account (cloud mind rebuilt) +npx agentvault@latest hypervault restore ar:// --to hypervault --key hv_NEW +``` + +## Security notes + +- **Key handling:** `hv_` keys never touch argv persistence; `--key` is vaulted immediately and scrubbed from any persisted config. `hypervault.json` refuses to store anything that looks like a plaintext key. +- **Encryption:** all bundle encryption uses `CanisterEncryption` (AES-256-GCM with validated auth tags). Private artifacts and conversations are **always** encrypted, regardless of `--encrypt`. +- **Integrity chain:** content SHA-256 per entry → Merkle root in the manifest → ed25519 manifest signature → Arweave tags → on-chain receipt commit. `hypervault verify` checks every link with a CI-friendly exit code. +- **Provenance:** hypervault `author_kind` / `author_key_prefix` travel into revision records, canister commit tags, and the manifest — "which agent wrote this memory" survives archival. + +See [`docs/architecture/hypervault-integration.md`](../architecture/hypervault-integration.md) for the three-tier architecture and the normative bundle-format spec. diff --git a/package.json b/package.json index acda0ce..fa09fa9 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,10 @@ "./backbone": { "types": "./dist/src/backbone/index.d.ts", "import": "./dist/src/backbone/index.js" + }, + "./hypervault": { + "types": "./dist/src/hypervault/index.d.ts", + "import": "./dist/src/hypervault/index.js" } }, "bin": { diff --git a/src/backup/thoughtform-bundle.ts b/src/backup/thoughtform-bundle.ts index 324f34b..6ca56d0 100644 --- a/src/backup/thoughtform-bundle.ts +++ b/src/backup/thoughtform-bundle.ts @@ -25,6 +25,34 @@ const gunzip = promisify(zlib.gunzip); export const THOUGHTFORM_BUNDLE_FORMAT = 'agentvault-thoughtform-bundle-v1'; +/** + * HyperVault account snapshots (`src/hypervault/snapshot.ts`) reuse this same + * gzip(JSON) envelope with a distinct format string. Recognizing it here lets + * `backup preview/verify` identify snapshot bundles instead of erroring on an + * unknown format. + */ +export const HYPERVAULT_SNAPSHOT_FORMAT = 'agentvault-hypervault-snapshot-v1'; + +/** Formats that share the gzipped-JSON bundle envelope. */ +export const RECOGNIZED_BUNDLE_FORMATS = [THOUGHTFORM_BUNDLE_FORMAT, HYPERVAULT_SNAPSHOT_FORMAT] as const; + +/** + * Inspect a gzipped bundle file and report its declared format without fully + * validating it. Returns null when the file is not a recognized bundle. + */ +export async function detectBundleFormat(filePath: string): Promise { + if (!fs.existsSync(filePath)) { + return null; + } + try { + const decompressed = await gunzip(fs.readFileSync(filePath)); + const parsed = JSON.parse(decompressed.toString('utf8')) as { format?: unknown }; + return typeof parsed.format === 'string' ? parsed.format : null; + } catch { + return null; + } +} + export interface ThoughtformBundle { format: typeof THOUGHTFORM_BUNDLE_FORMAT; createdAt: string; diff --git a/src/hypervault/auth.ts b/src/hypervault/auth.ts new file mode 100644 index 0000000..683f306 --- /dev/null +++ b/src/hypervault/auth.ts @@ -0,0 +1,142 @@ +/** + * HyperVault key resolution & local state + * + * Resolution chain (per the AGENTS.md wallet-secret policy — keys never + * persist in flags or config files): + * + * 1. explicit key passed by the caller (e.g. a `--key` flag — discouraged) + * 2. HYPERVAULT_API_KEY environment variable + * 3. secrets vault lookup (`hypervault_api_key` for the agent) + * + * Interactive prompting is the CLI layer's job; this module is + * non-interactive. `.agentvault/hypervault.json` stores a `keyRef` + * (vault pointer), never the key itself. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { atomicWriteFileSync } from '../utils/path-validation.js'; +import type { SecretProvider } from '../vault/provider.js'; +import { hypervaultStateSchema, type HypervaultState } from './types.js'; +import { DEFAULT_HYPERVAULT_API_URL } from './client.js'; + +export const HYPERVAULT_KEY_SECRET_NAME = 'hypervault_api_key'; +export const HYPERVAULT_STATE_FILENAME = 'hypervault.json'; + +export type HyperVaultKeySource = 'flag' | 'env' | 'vault'; + +export interface ResolvedHyperVaultKey { + key: string; + source: HyperVaultKeySource; + /** Vault pointer for persisting in hypervault.json (vault source only) */ + keyRef?: string; + /** Set when the key arrived via a CLI flag — callers should warn */ + insecureSource: boolean; +} + +type VaultBackend = 'hashicorp' | 'bitwarden'; + +function resolveVaultBackend(): VaultBackend { + const b = (process.env.AGENTVAULT_VAULT_BACKEND ?? 'hashicorp').toLowerCase(); + return b === 'bitwarden' || b === 'bw' ? 'bitwarden' : 'hashicorp'; +} + +async function buildVaultProvider(agentId: string, backend: VaultBackend): Promise { + if (backend === 'bitwarden') { + const { BitwardenProvider } = await import('../vault/bitwarden.js'); + return new BitwardenProvider({ agentId }); + } + const { HashiCorpVaultProvider } = await import('../vault/hashicorp-provider.js'); + return HashiCorpVaultProvider.forAgent(agentId); +} + +export function makeKeyRef(backend: VaultBackend, agentId: string): string { + return `vault:${backend}/${agentId}/${HYPERVAULT_KEY_SECRET_NAME}`; +} + +/** + * Resolve the HyperVault API key without prompting. + * + * @param options.flagKey - key passed on the command line (discouraged) + * @param options.agentId - agent identifier for the vault lookup + * @returns The resolved key, or null when no source produced one. + */ +export async function resolveHyperVaultKey(options: { + flagKey?: string; + agentId?: string; +} = {}): Promise { + if (options.flagKey) { + return { key: options.flagKey, source: 'flag', insecureSource: true }; + } + + const envKey = process.env.HYPERVAULT_API_KEY; + if (envKey) { + return { key: envKey, source: 'env', insecureSource: false }; + } + + if (options.agentId) { + const backend = resolveVaultBackend(); + try { + const provider = await buildVaultProvider(options.agentId, backend); + const value = await provider.getSecret(HYPERVAULT_KEY_SECRET_NAME); + if (value) { + return { + key: value, + source: 'vault', + keyRef: makeKeyRef(backend, options.agentId), + insecureSource: false, + }; + } + } catch { + // Vault not configured / unreachable — fall through to null. + } + } + + return null; +} + +/** + * Store the key in the secrets vault and return its keyRef. + * Throws when no vault backend is configured/reachable. + */ +export async function storeHyperVaultKey(agentId: string, key: string): Promise { + const backend = resolveVaultBackend(); + const provider = await buildVaultProvider(agentId, backend); + await provider.storeSecret(HYPERVAULT_KEY_SECRET_NAME, key); + return makeKeyRef(backend, agentId); +} + +// --------------------------------------------------------------------------- +// State file (.agentvault/hypervault.json) +// --------------------------------------------------------------------------- + +export function hypervaultStatePath(projectRoot: string = process.cwd()): string { + return path.join(projectRoot, '.agentvault', HYPERVAULT_STATE_FILENAME); +} + +export function loadHypervaultState(projectRoot: string = process.cwd()): HypervaultState | null { + const filePath = hypervaultStatePath(projectRoot); + if (!fs.existsSync(filePath)) return null; + try { + const raw = JSON.parse(fs.readFileSync(filePath, 'utf-8')) as unknown; + return hypervaultStateSchema.parse(raw); + } catch { + return null; + } +} + +/** + * Persist HyperVault state. Refuses to write anything that looks like a + * plaintext `hv_` key (defence in depth for §7.1). + */ +export function saveHypervaultState(state: HypervaultState, projectRoot: string = process.cwd()): void { + const serialized = JSON.stringify(hypervaultStateSchema.parse(state), null, 2) + '\n'; + if (/hv_[A-Za-z0-9]{8,}/.test(serialized)) { + throw new Error('Refusing to write a plaintext HyperVault key to hypervault.json (use the secrets vault)'); + } + atomicWriteFileSync(hypervaultStatePath(projectRoot), serialized, { encoding: 'utf8', mode: 0o600 }); +} + +export function defaultHypervaultState(): HypervaultState { + return hypervaultStateSchema.parse({ apiUrl: process.env.HYPERVAULT_API_URL ?? DEFAULT_HYPERVAULT_API_URL }); +} diff --git a/src/hypervault/client.ts b/src/hypervault/client.ts new file mode 100644 index 0000000..e45daf8 --- /dev/null +++ b/src/hypervault/client.ts @@ -0,0 +1,439 @@ +/** + * HyperVault typed REST client + * + * Thin, typed wrapper over the hypervault.store HTTP API (the same surface + * the upstream Python MCP server proxies). Authenticates with the + * `X-HyperVault-Key` header, retries transient failures with exponential + * backoff, and honours hypervault's 60 req/min per-key rate limit by + * backing off on 429 responses. + */ + +import { request } from 'undici'; +import { + hvExportManifestSchema, + hvExportRecordSchema, + HV_SUPPORTED_SCHEMA_VERSION, + type HvArtifact, + type HvExportManifest, + type HvExportRecord, + type HvMemory, + type HvMindBranch, + type HvMindCommit, +} from './types.js'; + +export const DEFAULT_HYPERVAULT_API_URL = 'https://hypervault.store'; + +/** Retry policy for transient failures */ +const MAX_RETRIES = 4; +const BASE_BACKOFF_MS = 500; + +export interface HyperVaultClientOptions { + apiKey: string; + apiUrl?: string; + /** Override for tests: milliseconds to wait between retries multiplier */ + backoffMs?: number; +} + +export interface ExportOptions { + include?: string[]; + since?: string; + branch?: string; +} + +export interface ExportResult { + records: HvExportRecord[]; + manifest: HvExportManifest; +} + +export interface MemorizeInput { + title?: string; + content: string; + tags?: string[]; + summary?: string; +} + +export interface RecallOptions { + limit?: number; + tags?: string[]; +} + +export class HyperVaultError extends Error { + constructor( + message: string, + public readonly statusCode?: number, + ) { + super(message); + this.name = 'HyperVaultError'; + } +} + +export class HyperVaultClient { + private readonly apiUrl: string; + private readonly apiKey: string; + private readonly backoffMs: number; + + constructor(options: HyperVaultClientOptions) { + if (!options.apiKey) { + throw new HyperVaultError('HyperVault API key is required'); + } + this.apiKey = options.apiKey; + this.apiUrl = (options.apiUrl ?? process.env.HYPERVAULT_API_URL ?? DEFAULT_HYPERVAULT_API_URL).replace(/\/+$/, ''); + this.backoffMs = options.backoffMs ?? BASE_BACKOFF_MS; + } + + getApiUrl(): string { + return this.apiUrl; + } + + // ------------------------------------------------------------------------- + // Low-level request with retry/backoff + // ------------------------------------------------------------------------- + + private async requestRaw( + method: 'GET' | 'POST' | 'PATCH' | 'DELETE', + path: string, + body?: unknown, + query?: Record, + ): Promise<{ statusCode: number; body: NodeJS.ReadableStream & { text(): Promise } }> { + const url = new URL(this.apiUrl + path); + for (const [k, v] of Object.entries(query ?? {})) { + if (v !== undefined && v !== '') url.searchParams.set(k, v); + } + + let lastError: unknown; + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + if (attempt > 0) { + await sleep(this.backoffMs * 2 ** (attempt - 1)); + } + try { + const res = await request(url, { + method, + headers: { + 'X-HyperVault-Key': this.apiKey, + accept: 'application/json', + ...(body !== undefined ? { 'content-type': 'application/json' } : {}), + }, + body: body !== undefined ? JSON.stringify(body) : undefined, + }); + + // Retry on rate limit and server errors; return everything else. + if (res.statusCode === 429 || res.statusCode >= 500) { + lastError = new HyperVaultError( + `HyperVault responded ${res.statusCode} for ${method} ${path}`, + res.statusCode, + ); + await res.body.text().catch(() => undefined); // drain + continue; + } + return { statusCode: res.statusCode, body: res.body }; + } catch (error) { + lastError = error; + } + } + const message = lastError instanceof Error ? lastError.message : String(lastError); + throw new HyperVaultError(`HyperVault request failed after ${MAX_RETRIES + 1} attempts: ${message}`); + } + + private async requestJSON( + method: 'GET' | 'POST' | 'PATCH' | 'DELETE', + path: string, + body?: unknown, + query?: Record, + ): Promise { + const res = await this.requestRaw(method, path, body, query); + const text = await res.body.text(); + if (res.statusCode === 401 || res.statusCode === 403) { + throw new HyperVaultError('HyperVault API key was rejected (check `agentvault hypervault connect`)', res.statusCode); + } + if (res.statusCode >= 400) { + throw new HyperVaultError(`HyperVault ${method} ${path} failed (${res.statusCode}): ${truncate(text)}`, res.statusCode); + } + if (!text) return undefined as T; + try { + return JSON.parse(text) as T; + } catch { + throw new HyperVaultError(`HyperVault ${method} ${path} returned invalid JSON: ${truncate(text)}`, res.statusCode); + } + } + + // ------------------------------------------------------------------------- + // Key / identity + // ------------------------------------------------------------------------- + + /** Validate the API key. Returns a user hint when the API provides one. */ + async validateKey(): Promise<{ valid: boolean; userIdHint?: string }> { + try { + const data = await this.requestJSON<{ user_id?: string; key_prefix?: string }>('GET', '/api/keys'); + return { valid: true, userIdHint: data?.user_id ?? data?.key_prefix }; + } catch (error) { + if (error instanceof HyperVaultError && (error.statusCode === 401 || error.statusCode === 403)) { + return { valid: false }; + } + throw error; + } + } + + // ------------------------------------------------------------------------- + // Memories + // ------------------------------------------------------------------------- + + async memorize(input: MemorizeInput): Promise { + return this.requestJSON('POST', '/api/memories', input); + } + + async listMemories(options: { limit?: number; tags?: string[] } = {}): Promise { + const data = await this.requestJSON('GET', '/api/memories', undefined, { + limit: options.limit?.toString(), + tags: options.tags?.join(','), + }); + return Array.isArray(data) ? data : (data?.memories ?? []); + } + + async getMemory(id: string): Promise { + try { + return await this.requestJSON('GET', `/api/memories/${encodeURIComponent(id)}`); + } catch (error) { + if (error instanceof HyperVaultError && error.statusCode === 404) return null; + throw error; + } + } + + async recall(query: string, options: RecallOptions = {}): Promise { + const data = await this.requestJSON('GET', '/api/recall', undefined, { + q: query, + limit: options.limit?.toString(), + tags: options.tags?.join(','), + }); + return Array.isArray(data) ? data : (data?.memories ?? []); + } + + async editMemory(id: string, patch: Partial): Promise { + return this.requestJSON('PATCH', `/api/memories/${encodeURIComponent(id)}`, patch); + } + + async forgetMemory(id: string): Promise { + try { + await this.requestJSON('DELETE', `/api/memories/${encodeURIComponent(id)}`); + return true; + } catch (error) { + if (error instanceof HyperVaultError && error.statusCode === 404) return false; + throw error; + } + } + + async memoryHistory(id: string): Promise>> { + return this.requestJSON('GET', `/api/memories/${encodeURIComponent(id)}/history`); + } + + // ------------------------------------------------------------------------- + // Mind DAG + // ------------------------------------------------------------------------- + + async mindLog(options: { branch?: string; since?: string; limit?: number } = {}): Promise { + const data = await this.requestJSON('GET', '/api/mind/log', undefined, { + branch: options.branch, + since: options.since, + limit: options.limit?.toString(), + }); + return Array.isArray(data) ? data : (data?.commits ?? []); + } + + async mindBranches(): Promise { + const data = await this.requestJSON('GET', '/api/mind/branches'); + return Array.isArray(data) ? data : (data?.branches ?? []); + } + + async mindBranch(name: string): Promise { + return this.requestJSON('POST', '/api/mind/branches', { name }); + } + + async mindDiff(from: string, to: string): Promise> { + return this.requestJSON('GET', '/api/mind/diff', undefined, { from, to }); + } + + async mindMerge(fromBranch: string, toBranch: string): Promise> { + return this.requestJSON('POST', '/api/mind/merge', { from: fromBranch, to: toBranch }); + } + + async mindRevert(commitId: string): Promise> { + return this.requestJSON('POST', '/api/mind/revert', { commit_id: commitId }); + } + + async mindState(branch?: string): Promise> { + return this.requestJSON('GET', '/api/mind/state', undefined, { branch }); + } + + // ------------------------------------------------------------------------- + // Artifacts & connections + // ------------------------------------------------------------------------- + + async saveArtifact(artifact: Partial & { content: string }): Promise { + return this.requestJSON('POST', '/api/artifacts', artifact); + } + + async listArtifacts(): Promise { + const data = await this.requestJSON('GET', '/api/artifacts'); + return Array.isArray(data) ? data : (data?.artifacts ?? []); + } + + async deleteArtifact(slug: string): Promise { + try { + await this.requestJSON('DELETE', `/api/artifacts/${encodeURIComponent(slug)}`); + return true; + } catch (error) { + if (error instanceof HyperVaultError && error.statusCode === 404) return false; + throw error; + } + } + + async connect(fromId: string, toId: string, kind = 'link'): Promise> { + return this.requestJSON('POST', '/api/connections', { from_id: fromId, to_id: toId, kind }); + } + + // ------------------------------------------------------------------------- + // Bulk export / import (companion hypervault PRs §4.1 / §4.3) + // ------------------------------------------------------------------------- + + /** + * Stream the full-account NDJSON export. + * + * Yields one validated record per row; the final manifest line is returned + * via the `onManifest` callback (and validated against the supported + * schema version). + */ + async *exportVaultStream( + options: ExportOptions = {}, + onManifest?: (manifest: HvExportManifest) => void, + ): AsyncGenerator { + const res = await this.requestRaw('GET', '/api/export', undefined, { + include: options.include?.join(','), + since: options.since, + branch: options.branch, + }); + if (res.statusCode >= 400) { + const text = await res.body.text(); + throw new HyperVaultError(`HyperVault export failed (${res.statusCode}): ${truncate(text)}`, res.statusCode); + } + + let buffer = ''; + for await (const chunk of res.body) { + buffer += typeof chunk === 'string' ? chunk : chunk.toString('utf-8'); + let newline: number; + while ((newline = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (!line) continue; + const record = parseExportLine(line, onManifest); + if (record) yield record; + } + } + const tail = buffer.trim(); + if (tail) { + const record = parseExportLine(tail, onManifest); + if (record) yield record; + } + } + + /** Collect a full export into memory (fine for typical accounts). */ + async exportVault(options: ExportOptions = {}): Promise { + let manifest: HvExportManifest | undefined; + const records: HvExportRecord[] = []; + for await (const record of this.exportVaultStream(options, (m) => (manifest = m))) { + records.push(record); + } + if (!manifest) { + // Tolerate servers that omit the manifest line by synthesizing one. + manifest = hvExportManifestSchema.parse({ manifest: true, row_counts: countRows(records) }); + } + return { records, manifest }; + } + + /** Restore a full account from export records (POST /api/import/vault). */ + async importVault(records: HvExportRecord[], manifest?: HvExportManifest): Promise<{ imported: number }> { + const ndjson = + records.map((r) => JSON.stringify(r)).join('\n') + (manifest ? '\n' + JSON.stringify(manifest) : '') + '\n'; + const url = new URL(this.apiUrl + '/api/import/vault'); + const res = await request(url, { + method: 'POST', + headers: { + 'X-HyperVault-Key': this.apiKey, + 'content-type': 'application/x-ndjson', + }, + body: ndjson, + }); + const text = await res.body.text(); + if (res.statusCode >= 400) { + throw new HyperVaultError(`HyperVault import failed (${res.statusCode}): ${truncate(text)}`, res.statusCode); + } + try { + const data = JSON.parse(text) as { imported?: number }; + return { imported: data.imported ?? records.length }; + } catch { + return { imported: records.length }; + } + } + + /** Post an archive receipt so the hypervault dashboard can show it (§4.4). */ + async postArchiveReceipt(receipt: { + kind: 'arweave' | 'icp'; + ref: string; + manifest_hash: string; + }): Promise { + try { + await this.requestJSON('POST', '/api/archive-receipts', receipt); + } catch (error) { + // Receipts are best-effort: older hypervault deployments don't have + // the endpoint yet, and archive success must not depend on it. + if (error instanceof HyperVaultError && (error.statusCode === 404 || error.statusCode === 405)) return; + throw error; + } + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function parseExportLine( + line: string, + onManifest?: (manifest: HvExportManifest) => void, +): HvExportRecord | null { + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + throw new HyperVaultError(`Export stream contained an invalid NDJSON line: ${truncate(line)}`); + } + if (parsed && typeof parsed === 'object' && 'manifest' in (parsed as Record)) { + const manifest = hvExportManifestSchema.parse(parsed); + if (Math.floor(manifest.schema_version) > HV_SUPPORTED_SCHEMA_VERSION) { + throw new HyperVaultError( + `Export schema version ${manifest.schema_version} is newer than supported (${HV_SUPPORTED_SCHEMA_VERSION}); upgrade agentvault`, + ); + } + onManifest?.(manifest); + return null; + } + const result = hvExportRecordSchema.safeParse(parsed); + if (!result.success) { + // Skip unknown tables rather than failing the whole export (schema drift). + return null; + } + return result.data; +} + +export function countRows(records: HvExportRecord[]): Record { + const counts: Record = {}; + for (const record of records) { + counts[record.table] = (counts[record.table] ?? 0) + 1; + } + return counts; +} + +function truncate(text: string, max = 200): string { + return text.length > max ? text.slice(0, max) + '…' : text; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/hypervault/index.ts b/src/hypervault/index.ts new file mode 100644 index 0000000..60c45c8 --- /dev/null +++ b/src/hypervault/index.ts @@ -0,0 +1,82 @@ +/** + * HyperVault integration — public surface (`agentvault/hypervault`) + * + * The living cloud mind (hypervault.store) made a first-class citizen of + * AgentVault: typed REST client, snapshot/bundle format, local indices, + * backbone/wiki adapters, on-chain mind-DAG mirror, and the flagship + * bootstrap/archive/restore pipeline. + */ + +export * from './types.js'; +export { + HyperVaultClient, + HyperVaultError, + DEFAULT_HYPERVAULT_API_URL, + countRows, + type HyperVaultClientOptions, + type ExportOptions, + type ExportResult, + type MemorizeInput, + type RecallOptions, +} from './client.js'; +export { + resolveHyperVaultKey, + storeHyperVaultKey, + makeKeyRef, + loadHypervaultState, + saveHypervaultState, + hypervaultStatePath, + defaultHypervaultState, + HYPERVAULT_KEY_SECRET_NAME, + HYPERVAULT_STATE_FILENAME, + type ResolvedHyperVaultKey, + type HyperVaultKeySource, +} from './auth.js'; +export { + buildSnapshot, + writeSnapshot, + readSnapshot, + validateSnapshot, + verifySnapshot, + readSnapshotEntry, + snapshotToRecords, + readEmbeddings, + snapshotEmbeddingModel, + canonicalManifestBytes, + HYPERVAULT_SNAPSHOT_FORMAT, + SNAPSHOT_FILE_EXTENSION, + type HypervaultSnapshot, + type HypervaultSnapshotManifest, + type SnapshotVerifyResult, + type BuildSnapshotOptions, +} from './snapshot.js'; +export { FtsIndex, tokenize, type FtsHit, type FtsDocInput } from './index/fts-index.js'; +export { VectorIndex, type VectorHit } from './index/vector-index.js'; +export { + buildIndices, + buildIndicesFromSnapshot, + saveIndices, + loadIndices, + snapshotMemories, + indexDir, + type BuiltIndices, +} from './index/builder.js'; +export { hybridRecall, type QueryEmbedder, type HybridRecallOptions } from './index/recall.js'; +export { HyperVaultMemoryStore } from './memory-store.js'; +export { HyperVaultKnowledgeStore } from './knowledge-store.js'; +export { HyperVaultWikiStore } from './wiki-store.js'; +export { + syncMindToCanister, + topologicalOrder, + collectSyncedCommitIds, + writeArchiveReceipt, + type MindSyncInput, + type MindSyncResult, +} from './mind-sync.js'; +export * from './pipeline.js'; +export { + getHyperVaultToolDefinitions, + handleHyperVaultToolCall, + serveMcp, + type McpServeOptions, +} from './mcp-server.js'; diff --git a/src/hypervault/index/builder.ts b/src/hypervault/index/builder.ts new file mode 100644 index 0000000..3f2c404 --- /dev/null +++ b/src/hypervault/index/builder.ts @@ -0,0 +1,122 @@ +/** + * Index builder — derives local FTS + vector indices from a snapshot + * + * Indices live under `.agentvault/index/` and are always rebuildable from + * the snapshot, so they are never part of an archived bundle's integrity + * surface (they're `derived` artifacts — §5.5). + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { atomicWriteFileSync } from '../../utils/path-validation.js'; +import { hvMemorySchema, type HvMemory } from '../types.js'; +import { readEmbeddings, snapshotEmbeddingModel, readSnapshotEntry, type HypervaultSnapshot } from '../snapshot.js'; +import { FtsIndex } from './fts-index.js'; +import { VectorIndex } from './vector-index.js'; + +export const INDEX_DIR_NAME = 'index'; +export const FTS_INDEX_FILENAME = 'fts.json'; +export const VECTOR_INDEX_FILENAME = 'vectors.json'; + +export interface BuiltIndices { + fts: FtsIndex; + vectors: VectorIndex | null; + memoriesIndexed: number; + vectorsIndexed: number; +} + +export function indexDir(projectRoot: string = process.cwd()): string { + return path.join(projectRoot, '.agentvault', INDEX_DIR_NAME); +} + +/** Build indices from in-memory rows. */ +export function buildIndices( + memories: HvMemory[], + embeddingsById: Map = new Map(), + embeddingModel?: string, +): BuiltIndices { + const fts = new FtsIndex(); + for (const memory of memories) { + fts.add({ + id: memory.id, + title: memory.title, + tags: memory.tags, + summary: memory.summary, + content: memory.content, + }); + } + + let vectors: VectorIndex | null = null; + let vectorsIndexed = 0; + for (const [id, embedding] of embeddingsById) { + if (!vectors) { + vectors = new VectorIndex(embedding.length, embeddingModel); + } + if (embedding.length !== vectors.dims) continue; // refuse mixed dims + vectors.add(id, embedding); + vectorsIndexed += 1; + } + + return { fts, vectors, memoriesIndexed: memories.length, vectorsIndexed }; +} + +/** Build indices straight from a snapshot bundle. */ +export async function buildIndicesFromSnapshot( + snapshot: HypervaultSnapshot, + passphrase?: string, +): Promise { + const memories = await snapshotMemories(snapshot, passphrase); + const embeddings = await readEmbeddings(snapshot, passphrase); + const model = await snapshotEmbeddingModel(snapshot, passphrase); + return buildIndices(memories, embeddings, model); +} + +/** Parse the memories entry of a snapshot into typed rows. */ +export async function snapshotMemories( + snapshot: HypervaultSnapshot, + passphrase?: string, +): Promise { + if (!('memories.ndjson' in snapshot.entries)) return []; + const text = (await readSnapshotEntry(snapshot, 'memories.ndjson', passphrase)).toString('utf-8'); + const memories: HvMemory[] = []; + for (const line of text.split('\n')) { + if (!line.trim()) continue; + const parsed = hvMemorySchema.safeParse(JSON.parse(line)); + if (parsed.success) memories.push(parsed.data); + } + return memories; +} + +/** Persist built indices under `.agentvault/index/`. */ +export function saveIndices(indices: BuiltIndices, projectRoot: string = process.cwd()): string { + const dir = indexDir(projectRoot); + atomicWriteFileSync(path.join(dir, FTS_INDEX_FILENAME), JSON.stringify(indices.fts.toJSON()), { + encoding: 'utf8', + }); + if (indices.vectors) { + atomicWriteFileSync(path.join(dir, VECTOR_INDEX_FILENAME), JSON.stringify(indices.vectors.toJSON()), { + encoding: 'utf8', + }); + } + return dir; +} + +/** Load previously built indices (nulls when absent). */ +export function loadIndices(projectRoot: string = process.cwd()): { + fts: FtsIndex | null; + vectors: VectorIndex | null; +} { + const dir = indexDir(projectRoot); + let fts: FtsIndex | null = null; + let vectors: VectorIndex | null = null; + + const ftsPath = path.join(dir, FTS_INDEX_FILENAME); + if (fs.existsSync(ftsPath)) { + fts = FtsIndex.fromJSON(JSON.parse(fs.readFileSync(ftsPath, 'utf-8'))); + } + const vectorPath = path.join(dir, VECTOR_INDEX_FILENAME); + if (fs.existsSync(vectorPath)) { + vectors = VectorIndex.fromJSON(JSON.parse(fs.readFileSync(vectorPath, 'utf-8'))); + } + return { fts, vectors }; +} diff --git a/src/hypervault/index/fts-index.ts b/src/hypervault/index/fts-index.ts new file mode 100644 index 0000000..b4604b8 --- /dev/null +++ b/src/hypervault/index/fts-index.ts @@ -0,0 +1,128 @@ +/** + * Full-text index — pure-TS inverted index with weighted fields + * + * Mirrors the field weighting of hypervault's generated tsvector + * (A = title, B = tags/summary, C = content) so offline recall ranks + * comparably to the cloud index. Scoring is TF-IDF with length + * normalization (BM25-lite). No native dependencies. + */ + +export interface FtsDocInput { + id: string; + title?: string; + tags?: string[]; + summary?: string; + content?: string; +} + +export interface FtsHit { + id: string; + score: number; +} + +/** tsvector-style weights: A/B/C */ +const FIELD_WEIGHTS = { + title: 1.0, + tags: 0.4, + summary: 0.4, + content: 0.2, +} as const; + +interface SerializedFtsIndex { + version: 1; + docCount: number; + docLengths: Record; + /** term -> { docId -> weighted term frequency } */ + postings: Record>; +} + +export function tokenize(text: string): string[] { + return text + .toLowerCase() + .split(/[^\p{L}\p{N}]+/u) + .filter((t) => t.length >= 2); +} + +export class FtsIndex { + private postings = new Map>(); + private docLengths = new Map(); + + get size(): number { + return this.docLengths.size; + } + + add(doc: FtsDocInput): void { + if (this.docLengths.has(doc.id)) { + this.remove(doc.id); + } + let length = 0; + const addTokens = (text: string | undefined, weight: number): void => { + for (const token of tokenize(text ?? '')) { + const docs = this.postings.get(token) ?? new Map(); + docs.set(doc.id, (docs.get(doc.id) ?? 0) + weight); + this.postings.set(token, docs); + length += 1; + } + }; + addTokens(doc.title, FIELD_WEIGHTS.title); + addTokens((doc.tags ?? []).join(' '), FIELD_WEIGHTS.tags); + addTokens(doc.summary, FIELD_WEIGHTS.summary); + addTokens(doc.content, FIELD_WEIGHTS.content); + this.docLengths.set(doc.id, Math.max(length, 1)); + } + + remove(id: string): void { + if (!this.docLengths.delete(id)) return; + for (const [term, docs] of this.postings) { + docs.delete(id); + if (docs.size === 0) this.postings.delete(term); + } + } + + search(query: string, limit = 10): FtsHit[] { + const terms = tokenize(query); + if (terms.length === 0 || this.docLengths.size === 0) return []; + + const scores = new Map(); + const totalDocs = this.docLengths.size; + for (const term of terms) { + const docs = this.postings.get(term); + if (!docs) continue; + const idf = Math.log(1 + (totalDocs - docs.size + 0.5) / (docs.size + 0.5)); + for (const [docId, weightedTf] of docs) { + const docLength = this.docLengths.get(docId) ?? 1; + const norm = weightedTf / (weightedTf + 1 + docLength / 500); + scores.set(docId, (scores.get(docId) ?? 0) + idf * norm); + } + } + + return [...scores.entries()] + .map(([id, score]) => ({ id, score })) + .sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)) + .slice(0, limit); + } + + toJSON(): SerializedFtsIndex { + const postings: Record> = {}; + for (const [term, docs] of this.postings) { + postings[term] = Object.fromEntries(docs); + } + return { + version: 1, + docCount: this.docLengths.size, + docLengths: Object.fromEntries(this.docLengths), + postings, + }; + } + + static fromJSON(data: SerializedFtsIndex): FtsIndex { + const index = new FtsIndex(); + for (const [docId, length] of Object.entries(data.docLengths)) { + index.docLengths.set(docId, length); + } + for (const [term, docs] of Object.entries(data.postings)) { + index.postings.set(term, new Map(Object.entries(docs))); + } + return index; + } +} diff --git a/src/hypervault/index/recall.ts b/src/hypervault/index/recall.ts new file mode 100644 index 0000000..8ef0035 --- /dev/null +++ b/src/hypervault/index/recall.ts @@ -0,0 +1,69 @@ +/** + * Hybrid recall — lexical + semantic fusion over local indices + * + * Ports the shape of hypervault's `hybridRecallMemories` fusion: both + * retrieval paths run independently and are merged with reciprocal-rank + * fusion (RRF). Falls back gracefully to FTS-only when no embedding + * provider or vector index is available — the same degradation hypervault + * itself applies. + */ + +import type { HvMemory, HvRecallResult } from '../types.js'; +import type { FtsIndex } from './fts-index.js'; +import type { VectorIndex } from './vector-index.js'; + +/** Embeds a query string; typically backed by an OpenAI-compatible endpoint */ +export type QueryEmbedder = (query: string) => Promise<{ embedding: number[]; model?: string }>; + +export interface HybridRecallOptions { + limit?: number; + /** Optional semantic path — omitted → FTS-only */ + embedQuery?: QueryEmbedder; + /** RRF constant (default 60) */ + rrfK?: number; +} + +export async function hybridRecall( + query: string, + indices: { fts: FtsIndex | null; vectors: VectorIndex | null }, + memoriesById: Map, + options: HybridRecallOptions = {}, +): Promise { + const limit = options.limit ?? 10; + const rrfK = options.rrfK ?? 60; + const candidateCount = Math.max(limit * 3, 30); + + const lexical = indices.fts ? indices.fts.search(query, candidateCount) : []; + + let semantic: Array<{ id: string; score: number }> = []; + if (indices.vectors && options.embedQuery) { + try { + const { embedding, model } = await options.embedQuery(query); + semantic = indices.vectors.search(embedding, candidateCount, model); + } catch { + // Provider unavailable or model mismatch — degrade to lexical-only. + semantic = []; + } + } + + const fused = new Map }>(); + const fuse = (hits: Array<{ id: string }>, kind: 'lexical' | 'semantic'): void => { + hits.forEach((hit, rank) => { + const entry = fused.get(hit.id) ?? { score: 0, matchedBy: new Set<'lexical' | 'semantic'>() }; + entry.score += 1 / (rrfK + rank + 1); + entry.matchedBy.add(kind); + fused.set(hit.id, entry); + }); + }; + fuse(lexical, 'lexical'); + fuse(semantic, 'semantic'); + + return [...fused.entries()] + .sort((a, b) => b[1].score - a[1].score || a[0].localeCompare(b[0])) + .flatMap(([id, entry]) => { + const memory = memoriesById.get(id); + if (!memory) return []; + return [{ memory, score: entry.score, matchedBy: [...entry.matchedBy] }]; + }) + .slice(0, limit); +} diff --git a/src/hypervault/index/vector-index.ts b/src/hypervault/index/vector-index.ts new file mode 100644 index 0000000..4eafdcc --- /dev/null +++ b/src/hypervault/index/vector-index.ts @@ -0,0 +1,120 @@ +/** + * Vector index — cosine similarity over exported embeddings + * + * Pure-TS exact (brute force) nearest-neighbour search. This is fine for + * typical accounts (well under ~20k vectors — plan risk #5); an optional + * `hnswlib-node` accelerator can be layered on later without changing the + * on-disk format. Vectors are stored packed as float32 (base64) with an + * id/model sidecar, matching the snapshot's `embeddings.bin` layout. + * + * The recorded `model` guards against cross-model mixing (plan risk #1): + * queries embedded with a different model are refused. + */ + +export interface VectorHit { + id: string; + /** Cosine similarity in [-1, 1] */ + score: number; +} + +interface SerializedVectorIndex { + version: 1; + dims: number; + model?: string; + ids: string[]; + /** base64-packed float32, row-major [ids.length x dims] */ + vectors: string; +} + +export class VectorIndex { + private ids: string[] = []; + private vectors: Float32Array; + private norms: number[] = []; + + constructor( + public readonly dims: number, + public readonly model?: string, + ) { + if (!Number.isInteger(dims) || dims <= 0) { + throw new Error(`Vector index dims must be a positive integer (got ${dims})`); + } + this.vectors = new Float32Array(0); + } + + get size(): number { + return this.ids.length; + } + + add(id: string, vector: number[]): void { + if (vector.length !== this.dims) { + throw new Error(`Vector for ${id} has ${vector.length} dims, index expects ${this.dims}`); + } + const next = new Float32Array((this.ids.length + 1) * this.dims); + next.set(this.vectors); + next.set(vector, this.ids.length * this.dims); + this.vectors = next; + this.ids.push(id); + this.norms.push(norm(vector)); + } + + /** + * Exact cosine-similarity search. + * @param queryModel - model that produced the query embedding; refused if + * it differs from the index's model (risk #1) + */ + search(query: number[], limit = 10, queryModel?: string): VectorHit[] { + if (queryModel && this.model && queryModel !== this.model) { + throw new Error( + `Embedding model mismatch: index was built with "${this.model}" but the query used "${queryModel}"`, + ); + } + if (query.length !== this.dims) { + throw new Error(`Query has ${query.length} dims, index expects ${this.dims}`); + } + const queryNorm = norm(query); + if (queryNorm === 0 || this.ids.length === 0) return []; + + const hits: VectorHit[] = []; + for (let i = 0; i < this.ids.length; i++) { + let dot = 0; + const offset = i * this.dims; + for (let j = 0; j < this.dims; j++) { + dot += this.vectors[offset + j]! * query[j]!; + } + const denominator = (this.norms[i] ?? 0) * queryNorm; + hits.push({ id: this.ids[i]!, score: denominator === 0 ? 0 : dot / denominator }); + } + return hits.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)).slice(0, limit); + } + + toJSON(): SerializedVectorIndex { + return { + version: 1, + dims: this.dims, + model: this.model, + ids: [...this.ids], + vectors: Buffer.from(this.vectors.buffer, this.vectors.byteOffset, this.vectors.byteLength).toString('base64'), + }; + } + + static fromJSON(data: SerializedVectorIndex): VectorIndex { + const index = new VectorIndex(data.dims, data.model); + const raw = Buffer.from(data.vectors, 'base64'); + index.vectors = new Float32Array(raw.buffer.slice(raw.byteOffset, raw.byteOffset + raw.byteLength)); + index.ids = [...data.ids]; + index.norms = data.ids.map((_, i) => { + let sum = 0; + for (let j = 0; j < data.dims; j++) { + sum += index.vectors[i * data.dims + j]! ** 2; + } + return Math.sqrt(sum); + }); + return index; + } +} + +function norm(vector: number[]): number { + let sum = 0; + for (const v of vector) sum += v * v; + return Math.sqrt(sum); +} diff --git a/src/hypervault/knowledge-store.ts b/src/hypervault/knowledge-store.ts new file mode 100644 index 0000000..7fb50cd --- /dev/null +++ b/src/hypervault/knowledge-store.ts @@ -0,0 +1,119 @@ +/** + * HyperVaultKnowledgeStore — backbone KnowledgeStore backed by the cloud mind + * + * `KnowledgeEntry` lifecycle (draft → proposed → ratified → archived) maps to + * memories tagged `knowledge:`. The full entry is serialized into + * the memory content so nothing is lost in the round trip; versions are + * backed upstream by `memory_history` (every edit is a mind commit). + */ + +import type { KnowledgeEntry } from '../backbone/types.js'; +import type { KnowledgeListFilters, KnowledgeStore } from '../backbone/services/knowledge.js'; +import type { HyperVaultClient } from './client.js'; +import type { HvMemory } from './types.js'; + +const TAG_PREFIX = 'av'; + +function companyTag(companyId: string): string { + return `${TAG_PREFIX}:company:${companyId}`; +} +function idTag(id: string): string { + return `${TAG_PREFIX}:kid:${id}`; +} +function statusTag(status: string): string { + return `knowledge:${status}`; +} +function categoryTag(category: string): string { + return `${TAG_PREFIX}:kcat:${category}`; +} + +export class HyperVaultKnowledgeStore implements KnowledgeStore { + constructor(private readonly client: HyperVaultClient) {} + + async list(companyId: string, filters?: KnowledgeListFilters): Promise { + const tags = [companyTag(companyId), `${TAG_PREFIX}:kind:knowledge`]; + if (filters?.status) tags.push(statusTag(filters.status)); + if (filters?.category) tags.push(categoryTag(filters.category)); + + const memories = await this.client.listMemories({ tags }); + let entries = memories + .map((m) => parseEntry(m)) + .filter((e): e is KnowledgeEntry => e !== null && e.companyId === companyId); + + if (filters?.status) entries = entries.filter((e) => e.status === filters.status); + if (filters?.category) entries = entries.filter((e) => e.category === filters.category); + if (filters?.search) { + const needle = filters.search.toLowerCase(); + entries = entries.filter( + (e) => e.title.toLowerCase().includes(needle) || e.content.toLowerCase().includes(needle), + ); + } + return entries; + } + + async get(id: string): Promise { + const memories = await this.client.listMemories({ tags: [idTag(id)] }); + const memory = memories.find((m) => m.tags.includes(idTag(id))); + return memory ? parseEntry(memory) : null; + } + + async create(entry: KnowledgeEntry): Promise { + await this.client.memorize({ + title: entry.title, + content: JSON.stringify(entry), + tags: entryTags(entry), + }); + return entry; + } + + async update(id: string, partial: Partial): Promise { + const memories = await this.client.listMemories({ tags: [idTag(id)] }); + const memory = memories.find((m) => m.tags.includes(idTag(id))); + if (!memory) return null; + const existing = parseEntry(memory); + if (!existing) return null; + + const updated: KnowledgeEntry = { + ...existing, + ...partial, + id: existing.id, + companyId: existing.companyId, + version: existing.version + 1, + updatedAt: new Date().toISOString(), + }; + await this.client.editMemory(memory.id, { + title: updated.title, + content: JSON.stringify(updated), + tags: entryTags(updated), + }); + return updated; + } + + async delete(id: string): Promise { + const memories = await this.client.listMemories({ tags: [idTag(id)] }); + const memory = memories.find((m) => m.tags.includes(idTag(id))); + if (!memory) return false; + return this.client.forgetMemory(memory.id); + } +} + +function entryTags(entry: KnowledgeEntry): string[] { + return [ + `${TAG_PREFIX}:kind:knowledge`, + companyTag(entry.companyId), + idTag(entry.id), + statusTag(entry.status), + categoryTag(entry.category), + ...(entry.tags ?? []), + ]; +} + +function parseEntry(memory: HvMemory): KnowledgeEntry | null { + try { + const parsed = JSON.parse(memory.content) as KnowledgeEntry; + if (!parsed || typeof parsed !== 'object' || typeof parsed.id !== 'string') return null; + return parsed; + } catch { + return null; + } +} diff --git a/src/hypervault/mcp-server.ts b/src/hypervault/mcp-server.ts new file mode 100644 index 0000000..e93c082 --- /dev/null +++ b/src/hypervault/mcp-server.ts @@ -0,0 +1,374 @@ +/** + * Native MCP server — `agentvault mcp serve` + * + * AgentVault's first real MCP *server* (it previously only consumed MCP via + * PolyticianMCPClient and had serverless wiki tool definitions). Implements + * the Model Context Protocol over stdio (default) or newline-delimited + * JSON-RPC — enough for `initialize`, `tools/list`, and `tools/call` — with + * no new runtime dependency. This makes the entire store→chain pipeline + * "MCP available": an agent with this server can archive its own mind. + * + * Tools exposed: + * Pipeline (net-new): hypervault_bootstrap, hypervault_pull, hypervault_push, + * hypervault_snapshot, hypervault_archive, hypervault_verify, + * hypervault_restore, hypervault_status, hypervault_recall_local + * Wiki (already defined, finally served): the 10 wiki_* tools + */ + +import * as readline from 'node:readline'; +import type { MCPToolCallResult, MCPToolDefinition } from '../orchestration/mcp-client.js'; +import { getWikiToolDefinitions, handleWikiToolCall, type WikiMCPConfig } from '../wiki/mcp-tools.js'; +import { clientFromProject } from './pipeline.js'; +import { + archiveHyperVault, + bootstrapHyperVault, + pullHyperVault, + pushHyperVault, + recallLocal, + restoreHyperVault, + snapshotHyperVault, + statusHyperVault, + verifySnapshotFile, +} from './pipeline.js'; + +const PROTOCOL_VERSION = '2024-11-05'; +const SERVER_NAME = 'agentvault'; +const SERVER_VERSION = '1.0.0'; + +// --------------------------------------------------------------------------- +// Tool definitions +// --------------------------------------------------------------------------- + +/** The pipeline (hypervault_*) tool definitions served natively. */ +export function getHyperVaultToolDefinitions(): MCPToolDefinition[] { + return [ + { + name: 'hypervault_bootstrap', + description: 'Scaffold a HyperVault-backed agent: connect, pull memories & mind, build indices, wire MCP.', + inputSchema: { + type: 'object', + properties: { + project: { type: 'string', description: 'Target project directory' }, + branch: { type: 'string', description: 'Mind branch to pull (default main)' }, + no_artifacts: { type: 'boolean', description: 'Skip artifact content' }, + no_index: { type: 'boolean', description: 'Skip building local indices' }, + soul: { type: 'string', description: 'Memory slug to use as the agent soul' }, + }, + required: ['project'], + }, + }, + { + name: 'hypervault_pull', + description: 'Incremental export from hypervault.store into the local snapshot, working tree, and indices.', + inputSchema: { + type: 'object', + properties: { + branch: { type: 'string' }, + since: { type: 'string', description: 'ISO cursor; defaults to the last sync cursor' }, + no_artifacts: { type: 'boolean' }, + no_index: { type: 'boolean' }, + }, + }, + }, + { + name: 'hypervault_push', + description: 'Push locally edited memories back up as provenance-stamped mind commits.', + inputSchema: { + type: 'object', + properties: { dry_run: { type: 'boolean', description: 'Print the diff-as-mind-commits without writing' } }, + }, + }, + { + name: 'hypervault_snapshot', + description: 'Full export → agentvault-hypervault-snapshot-v1 bundle on disk.', + inputSchema: { + type: 'object', + properties: { + output: { type: 'string', description: 'Output bundle path' }, + passphrase: { type: 'string', description: 'Encrypt entries with a passphrase-derived key' }, + include_conversations: { type: 'boolean' }, + }, + }, + }, + { + name: 'hypervault_archive', + description: 'Sovereign archive pipeline: snapshot → encrypt → canister replay → Arweave → receipts → verify.', + inputSchema: { + type: 'object', + properties: { + passphrase: { type: 'string' }, + canister_id: { type: 'string' }, + since: { type: 'string' }, + include_conversations: { type: 'boolean' }, + }, + }, + }, + { + name: 'hypervault_verify', + description: 'Verify a snapshot bundle: manifest hash, ed25519 signature, Merkle root, per-entry checksums.', + inputSchema: { + type: 'object', + properties: { + ref: { type: 'string', description: 'Snapshot bundle file path' }, + passphrase: { type: 'string' }, + }, + required: ['ref'], + }, + }, + { + name: 'hypervault_restore', + description: 'Restore a snapshot (ar:// or file) to a local project or a fresh hypervault account.', + inputSchema: { + type: 'object', + properties: { + ref: { type: 'string', description: 'ar:// or a snapshot file path' }, + to: { type: 'string', enum: ['local', 'hypervault'], description: 'Restore target' }, + passphrase: { type: 'string' }, + }, + required: ['ref'], + }, + }, + { + name: 'hypervault_status', + description: 'The three-tier picture: cloud counts, local sync/indices, canister commit, Arweave receipt.', + inputSchema: { type: 'object', properties: {} }, + }, + { + name: 'hypervault_recall_local', + description: 'Offline hybrid (lexical + semantic) recall over the local indices.', + inputSchema: { + type: 'object', + properties: { + query: { type: 'string' }, + limit: { type: 'number' }, + }, + required: ['query'], + }, + }, + ]; +} + +// --------------------------------------------------------------------------- +// Tool dispatch +// --------------------------------------------------------------------------- + +export interface McpServeOptions { + projectRoot?: string; + apiUrl?: string; + /** Wiki config; when omitted, wiki tools are still listed but return an error */ + wikiConfig?: WikiMCPConfig; +} + +function ok(text: string): MCPToolCallResult { + return { content: [{ type: 'text', text }] }; +} + +function fail(message: string): MCPToolCallResult { + return { content: [{ type: 'text', text: message }], isError: true }; +} + +/** Handle a hypervault_* tool call. */ +export async function handleHyperVaultToolCall( + toolName: string, + args: Record, + options: McpServeOptions = {}, +): Promise { + const projectRoot = options.projectRoot ?? process.cwd(); + try { + switch (toolName) { + case 'hypervault_bootstrap': { + const result = await bootstrapHyperVault({ + project: String(args.project), + branch: args.branch ? String(args.branch) : undefined, + includeArtifacts: args.no_artifacts ? false : undefined, + buildIndex: args.no_index ? false : undefined, + soulSlug: args.soul ? String(args.soul) : undefined, + cwd: projectRoot, + }); + return ok(JSON.stringify(result, null, 2)); + } + case 'hypervault_pull': { + const client = await clientFromProject({ apiUrl: options.apiUrl, projectRoot }); + const result = await pullHyperVault({ + client, + projectRoot, + branch: args.branch ? String(args.branch) : undefined, + since: args.since ? String(args.since) : undefined, + includeArtifacts: args.no_artifacts ? false : undefined, + buildIndex: args.no_index ? false : undefined, + }); + return ok(JSON.stringify(result, null, 2)); + } + case 'hypervault_push': { + const client = await clientFromProject({ apiUrl: options.apiUrl, projectRoot }); + const result = await pushHyperVault({ client, projectRoot, dryRun: Boolean(args.dry_run) }); + return ok(JSON.stringify(result, null, 2)); + } + case 'hypervault_snapshot': { + const client = await clientFromProject({ apiUrl: options.apiUrl, projectRoot }); + const result = await snapshotHyperVault({ + client, + projectRoot, + outputPath: args.output ? String(args.output) : undefined, + passphrase: args.passphrase ? String(args.passphrase) : undefined, + includeConversations: Boolean(args.include_conversations), + }); + return ok(JSON.stringify({ path: result.path, sizeBytes: result.sizeBytes, rowCounts: result.rowCounts }, null, 2)); + } + case 'hypervault_archive': { + const client = await clientFromProject({ apiUrl: options.apiUrl, projectRoot }); + const result = await archiveHyperVault({ + client, + projectRoot, + passphrase: args.passphrase ? String(args.passphrase) : undefined, + canisterId: args.canister_id ? String(args.canister_id) : undefined, + since: args.since ? String(args.since) : undefined, + includeConversations: Boolean(args.include_conversations), + }); + return ok(JSON.stringify(result, null, 2)); + } + case 'hypervault_verify': { + const result = await verifySnapshotFile(String(args.ref), { + passphrase: args.passphrase ? String(args.passphrase) : undefined, + }); + return result.valid ? ok(JSON.stringify(result, null, 2)) : fail(JSON.stringify(result, null, 2)); + } + case 'hypervault_restore': { + const to = args.to === 'hypervault' ? 'hypervault' : 'local'; + const client = to === 'hypervault' ? await clientFromProject({ apiUrl: options.apiUrl, projectRoot }) : undefined; + const result = await restoreHyperVault({ + ref: String(args.ref), + to, + passphrase: args.passphrase ? String(args.passphrase) : undefined, + projectRoot, + client, + }); + return ok(JSON.stringify(result, null, 2)); + } + case 'hypervault_status': { + let client; + try { + client = await clientFromProject({ apiUrl: options.apiUrl, projectRoot }); + } catch { + client = undefined; + } + const result = await statusHyperVault({ projectRoot, client }); + return ok(JSON.stringify(result, null, 2)); + } + case 'hypervault_recall_local': { + const results = await recallLocal(String(args.query), { + projectRoot, + limit: args.limit ? Number(args.limit) : undefined, + }); + return ok( + JSON.stringify( + results.map((r) => ({ id: r.memory.id, title: r.memory.title, score: r.score, matchedBy: r.matchedBy })), + null, + 2, + ), + ); + } + default: + return fail(`Unknown tool: ${toolName}`); + } + } catch (error) { + return fail(error instanceof Error ? error.message : String(error)); + } +} + +/** All tools this server exposes (pipeline + wiki). */ +export function getAllToolDefinitions(): MCPToolDefinition[] { + return [...getHyperVaultToolDefinitions(), ...getWikiToolDefinitions()]; +} + +/** Dispatch any tool call (pipeline or wiki). */ +export async function dispatchToolCall( + toolName: string, + args: Record, + options: McpServeOptions = {}, +): Promise { + if (toolName.startsWith('wiki_')) { + if (!options.wikiConfig) { + return fail('Wiki tools require a configured wiki (run this server from a project with a wiki).'); + } + return handleWikiToolCall(options.wikiConfig, toolName, args); + } + return handleHyperVaultToolCall(toolName, args, options); +} + +// --------------------------------------------------------------------------- +// JSON-RPC transport (stdio) +// --------------------------------------------------------------------------- + +interface JsonRpcRequest { + jsonrpc: '2.0'; + id?: string | number | null; + method: string; + params?: Record; +} + +/** + * Serve the MCP protocol over stdin/stdout (newline-delimited JSON-RPC). + * Resolves when the input stream closes. + */ +export async function serveMcp(options: McpServeOptions & { input?: NodeJS.ReadableStream; output?: NodeJS.WritableStream } = {}): Promise { + const input = options.input ?? process.stdin; + const output = options.output ?? process.stdout; + + const send = (message: Record): void => { + output.write(JSON.stringify(message) + '\n'); + }; + + const respond = (id: JsonRpcRequest['id'], result: unknown): void => { + if (id === undefined || id === null) return; // notification — no response + send({ jsonrpc: '2.0', id, result }); + }; + const respondError = (id: JsonRpcRequest['id'], code: number, message: string): void => { + if (id === undefined || id === null) return; + send({ jsonrpc: '2.0', id, error: { code, message } }); + }; + + const rl = readline.createInterface({ input, crlfDelay: Infinity }); + + for await (const line of rl) { + const trimmed = line.trim(); + if (!trimmed) continue; + let req: JsonRpcRequest; + try { + req = JSON.parse(trimmed) as JsonRpcRequest; + } catch { + respondError(null, -32700, 'Parse error'); + continue; + } + + switch (req.method) { + case 'initialize': + respond(req.id, { + protocolVersion: PROTOCOL_VERSION, + capabilities: { tools: {} }, + serverInfo: { name: SERVER_NAME, version: SERVER_VERSION }, + }); + break; + case 'notifications/initialized': + case 'initialized': + // notification, no reply + break; + case 'ping': + respond(req.id, {}); + break; + case 'tools/list': + respond(req.id, { tools: getAllToolDefinitions() }); + break; + case 'tools/call': { + const params = req.params ?? {}; + const name = String(params.name ?? ''); + const toolArgs = (params.arguments as Record) ?? {}; + const result = await dispatchToolCall(name, toolArgs, options); + respond(req.id, result); + break; + } + default: + respondError(req.id ?? null, -32601, `Method not found: ${req.method}`); + } + } +} diff --git a/src/hypervault/memory-store.ts b/src/hypervault/memory-store.ts new file mode 100644 index 0000000..83305af --- /dev/null +++ b/src/hypervault/memory-store.ts @@ -0,0 +1,158 @@ +/** + * HyperVaultMemoryStore — backbone MemoryStore backed by the cloud mind + * + * Maps `AgentMemoryEntry.{key,value,metadata}` ⇄ `HvMemory.{title,content, + * tags}`. Scoping (`companyId`/`agentId`), the memory key, scalar metadata + * and TTL expiry all travel as namespaced tags so they round-trip through + * hypervault. Every write goes through `memorize`/`editMemory`, so it lands + * as a provenance-stamped mind commit upstream. `vaultRef` records the + * cloud identity as `hypervault:`. + */ + +import type { AgentMemoryEntry } from '../backbone/types.js'; +import type { MemoryStore } from '../backbone/services/memory.js'; +import type { HyperVaultClient } from './client.js'; +import type { HvMemory } from './types.js'; + +const TAG_PREFIX = 'av'; + +function companyTag(companyId: string): string { + return `${TAG_PREFIX}:company:${companyId}`; +} +function agentTag(agentId: string): string { + return `${TAG_PREFIX}:agent:${agentId}`; +} +function keyTag(key: string): string { + return `${TAG_PREFIX}:key:${key}`; +} +function expiresTag(expiresAt: string): string { + return `${TAG_PREFIX}:expires:${expiresAt}`; +} +function metaTag(k: string, v: unknown): string { + return `${TAG_PREFIX}:meta:${k}=${JSON.stringify(v)}`; +} + +export class HyperVaultMemoryStore implements MemoryStore { + constructor(private readonly client: HyperVaultClient) {} + + async list(companyId: string, agentId: string): Promise { + const memories = await this.client.listMemories({ + tags: [companyTag(companyId), agentTag(agentId)], + }); + const now = Date.now(); + return memories + .map((m) => toEntry(m, companyId, agentId)) + .filter((entry): entry is AgentMemoryEntry => entry !== null) + .filter((entry) => !isExpired(entry, now)); + } + + async get(companyId: string, agentId: string, key: string): Promise { + const memory = await this.findByKey(companyId, agentId, key); + if (!memory) return null; + const entry = toEntry(memory, companyId, agentId); + if (!entry || isExpired(entry, Date.now())) return null; + return entry; + } + + async upsert(companyId: string, agentId: string, entry: AgentMemoryEntry): Promise { + const tags = entryTags(companyId, agentId, entry); + const existing = await this.findByKey(companyId, agentId, entry.key); + const saved = existing + ? await this.client.editMemory(existing.id, { title: entry.key, content: entry.value, tags }) + : await this.client.memorize({ title: entry.key, content: entry.value, tags }); + const id = saved?.id ?? existing?.id ?? entry.id; + return { + ...entry, + vaultRef: `hypervault:${id}`, + updatedAt: new Date().toISOString(), + }; + } + + async delete(companyId: string, agentId: string, key: string): Promise { + const memory = await this.findByKey(companyId, agentId, key); + if (!memory) return false; + return this.client.forgetMemory(memory.id); + } + + async purgeExpired(): Promise { + // Only entries written by this adapter carry expiry tags. + const memories = await this.client.listMemories({ tags: [] }); + const now = Date.now(); + let purged = 0; + for (const memory of memories) { + const expiresAt = readTag(memory.tags, `${TAG_PREFIX}:expires:`); + if (expiresAt && new Date(expiresAt).getTime() <= now) { + if (await this.client.forgetMemory(memory.id)) purged += 1; + } + } + return purged; + } + + private async findByKey(companyId: string, agentId: string, key: string): Promise { + const memories = await this.client.listMemories({ + tags: [companyTag(companyId), agentTag(agentId), keyTag(key)], + }); + return memories.find((m) => m.tags.includes(keyTag(key))) ?? null; + } +} + +// --------------------------------------------------------------------------- +// Mapping helpers +// --------------------------------------------------------------------------- + +function entryTags(companyId: string, agentId: string, entry: AgentMemoryEntry): string[] { + const tags = [companyTag(companyId), agentTag(agentId), keyTag(entry.key)]; + if (entry.expiresAt) tags.push(expiresTag(entry.expiresAt)); + for (const [k, v] of Object.entries(entry.metadata ?? {})) { + // Scalar metadata only — nested objects don't survive the tag encoding. + if (v === null || ['string', 'number', 'boolean'].includes(typeof v)) { + tags.push(metaTag(k, v)); + } + } + return tags; +} + +function toEntry(memory: HvMemory, companyId: string, agentId: string): AgentMemoryEntry | null { + const key = readTag(memory.tags, `${TAG_PREFIX}:key:`) ?? memory.title; + if (!key) return null; + if (!memory.tags.includes(companyTag(companyId)) || !memory.tags.includes(agentTag(agentId))) { + return null; + } + + const metadata: Record = {}; + for (const tag of memory.tags) { + const prefix = `${TAG_PREFIX}:meta:`; + if (!tag.startsWith(prefix)) continue; + const eq = tag.indexOf('=', prefix.length); + if (eq < 0) continue; + const k = tag.slice(prefix.length, eq); + try { + metadata[k] = JSON.parse(tag.slice(eq + 1)); + } catch { + metadata[k] = tag.slice(eq + 1); + } + } + + const now = new Date().toISOString(); + return { + id: memory.id, + companyId, + agentId, + key, + value: memory.content, + metadata: Object.keys(metadata).length > 0 ? metadata : undefined, + vaultRef: `hypervault:${memory.id}`, + expiresAt: readTag(memory.tags, `${TAG_PREFIX}:expires:`), + createdAt: memory.created_at ?? now, + updatedAt: memory.updated_at ?? now, + }; +} + +function readTag(tags: string[], prefix: string): string | undefined { + const tag = tags.find((t) => t.startsWith(prefix)); + return tag ? tag.slice(prefix.length) : undefined; +} + +function isExpired(entry: AgentMemoryEntry, nowMs: number): boolean { + return Boolean(entry.expiresAt && new Date(entry.expiresAt).getTime() <= nowMs); +} diff --git a/src/hypervault/mind-sync.ts b/src/hypervault/mind-sync.ts new file mode 100644 index 0000000..b15ee27 --- /dev/null +++ b/src/hypervault/mind-sync.ts @@ -0,0 +1,252 @@ +/** + * Mind sync — mirror the HyperVault mind DAG onto a memory_repo canister + * + * Replays hypervault `memory_commits` onto the on-chain repo commit-for- + * commit, in topological order (parents before children), idempotently: + * each on-chain commit is tagged `hv:commit:` and commits whose UUID + * is already on-chain are skipped. Mapping (§5.6): + * + * memory_branches.name → createBranch / switchBranch + * memory_commits → commit(message, diff, tags) with hypervault + * UUID, author provenance and merge-parent in tags + * memory_revisions → the commit `diff` payload (JSON revision ops) + * memory content at head → storeThoughtForm entries (content-addressed) + * + * Large blobs never go in the canister (64 MiB wasm_memory_limit): entries + * over the chunk threshold are replaced by `{content_hash}` pointers whose + * blobs live on Arweave. + */ + +import * as crypto from 'node:crypto'; +import type { _SERVICE } from '../canister/memory-repo-actor.js'; +import type { HvMemory, HvMindBranch, HvMindCommit, HvRevision } from './types.js'; + +/** Keep individual canister payloads comfortably under message limits. */ +const MAX_ONCHAIN_PAYLOAD_BYTES = 256 * 1024; + +export const HV_COMMIT_TAG_PREFIX = 'hv:commit:'; +export const HV_PARENT_TAG_PREFIX = 'hv:parent:'; +export const HV_MERGE_PARENT_TAG_PREFIX = 'hv:merge-parent:'; +export const HV_AUTHOR_TAG_PREFIX = 'hv:author:'; +export const ARCHIVE_RECEIPT_TAG_PREFIX = 'archive-receipt:'; + +export interface MindSyncInput { + commits: HvMindCommit[]; + /** memory_revisions grouped however they arrive; matched by commit_id */ + revisions: HvRevision[]; + branches: HvMindBranch[]; + /** memory heads to store as thoughtforms (content-addressed) */ + memories: HvMemory[]; +} + +export interface MindSyncResult { + commitsReplayed: number; + commitsSkipped: number; + thoughtformsStored: number; + /** hypervault UUID of the last replayed (or already-synced) commit */ + lastSyncedCommitId?: string; + errors: string[]; +} + +/** + * Order commits so every parent (and merge parent) precedes its children. + * Ties break on created_at then id for determinism. + */ +export function topologicalOrder(commits: HvMindCommit[]): HvMindCommit[] { + const byId = new Map(commits.map((c) => [c.id, c])); + const children = new Map(); + const inDegree = new Map(); + + for (const commit of commits) { + inDegree.set(commit.id, 0); + } + for (const commit of commits) { + for (const parent of [commit.parent_id, commit.merge_parent_id]) { + if (parent && byId.has(parent)) { + children.set(parent, [...(children.get(parent) ?? []), commit.id]); + inDegree.set(commit.id, (inDegree.get(commit.id) ?? 0) + 1); + } + } + } + + const compare = (a: string, b: string): number => { + const ca = byId.get(a)!; + const cb = byId.get(b)!; + return (ca.created_at ?? '').localeCompare(cb.created_at ?? '') || ca.id.localeCompare(cb.id); + }; + + const ready = [...inDegree.entries()].filter(([, d]) => d === 0).map(([id]) => id).sort(compare); + const ordered: HvMindCommit[] = []; + while (ready.length > 0) { + const id = ready.shift()!; + ordered.push(byId.get(id)!); + for (const child of children.get(id) ?? []) { + const remaining = (inDegree.get(child) ?? 1) - 1; + inDegree.set(child, remaining); + if (remaining === 0) { + ready.push(child); + ready.sort(compare); + } + } + } + + if (ordered.length !== commits.length) { + throw new Error('Mind DAG contains a cycle or dangling parent references; refusing to replay'); + } + return ordered; +} + +/** Collect hypervault commit UUIDs already replayed onto the canister. */ +export async function collectSyncedCommitIds(actor: _SERVICE): Promise> { + const synced = new Set(); + const branches = await actor.getBranches(); + const branchNames = branches.map(([name]) => name); + const targets: Array<[string] | []> = branchNames.length > 0 ? branchNames.map((n) => [n] as [string]) : [[]]; + for (const target of targets) { + const commits = await actor.log(target); + for (const commit of commits) { + for (const tag of commit.tags) { + if (tag.startsWith(HV_COMMIT_TAG_PREFIX)) { + synced.add(tag.slice(HV_COMMIT_TAG_PREFIX.length)); + } + } + } + } + return synced; +} + +/** + * Replay the mind DAG onto the canister. Idempotent and resumable. + */ +export async function syncMindToCanister(actor: _SERVICE, input: MindSyncInput): Promise { + const result: MindSyncResult = { + commitsReplayed: 0, + commitsSkipped: 0, + thoughtformsStored: 0, + errors: [], + }; + + // Ensure the repo is initialized. + const status = await actor.getRepoStatus(); + if (!status.initialized) { + const init = await actor.initRepo('HyperVault mind mirror'); + if ('err' in init) { + result.errors.push(`initRepo failed: ${init.err}`); + return result; + } + } + + const synced = await collectSyncedCommitIds(actor); + const revisionsByCommit = new Map(); + for (const revision of input.revisions) { + revisionsByCommit.set(revision.commit_id, [...(revisionsByCommit.get(revision.commit_id) ?? []), revision]); + } + + const existingBranches = new Set((await actor.getBranches()).map(([name]) => name)); + let currentBranch = (await actor.getRepoStatus()).currentBranch; + + const ensureBranch = async (name: string | undefined): Promise => { + const branch = name && name.length > 0 ? name : 'main'; + if (!existingBranches.has(branch)) { + const created = await actor.createBranch(branch); + if ('ok' in created) existingBranches.add(branch); + // "already exists" errors are fine — treat as present. + else existingBranches.add(branch); + } + if (currentBranch !== branch) { + const switched = await actor.switchBranch(branch); + if ('ok' in switched) currentBranch = branch; + } + }; + + // Replay commits in topological order. + for (const commit of topologicalOrder(input.commits)) { + if (synced.has(commit.id)) { + result.commitsSkipped += 1; + result.lastSyncedCommitId = commit.id; + continue; + } + await ensureBranch(commit.branch); + + const revisions = revisionsByCommit.get(commit.id) ?? []; + let diff = JSON.stringify(revisions.map(revisionOp)); + if (Buffer.byteLength(diff, 'utf-8') > MAX_ONCHAIN_PAYLOAD_BYTES) { + // Blob too large for the canister: store a content-hash pointer instead. + diff = JSON.stringify({ + pointer: true, + content_hash: sha256(diff), + note: 'revision payload exceeds on-chain chunk limit; blob archived off-chain', + }); + } + + const tags = [ + `${HV_COMMIT_TAG_PREFIX}${commit.id}`, + ...(commit.parent_id ? [`${HV_PARENT_TAG_PREFIX}${commit.parent_id}`] : []), + ...(commit.merge_parent_id ? [`${HV_MERGE_PARENT_TAG_PREFIX}${commit.merge_parent_id}`] : []), + ...(commit.author_kind || commit.author_key_prefix || commit.author_key_id + ? [`${HV_AUTHOR_TAG_PREFIX}${commit.author_kind ?? 'unknown'}/${commit.author_key_prefix ?? commit.author_key_id ?? ''}`] + : []), + ]; + + const committed = await actor.commit(commit.message || `hypervault commit ${commit.id}`, diff, tags); + if ('err' in committed) { + result.errors.push(`commit ${commit.id} failed: ${committed.err}`); + // Children depend on this commit's tag for idempotency — stop here so + // the next run resumes from the same point. + return result; + } + synced.add(commit.id); + result.commitsReplayed += 1; + result.lastSyncedCommitId = commit.id; + } + + // Store memory heads as content-addressed thoughtforms. + for (const memory of input.memories) { + const withoutEmbedding: Record = { ...memory }; + delete withoutEmbedding.embedding; + const json = JSON.stringify(withoutEmbedding); + if (Buffer.byteLength(json, 'utf-8') > MAX_ONCHAIN_PAYLOAD_BYTES) continue; + const hash = sha256(json); + const existing = await actor.getThoughtFormByHash(hash); + if (existing.length > 0) continue; + const stored = await actor.storeThoughtForm(json, BigInt(Date.now()), hash); + if ('ok' in stored) { + result.thoughtformsStored += 1; + } else { + result.errors.push(`storeThoughtForm ${memory.id} failed: ${stored.err}`); + } + } + + return result; +} + +/** + * Write an archive receipt commit: the chain itself attests where the cold + * copy lives (§5.6). + */ +export async function writeArchiveReceipt( + actor: _SERVICE, + arweaveTx: string, + manifestHash: string, +): Promise { + const result = await actor.commit( + `Archive receipt: ${arweaveTx}`, + JSON.stringify({ arweave_tx: arweaveTx, manifest_hash: manifestHash }), + [`${ARCHIVE_RECEIPT_TAG_PREFIX}${arweaveTx}`], + ); + return 'ok' in result; +} + +function revisionOp(revision: HvRevision): Record { + return { + id: revision.id, + memory_id: revision.memory_id, + operation: revision.operation, + snapshot: revision.snapshot, + created_at: revision.created_at, + }; +} + +function sha256(data: string): string { + return crypto.createHash('sha256').update(data, 'utf-8').digest('hex'); +} diff --git a/src/hypervault/pipeline.ts b/src/hypervault/pipeline.ts new file mode 100644 index 0000000..31f9184 --- /dev/null +++ b/src/hypervault/pipeline.ts @@ -0,0 +1,993 @@ +/** + * HyperVault pipeline — the flows behind the CLI commands and MCP tools + * + * Every flagship flow (§6) lives here as a plain async function so the + * `agentvault hypervault …` CLI, the native MCP server, and the SDK all + * share one implementation: + * + * connect → validate key, vault it, write hypervault.json (keyRef only) + * pull → incremental export → snapshot + working tree + indices + * snapshot → full export → snapshot bundle on disk + * archive → snapshot → encrypt → canister replay → Arweave → receipts + * verify → every link of the integrity chain + * restore → Arweave tx / snapshot file → local project or hypervault + * status → the whole three-tier picture + * recall → offline hybrid recall over local indices + * bootstrap → scaffold + connect + pull + MCP wiring + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { atomicWriteFileSync } from '../utils/path-validation.js'; +import { HyperVaultClient, countRows, type ExportResult } from './client.js'; +import { + defaultHypervaultState, + loadHypervaultState, + resolveHyperVaultKey, + saveHypervaultState, + storeHyperVaultKey, +} from './auth.js'; +import { + buildSnapshot, + readSnapshot, + snapshotToRecords, + verifySnapshot, + writeSnapshot, + readEmbeddings, + snapshotEmbeddingModel, + SNAPSHOT_FILE_EXTENSION, + type HypervaultSnapshot, + type SnapshotVerifyResult, +} from './snapshot.js'; +import { + buildIndices, + loadIndices, + saveIndices, + snapshotMemories, +} from './index/builder.js'; +import { hybridRecall, type QueryEmbedder } from './index/recall.js'; +import { syncMindToCanister, writeArchiveReceipt, type MindSyncResult } from './mind-sync.js'; +import { + hvMemorySchema, + hvMindBranchSchema, + hvMindCommitSchema, + hvRevisionSchema, + type HvExportRecord, + type HvMemory, + type HvRecallResult, + type HypervaultState, +} from './types.js'; +import type { _SERVICE } from '../canister/memory-repo-actor.js'; + +export const SNAPSHOT_DIR = path.join('.agentvault', 'canister'); +export const SNAPSHOT_BASENAME = `latest${SNAPSHOT_FILE_EXTENSION}`; +export const MEMORIES_DIR = path.join('.agentvault', 'memories'); + +export function snapshotPath(projectRoot: string = process.cwd()): string { + return path.join(projectRoot, SNAPSHOT_DIR, SNAPSHOT_BASENAME); +} + +// --------------------------------------------------------------------------- +// connect +// --------------------------------------------------------------------------- + +export interface ConnectOptions { + key?: string; + apiUrl?: string; + agentId?: string; + projectRoot?: string; +} + +export interface ConnectResult { + valid: boolean; + keyRef?: string; + vaulted: boolean; + userIdHint?: string; + warning?: string; +} + +export async function connectHyperVault(options: ConnectOptions = {}): Promise { + const projectRoot = options.projectRoot ?? process.cwd(); + const agentId = options.agentId ?? projectAgentId(projectRoot); + const resolved = await resolveHyperVaultKey({ flagKey: options.key, agentId }); + if (!resolved) { + throw new Error( + 'No HyperVault API key found. Pass one via HYPERVAULT_API_KEY, the secrets vault, or --key (discouraged).', + ); + } + + const client = new HyperVaultClient({ apiKey: resolved.key, apiUrl: options.apiUrl }); + const validation = await client.validateKey(); + if (!validation.valid) { + return { valid: false, vaulted: false }; + } + + // Vault the key so it never needs to be passed again; scrub any flag copy. + let keyRef = resolved.keyRef; + let vaulted = resolved.source === 'vault'; + let warning: string | undefined; + if (resolved.source !== 'vault' && agentId) { + try { + keyRef = await storeHyperVaultKey(agentId, resolved.key); + vaulted = true; + } catch { + warning = 'Secrets vault unavailable — key NOT persisted. Set HYPERVAULT_API_KEY or configure `agentvault vault`.'; + } + } + if (resolved.insecureSource) { + warning = [ + 'The --key flag leaks secrets into shell history and process lists; prefer HYPERVAULT_API_KEY or the secrets vault.', + warning, + ] + .filter(Boolean) + .join(' '); + } + + const state: HypervaultState = { + ...(loadHypervaultState(projectRoot) ?? defaultHypervaultState()), + apiUrl: client.getApiUrl(), + keyRef, + userIdHint: validation.userIdHint, + }; + ensureDir(path.join(projectRoot, '.agentvault')); + saveHypervaultState(state, projectRoot); + + return { valid: true, keyRef, vaulted, userIdHint: validation.userIdHint, warning }; +} + +/** Build an authenticated client from resolved key + saved state. */ +export async function clientFromProject(options: { + key?: string; + apiUrl?: string; + projectRoot?: string; +} = {}): Promise { + const projectRoot = options.projectRoot ?? process.cwd(); + const state = loadHypervaultState(projectRoot); + const resolved = await resolveHyperVaultKey({ + flagKey: options.key, + agentId: projectAgentId(projectRoot), + }); + if (!resolved) { + throw new Error('No HyperVault API key available. Run `agentvault hypervault connect` first.'); + } + return new HyperVaultClient({ + apiKey: resolved.key, + apiUrl: options.apiUrl ?? state?.apiUrl, + }); +} + +// --------------------------------------------------------------------------- +// pull (incremental export → snapshot + working tree + indices) +// --------------------------------------------------------------------------- + +export interface PullOptions { + client: HyperVaultClient; + projectRoot?: string; + branch?: string; + since?: string; + /** Skip artifact entries (--no-artifacts) */ + includeArtifacts?: boolean; + /** Skip index build (--no-index) */ + buildIndex?: boolean; +} + +export interface PullResult { + recordsPulled: number; + totalRecords: number; + memoriesWritten: number; + indicesBuilt: boolean; + snapshotFile: string; + cursor?: string; +} + +export async function pullHyperVault(options: PullOptions): Promise { + const projectRoot = options.projectRoot ?? process.cwd(); + const state = loadHypervaultState(projectRoot) ?? defaultHypervaultState(); + const since = options.since ?? state.lastExportCursor; + + const include = options.includeArtifacts === false + ? ['memories', 'mind', 'connections', 'embeddings'] + : undefined; + const fresh = await options.client.exportVault({ since, branch: options.branch ?? state.branch, include }); + + // Merge into the existing snapshot for incremental pulls. + let merged: HvExportRecord[] = fresh.records; + const file = snapshotPath(projectRoot); + if (since && fs.existsSync(file)) { + const existing = await snapshotToRecords(await readSnapshot(file)); + merged = mergeRecords(existing, fresh.records); + } + + const manifest = { ...fresh.manifest, row_counts: countRows(merged) }; + const snapshot = await buildSnapshot(merged, manifest); + ensureDir(path.dirname(file)); + await writeSnapshot(snapshot, file); + + const memories = await snapshotMemories(snapshot); + const memoriesWritten = writeMemoriesWorkingTree(memories, projectRoot); + + let indicesBuilt = false; + if (options.buildIndex !== false) { + const embeddings = await readEmbeddings(snapshot); + const model = await snapshotEmbeddingModel(snapshot); + saveIndices(buildIndices(memories, embeddings, model), projectRoot); + indicesBuilt = true; + } + + saveHypervaultState( + { + ...state, + branch: options.branch ?? state.branch, + lastExportCursor: fresh.manifest.cursor ?? state.lastExportCursor, + lastSync: new Date().toISOString(), + }, + projectRoot, + ); + + return { + recordsPulled: fresh.records.length, + totalRecords: merged.length, + memoriesWritten, + indicesBuilt, + snapshotFile: file, + cursor: fresh.manifest.cursor, + }; +} + +/** Merge export records, newer rows replacing older ones by (table, id). */ +export function mergeRecords(existing: HvExportRecord[], fresh: HvExportRecord[]): HvExportRecord[] { + const keyOf = (r: HvExportRecord): string => { + const id = typeof r.row.id === 'string' ? r.row.id : typeof r.row.name === 'string' ? r.row.name : JSON.stringify(r.row); + return `${r.table}${id}`; + }; + const merged = new Map(); + for (const record of existing) merged.set(keyOf(record), record); + for (const record of fresh) merged.set(keyOf(record), record); + return [...merged.values()]; +} + +/** Write the human-readable markdown working tree (`.agentvault/memories/`). */ +export function writeMemoriesWorkingTree(memories: HvMemory[], projectRoot: string = process.cwd()): number { + const dir = path.join(projectRoot, MEMORIES_DIR); + ensureDir(dir); + let written = 0; + for (const memory of memories) { + const safeName = memory.id.replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 100) || 'memory'; + const frontmatter = [ + '---', + `id: ${JSON.stringify(memory.id)}`, + `title: ${JSON.stringify(memory.title)}`, + `tags: ${JSON.stringify(memory.tags)}`, + ...(memory.branch ? [`branch: ${JSON.stringify(memory.branch)}`] : []), + ...(memory.summary ? [`summary: ${JSON.stringify(memory.summary)}`] : []), + ...(memory.updated_at ? [`updated_at: ${JSON.stringify(memory.updated_at)}`] : []), + '---', + '', + ].join('\n'); + atomicWriteFileSync(path.join(dir, `${safeName}.md`), frontmatter + memory.content + '\n', { + encoding: 'utf8', + }); + written += 1; + } + return written; +} + +// --------------------------------------------------------------------------- +// push (local working tree → cloud, via provenance-stamped writes) +// --------------------------------------------------------------------------- + +export interface PushOptions { + client: HyperVaultClient; + projectRoot?: string; + dryRun?: boolean; +} + +export interface PushChange { + id: string; + title: string; + kind: 'create' | 'update'; +} + +export interface PushResult { + changes: PushChange[]; + pushed: number; + dryRun: boolean; +} + +/** + * Push local working-tree edits up through `memorize`/`editMemory` so each + * lands as a mind commit. New files (no cloud id) become creates. + */ +export async function pushHyperVault(options: PushOptions): Promise { + const projectRoot = options.projectRoot ?? process.cwd(); + const dir = path.join(projectRoot, MEMORIES_DIR); + if (!fs.existsSync(dir)) { + return { changes: [], pushed: 0, dryRun: Boolean(options.dryRun) }; + } + + const file = snapshotPath(projectRoot); + const baseline = new Map(); + if (fs.existsSync(file)) { + for (const memory of await snapshotMemories(await readSnapshot(file))) { + baseline.set(memory.id, memory); + } + } + + const changes: PushChange[] = []; + const toPush: Array<{ change: PushChange; memory: Partial & { content: string } }> = []; + for (const entry of fs.readdirSync(dir)) { + if (!entry.endsWith('.md')) continue; + const parsed = parseMemoryMarkdown(fs.readFileSync(path.join(dir, entry), 'utf-8')); + if (!parsed) continue; + const existing = parsed.id ? baseline.get(parsed.id) : undefined; + if (existing) { + if (existing.content.trimEnd() !== parsed.content.trimEnd() || existing.title !== (parsed.title ?? existing.title)) { + const change: PushChange = { id: existing.id, title: parsed.title ?? existing.title, kind: 'update' }; + changes.push(change); + toPush.push({ change, memory: { id: existing.id, title: parsed.title, content: parsed.content, tags: parsed.tags } }); + } + } else { + const change: PushChange = { id: parsed.id ?? entry, title: parsed.title ?? entry, kind: 'create' }; + changes.push(change); + toPush.push({ change, memory: { title: parsed.title, content: parsed.content, tags: parsed.tags } }); + } + } + + if (options.dryRun) { + return { changes, pushed: 0, dryRun: true }; + } + + let pushed = 0; + for (const { change, memory } of toPush) { + if (change.kind === 'update' && memory.id) { + await options.client.editMemory(memory.id, { + title: memory.title, + content: memory.content, + tags: memory.tags, + }); + } else { + await options.client.memorize({ title: memory.title, content: memory.content, tags: memory.tags }); + } + pushed += 1; + } + return { changes, pushed, dryRun: false }; +} + +function parseMemoryMarkdown( + text: string, +): { id?: string; title?: string; tags?: string[]; content: string } | null { + if (!text.startsWith('---\n')) { + return { content: text }; + } + const end = text.indexOf('\n---\n', 4); + if (end < 0) return { content: text }; + const header = text.slice(4, end); + const content = text.slice(end + 5).replace(/^\n/, ''); + const fields: Record = {}; + for (const line of header.split('\n')) { + const colon = line.indexOf(':'); + if (colon < 0) continue; + const key = line.slice(0, colon).trim(); + try { + fields[key] = JSON.parse(line.slice(colon + 1).trim()); + } catch { + fields[key] = line.slice(colon + 1).trim(); + } + } + return { + id: typeof fields.id === 'string' ? fields.id : undefined, + title: typeof fields.title === 'string' ? fields.title : undefined, + tags: Array.isArray(fields.tags) ? fields.tags.filter((t): t is string => typeof t === 'string') : undefined, + content, + }; +} + +// --------------------------------------------------------------------------- +// snapshot (full export → bundle on disk) +// --------------------------------------------------------------------------- + +export interface SnapshotOptions { + client: HyperVaultClient; + outputPath?: string; + passphrase?: string; + includeConversations?: boolean; + branch?: string; + projectRoot?: string; + signingKeyPath?: string; +} + +export interface SnapshotResult { + snapshot: HypervaultSnapshot; + path: string; + sizeBytes: number; + rowCounts: Record; +} + +export async function snapshotHyperVault(options: SnapshotOptions): Promise { + const projectRoot = options.projectRoot ?? process.cwd(); + const include = options.includeConversations ? undefined : undefined; // server default: all + const exported: ExportResult = await options.client.exportVault({ branch: options.branch, include }); + const snapshot = await buildSnapshot(exported.records, exported.manifest, { + passphrase: options.passphrase, + includeConversations: options.includeConversations, + signingKeyPath: options.signingKeyPath, + }); + const outputPath = + options.outputPath ?? path.join(projectRoot, `hypervault-${timestampSlug()}${SNAPSHOT_FILE_EXTENSION}`); + ensureDir(path.dirname(outputPath)); + const sizeBytes = await writeSnapshot(snapshot, outputPath); + return { snapshot, path: outputPath, sizeBytes, rowCounts: snapshot.manifest.rowCounts }; +} + +// --------------------------------------------------------------------------- +// archive (§6.2) — snapshot → encrypt → canister → Arweave → receipts → verify +// --------------------------------------------------------------------------- + +export interface ArchiveOptions { + client: HyperVaultClient; + projectRoot?: string; + passphrase?: string; + /** memory_repo canister actor (omit to skip the warm tier) */ + actor?: _SERVICE; + canisterId?: string; + /** Arweave JWK (omit to skip the cold tier) */ + arweaveJwk?: Record; + /** Injected for tests */ + archiver?: ArchiverLike; + agentName?: string; + includeConversations?: boolean; + since?: string; + signingKeyPath?: string; + onStep?: (step: string, detail?: string) => void; +} + +export interface ArchiverLike { + archive( + state: Record, + jwk: Record, + ): Promise<{ success: boolean; transactionId?: string; error?: string }>; + fetchBundle(bundleId: string): Promise<{ state: string } | null>; + verifyBundle(bundle: never, expectedPublicKey?: string): Promise<{ valid: boolean; error?: string }>; +} + +export interface ArchiveResultSummary { + snapshotFile: string; + rowCounts: Record; + mindSync?: MindSyncResult; + arweaveTx?: string; + receiptOnChain: boolean; + receiptPosted: boolean; + verified: boolean; + errors: string[]; +} + +export async function archiveHyperVault(options: ArchiveOptions): Promise { + const projectRoot = options.projectRoot ?? process.cwd(); + const step = options.onStep ?? ((): void => undefined); + const errors: string[] = []; + + // 1. Export + step('export', options.since ? `incremental since ${options.since}` : 'full account'); + const exported = await options.client.exportVault({ since: options.since }); + + // 2. Bundle (+ 3. encrypt) + step('bundle', `${exported.records.length} records`); + const snapshot = await buildSnapshot(exported.records, exported.manifest, { + passphrase: options.passphrase, + includeConversations: options.includeConversations, + signingKeyPath: options.signingKeyPath, + }); + const file = path.join(projectRoot, SNAPSHOT_DIR, `archive-${timestampSlug()}${SNAPSHOT_FILE_EXTENSION}`); + ensureDir(path.dirname(file)); + await writeSnapshot(snapshot, file); + step('encrypt', options.passphrase ? 'aes-256-gcm (passphrase-wrapped)' : 'plaintext (--encrypt recommended)'); + + // 4. Canister commit — replay the mind DAG + let mindSync: MindSyncResult | undefined; + if (options.actor) { + step('canister', options.canisterId); + const records = exported.records; + mindSync = await syncMindToCanister(options.actor, { + commits: parseRows(records, 'memory_commits', hvMindCommitSchema), + revisions: parseRows(records, 'memory_revisions', hvRevisionSchema), + branches: parseRows(records, 'memory_branches', hvMindBranchSchema), + memories: parseRows(records, 'memories', hvMemorySchema), + }); + errors.push(...mindSync.errors); + } + + // 5. Arweave upload + let arweaveTx: string | undefined; + let verified = false; + if (options.arweaveJwk) { + step('arweave', 'uploading snapshot bundle'); + const archiver = options.archiver ?? (await defaultArchiver(options.agentName ?? projectAgentId(projectRoot) ?? 'agent', options.signingKeyPath)); + const upload = await archiver.archive( + { format: snapshot.format, snapshot: JSON.stringify(snapshot) }, + options.arweaveJwk, + ); + if (upload.success && upload.transactionId) { + arweaveTx = upload.transactionId; + } else { + errors.push(`Arweave upload failed: ${upload.error ?? 'unknown error'}`); + } + + // 7. Verify — re-fetch and check the whole chain + if (arweaveTx) { + step('verify', arweaveTx); + try { + const fetched = await archiver.fetchBundle(arweaveTx); + if (fetched) { + const state = JSON.parse(fetched.state) as { snapshot?: string }; + if (state.snapshot) { + const roundTripped = JSON.parse(state.snapshot) as HypervaultSnapshot; + const result = await verifySnapshot(roundTripped, { passphrase: options.passphrase }); + verified = result.signatureValid && result.checksumsValid; + if (!verified) errors.push(...result.errors); + } + } else { + errors.push('Could not re-fetch bundle from Arweave for verification'); + } + } catch (error) { + errors.push(`Verification failed: ${error instanceof Error ? error.message : String(error)}`); + } + } + } + + // 6. Receipts + let receiptOnChain = false; + let receiptPosted = false; + if (arweaveTx) { + step('receipts', arweaveTx); + if (options.actor) { + receiptOnChain = await writeArchiveReceipt(options.actor, arweaveTx, snapshot.manifest.merkleRoot); + } + try { + await options.client.postArchiveReceipt({ + kind: 'arweave', + ref: arweaveTx, + manifest_hash: snapshot.manifest.merkleRoot, + }); + receiptPosted = true; + } catch { + receiptPosted = false; + } + } + + // Update cursors + const state = loadHypervaultState(projectRoot) ?? defaultHypervaultState(); + saveHypervaultState( + { + ...state, + canisterId: options.canisterId ?? state.canisterId, + lastArweaveTx: arweaveTx ?? state.lastArweaveTx, + lastMindCommitSynced: mindSync?.lastSyncedCommitId ?? state.lastMindCommitSynced, + lastExportCursor: exported.manifest.cursor ?? state.lastExportCursor, + lastSync: new Date().toISOString(), + }, + projectRoot, + ); + + return { + snapshotFile: file, + rowCounts: snapshot.manifest.rowCounts, + mindSync, + arweaveTx, + receiptOnChain, + receiptPosted, + verified, + errors, + }; +} + +async function defaultArchiver(agentName: string, signingKeyPath?: string): Promise { + const { ArweaveArchiver } = await import('../archival/arweave-archiver.js'); + return new ArweaveArchiver({ agentName, signingKeyPath }) as unknown as ArchiverLike; +} + +function parseRows( + records: HvExportRecord[], + table: HvExportRecord['table'], + schema: { safeParse: (v: unknown) => { success: boolean; data?: T } }, +): T[] { + const rows: T[] = []; + for (const record of records) { + if (record.table !== table) continue; + const parsed = schema.safeParse(record.row); + if (parsed.success && parsed.data !== undefined) rows.push(parsed.data); + } + return rows; +} + +// --------------------------------------------------------------------------- +// verify / restore +// --------------------------------------------------------------------------- + +export async function verifySnapshotFile( + filePath: string, + options: { passphrase?: string; expectedPublicKey?: string } = {}, +): Promise { + const snapshot = await readSnapshot(filePath); + return verifySnapshot(snapshot, options); +} + +export interface RestoreOptions { + /** `ar://` or a snapshot file path */ + ref: string; + to: 'local' | 'hypervault'; + passphrase?: string; + projectRoot?: string; + /** Required for --to hypervault (a distinct explicit key — §7.4) */ + client?: HyperVaultClient; + /** Injected for tests / Arweave fetch */ + archiver?: ArchiverLike; + agentName?: string; +} + +export interface RestoreResult { + records: number; + memoriesWritten?: number; + importedToHypervault?: number; + verify: SnapshotVerifyResult; +} + +export async function restoreHyperVault(options: RestoreOptions): Promise { + const projectRoot = options.projectRoot ?? process.cwd(); + let snapshot: HypervaultSnapshot; + + if (options.ref.startsWith('ar://')) { + const txId = options.ref.slice('ar://'.length); + const archiver = options.archiver ?? (await defaultArchiver(options.agentName ?? 'agent')); + const bundle = await archiver.fetchBundle(txId); + if (!bundle) { + throw new Error(`Could not fetch Arweave bundle ${txId}`); + } + const state = JSON.parse(bundle.state) as { snapshot?: string }; + if (!state.snapshot) { + throw new Error('Arweave bundle does not contain a hypervault snapshot'); + } + snapshot = JSON.parse(state.snapshot) as HypervaultSnapshot; + } else { + snapshot = await readSnapshot(options.ref); + } + + const verify = await verifySnapshot(snapshot, { passphrase: options.passphrase }); + if (!verify.signatureValid || !verify.checksumsValid) { + throw new Error(`Snapshot failed verification: ${verify.errors.join('; ') || 'unknown error'}`); + } + + const records = await snapshotToRecords(snapshot, options.passphrase); + + if (options.to === 'hypervault') { + if (!options.client) { + throw new Error('Restoring to hypervault requires an explicit API key for the destination account'); + } + const result = await options.client.importVault(records); + return { records: records.length, importedToHypervault: result.imported, verify }; + } + + // --to local: rebuild snapshot copy, working tree, and indices + const file = snapshotPath(projectRoot); + ensureDir(path.dirname(file)); + await writeSnapshot(snapshot, file); + const memories = await snapshotMemories(snapshot, options.passphrase); + const memoriesWritten = writeMemoriesWorkingTree(memories, projectRoot); + const embeddings = await readEmbeddings(snapshot, options.passphrase); + const model = await snapshotEmbeddingModel(snapshot, options.passphrase); + saveIndices(buildIndices(memories, embeddings, model), projectRoot); + + const state = loadHypervaultState(projectRoot) ?? defaultHypervaultState(); + saveHypervaultState({ ...state, lastSync: new Date().toISOString() }, projectRoot); + + return { records: records.length, memoriesWritten, verify }; +} + +// --------------------------------------------------------------------------- +// status — the whole three-tier picture +// --------------------------------------------------------------------------- + +export interface StatusOptions { + projectRoot?: string; + client?: HyperVaultClient; + actor?: _SERVICE; +} + +export interface StatusResult { + configured: boolean; + apiUrl?: string; + keyRef?: string; + keyValid?: boolean; + cloud?: { memories: number; artifacts: number; branches: number }; + local: { + snapshotPresent: boolean; + memoriesInWorkingTree: number; + ftsIndexed: number; + vectorsIndexed: number; + lastSync?: string; + lastExportCursor?: string; + }; + canister?: { id: string; currentBranch?: string; totalCommits?: string; lastMindCommitSynced?: string }; + arweave?: { lastTx?: string }; +} + +export async function statusHyperVault(options: StatusOptions = {}): Promise { + const projectRoot = options.projectRoot ?? process.cwd(); + const state = loadHypervaultState(projectRoot); + + const local = { + snapshotPresent: fs.existsSync(snapshotPath(projectRoot)), + memoriesInWorkingTree: countMarkdown(path.join(projectRoot, MEMORIES_DIR)), + ftsIndexed: 0, + vectorsIndexed: 0, + lastSync: state?.lastSync, + lastExportCursor: state?.lastExportCursor, + }; + const indices = loadIndices(projectRoot); + local.ftsIndexed = indices.fts?.size ?? 0; + local.vectorsIndexed = indices.vectors?.size ?? 0; + + const result: StatusResult = { + configured: state !== null, + apiUrl: state?.apiUrl, + keyRef: state?.keyRef, + local, + arweave: state?.lastArweaveTx ? { lastTx: state.lastArweaveTx } : undefined, + }; + + if (options.client) { + try { + const validation = await options.client.validateKey(); + result.keyValid = validation.valid; + if (validation.valid) { + const [memories, artifacts, branches] = await Promise.all([ + options.client.listMemories(), + options.client.listArtifacts(), + options.client.mindBranches(), + ]); + result.cloud = { memories: memories.length, artifacts: artifacts.length, branches: branches.length }; + } + } catch { + result.keyValid = false; + } + } + + if (state?.canisterId) { + result.canister = { id: state.canisterId, lastMindCommitSynced: state.lastMindCommitSynced }; + if (options.actor) { + try { + const repo = await options.actor.getRepoStatus(); + result.canister.currentBranch = repo.currentBranch; + result.canister.totalCommits = repo.totalCommits.toString(); + } catch { + // canister unreachable — leave the cached values + } + } + } + + return result; +} + +// --------------------------------------------------------------------------- +// recall — offline hybrid recall over local indices +// --------------------------------------------------------------------------- + +export interface RecallLocalOptions { + projectRoot?: string; + limit?: number; + embedQuery?: QueryEmbedder; + passphrase?: string; +} + +export async function recallLocal(query: string, options: RecallLocalOptions = {}): Promise { + const projectRoot = options.projectRoot ?? process.cwd(); + const indices = loadIndices(projectRoot); + if (!indices.fts && !indices.vectors) { + throw new Error('No local indices found. Run `agentvault hypervault pull` or `reindex` first.'); + } + const file = snapshotPath(projectRoot); + if (!fs.existsSync(file)) { + throw new Error('No local snapshot found. Run `agentvault hypervault pull` first.'); + } + const memories = await snapshotMemories(await readSnapshot(file), options.passphrase); + const memoriesById = new Map(memories.map((m) => [m.id, m])); + return hybridRecall(query, indices, memoriesById, { + limit: options.limit, + embedQuery: options.embedQuery, + }); +} + +/** Rebuild local indices from the pulled snapshot. */ +export async function reindexHyperVault(options: { projectRoot?: string; passphrase?: string } = {}): Promise<{ + memoriesIndexed: number; + vectorsIndexed: number; +}> { + const projectRoot = options.projectRoot ?? process.cwd(); + const file = snapshotPath(projectRoot); + if (!fs.existsSync(file)) { + throw new Error('No local snapshot found. Run `agentvault hypervault pull` first.'); + } + const snapshot = await readSnapshot(file); + const memories = await snapshotMemories(snapshot, options.passphrase); + const embeddings = await readEmbeddings(snapshot, options.passphrase); + const model = await snapshotEmbeddingModel(snapshot, options.passphrase); + const built = buildIndices(memories, embeddings, model); + saveIndices(built, projectRoot); + return { memoriesIndexed: built.memoriesIndexed, vectorsIndexed: built.vectorsIndexed }; +} + +// --------------------------------------------------------------------------- +// bootstrap (§6.1) — scaffold + connect + pull + indices + MCP wiring +// --------------------------------------------------------------------------- + +export interface BootstrapOptions { + project: string; + key?: string; + apiUrl?: string; + branch?: string; + includeArtifacts?: boolean; + buildIndex?: boolean; + soulSlug?: string; + cwd?: string; + onStep?: (step: string, detail?: string) => void; +} + +export interface BootstrapResult { + projectPath: string; + connected: boolean; + pull?: PullResult; + mcpConfigPath: string; + soulDetected: boolean; + warning?: string; +} + +export async function bootstrapHyperVault(options: BootstrapOptions): Promise { + const step = options.onStep ?? ((): void => undefined); + const cwd = options.cwd ?? process.cwd(); + const projectPath = path.resolve(cwd, options.project); + const name = path.basename(projectPath); + + // 1. Scaffold + step('scaffold', projectPath); + ensureDir(projectPath); + const agentJsonPath = path.join(projectPath, 'agent.json'); + if (!fs.existsSync(agentJsonPath)) { + atomicWriteFileSync( + agentJsonPath, + JSON.stringify( + { + name, + version: '1.0.0', + description: `HyperVault-backed agent ${name}`, + type: 'generic', + entryPoint: 'index.js', + hypervault: { apiUrl: options.apiUrl ?? defaultHypervaultState().apiUrl, branch: options.branch ?? 'main' }, + }, + null, + 2, + ) + '\n', + { encoding: 'utf8' }, + ); + } + const indexJsPath = path.join(projectPath, 'index.js'); + if (!fs.existsSync(indexJsPath)) { + atomicWriteFileSync( + indexJsPath, + `// ${name} — HyperVault-backed AgentVault agent\nexport async function handleTask(task) {\n return { status: 'ok', task };\n}\n`, + { encoding: 'utf8' }, + ); + } + ensureDir(path.join(projectPath, '.agentvault')); + + // 2. Connect + step('connect'); + const connect = await connectHyperVault({ + key: options.key, + apiUrl: options.apiUrl, + agentId: name, + projectRoot: projectPath, + }); + if (!connect.valid) { + return { + projectPath, + connected: false, + mcpConfigPath: '', + soulDetected: false, + warning: 'HyperVault key was rejected — create one at your hypervault.store dashboard and re-run connect.', + }; + } + + // 3+4. Pull memories & mind, build indices + step('pull'); + const client = await clientFromProject({ key: options.key, apiUrl: options.apiUrl, projectRoot: projectPath }); + const pull = await pullHyperVault({ + client, + projectRoot: projectPath, + branch: options.branch, + includeArtifacts: options.includeArtifacts, + buildIndex: options.buildIndex, + }); + + // 5. Wire MCP — register both the native server and the upstream Python one + step('mcp'); + const mcpConfigPath = path.join(projectPath, '.mcp.json'); + const existing = fs.existsSync(mcpConfigPath) + ? (JSON.parse(fs.readFileSync(mcpConfigPath, 'utf-8')) as { mcpServers?: Record }) + : {}; + const mcpConfig = { + ...existing, + mcpServers: { + ...(existing.mcpServers ?? {}), + agentvault: { + command: 'npx', + args: ['-y', 'agentvault@latest', 'mcp', 'serve'], + }, + 'hypervault-mcp': { + command: 'hypervault-mcp', + args: [], + }, + }, + }; + atomicWriteFileSync(mcpConfigPath, JSON.stringify(mcpConfig, null, 2) + '\n', { encoding: 'utf8' }); + + // 6. Soul detection + step('soul'); + let soulDetected = false; + try { + const soulMemories = options.soulSlug + ? [await client.getMemory(options.soulSlug)].filter((m): m is HvMemory => m !== null) + : (await client.listMemories({ tags: ['soul'] })).filter((m) => m.tags.includes('soul')); + const soul = soulMemories[0]; + if (soul) { + atomicWriteFileSync(path.join(projectPath, 'soul.md'), soul.content + '\n', { encoding: 'utf8' }); + atomicWriteFileSync( + path.join(projectPath, '.agentvault', 'memory-repo.config.json'), + JSON.stringify({ soulDetected: true, soulFile: 'soul.md', detectedAt: Date.now() }, null, 2) + '\n', + { encoding: 'utf8' }, + ); + soulDetected = true; + } + } catch { + soulDetected = false; + } + + return { projectPath, connected: true, pull, mcpConfigPath, soulDetected, warning: connect.warning }; +} + +// --------------------------------------------------------------------------- +// shared helpers +// --------------------------------------------------------------------------- + +export function projectAgentId(projectRoot: string): string | undefined { + const agentJson = path.join(projectRoot, 'agent.json'); + if (fs.existsSync(agentJson)) { + try { + const parsed = JSON.parse(fs.readFileSync(agentJson, 'utf-8')) as { name?: string }; + if (parsed.name) return parsed.name; + } catch { + // fall through + } + } + const configPath = path.join(projectRoot, '.agentvault', 'config', 'agent.config.json'); + if (fs.existsSync(configPath)) { + try { + const parsed = JSON.parse(fs.readFileSync(configPath, 'utf-8')) as { name?: string }; + if (parsed.name) return parsed.name; + } catch { + // fall through + } + } + return undefined; +} + +function countMarkdown(dir: string): number { + if (!fs.existsSync(dir)) return 0; + return fs.readdirSync(dir).filter((f) => f.endsWith('.md')).length; +} + +function ensureDir(dir: string): void { + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); +} + +function timestampSlug(): string { + return new Date().toISOString().replace(/[:.]/g, '-'); +} diff --git a/src/hypervault/snapshot.ts b/src/hypervault/snapshot.ts new file mode 100644 index 0000000..81b6660 --- /dev/null +++ b/src/hypervault/snapshot.ts @@ -0,0 +1,633 @@ +/** + * HyperVault snapshot bundle — `agentvault-hypervault-snapshot-v1` + * + * A hypervault snapshot IS a thoughtform-bundle-style envelope (same + * `entries: path -> base64` layout, gzip(JSON) wire format, per-entry + * SHA-256 checksums) whose entries carry a full hypervault account export: + * + * memories.ndjson — live memories (all branches' heads) + * mind/commits.ndjson — memory_commits (full DAG incl. merge parents) + * mind/revisions.ndjson — memory_revisions (full history) + * mind/branches.json — memory_branches + * mind/links.ndjson — memory_links + memory_link_changes + * artifacts/.html — artifact content, content-addressed + * artifacts/index.ndjson — artifact metadata + * connections.ndjson — artifact graph edges + memory_artifact_links + * embeddings.bin — packed float32 vectors (+ embeddings.idx.json) + * conversations.ndjson — optional (--include conversations) + * + * Integrity chain: per-entry SHA-256 → Merkle root (src/backup/merkle.ts) + * → ed25519 manifest signature (~/.agentvault/arweave-signing.key), the + * same semantics `ArweaveArchiver.verifyBundle` uses. + * + * Encryption uses `CanisterEncryption` (AES-256-GCM with validated auth + * tags — audit-approved; explicitly NOT the legacy vetkeys.decryptJSON + * path, see audit finding C-1). Keys are derived from a passphrase with + * PBKDF2 (210k iterations). Private artifacts are always encrypted even + * when `encrypt` is not requested (§7.5). + */ + +import * as crypto from 'node:crypto'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as zlib from 'node:zlib'; +import { promisify } from 'node:util'; +import { computeMerkleRoot, type MerkleEntry } from '../backup/merkle.js'; +import { loadOrCreateSigningKey } from '../backup/backup.js'; +import { createCanisterEncryption, type CanisterEncryptedData } from '../canister/encryption.js'; +import { atomicWriteFileSync } from '../utils/path-validation.js'; +import { + hvArtifactSchema, + hvMemorySchema, + type HvExportManifest, + type HvExportRecord, +} from './types.js'; + +const gzip = promisify(zlib.gzip); +const gunzip = promisify(zlib.gunzip); + +export const HYPERVAULT_SNAPSHOT_FORMAT = 'agentvault-hypervault-snapshot-v1'; +export const SNAPSHOT_FILE_EXTENSION = '.hypervault-snapshot.json.gz'; + +const PBKDF2_ITERATIONS = 210_000; + +export interface SnapshotEncryptionInfo { + /** Key wrapping mode: passphrase (PBKDF2) or vetkeys (canister-derived) */ + mode: 'passphrase' | 'vetkeys'; + algorithm: 'aes-256-gcm'; + salt: string; + iterations: number; +} + +export interface HypervaultSnapshotManifest { + version: '1.0'; + format: typeof HYPERVAULT_SNAPSHOT_FORMAT; + createdAt: string; + schemaVersion: number; + cursor?: string; + branchHeads?: Record; + rowCounts: Record; + /** sha256 (hex) of each entry's plaintext bytes */ + checksums: Record; + /** Merkle root over all plaintext entries */ + merkleRoot: string; + /** Entries stored encrypted (JSON-wrapped AES-256-GCM payloads) */ + encryptedEntries: string[]; + encryption?: SnapshotEncryptionInfo; + /** Rebuild artifacts intentionally outside the integrity surface (§5.5) */ + derived: string[]; +} + +export interface HypervaultSnapshot { + format: typeof HYPERVAULT_SNAPSHOT_FORMAT; + createdAt: string; + manifest: HypervaultSnapshotManifest; + /** ed25519 signature (hex) over canonical manifest bytes */ + signature: string; + /** ed25519 public key (hex) */ + publicKey: string; + /** entry path -> base64 content (ciphertext JSON for encrypted entries) */ + entries: Record; +} + +export interface BuildSnapshotOptions { + /** Encrypt all entries with a passphrase-derived AES-256-GCM key */ + passphrase?: string; + /** Include conversations/messages tables (default false — §10.7) */ + includeConversations?: boolean; + /** Override signing key path (default ~/.agentvault/arweave-signing.key) */ + signingKeyPath?: string; +} + +export interface SnapshotVerifyResult { + valid: boolean; + checksumsValid: boolean; + merkleRootValid: boolean; + signatureValid: boolean; + errors: string[]; +} + +// --------------------------------------------------------------------------- +// Build +// --------------------------------------------------------------------------- + +interface EntryPayload { + path: string; + content: Buffer; + /** force encryption even without a snapshot-level passphrase */ + sensitive: boolean; +} + +/** + * Build a snapshot bundle from export records + export manifest. + */ +export async function buildSnapshot( + records: HvExportRecord[], + exportManifest: HvExportManifest, + options: BuildSnapshotOptions = {}, +): Promise { + const byTable = new Map>>(); + for (const record of records) { + const rows = byTable.get(record.table) ?? []; + rows.push(record.row); + byTable.set(record.table, rows); + } + + const payloads: EntryPayload[] = []; + const derived: string[] = []; + + const ndjson = (rows: Array>): Buffer => + Buffer.from(rows.map((r) => JSON.stringify(r)).join('\n') + (rows.length ? '\n' : ''), 'utf-8'); + + // Memories (embeddings split out into the packed sidecar) + const memories = byTable.get('memories') ?? []; + const embeddingRows: Array<{ id: string; embedding: number[]; model?: string }> = []; + const memoriesForEntry = memories.map((row) => { + const parsed = hvMemorySchema.safeParse(row); + if (parsed.success && parsed.data.embedding && parsed.data.embedding.length > 0) { + embeddingRows.push({ + id: parsed.data.id, + embedding: parsed.data.embedding, + model: parsed.data.embedding_model, + }); + return omitKeys(row, ['embedding']); + } + return row; + }); + payloads.push({ path: 'memories.ndjson', content: ndjson(memoriesForEntry), sensitive: false }); + + // Mind DAG + payloads.push({ path: 'mind/commits.ndjson', content: ndjson(byTable.get('memory_commits') ?? []), sensitive: false }); + payloads.push({ path: 'mind/revisions.ndjson', content: ndjson(byTable.get('memory_revisions') ?? []), sensitive: false }); + payloads.push({ + path: 'mind/branches.json', + content: Buffer.from(JSON.stringify(byTable.get('memory_branches') ?? [], null, 2) + '\n', 'utf-8'), + sensitive: false, + }); + payloads.push({ + path: 'mind/links.ndjson', + content: ndjson([...(byTable.get('memory_links') ?? []), ...(byTable.get('memory_link_changes') ?? [])]), + sensitive: false, + }); + + // Artifacts — content-addressed, private artifacts always encrypted (§7.5) + const artifactRows = byTable.get('artifacts') ?? []; + const artifactIndex: Array> = []; + for (const row of artifactRows) { + const parsed = hvArtifactSchema.safeParse(row); + if (!parsed.success) { + artifactIndex.push(row); + continue; + } + const artifact = parsed.data; + const contentHash = + artifact.content_hash ?? crypto.createHash('sha256').update(artifact.content, 'utf-8').digest('hex'); + const entryPath = `artifacts/${sanitizeHash(contentHash)}.html`; + payloads.push({ + path: entryPath, + content: Buffer.from(artifact.content, 'utf-8'), + sensitive: artifact.visibility === 'private', + }); + artifactIndex.push({ ...omitKeys(row, ['content']), content_hash: contentHash, entry: entryPath }); + } + payloads.push({ path: 'artifacts/index.ndjson', content: ndjson(artifactIndex), sensitive: false }); + + // Connections + memory<->artifact links + payloads.push({ + path: 'connections.ndjson', + content: ndjson([...(byTable.get('connections') ?? []), ...(byTable.get('memory_artifact_links') ?? [])]), + sensitive: false, + }); + + // Embeddings — packed float32 + id/model sidecar + if (embeddingRows.length > 0) { + const dims = embeddingRows[0]?.embedding.length ?? 0; + const packed = Buffer.alloc(embeddingRows.length * dims * 4); + embeddingRows.forEach((row, i) => { + row.embedding.forEach((v, j) => packed.writeFloatLE(v, (i * dims + j) * 4)); + }); + payloads.push({ path: 'embeddings.bin', content: packed, sensitive: false }); + payloads.push({ + path: 'embeddings.idx.json', + content: Buffer.from( + JSON.stringify( + { + dims, + model: embeddingRows[0]?.model, + ids: embeddingRows.map((r) => r.id), + }, + null, + 2, + ) + '\n', + 'utf-8', + ), + sensitive: false, + }); + } + + // Conversations — opt-in and always sensitive (§10.7) + if (options.includeConversations) { + const conversations = [...(byTable.get('conversations') ?? []), ...(byTable.get('messages') ?? [])]; + payloads.push({ path: 'conversations.ndjson', content: ndjson(conversations), sensitive: true }); + } + + // Integrity surface over PLAINTEXT bytes + const merkleEntries: MerkleEntry[] = payloads.map((p) => ({ path: p.path, content: p.content })); + const checksums: Record = {}; + for (const p of payloads) { + checksums[p.path] = sha256(p.content); + } + const merkleRoot = computeMerkleRoot(merkleEntries); + + // Encryption + const entries: Record = {}; + const encryptedEntries: string[] = []; + let encryption: SnapshotEncryptionInfo | undefined; + const mustEncrypt = payloads.some((p) => p.sensitive); + let key: Buffer | undefined; + + if (options.passphrase || mustEncrypt) { + if (!options.passphrase) { + throw new Error( + 'Snapshot contains private artifacts or conversations; a passphrase is required (they are always encrypted)', + ); + } + const salt = crypto.randomBytes(32); + key = crypto.pbkdf2Sync(options.passphrase, salt, PBKDF2_ITERATIONS, 32, 'sha256'); + encryption = { + mode: 'passphrase', + algorithm: 'aes-256-gcm', + salt: salt.toString('hex'), + iterations: PBKDF2_ITERATIONS, + }; + } + + const encryptAll = Boolean(options.passphrase); + const cipher = createCanisterEncryption({ algorithm: 'aes-256-gcm' }); + for (const p of payloads) { + if (key && (encryptAll || p.sensitive)) { + const result = await cipher.encrypt(p.content.toString('base64'), key); + if (!result.success || !result.encrypted) { + throw new Error(`Failed to encrypt snapshot entry ${p.path}: ${result.error ?? 'unknown error'}`); + } + entries[p.path] = Buffer.from(JSON.stringify(serializeEncrypted(result.encrypted)), 'utf-8').toString('base64'); + encryptedEntries.push(p.path); + } else { + entries[p.path] = p.content.toString('base64'); + } + } + + const manifest: HypervaultSnapshotManifest = { + version: '1.0', + format: HYPERVAULT_SNAPSHOT_FORMAT, + createdAt: new Date().toISOString(), + schemaVersion: exportManifest.schema_version, + cursor: exportManifest.cursor, + branchHeads: exportManifest.branch_heads, + rowCounts: exportManifest.row_counts, + checksums, + merkleRoot, + encryptedEntries, + encryption, + derived, + }; + + // Sign the canonical manifest with the archival ed25519 key + const { privateKey, publicKey } = await loadOrCreateSigningKey( + options.signingKeyPath ?? defaultSigningKeyPath(), + ); + const { ed25519 } = await import('@noble/curves/ed25519'); + const signature = Buffer.from(ed25519.sign(canonicalManifestBytes(manifest), privateKey)).toString('hex'); + + return { + format: HYPERVAULT_SNAPSHOT_FORMAT, + createdAt: manifest.createdAt, + manifest, + signature, + publicKey: publicKey.toString('hex'), + entries, + }; +} + +// --------------------------------------------------------------------------- +// Serialize / deserialize +// --------------------------------------------------------------------------- + +export async function writeSnapshot(snapshot: HypervaultSnapshot, filePath: string): Promise { + const compressed = await gzip(Buffer.from(JSON.stringify(snapshot), 'utf-8')); + atomicWriteFileSync(filePath, compressed, { mode: 0o600 }); + return fs.statSync(filePath).size; +} + +export async function readSnapshot(filePath: string): Promise { + const raw = fs.readFileSync(filePath); + const json = await gunzip(raw); + const data = JSON.parse(json.toString('utf-8')) as unknown; + validateSnapshot(data); + return data; +} + +export function validateSnapshot(data: unknown): asserts data is HypervaultSnapshot { + if (!data || typeof data !== 'object') { + throw new Error('Snapshot is not an object'); + } + const bundle = data as Partial; + if (bundle.format !== HYPERVAULT_SNAPSHOT_FORMAT) { + throw new Error(`Unknown snapshot format: ${String(bundle.format)} (expected ${HYPERVAULT_SNAPSHOT_FORMAT})`); + } + if (!bundle.manifest || typeof bundle.manifest !== 'object') { + throw new Error('Snapshot is missing its manifest'); + } + if (!bundle.entries || typeof bundle.entries !== 'object') { + throw new Error('Snapshot is missing its entries'); + } + if (typeof bundle.signature !== 'string' || typeof bundle.publicKey !== 'string') { + throw new Error('Snapshot is missing its manifest signature'); + } +} + +// --------------------------------------------------------------------------- +// Verify — checks every link of the integrity chain (§7.3) +// --------------------------------------------------------------------------- + +export async function verifySnapshot( + snapshot: HypervaultSnapshot, + options: { passphrase?: string; expectedPublicKey?: string } = {}, +): Promise { + const errors: string[] = []; + + // 1. ed25519 signature over the canonical manifest + let signatureValid = false; + try { + const { ed25519 } = await import('@noble/curves/ed25519'); + signatureValid = ed25519.verify( + Buffer.from(snapshot.signature, 'hex'), + canonicalManifestBytes(snapshot.manifest), + Buffer.from(snapshot.publicKey, 'hex'), + ); + if (!signatureValid) errors.push('Manifest signature verification failed'); + if (options.expectedPublicKey && options.expectedPublicKey !== snapshot.publicKey) { + signatureValid = false; + errors.push('Snapshot public key does not match the expected key'); + } + } catch (error) { + errors.push(`Signature check error: ${error instanceof Error ? error.message : String(error)}`); + } + + // 2. Per-entry checksums + 3. Merkle root — need plaintext, so decrypt + // encrypted entries when a passphrase is available. + let checksumsValid = true; + let merkleRootValid = false; + const plaintexts: MerkleEntry[] = []; + const encrypted = new Set(snapshot.manifest.encryptedEntries); + const canDecrypt = Boolean(options.passphrase && snapshot.manifest.encryption); + + for (const [entryPath, b64] of Object.entries(snapshot.entries)) { + let plaintext: Buffer | null = null; + if (encrypted.has(entryPath)) { + if (!canDecrypt) continue; // cannot check without the passphrase + try { + plaintext = await decryptEntry(snapshot, entryPath, options.passphrase!); + } catch (error) { + checksumsValid = false; + errors.push(`Failed to decrypt entry ${entryPath}: ${error instanceof Error ? error.message : String(error)}`); + continue; + } + } else { + plaintext = Buffer.from(b64, 'base64'); + } + const expected = snapshot.manifest.checksums[entryPath]; + if (!expected) { + checksumsValid = false; + errors.push(`Entry ${entryPath} is not listed in the manifest checksums`); + continue; + } + if (sha256(plaintext) !== expected) { + checksumsValid = false; + errors.push(`Checksum mismatch for entry ${entryPath}`); + } + plaintexts.push({ path: entryPath, content: plaintext }); + } + + for (const entryPath of Object.keys(snapshot.manifest.checksums)) { + if (!(entryPath in snapshot.entries)) { + checksumsValid = false; + errors.push(`Entry ${entryPath} referenced in manifest is missing from the snapshot`); + } + } + + const allPlaintextAvailable = plaintexts.length === Object.keys(snapshot.manifest.checksums).length; + if (allPlaintextAvailable) { + merkleRootValid = computeMerkleRoot(plaintexts) === snapshot.manifest.merkleRoot; + if (!merkleRootValid) errors.push('Merkle root mismatch'); + } else if (encrypted.size > 0 && !canDecrypt) { + // Signature still binds the Merkle root; report it as unverifiable. + merkleRootValid = checksumsValid; + errors.push('Encrypted entries present and no passphrase supplied; Merkle root only partially verified'); + } + + return { + valid: signatureValid && checksumsValid && merkleRootValid && errors.length === 0, + checksumsValid, + merkleRootValid, + signatureValid, + errors, + }; +} + +// --------------------------------------------------------------------------- +// Entry access & restore +// --------------------------------------------------------------------------- + +/** Read one entry's plaintext (decrypting when needed). */ +export async function readSnapshotEntry( + snapshot: HypervaultSnapshot, + entryPath: string, + passphrase?: string, +): Promise { + const b64 = snapshot.entries[entryPath]; + if (b64 === undefined) { + throw new Error(`Snapshot has no entry ${entryPath}`); + } + if (snapshot.manifest.encryptedEntries.includes(entryPath)) { + if (!passphrase) { + throw new Error(`Entry ${entryPath} is encrypted; a passphrase is required`); + } + return decryptEntry(snapshot, entryPath, passphrase); + } + return Buffer.from(b64, 'base64'); +} + +/** + * Convert a snapshot back into export records — the reverse of + * `buildSnapshot` — for `importVault` (chain → cloud resurrection) or a + * local working-tree rebuild. + */ +export async function snapshotToRecords( + snapshot: HypervaultSnapshot, + passphrase?: string, +): Promise { + const records: HvExportRecord[] = []; + + const readNdjson = async (entryPath: string): Promise>> => { + if (!(entryPath in snapshot.entries)) return []; + const text = (await readSnapshotEntry(snapshot, entryPath, passphrase)).toString('utf-8'); + return text + .split('\n') + .filter((l) => l.trim().length > 0) + .map((l) => JSON.parse(l) as Record); + }; + + // Memories, re-joined with embeddings + const embeddingsById = await readEmbeddings(snapshot, passphrase); + for (const row of await readNdjson('memories.ndjson')) { + const id = typeof row.id === 'string' ? row.id : undefined; + const embedding = id ? embeddingsById.get(id) : undefined; + records.push({ table: 'memories', row: embedding ? { ...row, embedding } : row }); + } + + for (const row of await readNdjson('mind/commits.ndjson')) records.push({ table: 'memory_commits', row }); + for (const row of await readNdjson('mind/revisions.ndjson')) records.push({ table: 'memory_revisions', row }); + for (const row of await readNdjson('mind/links.ndjson')) records.push({ table: 'memory_links', row }); + if ('mind/branches.json' in snapshot.entries) { + const branches = JSON.parse( + (await readSnapshotEntry(snapshot, 'mind/branches.json', passphrase)).toString('utf-8'), + ) as Array>; + for (const row of branches) records.push({ table: 'memory_branches', row }); + } + + // Artifacts, re-joined with their content entries + for (const row of await readNdjson('artifacts/index.ndjson')) { + const entry = typeof row.entry === 'string' ? row.entry : undefined; + let content: string | undefined; + if (entry && entry in snapshot.entries) { + content = (await readSnapshotEntry(snapshot, entry, passphrase)).toString('utf-8'); + } + const rest = omitKeys(row, ['entry']); + records.push({ table: 'artifacts', row: content !== undefined ? { ...rest, content } : rest }); + } + + for (const row of await readNdjson('connections.ndjson')) records.push({ table: 'connections', row }); + for (const row of await readNdjson('conversations.ndjson')) records.push({ table: 'conversations', row }); + + return records; +} + +/** Read the packed embeddings sidecar back into per-memory vectors. */ +export async function readEmbeddings( + snapshot: HypervaultSnapshot, + passphrase?: string, +): Promise> { + const result = new Map(); + if (!('embeddings.bin' in snapshot.entries) || !('embeddings.idx.json' in snapshot.entries)) { + return result; + } + const idx = JSON.parse( + (await readSnapshotEntry(snapshot, 'embeddings.idx.json', passphrase)).toString('utf-8'), + ) as { dims: number; ids: string[]; model?: string }; + const packed = await readSnapshotEntry(snapshot, 'embeddings.bin', passphrase); + idx.ids.forEach((id, i) => { + const vector: number[] = []; + for (let j = 0; j < idx.dims; j++) { + vector.push(packed.readFloatLE((i * idx.dims + j) * 4)); + } + result.set(id, vector); + }); + return result; +} + +/** The embedding model recorded in the snapshot sidecar, if any. */ +export async function snapshotEmbeddingModel( + snapshot: HypervaultSnapshot, + passphrase?: string, +): Promise { + if (!('embeddings.idx.json' in snapshot.entries)) return undefined; + const idx = JSON.parse( + (await readSnapshotEntry(snapshot, 'embeddings.idx.json', passphrase)).toString('utf-8'), + ) as { model?: string }; + return idx.model; +} + +// --------------------------------------------------------------------------- +// Internals +// --------------------------------------------------------------------------- + +async function decryptEntry( + snapshot: HypervaultSnapshot, + entryPath: string, + passphrase: string, +): Promise { + const info = snapshot.manifest.encryption; + if (!info) { + throw new Error('Snapshot manifest is missing encryption parameters'); + } + const key = crypto.pbkdf2Sync(passphrase, Buffer.from(info.salt, 'hex'), info.iterations, 32, 'sha256'); + const wrapped = JSON.parse(Buffer.from(snapshot.entries[entryPath]!, 'base64').toString('utf-8')) as SerializedEncrypted; + const cipher = createCanisterEncryption({ algorithm: 'aes-256-gcm' }); + const result = await cipher.decrypt(deserializeEncrypted(wrapped), key); + if (!result.success || result.decrypted === undefined) { + throw new Error(result.error ?? 'decryption failed (wrong passphrase or tampered entry)'); + } + return Buffer.from(result.decrypted, 'base64'); +} + +interface SerializedEncrypted { + ciphertext: string; + iv: string; + tag: string; + algorithm: 'aes-256-gcm' | 'chacha20-poly1305'; + timestamp: number; +} + +function serializeEncrypted(data: CanisterEncryptedData): SerializedEncrypted { + return { + ciphertext: Buffer.from(data.ciphertext).toString('hex'), + iv: Buffer.from(data.iv).toString('hex'), + tag: Buffer.from(data.tag).toString('hex'), + algorithm: data.algorithm, + timestamp: data.timestamp, + }; +} + +function deserializeEncrypted(data: SerializedEncrypted): CanisterEncryptedData { + return { + ciphertext: new Uint8Array(Buffer.from(data.ciphertext, 'hex')), + iv: new Uint8Array(Buffer.from(data.iv, 'hex')), + tag: new Uint8Array(Buffer.from(data.tag, 'hex')), + algorithm: data.algorithm, + timestamp: data.timestamp, + }; +} + +/** Canonical manifest bytes: alphabetically sorted keys (ArweaveArchiver idiom) */ +export function canonicalManifestBytes(manifest: HypervaultSnapshotManifest): Buffer { + const sorted = Object.keys(manifest) + .sort() + .reduce>((acc, k) => { + const value = (manifest as unknown as Record)[k]; + if (value !== undefined) acc[k] = value; + return acc; + }, {}); + return Buffer.from(JSON.stringify(sorted), 'utf-8'); +} + +function sha256(data: Buffer): string { + return crypto.createHash('sha256').update(data).digest('hex'); +} + +function omitKeys(row: Record, keys: string[]): Record { + const result: Record = {}; + for (const [k, v] of Object.entries(row)) { + if (!keys.includes(k)) result[k] = v; + } + return result; +} + +function sanitizeHash(hash: string): string { + return hash.replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 100); +} + +function defaultSigningKeyPath(): string { + const home = process.env['HOME'] ?? process.env['USERPROFILE'] ?? '/tmp'; + return path.join(home, '.agentvault', 'arweave-signing.key'); +} diff --git a/src/hypervault/types.ts b/src/hypervault/types.ts new file mode 100644 index 0000000..7d3e3ff --- /dev/null +++ b/src/hypervault/types.ts @@ -0,0 +1,179 @@ +/** + * HyperVault integration — schema types + * + * TypeScript mirrors of the hypervault.store schema (memories, mind DAG, + * artifacts, connections) plus the export/import wire format. Validators + * follow the src/backbone/validators.ts idiom (zod object schemas with + * inferred types). + */ + +import { z } from 'zod'; + +// --------------------------------------------------------------------------- +// Core rows +// --------------------------------------------------------------------------- + +/** A live memory row (`memories` table) */ +export const hvMemorySchema = z.object({ + id: z.string().min(1), + title: z.string().default(''), + content: z.string(), + summary: z.string().optional(), + tags: z.array(z.string()).default([]), + branch: z.string().optional(), + /** pgvector embedding (1536-d for text-embedding-3-small); optional in exports */ + embedding: z.array(z.number()).optional(), + embedding_model: z.string().optional(), + author_kind: z.string().optional(), + author_key_prefix: z.string().optional(), + created_at: z.string().optional(), + updated_at: z.string().optional(), +}); +export type HvMemory = z.infer; + +/** A saved artifact (`artifacts` table) */ +export const hvArtifactSchema = z.object({ + id: z.string().min(1), + slug: z.string().min(1), + title: z.string().default(''), + content: z.string().default(''), + original_content: z.string().optional(), + source_prompt: z.string().optional(), + content_hash: z.string().optional(), + tags: z.array(z.string()).default([]), + visibility: z.enum(['public', 'unlisted', 'private']).default('private'), + created_at: z.string().optional(), + updated_at: z.string().optional(), +}); +export type HvArtifact = z.infer; + +/** Artifact graph edge (`connections` table) */ +export const hvConnectionSchema = z.object({ + id: z.string().min(1), + from_id: z.string().min(1), + to_id: z.string().min(1), + kind: z.string().default('link'), + metadata: z.record(z.string(), z.unknown()).optional(), + created_at: z.string().optional(), +}); +export type HvConnection = z.infer; + +/** Memory knowledge-graph edge (`memory_links` table) */ +export const hvMemoryLinkSchema = z.object({ + id: z.string().min(1), + from_memory_id: z.string().min(1), + to_memory_id: z.string().min(1), + kind: z.string().default('related'), + created_at: z.string().optional(), +}); +export type HvMemoryLink = z.infer; + +// --------------------------------------------------------------------------- +// Mind DAG ("Git for a Mind") +// --------------------------------------------------------------------------- + +/** A mind branch (`memory_branches` table) */ +export const hvMindBranchSchema = z.object({ + name: z.string().min(1), + head_commit_id: z.string().optional(), + created_at: z.string().optional(), +}); +export type HvMindBranch = z.infer; + +/** A mind commit (`memory_commits` table) — full DAG incl. merge parents */ +export const hvMindCommitSchema = z.object({ + id: z.string().min(1), + parent_id: z.string().nullable().optional(), + merge_parent_id: z.string().nullable().optional(), + branch: z.string().optional(), + message: z.string().default(''), + author_kind: z.string().optional(), + author_key_id: z.string().optional(), + author_key_prefix: z.string().optional(), + created_at: z.string().optional(), +}); +export type HvMindCommit = z.infer; + +/** A memory revision (`memory_revisions` table) — one snapshot per change */ +export const hvRevisionSchema = z.object({ + id: z.string().min(1), + commit_id: z.string().min(1), + memory_id: z.string().min(1), + operation: z.enum(['create', 'update', 'delete']).default('update'), + /** Full memory snapshot at this revision (title/content/tags/summary) */ + snapshot: z.record(z.string(), z.unknown()).optional(), + created_at: z.string().optional(), +}); +export type HvRevision = z.infer; + +// --------------------------------------------------------------------------- +// Export wire format (GET /api/export — streamed NDJSON) +// --------------------------------------------------------------------------- + +export const HV_EXPORT_TABLES = [ + 'memories', + 'memory_branches', + 'memory_commits', + 'memory_revisions', + 'memory_heads', + 'memory_links', + 'memory_link_changes', + 'memory_artifact_links', + 'artifacts', + 'connections', + 'conversations', + 'messages', +] as const; +export type HvExportTable = (typeof HV_EXPORT_TABLES)[number]; + +/** One NDJSON line of the export stream: `{"table": "...", "row": {...}}` */ +export const hvExportRecordSchema = z.object({ + table: z.enum(HV_EXPORT_TABLES), + row: z.record(z.string(), z.unknown()), +}); +export type HvExportRecord = z.infer; + +/** Final NDJSON line of the export stream: the manifest */ +export const hvExportManifestSchema = z.object({ + manifest: z.literal(true), + schema_version: z.number().int().default(1), + exported_at: z.string().optional(), + cursor: z.string().optional(), + row_counts: z.record(z.string(), z.number().int().nonnegative()).default({}), + table_hashes: z.record(z.string(), z.string()).default({}), + branch_heads: z.record(z.string(), z.string()).optional(), +}); +export type HvExportManifest = z.infer; + +/** Highest export schema major version this client understands (risk #3) */ +export const HV_SUPPORTED_SCHEMA_VERSION = 1; + +// --------------------------------------------------------------------------- +// Local state file (.agentvault/hypervault.json) — never contains the key +// --------------------------------------------------------------------------- + +export const hypervaultStateSchema = z.object({ + apiUrl: z.string().min(1), + /** Vault reference, e.g. "vault:hashicorp//hypervault_api_key" */ + keyRef: z.string().optional(), + branch: z.string().default('main'), + userIdHint: z.string().optional(), + lastSync: z.string().optional(), + lastExportCursor: z.string().optional(), + lastMindCommitSynced: z.string().optional(), + canisterId: z.string().optional(), + lastArweaveTx: z.string().optional(), +}); +export type HypervaultState = z.infer; + +// --------------------------------------------------------------------------- +// Recall +// --------------------------------------------------------------------------- + +export interface HvRecallResult { + memory: HvMemory; + /** Fused relevance score (higher is better) */ + score: number; + /** Which retrieval paths matched */ + matchedBy: Array<'lexical' | 'semantic'>; +} diff --git a/src/hypervault/wiki-store.ts b/src/hypervault/wiki-store.ts new file mode 100644 index 0000000..1830561 --- /dev/null +++ b/src/hypervault/wiki-store.ts @@ -0,0 +1,208 @@ +/** + * HyperVaultWikiStore — WikiStore backed by the cloud mind + * + * Wiki pages persist as memories tagged `wiki:` (one memory per + * page, full page JSON in the content), so `agentvault wiki` can run + * against hypervault instead of `.agentvault/wiki/*.json`. Backlinks are + * derived from each page's `crossRefs` (mirrored upstream as + * `memory_links` by hypervault's auto-linking). The wiki log and schema + * are memories tagged `wiki-log:` / `wiki-schema:`. + */ + +import type { WikiLogEntry, WikiPage, WikiSchema } from '../backbone/types.js'; +import type { WikiListFilters, WikiStore } from '../wiki/wiki-store.js'; +import type { HyperVaultClient } from './client.js'; +import type { HvMemory } from './types.js'; + +const TAG_PREFIX = 'av'; + +function wikiTag(wikiId: string): string { + return `wiki:${wikiId}`; +} +function slugTag(slug: string): string { + return `${TAG_PREFIX}:slug:${slug}`; +} +function pageIdTag(id: string): string { + return `${TAG_PREFIX}:page-id:${id}`; +} +function logTag(wikiId: string): string { + return `wiki-log:${wikiId}`; +} +function schemaTag(wikiId: string): string { + return `wiki-schema:${wikiId}`; +} + +export class HyperVaultWikiStore implements WikiStore { + constructor(private readonly client: HyperVaultClient) {} + + // ------------------------------------------------------------------------- + // Pages + // ------------------------------------------------------------------------- + + async listPages(wikiId: string, filters?: WikiListFilters): Promise { + const memories = await this.client.listMemories({ tags: [wikiTag(wikiId)] }); + let pages = memories + .filter((m) => m.tags.includes(wikiTag(wikiId)) && !m.tags.includes(logTag(wikiId)) && !m.tags.includes(schemaTag(wikiId))) + .map((m) => parsePage(m)) + .filter((p): p is WikiPage => p !== null); + + if (filters?.category) pages = pages.filter((p) => p.category === filters.category); + if (filters?.status) pages = pages.filter((p) => p.status === filters.status); + if (filters?.staleness) pages = pages.filter((p) => p.staleness === filters.staleness); + if (filters?.tags && filters.tags.length > 0) { + pages = pages.filter((p) => filters.tags!.every((t) => (p.tags ?? []).includes(t))); + } + if (filters?.search) { + const needle = filters.search.toLowerCase(); + pages = pages.filter( + (p) => p.title.toLowerCase().includes(needle) || p.content.toLowerCase().includes(needle), + ); + } + return pages; + } + + async getPage(wikiId: string, slug: string): Promise { + const memory = await this.findPageMemory(wikiId, slug); + return memory ? parsePage(memory) : null; + } + + async getPageById(id: string): Promise { + const memories = await this.client.listMemories({ tags: [pageIdTag(id)] }); + const memory = memories.find((m) => m.tags.includes(pageIdTag(id))); + return memory ? parsePage(memory) : null; + } + + async createPage(page: WikiPage): Promise { + await this.client.memorize({ + title: page.title, + content: JSON.stringify(page), + tags: pageTags(page), + }); + return page; + } + + async updatePage(wikiId: string, slug: string, partial: Partial): Promise { + const memory = await this.findPageMemory(wikiId, slug); + if (!memory) return null; + const existing = parsePage(memory); + if (!existing) return null; + + const updated: WikiPage = { + ...existing, + ...partial, + id: existing.id, + companyId: existing.companyId, + slug: existing.slug, + version: existing.version + 1, + updatedAt: new Date().toISOString(), + }; + await this.client.editMemory(memory.id, { + title: updated.title, + content: JSON.stringify(updated), + tags: pageTags(updated), + }); + return updated; + } + + async deletePage(wikiId: string, slug: string): Promise { + const memory = await this.findPageMemory(wikiId, slug); + if (!memory) return false; + return this.client.forgetMemory(memory.id); + } + + // ------------------------------------------------------------------------- + // Cross-reference queries + // ------------------------------------------------------------------------- + + async getBacklinks(wikiId: string, slug: string): Promise { + const pages = await this.listPages(wikiId); + return pages.filter((p) => p.slug !== slug && p.crossRefs.includes(slug)); + } + + async getOrphans(wikiId: string): Promise { + const pages = await this.listPages(wikiId); + const referenced = new Set(pages.flatMap((p) => p.crossRefs)); + return pages.filter((p) => !referenced.has(p.slug)); + } + + // ------------------------------------------------------------------------- + // Log + // ------------------------------------------------------------------------- + + async appendLog(wikiId: string, entry: WikiLogEntry): Promise { + await this.client.memorize({ + title: `wiki log: ${entry.operation}`, + content: JSON.stringify(entry), + tags: [wikiTag(wikiId), logTag(wikiId)], + }); + } + + async getLog(wikiId: string, limit = 50): Promise { + const memories = await this.client.listMemories({ tags: [logTag(wikiId)] }); + return memories + .filter((m) => m.tags.includes(logTag(wikiId))) + .map((m) => { + try { + return JSON.parse(m.content) as WikiLogEntry; + } catch { + return null; + } + }) + .filter((e): e is WikiLogEntry => e !== null) + .sort((a, b) => b.timestamp.localeCompare(a.timestamp)) + .slice(0, limit); + } + + // ------------------------------------------------------------------------- + // Schema + // ------------------------------------------------------------------------- + + async getSchema(wikiId: string): Promise { + const memories = await this.client.listMemories({ tags: [schemaTag(wikiId)] }); + const memory = memories.find((m) => m.tags.includes(schemaTag(wikiId))); + if (!memory) return null; + try { + return JSON.parse(memory.content) as WikiSchema; + } catch { + return null; + } + } + + async setSchema(wikiId: string, schema: WikiSchema): Promise { + const memories = await this.client.listMemories({ tags: [schemaTag(wikiId)] }); + const existing = memories.find((m) => m.tags.includes(schemaTag(wikiId))); + if (existing) { + await this.client.editMemory(existing.id, { + title: `wiki schema: ${schema.name}`, + content: JSON.stringify(schema), + }); + } else { + await this.client.memorize({ + title: `wiki schema: ${schema.name}`, + content: JSON.stringify(schema), + tags: [wikiTag(wikiId), schemaTag(wikiId)], + }); + } + } + + // ------------------------------------------------------------------------- + + private async findPageMemory(wikiId: string, slug: string): Promise { + const memories = await this.client.listMemories({ tags: [wikiTag(wikiId), slugTag(slug)] }); + return memories.find((m) => m.tags.includes(wikiTag(wikiId)) && m.tags.includes(slugTag(slug))) ?? null; + } +} + +function pageTags(page: WikiPage): string[] { + return [wikiTag(page.companyId), slugTag(page.slug), pageIdTag(page.id), ...(page.tags ?? [])]; +} + +function parsePage(memory: HvMemory): WikiPage | null { + try { + const parsed = JSON.parse(memory.content) as WikiPage; + if (!parsed || typeof parsed !== 'object' || typeof parsed.slug !== 'string') return null; + return parsed; + } catch { + return null; + } +} diff --git a/src/security/types.ts b/src/security/types.ts index e8ff57e..791a04e 100644 --- a/src/security/types.ts +++ b/src/security/types.ts @@ -29,6 +29,13 @@ export interface EncryptedData { salt: string; /** Encrypted ciphertext */ ciphertext: string; + /** + * AEAD authentication tag (hex). Both supported algorithms are AEAD, so + * decryption requires a tag. Legacy payloads produced by writers that + * appended the 16-byte tag to `ciphertext` (combined layout) may omit + * this field; the tag is then split off the end of `ciphertext`. + */ + tag?: string; /** Timestamp of encryption */ encryptedAt: string; } diff --git a/src/security/vetkeys.ts b/src/security/vetkeys.ts index bf7b0d9..12521c6 100644 --- a/src/security/vetkeys.ts +++ b/src/security/vetkeys.ts @@ -27,6 +27,9 @@ import type { VetKeysDerivedKey as DerivedKey, } from './types.js'; +/** AEAD authentication tag length (bytes) for both supported algorithms */ +const GCM_TAG_BYTES = 16; + type CanisterAlgorithm = 'aes_256_gcm' | 'chacha20_poly1305'; function toCanisterAlgorithm(algorithm: EncryptionAlgorithm): CanisterAlgorithm { @@ -53,9 +56,55 @@ export class VetKeysImplementation { this.useCanister = options.useCanister ?? !!options.canisterId; } + /** + * Encrypt JSON data using seed phrase + * + * Counterpart to {@link decryptJSON}. Emits the AEAD authentication tag as + * a separate `tag` field so decryption can (and must) verify integrity. + * + * @param data - JSON-serializable value to encrypt + * @param seedPhrase - Seed phrase for key derivation + * @param algorithm - AEAD algorithm (default aes-256-gcm) + * @returns Encrypted data container including the auth tag + */ + public static async encryptJSON( + data: unknown, + seedPhrase: string, + algorithm: EncryptionAlgorithm = 'aes-256-gcm' + ): Promise { + const crypto = await import('node:crypto'); + const bip39 = await import('bip39'); + + const salt = crypto.randomBytes(16).toString('hex'); + const seed = await bip39.mnemonicToSeed(seedPhrase); + const key = crypto.pbkdf2Sync(seed, salt, 100000, 32, 'sha256'); + + const iv = crypto.randomBytes(12); + const cipher = + algorithm === 'aes-256-gcm' + ? crypto.createCipheriv('aes-256-gcm', key, iv) + : crypto.createCipheriv('chacha20-poly1305', key, iv, { authTagLength: 16 }); + + const plaintext = Buffer.from(JSON.stringify(data), 'utf-8'); + const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]); + + return { + algorithm, + iv: iv.toString('hex'), + salt, + ciphertext: ciphertext.toString('hex'), + tag: cipher.getAuthTag().toString('hex'), + encryptedAt: new Date().toISOString(), + }; + } + /** * Decrypt JSON data using seed phrase * + * Security (audit C-1): both supported algorithms are AEAD, so the + * authentication tag is always validated. Payloads without a usable tag + * are rejected rather than decrypted unauthenticated. + * * @param encrypted - Encrypted data to decrypt * @param seedPhrase - Seed phrase for key derivation * @returns Decrypted JSON object @@ -77,19 +126,31 @@ export class VetKeysImplementation { 'sha256', ); - // Decode IV and ciphertext + // Decode IV, ciphertext and auth tag. Legacy writers appended the + // 16-byte tag to the ciphertext (combined layout); support that, but + // never decrypt without a tag. const iv = Buffer.from(encrypted.iv, 'hex'); - const ciphertext = Buffer.from(encrypted.ciphertext, 'hex'); - - // Decrypt based on algorithm - let algorithm: string; - if (encrypted.algorithm === 'aes-256-gcm') { - algorithm = 'aes-256-gcm'; + let ciphertext = Buffer.from(encrypted.ciphertext, 'hex'); + let tag: Buffer; + if (encrypted.tag) { + tag = Buffer.from(encrypted.tag, 'hex'); + } else if (ciphertext.length > GCM_TAG_BYTES) { + tag = ciphertext.subarray(ciphertext.length - GCM_TAG_BYTES); + ciphertext = ciphertext.subarray(0, ciphertext.length - GCM_TAG_BYTES); } else { - algorithm = encrypted.algorithm.replace('-', ''); + throw new Error( + 'Encrypted payload is missing its authentication tag; refusing unauthenticated decryption' + ); + } + if (tag.length !== GCM_TAG_BYTES) { + throw new Error(`Authentication tag must be ${GCM_TAG_BYTES} bytes`); } - const decipher = crypto.createDecipheriv(algorithm, key, iv); + const decipher = + encrypted.algorithm === 'aes-256-gcm' + ? crypto.createDecipheriv('aes-256-gcm', key, iv) + : crypto.createDecipheriv('chacha20-poly1305', key, iv, { authTagLength: 16 }); + decipher.setAuthTag(tag); const decrypted = Buffer.concat([ decipher.update(ciphertext), @@ -663,6 +724,22 @@ export async function decryptJSON( return VetKeysImplementation.decryptJSON(encrypted, seedPhrase); } +/** + * Encrypt JSON data using seed phrase (AEAD, tag included in output) + * + * @param data - JSON-serializable value to encrypt + * @param seedPhrase - Seed phrase for key derivation + * @param algorithm - AEAD algorithm (default aes-256-gcm) + * @returns Encrypted data container including the auth tag + */ +export async function encryptJSON( + data: unknown, + seedPhrase: string, + algorithm: EncryptionAlgorithm = 'aes-256-gcm' +): Promise { + return VetKeysImplementation.encryptJSON(data, seedPhrase, algorithm); +} + // --------------------------------------------------------------------------- // Bundle encryption / decryption using principal-based VetKeys // --------------------------------------------------------------------------- diff --git a/tests/hypervault/auth.test.ts b/tests/hypervault/auth.test.ts new file mode 100644 index 0000000..8798d32 --- /dev/null +++ b/tests/hypervault/auth.test.ts @@ -0,0 +1,60 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + resolveHyperVaultKey, + saveHypervaultState, + loadHypervaultState, + hypervaultStatePath, + defaultHypervaultState, +} from '../../src/hypervault/auth.js'; + +describe('hypervault auth', () => { + let tmp: string; + const savedEnv = process.env.HYPERVAULT_API_KEY; + + beforeEach(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'hv-auth-')); + delete process.env.HYPERVAULT_API_KEY; + }); + + afterEach(() => { + fs.rmSync(tmp, { recursive: true, force: true }); + if (savedEnv === undefined) delete process.env.HYPERVAULT_API_KEY; + else process.env.HYPERVAULT_API_KEY = savedEnv; + }); + + it('resolves the flag key and flags it as insecure', async () => { + const resolved = await resolveHyperVaultKey({ flagKey: 'hv_flagkey' }); + expect(resolved?.key).toBe('hv_flagkey'); + expect(resolved?.source).toBe('flag'); + expect(resolved?.insecureSource).toBe(true); + }); + + it('prefers the env var over the vault, non-insecure', async () => { + process.env.HYPERVAULT_API_KEY = 'hv_envkey'; + const resolved = await resolveHyperVaultKey({ agentId: 'agent-x' }); + expect(resolved?.key).toBe('hv_envkey'); + expect(resolved?.source).toBe('env'); + expect(resolved?.insecureSource).toBe(false); + }); + + it('returns null when nothing supplies a key', async () => { + const resolved = await resolveHyperVaultKey({}); + expect(resolved).toBeNull(); + }); + + it('never writes a plaintext hv_ key to the state file', () => { + const state = { ...defaultHypervaultState(), keyRef: 'vault:hashicorp/agent/hypervault_api_key' }; + saveHypervaultState(state, tmp); + const raw = fs.readFileSync(hypervaultStatePath(tmp), 'utf-8'); + expect(raw).not.toMatch(/hv_[A-Za-z0-9]{8,}/); + expect(loadHypervaultState(tmp)?.keyRef).toBe('vault:hashicorp/agent/hypervault_api_key'); + }); + + it('refuses to persist an accidental plaintext key', () => { + const state = { ...defaultHypervaultState(), userIdHint: 'hv_abcdef1234567890' }; + expect(() => saveHypervaultState(state, tmp)).toThrow(/plaintext/i); + }); +}); diff --git a/tests/hypervault/client.test.ts b/tests/hypervault/client.test.ts new file mode 100644 index 0000000..e5e84da --- /dev/null +++ b/tests/hypervault/client.test.ts @@ -0,0 +1,85 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { MockAgent, setGlobalDispatcher } from 'undici'; +import { HyperVaultClient, HyperVaultError } from '../../src/hypervault/client.js'; +import { sampleManifest, sampleRecords, toNdjson } from './fixtures.js'; + +const BASE = 'https://hv.test'; + +describe('HyperVaultClient', () => { + let mockAgent: MockAgent; + + beforeEach(() => { + mockAgent = new MockAgent(); + mockAgent.disableNetConnect(); + setGlobalDispatcher(mockAgent); + }); + + afterEach(async () => { + await mockAgent.close(); + }); + + it('validates a good key and reports the user hint', async () => { + mockAgent + .get(BASE) + .intercept({ path: '/api/keys', method: 'GET' }) + .reply(200, { user_id: 'user-123' }); + + const client = new HyperVaultClient({ apiKey: 'hv_good', apiUrl: BASE }); + const result = await client.validateKey(); + expect(result.valid).toBe(true); + expect(result.userIdHint).toBe('user-123'); + }); + + it('reports an invalid key on 401 without throwing', async () => { + mockAgent.get(BASE).intercept({ path: '/api/keys', method: 'GET' }).reply(401, { error: 'nope' }); + const client = new HyperVaultClient({ apiKey: 'hv_bad', apiUrl: BASE }); + expect(await client.validateKey()).toEqual({ valid: false }); + }); + + it('sends the X-HyperVault-Key header', async () => { + // The interceptor only matches when the header is present with the right + // value; a missing/wrong header would fall through to a mock error. + mockAgent + .get(BASE) + .intercept({ + path: '/api/memories', + method: 'GET', + headers: { 'x-hypervault-key': 'hv_secret' }, + }) + .reply(200, []); + const client = new HyperVaultClient({ apiKey: 'hv_secret', apiUrl: BASE }); + await expect(client.listMemories()).resolves.toEqual([]); + }); + + it('retries on 429 and then succeeds', async () => { + const pool = mockAgent.get(BASE); + pool.intercept({ path: '/api/memories', method: 'GET' }).reply(429, 'slow down'); + pool.intercept({ path: '/api/memories', method: 'GET' }).reply(200, [{ id: 'm', content: 'x', tags: [] }]); + const client = new HyperVaultClient({ apiKey: 'hv_k', apiUrl: BASE, backoffMs: 1 }); + const memories = await client.listMemories(); + expect(memories).toHaveLength(1); + }); + + it('streams and validates an NDJSON export, surfacing the manifest', async () => { + mockAgent + .get(BASE) + .intercept({ path: '/api/export', method: 'GET' }) + .reply(200, toNdjson(sampleRecords(), sampleManifest())); + + const client = new HyperVaultClient({ apiKey: 'hv_k', apiUrl: BASE }); + const { records, manifest } = await client.exportVault(); + expect(records.length).toBe(sampleRecords().length); + expect(manifest.branch_heads).toEqual({ main: 'commit-2' }); + expect(records.filter((r) => r.table === 'memories')).toHaveLength(2); + }); + + it('rejects an export whose schema version is newer than supported', async () => { + const manifest = { ...sampleManifest(), schema_version: 99 }; + mockAgent + .get(BASE) + .intercept({ path: '/api/export', method: 'GET' }) + .reply(200, toNdjson(sampleRecords(), manifest)); + const client = new HyperVaultClient({ apiKey: 'hv_k', apiUrl: BASE }); + await expect(client.exportVault()).rejects.toBeInstanceOf(HyperVaultError); + }); +}); diff --git a/tests/hypervault/fixtures.ts b/tests/hypervault/fixtures.ts new file mode 100644 index 0000000..c033de8 --- /dev/null +++ b/tests/hypervault/fixtures.ts @@ -0,0 +1,109 @@ +/** + * Shared fixtures for the hypervault test suite. + */ + +import type { HvExportManifest, HvExportRecord } from '../../src/hypervault/types.js'; + +export function sampleRecords(): HvExportRecord[] { + return [ + { + table: 'memories', + row: { + id: 'mem-1', + title: 'First memory', + content: 'The quick brown fox jumps over the lazy dog.', + tags: ['soul', 'origin'], + summary: 'A pangram', + embedding: [0.1, 0.2, 0.3, 0.4], + embedding_model: 'text-embedding-3-small', + branch: 'main', + created_at: '2026-07-01T00:00:00.000Z', + updated_at: '2026-07-01T00:00:00.000Z', + }, + }, + { + table: 'memories', + row: { + id: 'mem-2', + title: 'Second memory', + content: 'Sphinx of black quartz, judge my vow.', + tags: ['origin'], + embedding: [0.4, 0.3, 0.2, 0.1], + embedding_model: 'text-embedding-3-small', + branch: 'main', + }, + }, + { + table: 'memory_branches', + row: { name: 'main', head_commit_id: 'commit-2' }, + }, + { + table: 'memory_commits', + row: { + id: 'commit-1', + parent_id: null, + message: 'genesis', + branch: 'main', + author_kind: 'agent', + author_key_prefix: 'hv_abc', + created_at: '2026-07-01T00:00:00.000Z', + }, + }, + { + table: 'memory_commits', + row: { + id: 'commit-2', + parent_id: 'commit-1', + message: 'add second memory', + branch: 'main', + author_kind: 'agent', + author_key_prefix: 'hv_abc', + created_at: '2026-07-02T00:00:00.000Z', + }, + }, + { + table: 'memory_revisions', + row: { id: 'rev-1', commit_id: 'commit-1', memory_id: 'mem-1', operation: 'create' }, + }, + { + table: 'artifacts', + row: { + id: 'art-1', + slug: 'hello-world', + title: 'Hello World', + content: '

Hello

', + content_hash: 'abc123', + tags: ['demo'], + visibility: 'public', + }, + }, + { + table: 'connections', + row: { id: 'conn-1', from_id: 'art-1', to_id: 'mem-1', kind: 'derived-from' }, + }, + ]; +} + +export function sampleManifest(): HvExportManifest { + return { + manifest: true, + schema_version: 1, + exported_at: '2026-07-15T00:00:00.000Z', + cursor: '2026-07-15T00:00:00.000Z', + row_counts: { + memories: 2, + memory_branches: 1, + memory_commits: 2, + memory_revisions: 1, + artifacts: 1, + connections: 1, + }, + table_hashes: {}, + branch_heads: { main: 'commit-2' }, + }; +} + +/** Serialize records + manifest as the NDJSON export wire format. */ +export function toNdjson(records: HvExportRecord[], manifest: HvExportManifest): string { + return records.map((r) => JSON.stringify(r)).join('\n') + '\n' + JSON.stringify(manifest) + '\n'; +} diff --git a/tests/hypervault/index.test.ts b/tests/hypervault/index.test.ts new file mode 100644 index 0000000..5bfedfb --- /dev/null +++ b/tests/hypervault/index.test.ts @@ -0,0 +1,82 @@ +import { describe, it, expect } from 'vitest'; +import { FtsIndex } from '../../src/hypervault/index/fts-index.js'; +import { VectorIndex } from '../../src/hypervault/index/vector-index.js'; +import { buildIndices } from '../../src/hypervault/index/builder.js'; +import { hybridRecall } from '../../src/hypervault/index/recall.js'; +import type { HvMemory } from '../../src/hypervault/types.js'; + +const MEMORIES: HvMemory[] = [ + { id: 'a', title: 'Coffee brewing', content: 'Pour over method with a gooseneck kettle', tags: ['coffee'] }, + { id: 'b', title: 'Tea steeping', content: 'Green tea steeps best at 80 degrees', tags: ['tea'] }, + { id: 'c', title: 'Espresso', content: 'A concentrated coffee shot under pressure', tags: ['coffee'] }, +]; + +describe('FtsIndex', () => { + it('ranks title matches above content matches', () => { + const index = new FtsIndex(); + for (const m of MEMORIES) index.add({ id: m.id, title: m.title, tags: m.tags, content: m.content }); + const hits = index.search('coffee'); + expect(hits[0]?.id).toBe('a'); // title match beats content-only match 'c' + expect(hits.map((h) => h.id)).toContain('c'); + }); + + it('serializes and restores deterministically', () => { + const index = new FtsIndex(); + for (const m of MEMORIES) index.add({ id: m.id, title: m.title, content: m.content }); + const restored = FtsIndex.fromJSON(index.toJSON()); + expect(restored.search('espresso')).toEqual(index.search('espresso')); + }); +}); + +describe('VectorIndex', () => { + it('finds the nearest vector by cosine similarity', () => { + const index = new VectorIndex(3, 'test-model'); + index.add('x', [1, 0, 0]); + index.add('y', [0, 1, 0]); + index.add('z', [0.9, 0.1, 0]); + const hits = index.search([1, 0, 0], 2, 'test-model'); + expect(hits[0]?.id).toBe('x'); + expect(hits[1]?.id).toBe('z'); + }); + + it('refuses cross-model queries', () => { + const index = new VectorIndex(3, 'model-a'); + index.add('x', [1, 0, 0]); + expect(() => index.search([1, 0, 0], 1, 'model-b')).toThrow(/model mismatch/i); + }); + + it('round-trips through JSON', () => { + const index = new VectorIndex(2, 'm'); + index.add('p', [3, 4]); + const restored = VectorIndex.fromJSON(index.toJSON()); + const hits = restored.search([3, 4], 1); + expect(hits[0]?.id).toBe('p'); + expect(hits[0]?.score).toBeCloseTo(1, 5); + }); +}); + +describe('hybridRecall', () => { + it('falls back to FTS-only when no embedder is available', async () => { + const built = buildIndices(MEMORIES); + const byId = new Map(MEMORIES.map((m) => [m.id, m])); + const results = await hybridRecall('coffee', built, byId, { limit: 3 }); + expect(results.length).toBeGreaterThan(0); + expect(results.every((r) => r.matchedBy.includes('lexical'))).toBe(true); + }); + + it('fuses lexical and semantic hits when an embedder is present', async () => { + const embeddingsById = new Map([ + ['a', [1, 0, 0]], + ['b', [0, 1, 0]], + ['c', [0.8, 0.2, 0]], + ]); + const built = buildIndices(MEMORIES, embeddingsById, 'test-model'); + const byId = new Map(MEMORIES.map((m) => [m.id, m])); + const results = await hybridRecall('coffee', built, byId, { + limit: 3, + embedQuery: async () => ({ embedding: [1, 0, 0], model: 'test-model' }), + }); + const topMatch = results.find((r) => r.matchedBy.includes('semantic')); + expect(topMatch).toBeDefined(); + }); +}); diff --git a/tests/hypervault/mind-sync.test.ts b/tests/hypervault/mind-sync.test.ts new file mode 100644 index 0000000..41689ca --- /dev/null +++ b/tests/hypervault/mind-sync.test.ts @@ -0,0 +1,106 @@ +import { describe, it, expect } from 'vitest'; +import { topologicalOrder, syncMindToCanister, HV_COMMIT_TAG_PREFIX } from '../../src/hypervault/mind-sync.js'; +import type { _SERVICE, Commit, OperationResult, RepoStatus } from '../../src/canister/memory-repo-actor.js'; +import type { HvMindCommit } from '../../src/hypervault/types.js'; + +function commit(id: string, parent?: string, merge?: string, at?: string): HvMindCommit { + return { + id, + parent_id: parent ?? null, + merge_parent_id: merge ?? null, + branch: 'main', + message: `commit ${id}`, + created_at: at ?? `2026-07-0${id.slice(-1)}T00:00:00.000Z`, + }; +} + +/** Minimal in-memory fake of the memory_repo canister actor. */ +function fakeActor(initialCommits: Commit[] = []): _SERVICE & { commits: Commit[] } { + const commits: Commit[] = [...initialCommits]; + const thoughtforms = new Map(); + let initialized = true; + const actor = { + commits, + async getRepoStatus(): Promise { + return { initialized, currentBranch: 'main', totalCommits: BigInt(commits.length), totalBranches: 1n, owner: 'test' }; + }, + async initRepo(): Promise { + initialized = true; + return { ok: 'initialized' }; + }, + async getBranches(): Promise<[string, string][]> { + return [['main', commits[commits.length - 1]?.id ?? '']]; + }, + async createBranch(): Promise { + return { ok: 'created' }; + }, + async switchBranch(): Promise { + return { ok: 'switched' }; + }, + async log(): Promise { + return commits; + }, + async commit(message: string, diff: string, tags: string[]): Promise { + const id = `chain-${commits.length + 1}`; + commits.push({ id, timestamp: 0n, message, diff, tags, parent: [], branch: 'main' }); + return { ok: id }; + }, + async getThoughtFormByHash(hash: string) { + const tf = thoughtforms.get(hash); + return (tf ? [tf] : []) as [{ json: string; timestamp: bigint; hash: string }] | []; + }, + async storeThoughtForm(json: string, timestamp: bigint, hash: string): Promise { + thoughtforms.set(hash, { json, timestamp, hash }); + return { ok: hash }; + }, + } as unknown as _SERVICE & { commits: Commit[] }; + return actor; +} + +describe('topologicalOrder', () => { + it('orders parents before children including merge parents', () => { + const commits = [ + commit('4', '2', '3'), + commit('2', '1'), + commit('3', '1'), + commit('1'), + ]; + const ordered = topologicalOrder(commits).map((c) => c.id); + expect(ordered.indexOf('1')).toBeLessThan(ordered.indexOf('2')); + expect(ordered.indexOf('1')).toBeLessThan(ordered.indexOf('3')); + expect(ordered.indexOf('2')).toBeLessThan(ordered.indexOf('4')); + expect(ordered.indexOf('3')).toBeLessThan(ordered.indexOf('4')); + }); + + it('throws on a cycle', () => { + const commits = [commit('1', '2'), commit('2', '1')]; + expect(() => topologicalOrder(commits)).toThrow(/cycle|dangling/i); + }); +}); + +describe('syncMindToCanister', () => { + it('replays all commits and is idempotent on a second run', async () => { + const actor = fakeActor(); + const input = { + commits: [commit('1'), commit('2', '1'), commit('3', '2')], + revisions: [{ id: 'r1', commit_id: '1', memory_id: 'm1', operation: 'create' as const }], + branches: [{ name: 'main' }], + memories: [{ id: 'm1', title: 'M1', content: 'hello', tags: [] }], + }; + + const first = await syncMindToCanister(actor, input); + expect(first.commitsReplayed).toBe(3); + expect(first.errors).toEqual([]); + expect(first.thoughtformsStored).toBe(1); + + const second = await syncMindToCanister(actor, input); + expect(second.commitsReplayed).toBe(0); + expect(second.commitsSkipped).toBe(3); + // No duplicate thoughtforms + expect(second.thoughtformsStored).toBe(0); + + // Every hypervault commit id is tagged exactly once on chain + const tagged = actor.commits.flatMap((c) => c.tags).filter((t) => t.startsWith(HV_COMMIT_TAG_PREFIX)); + expect(new Set(tagged).size).toBe(3); + }); +}); diff --git a/tests/hypervault/snapshot.test.ts b/tests/hypervault/snapshot.test.ts new file mode 100644 index 0000000..6dd6b4f --- /dev/null +++ b/tests/hypervault/snapshot.test.ts @@ -0,0 +1,108 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { + buildSnapshot, + writeSnapshot, + readSnapshot, + verifySnapshot, + snapshotToRecords, + readEmbeddings, +} from '../../src/hypervault/snapshot.js'; +import { sampleManifest, sampleRecords } from './fixtures.js'; + +describe('hypervault snapshot bundle', () => { + let tmp: string; + let keyPath: string; + + beforeEach(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'hv-snap-')); + keyPath = path.join(tmp, 'signing.key'); + }); + + afterEach(() => { + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + it('round-trips records → bundle → records (deep equal on core tables)', async () => { + const snapshot = await buildSnapshot(sampleRecords(), sampleManifest(), { signingKeyPath: keyPath }); + const file = path.join(tmp, 'snap.gz'); + await writeSnapshot(snapshot, file); + + const loaded = await readSnapshot(file); + const records = await snapshotToRecords(loaded); + + const memoriesIn = sampleRecords().filter((r) => r.table === 'memories'); + const memoriesOut = records.filter((r) => r.table === 'memories'); + expect(memoriesOut).toHaveLength(memoriesIn.length); + expect(memoriesOut.map((r) => r.row.id).sort()).toEqual(memoriesIn.map((r) => r.row.id).sort()); + + // artifact content survives the content-addressed split/rejoin + const artifact = records.find((r) => r.table === 'artifacts'); + expect(artifact?.row.content).toBe('

Hello

'); + }); + + it('preserves embeddings through the packed sidecar', async () => { + const snapshot = await buildSnapshot(sampleRecords(), sampleManifest(), { signingKeyPath: keyPath }); + const embeddings = await readEmbeddings(snapshot); + expect(embeddings.get('mem-1')).toEqual([ + expect.closeTo(0.1, 5), + expect.closeTo(0.2, 5), + expect.closeTo(0.3, 5), + expect.closeTo(0.4, 5), + ]); + }); + + it('verifies signature, checksums, and Merkle root', async () => { + const snapshot = await buildSnapshot(sampleRecords(), sampleManifest(), { signingKeyPath: keyPath }); + const result = await verifySnapshot(snapshot); + expect(result.valid).toBe(true); + expect(result.signatureValid).toBe(true); + expect(result.checksumsValid).toBe(true); + expect(result.merkleRootValid).toBe(true); + }); + + it('rejects a tampered entry (checksum + Merkle break)', async () => { + const snapshot = await buildSnapshot(sampleRecords(), sampleManifest(), { signingKeyPath: keyPath }); + // Corrupt the memories entry + snapshot.entries['memories.ndjson'] = Buffer.from('tampered', 'utf-8').toString('base64'); + const result = await verifySnapshot(snapshot); + expect(result.valid).toBe(false); + expect(result.checksumsValid).toBe(false); + }); + + it('rejects a tampered manifest (signature break)', async () => { + const snapshot = await buildSnapshot(sampleRecords(), sampleManifest(), { signingKeyPath: keyPath }); + snapshot.manifest.merkleRoot = crypto.randomBytes(32).toString('hex'); + const result = await verifySnapshot(snapshot); + expect(result.signatureValid).toBe(false); + expect(result.valid).toBe(false); + }); + + it('encrypts entries and round-trips with the passphrase', async () => { + const snapshot = await buildSnapshot(sampleRecords(), sampleManifest(), { + signingKeyPath: keyPath, + passphrase: 'correct horse battery staple', + }); + expect(snapshot.manifest.encryptedEntries).toContain('memories.ndjson'); + // Ciphertext should not contain the plaintext + const raw = Buffer.from(snapshot.entries['memories.ndjson']!, 'base64').toString('utf-8'); + expect(raw).not.toContain('quick brown fox'); + + const verified = await verifySnapshot(snapshot, { passphrase: 'correct horse battery staple' }); + expect(verified.valid).toBe(true); + + const records = await snapshotToRecords(snapshot, 'correct horse battery staple'); + expect(records.filter((r) => r.table === 'memories')).toHaveLength(2); + }); + + it('fails decryption with the wrong passphrase (auth tag rejects)', async () => { + const snapshot = await buildSnapshot(sampleRecords(), sampleManifest(), { + signingKeyPath: keyPath, + passphrase: 'right-passphrase', + }); + await expect(snapshotToRecords(snapshot, 'wrong-passphrase')).rejects.toThrow(); + }); +}); diff --git a/tests/mcp-server/protocol.test.ts b/tests/mcp-server/protocol.test.ts new file mode 100644 index 0000000..c0cc4b8 --- /dev/null +++ b/tests/mcp-server/protocol.test.ts @@ -0,0 +1,75 @@ +import { describe, it, expect } from 'vitest'; +import { PassThrough } from 'node:stream'; +import { + serveMcp, + getAllToolDefinitions, + getHyperVaultToolDefinitions, + handleHyperVaultToolCall, +} from '../../src/hypervault/mcp-server.js'; + +/** Drive the stdio JSON-RPC server with a scripted set of requests. */ +async function driveServer(requests: object[]): Promise[]> { + const input = new PassThrough(); + const output = new PassThrough(); + const responses: Record[] = []; + + output.on('data', (chunk: Buffer) => { + for (const line of chunk.toString('utf-8').split('\n')) { + if (line.trim()) responses.push(JSON.parse(line) as Record); + } + }); + + const done = serveMcp({ input, output }); + for (const req of requests) { + input.write(JSON.stringify(req) + '\n'); + } + input.end(); + await done; + return responses; +} + +describe('native MCP server protocol conformance', () => { + it('responds to initialize with protocol version and server info', async () => { + const responses = await driveServer([{ jsonrpc: '2.0', id: 1, method: 'initialize', params: {} }]); + const res = responses[0]!; + expect(res.id).toBe(1); + const result = res.result as { protocolVersion: string; serverInfo: { name: string } }; + expect(result.protocolVersion).toBeTruthy(); + expect(result.serverInfo.name).toBe('agentvault'); + }); + + it('lists both pipeline and wiki tools', async () => { + const responses = await driveServer([{ jsonrpc: '2.0', id: 2, method: 'tools/list' }]); + const result = responses[0]!.result as { tools: { name: string }[] }; + const names = result.tools.map((t) => t.name); + expect(names).toContain('hypervault_archive'); + expect(names).toContain('hypervault_bootstrap'); + expect(names).toContain('wiki_read'); + expect(result.tools.length).toBe(getAllToolDefinitions().length); + }); + + it('returns method-not-found for unknown methods', async () => { + const responses = await driveServer([{ jsonrpc: '2.0', id: 3, method: 'does/not/exist' }]); + const error = responses[0]!.error as { code: number }; + expect(error.code).toBe(-32601); + }); + + it('does not respond to notifications (no id)', async () => { + const responses = await driveServer([ + { jsonrpc: '2.0', method: 'notifications/initialized' }, + { jsonrpc: '2.0', id: 9, method: 'ping' }, + ]); + // Only the ping (id 9) produces a response + expect(responses).toHaveLength(1); + expect(responses[0]?.id).toBe(9); + }); + + it('exposes all 9 pipeline tools', () => { + expect(getHyperVaultToolDefinitions()).toHaveLength(9); + }); + + it('returns an error result for an unknown tool call', async () => { + const result = await handleHyperVaultToolCall('hypervault_nope', {}); + expect(result.isError).toBe(true); + }); +}); diff --git a/tests/security/vetkeys-decrypt-json.test.ts b/tests/security/vetkeys-decrypt-json.test.ts new file mode 100644 index 0000000..195513b --- /dev/null +++ b/tests/security/vetkeys-decrypt-json.test.ts @@ -0,0 +1,39 @@ +import { describe, it, expect } from 'vitest'; +import { encryptJSON, decryptJSON } from '../../src/security/vetkeys.js'; + +// A valid BIP39 12-word mnemonic (test vector — not a real wallet). +const SEED = 'legal winner thank year wave sausage worth useful legal winner thank yellow'; + +describe('vetkeys encryptJSON/decryptJSON (audit C-1)', () => { + it('round-trips a JSON value with an authenticated tag', async () => { + const data = { secret: 'value', count: 42, nested: { ok: true } }; + const encrypted = await encryptJSON(data, SEED); + expect(encrypted.tag).toBeTruthy(); + expect(encrypted.algorithm).toBe('aes-256-gcm'); + const decrypted = await decryptJSON(encrypted, SEED); + expect(decrypted).toEqual(data); + }); + + it('rejects ciphertext tampering (GCM auth tag now validated)', async () => { + const encrypted = await encryptJSON({ secret: 'value' }, SEED); + // Flip a byte in the ciphertext + const bytes = Buffer.from(encrypted.ciphertext, 'hex'); + bytes[0] = bytes[0]! ^ 0xff; + const tampered = { ...encrypted, ciphertext: bytes.toString('hex') }; + await expect(decryptJSON(tampered, SEED)).rejects.toThrow(); + }); + + it('rejects a tampered auth tag', async () => { + const encrypted = await encryptJSON({ secret: 'value' }, SEED); + const tag = Buffer.from(encrypted.tag!, 'hex'); + tag[0] = tag[0]! ^ 0xff; + await expect(decryptJSON({ ...encrypted, tag: tag.toString('hex') }, SEED)).rejects.toThrow(); + }); + + it('refuses to decrypt a payload with no usable auth tag', async () => { + const encrypted = await encryptJSON({ secret: 'value' }, SEED); + // Strip the tag and keep the ciphertext short so no tag can be split off. + const noTag = { ...encrypted, tag: undefined, ciphertext: '00' }; + await expect(decryptJSON(noTag, SEED)).rejects.toThrow(/authentication tag/i); + }); +});