From 4748c24c05e4c0cb0c3548454a7794b406e29b8b Mon Sep 17 00:00:00 2001 From: kunaldhongade Date: Sun, 2 Aug 2026 19:16:57 +0530 Subject: [PATCH] feat(runtime): ingest local telemetry evidence --- packages/cli/src/commands/registry.ts | 5 + packages/cli/src/commands/runtime.ts | 34 ++ .../cli/src/docs/command-docs/analysis.ts | 21 ++ packages/cli/src/docs/command-docs/order.ts | 2 +- packages/cli/src/parsers/args.ts | 1 + packages/cli/src/parsers/runtime.ts | 46 +++ packages/cli/src/types/index.ts | 1 + packages/cli/src/types/runtime.ts | 12 + packages/cli/test/runtime.test.ts | 66 ++++ packages/knowledge/src/index.ts | 11 + packages/knowledge/src/runtime/ingest.ts | 299 ++++++++++++++++++ packages/knowledge/src/runtime/render.ts | 26 ++ packages/knowledge/src/runtime/types.ts | 73 +++++ .../knowledge/test/runtime-evidence.test.ts | 106 +++++++ 14 files changed, 702 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/commands/runtime.ts create mode 100644 packages/cli/src/parsers/runtime.ts create mode 100644 packages/cli/src/types/runtime.ts create mode 100644 packages/cli/test/runtime.test.ts create mode 100644 packages/knowledge/src/runtime/ingest.ts create mode 100644 packages/knowledge/src/runtime/render.ts create mode 100644 packages/knowledge/src/runtime/types.ts create mode 100644 packages/knowledge/test/runtime-evidence.test.ts diff --git a/packages/cli/src/commands/registry.ts b/packages/cli/src/commands/registry.ts index b6bc8c6..976993f 100644 --- a/packages/cli/src/commands/registry.ts +++ b/packages/cli/src/commands/registry.ts @@ -19,6 +19,7 @@ import { runMcpCommand as runMcpCommandWithDependencies } from "./mcp"; import { runProductCommand as runProductCommandWithDependencies } from "./product"; import { runRedteamCommand as runRedteamCommandWithDependencies } from "./redteam"; import { runRevalidateCommand as runRevalidateCommandWithDependencies } from "./revalidate"; +import { runRuntimeCommand as runRuntimeCommandWithDependencies } from "./runtime"; import { runSessionCommand as runSessionCommandWithDependencies } from "./session"; import { runSnapshotCommand as runSnapshotCommandWithDependencies } from "./snapshot"; import { createProductTargetReport as createProductTargetReportWithRuntime } from "../product/runtime"; @@ -118,6 +119,10 @@ export function createCommandHandlers(options: CommandRegistryOptions): Record runRuntimeCommandWithDependencies(context, { + resolveRepoRoot: getRepoRootForCli, + writeOutput: writeCliOutput + }), session: (context) => runSessionCommandWithDependencies(context, { createAnalysisContext: createAnalysisContextForCli, resolveRepoRoot: getRepoRootForCli, diff --git a/packages/cli/src/commands/runtime.ts b/packages/cli/src/commands/runtime.ts new file mode 100644 index 0000000..dadd26e --- /dev/null +++ b/packages/cli/src/commands/runtime.ts @@ -0,0 +1,34 @@ +import { resolve } from "node:path"; +import { + ingestRuntimeEvidence, + loadServiceTopologyManifest, + renderRuntimeEvidenceMarkdown +} from "@submuxhq/codedecay-knowledge"; +import { parseRuntimeArgs } from "../parsers/args"; +import type { CliCommandContext, CliRuntime, RuntimeOptions } from "../types"; + +export interface RunRuntimeCommandDependencies { + resolveRepoRoot(cwd: string, options: RuntimeOptions): string; + writeOutput(input: { cwd: string; output?: string | undefined; rendered: string; runtime: CliRuntime }): void; +} + +export function runRuntimeCommand(context: CliCommandContext, dependencies: RunRuntimeCommandDependencies): void { + const options = parseRuntimeArgs(context.args); + const cwd = resolve(context.runtimeCwd, options.cwd ?? "."); + const rootDir = dependencies.resolveRepoRoot(cwd, options); + const topology = options.topology + ? loadServiceTopologyManifest({ rootDir, path: options.topology }) + : undefined; + const report = ingestRuntimeEvidence({ + rootDir, + otlpPath: options.telemetry, + errorsPath: options.errors, + topology, + headRevision: options.headRevision, + environment: options.environment + }); + const rendered = options.format === "json" + ? `${JSON.stringify(report, null, 2)}\n` + : renderRuntimeEvidenceMarkdown(report); + dependencies.writeOutput({ cwd: rootDir, output: options.output, rendered, runtime: context.runtime }); +} diff --git a/packages/cli/src/docs/command-docs/analysis.ts b/packages/cli/src/docs/command-docs/analysis.ts index 476487b..a8f0200 100644 --- a/packages/cli/src/docs/command-docs/analysis.ts +++ b/packages/cli/src/docs/command-docs/analysis.ts @@ -1,6 +1,27 @@ import type { CommandDoc } from "../../renderers/discovery"; export const ANALYSIS_COMMAND_DOCS: Record = { + runtime: { + name: "runtime", + summary: "Ingest local runtime exports as redacted engineering evidence.", + usage: ["codedecay runtime [options]"], + description: ["Read local OTLP JSON traces and structured error exports, correlate them with an optional service topology, and emit revision-aware investigation evidence."], + options: [ + { flag: "--telemetry ", description: "Repo-local OTLP JSON trace export" }, + { flag: "--errors ", description: "Repo-local structured error export" }, + { flag: "--topology ", description: "Optional repo-local service topology manifest" }, + { flag: "--head-revision ", description: "Current source revision used to classify evidence trust" }, + { flag: "--environment ", description: "Environment label when an export omits one" }, + { flag: "--cwd ", description: "Repository working directory (default: current directory)" }, + { flag: "--format ", description: "json or markdown (default: markdown)" }, + { flag: "--output ", description: "Write evidence report to a file instead of stdout" } + ], + examples: [ + "codedecay runtime --telemetry .codedecay/runtime/traces.json --head-revision $(git rev-parse HEAD)", + "codedecay runtime --errors .codedecay/runtime/errors.json --format json" + ], + notes: ["Inputs must resolve inside the repository. The command performs no network calls or project command execution."] + }, analyze: { name: "analyze", summary: "Deterministic PR risk, impact, and decay report.", diff --git a/packages/cli/src/docs/command-docs/order.ts b/packages/cli/src/docs/command-docs/order.ts index b2b761a..6b99775 100644 --- a/packages/cli/src/docs/command-docs/order.ts +++ b/packages/cli/src/docs/command-docs/order.ts @@ -1,3 +1,3 @@ -export const COMMAND_ORDER = ["ai", "session", "context", "analyze", "benchmark", "snapshot", "redteam", "revalidate", "llm-review", "agent", "loop", "doctor", "config", "memory", "memory-import", "memory-learn", "execute", "differential", "product", "dashboard", "mcp"] as const; +export const COMMAND_ORDER = ["ai", "session", "context", "analyze", "runtime", "benchmark", "snapshot", "redteam", "revalidate", "llm-review", "agent", "loop", "doctor", "config", "memory", "memory-import", "memory-learn", "execute", "differential", "product", "dashboard", "mcp"] as const; export const UTILITY_COMMAND_ORDER = ["help", "man", "update", "uninstall", "version"] as const; export const ROOT_FLAG_ALIASES = ["--help", "-h", "--version", "-V"] as const; diff --git a/packages/cli/src/parsers/args.ts b/packages/cli/src/parsers/args.ts index bbd6689..c2b5401 100644 --- a/packages/cli/src/parsers/args.ts +++ b/packages/cli/src/parsers/args.ts @@ -13,6 +13,7 @@ export { parseLoopArgs } from "./loop"; export { parseMcpArgs } from "./mcp"; export { parseMemoryArgs, parseMemoryImportArgs, parseMemoryLearnArgs, parseMemorySetupArgs } from "./memory"; export { parseRevalidateArgs } from "./revalidate"; +export { parseRuntimeArgs } from "./runtime"; export { parseProductArgs } from "./product"; export { parseRedteamArgs } from "./redteam"; export { parseSessionArgs } from "./session"; diff --git a/packages/cli/src/parsers/runtime.ts b/packages/cli/src/parsers/runtime.ts new file mode 100644 index 0000000..906a663 --- /dev/null +++ b/packages/cli/src/parsers/runtime.ts @@ -0,0 +1,46 @@ +import type { RuntimeOptions } from "../types"; +import { requireValue } from "./primitives"; +import { HelpRequested, throwUnknownOption } from "./shared"; + +export function parseRuntimeArgs(args: string[]): RuntimeOptions { + const options: RuntimeOptions = { format: "markdown" }; + const valueOptions = new Map([ + ["cwd", "cwd"], + ["telemetry", "telemetry"], + ["errors", "errors"], + ["topology", "topology"], + ["head-revision", "headRevision"], + ["environment", "environment"], + ["output", "output"] + ]); + + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (!arg) continue; + if (arg === "--help" || arg === "-h") throw new HelpRequested(); + if (arg.startsWith("--format=")) { + options.format = parseFormat(arg.slice("--format=".length)); + continue; + } + if (arg === "--format") { + options.format = parseFormat(requireValue(args, index, arg)); + index += 1; + continue; + } + const match = /^--([^=]+)(?:=(.*))?$/.exec(arg); + const key = match?.[1] ? valueOptions.get(match[1]) : undefined; + if (key) { + const value = match?.[2] ?? requireValue(args, index, arg); + options[key] = value as never; + if (match?.[2] === undefined) index += 1; + continue; + } + throwUnknownOption(arg, "runtime"); + } + return options; +} + +function parseFormat(value: string): RuntimeOptions["format"] { + if (value === "json" || value === "markdown") return value; + throw new Error(`Invalid runtime format "${value}". Expected json or markdown.`); +} diff --git a/packages/cli/src/types/index.ts b/packages/cli/src/types/index.ts index 71ced88..0a12fcc 100644 --- a/packages/cli/src/types/index.ts +++ b/packages/cli/src/types/index.ts @@ -17,5 +17,6 @@ export * from "./memory"; export * from "./product"; export * from "./redteam"; export * from "./revalidate"; +export * from "./runtime"; export * from "./session"; export * from "./snapshot"; diff --git a/packages/cli/src/types/runtime.ts b/packages/cli/src/types/runtime.ts new file mode 100644 index 0000000..791a2d3 --- /dev/null +++ b/packages/cli/src/types/runtime.ts @@ -0,0 +1,12 @@ +import type { ConfigFormat } from "./common"; + +export interface RuntimeOptions { + cwd?: string | undefined; + telemetry?: string | undefined; + errors?: string | undefined; + topology?: string | undefined; + headRevision?: string | undefined; + environment?: string | undefined; + format: ConfigFormat; + output?: string | undefined; +} diff --git a/packages/cli/test/runtime.test.ts b/packages/cli/test/runtime.test.ts new file mode 100644 index 0000000..759d360 --- /dev/null +++ b/packages/cli/test/runtime.test.ts @@ -0,0 +1,66 @@ +import { execFileSync } from "node:child_process"; +import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { runCli } from "../src/index"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +describe("codedecay runtime CLI", () => { + it("writes revision-aware, topology-correlated local evidence", async () => { + const root = createRepo(); + mkdirSync(join(root, ".codedecay", "runtime"), { recursive: true }); + writeFileSync(join(root, ".codedecay", "runtime", "traces.json"), JSON.stringify({ + resourceSpans: [{ + resource: { attributes: [attr("service.name", "api"), attr("service.version", "head")] }, + scopeSpans: [{ spans: [{ name: "GET /health", spanId: "abc", flags: 1, startTimeUnixNano: "0", endTimeUnixNano: "2000000", attributes: [attr("http.route", "/health")] }] }] + }] + }), "utf8"); + writeFileSync(join(root, "topology.json"), JSON.stringify({ + schemaVersion: 1, + nodes: [{ id: "service:api", kind: "service", label: "api", confidence: "declared", freshness: "unknown", trustClass: "declared-context", sources: [{ kind: "manifest", source: "fixture", repositoryId: "repo", revision: "head" }], limitations: [] }], + edges: [], + limitations: [] + }), "utf8"); + + const result = await run(["runtime", "--cwd", root, "--telemetry", ".codedecay/runtime/traces.json", "--topology", "topology.json", "--head-revision", "head", "--format", "json", "--output", "reports/runtime.json"]); + const report = JSON.parse(readFileSync(join(root, "reports", "runtime.json"), "utf8")) as { operations: Array<{ trust: string; topologyNodeIds: string[] }>; safety: { networkCalled: boolean } }; + + expect(result).toEqual({ exitCode: 0, stdout: "", stderr: "" }); + expect(report.operations[0]).toMatchObject({ trust: "current-revision", topologyNodeIds: ["service:api"] }); + expect(report.safety.networkCalled).toBe(false); + }); + + it("renders useful limitations with no configured exports", async () => { + const result = await run(["runtime", "--cwd", createRepo()]); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("No runtime operations were ingested."); + expect(result.stdout).toContain("No local OpenTelemetry export was configured"); + expect(result.stdout).toContain("No structured error export was configured"); + expect(result.stdout).toContain("no network or command execution"); + }); +}); + +async function run(args: string[]): Promise<{ exitCode: number; stdout: string; stderr: string }> { + let stdout = ""; + let stderr = ""; + const exitCode = await runCli(args, { stdout: (text) => { stdout += text; }, stderr: (text) => { stderr += text; } }); + return { exitCode, stdout, stderr }; +} + +function createRepo(): string { + const root = join(tmpdir(), `codedecay-runtime-cli-${Date.now()}-${Math.random().toString(16).slice(2)}`); + mkdirSync(root, { recursive: true }); + execFileSync("git", ["init", "-q"], { cwd: root }); + roots.push(root); + return root; +} + +function attr(key: string, value: string): unknown { + return { key, value: { stringValue: value } }; +} diff --git a/packages/knowledge/src/index.ts b/packages/knowledge/src/index.ts index cd07528..1aeb148 100644 --- a/packages/knowledge/src/index.ts +++ b/packages/knowledge/src/index.ts @@ -43,6 +43,17 @@ export { SERVICE_TOPOLOGY_NODE_KINDS, SERVICE_TOPOLOGY_SCHEMA_VERSION } from "./topology/types"; +export { ingestRuntimeEvidence } from "./runtime/ingest"; +export type { IngestRuntimeEvidenceOptions } from "./runtime/ingest"; +export { renderRuntimeEvidenceMarkdown } from "./runtime/render"; +export { RUNTIME_EVIDENCE_SCHEMA_VERSION } from "./runtime/types"; +export type { + RuntimeErrorEvidence, + RuntimeEvidenceReport, + RuntimeEvidenceSource, + RuntimeEvidenceTrust, + RuntimeOperationEvidence +} from "./runtime/types"; export type { ServiceTopologyConfidence, ServiceTopologyEdge, diff --git a/packages/knowledge/src/runtime/ingest.ts b/packages/knowledge/src/runtime/ingest.ts new file mode 100644 index 0000000..ab88f83 --- /dev/null +++ b/packages/knowledge/src/runtime/ingest.ts @@ -0,0 +1,299 @@ +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, realpathSync, statSync } from "node:fs"; +import { resolve } from "node:path"; +import type { ServiceTopologyGraph } from "../topology/types"; +import { + RUNTIME_EVIDENCE_SCHEMA_VERSION, + type RuntimeErrorEvidence, + type RuntimeEvidenceReport, + type RuntimeEvidenceSource, + type RuntimeEvidenceTrust, + type RuntimeOperationEvidence +} from "./types"; + +const SENSITIVE_KEY = /authorization|cookie|password|secret|token|api[-_]?key|request\.body|request_body|user\.email|client\.address/i; +const SENSITIVE_VALUE = /(?:bearer\s+[a-z0-9._~+/-]+=*|\b(?:sk|ghp|github_pat)_[a-z0-9_-]+|[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,})/gi; +const DEFAULT_MAX_SPANS = 5_000; +const DEFAULT_MAX_OPERATIONS = 200; +const DEFAULT_MAX_INPUT_BYTES = 10 * 1024 * 1024; + +export interface IngestRuntimeEvidenceOptions { + rootDir: string; + otlpPath?: string | undefined; + errorsPath?: string | undefined; + headRevision?: string | undefined; + environment?: string | undefined; + topology?: ServiceTopologyGraph | undefined; + maxSpans?: number | undefined; + maxOperations?: number | undefined; + maxInputBytes?: number | undefined; + generatedAt?: string | undefined; +} + +interface MutableStats { + spansRead: number; + spansDroppedByBounds: number; + malformedRecords: number; + redactedValues: number; +} + +interface SpanRecord { + service: string; + operation: string; + route?: string | undefined; + environment?: string | undefined; + revision?: string | undefined; + latencyMs: number; + error: boolean; + sampled: boolean; + sourceRef: string; +} + +export function ingestRuntimeEvidence(options: IngestRuntimeEvidenceOptions): RuntimeEvidenceReport { + const rootDir = realpathSync(options.rootDir); + const stats: MutableStats = { spansRead: 0, spansDroppedByBounds: 0, malformedRecords: 0, redactedValues: 0 }; + const sources: RuntimeEvidenceSource[] = []; + const limitations: string[] = []; + const maxInputBytes = options.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES; + const spans = options.otlpPath ? loadOtlp(rootDir, options.otlpPath, options.environment, options.maxSpans ?? DEFAULT_MAX_SPANS, maxInputBytes, stats, sources) : []; + const errors = options.errorsPath ? loadErrors(rootDir, options.errorsPath, options.headRevision, options.environment, maxInputBytes, stats, sources) : []; + if (!options.otlpPath) limitations.push("No local OpenTelemetry export was configured; runtime path exposure is unavailable."); + if (!options.errorsPath) limitations.push("No structured error export was configured; deployment-correlated errors are unavailable."); + const operations = aggregateOperations(spans, options.headRevision, options.topology, options.maxOperations ?? DEFAULT_MAX_OPERATIONS, stats); + const investigationTasks = createTasks(operations, errors); + if (stats.malformedRecords > 0) limitations.push(`${stats.malformedRecords} malformed runtime record(s) were ignored; the report may be incomplete.`); + if (stats.spansDroppedByBounds > 0) limitations.push(`${stats.spansDroppedByBounds} runtime record(s) were omitted by cardinality bounds.`); + + return { + tool: "CodeDecay", + schemaVersion: RUNTIME_EVIDENCE_SCHEMA_VERSION, + generatedAt: options.generatedAt ?? new Date().toISOString(), + headRevision: options.headRevision, + sources, + operations, + errors, + investigationTasks, + limitations, + stats, + safety: { + networkCalled: false, + commandsExecuted: false, + telemetrySent: false, + rawRequestBodiesPersisted: false, + secretsPersisted: false + } + }; +} + +function loadOtlp(rootDir: string, path: string, environment: string | undefined, maxSpans: number, maxInputBytes: number, stats: MutableStats, sources: RuntimeEvidenceSource[]): SpanRecord[] { + const sourcePath = resolveInput(rootDir, path); + const value = parseLocalJson(sourcePath, maxInputBytes, stats); + const resourceSpans = recordArray(value, "resourceSpans", stats); + const records: SpanRecord[] = []; + let sampled = false; + for (const resourceItem of resourceSpans) { + const resourceSpan = asRecord(resourceItem); + if (!resourceSpan) { stats.malformedRecords += 1; continue; } + const resourceAttributes = attributes(asRecord(resourceSpan.resource)?.attributes, stats); + const service = stringAttribute(resourceAttributes, "service.name") ?? "unknown-service"; + const revision = stringAttribute(resourceAttributes, "service.version") ?? stringAttribute(resourceAttributes, "vcs.ref.head.revision"); + const spanEnvironment = stringAttribute(resourceAttributes, "deployment.environment.name") ?? environment; + for (const scopeItem of recordArray(resourceSpan, "scopeSpans", stats)) { + const scopeSpan = asRecord(scopeItem); + if (!scopeSpan) { stats.malformedRecords += 1; continue; } + for (const spanItem of recordArray(scopeSpan, "spans", stats)) { + stats.spansRead += 1; + if (records.length >= maxSpans) { stats.spansDroppedByBounds += 1; continue; } + const span = asRecord(spanItem); + if (!span || typeof span.name !== "string") { stats.malformedRecords += 1; continue; } + const spanAttributes = attributes(span.attributes, stats); + const spanFlags = numberValue(span.flags); + const spanSampled = spanFlags !== undefined && (spanFlags & 1) === 1; + sampled ||= spanSampled; + records.push({ + service, + operation: redactText(span.name, stats), + route: stripQuery(stringAttribute(spanAttributes, "http.route") ?? stringAttribute(spanAttributes, "url.path")), + environment: spanEnvironment, + revision, + latencyMs: durationMs(span.startTimeUnixNano, span.endTimeUnixNano), + error: asRecord(span.status)?.code === 2 || Boolean(stringAttribute(spanAttributes, "error.type")), + sampled: spanSampled, + sourceRef: `${path}#span:${typeof span.spanId === "string" ? span.spanId : stats.spansRead}` + }); + } + } + } + sources.push({ kind: "otlp-json", path, environment, sampled, redacted: true, limitations: sampled ? ["Trace export is sampled and cannot prove absence of unobserved paths."] : [] }); + return records; +} + +function loadErrors(rootDir: string, path: string, headRevision: string | undefined, environment: string | undefined, maxInputBytes: number, stats: MutableStats, sources: RuntimeEvidenceSource[]): RuntimeErrorEvidence[] { + const value = parseLocalJson(resolveInput(rootDir, path), maxInputBytes, stats); + const records = recordArray(value, "errors", stats); + if (records.length > 500) stats.spansDroppedByBounds += records.length - 500; + const errors = records.slice(0, 500).flatMap((item, index) => { + const error = asRecord(item); + if (!error || typeof error.service !== "string" || typeof error.message !== "string") { stats.malformedRecords += 1; return []; } + const revision = optionalString(error.revision); + const group = optionalString(error.group) ?? `error-${index + 1}`; + return [{ + evidenceId: evidenceId(["error", path, group, revision ?? "unknown"]), + group: redactText(group, stats), + service: redactText(error.service, stats), + operation: optionalString(error.operation) ? redactText(String(error.operation), stats) : undefined, + message: redactText(error.message, stats), + count: Math.max(1, Math.floor(numberValue(error.count) ?? 1)), + environment: optionalString(error.environment) ?? environment, + revision, + firstSeen: validTimestamp(error.firstSeen), + lastSeen: validTimestamp(error.lastSeen), + trust: revisionTrust(revision, headRevision), + sourceRef: `${path}#error:${index + 1}`, + limitations: revision ? [] : ["Error export does not identify a deployment revision."] + } satisfies RuntimeErrorEvidence]; + }); + sources.push({ kind: "structured-errors", path, environment, sampled: false, redacted: true, limitations: [] }); + return errors; +} + +function aggregateOperations(spans: SpanRecord[], headRevision: string | undefined, topology: ServiceTopologyGraph | undefined, maxOperations: number, stats: MutableStats): RuntimeOperationEvidence[] { + const groups = new Map(); + for (const span of spans) { + const key = [span.service, span.operation, span.route ?? "", span.environment ?? "", span.revision ?? ""].join("\0"); + const existing = groups.get(key); + if (existing) existing.push(span); else if (groups.size < maxOperations) groups.set(key, [span]); else stats.spansDroppedByBounds += 1; + } + return [...groups.values()].map((items) => { + const first = items[0] as SpanRecord; + const topologyNodeIds = correlateTopology(topology, first.service, first.route); + const trust = revisionTrust(first.revision, headRevision); + const totalLatency = items.reduce((sum, item) => sum + item.latencyMs, 0); + return { + evidenceId: evidenceId(["operation", first.service, first.operation, first.route ?? "", first.revision ?? "unknown"]), + service: first.service, + operation: first.operation, + route: first.route, + environment: first.environment, + revision: first.revision, + spanCount: items.length, + errorCount: items.filter((item) => item.error).length, + maxLatencyMs: Math.max(...items.map((item) => item.latencyMs)), + averageLatencyMs: Math.round((totalLatency / items.length) * 100) / 100, + sampled: items.some((item) => item.sampled), + trust, + topologyNodeIds, + sourceRefs: items.slice(0, 20).map((item) => item.sourceRef), + limitations: [ + ...(items.some((item) => item.sampled) ? ["Sampled traces cannot prove absence of failures."] : []), + ...(trust !== "current-revision" ? ["Runtime evidence does not exactly match the current head revision."] : []) + ] + }; + }).sort((left, right) => right.errorCount - left.errorCount || right.maxLatencyMs - left.maxLatencyMs || left.evidenceId.localeCompare(right.evidenceId)); +} + +function correlateTopology(topology: ServiceTopologyGraph | undefined, service: string, route: string | undefined): string[] { + if (!topology) return []; + const normalizedService = service.toLowerCase(); + const normalizedRoute = route?.toLowerCase(); + return topology.nodes.filter((node) => { + const metadataRoute = typeof node.metadata?.route === "string" ? node.metadata.route.toLowerCase() : undefined; + return node.id.toLowerCase() === `service:${normalizedService}` || node.label.toLowerCase() === normalizedService || Boolean(normalizedRoute && metadataRoute === normalizedRoute); + }).map((node) => node.id).sort(); +} + +function createTasks(operations: RuntimeOperationEvidence[], errors: RuntimeErrorEvidence[]): string[] { + return [ + ...operations.filter((item) => item.errorCount > 0).map((item) => `Reproduce ${item.errorCount} observed error span(s) for ${item.service} ${item.route ?? item.operation} against the current tree.`), + ...operations.filter((item) => item.maxLatencyMs >= 1_000).map((item) => `Verify the ${item.maxLatencyMs}ms runtime hotspot for ${item.service} ${item.route ?? item.operation} with a bounded local performance check.`), + ...errors.map((item) => `Investigate runtime error group ${item.group} (${item.count} event(s)) for ${item.service}; do not treat the export as current-tree proof.`) + ]; +} + +function resolveInput(rootDir: string, path: string): string { + const lexical = resolve(rootDir, path); + if (lexical !== rootDir && !lexical.startsWith(`${rootDir}/`)) throw new Error(`Runtime evidence path must stay inside repository: ${path}`); + if (!existsSync(lexical)) throw new Error(`Runtime evidence file not found: ${path}`); + const real = realpathSync(lexical); + if (real !== rootDir && !real.startsWith(`${rootDir}/`)) throw new Error(`Runtime evidence path must stay inside repository: ${path}`); + return real; +} + +function parseLocalJson(path: string, maxInputBytes: number, stats: MutableStats): unknown { + const size = statSync(path).size; + if (size > maxInputBytes) throw new Error(`Runtime evidence file exceeds ${maxInputBytes} byte limit: ${path}`); + try { return JSON.parse(readFileSync(path, "utf8")) as unknown; } catch { stats.malformedRecords += 1; return {}; } +} + +function attributes(value: unknown, stats: MutableStats): Map { + const result = new Map(); + if (!Array.isArray(value)) return result; + for (const item of value) { + const attribute = asRecord(item); + if (!attribute || typeof attribute.key !== "string") { stats.malformedRecords += 1; continue; } + if (SENSITIVE_KEY.test(attribute.key)) { stats.redactedValues += 1; result.set(attribute.key, "[REDACTED]"); continue; } + const raw = asRecord(attribute.value); + const scalar = raw?.stringValue ?? raw?.intValue ?? raw?.doubleValue ?? raw?.boolValue; + if (["string", "number", "boolean"].includes(typeof scalar)) result.set(attribute.key, typeof scalar === "string" ? redactText(scalar, stats) : scalar as number | boolean); + } + return result; +} + +function stringAttribute(values: Map, key: string): string | undefined { + const value = values.get(key); + return typeof value === "string" && value !== "[REDACTED]" ? value : undefined; +} + +function redactText(value: string, stats: MutableStats): string { + let redacted = stripQuery(value) ?? ""; + redacted = redacted.replace(SENSITIVE_VALUE, () => { stats.redactedValues += 1; return "[REDACTED]"; }); + return redacted.slice(0, 500); +} + +function stripQuery(value: string | undefined): string | undefined { + if (!value) return value; + const queryIndex = value.indexOf("?"); + return queryIndex >= 0 ? value.slice(0, queryIndex) : value; +} + +function revisionTrust(revision: string | undefined, headRevision: string | undefined): RuntimeEvidenceTrust { + if (!revision || !headRevision) return revision ? "unmatched" : "inferred"; + return revision === headRevision ? "current-revision" : "historical"; +} + +function durationMs(start: unknown, end: unknown): number { + try { + const duration = Number(BigInt(String(end ?? 0)) - BigInt(String(start ?? 0))) / 1_000_000; + return Number.isFinite(duration) && duration >= 0 ? Math.round(duration * 100) / 100 : 0; + } catch { return 0; } +} + +function recordArray(value: unknown, key: string, stats: MutableStats): unknown[] { + const record = asRecord(value); + const items = record?.[key]; + if (items === undefined) return []; + if (!Array.isArray(items)) { stats.malformedRecords += 1; return []; } + return items; +} + +function asRecord(value: unknown): Record | undefined { + return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : undefined; +} + +function numberValue(value: unknown): number | undefined { + const number = typeof value === "number" ? value : typeof value === "string" ? Number(value) : Number.NaN; + return Number.isFinite(number) ? number : undefined; +} + +function optionalString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function validTimestamp(value: unknown): string | undefined { + const text = optionalString(value); + return text && Number.isFinite(new Date(text).getTime()) ? text : undefined; +} + +function evidenceId(parts: string[]): string { + return `runtime:${createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 20)}`; +} diff --git a/packages/knowledge/src/runtime/render.ts b/packages/knowledge/src/runtime/render.ts new file mode 100644 index 0000000..fe8631f --- /dev/null +++ b/packages/knowledge/src/runtime/render.ts @@ -0,0 +1,26 @@ +import type { RuntimeEvidenceReport } from "./types"; + +export function renderRuntimeEvidenceMarkdown(report: RuntimeEvidenceReport): string { + const lines = [ + "## CodeDecay Runtime Evidence", + "", + `Head revision: \`${report.headRevision ?? "unknown"}\``, + `Sources: ${report.sources.length}; spans read: ${report.stats.spansRead}; bounded drops: ${report.stats.spansDroppedByBounds}; malformed: ${report.stats.malformedRecords}`, + "", + "### Runtime Operations", + "" + ]; + if (report.operations.length === 0) lines.push("No runtime operations were ingested.", ""); + for (const item of report.operations) lines.push(`- **${item.service} ${item.route ?? item.operation}**: ${item.spanCount} span(s), ${item.errorCount} error(s), max ${item.maxLatencyMs}ms; trust \`${item.trust}\`.`); + lines.push("", "### Correlated Errors", ""); + if (report.errors.length === 0) lines.push("No structured error groups were ingested.", ""); + for (const item of report.errors) lines.push(`- **${item.group}**: ${item.count} event(s) for ${item.service}; trust \`${item.trust}\`; source \`${item.sourceRef}\`.`); + lines.push("", "### Investigation Tasks", ""); + if (report.investigationTasks.length === 0) lines.push("No runtime investigation task was generated."); + for (const task of report.investigationTasks) lines.push(`- ${task}`); + lines.push("", "### Limitations", ""); + if (report.limitations.length === 0) lines.push("No ingestion limitation was reported."); + for (const limitation of report.limitations) lines.push(`- ${limitation}`); + lines.push("", "### Safety", "", "- Local artifact ingestion only; no network or command execution.", "- Sensitive attributes, query strings, authorization data, request bodies, tokens, and email addresses are redacted before report assembly.", "- Historical or sampled runtime evidence cannot prove the current tree safe.", ""); + return `${lines.join("\n")}\n`; +} diff --git a/packages/knowledge/src/runtime/types.ts b/packages/knowledge/src/runtime/types.ts new file mode 100644 index 0000000..9abb7a0 --- /dev/null +++ b/packages/knowledge/src/runtime/types.ts @@ -0,0 +1,73 @@ +export const RUNTIME_EVIDENCE_SCHEMA_VERSION = 1 as const; + +export type RuntimeEvidenceTrust = "current-revision" | "historical" | "unmatched" | "inferred"; + +export interface RuntimeEvidenceSource { + kind: "otlp-json" | "structured-errors"; + path: string; + collectionStart?: string | undefined; + collectionEnd?: string | undefined; + environment?: string | undefined; + sampled: boolean; + redacted: true; + limitations: string[]; +} + +export interface RuntimeOperationEvidence { + evidenceId: string; + service: string; + operation: string; + route?: string | undefined; + environment?: string | undefined; + revision?: string | undefined; + spanCount: number; + errorCount: number; + maxLatencyMs: number; + averageLatencyMs: number; + sampled: boolean; + trust: RuntimeEvidenceTrust; + topologyNodeIds: string[]; + sourceRefs: string[]; + limitations: string[]; +} + +export interface RuntimeErrorEvidence { + evidenceId: string; + group: string; + service: string; + operation?: string | undefined; + message: string; + count: number; + environment?: string | undefined; + revision?: string | undefined; + firstSeen?: string | undefined; + lastSeen?: string | undefined; + trust: RuntimeEvidenceTrust; + sourceRef: string; + limitations: string[]; +} + +export interface RuntimeEvidenceReport { + tool: "CodeDecay"; + schemaVersion: typeof RUNTIME_EVIDENCE_SCHEMA_VERSION; + generatedAt: string; + headRevision?: string | undefined; + sources: RuntimeEvidenceSource[]; + operations: RuntimeOperationEvidence[]; + errors: RuntimeErrorEvidence[]; + investigationTasks: string[]; + limitations: string[]; + stats: { + spansRead: number; + spansDroppedByBounds: number; + malformedRecords: number; + redactedValues: number; + }; + safety: { + networkCalled: false; + commandsExecuted: false; + telemetrySent: false; + rawRequestBodiesPersisted: false; + secretsPersisted: false; + }; +} diff --git a/packages/knowledge/test/runtime-evidence.test.ts b/packages/knowledge/test/runtime-evidence.test.ts new file mode 100644 index 0000000..d8da6b0 --- /dev/null +++ b/packages/knowledge/test/runtime-evidence.test.ts @@ -0,0 +1,106 @@ +import { mkdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { ingestRuntimeEvidence, normalizeServiceTopologyGraph } from "../src/index"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +describe("runtime evidence ingestion", () => { + it("correlates current-revision traces while redacting sensitive data", () => { + const root = tempRoot(); + writeJson(join(root, "traces.json"), otlp([ + span("GET /users?token=secret", "route-1", 1, { + "http.route": "/users?authorization=Bearer abc.def", + "user.email": "person@example.com", + "error.type": "timeout" + }) + ], "api", "head-sha")); + writeJson(join(root, "errors.json"), { + errors: [{ service: "api", group: "users?token=secret", message: "failed for person@example.com with ghp_abcdefghijklmnopqrstuvwxyz", revision: "old-sha", count: 3 }] + }); + const topology = normalizeServiceTopologyGraph({ + schemaVersion: 1, + nodes: [{ id: "service:api", kind: "service", label: "api", confidence: "declared", freshness: "unknown", trustClass: "declared-context", sources: [{ kind: "manifest", source: "fixture", repositoryId: "repo", revision: "head-sha" }], limitations: [] }], + edges: [], + limitations: [] + }); + + const report = ingestRuntimeEvidence({ rootDir: root, otlpPath: "traces.json", errorsPath: "errors.json", headRevision: "head-sha", topology, generatedAt: "2026-08-02T00:00:00.000Z" }); + const serialized = JSON.stringify(report); + + expect(report.operations[0]).toMatchObject({ route: "/users", spanCount: 1, errorCount: 1, sampled: true, trust: "current-revision", topologyNodeIds: ["service:api"] }); + expect(report.errors[0]).toMatchObject({ group: "users", count: 3, trust: "historical" }); + expect(report.stats.redactedValues).toBeGreaterThanOrEqual(2); + expect(serialized).not.toContain("person@example.com"); + expect(serialized).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz"); + expect(serialized).not.toContain("Bearer abc.def"); + expect(report.safety).toEqual({ networkCalled: false, commandsExecuted: false, telemetrySent: false, rawRequestBodiesPersisted: false, secretsPersisted: false }); + }); + + it("bounds high-cardinality input and degrades on malformed records", () => { + const root = tempRoot(); + writeJson(join(root, "traces.json"), otlp([ + span("one", "1", 0), + span("two", "2", 0), + { spanId: "bad" } + ], "api", "head")); + writeFileSync(join(root, "errors.json"), "{not json", "utf8"); + + const report = ingestRuntimeEvidence({ rootDir: root, otlpPath: "traces.json", errorsPath: "errors.json", headRevision: "head", maxSpans: 1 }); + + expect(report.operations).toHaveLength(1); + expect(report.operations[0]?.sampled).toBe(false); + expect(report.stats).toMatchObject({ spansRead: 3, spansDroppedByBounds: 2, malformedRecords: 1 }); + expect(report.errors).toEqual([]); + expect(report.limitations).toEqual(expect.arrayContaining([ + expect.stringContaining("malformed runtime record"), + expect.stringContaining("omitted by cardinality bounds") + ])); + }); + + it("rejects oversized and symlinked evidence outside the repository", () => { + const root = tempRoot(); + const outside = tempRoot(); + writeFileSync(join(root, "large.json"), "12345", "utf8"); + writeJson(join(outside, "trace.json"), otlp([], "api", "head")); + symlinkSync(join(outside, "trace.json"), join(root, "linked.json")); + + expect(() => ingestRuntimeEvidence({ rootDir: root, otlpPath: "large.json", maxInputBytes: 4 })).toThrow("exceeds 4 byte limit"); + expect(() => ingestRuntimeEvidence({ rootDir: root, otlpPath: "linked.json" })).toThrow("must stay inside repository"); + }); + + it("reports explicit limitations when no providers are configured", () => { + const report = ingestRuntimeEvidence({ rootDir: tempRoot(), generatedAt: "2026-08-02T00:00:00.000Z" }); + expect(report.sources).toEqual([]); + expect(report.limitations).toHaveLength(2); + expect(report.investigationTasks).toEqual([]); + }); +}); + +function tempRoot(): string { + const root = join(tmpdir(), `codedecay-runtime-${Date.now()}-${Math.random().toString(16).slice(2)}`); + mkdirSync(root, { recursive: true }); + roots.push(root); + return root; +} + +function writeJson(path: string, value: unknown): void { + writeFileSync(path, JSON.stringify(value), "utf8"); +} + +function otlp(spans: unknown[], service: string, revision: string): unknown { + return { resourceSpans: [{ resource: { attributes: [attribute("service.name", service), attribute("service.version", revision)] }, scopeSpans: [{ spans }] }] }; +} + +function span(name: string, spanId: string, flags: number, values: Record = {}): unknown { + return { name, spanId, flags, startTimeUnixNano: "1000000", endTimeUnixNano: "6000000", status: { code: 0 }, attributes: Object.entries(values).map(([key, value]) => attribute(key, value)) }; +} + +function attribute(key: string, value: string): unknown { + return { key, value: { stringValue: value } }; +}