feat(provenance): content-derived Agent fingerprint for scoreboard

packages/cli/src/commands/provenance.ts

@@ -0,0 +1,70 @@
+/**
+ * `penguin provenance` — prints an Agent's content-derived reproducibility fingerprint.
+ *
+ *   penguin provenance [--agent-id <id>] [--project-id <id>] [--root <dir>]
+ *                      [--provider <p> --model-id <m>] [--format yaml|json]
+ *
+ * This is the deterministic capture path the self-evolution skills call: `agent-optimization`
+ * runs it against the Test Agent before writing a scoreboard record, and embeds the returned
+ * block as that evaluation's `provenance:` field. An LLM cannot reliably compute sha256, so the
+ * fingerprint must come from code — hence a CLI command rather than a skill instruction.
+ *
+ * Output goes to stdout only (no logs), so a skill can consume it verbatim; `--format yaml`
+ * (default) drops straight into scoreboard.yaml, `--format json` is available for programmatic use.
+ * Docs: /docs/cli § "penguin provenance".
+ */
+import path from "node:path";
+import type { Command } from "commander";
+import {
+  DEFAULT_AGENT_ID,
+  DEFAULT_PROJECT_ID,
+  buildAgentProvenance,
+  loadOrInitAgentState,
+  resolveRoot,
+  type ProvenanceModelRef,
+} from "@prismshadow/penguin-core";
+import { stringify as stringifyYaml } from "yaml";
+import type { Messages } from "../i18n.js";
+
+function resolveRootOption(root: string | undefined): string {
+  return root !== undefined ? path.resolve(root) : resolveRoot();
+}
+
+export function registerProvenanceCommand(program: Command, t: Messages): void {
+  program
+    .command("provenance")
+    .description(t.provenance.desc)
+    .option("--agent-id <id>", t.common.agentId, DEFAULT_AGENT_ID)
+    .option("--project-id <id>", t.common.projectId, DEFAULT_PROJECT_ID)
+    .option("--provider <group>", t.provenance.provider)
+    .option("--model-id <id>", t.provenance.modelId)
+    .option("--format <fmt>", t.provenance.format, "yaml")
+    .option("--root <dir>", t.common.root)
+    .action(async (opts) => {
+      const root = resolveRootOption(opts.root);
+      // A model reference is always the complete (provider, model_id) pair — reject a lone half.
+      if ((opts.provider === undefined) !== (opts.modelId === undefined)) {
+        throw new Error("--provider and --model-id must be given together (a model is a pair).");
+      }
+      const model: ProvenanceModelRef | undefined =
+        opts.provider !== undefined && opts.modelId !== undefined
+          ? { provider: opts.provider, model_id: opts.modelId }
+          : undefined;
+
+      const state = await loadOrInitAgentState({
+        root,
+        projectId: opts.projectId,
+        agentId: opts.agentId,
+      });
+      const provenance = await buildAgentProvenance(state, model ? { model } : undefined);
+
+      const format = String(opts.format).toLowerCase();
+      if (format === "json") {
+        process.stdout.write(`${JSON.stringify(provenance, null, 2)}\n`);
+      } else if (format === "yaml") {
+        process.stdout.write(stringifyYaml(provenance));
+      } else {
+        throw new Error(`Unknown --format ${JSON.stringify(opts.format)}: expected yaml or json.`);
+      }
+    });
+}

packages/cli/src/i18n.ts

@@ -73,6 +73,12 @@ export interface Messages {
     /** run's --goal: goal mode, with an optional token budget value (`--goal 500k`). */
     goal: string;
   };
+  provenance: {
+    desc: string;
+    format: string;
+    provider: string;
+    modelId: string;
+  };
   chat: { desc: string; resume: string };
   serve: {
     serverDesc: string;
@@ -319,6 +325,12 @@ const en: Messages = {
     message: "Prompt for this Task",
     goal: "Goal mode: loop until the goal completes; optional token budget (e.g. 500k, 2m)",
   },
+  provenance: {
+    desc: "Print the Agent's content-derived reproducibility fingerprint (for scoreboard records)",
+    format: "Output format: yaml (default) or json",
+    provider: "Provider group of the evaluation model (folded into the fingerprint; pairs with --model-id)",
+    modelId: "Upstream model id of the evaluation model (pairs with --provider)",
+  },
   chat: {
     desc: "Open the interactive REPL",
     resume:
@@ -537,6 +549,12 @@ const zh: Messages = {
     message: "本次 Task 的 Prompt",
     goal: "目标模式：循环运行直至目标完成；可选 token 预算（如 500k、2m）",
   },
+  provenance: {
+    desc: "打印 Agent 的内容派生可复现指纹（用于 scoreboard 记录）",
+    format: "输出格式：yaml（缺省）或 json",
+    provider: "评测模型的 provider 分组（并入指纹；与 --model-id 配对）",
+    modelId: "评测模型的上游 model id（与 --provider 配对）",
+  },
   chat: {
     desc: "打开交互式 REPL",
     resume:

packages/cli/src/index.ts

@@ -17,6 +17,7 @@ import { Command } from "commander";
 import { VERSION } from "@prismshadow/penguin-core";
 import { registerConfigCommand } from "./commands/config.js";
 import { registerRunCommand } from "./commands/run.js";
+import { registerProvenanceCommand } from "./commands/provenance.js";
 import { registerChatCommand } from "./commands/chat.js";
 import { registerServeCommands } from "./commands/serve.js";
 import { registerUpdateCommand } from "./commands/update.js";
@@ -34,6 +35,7 @@ program
 
 registerConfigCommand(program, t);
 registerRunCommand(program, t);
+registerProvenanceCommand(program, t);
 registerChatCommand(program, t);
 registerServeCommands(program, t);
 registerUpdateCommand(program, t);

packages/core/src/state/example-benchmark.ts

@@ -79,6 +79,46 @@ interface ExampleRun {
   session_id: string;
 }
 
+/**
+ * Example provenance fingerprints (illustrative only — not computed from a real Agent). The
+ * three rounds share the same tools/skills hashes (nothing was installed or uninstalled) while
+ * the system prompt hash and the top-level agent hash change each round: this is exactly the
+ * localization story provenance is meant to tell — "the prompt changed, the skills didn't". Real
+ * fingerprints come from `penguin provenance`.
+ */
+const EXAMPLE_TOOLS_SHA = "08910c743d74efbbe547657603461d72c8fea7361e3bbc88e6a9cac896374ef3";
+const EXAMPLE_SKILLS_SHA = "69bf7c0181bb8cc80044504c9681ff87b34b706811f1bb6a5f17eb642ceb497e";
+const EXAMPLE_AGENTS_MD_SHA =
+  "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+
+/** A shape-valid 64-char hex string seeded by `tag` + `v` (illustrative, not a real digest). */
+function fakeHex(tag: string, v: number): string {
+  const seed = `${tag}${v}`;
+  let hex = "";
+  for (let i = 0; hex.length < 64; i += 1) {
+    hex += (((seed.charCodeAt(i % seed.length) * 7 + i * 31 + v) % 16) & 0xf).toString(16);
+  }
+  return hex.slice(0, 64);
+}
+
+/** Builds an illustrative provenance block for example version `v` (fake but shape-valid hashes). */
+function exampleProvenance(v: number): Record<string, unknown> {
+  return {
+    provenance_version: 1,
+    version: v,
+    // Changes each round (the prompt was edited every version).
+    system_prompt_sha256: fakeHex("sysprompt", v),
+    agents_md_sha256: EXAMPLE_AGENTS_MD_SHA,
+    // Constant across rounds (no tool/skill install or uninstall).
+    tools_sha256: EXAMPLE_TOOLS_SHA,
+    skills_sha256: EXAMPLE_SKILLS_SHA,
+    model: { provider: "deepseek", model_id: "deepseek-v4-pro" },
+    thinking_level: "medium",
+    // Top-level fingerprint changes each round because the prompt hash feeds into it.
+    agent_sha256: fakeHex("agent", v),
+  };
+}
+
 /**
  * Raw runs for the three sample evaluations (case-level and evaluation-level metrics are
  * computed from these, keeping the numbers self-consistent). Each carries the model actually
@@ -279,6 +319,7 @@ export function buildExampleScoreboard(): {
     score: number;
     cost: number | null;
     duration_ms: number;
+    provenance: Record<string, unknown>;
     cases: Array<{
       case: string;
       score: number;
@@ -308,6 +349,7 @@ export function buildExampleScoreboard(): {
         score: averageTwo(cases.map((c) => c.score)),
         cost: averageKnownCost(cases.map((c) => c.cost)),
         duration_ms: averageDuration(cases.map((c) => c.duration_ms)),
+        provenance: exampleProvenance(e.version),
         cases,
       };
     }),

packages/core/src/state/index.ts

@@ -11,6 +11,7 @@ export * from "./project-config.js";
 export * from "./agent-state.js";
 export * from "./agent-vault.js";
 export * from "./example-benchmark.js";
+export * from "./provenance.js";
 
 // Skill library types and frontmatter parser (from the skills package; server reuses the same implementation via core).
 export { parseSkillFrontmatter, type SkillMetadata } from "@prismshadow/penguin-skills";

packages/core/src/state/provenance.ts

@@ -0,0 +1,163 @@
+/**
+ * Agent provenance fingerprint: a content-derived hash of the editable inputs that
+ * determine an Agent's behavior (the system prompt template, AGENTS.md, installed skills,
+ * tool contract, and model parameters).
+ *
+ * Why: the self-evolution loop appends an evaluation to `scoreboard.yaml` each round, but a
+ * record only carries `version` + `provider` + `model_id`. `version` is a human-assigned
+ * monotonic integer, not content-derived — so editing the system prompt / AGENTS.md / a skill
+ * without bumping `version` leaves two evaluations looking identical while testing different
+ * Agents, and score differences become unattributable. `agent_sha256` answers "did the config
+ * change?" in one glance; the per-part sub-hashes answer "what changed?".
+ *
+ * The fingerprint hashes the **raw config inputs** (before placeholder substitution), NOT the
+ * assembled `session_meta.system_prompt` — the assembled prompt embeds per-run values
+ * (`{{DATE}}`, `{{SESSION_ID}}`, `{{CWD}}`) that vary every run and would make the hash useless
+ * as a config identity.
+ *
+ * Reading follows the same degrade-to-null discipline as the rest of Agent State loading:
+ * a missing AGENTS.md / skill file hashes an empty string rather than throwing.
+ */
+import { createHash } from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { agentStateVersion } from "./default-config.js";
+import { agentsMdPath, skillsDir } from "./paths.js";
+import { buildToolConfig, listInstalledSkills, type AgentState } from "./agent-state.js";
+
+/** Fingerprint of a single installed skill. */
+export interface AgentSkillFingerprint {
+  name: string;
+  version: number;
+  /** sha256 of the skill's full SKILL.md content. */
+  sha256: string;
+}
+
+/** Optional model reference for the fingerprint (the model an evaluation ran on). */
+export interface ProvenanceModelRef {
+  provider: string;
+  model_id: string;
+}
+
+/**
+ * Content-derived provenance fingerprint of an Agent State.
+ * Docs: /docs/self-improvement § "Provenance".
+ */
+export interface AgentProvenance {
+  provenance_version: 1;
+  /** Agent State version number (the `version` in system_config.yaml; treated as 1 if missing). */
+  version: number;
+  /** sha256 of the raw system_prompt template (before placeholder substitution). */
+  system_prompt_sha256: string;
+  /** sha256 of the full AGENTS.md content (empty string when the file is missing). */
+  agents_md_sha256: string;
+  /** sha256 of the canonicalized builtin tool contract (name + description + parameters). */
+  tools_sha256: string;
+  /** Per-skill fingerprints, sorted by name. */
+  skills: AgentSkillFingerprint[];
+  /** Order-independent combined hash of the skills array. */
+  skills_sha256: string;
+  /** The model this fingerprint is paired with (optional; e.g. the evaluation's model). */
+  model?: ProvenanceModelRef;
+  /** The effective thinking level (optional). */
+  thinking_level?: string;
+  /** Top-level fingerprint combining every field above — the single "did config change?" answer. */
+  agent_sha256: string;
+}
+
+/** Recursively sort object keys so `JSON.stringify` is stable across machines/insertion order. */
+function canonicalize(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(canonicalize);
+  if (value !== null && typeof value === "object") {
+    const out: Record<string, unknown> = {};
+    for (const key of Object.keys(value as Record<string, unknown>).sort()) {
+      out[key] = canonicalize((value as Record<string, unknown>)[key]);
+    }
+    return out;
+  }
+  return value;
+}
+
+/** sha256 hex of a string. */
+function sha256(text: string): string {
+  return createHash("sha256").update(text, "utf8").digest("hex");
+}
+
+/** sha256 hex of a value's canonical JSON form (key-order independent). */
+function sha256Canonical(value: unknown): string {
+  return sha256(JSON.stringify(canonicalize(value)));
+}
+
+/**
+ * Builds the content-derived provenance fingerprint for an Agent State. The optional model /
+ * thinking level are folded into the top-level `agent_sha256` when provided (so an evaluation on
+ * a different model gets a distinct fingerprint), and echoed as their own fields.
+ */
+export async function buildAgentProvenance(
+  state: AgentState,
+  opts?: { model?: ProvenanceModelRef; thinkingLevel?: string },
+): Promise<AgentProvenance> {
+  const version = agentStateVersion(state.systemConfig);
+
+  // Raw template, NOT the assembled prompt — assembly injects per-run environment values.
+  const systemPromptSha = sha256(state.systemConfig.system_prompt);
+
+  // AGENTS.md: degrade to an empty-string hash when missing (matches agent-state loading).
+  const agentsMdText = await readFileOrEmpty(
+    agentsMdPath(state.root, state.projectId, state.agentId),
+  );
+  const agentsMdSha = sha256(agentsMdText);
+
+  // Tool contract: the full builtin tool definitions (name + description + parameters), since
+  // description-driven behavior is part of what the model sees.
+  const tools = buildToolConfig(state).customTools.map((t) => ({
+    name: t.name,
+    description: t.description,
+    parameters: t.parameters ?? null,
+  }));
+  const toolsSha = sha256Canonical(tools);
+
+  // Skills: full SKILL.md content per installed skill, sorted by name.
+  const installed = await listInstalledSkills(state.root, state.projectId, state.agentId);
+  const dir = skillsDir(state.root, state.projectId, state.agentId);
+  const skills: AgentSkillFingerprint[] = [];
+  for (const skill of installed) {
+    const content = await readFileOrEmpty(path.join(dir, skill.name, "SKILL.md"));
+    skills.push({ name: skill.name, version: skill.version, sha256: sha256(content) });
+  }
+  skills.sort((a, b) => a.name.localeCompare(b.name));
+  const skillsSha = sha256Canonical(skills);
+
+  const agentSha = sha256Canonical({
+    provenance_version: 1,
+    version,
+    system_prompt_sha256: systemPromptSha,
+    agents_md_sha256: agentsMdSha,
+    tools_sha256: toolsSha,
+    skills_sha256: skillsSha,
+    ...(opts?.model ? { model: opts.model } : {}),
+    ...(opts?.thinkingLevel !== undefined ? { thinking_level: opts.thinkingLevel } : {}),
+  });
+
+  return {
+    provenance_version: 1,
+    version,
+    system_prompt_sha256: systemPromptSha,
+    agents_md_sha256: agentsMdSha,
+    tools_sha256: toolsSha,
+    skills,
+    skills_sha256: skillsSha,
+    ...(opts?.model ? { model: opts.model } : {}),
+    ...(opts?.thinkingLevel !== undefined ? { thinking_level: opts.thinkingLevel } : {}),
+    agent_sha256: agentSha,
+  };
+}
+
+/** Reads a file's text, returning "" when it's missing (degrade-to-null discipline). */
+async function readFileOrEmpty(filePath: string): Promise<string> {
+  try {
+    return await fs.readFile(filePath, "utf8");
+  } catch {
+    return "";
+  }
+}