From e50f7601c8d853e2b6a49bb722fbec6ad3a23cc9 Mon Sep 17 00:00:00 2001 From: Sesh Nalla Date: Tue, 26 May 2026 01:58:37 -0400 Subject: [PATCH] feat(directed-evolution): stackoverflow-agents seed app + agent-user simulator - os-apps/stackoverflow-agents: Q&A seed organism (Question/Answer, pure IOA, Downvote deliberately ABSENT; cascade ALL PASSED L0-L3). The organism the directed-evolution loop grows. - scripts/so-simulator: deterministic Node ESM agent-user simulator that seeds Q&A, tries the (missing) Downvote, and emits the unmet intent + Evolution. Part of the directed-evolution system (see genesis + temper PRs). DRAFT: DST/code reviews + .proofs/ report pending before merge. Co-Authored-By: Claude Opus 4.7 --- os-apps/stackoverflow-agents/APP.md | 17 + os-apps/stackoverflow-agents/app.toml | 4 + .../policies/answer.cedar | 11 + .../policies/question.cedar | 10 + .../specs/answer.ioa.toml | 58 +++ .../stackoverflow-agents/specs/model.csdl.xml | 134 +++++ .../specs/question.ioa.toml | 49 ++ scripts/so-simulator/README.md | 111 ++++ scripts/so-simulator/index.mjs | 487 ++++++++++++++++++ scripts/so-simulator/lib.mjs | 191 +++++++ 10 files changed, 1072 insertions(+) create mode 100644 os-apps/stackoverflow-agents/APP.md create mode 100644 os-apps/stackoverflow-agents/app.toml create mode 100644 os-apps/stackoverflow-agents/policies/answer.cedar create mode 100644 os-apps/stackoverflow-agents/policies/question.cedar create mode 100644 os-apps/stackoverflow-agents/specs/answer.ioa.toml create mode 100644 os-apps/stackoverflow-agents/specs/model.csdl.xml create mode 100644 os-apps/stackoverflow-agents/specs/question.ioa.toml create mode 100644 scripts/so-simulator/README.md create mode 100755 scripts/so-simulator/index.mjs create mode 100644 scripts/so-simulator/lib.mjs diff --git a/os-apps/stackoverflow-agents/APP.md b/os-apps/stackoverflow-agents/APP.md new file mode 100644 index 000000000..c963afbe8 --- /dev/null +++ b/os-apps/stackoverflow-agents/APP.md @@ -0,0 +1,17 @@ +# stackoverflow-agents + +Q&A for AI agents — the **seed organism** for directed evolution. Deliberately minimal so the evolution loop can grow it. + +## Entities +- **Question** — `Open → Answered → Closed`. `has_accepted` (bool). Actions: `AcceptAnswer`, `Close`. +- **Answer** — `Active → Accepted → Deleted`. `upvotes` (counter). Actions: `Upvote`, `Accept`, `Delete`. + +## The deliberate gap +There is **no `Downvote`**. Agents will want to bury low-quality answers; the directed-evolution loop observes that unmet intent and grows a `Downvote` action + `downvotes` counter onto `Answer` — gated by the verification cascade before it deploys. This is the Phase-1 "first light" episode. + +## Invariants (cascade-checked) +- `AnsweredRequiresAccepted` — a Question in `Answered` has an accepted answer (`has_accepted`). +- `ClosedIsFinal` / `DeletedIsFinal` — terminal states (`no_further_transitions`). + +## Notes +Phase 1 is **pure IOA** (no WASM, no cross-entity effects). A bounty/escrow economy (with WASM: `lock_escrow`, `verify_award`) arrives later as the marquee evolution episode (Phase 2). diff --git a/os-apps/stackoverflow-agents/app.toml b/os-apps/stackoverflow-agents/app.toml new file mode 100644 index 000000000..27472aa26 --- /dev/null +++ b/os-apps/stackoverflow-agents/app.toml @@ -0,0 +1,4 @@ +name = "stackoverflow-agents" +description = "Q&A for AI agents — questions, answers, upvotes. The seed organism for directed evolution." +version = "0.1.0" +dependencies = [] diff --git a/os-apps/stackoverflow-agents/policies/answer.cedar b/os-apps/stackoverflow-agents/policies/answer.cedar new file mode 100644 index 000000000..7722b4d5c --- /dev/null +++ b/os-apps/stackoverflow-agents/policies/answer.cedar @@ -0,0 +1,11 @@ +// stackoverflow-agents — Answer authorization (v1: open Q&A for agents). +// +// Phase 1 keeps authz permissive. When the loop adds Downvote, a future +// evolution can gate it (e.g. require reputation) — and the Cedar fitness +// stage will check that gate. + +permit( + principal, + action, + resource is Answer +); diff --git a/os-apps/stackoverflow-agents/policies/question.cedar b/os-apps/stackoverflow-agents/policies/question.cedar new file mode 100644 index 000000000..38c029d7e --- /dev/null +++ b/os-apps/stackoverflow-agents/policies/question.cedar @@ -0,0 +1,10 @@ +// stackoverflow-agents — Question authorization (v1: open Q&A for agents). +// +// Phase 1 keeps authz permissive — the focus is the evolution loop. Finer +// gates (e.g. only the asker may AcceptAnswer/Close) are a later evolution. + +permit( + principal, + action, + resource is Question +); diff --git a/os-apps/stackoverflow-agents/specs/answer.ioa.toml b/os-apps/stackoverflow-agents/specs/answer.ioa.toml new file mode 100644 index 000000000..ab8378ca1 --- /dev/null +++ b/os-apps/stackoverflow-agents/specs/answer.ioa.toml @@ -0,0 +1,58 @@ +# Answer Entity — I/O Automaton Specification +# +# An answer to a Question. Created (via POST) in Active; can be upvoted, +# accepted as the solution, or deleted. +# +# NOTE: there is deliberately NO `Downvote` action and NO `downvotes` counter. +# Agents will want to bury low-quality answers; the directed-evolution loop +# observes that unmet intent and adds them here — verified before deploy. + +[automaton] +name = "Answer" +states = ["Active", "Accepted", "Deleted"] +initial = "Active" +allow_indefinite_states = ["Active", "Accepted", "Deleted"] + +# --- State Variables --- + +[[state]] +name = "upvotes" +type = "counter" +initial = "0" + +# --- Actions --- + +[[action]] +name = "Upvote" +kind = "input" +from = ["Active", "Accepted"] +effect = "increment upvotes" +params = ["VoterId"] +hint = "Upvote this answer. Increases its upvote tally." + +[[action]] +name = "Accept" +kind = "input" +from = ["Active"] +to = "Accepted" +hint = "Mark this answer as the accepted solution." + +[[action]] +name = "Delete" +kind = "input" +from = ["Active", "Accepted"] +to = "Deleted" +params = ["reason"] +hint = "Delete this answer. Terminal." + +# =========================================================================== +# DELIBERATE GAP: no `Downvote` action, no `downvotes` counter. +# The directed-evolution loop grows them here from an observed unmet intent. +# =========================================================================== + +# --- Safety Invariants --- + +[[invariant]] +name = "DeletedIsFinal" +when = ["Deleted"] +assert = "no_further_transitions" diff --git a/os-apps/stackoverflow-agents/specs/model.csdl.xml b/os-apps/stackoverflow-agents/specs/model.csdl.xml new file mode 100644 index 000000000..9b23ec807 --- /dev/null +++ b/os-apps/stackoverflow-agents/specs/model.csdl.xml @@ -0,0 +1,134 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + OpenAnsweredClosed + + + + + + + + + + + + + + + + + + + + + ActiveAcceptedDeleted + + + + + + + + + + + + + + Open + + + + + + + + + OpenAnswered + + + + + + + + + + ActiveAccepted + + + + + + + Active + + + + + + + + ActiveAccepted + + + + + + + + + + + + + + + + + diff --git a/os-apps/stackoverflow-agents/specs/question.ioa.toml b/os-apps/stackoverflow-agents/specs/question.ioa.toml new file mode 100644 index 000000000..aeaded1f7 --- /dev/null +++ b/os-apps/stackoverflow-agents/specs/question.ioa.toml @@ -0,0 +1,49 @@ +# Question Entity — I/O Automaton Specification +# +# A question in the agent Q&A. Created (via POST) in Open; an answer can be +# accepted (→ Answered); the question can be Closed (terminal). Seed organism +# for directed evolution — intentionally minimal. + +[automaton] +name = "Question" +states = ["Open", "Answered", "Closed"] +initial = "Open" +allow_indefinite_states = ["Open", "Answered", "Closed"] + +# --- State Variables --- + +[[state]] +name = "has_accepted" +type = "bool" +initial = "false" + +# --- Actions --- + +[[action]] +name = "AcceptAnswer" +kind = "input" +from = ["Open"] +to = "Answered" +effect = "set has_accepted true" +params = ["AnswerId"] +hint = "Accept an answer as the solution. Moves the question to Answered." + +[[action]] +name = "Close" +kind = "input" +from = ["Open", "Answered"] +to = "Closed" +params = ["reason"] +hint = "Close the question (resolved, duplicate, off-topic). Terminal." + +# --- Safety Invariants --- + +[[invariant]] +name = "AnsweredRequiresAccepted" +when = ["Answered"] +assert = "has_accepted" + +[[invariant]] +name = "ClosedIsFinal" +when = ["Closed"] +assert = "no_further_transitions" diff --git a/scripts/so-simulator/README.md b/scripts/so-simulator/README.md new file mode 100644 index 000000000..e86a4d505 --- /dev/null +++ b/scripts/so-simulator/README.md @@ -0,0 +1,111 @@ +# so-simulator — synthetic agent-user driver for stackoverflow-agents + +A lightweight, deterministic driver that exercises the +`stackoverflow-agents` seed app, intentionally bumps into the absent +`Downvote` action, and emits the resulting unmet intent so the +directed-evolution loop can pick it up. + +This is **Phase 1.5 (simulator half)** of the directed-evolution build. +The Evolution Studio UI is its visual counterpart, in +`genesis/web/src/routes/studio/`. + +## What it does + +1. Discovers the running stackoverflow-agents tenant via OData + (`GET /tdata/$metadata`). +2. Seeds N **Questions** and per-question M **Answers** under that + tenant. +3. Casts a few **Upvotes** on each answer to establish a baseline + "good answer" / "bad answer" split. +4. Each of K synthetic **user-agents** then tries to `Downvote` the + lowest-quality answer. Because the seed app has no `Downvote` + action, the OData server returns 404/400 — that *is* the unmet + intent. +5. On the first failed downvote, the simulator: + - `POST /api/evolution/trajectories/unmet` to the running temper + server (the canonical intake), AND + - `POST /tdata/Evolutions` against the genesis OData (target_app + = `stackoverflow-agents`, intent = "agents want to downvote + low-quality answers", autonomy = configurable). +6. Prints a single-line trajectory summary suitable for piping into + the proof reports under `.proofs/`. + +## Modes + +| flag | behavior | +|---|---| +| `--dry-run` | print the requests it *would* make, exit 0. No HTTP. | +| `--deterministic` (default) | seeded RNG (`SO_SIM_SEED`, default 42). Identical run shape every time. | +| `--llm` | use `ANTHROPIC_API_KEY` to let Claude pick the next action. Implies non-deterministic. | +| `--no-evolution` | skip the genesis Evolution creation (only emit unmet intent). | +| `--target-only` | just emit the unmet intent — don't create Evolution rows. | + +Determinism is the default because the demo needs to be repeatable. + +## Configuration + +Env vars (all optional, sane defaults): + +``` +SO_API_BASE http://127.0.0.1:3000 # temper-platform / temper-server +SO_TENANT_ID stackoverflow-agents # X-Tenant-Id header +GENESIS_API_BASE http://127.0.0.1:3000 # genesis OData base (same process in dev) +GENESIS_TENANT_ID default # tenant under which Evolution rows live +SO_SIM_SEED 42 +SO_SIM_QUESTIONS 3 # # questions to seed +SO_SIM_ANSWERS 3 # # answers per question +SO_SIM_AGENTS 4 # # synthetic user-agents +SO_SIM_INTENT_AUTONOMY 0 # autonomy level on the Evolution row +ANTHROPIC_API_KEY (required iff --llm) +ANTHROPIC_MODEL claude-sonnet-4-6 +``` + +## Run + +```bash +# 1. dry-run (no HTTP, prints request plan) +node scripts/so-simulator/index.mjs --dry-run + +# 2. against a running platform (deterministic) +SO_API_BASE=http://127.0.0.1:3000 node scripts/so-simulator/index.mjs + +# 3. LLM-driven user-agents +ANTHROPIC_API_KEY=sk-... node scripts/so-simulator/index.mjs --llm +``` + +The script exits non-zero only on simulator failure (network unreachable, +malformed CSDL, etc.). The *whole point* of this simulator is that the +downvote attempt fails — that failure is success. + +## Deterministic by design + +- Seeded RNG (mulberry32) chooses question/answer text and which + agents act. +- Stable IDs: `sim-q-{seed}-{idx}`, `sim-a-{seed}-{q}-{idx}`. +- No `Date.now()` — wall-clock timestamps are passed through but the + decision logic does not depend on them. + +## Files + +- `index.mjs` — the simulator (single file, Node 18+ native fetch, no + npm deps). +- `lib.mjs` — small helpers (OData wrappers, seeded RNG, scripted + decisions). +- `README.md` — this file. + +## How it integrates with the rest of Phase 1 + +``` +[so-simulator] ──Downvote (404)──▶ [stackoverflow-agents tenant] + │ ▲ + │ POST unmet intent │ later: hot-deploy of variant + ▼ │ adds Downvote action +[temper-platform: /api/evolution/ │ + trajectories/unmet] │ + │ │ + ▼ │ +[genesis: POST /tdata/Evolutions] ──────▶ [Evolution Studio UI] + │ + ▼ +[evolver engine: gen_variant → run_stage_caller → select_winner → merge_variant] +``` diff --git a/scripts/so-simulator/index.mjs b/scripts/so-simulator/index.mjs new file mode 100755 index 000000000..7a2f0d607 --- /dev/null +++ b/scripts/so-simulator/index.mjs @@ -0,0 +1,487 @@ +#!/usr/bin/env node +// +// so-simulator — synthetic agent-user driver for stackoverflow-agents. +// +// Phase 1.5 of the directed-evolution build. Exercises the running +// stackoverflow-agents tenant over OData, intentionally bumps into the +// absent `Downvote` action, and emits the resulting unmet intent so +// the evolver (in genesis) picks it up and grows the missing feature. +// +// See README.md for the full mode matrix. +// +// Determinism is the default. The whole point is that the +// "downvote-attempt-fails" trajectory is bit-stable across runs so the +// demo (and any reproducibility report under .proofs/) doesn't drift. + +import { + mulberry32, + parseArgs, + questionId, + answerId, + agentId, + odataGet, + odataPost, + scriptedDecide, + llmDecide, + fmtTrajectoryLine, +} from './lib.mjs'; + +// ─── Configuration ────────────────────────────────────────────────── + +const argv = parseArgs(process.argv.slice(2)); +const FLAGS = argv.flags; +const OPTS = argv.opts; + +const cfg = { + dryRun: FLAGS.has('dry-run'), + llm: FLAGS.has('llm'), + noEvolution: FLAGS.has('no-evolution'), + targetOnly: FLAGS.has('target-only'), + + soApiBase: process.env.SO_API_BASE ?? 'http://127.0.0.1:3000', + soTenant: process.env.SO_TENANT_ID ?? 'stackoverflow-agents', + genesisApiBase: process.env.GENESIS_API_BASE ?? 'http://127.0.0.1:3000', + genesisTenant: process.env.GENESIS_TENANT_ID ?? 'default', + + seed: Number(process.env.SO_SIM_SEED ?? OPTS.seed ?? 42), + questions: Number(process.env.SO_SIM_QUESTIONS ?? OPTS.questions ?? 3), + answersPerQuestion: Number( + process.env.SO_SIM_ANSWERS ?? OPTS.answers ?? 3, + ), + agents: Number(process.env.SO_SIM_AGENTS ?? OPTS.agents ?? 4), + intentAutonomy: Number(process.env.SO_SIM_INTENT_AUTONOMY ?? 0), + + anthropicApiKey: process.env.ANTHROPIC_API_KEY ?? '', + anthropicModel: process.env.ANTHROPIC_MODEL ?? 'claude-sonnet-4-6', +}; + +if (cfg.llm && !cfg.anthropicApiKey && !cfg.dryRun) { + log('!! --llm requires ANTHROPIC_API_KEY; falling back to scripted mode'); + cfg.llm = false; +} + +const rng = mulberry32(cfg.seed); +const trajectory = []; + +function log(msg) { + // Single sink so the proof report can grep ::SIM:: lines. + process.stdout.write(`::SIM:: ${msg}\n`); +} + +function record(entry) { + trajectory.push(entry); + log(fmtTrajectoryLine(entry)); +} + +// ─── Seed corpus ──────────────────────────────────────────────────── + +const sampleTitles = [ + 'How do I stream WASM logs to ClickHouse?', + 'Why does my Cedar policy deny everything?', + 'What is the canonical hash for an empty tree?', + 'Best way to fan out OData reads in Rust?', + 'How do agents reliably emit unmet intents?', +]; + +const sampleAnswerBodies = [ + 'You should just disable Cedar entirely.', // intentionally bad + 'Use a separate file watcher per partition.', + 'Read the spec, then call DispatchAction.', + 'I have no idea — please update the docs!', // intentionally bad + 'Try `temper verify --specs-dir specs`.', +]; + +// ─── Seed phase: questions + answers + upvotes ────────────────────── + +async function seedWorld() { + log(`seeding ${cfg.questions} questions × ${cfg.answersPerQuestion} answers @ tenant=${cfg.soTenant}`); + const questions = []; + + for (let q = 0; q < cfg.questions; q += 1) { + const qid = questionId(cfg.seed, q); + const title = sampleTitles[q % sampleTitles.length]; + const body = { + Id: qid, + Title: title, + Body: `seeded by so-simulator (seed=${cfg.seed})`, + AuthorId: `sim-author-${q}`, + Status: 'Open', + HasAccepted: false, + AcceptedAnswerId: null, + CreatedAt: new Date(0).toISOString(), + UpdatedAt: new Date(0).toISOString(), + }; + + if (cfg.dryRun) { + record({ + action: 'POST Questions', + target: qid, + success: true, + status: 200, + note: 'dry-run', + }); + } else { + const r = await odataPost( + cfg.soApiBase, + cfg.soTenant, + '/tdata/Questions', + body, + ); + record({ + action: 'POST Questions', + target: qid, + success: r.ok, + status: r.status, + note: r.ok ? '' : excerpt(r.body), + }); + } + + const answers = []; + for (let a = 0; a < cfg.answersPerQuestion; a += 1) { + const aid = answerId(cfg.seed, q, a); + const abody = { + Id: aid, + QuestionId: qid, + Body: sampleAnswerBodies[(q + a) % sampleAnswerBodies.length], + AuthorId: `sim-author-${q}-${a}`, + Status: 'Active', + Upvotes: 0, + CreatedAt: new Date(0).toISOString(), + }; + + if (cfg.dryRun) { + record({ + action: 'POST Answers', + target: aid, + success: true, + status: 200, + note: 'dry-run', + }); + answers.push({ id: aid, upvotes: 0 }); + } else { + const r = await odataPost( + cfg.soApiBase, + cfg.soTenant, + '/tdata/Answers', + abody, + ); + record({ + action: 'POST Answers', + target: aid, + success: r.ok, + status: r.status, + note: r.ok ? '' : excerpt(r.body), + }); + answers.push({ id: aid, upvotes: 0 }); + } + } + questions.push({ id: qid, title, answers }); + } + + // Cast a handful of Upvotes — deterministically distributed so one + // answer ends up clearly low-quality (which is the "bad answer" + // every agent then tries to downvote). + log(`casting upvotes (deterministic distribution)`); + for (const q of questions) { + for (let i = 0; i < q.answers.length; i += 1) { + // Answer 0: many upvotes; answer 1: a few; answer 2: zero (the + // intended downvote target). + const upvotes = Math.max(0, q.answers.length - 1 - i) * 2; + for (let u = 0; u < upvotes; u += 1) { + const voter = `sim-voter-${u}`; + const path = `/tdata/Answers('${q.answers[i].id}')/Soa.QA.Upvote`; + if (cfg.dryRun) { + record({ + action: 'Upvote', + target: q.answers[i].id, + success: true, + status: 200, + note: `dry-run voter=${voter}`, + }); + } else { + const r = await odataPost( + cfg.soApiBase, + cfg.soTenant, + path, + { VoterId: voter }, + ); + record({ + action: 'Upvote', + target: q.answers[i].id, + success: r.ok, + status: r.status, + agent: voter, + note: r.ok ? '' : excerpt(r.body), + }); + } + q.answers[i].upvotes += 1; + } + } + } + return questions; +} + +function excerpt(v) { + if (v == null) return ''; + const s = typeof v === 'string' ? v : JSON.stringify(v); + return s.slice(0, 140).replace(/\s+/g, ' '); +} + +// ─── Downvote phase: each agent attempts the absent action ────────── + +async function downvoteAttempts(world) { + log(`${cfg.agents} synthetic user-agents attempting Downvote (this MUST fail in Phase 1)`); + const failures = []; + + // We flatten the answers across all questions, then each agent + // picks the lowest-upvote one (scripted) or asks Claude (llm). + const allAnswers = world.flatMap((q) => q.answers); + const worldView = { answers: allAnswers }; + + for (let i = 0; i < cfg.agents; i += 1) { + const id = agentId(cfg.seed, i); + let decision; + if (cfg.llm) { + decision = await llmDecide({ + apiKey: cfg.anthropicApiKey, + model: cfg.anthropicModel, + world: worldView, + agentIdx: i, + }); + if (decision.error) { + record({ + agent: id, + action: 'llm-decide', + target: '', + success: false, + status: 0, + note: decision.error, + }); + decision = scriptedDecide(rng, worldView, i); + } + } else { + decision = scriptedDecide(rng, worldView, i); + } + + const target = decision.answerId; + const path = `/tdata/Answers('${target}')/Soa.QA.Downvote`; + if (cfg.dryRun) { + // We mark it as a deliberate failure for symmetry with the live + // path. The expected real-world response is 404/400 because the + // CSDL has no `Downvote` action declared on Soa.QA.Answer. + const entry = { + agent: id, + action: 'Downvote (absent)', + target, + success: false, + status: 404, + note: 'dry-run (CSDL has no Downvote action — expected)', + }; + record(entry); + failures.push(entry); + } else { + const r = await odataPost( + cfg.soApiBase, + cfg.soTenant, + path, + { VoterId: id, reason: decision.reason }, + ); + const entry = { + agent: id, + action: 'Downvote (absent)', + target, + success: r.ok, + status: r.status, + note: r.ok + ? '!! UNEXPECTED — Downvote already exists?' + : excerpt(r.body), + }; + record(entry); + if (!r.ok) failures.push(entry); + } + } + return failures; +} + +// ─── Intent emission phase ────────────────────────────────────────── + +async function emitUnmetIntent(failures) { + if (!failures.length) { + log('no failed downvotes recorded — skipping unmet intent emission'); + return; + } + const sample = failures[0]; + const intentBody = { + action: 'Downvote', + intent: 'agents want to downvote low-quality answers', + tenant: cfg.soTenant, + entity_type: 'Answer', + error: `${sample.status} on /tdata/Answers('${sample.target}')/Soa.QA.Downvote`, + source: 'platform', + metadata: { + sim_seed: cfg.seed, + sim_agents: cfg.agents, + sim_failures: failures.length, + }, + }; + + if (cfg.dryRun) { + record({ + action: 'POST /api/evolution/trajectories/unmet', + target: cfg.soTenant, + success: true, + status: 201, + note: `dry-run body=${excerpt(intentBody)}`, + }); + } else { + const r = await odataPost( + cfg.soApiBase, + cfg.soTenant, + '/api/evolution/trajectories/unmet', + intentBody, + ); + record({ + action: 'POST /api/evolution/trajectories/unmet', + target: cfg.soTenant, + success: r.ok, + status: r.status, + note: r.ok ? '' : excerpt(r.body), + }); + } +} + +async function createEvolutionRow(failures) { + if (cfg.noEvolution || cfg.targetOnly) { + log('skipping Evolution creation (--no-evolution / --target-only)'); + return; + } + if (!failures.length) { + log('no failed downvotes — skipping Evolution creation'); + return; + } + + // Stable, deterministic UUID-shaped id. We deliberately do not use + // crypto.randomUUID() so reruns produce the same Evolution row id + // (the OData store will treat the POST as an upsert via key). + const stableHex = (n) => n.toString(16).padStart(8, '0'); + const evolutionId = + `${stableHex(cfg.seed)}-0000-4000-8000-${stableHex(0)}0000`; + const body = { + Id: evolutionId, + TargetApp: 'stackoverflow-agents', + TargetTenant: cfg.soTenant, + Intent: 'agents want to downvote low-quality answers', + ProblemStatement: + 'Add a Downvote action and a downvotes counter to Answer; ' + + 'maintain ScoreConsistent across upvotes and downvotes.', + Autonomy: cfg.intentAutonomy, + VariantCount: 0, + Status: 'IntentObserved', + CreatedAt: new Date(0).toISOString(), + }; + + if (cfg.dryRun) { + record({ + action: 'POST Evolutions', + target: evolutionId, + success: true, + status: 201, + note: `dry-run body=${excerpt(body)}`, + }); + return; + } + const r = await odataPost( + cfg.genesisApiBase, + cfg.genesisTenant, + '/tdata/Evolutions', + body, + ); + record({ + action: 'POST Evolutions', + target: evolutionId, + success: r.ok, + status: r.status, + note: r.ok ? '' : excerpt(r.body), + }); +} + +// ─── Pre-flight: discover the running tenant ──────────────────────── + +async function preflight() { + if (cfg.dryRun) { + log('dry-run mode; skipping CSDL discovery'); + return { ok: true, dryRun: true }; + } + const r = await odataGet( + cfg.soApiBase, + cfg.soTenant, + '/tdata/$metadata', + ); + if (!r.ok) { + log(`!! CSDL fetch failed (${r.status}) at ${cfg.soApiBase} — server unreachable?`); + return { ok: false }; + } + const body = typeof r.body === 'string' ? r.body : JSON.stringify(r.body); + const hasQuestion = body.includes('EntityType Name="Question"'); + const hasAnswer = body.includes('EntityType Name="Answer"'); + const hasDownvote = /Action Name="Downvote"/.test(body); + log(`preflight: Question=${hasQuestion} Answer=${hasAnswer} Downvote=${hasDownvote}`); + if (hasDownvote) { + log('!! Downvote action ALREADY present in CSDL — the seed app has already evolved? Aborting.'); + return { ok: false, alreadyEvolved: true }; + } + if (!hasQuestion || !hasAnswer) { + log('!! tenant does not look like stackoverflow-agents'); + return { ok: false }; + } + return { ok: true }; +} + +// ─── Main ─────────────────────────────────────────────────────────── + +async function main() { + log(`so-simulator starting; mode=${cfg.dryRun ? 'dry-run' : cfg.llm ? 'llm' : 'scripted'} seed=${cfg.seed}`); + log(`config: questions=${cfg.questions} answers/q=${cfg.answersPerQuestion} agents=${cfg.agents}`); + log(`endpoints: so=${cfg.soApiBase} (tenant=${cfg.soTenant}); genesis=${cfg.genesisApiBase} (tenant=${cfg.genesisTenant})`); + + const pf = await preflight(); + if (!pf.ok && !cfg.dryRun) { + log('!! pre-flight failed — exit 2'); + process.exit(2); + } + + const world = await seedWorld(); + const failures = await downvoteAttempts(world); + await emitUnmetIntent(failures); + await createEvolutionRow(failures); + + const totalActions = trajectory.length; + const successCount = trajectory.filter((t) => t.success).length; + const downvoteFails = trajectory.filter( + (t) => t.action === 'Downvote (absent)' && !t.success, + ).length; + log(`done: ${totalActions} actions, ${successCount} ok, ${downvoteFails} downvote-absent (the point)`); + + // Single-line summary suitable for ::PROOF:: redirection + const summary = JSON.stringify({ + sim_seed: cfg.seed, + total_actions: totalActions, + successes: successCount, + downvote_failures: downvoteFails, + unmet_intent_emitted: failures.length > 0, + evolution_row_created: failures.length > 0 && !cfg.noEvolution && !cfg.targetOnly, + mode: cfg.dryRun ? 'dry-run' : cfg.llm ? 'llm' : 'scripted', + }); + process.stdout.write(`::SIM-SUMMARY:: ${summary}\n`); + + // Exit 0 if we got the expected failure signature; exit 3 if the + // seed app already has Downvote (no unmet intent to emit). + if (failures.length === 0 && !cfg.dryRun) { + process.exit(3); + } +} + +main().catch((err) => { + log(`!! uncaught error: ${err?.stack ?? err}`); + process.exit(1); +}); diff --git a/scripts/so-simulator/lib.mjs b/scripts/so-simulator/lib.mjs new file mode 100644 index 000000000..3cbaf217f --- /dev/null +++ b/scripts/so-simulator/lib.mjs @@ -0,0 +1,191 @@ +// Small helpers for the stackoverflow-agents simulator. +// +// No npm deps; Node 18+ native fetch only. Everything here is pure +// helpers — no top-level effects — so it stays cheap to import from +// both the runtime entry point and any unit-style smoke tests. + +// ─── Seeded RNG (mulberry32) ──────────────────────────────────────── +// We deliberately do NOT use Math.random — the simulator must be +// bit-stable across runs so the demo is repeatable. Same seed → same +// agent decisions → same trajectory shape. + +export function mulberry32(seed) { + let state = seed >>> 0; + return function next() { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +export function pick(rng, arr) { + return arr[Math.floor(rng() * arr.length)]; +} + +// ─── Tiny argv parser (no dep) ────────────────────────────────────── + +export function parseArgs(argv) { + const flags = new Set(); + const opts = {}; + for (let i = 0; i < argv.length; i += 1) { + const a = argv[i]; + if (a.startsWith('--')) { + const eq = a.indexOf('='); + if (eq > -1) { + opts[a.slice(2, eq)] = a.slice(eq + 1); + } else { + flags.add(a.slice(2)); + } + } + } + return { flags, opts }; +} + +// ─── Stable ID helpers ────────────────────────────────────────────── +// We pick sim-prefixed deterministic IDs so the simulator's writes are +// idempotent on reruns (PUT-like POST behavior) and post-mortem +// queries against the journal can grep for them easily. + +export function questionId(seed, idx) { + return `sim-q-${seed}-${idx}`; +} + +export function answerId(seed, q, idx) { + return `sim-a-${seed}-${q}-${idx}`; +} + +export function agentId(seed, idx) { + return `sim-agent-${seed}-${idx}`; +} + +// ─── HTTP helpers ─────────────────────────────────────────────────── +// Each helper returns { ok, status, body, error } — never throws on +// non-2xx, because we *want* to observe non-2xx responses (the whole +// downvote attempt is supposed to fail). + +export async function odataGet(base, tenant, path, headers = {}) { + const url = `${base.replace(/\/$/, '')}${path}`; + try { + const resp = await fetch(url, { + method: 'GET', + headers: { + Accept: 'application/json', + 'X-Tenant-Id': tenant, + ...headers, + }, + }); + const text = await resp.text(); + let body = text; + try { body = JSON.parse(text); } catch (_) { /* leave as text */ } + return { ok: resp.ok, status: resp.status, body }; + } catch (err) { + return { ok: false, status: 0, error: err.message ?? String(err) }; + } +} + +export async function odataPost(base, tenant, path, body, headers = {}) { + const url = `${base.replace(/\/$/, '')}${path}`; + try { + const resp = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-Tenant-Id': tenant, + ...headers, + }, + body: JSON.stringify(body), + }); + const text = await resp.text(); + let parsed = text; + try { parsed = JSON.parse(text); } catch (_) { /* leave as text */ } + return { ok: resp.ok, status: resp.status, body: parsed }; + } catch (err) { + return { ok: false, status: 0, error: err.message ?? String(err) }; + } +} + +// ─── Scripted agent decisions ─────────────────────────────────────── +// In deterministic mode, "what should this agent do next" is a +// function of (rng, world state). In LLM mode we substitute a Claude +// call; the *shape* of the returned decision is identical so the +// downstream code never branches on the agent brain. + +export function scriptedDecide(rng, world, agentIdx) { + // Phase 1 only has one interesting decision: which answer to try to + // downvote. We pick the one with the lowest upvote count, ties + // broken by the seeded RNG. + const answers = world.answers.slice(); + answers.sort((a, b) => { + if (a.upvotes !== b.upvotes) return a.upvotes - b.upvotes; + return rng() < 0.5 ? -1 : 1; + }); + const target = answers[0]; + return { + kind: 'downvote', + answerId: target.id, + reason: 'low-quality answer (scripted)', + }; +} + +// ─── Tiny LLM bridge (optional) ───────────────────────────────────── +// Wraps Claude's messages API. We deliberately keep this as a single +// fetch call so the simulator stays dep-free; if LLM mode is not +// enabled, this is never imported. + +export async function llmDecide({ apiKey, model, world, agentIdx }) { + const url = 'https://api.anthropic.com/v1/messages'; + const system = + 'You are agent ' + agentIdx + ' on a Q&A site for AI agents. ' + + 'You can ONLY upvote, accept, delete, or (try to) downvote answers. ' + + 'Pick ONE answer to downvote because it is low-quality. ' + + 'Reply as compact JSON: {"kind":"downvote","answer_id":"...","reason":"..."}'; + const userBody = JSON.stringify({ + answers: world.answers.map((a) => ({ id: a.id, upvotes: a.upvotes })), + }); + const body = { + model, + max_tokens: 200, + system, + messages: [{ role: 'user', content: userBody }], + }; + try { + const resp = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-api-key': apiKey, + 'anthropic-version': '2023-06-01', + }, + body: JSON.stringify(body), + }); + if (!resp.ok) { + const t = await resp.text(); + return { error: `claude ${resp.status}: ${t.slice(0, 200)}` }; + } + const json = await resp.json(); + const text = json?.content?.[0]?.text ?? ''; + try { + const parsed = JSON.parse(text); + return { + kind: 'downvote', + answerId: parsed.answer_id ?? parsed.answerId, + reason: parsed.reason ?? 'llm-driven', + }; + } catch (_) { + return { error: 'claude returned non-JSON: ' + text.slice(0, 120) }; + } + } catch (err) { + return { error: err.message ?? String(err) }; + } +} + +// ─── Pretty-printers ──────────────────────────────────────────────── + +export function fmtTrajectoryLine(entry) { + const status = entry.success ? 'OK' : 'FAIL'; + const code = entry.status ?? '-'; + return `[${status} ${code}] ${entry.agent ?? '-'} ${entry.action} ${entry.target ?? ''} :: ${entry.note ?? ''}`; +}