From d369759bec4a99b6fa801808cef96df945f29a56 Mon Sep 17 00:00:00 2001 From: ablancorobayna Date: Wed, 13 May 2026 10:05:40 +0100 Subject: [PATCH 1/2] chore: add concept registry drift check and bump version to 0.7.1 Add test/methodology/drift-check.test.ts that cross-validates concept registry entries against persona guidance prose (DM system prompt, SAP AEM plugin fragments). Uses keyword matching to catch real drift while tolerating natural wording differences between structured labels and prose. Covers Sprint 0 checklist, AEM addendum, AEM phases, document types, and persona roles (AC1.4). --- package-lock.json | 4 +- package.json | 2 +- test/methodology/drift-check.test.ts | 153 +++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 test/methodology/drift-check.test.ts diff --git a/package-lock.json b/package-lock.json index dba6b98..58373a2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mrvn-cli", - "version": "0.6.1", + "version": "0.7.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mrvn-cli", - "version": "0.6.1", + "version": "0.7.1", "license": "MIT", "dependencies": { "@anthropic-ai/claude-agent-sdk": "^0.2.37", diff --git a/package.json b/package.json index 685d0c5..90281f5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mrvn-cli", - "version": "0.7.0", + "version": "0.7.1", "description": "AI-powered software product development assistant with Product Owner, Delivery Manager, and Technical Lead personas", "type": "module", "bin": { diff --git a/test/methodology/drift-check.test.ts b/test/methodology/drift-check.test.ts new file mode 100644 index 0000000..3853b4b --- /dev/null +++ b/test/methodology/drift-check.test.ts @@ -0,0 +1,153 @@ +/** + * AC1.4 — Drift check between concept registry and persona guidance. + * + * Validates that the methodology knowledge encoded in persona guidance + * (DM system prompt, SAP AEM plugin fragments) stays in sync with the + * concept registry. If a concept registry entry diverges from the prose + * that originally seeded it, these tests fail — forcing a reconciliation + * before merge. + * + * Tests use keyword matching (not exact phrases) because registry items + * are structured labels while persona guidance is prose. + */ +import { describe, it, expect } from "vitest"; +import { deliveryManager } from "../../src/personas/builtin/delivery-manager.js"; +import { sapAemPlugin } from "../../src/plugins/builtin/sap-aem.js"; +import { getConceptRegistry } from "../../src/methodology/registry.js"; + +const registry = getConceptRegistry(); +const dmPrompt = deliveryManager.systemPrompt.toLowerCase(); +const aemDmFragment = sapAemPlugin.promptFragments!["delivery-manager"].toLowerCase(); +const aemWildcardFragment = sapAemPlugin.promptFragments!["*"].toLowerCase(); + +/** Extract significant keywords from a phrase (drops short filler words). */ +function keywords(phrase: string): string[] { + const stopWords = new Set(["to", "of", "the", "a", "an", "and", "or", "for", "in", "on", "with"]); + return phrase + .toLowerCase() + .split(/[\s,/]+/) + .filter((w) => w.length > 2 && !stopWords.has(w)); +} + +/** Check that most keywords from a phrase appear in the text. */ +function hasKeywordOverlap(text: string, phrase: string, minRatio = 0.5): boolean { + const kw = keywords(phrase); + if (kw.length === 0) return true; + const matched = kw.filter((w) => text.includes(w)); + return matched.length / kw.length >= minRatio; +} + +describe("Concept registry ↔ persona guidance drift check", () => { + describe("Sprint 0 checklist vs DM persona guidance", () => { + const sprint0 = registry.get("sprint-0")!; + + it("sprint-0 concept exists in registry", () => { + expect(sprint0).toBeDefined(); + }); + + it("DM persona mentions Sprint 0 as variable-duration bootstrapping", () => { + expect(dmPrompt).toContain("sprint 0"); + expect(dmPrompt).toContain("variable-duration"); + expect(dmPrompt).toContain("bootstrapping"); + }); + + it("DM persona states Sprint 0 exit criteria", () => { + expect(dmPrompt).toContain("ready to start sprint 1"); + }); + + const genericCategories = sprint0.checklist!.filter((c) => !c.appliesWhen); + + it("registry has the same number of generic checklist categories as DM persona sections", () => { + // DM persona has 4 bullet groups under Sprint 0: Infrastructure, Backlog, Ceremony, Integration + expect(genericCategories.length).toBe(4); + }); + + for (const section of genericCategories) { + it(`DM persona covers category "${section.category}" (keyword match)`, () => { + const matchedItems = section.items.filter((item) => hasKeywordOverlap(dmPrompt, item)); + expect(matchedItems.length).toBeGreaterThan( + 0, + `No items from "${section.category}" found in DM persona guidance. Items: ${section.items.join(", ")}`, + ); + }); + + for (const item of section.items) { + it(`DM persona covers "${item}" from ${section.category}`, () => { + expect(hasKeywordOverlap(dmPrompt, item)).toBe(true); + }); + } + } + }); + + describe("AEM addendum vs SAP AEM DM plugin fragment", () => { + const sprint0 = registry.get("sprint-0")!; + const aemAddendum = sprint0.checklist!.find((c) => c.appliesWhen === "methodology=aem"); + + it("AEM addendum section exists in registry", () => { + expect(aemAddendum).toBeDefined(); + }); + + it("SAP AEM DM fragment mentions Sprint 0 for AEM", () => { + expect(aemDmFragment).toContain("sprint 0"); + }); + + for (const item of aemAddendum!.items) { + it(`SAP AEM DM fragment covers "${item}" (keyword match)`, () => { + expect(hasKeywordOverlap(aemDmFragment, item)).toBe(true); + }); + } + }); + + describe("AEM phases vs SAP AEM plugin wildcard fragment", () => { + const phaseIds = ["assess-use-case", "assess-technology", "define-solution"]; + + for (const phaseId of phaseIds) { + const phase = registry.get(phaseId); + + it(`phase "${phaseId}" exists in registry`, () => { + expect(phase).toBeDefined(); + }); + + it(`SAP AEM wildcard fragment covers phase "${phase!.name}" (keyword match)`, () => { + expect(hasKeywordOverlap(aemWildcardFragment, phase!.name)).toBe(true); + }); + } + + it("SAP AEM wildcard fragment describes the 3-phase approach", () => { + expect(aemWildcardFragment).toContain("3-phase"); + }); + }); + + describe("AEM document types vs registry artifact-type concepts", () => { + const aemOnlyTypes = ["use-case", "tech-assessment", "extension-design"]; + + for (const typeId of aemOnlyTypes) { + const concept = registry.get(typeId); + + it(`artifact-type "${typeId}" exists in registry as AEM-scoped`, () => { + expect(concept).toBeDefined(); + expect(concept!.category).toBe("artifact-type"); + expect(concept!.methodology).toContain("aem"); + }); + + it(`SAP AEM wildcard fragment mentions "${concept!.name}"`, () => { + expect(aemWildcardFragment).toContain(concept!.name.toLowerCase()); + }); + } + }); + + describe("Persona roles vs registry role concepts", () => { + const roles = ["product-owner", "delivery-manager", "tech-lead"]; + + for (const roleId of roles) { + const concept = registry.get(roleId); + + it(`role "${roleId}" exists in registry for both methodologies`, () => { + expect(concept).toBeDefined(); + expect(concept!.category).toBe("role"); + expect(concept!.methodology).toContain("generic-agile"); + expect(concept!.methodology).toContain("aem"); + }); + } + }); +}); From dd3b05aa3175d9ece8e118f19327299086b5aa0e Mon Sep 17 00:00:00 2001 From: ablancorobayna Date: Wed, 13 May 2026 12:04:02 +0100 Subject: [PATCH 2/2] fix: guard prompt fragment access in drift check with beforeAll assertions --- test/methodology/drift-check.test.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/test/methodology/drift-check.test.ts b/test/methodology/drift-check.test.ts index 3853b4b..f8ba93a 100644 --- a/test/methodology/drift-check.test.ts +++ b/test/methodology/drift-check.test.ts @@ -10,15 +10,15 @@ * Tests use keyword matching (not exact phrases) because registry items * are structured labels while persona guidance is prose. */ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, beforeAll } from "vitest"; import { deliveryManager } from "../../src/personas/builtin/delivery-manager.js"; import { sapAemPlugin } from "../../src/plugins/builtin/sap-aem.js"; import { getConceptRegistry } from "../../src/methodology/registry.js"; const registry = getConceptRegistry(); -const dmPrompt = deliveryManager.systemPrompt.toLowerCase(); -const aemDmFragment = sapAemPlugin.promptFragments!["delivery-manager"].toLowerCase(); -const aemWildcardFragment = sapAemPlugin.promptFragments!["*"].toLowerCase(); +let dmPrompt: string; +let aemDmFragment: string; +let aemWildcardFragment: string; /** Extract significant keywords from a phrase (drops short filler words). */ function keywords(phrase: string): string[] { @@ -38,6 +38,17 @@ function hasKeywordOverlap(text: string, phrase: string, minRatio = 0.5): boolea } describe("Concept registry ↔ persona guidance drift check", () => { + beforeAll(() => { + dmPrompt = deliveryManager.systemPrompt.toLowerCase(); + + expect(sapAemPlugin.promptFragments).toBeDefined(); + expect(sapAemPlugin.promptFragments!["delivery-manager"]).toBeDefined(); + expect(sapAemPlugin.promptFragments!["*"]).toBeDefined(); + + aemDmFragment = sapAemPlugin.promptFragments!["delivery-manager"].toLowerCase(); + aemWildcardFragment = sapAemPlugin.promptFragments!["*"].toLowerCase(); + }); + describe("Sprint 0 checklist vs DM persona guidance", () => { const sprint0 = registry.get("sprint-0")!;