Merge pull request #19 from firstsun-dev/worktree-personality-scoring-fix

fix(personality): relative-frequency cognitive-function scoring + opt-in LLM-vote mode

Author: ClaudiaFang

cli/src/analysis/__tests__/personality.test.ts

@@ -294,19 +294,24 @@ const PATTERN_TO_FUNCTION: Record<string, string> = {
 };
 
 describe('computePersonalityProfile — cognitive functions', () => {
-  it('scores all 8 functions from mean confidence of their mapped pattern category', () => {
+  it('scores each function by its relative share of pattern instances, not mean confidence', () => {
+    // 8 instances total: ni gets 2 (2x the 1/8 "fair share" -> capped at 100), six other
+    // functions get exactly 1 each (exactly fair share -> 50/moderate), fe gets 0 (-> null).
+    // Confidence is deliberately uniform (all 80) to prove the score no longer tracks it —
+    // this is the fix for the old formula's "everything lands >=65 because confidence has a
+    // 70 floor" bug (see computeCognitiveFunctions' doc comment in ../personality.ts).
     const facets: PersonalityFacetInput[] = [
       facet({
         effectivePatterns: [
-          ep({ category: 'structured-planning', confidence: 90 }),
-          ep({ category: 'structured-planning', confidence: 70 }), // ni: (90+70)/2 = 80
-          ep({ category: 'context-gathering', confidence: 60 }),   // ne: 60
-          ep({ category: 'domain-expertise', confidence: 40 }),    // si: 40
-          ep({ category: 'incremental-implementation', confidence: 100 }), // se: 100
-          ep({ category: 'systematic-debugging', confidence: 55 }), // ti: 55
-          ep({ category: 'verification-workflow', confidence: 65 }), // te: 65
-          ep({ category: 'self-correction', confidence: 20 }),      // fi: 20
-          ep({ category: 'effective-tooling', confidence: 75 }),    // fe: 75
+          ep({ category: 'structured-planning', confidence: 80 }), // ni
+          ep({ category: 'structured-planning', confidence: 80 }), // ni (2nd instance)
+          ep({ category: 'context-gathering', confidence: 80 }),   // ne
+          ep({ category: 'domain-expertise', confidence: 80 }),    // si
+          ep({ category: 'incremental-implementation', confidence: 80 }), // se
+          ep({ category: 'systematic-debugging', confidence: 80 }), // ti
+          ep({ category: 'verification-workflow', confidence: 80 }), // te
+          ep({ category: 'self-correction', confidence: 80 }),      // fi
+          // effective-tooling (fe) deliberately absent
         ],
       }),
     ];
@@ -316,15 +321,44 @@ describe('computePersonalityProfile — cognitive functions', () => {
     // Stable order check
     expect(profile.cognitiveFunctions.map(f => f.key)).toEqual(['ni', 'ne', 'si', 'se', 'ti', 'te', 'fi', 'fe']);
 
-    expect(cogFn(profile.cognitiveFunctions, 'ni').score).toBe(80);
+    expect(cogFn(profile.cognitiveFunctions, 'ni').score).toBe(100);
     expect(cogFn(profile.cognitiveFunctions, 'ni').sampleSize).toBe(2);
-    expect(cogFn(profile.cognitiveFunctions, 'ne').score).toBe(60);
-    expect(cogFn(profile.cognitiveFunctions, 'si').score).toBe(40);
-    expect(cogFn(profile.cognitiveFunctions, 'se').score).toBe(100);
-    expect(cogFn(profile.cognitiveFunctions, 'ti').score).toBe(55);
-    expect(cogFn(profile.cognitiveFunctions, 'te').score).toBe(65);
-    expect(cogFn(profile.cognitiveFunctions, 'fi').score).toBe(20);
-    expect(cogFn(profile.cognitiveFunctions, 'fe').score).toBe(75);
+    for (const key of ['ne', 'si', 'se', 'ti', 'te', 'fi']) {
+      expect(cogFn(profile.cognitiveFunctions, key).score).toBe(50);
+      expect(cogFn(profile.cognitiveFunctions, key).sampleSize).toBe(1);
+    }
+
+    const fe = cogFn(profile.cognitiveFunctions, 'fe');
+    expect(fe.score).toBeNull();
+    expect(fe.sampleSize).toBe(0);
+    expect(fe.band).toBeUndefined();
+  });
+
+  it('differentiates functions purely by relative frequency even when confidence is identical', () => {
+    // se: 2 instances, fi: 1 instance, plus 6 one-off fillers spread across the other
+    // categories so neither se nor fi's share hits the 2x-fair-share cap (which would make
+    // both saturate at 100 and hide the ordering this test is checking for).
+    const facets: PersonalityFacetInput[] = [
+      facet({
+        effectivePatterns: [
+          ep({ category: 'incremental-implementation', confidence: 95 }), // se
+          ep({ category: 'incremental-implementation', confidence: 95 }), // se
+          ep({ category: 'self-correction', confidence: 95 }),            // fi
+          ep({ category: 'structured-planning', confidence: 95 }),        // ni filler
+          ep({ category: 'context-gathering', confidence: 95 }),          // ne filler
+          ep({ category: 'domain-expertise', confidence: 95 }),           // si filler
+          ep({ category: 'systematic-debugging', confidence: 95 }),       // ti filler
+          ep({ category: 'verification-workflow', confidence: 95 }),      // te filler
+          ep({ category: 'effective-tooling', confidence: 95 }),          // fe filler
+        ],
+      }),
+    ];
+    const profile = computePersonalityProfile(facets, [], '2026-W29', '__all__');
+    const se = cogFn(profile.cognitiveFunctions, 'se');
+    const fi = cogFn(profile.cognitiveFunctions, 'fi');
+    expect(se.score).not.toBeNull();
+    expect(fi.score).not.toBeNull();
+    expect(se.score!).toBeGreaterThan(fi.score!);
   });
 
   it('is null with sampleSize 0 for a function whose category has zero pattern instances', () => {
@@ -338,15 +372,29 @@ describe('computePersonalityProfile — cognitive functions', () => {
     expect(fe.band).toBeUndefined();
   });
 
-  it('maps each of the 8 known categories to its documented function independently', () => {
+  it('scores an isolated single-category sample at 100 — its entire share of the total', () => {
     for (const [category, fn] of Object.entries(PATTERN_TO_FUNCTION)) {
       const facets: PersonalityFacetInput[] = [
         facet({ effectivePatterns: [ep({ category, confidence: 88 })] }),
       ];
       const profile = computePersonalityProfile(facets, [], '2026-W29', '__all__');
-      expect(cogFn(profile.cognitiveFunctions, fn).score).toBe(88);
+      expect(cogFn(profile.cognitiveFunctions, fn).score).toBe(100);
     }
   });
+
+  it('returns all-null functions when there are zero effective pattern instances', () => {
+    const facets: PersonalityFacetInput[] = [facet({ effectivePatterns: [] })];
+    const profile = computePersonalityProfile(facets, [], '2026-W29', '__all__');
+    for (const f of profile.cognitiveFunctions) {
+      expect(f.score).toBeNull();
+      expect(f.sampleSize).toBe(0);
+    }
+  });
+
+  it('sets cognitiveFunctionScoringMode to formula (the only mode this pure function knows about)', () => {
+    const profile = computePersonalityProfile([facet()], [], '2026-W29', '__all__');
+    expect(profile.cognitiveFunctionScoringMode).toBe('formula');
+  });
 });
 
 // ── MBTI derivation ──────────────────────────────────────────────────────────

cli/src/analysis/personality.ts

@@ -5,10 +5,19 @@
 // the server route (server/src/routes/personality.ts) import this module — no
 // reimplementation of scoring logic in either caller.
 //
-// The LLM is used ONLY for the optional `archetype` prose (see server/src/llm/
-// reflect-prompts.ts generatePersonalityPrompt). Every numeric field on
-// PersonalityProfile produced here is deterministic and reproducible from the same
-// inputs — the LLM never contributes a number to this profile.
+// Every numeric field THIS MODULE produces is deterministic and reproducible from the
+// same inputs — this file itself never calls an LLM. Two callers layer optional
+// LLM-authored numbers on top of what this module computes, both in server/src/llm/:
+//   - reflect-prompts.ts generatePersonalityPrompt: the `archetype` prose plus
+//     `mbti.topCandidates[].likelihood`, a top-5 ranked MBTI guess.
+//   - personality-vote.ts scoreCognitiveFunctionsByLlmVote: an OPT-IN alternative to
+//     computeCognitiveFunctions below, gated on dashboard.analysis.personality.
+//     cognitiveFunctionScoring === 'llm-vote' in config.json. When active, it replaces
+//     `cognitiveFunctions` (and therefore `mbti`, re-derived from those scores) with the
+//     average of N independent LLM scoring rounds instead of the formula in this file —
+//     see PersonalityProfile.cognitiveFunctionScoringMode, which records which path ran.
+//     computePersonalityProfile below always returns the deterministic 'formula' scores;
+//     only the server route (POST /generate) can override them post hoc.
 
 import type {
   FrictionPoint,
@@ -26,8 +35,12 @@ import type {
 
 /** Formula version for the deterministic scoring below. Bump when any formula changes
  * so cached personality_snapshots rows can be identified as stale by consumers that care.
- * Bumped to 2.0.0 for the cognitiveFunctions + mbti addition (profileVersion 2). */
-export const PERSONALITY_ANALYSIS_VERSION = '2.0.0';
+ * Bumped to 2.0.0 for the cognitiveFunctions + mbti addition (profileVersion 2).
+ * Bumped to 2.1.0 when computeCognitiveFunctions switched from mean-confidence to
+ * relative-frequency-share scoring (see that function's doc comment) — readSnapshot in
+ * server/src/routes/personality.ts treats any cached row below this version as stale so
+ * old confidence-scored rows get recomputed instead of served forever. */
+export const PERSONALITY_ANALYSIS_VERSION = '2.1.0';
 
 /**
  * Per-session facet input. Deliberately a flattened, caller-friendly shape rather than
@@ -91,7 +104,7 @@ function normalizeConfidence(raw: number): number {
   return raw;
 }
 
-function bandFor(score: number): 'low' | 'moderate' | 'high' {
+export function bandFor(score: number): 'low' | 'moderate' | 'high' {
   if (score >= 65) return 'high';
   if (score >= 35) return 'moderate';
   return 'low';
@@ -276,7 +289,7 @@ function computePace(facets: PersonalityFacetInput[]): PersonalityPace {
 //   self-correction           -> Fi (Introverted Feeling — internally-driven correction against one's own standard)
 //   effective-tooling         -> Fe (Extraverted Feeling — attunement to and effective use of the
 //                                    external/collaborative environment)
-const EFFECTIVE_PATTERN_TO_FUNCTION: Record<string, CognitiveFunctionKey> = {
+export const EFFECTIVE_PATTERN_TO_FUNCTION: Record<string, CognitiveFunctionKey> = {
   'structured-planning': 'ni',
   'context-gathering': 'ne',
   'domain-expertise': 'si',
@@ -288,37 +301,62 @@ const EFFECTIVE_PATTERN_TO_FUNCTION: Record<string, CognitiveFunctionKey> = {
 };
 
 /** Stable, fixed display/serialization order for the 8 cognitive functions. */
-const COGNITIVE_FUNCTION_ORDER: CognitiveFunctionKey[] = ['ni', 'ne', 'si', 'se', 'ti', 'te', 'fi', 'fe'];
+export const COGNITIVE_FUNCTION_ORDER: CognitiveFunctionKey[] = ['ni', 'ne', 'si', 'se', 'ti', 'te', 'fi', 'fe'];
 
 /**
- * One score per Jungian cognitive function: mean confidence (normalized 0-100) of
- * effective-pattern instances whose category maps to that function, via
- * EFFECTIVE_PATTERN_TO_FUNCTION above. Same aggregation style as computeCraft — a flat
- * mean over all matching pattern instances, not a per-session average. Zero instances
- * for a function -> null score, sampleSize 0 (never defaults to 0/neutral — "no signal"
- * and "measured and low" are different things, same convention as every other score
- * in this file).
+ * One score per Jungian cognitive function: RELATIVE FREQUENCY SHARE of effective-pattern
+ * instances mapped to that function (via EFFECTIVE_PATTERN_TO_FUNCTION above) — NOT mean
+ * confidence, despite that being the v1 (analysisVersion 2.0.0) formula.
+ *
+ * Why the change: effective-pattern confidence is written by the analysis prompts with a
+ * hard floor of 70 ("Require a minimum confidence score of 70 for any decision or
+ * learning. Drop insights below this threshold." — cli/src/analysis/prompts.ts) and
+ * clusters in 70-95 in practice. Averaging confidence therefore made every function that
+ * had any samples at all land at or above 65 (the "high" band threshold from bandFor)
+ * almost by construction — it measured "how sure the LLM was when it flagged an
+ * instance," not "how strongly this function shows up relative to the other 7." Jungian
+ * functions (and MBTI more broadly) are an ipsative/competing construct by design —
+ * strength in one function implies relatively less reliance on its opposite — so a score
+ * that can't differentiate across the 8 functions can't produce a meaningful profile.
+ *
+ * Formula: share = count(function) / totalCount(all mapped instances in scope).
+ * fairShare = 1 / 8 (the uniform baseline if all 8 functions were equally represented).
+ * score = round(min(100, (share / fairShare) * 50)) — a function sitting exactly at its
+ * fair share scores 50 (moderate); one at 2x fair share or more scores 100 (high,
+ * capped); one entirely absent relative to the total scores toward 0. This directly
+ * reflects "how often this function's behavior shows up relative to the others," which
+ * is the actual claim a Jungian function score makes, and — unlike mean confidence —
+ * guarantees differentiation across the 8 scores whenever pattern-category usage isn't
+ * perfectly uniform (the overwhelmingly common case). Zero total instances -> every
+ * function null (never a fabricated midpoint); zero instances for one function specifically
+ * -> null score, sampleSize 0 for that function only (same "no signal" convention as every
+ * other score in this file).
  */
 function computeCognitiveFunctions(facets: PersonalityFacetInput[]): CognitiveFunctionScore[] {
-  const sums = new Map<CognitiveFunctionKey, number>();
   const counts = new Map<CognitiveFunctionKey, number>();
+  let totalCount = 0;
 
   for (const facet of facets) {
     for (const ep of facet.effectivePatterns) {
       const fn = EFFECTIVE_PATTERN_TO_FUNCTION[ep.category];
       if (!fn) continue; // unmapped/unknown category — not one of the 8 known effective-pattern categories
-      if (typeof ep.confidence !== 'number' || !Number.isFinite(ep.confidence)) continue;
-      sums.set(fn, (sums.get(fn) ?? 0) + normalizeConfidence(ep.confidence));
       counts.set(fn, (counts.get(fn) ?? 0) + 1);
+      totalCount++;
     }
   }
 
+  if (totalCount === 0) {
+    return COGNITIVE_FUNCTION_ORDER.map(key => ({ key, score: null, sampleSize: 0 }));
+  }
+
+  const fairShare = 1 / COGNITIVE_FUNCTION_ORDER.length;
   return COGNITIVE_FUNCTION_ORDER.map(key => {
     const count = counts.get(key) ?? 0;
     if (count === 0) {
       return { key, score: null, sampleSize: 0 };
     }
-    const score = Math.round((sums.get(key) ?? 0) / count);
+    const share = count / totalCount;
+    const score = Math.round(Math.min(100, (share / fairShare) * 50));
     return { key, score, band: bandFor(score), sampleSize: count };
   });
 }
@@ -455,6 +493,7 @@ export function computePersonalityProfile(
     axis,
     pace,
     cognitiveFunctions,
+    cognitiveFunctionScoringMode: 'formula',
     mbti,
     computedAt: new Date().toISOString(),
     analysisVersion: PERSONALITY_ANALYSIS_VERSION,

cli/src/commands/config.ts

@@ -84,6 +84,17 @@ function showConfigAction(): void {
     console.log(chalk.gray(`    Same-proj:  ${r.sameProjectOnly !== false ? 'yes' : 'no'}`));
   }
 
+  // Personality — cognitive function scoring mode
+  {
+    const p = config.dashboard?.analysis?.personality;
+    const mode = p?.cognitiveFunctionScoring ?? 'formula';
+    console.log(chalk.white('\n  Personality (cognitive functions):'));
+    console.log(chalk.gray(`    Scoring:    ${mode}${mode === 'formula' ? ' (deterministic, default)' : ''}`));
+    if (mode === 'llm-vote') {
+      console.log(chalk.gray(`    Vote rounds: ${p?.llmVoteRounds ?? 3}`));
+    }
+  }
+
   // Telemetry — default is enabled; env vars can override at runtime
   console.log(chalk.white('\n  Telemetry:'));
   const telemetryEnabled = config.telemetry !== false;
@@ -105,7 +116,7 @@ export const configCommand = new Command('config')
 
 configCommand
   .command('set <key> <value>')
-  .description('Set a configuration value (telemetry)')
+  .description('Set a configuration value (telemetry, personality-scoring, personality-vote-rounds)')
   .action((key: string, value: string) => {
     if (key === 'telemetry') {
       if (value !== 'true' && value !== 'false') {
@@ -124,8 +135,52 @@ configCommand
       }
       console.log(chalk.green(`\nTelemetry ${value === 'true' ? 'enabled' : 'disabled'}.\n`));
       trackEvent('cli_config', { subcommand: 'set', success: true });
+    } else if (key === 'personality-scoring') {
+      if (value !== 'formula' && value !== 'llm-vote') {
+        console.error(chalk.red(`\nInvalid value "${value}". Must be "formula" or "llm-vote".\n`));
+        process.exit(1);
+      }
+      const existing = loadConfig() ?? { sync: { claudeDir: '~/.claude/projects', excludeProjects: [] } };
+      existing.dashboard = {
+        ...existing.dashboard,
+        analysis: {
+          ...existing.dashboard?.analysis,
+          personality: {
+            ...existing.dashboard?.analysis?.personality,
+            cognitiveFunctionScoring: value,
+          },
+        },
+      };
+      saveConfig(existing);
+      console.log(chalk.green(`\nCognitive function scoring set to "${value}".`));
+      if (value === 'llm-vote') {
+        console.log(chalk.gray('  Only applies when generating a new snapshot (Generate button / POST /generate) — requires an LLM configured via `code-insights config llm`.\n'));
+      } else {
+        console.log('');
+      }
+      trackEvent('cli_config', { subcommand: 'set', success: true });
+    } else if (key === 'personality-vote-rounds') {
+      const rounds = parseInt(value, 10);
+      if (!Number.isFinite(rounds) || rounds < 1 || rounds > 7) {
+        console.error(chalk.red(`\nInvalid value "${value}". Must be an integer between 1 and 7.\n`));
+        process.exit(1);
+      }
+      const existing = loadConfig() ?? { sync: { claudeDir: '~/.claude/projects', excludeProjects: [] } };
+      existing.dashboard = {
+        ...existing.dashboard,
+        analysis: {
+          ...existing.dashboard?.analysis,
+          personality: {
+            ...existing.dashboard?.analysis?.personality,
+            llmVoteRounds: rounds,
+          },
+        },
+      };
+      saveConfig(existing);
+      console.log(chalk.green(`\nLLM vote rounds set to ${rounds}.\n`));
+      trackEvent('cli_config', { subcommand: 'set', success: true });
     } else {
-      console.error(chalk.red(`\nUnknown config key "${key}". Available: telemetry.\n`));
+      console.error(chalk.red(`\nUnknown config key "${key}". Available: telemetry, personality-scoring, personality-vote-rounds.\n`));
       process.exit(1);
     }
   });