diff --git a/adapter/codex/hooks/on-session-start.ps1 b/adapter/codex/hooks/on-session-start.ps1 index e8bcb0d..04f2d18 100644 --- a/adapter/codex/hooks/on-session-start.ps1 +++ b/adapter/codex/hooks/on-session-start.ps1 @@ -348,32 +348,118 @@ if ($selfEvalFound) { Register-Section 'self_eval_head' 'Self-evaluation log head (most recent)' $selfEvalHead # promotion candidates (memory -> Li+ source) +# Evolution Loop observe stage: surface pattern-detection candidates at cold-start +# so that AI sees promotion candidates without waiting for passive noticing. +# rules/evolution/evolution.md "Pattern Detection Surfacing At Cold-start" fixes +# the three detection targets (self-evaluation log repetition / recent memory +# additions / keyword overlap with Li+ source) and delegates the thresholds and +# the concrete logic to the adapter, which is this block. +# +# #1632 F3 / #1636: every detector below used to read a format nothing writes, so +# the section was empty on every session and an empty surface could not be told +# apart from "nothing crossed the noise floor". Detector 1 matched a +# `root_cause:` / `tags:` line syntax that no spec defines and the log has never +# used; detectors 2 and 3 scanned the flat feedback.md / project.md pair that the +# one-memory-per-file host layout replaced. The formats read below are the ones +# the live artifacts actually carry, and they are the same formats the sibling +# ports read — the detection behavior is deliberately identical across +# adapter/claude/hooks/on-session-start.sh, on-session-start.sh and this file. +# +# Case sensitivity: the bash ports match with awk / sed / `case`, which are +# case-sensitive, so every comparison here that stands in for one of those uses +# the case-sensitive operator (`-cmatch` / `-ccontains` / `-cnotcontains`) or an +# ordinal comparer. `ToLowerInvariant` stands in for awk `tolower`, which is +# ASCII-only and culture-independent. $thresholdN = 2 +# $surfaceCap bounds the two list-shaped detectors. This is an orientation +# surface read at session opening, and a list past roughly this length stops +# being scannable; the full count is still printed, so a truncated list never +# hides that more exists. +$surfaceCap = 10 + # A candidate qualifies as $memoryDir only when it holds at least one file that # some $memoryDir consumer reads. Directory existence alone is not the criterion: # an empty higher-precedence directory would otherwise shadow a populated # lower-precedence one and silence every consumer at once. -# The marker set is the files $memoryDir consumers read (feedback / project -# detectors, self-evolution observation surface), plus self-evaluation_log.md so -# that both resolution paths agree on what counts as a memory directory. That -# added member never decides a case in practice: the self-eval lookup above -# scans the same candidate directories, so whenever that file exists the primary -# path has already claimed the directory before this check runs. +# The marker set is the files $memoryDir consumers read: the observation surface, +# the per-topic entry-file prefixes the promotion detectors scan, plus +# self-evaluation_log.md so that both resolution paths agree on what counts as a +# memory directory. That last member never decides a case in practice: the +# self-eval lookup above scans the same candidate directories, so whenever that +# file exists the primary path has already claimed the directory before this +# check runs. +# The prefixes replace the former flat feedback.md / project.md pair: the host +# auto-memory layout is one memory per file, so a live memory directory holds +# feedback_.md / project_.md / reference_.md / +# user_.md and neither flat name exists. Matching prefixes rather than any +# *.md is deliberate — an unrelated file must not let a directory claim the slot. function Test-MemoryDirPopulated { param([string]$Dir) foreach ($markerFile in @( 'self-evaluation_log.md', - 'feedback.md', - 'project.md', 'self-evolution-observation.md')) { if (Test-Path -LiteralPath (Join-Path $Dir $markerFile) -PathType Leaf) { return $true } } + foreach ($markerPrefix in @('feedback', 'project', 'reference', 'user')) { + $hit = Get-ChildItem -LiteralPath $Dir -Filter "$markerPrefix*.md" -File -ErrorAction SilentlyContinue | + Select-Object -First 1 + if ($hit) { return $true } + } return $false } +# Memory entry files inside a resolved $memoryDir, under the same one-memory- +# per-file layout. Excluded are the index and the three transient operational +# files, each of which has its own dedicated reader. Flat feedback.md / +# project.md are NOT excluded, so a workspace that has not migrated is still +# scanned. Ordinal sort, because the detector output is sha256-fingerprinted for +# diff-only emission and must not depend on directory order — and to match the +# bash ports, whose plain `sort` is bytewise under a C locale. No bash port pins +# LC_ALL=C, so that match holds by environment rather than by enforcement; under +# a culture-aware locale the bash order can differ from this one. +function Get-MemoryEntryFiles { + param([string]$Dir) + $skip = @('MEMORY.md', 'promotion_tally.md', 'self-evaluation_log.md', 'self-evolution-observation.md') + $names = [string[]]@( + Get-ChildItem -LiteralPath $Dir -Filter '*.md' -File -ErrorAction SilentlyContinue | + Where-Object { $skip -cnotcontains $_.Name } | + ForEach-Object { $_.Name }) + if ($names.Count -eq 0) { return @() } + [Array]::Sort($names, [System.StringComparer]::Ordinal) + return @($names | ForEach-Object { Join-Path $Dir $_ }) +} + +# Title of one memory entry = its frontmatter `name:` (written by the host +# auto-memory) when present, else the filename stem. Under the flat-file layout +# the equivalent unit was a `## ` section header; with one memory per file the +# file itself is the entry, so the title moves to the frontmatter. +function Get-MemoryEntryTitle { + param([string]$Path) + $title = '' + foreach ($line in @(Get-Content -LiteralPath $Path -TotalCount 10 -ErrorAction SilentlyContinue)) { + if ($line -cmatch '^name:[ \t]*(.*)$') { $title = $matches[1]; break } + } + $title = ($title -replace "`r", '') -replace '\s+$', '' + if (-not $title) { return [System.IO.Path]::GetFileNameWithoutExtension($Path) } + return $title +} + +# One (axis, verdict) pair out of a self-evaluation entry's axis-tag list. +# Counted only when the verdict carries the word `miss`. +function Add-AxisMiss { + param([hashtable]$Tally, [string]$Pair) + $sep = $Pair.IndexOf(':') + if ($sep -lt 0) { return } + $axis = ($Pair.Substring(0, $sep) -replace '\*', '').Trim().ToLowerInvariant() + $verdict = $Pair.Substring($sep + 1) + if (-not $axis) { return } + if ($verdict.ToLowerInvariant().IndexOf('miss') -lt 0) { return } + if ($Tally.ContainsKey($axis)) { $Tally[$axis]++ } else { $Tally[$axis] = 1 } +} + # Probe the directory directly when self-evaluation_log.md is absent: the other -# $memoryDir readers (feedback/project detectors, self-evolution observation -# surface) must not be silenced by the absence of an unrelated file. +# $memoryDir readers (promotion detectors, self-evolution observation surface) +# must not be silenced by the absence of an unrelated file. $memoryDir = '' if ($selfEvalFound) { $memoryDir = Split-Path -Parent $selfEvalFound @@ -384,75 +470,194 @@ if ($selfEvalFound) { } $promotionBody = '' -# Detector 1: repeated (root_cause, first-tag) pairs in self-evaluation_log.md. +# Detector 1: the same observational axis tagged `miss` across several +# self-evaluation entries. That repetition is the one the spec names: +# `skills/evolution-self-eval/SKILL.md` Recording — "Repeated miss on the same +# axis across entries = weakness region = distill candidate for evolution loop". +# +# Two entry layouts are in live use and both are read: +# **Axis tags**: : / : / ... (one line) +# **Axis tags (10-axis)**: (header, then) +# - : (bullets) +# A verdict counts as a miss when the word appears anywhere in it, so +# `**miss (primary)**` and `miss→hit` both register as an observed miss. +# Axis names are lowercased before tallying; `**` emphasis is stripped. if ($selfEvalFound -and (Test-Path -LiteralPath $selfEvalFound)) { - $pairCount = @{} - $rc = '' - foreach ($l in (Get-Content -LiteralPath $selfEvalFound -ErrorAction SilentlyContinue)) { - if ($l -match '^\s*root_cause:\s*(.*)$') { $rc = $matches[1].Trim(); continue } - if ($l -match '^\s*tags:\s*(.*)$' -and $rc -ne '') { - $tag = (($matches[1] -split ',')[0]).Trim() - if ($tag) { $key = "$rc|$tag"; if ($pairCount.ContainsKey($key)) { $pairCount[$key]++ } else { $pairCount[$key] = 1 } } - $rc = '' + $axisCount = @{} + $inAxisBlock = $false + foreach ($l in @(Get-Content -LiteralPath $selfEvalFound -ErrorAction SilentlyContinue)) { + if ($l -cmatch '^\s*\*\*Axis tags') { + # Everything past the closing "**:" of the label is the inline pair list; + # an empty remainder means the bullet layout follows. + $labelEnd = $l.IndexOf('**:') + $rest = if ($labelEnd -ge 0) { $l.Substring($labelEnd + 3) } else { '' } + if ($rest -match '\S') { + foreach ($part in ($rest -split ' / ')) { Add-AxisMiss $axisCount $part } + $inAxisBlock = $false + } else { + $inAxisBlock = $true + } + continue } - } - $dupes = '' - foreach ($k in $pairCount.Keys) { - if ($pairCount[$k] -ge $thresholdN) { - $p = $k -split '\|' - $dupes += " - ($($p[0]), $($p[1])) x$($pairCount[$k])`n" + if ($inAxisBlock -and ($l -match '^\s*-\s')) { + Add-AxisMiss $axisCount ($l -replace '^\s*-\s*', '') + continue } + if ($inAxisBlock) { $inAxisBlock = $false } + } + $axisLines = [string[]]@( + $axisCount.Keys | + Where-Object { $axisCount[$_] -ge $thresholdN } | + ForEach-Object { ' - axis "{0}" tagged miss x{1}' -f $_, $axisCount[$_] }) + if ($axisLines.Count -gt 0) { + [Array]::Sort($axisLines, [System.StringComparer]::Ordinal) + $promotionBody += "repeated self-evaluation axis misses:`n" + ($axisLines -join "`n") + "`n" } - if ($dupes) { $promotionBody += "repeated (root_cause, domain-tag) pairs:`n$dupes" } } -# Detector 2: recent (<=7d) memory section additions in feedback.md / project.md. +# Detector 2: memory entries written or rewritten within the last 7 days. +# One memory is one file, so the entry is the unit of recency and file mtime is +# the signal; the flat-file era counted '## ' sections inside two files instead. +# Flagged when the count reaches $thresholdN. +# +# Note: this 7d window is the memory-scan recency window (Cold-start observe +# stage surface), independent from the 3d cluster window in +# rules/evolution/promotion-judgment.md. The two timers serve different axes: +# - 7d here = "did anything new land in memory recently? show it for AI review" +# - 3d there = "has the same cluster crossed the noise floor for promotion?" +# Do not unify the two values; they intentionally sit on different axes. if ($memoryDir -and (Test-Path -LiteralPath $memoryDir)) { - $recentSections = '' - foreach ($mf in @((Join-Path $memoryDir 'feedback.md'), (Join-Path $memoryDir 'project.md'))) { - if (-not (Test-Path -LiteralPath $mf)) { continue } + $recentEntries = '' + $recentCount = 0 + $recentCutoff = (Get-Date).AddDays(-7) + foreach ($mf in (Get-MemoryEntryFiles $memoryDir)) { $mtime = (Get-Item -LiteralPath $mf -ErrorAction SilentlyContinue).LastWriteTime - if ($mtime -and $mtime -ge (Get-Date).AddDays(-7)) { - $secs = Select-String -LiteralPath $mf -Pattern '^## ' -ErrorAction SilentlyContinue | ForEach-Object { ' - ' + ($_.Line -replace '^## ', '') } - if ($secs -and $secs.Count -ge $thresholdN) { - $recentSections += "$(Split-Path -Leaf $mf) (modified within 7d, $($secs.Count) sections):`n" + ($secs -join "`n") + "`n" + if ($mtime -and $mtime -ge $recentCutoff) { + $recentCount++ + if ($recentCount -le $surfaceCap) { + $recentEntries += " - $(Split-Path -Leaf $mf) [$(Get-MemoryEntryTitle $mf)]`n" } } } - if ($recentSections) { $promotionBody += "recent memory additions (<= 7d):`n$recentSections" } + if ($recentCount -ge $thresholdN) { + # A consolidate pass rewrites every entry at once, so the cap is a normal + # occurrence rather than an edge case. + if ($recentCount -gt $surfaceCap) { + $recentEntries += " - ... and $($recentCount - $surfaceCap) more`n" + } + $promotionBody += "recent memory additions (<= 7d, $recentCount entries):`n$recentEntries" + } } -# Detector 3: keyword overlap between memory section titles and Li+ source files. +# Detector 3: keyword overlap between memory entry titles and Li+ source files. +# Tokens are the >= 4-char ASCII alphanumeric words of an entry title; a hit +# means the entry's topic already has a surface in rules/ or skills/. Surfaced +# as a "possible overlap" hint — not a promotion decision. +# +# The four entry-type prefixes are dropped: under the per-topic naming scheme +# they classify the entry rather than name its topic, so they would match nearly +# every source file and drown the real signal. A non-ASCII title yields no +# tokens and is simply skipped, as before. +# +# A pair is reported only once at least $thresholdN distinct title tokens land in +# the same source file. One shared common word ("identity", "answer") is +# coincidence at this corpus size and produced hundreds of lines when measured +# against the live memory set; two independent words of one title meeting in one +# file is topical adjacency, which is what this detector is looking for. +# +# The source is named by its path relative to the clone, not by basename: every +# skill file is called SKILL.md, so a basename label identifies nothing. +# +# Matching is on whole words, not substrings: each source file is split on +# non-alphanumeric runs exactly as the titles are, so `user` no longer hits +# inside `users` and the two bash ports and this one agree on what a hit is. +# The rank prefix (inverted token count, zero padded) makes a plain ordinal sort +# order strongest adjacency first, and the cap is applied after the sort — +# capping an unordered set would pick a different subset per run and churn the +# sha256 the diff-only emission compares against. if ($memoryDir -and (Test-Path -LiteralPath $memoryDir)) { - $overlap = '' + # Parallel arrays, deliberately not de-duplicated: a token repeated inside one + # title counts twice, matching the bash ports. + $tokenLabels = @() + $tokenNames = @() + foreach ($mf in (Get-MemoryEntryFiles $memoryDir)) { + $entryTitle = Get-MemoryEntryTitle $mf + $entryLabel = "$(Split-Path -Leaf $mf) [$entryTitle]" + foreach ($tok in ($entryTitle.ToLowerInvariant() -split '[^a-z0-9]+')) { + if ($tok.Length -lt 4) { continue } + if (@('feedback', 'project', 'reference', 'user') -ccontains $tok) { continue } + $tokenLabels += $entryLabel + $tokenNames += $tok + } + } $srcFiles = @() - $srcFiles += Get-ChildItem -LiteralPath $rulesRoot -Recurse -Filter '*.md' -File -ErrorAction SilentlyContinue + if (Test-Path -LiteralPath $rulesRoot) { + $srcFiles += @(Get-ChildItem -LiteralPath $rulesRoot -Recurse -Filter '*.md' -File -ErrorAction SilentlyContinue) + } $skillsRoot = Join-Path $liplusDir 'skills' if (Test-Path -LiteralPath $skillsRoot) { - $srcFiles += Get-ChildItem -LiteralPath $skillsRoot -Recurse -Filter 'SKILL.md' -File -ErrorAction SilentlyContinue - } - $srcLc = @{} - foreach ($sf in $srcFiles) { - $c = Get-Content -LiteralPath $sf.FullName -Raw -ErrorAction SilentlyContinue - if ($c) { $srcLc[$sf.FullName] = $c.ToLower() } + $srcFiles += @(Get-ChildItem -LiteralPath $skillsRoot -Recurse -Depth 1 -Filter 'SKILL.md' -File -ErrorAction SilentlyContinue) } - foreach ($mf in @((Join-Path $memoryDir 'feedback.md'), (Join-Path $memoryDir 'project.md'))) { - if (-not (Test-Path -LiteralPath $mf)) { continue } - $headers = Select-String -LiteralPath $mf -Pattern '^## ' -ErrorAction SilentlyContinue - foreach ($h in $headers) { - $title = ($h.Line -replace '^## ', '') - $tokens = ($title.ToLower() -split '[^a-z0-9]+') | Where-Object { $_.Length -ge 4 } | Select-Object -Unique - if (-not $tokens) { continue } - foreach ($sfPath in $srcLc.Keys) { - $hit = @() - foreach ($tok in $tokens) { if ($srcLc[$sfPath].Contains($tok)) { $hit += $tok } } - if ($hit.Count -gt 0) { - $overlap += " - $(Split-Path -Leaf $mf) [$title] ~ $(Split-Path -Leaf $sfPath) (tokens: $($hit -join ' '))`n" + if ($tokenNames.Count -gt 0 -and $srcFiles.Count -gt 0) { + $wanted = @{} + foreach ($tok in $tokenNames) { $wanted[$tok] = $true } + $srcHits = @{} + $rootPrefix = ($liplusDir -replace '\\', '/').TrimEnd('/') + '/' + foreach ($sf in $srcFiles) { + $rel = $sf.FullName -replace '\\', '/' + if ($rel.StartsWith($rootPrefix)) { $rel = $rel.Substring($rootPrefix.Length) } + $content = Get-Content -LiteralPath $sf.FullName -Raw -ErrorAction SilentlyContinue + if (-not $content) { continue } + $seen = @{} + foreach ($word in ($content.ToLowerInvariant() -split '[^a-z0-9]+')) { + if (-not $wanted.ContainsKey($word)) { continue } + if ($seen.ContainsKey($word)) { continue } + $seen[$word] = $true + if (-not $srcHits.ContainsKey($word)) { $srcHits[$word] = @() } + $srcHits[$word] += $rel + } + } + # Ordinal comparer, not the `@{}` literal: a PowerShell hashtable literal + # keys case-insensitively, while the awk arrays the bash ports use are + # case-sensitive. This key carries the entry title and the source path at + # their original case, so the literal would merge two pairs the bash ports + # keep apart. ($wanted / $seen / $srcHits key on already-lowercased tokens, + # so the default comparer is equivalent there.) + $pairs = New-Object System.Collections.Hashtable ([System.StringComparer]::Ordinal) + for ($j = 0; $j -lt $tokenNames.Count; $j++) { + $tok = $tokenNames[$j] + if (-not $srcHits.ContainsKey($tok)) { continue } + foreach ($rel in $srcHits[$tok]) { + $key = $tokenLabels[$j] + [char]28 + $rel + if ($pairs.ContainsKey($key)) { + $pairs[$key].Tokens += " $tok" + $pairs[$key].Depth++ + } else { + $pairs[$key] = [pscustomobject]@{ + Label = $tokenLabels[$j] + Rel = $rel + Tokens = " $tok" + Depth = 1 + } } } } + $overlapRanked = [string[]]@( + $pairs.Values | + Where-Object { $_.Depth -ge $thresholdN } | + ForEach-Object { ('{0:d3}' -f (999 - $_.Depth)) + "`t" + " - $($_.Label) ~ $($_.Rel) (tokens:$($_.Tokens))" }) + if ($overlapRanked.Count -gt 0) { + [Array]::Sort($overlapRanked, [System.StringComparer]::Ordinal) + $overlapCount = $overlapRanked.Count + $overlapText = (@($overlapRanked | + Select-Object -First $surfaceCap | + ForEach-Object { $_.Substring($_.IndexOf("`t") + 1) }) -join "`n") + if ($overlapCount -gt $surfaceCap) { + $overlapText += "`n - ... and $($overlapCount - $surfaceCap) more" + } + $promotionBody += "possible keyword overlap with Li+ source ($overlapCount pairs):`n$overlapText`n" + } } - if ($overlap) { $promotionBody += "possible keyword overlap with Li+ source:`n$overlap" } } Register-Section 'promotion_candidates' 'Promotion candidates (memory → Li+ source)' $promotionBody diff --git a/adapter/codex/hooks/on-session-start.sh b/adapter/codex/hooks/on-session-start.sh index b775aeb..737f381 100644 --- a/adapter/codex/hooks/on-session-start.sh +++ b/adapter/codex/hooks/on-session-start.sh @@ -289,31 +289,92 @@ SELFEVAL_HEAD="" register_section "self_eval_head" "Self-evaluation log head (most recent)" "$SELFEVAL_HEAD" # promotion candidates +# Evolution Loop observe stage: surface pattern-detection candidates at cold-start +# so that AI sees promotion candidates without waiting for passive noticing. +# rules/evolution/evolution.md "Pattern Detection Surfacing At Cold-start" fixes +# the three detection targets (self-evaluation log repetition / recent memory +# additions / keyword overlap with Li+ source) and delegates the thresholds and +# the concrete logic to the adapter, which is this block. +# All three detectors are best-effort; silent skip when sources are absent. +# Threshold is adjustable via THRESHOLD_N (initial value = 2, see issue #1080). +# SURFACE_CAP bounds the two list-shaped detectors. This is an orientation +# surface read at session opening, and a list past roughly this length stops +# being scannable; the full count is still printed, so a truncated list never +# hides that more exists. +# +# #1632 F3 / #1636: every detector below used to read a format nothing writes, so +# the section was empty on every session and an empty surface could not be told +# apart from "nothing crossed the noise floor". Detector 1 matched a +# `root_cause:` / `tags:` line syntax that no spec defines and the log has never +# used; detectors 2 and 3 scanned the flat feedback.md / project.md pair that the +# one-memory-per-file host layout replaced. The formats read below are the ones +# the live artifacts actually carry, and they are the same formats the sibling +# ports read — the detection behavior is deliberately identical across +# adapter/claude/hooks/on-session-start.sh, this file and on-session-start.ps1. THRESHOLD_N=2 +SURFACE_CAP=10 + # A candidate qualifies as MEMORY_DIR only when it holds at least one file that # some MEMORY_DIR consumer reads. Directory existence alone is not the criterion: # an empty higher-precedence directory would otherwise shadow a populated # lower-precedence one and silence every consumer at once. -# The marker set is the files MEMORY_DIR consumers read (feedback / project -# detectors, self-evolution observation surface), plus self-evaluation_log.md so -# that both resolution paths agree on what counts as a memory directory. That -# added member never decides a case in practice: the self-eval lookup above -# scans the same candidate directories, so whenever that file exists the primary -# path has already claimed the directory before this check runs. +# The marker set is the files MEMORY_DIR consumers read: the observation surface, +# the per-topic entry-file prefixes the promotion detectors scan, plus +# self-evaluation_log.md so that both resolution paths agree on what counts as a +# memory directory. That last member never decides a case in practice: the +# self-eval lookup above scans the same candidate directories, so whenever that +# file exists the primary path has already claimed the directory before this +# check runs. +# The prefixes replace the former flat feedback.md / project.md pair: the host +# auto-memory layout is one memory per file, so a live memory directory holds +# feedback_.md / project_.md / reference_.md / +# user_.md and neither flat name exists. Matching prefixes rather than any +# *.md is deliberate — an unrelated file must not let a directory claim the slot. memory_dir_populated() { for markerfile in \ self-evaluation_log.md \ - feedback.md \ - project.md \ self-evolution-observation.md; do [ -f "$1/$markerfile" ] && return 0 done + for markerglob in "$1"/feedback*.md "$1"/project*.md "$1"/reference*.md "$1"/user*.md; do + [ -f "$markerglob" ] && return 0 + done return 1 } +# Memory entry files inside a resolved MEMORY_DIR, under the same one-memory- +# per-file layout. Excluded are the index and the three transient operational +# files, each of which has its own dedicated reader. Flat feedback.md / +# project.md are NOT excluded, so a workspace that has not migrated is still +# scanned. Sorted, because the detector output is sha256-fingerprinted for +# diff-only emission and must not depend on directory order. +memory_entry_files() { + find "$1" -maxdepth 1 -type f -name '*.md' 2>/dev/null | sort | while IFS= read -r entryfile; do + case "${entryfile##*/}" in + MEMORY.md|promotion_tally.md|self-evaluation_log.md|self-evolution-observation.md) ;; + *) printf '%s\n' "$entryfile" ;; + esac + done +} + +# Title of one memory entry = its frontmatter `name:` (written by the host +# auto-memory) when present, else the filename stem. Under the flat-file layout +# the equivalent unit was a `## ` section header; with one memory per file the +# file itself is the entry, so the title moves to the frontmatter. +memory_entry_title() { + local title + title=$(head -n 10 "$1" 2>/dev/null | sed -n 's/^name:[[:space:]]*//p' | head -n 1 \ + | tr -d '\r' | sed 's/[[:space:]]*$//') + if [ -z "$title" ]; then + title="${1##*/}" + title="${title%.md}" + fi + printf '%s' "$title" +} + # Probe the directory directly when self-evaluation_log.md is absent: the other -# MEMORY_DIR readers (feedback/project detectors, self-evolution observation -# surface) must not be silenced by the absence of an unrelated file. +# MEMORY_DIR readers (promotion detectors, self-evolution observation surface) +# must not be silenced by the absence of an unrelated file. MEMORY_DIR="" if [ -n "$SELFEVAL_FOUND" ]; then MEMORY_DIR=$(dirname "$SELFEVAL_FOUND") @@ -324,68 +385,216 @@ else fi PROMOTION_BODY="" +# Detector 1: the same observational axis tagged `miss` across several +# self-evaluation entries. That repetition is the one the spec names: +# `skills/evolution-self-eval/SKILL.md` Recording — "Repeated miss on the same +# axis across entries = weakness region = distill candidate for evolution loop". +# +# Two entry layouts are in live use and both are read: +# **Axis tags**: : / : / ... (one line) +# **Axis tags (10-axis)**: (header, then) +# - : (bullets) +# A verdict counts as a miss when the word appears anywhere in it, so +# `**miss (primary)**` and `miss→hit` both register as an observed miss. +# Axis names are lowercased before tallying; `**` emphasis is stripped. if [ -n "$SELFEVAL_FOUND" ] && [ -f "$SELFEVAL_FOUND" ]; then - PAIR_DUPES=$(awk -v n="$THRESHOLD_N" ' - /^[[:space:]]*root_cause:[[:space:]]*/ { sub(/^[[:space:]]*root_cause:[[:space:]]*/, ""); rc=$0; next } - /^[[:space:]]*tags:[[:space:]]*/ && rc != "" { - sub(/^[[:space:]]*tags:[[:space:]]*/, ""); split($0, t, /,[[:space:]]*/); tag=t[1] - gsub(/[[:space:]]+$/, "", tag) - if (tag != "") { key=rc "|" tag; count[key]++ } - rc="" + AXIS_MISSES=$(awk -v n="$THRESHOLD_N" ' + function record(pair, sep, axis, verdict) { + sep = index(pair, ":") + if (sep == 0) return + axis = substr(pair, 1, sep - 1) + verdict = substr(pair, sep + 1) + gsub(/\*/, "", axis) + sub(/^[[:space:]]+/, "", axis) + sub(/[[:space:]]+$/, "", axis) + axis = tolower(axis) + if (axis == "") return + if (index(tolower(verdict), "miss") == 0) return + count[axis]++ } - END { for (k in count) if (count[k] >= n) { split(k, p, "|"); printf " - (%s, %s) x%d\n", p[1], p[2], count[k] } } - ' "$SELFEVAL_FOUND") - [ -n "$PAIR_DUPES" ] && PROMOTION_BODY="${PROMOTION_BODY}repeated (root_cause, domain-tag) pairs: -${PAIR_DUPES} + /^[[:space:]]*\*\*Axis tags/ { + # Everything past the closing "**:" of the label is the inline pair list; + # an empty remainder means the bullet layout follows. + label_end = index($0, "**:") + rest = (label_end > 0) ? substr($0, label_end + 3) : "" + if (rest ~ /[^[:space:]]/) { + pairs = split(rest, part, / \/ /) + for (i = 1; i <= pairs; i++) record(part[i]) + in_axis_block = 0 + } else { + in_axis_block = 1 + } + next + } + in_axis_block && /^[[:space:]]*-[[:space:]]/ { + bullet = $0 + sub(/^[[:space:]]*-[[:space:]]*/, "", bullet) + record(bullet) + next + } + in_axis_block { in_axis_block = 0 } + END { + for (axis in count) { + if (count[axis] >= n) { + printf " - axis \"%s\" tagged miss x%d\n", axis, count[axis] + } + } + } + ' "$SELFEVAL_FOUND" | sort) + [ -n "$AXIS_MISSES" ] && PROMOTION_BODY="${PROMOTION_BODY}repeated self-evaluation axis misses: +${AXIS_MISSES} " fi +# Detector 2: memory entries written or rewritten within the last 7 days. +# One memory is one file, so the entry is the unit of recency and file mtime is +# the signal; the flat-file era counted '## ' sections inside two files instead. +# Flagged when the count reaches THRESHOLD_N. +# +# Note: this 7d window is the memory-scan recency window (Cold-start observe +# stage surface), independent from the 3d cluster window in +# rules/evolution/promotion-judgment.md. The two timers serve different axes: +# - 7d here = "did anything new land in memory recently? show it for AI review" +# - 3d there = "has the same cluster crossed the noise floor for promotion?" +# Do not unify the two values; they intentionally sit on different axes. if [ -n "$MEMORY_DIR" ] && [ -d "$MEMORY_DIR" ]; then - RECENT_SECTIONS="" - for memfile in "$MEMORY_DIR/feedback.md" "$MEMORY_DIR/project.md"; do - [ -f "$memfile" ] || continue + RECENT_ENTRIES="" + RECENT_COUNT=0 + while IFS= read -r memfile; do + [ -n "$memfile" ] || continue if find "$memfile" -mtime -7 -print 2>/dev/null | grep -q .; then - SECTIONS=$(grep -E '^## ' "$memfile" 2>/dev/null | sed 's/^## / - /') - SEC_COUNT=$(printf '%s\n' "$SECTIONS" | grep -c '^ - ' 2>/dev/null) - if [ "${SEC_COUNT:-0}" -ge "$THRESHOLD_N" ]; then - RECENT_SECTIONS="${RECENT_SECTIONS}$(basename "$memfile") (modified within 7d, ${SEC_COUNT} sections): -${SECTIONS} + RECENT_COUNT=$((RECENT_COUNT + 1)) + if [ "$RECENT_COUNT" -le "$SURFACE_CAP" ]; then + RECENT_ENTRIES="${RECENT_ENTRIES} - ${memfile##*/} [$(memory_entry_title "$memfile")] " fi fi - done - [ -n "$RECENT_SECTIONS" ] && PROMOTION_BODY="${PROMOTION_BODY}recent memory additions (<= 7d): -${RECENT_SECTIONS}" + done < <(memory_entry_files "$MEMORY_DIR") + if [ "$RECENT_COUNT" -ge "$THRESHOLD_N" ]; then + # A consolidate pass rewrites every entry at once, so the cap is a normal + # occurrence rather than an edge case. + if [ "$RECENT_COUNT" -gt "$SURFACE_CAP" ]; then + RECENT_ENTRIES="${RECENT_ENTRIES} - ... and $((RECENT_COUNT - SURFACE_CAP)) more +" + fi + PROMOTION_BODY="${PROMOTION_BODY}recent memory additions (<= 7d, ${RECENT_COUNT} entries): +${RECENT_ENTRIES}" + fi fi +# Detector 3: keyword overlap between memory entry titles and Li+ source files. +# Tokens are the >= 4-char ASCII alphanumeric words of an entry title; a hit +# means the entry's topic already has a surface in rules/ or skills/. Surfaced +# as a "possible overlap" hint — not a promotion decision. +# +# The four entry-type prefixes are dropped: under the per-topic naming scheme +# they classify the entry rather than name its topic, so they would match nearly +# every source file and drown the real signal. A non-ASCII title yields no +# tokens and is simply skipped, as before. +# +# A pair is reported only once at least THRESHOLD_N distinct title tokens land in +# the same source file. One shared common word ("identity", "answer") is +# coincidence at this corpus size and produced hundreds of lines when measured +# against the live memory set; two independent words of one title meeting in one +# file is topical adjacency, which is what this detector is looking for. +# +# The source is named by its path relative to the clone, not by basename: every +# skill file is called SKILL.md, so a basename label identifies nothing. +# +# One awk pass over the whole source set replaces the previous nested loop, +# which re-read every source file once per title. Source paths are handed over +# as a list file and read with getline, so a workspace path containing spaces +# stays intact. awk emits every qualifying pair; the sort and the cap are +# applied after it, in that order. Sorting first is what makes the truncation +# stable — awk's `for (k in array)` order is unspecified, so capping inside awk +# would pick a different subset per run and churn the sha256 the diff-only +# emission compares against. if [ -n "$MEMORY_DIR" ] && [ -d "$MEMORY_DIR" ]; then OVERLAP="" - TMP_HEADERS=$(mktemp 2>/dev/null || echo "/tmp/liplus-headers-$$") + OVERLAP_ALL="" TMP_TOKENS=$(mktemp 2>/dev/null || echo "/tmp/liplus-tokens-$$") - for memfile in "$MEMORY_DIR/feedback.md" "$MEMORY_DIR/project.md"; do - [ -f "$memfile" ] || continue - grep -E '^## ' "$memfile" 2>/dev/null > "$TMP_HEADERS" || true - while IFS= read -r header; do - [ -n "$header" ] || continue - title=$(printf '%s' "$header" | sed 's/^## //') - printf '%s' "$title" | tr 'A-Z' 'a-z' | tr -cs 'a-z0-9' '\n' | awk 'length($0) >= 4' > "$TMP_TOKENS" - [ -s "$TMP_TOKENS" ] || continue - while IFS= read -r src; do - [ -f "$src" ] || continue - HIT="" - SRC_LC=$(tr 'A-Z' 'a-z' < "$src") - while IFS= read -r tok; do - [ -n "$tok" ] || continue - printf '%s' "$SRC_LC" | grep -qF "$tok" 2>/dev/null && HIT="${HIT}${tok} " - done < "$TMP_TOKENS" - [ -n "$HIT" ] && OVERLAP="${OVERLAP} - $(basename "$memfile") [${title}] ~ $(basename "$src") (tokens: ${HIT% }) + TMP_SRCLIST=$(mktemp 2>/dev/null || echo "/tmp/liplus-srclist-$$") + : > "$TMP_TOKENS" + while IFS= read -r memfile; do + [ -n "$memfile" ] || continue + entry_title=$(memory_entry_title "$memfile") + entry_label="${memfile##*/} [${entry_title}]" + printf '%s' "$entry_title" | tr 'A-Z' 'a-z' | tr -cs 'a-z0-9' '\n' \ + | awk -v lbl="$entry_label" ' + length($0) >= 4 && + $0 != "feedback" && $0 != "project" && $0 != "reference" && $0 != "user" { + print lbl "\t" $0 + }' >> "$TMP_TOKENS" + done < <(memory_entry_files "$MEMORY_DIR") + find "$LIPLUS_DIR/rules" -type f -name '*.md' 2>/dev/null > "$TMP_SRCLIST" + find "$LIPLUS_DIR/skills" -maxdepth 2 -type f -name 'SKILL.md' 2>/dev/null >> "$TMP_SRCLIST" + if [ -s "$TMP_TOKENS" ] && [ -s "$TMP_SRCLIST" ]; then + OVERLAP_ALL=$(awk -v n="$THRESHOLD_N" -v root="$LIPLUS_DIR/" ' + # pass 1: "\t" lines + NR == FNR { + sep = index($0, "\t") + if (sep == 0) next + tn++ + tlabel[tn] = substr($0, 1, sep - 1) + ttok[tn] = substr($0, sep + 1) + wanted[ttok[tn]] = 1 + next + } + # pass 2: one Li+ source path per line + { + path = $0 + if (path == "") next + src = path + if (substr(src, 1, length(root)) == root) src = substr(src, length(root) + 1) + gsub(/\\/, "/", src) + while ((getline srcline < path) > 0) { + words = tolower(srcline) + gsub(/[^a-z0-9]+/, " ", words) + wc = split(words, w, " ") + for (i = 1; i <= wc; i++) { + if (!(w[i] in wanted)) continue + if ((src SUBSEP w[i]) in seen) continue + seen[src SUBSEP w[i]] = 1 + srcs[w[i]] = srcs[w[i]] " " src + } + } + close(path) + } + END { + for (j = 1; j <= tn; j++) { + if (!(ttok[j] in srcs)) continue + fc = split(srcs[ttok[j]], f, " ") + for (i = 1; i <= fc; i++) { + if (f[i] == "") continue + key = tlabel[j] SUBSEP f[i] + hit[key] = hit[key] " " ttok[j] + depth[key]++ + } + } + for (key in hit) { + if (depth[key] < n) continue + split(key, part, SUBSEP) + # Rank prefix = inverted token count, zero padded, so a plain + # lexicographic sort orders strongest adjacency first and falls back + # to the line text for ties. Keeping the ordering inside plain `sort` + # avoids depending on a field-separator flag; `cut` strips the prefix. + printf "%03d\t - %s ~ %s (tokens:%s)\n", 999 - depth[key], part[1], part[2], hit[key] + } + } + ' "$TMP_TOKENS" "$TMP_SRCLIST" | sort | cut -f2-) + fi + rm -f "$TMP_TOKENS" "$TMP_SRCLIST" + if [ -n "$OVERLAP_ALL" ]; then + OVERLAP_COUNT=$(printf '%s\n' "$OVERLAP_ALL" | grep -c '^ - ') + OVERLAP=$(printf '%s\n' "$OVERLAP_ALL" | head -n "$SURFACE_CAP") + if [ "${OVERLAP_COUNT:-0}" -gt "$SURFACE_CAP" ]; then + OVERLAP="${OVERLAP} + - ... and $((OVERLAP_COUNT - SURFACE_CAP)) more" + fi + PROMOTION_BODY="${PROMOTION_BODY}possible keyword overlap with Li+ source (${OVERLAP_COUNT} pairs): +${OVERLAP} " - done < <(find "$LIPLUS_DIR/rules" -type f -name '*.md' 2>/dev/null; find "$LIPLUS_DIR/skills" -maxdepth 2 -type f -name 'SKILL.md' 2>/dev/null) - done < "$TMP_HEADERS" - done - rm -f "$TMP_HEADERS" "$TMP_TOKENS" - [ -n "$OVERLAP" ] && PROMOTION_BODY="${PROMOTION_BODY}possible keyword overlap with Li+ source: -${OVERLAP}" + fi fi register_section "promotion_candidates" "Promotion candidates (memory → Li+ source)" "$PROMOTION_BODY" diff --git a/docs/2.-Evolution.md b/docs/2.-Evolution.md index a3d88c2..ac007b5 100644 --- a/docs/2.-Evolution.md +++ b/docs/2.-Evolution.md @@ -280,7 +280,7 @@ state file は `{workspace_root}/.claude/state/last-cold-start-emit.json`(sha2 (→ `rules/evolution/memory-entry-format.md`) -memory file 群(`feedback.md` / `project.md` / `MEMORY.md` / `promotion_tally.md` / `self-evaluation_log.md` 等)の entry 書式とメンテナンス規律を、各 file 内ローカル運用メモから L2 Evolution Layer の正規ルールに昇格する。横断規律として single source で扱い、各 memory file 冒頭の運用メモはこのルールへの参照に置き換える。 +memory file 群(per-topic entry file の `feedback_.md` / `project_.md` / `reference_.md` / `user_.md` = 1 memory 1 file、および index と運用ファイルの `MEMORY.md` / `promotion_tally.md` / `self-evaluation_log.md` / `self-evolution-observation.md`)の entry 書式とメンテナンス規律を、各 file 内ローカル運用メモから L2 Evolution Layer の正規ルールに昇格する。横断規律として single source で扱い、各 memory file 冒頭の運用メモはこのルールへの参照に置き換える。 **スコープ:memory は transient のみ。** memory が扱うのは cluster tally・self-evaluation log・reference に限る。永続情報は memory に置かない。 diff --git a/rules/evolution/memory-entry-format.md b/rules/evolution/memory-entry-format.md index 2171fd4..9bbf57a 100644 --- a/rules/evolution/memory-entry-format.md +++ b/rules/evolution/memory-entry-format.md @@ -13,7 +13,7 @@ layer: L2-evolution ## Position Layer = L2 Evolution Layer -Entry format and maintenance discipline for the memory file set (`feedback.md` / `project.md` / `MEMORY.md` / `promotion_tally.md` / `self-evaluation_log.md` etc.). +Entry format and maintenance discipline for the memory file set: the per-topic entry files (`feedback_.md` / `project_.md` / `reference_.md` / `user_.md` — one memory per file) plus the index and the operational files (`MEMORY.md` / `promotion_tally.md` / `self-evaluation_log.md` / `self-evolution-observation.md`). Requires = L2 Evolution Layer (persistence-tiering / promotion-judgment surroundings) Load timing = always-on (memory writes occur across the entire session) Single source. Replace the operational note at the head of each memory file with a reference to this rule (avoid double-holding drift). diff --git a/rules/evolution/promotion-judgment.md b/rules/evolution/promotion-judgment.md index 822cbb9..8e42c96 100644 --- a/rules/evolution/promotion-judgment.md +++ b/rules/evolution/promotion-judgment.md @@ -83,7 +83,7 @@ No past-occurrence carryover. Expired clusters are deleted in full. The AI holds no exception criteria internally. Future-reoccurrence prediction at observation time invites over-judgment (retaining "this is important" from one observation), so it is prohibited. Exception retention is permitted only when human explicitly overrides. -Override storage = a memory area outside the tally (e.g. an override section in `memory/feedback.md`). Do not write into the tally. +Override storage = a memory file outside the tally (e.g. a `memory/feedback_.md` entry). Do not write into the tally. diff --git a/skills/model-agentic-search/SKILL.md b/skills/model-agentic-search/SKILL.md index dd3b1d9..cfe1f2e 100644 --- a/skills/model-agentic-search/SKILL.md +++ b/skills/model-agentic-search/SKILL.md @@ -370,7 +370,7 @@ Naive single-shot RAG consumption fails on corpus boundary, recognition bias, an ## Observation and evolution Single environment cannot benchmark this skill against alternatives. Observation loop instead: -- log failure cases (hit State C, escalation chosen, outcome) to `memory/feedback.md` or self-evaluation log when notable +- log failure cases (hit State C, escalation chosen, outcome) to a `memory/feedback_.md` entry or the self-evaluation log when notable - side-by-side compare with naive single-shot consumption when retrospectively visible - feed observations into evolution loop observe stage (`skills/evolution-loop/SKILL.md`) diff --git a/skills/task-subagent-prompt/SKILL.md b/skills/task-subagent-prompt/SKILL.md index 88538e7..9b26f2c 100644 --- a/skills/task-subagent-prompt/SKILL.md +++ b/skills/task-subagent-prompt/SKILL.md @@ -70,7 +70,7 @@ Detection signs: # Memory-only knowledge does not transfer to subagent -Parent-side memory (workspace memory/feedback.md, memory/project.md, in-session corrections) is NOT auto-loaded into the subagent's context. The subagent only sees the issue body, the auto-loaded Li+ rules and skills, and the delegation prompt itself. +Parent-side memory (the per-topic entry files `memory/feedback_.md`, `memory/project_.md` and their siblings, plus in-session corrections) is NOT auto-loaded into the subagent's context. The subagent only sees the issue body, the auto-loaded Li+ rules and skills, and the delegation prompt itself. If subagent behavior depends on memory content, the parent MUST inject the relevant literal into the delegation prompt. "Memory has it, so subagent will pick it up" has failed multiple times in past sessions; pattern-match this assumption and reject it at delegation-construction time. diff --git a/tests/test_on_session_start_observation_surface.py b/tests/test_on_session_start_observation_surface.py index e47a0fd..b8be2c9 100644 --- a/tests/test_on_session_start_observation_surface.py +++ b/tests/test_on_session_start_observation_surface.py @@ -153,6 +153,106 @@ def no_new_material_marker(hook_output: str) -> str | None: return None +def promotion_section(hook_output: str) -> str | None: + """Body of the promotion-candidates section, or None when it was empty. + + Located by topic for the same reason `observation_section` is: the banner + wording is an adapter choice. An empty body is never emitted at all, so + None is also how "every detector stayed silent" reads. + """ + for banner, body in emitted_sections(hook_output): + if "promotion" in banner.lower(): + return body + return None + + +class PromotionSurface(NamedTuple): + """The judgment reported by the three promotion-candidate detectors. + + Presentation is delegated to the adapter by `rules/evolution/evolution.md` + ("Threshold values and concrete detection logic belong to the adapter"), so + this reads out what was *judged* — which axis crossed the repeat threshold + with what tally, which memory entries counted as recent, which entry/source + pairs are adjacent over which tokens, and the totals each list declares — + and not the bullet shape, the header wording or the order of the lines. + """ + + axis_misses: dict[str, int] + recent_total: int | None + recent_listed: frozenset[str] + overlap_total: int | None + overlap_listed: frozenset[tuple[str, str, frozenset[str]]] + truncated: dict[str, int] + + +_AXIS_MISS_RE = re.compile(r'axis\s+"([^"]+)".*?(\d+)\s*$') +_TOTAL_RE = re.compile(r"(\d+)\s+(entries|pairs)") +_MORE_RE = re.compile(r"\.\.\.\s*and\s+(\d+)\s+more") +_ENTRY_RE = re.compile(r"(\S+\.md)\s*\[") +_TOKENS_RE = re.compile(r"\(tokens:([^)]*)\)") + + +def promotion_surface(section_body: str | None) -> PromotionSurface: + """Parse a promotion section into its judgments, layout-agnostically.""" + axis_misses: dict[str, int] = {} + recent_total: int | None = None + recent_listed: set[str] = set() + overlap_total: int | None = None + overlap_listed: set[tuple[str, str, frozenset[str]]] = set() + truncated: dict[str, int] = {} + bucket = "" + for line in (section_body or "").split("\n"): + lowered = line.lower() + if not line.startswith(" ") and lowered.strip(): + # A detector's own header line: it names the detector and, for the + # two list-shaped ones, declares the full count. + total = _TOTAL_RE.search(line) + if "axis" in lowered: + bucket = "axis" + elif "recent memory" in lowered: + bucket = "recent" + recent_total = int(total.group(1)) if total else None + elif "overlap" in lowered: + bucket = "overlap" + overlap_total = int(total.group(1)) if total else None + else: + bucket = "" + continue + more = _MORE_RE.search(line) + if more: + truncated[bucket] = int(more.group(1)) + continue + if bucket == "axis": + match = _AXIS_MISS_RE.search(line) + if match: + axis_misses[match.group(1)] = int(match.group(2)) + elif bucket == "recent": + match = _ENTRY_RE.search(line) + if match: + recent_listed.add(match.group(1)) + elif bucket == "overlap": + entry, _, source = line.partition("~") + entry_match = _ENTRY_RE.search(entry) + tokens = _TOKENS_RE.search(source) + source_path = source.split("(tokens:")[0].strip() + if entry_match and source_path: + overlap_listed.add( + ( + entry_match.group(1), + source_path, + frozenset(tokens.group(1).split()) if tokens else frozenset(), + ) + ) + return PromotionSurface( + axis_misses=axis_misses, + recent_total=recent_total, + recent_listed=frozenset(recent_listed), + overlap_total=overlap_total, + overlap_listed=frozenset(overlap_listed), + truncated=truncated, + ) + + class SurfacedEntry(NamedTuple): """The judgment reported for one observation entry.""" @@ -895,5 +995,196 @@ def test_non_startup_matcher_reanchors_only_and_leaves_state_untouched(self) -> ) +class PromotionCandidateDetectorTest(ObservationSurfaceTestCase): + """Coverage area 6: the three promotion-candidate detectors (#1632 F3 / #1636). + + `rules/evolution/evolution.md` "Pattern Detection Surfacing At Cold-start" + fixes the three detection targets — self-evaluation log repetition, recent + memory additions, keyword overlap with Li+ source — and requires them to be + surfaced as observable material rather than left to passive noticing. It + delegates the threshold values and the concrete logic to the adapter, so the + assertions below read the judgment out of the emission and leave the + presentation alone (`promotion_surface`). + + Every case runs all three ports against one fixture. That is the shape the + defect needed: #1635 repaired the claude port while both codex ports kept + reading the flat `feedback.md` / `project.md` pair and the invented + `root_cause:` line syntax, and a suite that exercised one port could not see + it. The live host layout is one memory per file, so a fixture written in + that layout is silent on every unrepaired port. + """ + + SOURCE_TOKEN_TEXT = "widget calibration harness notes\n" + + def seed_source(self, workspace: Workspace) -> None: + """Two Li+ source files for the overlap detector to match against.""" + workspace.write( + workspace.liplus / "rules" / "evolution", + "widgets.md", + f"# widgets\n\n{self.SOURCE_TOKEN_TEXT}", + ) + workspace.write( + workspace.liplus / "skills" / "sample-skill", + "SKILL.md", + "# sample\n\nwidget calibration notes\n", + ) + + def seed_entry(self, workspace: Workspace, filename: str, title: str) -> None: + """One per-topic memory entry, titled through its frontmatter `name:`.""" + workspace.write( + workspace.shared_memory, + filename, + f"---\nname: {title}\n---\n\nbody\n", + ) + + def seed_self_eval(self, workspace: Workspace, *entries: str) -> None: + workspace.write( + workspace.shared_memory, + "self-evaluation_log.md", + "# Self-Evaluation Log\n\n" + "\n\n".join(entries) + "\n", + ) + + def surfaces_for_all_adapters(self) -> dict[str, PromotionSurface]: + surfaces: dict[str, PromotionSurface] = {} + for adapter in ADAPTERS: + self.ws.clear_state() + surfaces[adapter] = promotion_surface( + promotion_section(self.run_hook(adapter)) + ) + return surfaces + + def assert_ports_agree(self, surfaces: dict[str, PromotionSurface]) -> None: + reference = surfaces["claude_sh"] + for adapter, surface in surfaces.items(): + with self.subTest(adapter=adapter): + self.assertEqual( + surface, + reference, + f"{adapter} disagrees with claude_sh on identical input", + ) + + def test_all_three_ports_read_the_live_memory_layout(self) -> None: + """All three detectors fire, on the same input, in the same way. + + The fixture carries no flat `feedback.md` / `project.md` and no + `root_cause:` line, which is what the live artifacts look like. A port + still reading either format reports nothing at all here. + """ + self.seed_source(self.ws) + self.seed_self_eval( + self.ws, + "## entry 1\n**Axis tags**: character-drift: miss / source-check: hit", + "## entry 2\n**Axis tags (10-axis)**:\n" + "- character-drift: **miss (primary)**\n- frame-check: hit", + ) + self.seed_entry( + self.ws, "feedback_widget_calibration.md", "widget calibration harness" + ) + self.seed_entry(self.ws, "project_alpha.md", "alpha unrelated topic") + + surfaces = self.surfaces_for_all_adapters() + self.assert_ports_agree(surfaces) + + surface = surfaces["claude_sh"] + self.assertEqual( + surface.axis_misses, + {"character-drift": 2}, + "the repeated axis is the one the self-eval spec names as a weakness " + "region; the inline and the bullet layout must both count", + ) + self.assertEqual(surface.recent_total, 2) + self.assertEqual( + surface.recent_listed, + frozenset({"feedback_widget_calibration.md", "project_alpha.md"}), + "one memory is one file, so every freshly written entry counts", + ) + self.assertEqual( + surface.overlap_listed, + frozenset( + { + ( + "feedback_widget_calibration.md", + "rules/evolution/widgets.md", + frozenset({"widget", "calibration", "harness"}), + ), + ( + "feedback_widget_calibration.md", + "skills/sample-skill/SKILL.md", + frozenset({"widget", "calibration"}), + ), + } + ), + "the adjacent entry must be named against the source path, and the " + "unrelated entry must not be reported", + ) + self.assertEqual(surface.overlap_total, 2) + self.assertEqual(surface.truncated, {}) + + def test_single_observation_stays_below_every_threshold(self) -> None: + """One occurrence is noise on all three axes, on all three ports.""" + self.seed_source(self.ws) + self.seed_self_eval( + self.ws, + "## entry 1\n**Axis tags**: character-drift: miss / source-check: hit", + "## entry 2\n**Axis tags**: source-check: hit / frame-check: hit", + ) + # `widget` alone reaches the source files; one shared token is + # coincidence at corpus scale, so the pair must not be reported. + self.seed_entry(self.ws, "feedback_widget_only.md", "widget") + + surfaces = self.surfaces_for_all_adapters() + self.assert_ports_agree(surfaces) + + surface = surfaces["claude_sh"] + self.assertEqual( + surface.axis_misses, {}, "an axis missed once is not a weakness region" + ) + self.assertIsNone( + surface.recent_total, "a single recent entry is below the threshold" + ) + self.assertEqual(surface.overlap_listed, frozenset()) + + def test_surface_cap_truncates_the_list_and_still_reports_the_total(self) -> None: + """The cap bounds the list; the declared total keeps it honest. + + A consolidate pass rewrites every entry at once, so hitting the cap is + normal operation rather than an edge case, and a truncated list that + dropped the count would hide how much it left out. + """ + for index in range(14): + self.seed_entry( + self.ws, f"reference_topic{index:02d}.md", f"topic {index:02d}" + ) + + surfaces = self.surfaces_for_all_adapters() + self.assert_ports_agree(surfaces) + + surface = surfaces["claude_sh"] + self.assertEqual(surface.recent_total, 14) + self.assertLess( + len(surface.recent_listed), + 14, + "a list this long is past the point of being scannable; the cap " + "exists so the orientation surface stays readable", + ) + self.assertEqual( + len(surface.recent_listed) + surface.truncated.get("recent", 0), + 14, + "the surfaced entries plus the omitted count must add up to the " + "declared total, or the omission is hiding something", + ) + + def test_empty_memory_directory_is_a_silent_skip(self) -> None: + """No sources, no section — on every port.""" + self.ws.write(self.ws.shared_memory, "self-evaluation_log.md", "# log\n") + for adapter in ADAPTERS: + with self.subTest(adapter=adapter): + self.ws.clear_state() + self.assertIsNone( + promotion_section(self.run_hook(adapter)), + f"{adapter} emitted a promotion section with nothing to report", + ) + + if __name__ == "__main__": unittest.main()