diff --git a/docs/paper/fig1-gate-contrasts.png b/docs/paper/fig1-gate-contrasts.png new file mode 100644 index 0000000..95dfcc3 Binary files /dev/null and b/docs/paper/fig1-gate-contrasts.png differ diff --git a/docs/paper/fig2-localization.png b/docs/paper/fig2-localization.png new file mode 100644 index 0000000..e7a55b9 Binary files /dev/null and b/docs/paper/fig2-localization.png differ diff --git a/docs/paper/fig3-dose.png b/docs/paper/fig3-dose.png new file mode 100644 index 0000000..b5b58e6 Binary files /dev/null and b/docs/paper/fig3-dose.png differ diff --git a/docs/paper/fig4-matrix.png b/docs/paper/fig4-matrix.png new file mode 100644 index 0000000..4be9e66 Binary files /dev/null and b/docs/paper/fig4-matrix.png differ diff --git a/docs/paper/fig5-m4-floors.png b/docs/paper/fig5-m4-floors.png new file mode 100644 index 0000000..7b60d3a Binary files /dev/null and b/docs/paper/fig5-m4-floors.png differ diff --git a/docs/paper/fig6-collateral-asymmetry.png b/docs/paper/fig6-collateral-asymmetry.png new file mode 100644 index 0000000..41c07b4 Binary files /dev/null and b/docs/paper/fig6-collateral-asymmetry.png differ diff --git a/docs/paper/figures.py b/docs/paper/figures.py new file mode 100644 index 0000000..61d50c6 --- /dev/null +++ b/docs/paper/figures.py @@ -0,0 +1,582 @@ +#!/usr/bin/env python3 +"""Render the paper's figures from mute-map's committed result JSONs. + +Run +--- + uv run --with matplotlib docs/paper/figures.py + +matplotlib is injected for that run only and is deliberately NOT added to +``pyproject.toml``: the project's dependency set is the one the measurements ran +under, and a write-up must not change it. + +Reads (read-only, never written) +-------------------------------- + results/m1-battery-qwen2.5-{0.5b,1.5b,3b}-instruct.json + results/m2-depth-qwen2.5-{0.5b,1.5b,3b}-instruct.json + results/m3-matrix-qwen2.5-{0.5b,1.5b,3b}-instruct.json + results/m4-strip-qwen2.5-{0.5b,1.5b,3b}-instruct.json + +Writes +------ + docs/paper/fig1-gate-contrasts.png the five pre-committed gate contrasts (M1, M2 x2, M3 x2), + recorded Newcombe point estimate and 95% interval, x3 subjects + docs/paper/fig2-localization.png M2 sliding-window sweep: naming survival per window start, + recorded Wilson 95% intervals, workspace band shaded + docs/paper/fig3-dose.png M2 dose grid: naming rate (recorded Wilson 95%) and mean + concept mass at the five frozen lambda values + docs/paper/fig4-matrix.png M3 12 x 12 prime x probe matrix, cell = recorded naming + survival rate, annotated with the recorded hits/n + docs/paper/fig5-m4-floors.png M4's three pre-registered floor reads against the frozen + 0.5 bar, recorded Wilson 95% intervals + docs/paper/fig6-collateral-asymmetry.png + M4 per-prime row survival vs per-probe column survival, + both recorded rates, one mark per concept + +What this script computes -- and does not +----------------------------------------- +It computes NOTHING beyond reading recorded values and, where a JSON records a +count pair rather than a rate, the single division ``hits / n``. Every point, every +interval endpoint and every axis value (lambda, layer, band, subject) is lifted +verbatim from the files above. + +It does NOT smooth, interpolate, fit, re-bin, pool across cells the repo did not +pool, or compute an interval of its own. No line is drawn between grid points on +the sweep or the dose figures: the values between two measured positions were never +measured, and connecting them would assert them. Error bars are the recorded +``wilson_95`` / ``newcombe_*`` endpoints re-expressed as distances from their own +recorded point estimate, which is what matplotlib's API takes -- the interval drawn +is the recorded interval. Ordering marks by rank (figure 6) orders recorded values; +it creates none. + +Every plotted number is printed to stdout with the file and JSON key it came from, +so the figures can be checked against the paper's tables without opening a PNG. + +Deterministic and headless: same inputs, same PNGs, on any re-run. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") # headless: no display, no interactive backend + +import matplotlib.pyplot as plt +from matplotlib.colors import LinearSegmentedColormap +from matplotlib.lines import Line2D + +REPO = Path(__file__).resolve().parents[2] +RESULTS = REPO / "results" +OUT = Path(__file__).resolve().parent + +SUBJECTS = [ + ("0.5B", "qwen2.5-0.5b-instruct"), + ("1.5B", "qwen2.5-1.5b-instruct"), + ("3B", "qwen2.5-3b-instruct"), +] + +# --- dataviz palette: categorical slots 1-3, light surface ------------------- +# Validated with the dataviz skill's validate_palette.js (light, surface #fcfcfb, +# --pairs all): lightness band PASS, chroma floor PASS, worst all-pairs CVD dE 9.2, +# worst normal-vision dE 24.0. Aqua sits below 3:1 on the light surface, so the +# relief rule applies -- every series is direct-labelled or legended, and the paper +# carries the full table view beside each figure. +SERIES = {"0.5B": "#2a78d6", "1.5B": "#eb6834", "3B": "#1baf7a"} + +SURFACE = "#fcfcfb" +INK = "#0b0b0b" +INK_2 = "#52514e" +MUTED = "#898781" +GRID = "#e1e0d9" +AXIS = "#c3c2b7" +BAND_FILL = "#f0efec" + +# Sequential blue ramp (dataviz reference palette, steps 100 -> 700), reversed so +# that a muted cell (survival 0) is darkest and a spared cell (survival 1) recedes +# toward the surface -- one hue, monotonic in lightness. +BLUE_RAMP = [ + "#cde2fb", "#b7d3f6", "#9ec5f4", "#86b6ef", "#6da7ec", "#5598e7", + "#3987e5", "#2a78d6", "#256abf", "#1c5cab", "#184f95", "#104281", "#0d366b", +] +SEQ = LinearSegmentedColormap.from_list("mute_blue_r", list(reversed(BLUE_RAMP))) + + +def load(stage: str, slug: str) -> dict: + with (RESULTS / f"{stage}-{slug}.json").open() as fh: + return json.load(fh) + + +def style(ax) -> None: + ax.set_facecolor(SURFACE) + for side in ("top", "right"): + ax.spines[side].set_visible(False) + for side in ("left", "bottom"): + ax.spines[side].set_color(AXIS) + ax.spines[side].set_linewidth(0.8) + ax.tick_params(colors=MUTED, labelsize=8, length=3, width=0.8) + for label in ax.get_xticklabels() + ax.get_yticklabels(): + label.set_color(INK_2) + + +def new_figure(*args, **kwargs): + fig, axes = plt.subplots(*args, **kwargs) + fig.patch.set_facecolor(SURFACE) + return fig, axes + + +def save(fig, name: str) -> None: + path = OUT / name + fig.savefig(path, dpi=200, facecolor=SURFACE, bbox_inches="tight") + plt.close(fig) + print(f"\n wrote {path.relative_to(REPO)}") + + +def band(label: str) -> str: + return f"[{label}]" + + +# --------------------------------------------------------------------------- +# Figure 1 -- the five pre-committed gate contrasts +# --------------------------------------------------------------------------- + +def figure_1() -> None: + print("\n=== FIGURE 1: pre-committed gate contrasts " + "(recorded Newcombe 95% intervals) ===") + + # (row label, stage, JSON key path into the result file) + ROWS = [ + ("M3 clause (2)\nwithin-category off-diag - diagonal", "m3-matrix", + ("specificity_contrast", "clause_2_within_category", + "newcombe_offdiagonal_minus_diagonal_naming")), + ("M3 clause (1)\npooled off-diagonal - diagonal", "m3-matrix", + ("specificity_contrast", "clause_1_pooled", + "newcombe_offdiagonal_minus_diagonal_naming")), + ("M2 middle - late", "m2-depth", + ("localization_contrast", "newcombe_primed_middle_minus_primed_late_naming")), + ("M2 early - late", "m2-depth", + ("localization_contrast", "newcombe_primed_early_minus_primed_late_naming")), + ("M1 control_late - primed_late", "m1-battery", + ("breadth_contrast", "newcombe_control_minus_primed_late_naming")), + ] + + fig, ax = new_figure(figsize=(7.6, 5.0)) + style(ax) + ax.set_axisbelow(True) + ax.xaxis.grid(True, color=GRID, linewidth=0.8) + ax.axvline(0.0, color=INK_2, linewidth=1.0) + + offsets = {"0.5B": +0.24, "1.5B": 0.0, "3B": -0.24} + for row, (label, stage, keys) in enumerate(ROWS): + for subject, slug in SUBJECTS: + data = load(stage, slug) + node = data + for key in keys: + node = node[key] + point, lo, hi = node + y = row + offsets[subject] + ax.errorbar( + point, y, xerr=[[point - lo], [hi - point]], + fmt="o", markersize=6.5, elinewidth=1.6, capsize=0, + color=SERIES[subject], markeredgecolor=SURFACE, markeredgewidth=1.6, + zorder=3, + ) + print(f" {stage:10s} {subject:5s} {label.replace(chr(10), ' '):52s} " + f"{point:+.4f} [{lo:+.4f}, {hi:+.4f}] " + f"<- {stage}-{slug}.json : {'.'.join(keys)}") + + ax.set_yticks(range(len(ROWS))) + ax.set_yticklabels([label for label, _, _ in ROWS], fontsize=8.5) + ax.set_ylim(-0.6, len(ROWS) - 0.4) + ax.set_xlim(-0.05, 1.05) + ax.set_xlabel("difference in naming-survival rate (Newcombe 95%)", + fontsize=9, color=INK_2) + ax.set_title( + "Every pre-committed gate contrast excludes zero, on all three subjects", + fontsize=11, color=INK, loc="left", pad=12, + ) + handles = [ + Line2D([], [], marker="o", linestyle="none", markersize=6.5, + color=SERIES[s], markeredgecolor=SURFACE, markeredgewidth=1.6, + label=f"{s}{' (off-gate)' if s == '0.5B' else ''}") + for s, _ in SUBJECTS + ] + ax.legend(handles=handles, frameon=False, fontsize=8.5, loc="lower left", + labelcolor=INK_2) + save(fig, "fig1-gate-contrasts.png") + + +# --------------------------------------------------------------------------- +# Figure 2 -- M2 sliding-window localization sweep +# --------------------------------------------------------------------------- + +def figure_2() -> None: + print("\n=== FIGURE 2: M2 window sweep " + "(naming survival per window start, recorded Wilson 95%) ===") + + fig, axes = new_figure(3, 1, figsize=(7.8, 7.6), sharey=True) + for ax, (subject, slug) in zip(axes, SUBJECTS): + data = load("m2-depth", slug) + layers = data["band"] + width = data["window_width"] + gated_n = data["tier_cells"]["primed_late"]["n"] + style(ax) + ax.set_axisbelow(True) + ax.yaxis.grid(True, color=GRID, linewidth=0.8) + ax.axvspan(min(layers) - 0.5, max(layers) + 0.5, color=BAND_FILL, + zorder=0, linewidth=0) + + print(f"\n -- {subject} ({slug}) band L{min(layers)}-L{max(layers)}, " + f"window width {width}, gated n = {gated_n}" + f" <- m2-depth-{slug}.json : band / window_width / " + f"tier_cells.primed_late.n") + for window in data["window_map"]: + cell = window["cell"] + rate, lo, hi = cell["rate"], cell["wilson_95"][0], cell["wilson_95"][1] + gate = window["is_gate_cell"] + ax.errorbar( + window["start"], rate, yerr=[[rate - lo], [hi - rate]], + fmt="D" if gate else "o", markersize=7.0 if gate else 6.0, + elinewidth=1.4, capsize=0, color=SERIES[subject], + markeredgecolor=SURFACE, markeredgewidth=1.5, zorder=3, + ) + flag = " * late-third gate cell" if gate else ( + " (no layer in band)" if window["outside_band"] else "") + print(f" {window['name']:16s} start L{window['start']:<3d} " + f"{cell['hits']:3d}/{cell['n']:<3d} rate {rate:.4f} " + f"[{lo:.4f}, {hi:.4f}]{flag}" + f" <- window_map[{window['name']}].cell") + + gate_x = next(w["start"] for w in data["window_map"] if w["is_gate_cell"]) + ax.annotate( + "late-third\ngate cell", xy=(gate_x, 0.0), xytext=(gate_x, 0.30), + fontsize=8, color=INK_2, ha="center", + arrowprops=dict(arrowstyle="-", color=AXIS, linewidth=0.8), + ) + ax.text(0.995, 0.93, f"{subject} window width {width}, n = {gated_n}", + transform=ax.transAxes, ha="right", va="top", fontsize=9, + color=SERIES[subject], fontweight="bold") + ax.text(min(layers) + 0.2, 0.06, "workspace band", fontsize=7.5, + color=MUTED, va="bottom") + ax.set_ylim(-0.06, 1.10) + ax.set_ylabel("naming survival", fontsize=9, color=INK_2) + + axes[-1].set_xlabel("window start layer (stride 2; marks are measured " + "positions only — no line is drawn between them)", + fontsize=9, color=INK_2) + axes[0].set_title( + "The switch is a late cliff on a floor, not a band-wide effect", + fontsize=11, color=INK, loc="left", pad=12, + ) + fig.tight_layout() + save(fig, "fig2-localization.png") + + +# --------------------------------------------------------------------------- +# Figure 3 -- M2 dose grid +# --------------------------------------------------------------------------- + +def figure_3() -> None: + print("\n=== FIGURE 3: M2 dose grid at the five frozen lambda values ===") + + fig, axes = new_figure(1, 2, figsize=(8.4, 3.9)) + for ax in axes: + style(ax) + ax.set_axisbelow(True) + ax.yaxis.grid(True, color=GRID, linewidth=0.8) + ax.set_xlim(-0.08, 1.08) + ax.set_ylim(-0.06, 1.10) + ax.set_xticks([0.0, 0.25, 0.5, 0.75, 1.0]) + ax.set_xlabel("$\\lambda$ (fraction of the direction removed)", + fontsize=9, color=INK_2) + + # Subjects are dodged horizontally at each lambda purely so overlapping marks + # stay visible (three subjects sit at exactly 1.0 naming at lambda = 0). The + # ticks are the five frozen grid values; no mark is placed at an unmeasured + # lambda, and the dodge carries no information. + dodge = {"0.5B": -0.022, "1.5B": 0.0, "3B": +0.022} + for subject, slug in SUBJECTS: + data = load("m2-depth", slug) + print(f"\n -- {subject} ({slug}) <- m2-depth-{slug}.json : dose_curve") + for entry in data["dose_curve"]: + cell = entry["cell"] + lam = entry["lambda"] + x = lam + dodge[subject] + rate, lo, hi = cell["rate"], cell["wilson_95"][0], cell["wilson_95"][1] + mass = entry["mean_concept_mass_eligible"] + axes[0].errorbar( + x, rate, yerr=[[rate - lo], [hi - rate]], fmt="o", markersize=6.0, + elinewidth=1.4, capsize=0, color=SERIES[subject], + markeredgecolor=SURFACE, markeredgewidth=1.5, zorder=3, + ) + axes[1].plot(x, mass, "o", markersize=6.0, color=SERIES[subject], + markeredgecolor=SURFACE, markeredgewidth=1.5, zorder=3) + reused = f" (reused: {entry['reused_from']})" if entry["reused_from"] else "" + print(f" lambda {lam:<5.2f} naming {cell['hits']:3d}/{cell['n']:<3d} " + f"rate {rate:.4f} [{lo:.4f}, {hi:.4f}] mean concept mass " + f"{mass:.4f} (mass n = {entry['mass_channel_n']}){reused}") + + axes[0].set_ylabel("naming survival (Wilson 95%)", fontsize=9, color=INK_2) + axes[0].set_title("binary readout", fontsize=9.5, color=INK, loc="left") + axes[1].set_ylabel("mean concept softmax mass", fontsize=9, color=INK_2) + axes[1].set_title("graded readout", fontsize=9.5, color=INK, loc="left") + + handles = [ + Line2D([], [], marker="o", linestyle="none", markersize=6.0, + color=SERIES[s], markeredgecolor=SURFACE, markeredgewidth=1.5, + label=f"{s}{' (off-gate)' if s == '0.5B' else ''}") + for s, _ in SUBJECTS + ] + axes[1].legend(handles=handles, frameon=False, fontsize=8.5, loc="upper right", + labelcolor=INK_2) + fig.suptitle("A dimmer, not a step — and the knee moves right with scale", + fontsize=11, color=INK, x=0.005, ha="left", y=1.02) + fig.tight_layout() + save(fig, "fig3-dose.png") + + +# --------------------------------------------------------------------------- +# Figure 4 -- M3 prime x probe matrix (the killer figure) +# --------------------------------------------------------------------------- + +def figure_4() -> None: + print("\n=== FIGURE 4: M3 12 x 12 prime x probe matrix " + "(cell = recorded naming survival rate) ===") + + fig, axes = new_figure(1, 3, figsize=(13.4, 5.0)) + for ax, (subject, slug) in zip(axes, SUBJECTS): + data = load("m3-matrix", slug) + cells = data["matrix"] + primes, probes = [], [] + for cell in cells: + if cell["prime"] not in primes: + primes.append(cell["prime"]) + if cell["probe"] not in probes: + probes.append(cell["probe"]) + lookup = {(c["prime"], c["probe"]): c for c in cells} + + grid = [[lookup[(a, b)]["cell"]["rate"] for b in probes] for a in primes] + ax.imshow(grid, cmap=SEQ, vmin=0.0, vmax=1.0, aspect="equal", + interpolation="nearest") + + print(f"\n -- {subject} ({slug}) <- m3-matrix-{slug}.json : matrix") + for i, prime in enumerate(primes): + row_text = [] + for j, probe in enumerate(probes): + cell = lookup[(prime, probe)]["cell"] + hits, n, rate = cell["hits"], cell["n"], cell["rate"] + label = f"{hits}/{n}" if n else "—" + row_text.append(f"{probe}:{label}") + colour = SURFACE if (rate is not None and rate < 0.45) else INK + weight = "bold" if lookup[(prime, probe)]["is_diagonal"] else "normal" + ax.text(j, i, label, ha="center", va="center", fontsize=6.4, + color=colour, fontweight=weight) + print(f" A = {prime:9s} | " + " ".join(row_text)) + + ax.set_xticks(range(len(probes))) + ax.set_xticklabels(probes, rotation=60, ha="right", fontsize=7.2) + ax.set_yticks(range(len(primes))) + ax.set_yticklabels(primes, fontsize=7.2) + ax.tick_params(colors=MUTED, length=0) + for label in ax.get_xticklabels() + ax.get_yticklabels(): + label.set_color(INK_2) + for side in ax.spines.values(): + side.set_visible(False) + gated = data["pooled_arms"]["diagonal"]["n"] + ax.set_title(f"{subject}{' (off-gate)' if subject == '0.5B' else ''}" + f" gated n = {gated}", + fontsize=9.5, color=SERIES[subject], loc="left", pad=8) + if ax is axes[0]: + ax.set_ylabel("deleted direction A", fontsize=9, color=INK_2) + ax.set_xlabel("probed concept B", fontsize=9, color=INK_2) + + mappable = plt.cm.ScalarMappable(cmap=SEQ) + mappable.set_clim(0.0, 1.0) + bar = fig.colorbar(mappable, ax=axes, fraction=0.016, pad=0.015) + bar.set_label("naming survival rate (recorded)", fontsize=8.5, color=INK_2) + bar.ax.tick_params(colors=MUTED, labelsize=7.5) + bar.outline.set_visible(False) + + fig.suptitle("A dark diagonal on a near-white grid: deleting A silences A " + "and spares B", fontsize=11, color=INK, x=0.005, ha="left", y=1.0) + save(fig, "fig4-matrix.png") + + +# --------------------------------------------------------------------------- +# Figure 5 -- M4's three floor reads against the frozen 0.5 bar +# --------------------------------------------------------------------------- + +def figure_5() -> None: + print("\n=== FIGURE 5: M4 floor reads vs the pre-registered 0.5 bar " + "(recorded Wilson 95%) ===") + + fig, ax = new_figure(figsize=(7.6, 4.2)) + style(ax) + ax.set_axisbelow(True) + ax.yaxis.grid(True, color=GRID, linewidth=0.8) + + READS = ["item-level\n(the gate)", "residual-conservative", "concept-level"] + positions = {"0.5B": -0.26, "1.5B": 0.0, "3B": +0.26} + + for subject, slug in SUBJECTS: + data = load("m4-strip", slug) + sparing = data["vocabulary_sparing"] + bar_value = sparing["bar"] + reads = [("gate_arm", sparing["gate_arm"])] + for entry in sparing["conservative_reads"]: + reads.append((entry["read"], entry)) + print(f"\n -- {subject} ({slug}) <- m4-strip-{slug}.json : " + f"vocabulary_sparing") + for idx, (name, cell) in enumerate(reads): + rate = cell["rate"] + lo, hi = cell["wilson_95"] + x = idx + positions[subject] + ax.errorbar( + x, rate, yerr=[[rate - lo], [hi - rate]], fmt="o", markersize=6.5, + elinewidth=1.6, capsize=0, color=SERIES[subject], + markeredgecolor=SURFACE, markeredgewidth=1.6, zorder=3, + ) + clears = "clears" if lo >= bar_value else "BELOW THE BAR" + print(f" {name:24s} {cell['k']:3d}/{cell['n']:<3d} = {rate:.4f} " + f"[{lo:.4f}, {hi:.4f}] lower bound {clears} {bar_value}") + + bar_value = load("m4-strip", SUBJECTS[1][1])["vocabulary_sparing"]["bar"] + ax.axhline(bar_value, color=INK_2, linewidth=1.2, zorder=2) + ax.text(-0.5, bar_value + 0.02, f"pre-registered bar {bar_value}", + fontsize=8.5, color=INK_2, ha="left") + + ax.set_xticks(range(len(READS))) + ax.set_xticklabels(READS, fontsize=8.5) + ax.set_xlim(-0.55, len(READS) - 0.45) + ax.set_ylim(0.0, 1.0) + ax.set_ylabel("proportion surviving all 12 deletions\n(Wilson 95%)", + fontsize=9, color=INK_2) + ax.set_title("Why the verdict carries AS-SCORED ONLY: one read clears the " + "bar, another does not", fontsize=10.5, color=INK, loc="left", + pad=12) + handles = [ + Line2D([], [], marker="o", linestyle="none", markersize=6.5, + color=SERIES[s], markeredgecolor=SURFACE, markeredgewidth=1.6, + label=f"{s}{' (off-gate)' if s == '0.5B' else ''}") + for s, _ in SUBJECTS + ] + ax.legend(handles=handles, frameon=False, fontsize=8.5, loc="upper left", + labelcolor=INK_2) + fig.tight_layout() + save(fig, "fig5-m4-floors.png") + + +# --------------------------------------------------------------------------- +# Figure 6 -- M4 collateral asymmetry: safe primes, fragile probes +# --------------------------------------------------------------------------- + +def figure_6() -> None: + print("\n=== FIGURE 6: M4 collateral asymmetry " + "(per-prime row survival vs per-probe column survival) ===") + + gate_bearing = [s for s in SUBJECTS if s[0] != "0.5B"] + fig, axes = new_figure(2, 2, figsize=(9.2, 6.2), sharex=True, + gridspec_kw={"width_ratios": [1, 2.2]}) + + for (ax_primes, ax_probes), (subject, slug) in zip(axes, gate_bearing): + data = load("m4-strip", slug) + for ax in (ax_primes, ax_probes): + style(ax) + ax.set_axisbelow(True) + ax.xaxis.grid(True, color=GRID, linewidth=0.8) + + rows = [] + for prime, profile in data["row_profiles"].items(): + cell = profile["collateral_non_subset"] + rows.append((prime, cell["rate"], cell["hits"], cell["n"])) + rows.sort(key=lambda item: item[1]) + + columns = [] + for probe, profile in data["column_profiles"].items(): + cell = profile["fragility"] + if not cell["n"] or not profile["gated_items"]: + continue + columns.append((probe, cell["rate"], cell["hits"], cell["n"], + profile["in_subset"])) + columns.sort(key=lambda item: item[1]) + + print(f"\n -- {subject} ({slug}) deleted directions (rows), arm = the " + f"gated non-subset items" + f" <- m4-strip-{slug}.json : row_profiles[*].collateral_non_subset") + for prime, rate, hits, n in rows: + print(f" A = {prime:9s} {hits:3d}/{n:<3d} = {rate:.4f}") + print(f" -- {subject} probed concepts (columns), arm = the off-target " + f"deletions of that concept" + f" <- m4-strip-{slug}.json : column_profiles[*].fragility") + for probe, rate, hits, n, in_subset in columns: + tag = " (subset probe: 11 off-target deletions)" if in_subset else "" + print(f" B = {probe:11s} {hits:3d}/{n:<3d} = {rate:.4f}{tag}") + + n_rows, n_cols = len(rows), len(columns) + untouched = sum(1 for c in columns if c[2] == c[3]) + print(f" probes taking zero collateral: {untouched} of {n_cols}") + + ax_primes.plot([r[1] for r in rows], range(n_rows), "o", markersize=6.5, + color=SERIES[subject], markeredgecolor=SURFACE, + markeredgewidth=1.4, zorder=4) + ax_probes.plot([c[1] for c in columns], range(n_cols), "s", markersize=4.4, + color=SERIES[subject], markeredgecolor=SURFACE, + markeredgewidth=1.0, zorder=4) + + # Selective direct labels: the three most fragile probes only, fanned + # upward into empty space with hairline leaders so they cannot collide. + for rank, (probe, rate, hits, n, _) in enumerate(columns[:3]): + ax_probes.annotate( + f"{probe} {hits}/{n}", xy=(rate, rank), + xytext=(9, 6 + 15 * rank), textcoords="offset points", + fontsize=7.5, color=INK_2, va="center", + arrowprops=dict(arrowstyle="-", color=AXIS, linewidth=0.7, + shrinkA=0, shrinkB=2), + ) + worst_prime = rows[0] + ax_primes.annotate(f"{worst_prime[0]} {worst_prime[2]}/{worst_prime[3]}", + xy=(worst_prime[1], 0), xytext=(7, 0), + textcoords="offset points", fontsize=7.5, color=INK_2, + va="center") + + for ax, count, label in ((ax_primes, n_rows, "primes"), + (ax_probes, n_cols, "probes")): + ax.set_yticks([]) + ax.set_ylim(-1.5, count + 0.5) + ax_primes.set_ylabel(f"{subject}\n{n_rows} deleted directions", + fontsize=8.5, color=SERIES[subject], fontweight="bold") + ax_probes.set_ylabel(f"{n_cols} probed concepts", fontsize=8.5, color=INK_2) + ax_primes.set_xlim(0.35, 1.06) + if subject == gate_bearing[0][0]: + ax_primes.set_title("what deleting A spares", fontsize=9.5, color=INK, + loc="left", pad=8) + ax_probes.set_title("what B survives", fontsize=9.5, color=INK, + loc="left", pad=8) + ax_probes.text(0.36, n_cols - 1, + f"{untouched} of {n_cols} probes take zero collateral", + fontsize=8.5, color=INK_2, va="top") + + for ax in axes[-1]: + ax.set_xlabel("naming-survival rate (recorded)", fontsize=9, color=INK_2) + fig.suptitle("Collateral concentrates on fragile probes, not on damaging primes", + fontsize=11, color=INK, x=0.005, ha="left", y=1.0) + fig.tight_layout() + save(fig, "fig6-collateral-asymmetry.png") + + +def main() -> None: + print("mute-map — paper figures") + print("Every value below is read from a committed file in results/; the only " + "arithmetic\nis hits/n where a file records the pair rather than the rate.") + figure_1() + figure_2() + figure_3() + figure_4() + figure_5() + figure_6() + print("\nDone. 6 figures written to docs/paper/.") + + +if __name__ == "__main__": + main() diff --git a/docs/paper/mute-map-paper.md b/docs/paper/mute-map-paper.md new file mode 100644 index 0000000..2a7b9ed --- /dev/null +++ b/docs/paper/mute-map-paper.md @@ -0,0 +1,761 @@ +# Cartography of a late-band output off-switch in small language models + +**Breadth, localization, dose, specificity, and vocabulary collateral of a +single-direction concept mute in Qwen2.5-0.5B/1.5B/3B-Instruct** + +*mute-map, 2026-07-29. All measurements local, forward-only, $0.* + +> **A note on the figures.** All six are drawn by [`figures.py`](figures.py) from the +> committed artifacts in `results/`, and plot nothing but recorded values — counts, +> rates, and the intervals the runners themselves recorded. Nothing is smoothed, +> fitted or interpolated: on the sweep and dose figures no line joins the marks, +> because the values between them were never measured. The script prints every +> plotted number, so each figure can be checked against the tables without opening a +> PNG. Nothing was re-run or re-measured for this write-up. + +--- + +## Abstract + +During an independent rebuild of Anthropic's Jacobian-lens work +([transformer-circuits.pub/2026/workspace](https://transformer-circuits.pub/2026/workspace/index.html)), +the predecessor project *dim-stage* observed an effect that survived every control it +ran: removing a single concept's lens direction — a rank-one projection removal — at the +late third of that model's "workspace band" left the model unable to say that word, while +the same removal of a same-category control direction did not (+0.727 [+0.471, +0.868] at +1.5B). That rested on one gated cell of 22 items and one control. This paper characterizes +it. Over five pre-registered stages on Qwen2.5-0.5B/1.5B/3B-Instruct — an exact anchor +re-run, a 60-concept / 180-item battery, a window and dose sweep, a 12 × 12 prime × probe +matrix, and a 12-prime × 180-item collateral strip — the switch is broad +(+0.656 [+0.517, +0.763] at 1.5B; +0.636 [+0.443, +0.759] at 3B), localized to the late +third rather than the band (early − late +0.853 [+0.668, +0.936] at 1.5B), graded rather +than binary in dose, and specific across the matrix (+0.971 [+0.867, +0.983] at 1.5B). +The close-out stage asks whether deleting one concept spares the *other 48*: the +item-level floor clears the pre-registered 0.5 bar (51/71 = 0.718 [0.605, 0.810] at 1.5B; +63/84 = 0.750 [0.648, 0.830] at 3B), but the pre-committed concept-level floor's Wilson +lower bound does not (0.434 and 0.456), so both verdicts carry a pre-declared **AS-SCORED +ONLY** qualifier. We report every null and owned bound, plus two findings that re-scope — +rather than retract — earlier stages. The anchor throughout is our own recorded result, +never a published claim. + +--- + +## 1. Introduction + +Interpretability results are easy to state and hard to bound. An intervention that +produces a dramatic behavioural change — a model that suddenly cannot say "France" — +reads as a mechanism. Whether it *is* one depends on questions a single cell cannot +answer: does it work for other words, or only the ones you tried? Does it need that +exact location? Is it a switch or a dial? Does it damage only its target? This project +answers those four questions for one specific effect, and then a fifth the first four +leave open. + +**The honest contribution.** This is neither a reproduction of a published claim nor a +novel mechanism. It is the lineage's first **original characterization**: an effect +found during a replication, characterized here. The anchor is the predecessor project +*dim-stage*'s own recorded S4b result (`docs/S4-BRIEF.md` there), which this project +re-runs bit-for-bit before measuring anything new. The seed paper that motivated the +lineage — Anthropic's workspace / Jacobian-lens write-up at +[transformer-circuits.pub/2026/workspace](https://transformer-circuits.pub/2026/workspace/index.html), +which has no arXiv identifier and is cited by URL — supplies intellectual context and +nothing more. No result below is offered as reproducing it. + +**What is being deleted.** A *Jacobian lens* maps a model's internal activations onto +per-word directions. *Projection removal* at rank one (`k = 1`) subtracts out exactly +the component of the activation pointing along one such direction and leaves everything +else untouched. Applied at a contiguous set of layers it is a surgical, forward-only +edit: no weights change, no gradients, no fine-tuning. + +**The discipline.** Every stage froze its gate — verdict wording included — as +executable code before its first real run, and dry-ran it on deliberately wrong inputs +to confirm it exits INVALID. A cell under 20 trials is pre-declared UNDERPOWERED and +makes no claim; a pre-committed null is a reportable result. Each stage re-certifies its +predecessors' recorded cells bit-for-bit before reading a single new one, so a drifting +instrument cannot masquerade as a finding. + +--- + +## 2. Background and method + +### 2.1 The anchor + +At the late third of each model's workspace band, the predecessor project recorded naming +under the concept's own direction removal at 0/5, 0/22 and 0/8 gated items (0.5B / 1.5B / +3B), against a same-category control that left naming largely intact. Its specificity +readout was concept-SPECIFIC at 1.5B (+0.727 [+0.471, +0.868]), not shown and +UNDERPOWERED at 0.5B (+0.200 [−0.264, +0.624]), and concept-SPECIFIC but UNDERPOWERED at +3B (+1.000 [+0.541, +1.000]). The 0.5B and 3B cells were starved by a dual competence +gate requiring the model both to name the concept correctly and to *avoid* it correctly +on request. + +### 2.2 The instrument, and the design change + +mute-map inherits the fitted lens artifacts (never refit for the core chain — copied with +SHA256 provenance, decision K3), the projection-removal operator, and the anchor +protocol. The one standing design change is a **naming-only competence gate** (K2): +because the off-switch is a claim about naming, an item enters the measured set iff the +model names the concept correctly on the clean arm. Dropping the avoidance half is what +takes the gated cells from 5 / 22 / 8 to powered sizes at every scale. It is an owned +deviation, and anchor comparability is preserved by re-running the original dual-gate +protocol exactly (§4.1). + +### 2.3 The oracle + +The readout is deterministic by standing guardrail: never an LLM judge, never free-text +parsing. Through M0 and M1 the rule was the **greedy first token** — the model's single +highest-probability next token must be one of the concept's single-token spellings, +case-exact. M2 widened it once, by decision D9(b), frozen as code in `oracle.py` before +any M2 run: the concept is produced iff the decoded first-3-greedy span, after stripping +leading whitespace, **opens with the concept's spelling at a word boundary, +case-insensitive**. It is a prefix rule, never containment ("not France" is a miss); the +boundary test is "the next character is not a letter or digit" ("Marseille" is not Mars). +`oracle.py` is byte-shared rather than copied by four consumers, so the rule cannot drift +between them. The concept's softmax mass and the first-token outcome are recorded beside +every cell, so anchor comparability never degrades. + +### 2.4 The statistical rules + +- **Wilson 95% intervals** on every cell; **Newcombe 95% intervals** on every + between-arm difference. A difference whose interval includes zero is a null and is + stated as one. **A cell whose interval overlaps its neighbour is not a result.** +- **MIN_N = 20** per cell; below it the verdict is pre-declared UNDERPOWERED. +- **Degeneracy guards.** Every arm's most common wrong opening token is recorded; a + share at or above COLLAPSE_SHARE = 0.5 on a dispositive arm pre-declares the run + DEGENERATE. A guard on the intervened arm is a tag only, since a shared attractor + under the concept's own deletion is the expected signature of the switch. +- **Environment-scoped cross-checks.** Bit-for-bit reproduction is a property of the + certified stack (device `mps`, `torch==2.13.0`, `transformers==5.13.1`). On it a + mismatch exits INVALID; off it the run is pre-declared NOT A RESULT. +- **Precedence, frozen in each verdict function:** NOT A RESULT > DEGENERATE > + UNDERPOWERED > the contrast. + +--- + +## 3. Experimental setup + +Three subjects throughout: **Qwen2.5-0.5B-Instruct**, **Qwen2.5-1.5B-Instruct**, +**Qwen2.5-3B-Instruct**, run locally on Apple MPS, forward-only, at $0. Every gate is +the AND over the two gate-bearing subjects, 1.5B and 3B. **0.5B is never gate-bearing** +and is read only under a standing any-direction-damage frame, because the predecessor +project had already recorded non-specific damage at that scale. + +Lens artifacts were fitted by dim-stage's `fitter.py` at n_prompts = 100 on WikiText +prompts (0.5B and 1.5B on local MPS, 3B on a rented RTX 4090) and copied here with +recorded SHA256 fingerprints. Workspace bands, ported unchanged, are L9–L21, L11–L24 and +L14–L32; the band thirds follow the ported convention, which gives the late tier the +band's remainder (4/4/**5**, 4/4/**6**, 6/6/**7**). + +| Stage | Question | Design | Cells / subject | +|---|---|---|---| +| **M0** | Is the ported instrument the same instrument? | Exact re-run of the anchor protocol, dual competence gate retained | 840 | +| **M1** | How much of the measurable vocabulary has an off-switch? | 60 concepts × 10 categories × 3 clue items = 180 items; conditions clean / primed_late / control_late | 540 | +| **M2** | Where does the switch live, and how much removal does it take? | Pre-registered 12-concept subset (36 items); three tiers, a stride-2 sliding window sweep including outside-band positions, and a partial-ablation dose curve λ ∈ {0, .25, .5, .75, 1} | 720 / 756 / 864 | +| **M3** | Does deleting A damage B? | Full 12 × 12 prime × probe matrix at the switch's home band, plus 18 out-of-subset control-direction cells | 486 | +| **M4** | Does deleting one concept spare the *other 48*? | The 12 characterized directions as primes; **all 180 battery items** as probes | 2,340 | + +The 180-item battery was frozen in `items/m1-battery.json` before any M1 run: 10 +categories × 6 concepts × 3 clue sentences, 53 concepts drawn from vocabularies this +lineage had already measured and 7 new-list top-ups (marked in the frozen file) filling +gaps where a shipped list ran dry. 60 of the 180 items are the predecessor project's own +frozen items, reused verbatim so that every later run carries a live anchor check inside +it. **The item sets are constructed, not naturally occurring** — an owned deviation +carried from the first stage onward. + +--- + +## 4. Results + +Verdicts are quoted as the runners emitted them. Bold marks the gate-bearing subjects. + +![The five pre-committed gate contrasts, all three subjects. Each mark is the +recorded Newcombe point estimate and each bar the recorded Newcombe 95% interval; +none touches zero.](fig1-gate-contrasts.png) + +**Figure 1 — every pre-committed gate contrast excludes zero, on all three subjects.** +Point estimates and intervals are the `newcombe_*` triples recorded by `m1_battery.py`, +`m2_depth.py` and `m3_matrix.py`; nothing here is recomputed. Pooled gated n = 38 / 61 / +44 for M1 and 28 / 34 / 32 for M2 and M3 (0.5B / 1.5B / 3B); the arm sizes behind each +contrast are in §§4.2–4.4. Subjects are dodged vertically within each row for legibility. +M4's gate is a **level** bar rather than a contrast and so does not share this axis — it +appears in Figure 5. + +### 4.1 M0 — the instrument is the same instrument + +| Subject | Cells compared | Mismatches | Gated n | Anchor specificity readout reproduced | +|---|---|---|---|---| +| 0.5B | 840 | **0** | 5 | not shown, UNDERPOWERED (+0.200 [−0.264, +0.624]) | +| 1.5B | 840 | **0** | 22 | concept-SPECIFIC (+0.727 [+0.471, +0.868]) | +| 3B | 840 | **0** | 8 | concept-SPECIFIC, UNDERPOWERED (+1.000 [+0.541, +1.000]) | + +Beyond the gate's bar, the recorded `concept_mass` softmax floats reproduced **exactly, +840/840 cells on every subject** — the pinned environment preserved not just the greedy +argmax but the full computed distribution to the last bit. Every later stage embeds a +subset of previously recorded cells and grades them *first*: M2 and M3 each re-certified +108/108 M1 cells, and M4 re-certified **two** artifact sets at once — **255/255 M1 cells +and 468/468 M3 cells, `concept_mass` exact on all 723 comparisons, ×3 subjects** — +before reading a single new cell. + +### 4.2 M1 — breadth + +| Subject | Gated n / 180 | `primed_late` | `control_late` | control − primed [Newcombe 95%] | Verdict | +|---|---|---|---|---|---| +| 0.5B *(off-gate)* | 38 | 0/38 | 17/38 | +0.447 [+0.275, +0.603] | BREADTH-SPECIFIC | +| **1.5B** | 61 | 0/61 | 40/61 | **+0.656 [+0.517, +0.763]** | BREADTH-SPECIFIC | +| **3B** | 44 | 6/44 | 34/44 | **+0.636 [+0.443, +0.759]** | BREADTH-SPECIFIC | + +**M1 verdict: BREADTH-SPECIFIC at 1.5B AND 3B.** The effect is not idiosyncratic to the +handful of items it was first seen on. + +**Prevalence is UNDERPOWERED, exactly as pre-declared.** On the fixed-denominator +concept set, 4/8 (0.5B), 9/11 (1.5B) and 6/8 (3B) concepts show the full hard-switch +profile. All three carry the pre-declared UNDERPOWERED tag — the concept-set cell was +named in advance as the single sub-MIN_N cell in the stage — and none supports a claim. + +**0.5B came in BREADTH-SPECIFIC too, and that weakens a story we inherited.** The +anchor's 0.5B cell did not show specificity, on a gated n of 5. The naming-only gate +lifts 0.5B to n = 38, and the contrast is CI-clean. The honest reading, forecast in the +deviations table before the run: the anchor's 0.5B null was **underpowered, not +evidence of absence**, and the lineage's "specificity emerges by scale" narrative +weakens accordingly. M1 makes no scale claim; it reports that all three subjects show +the switch once each has the power to see it. + +#### 4.2.1 The owned bound: the readout, not the model, sets the coverage + +The competence gate admitted only 38 / 61 / 44 of 180 items, and the dominant reason is +a property of the readout rather than of the models. Under the first-token oracle a +concept is scorable only if its spelling at the answer position is a single token; +**26 of the 60 roster words have a multi-token bare form**, and the model usually spells +them without a leading space, so its first token is a fragment (`'Mer'`, `'Viol'`, +`'Fl'`) that the gate scores as a miss. Whole categories therefore gate near zero — +**planets and musical instruments gated 0 items on all three subjects** — so the +per-category map is in part a map of tokenizer geometry rather than of concepts. The +first-3-greedy texture measures the cost rather than assuming it: among *ungated* items +the model still said the concept within three tokens in **35/142 (0.5B), 54/119 (1.5B) +and 80/136 (3B)** — competence the primary readout cannot see. + +Two things bound this bound. It does not threaten the gate: the contrast is computed +*within* the gated set, and gating is a property of the clean arm alone, decided before +any ablation. And the same texture shows the bias **runs against the finding, not for it** +— on the gated cell the model said the concept within three tokens in `control_late` +17/17, 46/40 and 36/34, so at both gate-bearing subjects the primary readout *understates* +control-arm survival, while under `primed_late` it said the concept 0/38, 0/61 and 6/44, +exactly matching the primary count. The comparison arm is, if anything, scored too +harshly. + +#### 4.2.2 The re-score, published beside — not instead of — M1's numbers + +The instrument fix belonged in a decision, not a results section. M2 opened by widening +the oracle (D9(b)) and publishing a **labelled reanalysis** of M1's *same recorded cells* +under it (D10(a)). No model was run: the re-score is a pure function of the committed M1 +artifacts, and the script refuses to write anything unless it first reproduces M1's +published first-token contrast exactly — which it does, on all three subjects. **M1's +verdict of record is unchanged.** + +| Subject | Gated n (first-token → widened) | `primed_late` | `control_late` | control − primed [Newcombe 95%] | +|---|---|---|---|---| +| 0.5B *(off-gate)* | 38 → 69 | 0/69 | 33/69 | +0.478 [+0.353, +0.594] | +| **1.5B** | 61 → 105 | 0/105 | 80/105 | **+0.762 [+0.665, +0.833]** | +| **3B** | 44 → 116 | 12/116 | 92/116 | **+0.690 [+0.582, +0.767]** | + +The two dark categories light up — planets 0 → 7 / 8 / 15 and musical instruments +0 → 2 / 8 / 13 of 18 items each — and the contrast survives in the *harder* direction: +`control_late` gains far more items than `primed_late` does, and at 1.5B primed stays at +exactly 0 across all 105 gated items. Under the first-token readout one could argue the +mute was partly an artifact of fragment-scoring. It is not. + +A separate worry — that the contrast might be carried by the 60 items the predecessor +project itself selected — closes here under **both** oracles. On the **120 newly authored +items alone** the contrast is CI-clean on all three subjects: +0.278 / +0.545 / +0.478 +under the first-token oracle and +0.389 / +0.714 / +0.629 under the widened one. The +reused stratum runs higher, as expected for items chosen against a model that could +already name them, but the new stratum stands on its own. + +### 4.3 M2 — localization and dose + +| Subject | Gated n / 36 | `primed_early` | `primed_middle` | `primed_late` | early − late [Newcombe 95%] | middle − late [Newcombe 95%] | Verdict | +|---|---|---|---|---|---|---|---| +| 0.5B *(off-gate)* | 28 | 17/28 | 17/28 | 0/28 | +0.607 [+0.388, +0.764] | +0.607 [+0.388, +0.764] | LATE-LOCALIZED | +| **1.5B** | 34 | 29/34 | 27/34 | 0/34 | **+0.853 [+0.668, +0.936]** | **+0.794 [+0.603, +0.897]** | LATE-LOCALIZED | +| **3B** | 32 | 27/32 | 25/32 | 3/32 | **+0.750 [+0.531, +0.857]** | **+0.688 [+0.463, +0.812]** | LATE-LOCALIZED | + +**M2 verdict: LATE-LOCALIZED at 1.5B AND 3B.** The gated ns were *predicted before the +runs* — 28 / 34 / 32, because gating is a property of the deterministic clean arm M1 had +already recorded — and came in at 28 / 34 / 32. A disagreement would have been an INVALID +cross-check, not a power surprise. + +![Naming survival at each sliding-window position, three subjects. Marks are the +recorded rates with their recorded Wilson 95% intervals; the shaded region is each +subject's workspace band; the diamond is the reused late-third gate cell.](fig2-localization.png) + +**Figure 2 — the switch is a late cliff on a floor, not a band-wide effect.** +Each mark is one recorded window cell: the naming-survival rate with its recorded Wilson +95% interval, on the same gated items throughout (n = 28 / 34 / 32 at 0.5B / 1.5B / 3B), +so the positions are within-item correlated. Window width is 5 / 6 / 7 layers and the +stride is 2, both as recorded; the shading is the recorded workspace band (L9–L21, +L11–L24, L14–L32) and the diamond is the late-third cell the gate is computed on. **No +line joins the marks** — the positions between two window starts were never measured. +This sweep is descriptive and was never gate-bearing. + +| Subject | naming / gated n by window start | +|---|---| +| 0.5B (width 5, n = 28) | L0°15, L1°15, L3°14, L5 16, L7 16, L9 16, L11 19, L13 15, L15 13, **L17\* 0**, L18 0 | +| 1.5B (width 6, n = 34) | L0°33, L1°32, L3°27, L5°28, L7 28, L9 27, L11 25, L13 23, L15 23, L17 1, **L19\* 0**, L21 0 | +| 3B (width 7, n = 32) | L0°32, L2°32, L4°31, L6°30, L8 30, L10 29, L12 29, L14 25, L16 25, L18 25, L20 24, L22 16, L24 11, **L26\* 3**, L28 1 | + +*(° marks a window with no layer inside the band, \* the reused late-third gate cell.)* + +Removing the *same* direction at the *same* strength anywhere before the late third +leaves most naming intact; only the late window drives it to floor. The transition is +sharp at 0.5B and 1.5B and noticeably more gradual at 3B, which descends 24 → 16 → 11 → +3 over four positions — visible in Figure 2 as a staircase rather than a step. Stride 2 +localizes the 1.5B edge to between window starts L15 and L17. + +**Out-of-band ablation is cheap at the larger subjects and expensive at 0.5B**, quoted +as ranges rather than best cases: over windows with no layer in the band at all, naming +survives 27–33 of 34 at 1.5B (3–21% lost), 30–32 of 32 at 3B (0–6% lost) and 14–15 of 28 +at 0.5B (46–50% lost). Only 3B has a genuinely free out-of-band position; 1.5B's best +still costs one item and its worst loses 21% — the same depth-nonspecific damage 0.5B +shows, an order of magnitude smaller but not absent. **This is why 0.5B's LATE-LOCALIZED +reading sits on a raised floor**: its late cell is a genuine cliff (0/28 against a +~15/28 baseline), so the localization shape is real, but the "everywhere else is benign" +half of the story fails there — which is what the three panels' differing floor heights +in Figure 2 are. + +![Naming survival and mean concept mass at the five frozen λ values, three subjects. +Marks only; no curve is fitted through them.](fig3-dose.png) + +**Figure 3 — a dimmer, not a step; and the knee moves right with scale.** Left: the +recorded naming rate at each λ with its recorded Wilson 95% interval, on the same gated +items as the sweep (n = 28 / 34 / 32). Right: the recorded mean concept softmax mass, +scoped as recorded to the gated items whose bare spelling is single-token +(n = 22 / 24 / 21) — plotted as a bare point because no interval for it is on record. +λ = 0 is the reused `clean` cell and λ = 1 the reused `primed_late` cell. The λ axis is +the frozen five-value grid; **nothing is drawn between grid points**, and marks are +dodged horizontally only so three subjects sitting at 1.0 at λ = 0 stay visible. + +| λ | 0.5B naming (mass) | 1.5B naming (mass) | 3B naming (mass) | +|---|---|---|---| +| 0 | 28/28 (0.833) | 34/34 (0.913) | 32/32 (0.942) | +| 0.25 | 13/28 (0.362) | 20/34 (0.594) | 21/32 (0.782) | +| 0.5 | 0/28 (0.022) | 3/34 (0.115) | 10/32 (0.342) | +| 0.75 | 0/28 (0.001) | 1/34 (0.037) | 4/32 (0.197) | +| 1 | 0/28 (0.000) | 0/34 (0.017) | 3/32 (0.120) | + +Partial removal produces intermediate naming rates and intermediate probability mass at +every subject. **Nothing here behaves like a binary switch that flips at a threshold**, +which answers a question the kickoff brief left open. The knee is steep and appears to +move right with scale — the brief's half-mute points are λ ≈ 0.23 / 0.29 / 0.36 — but +those three figures are **linear interpolations between two grid points, not +measurements**: the grid is frozen at five values and nothing was re-fit. They are quoted +here and plotted nowhere, which is why Figure 3 shows five points and no curve. The mass +channel tells the same story without interpolation: at λ = 0.5 the retained mass is +0.022 / 0.115 / 0.342. The binary channel can only step and the mass channel moves +continuously; they fall together, so the dimmer reading does not rest on the binary +readout alone. + +**The pre-registered strata did their jobs**, including the one selected to fail. The +hard-switch core sat at 0/3 naming under `primed_late` on every subject; the +readout-unlocked stratum muted throughout; the leaky stratum leaked at 3B where +predicted. And the non-specific anti-example `silver` failed the pattern as designed: at +1.5B it gates 3/3 and reads `primed_late` 0/3 **and `control_late` 0/3** — the control +direction mutes it too — with `primed_early` 1/3 and `primed_middle` 0/3, i.e. damaged at +*every* depth. The pooled curves include it. + +### 4.4 M3 — the specificity matrix + +| Subject | Gated n / 36 | Diagonal | Off-diagonal | clause (1) off − diag [Newcombe 95%] | Within-category | Restricted diagonal | clause (2) [Newcombe 95%] | Verdict | +|---|---|---|---|---|---|---|---|---| +| 0.5B *(off-gate)* | 28 | 0/28 | 279/308 | +0.906 [+0.779, +0.934] | 80/96 | 0/24 | +0.833 [+0.670, +0.895] | MATRIX-SPECIFIC | +| **1.5B** | 34 | 0/34 | 363/374 | **+0.971 [+0.867, +0.983]** | 95/100 | 0/28 | **+0.950 [+0.814, +0.978]** | MATRIX-SPECIFIC | +| **3B** | 32 | 3/32 | 343/352 | **+0.881 [+0.731, +0.943]** | 97/101 | 2/29 | **+0.891 [+0.730, +0.947]** | MATRIX-SPECIFIC | + +**M3 verdict: MATRIX-SPECIFIC at 1.5B AND 3B**, on both pre-committed clauses. Clause (2) +restricts the contrast to *same-category* pairs — the arm the predecessor's single +control actually tested — so the pooled arm's cross-category-heavy composition did not +carry the verdict. **No subject carries the ON A DAMAGED FLOOR qualifier**: the +collateral floor reads 25/28 [0.728, 0.963], 33/34 [0.851, 0.995] and 31/32 [0.843, +0.994], all far above the pre-registered 0.5 floor — including at 0.5B, which the brief +had left open. Every pre-registered n came in exactly, every pooled cell clears MIN_N, and +no arm collapsed. **126 of the matrix's 132 ordered off-diagonal pairs had never been +measured before**; the other 6 are the earlier stage's own control cells. + +![The 12 × 12 prime × probe matrix at each of the three subjects. Rows are the deleted +direction, columns the probed concept; each cell is annotated with its recorded hits over +n, and shaded by the recorded naming-survival rate.](fig4-matrix.png) + +**Figure 4 — a dark diagonal on a near-white grid.** Every cell is one recorded matrix +cell: rows are the deleted direction A, columns the probed concept B, the annotation is +the recorded `hits/n`, and the shade is the recorded rate (dark = muted, light = spared). +Per-cell n is the number of B's gated items, **n ≤ 3 throughout, so no individual cell is +verdict-bearing** — the gate is computed on the pooled arms in the table above (gated +n = 28 / 34 / 32). Because the annotation gives every cell's counts, this figure is its +own table view. Summed over the grid, the pooled off-diagonal arm misses **11 of its 374 +item observations at 1.5B, 9 of 352 at 3B and 29 of 308 at 0.5B** (each grid square holds +n ≤ 3 of those observations). 0.5B's extra colour is the category block described in +§4.4.2; `silver`'s column is the visible fragile stripe at 1.5B and 3B. + +An effective-n sanity check — collapsing each item to "survives *all 11* off-diagonal +deletions" — agrees with the gate everywhere: 19/28, 29/34, 27/32 against diagonal 0/28, +0/34, 3/32, giving +0.679 [+0.458, +0.821], +0.853 [+0.668, +0.936] and +0.750 [+0.531, ++0.857]. There is no case where the honest per-item numbers would have had to be quoted +instead of the pooled ones. The graded channel agrees with the binary one: mean concept +mass reads clean 0.833 / 0.913 / 0.942, diagonal 0.0001 / 0.017 / 0.120, off-diagonal +0.773 / 0.889 / 0.937. + +#### 4.4.1 Re-attribution (a): non-specificity has a direction + +`silver` entered the subset as the pre-registered **non-specific anti-example** — the one +concept whose *control* direction had muted it in M1 — and the brief expected its row to +drag the pooled off-diagonal down. Its row does no such thing. **Deleting silver's +direction damages nothing, at any scale** (27/27, 31/31, 31/31) — its row in Figure 4 is +uniformly light. What is true is the transpose: silver's *column* is the most fragile in +the matrix (7/11, 27/33, 6/11 under other concepts' deletions). The misses pile onto a +few fragile probes rather than spreading from a few damaging primes — at 1.5B all 11 +off-diagonal misses land on `silver` (6), `Canada` (3), `piano` (1) and `violin` (1). + +This is a **re-scoping, not a retraction**. M1 and M2 each sampled *one cell of silver's +column* and read it as a property of silver's row; the single-control design could not +have distinguished the two. M1's and M2's published numbers stand; what changes is what +the label "non-specific" was ever a fact about. A row and a column are indistinguishable +in a single cell and nothing alike in a grid, which is why Figure 4 is what makes the +distinction visible. + +#### 4.4.2 The nulls M3 recorded + +**Category-block collateral is CI-clean at 0.5B and dissolves by 1.5B.** Within- versus +cross-category collateral gives a Newcombe difference of **+0.105 [+0.032, +0.196]** at +0.5B — CI-clean — but **+0.028 [−0.010, +0.091]** at 1.5B and **+0.020 [−0.016, +0.079]** +at 3B. Both straddle zero, and by this project's own rule a cell whose interval overlaps +its neighbour is not a result. So: at 0.5B, deleting a country's direction measurably +damages *other countries*; by 1.5B that block has dissolved into noise. §4.5.1 revisits +this, and the revisit is the reason this null is stated at exactly this strength. + +**The leak stratum replicated, on the diagonal, at 3B only.** The only diagonal cells +anywhere that are not zero are `Egypt` 2/3 and `October` 1/2 at 3B — exactly the two +concepts pre-registered as the leaky-switch stratum, and the only two dark-but-not-black +diagonal cells in Figure 4's right panel. The mute is not perfect for those two words at +the largest subject, and the pre-registration named them in advance. + +**Asymmetry is real but sparse**: 19 / 7 / 8 of the 66 unordered pairs differ at all +between A→B and B→A, and at both gate-bearing subjects the largest gaps are dominated by +`silver` on the probe side. + +### 4.5 M4 — the vocabulary collateral strip (the close-out stage) + +M3's near-white grid showed that deleting France spares the other *eleven* subset +concepts. It showed nothing about the other 48. M4 keeps the 12 characterized directions +as the **primes** and widens the **probes** to all 60 battery concepts: **2,340 cells per +subject**, every one at the identical late third, λ = 1, k = 1. The gate is a **level** +bar, not an ordering one — M3 settled the ordering — and it is single-clause: + +> **VOCAB-SPARING** iff, per subject: among the gated **non-subset** items, the +> proportion that **survives all 12** subset-direction deletions has its Wilson 95% +> lower bound at or above **0.5**. The bar is read **only when** the 468 M3-recorded and +> 255 M1-recorded cells reproduce their recorded outcomes bit-for-bit. + +| Readout | 0.5B *(off-gate)* | **1.5B** | **3B** | +|---|---|---|---| +| Gated items (full roster) | 69 | 105 | 116 | +| **Gate arm** (gated non-subset) | 41 | 71 | 84 | +| **Survives all 12** | 11/41 = 0.268 | **51/71 = 0.718** | **63/84 = 0.750** | +| Wilson 95% | [0.157, 0.419] | **[0.605, 0.810]** | **[0.648, 0.830]** | +| vs the pre-registered bar 0.5 | fails | **clears** | **clears** | +| Residual-conservative (fail in place) | 11/41 = 0.268 | 49/71 = 0.690, lower 0.575 | 62/84 = 0.738, lower 0.635 | +| **Concept-level collapse** | 4/23 = 0.174, lower 0.070 | **24/41 = 0.585, lower 0.434** | **26/43 = 0.605, lower 0.456** | +| Pre-registered ceiling | 35/41 ✓ | 69/71 ✓ | 82/84 ✓ | +| Verdict | `not shown` | **VOCAB-SPARING — AS-SCORED ONLY** | **VOCAB-SPARING — AS-SCORED ONLY** | + +![The three pre-registered floor reads at each subject, against the frozen 0.5 bar. +The item-level interval's lower bound sits above the bar at both gate-bearing subjects; +the concept-level interval's lower bound sits below it.](fig5-m4-floors.png) + +**Figure 5 — why the verdict carries AS-SCORED ONLY.** Each mark is one recorded floor +read with its recorded Wilson 95% interval, against the recorded bar of 0.5. The +denominators are the ones in the table above: item-level and residual-conservative are +out of 41 / 71 / 84 gate-arm items, concept-level out of 23 / 41 / 43 non-subset concepts +with at least one gated item. **The gate reads the lower bound, not the point estimate** — +so what matters visually is where each bar *ends*, not where its dot sits. At 1.5B and 3B +the item-level bar ends above the line and the concept-level bar ends below it. Subjects +are dodged horizontally within each read. + +**M4 verdict: VOCAB-SPARING at 1.5B AND 3B — AS-SCORED ONLY.** The scope is the result, +not a footnote to it. **The item-level floor clears the 0.5 bar; the concept-level +floor's Wilson lower bound does not (0.434 at 1.5B and 0.456 at 3B).** The concept-level +read — one binary per concept instead of per item — was pre-registered before any new +cell ran, and the brief pre-committed those as **the honest numbers to quote**. A +post-freeze amendment, ratified before the run, is what puts them *inside* the verdict +string rather than in prose: prose is not what gets quoted, the label is. The +residual-conservative read clears at both subjects, so exactly one of the two +pre-registered conservative reads fired. + +Every pre-registered n landed exactly: gate arms 41/71/84, concept counts 23/41/43, +ceilings 35/41, 69/71 and 82/84 with precisely the named misses (`july-1`, `april-1`, +`april-3`, `gold-1/2/3` at 0.5B; `july-3`, `venus-3` at 1.5B; `guitar-2`, `neptune-1` at +3B) — knowable in advance because the strip physically re-runs the 255 M1-recorded and +468 M3-recorded cells (**633 of the 2,340** once the 90-cell overlap is counted once) and +grades them first. No degeneracy fired on the dispositive arm: wrong-opening shares +0.022 / 0.011 / 0.011 against a 0.5 threshold. + +#### 4.5.1 Re-attribution (b): category-block collateral does not dissolve with scale + +M4 reverses M3's §4.4.2 null, and the reason is arm composition. + +| Subject | Within-category | Cross-category | +|---|---|---| +| 0.5B | 5/22 | 382/470 | +| **1.5B** | **22/29 = 0.759** | **769/823 = 0.934** | +| **3B** | **35/53 = 0.660** | **913/955 = 0.956** | + +**Category-block collateral is real in the wider vocabulary and does not dissolve with +scale.** M3 saw it dissolve by 1.5B — but M3's within-category arm was **30 of 34 pairs +countries**, a single tight block sampled twelve ways. The strip's within-category arm +samples ten categories. Again: a **re-scoping, not a retraction**. M3's published numbers +stand; what was measured there was a fact about that arm's composition. + +#### 4.5.2 Finding 1 generalizes out of sample + +![Per-prime and per-probe survival rates at the two gate-bearing subjects. The twelve +deleted directions cluster tightly near the top of the scale; the probed concepts spread +down to 0.5.](fig6-collateral-asymmetry.png) + +**Figure 6 — collateral concentrates on fragile probes, not on damaging primes.** Left +panels: each of the 12 deleted directions at its recorded survival rate over the gate arm +(`row_profiles[A].collateral_non_subset`, n = 71 at 1.5B and 84 at 3B). Right panels: +each gated probed concept at its recorded survival rate under the other concepts' +deletions (`column_profiles[B].fragility`; n is 12 deletions × that concept's gated items, +or 11 × for a subset concept, whose own direction is excluded — so denominators differ per +mark and every labelled mark carries its own). Marks are ordered by their own recorded +rate; the ordering creates no value. Only the three most fragile probes per subject are +labelled; the rest are in the recorded profiles and the script's printout. + +Collateral still concentrates on fragile **probes**, not damaging **primes**. **No prime +is a wrecking ball**: at 1.5B every row lands between 63/71 and 67/71, at 3B between +77/84 and 83/84 — the tight left-hand clusters. But specific probe columns collapse — +`copper` 6/12 and `mosquito` 8/12 at 1.5B; `eagle` 8/12, `platinum` 9/12 and `trumpet` +18/36 at 3B — while **32 of 53 gated columns at 1.5B and 33 of 55 at 3B take zero +collateral across all 12 deletions**. That bimodality — the dense stack at 1.0 with a thin +tail reaching 0.5 — is exactly what makes the item-level and concept-level statistics +diverge, because a concept with one fragile item fails the concept-level binary outright. +It is the visual answer to why Figure 5's two reads land on opposite sides of the bar. + +#### 4.5.3 The nulls and divergences M4 recorded + +**0.5B reads `not shown` off-gate at 11/41 = 0.268 [0.157, 0.419]** — the **first +measured divergence between the pre-registered subset's robustness and the wider +roster's**. 30 of 0.5B's 41 gate-arm items (73%) are damaged by at least one deletion, +against 28% at 1.5B and 25% at 3B. Read under the standing any-direction-damage frame, +never as a gate claim, and consistent in advance with M3's own 0.5B subset failing this +bar in-statistic (19/28, lower 0.4934). + +**The two statistics disagree by design, and 0.5B shows it starkly.** The M3-comparable +cluster-mean per-cell floor on the same 0.5B cells reads **32/41 → [0.633, 0.880]** — +comfortably above 0.5 — while the 12-fold conjunction on those same cells reads 0.268. +Same subject, same cells, opposite sides of the same constant. That is why the stage +refused to inherit M3's 0.5 and wrote the per-cell equivalence (0.5^(1/12) ≈ 0.944) into +its frozen wording. + +**The five pre-registered cross-mention cells did not carry the verdict, as predicted.** +At 3B all four gate-bearing cells named their concept; at 1.5B three named and +`China→jade-1` missed; `Egypt→beetle-2` remains ungated on all three subjects. + +**The residual set was larger in the ablated arm than in the clean arm.** The clean-arm +gate-arm residuals were the pre-computed 0 / 2 / 2, but the run recorded **0 / 27 / 21** +residual cells in total (0 / 26 / 21 in the gate arm), all on `beetle`, `butterfly` and +`trumpet` — the three concepts the frozen oracle docstring names. That is what the +pre-registered selector was written for, and why the conservative read moves the number +at all: 51 → 49 at 1.5B, 63 → 62 at 3B. + +**The claim is sparing across the *measurable* vocabulary, said exactly that way.** +**25 / 7 / 5 of the 48 non-subset concepts gate zero items.** That is a *competence +selection* — the model answers something else, or answers correctly behind a modifier the +opening-word rule refuses, or misses on morphology — and it plausibly enriches for robust +concepts, biasing the floor **upward**. The 0.5 bar itself is new, deliberately lenient +and uncalibrated: pre-registered before any new cell and fitted to none, with the per-cell +equivalence written into the frozen wording so it cannot be quoted as M3's floor. + +--- + +## 5. Discussion + +**What was measured.** A rank-one projection removal of one concept's lens direction, at +the late third of the workspace band, reliably prevents small Qwen2.5 models from saying +that word. The effect is broad over a 60-concept battery, localized to the late third +rather than the band, graded rather than binary in dose, specific across a full 12 × 12 +grid on both clauses — and it mostly spares the wider vocabulary, with the scope stated +on the label. + +**What the scope means.** The bar VOCAB-SPARING names permits real damage: at the +realized 1.5B rate, 20 of 71 measurable items are still damaged by at least one of the +twelve deletions. The concept-level read — "is *this concept* untouched?" rather than "is +*this item* untouched?" — sits below the bar at both gate-bearing subjects. Both readings +were pre-registered, neither was chosen after seeing the data, and the honest one-line +summary is the one this paper leads with. + +**What the nulls mean.** Three matter. (i) The dose curve's null against a step function +is a positive statement about mechanism: the ability to emit the word degrades +continuously with how much of the direction is removed. (ii) M3's category-block null was +a true null *for that arm*, and M4 shows why reading it as a general one would have been +wrong. (iii) M1's prevalence cells are UNDERPOWERED by pre-declaration and support no +claim about how many concepts have switches — only that the pooled contrast holds. + +**The two re-attributions are the most transferable result here.** Neither is a +discovery; both correct what an earlier measurement was a fact *about*. A single control +cell told us `silver` was non-specific; the matrix showed that was a fact about silver's +*column*, and that its row damages nothing at any scale. A countries-dominated +within-category arm told us category-block collateral dissolves with scale; the strip +showed that was a fact about *arm composition*. The lesson is structural rather than +mechanistic: **a single control cell measures a cell, not a row**, and an arm's +composition is part of what it measured. Both were findable only because every stage +re-ran its predecessor's recorded cells instead of trusting them. + +**The un-validatable residual.** Nothing here establishes *why* the late third is +special, whether the direction is the same object the seed paper's lens is about, or +whether any of this holds above 3B. The sweep's above-band coverage is structurally thin +— no window is ever fully above the band — so "the switch is late" is well-measured on +its early side and only weakly probed on its far side. Prime-side correlation structure +is measured but not modelled. The 7B scale extension and the lexical-versus-semantic +scope question ("can the model still say *French*, *Paris*?") were designed, declined for +this repository, and banked. + +--- + +## 6. Threats to validity + +Each stage's owned deviations table survives here in full. These are disclosures made +before or at the time of the runs, not concessions extracted afterward. + +### 6.1 Standing deviations (all stages) + +| Deviation | From | Owned reason | +|---|---|---| +| Model scale 0.5B–3B rather than a frontier model | the seed paper's setting | The lineage's standing frame; every claim is scoped to these three subjects | +| Anchor is our own recorded result, not a paper claim | lineage precedent | This is the lineage's first original characterization; framing stated in the kickoff brief and never softened | +| Lenses copied, never refit for the core chain | — | Decision K3; SHA256 provenance recorded; a hash mismatch after any refit means it is not the anchor instrument | +| **Naming-only competence gate** | the anchor's dual gate | Decision K2: the switch is a naming claim, and the avoidance half is what starved the anchor's cells. Measures the switch, not exclusion capacity; comparability preserved by M0's exact dual-gate re-run | +| **Constructed item sets**, frozen pre-run | naturally occurring text | 60 items reused verbatim from the predecessor's frozen set (and used as a live anchor check); 120 newly authored to a fixed recipe. The reuse stratum runs higher — reported separately (§4.2.2) | +| Item-level pooling in gate cells | an independence assumption | Items within a concept correlate; per-concept and per-category views reported beside, and effective-n collapses reported at M3 and M4 | +| Paired arms scored with an independent-samples Newcombe | the paired design | For positively correlated paired arms this **widens** the interval, so it can cost power but cannot manufacture a false positive | +| MIN_N applied to raw n | an effective n | Not discounted for the within-concept clustering the row above already owns | + +### 6.2 Stage-specific deviations + +| Stage | Deviation | Owned reason | +|---|---|---| +| M1 | 7 new-list concept words beyond the measured-only rule | The kickoff sanctions "extended by new frozen lists"; all 7 marked in the frozen file; the competence gate does the honest filtering | +| M1 | Word-prefix + explicit forbidden-form leak guard | With 60 short concepts a substring test makes "plant" a leak for "ant"; all 60 reused items pass the stricter guard unchanged | +| M2 | **Widened primary oracle** (span prefix, case-insensitive) | Fixes tokenizer geometry, not semantics; the first-token outcome is recorded beside every cell; M1's published numbers stand and the re-score is published beside them as a labelled reanalysis | +| M2 | Partial-projection operator is new code | Lives in the M2 runner, read-back generalized to survivor = (1−λ)·original within tolerance, unit-covered | +| M2 | Sliding windows have no precedent in the anchor protocol | The point of the stage; the three tier cells keep the anchor-comparable frame beside the new map | +| M2 | **The tier arms do not ablate the same number of layers** (4/4/5, 4/4/6, 6/6/7) | The ported band-thirds convention gives the late tier the remainder, so the localization gate compares a 4-layer ablation against a 6-layer one at 1.5B: what differs is depth **and** intervention size. Retired descriptively by the constant-width sweep — at 1.5B the width-6 window at L11–L16 scores 25/34 against the late window's 0/34, and at 3B the width-7 L12–L18 scores 29/32 against 3/32 — but the *gate* was computed on unequal-width arms, and that is stated rather than implied | +| M2 | Mass channel scoped to single-token bare spellings | The readout-unlocked stratum has no single-token bare form, so its mass is floor-pinned by construction; that stratum's dose curve is binary-only | +| M2 | Directions keyed to the leading-space unembed row for multi-token-bare concepts | No bare single token exists for 26 roster words; the space-keyed late ablation is measured to mute the bare emission all the same | +| M3 | Full prime × probe matrix has no precedent in the anchor protocol | The point of the stage; the diagonal and the 36 control cells keep the anchor-comparable frame inside the matrix | +| M3 | Pooled off-diagonal counts each gated item 11 times | Within-item correlation, owned; per-direction, per-pair and effective-n views reported beside | +| M3 | **The frozen degeneracy scope guards only one of the gate's two surviving arms** | Found at post-run adversarial review and owned rather than patched: a wrong-opening collapse confined to the within-category arm could in principle let clause (2) rest on a degenerate cell. At the two smaller subjects the unguarded arm runs 3–4× the guarded arm's share (0.052 vs 0.016 at 0.5B; 0.030 vs 0.008 at 1.5B); at 3B the ordering inverts (0.010 vs 0.014). **Nothing here is affected** — 0.052 is an order of magnitude below the 0.5 threshold and no arm collapsed on any subject — and the wording was **not** amended, because editing a pre-registration after seeing results is the exact move the frozen-wording rule exists to prevent | +| M4 | **The stage exists at all**, after the kickoff's frozen chain | A close-out stage picked after M3, to close M3's own stated bound before write-up; the kickoff's scope decisions were not relitigated | +| M4 | **A new, uncalibrated, sole-dispositive 0.5 constant** | M3's 0.5 was a per-cell floor and never dispositive; M4's is a bar on a 12-fold conjunction and is the single gate, so no provenance transfers. Owned as deliberately lenient, pre-registered before any new cell, fitted to none, with the per-cell equivalence (≈ 0.944) frozen into the gate wording itself so no write-up can quote it as M3's floor | +| M4 | A level-bar gate rather than the lineage's ordering gates | The ordering is M3's settled result; the strip's question is a level question | +| M4 | Five cross-mention (prime, item) pairs kept in gate-bearing pools | The confound biases *against* the gate; each is named and reported per cell (§4.5.3) | +| M4 | `oracle.py` byte-shared by a **fourth** consumer | The rule's entire purpose is byte-identity across consumers; pinned by the existing shared-oracle test pattern. The standing convention that each runner is *cut* from its predecessor is otherwise unbroken — no certified file is ever edited to serve a later stage | +| M4 | **The oracle's span-truncation residual now sits in a gate-bearing arm** | The 3-token span cannot observe the closing boundary for the three concepts whose bare form fills it (`beetle`, `butterfly`, `trumpet`), and M4 scores all 60 probes, so they are gated again for the first time since M1. The bias runs *toward* the gate, so it is disclosed per subject (0/2/2 clean-arm cells; 0/27/21 recorded overall) and carried by the pre-registered residual-conservative recomputation — never by editing the frozen oracle | +| M4 | **Probe-side reach is still the oracle-visible roster** | 25 / 7 / 5 of the 48 non-subset concepts gate zero items. This is a **competence selection**, and it plausibly enriches for robust concepts and biases the floor **upward**. The claim is sparing across the *measurable* vocabulary, and it is said exactly that way | +| M4 | The frozen gate wording promises per-pair-cell degeneracy texture the runner does not compute | Found at post-run adversarial review; the wording is byte-frozen with three subjects' artifacts and cannot be edited, and the readout is pre-declared non-verdict-bearing at n ≤ 3. Disclosed here rather than patched, and **that clause must not be quoted as if the field exists** | + +### 6.3 Bounds on what the numbers can carry + +- **The coverage bound is a readout bound, twice over.** M1's first-token oracle could + see 38 / 61 / 44 of 180 items, the widened oracle 69 / 105 / 116. M4's gate arm is what + the widened oracle can see; no number here speaks for the invisible remainder. +- **Every 0.5B reading is off-gate**, under a standing any-direction-damage frame. +- **Per-concept, per-window and per-pair cells are n ≤ 3**, never verdict-bearing. This + includes every cell of Figure 4. +- **Curves are within-item correlated**: the same 28 / 34 / 32 gated items appear in + every window and dose cell, so Figures 2 and 3 are not independent samples across + positions. +- **The 3B diagonal is not perfectly zero** (3/32) — the two responsible concepts were + pre-registered as the leaky stratum. + +--- + +## 7. Reproducibility + +Everything is local, forward-only, and free. **Whole-project compute cost: $0.** The +close-out stage's three subjects — 2,340 cells each — took roughly 50 minutes in total on +Apple MPS. + +**To re-run the measurements.** `uv` (Python 3.12+) manages the environment. `uv run +pytest` greens the suite — **396 tests**, recorded green locally and in CI in +`HANDOFF.md`; this write-up quotes that record rather than re-running it. Runners +live at the repository root and are invoked per subject, e.g. `uv run python -u +m4_strip.py --model-id Qwen/Qwen2.5-1.5B-Instruct --lens +lenses/qwen2.5-1.5b-instruct-n100.pt`. Every runner supports `--dry-run` and `--limit` +(smoke only, never a result), and every gate exits INVALID on wrong-arm input. + +**To re-render the figures.** `uv run --with matplotlib docs/paper/figures.py`. The +script is deterministic and headless: it reads only the committed JSONs in `results/`, +writes only the six PNGs beside it, and prints every plotted number with the file and +JSON key it came from. matplotlib is injected for that run alone and is deliberately not +a project dependency — the manifest is the one the measurements ran under. **The script +computes nothing beyond `hits / n` where a file records the count pair rather than the +rate**; every interval it draws is a recorded `wilson_95` or `newcombe_*` endpoint, and +it never smooths, interpolates, fits, re-bins, or pools across cells the runners did not +pool. + +**What is and is not in the repository.** Frozen item sets in `items/`; per-run JSONs in +`results/` (18 files: anchor, M1 battery, M1 re-score, M2 depth, M3 matrix and M4 strip, +×3 subjects); decisions D1–D22 in `docs/DECISIONS.md`; per-stage briefs in `docs/`. The +`.pt` lens artifacts are **gitignored** by decision K3 — sourced from the predecessor +project's local copies, with `lenses/PROVENANCE.md` recording each file's SHA256, its fit +provenance and the exact regeneration command. Models pull from HuggingFace on first use; +no API keys, no `.env`. + +**Environment is load-bearing.** Bit-for-bit reproduction depends on the certified stack +— device `mps`, `torch==2.13.0`, `transformers==5.13.1`; off it the run is pre-declared +NOT A RESULT. Re-certifying the anchor after touching the harness, the operator, the +subject loader or the pins requires regenerating the left-hand side first (`m0_anchor.py` +per subject) before `m0_port_gate.py --all`, which otherwise compares two committed files +and is tautological. + +**Three known, unfixed follow-ups**, recorded rather than repaired, none affecting any +number above: + +1. The close-out stage's frozen gate wording promises per-pair-cell degeneracy texture + the runner does not compute (its `tokenizer` parameter is unused). The wording is + byte-frozen with three subjects' artifacts; honouring it would change the JSONs and + cost a full re-run for a readout pre-declared non-verdict-bearing at n ≤ 3. +2. That runner's `main()` re-parses the battery outside the `try/except` that turns + battery drift into a clean INVALID exit, so those guards would raise a bare traceback + rather than exit 2. Unreachable in practice — the file cannot change between the two + calls in one process. +3. The CI job is still *named* `offline-suites`, but now that pytest genuinely runs it + fetches four Qwen2.5 tokenizer repositories on every push. **390 of the 396 tests pass + with no network at all; a red build there is network, not logic.** The two remedies + trade off against each other, so it is a workflow design call, not a correctness fix. + +A related disclosure: until 2026-07-29 the CI workflow executed each test file as a plain +script, and with no `__main__` guard those files imported, defined their functions and +exited 0 — so CI ran zero tests. It now runs `pytest` per file, green with all 396 cases +collected. **Any green CI badge dated before 2026-07-29 certifies syntax, not behaviour.** + +--- + +## 8. References + +1. Anthropic. *Workspace / Jacobian lens.* Transformer Circuits, 2026. + — **cited by URL; this + publication has no arXiv identifier, and the repository records no author list or + venue beyond the URL.** Intellectual context for the lineage only; no result here is + offered as reproducing any claim in it. +2. *dim-stage* — the predecessor project: an independent rebuild and small-scale + measurement of the Jacobian lens, in which the effect characterized here was first + observed (its S4b result, recorded in `docs/S4-BRIEF.md` there and in its committed + result JSONs). . **This is the anchor for every + comparison in this paper.** +3. *mute-map* — this project. Approved brief `docs/KICKOFF.md`; decisions D1–D22 in + `docs/DECISIONS.md`; per-stage briefs `docs/M0-BRIEF.md` … `docs/M4-BRIEF.md`; + recorded results in `results/`; figure script `docs/paper/figures.py`. + . +4. Models: Qwen2.5-0.5B-Instruct, Qwen2.5-1.5B-Instruct, Qwen2.5-3B-Instruct, as + published on HuggingFace under those identifiers. The repository records the model + identifiers and the pinned inference stack; it records no citation for the Qwen + technical report, and none is invented here. diff --git a/docs/paper/mute-map-presenter-pack.md b/docs/paper/mute-map-presenter-pack.md new file mode 100644 index 0000000..69d45c2 --- /dev/null +++ b/docs/paper/mute-map-presenter-pack.md @@ -0,0 +1,256 @@ +# mute-map — presenter pack + +*Companion to `mute-map-paper.md`. Everything here traces to a recorded file; the +provenance table is the map. The paper's six figures are drawn by +`docs/paper/figures.py` from those same files and plot nothing but recorded values — +if someone points at a figure, the number behind it is in the table below and in the +script's own printout.* + +--- + +## The 60-second story + +"In a previous project I rebuilt Anthropic's Jacobian lens at hobby scale, and one effect +survived every control I threw at it: if you take a single concept's lens direction and +subtract it out of the model's activations at the late third of its workspace band, the +model becomes unable to say that word. The evidence was one cell — twenty-two items, one +control comparison. mute-map is the characterization. I mapped it along five axes with +pre-registered gates frozen as code before every run: is it broad, where does it live, +is it a switch or a dial, does it damage the concepts near it, and does deleting one +concept spare the rest of the vocabulary. It's broad, it's genuinely *late* rather than +band-wide, it's a dimmer rather than a step, and it's clean across a full twelve-by-twelve +grid. The close-out result is scoped, and the scope is the point: at the item level the +sparing floor clears my pre-registered bar — 51 of 71 at 1.5B, 63 of 84 at 3B — but +collapsed to one binary per *concept*, the lower bound doesn't (0.434 and 0.456). Both +verdicts ship with an AS-SCORED ONLY tag that was pre-registered before the run, not +added afterward. Everything ran locally on my Mac, forward-only, for $0." + +--- + +## Results at a glance + +Bold = the two gate-bearing subjects. 0.5B is never gate-bearing. + +| Stage | Gate (frozen as code before the run) | 0.5B | **1.5B** | **3B** | Verdict | +|---|---|---|---|---|---| +| **M0** anchor | 0 mismatches vs the recorded predecessor JSONs | 0/840 | 0/840 | 0/840 | **PASSED** | +| **M1** breadth | pooled `control_late` − `primed_late` CI-clean | +0.447 [+0.275, +0.603] | **+0.656 [+0.517, +0.763]** | **+0.636 [+0.443, +0.759]** | **BREADTH-SPECIFIC** | +| **M2** localization | early − late and middle − late both CI-clean | +0.607 / +0.607 | **+0.853 / +0.794** | **+0.750 / +0.688** | **LATE-LOCALIZED** | +| **M3** specificity | off-diagonal − diagonal CI-clean, pooled AND within-category | +0.906 / +0.833 | **+0.971 / +0.950** | **+0.881 / +0.891** | **MATRIX-SPECIFIC** | +| **M4** vocab collateral | survives-all-12 Wilson lower ≥ 0.5 | 11/41 = 0.268 [0.157, 0.419] | **51/71 = 0.718 [0.605, 0.810]** | **63/84 = 0.750 [0.648, 0.830]** | **VOCAB-SPARING — AS-SCORED ONLY** | + +**The M4 scope, said the honest way:** *the item-level floor clears the 0.5 bar; the +concept-level floor's Wilson lower bound does not — 0.434 at 1.5B and 0.456 at 3B.* +Lead with that sentence. Never lead with a bare "VOCAB-SPARING." + +**Every null and owned bound, on one card:** + +- 0.5B `not shown` off-gate at M4 (0.268) — the first measured divergence between the + subset's robustness and the wider roster's; 73% of its gate-arm items damaged vs 28% + and 25%. +- M1's prevalence cells (4/8, 9/11, 6/8) are **pre-declared UNDERPOWERED** — no claim. +- M3's category-block collateral **straddles zero** at both gate-bearing subjects + (+0.028 [−0.010, +0.091] and +0.020 [−0.016, +0.079]) — a null, and M4 later shows why + reading it as general would have been wrong. +- The dose curve is a **dimmer, not a step** — the null against a threshold model. +- 25 / 7 / 5 of the 48 non-subset concepts **gate zero items** — a competence selection + that biases the floor **upward**. The claim is sparing across the *measurable* + vocabulary, said exactly that way. +- The oracle's **span-truncation residual** now sits in a gate-bearing arm: 0/2/2 + clean-arm cells, 0/27/21 recorded overall, carried by a pre-registered conservative + recomputation rather than by editing the frozen oracle. +- M1's **first-token coverage bound**: 26 of 60 roster words are multi-token bare, so + planets and instruments gated 0 items on every subject. The re-score under the widened + oracle is published *beside* M1's numbers, never instead of them. + +--- + +## The six figures — what each one is for + +If you get one minute at a whiteboard, it's Figure 4. If you get one slide, it's Figure 5. + +| Figure | The one sentence it exists to say | The trap it avoids | +|---|---|---| +| **1** gate contrasts | Every pre-committed gate cleared zero, on every subject, and you can see how wide each interval is. | It plots the *recorded* Newcombe triples — nothing is recomputed. M4 is absent because its gate is a level bar, not a contrast. | +| **2** window sweep | The switch is a late **cliff on a floor**, and 0.5B's floor is visibly raised. | No line joins the marks: positions between two window starts were never measured. | +| **3** dose grid | A **dimmer, not a step** — and the binary and mass channels fall together. | The half-mute λs (0.23/0.29/0.36) are interpolations and are **plotted nowhere**. Five frozen grid points, no curve. | +| **4** the 12 × 12 matrix | A dark diagonal on a near-white grid — and `silver`'s *column* is the fragile stripe, not its row. | Every cell is annotated with its own hits/n, and **every cell is n ≤ 3** — the gate lives in the pooled arms, never in a cell. | +| **5** M4 floor reads | Why the verdict says AS-SCORED ONLY: the item-level bar ends above 0.5, the concept-level bar ends below it. | The gate reads the *lower bound*, so point on the bar ends, not the dots. | +| **6** collateral asymmetry | Primes cluster tight and high; probes are bimodal with a tail to 0.5. That bimodality is *why* Figure 5's two reads disagree. | Denominators differ per mark (12 deletions × the concept's gated items; 11 × for a subset concept) — the labelled marks carry their own. | + +**If asked "did you draw these to flatter the result?"** — the script is committed, it is +deterministic and headless, it reads only the committed result JSONs, and it prints every +plotted number with the JSON key it came from. It computes exactly one thing: `hits / n`, +where a file records the pair instead of the rate. It never smooths, fits, interpolates, +re-bins, pools arms the runners didn't pool, or invents an error bar — every interval +drawn is a recorded `wilson_95` or `newcombe_*` endpoint. Run +`uv run --with matplotlib docs/paper/figures.py` and diff the printout against the +paper's tables. + +--- + +## Provenance table — claim → number → source + +Any of these can be pulled up live. JSON paths are the field to read. + +| Claim | Number | Source | +|---|---|---| +| Anchor reproduced bit-for-bit | 0 mismatches / 840 cells ×3; mass exact 840/840 | `docs/M0-BRIEF.md` results; `results/anchor-*.json` | +| Anchor specificity at 1.5B | +0.727 [+0.471, +0.868] | `results/anchor-qwen2.5-1.5b-instruct.json` → `late_switch_specificity` | +| M1 gate *(Fig 1)* | +0.656 [+0.517, +0.763] / +0.636 [+0.443, +0.759] | `results/m1-battery-*.json` → `breadth_contrast.newcombe_control_minus_primed_late_naming` | +| M1 gated n | 38 / 61 / 44 of 180 | same → `competence.gate_greedy` | +| M1 arms | `primed_late` 0/38, 0/61, 6/44; `control_late` 17/38, 40/61, 34/44 | same → `naming_success_gated` | +| M1 prevalence, UNDERPOWERED | 4/8, 9/11, 6/8 | same → `prevalence.cell` | +| Bias runs against the finding | control said concept in 3 tokens 17/17, 46/40, 36/34 | same → `greedy_3_texture` vs `naming_success_gated` | +| Re-score (labelled reanalysis) | gated 38→69, 61→105, 44→116; +0.478 / +0.762 / +0.690 | `results/m1-rescore-*.json` → `oracles.prefix` | +| Re-score reproduces M1 exactly | `true` ×3 | same → `reproduces_published_first_token_cell` | +| New-items-only contrast | +0.278 / +0.545 / +0.478 (first-token) | same → `per_source.first_token.m1-new` | +| M2 gate *(Fig 1)* | +0.853 / +0.794 at 1.5B; +0.750 / +0.688 at 3B | `results/m2-depth-*.json` → `localization_contrast` | +| M2 window sweep *(Fig 2)* | 1.5B: 33,32,27,28,28,27,25,23,23,1,**0**,0 of 34 | same → `window_map[*].cell` | +| Band shading in Fig 2 | L9–L21 / L11–L24 / L14–L32 | same → `band` | +| M2 dose curve *(Fig 3)* | 1.5B 34,20,3,1,0 of 34 (mass .913,.594,.115,.037,.017) | same → `dose_curve` | +| Half-mute λ ≈ 0.23/0.29/0.36 | **INTERPOLATED, not measured — quoted in prose, plotted nowhere** | `docs/M2-BRIEF.md` — the brief says so itself | +| Unequal band thirds | 4/4/**5**, 4/4/**6**, 6/6/**7** | `results/m2-depth-*.json` → `thirds` | +| Equal-width control | 1.5B L11–L16 25/34 vs late 0/34; 3B L12–L18 29/32 vs 3/32 | same → `window_map` | +| M3 clause (1) *(Fig 1)* | +0.971 [+0.867, +0.983] / +0.881 [+0.731, +0.943] | `results/m3-matrix-*.json` → `specificity_contrast.clause_1_pooled` | +| M3 clause (2) *(Fig 1)* | +0.950 [+0.814, +0.978] / +0.891 [+0.730, +0.947] | same → `clause_2_within_category` | +| M3 arms | diagonal 0/28, 0/34, 3/32; off-diagonal 279/308, 363/374, 343/352 | same → `pooled_arms` | +| **Every cell of Fig 4** | 144 cells × 3 subjects, each `hits`/`n`/`rate` | same → `matrix[*].cell` | +| `silver` row damages nothing | 27/27, 31/31, 31/31 | same → `row_profiles.silver.collateral_all` | +| `silver` column is the most fragile | 7/11, 27/33, 6/11 | same → `column_profiles.silver.fragility_all` | +| Category-block **null** at 1.5B/3B | +0.028 [−0.010, +0.091]; +0.020 [−0.016, +0.079] | `docs/M3-BRIEF.md` results (**the Newcombe is brief-only; the arms are in `pooled_arms`**) | +| M4 gate arm | 41 / 71 / 84 | `results/m4-strip-*.json` → `competence.gate_arm_n` | +| **M4 headline** *(Fig 5)* | 11/41 = 0.268; **51/71 = 0.718 [0.605, 0.810]**; **63/84 = 0.750 [0.648, 0.830]** | same → `vocabulary_sparing.gate_arm` | +| **M4 concept-level (the honest quote)** *(Fig 5)* | **24/41 = 0.585, lower 0.434**; **26/43 = 0.605, lower 0.456** | same → `vocabulary_sparing.conservative_reads[concept-level]` | +| M4 verdict strings, verbatim | `VOCAB-SPARING (…) — AS-SCORED ONLY (concept-level …)` | same → `vocabulary_sparing.verdict` | +| Residual-conservative read (clears) *(Fig 5)* | 49/71 lower 0.575; 62/84 lower 0.635 | same → `conservative_reads[residual-conservative]` | +| The 0.5 bar drawn in Fig 5 | 0.5 | same → `vocabulary_sparing.bar` | +| Two cross-checks, two generations | M1 255/255 and M3 468/468 cells, mass exact, ×3 | same → `m1_crosscheck`, `m3_crosscheck` | +| No prime is a wrecking ball *(Fig 6, left)* | 1.5B rows 63/71–67/71; 3B 77/84–83/84 | same → `row_profiles[*].collateral_non_subset` | +| Fragile columns *(Fig 6, right)* | `copper` 6/12, `mosquito` 8/12; `eagle` 8/12, `platinum` 9/12 | same → `column_profiles[*].fragility` | +| Zero-collateral columns *(Fig 6)* | 32 of 53 at 1.5B; 33 of 55 at 3B | same → `column_profiles[*].fragility` (hits = n) | +| Category block **does not** dissolve | 22/29 vs 769/823; 35/53 vs 913/955 | same → `new_pool_arms.{within_category, cross_category}` | +| Zero-gated concepts (upward bias) | 25 / 7 / 5 | same → `competence.zero_gated_non_subset_concepts` | +| Span residual cells | 0 / 27 / 21 (0/26/21 in the gate arm) | same → `residual_cells` | +| Cost, suite | $0; 396 tests green (390 pass with no network) | briefs' wall-clock sections; `HANDOFF.md` records the suite count and CI state | +| Every figure's plotted values | printed with its JSON key on every run | `uv run --with matplotlib docs/paper/figures.py` | + +--- + +## Anticipated Q&A + +**"Why is this not just a reproduction?"** +Because there is nothing published to reproduce. The effect was found *inside* my own +replication of the Jacobian lens, in a stage the seed paper does not contain. The anchor +is dim-stage's own recorded S4b result, and I re-run it bit-for-bit before measuring +anything new. The seed paper is cited by URL as intellectual context — it has no arXiv +ID, and I claim nothing of it. + +**"Your headline says VOCAB-SPARING. Isn't that overclaiming?"** +That's exactly why the label carries AS-SCORED ONLY *inside* the verdict string. The +item-level floor clears the bar; the concept-level floor's lower bound does not — 0.434 +and 0.456. Both reads were pre-registered before any new cell ran, and the brief +pre-committed the concept-level numbers as the ones to quote. The qualifier is attached +by the runner, conditionally, not written by me after seeing the result. Figure 5 is the +whole argument in one picture: same cells, two pre-registered ways to count them, and the +lower bounds land on opposite sides of the line. + +**"Why is a null a result?"** +Because the gate was written before the run and can only be read one way afterward. The +dose curve's null against a step function is the answer to the kickoff's own question: +whatever the direction carries, the ability to emit the word degrades continuously. +M3's category-block null is a real null *for that arm* — and M4 shows exactly why calling +it general would have been wrong. If I only reported nulls I liked, none of my gates would +mean anything. + +**"Why Wilson intervals rather than a normal approximation?"** +Because most of these cells are at or near 0 or 1 — diagonal cells of 0/34, clean cells of +105/105. The normal approximation gives intervals that run past 0 and 1 and are badly +wrong at the extremes; Wilson stays inside [0, 1] and behaves at small n. For differences +between two arms I use Newcombe's method, built from the two Wilson intervals. One honest +caveat: my arms are *paired* (the same items in both) and Newcombe assumes independence. +For positively correlated arms that **widens** the interval — it can cost me power, it +cannot manufacture a positive. + +**"Why don't your sweep and dose figures have lines through them?"** +Because I didn't measure the points between the points. The window sweep is a stride-2 +grid and the dose curve is five frozen λ values; a line through them would draw values +that were never run, and the smoothed version is exactly how a five-point grid gets +mistaken for a measured curve. Same reason the half-mute λs (≈ 0.23 / 0.29 / 0.36) are in +the prose and on no axis — they're linear interpolations between two grid points and the +brief labels them that way. The mass channel in Figure 3 is the non-interpolated +companion: it moves continuously because it *is* continuous, not because I drew it that +way. + +**"What's the un-validatable residual?"** +I never measured *why* the late third is special, whether this direction is the same +object the seed paper's lens is about, or anything above 3B. The window sweep is +structurally thin above the band — no window is ever fully above it — so "the switch is +late" is well-measured on its early side and weakly probed on its far side. And the claim +is scoped to the *measurable* vocabulary: 25 / 7 / 5 concepts gate zero items, a +selection that plausibly enriches for robust concepts and biases my floor upward. + +**"Why these models?"** +Qwen2.5-0.5B/1.5B/3B-Instruct because the anchor was measured on exactly them, the lenses +were already fitted for them, and all three run locally on MPS forward-only at $0. 0.5B is +carried through every stage but is never gate-bearing — the predecessor had already +recorded non-specific damage at that scale, so it is read only under a standing +any-direction-damage frame. + +**"Two of your findings correct earlier stages. Doesn't that undermine them?"** +They re-scope, they don't retract — and both earlier stages' published numbers stand +untouched. `silver` was labelled non-specific from a single control cell; the matrix +showed that was a fact about silver's *column* (its row damages nothing at any scale) — +look at Figure 4 and the row is uniformly light while the column is the visible stripe. +Category-block collateral looked like it dissolved with scale; the strip showed M3's +within-category arm was 30 of 34 pairs *countries*, and over ten categories it doesn't +dissolve. The transferable lesson: **a single control cell measures a cell, not a row**, +and an arm's composition is part of what it measured. Both were findable only because +every stage re-runs its predecessor's recorded cells rather than trusting them. + +**"What would you do next?"** +Two designed-and-declined stretches are banked: 7B on a rented GPU for the +specificity-versus-scale curve, and a lexical-versus-semantic scope test — if I delete +`France`, can the model still say *French*, *Paris*, a translation? That second one is the +question that decides whether this is a **token** mute button or a **concept** mute +button, and I genuinely don't know the answer. Its brief also owes one decision before it +can freeze anything: what counts as a word boundary for non-ASCII forms, which +`oracle.py`'s `_BOUNDARY` currently leaves open. + +**"Anything broken you're not hiding?"** +Three, all recorded and none affecting a number: M4's frozen gate wording promises a +per-pair degeneracy readout the runner doesn't compute (byte-frozen, so it's disclosed +rather than patched — and I don't quote that clause); a battery re-parse sits outside its +`try/except` and would traceback instead of exiting cleanly (unreachable in one process); +and the CI job is still *named* `offline-suites` while now fetching four tokenizer repos +per push — 390 of 396 tests pass with no network, so a red build there is network, not +logic. Also: CI genuinely ran zero tests until 2026-07-29, so any green badge before that +date certifies syntax, not behaviour. + +--- + +## Vocabulary crib + +| Term | One plain line | +|---|---| +| **Jacobian lens** | A fitted map from a model's internal activations to per-word directions. | +| **Direction / concept vector** | The lens's row for one word — the pattern that word's presence writes into the activations. | +| **Projection removal (k = 1)** | Subtract out exactly the component of the activation pointing along one direction; leave everything else. Forward-only, no retraining. | +| **λ (dose)** | How much of that component to remove. λ = 1 is all of it, λ = 0.25 is a quarter. | +| **Workspace band** | The contiguous layer range the lens was fitted over; "late third" is its last third. | +| **Prime / probe** | The concept whose direction you *delete* (prime) versus the concept you then *ask about* (probe). | +| **Diagonal / off-diagonal** | Diagonal = delete A, ask about A (the mute). Off-diagonal = delete A, ask about B (the collateral). | +| **Competence gate** | An item counts only if the model answers it correctly with nothing ablated. Decided on the clean arm, before any intervention. | +| **Greedy decoding** | Always take the single highest-probability next token — no sampling, so the readout is deterministic and reruns are bit-identical. | +| **Oracle** | The fixed rule deciding "did the model say the word". Here: does the 3-token greedy span *open with* the spelling at a word boundary, case-insensitive. Never an LLM judge. | +| **Concept mass** | The softmax probability assigned to the concept's token — a graded channel beside the yes/no one. | +| **Wilson interval** | A confidence interval for a proportion that stays inside [0, 1] and behaves at small n and at 0/1. | +| **Newcombe interval** | A confidence interval for the *difference* of two proportions, built from their two Wilson intervals. | +| **CI-clean** | The difference's interval excludes zero. If it includes zero, it's a null. | +| **UNDERPOWERED** | Fewer than 20 trials in the cell — pre-declared to support no claim, whatever it shows. | +| **Degeneracy guard** | A check that an arm hasn't collapsed onto one repeated wrong answer, which would fake a clean result. | +| **Pre-registration / frozen gate** | The pass/fail rule *and its exact verdict wording* written as code before the first run, then never edited. | +| **Bit-for-bit cross-check** | Re-running previously recorded cells and demanding identical outputs, before reading any new cell. | +| **AS-SCORED ONLY** | This project's pre-declared qualifier: the headline holds under the scoring rule used, and a pre-registered alternative scoring does not clear the bar. | +| **`not shown`** | The lineage's null label — failing a lower-bound test does not establish the opposite, so the failing verdict never asserts one. | +| **Interpolation (and why it's absent)** | Reading a value *between* two measured grid points. It is quoted in prose where the brief did so and labelled as such — and never plotted, because a drawn line asserts unmeasured values. |