diff --git a/.gitignore b/.gitignore index 4d397bf..1b3f465 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ models/qwopus-coder.md cards/qwopus-coder.html cards/img/qwopus-coder.png + +# python build artifacts +__pycache__/ +*.pyc diff --git a/scripts/thinking-probes/README.md b/scripts/thinking-probes/README.md index 1339bc5..4053733 100644 --- a/scripts/thinking-probes/README.md +++ b/scripts/thinking-probes/README.md @@ -65,15 +65,46 @@ cannot make: | signal | verdict | |---|---| -| `reasoning_content` non-empty | `REASONED_PARSED` | +| `reasoning_content` or `reasoning` non-empty | `REASONED_PARSED` | | `` markers in content | `REASONED_UNPARSED` | | neither | `NO_REASONING` | `REASONED_UNPARSED` is the important one. Any occurrence means a stack reading -only `reasoning_content` would report that arm as not firing when it did, so a +only the reasoning field would report that arm as not firing when it did, so a published firing rate from such a stack is measuring the parser as much as the model. +**Servers disagree on the field name.** llama.cpp populates `reasoning_content`; +vLLM exposes `reasoning`. These tools read both, because reading only the first +makes every arm on a vLLM lane look like `NO_REASONING`, which is this tool +inventing the very answer it exists to measure. Reported by @Blackwellboy in +offlabel#8 after a run where all 42 rows logged zero reasoning. + +If a run reports `NO_REASONING` across **every** arm including `true`, do not +conclude the model does not reason. Check the field name first: a stack that +exposes thinking under a third name would still read as silent here. + +**Both sampling tools now say this themselves.** A guard rail only works on +someone who reads it at the moment they need it, and this condition is +machine-checkable, so the tools print a warning at summary time rather than +leaving it to the reader: + +``` + WARNING: every arm returned NO_REASONING, including true where thinking is + explicitly enabled. That is either a genuinely non-reasoning model or a + reasoning field this tool does not read. +``` + +That makes it a positive control rather than advice, which matters more here +than usual: the thing being measured is absence, and a wrong key produces +absence too. + +It deliberately stays quiet when it would be guessing. If the enabled arm +errored rather than returning nothing, or every request failed, nothing was +measured and no field-name conclusion is available. `persona_ab.py` has no +`true` arm at all, so there it keys on the `absent` cells, since zero in the +`false` control cells is the expected result rather than a symptom. + ## `persona_ab.py`: does a system prompt change it Crosses a persona system prompt against the kwarg, four cells: diff --git a/scripts/thinking-probes/persona_ab.py b/scripts/thinking-probes/persona_ab.py index 4838104..19be042 100755 --- a/scripts/thinking-probes/persona_ab.py +++ b/scripts/thinking-probes/persona_ab.py @@ -59,6 +59,32 @@ THINK_MARKER = re.compile(r"", re.I) +def field_name_warning(rows): + """Positive control for the one failure this tool cannot otherwise see. + + What is being measured is absence, and a reasoning field this tool does + not read produces absence too. Note this grid has no explicit `true` + arm: the `false` cells are the structural control, where zero is the + expected result, so the signal is zero in the `absent` cells as well. + + Returns a warning string, or None when the check does not apply. + """ + scored = [r for r in rows if r["reasoned"] is not None] + if not scored: + return None # nothing was measured; not this failure + if any(r["reasoned"] for r in scored): + return None # something reasoned; the reader works + available = [k for (_, k) in CELLS if k != "false"] + if not any(r["kwarg"] in available for r in scored): + return None # only control cells scored; says nothing + return (" WARNING: no cell reasoned, including " + f"{'/'.join(sorted(set(available)))} where thinking is available\n" + " rather than structurally suppressed. That is either a genuinely\n" + " non-reasoning model or a reasoning field this tool does not read.\n" + " Check the field name against the server's response before\n" + " believing these numbers.") + + def post(url, body, timeout): req = urllib.request.Request( url, data=json.dumps(body).encode(), @@ -104,7 +130,13 @@ def main(): d = post(args.base + "/v1/chat/completions", body, args.timeout) ch = d["choices"][0] content = ch["message"].get("content") or "" - reasoning = ch["message"].get("reasoning_content") or "" + # Servers disagree on the field name: llama.cpp populates + # reasoning_content, vLLM exposes reasoning. Reading only the first + # makes every arm look like NO_REASONING on a vLLM lane, which is + # this tool inventing the answer it exists to measure + # (offlabel#8, @Blackwellboy). + reasoning = (ch["message"].get("reasoning_content") + or ch["message"].get("reasoning") or "") finish = ch.get("finish_reason") reasoned = bool(reasoning.strip()) or bool(THINK_MARKER.search(content)) except Exception as exc: # noqa: BLE001 @@ -129,6 +161,11 @@ def main(): mr = sum(r["reasoning_chars"] for r in rs) / max(len(rs), 1) print(f" {persona:<11} {kwarg:<7} {n:>4}/{len(rs):<5} {mr:>18.0f}") + warning = field_name_warning(rows) + if warning: + print() + print(warning) + print("\n=== persona effect WITHIN the absent arm, per prompt shape ===") for pname in PROMPTS: line = f" {pname:<15}" diff --git a/scripts/thinking-probes/thinking_ab.py b/scripts/thinking-probes/thinking_ab.py index a038ce6..beddeea 100755 --- a/scripts/thinking-probes/thinking_ab.py +++ b/scripts/thinking-probes/thinking_ab.py @@ -75,6 +75,33 @@ def rendered_tail(base, arm, timeout): return p[-28:], p.rstrip().endswith("") +def field_name_warning(rows): + """Positive control for the one failure this tool cannot otherwise see. + + What is being measured is absence, and a reasoning field this tool does + not read produces absence too. The two are indistinguishable in the + numbers, so when nothing reasoned anywhere, including the arm that + explicitly enables thinking, say so rather than let the zeros stand. + + Returns a warning string, or None when the check does not apply. + """ + scored = [r for r in rows if r["verdict"] != "ERROR"] + if not scored: + return None # nothing was measured; not this failure + if any(r["verdict"].startswith("REASONED_") for r in scored): + return None # something reasoned; the reader works + # Derived, not hardcoded, so it stays correct if ARMS changes. + enabled = [a for a, kw in ARMS.items() + if kw and kw.get("enable_thinking") is True] + if not any(r["arm"] in enabled for r in scored): + return None # the enabled arm produced no scored sample + return (" WARNING: every arm returned NO_REASONING, including " + f"{'/'.join(enabled)} where thinking is explicitly enabled.\n" + " That is either a genuinely non-reasoning model or a reasoning\n" + " field this tool does not read. Check the field name against the\n" + " server's response before believing these numbers.") + + def verdict(content, reasoning): if reasoning.strip(): return "REASONED_PARSED" @@ -128,7 +155,13 @@ def main(): d = post(args.base + "/v1/chat/completions", body, args.timeout) ch = d["choices"][0] content = ch["message"].get("content") or "" - reasoning = ch["message"].get("reasoning_content") or "" + # Servers disagree on the field name: llama.cpp populates + # reasoning_content, vLLM exposes reasoning. Reading only the first + # makes every arm look like NO_REASONING on a vLLM lane, which is + # this tool inventing the answer it exists to measure + # (offlabel#8, @Blackwellboy). + reasoning = (ch["message"].get("reasoning_content") + or ch["message"].get("reasoning") or "") finish = ch.get("finish_reason") v = verdict(content, reasoning) except Exception as exc: # noqa: BLE001 @@ -153,6 +186,11 @@ def main(): print(f" {arm:<8} {c('REASONED_PARSED'):>7} {c('REASONED_UNPARSED'):>9} " f"{c('NO_REASONING'):>6} {c('ERROR'):>4} {mr:>10.0f}") + warning = field_name_warning(rows) + if warning: + print() + print(warning) + print("\n=== per prompt shape (is the effect prompt-specific?) ===") for pname in PROMPTS: line = f" {pname:<15}"