diff --git a/marc/eval/baselines/README.md b/marc/eval/baselines/README.md new file mode 100644 index 0000000..60097da --- /dev/null +++ b/marc/eval/baselines/README.md @@ -0,0 +1,49 @@ +# CoT baseline + +Token-level chain-of-thought baseline for H1 comparison against MARC's value-diffusion solver. + +## Model + +`gpt-5` via OpenAI API. Fixed internal temperature — this model does not support +temperature overrides (raises `BadRequestError` if passed). `max_completion_tokens=4096`; +reasoning tokens count against this budget, and problems that exhaust it before producing +a visible answer are scored as unparseable (see Known limitations). + +## Prompt + +Equations are rendered from `problem.graph.factors` (not the `description` string) so +the model sees exactly what the checker verifies. Answers are constrained to decimal +format (`ANSWER: x=, y=`) to keep parsing unambiguous — fractions +(e.g. `7/2`) are explicitly disallowed in the instruction. + +## Run config + +- N = 25 problems per split (in_distribution, held_out_structure) +- perturb_delta = 0.1 +- n_samples (k) = 1 (pass@1) + +## Running + + python -m marc.eval.baselines.cot_baseline + +Output: results/p2_main/cot_baseline.json + +## Result summary (N=25) + +- solve_rate: 1.0 on both splits (generalization_gap = 0.0) +- perturbation_robustness: 0.0 (in-distribution) vs 0.12 (held-out structure) + +Raw solve rate saturates at this problem size/difficulty, so it does not separate +the two splits — but perturbation_robustness does show a gap, suggesting this suite +may still be too easy for solve_rate itself to be an informative H1 signal. +Flagged to Sparsh/Davin: consider a harder suite (e.g. length_extrapolation with +higher n_vars) if solve_rate saturation persists once MARC's model is compared. + +## Known limitations + +- gpt-5's reasoning-token usage varies per problem; if it exhausts max_completion_tokens + on internal reasoning before emitting an ANSWER line, parse_answer returns None and + the Checker correctly scores it as a failure. This is a legitimate baseline failure + mode (a token-budget design choice), not a parsing bug. +- Output JSON can contain NaN/Infinity for unparseable answers — not valid strict JSON; + confirm downstream consumers (e.g. summary table merge) handle this or coerce to null. diff --git a/marc/eval/baselines/__init__.py b/marc/eval/baselines/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/marc/eval/baselines/cot_baseline.py b/marc/eval/baselines/cot_baseline.py new file mode 100644 index 0000000..3b90670 --- /dev/null +++ b/marc/eval/baselines/cot_baseline.py @@ -0,0 +1,106 @@ +import re + + +import sympy as sp + +def _split_residual(expression: str) -> tuple[str, str]: + expr = sp.sympify(expression) + const, var_part = expr.as_coeff_Add() + rhs = -const + return str(var_part), str(rhs) + +def render_prompt(problem) -> str: + var_names = [v.id for v in problem.graph.variables] + + equations = [] + for factor in problem.graph.factors: + lhs, rhs = _split_residual(factor.expression) + equations.append(f"{lhs} = {rhs}") + + system = "\n".join(equations) + answer_line = ", ".join(f"{v}=" for v in var_names) + + return ( + f"Solve this system of equations for {', '.join(var_names)}:\n\n" + f"{system}\n\n" + "Reason step by step, then give your final answer as the very last line " + "in exactly this format, using decimal numbers (e.g. 3.5, not 7/2):\n" + f"ANSWER: {answer_line}" + ) +import openai + +_client = openai.OpenAI() # reads OPENAI_API_KEY from env automatically + + +def call_model(prompt: str, model: str = "gpt-5") -> str: + response = _client.chat.completions.create( + model=model, + max_completion_tokens=4096, + messages=[{"role": "user", "content": prompt}], + ) + return response.choices[0].message.content + +import re + + +def parse_answer(text: str, n_vars: int) -> list[float] | None: + match = re.search(r"ANSWER:\s*(.+)", text) + if match is None: + return None + + numbers = re.findall(r"-?\d+\.?\d*", match.group(1)) + if len(numbers) != n_vars: + return None + + return [float(n) for n in numbers] + + +class CoTSolver: + def __init__(self, model: str = "gpt-5"): + self.model = model + + def sample(self, problem, k: int) -> list[list[float]]: + prompt = render_prompt(problem) + n_vars = len(problem.solution) + + candidates = [] + for _ in range(k): + text = call_model(prompt, self.model) + parsed = parse_answer(text, n_vars) + if parsed is None: + parsed = [float("nan")] * n_vars + candidates.append(parsed) + return candidates + +import json +from marc.eval.problems import in_distribution, held_out_structure +from marc.eval.runner import run_split_eval + +import json +from marc.eval.problems import in_distribution, held_out_structure +from marc.eval.runner import run_split_eval + +if __name__ == "__main__": + N = 25 # TODO: confirm with Sparsh + PERTURB_DELTA = 0.1 # TODO: confirm with Sparsh + K = 1 + + solver = CoTSolver(model="gpt-5") + + metrics = run_split_eval( + in_distribution(N), + held_out_structure(N), + solver=solver, + perturb_delta=PERTURB_DELTA, + n_samples=K, + solver_name="cot_baseline", + ) + + import os + os.makedirs("results/p2_main", exist_ok=True) + with open("results/p2_main/cot_baseline.json", "w") as f: + json.dump(metrics, f, indent=2) + + print(f"solve rate (in-dist): {metrics['splits']['in_distribution']['solve_rate']}") + print(f"solve rate (held-out): {metrics['splits']['held_out_structure']['solve_rate']}") + print(f"generalization gap: {metrics['generalization_gap']}") \ No newline at end of file