From ca9d864fa2c7b65cde75e67d426d6be41adeaca8 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Wed, 30 Sep 2026 12:44:55 +0100 Subject: [PATCH 01/10] Entry-resolution tasks: prompts that name a protein, graded against all its entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every MVS task handed the model the PDB id. Real users don't: the first evaluator prompt to fail live was "I wanna see a structure of PDE5A", answered with 1UJ7 — an entry that does not exist. Probing found the quieter failure too: real ids for the wrong protein (a SARS-CoV-2 RBD for "nanobody bound to GFP"). - tasks/mvs_resolve/: six tasks (PDE5A, PCSK9, CFTR, GFP, myoglobin, lysozyme), each with `accepted_ids` = every PDB entry mapped to the protein's UniProt accession (RCSB search, recorded under provenance) and a minimal polymer-cartoon reference on one canonical entry. scripts/generate_resolve_tasks.py regenerates. - grade_mvs(accepted_refs=…): any accepted id in the prediction is folded onto the reference's; anything else (1UJ7) mismatches as before. Runner and escalation pass the task's set through. - Tests: accepted set sanity, any-accepted-entry == 1.0, the 1UJ7 case scores below it, multi-structure references are left alone. 22 pass. - ROADMAP: B4 resolution, B5 colour-scheme prompt drift, B6 structured outputs. Co-Authored-By: Claude Fable 5.1 --- ROADMAP.md | 16 + molbench/escalate.py | 3 +- molbench/mvs.py | 37 +- molbench/runner.py | 6 +- scripts/generate_resolve_tasks.py | 106 ++ tasks/mvs_resolve/res-cftr.json | 130 +++ tasks/mvs_resolve/res-gfp.json | 707 +++++++++++++ tasks/mvs_resolve/res-lysozyme.json | 1468 ++++++++++++++++++++++++++ tasks/mvs_resolve/res-myoglobin.json | 427 ++++++++ tasks/mvs_resolve/res-pcsk9.json | 136 +++ tasks/mvs_resolve/res-pde5a.json | 120 +++ tests/test_harness.py | 57 + 12 files changed, 3208 insertions(+), 5 deletions(-) create mode 100644 scripts/generate_resolve_tasks.py create mode 100644 tasks/mvs_resolve/res-cftr.json create mode 100644 tasks/mvs_resolve/res-gfp.json create mode 100644 tasks/mvs_resolve/res-lysozyme.json create mode 100644 tasks/mvs_resolve/res-myoglobin.json create mode 100644 tasks/mvs_resolve/res-pcsk9.json create mode 100644 tasks/mvs_resolve/res-pde5a.json diff --git a/ROADMAP.md b/ROADMAP.md index a7aef6f..249b5e5 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -78,6 +78,22 @@ strong judge, structured protocol, self-variance reporting); rubric quality search to extract interface/contact residues (ligand–protein, protein–protein), giving grounded "show the binding-site residues" / "show the PPI interface" tasks — a realistic, high-value class we currently lack. +* **B4. Entry resolution (STARTED — `tasks/mvs_resolve/`).** Every task used to hand + the model the PDB id; real users name the protein. The first live failure was + "a structure of PDE5A" → PDB 1UJ7 (does not exist), and probing found real ids + for the wrong protein too. Six tasks now grade against *all* entries for the + protein's UniProt accession (`accepted_ids`, folded by the grader). Next: more + targets, and an `exists` sub-metric (did the id resolve at RCSB at all?) — a + hallucinated id and a wrong-but-real id are different failures. +* **B5. Colour schemes.** The chat-driver's prompt now teaches Mol*'s + `custom.molstar_color_theme_name` on a `color` node for rainbow / by-chain / + by-SS colouring (models kept writing `"spectrum"` as a colour). Porting that here + needs the grader to treat a theme-coloured representation as equivalent to the + per-chain component references it would replace; until then the two prompts differ. +* **B6. Structured outputs.** Constrained decoding (`response_format: json_schema`, + `strict`) with a trimmed MVS schema makes invalid trees impossible by + construction. Measure validation failures + F1 against the prompt-only path on + the cheap models. * **B3. Corpus scaling & de-correlation.** More clinical targets (BRCA1, CFTR, kinases, more p53 structures incl. one with a numbering *offset* to exercise the SIFTS bridge); more Tier-1 template diversity so per-skill items are less diff --git a/molbench/escalate.py b/molbench/escalate.py index e2270cf..dbf73a0 100644 --- a/molbench/escalate.py +++ b/molbench/escalate.py @@ -37,8 +37,9 @@ def escalating_grade( tree_threshold: float = 0.999, visual_threshold: float = 0.97, workdir: str | pathlib.Path | None = None, + accepted_refs: set[str] | list[str] | None = None, ) -> dict[str, Any]: - g = grade_mvs(reference_tree, predicted_tree) + g = grade_mvs(reference_tree, predicted_tree, accepted_refs=accepted_refs) out: dict[str, Any] = {"tree_f1": g["f1"], "tier": "tree"} # T0 — tree match is conclusive (identical / normalised-equivalent). diff --git a/molbench/mvs.py b/molbench/mvs.py index 3dc5ebd..3499e74 100644 --- a/molbench/mvs.py +++ b/molbench/mvs.py @@ -170,8 +170,39 @@ def _path_similarity(p: tuple, q: tuple) -> float: # --- grading ---------------------------------------------------------------------- -def grade_mvs(reference: Any, predicted: Any) -> dict[str, Any]: - """Grade a predicted MVS scene against a reference. Returns precision/recall/f1.""" +def _fold_accepted_refs(ref_paths: list[tuple], pred_paths: list[tuple], + accepted: set[str]) -> list[tuple]: + """Treat any accepted structure id in the prediction as the reference's own. + + A prompt that names a protein ("a structure of PDE5A") has many right answers; + the task lists them in ``accepted_ids``. The reference carries one canonical id, + so a prediction that downloaded another accepted entry is rewritten to that id + before matching. An id outside the set (a made-up one, say) stays as it is and + mismatches the download segment like any other wrong answer. + """ + ref_ids = {seg[1][1] for path in ref_paths for seg in path + if seg[0] == "download" and isinstance(seg[1], tuple) and seg[1][0] == "ref"} + if len(ref_ids) != 1: + return pred_paths # several structures: no single canonical id to fold onto + canonical = next(iter(ref_ids)) + accepted = {a.lower() for a in accepted} + + def fold(seg: tuple) -> tuple: + if seg[0] == "download" and isinstance(seg[1], tuple) and seg[1][0] == "ref" \ + and seg[1][1] in accepted: + return ("download", ("ref", canonical)) + return seg + + return [tuple(fold(seg) for seg in path) for path in pred_paths] + + +def grade_mvs(reference: Any, predicted: Any, + accepted_refs: set[str] | list[str] | None = None) -> dict[str, Any]: + """Grade a predicted MVS scene against a reference. Returns precision/recall/f1. + + ``accepted_refs``: other structure ids that count as the reference's (see + ``_fold_accepted_refs``); used by tasks that name a protein instead of an id. + """ ref_root, pred_root = extract_root(reference), extract_root(predicted) if ref_root is None: return {"precision": 0.0, "recall": 0.0, "f1": 0.0, "error": "bad reference tree"} @@ -181,6 +212,8 @@ def grade_mvs(reference: Any, predicted: Any) -> dict[str, Any]: ref_paths = flatten_paths(ref_root) pred_paths = flatten_paths(pred_root) + if accepted_refs: + pred_paths = _fold_accepted_refs(ref_paths, pred_paths, set(accepted_refs)) n_ref, n_pred = len(ref_paths), len(pred_paths) if n_ref == 0: f1 = 1.0 if n_pred == 0 else 0.0 diff --git a/molbench/runner.py b/molbench/runner.py index 9298044..9a1783f 100644 --- a/molbench/runner.py +++ b/molbench/runner.py @@ -196,7 +196,8 @@ def run_mvs_task(model, system: str, task: dict) -> dict[str, Any]: tree, err = extract_json_object(raw) if err: return {"f1": 0.0, "precision": 0.0, "recall": 0.0, "error": err, "raw": raw} - result = mvs_mod.grade_mvs(task["reference_mvs"], tree) + result = mvs_mod.grade_mvs(task["reference_mvs"], tree, + accepted_refs=task.get("accepted_ids")) result["schema_errors"] = mvs_mod.validate_mvs(tree) result["predicted"] = tree result["raw"] = raw @@ -305,7 +306,8 @@ def run(models: list[str], categories: list[str] | None, wd = gallery / model.name.replace("/", "_") / task["id"] pt["escalation"] = escalating_grade( task["reference_mvs"], pt["predicted"], prompt=task.get("prompt"), - render=render_fn, vlm=vlm_fn, workdir=str(wd)) + render=render_fn, vlm=vlm_fn, workdir=str(wd), + accepted_refs=task.get("accepted_ids")) per_task.append(pt) raw_samples[model.name][task["id"]] = [ {"f1": g["f1"], "predicted": g.get("predicted"), diff --git a/scripts/generate_resolve_tasks.py b/scripts/generate_resolve_tasks.py new file mode 100644 index 0000000..11875c2 --- /dev/null +++ b/scripts/generate_resolve_tasks.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +""" +Entry-resolution tasks: the prompt names a protein, not a PDB id. + +Every other MVS task hands the model the id. Real users don't: the first evaluator +prompt to fail in the wild was "I wanna see a structure of PDE5A", answered with +PDB 1UJ7 — an entry that does not exist. Probing further found the quieter failure: +real ids for the wrong protein (a SARS-CoV-2 RBD for "nanobody bound to GFP"). + +Many entries are a right answer here, so each task lists ``accepted_ids``: every PDB +entry whose polymer maps to the protein's UniProt accession, pulled from RCSB search +at authoring time and recorded under ``provenance``. The reference carries one +canonical entry; the grader folds any accepted id onto it (``mvs._fold_accepted_refs``). +The scene itself is deliberately minimal (a polymer cartoon) so the task measures +resolution, not styling. + + python scripts/generate_resolve_tasks.py # -> tasks/mvs_resolve/*.json +""" + +from __future__ import annotations + +import datetime as dt +import json +import pathlib +import urllib.request + +from molbench.mvs import categorize + +REPO = pathlib.Path(__file__).resolve().parent.parent +OUT = REPO / "tasks" / "mvs_resolve" +CIF = "https://files.rcsb.org/download/{}.cif" +SEARCH = "https://search.rcsb.org/rcsbsearch/v2/query" + +# (slug, prompt, UniProt accession, canonical PDB id, note). Prompts are verbatim or +# lightly edited from the evaluator corpus; the canonical id is a well-known entry. +SPECS = [ + ("pde5a", "I wanna see a structure of PDE5A", "O76074", "1udt", + "Evaluator prompt (2026-09-30). Haiku answered 1UJ7, which does not exist."), + ("pcsk9", "show the structure of PCSK9", "Q8NBP7", "2p4e", + "Probe: Haiku answered 1D0G (death receptor 5 / TRAIL)."), + ("cftr", "show me the CFTR channel", "P13569", "5uak", + "Probe: Haiku answered 5UAY (a plant Toc75 POTRA domain)."), + ("gfp", "show green fluorescent protein", "P42212", "1ema", + "The starter chip's target; the model tends to be right here — a control."), + ("myoglobin", "Show me sperm whale myoglobin", "P02185", "1mbn", + "Corpus prompt asked for 'crab myoglobin' (no such entry); sperm whale is the classic."), + ("lysozyme", "Show me hen egg white lysozyme", "P00698", "1lyz", + "Starter chip; hundreds of valid entries — a control."), +] + + +def entries_for(accession: str) -> list[str]: + """All PDB entries with a polymer entity mapped to this UniProt accession.""" + query = { + "query": {"type": "terminal", "service": "text", "parameters": { + "attribute": "rcsb_polymer_entity_container_identifiers." + "reference_sequence_identifiers.database_accession", + "operator": "exact_match", "value": accession}}, + "return_type": "entry", + "request_options": {"paginate": {"start": 0, "rows": 10000}}, + } + req = urllib.request.Request(SEARCH, data=json.dumps(query).encode(), + headers={"content-type": "application/json"}) + with urllib.request.urlopen(req, timeout=60) as resp: + if resp.status == 204: + return [] + return sorted(hit["identifier"].lower() for hit in json.load(resp)["result_set"]) + + +def reference(pdb: str) -> dict: + return {"kind": "root", "children": [{ + "kind": "download", "params": {"url": CIF.format(pdb)}, "children": [{ + "kind": "parse", "params": {"format": "mmcif"}, "children": [{ + "kind": "structure", "params": {"type": "model"}, "children": [{ + "kind": "component", "params": {"selector": "polymer"}, "children": [{ + "kind": "representation", "params": {"type": "cartoon"}}]}]}]}]}]} + + +def main() -> None: + OUT.mkdir(parents=True, exist_ok=True) + today = dt.date.today().isoformat() + for slug, prompt, accession, canonical, note in SPECS: + ids = entries_for(accession) + if canonical not in ids: + raise SystemExit(f"{slug}: canonical {canonical} is not mapped to {accession}") + ref = reference(canonical) + task = { + "id": f"res-{slug}", + "category": "mvs", + "source": "resolve", + "title": f"Resolve by name: {slug}", + "prompt": prompt, + "reference_mvs": ref, + "accepted_ids": ids, + "categories": categorize(ref) + ["resolve"], + "skills": ["resolve", "download", "component", "representation"], + "notes": note, + "provenance": {"uniprot": accession, "n_entries": len(ids), + "source": "RCSB search, reference_sequence_identifiers", "fetched": today}, + } + (OUT / f"res-{slug}.json").write_text(json.dumps(task, indent=2) + "\n") + print(f"res-{slug:10s} {accession} {len(ids):4d} accepted entries canonical {canonical}") + + +if __name__ == "__main__": + main() diff --git a/tasks/mvs_resolve/res-cftr.json b/tasks/mvs_resolve/res-cftr.json new file mode 100644 index 0000000..3b75542 --- /dev/null +++ b/tasks/mvs_resolve/res-cftr.json @@ -0,0 +1,130 @@ +{ + "id": "res-cftr", + "category": "mvs", + "source": "resolve", + "title": "Resolve by name: cftr", + "prompt": "show me the CFTR channel", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/5uak.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "accepted_ids": [ + "1xmi", + "1xmj", + "2bbo", + "2bbs", + "2bbt", + "2lob", + "2pze", + "2pzf", + "2pzg", + "3gd7", + "3isw", + "4wz6", + "5d2d", + "5d3e", + "5d3f", + "5tf7", + "5tf8", + "5tfa", + "5tfb", + "5tfc", + "5tfd", + "5tff", + "5tfg", + "5tfi", + "5tfj", + "5tgk", + "5uak", + "6gjq", + "6gjs", + "6gju", + "6gk4", + "6gkd", + "6hep", + "6msm", + "6o1v", + "6o2p", + "6uk1", + "6wbs", + "6ze1", + "7qi1", + "7sv7", + "7svd", + "7svr", + "8eig", + "8eio", + "8eiq", + "8ej1", + "8fzq", + "8gls", + "8ubr", + "8v7z", + "8v81", + "9dw4", + "9dw5", + "9dw7", + "9dw8", + "9dw9", + "9mxl", + "9y1q", + "9y4t" + ], + "categories": [ + "load", + "representation", + "resolve" + ], + "skills": [ + "resolve", + "download", + "component", + "representation" + ], + "notes": "Probe: Haiku answered 5UAY (a plant Toc75 POTRA domain).", + "provenance": { + "uniprot": "P13569", + "n_entries": 60, + "source": "RCSB search, reference_sequence_identifiers", + "fetched": "2026-09-30" + } +} diff --git a/tasks/mvs_resolve/res-gfp.json b/tasks/mvs_resolve/res-gfp.json new file mode 100644 index 0000000..9eb9385 --- /dev/null +++ b/tasks/mvs_resolve/res-gfp.json @@ -0,0 +1,707 @@ +{ + "id": "res-gfp", + "category": "mvs", + "source": "resolve", + "title": "Resolve by name: gfp", + "prompt": "show green fluorescent protein", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1ema.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "accepted_ids": [ + "13lj", + "13lk", + "13ll", + "13lm", + "13ln", + "13lo", + "13lp", + "13lq", + "13lr", + "13ls", + "13lt", + "13lu", + "1b9c", + "1bfp", + "1c4f", + "1cv7", + "1ema", + "1emb", + "1emc", + "1eme", + "1emf", + "1emg", + "1emk", + "1eml", + "1emm", + "1f09", + "1f0b", + "1gfl", + "1h6r", + "1hcj", + "1huy", + "1jby", + "1jbz", + "1jc0", + "1jc1", + "1kp5", + "1kyp", + "1kyr", + "1kys", + "1myw", + "1q4a", + "1q4b", + "1q4c", + "1q4d", + "1q4e", + "1q73", + "1qxt", + "1qy3", + "1qyf", + "1qyo", + "1qyq", + "1rm9", + "1rmm", + "1rmo", + "1rmp", + "1rrx", + "1s6z", + "1w7s", + "1w7t", + "1w7u", + "1yfp", + "1yhg", + "1yhh", + "1yhi", + "1yj2", + "1yjf", + "1z1p", + "1z1q", + "24vi", + "24vl", + "24vm", + "24vn", + "24vo", + "24vp", + "24vq", + "24vr", + "24vs", + "25ex", + "29te", + "2ah8", + "2aha", + "2awj", + "2awk", + "2awl", + "2awm", + "2b3p", + "2b3q", + "2due", + "2duf", + "2dug", + "2duh", + "2dui", + "2emd", + "2emn", + "2emo", + "2fwq", + "2fzu", + "2g16", + "2g2s", + "2g3d", + "2g5z", + "2g6e", + "2h6v", + "2h9w", + "2hcg", + "2hfc", + "2hgd", + "2hgy", + "2hjo", + "2hqz", + "2hrs", + "2jad", + "2o24", + "2o29", + "2o2b", + "2okw", + "2oky", + "2q57", + "2q6p", + "2qrf", + "2qt2", + "2qu1", + "2qz0", + "2wsn", + "2wso", + "2wur", + "2y0g", + "2ydz", + "2ye0", + "2ye1", + "2yfp", + "31fw", + "38oa", + "38rk", + "38rx", + "3ai4", + "3ai5", + "3cb9", + "3cbe", + "3cd1", + "3cd9", + "3dpw", + "3dpx", + "3dpz", + "3dq1", + "3dq2", + "3dq3", + "3dq4", + "3dq5", + "3dq6", + "3dq7", + "3dq8", + "3dq9", + "3dqa", + "3dqc", + "3dqd", + "3dqe", + "3dqf", + "3dqh", + "3dqi", + "3dqj", + "3dqk", + "3dql", + "3dqm", + "3dqn", + "3dqo", + "3dqu", + "3ed8", + "3evp", + "3evr", + "3g9a", + "3gex", + "3gj1", + "3gj2", + "3i19", + "3k1k", + "3la1", + "3ogo", + "3p28", + "3sry", + "3ss0", + "3ssh", + "3ssk", + "3ssl", + "3ssp", + "3sst", + "3ssv", + "3ssy", + "3st0", + "3sv5", + "3svb", + "3svc", + "3svd", + "3sve", + "3u8p", + "3ufz", + "3ug0", + "3v3d", + "3vht", + "3w1c", + "3w1d", + "3ztf", + "4anj", + "4ar7", + "4as8", + "4b5y", + "4bdu", + "4en1", + "4eul", + "4ges", + "4gf6", + "4h47", + "4h48", + "4ik1", + "4ik3", + "4ik4", + "4ik5", + "4ik8", + "4ik9", + "4j88", + "4j89", + "4j8a", + "4jfg", + "4jrb", + "4ka9", + "4kag", + "4kex", + "4kf5", + "4kw4", + "4kw8", + "4kw9", + "4l12", + "4l13", + "4l1i", + "4lqt", + "4lqu", + "4lw5", + "4n3d", + "4ndj", + "4ndk", + "4ogs", + "4orn", + "4p1q", + "4p7h", + "4pa0", + "4pfe", + "4u2v", + "4xbi", + "4xgy", + "4xl5", + "4xov", + "4xow", + "4xvp", + "4z4k", + "4z4m", + "4zf3", + "4zf4", + "4zf5", + "5aqb", + "5b61", + "5bkf", + "5bkg", + "5dpg", + "5dph", + "5dpi", + "5dpj", + "5dqb", + "5dqm", + "5drf", + "5drg", + "5dtx", + "5dty", + "5dtz", + "5du0", + "5ehu", + "5f9g", + "5fgu", + "5hbd", + "5hge", + "5hw9", + "5hzo", + "5j2o", + "5j3n", + "5jzk", + "5jzl", + "5ktg", + "5lel", + "5lem", + "5ma3", + "5ma4", + "5ma5", + "5ma6", + "5ma8", + "5ma9", + "5mad", + "5mak", + "5mse", + "5n9o", + "5nhn", + "5ni3", + "5o89", + "5o8b", + "5o8c", + "5ox8", + "5ox9", + "5oxa", + "5oxb", + "5oxc", + "5t3i", + "5wj2", + "5wj3", + "5wj4", + "5z6y", + "6aa2", + "6aa6", + "6as9", + "6b7r", + "6b7t", + "6b9c", + "6dq0", + "6dq1", + "6efr", + "6fll", + "6fww", + "6gel", + "6gez", + "6go8", + "6go9", + "6gqg", + "6gqh", + "6grm", + "6hr1", + "6hut", + "6ir6", + "6ir7", + "6itc", + "6jgh", + "6jgi", + "6jgj", + "6kkz", + "6kl0", + "6kl1", + "6krg", + "6l27", + "6lnp", + "6lr7", + "6m63", + "6mb2", + "6mdr", + "6mks", + "6mwq", + "6oa8", + "6ofk", + "6ofl", + "6ofm", + "6ofn", + "6ofo", + "6og8", + "6og9", + "6oga", + "6ogb", + "6ogc", + "6pfr", + "6pfs", + "6pft", + "6pfu", + "6qq8", + "6qq9", + "6qqa", + "6qqb", + "6quh", + "6qui", + "6quj", + "6sm0", + "6t39", + "6t3a", + "6t90", + "6uhj", + "6uhk", + "6uhl", + "6uhm", + "6uhn", + "6uho", + "6uhp", + "6uhq", + "6uhr", + "6un4", + "6un5", + "6un6", + "6un7", + "6uz0", + "6uz3", + "6val", + "6vam", + "6vio", + "6wrf", + "6wsg", + "6wv9", + "6wva", + "6wvd", + "6wvf", + "6wvg", + "6xzf", + "6yov", + "6zsm", + "6zsn", + "6zui", + "7a7k", + "7a7l", + "7aa5", + "7amb", + "7amf", + "7amu", + "7bwn", + "7byl", + "7bym", + "7byn", + "7cd7", + "7cd8", + "7edj", + "7fhk", + "7fhl", + "7fhn", + "7fho", + "7k18", + "7km4", + "7ks0", + "7ks3", + "7kuy", + "7l31", + "7o7c", + "7o7d", + "7o7e", + "7o7h", + "7o7u", + "7o7v", + "7o7w", + "7o7x", + "7pca", + "7pcz", + "7pd0", + "7phr", + "7pnn", + "7sah", + "7sai", + "7sqy", + "7ssv", + "7ssx", + "7ssy", + "7ssz", + "7sun", + "7vcm", + "7x5v", + "7y96", + "7ydq", + "7yeu", + "7yv3", + "7yv5", + "8a6g", + "8a6n", + "8a6o", + "8a6p", + "8a6q", + "8a6r", + "8a6s", + "8a7v", + "8a83", + "8aha", + "8ahb", + "8am4", + "8b6s", + "8b6t", + "8ban", + "8bav", + "8bbj", + "8bvg", + "8bxp", + "8c1x", + "8c7i", + "8dfl", + "8dhr", + "8dn2", + "8dn3", + "8dn4", + "8dn5", + "8dpd", + "8dta", + "8et3", + "8f6p", + "8fck", + "8fed", + "8fee", + "8fef", + "8g0i", + "8g4e", + "8hco", + "8iyy", + "8iyz", + "8iz0", + "8iz1", + "8iz2", + "8iz3", + "8j0j", + "8j1e", + "8j6o", + "8j7h", + "8jf7", + "8jkc", + "8jkg", + "8jki", + "8jl2", + "8jl5", + "8jl6", + "8jl7", + "8jll", + "8jlm", + "8jls", + "8jlt", + "8jlu", + "8k4s", + "8kex", + "8ots", + "8ovn", + "8ovo", + "8ovp", + "8ovy", + "8qwj", + "8rl9", + "8rlb", + "8rlc", + "8rle", + "8sfs", + "8sfv", + "8sfx", + "8sfz", + "8sg3", + "8slc", + "8smu", + "8syg", + "8t15", + "8t17", + "8t18", + "8t6k", + "8t6l", + "8t6q", + "8tlm", + "8tu9", + "8u4n", + "8u4o", + "8u4p", + "8u4q", + "8u4r", + "8u4s", + "8uqq", + "8vdp", + "8vdq", + "8vdr", + "8w2l", + "8wg3", + "8wg4", + "8x9p", + "8xtw", + "8xtx", + "8xty", + "8ya0", + "8ya2", + "8zup", + "8zuq", + "8zur", + "8zus", + "8zut", + "9bcf", + "9boi", + "9c74", + "9c7c", + "9c7e", + "9c7p", + "9c7q", + "9c7r", + "9cla", + "9clv", + "9con", + "9cty", + "9d0y", + "9f22", + "9f23", + "9f24", + "9f33", + "9f34", + "9fen", + "9feq", + "9fgi", + "9haa", + "9hum", + "9irv", + "9k8w", + "9k8x", + "9lk2", + "9lk6", + "9lyl", + "9mus", + "9mut", + "9muu", + "9muv", + "9oo8", + "9osf", + "9osi", + "9pqc", + "9qks", + "9qxl", + "9rgb", + "9rl4", + "9rl5", + "9rmc", + "9rn1", + "9rn2", + "9rpb", + "9rpd", + "9s0t", + "9s2u", + "9sdl", + "9spe", + "9t9p", + "9tpa", + "9u9d", + "9ved", + "9vee", + "9vef", + "9w1y", + "9w24", + "9w25", + "9yop", + "9yp4", + "9yp9", + "9yr7", + "9yrg", + "9yrh" + ], + "categories": [ + "load", + "representation", + "resolve" + ], + "skills": [ + "resolve", + "download", + "component", + "representation" + ], + "notes": "The starter chip's target; the model tends to be right here \u2014 a control.", + "provenance": { + "uniprot": "P42212", + "n_entries": 637, + "source": "RCSB search, reference_sequence_identifiers", + "fetched": "2026-09-30" + } +} diff --git a/tasks/mvs_resolve/res-lysozyme.json b/tasks/mvs_resolve/res-lysozyme.json new file mode 100644 index 0000000..4845439 --- /dev/null +++ b/tasks/mvs_resolve/res-lysozyme.json @@ -0,0 +1,1468 @@ +{ + "id": "res-lysozyme", + "category": "mvs", + "source": "resolve", + "title": "Resolve by name: lysozyme", + "prompt": "Show me hen egg white lysozyme", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1lyz.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "accepted_ids": [ + "132l", + "193l", + "194l", + "1a2y", + "1aki", + "1at5", + "1at6", + "1azf", + "1b0d", + "1b2k", + "1bgi", + "1bhz", + "1bvk", + "1bvx", + "1bwh", + "1bwi", + "1bwj", + "1c08", + "1c10", + "1dpw", + "1dpx", + "1dqj", + "1e8l", + "1f0w", + "1f10", + "1f3j", + "1fdl", + "1flq", + "1flu", + "1flw", + "1fly", + "1fn5", + "1g7h", + "1g7i", + "1g7j", + "1g7l", + "1g7m", + "1gpq", + "1gwd", + "1gxv", + "1gxx", + "1h6m", + "1h87", + "1hc0", + "1hel", + "1hem", + "1hen", + "1heo", + "1hep", + "1heq", + "1her", + "1hew", + "1hf4", + "1hsw", + "1hsx", + "1ic4", + "1ic5", + "1ic7", + "1iee", + "1io5", + "1ioq", + "1ior", + "1ios", + "1iot", + "1ir7", + "1ir8", + "1ir9", + "1j1o", + "1j1p", + "1j1x", + "1ja2", + "1ja4", + "1ja6", + "1ja7", + "1jis", + "1jit", + "1jiy", + "1jj0", + "1jj1", + "1jj3", + "1jpo", + "1jto", + "1jtt", + "1kip", + "1kiq", + "1kir", + "1kxw", + "1kxx", + "1kxy", + "1lcn", + "1lj3", + "1lj4", + "1lje", + "1ljf", + "1ljg", + "1ljh", + "1lji", + "1ljj", + "1ljk", + "1lkr", + "1lks", + "1lma", + "1lpi", + "1lsa", + "1lsb", + "1lsc", + "1lsd", + "1lse", + "1lsf", + "1lsg", + "1lsm", + "1lsn", + "1lsy", + "1lsz", + "1lyo", + "1lys", + "1lyz", + "1lz8", + "1lz9", + "1lza", + "1lzb", + "1lzc", + "1lzd", + "1lze", + "1lzg", + "1lzh", + "1lzn", + "1lzt", + "1mel", + "1mlc", + "1n4f", + "1nby", + "1nbz", + "1ndg", + "1ndm", + "1p2c", + "1ps5", + "1qio", + "1qtk", + "1rcm", + "1rfp", + "1ri8", + "1rjc", + "1sf4", + "1sf6", + "1sf7", + "1sfb", + "1sfg", + "1sq2", + "1t3p", + "1t6v", + "1ua6", + "1uc0", + "1uco", + "1uia", + "1uib", + "1uic", + "1uid", + "1uie", + "1uif", + "1uig", + "1uih", + "1uuz", + "1v7s", + "1v7t", + "1vat", + "1vau", + "1vdp", + "1vdq", + "1vds", + "1vdt", + "1ved", + "1vfb", + "1w6z", + "1wtm", + "1wtn", + "1xei", + "1xej", + "1xek", + "1xfp", + "1xgp", + "1xgq", + "1xgr", + "1xgt", + "1xgu", + "1yik", + "1yil", + "1ykx", + "1yky", + "1ykz", + "1yl0", + "1yl1", + "1yqv", + "1z55", + "1zmy", + "1zv5", + "1zvh", + "1zvy", + "23si", + "29lj", + "29mn", + "29mr", + "29my", + "29nx", + "29oh", + "29ol", + "2a6u", + "2a7d", + "2a7f", + "2aub", + "2b5z", + "2blx", + "2bly", + "2bpu", + "2c8o", + "2c8p", + "2cds", + "2cgi", + "2d4i", + "2d4j", + "2d4k", + "2d6b", + "2d91", + "2dqc", + "2dqd", + "2dqe", + "2dqf", + "2dqg", + "2dqh", + "2dqi", + "2dqj", + "2eiz", + "2eks", + "2epe", + "2f2n", + "2f30", + "2f4a", + "2f4g", + "2fbb", + "2g4p", + "2g4q", + "2h9j", + "2h9k", + "2hs7", + "2hs9", + "2hso", + "2htx", + "2hu1", + "2hu3", + "2hub", + "2i25", + "2i26", + "2i6z", + "2iff", + "2lym", + "2lyo", + "2lyz", + "2lzh", + "2lzt", + "2pc2", + "2q0m", + "2vb1", + "2w1l", + "2w1m", + "2w1x", + "2w1y", + "2war", + "2x0a", + "2xbr", + "2xbs", + "2xjw", + "2xth", + "2ybh", + "2ybi", + "2ybj", + "2ybl", + "2ybm", + "2ybn", + "2ydg", + "2yss", + "2yvb", + "2z12", + "2z18", + "2z19", + "2znw", + "2znx", + "2zq3", + "2zq4", + "2zyp", + "30ig", + "30js", + "30ju", + "30jx", + "30jy", + "3a34", + "3a3q", + "3a3r", + "3a67", + "3a6b", + "3a6c", + "3a8z", + "3a90", + "3a91", + "3a92", + "3a93", + "3a94", + "3a95", + "3a96", + "3agg", + "3agh", + "3agi", + "3ajn", + "3atn", + "3ato", + "3aw6", + "3aw7", + "3az4", + "3az5", + "3az6", + "3az7", + "3b6l", + "3b72", + "3d9a", + "3e3d", + "3ems", + "3exd", + "3f6z", + "3g3a", + "3g3b", + "3hfm", + "3iju", + "3ijv", + "3j4g", + "3j6k", + "3kam", + "3lym", + "3lyo", + "3lyt", + "3lyz", + "3lzt", + "3m18", + "3m3u", + "3mbe", + "3n9a", + "3n9c", + "3n9e", + "3ojp", + "3ok0", + "3otp", + "3p4z", + "3p64", + "3p65", + "3p66", + "3p68", + "3qe8", + "3qng", + "3qy4", + "3rnx", + "3rt5", + "3ru5", + "3rw8", + "3rz4", + "3sp3", + "3t6u", + "3tmu", + "3tmv", + "3tmw", + "3tmx", + "3txb", + "3txd", + "3txe", + "3txf", + "3txg", + "3txh", + "3txi", + "3txj", + "3txk", + "3ulr", + "3w6a", + "3wl2", + "3wmk", + "3wpj", + "3wpk", + "3wpl", + "3wu7", + "3wu8", + "3wu9", + "3wua", + "3wul", + "3wum", + "3wun", + "3wvx", + "3wvy", + "3ww5", + "3ww6", + "3wxt", + "3wxu", + "3zek", + "3zvq", + "45kk", + "4a7d", + "4a8a", + "4a8b", + "4aga", + "4axt", + "4b0d", + "4b1a", + "4b49", + "4b4e", + "4b4i", + "4b4j", + "4bad", + "4baf", + "4bap", + "4bs7", + "4c3w", + "4cj2", + "4d9z", + "4dc4", + "4dd0", + "4dd1", + "4dd2", + "4dd3", + "4dd7", + "4dd9", + "4ddb", + "4ddc", + "4dt3", + "4e3u", + "4eof", + "4et8", + "4et9", + "4eta", + "4etb", + "4etc", + "4etd", + "4ete", + "4fjr", + "4g49", + "4g4b", + "4g4c", + "4g4h", + "4gcb", + "4gcc", + "4gcd", + "4gce", + "4gcf", + "4gla", + "4glv", + "4gn3", + "4gn4", + "4gn5", + "4h1p", + "4h8x", + "4h8y", + "4h8z", + "4h90", + "4h91", + "4h92", + "4h93", + "4h94", + "4h9a", + "4h9b", + "4h9c", + "4h9e", + "4h9f", + "4h9h", + "4h9i", + "4hp0", + "4hpi", + "4hsf", + "4htk", + "4htn", + "4htq", + "4hv1", + "4hv2", + "4i8s", + "4ias", + "4iat", + "4ii8", + "4j1a", + "4j1b", + "4j7v", + "4kxi", + "4lfp", + "4lfx", + "4lgk", + "4lt0", + "4lt1", + "4lt2", + "4lt3", + "4lyb", + "4lyc", + "4lym", + "4lyo", + "4lyt", + "4lyz", + "4lzt", + "4m4o", + "4m6d", + "4mr1", + "4n0j", + "4n1c", + "4n1e", + "4n5r", + "4n8z", + "4n9r", + "4neb", + "4nfv", + "4ng1", + "4ng8", + "4ngi", + "4ngj", + "4ngk", + "4ngl", + "4ngo", + "4ngv", + "4ngw", + "4ngy", + "4ngz", + "4nhi", + "4nhp", + "4nhq", + "4nhs", + "4nht", + "4nij", + "4nsg", + "4nsh", + "4nsi", + "4nsj", + "4nwe", + "4nwh", + "4ny5", + "4o34", + "4ooo", + "4oot", + "4owa", + "4owb", + "4owc", + "4owe", + "4owh", + "4p2e", + "4pgj", + "4phi", + "4ppo", + "4prq", + "4pru", + "4qeq", + "4qgz", + "4qy9", + "4r0f", + "4r6c", + "4rds", + "4rlm", + "4rln", + "4rw1", + "4rw2", + "4tsa", + "4tsb", + "4tsc", + "4ttd", + "4tun", + "4tws", + "4u3x", + "4uwn", + "4uwu", + "4uwv", + "4w94", + "4w96", + "4wg1", + "4wg7", + "4wl6", + "4wl7", + "4wld", + "4wlt", + "4wlx", + "4wly", + "4wm1", + "4wm2", + "4wm3", + "4wm4", + "4wm5", + "4wm6", + "4wmg", + "4wo6", + "4wo9", + "4woa", + "4x3b", + "4xad", + "4xen", + "4xjb", + "4xjd", + "4xjf", + "4xjg", + "4xjh", + "4xji", + "4xn6", + "4xyy", + "4yem", + "4yen", + "4yeo", + "4ym8", + "4yop", + "4z3m", + "4z41", + "4z46", + "4z98", + "4zee", + "4zfp", + "4zix", + "5a3e", + "5a3z", + "5amy", + "5apc", + "5apd", + "5ape", + "5apf", + "5b05", + "5b06", + "5b07", + "5b1f", + "5b1g", + "5b59", + "5b5j", + "5c6i", + "5c6j", + "5c6l", + "5d5c", + "5d5f", + "5dl9", + "5dla", + "5dm9", + "5e4p", + "5e9r", + "5ebh", + "5f14", + "5f16", + "5f81", + "5f9u", + "5f9x", + "5fcp", + "5fdj", + "5fek", + "5fel", + "5fhw", + "5fst", + "5gwb", + "5hll", + "5hmj", + "5hmv", + "5hnc", + "5hnl", + "5hq1", + "5i4w", + "5i4x", + "5i4y", + "5i53", + "5i54", + "5i5q", + "5idd", + "5iel", + "5ihg", + "5ii3", + "5ilc", + "5ilf", + "5j7c", + "5jen", + "5k2k", + "5k2n", + "5k2p", + "5k2q", + "5k2r", + "5k2s", + "5k7o", + "5kj9", + "5kki", + "5kkj", + "5kxk", + "5kxl", + "5kxm", + "5kxn", + "5kxo", + "5kxp", + "5kxr", + "5kxs", + "5kxt", + "5kxw", + "5kxx", + "5kxy", + "5kxz", + "5ky1", + "5l3h", + "5l3i", + "5l9j", + "5la5", + "5la8", + "5laf", + "5lag", + "5lan", + "5lin", + "5lio", + "5lvg", + "5lvh", + "5lvi", + "5lvj", + "5lxw", + "5lym", + "5lyt", + "5lyz", + "5m1y", + "5mjj", + "5myy", + "5nbj", + "5ne0", + "5nj1", + "5njm", + "5njp", + "5njq", + "5njr", + "5njs", + "5o4w", + "5o4x", + "5o6q", + "5ob6", + "5ob7", + "5ob8", + "5ob9", + "5ocv", + "5oer", + "5ole", + "5oov", + "5ooy", + "5op1", + "5t3f", + "5tk0", + "5uvj", + "5v4g", + "5v4h", + "5v4i", + "5wr9", + "5wra", + "5wrb", + "5yin", + "5yky", + "6a10", + "6a4n", + "6a4o", + "6a4p", + "6a4q", + "6abn", + "6abz", + "6ac2", + "6ad5", + "6adf", + "6aea", + "6agn", + "6agr", + "6ahh", + "6ahl", + "6apm", + "6b7u", + "6b7v", + "6b7w", + "6bo1", + "6bo2", + "6bre", + "6ciw", + "6d6e", + "6d6f", + "6d6g", + "6d6h", + "6f1l", + "6f1m", + "6f1o", + "6f1p", + "6f1r", + "6f2i", + "6f2j", + "6f2k", + "6f9x", + "6f9y", + "6f9z", + "6fa0", + "6fro", + "6frq", + "6fsj", + "6ftr", + "6g5c", + "6g5v", + "6g5y", + "6g8a", + "6gf0", + "6gnl", + "6gob", + "6goh", + "6goi", + "6goj", + "6h0k", + "6h0l", + "6h3b", + "6h79", + "6ht2", + "6hu5", + "6hy4", + "6hy6", + "6hy8", + "6hyb", + "6ig6", + "6irj", + "6jb2", + "6jb5", + "6jb8", + "6jxp", + "6jxq", + "6jzi", + "6k5q", + "6k8g", + "6kcb", + "6kcd", + "6kd1", + "6lav", + "6ll3", + "6lt5", + "6lyt", + "6lyz", + "6mqv", + "6muz", + "6mx9", + "6o2h", + "6p4a", + "6p4b", + "6p4d", + "6pbb", + "6pl9", + "6pla", + "6plb", + "6q88", + "6q8r", + "6q8t", + "6qea", + "6qnb", + "6qq3", + "6qqc", + "6qqd", + "6qqe", + "6qqf", + "6qww", + "6qwx", + "6qwy", + "6qwz", + "6qx0", + "6qxw", + "6qxx", + "6qxy", + "6qy0", + "6qy1", + "6qy2", + "6qy4", + "6qy5", + "6rnb", + "6rnc", + "6ro3", + "6ro5", + "6rt1", + "6rt3", + "6rt9", + "6rta", + "6rxi", + "6s2n", + "6s7n", + "6set", + "6seu", + "6sew", + "6sex", + "6sez", + "6sij", + "6sik", + "6sil", + "6sim", + "6sr0", + "6sr1", + "6sr2", + "6sr3", + "6sr4", + "6sr5", + "6syc", + "6syd", + "6sye", + "6tvl", + "6tvy", + "6w7p", + "6w8e", + "6web", + "6wec", + "6wgo", + "6wrl", + "6wrm", + "6wx5", + "6ybf", + "6ybi", + "6yjx", + "6yob", + "6yoc", + "6yod", + "6yoe", + "6zs8", + "7a70", + "7ac2", + "7ave", + "7avf", + "7avg", + "7b9j", + "7baz", + "7bb1", + "7bcu", + "7bcx", + "7bd0", + "7bdm", + "7bdz", + "7be0", + "7be1", + "7be2", + "7beb", + "7bec", + "7bhk", + "7bhl", + "7bhm", + "7bhn", + "7bmo", + "7bmp", + "7bmq", + "7bmr", + "7bms", + "7bmt", + "7br5", + "7bvm", + "7bvo", + "7byo", + "7byp", + "7c09", + "7cdk", + "7cdm", + "7cdn", + "7cdo", + "7cdp", + "7cdq", + "7cdr", + "7cds", + "7cdt", + "7cdu", + "7cjz", + "7cqo", + "7cvj", + "7cvl", + "7d01", + "7d02", + "7d04", + "7d05", + "7d0w", + "7deq", + "7der", + "7dln", + "7dtb", + "7dtf", + "7e02", + "7f26", + "7fcu", + "7fcw", + "7fg8", + "7fgu", + "7fgv", + "7jmu", + "7kh5", + "7knk", + "7l84", + "7llp", + "7ln8", + "7ln9", + "7loq", + "7lor", + "7lp6", + "7lpl", + "7lpm", + "7lyz", + "7mrp", + "7njf", + "7nkf", + "7o2o", + "7ol5", + "7ol6", + "7ol7", + "7ol8", + "7p6m", + "7pnh", + "7q04", + "7q05", + "7q06", + "7q0t", + "7q0u", + "7q0v", + "7qnp", + "7qq1", + "7r1q", + "7rxy", + "7ryd", + "7ryk", + "7rz0", + "7rz1", + "7rz2", + "7s27", + "7s28", + "7s29", + "7s2a", + "7s2b", + "7s2c", + "7s2d", + "7s2e", + "7s2f", + "7s2g", + "7s2q", + "7s2u", + "7s2v", + "7s2w", + "7s30", + "7s31", + "7s32", + "7s33", + "7s34", + "7s35", + "7s4w", + "7skw", + "7tum", + "7uly", + "7vei", + "7vgo", + "7vgp", + "7wbd", + "7wbe", + "7wbf", + "7wkr", + "7wuc", + "7wxs", + "7y6a", + "7ynu", + "7ynv", + "7yrk", + "7z6j", + "7zu6", + "8a9d", + "8a9e", + "8a9r", + "8aaz", + "8aj3", + "8aj4", + "8aj5", + "8b3l", + "8b3t", + "8b3u", + "8b3v", + "8bov", + "8boy", + "8bph", + "8bpj", + "8bpu", + "8bqm", + "8bqp", + "8bqq", + "8bqr", + "8bqt", + "8c39", + "8cl5", + "8cl6", + "8cpc", + "8cvu", + "8cvv", + "8cvw", + "8cw0", + "8cw1", + "8cw3", + "8cw5", + "8cw6", + "8cw7", + "8cw8", + "8cwb", + "8cwc", + "8cwd", + "8cwe", + "8cwf", + "8cwg", + "8cwh", + "8d5s", + "8d5t", + "8d5u", + "8d5w", + "8d5x", + "8d5z", + "8d60", + "8d61", + "8d62", + "8d69", + "8d6b", + "8d6i", + "8d75", + "8d77", + "8d7a", + "8d7b", + "8d7c", + "8d7d", + "8d7j", + "8d7l", + "8d7q", + "8d7s", + "8d8a", + "8d8b", + "8d8c", + "8d8d", + "8d8e", + "8d8f", + "8d8g", + "8d8h", + "8dct", + "8dcu", + "8dcv", + "8dcw", + "8dyz", + "8dz7", + "8e54", + "8ezo", + "8ezp", + "8ezu", + "8ezx", + "8f00", + "8f0b", + "8f28", + "8f2g", + "8fp6", + "8fp7", + "8fp8", + "8fpb", + "8fpd", + "8fpm", + "8fpn", + "8fpp", + "8fpr", + "8fpu", + "8fry", + "8fs0", + "8fs9", + "8fsa", + "8fsc", + "8fsf", + "8fsg", + "8fsh", + "8fsm", + "8fsn", + "8fst", + "8fsu", + "8fsv", + "8fsw", + "8fsx", + "8fsy", + "8ft0", + "8ft1", + "8ft2", + "8ft3", + "8gmv", + "8gmw", + "8gq1", + "8h3w", + "8h8t", + "8h8u", + "8h8v", + "8h8w", + "8jqv", + "8k8m", + "8lyz", + "8om8", + "8oms", + "8omt", + "8owc", + "8p1a", + "8p1b", + "8p1c", + "8p1d", + "8p2q", + "8pft", + "8pfu", + "8pfv", + "8pfw", + "8pfx", + "8pfy", + "8ph5", + "8ph6", + "8ph7", + "8ph8", + "8piw", + "8pob", + "8pte", + "8qcu", + "8qv8", + "8reg", + "8reh", + "8rge", + "8rlf", + "8rlg", + "8rlh", + "8rli", + "8rnv", + "8rnw", + "8rnx", + "8rny", + "8rpm", + "8rsh", + "8rtj", + "8rtk", + "8rus", + "8s2u", + "8s2v", + "8scy", + "8sil", + "8w6k", + "8wdi", + "8wzf", + "8wzg", + "8wzr", + "8wzt", + "8wzv", + "8y1r", + "8y1s", + "8y9w", + "8ya1", + "8ya4", + "8ybg", + "8ybh", + "8z51", + "8z52", + "8zst", + "8zsu", + "9b7c", + "9b7e", + "9b7f", + "9c9h", + "9enz", + "9eo2", + "9evu", + "9ex0", + "9ex1", + "9ex2", + "9fa5", + "9fc3", + "9fc4", + "9fc5", + "9fc6", + "9fc7", + "9fc8", + "9fc9", + "9fca", + "9fcb", + "9fcc", + "9fgx", + "9fgy", + "9fmy", + "9fud", + "9fue", + "9gcv", + "9ghx", + "9gi7", + "9gog", + "9gok", + "9gyh", + "9h3e", + "9h7w", + "9hc2", + "9hlk", + "9hmk", + "9hmq", + "9hru", + "9hti", + "9htj", + "9i6j", + "9i6k", + "9i6l", + "9i6m", + "9i6n", + "9i6o", + "9i6p", + "9i8l", + "9ig6", + "9j0l", + "9j0m", + "9k9z", + "9ka0", + "9kj7", + "9kot", + "9l1l", + "9l71", + "9l9r", + "9lmk", + "9lml", + "9lyz", + "9mp3", + "9mp4", + "9mp5", + "9mp6", + "9mp7", + "9mp8", + "9mp9", + "9mpa", + "9orv", + "9orw", + "9orx", + "9ory", + "9orz", + "9os0", + "9os1", + "9os8", + "9oz5", + "9p0y", + "9p0z", + "9p10", + "9p12", + "9p13", + "9p1w", + "9p56", + "9p57", + "9p58", + "9p59", + "9p5a", + "9p5b", + "9p5d", + "9p5e", + "9p5f", + "9p5g", + "9p5q", + "9p5r", + "9p5s", + "9p5t", + "9p5u", + "9p5v", + "9p5w", + "9p5x", + "9p5y", + "9p5z", + "9p60", + "9p61", + "9p62", + "9p63", + "9p64", + "9p65", + "9p66", + "9p67", + "9p68", + "9p69", + "9p6r", + "9qa9", + "9qsq", + "9qtl", + "9qum", + "9rbg", + "9rbt", + "9rbv", + "9rrk", + "9rrl", + "9ruv", + "9rv6", + "9rvi", + "9rvo", + "9rzl", + "9rzr", + "9rzs", + "9rzt", + "9s2a", + "9sm9", + "9smc", + "9tbl", + "9tej", + "9tek", + "9toq", + "9tor", + "9tos", + "9tot", + "9ugi", + "9uyr", + "9v3d", + "9v3g", + "9v3h", + "9v3i", + "9v3j", + "9v3k", + "9v3l", + "9v3m", + "9v3n", + "9v3o", + "9v3p", + "9v3q", + "9v3r", + "9wks", + "9xua" + ], + "categories": [ + "load", + "representation", + "resolve" + ], + "skills": [ + "resolve", + "download", + "component", + "representation" + ], + "notes": "Starter chip; hundreds of valid entries \u2014 a control.", + "provenance": { + "uniprot": "P00698", + "n_entries": 1398, + "source": "RCSB search, reference_sequence_identifiers", + "fetched": "2026-09-30" + } +} diff --git a/tasks/mvs_resolve/res-myoglobin.json b/tasks/mvs_resolve/res-myoglobin.json new file mode 100644 index 0000000..3c7d0be --- /dev/null +++ b/tasks/mvs_resolve/res-myoglobin.json @@ -0,0 +1,427 @@ +{ + "id": "res-myoglobin", + "category": "mvs", + "source": "resolve", + "title": "Resolve by name: myoglobin", + "prompt": "Show me sperm whale myoglobin", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1mbn.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "accepted_ids": [ + "101m", + "102m", + "103m", + "104m", + "105m", + "106m", + "107m", + "108m", + "109m", + "110m", + "111m", + "112m", + "1a6g", + "1a6k", + "1a6m", + "1a6n", + "1abs", + "1ajg", + "1ajh", + "1bvc", + "1bvd", + "1bz6", + "1bzp", + "1bzr", + "1ch1", + "1ch2", + "1ch3", + "1ch5", + "1ch7", + "1ch9", + "1cik", + "1cio", + "1co8", + "1co9", + "1cp0", + "1cp5", + "1cpw", + "1cq2", + "1do1", + "1do3", + "1do4", + "1do7", + "1dti", + "1dtm", + "1duk", + "1duo", + "1dxc", + "1dxd", + "1ebc", + "1f63", + "1f65", + "1f6h", + "1fcs", + "1h1x", + "1hjt", + "1iop", + "1irc", + "1j3f", + "1j52", + "1jdo", + "1jp6", + "1jp8", + "1jp9", + "1jpb", + "1jw8", + "1l2k", + "1ltw", + "1lue", + "1mbc", + "1mbd", + "1mbi", + "1mbn", + "1mbo", + "1mcy", + "1mgn", + "1mlf", + "1mlg", + "1mlh", + "1mlj", + "1mlk", + "1mll", + "1mlm", + "1mln", + "1mlo", + "1mlq", + "1mlr", + "1mls", + "1mlu", + "1moa", + "1mob", + "1moc", + "1mod", + "1mti", + "1mtj", + "1mtk", + "1myf", + "1mym", + "1myz", + "1mz0", + "1n9f", + "1n9h", + "1n9i", + "1n9x", + "1naz", + "1o16", + "1obm", + "1ofj", + "1ofk", + "1spe", + "1swm", + "1tes", + "1u7r", + "1u7s", + "1ufj", + "1ufp", + "1v9q", + "1vxa", + "1vxb", + "1vxc", + "1vxd", + "1vxe", + "1vxf", + "1vxg", + "1vxh", + "1wvp", + "1yog", + "1yoh", + "1yoi", + "2blh", + "2bli", + "2blj", + "2bw9", + "2bwh", + "2cmm", + "2d6c", + "2e2y", + "2eb8", + "2eb9", + "2ef2", + "2ekt", + "2eku", + "2evk", + "2evp", + "2g0r", + "2g0s", + "2g0v", + "2g0x", + "2g0z", + "2g10", + "2g11", + "2g12", + "2g14", + "2jho", + "2mb5", + "2mbw", + "2mga", + "2mgb", + "2mgc", + "2mgd", + "2mge", + "2mgf", + "2mgg", + "2mgh", + "2mgi", + "2mgj", + "2mgk", + "2mgl", + "2mgm", + "2mya", + "2myb", + "2myc", + "2myd", + "2mye", + "2oh8", + "2oh9", + "2oha", + "2ohb", + "2spl", + "2spm", + "2spn", + "2spo", + "2w6w", + "2w6x", + "2w6y", + "2z6s", + "2z6t", + "2zsn", + "2zso", + "2zsp", + "2zsq", + "2zsr", + "2zss", + "2zst", + "2zsx", + "2zsy", + "2zsz", + "2zt0", + "2zt1", + "2zt2", + "2zt3", + "2zt4", + "3a2g", + "3ase", + "3e4n", + "3e55", + "3e5i", + "3e5o", + "3ecl", + "3ecx", + "3ecz", + "3ed9", + "3eda", + "3edb", + "3h57", + "3h58", + "3k9z", + "3m38", + "3m39", + "3m3a", + "3m3b", + "3mn0", + "3nml", + "3o89", + "3ogb", + "3sdn", + "3u3e", + "4fwx", + "4fwy", + "4fwz", + "4h07", + "4h0b", + "4it8", + "4lpi", + "4mbn", + "4mxk", + "4mxl", + "4nxa", + "4nxc", + "4of9", + "4ood", + "4pnj", + "4pq6", + "4pqb", + "4pqc", + "4qau", + "4tyx", + "5b84", + "5b85", + "5c6y", + "5hav", + "5hlq", + "5hlu", + "5hlx", + "5iks", + "5ile", + "5ilm", + "5ilp", + "5ilr", + "5jom", + "5kd1", + "5kkk", + "5m3s", + "5mbn", + "5o41", + "5oj9", + "5oja", + "5ojb", + "5ojc", + "5ut7", + "5ut8", + "5ut9", + "5uta", + "5utb", + "5utc", + "5utd", + "5vnu", + "5vrt", + "5vzn", + "5vzo", + "5vzp", + "5vzq", + "5wjk", + "5xkv", + "5xkw", + "5xl0", + "5yce", + "5ych", + "5yzf", + "5zeo", + "5zzf", + "5zzg", + "6cf0", + "6d45", + "6e02", + "6e03", + "6e04", + "6f17", + "6f18", + "6f19", + "6f1a", + "6g5a", + "6g5b", + "6g5t", + "6jp1", + "6krc", + "6krf", + "6m8f", + "6mv0", + "6n02", + "6n03", + "6z4r", + "6z4t", + "7a44", + "7a45", + "7cen", + "7cez", + "7ehx", + "7kyr", + "7l3u", + "7l3y", + "7slh", + "7sli", + "7spe", + "7spf", + "7spg", + "7sph", + "7vdn", + "7vuc", + "7vw4", + "7xc9", + "7xcf", + "7xcq", + "7ylk", + "8eko", + "8ess", + "8esu", + "8f9h", + "8f9i", + "8f9j", + "8f9n", + "8fb0", + "8fdj", + "8j4k", + "8j4l", + "8kfh", + "8kfi", + "8kfj", + "8qba", + "8qbc", + "8xbj", + "8xc2", + "9dth", + "9dti", + "9ijo", + "9lmb", + "9p1e", + "9p1f" + ], + "categories": [ + "load", + "representation", + "resolve" + ], + "skills": [ + "resolve", + "download", + "component", + "representation" + ], + "notes": "Corpus prompt asked for 'crab myoglobin' (no such entry); sperm whale is the classic.", + "provenance": { + "uniprot": "P02185", + "n_entries": 357, + "source": "RCSB search, reference_sequence_identifiers", + "fetched": "2026-09-30" + } +} diff --git a/tasks/mvs_resolve/res-pcsk9.json b/tasks/mvs_resolve/res-pcsk9.json new file mode 100644 index 0000000..4712907 --- /dev/null +++ b/tasks/mvs_resolve/res-pcsk9.json @@ -0,0 +1,136 @@ +{ + "id": "res-pcsk9", + "category": "mvs", + "source": "resolve", + "title": "Resolve by name: pcsk9", + "prompt": "show the structure of PCSK9", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/2p4e.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "accepted_ids": [ + "10sb", + "2p4e", + "2pmw", + "2qtw", + "2w2m", + "2w2n", + "2w2o", + "2w2p", + "2w2q", + "2xtj", + "37hf", + "3bps", + "3gcw", + "3gcx", + "3h42", + "3m0c", + "3p5b", + "3p5c", + "3sqo", + "4k8r", + "4ne9", + "4nmx", + "4ov6", + "5oca", + "5vl7", + "5vla", + "5vlh", + "5vlk", + "5vll", + "5vlp", + "6e4y", + "6e4z", + "6mv5", + "6olz", + "6om0", + "6om7", + "6u26", + "6u2f", + "6u2n", + "6u2p", + "6u36", + "6u38", + "6u3i", + "6u3x", + "6xib", + "6xic", + "6xid", + "6xie", + "6xif", + "7anq", + "7kev", + "7kfa", + "7s5g", + "7s5h", + "8fpo", + "8fpq", + "8fvl", + "8fvm", + "8fvn", + "8fvo", + "8fvp", + "8fvq", + "8vdv", + "8wfr", + "9hz3", + "9szy" + ], + "categories": [ + "load", + "representation", + "resolve" + ], + "skills": [ + "resolve", + "download", + "component", + "representation" + ], + "notes": "Probe: Haiku answered 1D0G (death receptor 5 / TRAIL).", + "provenance": { + "uniprot": "Q8NBP7", + "n_entries": 66, + "source": "RCSB search, reference_sequence_identifiers", + "fetched": "2026-09-30" + } +} diff --git a/tasks/mvs_resolve/res-pde5a.json b/tasks/mvs_resolve/res-pde5a.json new file mode 100644 index 0000000..8b6c508 --- /dev/null +++ b/tasks/mvs_resolve/res-pde5a.json @@ -0,0 +1,120 @@ +{ + "id": "res-pde5a", + "category": "mvs", + "source": "resolve", + "title": "Resolve by name: pde5a", + "prompt": "I wanna see a structure of PDE5A", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1udt.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "accepted_ids": [ + "1rkp", + "1t9r", + "1t9s", + "1tbf", + "1udt", + "1udu", + "1uho", + "1xoz", + "1xp0", + "2chm", + "2h40", + "2h42", + "2h44", + "2xss", + "3b2r", + "3bjc", + "3hc8", + "3hdz", + "3jwq", + "3jwr", + "3lfv", + "3mf0", + "3shy", + "3shz", + "3sie", + "3tge", + "3tgg", + "4g2w", + "4g2y", + "4i9z", + "4ia0", + "4md6", + "4oew", + "4oex", + "5jo3", + "5zz2", + "6acb", + "6iwi", + "6l6e", + "6vbi", + "7faq", + "7far", + "8w4s", + "8w4t", + "8wkf", + "8wkg", + "8xww", + "9lsl", + "9lsm", + "9lss" + ], + "categories": [ + "load", + "representation", + "resolve" + ], + "skills": [ + "resolve", + "download", + "component", + "representation" + ], + "notes": "Evaluator prompt (2026-09-30). Haiku answered 1UJ7, which does not exist.", + "provenance": { + "uniprot": "O76074", + "n_entries": 50, + "source": "RCSB search, reference_sequence_identifiers", + "fetched": "2026-09-30" + } +} diff --git a/tests/test_harness.py b/tests/test_harness.py index c64dd94..5a89d16 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -196,3 +196,60 @@ def test_run_archives_raw_samples_and_meta(): test_render_state_wrapping() test_run_archives_raw_samples_and_meta() print("all sanity checks passed") + + +# --- entry resolution (tasks/mvs_resolve) ------------------------------------------ + +def _resolve_tasks(): + return [t for t in load_tasks(["mvs"]) if t.get("source") == "resolve"] + + +def _with_pdb(task: dict, pdb: str) -> dict: + """The task's own reference scene, downloading a different entry.""" + tree = json.loads(json.dumps(task["reference_mvs"])) + tree["children"][0]["params"]["url"] = f"https://files.rcsb.org/download/{pdb}.cif" + return tree + + +def test_resolve_tasks_have_a_valid_accepted_set(): + tasks = _resolve_tasks() + assert tasks, "no resolve tasks found" + for task in tasks: + canonical = task["reference_mvs"]["children"][0]["params"]["url"].rsplit("/", 1)[-1][:4] + assert canonical in task["accepted_ids"], f"{task['id']}: canonical id not in accepted_ids" + assert len(task["accepted_ids"]) > 1, f"{task['id']}: a one-entry set defeats the point" + + +def test_any_accepted_entry_scores_as_the_reference(): + task = next(t for t in _resolve_tasks() if t["id"] == "res-pde5a") + other = next(i for i in task["accepted_ids"] if i != "1udt") # e.g. 1tbf, PDE5 + tadalafil + result = mvs.grade_mvs(task["reference_mvs"], _with_pdb(task, other), + accepted_refs=task["accepted_ids"]) + assert result["f1"] == 1.0, result + + +def test_made_up_entry_is_penalised(): + """The evaluator case: 'a structure of PDE5A' answered with PDB 1UJ7, which does not exist.""" + task = next(t for t in _resolve_tasks() if t["id"] == "res-pde5a") + assert "1uj7" not in task["accepted_ids"] + wrong = mvs.grade_mvs(task["reference_mvs"], _with_pdb(task, "1uj7"), + accepted_refs=task["accepted_ids"]) + right = mvs.grade_mvs(task["reference_mvs"], _with_pdb(task, "1udt"), + accepted_refs=task["accepted_ids"]) + assert right["f1"] == 1.0 + assert wrong["f1"] < 1.0, wrong + # Without an accepted set the same wrong id scores the same: the fold only ever helps. + assert mvs.grade_mvs(task["reference_mvs"], _with_pdb(task, "1uj7"))["f1"] == wrong["f1"] + + +def test_accepted_fold_ignores_multi_structure_references(): + ref = {"kind": "root", "children": [ + {"kind": "download", "params": {"url": "https://files.rcsb.org/download/1hho.cif"}}, + {"kind": "download", "params": {"url": "https://files.rcsb.org/download/2hhb.cif"}}, + ]} + pred = {"kind": "root", "children": [ + {"kind": "download", "params": {"url": "https://files.rcsb.org/download/1a3n.cif"}}, + {"kind": "download", "params": {"url": "https://files.rcsb.org/download/2hhb.cif"}}, + ]} + # Two canonical ids → nothing to fold onto; grading is unchanged. + assert mvs.grade_mvs(ref, pred, accepted_refs=["1a3n"]) == mvs.grade_mvs(ref, pred) From 431268369a64c4211ad012c66d8e8bfb4276c489 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Wed, 30 Sep 2026 13:14:13 +0100 Subject: [PATCH 02/10] Address review: run resolve tests from the CI entry point; survive a download node without params Co-Authored-By: Claude Fable 5.1 --- molbench/mvs.py | 15 ++++++++--- tests/test_harness.py | 60 +++++++++++++++++++++++++++---------------- 2 files changed, 49 insertions(+), 26 deletions(-) diff --git a/molbench/mvs.py b/molbench/mvs.py index 3499e74..dfe4044 100644 --- a/molbench/mvs.py +++ b/molbench/mvs.py @@ -170,6 +170,15 @@ def _path_similarity(p: tuple, q: tuple) -> float: # --- grading ---------------------------------------------------------------------- +def _download_id(seg: tuple) -> str | None: + """The structure id in a ``download`` path segment, or None (a download node with + no/malformed params has an empty signature — a model can emit that).""" + kind, sig = seg + if kind == "download" and isinstance(sig, tuple) and len(sig) == 2 and sig[0] == "ref": + return sig[1] + return None + + def _fold_accepted_refs(ref_paths: list[tuple], pred_paths: list[tuple], accepted: set[str]) -> list[tuple]: """Treat any accepted structure id in the prediction as the reference's own. @@ -180,16 +189,14 @@ def _fold_accepted_refs(ref_paths: list[tuple], pred_paths: list[tuple], before matching. An id outside the set (a made-up one, say) stays as it is and mismatches the download segment like any other wrong answer. """ - ref_ids = {seg[1][1] for path in ref_paths for seg in path - if seg[0] == "download" and isinstance(seg[1], tuple) and seg[1][0] == "ref"} + ref_ids = {_download_id(seg) for path in ref_paths for seg in path} - {None} if len(ref_ids) != 1: return pred_paths # several structures: no single canonical id to fold onto canonical = next(iter(ref_ids)) accepted = {a.lower() for a in accepted} def fold(seg: tuple) -> tuple: - if seg[0] == "download" and isinstance(seg[1], tuple) and seg[1][0] == "ref" \ - and seg[1][1] in accepted: + if _download_id(seg) in accepted: return ("download", ("ref", canonical)) return seg diff --git a/tests/test_harness.py b/tests/test_harness.py index 5a89d16..8c32069 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -176,28 +176,6 @@ def test_run_archives_raw_samples_and_meta(): assert "raw" in tasks[tid][0], "raw model output must be archived" -if __name__ == "__main__": - test_all_reference_answers_are_schema_valid() - test_self_grading_is_perfect() - test_wrong_prediction_scores_low() - test_partial_credit_for_right_residues_wrong_colour() - test_load_modifiers_are_scored() - test_tooltip_text_is_scored() - test_sidechain_flag_is_scored() - test_mvs_self_grading_is_perfect() - test_mvs_is_metadata_insensitive() - test_mvs_partial_credit_for_wrong_colour() - test_mvs_grey_gray_fold() - test_mvs_non_tree_scores_zero() - test_mvs_grader_survives_non_dict_params() - test_mvs_download_variant_is_normalised() - test_mvs_downstream_credit_survives_upstream_mismatch() - test_escalate_tree_tier() - test_render_state_wrapping() - test_run_archives_raw_samples_and_meta() - print("all sanity checks passed") - - # --- entry resolution (tasks/mvs_resolve) ------------------------------------------ def _resolve_tasks(): @@ -253,3 +231,41 @@ def test_accepted_fold_ignores_multi_structure_references(): ]} # Two canonical ids → nothing to fold onto; grading is unchanged. assert mvs.grade_mvs(ref, pred, accepted_refs=["1a3n"]) == mvs.grade_mvs(ref, pred) + + +def test_accepted_fold_survives_download_without_params(): + """A model may emit a download node with no params; the fold must not crash on it.""" + task = next(t for t in _resolve_tasks() if t["id"] == "res-pde5a") + pred = {"kind": "root", "children": [{"kind": "download", "children": [ + {"kind": "parse", "params": {"format": "mmcif"}}]}]} + result = mvs.grade_mvs(task["reference_mvs"], pred, accepted_refs=task["accepted_ids"]) + assert 0.0 <= result["f1"] < 1.0 + bare_ref = {"kind": "root", "children": [{"kind": "download"}]} + assert mvs.grade_mvs(bare_ref, pred, accepted_refs=["1udt"])["f1"] >= 0.0 + + +if __name__ == "__main__": + test_all_reference_answers_are_schema_valid() + test_self_grading_is_perfect() + test_wrong_prediction_scores_low() + test_partial_credit_for_right_residues_wrong_colour() + test_load_modifiers_are_scored() + test_tooltip_text_is_scored() + test_sidechain_flag_is_scored() + test_mvs_self_grading_is_perfect() + test_mvs_is_metadata_insensitive() + test_mvs_partial_credit_for_wrong_colour() + test_mvs_grey_gray_fold() + test_mvs_non_tree_scores_zero() + test_mvs_grader_survives_non_dict_params() + test_mvs_download_variant_is_normalised() + test_mvs_downstream_credit_survives_upstream_mismatch() + test_escalate_tree_tier() + test_render_state_wrapping() + test_run_archives_raw_samples_and_meta() + test_resolve_tasks_have_a_valid_accepted_set() + test_any_accepted_entry_scores_as_the_reference() + test_made_up_entry_is_penalised() + test_accepted_fold_ignores_multi_structure_references() + test_accepted_fold_survives_download_without_params() + print("all sanity checks passed") From 4e4e40be3ba5ac945fb57039b0f4b3ea50d4e2ca Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Sun, 4 Oct 2026 16:53:55 +0100 Subject: [PATCH 03/10] Spec reference v2: primitives, opacity, interactions; bare/spec conditions The MVS reference listed 8 node kinds (one wrong), so no model could draw an H-bond. It now covers MVS as Mol* 5.9.0 implements it: primitives (distance_measurement, dashed tube, angle, arrow, 3D label), opacity, tooltip, camera, canvas, transform, the full representation list, and the Mol* custom keys for computed non-covalent interactions and colour themes. - Grader: a primitive is keyed on its shape and the atoms it joins (styling and direction ignored); the interactions flag and colour theme are graded. - tasks/mvs_interactions: 6 tasks (H-bonds, Fe-His distance, catalytic triad, computed interactions, transparent surface). Every atom is resolved against the mmCIF with gemmi before a task is written. - runner --condition bare|spec, recorded in run meta. - ROADMAP: revised plan (2x2 reference x grounding, closed-book core, chat driver owns the tuned prompt). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017XGmcrKZRD9R8nqM8HDUaY --- .gitignore | 3 + ROADMAP.md | 74 ++++++++- molbench/mvs.py | 39 ++++- molbench/mvs_reference.md | 104 ++++++++++--- molbench/runner.py | 24 ++- scripts/author_interaction_tasks.py | 229 ++++++++++++++++++++++++++++ tasks/mvs_interactions/int_001.json | 148 ++++++++++++++++++ tasks/mvs_interactions/int_002.json | 148 ++++++++++++++++++ tasks/mvs_interactions/int_003.json | 197 ++++++++++++++++++++++++ tasks/mvs_interactions/int_004.json | 85 +++++++++++ tasks/mvs_interactions/int_005.json | 85 +++++++++++ tasks/mvs_interactions/int_006.json | 89 +++++++++++ tests/test_harness.py | 55 +++++++ 13 files changed, 1244 insertions(+), 36 deletions(-) create mode 100644 scripts/author_interaction_tasks.py create mode 100644 tasks/mvs_interactions/int_001.json create mode 100644 tasks/mvs_interactions/int_002.json create mode 100644 tasks/mvs_interactions/int_003.json create mode 100644 tasks/mvs_interactions/int_004.json create mode 100644 tasks/mvs_interactions/int_005.json create mode 100644 tasks/mvs_interactions/int_006.json diff --git a/.gitignore b/.gitignore index a9c84d6..d663cce 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ runs/ # Fetched Mol* bundle (run scripts/fetch_molstar.py) molbench/static/molstar/ + +# mmCIF cache for task authoring (scripts/author_interaction_tasks.py) +data/cif/ diff --git a/ROADMAP.md b/ROADMAP.md index 249b5e5..381bb40 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -7,6 +7,66 @@ sharpen Component 1, and validate the instrument. --- +## Revised plan (2026-10-04) — read this first + +Three decisions, each backed by a test this round. + +**1. MVS stays the target. The bottleneck was our reference, not MVS.** +The chat driver told a user "MVS cannot show hydrogen bonds". Checked against the +Mol\* 5.9.0 source and real renders, that is wrong: `primitives` draws dashed lines +between named atoms (`distance_measurement`, labelled with the length), and Mol\* +computes H-bonds/metal contacts itself via `custom.molstar_show_non_covalent_interactions` +on a component. Our reference listed only 8 node kinds (one of them wrong), so no +model could ever draw an H-bond. A 4-prompt × 2-model A/B: old reference 0/4 +lines drawn and 2 Mol\* crashes on "50% transparent"; new reference 4/4 lines and +2/2 transparency. Models still pick wrong chains/atoms (see grounding below). +What MVS truly cannot do: select by distance or property, compute a +superposition, style the computed interactions. Those stay in the API track. + +**2. Prompting conditions: a 2×2, no public "tuned" row.** + +| | no grounding | grounding pack | +|---|---|---| +| **bare** (output contract only) | what the model knows | knows facts, not MVS | +| **spec** (vendored MVS reference) | **headline** | upper bound for an offline pipeline | + +* *Spec* = `molbench/mvs_reference.md`, a neutral, complete snapshot of MVS as + Mol\* 5.9.0 implements it. `--condition bare|spec` selects the row (done). +* *Grounding pack* = facts about the task's structure prepared **offline** and + frozen with the task: chain ids and residue ranges, ligand codes and atom names, + SIFTS/UniProt mapping, ClinVar positions, the accepted entry ids. Reproducible + by construction; no live calls during a run. Can be ablated separately + (MVS-reference vs molecular-grounding). +* The chat driver's own prompt is **tuned** on everything it sees and is owned by + `molstar-chat-driver`. It is not scored as a model comparison. Product claims + come from real user prompts that arrive *after* a prompt version is frozen + (graded by thumbs up/down or the C2 judge). Curated user prompts become new + benchmark tasks — after which they are dev data for the next prompt version. + +**3. External resources (PubMed, PDB, UniProt, ClinVar): closed-book core.** +Lookups enter only as the frozen grounding pack above. Build a live tool loop +only if grounding lifts the lookup-heavy tasks (`mvs_clinical`, `mvs_resolve`) +by ≥10 F1 points on a few cheap models. PubMed is out of scope. This keeps +MolBench a visualization benchmark that *measures* the value of lookups rather +than an API-calling benchmark. + +### Work list + +| # | item | status | +|---|---|---| +| R1 | Spec reference v2 (primitives, opacity, interactions, colour themes, fixed rep types) | done | +| R2 | Grader: primitives keyed on shape + atom pair (styling/direction ignored); interactions flag and colour theme graded | done | +| R3 | `tasks/mvs_interactions/` — 6 tasks, every atom checked against the mmCIF with gemmi (`scripts/author_interaction_tasks.py`) | done | +| R4 | `--condition bare\|spec` in the runner, recorded in run meta | done | +| R5 | Regression run: old vs spec vs bare on 3 cheap models, all MVS tasks | blocked (API credit) | +| R6 | Grounding packs: offline builder (gemmi + SIFTS + existing resolve/clinical provenance) and `--grounding` flag | todo | +| R7 | Grader: resolve selections to atom sets against the mmCIF, so equivalent namings (`label_comp_id`+atom vs residue number+atom) match and empty selections score zero | todo | +| R8 | Chat driver v2: new reference in its prompt + empty-selection lint (a primitive end that matches nothing is drawn to the origin by Mol\*) | in PR | +| R9 | Chat driver: thumbs up/down per turn, for grading real traffic | todo | +| R10 | Leaderboard: a regime for `source: interactions` (unknown sources currently fall into "Clinical") | todo | + +--- + ## Phase A — Component 2: render + VLM judge ← the headline Open-ended requests ("show the key hydrogen bond", "highlight the allosteric @@ -74,7 +134,8 @@ strong judge, structured protocol, self-variance reporting); rubric quality ~0.5–0.67 because the `polymer cartoon` scaffold dominates F1. Add a metric that isolates "did you select the right residues" from scene scaffolding, and report it for the grounded/clinical tiers. Re-score the clinical regime. -* **B2. Interaction/interface task type** (ChatMol-inspired). Use gemmi neighbour +* **B2. Interaction/interface task type** (ChatMol-inspired; named-atom H-bond and + computed-interaction tasks STARTED, see R3). Use gemmi neighbour search to extract interface/contact residues (ligand–protein, protein–protein), giving grounded "show the binding-site residues" / "show the PPI interface" tasks — a realistic, high-value class we currently lack. @@ -85,11 +146,10 @@ strong judge, structured protocol, self-variance reporting); rubric quality protein's UniProt accession (`accepted_ids`, folded by the grader). Next: more targets, and an `exists` sub-metric (did the id resolve at RCSB at all?) — a hallucinated id and a wrong-but-real id are different failures. -* **B5. Colour schemes.** The chat-driver's prompt now teaches Mol*'s - `custom.molstar_color_theme_name` on a `color` node for rainbow / by-chain / - by-SS colouring (models kept writing `"spectrum"` as a colour). Porting that here - needs the grader to treat a theme-coloured representation as equivalent to the - per-chain component references it would replace; until then the two prompts differ. +* **B5. Colour schemes (PARTLY DONE).** The Spec reference now teaches Mol*'s + `custom.molstar_color_theme_name` on a `color` node, and the grader scores the + theme instead of the placeholder colour. Still missing: treating a theme-coloured + representation as equivalent to the per-chain components it would replace. * **B6. Structured outputs.** Constrained decoding (`response_format: json_schema`, `strict`) with a trimmed MVS schema makes invalid trees impossible by construction. Measure validation failures + F1 against the prompt-only path on @@ -147,6 +207,8 @@ The grader is a system with its own error rate; validate it against ground truth ## Suggested sequence +*Superseded in part by the revised plan at the top: R5 → R6 → R7 come first.* + 1. **A0 renderer** (unblocks everything visual) → **A1 judge** → **A2 wire-in** → **A3 corpus** → **A4 validation**. This delivers a working Component 2. 2. In parallel / between: **B1 selection-accuracy** (cheap, sharpens current diff --git a/molbench/mvs.py b/molbench/mvs.py index dfe4044..c7574ca 100644 --- a/molbench/mvs.py +++ b/molbench/mvs.py @@ -70,10 +70,30 @@ def _color_signature(value: Any) -> Any: return value -def _param_signature(kind: str, params: dict | None) -> Any: +_PRIMITIVE_POINTS = ("start", "end", "position", "a", "b", "c") + + +def _param_signature(kind: str, params: dict | None, custom: dict | None = None) -> Any: params = params or {} + custom = custom if isinstance(custom, dict) else {} if kind == "component" and "selector" in params: - return ("selector", _selector_signature(params["selector"])) + sig = ("selector", _selector_signature(params["selector"])) + # Mol* computes and draws the interactions itself when this flag is set, so + # it is part of what the component shows, not a styling detail. + if custom.get("molstar_show_non_covalent_interactions"): + sig += ("interactions",) + return sig + if kind == "primitive": + # Key on the shape and the atoms it joins. Radius, dash length and label text + # are styling; a line is the same line whichever end the model starts from. + points = frozenset(_selector_signature(params[k]) for k in _PRIMITIVE_POINTS + if k in params) + return ("primitive", params.get("kind"), points) + if kind == "primitives": + return () # a grouping node; its colour is styling + if kind == "color" and custom.get("molstar_color_theme_name"): + # The theme overrides the placeholder colour, so grade the theme. + return ("theme", custom["molstar_color_theme_name"]) if kind == "representation": # Key on the representation *type* only. The builder injects secondary # defaults (e.g. surface_type='molecular') that a model shouldn't be @@ -126,7 +146,7 @@ def walk(node: dict, prefix: tuple) -> None: and len(params["selector"]) > 1: variants = [("selector", _selector_signature(e)) for e in params["selector"]] else: - variants = [_param_signature(kind, params)] + variants = [_param_signature(kind, params, node.get("custom"))] for vsig in variants: here = prefix + ((kind, vsig),) @@ -263,14 +283,17 @@ def categorize(root: dict) -> list[str]: kinds: dict[str, int] = {} has_expr = False reps: set[str] = set() + interactions = False def walk(node: dict) -> None: - nonlocal has_expr + nonlocal has_expr, interactions k = node.get("kind") kinds[k] = kinds.get(k, 0) + 1 params = node.get("params") or {} if k == "component" and not isinstance(params.get("selector"), str): has_expr = True + if k == "component" and (node.get("custom") or {}).get("molstar_show_non_covalent_interactions"): + interactions = True if k == "representation": reps.add(params.get("type")) for c in node.get("children") or []: @@ -295,7 +318,13 @@ def walk(node: dict) -> None: cats.append("annotation") if reps & {"surface", "isosurface", "gaussian-surface", "molecular-surface"}: cats.append("surface") - if kinds.get("volume") or kinds.get("primitives"): + if kinds.get("opacity"): + cats.append("transparency") + if kinds.get("primitive"): + cats.append("measurement") + if interactions: + cats.append("interactions") + if kinds.get("volume"): cats.append("volume") return cats diff --git a/molbench/mvs_reference.md b/molbench/mvs_reference.md index 801348b..80d8d2a 100644 --- a/molbench/mvs_reference.md +++ b/molbench/mvs_reference.md @@ -1,12 +1,15 @@ # MolViewSpec (MVS) — scene-tree reference (vendored) -> **Frozen snapshot** targeting the `molviewspec` package (v1.8.x) and the MVS -> state format rendered by `ipymolstar.MolViewSpec`. This is both the context the -> model-under-test receives and the structure the grader trusts. +> **Frozen snapshot** of the MVS scene tree as implemented by Mol\* 5.9.0 (the +> version vendored in `molbench/static/molstar/`). This is both the context the +> model-under-test receives (the **Spec** condition) and the structure the grader +> trusts. It covers the structure-scene subset of MVS; volumes, trajectories, +> annotation files (`*_from_uri`/`*_from_source`) and multi-snapshot animation are +> left out. -MVS describes a 3D molecular scene as a **tree of nodes**, independent of any -specific viewer. A scene is built by nesting: you download data, parse it, derive -a structure, carve out components, and give each a representation and colour. +MVS describes a 3D molecular scene as a **tree of nodes**. A scene is built by +nesting: you download data, parse it, derive a structure, carve out components, +and give each a representation and colour. ## State shape @@ -22,20 +25,28 @@ a structure, carve out components, and give each a representation and colour. Every node is `{"kind": , "params": {<...>}, "children": [, ...]}` (`params`/`children` omitted when empty). Nesting encodes meaning — a `color` node is a child of the `representation` it colours, which is a child of the `component` -it draws. +it draws. A node may also carry `"custom": {...}`: viewer-specific extras (see +*Mol\* extensions* below). -## Node kinds (the ones this benchmark uses) +## Node kinds -| kind | params | typical parent | meaning | +| kind | params | parent | meaning | |---|---|---|---| | `download` | `url` | root | fetch a structure file | | `parse` | `format` (`mmcif`,`bcif`,`pdb`) | download | parse the data | -| `structure` | `type` (`model`,`assembly`) | parse | build a structure | +| `structure` | `type` (`model`,`assembly`,`symmetry`), `assembly_id`, `model_index` | parse | build a structure | +| `transform` | `rotation` (3×3, flat), `translation`, or `matrix` (4×4, flat) | structure | move it by a known matrix | | `component` | `selector` | structure | select a subset to draw | -| `representation` | `type` | component | how to draw it | -| `color` | `color` | representation | colour it | -| `focus` | *(none)* | component | aim the camera at it | -| `label` | `text` | component | attach a text label | +| `representation` | `type`, `size_factor` | component | how to draw it | +| `color` | `color`, optional `selector` | representation | colour it (or only the selected part) | +| `opacity` | `opacity` (0–1; 0.5 = 50% transparent) | representation | transparency | +| `label` | `text` | component | text label on the component | +| `tooltip` | `text` | component | hover text | +| `focus` | optional `radius` | component (or root) | aim the camera at it | +| `camera` | `target`, `position`, `up` | root | set the camera explicitly | +| `canvas` | `background_color` | root | background colour | +| `primitives` | optional `color`, `label_color`, `opacity` | structure | a group of drawn shapes | +| `primitive` | `kind` + params below | primitives | one shape | **Component selectors** — a string for whole classes: `all`, `polymer`, `protein`, `nucleic`, `ligand`, `ion`, `water`, `branched`. @@ -46,14 +57,21 @@ Or a **ComponentExpression** object (or list of them) to pick specific atoms/res | `auth_asym_id` | author chain id (e.g. `"A"`) | | `auth_seq_id` | author residue number | | `label_asym_id`, `label_seq_id` | label (mmCIF) chain / residue | -| `beg_auth_seq_id`, `end_auth_seq_id` | inclusive residue range | -| `label_comp_id` | residue/ligand 3-letter code (`"HEM"`) | -| `label_atom_id` | atom name (`"FE"`) | +| `beg_auth_seq_id`, `end_auth_seq_id` | inclusive residue range (also `beg_/end_label_seq_id`) | +| `label_comp_id`, `auth_comp_id` | residue/ligand 3-letter code (`"HEM"`) | +| `label_atom_id`, `auth_atom_id` | atom name (`"FE"`, `"NE2"`) | +| `type_symbol` | element (`"FE"`) | +| `residue_index` | 0-based residue index in the file | -**Representation types:** `cartoon`, `ball_and_stick`, `spacefill`, `surface`, -`carbohydrate`, `isosurface`. +Fields in one expression are AND-ed; a list of expressions is OR-ed. A selection +that matches nothing draws nothing — it is not an error, so use real chain ids, +residue numbers and atom names. -**Colours:** CSS names (`"red"`, `"orange"`, `"steelblue"`) or hex (`"#FF6699"`). +**Representation types:** `cartoon`, `backbone`, `ball_and_stick`, `line`, +`spacefill`, `surface`, `putty`, `carbohydrate`. + +**Colours:** lowercase CSS names (`"red"`, `"orange"`, `"steelblue"`) or hex +(`"#FF6699"`). A `color` node takes ONE colour. ## Worked example @@ -77,3 +95,49 @@ Prompt: *"Load 1HHO; protein as grey cartoon, heme groups as orange ball-and-sti To select specific residues, replace the selector string with an expression, e.g. `"selector": {"auth_asym_id": "A", "auth_seq_id": 35}`. + +**Transparency:** `{"kind": "opacity", "params": {"opacity": 0.5}}` as a child of +the `representation`, beside its `color`. + +## Primitives: lines, distances, angles, 3D text + +A primitive position (`start`, `end`, `position`, `a`/`b`/`c`) is either +`[x, y, z]` or a ComponentExpression naming **one atom**, e.g. +`{"auth_asym_id": "A", "auth_seq_id": 64, "label_atom_id": "NE2"}` +(if it matches several atoms, their centre is used). + +| primitive `kind` | params | draws | +|---|---|---| +| `distance_measurement` | `start`, `end`, `color`, `radius`, `dash_length`, `label_template` | a dashed line labelled with its length (`"{{distance}}"` is replaced by it) | +| `tube` | `start`, `end`, `radius`, `dash_length` | a line, dashed if `dash_length` is set | +| `angle_measurement` | `a`, `b`, `c` | the angle at `b`, labelled | +| `arrow` | `start`, `end`, `tube_radius`, `color` | an arrow | +| `label` | `position`, `text`, `label_size` | 3D text | + +Example — *"1A6M: show the distance from the distal His64 to the bound O2"* +(the ligand code is `OXY`). The `primitives` node is a child of `structure`, next +to its `component` nodes: + +```json +{"kind": "primitives", "params": {"color": "yellow", "label_color": "yellow"}, "children": [ + {"kind": "primitive", "params": {"kind": "distance_measurement", + "start": {"auth_asym_id": "A", "auth_seq_id": 64, "label_atom_id": "NE2"}, + "end": {"auth_asym_id": "A", "label_comp_id": "OXY", "label_atom_id": "O2"}, + "radius": 0.06, "dash_length": 0.15, "label_template": "{{distance}}"}}]} +``` + +## Mol\* extensions (`custom`) + +Mol\* reads these `custom` keys; other MVS viewers ignore them. + +* **Non-covalent interactions** (H-bonds, salt bridges, metal coordination, π + stacking…), computed by Mol\*: on a `component`, set + `"custom": {"molstar_show_non_covalent_interactions": true, + "molstar_non_covalent_interactions_radius_ang": 5}`. Mol\* draws the residues + within the radius as ball-and-stick plus dashed interaction lines coloured by + type. The component needs no `representation` child for this. +* **Colour schemes** (rainbow, by chain, by secondary structure, by element): give + the representation ONE `color` child carrying + `"custom": {"molstar_color_theme_name": }` (its `color` param is a + placeholder the theme overrides). Themes: `"sequence-id"`, `"chain-id"`, + `"secondary-structure"`, `"element-symbol"`. diff --git a/molbench/runner.py b/molbench/runner.py index 9a1783f..7d743a7 100644 --- a/molbench/runner.py +++ b/molbench/runner.py @@ -50,6 +50,11 @@ def _git_commit() -> str | None: API_REFERENCE = ROOT / "api_reference.md" MVS_REFERENCE = ROOT / "mvs_reference.md" +# Prompting conditions for the scene-tree track. 'spec' hands the model the vendored +# MVS reference (the headline number); 'bare' gives only the output contract, so the +# gap between them measures what a model gains from documentation. +CONDITIONS = ("spec", "bare") + # Categories whose tasks produce a numeric F1 (vs. visual_rubric, which is VLM-judged). GRADED_CATEGORIES = ("mvs", "api_calling") @@ -123,7 +128,7 @@ def load_tasks(categories: list[str] | None) -> list[dict[str, Any]]: return tasks -def build_system_prompts() -> dict[str, str]: +def build_system_prompts(condition: str = "spec") -> dict[str, str]: """One assembled system prompt per task category. Each target speaks a different IR, so each gets its own instructions + vendored @@ -133,7 +138,11 @@ def build_system_prompts() -> dict[str, str]: api_prompt = (PROMPT_API.read_text() .replace("{{API_REFERENCE}}", API_REFERENCE.read_text()) .replace("{{JSON_SCHEMA}}", json.dumps(schema.build_json_schema(), indent=2))) - mvs_prompt = PROMPT_MVS.read_text().replace("{{MVS_REFERENCE}}", MVS_REFERENCE.read_text()) + mvs_prompt = PROMPT_MVS.read_text() + if condition == "bare": + mvs_prompt = mvs_prompt.replace("## MVS reference\n\n{{MVS_REFERENCE}}", "").rstrip() + "\n" + else: + mvs_prompt = mvs_prompt.replace("{{MVS_REFERENCE}}", MVS_REFERENCE.read_text()) return {"api_calling": api_prompt, "mvs": mvs_prompt, "visual_rubric": mvs_prompt} @@ -232,7 +241,7 @@ def _aggregate(samples: list[dict]) -> dict[str, Any]: def run(models: list[str], categories: list[str] | None, samples: int = 1, escalate: bool = False, - judge: str | None = None) -> tuple[dict[str, Any], dict[str, Any]]: + judge: str | None = None, condition: str = "spec") -> tuple[dict[str, Any], dict[str, Any]]: """Returns (report, raw_samples). `report` is the lightweight scorecard (aggregates + best-sample prediction). @@ -250,7 +259,7 @@ def run(models: list[str], categories: list[str] | None, tasks = load_tasks(categories) if not tasks: sys.exit("no tasks matched; check tasks/ and --categories") - prompts = build_system_prompts() + prompts = build_system_prompts(condition) vlm = StubVLMJudge() report: dict[str, Any] = { @@ -262,6 +271,7 @@ def run(models: list[str], categories: list[str] | None, "categories": categories, "escalate": escalate, "judge": judge, + "condition": condition, }, } @@ -422,6 +432,9 @@ def main(argv: list[str] | None = None) -> None: ap.add_argument("--judge", default=None, help="vision model id for the T2 VLM tier (e.g. claude-opus-4-8); " "off by default (T0/T1 only)") + ap.add_argument("--condition", choices=CONDITIONS, default="spec", + help="MVS prompting condition: 'spec' (with the MVS reference) or " + "'bare' (output contract only)") ap.add_argument("--out", default=str(RESULTS_DIR / "scorecard.json")) ap.add_argument("--no-archive", action="store_true", help="skip writing the timestamped per-run archive in runs/") @@ -429,7 +442,8 @@ def main(argv: list[str] | None = None) -> None: load_dotenv() # make keys in .env available before any model is built report, raw_samples = run(args.models, args.categories, samples=args.samples, - escalate=args.escalate, judge=args.judge) + escalate=args.escalate, judge=args.judge, + condition=args.condition) RESULTS_DIR.mkdir(exist_ok=True) pathlib.Path(args.out).write_text(json.dumps(report, indent=2)) print_scorecard(report) diff --git a/scripts/author_interaction_tasks.py b/scripts/author_interaction_tasks.py new file mode 100644 index 0000000..aa8c360 --- /dev/null +++ b/scripts/author_interaction_tasks.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +""" +Author the interaction / measurement MVS tasks (``tasks/mvs_interactions/``). + +These exercise the parts of MVS the original corpus never asked for: ``primitives`` +(dashed lines between named atoms, labelled with their length), Mol*'s computed +non-covalent interactions (``custom.molstar_show_non_covalent_interactions``) and +``opacity``. A user asking to "show the hydrogen bond" is a common viewer request +that the old reference could not express. + +Like ``author_mvs_tasks.py``, each reference is built with the ``molviewspec`` +builder. In addition, every ComponentExpression in a reference is resolved against +the real mmCIF with gemmi before the task is written: a selection that matches no +atom is not an error in Mol* (a primitive end silently lands at the origin), so the +ground truth must be checked here. Each measured distance is recorded in the task's +provenance. + + python scripts/author_interaction_tasks.py # (re)generate tasks/mvs_interactions/*.json +""" + +from __future__ import annotations + +import json +import pathlib +import urllib.request + +import gemmi +from molviewspec import ComponentExpression, create_builder + +from molbench.mvs import categorize + +REPO = pathlib.Path(__file__).resolve().parent.parent +OUT = REPO / "tasks" / "mvs_interactions" +CACHE = REPO / "data" / "cif" +CIF = "https://files.rcsb.org/download/{}.cif" + + +def _structure(pdb: str): + b = create_builder() + s = b.download(url=CIF.format(pdb)).parse(format="mmcif").model_structure() + return b, s + + +def _atom(chain: str, atom: str, seq: int | None = None, comp: str | None = None) -> dict: + """A ComponentExpression naming one atom (by residue number or by ligand code).""" + e = {"auth_asym_id": chain, "label_atom_id": atom} + if seq is not None: + e["auth_seq_id"] = seq + if comp is not None: + e["label_comp_id"] = comp + return e + + +def _sticks(s, chain: str, seq: int | None = None, comp: str | None = None) -> None: + kw = {"auth_seq_id": seq} if seq is not None else {"label_comp_id": comp} + s.component(selector=ComponentExpression(auth_asym_id=chain, **kw)) \ + .representation(type="ball_and_stick") + + +def _cartoon(s) -> None: + s.component(selector="polymer").representation(type="cartoon").color(color="gray") + + +# --- task definitions ------------------------------------------------------------- + +def t_his64_o2(): + b, s = _structure("1a6m") + _cartoon(s) + _sticks(s, "A", seq=64) + _sticks(s, "A", comp="OXY") + s.primitives().distance(start=_atom("A", "NE2", seq=64), end=_atom("A", "O2", comp="OXY"), + dash_length=0.15, label_template="{{distance}}") + return b + + +def t_fe_his93(): + b, s = _structure("1a6m") + _cartoon(s) + _sticks(s, "A", comp="HEM") + _sticks(s, "A", seq=93) + s.primitives().distance(start=_atom("A", "FE", comp="HEM"), end=_atom("A", "NE2", seq=93), + dash_length=0.15, label_template="{{distance}}") + return b + + +def t_triad(): + b, s = _structure("3ptb") + _cartoon(s) + for r in (57, 102, 195): + _sticks(s, "A", seq=r) + p = s.primitives() + p.distance(start=_atom("A", "OG", seq=195), end=_atom("A", "NE2", seq=57), + dash_length=0.15, label_template="{{distance}}") + p.distance(start=_atom("A", "ND1", seq=57), end=_atom("A", "OD2", seq=102), + dash_length=0.15, label_template="{{distance}}") + return b + + +def t_o2_interactions(): + b, s = _structure("1a6m") + _cartoon(s) + s.component(selector=ComponentExpression(label_comp_id="OXY"), + custom={"molstar_show_non_covalent_interactions": True}) + return b + + +def t_benzamidine_interactions(): + b, s = _structure("3ptb") + _cartoon(s) + s.component(selector=ComponentExpression(label_comp_id="BEN"), + custom={"molstar_show_non_covalent_interactions": True}) + return b + + +def t_transparent_surface(): + b, s = _structure("1hho") + s.component(selector="polymer").representation(type="surface").opacity(opacity=0.5) + s.component(selector="ligand").representation(type="ball_and_stick") + return b + + +TASKS = [ + {"id": "int-001", "pdb": "1a6m", "build": t_his64_o2, + "title": "H-bond: distal His64 to bound O2", + "prompt": "In oxymyoglobin 1A6M, show the protein as a grey cartoon, His64 and the bound " + "O2 as ball-and-stick, and draw the hydrogen bond from His64 NE2 to the O2 " + "molecule's terminal oxygen (atom O2) as a dashed line labelled with its length.", + "skills": ["primitives", "atom-selection", "hbond"]}, + {"id": "int-002", "pdb": "1a6m", "build": t_fe_his93, + "title": "Distance: heme iron to proximal histidine", + "prompt": "Load 1A6M with the protein as a grey cartoon. Show the heme and the proximal " + "histidine as ball-and-stick and measure the distance from the heme iron to the " + "histidine's coordinating nitrogen.", + "skills": ["primitives", "atom-selection", "domain-knowledge"]}, + {"id": "int-003", "pdb": "3ptb", "build": t_triad, + "title": "Catalytic-triad H-bonds in trypsin", + "prompt": "In trypsin (3PTB), show the protein as a grey cartoon and the catalytic triad " + "(His57, Asp102, Ser195) as ball-and-stick. Draw the two triad hydrogen bonds as " + "dashed lines with their lengths.", + "skills": ["primitives", "atom-selection", "hbond", "domain-knowledge"]}, + {"id": "int-004", "pdb": "1a6m", "build": t_o2_interactions, + "title": "Computed interactions around bound O2", + "prompt": "Load 1A6M with the protein as a grey cartoon and show all the non-covalent " + "interactions (hydrogen bonds, metal contacts) around the bound oxygen molecule.", + "skills": ["interactions"]}, + {"id": "int-005", "pdb": "3ptb", "build": t_benzamidine_interactions, + "title": "Computed interactions around benzamidine", + "prompt": "Show trypsin 3PTB as a grey cartoon and display the interactions between the " + "benzamidine inhibitor and the protein.", + "skills": ["interactions"]}, + {"id": "int-006", "pdb": "1hho", "build": t_transparent_surface, + "title": "Half-transparent surface over the hemes", + "prompt": "Load haemoglobin 1HHO; show the protein as a 50% transparent surface and the " + "heme groups as ball-and-stick.", + "skills": ["opacity", "surface"]}, +] + + +# --- ground-truth checks ---------------------------------------------------------- + +def _load(pdb: str) -> gemmi.Structure: + CACHE.mkdir(parents=True, exist_ok=True) + path = CACHE / f"{pdb}.cif" + if not path.exists(): + urllib.request.urlretrieve(CIF.format(pdb), path) + return gemmi.read_structure(str(path)) + + +def _match(st: gemmi.Structure, e: dict) -> list[gemmi.Atom]: + """Atoms a ComponentExpression selects (the fields this script uses).""" + out = [] + for chain in st[0]: + if e.get("auth_asym_id") not in (None, chain.name): + continue + for res in chain: + if e.get("auth_seq_id") not in (None, res.seqid.num): + continue + if e.get("label_comp_id") not in (None, res.name): + continue + out += [a for a in res if e.get("label_atom_id") in (None, a.name)] + return out + + +def _check(root: dict, st: gemmi.Structure, task_id: str) -> list[dict]: + """Fail on any selection that matches nothing; return each measured distance.""" + measured = [] + + def walk(node: dict) -> None: + params = node.get("params") or {} + sel = params.get("selector") + if node.get("kind") == "component" and isinstance(sel, dict): + assert _match(st, sel), f"{task_id}: selector matches nothing: {sel}" + if node.get("kind") == "primitive": + ends = [_match(st, params[k]) for k in ("start", "end")] + assert all(ends), f"{task_id}: primitive end matches nothing: {params}" + # Mol* uses the centre of an end's atoms (alt locs included); record the first. + measured.append({"start": params["start"], "end": params["end"], + "distance_A": round(ends[0][0].pos.dist(ends[1][0].pos), 2)}) + for c in node.get("children") or []: + walk(c) + + walk(root) + return measured + + +def main() -> None: + OUT.mkdir(parents=True, exist_ok=True) + for t in TASKS: + root = json.loads(t["build"]().get_state().dumps())["root"] + measured = _check(root, _load(t["pdb"]), t["id"]) + doc = { + "id": t["id"], + "category": "mvs", + "source": "interactions", + "pdb": t["pdb"], + "title": t["title"], + "prompt": t["prompt"], + "reference_mvs": root, + "categories": categorize(root), + "skills": t["skills"], + "provenance": {"checked_against": CIF.format(t["pdb"]), "measured": measured}, + } + path = OUT / f"{t['id'].replace('-', '_')}.json" + path.write_text(json.dumps(doc, indent=2) + "\n") + print(f"wrote {path.relative_to(REPO)} {[m['distance_A'] for m in measured]}") + + +if __name__ == "__main__": + main() diff --git a/tasks/mvs_interactions/int_001.json b/tasks/mvs_interactions/int_001.json new file mode 100644 index 0000000..39412ee --- /dev/null +++ b/tasks/mvs_interactions/int_001.json @@ -0,0 +1,148 @@ +{ + "id": "int-001", + "category": "mvs", + "source": "interactions", + "pdb": "1a6m", + "title": "H-bond: distal His64 to bound O2", + "prompt": "In oxymyoglobin 1A6M, show the protein as a grey cartoon, His64 and the bound O2 as ball-and-stick, and draw the hydrogen bond from His64 NE2 to the O2 molecule's terminal oxygen (atom O2) as a dashed line labelled with its length.", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1a6m.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + }, + "children": [ + { + "kind": "color", + "params": { + "color": "gray" + } + } + ] + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "auth_asym_id": "A", + "auth_seq_id": 64 + } + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "auth_asym_id": "A", + "label_comp_id": "OXY" + } + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + }, + { + "kind": "primitives", + "params": {}, + "children": [ + { + "kind": "primitive", + "params": { + "start": { + "auth_asym_id": "A", + "label_atom_id": "NE2", + "auth_seq_id": 64 + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "O2", + "label_comp_id": "OXY" + }, + "dash_length": 0.15, + "kind": "distance_measurement", + "label_template": "{{distance}}" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "categories": [ + "load", + "selection", + "representation", + "color", + "multi-component", + "measurement" + ], + "skills": [ + "primitives", + "atom-selection", + "hbond" + ], + "provenance": { + "checked_against": "https://files.rcsb.org/download/1a6m.cif", + "measured": [ + { + "start": { + "auth_asym_id": "A", + "label_atom_id": "NE2", + "auth_seq_id": 64 + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "O2", + "label_comp_id": "OXY" + }, + "distance_A": 2.97 + } + ] + } +} diff --git a/tasks/mvs_interactions/int_002.json b/tasks/mvs_interactions/int_002.json new file mode 100644 index 0000000..0ec1b88 --- /dev/null +++ b/tasks/mvs_interactions/int_002.json @@ -0,0 +1,148 @@ +{ + "id": "int-002", + "category": "mvs", + "source": "interactions", + "pdb": "1a6m", + "title": "Distance: heme iron to proximal histidine", + "prompt": "Load 1A6M with the protein as a grey cartoon. Show the heme and the proximal histidine as ball-and-stick and measure the distance from the heme iron to the histidine's coordinating nitrogen.", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1a6m.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + }, + "children": [ + { + "kind": "color", + "params": { + "color": "gray" + } + } + ] + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "auth_asym_id": "A", + "label_comp_id": "HEM" + } + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "auth_asym_id": "A", + "auth_seq_id": 93 + } + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + }, + { + "kind": "primitives", + "params": {}, + "children": [ + { + "kind": "primitive", + "params": { + "start": { + "auth_asym_id": "A", + "label_atom_id": "FE", + "label_comp_id": "HEM" + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "NE2", + "auth_seq_id": 93 + }, + "dash_length": 0.15, + "kind": "distance_measurement", + "label_template": "{{distance}}" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "categories": [ + "load", + "selection", + "representation", + "color", + "multi-component", + "measurement" + ], + "skills": [ + "primitives", + "atom-selection", + "domain-knowledge" + ], + "provenance": { + "checked_against": "https://files.rcsb.org/download/1a6m.cif", + "measured": [ + { + "start": { + "auth_asym_id": "A", + "label_atom_id": "FE", + "label_comp_id": "HEM" + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "NE2", + "auth_seq_id": 93 + }, + "distance_A": 2.06 + } + ] + } +} diff --git a/tasks/mvs_interactions/int_003.json b/tasks/mvs_interactions/int_003.json new file mode 100644 index 0000000..8af2f36 --- /dev/null +++ b/tasks/mvs_interactions/int_003.json @@ -0,0 +1,197 @@ +{ + "id": "int-003", + "category": "mvs", + "source": "interactions", + "pdb": "3ptb", + "title": "Catalytic-triad H-bonds in trypsin", + "prompt": "In trypsin (3PTB), show the protein as a grey cartoon and the catalytic triad (His57, Asp102, Ser195) as ball-and-stick. Draw the two triad hydrogen bonds as dashed lines with their lengths.", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/3ptb.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + }, + "children": [ + { + "kind": "color", + "params": { + "color": "gray" + } + } + ] + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "auth_asym_id": "A", + "auth_seq_id": 57 + } + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "auth_asym_id": "A", + "auth_seq_id": 102 + } + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "auth_asym_id": "A", + "auth_seq_id": 195 + } + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + }, + { + "kind": "primitives", + "params": {}, + "children": [ + { + "kind": "primitive", + "params": { + "start": { + "auth_asym_id": "A", + "label_atom_id": "OG", + "auth_seq_id": 195 + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "NE2", + "auth_seq_id": 57 + }, + "dash_length": 0.15, + "kind": "distance_measurement", + "label_template": "{{distance}}" + } + }, + { + "kind": "primitive", + "params": { + "start": { + "auth_asym_id": "A", + "label_atom_id": "ND1", + "auth_seq_id": 57 + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "OD2", + "auth_seq_id": 102 + }, + "dash_length": 0.15, + "kind": "distance_measurement", + "label_template": "{{distance}}" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "categories": [ + "load", + "selection", + "representation", + "color", + "multi-component", + "measurement" + ], + "skills": [ + "primitives", + "atom-selection", + "hbond", + "domain-knowledge" + ], + "provenance": { + "checked_against": "https://files.rcsb.org/download/3ptb.cif", + "measured": [ + { + "start": { + "auth_asym_id": "A", + "label_atom_id": "OG", + "auth_seq_id": 195 + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "NE2", + "auth_seq_id": 57 + }, + "distance_A": 2.96 + }, + { + "start": { + "auth_asym_id": "A", + "label_atom_id": "ND1", + "auth_seq_id": 57 + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "OD2", + "auth_seq_id": 102 + }, + "distance_A": 2.67 + } + ] + } +} diff --git a/tasks/mvs_interactions/int_004.json b/tasks/mvs_interactions/int_004.json new file mode 100644 index 0000000..fd8552e --- /dev/null +++ b/tasks/mvs_interactions/int_004.json @@ -0,0 +1,85 @@ +{ + "id": "int-004", + "category": "mvs", + "source": "interactions", + "pdb": "1a6m", + "title": "Computed interactions around bound O2", + "prompt": "Load 1A6M with the protein as a grey cartoon and show all the non-covalent interactions (hydrogen bonds, metal contacts) around the bound oxygen molecule.", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1a6m.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + }, + "children": [ + { + "kind": "color", + "params": { + "color": "gray" + } + } + ] + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "label_comp_id": "OXY" + } + }, + "custom": { + "molstar_show_non_covalent_interactions": true + } + } + ] + } + ] + } + ] + } + ] + }, + "categories": [ + "load", + "selection", + "representation", + "color", + "multi-component", + "interactions" + ], + "skills": [ + "interactions" + ], + "provenance": { + "checked_against": "https://files.rcsb.org/download/1a6m.cif", + "measured": [] + } +} diff --git a/tasks/mvs_interactions/int_005.json b/tasks/mvs_interactions/int_005.json new file mode 100644 index 0000000..723f50b --- /dev/null +++ b/tasks/mvs_interactions/int_005.json @@ -0,0 +1,85 @@ +{ + "id": "int-005", + "category": "mvs", + "source": "interactions", + "pdb": "3ptb", + "title": "Computed interactions around benzamidine", + "prompt": "Show trypsin 3PTB as a grey cartoon and display the interactions between the benzamidine inhibitor and the protein.", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/3ptb.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + }, + "children": [ + { + "kind": "color", + "params": { + "color": "gray" + } + } + ] + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "label_comp_id": "BEN" + } + }, + "custom": { + "molstar_show_non_covalent_interactions": true + } + } + ] + } + ] + } + ] + } + ] + }, + "categories": [ + "load", + "selection", + "representation", + "color", + "multi-component", + "interactions" + ], + "skills": [ + "interactions" + ], + "provenance": { + "checked_against": "https://files.rcsb.org/download/3ptb.cif", + "measured": [] + } +} diff --git a/tasks/mvs_interactions/int_006.json b/tasks/mvs_interactions/int_006.json new file mode 100644 index 0000000..9321636 --- /dev/null +++ b/tasks/mvs_interactions/int_006.json @@ -0,0 +1,89 @@ +{ + "id": "int-006", + "category": "mvs", + "source": "interactions", + "pdb": "1hho", + "title": "Half-transparent surface over the hemes", + "prompt": "Load haemoglobin 1HHO; show the protein as a 50% transparent surface and the heme groups as ball-and-stick.", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1hho.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "surface", + "surface_type": "molecular" + }, + "children": [ + { + "kind": "opacity", + "params": { + "opacity": 0.5 + } + } + ] + } + ] + }, + { + "kind": "component", + "params": { + "selector": "ligand" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "categories": [ + "load", + "representation", + "multi-component", + "surface", + "transparency" + ], + "skills": [ + "opacity", + "surface" + ], + "provenance": { + "checked_against": "https://files.rcsb.org/download/1hho.cif", + "measured": [] + } +} diff --git a/tests/test_harness.py b/tests/test_harness.py index 8c32069..e73bd9e 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -269,3 +269,58 @@ def test_accepted_fold_survives_download_without_params(): test_accepted_fold_ignores_multi_structure_references() test_accepted_fold_survives_download_without_params() print("all sanity checks passed") + + +def _with_structure_child(child: dict) -> dict: + s = _scene() + s["root"]["children"][0]["children"][0]["children"][0]["children"].append(child) + return s + + +def _hbond(start: dict, end: dict, **style) -> dict: + return {"kind": "primitives", "params": style.pop("group", {}), "children": [ + {"kind": "primitive", "params": {"kind": "distance_measurement", + "start": start, "end": end, **style}}]} + + +_HIS = {"auth_asym_id": "A", "auth_seq_id": 64, "label_atom_id": "NE2"} +_O2 = {"auth_asym_id": "A", "label_comp_id": "OXY", "label_atom_id": "O2"} + + +def test_primitive_ignores_styling_and_direction(): + ref = _with_structure_child(_hbond(_HIS, _O2)) + pred = _with_structure_child(_hbond(_O2, _HIS, radius=0.1, dash_length=0.3, + label_template="H-bond {{distance}}", + group={"color": "yellow"})) + assert mvs.grade_mvs(ref, pred)["f1"] == 1.0 + + +def test_primitive_wrong_atom_loses_credit(): + ref = _with_structure_child(_hbond(_HIS, _O2)) + pred = _with_structure_child(_hbond({**_HIS, "label_atom_id": "ND1"}, _O2)) + assert mvs.grade_mvs(ref, pred)["f1"] < 1.0 + + +def test_interactions_flag_is_graded(): + comp = {"kind": "component", "params": {"selector": {"label_comp_id": "OXY"}}} + ref = _with_structure_child({**comp, "custom": {"molstar_show_non_covalent_interactions": True}}) + assert mvs.grade_mvs(ref, ref)["f1"] == 1.0 + assert mvs.grade_mvs(ref, _with_structure_child(comp))["f1"] < 1.0 + assert "interactions" in mvs.categorize(mvs.extract_root(ref)) + + +def test_colour_theme_grades_the_theme_not_the_placeholder(): + def themed(theme, placeholder): + s = _scene(placeholder) + rep = s["root"]["children"][0]["children"][0]["children"][0]["children"][0]["children"][0] + rep["children"][0]["custom"] = {"molstar_color_theme_name": theme} + return s + assert mvs.grade_mvs(themed("chain-id", "gray"), themed("chain-id", "white"))["f1"] == 1.0 + assert mvs.grade_mvs(themed("chain-id", "gray"), themed("sequence-id", "gray"))["f1"] < 1.0 + + +def test_bare_condition_drops_the_reference(): + from molbench.runner import build_system_prompts + spec, bare = build_system_prompts("spec")["mvs"], build_system_prompts("bare")["mvs"] + assert "Node kinds" in spec and "Node kinds" not in bare + assert "{{MVS_REFERENCE}}" not in bare From 7f834ab9be1d3930130340b348b37b02c048427e Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Tue, 6 Oct 2026 14:54:03 +0100 Subject: [PATCH 04/10] ROADMAP: R5 regression results; chat driver v2 shipped Co-Authored-By: Claude Opus 5.5 (1M context) --- ROADMAP.md | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 381bb40..031b3d1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -58,13 +58,34 @@ than an API-calling benchmark. | R2 | Grader: primitives keyed on shape + atom pair (styling/direction ignored); interactions flag and colour theme graded | done | | R3 | `tasks/mvs_interactions/` — 6 tasks, every atom checked against the mmCIF with gemmi (`scripts/author_interaction_tasks.py`) | done | | R4 | `--condition bare\|spec` in the runner, recorded in run meta | done | -| R5 | Regression run: old vs spec vs bare on 3 cheap models, all MVS tasks | blocked (API credit) | +| R5 | Regression run: old vs spec vs bare on 3 cheap models, all MVS tasks | done — see below | | R6 | Grounding packs: offline builder (gemmi + SIFTS + existing resolve/clinical provenance) and `--grounding` flag | todo | | R7 | Grader: resolve selections to atom sets against the mmCIF, so equivalent namings (`label_comp_id`+atom vs residue number+atom) match and empty selections score zero | todo | -| R8 | Chat driver v2: new reference in its prompt + empty-selection lint (a primitive end that matches nothing is drawn to the origin by Mol\*) | in PR | +| R8 | Chat driver v2: new reference in its prompt + empty-selection lint (a primitive end that matches nothing is drawn to the origin by Mol\*) | shipped (chat-driver #18, deployed 2026-10-06) | | R9 | Chat driver: thumbs up/down per turn, for grading real traffic | todo | | R10 | Leaderboard: a regime for `source: interactions` (unknown sources currently fall into "Clinical") | todo | +**R5 result (2026-10-06, 1 sample, 76 tasks, ~$0.85).** Mean F1: + +| model | old ref | spec | bare | interaction tasks old → spec | +|---|---|---|---|---| +| Claude Haiku 4.5 | 0.800 | 0.806 | 0.061 | 0.52 → 0.72 | +| DeepSeek V3.2 | 0.827 | 0.814 | 0.018 | 0.58 → 0.56 | +| Qwen3-30B-A3B | 0.576 | 0.522 | 0.000 | 0.22 → 0.18 | + +* **Bare ≈ 0 for every model.** MVS is not in pretraining in any usable form; + the bare row measures nothing but "can't". Keep it as one line in the paper, + not a leaderboard column. +* **Spec costs ~0.01 on the old tasks for the strong models** (within 1-sample + noise: 57 tasks better, 63 worse of 228). Part of it is real: with a richer + reference, Haiku adds unrequested ball-and-stick on the resolve tasks + (same, correct entries; precision drops). +* **The small model loses ~0.05, almost all to JSON bracket-count errors** on a + ~2.4× longer prompt (parse success 0.78 → 0.67). Longer documentation hurts + small models — a finding, and an argument for B6 (structured outputs). +* Interaction tasks are not yet solved by anyone: line endpoints are named with + the wrong atoms or chains. That is the grounding gap (R6). + --- ## Phase A — Component 2: render + VLM judge ← the headline From ebd8877ef863552988e1a6487c3a6e16afc12e5a Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Tue, 6 Oct 2026 19:34:50 +0100 Subject: [PATCH 05/10] Address review: keep [x,y,z] primitive points ordered; guard non-dict custom; author script checks every primitive point Co-Authored-By: Claude Opus 5.5 (1M context) --- molbench/mvs.py | 13 +++++++++++-- scripts/author_interaction_tasks.py | 15 ++++++++++----- tests/test_harness.py | 14 ++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/molbench/mvs.py b/molbench/mvs.py index c7574ca..89960e6 100644 --- a/molbench/mvs.py +++ b/molbench/mvs.py @@ -73,6 +73,13 @@ def _color_signature(value: Any) -> Any: _PRIMITIVE_POINTS = ("start", "end", "position", "a", "b", "c") +def _point_signature(point: Any) -> Any: + """A primitive position: an [x, y, z] coordinate (order matters) or an atom selection.""" + if isinstance(point, list) and all(isinstance(v, (int, float)) for v in point): + return ("xyz", tuple(float(v) for v in point)) + return _selector_signature(point) + + def _param_signature(kind: str, params: dict | None, custom: dict | None = None) -> Any: params = params or {} custom = custom if isinstance(custom, dict) else {} @@ -86,7 +93,7 @@ def _param_signature(kind: str, params: dict | None, custom: dict | None = None) if kind == "primitive": # Key on the shape and the atoms it joins. Radius, dash length and label text # are styling; a line is the same line whichever end the model starts from. - points = frozenset(_selector_signature(params[k]) for k in _PRIMITIVE_POINTS + points = frozenset(_point_signature(params[k]) for k in _PRIMITIVE_POINTS if k in params) return ("primitive", params.get("kind"), points) if kind == "primitives": @@ -292,7 +299,9 @@ def walk(node: dict) -> None: params = node.get("params") or {} if k == "component" and not isinstance(params.get("selector"), str): has_expr = True - if k == "component" and (node.get("custom") or {}).get("molstar_show_non_covalent_interactions"): + custom = node.get("custom") + if k == "component" and isinstance(custom, dict) \ + and custom.get("molstar_show_non_covalent_interactions"): interactions = True if k == "representation": reps.add(params.get("type")) diff --git a/scripts/author_interaction_tasks.py b/scripts/author_interaction_tasks.py index aa8c360..458d835 100644 --- a/scripts/author_interaction_tasks.py +++ b/scripts/author_interaction_tasks.py @@ -168,6 +168,8 @@ def _load(pdb: str) -> gemmi.Structure: def _match(st: gemmi.Structure, e: dict) -> list[gemmi.Atom]: """Atoms a ComponentExpression selects (the fields this script uses).""" + if not isinstance(e, dict): + return [] # an [x, y, z] coordinate, not a selection out = [] for chain in st[0]: if e.get("auth_asym_id") not in (None, chain.name): @@ -191,11 +193,14 @@ def walk(node: dict) -> None: if node.get("kind") == "component" and isinstance(sel, dict): assert _match(st, sel), f"{task_id}: selector matches nothing: {sel}" if node.get("kind") == "primitive": - ends = [_match(st, params[k]) for k in ("start", "end")] - assert all(ends), f"{task_id}: primitive end matches nothing: {params}" - # Mol* uses the centre of an end's atoms (alt locs included); record the first. - measured.append({"start": params["start"], "end": params["end"], - "distance_A": round(ends[0][0].pos.dist(ends[1][0].pos), 2)}) + points = {k: params[k] for k in ("start", "end", "position", "a", "b", "c") + if isinstance(params.get(k), dict)} + atoms = {k: _match(st, e) for k, e in points.items()} + assert all(atoms.values()), f"{task_id}: primitive point matches nothing: {params}" + if "start" in atoms and "end" in atoms: + # Mol* uses the centre of an end's atoms (alt locs included); record the first. + measured.append({"start": params["start"], "end": params["end"], + "distance_A": round(atoms["start"][0].pos.dist(atoms["end"][0].pos), 2)}) for c in node.get("children") or []: walk(c) diff --git a/tests/test_harness.py b/tests/test_harness.py index e73bd9e..cbc2670 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -324,3 +324,17 @@ def test_bare_condition_drops_the_reference(): spec, bare = build_system_prompts("spec")["mvs"], build_system_prompts("bare")["mvs"] assert "Node kinds" in spec and "Node kinds" not in bare assert "{{MVS_REFERENCE}}" not in bare + + +def test_primitive_coordinates_keep_their_order(): + a = _with_structure_child({"kind": "primitives", "children": [{"kind": "primitive", "params": { + "kind": "label", "position": [1.0, 2.0, 3.0], "text": "x"}}]}) + b = _with_structure_child({"kind": "primitives", "children": [{"kind": "primitive", "params": { + "kind": "label", "position": [3.0, 2.0, 1.0], "text": "x"}}]}) + assert mvs.grade_mvs(a, a)["f1"] == 1.0 + assert mvs.grade_mvs(a, b)["f1"] < 1.0 + + +def test_categorize_survives_non_dict_custom(): + s = _with_structure_child({"kind": "component", "params": {"selector": "ligand"}, "custom": ["x"]}) + assert "interactions" not in mvs.categorize(mvs.extract_root(s)) From 8c4e6aa00f1ecca26112e6d5a6d666f36e1fe12d Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Tue, 6 Oct 2026 19:40:55 +0100 Subject: [PATCH 06/10] Address Codex review: arrows and angles keep point roles; interactions flag kept when a list selector is expanded Co-Authored-By: Claude Opus 5.5 (1M context) --- molbench/mvs.py | 16 ++++++++++++---- tests/test_harness.py | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/molbench/mvs.py b/molbench/mvs.py index 89960e6..93072e9 100644 --- a/molbench/mvs.py +++ b/molbench/mvs.py @@ -71,6 +71,7 @@ def _color_signature(value: Any) -> Any: _PRIMITIVE_POINTS = ("start", "end", "position", "a", "b", "c") +_UNDIRECTED_PRIMITIVES = {"tube", "distance_measurement"} def _point_signature(point: Any) -> Any: @@ -92,10 +93,16 @@ def _param_signature(kind: str, params: dict | None, custom: dict | None = None) return sig if kind == "primitive": # Key on the shape and the atoms it joins. Radius, dash length and label text - # are styling; a line is the same line whichever end the model starts from. - points = frozenset(_point_signature(params[k]) for k in _PRIMITIVE_POINTS + # are styling. A line or distance is the same whichever end comes first; an + # arrow's direction and an angle's vertex are not, so those keep their roles. + pkind = params.get("kind") + if pkind in _UNDIRECTED_PRIMITIVES: + points = frozenset(_point_signature(params[k]) for k in _PRIMITIVE_POINTS + if k in params) + else: + points = tuple((k, _point_signature(params[k])) for k in _PRIMITIVE_POINTS if k in params) - return ("primitive", params.get("kind"), points) + return ("primitive", pkind, points) if kind == "primitives": return () # a grouping node; its colour is styling if kind == "color" and custom.get("molstar_color_theme_name"): @@ -151,7 +158,8 @@ def walk(node: dict, prefix: tuple) -> None: # "group residues in one component" and "one component each" score alike. if kind == "component" and isinstance(params.get("selector"), list) \ and len(params["selector"]) > 1: - variants = [("selector", _selector_signature(e)) for e in params["selector"]] + flag = _param_signature(kind, {"selector": "all"}, node.get("custom"))[2:] + variants = [("selector", _selector_signature(e)) + flag for e in params["selector"]] else: variants = [_param_signature(kind, params, node.get("custom"))] diff --git a/tests/test_harness.py b/tests/test_harness.py index cbc2670..b6229a7 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -338,3 +338,21 @@ def test_primitive_coordinates_keep_their_order(): def test_categorize_survives_non_dict_custom(): s = _with_structure_child({"kind": "component", "params": {"selector": "ligand"}, "custom": ["x"]}) assert "interactions" not in mvs.categorize(mvs.extract_root(s)) + + +def test_arrow_direction_and_angle_vertex_are_graded(): + def prim(**params): + return _with_structure_child({"kind": "primitives", "children": [ + {"kind": "primitive", "params": params}]}) + a, b, c = ({"auth_seq_id": n} for n in (1, 2, 3)) + assert mvs.grade_mvs(prim(kind="arrow", start=a, end=b), prim(kind="arrow", start=b, end=a))["f1"] < 1.0 + assert mvs.grade_mvs(prim(kind="angle_measurement", a=a, b=b, c=c), + prim(kind="angle_measurement", a=b, b=a, c=c))["f1"] < 1.0 + + +def test_interactions_flag_survives_list_selector_expansion(): + sel = [{"label_comp_id": "OXY"}, {"label_comp_id": "HEM"}] + comp = {"kind": "component", "params": {"selector": sel}} + ref = _with_structure_child({**comp, "custom": {"molstar_show_non_covalent_interactions": True}}) + assert mvs.grade_mvs(ref, ref)["f1"] == 1.0 + assert mvs.grade_mvs(ref, _with_structure_child(comp))["f1"] < 1.0 From abf42251706006dc7ac44f1297b96f9d736e93c2 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Tue, 6 Oct 2026 19:42:02 +0100 Subject: [PATCH 07/10] Address Codex review: CFTR accepts only near-full-length entries; escalation renders the accepted entry; CI runs every test - generate_resolve_tasks: optional per-task min coverage (entity-level RCSB query); res-cftr drops isolated-domain entries such as 1XMI/2BBO. - escalate: the visual tiers re-point the reference at the accepted entry the prediction loaded, as the tree tier already does. - test_harness: the __main__ block runs every test_* function, so CI (which runs this file directly) can no longer skip a newly added test. Co-Authored-By: Claude Opus 5.5 (1M context) --- molbench/escalate.py | 40 +++++++++++++++++++++++-- scripts/generate_resolve_tasks.py | 49 ++++++++++++++++++++++--------- tasks/mvs_resolve/res-cftr.json | 42 +++----------------------- tests/test_harness.py | 44 +++++++++++++-------------- 4 files changed, 97 insertions(+), 78 deletions(-) diff --git a/molbench/escalate.py b/molbench/escalate.py index dbf73a0..6d383eb 100644 --- a/molbench/escalate.py +++ b/molbench/escalate.py @@ -19,14 +19,50 @@ from __future__ import annotations +import copy import tempfile import pathlib from typing import Any, Callable -from .mvs import grade_mvs +from .mvs import _download_id, extract_root, flatten_paths, grade_mvs from .visual import image_similarity +def _download_ids(tree: Any) -> set[str]: + root = extract_root(tree) + if root is None: + return set() + return {_download_id(seg) for path in flatten_paths(root) for seg in path} - {None} + + +def _reference_on_accepted(reference_tree: dict, predicted_tree: Any, + accepted: set[str] | list[str] | None) -> dict: + """The reference re-pointed at the accepted entry the prediction loaded. + + The tree tier folds an accepted id onto the canonical one; the visual tiers must + do the same, or they compare two different structures and an otherwise + equivalent scene can never be rescued. + """ + ref_ids, pred_ids = _download_ids(reference_tree), _download_ids(predicted_tree) + if not accepted or len(ref_ids) != 1 or len(pred_ids) != 1: + return reference_tree + (ref_id,), (pred_id,) = ref_ids, pred_ids + if pred_id == ref_id or pred_id not in {a.lower() for a in accepted}: + return reference_tree + moved = copy.deepcopy(reference_tree) + + def walk(node: dict) -> None: + params = node.get("params") + if node.get("kind") == "download" and isinstance(params, dict) and "url" in params: + params["url"] = params["url"].replace(f"/{ref_id}.", f"/{pred_id}.") + for c in node.get("children") or []: + if isinstance(c, dict): + walk(c) + + walk(extract_root(moved)) + return moved + + def escalating_grade( reference_tree: dict, predicted_tree: Any, @@ -54,7 +90,7 @@ def escalating_grade( wd.mkdir(parents=True, exist_ok=True) ref_png, pred_png = wd / "ref.png", wd / "pred.png" try: - render(reference_tree, str(ref_png)) + render(_reference_on_accepted(reference_tree, predicted_tree, accepted_refs), str(ref_png)) render(predicted_tree, str(pred_png)) except Exception as e: # noqa: BLE001 - a render failure is itself informative return {**out, "decision": "render-failed", "note": str(e)} diff --git a/scripts/generate_resolve_tasks.py b/scripts/generate_resolve_tasks.py index 11875c2..2a936ac 100644 --- a/scripts/generate_resolve_tasks.py +++ b/scripts/generate_resolve_tasks.py @@ -31,15 +31,18 @@ CIF = "https://files.rcsb.org/download/{}.cif" SEARCH = "https://search.rcsb.org/rcsbsearch/v2/query" -# (slug, prompt, UniProt accession, canonical PDB id, note). Prompts are verbatim or -# lightly edited from the evaluator corpus; the canonical id is a well-known entry. +# (slug, prompt, UniProt accession, canonical PDB id, note[, min coverage]). Prompts are +# verbatim or lightly edited from the evaluator corpus; the canonical id is a well-known +# entry. A min coverage (fraction of the UniProt sequence one polymer entity must span) +# is set where the prompt asks for the whole protein: "the CFTR channel" is not answered +# by an isolated NBD1 domain. SPECS = [ ("pde5a", "I wanna see a structure of PDE5A", "O76074", "1udt", "Evaluator prompt (2026-09-30). Haiku answered 1UJ7, which does not exist."), ("pcsk9", "show the structure of PCSK9", "Q8NBP7", "2p4e", "Probe: Haiku answered 1D0G (death receptor 5 / TRAIL)."), ("cftr", "show me the CFTR channel", "P13569", "5uak", - "Probe: Haiku answered 5UAY (a plant Toc75 POTRA domain)."), + "Probe: Haiku answered 5UAY (a plant Toc75 POTRA domain).", 0.7), ("gfp", "show green fluorescent protein", "P42212", "1ema", "The starter chip's target; the model tends to be right here — a control."), ("myoglobin", "Show me sperm whale myoglobin", "P02185", "1mbn", @@ -49,14 +52,27 @@ ] -def entries_for(accession: str) -> list[str]: - """All PDB entries with a polymer entity mapped to this UniProt accession.""" +def uniprot_length(accession: str) -> int: + url = f"https://rest.uniprot.org/uniprotkb/{accession}.json?fields=length" + with urllib.request.urlopen(url, timeout=60) as resp: + return json.load(resp)["sequence"]["length"] + + +def entries_for(accession: str, min_length: int | None = None) -> list[str]: + """All PDB entries with a polymer entity mapped to this UniProt accession (and, if + ``min_length`` is set, at least that many residues in the same entity).""" + nodes = [{"type": "terminal", "service": "text", "parameters": { + "attribute": "rcsb_polymer_entity_container_identifiers." + "reference_sequence_identifiers.database_accession", + "operator": "exact_match", "value": accession}}] + if min_length: + nodes.append({"type": "terminal", "service": "text", "parameters": { + "attribute": "entity_poly.rcsb_sample_sequence_length", + "operator": "greater_or_equal", "value": min_length}}) query = { - "query": {"type": "terminal", "service": "text", "parameters": { - "attribute": "rcsb_polymer_entity_container_identifiers." - "reference_sequence_identifiers.database_accession", - "operator": "exact_match", "value": accession}}, - "return_type": "entry", + "query": {"type": "group", "logical_operator": "and", "nodes": nodes}, + # Entity-level, so both conditions hold for the same polymer entity. + "return_type": "polymer_entity", "request_options": {"paginate": {"start": 0, "rows": 10000}}, } req = urllib.request.Request(SEARCH, data=json.dumps(query).encode(), @@ -64,7 +80,8 @@ def entries_for(accession: str) -> list[str]: with urllib.request.urlopen(req, timeout=60) as resp: if resp.status == 204: return [] - return sorted(hit["identifier"].lower() for hit in json.load(resp)["result_set"]) + hits = json.load(resp)["result_set"] + return sorted({hit["identifier"].split("_")[0].lower() for hit in hits}) def reference(pdb: str) -> dict: @@ -79,8 +96,10 @@ def reference(pdb: str) -> dict: def main() -> None: OUT.mkdir(parents=True, exist_ok=True) today = dt.date.today().isoformat() - for slug, prompt, accession, canonical, note in SPECS: - ids = entries_for(accession) + for slug, prompt, accession, canonical, note, *rest in SPECS: + min_coverage = rest[0] if rest else None + min_length = round(min_coverage * uniprot_length(accession)) if min_coverage else None + ids = entries_for(accession, min_length) if canonical not in ids: raise SystemExit(f"{slug}: canonical {canonical} is not mapped to {accession}") ref = reference(canonical) @@ -96,7 +115,9 @@ def main() -> None: "skills": ["resolve", "download", "component", "representation"], "notes": note, "provenance": {"uniprot": accession, "n_entries": len(ids), - "source": "RCSB search, reference_sequence_identifiers", "fetched": today}, + "source": "RCSB search, reference_sequence_identifiers", "fetched": today, + **({"min_coverage": min_coverage, "min_length": min_length} + if min_coverage else {})}, } (OUT / f"res-{slug}.json").write_text(json.dumps(task, indent=2) + "\n") print(f"res-{slug:10s} {accession} {len(ids):4d} accepted entries canonical {canonical}") diff --git a/tasks/mvs_resolve/res-cftr.json b/tasks/mvs_resolve/res-cftr.json index 3b75542..71a6962 100644 --- a/tasks/mvs_resolve/res-cftr.json +++ b/tasks/mvs_resolve/res-cftr.json @@ -48,46 +48,10 @@ ] }, "accepted_ids": [ - "1xmi", - "1xmj", - "2bbo", - "2bbs", - "2bbt", - "2lob", - "2pze", - "2pzf", - "2pzg", - "3gd7", - "3isw", - "4wz6", - "5d2d", - "5d3e", - "5d3f", - "5tf7", - "5tf8", - "5tfa", - "5tfb", - "5tfc", - "5tfd", - "5tff", - "5tfg", - "5tfi", - "5tfj", - "5tgk", "5uak", - "6gjq", - "6gjs", - "6gju", - "6gk4", - "6gkd", - "6hep", "6msm", "6o1v", "6o2p", - "6uk1", - "6wbs", - "6ze1", - "7qi1", "7sv7", "7svd", "7svr", @@ -123,8 +87,10 @@ "notes": "Probe: Haiku answered 5UAY (a plant Toc75 POTRA domain).", "provenance": { "uniprot": "P13569", - "n_entries": 60, + "n_entries": 24, "source": "RCSB search, reference_sequence_identifiers", - "fetched": "2026-09-30" + "fetched": "2026-10-06", + "min_coverage": 0.7, + "min_length": 1036 } } diff --git a/tests/test_harness.py b/tests/test_harness.py index 8c32069..97b6c3c 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -244,28 +244,24 @@ def test_accepted_fold_survives_download_without_params(): assert mvs.grade_mvs(bare_ref, pred, accepted_refs=["1udt"])["f1"] >= 0.0 + + +def test_escalation_renders_the_reference_on_the_accepted_entry(): + from molbench import escalate + task = _resolve_tasks()[0] + other = next(i for i in task["accepted_ids"] if i not in json.dumps(task["reference_mvs"])) + pred = _with_pdb(task, other) + moved = escalate._reference_on_accepted(task["reference_mvs"], pred, task["accepted_ids"]) + assert escalate._download_ids(moved) == {other} + assert escalate._download_ids(task["reference_mvs"]) != {other} # the task is untouched + made_up = _with_pdb(task, "9zzz") + assert escalate._reference_on_accepted(task["reference_mvs"], made_up, task["accepted_ids"]) \ + is task["reference_mvs"] + + if __name__ == "__main__": - test_all_reference_answers_are_schema_valid() - test_self_grading_is_perfect() - test_wrong_prediction_scores_low() - test_partial_credit_for_right_residues_wrong_colour() - test_load_modifiers_are_scored() - test_tooltip_text_is_scored() - test_sidechain_flag_is_scored() - test_mvs_self_grading_is_perfect() - test_mvs_is_metadata_insensitive() - test_mvs_partial_credit_for_wrong_colour() - test_mvs_grey_gray_fold() - test_mvs_non_tree_scores_zero() - test_mvs_grader_survives_non_dict_params() - test_mvs_download_variant_is_normalised() - test_mvs_downstream_credit_survives_upstream_mismatch() - test_escalate_tree_tier() - test_render_state_wrapping() - test_run_archives_raw_samples_and_meta() - test_resolve_tasks_have_a_valid_accepted_set() - test_any_accepted_entry_scores_as_the_reference() - test_made_up_entry_is_penalised() - test_accepted_fold_ignores_multi_structure_references() - test_accepted_fold_survives_download_without_params() - print("all sanity checks passed") + # Run every test_* function, so a new test can't be missed by CI (which runs this file). + tests = [f for name, f in sorted(globals().items()) if name.startswith("test_") and callable(f)] + for t in tests: + t() + print(f"all {len(tests)} harness checks passed") From 782c7a12e27d2369126ef13d559cca523bdf1307 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Tue, 6 Oct 2026 19:46:27 +0100 Subject: [PATCH 08/10] Address Codex review: resolve tasks get their own leaderboard regime; unknown sources fail loudly Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/build_leaderboard.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/scripts/build_leaderboard.py b/scripts/build_leaderboard.py index 3641ea0..c393487 100644 --- a/scripts/build_leaderboard.py +++ b/scripts/build_leaderboard.py @@ -34,18 +34,25 @@ NL = "\n" # How a task's `source` maps to a difficulty regime, ordered easy -> hard. -REGIMES = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)"] +REGIMES = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)", "Resolution (by name)"] +# The first three form the difficulty gradient (Finding 03); later regimes test other skills. +_SOURCE_REGIME = { + "generated": "Translation", "curated": "Translation", + "grounded": "Grounded (ligand/SS)", + "clinvar-variant": "Clinical (SIFTS/ClinVar)", "clinvar-hotspots": "Clinical (SIFTS/ClinVar)", + "sifts-named": "Clinical (SIFTS/ClinVar)", + "resolve": "Resolution (by name)", +} # --------------------------------------------------------------------------- # # Data helpers # --------------------------------------------------------------------------- # def _regime_of(source: str) -> str: - if source in ("generated", "curated"): - return "Translation" - if source == "grounded": - return "Grounded (ligand/SS)" - return "Clinical (SIFTS/ClinVar)" + # Explicit, so a new task source can't silently land in another regime's numbers. + if source not in _SOURCE_REGIME: + raise ValueError(f"task source {source!r} has no regime; add it to _SOURCE_REGIME") + return _SOURCE_REGIME[source] @functools.lru_cache(maxsize=1) @@ -340,7 +347,7 @@ def findings(ranked: list) -> str: # Compute it live — with a wider panel this can flip, because format-unreliable # models inject parse-driven spread that swamps the difficulty signal. regimes = _task_regimes() - reg_order = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)"] + reg_order = REGIMES[:3] reg_spread: dict[str, float] = {} for rg in reg_order: per = [] From 8cd7e9158efbc450eae3b0e520c8c6e2eaf0f1c9 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Tue, 6 Oct 2026 19:46:52 +0100 Subject: [PATCH 09/10] Address Codex review: 3D label text and element selectors are graded; interactions regime on the leaderboard Co-Authored-By: Claude Opus 5.5 (1M context) --- ROADMAP.md | 2 +- molbench/mvs.py | 7 ++++++- scripts/build_leaderboard.py | 4 +++- tests/test_harness.py | 13 +++++++++++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 031b3d1..07d8c99 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -63,7 +63,7 @@ than an API-calling benchmark. | R7 | Grader: resolve selections to atom sets against the mmCIF, so equivalent namings (`label_comp_id`+atom vs residue number+atom) match and empty selections score zero | todo | | R8 | Chat driver v2: new reference in its prompt + empty-selection lint (a primitive end that matches nothing is drawn to the origin by Mol\*) | shipped (chat-driver #18, deployed 2026-10-06) | | R9 | Chat driver: thumbs up/down per turn, for grading real traffic | todo | -| R10 | Leaderboard: a regime for `source: interactions` (unknown sources currently fall into "Clinical") | todo | +| R10 | Leaderboard: regimes for `resolve` and `interactions` sources (unknown sources now raise) | done | **R5 result (2026-10-06, 1 sample, 76 tasks, ~$0.85).** Mean F1: diff --git a/molbench/mvs.py b/molbench/mvs.py index 93072e9..5ed0dbe 100644 --- a/molbench/mvs.py +++ b/molbench/mvs.py @@ -53,12 +53,15 @@ def _selector_signature(sel: Any) -> Any: comp = e.get("label_comp_id") or e.get("auth_comp_id") comp = comp.upper() if isinstance(comp, str) else comp atom = e.get("label_atom_id") or e.get("auth_atom_id") + element = e.get("type_symbol") + element = element.upper() if isinstance(element, str) else element sigs.append(( ("chain", str(chain) if chain is not None else None), ("resnum", resnum), ("range", (beg, end) if (beg is not None or end is not None) else None), ("comp", comp), ("atom", atom), + ("element", element), )) return tuple(sorted(map(str, sigs))) @@ -102,7 +105,9 @@ def _param_signature(kind: str, params: dict | None, custom: dict | None = None) else: points = tuple((k, _point_signature(params[k])) for k in _PRIMITIVE_POINTS if k in params) - return ("primitive", pkind, points) + # A 3D label's text is its content (unlike a measurement's label template). + text = (" ".join(str(params.get("text", "")).split()).casefold(),) if pkind == "label" else () + return ("primitive", pkind, points) + text if kind == "primitives": return () # a grouping node; its colour is styling if kind == "color" and custom.get("molstar_color_theme_name"): diff --git a/scripts/build_leaderboard.py b/scripts/build_leaderboard.py index c393487..6a01e21 100644 --- a/scripts/build_leaderboard.py +++ b/scripts/build_leaderboard.py @@ -34,7 +34,8 @@ NL = "\n" # How a task's `source` maps to a difficulty regime, ordered easy -> hard. -REGIMES = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)", "Resolution (by name)"] +REGIMES = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)", "Resolution (by name)", + "Interactions (atoms)"] # The first three form the difficulty gradient (Finding 03); later regimes test other skills. _SOURCE_REGIME = { "generated": "Translation", "curated": "Translation", @@ -42,6 +43,7 @@ "clinvar-variant": "Clinical (SIFTS/ClinVar)", "clinvar-hotspots": "Clinical (SIFTS/ClinVar)", "sifts-named": "Clinical (SIFTS/ClinVar)", "resolve": "Resolution (by name)", + "interactions": "Interactions (atoms)", } diff --git a/tests/test_harness.py b/tests/test_harness.py index 14aaf5f..4b962aa 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -346,6 +346,19 @@ def test_interactions_flag_survives_list_selector_expansion(): assert mvs.grade_mvs(ref, _with_structure_child(comp))["f1"] < 1.0 +def test_label_text_and_element_selectors_are_graded(): + def prim(**params): + return _with_structure_child({"kind": "primitives", "children": [ + {"kind": "primitive", "params": params}]}) + at = {"auth_seq_id": 64} + assert mvs.grade_mvs(prim(kind="label", position=at, text="His64"), + prim(kind="label", position=at, text="his64 "))["f1"] == 1.0 + assert mvs.grade_mvs(prim(kind="label", position=at, text="His64"), + prim(kind="label", position=at, text="Gly65"))["f1"] < 1.0 + o, n = ({"auth_seq_id": 64, "type_symbol": el} for el in ("O", "N")) + assert mvs.grade_mvs(prim(kind="tube", start=o, end=at), prim(kind="tube", start=n, end=at))["f1"] < 1.0 + + if __name__ == "__main__": # Run every test_* function, so a new test can't be missed by CI (which runs this file). tests = [f for name, f in sorted(globals().items()) if name.startswith("test_") and callable(f)] From 4df4c990ce5bd76c964f4e65ce38d25e0038c9b5 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Tue, 6 Oct 2026 19:51:57 +0100 Subject: [PATCH 10/10] Address Codex review: non-gradient regimes shown apart from the difficulty gradient; GFP prompt names the species Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/build_leaderboard.py | 16 ++++++++++++---- scripts/generate_resolve_tasks.py | 2 +- tasks/mvs_resolve/res-gfp.json | 4 ++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/build_leaderboard.py b/scripts/build_leaderboard.py index c393487..3ed0a11 100644 --- a/scripts/build_leaderboard.py +++ b/scripts/build_leaderboard.py @@ -34,8 +34,10 @@ NL = "\n" # How a task's `source` maps to a difficulty regime, ordered easy -> hard. -REGIMES = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)", "Resolution (by name)"] -# The first three form the difficulty gradient (Finding 03); later regimes test other skills. +GRADIENT = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)"] +# Regimes that test another skill, not a harder step on the gradient: shown below it, unordered. +OTHER_REGIMES = ["Resolution (by name)"] +REGIMES = GRADIENT + OTHER_REGIMES _SOURCE_REGIME = { "generated": "Translation", "curated": "Translation", "grounded": "Grounded (ligand/SS)", @@ -347,7 +349,7 @@ def findings(ranked: list) -> str: # Compute it live — with a wider panel this can flip, because format-unreliable # models inject parse-driven spread that swamps the difficulty signal. regimes = _task_regimes() - reg_order = REGIMES[:3] + reg_order = GRADIENT reg_spread: dict[str, float] = {} for rg in reg_order: per = [] @@ -456,10 +458,15 @@ def regime_table(ranked: list) -> str: f"{html.escape(n.split('/')[-1])}" for n, _ in cols ) body = [] + other_header = False for rg in REGIMES: n_tasks = sum(1 for v in reg.values() if v == rg) if not n_tasks: continue + if rg in OTHER_REGIMES and not other_header: + other_header = True + body.append(f"Other skills " + "(not part of the gradient)") vals = {} for n, m in cols: fs = [t["f1"] for t in m["tasks"] if reg.get(t["id"]) == rg] @@ -483,7 +490,8 @@ def regime_table(ranked: list) -> str:

Difficulty gradient by task regime

Mean MVS F1 by how the answer key is grounded, easy → hard. Harder regimes pull the models apart — the - spread you want from a benchmark.

+ spread you want from a benchmark. Rows under Other skills test a + different ability and are not ordered by difficulty.

{head_cells} diff --git a/scripts/generate_resolve_tasks.py b/scripts/generate_resolve_tasks.py index 2a936ac..c63ef3d 100644 --- a/scripts/generate_resolve_tasks.py +++ b/scripts/generate_resolve_tasks.py @@ -43,7 +43,7 @@ "Probe: Haiku answered 1D0G (death receptor 5 / TRAIL)."), ("cftr", "show me the CFTR channel", "P13569", "5uak", "Probe: Haiku answered 5UAY (a plant Toc75 POTRA domain).", 0.7), - ("gfp", "show green fluorescent protein", "P42212", "1ema", + ("gfp", "show green fluorescent protein from Aequorea victoria", "P42212", "1ema", "The starter chip's target; the model tends to be right here — a control."), ("myoglobin", "Show me sperm whale myoglobin", "P02185", "1mbn", "Corpus prompt asked for 'crab myoglobin' (no such entry); sperm whale is the classic."), diff --git a/tasks/mvs_resolve/res-gfp.json b/tasks/mvs_resolve/res-gfp.json index 9eb9385..d805057 100644 --- a/tasks/mvs_resolve/res-gfp.json +++ b/tasks/mvs_resolve/res-gfp.json @@ -3,7 +3,7 @@ "category": "mvs", "source": "resolve", "title": "Resolve by name: gfp", - "prompt": "show green fluorescent protein", + "prompt": "show green fluorescent protein from Aequorea victoria", "reference_mvs": { "kind": "root", "children": [ @@ -702,6 +702,6 @@ "uniprot": "P42212", "n_entries": 637, "source": "RCSB search, reference_sequence_identifiers", - "fetched": "2026-09-30" + "fetched": "2026-10-06" } }
Regime