diff --git a/.gitignore b/.gitignore index a9c84d6..d663cce 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ runs/ # Fetched Mol* bundle (run scripts/fetch_molstar.py) molbench/static/molstar/ + +# mmCIF cache for task authoring (scripts/author_interaction_tasks.py) +data/cif/ diff --git a/ROADMAP.md b/ROADMAP.md index 249b5e5..07d8c99 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -7,6 +7,87 @@ sharpen Component 1, and validate the instrument. --- +## Revised plan (2026-10-04) — read this first + +Three decisions, each backed by a test this round. + +**1. MVS stays the target. The bottleneck was our reference, not MVS.** +The chat driver told a user "MVS cannot show hydrogen bonds". Checked against the +Mol\* 5.9.0 source and real renders, that is wrong: `primitives` draws dashed lines +between named atoms (`distance_measurement`, labelled with the length), and Mol\* +computes H-bonds/metal contacts itself via `custom.molstar_show_non_covalent_interactions` +on a component. Our reference listed only 8 node kinds (one of them wrong), so no +model could ever draw an H-bond. A 4-prompt × 2-model A/B: old reference 0/4 +lines drawn and 2 Mol\* crashes on "50% transparent"; new reference 4/4 lines and +2/2 transparency. Models still pick wrong chains/atoms (see grounding below). +What MVS truly cannot do: select by distance or property, compute a +superposition, style the computed interactions. Those stay in the API track. + +**2. Prompting conditions: a 2×2, no public "tuned" row.** + +| | no grounding | grounding pack | +|---|---|---| +| **bare** (output contract only) | what the model knows | knows facts, not MVS | +| **spec** (vendored MVS reference) | **headline** | upper bound for an offline pipeline | + +* *Spec* = `molbench/mvs_reference.md`, a neutral, complete snapshot of MVS as + Mol\* 5.9.0 implements it. `--condition bare|spec` selects the row (done). +* *Grounding pack* = facts about the task's structure prepared **offline** and + frozen with the task: chain ids and residue ranges, ligand codes and atom names, + SIFTS/UniProt mapping, ClinVar positions, the accepted entry ids. Reproducible + by construction; no live calls during a run. Can be ablated separately + (MVS-reference vs molecular-grounding). +* The chat driver's own prompt is **tuned** on everything it sees and is owned by + `molstar-chat-driver`. It is not scored as a model comparison. Product claims + come from real user prompts that arrive *after* a prompt version is frozen + (graded by thumbs up/down or the C2 judge). Curated user prompts become new + benchmark tasks — after which they are dev data for the next prompt version. + +**3. External resources (PubMed, PDB, UniProt, ClinVar): closed-book core.** +Lookups enter only as the frozen grounding pack above. Build a live tool loop +only if grounding lifts the lookup-heavy tasks (`mvs_clinical`, `mvs_resolve`) +by ≥10 F1 points on a few cheap models. PubMed is out of scope. This keeps +MolBench a visualization benchmark that *measures* the value of lookups rather +than an API-calling benchmark. + +### Work list + +| # | item | status | +|---|---|---| +| R1 | Spec reference v2 (primitives, opacity, interactions, colour themes, fixed rep types) | done | +| R2 | Grader: primitives keyed on shape + atom pair (styling/direction ignored); interactions flag and colour theme graded | done | +| R3 | `tasks/mvs_interactions/` — 6 tasks, every atom checked against the mmCIF with gemmi (`scripts/author_interaction_tasks.py`) | done | +| R4 | `--condition bare\|spec` in the runner, recorded in run meta | done | +| R5 | Regression run: old vs spec vs bare on 3 cheap models, all MVS tasks | done — see below | +| R6 | Grounding packs: offline builder (gemmi + SIFTS + existing resolve/clinical provenance) and `--grounding` flag | todo | +| R7 | Grader: resolve selections to atom sets against the mmCIF, so equivalent namings (`label_comp_id`+atom vs residue number+atom) match and empty selections score zero | todo | +| R8 | Chat driver v2: new reference in its prompt + empty-selection lint (a primitive end that matches nothing is drawn to the origin by Mol\*) | shipped (chat-driver #18, deployed 2026-10-06) | +| R9 | Chat driver: thumbs up/down per turn, for grading real traffic | todo | +| R10 | Leaderboard: regimes for `resolve` and `interactions` sources (unknown sources now raise) | done | + +**R5 result (2026-10-06, 1 sample, 76 tasks, ~$0.85).** Mean F1: + +| model | old ref | spec | bare | interaction tasks old → spec | +|---|---|---|---|---| +| Claude Haiku 4.5 | 0.800 | 0.806 | 0.061 | 0.52 → 0.72 | +| DeepSeek V3.2 | 0.827 | 0.814 | 0.018 | 0.58 → 0.56 | +| Qwen3-30B-A3B | 0.576 | 0.522 | 0.000 | 0.22 → 0.18 | + +* **Bare ≈ 0 for every model.** MVS is not in pretraining in any usable form; + the bare row measures nothing but "can't". Keep it as one line in the paper, + not a leaderboard column. +* **Spec costs ~0.01 on the old tasks for the strong models** (within 1-sample + noise: 57 tasks better, 63 worse of 228). Part of it is real: with a richer + reference, Haiku adds unrequested ball-and-stick on the resolve tasks + (same, correct entries; precision drops). +* **The small model loses ~0.05, almost all to JSON bracket-count errors** on a + ~2.4× longer prompt (parse success 0.78 → 0.67). Longer documentation hurts + small models — a finding, and an argument for B6 (structured outputs). +* Interaction tasks are not yet solved by anyone: line endpoints are named with + the wrong atoms or chains. That is the grounding gap (R6). + +--- + ## Phase A — Component 2: render + VLM judge ← the headline Open-ended requests ("show the key hydrogen bond", "highlight the allosteric @@ -74,7 +155,8 @@ strong judge, structured protocol, self-variance reporting); rubric quality ~0.5–0.67 because the `polymer cartoon` scaffold dominates F1. Add a metric that isolates "did you select the right residues" from scene scaffolding, and report it for the grounded/clinical tiers. Re-score the clinical regime. -* **B2. Interaction/interface task type** (ChatMol-inspired). Use gemmi neighbour +* **B2. Interaction/interface task type** (ChatMol-inspired; named-atom H-bond and + computed-interaction tasks STARTED, see R3). Use gemmi neighbour search to extract interface/contact residues (ligand–protein, protein–protein), giving grounded "show the binding-site residues" / "show the PPI interface" tasks — a realistic, high-value class we currently lack. @@ -85,11 +167,10 @@ strong judge, structured protocol, self-variance reporting); rubric quality protein's UniProt accession (`accepted_ids`, folded by the grader). Next: more targets, and an `exists` sub-metric (did the id resolve at RCSB at all?) — a hallucinated id and a wrong-but-real id are different failures. -* **B5. Colour schemes.** The chat-driver's prompt now teaches Mol*'s - `custom.molstar_color_theme_name` on a `color` node for rainbow / by-chain / - by-SS colouring (models kept writing `"spectrum"` as a colour). Porting that here - needs the grader to treat a theme-coloured representation as equivalent to the - per-chain component references it would replace; until then the two prompts differ. +* **B5. Colour schemes (PARTLY DONE).** The Spec reference now teaches Mol*'s + `custom.molstar_color_theme_name` on a `color` node, and the grader scores the + theme instead of the placeholder colour. Still missing: treating a theme-coloured + representation as equivalent to the per-chain components it would replace. * **B6. Structured outputs.** Constrained decoding (`response_format: json_schema`, `strict`) with a trimmed MVS schema makes invalid trees impossible by construction. Measure validation failures + F1 against the prompt-only path on @@ -147,6 +228,8 @@ The grader is a system with its own error rate; validate it against ground truth ## Suggested sequence +*Superseded in part by the revised plan at the top: R5 → R6 → R7 come first.* + 1. **A0 renderer** (unblocks everything visual) → **A1 judge** → **A2 wire-in** → **A3 corpus** → **A4 validation**. This delivers a working Component 2. 2. In parallel / between: **B1 selection-accuracy** (cheap, sharpens current diff --git a/molbench/mvs.py b/molbench/mvs.py index dfe4044..3d7d6f8 100644 --- a/molbench/mvs.py +++ b/molbench/mvs.py @@ -53,12 +53,15 @@ def _selector_signature(sel: Any) -> Any: comp = e.get("label_comp_id") or e.get("auth_comp_id") comp = comp.upper() if isinstance(comp, str) else comp atom = e.get("label_atom_id") or e.get("auth_atom_id") + element = e.get("type_symbol") + element = element.upper() if isinstance(element, str) else element sigs.append(( ("chain", str(chain) if chain is not None else None), ("resnum", resnum), ("range", (beg, end) if (beg is not None or end is not None) else None), ("comp", comp), ("atom", atom), + ("element", element), )) return tuple(sorted(map(str, sigs))) @@ -70,10 +73,55 @@ def _color_signature(value: Any) -> Any: return value -def _param_signature(kind: str, params: dict | None) -> Any: +_PRIMITIVE_POINTS = ("start", "end", "position", "a", "b", "c") +_UNDIRECTED_PRIMITIVES = {"tube", "distance_measurement"} + + +def _point_signature(point: Any) -> Any: + """A primitive position: an [x, y, z] coordinate (order matters) or an atom selection.""" + if isinstance(point, list) and all(isinstance(v, (int, float)) for v in point): + return ("xyz", tuple(float(v) for v in point)) + return _selector_signature(point) + + +def _param_signature(kind: str, params: dict | None, custom: dict | None = None) -> Any: params = params or {} + custom = custom if isinstance(custom, dict) else {} if kind == "component" and "selector" in params: - return ("selector", _selector_signature(params["selector"])) + sig = ("selector", _selector_signature(params["selector"])) + # Mol* computes and draws the interactions itself when this flag is set, so + # it is part of what the component shows, not a styling detail. + if custom.get("molstar_show_non_covalent_interactions"): + # The radius decides which neighbours are drawn; Mol*'s default is 5 Å. + radius = custom.get("molstar_non_covalent_interactions_radius_ang", 5) + try: + radius = round(float(radius), 1) + except (TypeError, ValueError): + pass + sig += ("interactions", radius) + return sig + if kind == "primitive": + # Key on the shape and the atoms it joins. Radius, dash length and label text + # are styling. A line or distance is the same whichever end comes first; an + # arrow's direction and an angle's vertex are not, so those keep their roles. + pkind = params.get("kind") + if pkind in _UNDIRECTED_PRIMITIVES: + points = frozenset(_point_signature(params[k]) for k in _PRIMITIVE_POINTS + if k in params) + else: + points = tuple((k, _point_signature(params[k])) for k in _PRIMITIVE_POINTS + if k in params) + # A 3D label's text is its content (unlike a measurement's label template). + text = (" ".join(str(params.get("text", "")).split()).casefold(),) if pkind == "label" else () + return ("primitive", pkind, points) + text + if kind == "primitives": + return () # a grouping node; its colour is styling + if kind == "color" and custom.get("molstar_color_theme_name"): + # The theme overrides the placeholder colour, so grade the theme (and where it applies). + sig = ("theme", custom["molstar_color_theme_name"]) + if "selector" in params: + sig += (_selector_signature(params["selector"]),) + return sig if kind == "representation": # Key on the representation *type* only. The builder injects secondary # defaults (e.g. surface_type='molecular') that a model shouldn't be @@ -124,9 +172,10 @@ def walk(node: dict, prefix: tuple) -> None: # "group residues in one component" and "one component each" score alike. if kind == "component" and isinstance(params.get("selector"), list) \ and len(params["selector"]) > 1: - variants = [("selector", _selector_signature(e)) for e in params["selector"]] + flag = _param_signature(kind, {"selector": "all"}, node.get("custom"))[2:] + variants = [("selector", _selector_signature(e)) + flag for e in params["selector"]] else: - variants = [_param_signature(kind, params)] + variants = [_param_signature(kind, params, node.get("custom"))] for vsig in variants: here = prefix + ((kind, vsig),) @@ -263,14 +312,19 @@ def categorize(root: dict) -> list[str]: kinds: dict[str, int] = {} has_expr = False reps: set[str] = set() + interactions = False def walk(node: dict) -> None: - nonlocal has_expr + nonlocal has_expr, interactions k = node.get("kind") kinds[k] = kinds.get(k, 0) + 1 params = node.get("params") or {} if k == "component" and not isinstance(params.get("selector"), str): has_expr = True + custom = node.get("custom") + if k == "component" and isinstance(custom, dict) \ + and custom.get("molstar_show_non_covalent_interactions"): + interactions = True if k == "representation": reps.add(params.get("type")) for c in node.get("children") or []: @@ -295,7 +349,13 @@ def walk(node: dict) -> None: cats.append("annotation") if reps & {"surface", "isosurface", "gaussian-surface", "molecular-surface"}: cats.append("surface") - if kinds.get("volume") or kinds.get("primitives"): + if kinds.get("opacity"): + cats.append("transparency") + if kinds.get("primitive"): + cats.append("measurement") + if interactions: + cats.append("interactions") + if kinds.get("volume"): cats.append("volume") return cats diff --git a/molbench/mvs_reference.md b/molbench/mvs_reference.md index 801348b..80d8d2a 100644 --- a/molbench/mvs_reference.md +++ b/molbench/mvs_reference.md @@ -1,12 +1,15 @@ # MolViewSpec (MVS) — scene-tree reference (vendored) -> **Frozen snapshot** targeting the `molviewspec` package (v1.8.x) and the MVS -> state format rendered by `ipymolstar.MolViewSpec`. This is both the context the -> model-under-test receives and the structure the grader trusts. +> **Frozen snapshot** of the MVS scene tree as implemented by Mol\* 5.9.0 (the +> version vendored in `molbench/static/molstar/`). This is both the context the +> model-under-test receives (the **Spec** condition) and the structure the grader +> trusts. It covers the structure-scene subset of MVS; volumes, trajectories, +> annotation files (`*_from_uri`/`*_from_source`) and multi-snapshot animation are +> left out. -MVS describes a 3D molecular scene as a **tree of nodes**, independent of any -specific viewer. A scene is built by nesting: you download data, parse it, derive -a structure, carve out components, and give each a representation and colour. +MVS describes a 3D molecular scene as a **tree of nodes**. A scene is built by +nesting: you download data, parse it, derive a structure, carve out components, +and give each a representation and colour. ## State shape @@ -22,20 +25,28 @@ a structure, carve out components, and give each a representation and colour. Every node is `{"kind": , "params": {<...>}, "children": [, ...]}` (`params`/`children` omitted when empty). Nesting encodes meaning — a `color` node is a child of the `representation` it colours, which is a child of the `component` -it draws. +it draws. A node may also carry `"custom": {...}`: viewer-specific extras (see +*Mol\* extensions* below). -## Node kinds (the ones this benchmark uses) +## Node kinds -| kind | params | typical parent | meaning | +| kind | params | parent | meaning | |---|---|---|---| | `download` | `url` | root | fetch a structure file | | `parse` | `format` (`mmcif`,`bcif`,`pdb`) | download | parse the data | -| `structure` | `type` (`model`,`assembly`) | parse | build a structure | +| `structure` | `type` (`model`,`assembly`,`symmetry`), `assembly_id`, `model_index` | parse | build a structure | +| `transform` | `rotation` (3×3, flat), `translation`, or `matrix` (4×4, flat) | structure | move it by a known matrix | | `component` | `selector` | structure | select a subset to draw | -| `representation` | `type` | component | how to draw it | -| `color` | `color` | representation | colour it | -| `focus` | *(none)* | component | aim the camera at it | -| `label` | `text` | component | attach a text label | +| `representation` | `type`, `size_factor` | component | how to draw it | +| `color` | `color`, optional `selector` | representation | colour it (or only the selected part) | +| `opacity` | `opacity` (0–1; 0.5 = 50% transparent) | representation | transparency | +| `label` | `text` | component | text label on the component | +| `tooltip` | `text` | component | hover text | +| `focus` | optional `radius` | component (or root) | aim the camera at it | +| `camera` | `target`, `position`, `up` | root | set the camera explicitly | +| `canvas` | `background_color` | root | background colour | +| `primitives` | optional `color`, `label_color`, `opacity` | structure | a group of drawn shapes | +| `primitive` | `kind` + params below | primitives | one shape | **Component selectors** — a string for whole classes: `all`, `polymer`, `protein`, `nucleic`, `ligand`, `ion`, `water`, `branched`. @@ -46,14 +57,21 @@ Or a **ComponentExpression** object (or list of them) to pick specific atoms/res | `auth_asym_id` | author chain id (e.g. `"A"`) | | `auth_seq_id` | author residue number | | `label_asym_id`, `label_seq_id` | label (mmCIF) chain / residue | -| `beg_auth_seq_id`, `end_auth_seq_id` | inclusive residue range | -| `label_comp_id` | residue/ligand 3-letter code (`"HEM"`) | -| `label_atom_id` | atom name (`"FE"`) | +| `beg_auth_seq_id`, `end_auth_seq_id` | inclusive residue range (also `beg_/end_label_seq_id`) | +| `label_comp_id`, `auth_comp_id` | residue/ligand 3-letter code (`"HEM"`) | +| `label_atom_id`, `auth_atom_id` | atom name (`"FE"`, `"NE2"`) | +| `type_symbol` | element (`"FE"`) | +| `residue_index` | 0-based residue index in the file | -**Representation types:** `cartoon`, `ball_and_stick`, `spacefill`, `surface`, -`carbohydrate`, `isosurface`. +Fields in one expression are AND-ed; a list of expressions is OR-ed. A selection +that matches nothing draws nothing — it is not an error, so use real chain ids, +residue numbers and atom names. -**Colours:** CSS names (`"red"`, `"orange"`, `"steelblue"`) or hex (`"#FF6699"`). +**Representation types:** `cartoon`, `backbone`, `ball_and_stick`, `line`, +`spacefill`, `surface`, `putty`, `carbohydrate`. + +**Colours:** lowercase CSS names (`"red"`, `"orange"`, `"steelblue"`) or hex +(`"#FF6699"`). A `color` node takes ONE colour. ## Worked example @@ -77,3 +95,49 @@ Prompt: *"Load 1HHO; protein as grey cartoon, heme groups as orange ball-and-sti To select specific residues, replace the selector string with an expression, e.g. `"selector": {"auth_asym_id": "A", "auth_seq_id": 35}`. + +**Transparency:** `{"kind": "opacity", "params": {"opacity": 0.5}}` as a child of +the `representation`, beside its `color`. + +## Primitives: lines, distances, angles, 3D text + +A primitive position (`start`, `end`, `position`, `a`/`b`/`c`) is either +`[x, y, z]` or a ComponentExpression naming **one atom**, e.g. +`{"auth_asym_id": "A", "auth_seq_id": 64, "label_atom_id": "NE2"}` +(if it matches several atoms, their centre is used). + +| primitive `kind` | params | draws | +|---|---|---| +| `distance_measurement` | `start`, `end`, `color`, `radius`, `dash_length`, `label_template` | a dashed line labelled with its length (`"{{distance}}"` is replaced by it) | +| `tube` | `start`, `end`, `radius`, `dash_length` | a line, dashed if `dash_length` is set | +| `angle_measurement` | `a`, `b`, `c` | the angle at `b`, labelled | +| `arrow` | `start`, `end`, `tube_radius`, `color` | an arrow | +| `label` | `position`, `text`, `label_size` | 3D text | + +Example — *"1A6M: show the distance from the distal His64 to the bound O2"* +(the ligand code is `OXY`). The `primitives` node is a child of `structure`, next +to its `component` nodes: + +```json +{"kind": "primitives", "params": {"color": "yellow", "label_color": "yellow"}, "children": [ + {"kind": "primitive", "params": {"kind": "distance_measurement", + "start": {"auth_asym_id": "A", "auth_seq_id": 64, "label_atom_id": "NE2"}, + "end": {"auth_asym_id": "A", "label_comp_id": "OXY", "label_atom_id": "O2"}, + "radius": 0.06, "dash_length": 0.15, "label_template": "{{distance}}"}}]} +``` + +## Mol\* extensions (`custom`) + +Mol\* reads these `custom` keys; other MVS viewers ignore them. + +* **Non-covalent interactions** (H-bonds, salt bridges, metal coordination, π + stacking…), computed by Mol\*: on a `component`, set + `"custom": {"molstar_show_non_covalent_interactions": true, + "molstar_non_covalent_interactions_radius_ang": 5}`. Mol\* draws the residues + within the radius as ball-and-stick plus dashed interaction lines coloured by + type. The component needs no `representation` child for this. +* **Colour schemes** (rainbow, by chain, by secondary structure, by element): give + the representation ONE `color` child carrying + `"custom": {"molstar_color_theme_name": }` (its `color` param is a + placeholder the theme overrides). Themes: `"sequence-id"`, `"chain-id"`, + `"secondary-structure"`, `"element-symbol"`. diff --git a/molbench/runner.py b/molbench/runner.py index 9a1783f..7d743a7 100644 --- a/molbench/runner.py +++ b/molbench/runner.py @@ -50,6 +50,11 @@ def _git_commit() -> str | None: API_REFERENCE = ROOT / "api_reference.md" MVS_REFERENCE = ROOT / "mvs_reference.md" +# Prompting conditions for the scene-tree track. 'spec' hands the model the vendored +# MVS reference (the headline number); 'bare' gives only the output contract, so the +# gap between them measures what a model gains from documentation. +CONDITIONS = ("spec", "bare") + # Categories whose tasks produce a numeric F1 (vs. visual_rubric, which is VLM-judged). GRADED_CATEGORIES = ("mvs", "api_calling") @@ -123,7 +128,7 @@ def load_tasks(categories: list[str] | None) -> list[dict[str, Any]]: return tasks -def build_system_prompts() -> dict[str, str]: +def build_system_prompts(condition: str = "spec") -> dict[str, str]: """One assembled system prompt per task category. Each target speaks a different IR, so each gets its own instructions + vendored @@ -133,7 +138,11 @@ def build_system_prompts() -> dict[str, str]: api_prompt = (PROMPT_API.read_text() .replace("{{API_REFERENCE}}", API_REFERENCE.read_text()) .replace("{{JSON_SCHEMA}}", json.dumps(schema.build_json_schema(), indent=2))) - mvs_prompt = PROMPT_MVS.read_text().replace("{{MVS_REFERENCE}}", MVS_REFERENCE.read_text()) + mvs_prompt = PROMPT_MVS.read_text() + if condition == "bare": + mvs_prompt = mvs_prompt.replace("## MVS reference\n\n{{MVS_REFERENCE}}", "").rstrip() + "\n" + else: + mvs_prompt = mvs_prompt.replace("{{MVS_REFERENCE}}", MVS_REFERENCE.read_text()) return {"api_calling": api_prompt, "mvs": mvs_prompt, "visual_rubric": mvs_prompt} @@ -232,7 +241,7 @@ def _aggregate(samples: list[dict]) -> dict[str, Any]: def run(models: list[str], categories: list[str] | None, samples: int = 1, escalate: bool = False, - judge: str | None = None) -> tuple[dict[str, Any], dict[str, Any]]: + judge: str | None = None, condition: str = "spec") -> tuple[dict[str, Any], dict[str, Any]]: """Returns (report, raw_samples). `report` is the lightweight scorecard (aggregates + best-sample prediction). @@ -250,7 +259,7 @@ def run(models: list[str], categories: list[str] | None, tasks = load_tasks(categories) if not tasks: sys.exit("no tasks matched; check tasks/ and --categories") - prompts = build_system_prompts() + prompts = build_system_prompts(condition) vlm = StubVLMJudge() report: dict[str, Any] = { @@ -262,6 +271,7 @@ def run(models: list[str], categories: list[str] | None, "categories": categories, "escalate": escalate, "judge": judge, + "condition": condition, }, } @@ -422,6 +432,9 @@ def main(argv: list[str] | None = None) -> None: ap.add_argument("--judge", default=None, help="vision model id for the T2 VLM tier (e.g. claude-opus-4-8); " "off by default (T0/T1 only)") + ap.add_argument("--condition", choices=CONDITIONS, default="spec", + help="MVS prompting condition: 'spec' (with the MVS reference) or " + "'bare' (output contract only)") ap.add_argument("--out", default=str(RESULTS_DIR / "scorecard.json")) ap.add_argument("--no-archive", action="store_true", help="skip writing the timestamped per-run archive in runs/") @@ -429,7 +442,8 @@ def main(argv: list[str] | None = None) -> None: load_dotenv() # make keys in .env available before any model is built report, raw_samples = run(args.models, args.categories, samples=args.samples, - escalate=args.escalate, judge=args.judge) + escalate=args.escalate, judge=args.judge, + condition=args.condition) RESULTS_DIR.mkdir(exist_ok=True) pathlib.Path(args.out).write_text(json.dumps(report, indent=2)) print_scorecard(report) diff --git a/scripts/author_interaction_tasks.py b/scripts/author_interaction_tasks.py new file mode 100644 index 0000000..458d835 --- /dev/null +++ b/scripts/author_interaction_tasks.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +""" +Author the interaction / measurement MVS tasks (``tasks/mvs_interactions/``). + +These exercise the parts of MVS the original corpus never asked for: ``primitives`` +(dashed lines between named atoms, labelled with their length), Mol*'s computed +non-covalent interactions (``custom.molstar_show_non_covalent_interactions``) and +``opacity``. A user asking to "show the hydrogen bond" is a common viewer request +that the old reference could not express. + +Like ``author_mvs_tasks.py``, each reference is built with the ``molviewspec`` +builder. In addition, every ComponentExpression in a reference is resolved against +the real mmCIF with gemmi before the task is written: a selection that matches no +atom is not an error in Mol* (a primitive end silently lands at the origin), so the +ground truth must be checked here. Each measured distance is recorded in the task's +provenance. + + python scripts/author_interaction_tasks.py # (re)generate tasks/mvs_interactions/*.json +""" + +from __future__ import annotations + +import json +import pathlib +import urllib.request + +import gemmi +from molviewspec import ComponentExpression, create_builder + +from molbench.mvs import categorize + +REPO = pathlib.Path(__file__).resolve().parent.parent +OUT = REPO / "tasks" / "mvs_interactions" +CACHE = REPO / "data" / "cif" +CIF = "https://files.rcsb.org/download/{}.cif" + + +def _structure(pdb: str): + b = create_builder() + s = b.download(url=CIF.format(pdb)).parse(format="mmcif").model_structure() + return b, s + + +def _atom(chain: str, atom: str, seq: int | None = None, comp: str | None = None) -> dict: + """A ComponentExpression naming one atom (by residue number or by ligand code).""" + e = {"auth_asym_id": chain, "label_atom_id": atom} + if seq is not None: + e["auth_seq_id"] = seq + if comp is not None: + e["label_comp_id"] = comp + return e + + +def _sticks(s, chain: str, seq: int | None = None, comp: str | None = None) -> None: + kw = {"auth_seq_id": seq} if seq is not None else {"label_comp_id": comp} + s.component(selector=ComponentExpression(auth_asym_id=chain, **kw)) \ + .representation(type="ball_and_stick") + + +def _cartoon(s) -> None: + s.component(selector="polymer").representation(type="cartoon").color(color="gray") + + +# --- task definitions ------------------------------------------------------------- + +def t_his64_o2(): + b, s = _structure("1a6m") + _cartoon(s) + _sticks(s, "A", seq=64) + _sticks(s, "A", comp="OXY") + s.primitives().distance(start=_atom("A", "NE2", seq=64), end=_atom("A", "O2", comp="OXY"), + dash_length=0.15, label_template="{{distance}}") + return b + + +def t_fe_his93(): + b, s = _structure("1a6m") + _cartoon(s) + _sticks(s, "A", comp="HEM") + _sticks(s, "A", seq=93) + s.primitives().distance(start=_atom("A", "FE", comp="HEM"), end=_atom("A", "NE2", seq=93), + dash_length=0.15, label_template="{{distance}}") + return b + + +def t_triad(): + b, s = _structure("3ptb") + _cartoon(s) + for r in (57, 102, 195): + _sticks(s, "A", seq=r) + p = s.primitives() + p.distance(start=_atom("A", "OG", seq=195), end=_atom("A", "NE2", seq=57), + dash_length=0.15, label_template="{{distance}}") + p.distance(start=_atom("A", "ND1", seq=57), end=_atom("A", "OD2", seq=102), + dash_length=0.15, label_template="{{distance}}") + return b + + +def t_o2_interactions(): + b, s = _structure("1a6m") + _cartoon(s) + s.component(selector=ComponentExpression(label_comp_id="OXY"), + custom={"molstar_show_non_covalent_interactions": True}) + return b + + +def t_benzamidine_interactions(): + b, s = _structure("3ptb") + _cartoon(s) + s.component(selector=ComponentExpression(label_comp_id="BEN"), + custom={"molstar_show_non_covalent_interactions": True}) + return b + + +def t_transparent_surface(): + b, s = _structure("1hho") + s.component(selector="polymer").representation(type="surface").opacity(opacity=0.5) + s.component(selector="ligand").representation(type="ball_and_stick") + return b + + +TASKS = [ + {"id": "int-001", "pdb": "1a6m", "build": t_his64_o2, + "title": "H-bond: distal His64 to bound O2", + "prompt": "In oxymyoglobin 1A6M, show the protein as a grey cartoon, His64 and the bound " + "O2 as ball-and-stick, and draw the hydrogen bond from His64 NE2 to the O2 " + "molecule's terminal oxygen (atom O2) as a dashed line labelled with its length.", + "skills": ["primitives", "atom-selection", "hbond"]}, + {"id": "int-002", "pdb": "1a6m", "build": t_fe_his93, + "title": "Distance: heme iron to proximal histidine", + "prompt": "Load 1A6M with the protein as a grey cartoon. Show the heme and the proximal " + "histidine as ball-and-stick and measure the distance from the heme iron to the " + "histidine's coordinating nitrogen.", + "skills": ["primitives", "atom-selection", "domain-knowledge"]}, + {"id": "int-003", "pdb": "3ptb", "build": t_triad, + "title": "Catalytic-triad H-bonds in trypsin", + "prompt": "In trypsin (3PTB), show the protein as a grey cartoon and the catalytic triad " + "(His57, Asp102, Ser195) as ball-and-stick. Draw the two triad hydrogen bonds as " + "dashed lines with their lengths.", + "skills": ["primitives", "atom-selection", "hbond", "domain-knowledge"]}, + {"id": "int-004", "pdb": "1a6m", "build": t_o2_interactions, + "title": "Computed interactions around bound O2", + "prompt": "Load 1A6M with the protein as a grey cartoon and show all the non-covalent " + "interactions (hydrogen bonds, metal contacts) around the bound oxygen molecule.", + "skills": ["interactions"]}, + {"id": "int-005", "pdb": "3ptb", "build": t_benzamidine_interactions, + "title": "Computed interactions around benzamidine", + "prompt": "Show trypsin 3PTB as a grey cartoon and display the interactions between the " + "benzamidine inhibitor and the protein.", + "skills": ["interactions"]}, + {"id": "int-006", "pdb": "1hho", "build": t_transparent_surface, + "title": "Half-transparent surface over the hemes", + "prompt": "Load haemoglobin 1HHO; show the protein as a 50% transparent surface and the " + "heme groups as ball-and-stick.", + "skills": ["opacity", "surface"]}, +] + + +# --- ground-truth checks ---------------------------------------------------------- + +def _load(pdb: str) -> gemmi.Structure: + CACHE.mkdir(parents=True, exist_ok=True) + path = CACHE / f"{pdb}.cif" + if not path.exists(): + urllib.request.urlretrieve(CIF.format(pdb), path) + return gemmi.read_structure(str(path)) + + +def _match(st: gemmi.Structure, e: dict) -> list[gemmi.Atom]: + """Atoms a ComponentExpression selects (the fields this script uses).""" + if not isinstance(e, dict): + return [] # an [x, y, z] coordinate, not a selection + out = [] + for chain in st[0]: + if e.get("auth_asym_id") not in (None, chain.name): + continue + for res in chain: + if e.get("auth_seq_id") not in (None, res.seqid.num): + continue + if e.get("label_comp_id") not in (None, res.name): + continue + out += [a for a in res if e.get("label_atom_id") in (None, a.name)] + return out + + +def _check(root: dict, st: gemmi.Structure, task_id: str) -> list[dict]: + """Fail on any selection that matches nothing; return each measured distance.""" + measured = [] + + def walk(node: dict) -> None: + params = node.get("params") or {} + sel = params.get("selector") + if node.get("kind") == "component" and isinstance(sel, dict): + assert _match(st, sel), f"{task_id}: selector matches nothing: {sel}" + if node.get("kind") == "primitive": + points = {k: params[k] for k in ("start", "end", "position", "a", "b", "c") + if isinstance(params.get(k), dict)} + atoms = {k: _match(st, e) for k, e in points.items()} + assert all(atoms.values()), f"{task_id}: primitive point matches nothing: {params}" + if "start" in atoms and "end" in atoms: + # Mol* uses the centre of an end's atoms (alt locs included); record the first. + measured.append({"start": params["start"], "end": params["end"], + "distance_A": round(atoms["start"][0].pos.dist(atoms["end"][0].pos), 2)}) + for c in node.get("children") or []: + walk(c) + + walk(root) + return measured + + +def main() -> None: + OUT.mkdir(parents=True, exist_ok=True) + for t in TASKS: + root = json.loads(t["build"]().get_state().dumps())["root"] + measured = _check(root, _load(t["pdb"]), t["id"]) + doc = { + "id": t["id"], + "category": "mvs", + "source": "interactions", + "pdb": t["pdb"], + "title": t["title"], + "prompt": t["prompt"], + "reference_mvs": root, + "categories": categorize(root), + "skills": t["skills"], + "provenance": {"checked_against": CIF.format(t["pdb"]), "measured": measured}, + } + path = OUT / f"{t['id'].replace('-', '_')}.json" + path.write_text(json.dumps(doc, indent=2) + "\n") + print(f"wrote {path.relative_to(REPO)} {[m['distance_A'] for m in measured]}") + + +if __name__ == "__main__": + main() diff --git a/scripts/build_leaderboard.py b/scripts/build_leaderboard.py index 3ed0a11..80326a5 100644 --- a/scripts/build_leaderboard.py +++ b/scripts/build_leaderboard.py @@ -36,7 +36,7 @@ # How a task's `source` maps to a difficulty regime, ordered easy -> hard. GRADIENT = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)"] # Regimes that test another skill, not a harder step on the gradient: shown below it, unordered. -OTHER_REGIMES = ["Resolution (by name)"] +OTHER_REGIMES = ["Resolution (by name)", "Interactions (atoms)"] REGIMES = GRADIENT + OTHER_REGIMES _SOURCE_REGIME = { "generated": "Translation", "curated": "Translation", @@ -44,6 +44,7 @@ "clinvar-variant": "Clinical (SIFTS/ClinVar)", "clinvar-hotspots": "Clinical (SIFTS/ClinVar)", "sifts-named": "Clinical (SIFTS/ClinVar)", "resolve": "Resolution (by name)", + "interactions": "Interactions (atoms)", } diff --git a/tasks/mvs_interactions/int_001.json b/tasks/mvs_interactions/int_001.json new file mode 100644 index 0000000..39412ee --- /dev/null +++ b/tasks/mvs_interactions/int_001.json @@ -0,0 +1,148 @@ +{ + "id": "int-001", + "category": "mvs", + "source": "interactions", + "pdb": "1a6m", + "title": "H-bond: distal His64 to bound O2", + "prompt": "In oxymyoglobin 1A6M, show the protein as a grey cartoon, His64 and the bound O2 as ball-and-stick, and draw the hydrogen bond from His64 NE2 to the O2 molecule's terminal oxygen (atom O2) as a dashed line labelled with its length.", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1a6m.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + }, + "children": [ + { + "kind": "color", + "params": { + "color": "gray" + } + } + ] + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "auth_asym_id": "A", + "auth_seq_id": 64 + } + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "auth_asym_id": "A", + "label_comp_id": "OXY" + } + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + }, + { + "kind": "primitives", + "params": {}, + "children": [ + { + "kind": "primitive", + "params": { + "start": { + "auth_asym_id": "A", + "label_atom_id": "NE2", + "auth_seq_id": 64 + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "O2", + "label_comp_id": "OXY" + }, + "dash_length": 0.15, + "kind": "distance_measurement", + "label_template": "{{distance}}" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "categories": [ + "load", + "selection", + "representation", + "color", + "multi-component", + "measurement" + ], + "skills": [ + "primitives", + "atom-selection", + "hbond" + ], + "provenance": { + "checked_against": "https://files.rcsb.org/download/1a6m.cif", + "measured": [ + { + "start": { + "auth_asym_id": "A", + "label_atom_id": "NE2", + "auth_seq_id": 64 + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "O2", + "label_comp_id": "OXY" + }, + "distance_A": 2.97 + } + ] + } +} diff --git a/tasks/mvs_interactions/int_002.json b/tasks/mvs_interactions/int_002.json new file mode 100644 index 0000000..0ec1b88 --- /dev/null +++ b/tasks/mvs_interactions/int_002.json @@ -0,0 +1,148 @@ +{ + "id": "int-002", + "category": "mvs", + "source": "interactions", + "pdb": "1a6m", + "title": "Distance: heme iron to proximal histidine", + "prompt": "Load 1A6M with the protein as a grey cartoon. Show the heme and the proximal histidine as ball-and-stick and measure the distance from the heme iron to the histidine's coordinating nitrogen.", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1a6m.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + }, + "children": [ + { + "kind": "color", + "params": { + "color": "gray" + } + } + ] + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "auth_asym_id": "A", + "label_comp_id": "HEM" + } + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "auth_asym_id": "A", + "auth_seq_id": 93 + } + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + }, + { + "kind": "primitives", + "params": {}, + "children": [ + { + "kind": "primitive", + "params": { + "start": { + "auth_asym_id": "A", + "label_atom_id": "FE", + "label_comp_id": "HEM" + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "NE2", + "auth_seq_id": 93 + }, + "dash_length": 0.15, + "kind": "distance_measurement", + "label_template": "{{distance}}" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "categories": [ + "load", + "selection", + "representation", + "color", + "multi-component", + "measurement" + ], + "skills": [ + "primitives", + "atom-selection", + "domain-knowledge" + ], + "provenance": { + "checked_against": "https://files.rcsb.org/download/1a6m.cif", + "measured": [ + { + "start": { + "auth_asym_id": "A", + "label_atom_id": "FE", + "label_comp_id": "HEM" + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "NE2", + "auth_seq_id": 93 + }, + "distance_A": 2.06 + } + ] + } +} diff --git a/tasks/mvs_interactions/int_003.json b/tasks/mvs_interactions/int_003.json new file mode 100644 index 0000000..8af2f36 --- /dev/null +++ b/tasks/mvs_interactions/int_003.json @@ -0,0 +1,197 @@ +{ + "id": "int-003", + "category": "mvs", + "source": "interactions", + "pdb": "3ptb", + "title": "Catalytic-triad H-bonds in trypsin", + "prompt": "In trypsin (3PTB), show the protein as a grey cartoon and the catalytic triad (His57, Asp102, Ser195) as ball-and-stick. Draw the two triad hydrogen bonds as dashed lines with their lengths.", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/3ptb.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + }, + "children": [ + { + "kind": "color", + "params": { + "color": "gray" + } + } + ] + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "auth_asym_id": "A", + "auth_seq_id": 57 + } + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "auth_asym_id": "A", + "auth_seq_id": 102 + } + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "auth_asym_id": "A", + "auth_seq_id": 195 + } + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + }, + { + "kind": "primitives", + "params": {}, + "children": [ + { + "kind": "primitive", + "params": { + "start": { + "auth_asym_id": "A", + "label_atom_id": "OG", + "auth_seq_id": 195 + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "NE2", + "auth_seq_id": 57 + }, + "dash_length": 0.15, + "kind": "distance_measurement", + "label_template": "{{distance}}" + } + }, + { + "kind": "primitive", + "params": { + "start": { + "auth_asym_id": "A", + "label_atom_id": "ND1", + "auth_seq_id": 57 + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "OD2", + "auth_seq_id": 102 + }, + "dash_length": 0.15, + "kind": "distance_measurement", + "label_template": "{{distance}}" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "categories": [ + "load", + "selection", + "representation", + "color", + "multi-component", + "measurement" + ], + "skills": [ + "primitives", + "atom-selection", + "hbond", + "domain-knowledge" + ], + "provenance": { + "checked_against": "https://files.rcsb.org/download/3ptb.cif", + "measured": [ + { + "start": { + "auth_asym_id": "A", + "label_atom_id": "OG", + "auth_seq_id": 195 + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "NE2", + "auth_seq_id": 57 + }, + "distance_A": 2.96 + }, + { + "start": { + "auth_asym_id": "A", + "label_atom_id": "ND1", + "auth_seq_id": 57 + }, + "end": { + "auth_asym_id": "A", + "label_atom_id": "OD2", + "auth_seq_id": 102 + }, + "distance_A": 2.67 + } + ] + } +} diff --git a/tasks/mvs_interactions/int_004.json b/tasks/mvs_interactions/int_004.json new file mode 100644 index 0000000..fd8552e --- /dev/null +++ b/tasks/mvs_interactions/int_004.json @@ -0,0 +1,85 @@ +{ + "id": "int-004", + "category": "mvs", + "source": "interactions", + "pdb": "1a6m", + "title": "Computed interactions around bound O2", + "prompt": "Load 1A6M with the protein as a grey cartoon and show all the non-covalent interactions (hydrogen bonds, metal contacts) around the bound oxygen molecule.", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1a6m.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + }, + "children": [ + { + "kind": "color", + "params": { + "color": "gray" + } + } + ] + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "label_comp_id": "OXY" + } + }, + "custom": { + "molstar_show_non_covalent_interactions": true + } + } + ] + } + ] + } + ] + } + ] + }, + "categories": [ + "load", + "selection", + "representation", + "color", + "multi-component", + "interactions" + ], + "skills": [ + "interactions" + ], + "provenance": { + "checked_against": "https://files.rcsb.org/download/1a6m.cif", + "measured": [] + } +} diff --git a/tasks/mvs_interactions/int_005.json b/tasks/mvs_interactions/int_005.json new file mode 100644 index 0000000..723f50b --- /dev/null +++ b/tasks/mvs_interactions/int_005.json @@ -0,0 +1,85 @@ +{ + "id": "int-005", + "category": "mvs", + "source": "interactions", + "pdb": "3ptb", + "title": "Computed interactions around benzamidine", + "prompt": "Show trypsin 3PTB as a grey cartoon and display the interactions between the benzamidine inhibitor and the protein.", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/3ptb.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + }, + "children": [ + { + "kind": "color", + "params": { + "color": "gray" + } + } + ] + } + ] + }, + { + "kind": "component", + "params": { + "selector": { + "label_comp_id": "BEN" + } + }, + "custom": { + "molstar_show_non_covalent_interactions": true + } + } + ] + } + ] + } + ] + } + ] + }, + "categories": [ + "load", + "selection", + "representation", + "color", + "multi-component", + "interactions" + ], + "skills": [ + "interactions" + ], + "provenance": { + "checked_against": "https://files.rcsb.org/download/3ptb.cif", + "measured": [] + } +} diff --git a/tasks/mvs_interactions/int_006.json b/tasks/mvs_interactions/int_006.json new file mode 100644 index 0000000..9321636 --- /dev/null +++ b/tasks/mvs_interactions/int_006.json @@ -0,0 +1,89 @@ +{ + "id": "int-006", + "category": "mvs", + "source": "interactions", + "pdb": "1hho", + "title": "Half-transparent surface over the hemes", + "prompt": "Load haemoglobin 1HHO; show the protein as a 50% transparent surface and the heme groups as ball-and-stick.", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1hho.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "surface", + "surface_type": "molecular" + }, + "children": [ + { + "kind": "opacity", + "params": { + "opacity": 0.5 + } + } + ] + } + ] + }, + { + "kind": "component", + "params": { + "selector": "ligand" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "ball_and_stick" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "categories": [ + "load", + "representation", + "multi-component", + "surface", + "transparency" + ], + "skills": [ + "opacity", + "surface" + ], + "provenance": { + "checked_against": "https://files.rcsb.org/download/1hho.cif", + "measured": [] + } +} diff --git a/tests/test_harness.py b/tests/test_harness.py index 97b6c3c..e9df5b6 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -259,6 +259,123 @@ def test_escalation_renders_the_reference_on_the_accepted_entry(): is task["reference_mvs"] +def _with_structure_child(child: dict) -> dict: + s = _scene() + s["root"]["children"][0]["children"][0]["children"][0]["children"].append(child) + return s + + +def _hbond(start: dict, end: dict, **style) -> dict: + return {"kind": "primitives", "params": style.pop("group", {}), "children": [ + {"kind": "primitive", "params": {"kind": "distance_measurement", + "start": start, "end": end, **style}}]} + + +_HIS = {"auth_asym_id": "A", "auth_seq_id": 64, "label_atom_id": "NE2"} +_O2 = {"auth_asym_id": "A", "label_comp_id": "OXY", "label_atom_id": "O2"} + + +def test_primitive_ignores_styling_and_direction(): + ref = _with_structure_child(_hbond(_HIS, _O2)) + pred = _with_structure_child(_hbond(_O2, _HIS, radius=0.1, dash_length=0.3, + label_template="H-bond {{distance}}", + group={"color": "yellow"})) + assert mvs.grade_mvs(ref, pred)["f1"] == 1.0 + + +def test_primitive_wrong_atom_loses_credit(): + ref = _with_structure_child(_hbond(_HIS, _O2)) + pred = _with_structure_child(_hbond({**_HIS, "label_atom_id": "ND1"}, _O2)) + assert mvs.grade_mvs(ref, pred)["f1"] < 1.0 + + +def test_interactions_flag_is_graded(): + comp = {"kind": "component", "params": {"selector": {"label_comp_id": "OXY"}}} + ref = _with_structure_child({**comp, "custom": {"molstar_show_non_covalent_interactions": True}}) + assert mvs.grade_mvs(ref, ref)["f1"] == 1.0 + assert mvs.grade_mvs(ref, _with_structure_child(comp))["f1"] < 1.0 + assert "interactions" in mvs.categorize(mvs.extract_root(ref)) + + +def test_colour_theme_grades_the_theme_not_the_placeholder(): + def themed(theme, placeholder): + s = _scene(placeholder) + rep = s["root"]["children"][0]["children"][0]["children"][0]["children"][0]["children"][0] + rep["children"][0]["custom"] = {"molstar_color_theme_name": theme} + return s + assert mvs.grade_mvs(themed("chain-id", "gray"), themed("chain-id", "white"))["f1"] == 1.0 + assert mvs.grade_mvs(themed("chain-id", "gray"), themed("sequence-id", "gray"))["f1"] < 1.0 + + +def test_bare_condition_drops_the_reference(): + from molbench.runner import build_system_prompts + spec, bare = build_system_prompts("spec")["mvs"], build_system_prompts("bare")["mvs"] + assert "Node kinds" in spec and "Node kinds" not in bare + assert "{{MVS_REFERENCE}}" not in bare + + +def test_primitive_coordinates_keep_their_order(): + a = _with_structure_child({"kind": "primitives", "children": [{"kind": "primitive", "params": { + "kind": "label", "position": [1.0, 2.0, 3.0], "text": "x"}}]}) + b = _with_structure_child({"kind": "primitives", "children": [{"kind": "primitive", "params": { + "kind": "label", "position": [3.0, 2.0, 1.0], "text": "x"}}]}) + assert mvs.grade_mvs(a, a)["f1"] == 1.0 + assert mvs.grade_mvs(a, b)["f1"] < 1.0 + + +def test_categorize_survives_non_dict_custom(): + s = _with_structure_child({"kind": "component", "params": {"selector": "ligand"}, "custom": ["x"]}) + assert "interactions" not in mvs.categorize(mvs.extract_root(s)) + + +def test_arrow_direction_and_angle_vertex_are_graded(): + def prim(**params): + return _with_structure_child({"kind": "primitives", "children": [ + {"kind": "primitive", "params": params}]}) + a, b, c = ({"auth_seq_id": n} for n in (1, 2, 3)) + assert mvs.grade_mvs(prim(kind="arrow", start=a, end=b), prim(kind="arrow", start=b, end=a))["f1"] < 1.0 + assert mvs.grade_mvs(prim(kind="angle_measurement", a=a, b=b, c=c), + prim(kind="angle_measurement", a=b, b=a, c=c))["f1"] < 1.0 + + +def test_interactions_flag_survives_list_selector_expansion(): + sel = [{"label_comp_id": "OXY"}, {"label_comp_id": "HEM"}] + comp = {"kind": "component", "params": {"selector": sel}} + ref = _with_structure_child({**comp, "custom": {"molstar_show_non_covalent_interactions": True}}) + assert mvs.grade_mvs(ref, ref)["f1"] == 1.0 + assert mvs.grade_mvs(ref, _with_structure_child(comp))["f1"] < 1.0 + + +def test_label_text_and_element_selectors_are_graded(): + def prim(**params): + return _with_structure_child({"kind": "primitives", "children": [ + {"kind": "primitive", "params": params}]}) + at = {"auth_seq_id": 64} + assert mvs.grade_mvs(prim(kind="label", position=at, text="His64"), + prim(kind="label", position=at, text="his64 "))["f1"] == 1.0 + assert mvs.grade_mvs(prim(kind="label", position=at, text="His64"), + prim(kind="label", position=at, text="Gly65"))["f1"] < 1.0 + o, n = ({"auth_seq_id": 64, "type_symbol": el} for el in ("O", "N")) + assert mvs.grade_mvs(prim(kind="tube", start=o, end=at), prim(kind="tube", start=n, end=at))["f1"] < 1.0 + + + +def test_interaction_radius_and_theme_selector_are_graded(): + def comp(**custom): + return _with_structure_child({"kind": "component", "params": {"selector": {"label_comp_id": "OXY"}}, + "custom": {"molstar_show_non_covalent_interactions": True, **custom}}) + assert mvs.grade_mvs(comp(), comp(molstar_non_covalent_interactions_radius_ang=5))["f1"] == 1.0 + assert mvs.grade_mvs(comp(), comp(molstar_non_covalent_interactions_radius_ang=12))["f1"] < 1.0 + + def themed(sel): + s = _scene() + color = s["root"]["children"][0]["children"][0]["children"][0]["children"][0]["children"][0]["children"][0] + color["custom"] = {"molstar_color_theme_name": "chain-id"} + color["params"]["selector"] = sel + return s + assert mvs.grade_mvs(themed({"auth_seq_id": 1}), themed({"auth_seq_id": 1}))["f1"] == 1.0 + assert mvs.grade_mvs(themed({"auth_seq_id": 1}), themed({"auth_seq_id": 9}))["f1"] < 1.0 + if __name__ == "__main__": # Run every test_* function, so a new test can't be missed by CI (which runs this file). tests = [f for name, f in sorted(globals().items()) if name.startswith("test_") and callable(f)]