From 7293fa77c4f44712df3375a7a92879df24c4fd57 Mon Sep 17 00:00:00 2001 From: ocean-debug <59615229+ocean-debug@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:13:09 +0800 Subject: [PATCH] ready adapter --- CHANGELOG.md | 2 + README.md | 6 +- .../generators/codex_skill_generator.py | 141 +++++++++++++++ paper2skill/inference/infer_bio_contract.py | 58 ++++-- paper2skill/inference/infer_workflow.py | 38 +++- paper2skill/miners/api_miner.py | 24 +++ paper2skill/schemas/adapter_spec_schema.yaml | 14 ++ paper2skill/schemas/workflow_dag_schema.yaml | 3 + paper2skill/templates/codex_skill/SKILL.md.j2 | 4 +- .../scripts/adapters/cli_adapter.py.j2 | 4 +- .../scripts/adapters/notebook_adapter.py.j2 | 4 +- .../scripts/adapters/python_api_adapter.py.j2 | 54 +++++- .../templates/codex_skill/scripts/run.py.j2 | 81 +++++++-- paper2skill/validators/skill_validator.py | 90 +++++++++- .../test_bioinformatics_benchmarks.py | 111 ++++++++++++ tests/test_bio_contract_inference.py | 33 ++++ tests/test_generated_skill.py | 170 ++++++++++++++++++ tests/test_generator.py | 56 ++++++ tests/test_workflow_dag.py | 14 +- 19 files changed, 859 insertions(+), 48 deletions(-) create mode 100644 paper2skill/schemas/adapter_spec_schema.yaml create mode 100644 paper2skill/schemas/workflow_dag_schema.yaml create mode 100644 tests/benchmarks/test_bioinformatics_benchmarks.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 371a517..512abd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,3 +5,5 @@ - Added resolved repository collection as the main build input path. - Added workflow DAG, bio IO contract enrichment, evidence decisions, and adapter scaffolding. - Added dependency/API miner coverage for environment files, CLI entrypoints, R exports, and workflow engines. +- Added `adapter_spec.yaml`, blocked non-demo adapter semantics, and a ready toy Python API adapter. +- Added mini offline bioinformatics benchmarks and stricter generated skill validation for adapter, workflow, and environment specs. diff --git a/README.md b/README.md index c3ad260..898597c 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,7 @@ references/tutorial_candidates.json references/tutorial_scanner_report.json references/io_contract.yaml references/evidence_graph.json +references/adapter_spec.yaml references/build_report.json ``` @@ -114,6 +115,8 @@ Supported now: - Python and R toy skill generation. - Tutorial scanning, dependency mining, workflow DAG inference, and basic bio IO contracts. - Environment preflight, install planning, and gated disposable conda environment creation. +- Adapter readiness tracking via `references/adapter_spec.yaml`. +- Demo-only execution and blocked non-demo runs when adapters are not ready. Experimental: @@ -121,6 +124,7 @@ Experimental: - Evidence graph conflict decisions. - Source-aware bio contract inference. - Adapter-based execution scaffolding. +- Mini offline bioinformatics benchmark coverage for Scanpy, Seurat, DESeq2-like, CLI, and workflow-engine repository shapes. Not yet: @@ -129,7 +133,7 @@ Not yet: - CUDA/system library automatic configuration. - Large dataset download automation. - Arbitrary install script execution. -- Fully automatic real algorithm execution for unknown repositories. +- Fully automatic real algorithm execution for unknown repositories. Unknown adapters stay `candidate` or `blocked` and do not run as successful demo fallbacks. ## Maturity Levels diff --git a/paper2skill/generators/codex_skill_generator.py b/paper2skill/generators/codex_skill_generator.py index 365ed08..0b385d6 100644 --- a/paper2skill/generators/codex_skill_generator.py +++ b/paper2skill/generators/codex_skill_generator.py @@ -3,6 +3,7 @@ from dataclasses import asdict, dataclass import json from pathlib import Path +import shutil from typing import Any import jinja2 @@ -141,12 +142,117 @@ def infer_adapter_type(repo_evidence: dict[str, Any], tutorial_trace: dict[str, return "workflow_engine" if repo_evidence.get("package_type") == "r_package" or classification.get("language") == "r": return "r_script" + if classification.get("language") == "python" and repo_evidence.get("api_functions"): + return "python_api" tutorials = tutorial_trace.get("tutorials", []) if tutorials and all(trace.get("language") == "python" for trace in tutorials) and not repo_evidence.get("api_functions"): return "notebook" return "demo_only" +TRUSTED_READY_PYTHON_FIXTURES = { + (PROJECT_ROOT / "tests" / "fixtures" / "toy_python_algorithm").resolve(), +} + + +def allow_ready_python_adapter(repo_path: Path | None) -> bool: + if repo_path is None: + return False + try: + resolved = repo_path.resolve() + except OSError: + return False + return resolved in TRUSTED_READY_PYTHON_FIXTURES + + +def build_adapter_spec( + adapter_type: str, + repo_evidence: dict[str, Any], + maturity_level: str, + *, + allow_ready_adapter: bool = False, +) -> dict[str, Any]: + spec = { + "adapter_type": adapter_type, + "status": "candidate", + "entrypoint": None, + "command": None, + "module": None, + "function": None, + "evidence": [], + "caveats": [], + } + if adapter_type == "demo_only": + spec["status"] = "demo_only" + spec["caveats"] = ["demo summary runner only; no real algorithm adapter was inferred"] + return spec + if adapter_type == "python_api": + candidate = select_python_api_candidate(repo_evidence) + if candidate: + module = candidate.get("public_module") or candidate.get("module") + function = candidate.get("name") + spec.update( + { + "entrypoint": f"{module}:{function}", + "module": module, + "function": function, + "evidence": [candidate.get("path", "api_miner")], + "caveats": ["Python API adapter remains candidate-only until the entrypoint is explicitly reviewed"], + } + ) + if allow_ready_adapter and maturity_level == "L2" and function == "summarize": + spec["status"] = "ready" + spec["caveats"] = ["Ready adapter uses a generated source snapshot under sources/repo"] + return spec + spec["status"] = "blocked" + spec["caveats"] = ["No importable Python API function was inferred"] + return spec + if adapter_type == "cli": + entrypoint = first_item(repo_evidence.get("entrypoints", [])) + command = entrypoint.get("name") if entrypoint else None + spec.update( + { + "entrypoint": entrypoint.get("target") if entrypoint else None, + "command": command, + "evidence": [entrypoint.get("source")] if entrypoint else [], + "caveats": ["CLI command is a candidate and must be user-reviewed before execution"], + } + ) + return spec + if adapter_type == "workflow_engine": + engine = first_item(repo_evidence.get("workflow_engines", [])) + spec.update( + { + "entrypoint": engine.get("engine") if engine else None, + "command": engine.get("engine") if engine else None, + "evidence": engine.get("files", []) if engine else [], + "caveats": ["Workflow engine execution is candidate-only and is not run automatically"], + } + ) + return spec + if adapter_type == "r_script": + spec["caveats"] = ["R script adapter requires user-reviewed function or script wiring"] + return spec + if adapter_type == "notebook": + spec["caveats"] = ["Notebook adapter requires user-reviewed execution wiring"] + return spec + spec["status"] = "blocked" + spec["caveats"] = ["Unsupported adapter type"] + return spec + + +def select_python_api_candidate(repo_evidence: dict[str, Any]) -> dict[str, Any] | None: + functions = [item for item in repo_evidence.get("api_functions", []) if item.get("module") and item.get("name")] + for item in functions: + if item.get("name") == "summarize": + return item + return first_item(functions) + + +def first_item(values: list[dict[str, Any]] | None) -> dict[str, Any] | None: + return values[0] if values else None + + def build_context( *, skill_name: str | None = None, @@ -204,6 +310,12 @@ def build_context( classification["language"] = language classification["execution_mode"] = "python_api" if language == "python" else "r_script" adapter_type = infer_adapter_type(repo_evidence, tutorial_trace, classification) + adapter_spec = build_adapter_spec( + adapter_type, + repo_evidence, + maturity_level, + allow_ready_adapter=allow_ready_python_adapter(resolved.repo_path), + ) environment_spec = infer_environment_spec(dependency_evidence, classification["language"]) if classification["language"] == "python" and not environment_spec["python"]["packages"]: environment_spec["python"]["packages"] = [] @@ -225,6 +337,7 @@ def build_context( "language": classification["language"], "execution_mode": classification["execution_mode"], "adapter_type": adapter_type, + "adapter_status": adapter_spec["status"], "maturity_level": maturity_level, }, **io_contract, @@ -265,6 +378,7 @@ def build_context( "environment_report": environment_report, "install_plan": install_plan, "install_plan_markdown": render_install_plan_markdown(install_plan), + "adapter_spec": adapter_spec, "algorithm_contract": algorithm_contract, "bio_contract": bio_contract, "evidence_graph": evidence_graph, @@ -324,6 +438,7 @@ def generate_skill(context: dict[str, Any], out_dir: str | Path) -> Path: write_json(root / "references" / "paper_evidence.json", public_context["paper_evidence"]) write_json(root / "references" / "repo_evidence.json", public_context["repo_evidence"]) write_yaml(root / "references" / "algorithm_contract.yaml", public_context["algorithm_contract"]) + write_yaml(root / "references" / "adapter_spec.yaml", public_context["adapter_spec"]) write_yaml(root / "references" / "bio_contract.yaml", public_context["bio_contract"]) write_yaml(root / "references" / "io_contract.yaml", {"input_contract": public_context["algorithm_contract"].get("input_contract"), "output_contract": public_context["algorithm_contract"].get("output_contract")}) write_json(root / "references" / "evidence_graph.json", public_context["evidence_graph"]) @@ -336,9 +451,34 @@ def generate_skill(context: dict[str, Any], out_dir: str | Path) -> Path: write_text(root / "assets" / "environment.yml", _conda_environment(public_context)) write_text(root / "assets" / "renv.lock.placeholder", "{}\n") write_text(root / "assets" / "demo_input.csv", context["demo_data"]) + write_source_snapshot(context, root) return root +def write_source_snapshot(context: dict[str, Any], root: Path) -> None: + adapter_spec = context.get("adapter_spec") or {} + if adapter_spec.get("adapter_type") != "python_api" or adapter_spec.get("status") != "ready": + return + repo_path = (((context.get("source_manifest") or {}).get("repo") or {}).get("resolved_path")) + if not repo_path: + return + source_root = Path(repo_path) + if not source_root.exists(): + return + dest = root / "sources" / "repo" + if dest.exists(): + shutil.rmtree(dest) + ensure_dir(dest) + for item in source_root.iterdir(): + if item.name.startswith(".") or item.name in {"__pycache__", ".pytest_cache", "tests", "docs", "examples", "data"}: + continue + target = dest / item.name + if item.is_dir() and (item / "__init__.py").exists(): + shutil.copytree(item, target, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + elif item.is_file() and (item.suffix == ".py" or item.name in {"pyproject.toml", "setup.py", "setup.cfg"}): + shutil.copyfile(item, target) + + def write_optional_collection_outputs(public_context: dict[str, Any], root: Path) -> None: paper = (public_context.get("source_manifest") or {}).get("paper") or {} parsed = paper.get("parsed_document") or {} @@ -412,6 +552,7 @@ def plan_outputs(context: dict[str, Any], out_dir: str | Path) -> Path: write_json(root / "repo_evidence.json", public_context["repo_evidence"]) write_json(root / "tutorial_trace.json", public_context["tutorial_trace"]) write_json(root / "workflow_dag.json", public_context["workflow"].get("workflow_dag", {"nodes": [], "edges": []})) + write_yaml(root / "adapter_spec.preview.yaml", public_context["adapter_spec"]) write_yaml(root / "algorithm_contract.preview.yaml", public_context["algorithm_contract"]) write_json(root / "environment_report.json", _public_environment_report(context["environment_report"])) write_text(root / "build_plan.md", _build_plan_markdown(public_context)) diff --git a/paper2skill/inference/infer_bio_contract.py b/paper2skill/inference/infer_bio_contract.py index e4d9995..f9442f7 100644 --- a/paper2skill/inference/infer_bio_contract.py +++ b/paper2skill/inference/infer_bio_contract.py @@ -60,7 +60,7 @@ def infer_bio_contract( strict_evidence: bool = False, ) -> dict[str, Any]: text_items = evidence_texts(tutorial_trace, paper_sections or []) - all_text = "\n".join(text for _eid, text, _source in text_items) + all_text = "\n".join(item["text"] for item in text_items) modality = first_with_evidence(text_items, MODALITY_RULES, strict_evidence) species = first_with_evidence(text_items, SPECIES_RULES, strict_evidence) gene_id = first_with_evidence(text_items, GENE_ID_RULES, strict_evidence) @@ -92,30 +92,36 @@ def infer_bio_contract( return {"bio_contract": base} -def evidence_texts(tutorial_trace: dict[str, Any], paper_sections: list[dict[str, Any]]) -> list[tuple[str, str, str]]: +def evidence_texts(tutorial_trace: dict[str, Any], paper_sections: list[dict[str, Any]]) -> list[dict[str, Any]]: items = [] for tutorial in tutorial_trace.get("tutorials", []): for step in tutorial.get("steps", tutorial.get("workflow_steps", [])): text = "\n".join([step.get("code_preview", ""), step.get("command_or_code", ""), " ".join(step.get("function_calls", [])), " ".join(step.get("imports", []))]) - items.append((step.get("evidence_id") or step.get("step_id", "tutorial:unknown"), text, "tutorial")) + source = "tutorial" + items.append({"evidence_id": step.get("evidence_id") or step.get("step_id", "tutorial:unknown"), "text": text, "source_type": source, "section_role": "code", "weight": source_weight(source)}) for section in paper_sections: role = section_role(section.get("section_id", ""), section.get("title", "")) - items.append((section.get("section_id", "paper:unknown"), section.get("text", ""), f"paper:{role}")) + source = f"paper:{role}" + items.append({"evidence_id": section.get("section_id", "paper:unknown"), "text": section.get("text", ""), "source_type": source, "section_role": role, "weight": source_weight(source)}) return items -def first_with_evidence(items: list[tuple[str, str, str]], rules: dict[str, list[str]], strict_evidence: bool = False) -> tuple[str, list[str], str] | None: - for evidence_id, text, source in items: - matches = match_rules(text, rules) +def first_with_evidence(items: list[dict[str, Any]], rules: dict[str, list[str]], strict_evidence: bool = False) -> tuple[str, list[str], str] | None: + candidates = [] + for item in items: + matches = match_rules(item["text"], rules) if matches: - confidence = confidence_for_source(source) + confidence = confidence_for_source(item["source_type"]) if strict_evidence and confidence == "low": continue - return matches[0], [evidence_id], confidence - return None + candidates.append((item["weight"], confidence_rank(confidence), matches[0], [item["evidence_id"]], confidence)) + if not candidates: + return None + _weight, _rank, value, evidence, confidence = sorted(candidates, reverse=True)[0] + return value, evidence, confidence -def transformation_chain(items: list[tuple[str, str, str]], strict_evidence: bool = False) -> list[str]: +def transformation_chain(items: list[dict[str, Any]], strict_evidence: bool = False) -> list[str]: found = [] for value in MATRIX_STATE_RULES: if evidence_for_value(items, MATRIX_STATE_RULES, value, strict_evidence): @@ -123,15 +129,15 @@ def transformation_chain(items: list[tuple[str, str, str]], strict_evidence: boo return found -def evidence_for_value(items: list[tuple[str, str, str]], rules: dict[str, list[str]], value: str, strict_evidence: bool = False) -> list[str]: +def evidence_for_value(items: list[dict[str, Any]], rules: dict[str, list[str]], value: str, strict_evidence: bool = False) -> list[str]: words = rules[value] - evidence = [] - for evidence_id, text, source in items: - if strict_evidence and confidence_for_source(source) == "low": + matches = [] + for item in items: + if strict_evidence and confidence_for_source(item["source_type"]) == "low": continue - if any(word.lower() in text.lower() for word in words): - evidence.append(evidence_id) - return evidence + if any(word.lower() in item["text"].lower() for word in words): + matches.append(item) + return [item["evidence_id"] for item in sorted(matches, key=lambda value: value["weight"], reverse=True)] def metadata_key(text: str, _field: str, candidates: list[str]) -> str | None: @@ -158,3 +164,19 @@ def confidence_for_source(source: str) -> str: if source in {"paper:methods", "paper:data", "paper:code"}: return "medium" return "low" + + +def source_weight(source: str) -> float: + if source == "tutorial": + return 1.0 + if source.startswith("api") or source.startswith("docs"): + return 0.9 + if source in {"paper:methods", "paper:data", "paper:code"}: + return 0.6 + if source.startswith("paper:"): + return 0.2 + return 0.0 + + +def confidence_rank(confidence: str) -> int: + return {"low": 0, "medium": 1, "high": 2}.get(confidence, 0) diff --git a/paper2skill/inference/infer_workflow.py b/paper2skill/inference/infer_workflow.py index 535c7c7..907f520 100644 --- a/paper2skill/inference/infer_workflow.py +++ b/paper2skill/inference/infer_workflow.py @@ -4,9 +4,6 @@ STEP_TYPE_RULES = { - "load_data": ["read", "read_csv", "read_h5ad", "read_10x", "read10x", "load", "import"], - "qc": ["calculate_qc", "filter_cells", "filter_genes", "mito", "qc"], - "filtering": ["filter", "subset"], "normalization": ["normalize", "normalizedata", "normalize_total"], "transformation": ["log1p", "lognormalize", "scale", "scaledata"], "feature_selection": ["highly_variable", "variablefeatures"], @@ -15,8 +12,11 @@ "differential_expression": ["rank_genes", "findmarkers", "deseq", "edger", "limma"], "model_training": ["fit", "train"], "prediction": ["predict", "classify"], + "qc": ["calculate_qc", "filter_cells", "filter_genes", "mito", "qc"], + "filtering": ["filter", "subset"], "visualization": ["plot", "scatter", "umap", "tsne", "savefig", "ggsave"], "save_output": ["write", "to_csv", "save", "saverds", "write_h5ad"], + "load_data": ["read", "read_csv", "read_h5ad", "read_10x", "read10x", "load"], } @@ -35,6 +35,7 @@ def build_workflow_dag(steps: list[dict[str, Any]]) -> dict[str, Any]: edges = [] last_object_producer: dict[str, str] = {} last_file_producer: dict[str, str] = {} + object_states: dict[str, dict[str, Any]] = {} for index, step in enumerate(steps): step_id = step.get("step_id") or step.get("id") or f"step_{index + 1:03d}" node_type = classify_step_type(step) @@ -50,7 +51,14 @@ def build_workflow_dag(steps: list[dict[str, Any]]) -> dict[str, Any]: producer = last_file_producer.get(name) if producer and producer != step_id: edges.append({"from": producer, "to": step_id, "reason": f"shared_file:{name}"}) - for name in output_objects: + state_objects = list(dict.fromkeys(output_objects + input_objects)) + state_after = object_state_after(step, state_objects, node_type, object_states) + for name, state in state_after.items(): + object_states[name] = state + mutated_objects = output_objects + if node_type in MUTATING_STEP_TYPES: + mutated_objects = output_objects or input_objects + for name in mutated_objects: last_object_producer[name] = step_id for name in output_files: last_file_producer[name] = step_id @@ -64,7 +72,7 @@ def build_workflow_dag(steps: list[dict[str, Any]]) -> dict[str, Any]: "output_files": output_files, "input_objects": input_objects, "output_objects": output_objects, - "object_state_after": object_state_after(step, output_objects or input_objects, node_type), + "object_state_after": state_after, "parameters": step.get("parameters", {}), "evidence": [step.get("evidence_id") or step_id], "confidence": step.get("confidence", "medium"), @@ -92,11 +100,27 @@ def classify_step_type(step: dict[str, Any]) -> str: return "other" -def object_state_after(step: dict[str, Any], objects: list[str], node_type: str) -> dict[str, dict[str, Any]]: +MUTATING_STEP_TYPES = { + "load_data", + "qc", + "filtering", + "normalization", + "transformation", + "feature_selection", + "dimensionality_reduction", + "clustering", + "differential_expression", + "model_training", + "prediction", +} + + +def object_state_after(step: dict[str, Any], objects: list[str], node_type: str, previous_states: dict[str, dict[str, Any]] | None = None) -> dict[str, dict[str, Any]]: state: dict[str, dict[str, Any]] = {} signals = set(step.get("bio_signals", []) or []) + previous_states = previous_states or {} for name in objects: - value: dict[str, Any] = {} + value: dict[str, Any] = dict(previous_states.get(name, {})) if "single_cell" in signals: value["modality"] = "scRNA-seq" if node_type == "load_data" or "raw_counts" in signals: diff --git a/paper2skill/miners/api_miner.py b/paper2skill/miners/api_miner.py index 3756575..cfdb634 100644 --- a/paper2skill/miners/api_miner.py +++ b/paper2skill/miners/api_miner.py @@ -28,6 +28,10 @@ def mine_api(repo_path: str | Path | None) -> dict[str, Any]: for item in mined.get("functions", []): item = dict(item) item["path"] = public_local_path(path, root) + item["module"] = python_module_name(path, root) + public_module = public_python_module_name(path, root, item["name"]) + if public_module: + item["public_module"] = public_module api_functions.append(item) for item in mined.get("classes", []): item = dict(item) @@ -113,6 +117,26 @@ def python_cli_commands(source: str, path: Path, root: Path) -> list[dict[str, A return commands +def python_module_name(path: Path, root: Path) -> str: + rel = path.relative_to(root).with_suffix("") + parts = list(rel.parts) + if parts and parts[-1] == "__init__": + parts = parts[:-1] + return ".".join(parts) + + +def public_python_module_name(path: Path, root: Path, function_name: str) -> str | None: + if path.name == "__init__.py": + return python_module_name(path, root) + package_init = path.parent / "__init__.py" + if not package_init.exists(): + return None + text = package_init.read_text(encoding="utf-8", errors="replace") + if re.search(rf"\b{re.escape(function_name)}\b", text): + return python_module_name(package_init, root) + return None + + def r_namespace_exports(root: Path) -> list[dict[str, Any]]: namespace = root / "NAMESPACE" if not namespace.exists(): diff --git a/paper2skill/schemas/adapter_spec_schema.yaml b/paper2skill/schemas/adapter_spec_schema.yaml new file mode 100644 index 0000000..c19f9ee --- /dev/null +++ b/paper2skill/schemas/adapter_spec_schema.yaml @@ -0,0 +1,14 @@ +required: + - adapter_type + - status + - entrypoint + - command + - module + - function + - evidence + - caveats +properties: + adapter_type: + enum: [demo_only, python_api, cli, notebook, r_script, workflow_engine] + status: + enum: [demo_only, candidate, ready, blocked] diff --git a/paper2skill/schemas/workflow_dag_schema.yaml b/paper2skill/schemas/workflow_dag_schema.yaml new file mode 100644 index 0000000..2c5b61b --- /dev/null +++ b/paper2skill/schemas/workflow_dag_schema.yaml @@ -0,0 +1,3 @@ +required: + - nodes + - edges diff --git a/paper2skill/templates/codex_skill/SKILL.md.j2 b/paper2skill/templates/codex_skill/SKILL.md.j2 index 6238001..cdeba01 100644 --- a/paper2skill/templates/codex_skill/SKILL.md.j2 +++ b/paper2skill/templates/codex_skill/SKILL.md.j2 @@ -67,7 +67,7 @@ Run: python scripts/run.py --manifest input_manifest.yaml ``` -`run.py` calls preflight first and stops if preflight fails. It records the selected adapter in `workflow/adapter_report.json`. Demo execution only proves the tutorial path, not biological validity on user data, and must not be treated as real algorithm execution unless the adapter report explicitly passes. +`run.py` calls preflight first and stops if preflight fails. It records the selected adapter in `workflow/adapter_report.json`. Adapter readiness is declared in `references/adapter_spec.yaml`. Non-demo runs do not fall back to the demo summary runner: if the adapter is not ready, the run is blocked. Demo execution only proves the tutorial path, not biological validity on user data, and must not be treated as real algorithm execution unless the adapter report explicitly passes. ## Output contract @@ -90,6 +90,7 @@ Dry-run results are not biological results. Demo runs only prove that the tutori - `blocked_dependencies_missing`: required package or executable is missing. - `blocked_runtime_missing`: required runtime such as `Rscript` is unavailable. - `blocked_input_invalid`: manifest or input files are invalid. +- `blocked_adapter_not_ready`: a non-demo run requested an adapter that is candidate, blocked, or missing required wiring. - `fail`: execution or output validation failed. ## Evidence sources @@ -97,6 +98,7 @@ Dry-run results are not biological results. Demo runs only prove that the tutori - `references/evidence_report.md` - `references/tutorial_trace.md` - `references/workflow_dag.json` +- `references/adapter_spec.yaml` - `references/api_reference.md` - `references/paper_summary.md` - `references/repo_summary.md` diff --git a/paper2skill/templates/codex_skill/scripts/adapters/cli_adapter.py.j2 b/paper2skill/templates/codex_skill/scripts/adapters/cli_adapter.py.j2 index 3c50bf0..f37f180 100644 --- a/paper2skill/templates/codex_skill/scripts/adapters/cli_adapter.py.j2 +++ b/paper2skill/templates/codex_skill/scripts/adapters/cli_adapter.py.j2 @@ -5,7 +5,7 @@ from pathlib import Path def run(manifest: dict, out: Path) -> dict: return { - "status": "not_implemented", + "status": "blocked", "adapter_type": "cli", - "message": "CLI execution requires a verified command mapping and is not auto-generated.", + "message": "CLI adapter is candidate-only and requires user-reviewed command wiring.", } diff --git a/paper2skill/templates/codex_skill/scripts/adapters/notebook_adapter.py.j2 b/paper2skill/templates/codex_skill/scripts/adapters/notebook_adapter.py.j2 index 5c9ea7a..71f0ed1 100644 --- a/paper2skill/templates/codex_skill/scripts/adapters/notebook_adapter.py.j2 +++ b/paper2skill/templates/codex_skill/scripts/adapters/notebook_adapter.py.j2 @@ -5,7 +5,7 @@ from pathlib import Path def run(manifest: dict, out: Path) -> dict: return { - "status": "not_implemented", + "status": "blocked", "adapter_type": "notebook", - "message": "Notebook execution is intentionally gated and not run automatically.", + "message": "Notebook adapter is candidate-only and is not run automatically.", } diff --git a/paper2skill/templates/codex_skill/scripts/adapters/python_api_adapter.py.j2 b/paper2skill/templates/codex_skill/scripts/adapters/python_api_adapter.py.j2 index 786ad30..9c696e0 100644 --- a/paper2skill/templates/codex_skill/scripts/adapters/python_api_adapter.py.j2 +++ b/paper2skill/templates/codex_skill/scripts/adapters/python_api_adapter.py.j2 @@ -1,11 +1,61 @@ from __future__ import annotations +import importlib +import json +import sys from pathlib import Path +ROOT = Path(__file__).resolve().parents[2] +SCRIPT_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPT_DIR)) +import env_manager + + +def write_json(path: Path, data: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def input_path(manifest: dict) -> Path: + value = ((manifest.get("inputs") or {}).get("primary_data") or {}).get("path") + if not value: + raise ValueError("inputs.primary_data.path is required") + path = Path(value) + return path if path.is_absolute() else ROOT / path + def run(manifest: dict, out: Path) -> dict: + spec = env_manager.load_json_yaml(ROOT / "references" / "adapter_spec.yaml") + if spec.get("status") != "ready": + return { + "status": "blocked", + "adapter_type": "python_api", + "entrypoint": spec.get("entrypoint"), + "message": "Python API adapter is not ready; user-reviewed wiring is required.", + "caveats": spec.get("caveats", []), + } + source_root = ROOT / "sources" / "repo" + if source_root.exists(): + sys.path.insert(0, str(source_root)) + module_name = spec.get("module") + function_name = spec.get("function") + if not module_name or not function_name: + return { + "status": "blocked", + "adapter_type": "python_api", + "entrypoint": spec.get("entrypoint"), + "message": "Python API adapter is missing module/function wiring.", + "caveats": spec.get("caveats", []), + } + module = importlib.import_module(module_name) + function = getattr(module, function_name) + result = function(input_path(manifest)) + output_path = out / "results" / "summary.json" + write_json(output_path, env_manager.public_data(result)) return { - "status": "not_implemented", + "status": "pass", "adapter_type": "python_api", - "message": "Python API execution requires user-reviewed adapter wiring.", + "entrypoint": spec.get("entrypoint"), + "outputs": ["results/summary.json"], + "message": "Python API adapter executed.", } diff --git a/paper2skill/templates/codex_skill/scripts/run.py.j2 b/paper2skill/templates/codex_skill/scripts/run.py.j2 index aeae483..7b39c99 100644 --- a/paper2skill/templates/codex_skill/scripts/run.py.j2 +++ b/paper2skill/templates/codex_skill/scripts/run.py.j2 @@ -52,8 +52,29 @@ def summarize_csv(path: Path) -> dict: def adapter_type() -> str: + return adapter_spec().get("adapter_type", "demo_only") + + +def adapter_spec() -> dict: + spec_path = ROOT / "references" / "adapter_spec.yaml" + if spec_path.exists(): + return load_json_yaml(spec_path) contract = load_json_yaml(ROOT / "references" / "algorithm_contract.yaml") - return contract.get("algorithm", {}).get("adapter_type", "demo_only") + return { + "adapter_type": contract.get("algorithm", {}).get("adapter_type", "demo_only"), + "status": "demo_only", + "entrypoint": None, + "command": None, + "module": None, + "function": None, + "evidence": [], + "caveats": ["adapter_spec.yaml was not generated"], + } + + +def is_demo_run(manifest: dict, spec: dict) -> bool: + mode = (((manifest.get("inputs") or {}).get("algorithm") or {}).get("mode")) + return mode == "demo" or spec.get("adapter_type") == "demo_only" def run_adapter(kind: str, manifest: dict, out: Path) -> dict: @@ -69,7 +90,28 @@ def run_adapter(kind: str, manifest: dict, out: Path) -> dict: from adapters import notebook_adapter return notebook_adapter.run(manifest, out) - return {"status": "not_implemented", "adapter_type": kind, "message": "No executable adapter is configured."} + return {"status": "blocked", "adapter_type": kind, "message": "No executable adapter is configured."} + + +def write_blocked_result(out: Path, selected_adapter: str, adapter_report: dict, manifest: dict) -> dict: + result = { + "status": "blocked_adapter_not_ready", + "skill_name": "{{ skill_name }}", + "algorithm": "{{ algorithm_name }}", + "maturity_level": "{{ maturity_level }}", + "input_validation": load_json(out / "qc" / "input_validation.json"), + "environment": load_json(out / "qc" / "environment_report.json"), + "parameter_resolution": env_manager.public_data(((manifest.get("inputs") or {}).get("algorithm") or {}).get("parameters", {})), + "workflow_summary": load_json(out / "workflow" / "plan.json"), + "adapter": adapter_report, + "outputs": [], + "caveats": ["adapter is not ready; demo fallback is disabled for non-demo runs"], + "evidence_used": load_json(out / "workflow" / "plan.json").get("evidence_used", []), + } + write_json(out / "result.json", env_manager.public_data(result)) + (out / "logs").mkdir(parents=True, exist_ok=True) + (out / "logs" / "run.log").write_text("blocked; adapter_type=%s; adapter_status=%s\n" % (selected_adapter, adapter_report.get("status")), encoding="utf-8") + return result def main(argv: list[str] | None = None) -> int: @@ -84,31 +126,44 @@ def main(argv: list[str] | None = None) -> int: return preflight.returncode subprocess.run([sys.executable, str(ROOT / "scripts" / "plan.py"), "--manifest", args.manifest, "--out", str(out)], check=True) manifest = load_json_yaml(Path(args.manifest)) - selected_adapter = adapter_type() + spec = adapter_spec() + selected_adapter = spec.get("adapter_type", adapter_type()) adapter_report = {"status": "demo_only", "adapter_type": selected_adapter, "message": "Demo summary runner was used."} - if selected_adapter != "demo_only": + demo_run = is_demo_run(manifest, spec) + if not demo_run: adapter_report = run_adapter(selected_adapter, manifest, out) if adapter_report.get("status") != "pass": - adapter_report["fallback"] = "demo_summary_runner" + if adapter_report.get("status") in {"not_implemented", "candidate"}: + adapter_report["status"] = "blocked" + write_json(out / "workflow" / "adapter_report.json", env_manager.public_data(adapter_report)) + blocked = write_blocked_result(out, selected_adapter, adapter_report, manifest) + print(json.dumps(blocked, indent=2)) + return 2 write_json(out / "workflow" / "adapter_report.json", env_manager.public_data(adapter_report)) - rel_input = manifest["inputs"]["primary_data"]["path"] - summary = summarize_csv(ROOT / rel_input) - write_json(out / "results" / "summary.json", summary) + outputs = adapter_report.get("outputs", []) + if demo_run: + rel_input = manifest["inputs"]["primary_data"]["path"] + summary = summarize_csv(ROOT / rel_input) + write_json(out / "results" / "summary.json", summary) + outputs = ["results/summary.json"] write_json(out / "parameters" / "resolved_parameters.json", env_manager.public_data(manifest["inputs"]["algorithm"].get("parameters", {}))) write_json(out / "parameters" / "parameter_sources.json", {"source": "input_manifest"}) write_json(out / "workflow" / "executed_steps.json", {"status": "pass", "executed": [step.get("id") for step in load_json(out / "workflow" / "plan.json").get("steps", [])]}) repro = out / "reproducibility" repro.mkdir(parents=True, exist_ok=True) - for name in ["source_manifest.json", "algorithm_contract.yaml", "bio_contract.yaml"]: + for name in ["source_manifest.json", "algorithm_contract.yaml", "adapter_spec.yaml", "bio_contract.yaml"]: src = ROOT / "references" / name if src.exists(): shutil.copyfile(src, repro / name) shutil.copyfile(ROOT / "assets" / "environment_spec.yaml", repro / "environment_spec.yaml") (repro / "command_history.sh").write_text("python scripts/run.py --manifest %s\n" % public_cli_path(args.manifest), encoding="utf-8") (out / "logs").mkdir(parents=True, exist_ok=True) - (out / "logs" / "run.log").write_text("demo run completed; adapter_type=%s\n" % selected_adapter, encoding="utf-8") - (out / "README.md").write_text("# Result\n\nDemo run completed.\n", encoding="utf-8") - (out / "report.md").write_text("# Report\n\nDemo summary: `%s`.\n" % summary, encoding="utf-8") + (out / "logs" / "run.log").write_text("run completed; adapter_type=%s; adapter_status=%s\n" % (selected_adapter, adapter_report.get("status")), encoding="utf-8") + (out / "README.md").write_text("# Result\n\nRun completed with adapter status `%s`.\n" % adapter_report.get("status"), encoding="utf-8") + if demo_run: + (out / "report.md").write_text("# Report\n\nDemo summary: `%s`.\n" % summary, encoding="utf-8") + else: + (out / "report.md").write_text("# Report\n\nAdapter execution completed with status `%s`.\n" % adapter_report.get("status"), encoding="utf-8") result = { "status": "pass", "skill_name": "{{ skill_name }}", @@ -119,7 +174,7 @@ def main(argv: list[str] | None = None) -> int: "parameter_resolution": load_json(out / "parameters" / "resolved_parameters.json"), "workflow_summary": load_json(out / "workflow" / "plan.json"), "adapter": load_json(out / "workflow" / "adapter_report.json"), - "outputs": ["results/summary.json"], + "outputs": outputs, "caveats": ["demo run is not user-data biological validation", "demo summary output is not a real algorithm execution unless adapter.status is pass"], "evidence_used": load_json(out / "workflow" / "plan.json").get("evidence_used", []), } diff --git a/paper2skill/validators/skill_validator.py b/paper2skill/validators/skill_validator.py index 91c027e..7f4b8e5 100644 --- a/paper2skill/validators/skill_validator.py +++ b/paper2skill/validators/skill_validator.py @@ -35,6 +35,7 @@ "references/source_manifest.json", "references/paper_evidence.json", "references/repo_evidence.json", + "references/adapter_spec.yaml", "references/paper.md", "references/paper_sections.json", "references/paper_parser_report.json", @@ -93,7 +94,13 @@ def validate_skill(skill_dir: str | Path) -> dict[str, Any]: if "Do not install anything unless the user explicitly approves" not in text: errors.append("SKILL.md missing explicit install confirmation policy") _validate_yaml_contract(root / "references/algorithm_contract.yaml", PROJECT_ROOT / "paper2skill/schemas/algorithm_skill_schema.yaml", "algorithm_contract", errors) + _validate_yaml_contract(root / "references/adapter_spec.yaml", PROJECT_ROOT / "paper2skill/schemas/adapter_spec_schema.yaml", "adapter_spec", errors) _validate_yaml_contract(root / "references/bio_contract.yaml", PROJECT_ROOT / "paper2skill/schemas/bio_contract_schema.yaml", "bio_contract", errors) + _validate_yaml_contract(root / "references/io_contract.yaml", PROJECT_ROOT / "paper2skill/schemas/algorithm_skill_schema.yaml", "io_contract", errors, required_only=("input_contract", "output_contract")) + _validate_yaml_contract(root / "assets/environment_spec.yaml", PROJECT_ROOT / "paper2skill/schemas/environment_schema.yaml", "environment_spec", errors) + _validate_environment_spec(root / "assets/environment_spec.yaml", errors) + _validate_workflow_dag(root / "references/workflow_dag.json", errors) + _validate_adapter_spec(root / "references/adapter_spec.yaml", errors) trace_path = root / "references/tutorial_trace.json" if trace_path.exists(): try: @@ -105,7 +112,7 @@ def validate_skill(skill_dir: str | Path) -> dict[str, Any]: return {"status": "pass" if not errors else "fail", "errors": errors, "warnings": warnings} -def _validate_yaml_contract(contract_path: Path, schema_path: Path, label: str, errors: list[str]) -> None: +def _validate_yaml_contract(contract_path: Path, schema_path: Path, label: str, errors: list[str], required_only: tuple[str, ...] | None = None) -> None: if not contract_path.exists() or not schema_path.exists(): return try: @@ -114,4 +121,85 @@ def _validate_yaml_contract(contract_path: Path, schema_path: Path, label: str, except yaml.YAMLError as exc: errors.append(f"{label}: invalid YAML: {exc}") return + if required_only is not None: + for key in required_only: + if key not in data: + errors.append(f"{label}: missing required key '{key}'") + return errors.extend(validate_simple_schema(data, schema, label)) + + +def _validate_workflow_dag(path: Path, errors: list[str]) -> None: + if not path.exists(): + return + try: + data = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + errors.append(f"workflow_dag: invalid JSON: {exc}") + return + if not isinstance(data.get("nodes"), list): + errors.append("workflow_dag.nodes: expected list") + return + node_ids = set() + for index, node in enumerate(data["nodes"]): + if not isinstance(node, dict): + errors.append(f"workflow_dag.nodes[{index}]: expected object") + continue + if not node.get("step_id"): + errors.append(f"workflow_dag.nodes[{index}]: missing required key 'step_id'") + if not node.get("type"): + errors.append(f"workflow_dag.nodes[{index}]: missing required key 'type'") + if node.get("step_id"): + node_ids.add(node["step_id"]) + for index, edge in enumerate(data.get("edges", []) or []): + if edge.get("from") not in node_ids: + errors.append(f"workflow_dag.edges[{index}]: unknown from node {edge.get('from')!r}") + if edge.get("to") not in node_ids: + errors.append(f"workflow_dag.edges[{index}]: unknown to node {edge.get('to')!r}") + + +def _validate_adapter_spec(path: Path, errors: list[str]) -> None: + if not path.exists(): + return + try: + spec = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError as exc: + errors.append(f"adapter_spec: invalid YAML: {exc}") + return + status = spec.get("status") + adapter_type = spec.get("adapter_type") + if status == "ready" and adapter_type == "python_api": + if not spec.get("module"): + errors.append("adapter_spec.module: required when python_api status is ready") + if not spec.get("function"): + errors.append("adapter_spec.function: required when python_api status is ready") + if status == "demo_only" and adapter_type != "demo_only": + errors.append("adapter_spec.status: demo_only status requires demo_only adapter_type") + + +def _validate_environment_spec(path: Path, errors: list[str]) -> None: + if not path.exists(): + return + try: + spec = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError as exc: + errors.append(f"environment_spec: invalid YAML: {exc}") + return + python_packages = ((spec.get("python") or {}).get("packages") or []) + for index, item in enumerate(python_packages): + if not isinstance(item, dict): + errors.append(f"environment_spec.python.packages[{index}]: expected object") + continue + if not item.get("spec"): + errors.append(f"environment_spec.python.packages[{index}]: missing required key 'spec'") + if item.get("required") is not None and not isinstance(item.get("required"), bool): + errors.append(f"environment_spec.python.packages[{index}].required: expected boolean") + r_packages = ((spec.get("r") or {}).get("packages") or []) + for index, item in enumerate(r_packages): + if not isinstance(item, dict): + errors.append(f"environment_spec.r.packages[{index}]: expected object") + continue + if not item.get("name"): + errors.append(f"environment_spec.r.packages[{index}]: missing required key 'name'") + if item.get("required") is not None and not isinstance(item.get("required"), bool): + errors.append(f"environment_spec.r.packages[{index}].required: expected boolean") diff --git a/tests/benchmarks/test_bioinformatics_benchmarks.py b/tests/benchmarks/test_bioinformatics_benchmarks.py new file mode 100644 index 0000000..0ba9663 --- /dev/null +++ b/tests/benchmarks/test_bioinformatics_benchmarks.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import yaml + +from paper2skill.generators.codex_skill_generator import build_context, generate_skill +from paper2skill.validators.skill_validator import validate_skill + + +def test_scanpy_10x_benchmark_generates_contracts_and_adapter_spec(tmp_path: Path): + repo = tmp_path / "scanpy_repo" + (repo / "docs").mkdir(parents=True) + write_notebook(repo / "docs" / "pbmc.ipynb", "import scanpy as sc\nadata = sc.read_10x_mtx('data/')\nsc.pp.normalize_total(adata)\nsc.pp.log1p(adata)\n") + (repo / "pyproject.toml").write_text("[project]\nname='scanpy-demo'\nversion='0.1.0'\ndependencies=['scanpy']\n", encoding="utf-8") + + out = build_benchmark_skill(tmp_path, repo) + + assert validate_skill(out)["status"] == "pass" + env = yaml.safe_load((out / "assets" / "environment_spec.yaml").read_text(encoding="utf-8")) + io_contract = yaml.safe_load((out / "references" / "io_contract.yaml").read_text(encoding="utf-8")) + dag = json.loads((out / "references" / "workflow_dag.json").read_text(encoding="utf-8")) + adapter = yaml.safe_load((out / "references" / "adapter_spec.yaml").read_text(encoding="utf-8")) + assert env["python"]["packages"][0]["name"] == "scanpy" + assert io_contract["input_contract"]["required"]["primary_data"]["format"]["value"] == "10x_mtx" + assert dag["nodes"] + assert adapter["status"] in {"candidate", "demo_only"} + + +def test_seurat_rds_benchmark_generates_r_contracts(tmp_path: Path): + repo = tmp_path / "seurat_repo" + (repo / "examples").mkdir(parents=True) + (repo / "DESCRIPTION").write_text("Package: demo\nVersion: 0.1.0\nImports: Seurat\n", encoding="utf-8") + (repo / "examples" / "demo.R").write_text("library(Seurat)\nobj <- readRDS('input.rds')\nobj <- NormalizeData(obj)\n", encoding="utf-8") + + out = build_benchmark_skill(tmp_path, repo) + + assert validate_skill(out)["status"] == "pass" + env = yaml.safe_load((out / "assets" / "environment_spec.yaml").read_text(encoding="utf-8")) + io_contract = yaml.safe_load((out / "references" / "io_contract.yaml").read_text(encoding="utf-8")) + assert env["r"]["required"] is True + assert env["r"]["packages"][0]["name"] == "Seurat" + assert io_contract["input_contract"]["required"]["primary_data"]["format"]["value"] == "rds" + + +def test_deseq_count_matrix_benchmark_keeps_bioconductor_dependency_required(tmp_path: Path): + repo = tmp_path / "deseq_repo" + (repo / "examples").mkdir(parents=True) + (repo / "DESCRIPTION").write_text("Package: demo\nVersion: 0.1.0\nImports: DESeq2\nSuggests: testthat\n", encoding="utf-8") + (repo / "examples" / "demo.R").write_text("library(DESeq2)\ncounts <- read.csv('counts.csv')\n", encoding="utf-8") + + out = build_benchmark_skill(tmp_path, repo) + + env = yaml.safe_load((out / "assets" / "environment_spec.yaml").read_text(encoding="utf-8")) + dep_names = [item["name"] for item in env["r"]["packages"]] + assert "DESeq2" in dep_names + assert "testthat" not in dep_names + assert validate_skill(out)["status"] == "pass" + + +def test_python_cli_benchmark_marks_adapter_candidate(tmp_path: Path): + repo = tmp_path / "cli_repo" + package = repo / "demo_cli" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "cli.py").write_text("def main():\n return 0\n", encoding="utf-8") + (repo / "pyproject.toml").write_text( + "[project]\nname='demo-cli'\nversion='0.1.0'\ndependencies=[]\n[project.scripts]\ndemo-cli='demo_cli.cli:main'\n", + encoding="utf-8", + ) + + out = build_benchmark_skill(tmp_path, repo) + + adapter = yaml.safe_load((out / "references" / "adapter_spec.yaml").read_text(encoding="utf-8")) + assert adapter["adapter_type"] == "cli" + assert adapter["status"] == "candidate" + assert validate_skill(out)["status"] == "pass" + + +def test_workflow_engine_benchmark_marks_adapter_candidate(tmp_path: Path): + repo = tmp_path / "workflow_repo" + repo.mkdir() + (repo / "Snakefile").write_text("rule all:\n input: 'results/done.txt'\n", encoding="utf-8") + + out = build_benchmark_skill(tmp_path, repo) + + adapter = yaml.safe_load((out / "references" / "adapter_spec.yaml").read_text(encoding="utf-8")) + assert adapter["adapter_type"] == "workflow_engine" + assert adapter["status"] == "candidate" + assert validate_skill(out)["status"] == "pass" + + +def build_benchmark_skill(tmp_path: Path, repo: Path) -> Path: + paper = tmp_path / f"{repo.name}_paper.md" + paper.write_text("# Methods\n\nBenchmark method with official tutorial evidence.\n", encoding="utf-8") + context = build_context(paper=str(paper), repo=str(repo), no_execute_tutorials=True) + return generate_skill(context, tmp_path / f"{repo.name}_skill") + + +def write_notebook(path: Path, source: str) -> None: + notebook = { + "cells": [ + {"cell_type": "markdown", "source": ["# Tutorial\n"], "metadata": {}}, + {"cell_type": "code", "source": [source], "metadata": {}, "outputs": [], "execution_count": None}, + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5, + } + path.write_text(json.dumps(notebook), encoding="utf-8") diff --git a/tests/test_bio_contract_inference.py b/tests/test_bio_contract_inference.py index 8f07d9b..0a3f0be 100644 --- a/tests/test_bio_contract_inference.py +++ b/tests/test_bio_contract_inference.py @@ -45,3 +45,36 @@ def test_strict_bio_contract_does_not_promote_background_species_keyword(): contract = infer_bio_contract({"tutorials": [], "workflow_steps": []}, paper_sections=sections, strict_evidence=True) species = contract["bio_contract"]["organism"]["species_supported"] assert species["value"] == "not_confirmed" + + +def test_bio_contract_prefers_tutorial_evidence_over_paper_background_conflict(): + trace = { + "tutorials": [ + { + "path": "docs/tutorial.ipynb", + "steps": [ + { + "step_id": "tutorial_001:cell_001", + "evidence_id": "tutorial.ipynb:cell:1", + "code_preview": "adata.uns['species'] = 'human'", + "command_or_code": "adata.uns['species'] = 'human'", + "function_calls": [], + } + ], + } + ] + } + sections = [ + { + "section_id": "paper:introduction", + "title": "Introduction", + "text": "Mouse examples are often used as background motivation.", + } + ] + + contract = infer_bio_contract(trace, paper_sections=sections) + + species = contract["bio_contract"]["organism"]["species_supported"] + assert species["value"] == "human" + assert species["confidence"] == "high" + assert species["evidence"] == ["tutorial.ipynb:cell:1"] diff --git a/tests/test_generated_skill.py b/tests/test_generated_skill.py index 4d0863d..7099ad5 100644 --- a/tests/test_generated_skill.py +++ b/tests/test_generated_skill.py @@ -8,6 +8,8 @@ import importlib.util from pathlib import Path +import yaml + def test_generated_toy_python_skill_smoke(tmp_path: Path): out = tmp_path / "toy-python-skill" @@ -70,6 +72,174 @@ def test_generated_skill_runs_with_regular_yaml_manifest(tmp_path: Path): assert (out / "yaml-result" / "result.json").exists() +def test_generated_candidate_adapter_blocks_non_demo_run(tmp_path: Path): + repo = tmp_path / "cli-repo" + package = repo / "demo_cli" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "cli.py").write_text( + "def main():\n print('candidate only')\n", + encoding="utf-8", + ) + (repo / "pyproject.toml").write_text( + "[project]\nname='demo-cli'\nversion='0.1.0'\ndependencies=[]\n[project.scripts]\ndemo-cli='demo_cli.cli:main'\n", + encoding="utf-8", + ) + paper = tmp_path / "paper.md" + paper.write_text("# Methods\n\nDemo CLI method.\n", encoding="utf-8") + out = tmp_path / "candidate-cli-skill" + build = subprocess.run( + [ + sys.executable, + "-m", + "paper2skill.cli", + "build", + "--paper", + str(paper), + "--repo", + str(repo), + "--out", + str(out), + "--no-execute-tutorials", + ], + text=True, + capture_output=True, + check=False, + ) + assert build.returncode == 0, build.stdout + build.stderr + adapter_spec = yaml.safe_load((out / "references" / "adapter_spec.yaml").read_text(encoding="utf-8")) + assert adapter_spec["adapter_type"] == "cli" + assert adapter_spec["status"] == "candidate" + manifest = out / "run_manifest.yaml" + manifest.write_text( + """ +inputs: + primary_data: + path: assets/demo_input.csv + format: csv + exists: true + algorithm: + mode: run +environment: + install_policy: ask +""".strip() + + "\n", + encoding="utf-8", + ) + result_dir = out / "candidate-result" + + result = subprocess.run( + [sys.executable, str(out / "scripts" / "run.py"), "--manifest", str(manifest), "--out", str(result_dir)], + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 2 + result_json = json.loads((result_dir / "result.json").read_text(encoding="utf-8")) + adapter_report = json.loads((result_dir / "workflow" / "adapter_report.json").read_text(encoding="utf-8")) + assert result_json["status"] == "blocked_adapter_not_ready" + assert adapter_report["status"] == "blocked" + assert not (result_dir / "results" / "summary.json").exists() + + +def test_generated_arbitrary_l2_python_api_summarize_stays_candidate(tmp_path: Path): + repo = tmp_path / "api-repo" + package = repo / "arbitrary_pkg" + package.mkdir(parents=True) + (package / "__init__.py").write_text("from .core import summarize\n", encoding="utf-8") + (package / "core.py").write_text( + "def summarize(path):\n raise RuntimeError('should not be executed without review')\n", + encoding="utf-8", + ) + (repo / "pyproject.toml").write_text( + "[project]\nname='arbitrary-pkg'\nversion='0.1.0'\ndependencies=[]\n", + encoding="utf-8", + ) + paper = tmp_path / "paper.md" + paper.write_text("# Methods\n\nArbitrary Python API method.\n", encoding="utf-8") + out = tmp_path / "candidate-api-skill" + + build = subprocess.run( + [ + sys.executable, + "-m", + "paper2skill.cli", + "build", + "--paper", + str(paper), + "--repo", + str(repo), + "--out", + str(out), + "--maturity-target", + "L2", + "--no-execute-tutorials", + ], + text=True, + capture_output=True, + check=False, + ) + + assert build.returncode == 0, build.stdout + build.stderr + adapter_spec = yaml.safe_load((out / "references" / "adapter_spec.yaml").read_text(encoding="utf-8")) + assert adapter_spec["adapter_type"] == "python_api" + assert adapter_spec["status"] == "candidate" + assert adapter_spec["module"] == "arbitrary_pkg" + assert adapter_spec["function"] == "summarize" + assert not (out / "sources" / "repo").exists() + + +def test_generated_ready_python_adapter_runs_without_demo_summary_fallback(tmp_path: Path): + out = tmp_path / "toy-python-skill" + build = subprocess.run( + [sys.executable, "-m", "paper2skill.cli", "build", "--example", "toy_python", "--out", str(out)], + text=True, + capture_output=True, + check=False, + ) + assert build.returncode == 0, build.stdout + build.stderr + adapter_spec = yaml.safe_load((out / "references" / "adapter_spec.yaml").read_text(encoding="utf-8")) + assert adapter_spec["adapter_type"] == "python_api" + assert adapter_spec["status"] == "ready" + assert adapter_spec["module"] == "toy_algorithm" + assert adapter_spec["function"] == "summarize" + manifest = out / "run_manifest.yaml" + manifest.write_text( + """ +inputs: + primary_data: + path: assets/demo_input.csv + format: csv + exists: true + algorithm: + mode: run +environment: + install_policy: ask +""".strip() + + "\n", + encoding="utf-8", + ) + result_dir = out / "adapter-result" + + result = subprocess.run( + [sys.executable, str(out / "scripts" / "run.py"), "--manifest", str(manifest), "--out", str(result_dir)], + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr + result_json = json.loads((result_dir / "result.json").read_text(encoding="utf-8")) + adapter_report = json.loads((result_dir / "workflow" / "adapter_report.json").read_text(encoding="utf-8")) + summary = json.loads((result_dir / "results" / "summary.json").read_text(encoding="utf-8")) + assert result_json["status"] == "pass" + assert adapter_report["status"] == "pass" + assert adapter_report["adapter_type"] == "python_api" + assert adapter_report.get("fallback") is None + assert summary == {"rows": 3, "value_mean": 2.0} + + def test_generated_plan_accepts_regular_yaml_manifest(tmp_path: Path): out = tmp_path / "toy-python-skill" subprocess.run([sys.executable, "-m", "paper2skill.cli", "build", "--example", "toy_python", "--out", str(out)], check=True) diff --git a/tests/test_generator.py b/tests/test_generator.py index eb2ac66..3d5f691 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -6,6 +6,7 @@ from pathlib import Path import pytest +import yaml from paper2skill.common import PROJECT_ROOT from paper2skill.generators.codex_skill_generator import build_context, example_inputs, generate_skill, plan_outputs @@ -18,6 +19,7 @@ def test_generator_creates_complete_toy_python_skill(tmp_path: Path): result = validate_skill(out) assert result["status"] == "pass", result assert (out / "references" / "algorithm_contract.yaml").exists() + assert (out / "references" / "adapter_spec.yaml").exists() assert (out / "references" / "bio_contract.yaml").exists() assert (out / "references" / "workflow_dag.json").exists() assert (out / "scripts" / "adapters" / "python_api_adapter.py").exists() @@ -75,6 +77,60 @@ def test_plan_outputs_do_not_include_absolute_paths(tmp_path: Path): assert_no_absolute_path_markers(out) +def test_validate_skill_rejects_missing_or_invalid_adapter_spec(tmp_path: Path): + context = build_context(**example_inputs("toy_python")) + out = generate_skill(context, tmp_path / "toy-python-skill") + adapter_spec = out / "references" / "adapter_spec.yaml" + adapter_spec.unlink() + + missing = validate_skill(out) + assert missing["status"] == "fail" + assert any("adapter_spec.yaml" in error for error in missing["errors"]) + + out = generate_skill(context, tmp_path / "toy-python-skill-invalid") + adapter_spec = out / "references" / "adapter_spec.yaml" + adapter_spec.write_text( + """ +adapter_type: python_api +status: unsafe +entrypoint: null +command: null +module: toy_algorithm +function: summarize +evidence: [] +caveats: [] +""".lstrip(), + encoding="utf-8", + ) + + invalid = validate_skill(out) + assert invalid["status"] == "fail" + assert any("adapter_spec.status" in error for error in invalid["errors"]) + + +def test_validate_skill_rejects_invalid_workflow_dag_and_environment_records(tmp_path: Path): + context = build_context(**example_inputs("toy_python")) + out = generate_skill(context, tmp_path / "toy-python-skill") + dag_path = out / "references" / "workflow_dag.json" + dag = json.loads(dag_path.read_text(encoding="utf-8")) + dag["nodes"][0].pop("step_id") + dag_path.write_text(json.dumps(dag, indent=2) + "\n", encoding="utf-8") + + invalid_dag = validate_skill(out) + assert invalid_dag["status"] == "fail" + assert any("workflow_dag.nodes[0]" in error and "step_id" in error for error in invalid_dag["errors"]) + + out = generate_skill(context, tmp_path / "toy-python-skill-invalid-env") + env_path = out / "assets" / "environment_spec.yaml" + env = yaml.safe_load(env_path.read_text(encoding="utf-8")) + env["python"]["packages"] = [{"name": "missing-spec"}] + env_path.write_text(yaml.safe_dump(env, sort_keys=False), encoding="utf-8") + + invalid_env = validate_skill(out) + assert invalid_env["status"] == "fail" + assert any("environment_spec.python.packages[0]" in error and "spec" in error for error in invalid_env["errors"]) + + @pytest.mark.skipif(shutil.which("git") is None, reason="git is required for file:// remote build") def test_remote_file_repo_build_uses_cloned_path_for_mining(tmp_path: Path): repo = tmp_path / "remote-source" diff --git a/tests/test_workflow_dag.py b/tests/test_workflow_dag.py index 2612c2a..bc408e2 100644 --- a/tests/test_workflow_dag.py +++ b/tests/test_workflow_dag.py @@ -37,10 +37,22 @@ def test_workflow_dag_infers_shared_object_edges_and_step_types(): "evidence_id": "tutorial.ipynb:cell:3", "confidence": "high", }, + { + "step_id": "tutorial_001:cell_004", + "language": "python", + "command_or_code": "import scanpy as sc\nsc.tl.pca(adata)", + "function_calls": ["sc.tl.pca"], + "input_objects": ["adata"], + "bio_signals": [], + "evidence_id": "tutorial.ipynb:cell:4", + "confidence": "high", + }, ] } workflow = infer_workflow(trace) dag = workflow["workflow_dag"] - assert [node["type"] for node in dag["nodes"]] == ["load_data", "normalization", "transformation"] + assert [node["type"] for node in dag["nodes"]] == ["load_data", "normalization", "transformation", "dimensionality_reduction"] assert {"from": "tutorial_001:cell_001", "to": "tutorial_001:cell_002", "reason": "shared_object:adata"} in dag["edges"] + assert {"from": "tutorial_001:cell_003", "to": "tutorial_001:cell_004", "reason": "shared_object:adata"} in dag["edges"] assert dag["nodes"][1]["object_state_after"]["adata"]["matrix_state"] == "normalized" + assert dag["nodes"][3]["object_state_after"]["adata"]["matrix_state"] == "log1p"