diff --git a/CHANGELOG.md b/CHANGELOG.md index 512abd5..e43b2b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,3 +7,4 @@ - Added dependency/API miner coverage for environment files, CLI entrypoints, R exports, and workflow engines. - Added `adapter_spec.yaml`, blocked non-demo adapter semantics, and a ready toy Python API adapter. - Added mini offline bioinformatics benchmarks and stricter generated skill validation for adapter, workflow, and environment specs. +- Added adapter lifecycle states, `adapter_review.yaml`, notebook execution policy reports, richer R/Bioconductor parsing, and modality-specific Bio/IO contract sections. diff --git a/README.md b/README.md index 898597c..756acd5 100644 --- a/README.md +++ b/README.md @@ -113,10 +113,11 @@ Supported now: - Local repository builds and basic `file://` remote clone/index flows. - Markdown, plain text, HTML, and optional MarkItDown document parsing. - Python and R toy skill generation. -- Tutorial scanning, dependency mining, workflow DAG inference, and basic bio IO contracts. +- Tutorial scanning, dependency mining, workflow DAG inference, and modality-aware bio IO contracts. - Environment preflight, install planning, and gated disposable conda environment creation. -- Adapter readiness tracking via `references/adapter_spec.yaml`. -- Demo-only execution and blocked non-demo runs when adapters are not ready. +- Adapter lifecycle tracking via `references/adapter_spec.yaml` and `references/adapter_review.yaml`. +- Static notebook execution policy reports via `references/notebook_execution_policy.json`. +- Demo-only execution and blocked non-demo runs when adapters are not `ready`, `reviewed`, or `verified`. Experimental: @@ -124,16 +125,18 @@ Experimental: - Evidence graph conflict decisions. - Source-aware bio contract inference. - Adapter-based execution scaffolding. +- R/Bioconductor metadata hints from DESCRIPTION, NAMESPACE, and R source. +- IPython notebook shell/cell magic, parameter, path, and risk detection. - Mini offline bioinformatics benchmark coverage for Scanpy, Seurat, DESeq2-like, CLI, and workflow-engine repository shapes. Not yet: - Robust large benchmark coverage. -- Complete Bioconductor metadata resolution. +- Complete online Bioconductor metadata resolution. - CUDA/system library automatic configuration. - Large dataset download automation. - Arbitrary install script execution. -- Fully automatic real algorithm execution for unknown repositories. Unknown adapters stay `candidate` or `blocked` and do not run as successful demo fallbacks. +- Fully automatic real algorithm execution for unknown repositories. Unknown adapters stay `candidate` or `blocked`; only `ready`, `reviewed`, or `verified` adapters execute. ## Maturity Levels diff --git a/paper2skill/cli.py b/paper2skill/cli.py index 8925326..54b5731 100644 --- a/paper2skill/cli.py +++ b/paper2skill/cli.py @@ -55,6 +55,7 @@ def add_input_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--no-execute-tutorials", action="store_true") parser.add_argument("--strict-evidence", action="store_true") parser.add_argument("--tutorial-filter", default=None) + parser.add_argument("--adapter-review", default=None) parser.add_argument("--tutorial", action="append", default=[]) parser.add_argument("--maturity-target", default=None) @@ -73,6 +74,7 @@ def command_plan(args: argparse.Namespace) -> int: no_execute_tutorials=args.no_execute_tutorials, strict_evidence=args.strict_evidence, tutorial_filter=args.tutorial_filter, + adapter_review=args.adapter_review, collection_dir=Path(args.out) / ".paper2skill_collection", maturity_level=args.maturity_target or "L1", ) @@ -100,6 +102,7 @@ def command_build(args: argparse.Namespace) -> int: "no_execute_tutorials": args.no_execute_tutorials, "strict_evidence": args.strict_evidence, "tutorial_filter": args.tutorial_filter, + "adapter_review": args.adapter_review, }.items(): if value is not None and value != "" and value != [] and value is not False: values[key] = value diff --git a/paper2skill/generators/codex_skill_generator.py b/paper2skill/generators/codex_skill_generator.py index 0b385d6..ed61bc6 100644 --- a/paper2skill/generators/codex_skill_generator.py +++ b/paper2skill/generators/codex_skill_generator.py @@ -3,6 +3,7 @@ from dataclasses import asdict, dataclass import json from pathlib import Path +import re import shutil from typing import Any @@ -29,6 +30,8 @@ TEMPLATE_ROOT = PROJECT_ROOT / "paper2skill" / "templates" / "codex_skill" +EXECUTABLE_ADAPTER_STATUSES = {"ready", "reviewed", "verified"} +ADAPTER_STATUSES = {"demo_only", "candidate", "blocked", "ready", "reviewed", "verified"} @dataclass @@ -165,6 +168,132 @@ def allow_ready_python_adapter(repo_path: Path | None) -> bool: return resolved in TRUSTED_READY_PYTHON_FIXTURES +def load_adapter_review(path: str | Path | None) -> dict[str, Any] | None: + if not path: + return None + review_path = Path(path) + if not review_path.exists(): + return None + data = yaml.safe_load(review_path.read_text(encoding="utf-8")) or {} + return data if isinstance(data, dict) else None + + +def default_adapter_review(spec: dict[str, Any]) -> dict[str, Any]: + return { + "adapter_type": spec.get("adapter_type"), + "status": spec.get("status"), + "entrypoint": spec.get("entrypoint"), + "command": spec.get("command"), + "module": spec.get("module"), + "function": spec.get("function"), + "human_approved": False, + "dry_run": {"status": "trusted_fixture" if spec.get("status") == "ready" else "not_run"}, + "expected_outputs": [], + "evidence": list(spec.get("evidence", []) or []), + "caveats": ["No human approval was provided; candidate adapters remain non-executable"], + } + + +def apply_adapter_review(spec: dict[str, Any], review: dict[str, Any] | None) -> tuple[dict[str, Any], dict[str, Any]]: + if not review: + return spec, default_adapter_review(spec) + review_data = {**default_adapter_review(spec), **review} + requested_status = review_data.get("status") + if requested_status not in ADAPTER_STATUSES: + reviewed_spec = dict(spec) + reviewed_spec["status"] = "blocked" + reviewed_spec["caveats"] = list(spec.get("caveats", []) or []) + ["adapter_review.yaml requested an invalid status"] + review_data["status"] = "blocked" + return reviewed_spec, review_data + if requested_status in EXECUTABLE_ADAPTER_STATUSES: + if requested_status == "reviewed" and review.get("human_approved") is not True: + reviewed_spec = dict(spec) + reviewed_spec["caveats"] = list(spec.get("caveats", []) or []) + ["adapter_review.yaml did not set human_approved=true"] + review_data["status"] = reviewed_spec["status"] + return reviewed_spec, review_data + if missing := missing_explicit_adapter_mapping(spec, review): + reviewed_spec = dict(spec) + reviewed_spec["status"] = "blocked" + reviewed_spec["caveats"] = list(spec.get("caveats", []) or []) + [f"adapter_review.yaml must provide explicit adapter mapping: {', '.join(missing)}"] + review_data["status"] = "blocked" + return reviewed_spec, review_data + if not adapter_review_matches(spec, review): + reviewed_spec = dict(spec) + reviewed_spec["status"] = "blocked" + reviewed_spec["caveats"] = list(spec.get("caveats", []) or []) + ["adapter_review.yaml mapping does not match the inferred adapter"] + review_data["status"] = "blocked" + return reviewed_spec, review_data + if requested_status == "ready" and not adapter_review_has_ready_evidence(review_data): + reviewed_spec = dict(spec) + reviewed_spec["status"] = "blocked" + reviewed_spec["caveats"] = list(spec.get("caveats", []) or []) + ["adapter_review.yaml ready status requires a passing dry_run"] + review_data["status"] = "blocked" + return reviewed_spec, review_data + if requested_status == "verified" and not adapter_review_has_verified_evidence(review_data): + reviewed_spec = dict(spec) + reviewed_spec["status"] = "blocked" + reviewed_spec["caveats"] = list(spec.get("caveats", []) or []) + ["adapter_review.yaml verified status requires passing dry_run and output_validation evidence"] + review_data["status"] = "blocked" + return reviewed_spec, review_data + reviewed_spec = dict(spec) + for key in ["adapter_type", "entrypoint", "command", "module", "function"]: + if key in review_data: + reviewed_spec[key] = review_data.get(key) + reviewed_spec["status"] = requested_status + reviewed_spec["evidence"] = sorted(dict.fromkeys((spec.get("evidence", []) or []) + (review_data.get("evidence", []) or []))) + reviewed_spec["caveats"] = list(review_data.get("caveats", []) or reviewed_spec.get("caveats", []) or []) + return reviewed_spec, review_data + + +def adapter_review_matches(spec: dict[str, Any], review: dict[str, Any]) -> bool: + if review.get("adapter_type") != spec.get("adapter_type"): + return False + for key in ["entrypoint", "command", "module", "function"]: + expected = spec.get(key) + approved = review.get(key) + if expected and approved and expected != approved: + return False + if review.get("module") and not safe_python_name(str(review["module"])): + return False + if review.get("function") and not safe_python_name(str(review["function"])): + return False + command = review.get("command") + if isinstance(command, str) and ("\n" in command or "\r" in command): + return False + return True + + +def missing_explicit_adapter_mapping(spec: dict[str, Any], review: dict[str, Any]) -> list[str]: + adapter_type = spec.get("adapter_type") + if adapter_type == "python_api": + required = ["adapter_type", "entrypoint", "module", "function"] + elif adapter_type in {"cli", "workflow_engine"}: + required = ["adapter_type", "entrypoint", "command"] + elif adapter_type in {"notebook", "r_script"}: + required = ["adapter_type", "entrypoint"] + else: + required = ["adapter_type"] + return [key for key in required if missing_adapter_review_mapping_value(review.get(key))] + + +def missing_adapter_review_mapping_value(value: Any) -> bool: + return not isinstance(value, str) or value == "" + + +def adapter_review_has_ready_evidence(review: dict[str, Any]) -> bool: + dry_run = review.get("dry_run") or {} + return dry_run.get("status") in {"pass", "trusted_fixture"} + + +def adapter_review_has_verified_evidence(review: dict[str, Any]) -> bool: + output_validation = review.get("output_validation") or {} + return adapter_review_has_ready_evidence(review) and output_validation.get("status") == "pass" and bool(review.get("expected_outputs")) + + +def safe_python_name(value: str) -> bool: + return bool(re.match(r"^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$", value)) + + def build_adapter_spec( adapter_type: str, repo_evidence: dict[str, Any], @@ -253,6 +382,37 @@ def first_item(values: list[dict[str, Any]] | None) -> dict[str, Any] | None: return values[0] if values else None +def notebook_execution_policy(tutorial_trace: dict[str, Any]) -> dict[str, Any]: + combined = { + "will_execute": False, + "reason": "static_analysis_only", + "notebooks": [], + "shell_magics": [], + "line_magics": [], + "cell_magics": [], + "parameter_cells": [], + "large_outputs": [], + "risks": [], + } + risks: set[str] = set() + for tutorial in tutorial_trace.get("tutorials", []): + policy = tutorial.get("execution_policy") + if not policy: + continue + path = tutorial.get("path") + combined["notebooks"].append(path) + for key in ["shell_magics", "line_magics", "cell_magics", "large_outputs"]: + for item in policy.get(key, []) or []: + record = dict(item) + record["notebook"] = path + combined[key].append(record) + for cell in policy.get("parameter_cells", []) or []: + combined["parameter_cells"].append({"notebook": path, "cell": cell}) + risks.update(policy.get("risks", []) or []) + combined["risks"] = sorted(risks) + return combined + + def build_context( *, skill_name: str | None = None, @@ -270,6 +430,7 @@ def build_context( no_execute_tutorials: bool = True, strict_evidence: bool = False, tutorial_filter: str | None = None, + adapter_review: str | Path | None = None, collection_dir: str | Path | None = None, ) -> dict[str, Any]: skill_name = slugify(skill_name or algorithm_name or "generated-skill") @@ -316,6 +477,7 @@ def build_context( maturity_level, allow_ready_adapter=allow_ready_python_adapter(resolved.repo_path), ) + adapter_spec, adapter_review_data = apply_adapter_review(adapter_spec, load_adapter_review(adapter_review)) environment_spec = infer_environment_spec(dependency_evidence, classification["language"]) if classification["language"] == "python" and not environment_spec["python"]["packages"]: environment_spec["python"]["packages"] = [] @@ -379,6 +541,8 @@ def build_context( "install_plan": install_plan, "install_plan_markdown": render_install_plan_markdown(install_plan), "adapter_spec": adapter_spec, + "adapter_review": adapter_review_data, + "notebook_execution_policy": notebook_execution_policy(tutorial_trace), "algorithm_contract": algorithm_contract, "bio_contract": bio_contract, "evidence_graph": evidence_graph, @@ -391,6 +555,7 @@ def build_context( "no_execute_tutorials": no_execute_tutorials, "strict_evidence": strict_evidence, "tutorial_filter": tutorial_filter, + "adapter_review": str(adapter_review) if adapter_review else None, }, "warnings": resolved.warnings, "demo_data": _demo_data_for_language(classification["language"]), @@ -439,6 +604,8 @@ def generate_skill(context: dict[str, Any], out_dir: str | Path) -> Path: write_json(root / "references" / "repo_evidence.json", public_context["repo_evidence"]) write_yaml(root / "references" / "algorithm_contract.yaml", public_context["algorithm_contract"]) write_yaml(root / "references" / "adapter_spec.yaml", public_context["adapter_spec"]) + write_yaml(root / "references" / "adapter_review.yaml", public_context["adapter_review"]) + write_json(root / "references" / "notebook_execution_policy.json", public_context["notebook_execution_policy"]) write_yaml(root / "references" / "bio_contract.yaml", public_context["bio_contract"]) write_yaml(root / "references" / "io_contract.yaml", {"input_contract": public_context["algorithm_contract"].get("input_contract"), "output_contract": public_context["algorithm_contract"].get("output_contract")}) write_json(root / "references" / "evidence_graph.json", public_context["evidence_graph"]) @@ -457,7 +624,7 @@ def generate_skill(context: dict[str, Any], out_dir: str | Path) -> Path: def write_source_snapshot(context: dict[str, Any], root: Path) -> None: adapter_spec = context.get("adapter_spec") or {} - if adapter_spec.get("adapter_type") != "python_api" or adapter_spec.get("status") != "ready": + if adapter_spec.get("adapter_type") != "python_api" or adapter_spec.get("status") not in EXECUTABLE_ADAPTER_STATUSES: return repo_path = (((context.get("source_manifest") or {}).get("repo") or {}).get("resolved_path")) if not repo_path: @@ -553,6 +720,8 @@ def plan_outputs(context: dict[str, Any], out_dir: str | Path) -> Path: write_json(root / "tutorial_trace.json", public_context["tutorial_trace"]) write_json(root / "workflow_dag.json", public_context["workflow"].get("workflow_dag", {"nodes": [], "edges": []})) write_yaml(root / "adapter_spec.preview.yaml", public_context["adapter_spec"]) + write_yaml(root / "adapter_review.preview.yaml", public_context["adapter_review"]) + write_json(root / "notebook_execution_policy.json", public_context["notebook_execution_policy"]) write_yaml(root / "algorithm_contract.preview.yaml", public_context["algorithm_contract"]) write_json(root / "environment_report.json", _public_environment_report(context["environment_report"])) write_text(root / "build_plan.md", _build_plan_markdown(public_context)) diff --git a/paper2skill/inference/bio_rules.py b/paper2skill/inference/bio_rules.py index e5225a2..fec93bf 100644 --- a/paper2skill/inference/bio_rules.py +++ b/paper2skill/inference/bio_rules.py @@ -3,13 +3,13 @@ MODALITY_RULES = { "scRNA-seq": ["single-cell RNA", "scRNA-seq", "AnnData", "Seurat", "scanpy", "h5ad", "10x"], "spatial_transcriptomics": ["spatial transcriptomics", "Visium", "Slide-seq", "MERFISH", "spatial", "Squidpy"], - "bulk_RNA-seq": ["bulk RNA", "DESeq2", "edgeR", "limma", "counts matrix"], + "bulk_RNA-seq": ["bulk RNA", "bulk RNA-seq", "DESeq2", "edgeR", "limma", "counts matrix", "countData"], "scATAC-seq": ["scATAC", "Signac", "ArchR", "peak matrix", "fragments.tsv"], "multiome": ["multiome", "RNA + ATAC", "paired RNA and ATAC"], } MATRIX_STATE_RULES = { - "raw_counts_loaded": ["raw counts", "count matrix", "read_10x_mtx", "Read10X", "counts slot", "layers['counts']"], + "raw_counts_loaded": ["raw counts", "count matrix", "counts matrix", "countData", "read_10x_mtx", "Read10X", "DESeqDataSetFromMatrix", "counts slot", "layers['counts']"], "normalized": ["NormalizeData", "normalize_total", "CPM", "TPM", "size factor"], "log1p_transformed": ["log1p", "log-normalized", "LogNormalize"], "scaled": ["ScaleData", "scale", "z-score", "standardized"], diff --git a/paper2skill/inference/infer_bio_contract.py b/paper2skill/inference/infer_bio_contract.py index f9442f7..154d99f 100644 --- a/paper2skill/inference/infer_bio_contract.py +++ b/paper2skill/inference/infer_bio_contract.py @@ -68,7 +68,7 @@ def infer_bio_contract( celltype_key = metadata_key(all_text, "celltype_key", ["cell_type", "celltype", "celltypes"]) base = default_bio_contract()["bio_contract"] base["modality"] = { - "primary": field_value(modality[0], modality[2], modality[1]) if modality else field_value(), + "primary": field_value(normalize_modality(modality[0]), modality[2], modality[1]) if modality else field_value(), "secondary": field_value(), } base["organism"] = { @@ -89,16 +89,25 @@ def infer_bio_contract( "batch_key": field_value(metadata_key(all_text, "batch_key", ["batch"]), "medium", ["tutorial_metadata_key"]) if metadata_key(all_text, "batch_key", ["batch"]) else field_value(), "condition_key": field_value(metadata_key(all_text, "condition_key", ["condition"]), "medium", ["tutorial_metadata_key"]) if metadata_key(all_text, "condition_key", ["condition"]) else field_value(), } + base["modality_contracts"] = modality_contracts(base, all_text) return {"bio_contract": base} def evidence_texts(tutorial_trace: dict[str, Any], paper_sections: list[dict[str, Any]]) -> list[dict[str, Any]]: items = [] + seen_steps: set[str] = set() for tutorial in tutorial_trace.get("tutorials", []): for step in tutorial.get("steps", tutorial.get("workflow_steps", [])): - text = "\n".join([step.get("code_preview", ""), step.get("command_or_code", ""), " ".join(step.get("function_calls", [])), " ".join(step.get("imports", []))]) + seen_steps.add(step.get("step_id", step.get("evidence_id", ""))) + text = step_text(step) source = "tutorial" items.append({"evidence_id": step.get("evidence_id") or step.get("step_id", "tutorial:unknown"), "text": text, "source_type": source, "section_role": "code", "weight": source_weight(source)}) + for step in tutorial_trace.get("workflow_steps", []): + step_id = step.get("step_id", step.get("evidence_id", "")) + if step_id in seen_steps: + continue + source = "tutorial" + items.append({"evidence_id": step.get("evidence_id") or step.get("step_id", "tutorial:unknown"), "text": step_text(step), "source_type": source, "section_role": "code", "weight": source_weight(source)}) for section in paper_sections: role = section_role(section.get("section_id", ""), section.get("title", "")) source = f"paper:{role}" @@ -106,6 +115,17 @@ def evidence_texts(tutorial_trace: dict[str, Any], paper_sections: list[dict[str return items +def step_text(step: dict[str, Any]) -> str: + return "\n".join( + [ + step.get("code_preview", ""), + step.get("command_or_code", ""), + " ".join(str(call) for call in step.get("function_calls", [])), + " ".join(step.get("imports", [])), + ] + ) + + def first_with_evidence(items: list[dict[str, Any]], rules: dict[str, list[str]], strict_evidence: bool = False) -> tuple[str, list[str], str] | None: candidates = [] for item in items: @@ -147,6 +167,85 @@ def metadata_key(text: str, _field: str, candidates: list[str]) -> str | None: return None +def normalize_modality(value: str) -> str: + return { + "bulk_RNA-seq": "bulk RNA-seq", + "spatial_transcriptomics": "spatial", + }.get(value, value) + + +def matrix_state_value(base: dict[str, Any]) -> dict[str, Any]: + matrix = base.get("input_matrix_state", {}) + transformations = matrix.get("matrix_transformations", []) or [] + if "log1p_transformed" in transformations: + return field_value("log1p", "high", matrix.get("log_transformed_allowed", {}).get("evidence", [])) + if "normalized" in transformations: + return field_value("normalized", "high", matrix.get("normalized_allowed", {}).get("evidence", [])) + if "raw_counts_loaded" in transformations or matrix.get("raw_counts_required", {}).get("value") is True: + return field_value("raw_counts", "high", matrix.get("raw_counts_required", {}).get("evidence", [])) + return field_value() + + +def modality_contracts(base: dict[str, Any], all_text: str) -> dict[str, Any]: + modality = ((base.get("modality") or {}).get("primary") or {}).get("value") + metadata = base.get("metadata_requirements", {}) + contracts = { + "scrna_seq": { + "input_state": {"matrix_state": field_value(), "formats": ["10x_mtx", "h5ad", "rds"]}, + "metadata": {"celltype_key": field_value(), "sample_key": field_value(), "batch_key": field_value(), "condition_key": field_value()}, + "reference_resources": base.get("reference_resources", {}), + "outputs": {"required": []}, + }, + "bulk_rna_seq": { + "input_state": {"matrix_state": field_value(), "formats": ["count_matrix", "csv", "tsv"]}, + "metadata": {"sample_key": field_value(), "condition_key": field_value(), "batch_key": field_value()}, + "statistical": {"design_formula": field_value(), "replicates": field_value()}, + "reference_resources": base.get("reference_resources", {}), + "outputs": {"required": []}, + }, + "spatial": { + "input_state": {"matrix_state": field_value(), "formats": ["h5ad", "rds", "spatial_directory"]}, + "metadata": {"spatial_coordinates": field_value(), "image": field_value()}, + "reference_resources": base.get("reference_resources", {}), + "outputs": {"required": []}, + }, + "proteomics": { + "input_state": {"matrix_state": field_value(), "formats": ["csv", "tsv"]}, + "metadata": {}, + "reference_resources": {}, + "outputs": {"required": []}, + }, + "general": { + "input_state": {"matrix_state": matrix_state_value(base), "formats": []}, + "metadata": metadata, + "reference_resources": base.get("reference_resources", {}), + "outputs": {"required": []}, + }, + } + if modality == "scRNA-seq": + contracts["scrna_seq"]["input_state"]["matrix_state"] = matrix_state_value(base) + contracts["scrna_seq"]["metadata"] = { + key: metadata.get(key, field_value()) + for key in ["celltype_key", "sample_key", "batch_key", "condition_key"] + } + if modality == "bulk RNA-seq": + contracts["bulk_rna_seq"]["input_state"]["matrix_state"] = matrix_state_value(base) + contracts["bulk_rna_seq"]["metadata"]["condition_key"] = metadata.get("condition_key", field_value()) + formula = design_formula(all_text) + if formula: + contracts["bulk_rna_seq"]["statistical"]["design_formula"] = field_value(formula, "high", ["tutorial_design_formula"]) + if modality == "spatial": + contracts["spatial"]["input_state"]["matrix_state"] = matrix_state_value(base) + return contracts + + +def design_formula(text: str) -> str | None: + match = re.search(r"design\s*=\s*(~\s*[A-Za-z0-9_+. ]+)", text) + if not match: + return None + return match.group(1).strip() + + def section_role(section_id: str, title: str) -> str: text = f"{section_id} {title}".lower() if any(word in text for word in ["methods", "method", "data", "code", "software"]): diff --git a/paper2skill/inference/infer_io_contract.py b/paper2skill/inference/infer_io_contract.py index 9cc8717..44657ca 100644 --- a/paper2skill/inference/infer_io_contract.py +++ b/paper2skill/inference/infer_io_contract.py @@ -8,13 +8,16 @@ def infer_io_contract(tutorial_trace: dict[str, Any], bio_contract: dict[str, An inputs = [] outputs = [] format_evidence = [] + text_items = [] for step in tutorial_trace.get("workflow_steps", []): inputs.extend(step.get("inputs", [])) inputs.extend(step.get("read_files", [])) outputs.extend(step.get("outputs", [])) outputs.extend(step.get("write_files", [])) format_evidence.extend(step_format_evidence(step)) + text_items.extend(step_format_evidence(step)) bio = (bio_contract or {}).get("bio_contract", {}) + modality = (((bio.get("modality") or {}).get("primary") or {}).get("value")) return { "input_contract": { "required": { @@ -22,10 +25,10 @@ def infer_io_contract(tutorial_trace: dict[str, Any], bio_contract: dict[str, An "primary_data": { "type": "file", "state": "not_confirmed" if not inputs else "tutorial_confirmed", - "format": file_format_field(format_evidence), + "format": file_format_field(format_evidence, modality), "matrix_state": matrix_state_field(bio), "matrix_orientation": field_value("unknown", "low", []), - "metadata_keys": metadata_keys(bio), + "metadata_keys": metadata_keys(bio, text_items), "organism": organism_fields(bio), "external_resources": external_resources(bio), "evidence": sorted({str(value) for value in inputs if value}), @@ -68,17 +71,31 @@ def field_value(value: Any, confidence: str = "low", evidence: list[str] | None return {"value": value, "confidence": confidence, "evidence": evidence or []} -def file_format_field(paths: list[Any]) -> dict[str, Any]: +def file_format_field(paths: list[Any], modality: str | None = None) -> dict[str, Any]: + detected_values: dict[str, list[str]] = {} for value in paths: - detected = detect_file_format(str(value)) + detected = detect_file_format(str(value), modality) if detected != "unknown": - return field_value(detected, "high", [str(value)]) + detected_values.setdefault(detected, []).append(str(value)) + primary_values = {key: value for key, value in detected_values.items() if key not in {"csv", "tsv"}} + if primary_values: + detected_values = primary_values + if len(detected_values) == 1: + detected, evidence = next(iter(detected_values.items())) + return field_value(detected, "high", evidence) + if len(detected_values) > 1: + evidence = [item for values in detected_values.values() for item in values] + result = field_value("not_confirmed", "low", evidence) + result["conflicts"] = sorted(detected_values) + return result return field_value("unknown", "low", []) -def detect_file_format(value: str) -> str: +def detect_file_format(value: str, modality: str | None = None) -> str: lower = value.lower() name = PurePosixPath(lower.replace("\\", "/")).name + if modality == "bulk RNA-seq" and ("deseqdatasetfrommatrix" in lower or "countdata" in lower or "counts" in name): + return "count_matrix" if lower.endswith(".h5ad"): return "h5ad" if lower.endswith((".rds", ".rda")): @@ -87,6 +104,8 @@ def detect_file_format(value: str) -> str: return "10x_mtx" if lower.endswith(".mtx"): return "mtx" + if modality == "bulk RNA-seq" and lower.endswith((".csv", ".tsv", ".txt")): + return "count_matrix" if lower.endswith((".csv", ".tsv")): return "csv" if lower.endswith(".csv") else "tsv" if lower.endswith(".loom"): @@ -108,9 +127,15 @@ def matrix_state_field(bio: dict[str, Any]) -> dict[str, Any]: return field_value("unknown", "low", []) -def metadata_keys(bio: dict[str, Any]) -> dict[str, Any]: +def metadata_keys(bio: dict[str, Any], text_items: list[str] | None = None) -> dict[str, Any]: metadata = bio.get("metadata_requirements", {}) - return {key: metadata.get(key, field_value("not_confirmed")) for key in ["sample_key", "batch_key", "condition_key", "celltype_key", "perturbation_key"]} + values = {key: metadata.get(key, field_value("not_confirmed")) for key in ["sample_key", "batch_key", "condition_key", "celltype_key", "perturbation_key"]} + text = "\n".join(text_items or []) + if values["condition_key"]["value"] == "not_confirmed" and "condition" in text.lower(): + values["condition_key"] = field_value("condition", "medium", ["tutorial_metadata_key"]) + if values["sample_key"]["value"] == "not_confirmed" and "sample" in text.lower(): + values["sample_key"] = field_value("sample", "medium", ["tutorial_metadata_key"]) + return values def organism_fields(bio: dict[str, Any]) -> dict[str, Any]: diff --git a/paper2skill/miners/api_miner.py b/paper2skill/miners/api_miner.py index cfdb634..a6a49b6 100644 --- a/paper2skill/miners/api_miner.py +++ b/paper2skill/miners/api_miner.py @@ -43,7 +43,8 @@ def mine_api(repo_path: str | Path | None) -> dict[str, Any]: item = dict(item) item["path"] = public_local_path(path, root) r_functions.append(item) - r_exports = r_namespace_exports(root) + r_namespace = r_namespace_imports(root) + r_exports = r_namespace["exports"] language = "python" if api_functions or classes else ("r" if r_functions else "unknown") return { "language": language, @@ -54,6 +55,7 @@ def mine_api(repo_path: str | Path | None) -> dict[str, Any]: "cli_commands": cli_commands, "api_functions": api_functions or r_functions, "r_exports": r_exports, + "r_namespace": r_namespace, "classes": classes, "workflow_engines": workflow_engines(root), "tutorials": public_local_paths([p for p in root.rglob("*") if p.suffix.lower() in {".ipynb", ".py", ".r", ".rmd"}], root), @@ -138,18 +140,36 @@ def public_python_module_name(path: Path, root: Path, function_name: str) -> str def r_namespace_exports(root: Path) -> list[dict[str, Any]]: + return r_namespace_imports(root)["exports"] + + +def r_namespace_imports(root: Path) -> dict[str, list[dict[str, Any]]]: namespace = root / "NAMESPACE" + result = {"imports": [], "import_from": [], "exports": [], "s3methods": []} if not namespace.exists(): - return [] + return result text = namespace.read_text(encoding="utf-8", errors="replace") - names = re.findall(r"export\(([^)]+)\)", text) - exports = [] - for group in names: + for group in re.findall(r"\bimport\(([^)]+)\)", text): + for name in group.split(","): + clean = name.strip().strip("'\"") + if clean: + result["imports"].append({"package": clean, "source": "NAMESPACE:import"}) + for group in re.findall(r"\bimportFrom\(([^)]+)\)", text): + parts = [part.strip().strip("'\"") for part in group.split(",") if part.strip()] + if len(parts) >= 2: + package = parts[0] + for function in parts[1:]: + result["import_from"].append({"package": package, "function": function, "source": "NAMESPACE:importFrom"}) + for group in re.findall(r"\bexport\(([^)]+)\)", text): for name in group.split(","): clean = name.strip().strip("'\"") if clean: - exports.append({"name": clean, "source": "NAMESPACE"}) - return exports + result["exports"].append({"name": clean, "source": "NAMESPACE:export"}) + for group in re.findall(r"\bS3method\(([^)]+)\)", text): + parts = [part.strip().strip("'\"") for part in group.split(",") if part.strip()] + if len(parts) >= 2: + result["s3methods"].append({"generic": parts[0], "class": parts[1], "source": "NAMESPACE:S3method"}) + return result def workflow_engines(root: Path) -> list[dict[str, Any]]: diff --git a/paper2skill/miners/dependency_miner.py b/paper2skill/miners/dependency_miner.py index 8ca22f9..932f4fd 100644 --- a/paper2skill/miners/dependency_miner.py +++ b/paper2skill/miners/dependency_miner.py @@ -15,7 +15,20 @@ from paper2skill.miners.script_miner import R_LIBRARY_RE R_BASE_PACKAGES = {"base", "compiler", "datasets", "graphics", "grDevices", "grid", "methods", "parallel", "splines", "stats", "tools", "utils"} -BIOCONDUCTOR_HINTS = {"DESeq2", "edgeR", "limma", "clusterProfiler", "SingleCellExperiment", "SummarizedExperiment", "AnnotationDbi", "BiocGenerics", "scran", "scater", "ComplexHeatmap"} +BIOCONDUCTOR_HINTS = { + "DESeq2", + "edgeR", + "limma", + "clusterProfiler", + "SingleCellExperiment", + "SummarizedExperiment", + "AnnotationDbi", + "BiocGenerics", + "BiocStyle", + "scran", + "scater", + "ComplexHeatmap", +} def parse_requirements(path: Path) -> list[str]: @@ -139,11 +152,22 @@ def parse_description(path: Path) -> list[str]: def parse_description_fields(path: Path) -> tuple[list[str], dict[str, list[str]]]: + records, optional_records = parse_description_field_records(path) + required = sorted(dict.fromkeys(record["name"] for record in records)) + optional = { + key: sorted(dict.fromkeys(record["name"] for record in value)) + for key, value in optional_records.items() + if value + } + return required, optional + + +def parse_description_field_records(path: Path) -> tuple[list[dict[str, Any]], dict[str, list[dict[str, Any]]]]: text = path.read_text(encoding="utf-8", errors="replace") - fields: dict[str, list[str]] = {"Imports": [], "Depends": [], "Suggests": [], "Enhances": []} + fields: dict[str, list[dict[str, Any]]] = {"Imports": [], "Depends": [], "LinkingTo": [], "Suggests": [], "Enhances": []} capture: str | None = None for line in text.splitlines(): - match = re.match(r"^(Imports|Depends|Suggests|Enhances):", line) + match = re.match(r"^(Imports|Depends|LinkingTo|Suggests|Enhances):", line) if match: capture = match.group(1) line = line.split(":", 1)[1] @@ -151,23 +175,39 @@ def parse_description_fields(path: Path) -> tuple[list[str], dict[str, list[str] capture = None if capture: for item in line.split(","): - name = re.sub(r"\(.+?\)", "", item).strip() - if name and name not in {"R"} and name not in R_BASE_PACKAGES: - fields[capture].append(name) - required = sorted(dict.fromkeys(fields["Imports"] + fields["Depends"])) + record = _parse_r_package_item(item, capture) + if record: + fields[capture].append(record) + required = _dedupe_records(fields["Imports"] + fields["Depends"] + fields["LinkingTo"], "name") optional = { - "DESCRIPTION:Suggests": sorted(dict.fromkeys(fields["Suggests"])), - "DESCRIPTION:Enhances": sorted(dict.fromkeys(fields["Enhances"])), + "DESCRIPTION:Suggests": _dedupe_records(fields["Suggests"], "name"), + "DESCRIPTION:Enhances": _dedupe_records(fields["Enhances"], "name"), } return required, {key: value for key, value in optional.items() if value} +def _parse_r_package_item(item: str, field: str) -> dict[str, Any] | None: + item = item.strip() + if not item: + return None + match = re.match(r"^([A-Za-z][A-Za-z0-9_.]*)(?:\s*\(([^)]+)\))?$", item) + if not match: + return None + name, version = match.groups() + if name in {"R"} or name in R_BASE_PACKAGES: + return None + record = {"name": name, "field": field} + if version: + record["version_spec"] = version.strip() + return record + + def parse_description_metadata(path: Path) -> dict[str, list[str]]: text = path.read_text(encoding="utf-8", errors="replace") - fields: dict[str, list[str]] = {"SystemRequirements": [], "Remotes": []} + fields: dict[str, list[str]] = {"SystemRequirements": [], "Remotes": [], "biocViews": []} capture: str | None = None for line in text.splitlines(): - match = re.match(r"^(SystemRequirements|Remotes):", line) + match = re.match(r"^(SystemRequirements|Remotes|biocViews):", line) if match: capture = match.group(1) line = line.split(":", 1)[1] @@ -283,13 +323,25 @@ def mine_dependencies(repo_path: str | Path | None, tutorial_paths: list[str | P description = root / "DESCRIPTION" if description.exists(): dependency_files.append(public_local_path(description, root)) - required_r, optional_r = parse_description_fields(description) + required_r_records, optional_r_records = parse_description_field_records(description) + required_r = [record["name"] for record in required_r_records] r_packages.extend(required_r) - r_records.extend({"name": name, "source": "Bioconductor_or_unknown" if name in BIOCONDUCTOR_HINTS else "DESCRIPTION", "evidence": "Imports/Depends", "required": True, "category": "runtime"} for name in required_r) - optional["r"].update(optional_r) + for record in required_r_records: + item = { + "name": record["name"], + "source": "Bioconductor_or_unknown" if record["name"] in BIOCONDUCTOR_HINTS else "DESCRIPTION", + "evidence": f"DESCRIPTION:{record['field']}", + "required": True, + "category": "runtime", + } + if "version_spec" in record: + item["version_spec"] = record["version_spec"] + r_records.append(item) + optional["r"].update({key: sorted(record["name"] for record in value) for key, value in optional_r_records.items()}) metadata = parse_description_metadata(description) system_requirements.extend({"value": item, "source": "DESCRIPTION", "required": True, "install": "plan_only"} for item in metadata.get("SystemRequirements", [])) external_resources.extend({"name": item, "type": "r_remote", "source": "DESCRIPTION:Remotes", "required": False, "downloadable": False} for item in metadata.get("Remotes", [])) + bioconductor = {"biocViews": metadata.get("biocViews", []), "hinted_packages": sorted(name for name in required_r if name in BIOCONDUCTOR_HINTS)} renv = root / "renv.lock" if renv.exists(): dependency_files.append(public_local_path(renv, root)) @@ -317,6 +369,7 @@ def mine_dependencies(repo_path: str | Path | None, tutorial_paths: list[str | P "conda_records": sorted(conda_records, key=lambda item: item.get("package", item.get("name", ""))), "system_requirements": system_requirements, "external_resources": external_resources, + "bioconductor": bioconductor if "bioconductor" in locals() else {"biocViews": [], "hinted_packages": []}, "executables": [], } diff --git a/paper2skill/miners/notebook_miner.py b/paper2skill/miners/notebook_miner.py index 295953c..ff57f53 100644 --- a/paper2skill/miners/notebook_miner.py +++ b/paper2skill/miners/notebook_miner.py @@ -1,10 +1,23 @@ from __future__ import annotations +import ast import json from pathlib import Path +import re from typing import Any from paper2skill.miners.python_ast import classify_bio_signals, infer_object_flow, mine_python_source +from paper2skill.miners.script_miner import mine_r_source + + +READ_NAMES = {"open", "read_csv", "read_table", "read_excel", "read_h5ad", "load", "loadtxt", "read_10x_mtx"} +WRITE_NAMES = {"to_csv", "write", "write_csv", "save", "savefig", "save_npz", "dump"} +CELL_MAGIC_RE = re.compile(r"^\s*%%([A-Za-z0-9_]+)\b[^\n]*\n?(.*)$", re.S) +LINE_MAGIC_RE = re.compile(r"^\s*%([A-Za-z0-9_]+)\b(.*)$") +SHELL_LINE_RE = re.compile(r"^\s*!(.+)$") +NETWORK_WORDS = ("wget ", "curl ", "gsutil ", "aws s3", "http://", "https://") +INSTALL_WORDS = ("pip install", "conda install", "mamba install") +PYTHON_WRAPPER_CELL_MAGICS = {"capture", "time", "timeit", "prun"} def _source_to_text(source: str | list[str]) -> str: @@ -13,14 +26,47 @@ def _source_to_text(source: str | list[str]) -> str: return source +def default_execution_policy() -> dict[str, Any]: + return { + "will_execute": False, + "reason": "static_analysis_only", + "shell_magics": [], + "line_magics": [], + "cell_magics": [], + "parameter_cells": [], + "large_outputs": [], + "risks": [], + } + + def mine_notebook(path: str | Path) -> dict[str, Any]: notebook_path = Path(path) data = json.loads(notebook_path.read_text(encoding="utf-8")) cells = [] steps = [] title = None + parameters: dict[str, Any] = {} + known_values: dict[str, Any] = {} + policy = default_execution_policy() for index, cell in enumerate(data.get("cells", [])): source = _source_to_text(cell.get("source", "")) + metadata = cell.get("metadata", {}) or {} + tags = set(metadata.get("tags", []) or []) + parameter_cell = "parameters" in tags or _looks_like_parameter_cell(source, index) + if parameter_cell: + policy["parameter_cells"].append(index) + _record_output_policy(index, cell.get("outputs", []) or [], policy) + shell_lines, line_magics, clean_source = _split_python_magics(source) + for command in shell_lines: + policy["shell_magics"].append({"cell": index, "command": command}) + _record_command_risks(command, policy) + for magic in line_magics: + policy["line_magics"].append({"cell": index, **magic}) + cell_magic = _cell_magic(source) + if cell_magic: + policy["cell_magics"].append({"cell": index, "magic": cell_magic["magic"]}) + if cell_magic["magic"].lower() in {"bash", "sh"}: + _record_command_risks(cell_magic["body"], policy) record: dict[str, Any] = { "index": index, "cell_type": cell.get("cell_type", "unknown"), @@ -34,42 +80,205 @@ def mine_notebook(path: str | Path) -> dict[str, Any]: "plots": [], } if record["cell_type"] == "code": - mined = mine_python_source(source) + language = "python" + command_or_code = clean_source + if cell_magic and cell_magic["magic"].lower() == "r": + language = "r" + command_or_code = cell_magic["body"] + mined = mine_r_source(command_or_code) + elif cell_magic and cell_magic["magic"].lower() in {"bash", "sh"}: + language = "bash" + command_or_code = cell_magic["body"] + mined = {"imports": [], "assignments": [], "function_calls": [], "file_reads": [], "file_writes": [], "parameters": {}, "plots": []} + else: + mined = mine_python_source(clean_source) + io_summary = _python_io_with_context(clean_source, known_values) + mined["file_reads"] = sorted(dict.fromkeys((mined.get("file_reads", []) or []) + io_summary["file_reads"])) + mined["file_writes"] = sorted(dict.fromkeys((mined.get("file_writes", []) or []) + io_summary["file_writes"])) + mined["assignments"] = (mined.get("assignments", []) or []) + io_summary["assignments"] + mined["parameters"] = {**(mined.get("parameters", {}) or {}), **io_summary["parameters"]} + known_values.update(io_summary["parameters"]) + if io_summary["hidden_state"]: + _add_risk(policy, "hidden_state") record.update({key: mined.get(key, record.get(key)) for key in record if key in mined}) - flow = infer_object_flow(source) - code_preview = source.strip().splitlines()[0] if source.strip() else "Empty code cell" + if parameter_cell: + parameters.update({key: value for key, value in record["parameters"].items() if isinstance(value, (str, int, float, bool)) or value is None}) + known_values.update(parameters) + flow = infer_object_flow(command_or_code) if language == "python" else {"input_objects": [], "output_objects": []} + code_preview = command_or_code.strip().splitlines()[0] if command_or_code.strip() else "Empty code cell" step_id = f"tutorial_001:cell_{index:03d}" - function_calls = [call["name"] for call in record["function_calls"]] - steps.append( - { - "id": step_id, - "step_id": step_id, - "name": f"Notebook cell {index}", - "description": code_preview, - "source": f"{notebook_path}:cell:{index}", - "source_type": "tutorial_notebook", - "evidence_id": f"{notebook_path.name}:cell:{index}", - "language": "python", - "code_preview": code_preview, - "imports": record["imports"], - "function_calls": function_calls, - "read_files": record["file_reads"], - "write_files": record["file_writes"], - "inputs": record["file_reads"], - "outputs": record["file_writes"], - "input_objects": flow["input_objects"], - "output_objects": flow["output_objects"], - "parameters": record["parameters"], - "figures": [call["name"] for call in record["plots"]], - "bio_signals": classify_bio_signals(source, record["function_calls"]), - "command_or_code": source, - "confidence": "high", - } - ) + if command_or_code.strip(): + function_calls = [call["name"] if isinstance(call, dict) else str(call) for call in record["function_calls"]] + steps.append( + { + "id": step_id, + "step_id": step_id, + "name": f"Notebook cell {index}", + "description": code_preview, + "source": f"{notebook_path}:cell:{index}", + "source_type": "tutorial_notebook", + "evidence_id": f"{notebook_path.name}:cell:{index}", + "language": language, + "code_preview": code_preview, + "imports": record["imports"], + "function_calls": function_calls, + "read_files": record["file_reads"], + "write_files": record["file_writes"], + "inputs": record["file_reads"], + "outputs": record["file_writes"], + "input_objects": flow["input_objects"], + "output_objects": flow["output_objects"], + "parameters": record["parameters"], + "figures": [call["name"] if isinstance(call, dict) else str(call) for call in record["plots"]], + "bio_signals": classify_bio_signals(command_or_code, record["function_calls"]), + "command_or_code": command_or_code, + "confidence": "high" if language in {"python", "r"} else "low", + } + ) elif record["cell_type"] == "markdown" and title is None: for line in source.splitlines(): if line.strip().startswith("#"): title = line.strip().lstrip("#").strip() break cells.append(record) - return {"path": str(notebook_path), "title": title, "language": "python", "cells": cells, "steps": steps, "workflow_steps": steps} + policy["risks"] = sorted(policy["risks"]) + return { + "path": str(notebook_path), + "title": title, + "language": "python", + "cells": cells, + "steps": steps, + "workflow_steps": steps, + "parameters": parameters, + "execution_policy": policy, + } + + +def _cell_magic(source: str) -> dict[str, str] | None: + match = CELL_MAGIC_RE.match(source) + if not match: + return None + return {"magic": match.group(1), "body": match.group(2)} + + +def _split_python_magics(source: str) -> tuple[list[str], list[dict[str, str]], str]: + shell_lines = [] + line_magics = [] + clean_lines = [] + cell_magic = _cell_magic(source) + if cell_magic: + if cell_magic["magic"].lower() in {"r", "bash", "sh"}: + return shell_lines, line_magics, "" + if cell_magic["magic"].lower() in PYTHON_WRAPPER_CELL_MAGICS: + return shell_lines, line_magics, cell_magic["body"] + return shell_lines, line_magics, "" + for line in source.splitlines(): + shell = SHELL_LINE_RE.match(line) + if shell: + shell_lines.append(shell.group(1).strip()) + continue + magic = LINE_MAGIC_RE.match(line) + if magic: + line_magics.append({"magic": magic.group(1), "args": magic.group(2).strip()}) + continue + clean_lines.append(line) + return shell_lines, line_magics, "\n".join(clean_lines) + ("\n" if clean_lines else "") + + +def _looks_like_parameter_cell(source: str, index: int) -> bool: + if index != 0: + return False + mined = mine_python_source(source) + return bool(mined.get("parameters")) and not mined.get("function_calls") + + +def _record_output_policy(index: int, outputs: list[dict[str, Any]], policy: dict[str, Any]) -> None: + for output in outputs: + data = output.get("data", {}) if isinstance(output, dict) else {} + for mime, value in data.items(): + size = len(value) if isinstance(value, str) else len(json.dumps(value)) + if mime.startswith("image/") and size > 100_000: + policy["large_outputs"].append({"cell": index, "mime": mime, "size": size}) + _add_risk(policy, "large_output") + elif size > 200_000: + policy["large_outputs"].append({"cell": index, "mime": mime, "size": size}) + _add_risk(policy, "large_output") + + +def _record_command_risks(command: str, policy: dict[str, Any]) -> None: + lower = command.lower() + if any(word in lower for word in INSTALL_WORDS): + _add_risk(policy, "install_command") + if any(word in lower for word in NETWORK_WORDS): + _add_risk(policy, "network_download") + if re.search(r">\s*\S+", command): + _add_risk(policy, "shell_file_write") + + +def _add_risk(policy: dict[str, Any], risk: str) -> None: + if risk not in policy["risks"]: + policy["risks"].append(risk) + + +def _python_io_with_context(source: str, context: dict[str, Any]) -> dict[str, Any]: + summary = {"file_reads": [], "file_writes": [], "parameters": {}, "assignments": [], "hidden_state": False} + if not source.strip(): + return summary + try: + tree = ast.parse(source) + except SyntaxError: + return summary + values = dict(context) + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + value = _resolve_py_value(node.value, values) + for target in node.targets: + if isinstance(target, ast.Name): + values[target.id] = value + if isinstance(value, (str, int, float, bool)) or value is None: + summary["parameters"][target.id] = value + summary["assignments"].append({"name": target.id, "lineno": node.lineno, "value": value}) + for node in ast.walk(tree): + if isinstance(node, ast.Call): + name = _call_name(node.func) + if not name: + continue + short = name.split(".")[-1] + if not node.args: + continue + first = _resolve_py_value(node.args[0], values) + if short in READ_NAMES and isinstance(first, str): + summary["file_reads"].append(first) + if short in WRITE_NAMES and isinstance(first, str): + summary["file_writes"].append(first) + for arg in node.args: + if isinstance(arg, ast.Name) and arg.id not in values: + summary["hidden_state"] = True + summary["file_reads"] = sorted(dict.fromkeys(summary["file_reads"])) + summary["file_writes"] = sorted(dict.fromkeys(summary["file_writes"])) + return summary + + +def _call_name(node: ast.AST) -> str | None: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + parent = _call_name(node.value) + return f"{parent}.{node.attr}" if parent else node.attr + return None + + +def _resolve_py_value(node: ast.AST, values: dict[str, Any]) -> Any: + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Name): + return values.get(node.id) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + left = _resolve_py_value(node.left, values) + right = _resolve_py_value(node.right, values) + if isinstance(left, str) and isinstance(right, str): + return left + right + try: + return ast.literal_eval(node) + except Exception: + return None diff --git a/paper2skill/miners/script_miner.py b/paper2skill/miners/script_miner.py index 0e6dd53..ebb7066 100644 --- a/paper2skill/miners/script_miner.py +++ b/paper2skill/miners/script_miner.py @@ -8,11 +8,14 @@ R_LIBRARY_RE = re.compile(r"\b(?:library|require)\s*\(\s*['\"]?([A-Za-z0-9_.]+)['\"]?\s*\)") -R_FUNCTION_RE = re.compile(r"\b([A-Za-z][A-Za-z0-9_.]*)\s*\(") +R_FUNCTION_RE = re.compile(r"\b(?:(?P[A-Za-z][A-Za-z0-9_.]*):::{0,1})?(?P[A-Za-z][A-Za-z0-9_.]*)\s*\(") R_ASSIGN_RE = re.compile(r"^\s*([A-Za-z][A-Za-z0-9_.]*)\s*(?:<-|=)\s*(.+?)\s*$") R_READ_RE = re.compile(r"\b(read\.[A-Za-z0-9_.]+|readRDS|read_h5ad)\s*\(\s*([A-Za-z][A-Za-z0-9_.]*|['\"][^'\"]+['\"])") R_WRITE_RE = re.compile(r"\b(write\.[A-Za-z0-9_.]+|saveRDS|ggsave)\s*\((?:[^,\n]+,\s*)?([A-Za-z][A-Za-z0-9_.]*|['\"][^'\"]+['\"])") R_FIGURE_RE = re.compile(r"\b(ggsave|pdf|png|jpeg|tiff|svg)\s*\(") +R_SOURCE_RE = re.compile(r"\bsource\s*\(\s*([^)]+)\)") +R_READ_NAMES = {"read.csv", "read.table", "read.delim", "readRDS", "read_h5ad", "Read10X"} +R_WRITE_NAMES = {"write.csv", "write.table", "saveRDS", "ggsave"} def mine_r_source(source: str) -> dict[str, Any]: @@ -21,17 +24,27 @@ def mine_r_source(source: str) -> dict[str, Any]: assignments = [] parameters: dict[str, Any] = {} for lineno, line in enumerate(source.splitlines(), start=1): - for call in R_FUNCTION_RE.findall(line): - if call not in {"if", "for", "while", "function"}: - calls.append({"name": call, "lineno": lineno}) + for match in R_FUNCTION_RE.finditer(line): + package = match.group("package") + function = match.group("function") + if function not in {"if", "for", "while", "function"}: + record = {"name": f"{package}::{function}" if package else function, "lineno": lineno, "function": function, "package": package} + if package: + record["package"] = package + if package not in imports: + imports.append(package) + calls.append(record) match = R_ASSIGN_RE.match(line) if match: name, value = match.groups() - clean_value = value.strip().strip("'\"") + clean_value = _resolve_r_value(value, parameters) assignments.append({"name": name, "lineno": lineno, "value": clean_value}) parameters[name] = clean_value - file_reads = [_resolve_r_arg(match[1], parameters) for match in R_READ_RE.findall(source)] - file_writes = [_resolve_r_arg(match[1], parameters) for match in R_WRITE_RE.findall(source)] + file_reads = _r_call_paths(source, R_READ_NAMES, parameters, mode="read") + file_reads.extend(_resolve_r_arg(match[1], parameters) for match in R_READ_RE.findall(source)) + file_writes = _r_call_paths(source, R_WRITE_NAMES, parameters, mode="write") + file_writes.extend(_resolve_r_arg(match[1], parameters) for match in R_WRITE_RE.findall(source)) + source_files = [_resolve_r_value(match, parameters) for match in R_SOURCE_RE.findall(source)] return { "imports": imports, "functions": [], @@ -39,20 +52,110 @@ def mine_r_source(source: str) -> dict[str, Any]: "function_calls": calls, "assignments": assignments, "parameters": parameters, - "file_reads": [item for item in file_reads if item], - "file_writes": [item for item in file_writes if item], + "file_reads": sorted(dict.fromkeys(item for item in file_reads if item)), + "file_writes": sorted(dict.fromkeys(item for item in file_writes if item)), + "source_files": [item for item in source_files if isinstance(item, str)], "figures": R_FIGURE_RE.findall(source), } def _resolve_r_arg(value: str, parameters: dict[str, Any]) -> str | None: - value = value.strip().strip("'\"") - resolved = parameters.get(value, value) + resolved = _resolve_r_value(value, parameters) if isinstance(resolved, str): return resolved.strip().strip("'\"") return None +def _resolve_r_value(value: str, parameters: dict[str, Any]) -> Any: + value = value.strip() + if value.startswith("file.path(") and value.endswith(")"): + inner = value[len("file.path(") : -1] + parts = [_resolve_r_value(part, parameters) for part in _split_r_args(inner)] + return "/".join(str(part).strip("/").strip("'\"") for part in parts if part is not None) + if (value.startswith("'") and value.endswith("'")) or (value.startswith('"') and value.endswith('"')): + return value[1:-1] + return parameters.get(value, value.strip("'\"")) + + +def _split_r_args(value: str) -> list[str]: + args = [] + current = [] + quote: str | None = None + depth = 0 + for char in value: + if quote: + current.append(char) + if char == quote: + quote = None + continue + if char in {"'", '"'}: + quote = char + current.append(char) + elif char == "(": + depth += 1 + current.append(char) + elif char == ")": + depth -= 1 + current.append(char) + elif char == "," and depth == 0: + args.append("".join(current).strip()) + current = [] + else: + current.append(char) + if current: + args.append("".join(current).strip()) + return args + + +def _r_call_paths(source: str, names: set[str], parameters: dict[str, Any], mode: str) -> list[str]: + paths = [] + for match in R_FUNCTION_RE.finditer(source): + package = match.group("package") + function = match.group("function") + if function not in names: + continue + start = match.end() + args_text = _balanced_call_args(source, start) + if args_text is None: + continue + args = _split_r_args(args_text) + if not args: + continue + index = 0 + if mode == "write" and function == "saveRDS" and len(args) > 1: + index = 1 + target = _resolve_r_value(args[index], parameters) + if isinstance(target, str): + paths.append(target.strip("'\"")) + return paths + + +def _balanced_call_args(source: str, start: int) -> str | None: + depth = 1 + quote: str | None = None + chars = [] + for char in source[start:]: + if quote: + chars.append(char) + if char == quote: + quote = None + continue + if char in {"'", '"'}: + quote = char + chars.append(char) + elif char == "(": + depth += 1 + chars.append(char) + elif char == ")": + depth -= 1 + if depth == 0: + return "".join(chars) + chars.append(char) + else: + chars.append(char) + return None + + def mine_script(path: str | Path) -> dict[str, Any]: script_path = Path(path) source = script_path.read_text(encoding="utf-8", errors="replace") @@ -120,6 +223,7 @@ def mine_script(path: str | Path) -> dict[str, Any]: "function_calls": mined.get("function_calls", []), "assignments": mined.get("assignments", []), "parameters": mined.get("parameters", {}), + "source_files": mined.get("source_files", []), "file_reads": mined.get("file_reads", []), "file_writes": mined.get("file_writes", []), "steps": steps, diff --git a/paper2skill/schemas/adapter_spec_schema.yaml b/paper2skill/schemas/adapter_spec_schema.yaml index c19f9ee..ae99b70 100644 --- a/paper2skill/schemas/adapter_spec_schema.yaml +++ b/paper2skill/schemas/adapter_spec_schema.yaml @@ -11,4 +11,4 @@ properties: adapter_type: enum: [demo_only, python_api, cli, notebook, r_script, workflow_engine] status: - enum: [demo_only, candidate, ready, blocked] + enum: [demo_only, candidate, blocked, ready, reviewed, verified] diff --git a/paper2skill/schemas/bio_contract_schema.yaml b/paper2skill/schemas/bio_contract_schema.yaml index 2fb34d9..bab6869 100644 --- a/paper2skill/schemas/bio_contract_schema.yaml +++ b/paper2skill/schemas/bio_contract_schema.yaml @@ -7,4 +7,5 @@ properties: - organism - input_matrix_state - metadata_requirements + - modality_contracts - interpretation_boundary diff --git a/paper2skill/templates/codex_skill/SKILL.md.j2 b/paper2skill/templates/codex_skill/SKILL.md.j2 index cdeba01..7e3acce 100644 --- a/paper2skill/templates/codex_skill/SKILL.md.j2 +++ b/paper2skill/templates/codex_skill/SKILL.md.j2 @@ -67,7 +67,16 @@ Run: python scripts/run.py --manifest input_manifest.yaml ``` -`run.py` calls preflight first and stops if preflight fails. It records the selected adapter in `workflow/adapter_report.json`. Adapter readiness is declared in `references/adapter_spec.yaml`. Non-demo runs do not fall back to the demo summary runner: if the adapter is not ready, the run is blocked. Demo execution only proves the tutorial path, not biological validity on user data, and must not be treated as real algorithm execution unless the adapter report explicitly passes. +`run.py` calls preflight first and stops if preflight fails. It records the selected adapter in `workflow/adapter_report.json`. Adapter readiness is declared in `references/adapter_spec.yaml` and review evidence is recorded in `references/adapter_review.yaml`. Non-demo runs only execute adapters with status `ready`, `reviewed`, or `verified`; `candidate` and `blocked` adapters stop with `blocked_adapter_not_ready`. Demo execution only proves the tutorial path, not biological validity on user data, and must not be treated as real algorithm execution unless the adapter report explicitly passes. + +Adapter lifecycle: + +- `demo_only`: demo mode only. +- `candidate`: inferred but not reviewed; non-demo run is blocked. +- `blocked`: missing or unsafe wiring; non-demo run is blocked. +- `reviewed`: human-approved mapping passed static checks. +- `ready`: trusted fixture or minimal dry-run evidence passed. +- `verified`: executable adapter plus expected output contract validation. ## Output contract @@ -99,6 +108,8 @@ Dry-run results are not biological results. Demo runs only prove that the tutori - `references/tutorial_trace.md` - `references/workflow_dag.json` - `references/adapter_spec.yaml` +- `references/adapter_review.yaml` +- `references/notebook_execution_policy.json` - `references/api_reference.md` - `references/paper_summary.md` - `references/repo_summary.md` diff --git a/paper2skill/templates/codex_skill/scripts/adapters/python_api_adapter.py.j2 b/paper2skill/templates/codex_skill/scripts/adapters/python_api_adapter.py.j2 index 9c696e0..827d91c 100644 --- a/paper2skill/templates/codex_skill/scripts/adapters/python_api_adapter.py.j2 +++ b/paper2skill/templates/codex_skill/scripts/adapters/python_api_adapter.py.j2 @@ -10,6 +10,8 @@ SCRIPT_DIR = Path(__file__).resolve().parents[1] sys.path.insert(0, str(SCRIPT_DIR)) import env_manager +EXECUTABLE_ADAPTER_STATUSES = {"ready", "reviewed", "verified"} + def write_json(path: Path, data: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) @@ -26,12 +28,12 @@ def input_path(manifest: dict) -> Path: def run(manifest: dict, out: Path) -> dict: spec = env_manager.load_json_yaml(ROOT / "references" / "adapter_spec.yaml") - if spec.get("status") != "ready": + if spec.get("status") not in EXECUTABLE_ADAPTER_STATUSES: return { "status": "blocked", "adapter_type": "python_api", "entrypoint": spec.get("entrypoint"), - "message": "Python API adapter is not ready; user-reviewed wiring is required.", + "message": "Python API adapter is not executable; reviewed, ready, or verified status is required.", "caveats": spec.get("caveats", []), } source_root = ROOT / "sources" / "repo" diff --git a/paper2skill/templates/codex_skill/scripts/run.py.j2 b/paper2skill/templates/codex_skill/scripts/run.py.j2 index 7b39c99..6e96c0b 100644 --- a/paper2skill/templates/codex_skill/scripts/run.py.j2 +++ b/paper2skill/templates/codex_skill/scripts/run.py.j2 @@ -14,6 +14,8 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(SCRIPT_DIR)) import env_manager +EXECUTABLE_ADAPTER_STATUSES = {"ready", "reviewed", "verified"} + def write_json(path: Path, data: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) @@ -131,6 +133,19 @@ def main(argv: list[str] | None = None) -> int: adapter_report = {"status": "demo_only", "adapter_type": selected_adapter, "message": "Demo summary runner was used."} demo_run = is_demo_run(manifest, spec) if not demo_run: + if spec.get("status") not in EXECUTABLE_ADAPTER_STATUSES: + adapter_report = { + "status": "blocked", + "adapter_type": selected_adapter, + "adapter_status": spec.get("status"), + "entrypoint": spec.get("entrypoint"), + "message": "Adapter status is not executable; reviewed, ready, or verified status is required.", + "caveats": spec.get("caveats", []), + } + write_json(out / "workflow" / "adapter_report.json", env_manager.public_data(adapter_report)) + blocked = write_blocked_result(out, selected_adapter, adapter_report, manifest) + print(json.dumps(blocked, indent=2)) + return 2 adapter_report = run_adapter(selected_adapter, manifest, out) if adapter_report.get("status") != "pass": if adapter_report.get("status") in {"not_implemented", "candidate"}: @@ -151,7 +166,7 @@ def main(argv: list[str] | None = None) -> int: write_json(out / "workflow" / "executed_steps.json", {"status": "pass", "executed": [step.get("id") for step in load_json(out / "workflow" / "plan.json").get("steps", [])]}) repro = out / "reproducibility" repro.mkdir(parents=True, exist_ok=True) - for name in ["source_manifest.json", "algorithm_contract.yaml", "adapter_spec.yaml", "bio_contract.yaml"]: + for name in ["source_manifest.json", "algorithm_contract.yaml", "adapter_spec.yaml", "adapter_review.yaml", "bio_contract.yaml", "notebook_execution_policy.json"]: src = ROOT / "references" / name if src.exists(): shutil.copyfile(src, repro / name) diff --git a/paper2skill/validators/skill_validator.py b/paper2skill/validators/skill_validator.py index 7f4b8e5..699fe85 100644 --- a/paper2skill/validators/skill_validator.py +++ b/paper2skill/validators/skill_validator.py @@ -36,6 +36,8 @@ "references/paper_evidence.json", "references/repo_evidence.json", "references/adapter_spec.yaml", + "references/adapter_review.yaml", + "references/notebook_execution_policy.json", "references/paper.md", "references/paper_sections.json", "references/paper_parser_report.json", @@ -77,6 +79,10 @@ "## Evidence sources", ] +ADAPTER_STATUSES = {"demo_only", "candidate", "blocked", "ready", "reviewed", "verified"} +EXECUTABLE_ADAPTER_STATUSES = {"ready", "reviewed", "verified"} +READY_DRY_RUN_STATUSES = {"pass", "trusted_fixture"} + def validate_skill(skill_dir: str | Path) -> dict[str, Any]: root = Path(skill_dir) @@ -101,6 +107,7 @@ def validate_skill(skill_dir: str | Path) -> dict[str, Any]: _validate_environment_spec(root / "assets/environment_spec.yaml", errors) _validate_workflow_dag(root / "references/workflow_dag.json", errors) _validate_adapter_spec(root / "references/adapter_spec.yaml", errors) + _validate_adapter_review(root / "references/adapter_review.yaml", errors) trace_path = root / "references/tutorial_trace.json" if trace_path.exists(): try: @@ -168,15 +175,93 @@ def _validate_adapter_spec(path: Path, errors: list[str]) -> None: return status = spec.get("status") adapter_type = spec.get("adapter_type") - if status == "ready" and adapter_type == "python_api": + if status in {"ready", "reviewed", "verified"} and adapter_type == "python_api": if not spec.get("module"): - errors.append("adapter_spec.module: required when python_api status is ready") + errors.append("adapter_spec.module: required when python_api status is executable") if not spec.get("function"): - errors.append("adapter_spec.function: required when python_api status is ready") + errors.append("adapter_spec.function: required when python_api status is executable") if status == "demo_only" and adapter_type != "demo_only": errors.append("adapter_spec.status: demo_only status requires demo_only adapter_type") +def _validate_adapter_review(path: Path, errors: list[str]) -> None: + if not path.exists(): + return + try: + review = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError as exc: + errors.append(f"adapter_review: invalid YAML: {exc}") + return + for key in ["adapter_type", "status", "entrypoint", "command", "module", "function", "human_approved", "dry_run", "expected_outputs", "evidence", "caveats"]: + if key not in review: + errors.append(f"adapter_review: missing required key '{key}'") + status = review.get("status") + if status not in ADAPTER_STATUSES: + errors.append("adapter_review.status: invalid adapter status") + if review.get("human_approved") is not None and not isinstance(review.get("human_approved"), bool): + errors.append("adapter_review.human_approved: expected boolean") + if "dry_run" in review and not isinstance(review.get("dry_run"), dict): + errors.append("adapter_review.dry_run: expected mapping") + for key in ["expected_outputs", "evidence", "caveats"]: + if key in review and not isinstance(review.get(key), list): + errors.append(f"adapter_review.{key}: expected list") + command = review.get("command") + if isinstance(command, str) and ("\n" in command or "\r" in command): + errors.append("adapter_review.command: must not contain newlines") + adapter_spec = _load_adapter_spec_for_review(path) + if adapter_spec: + if status and adapter_spec.get("status") != status: + errors.append("adapter_review.status: must match adapter_spec.status") + if review.get("adapter_type") and adapter_spec.get("adapter_type") != review.get("adapter_type"): + errors.append("adapter_review.adapter_type: must match adapter_spec.adapter_type") + for key in ["entrypoint", "command", "module", "function"]: + expected = adapter_spec.get(key) + approved = review.get(key) + if expected and approved and expected != approved: + errors.append(f"adapter_review.{key}: must match adapter_spec.{key}") + if status in EXECUTABLE_ADAPTER_STATUSES: + for key in _required_adapter_review_mapping_keys(review.get("adapter_type")): + if _missing_adapter_review_mapping_value(review.get(key)): + errors.append(f"adapter_review.{key}: required when adapter status is executable") + if status == "reviewed" and review.get("human_approved") is not True: + errors.append("adapter_review.human_approved: required when adapter status is reviewed") + if status in {"ready", "verified"}: + dry_run = review.get("dry_run") or {} + if not isinstance(dry_run, dict) or dry_run.get("status") not in READY_DRY_RUN_STATUSES: + errors.append("adapter_review.dry_run.status: ready or verified status requires pass or trusted_fixture") + if status == "verified": + output_validation = review.get("output_validation") or {} + if not isinstance(output_validation, dict) or output_validation.get("status") != "pass": + errors.append("adapter_review.output_validation.status: verified status requires pass") + if not review.get("expected_outputs"): + errors.append("adapter_review.expected_outputs: verified status requires at least one expected output") + + +def _load_adapter_spec_for_review(review_path: Path) -> dict[str, Any]: + spec_path = review_path.with_name("adapter_spec.yaml") + if not spec_path.exists(): + return {} + try: + data = yaml.safe_load(spec_path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError: + return {} + return data if isinstance(data, dict) else {} + + +def _required_adapter_review_mapping_keys(adapter_type: str | None) -> list[str]: + if adapter_type == "python_api": + return ["adapter_type", "entrypoint", "module", "function"] + if adapter_type in {"cli", "workflow_engine"}: + return ["adapter_type", "entrypoint", "command"] + if adapter_type in {"notebook", "r_script"}: + return ["adapter_type", "entrypoint"] + return ["adapter_type"] + + +def _missing_adapter_review_mapping_value(value: Any) -> bool: + return not isinstance(value, str) or value == "" + + def _validate_environment_spec(path: Path, errors: list[str]) -> None: if not path.exists(): return diff --git a/tests/test_bio_contract_inference.py b/tests/test_bio_contract_inference.py index 0a3f0be..5f9077d 100644 --- a/tests/test_bio_contract_inference.py +++ b/tests/test_bio_contract_inference.py @@ -32,6 +32,9 @@ def test_bio_contract_infers_single_cell_transform_chain_and_metadata_key(): assert bio["modality"]["primary"]["value"] == "scRNA-seq" assert bio["input_matrix_state"]["matrix_transformations"] == ["raw_counts_loaded", "normalized", "log1p_transformed"] assert bio["metadata_requirements"]["celltype_key"]["value"] == "cell_type" + scrna = bio["modality_contracts"]["scrna_seq"] + assert scrna["input_state"]["matrix_state"]["value"] == "log1p" + assert scrna["metadata"]["celltype_key"]["value"] == "cell_type" def test_strict_bio_contract_does_not_promote_background_species_keyword(): @@ -78,3 +81,26 @@ def test_bio_contract_prefers_tutorial_evidence_over_paper_background_conflict() assert species["value"] == "human" assert species["confidence"] == "high" assert species["evidence"] == ["tutorial.ipynb:cell:1"] + + +def test_bio_contract_adds_bulk_rna_seq_modality_contract(): + trace = { + "workflow_steps": [ + { + "step_id": "tutorial_001:cell_001", + "evidence_id": "tutorial.ipynb:cell:1", + "code_preview": "dds <- DESeqDataSetFromMatrix(countData = counts, colData = sample_table, design = ~ condition)", + "command_or_code": "DESeq2::DESeqDataSetFromMatrix(countData = counts, colData = sample_table, design = ~ condition)", + "function_calls": ["DESeq2::DESeqDataSetFromMatrix"], + } + ] + } + + contract = infer_bio_contract(trace, paper_sections=[]) + + bio = contract["bio_contract"] + assert bio["modality"]["primary"]["value"] == "bulk RNA-seq" + bulk = bio["modality_contracts"]["bulk_rna_seq"] + assert bulk["input_state"]["matrix_state"]["value"] == "raw_counts" + assert bulk["metadata"]["condition_key"]["value"] == "condition" + assert bulk["statistical"]["design_formula"]["value"] == "~ condition" diff --git a/tests/test_dependency_miner.py b/tests/test_dependency_miner.py index ae83bb3..89a5fe1 100644 --- a/tests/test_dependency_miner.py +++ b/tests/test_dependency_miner.py @@ -171,3 +171,37 @@ def test_environment_yml_setup_cfg_and_description_system_requirements(tmp_path) assert deps["r_records"][0]["source"] == "Bioconductor_or_unknown" assert {"value": "libxml2", "source": "DESCRIPTION", "required": True, "install": "plan_only"} in deps["system_requirements"] assert deps["external_resources"][0]["name"] == "user/custompkg" + + +def test_description_bioconductor_metadata_multiline_fields_and_versions(tmp_path): + (tmp_path / "DESCRIPTION").write_text( + """ +Package: demo +Imports: + DESeq2 (>= 1.40), + SummarizedExperiment, + ggplot2 +LinkingTo: + Rcpp +biocViews: + RNASeq, + DifferentialExpression +SystemRequirements: + libxml2, + hdf5 +Enhances: + BiocStyle +""".strip() + + "\n", + encoding="utf-8", + ) + + deps = mine_dependencies(tmp_path) + + assert deps["r"] == ["DESeq2", "Rcpp", "SummarizedExperiment", "ggplot2"] + assert deps["optional"]["r"]["DESCRIPTION:Enhances"] == ["BiocStyle"] + deseq = next(record for record in deps["r_records"] if record["name"] == "DESeq2") + assert deseq["version_spec"] == ">= 1.40" + assert deseq["source"] == "Bioconductor_or_unknown" + assert "RNASeq" in deps["bioconductor"]["biocViews"] + assert {"value": "hdf5", "source": "DESCRIPTION", "required": True, "install": "plan_only"} in deps["system_requirements"] diff --git a/tests/test_generated_skill.py b/tests/test_generated_skill.py index 7099ad5..0300436 100644 --- a/tests/test_generated_skill.py +++ b/tests/test_generated_skill.py @@ -240,6 +240,48 @@ def test_generated_ready_python_adapter_runs_without_demo_summary_fallback(tmp_p assert summary == {"rows": 3, "value_mean": 2.0} +def test_generated_reviewed_python_adapter_is_executable(tmp_path: Path): + out = tmp_path / "toy-python-skill" + build = subprocess.run( + [sys.executable, "-m", "paper2skill.cli", "build", "--example", "toy_python", "--out", str(out)], + text=True, + capture_output=True, + check=False, + ) + assert build.returncode == 0, build.stdout + build.stderr + adapter_spec_path = out / "references" / "adapter_spec.yaml" + adapter_spec = yaml.safe_load(adapter_spec_path.read_text(encoding="utf-8")) + adapter_spec["status"] = "reviewed" + adapter_spec_path.write_text(yaml.safe_dump(adapter_spec, sort_keys=False), encoding="utf-8") + manifest = out / "run_manifest.yaml" + manifest.write_text( + """ +inputs: + primary_data: + path: assets/demo_input.csv + format: csv + exists: true + algorithm: + mode: run +environment: + install_policy: ask +""".strip() + + "\n", + encoding="utf-8", + ) + + result = subprocess.run( + [sys.executable, str(out / "scripts" / "run.py"), "--manifest", str(manifest), "--out", str(out / "reviewed-result")], + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr + adapter_report = json.loads((out / "reviewed-result" / "workflow" / "adapter_report.json").read_text(encoding="utf-8")) + assert adapter_report["status"] == "pass" + + def test_generated_plan_accepts_regular_yaml_manifest(tmp_path: Path): out = tmp_path / "toy-python-skill" subprocess.run([sys.executable, "-m", "paper2skill.cli", "build", "--example", "toy_python", "--out", str(out)], check=True) diff --git a/tests/test_generator.py b/tests/test_generator.py index 3d5f691..90cb8ff 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -20,6 +20,8 @@ def test_generator_creates_complete_toy_python_skill(tmp_path: Path): assert result["status"] == "pass", result assert (out / "references" / "algorithm_contract.yaml").exists() assert (out / "references" / "adapter_spec.yaml").exists() + assert (out / "references" / "adapter_review.yaml").exists() + assert (out / "references" / "notebook_execution_policy.json").exists() assert (out / "references" / "bio_contract.yaml").exists() assert (out / "references" / "workflow_dag.json").exists() assert (out / "scripts" / "adapters" / "python_api_adapter.py").exists() @@ -108,6 +110,241 @@ def test_validate_skill_rejects_missing_or_invalid_adapter_spec(tmp_path: Path): assert any("adapter_spec.status" in error for error in invalid["errors"]) +def test_validate_skill_accepts_reviewed_and_verified_adapter_statuses(tmp_path: Path): + context = build_context(**example_inputs("toy_python")) + out = generate_skill(context, tmp_path / "toy-python-skill") + adapter_spec = out / "references" / "adapter_spec.yaml" + adapter_review = out / "references" / "adapter_review.yaml" + spec = yaml.safe_load(adapter_spec.read_text(encoding="utf-8")) + review = yaml.safe_load(adapter_review.read_text(encoding="utf-8")) + + spec["status"] = "reviewed" + review.update({"status": "reviewed", "human_approved": True, "dry_run": {"status": "not_run"}, "expected_outputs": []}) + adapter_spec.write_text(yaml.safe_dump(spec, sort_keys=False), encoding="utf-8") + adapter_review.write_text(yaml.safe_dump(review, sort_keys=False), encoding="utf-8") + assert validate_skill(out)["status"] == "pass" + + spec["status"] = "verified" + review.update({"status": "verified", "human_approved": True, "dry_run": {"status": "pass"}, "expected_outputs": ["results/summary.json"], "output_validation": {"status": "pass"}}) + adapter_spec.write_text(yaml.safe_dump(spec, sort_keys=False), encoding="utf-8") + adapter_review.write_text(yaml.safe_dump(review, sort_keys=False), encoding="utf-8") + assert validate_skill(out)["status"] == "pass" + + +def test_validate_skill_rejects_invalid_adapter_review_lifecycle(tmp_path: Path): + context = build_context(**example_inputs("toy_python")) + out = generate_skill(context, tmp_path / "toy-python-skill") + adapter_review = out / "references" / "adapter_review.yaml" + review = yaml.safe_load(adapter_review.read_text(encoding="utf-8")) + + review.update({"status": "ready", "dry_run": {"status": "not_run"}}) + adapter_review.write_text(yaml.safe_dump(review, sort_keys=False), encoding="utf-8") + + invalid = validate_skill(out) + assert invalid["status"] == "fail" + assert any("adapter_review.dry_run.status" in error for error in invalid["errors"]) + + review.update({"status": "verified", "dry_run": {"status": "pass"}, "expected_outputs": []}) + review.pop("output_validation", None) + adapter_review.write_text(yaml.safe_dump(review, sort_keys=False), encoding="utf-8") + + invalid = validate_skill(out) + assert invalid["status"] == "fail" + assert any("adapter_review.output_validation.status" in error for error in invalid["errors"]) + + review.update({"status": "verified", "dry_run": {"status": "pass"}, "expected_outputs": "results/summary.json", "output_validation": {"status": "pass"}}) + adapter_review.write_text(yaml.safe_dump(review, sort_keys=False), encoding="utf-8") + + invalid = validate_skill(out) + assert invalid["status"] == "fail" + assert any("adapter_review.expected_outputs: expected list" in error for error in invalid["errors"]) + + +def test_adapter_review_human_approval_promotes_candidate_to_reviewed(tmp_path: Path): + repo = tmp_path / "api-repo" + package = repo / "reviewed_pkg" + package.mkdir(parents=True) + (package / "__init__.py").write_text("from .core import summarize\n", encoding="utf-8") + (package / "core.py").write_text("def summarize(path):\n return {'rows': 0}\n", encoding="utf-8") + (repo / "pyproject.toml").write_text("[project]\nname='reviewed-pkg'\nversion='0.1.0'\ndependencies=[]\n", encoding="utf-8") + paper = tmp_path / "paper.md" + paper.write_text("# Methods\n\nReviewed Python API method.\n", encoding="utf-8") + review = tmp_path / "adapter_review.yaml" + review.write_text( + """ +adapter_type: python_api +status: reviewed +entrypoint: reviewed_pkg:summarize +command: null +module: reviewed_pkg +function: summarize +human_approved: true +dry_run: + status: not_run +expected_outputs: [] +evidence: + - human_review +caveats: [] +""".strip() + + "\n", + encoding="utf-8", + ) + + context = build_context(paper=str(paper), repo=str(repo), maturity_level="L2", adapter_review=str(review)) + + assert context["adapter_spec"]["status"] == "reviewed" + assert context["adapter_spec"]["module"] == "reviewed_pkg" + + +def test_adapter_review_dry_run_promotes_candidate_to_ready_without_human_approval(tmp_path: Path): + repo = tmp_path / "api-repo" + package = repo / "ready_pkg" + package.mkdir(parents=True) + (package / "__init__.py").write_text("from .core import summarize\n", encoding="utf-8") + (package / "core.py").write_text("def summarize(path):\n return {'rows': 0}\n", encoding="utf-8") + (repo / "pyproject.toml").write_text("[project]\nname='ready-pkg'\nversion='0.1.0'\ndependencies=[]\n", encoding="utf-8") + paper = tmp_path / "paper.md" + paper.write_text("# Methods\n\nDry-run validated Python API method.\n", encoding="utf-8") + review = tmp_path / "adapter_review.yaml" + review.write_text( + """ +adapter_type: python_api +status: ready +entrypoint: ready_pkg:summarize +command: null +module: ready_pkg +function: summarize +human_approved: false +dry_run: + status: pass +expected_outputs: [] +evidence: + - dry_run +caveats: [] +""".strip() + + "\n", + encoding="utf-8", + ) + + context = build_context(paper=str(paper), repo=str(repo), maturity_level="L2", adapter_review=str(review)) + + assert context["adapter_spec"]["status"] == "ready" + assert context["adapter_spec"]["module"] == "ready_pkg" + + +def test_adapter_review_malformed_mapping_blocks_without_crashing(tmp_path: Path): + repo = tmp_path / "api-repo" + package = repo / "malformed_pkg" + package.mkdir(parents=True) + (package / "__init__.py").write_text("from .core import summarize\n", encoding="utf-8") + (package / "core.py").write_text("def summarize(path):\n return {'rows': 0}\n", encoding="utf-8") + (repo / "pyproject.toml").write_text("[project]\nname='malformed-pkg'\nversion='0.1.0'\ndependencies=[]\n", encoding="utf-8") + paper = tmp_path / "paper.md" + paper.write_text("# Methods\n\nMalformed review mapping.\n", encoding="utf-8") + review = tmp_path / "adapter_review.yaml" + review.write_text( + """ +adapter_type: python_api +status: ready +entrypoint: malformed_pkg:summarize +command: null +module: + - malformed_pkg +function: summarize +human_approved: false +dry_run: + status: pass +expected_outputs: [] +evidence: + - dry_run +caveats: [] +""".strip() + + "\n", + encoding="utf-8", + ) + + context = build_context(paper=str(paper), repo=str(repo), maturity_level="L2", adapter_review=str(review)) + + assert context["adapter_spec"]["status"] == "blocked" + assert "explicit adapter mapping" in " ".join(context["adapter_spec"]["caveats"]) + + +def test_adapter_review_requires_explicit_mapping_for_executable_status(tmp_path: Path): + repo = tmp_path / "api-repo" + package = repo / "reviewed_pkg" + package.mkdir(parents=True) + (package / "__init__.py").write_text("from .core import summarize\n", encoding="utf-8") + (package / "core.py").write_text("def summarize(path):\n return {'rows': 0}\n", encoding="utf-8") + (repo / "pyproject.toml").write_text("[project]\nname='reviewed-pkg'\nversion='0.1.0'\ndependencies=[]\n", encoding="utf-8") + paper = tmp_path / "paper.md" + paper.write_text("# Methods\n\nReviewed Python API method.\n", encoding="utf-8") + review = tmp_path / "adapter_review.yaml" + review.write_text( + """ +adapter_type: python_api +status: reviewed +human_approved: true +dry_run: + status: not_run +expected_outputs: [] +evidence: + - human_review +caveats: [] +""".strip() + + "\n", + encoding="utf-8", + ) + + context = build_context(paper=str(paper), repo=str(repo), maturity_level="L2", adapter_review=str(review)) + + assert context["adapter_spec"]["status"] == "blocked" + assert "explicit adapter mapping" in " ".join(context["adapter_spec"]["caveats"]) + + +def test_adapter_review_verified_requires_output_validation(tmp_path: Path): + repo = tmp_path / "api-repo" + package = repo / "verified_pkg" + package.mkdir(parents=True) + (package / "__init__.py").write_text("from .core import summarize\n", encoding="utf-8") + (package / "core.py").write_text("def summarize(path):\n return {'rows': 0}\n", encoding="utf-8") + (repo / "pyproject.toml").write_text("[project]\nname='verified-pkg'\nversion='0.1.0'\ndependencies=[]\n", encoding="utf-8") + paper = tmp_path / "paper.md" + paper.write_text("# Methods\n\nVerified Python API method.\n", encoding="utf-8") + review = tmp_path / "adapter_review.yaml" + review.write_text( + """ +adapter_type: python_api +status: verified +entrypoint: verified_pkg:summarize +command: null +module: verified_pkg +function: summarize +human_approved: true +dry_run: + status: pass +expected_outputs: + - results/summary.json +evidence: + - human_review +caveats: [] +""".strip() + + "\n", + encoding="utf-8", + ) + + context = build_context(paper=str(paper), repo=str(repo), maturity_level="L2", adapter_review=str(review)) + assert context["adapter_spec"]["status"] == "blocked" + + review.write_text( + review.read_text(encoding="utf-8") + + "output_validation:\n" + + " status: pass\n", + encoding="utf-8", + ) + context = build_context(paper=str(paper), repo=str(repo), maturity_level="L2", adapter_review=str(review)) + assert context["adapter_spec"]["status"] == "verified" + + def test_validate_skill_rejects_invalid_workflow_dag_and_environment_records(tmp_path: Path): context = build_context(**example_inputs("toy_python")) out = generate_skill(context, tmp_path / "toy-python-skill") diff --git a/tests/test_io_contract_inference.py b/tests/test_io_contract_inference.py index 863ba8a..6727daf 100644 --- a/tests/test_io_contract_inference.py +++ b/tests/test_io_contract_inference.py @@ -23,3 +23,68 @@ def test_io_contract_infers_10x_format_from_scanpy_reader_call(): assert primary_data["format"]["value"] == "10x_mtx" assert primary_data["format"]["confidence"] == "high" assert "sc.read_10x_mtx" in primary_data["format"]["evidence"] + + +def test_io_contract_uses_bulk_modality_rules_for_count_matrix(): + trace = { + "workflow_steps": [ + { + "step_id": "tutorial_001:cell_001", + "code_preview": "counts <- read.csv('counts.csv')\ndds <- DESeqDataSetFromMatrix(countData=counts, colData=meta, design=~condition)", + "command_or_code": "DESeq2::DESeqDataSetFromMatrix(countData=counts, colData=meta, design=~condition)", + "function_calls": ["DESeq2::DESeqDataSetFromMatrix"], + "read_files": ["counts.csv", "metadata.tsv"], + "inputs": [], + } + ] + } + bio_contract = {"bio_contract": {"modality": {"primary": {"value": "bulk RNA-seq"}}}} + + contract = infer_io_contract(trace, bio_contract) + + primary_data = contract["input_contract"]["required"]["primary_data"] + assert primary_data["format"]["value"] == "count_matrix" + assert primary_data["metadata_keys"]["condition_key"]["value"] == "condition" + + +def test_io_contract_marks_conflicting_primary_formats_not_confirmed(): + trace = { + "workflow_steps": [ + { + "step_id": "tutorial_001:cell_001", + "code_preview": "adata = sc.read_10x_mtx('data/')\nobj <- readRDS('object.rds')", + "command_or_code": "sc.read_10x_mtx('data/')\nreadRDS('object.rds')", + "function_calls": ["sc.read_10x_mtx", "readRDS"], + "read_files": ["object.rds"], + "inputs": [], + } + ] + } + + contract = infer_io_contract(trace) + + fmt = contract["input_contract"]["required"]["primary_data"]["format"] + assert fmt["value"] == "not_confirmed" + assert fmt["confidence"] == "low" + assert set(fmt["conflicts"]) == {"10x_mtx", "rds"} + + +def test_io_contract_metadata_csv_does_not_conflict_with_primary_h5ad(): + trace = { + "workflow_steps": [ + { + "step_id": "tutorial_001:cell_001", + "code_preview": "adata = sc.read_h5ad('adata.h5ad')\nmetadata = pd.read_csv('metadata.csv')", + "command_or_code": "adata = sc.read_h5ad('adata.h5ad')\nmetadata = pd.read_csv('metadata.csv')", + "function_calls": ["sc.read_h5ad", "pd.read_csv"], + "read_files": ["adata.h5ad", "metadata.csv"], + "inputs": [], + } + ] + } + + contract = infer_io_contract(trace) + + fmt = contract["input_contract"]["required"]["primary_data"]["format"] + assert fmt["value"] == "h5ad" + assert "conflicts" not in fmt diff --git a/tests/test_notebook_miner.py b/tests/test_notebook_miner.py index a0348fb..75ec61d 100644 --- a/tests/test_notebook_miner.py +++ b/tests/test_notebook_miner.py @@ -1,5 +1,8 @@ from __future__ import annotations +import json +from pathlib import Path + from paper2skill.miners.notebook_miner import mine_notebook @@ -10,3 +13,44 @@ def test_notebook_trace_includes_ordered_code_cells(): assert code_cells[0]["index"] == 1 assert "csv" in code_cells[0]["imports"] assert trace["workflow_steps"][0]["evidence_id"].endswith("cell:1") + + +def test_notebook_trace_detects_magic_parameters_paths_and_policy(tmp_path: Path): + notebook = { + "cells": [ + { + "cell_type": "code", + "source": ["input_dir = 'data'\noutput_file = 'results/out.csv'\n"], + "metadata": {"tags": ["parameters"]}, + "outputs": [], + }, + {"cell_type": "code", "source": ["!pip install scanpy\n!wget https://example.org/data.h5ad\n"], "metadata": {}, "outputs": []}, + {"cell_type": "code", "source": ["%%R\nlibrary(Seurat)\nobj <- readRDS('data/pbmc.rds')\nsaveRDS(obj, 'results/pbmc.rds')\n"], "metadata": {}, "outputs": []}, + {"cell_type": "code", "source": ["%%capture\nimport pandas as pd\ncaptured = input_dir + '/captured.csv'\ndf = pd.read_csv(captured)\n"], "metadata": {}, "outputs": []}, + {"cell_type": "code", "source": ["%%html\n
not python
\n"], "metadata": {}, "outputs": []}, + {"cell_type": "code", "source": ["path = input_dir + '/matrix.csv'\ndef load():\n return pd.read_csv(path)\n"], "metadata": {}, "outputs": [{"output_type": "display_data", "data": {"image/png": "x" * 200000}}]}, + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5, + } + path = tmp_path / "rich.ipynb" + path.write_text(json.dumps(notebook), encoding="utf-8") + + trace = mine_notebook(path) + + policy = trace["execution_policy"] + assert policy["will_execute"] is False + assert policy["parameter_cells"] == [0] + assert policy["shell_magics"][0]["command"].startswith("pip install") + assert policy["cell_magics"][0]["magic"] == "R" + assert any(item["magic"] == "capture" for item in policy["cell_magics"]) + assert any(item["magic"] == "html" for item in policy["cell_magics"]) + assert "install_command" in policy["risks"] + assert "network_download" in policy["risks"] + assert "large_output" in policy["risks"] + assert trace["parameters"]["input_dir"] == "data" + assert any(step["language"] == "r" and "data/pbmc.rds" in step["read_files"] for step in trace["workflow_steps"]) + assert any("data/captured.csv" in step["read_files"] and "pandas" in step["imports"] for step in trace["workflow_steps"]) + assert not any("
not python
" in step["command_or_code"] for step in trace["workflow_steps"]) + assert any("data/matrix.csv" in step["read_files"] for step in trace["workflow_steps"]) diff --git a/tests/test_script_miner.py b/tests/test_script_miner.py index 0ef5ea3..aa0e3d9 100644 --- a/tests/test_script_miner.py +++ b/tests/test_script_miner.py @@ -1,5 +1,8 @@ from __future__ import annotations +from pathlib import Path + +from paper2skill.miners.api_miner import r_namespace_imports from paper2skill.miners.script_miner import mine_script @@ -17,3 +20,51 @@ def test_r_script_trace_extracts_library_and_io(): assert "stats" in trace["imports"] assert "data/demo_input.csv" in trace["file_reads"] assert "results/summary.csv" in trace["file_writes"] + + +def test_r_script_trace_extracts_namespace_calls_file_path_and_source(tmp_path: Path): + script = tmp_path / "analysis.R" + script.write_text( + """ +library(Seurat) +source("R/helpers.R") +data_dir <- "data" +out_dir <- "results" +input_file <- file.path(data_dir, "pbmc.rds") +obj <- Seurat::Read10X(input_file) +saveRDS(obj, file.path(out_dir, "pbmc.rds")) +ggsave(file.path(out_dir, "umap.png")) +""".strip() + + "\n", + encoding="utf-8", + ) + + trace = mine_script(script) + + assert "Seurat" in trace["imports"] + assert "R/helpers.R" in trace["source_files"] + assert "data/pbmc.rds" in trace["file_reads"] + assert "results/pbmc.rds" in trace["file_writes"] + assert "results/umap.png" in trace["file_writes"] + assert any(call["package"] == "Seurat" and call["function"] == "Read10X" for call in trace["function_calls"]) + + +def test_r_namespace_imports_exports_and_s3_methods(tmp_path: Path): + namespace = tmp_path / "NAMESPACE" + namespace.write_text( + """ +import(Seurat) +importFrom(DESeq2, DESeqDataSetFromMatrix, results) +export(run_demo) +S3method(plot, demo) +""".strip() + + "\n", + encoding="utf-8", + ) + + parsed = r_namespace_imports(tmp_path) + + assert {"package": "Seurat", "source": "NAMESPACE:import"} in parsed["imports"] + assert {"package": "DESeq2", "function": "results", "source": "NAMESPACE:importFrom"} in parsed["import_from"] + assert {"name": "run_demo", "source": "NAMESPACE:export"} in parsed["exports"] + assert {"generic": "plot", "class": "demo", "source": "NAMESPACE:S3method"} in parsed["s3methods"]