diff --git a/CITATION.cff b/CITATION.cff index 0e427b1..a1c4453 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -18,8 +18,8 @@ abstract: > (RRRM-1, RRRM-2). The v7 release also includes unified scPRINT2 and graph neural network benchmarking, plus cross-method synthesis across classical, foundation-model, and graph-based approaches. The v7.1.2 release is a - documentation, public-card, metadata, and evidence-visibility patch; it does not - introduce new benchmark result generation. + documentation, public-card, metadata, and evidence-visibility patch; it does + not introduce new benchmark result generation. authors: - family-names: "Kim" diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d617dd4 --- /dev/null +++ b/Makefile @@ -0,0 +1,53 @@ +PYTHON ?= python3 +HF_REPO ?= jang1563/genelab-benchmark +HF_UPLOAD_PLAN ?= output/hf_upload_plan.json +HF_TASK ?= all + +.PHONY: help +help: + @echo "SpaceBio-Bench release helpers" + @echo "" + @echo " make release-qa Run local public release validation" + @echo " make hpc-public-qa Run the public-surface slice of the HPC gate" + @echo " make public-docs-qa Validate README, HF card, citation, and manifest" + @echo " make hf-upload-plan Write a dry-run HF upload plan JSON" + @echo " make hf-card-plan Dry-run only the HF dataset card upload" + @echo " make hf-manifest-plan Dry-run only HF release metadata upload" + @echo " make validate-hf-remote Post-upload smoke-check remote HF files" + @echo "" + @echo "Variables:" + @echo " PYTHON=$(PYTHON)" + @echo " HF_REPO=$(HF_REPO)" + @echo " HF_TASK=$(HF_TASK)" + @echo " HF_UPLOAD_PLAN=$(HF_UPLOAD_PLAN)" + +.PHONY: public-docs-qa +public-docs-qa: + $(PYTHON) scripts/validate_release_manifest.py + $(PYTHON) scripts/validate_public_docs_consistency.py + $(PYTHON) -m unittest tests/test_release_manifest.py tests/test_public_docs_consistency.py + +.PHONY: release-qa +release-qa: public-docs-qa + $(PYTHON) -m unittest tests/test_hf_upload_workflow.py + $(PYTHON) -m py_compile scripts/upload_to_hf.py scripts/validate_release_manifest.py scripts/validate_public_docs_consistency.py + +.PHONY: hpc-public-qa +hpc-public-qa: + PYTHON_BIN=$(PYTHON) bash scripts/hpc_release_validate.sh --public-only + +.PHONY: hf-upload-plan +hf-upload-plan: + $(PYTHON) scripts/upload_to_hf.py --task $(HF_TASK) --repo $(HF_REPO) --dry-run --write-upload-plan $(HF_UPLOAD_PLAN) + +.PHONY: hf-card-plan +hf-card-plan: + $(PYTHON) scripts/upload_to_hf.py --repo $(HF_REPO) --card-only --dry-run + +.PHONY: hf-manifest-plan +hf-manifest-plan: + $(PYTHON) scripts/upload_to_hf.py --repo $(HF_REPO) --manifest-only --dry-run + +.PHONY: validate-hf-remote +validate-hf-remote: + $(PYTHON) scripts/upload_to_hf.py --task $(HF_TASK) --repo $(HF_REPO) --dry-run --remote-diff --validate-remote diff --git a/README.md b/README.md index 5617357..11e4352 100644 --- a/README.md +++ b/README.md @@ -1,296 +1,89 @@ -# GeneLab Benchmark +# SpaceBio-Bench -**A comprehensive benchmark for evaluating AI/ML and Foundation Models on NASA OSDR spaceflight transcriptomics data.** +**Mission-held-out transcriptomics benchmarks for evaluating whether AI/ML and +foundation models generalize spaceflight biological signatures across missions, +tissues, and model systems.** -Version: v7.1.2 public-card/metadata/evidence-visibility patch (2026-06-05) | Canonical results: v7.1 | Dataset freeze: 2026-03-01 -Status: **v1-v7 Complete; public-review card pack ready** +Former public name: **GeneLab Benchmark**. The v1-v7 historical benchmark +surface keeps that name; SpaceBio-Bench is the forward-looking platform name. -Canonical v7.1 documentation source: [`docs/CANONICAL_RESULTS_V7_1.md`](docs/CANONICAL_RESULTS_V7_1.md) records the locked scope accounting, headline result table, and v8 boundary notes for public release text. - -Live dataset card: [Hugging Face / `jang1563/genelab-benchmark`](https://huggingface.co/datasets/jang1563/genelab-benchmark) - -## Reviewer Fast Path - -For portfolio, job-application, or external-review contexts, start here: - -1. [`docs/SPACEBIOBENCH_PORTFOLIO_BRIEF.md`](docs/SPACEBIOBENCH_PORTFOLIO_BRIEF.md) - concise project contribution and role-relevant signal. -2. [`docs/SPACEBIOBENCH_TRANSPARENCY_CARD_PACK.md`](docs/SPACEBIOBENCH_TRANSPARENCY_CARD_PACK.md) - map of the system card, evaluation card, release readiness card, and claim register. -3. [`docs/SPACEBIOBENCH_SYSTEM_CARD.md`](docs/SPACEBIOBENCH_SYSTEM_CARD.md) - benchmark/system boundary, intended use, out-of-scope claims, and provenance controls. -4. [`docs/SPACEBIOBENCH_EVALUATION_CARD.md`](docs/SPACEBIOBENCH_EVALUATION_CARD.md) - task/fold/metric interpretation and leakage/confounding controls. -5. [`docs/SPACEBIOBENCH_CLAIM_REGISTER.md`](docs/SPACEBIOBENCH_CLAIM_REGISTER.md) - supported, blocked, and future-only claim language. -6. [Hugging Face dataset card](https://huggingface.co/datasets/jang1563/genelab-benchmark) - live public dataset card and feature-matrix package. - -Release archive pack: [`docs/RELEASE_ARCHIVE_CARD.md`](docs/RELEASE_ARCHIVE_CARD.md) -defines paper-archive scope. [`docs/RELEASE_ARCHIVE_MANIFEST.md`](docs/RELEASE_ARCHIVE_MANIFEST.md) -and [`docs/RELEASE_ARCHIVE_CHECKLIST.md`](docs/RELEASE_ARCHIVE_CHECKLIST.md) -track Zenodo metadata readiness and final DOI/tag gates. - -Release note: `v7.1.2` is a documentation, public-card, metadata, and -evidence-visibility patch on top of the canonical v7.1 result surface. It does -not introduce new benchmark result generation. DOI deposition remains pending. - -v9 note: `main` includes a curated metadata-only public bulk evidence subset -under [`v9/`](v9/) for task/source/checksum registries, the draft Data Package, -alpha-boundary reports, and baseline summaries. It excludes payload matrices -and draft extension lanes. - -[![Dataset on HuggingFace](https://img.shields.io/badge/HuggingFace-Dataset-yellow)](https://huggingface.co/datasets/jang1563/genelab-benchmark) +[![Dataset on Hugging Face](https://img.shields.io/badge/Hugging%20Face-Dataset-yellow)](https://huggingface.co/datasets/jang1563/genelab-benchmark) [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) ---- - -## What Is This? - -GeneLab Benchmark provides standardized tasks for evaluating how well machine learning models — from classical baselines to gene expression foundation models (Geneformer, scGPT, UCE, scFoundation) and text-based LLMs (GPT-4o, Claude, Llama) — generalize **spaceflight transcriptomic signatures across missions**. - -The reviewer-facing transparency layer is documented as **SpaceBio-Bench**: -system, evaluation, release-readiness, and claim-boundary cards for the public -benchmark surface. - -**Core challenge**: Train a model on one spaceflight mission's RNA-seq data. Can it classify samples from a different mission it has never seen? - -**Data source**: [NASA Open Science Data Repository (OSDR)](https://osdr.nasa.gov/bio/repo/) — mouse multi-tissue bulk RNA-seq from ISS and ground control missions. - -### Benchmark Scope - -- **8 tissues**: Liver, Gastrocnemius, Kidney, Thymus, Skin, Eye, Lung, Colon -- **17+ ISS missions**: RR-1 through RR-9, MHU-1, MHU-2, and more -- **24+ verified OSD studies**, ~600+ samples (binary Flight/Ground, with BC/VC controls in v4) -- **25+ evaluation tasks** across 7 categories (A-D, J, NC, Validation) -- **v4**: 256 evaluations (8 tissues x 8 classifiers x 4 feature types) -- **Multi-species**: Mouse bulk, Drosophila KEGG, mouse scRNA-seq (RRRM-1/RRRM-2), spatial Visium -- **4 gene-expression foundation models + 3 text LLMs evaluated**: Geneformer, scGPT, UCE, scFoundation, plus GPT-4o/Claude/Llama - ---- - -## Key Features - -- **Leave-One-Mission-Out (LOMO)** cross-validation — mission = independence unit, preventing cross-mission data leakage -- **Category A**: Spaceflight detection per tissue (binary: Flight vs. Ground) -- **Category B**: Cross-mission transfer matrix (train on mission i, test on mission j) for all tissues -- **Category C**: Cross-tissue transfer (3 methods: gene, DEG, pathway) -- **Category D**: Condition/confounder prediction (mission, strain, hardware, gravity) -- **3-tier model evaluation**: Classical ML -> Gene Expression Foundation Models -> Text LLMs -- **Standardized submission format** with automatic AUROC/CI/p-value evaluation -- **Biological validation**: NES pathway conservation, Cell 2020 concordance, negative controls -- **v4 multi-method**: 8 classifiers (PCA-LR, ElasticNet-LR, RF, XGBoost, SVM-Linear, SVM-RBF, TabNet, LightGBM) -- **v4 multi-feature**: 4 feature types (gene, Hallmark, KEGG, pathway-combined) - ---- +Maintainer / citation author: JangKeun Kim, Weill Cornell Medicine. -## Results Overview +Current public release note: **v7.1.2 public-card/metadata/evidence-visibility +patch** over canonical v7.1 results. The patch updates documentation, public +metadata, and evidence visibility; it does not introduce new benchmark result +generation. Dataset freeze: **2026-03-01**. -### v4 Phase 1 — Multi-Method Evaluation (256 evaluations) +## What This Is -**Best overall classifier**: PCA-LR (gene mean AUROC 0.776), ElasticNet-LR 2nd (0.762) +SpaceBio-Bench evaluates a practical space-biology question: -Canonical v7.1 result source: [`docs/CANONICAL_RESULTS_V7_1.md`](docs/CANONICAL_RESULTS_V7_1.md). Keep README, Hugging Face card, and paper outline synchronized with that table. +> If a model learns a transcriptomic spaceflight signature from one mission, +> can it recognize that signature in a different mission it has never seen? -| Tissue | Best AUROC | Best Method | Best Feature | Significant (p<0.05) | -|--------|-----------|-------------|-------------|---------------------| -| **Thymus** | **0.948*** | PCA-LR | KEGG | Yes | -| **Colon** | **0.921*** | PCA-LR | KEGG | Yes | -| **Lung** | **0.901*** | PCA-LR | Gene | Yes | -| **Kidney** | **0.829**** | ElasticNet-LR | Hallmark | Yes | -| **Skin** | 0.819 | PCA-LR | Gene | - | -| **Eye** | 0.823 | PCA-LR | Hallmark | - | -| **Liver** | 0.670 | PCA-LR | Gene | - | -| **Gastrocnemius** | 0.776 | PCA-LR | Gene | - | +The current public benchmark uses NASA Open Science Data Repository (OSDR) +spaceflight transcriptomics, with emphasis on mouse multi-tissue bulk RNA-seq, +mission-held-out validation, and transparent release boundaries. -*p<0.05, **p<0.01. 40/256 evaluations significant at p<0.05; 6/8 tissues have at least 1 significant result. +## Current Public Surfaces -**Key v4 findings**: -- SVM-RBF worst on high-dimensional gene data (0.510 mean) -- TabNet disappointing (0.527 mean) — deep learning doesn't help small-n transcriptomics -- Pathway features improve kidney (0.584->0.829), thymus (0.908->0.948), eye (0.697->0.823) -- Gene features better for skin (0.819) and lung (0.901) -- v4 expanded controls: BC/VC included (e.g., liver 261 vs v1's 193 samples) - -### v1 Results — Spaceflight Detection (LOMO, 6 tissues) - -**Gene-level (primary)**: - -| Task | Tissue | Missions | Method | Mean AUROC | 95% CI lower | perm_p | Decision | -|------|--------|---------|--------|-----------|-------------|--------|----------| -| A4 | Thymus | 4 | PCA-LR | **0.923** | 0.878 | 0.037 | GO | -| A2 | Gastrocnemius | 3 | LR | **0.907** | 0.717 | 0.026 | GO | -| A5 | Skin | 3 | LR | **0.821** | 0.637 | 0.0023 | GO | -| A6 | Eye | 3 | LR | 0.811 | 0.470 | 0.063 | NO-GO | -| A1 | Liver | 6 | LR | 0.653 | 0.457 | 0.091 | NO-GO | -| A3 | Kidney | 3 | LR | 0.593 | 0.431 | 0.281 | NO-GO | - -**Pathway-level rescue**: Eye gene-level NO-GO -> GSVA Hallmark AUROC **0.915** (GO). Kidney 0.43->0.74. - -### Cross-Mission Transfer (Category B) - -| Tissue | Mean AUROC | 95% CI | AUROC>=0.70 | Tier | -|--------|-----------|--------|-----------|------| -| Thymus | **0.860** | [0.763, 0.953] | 9/12 | 1 | -| Gastrocnemius | **0.801** | [0.653, 0.944] | 4/6 | 1 | -| Skin | **0.772** | [0.691, 0.834] | 5/6 | 2 | -| Eye | 0.754 | [0.688, 0.838] | 5/6 | 2 | -| Liver | 0.577 | [0.492, 0.666] | 13/30 | 3 | -| Kidney | 0.555 | [0.397, 0.681] | 2/6 | 3 | - -### v5 — Biological Interpretation - -| Analysis | Key Finding | -|----------|------------| -| **Immune deconvolution** (mMCP-counter, 8 tissues) | Skin strongest: 6/14 cell types FDR<0.05. Kidney, thymus 2/14 each. Most tissues 0 significant | -| **Cross-organ signaling** (OmniPath, 111 L–R pairs) | 1 SHAP-active L–R pair. TF activity: thymus 240 sig, skin 241, kidney 177, liver 105 | -| **Metabolic flux** (iMM1865 E-Flux + pFBA) | Flight vs ground objectives differ in all 6 tissues (largest: thymus FLT 15,695 vs GC 14,696) | -| **Drug targets** (DGIdb + ChEMBL) | 271/834 spaceflight genes druggable; 1,284 FDA-approved drug–gene interactions | -| **Consensus biomarker panel** (20 genes) | Top genes: MUP22, Thrsp, Apoa1, NPAS2, PER2. AUROC: gastro 0.806, liver 0.754, eye 0.728 | - -### Foundation / LLM Comparison Snapshot - -| Model | Architecture | Mean AUROC (6 tissues) | vs PCA-LR | -|-------|------------|----------------------|-----------| -| **PCA-LR baseline** | Classical ML | **0.758** | — | -| scGPT | 12L Transformer, 33M human cells | 0.666 | -0.093 | -| scFoundation | 100M params, 50M cells | 0.635† (liver best, p<0.01) | Below baseline | -| UCE | 33L, 36M cells | 0.632† (thymus best, p=0.031) | Below baseline | -| Mouse-Geneformer | 6L BERT, 30M mouse cells | 0.476 | -0.283 | -| Text LLMs (3x) | GPT-4o, Claude, Llama 3 | 0.47-0.51 | Chance level | - -**FM verdict**: All foundation models underperform classical ML. The scGPT and Mouse-Geneformer rows report 6-tissue v1 means against the 0.758 PCA-LR baseline; the scFoundation and UCE rows show their best single-tissue v3 results, and their full cross-tissue means also remain below baseline. †Best single-tissue AUROC shown; cross-tissue means are lower. - -### Independent Held-Out Validation - -| Tissue | Mission | AUROC | 95% CI | p-value | Duration | n_test | -|--------|---------|-------|--------|---------|---------|--------| -| Thymus | RR-23 | **0.905** | [0.672, 1.000] | 0.005 | 30 days | 18 | -| Skin | RR-7 | **0.885** | [0.745, 0.986] | <0.001 | 75 days | 30 | - ---- - -## Key Scientific Findings - -Interpretation boundary: the findings below are benchmark and synthesis -evidence from the documented result surfaces. They should not be read as -biological mechanism proof, clinical guidance, crew-health prediction, or -countermeasure evidence without additional validation. - -### Pre-registered Hypotheses - -| Hypothesis | Statement | Verdict | Key Evidence | +| Surface | Public status | Use it for | Entry point | |---|---|---|---| -| **H1** | Liver has the most consistent cross-mission transcriptome | **REFUTED** | Thymus (0.860) >> Liver (0.577). Thymus and Gastrocnemius = Tier 1. | -| **H2** | Transfer failure from biological diversity, not batch effects | **SUPPORTED** | NES conservation r=0.9 (5 tissues). D3 pathway F1=0.06 (batch-invariant). | -| **H3** | Pathway-level preserves spaceflight response better than gene-level | **CONDITIONALLY SUPPORTED** | Kidney rescue (0.43->0.74), Eye (0.79->0.92). But tissue-pair dependent. | - -### Novel Findings - -- **Tissue hierarchy**: Thymus > Gastrocnemius > Skin > Eye >> Liver > Kidney for spaceflight detection -- **Pathway rescue**: Pathway features rescue NO-GO tissues (kidney +0.31, eye +0.13) -- **FM failure mode**: scRNA-seq-pretrained FMs don't transfer to small-n bulk transcriptomics -- **Spaceflight amplifies aging**: OLD AUROC=0.945 vs YNG=0.679 (Delta=+0.266, RR-8 liver) -- **PBMC NK cells**: Strongest single-cell spaceflight signal (AUROC=0.845, RRRM-2) -- **Brain spatial**: Genuine negative — no detectable spaceflight signal (Visium AUROC=0.139) -- **Cross-species**: Drosophila-mouse NES correlations significant but negative (r=-0.19 to -0.59) - ---- - -## Version Structure - -| Version | Scope | Status | Location | -|---------|-------|--------|----------| -| **v1.0** | Mouse bulk RNA-seq, 6 tissues, 25+ tasks, 3 model tiers, 2 held-out validations | **Complete** | Project root | -| **v2.0** | Temporal dynamics (T1-T3), cross-species (E1-E3), single-cell (F1, F2 RRRM-1) | **Complete** | `v2/` | -| **v3.0** | Multi-species (E4), Spatial Visium (F3), RRRM-2 scRNA-seq (F5), UCE/scFoundation FM, radiation analogs, cross-tissue transfer 7×7 | **Complete** | `v3/` | -| **v4.0** | Multi-method benchmark: 8 tissues × 8 classifiers × 4 features (256 evals), ablation, SHAP, WGCNA, module preservation, STRING PPI | **Complete** | `v4/` | -| **v5.0** | Biological interpretation: immune deconvolution (mMCP-counter), metabolic flux (iMM1865 E-Flux), drug targets (DGIdb/ChEMBL), consensus 20-gene biomarker panel | **Complete** | `v5/` | -| **v6.0** | Cross-species validation: gene/pathway conservation, cross-species transfer, TF conservation, biomarker/drug target validation across species | **Complete** | `v6/` | -| **v7.0** | Unified/foundation-model benchmarking: scPRINT2, GNN/WGCNA graph baselines, cross-method synthesis, and signal hierarchy analysis | **Complete** | `v7/` | - ---- - -## Repository Structure - -``` -GeneLab_benchmark/ -├── README.md <- This file -├── DATA_CATALOG.md <- OSDR inventory (24+ studies) -├── CITATION.cff <- Citation metadata -│ -├── tasks/ <- Public task inputs (benchmark + sensitivity tasks) -│ ├── A1_liver_lomo/ <- 6 folds + 3 variants -│ ├── A2_gastrocnemius_lomo/ <- 3 folds -│ ├── A3_kidney_lomo/ <- 3 folds -│ ├── A4_thymus_lomo/ <- 4 folds + holdout -│ ├── A5_skin_lomo/ <- 3 folds -│ ├── A6_eye_lomo/ <- 3 folds -│ └── B1-B6_*_cross_mission/ <- N x (N-1) mission pairs per tissue -│ -├── scripts/ <- v1 pipeline scripts -│ ├── run_baselines.py <- Classical ML baseline runner -│ ├── evaluate_submission.py <- Submission evaluator (AUROC, CI, perm_p) -│ ├── generate_tasks.py <- LOMO split generator -│ └── ... <- See scripts/ for full list -│ -├── docs/ -│ ├── hf_dataset_card.md <- HuggingFace dataset documentation -│ ├── submission_format.md <- JSON submission specification -│ ├── text_llm_format.md <- Text LLM evaluation format -│ └── BIOLOGICAL_GROUND_TRUTH.md <- Validation reference (Cell 2020, SOMA 2024) -│ -├── evaluation/ <- Result JSON files (v1 baseline) -│ ├── RESULTS_SUMMARY.md <- Comprehensive results table (v1–v5) -│ └── *.json <- Per-task results -│ -├── v2/ <- Temporal dynamics, cross-species, scRNA-seq -│ ├── README.md -│ ├── scripts/ <- v2 analysis and runner scripts -│ ├── evaluation/ <- v2 results -│ └── figures/ <- v2 interactive HTML figures -│ -├── v3/ <- Multi-species, spatial Visium, FM evaluation -│ ├── README.md -│ ├── scripts/ <- v3 scripts (Python/Bash/R) -│ ├── evaluation/ <- v3 result summaries -│ └── figures/ <- v3 interactive HTML figures -│ -├── v4/ <- Multi-method benchmark + network biology -│ ├── scripts/ <- Classifiers, SHAP, WGCNA, and PPI analysis -│ ├── evaluation/ <- v4 result JSONs + SHAP/WGCNA outputs -│ ├── wgcna_outputs/ <- Per-tissue WGCNA module data -│ └── figures/html/ <- v4 interactive HTML figures -│ -├── v5/ <- Biological interpretation layer -│ ├── scripts/ <- Immune deconv, metabolic flux, drug targets -│ ├── evaluation/ <- Immune, TF, metabolic, and drug-target results -│ └── figures/html/ <- v5 interactive HTML figures -│ -├── v6/ <- Cross-species validation -│ ├── scripts/ <- Phase A-F Python scripts -│ └── evaluation/ <- v6 result summaries -│ -├── v7/ <- Unified/foundation-model benchmarking -│ ├── scripts/ <- HPC setup and launcher scripts -│ ├── unified/ <- scPRINT2, GNN/WGCNA, and synthesis drivers -│ ├── evaluation/ <- v7 result JSONs -│ └── figures/html/ <- v7 interactive HTML figures -│ -├── v9/ <- Public bulk metadata-alpha evidence subset -│ ├── task_manifests/ <- 8 public bulk LOMO task manifests -│ ├── reports/ <- scaffold baseline and alpha-boundary summaries -│ └── datapackage.draft.json <- metadata-only draft package descriptor -│ -└── processed/ <- Intermediate analysis outputs - ├── A_detection/ <- Per-tissue LOMO data - ├── B_cross_mission/ <- Transfer matrices - ├── fgsea/ <- 60+ fGSEA results - └── pathway_scores/ <- 48 ssGSEA pathway score files -``` - ---- - -## Getting Started - -Feature matrices (train_X.csv, test_X.csv) are hosted on HuggingFace due to size (~2 GB). -Labels, metadata, and fold structure are in this repository. - -### Option A -- Load from HuggingFace (recommended) +| v7.1 GeneLab Benchmark | Canonical historical result surface | v1-v7 results, public fold package, citation | [Canonical results](docs/CANONICAL_RESULTS_V7_1.md) | +| Hugging Face dataset | Public processed fold package | Download selected LOMO feature matrices and result artifacts | [HF dataset card](docs/hf_dataset_card.md) | +| v9 public bulk | Metadata-only public bulk alpha | Task catalog, source inventory, and baseline summaries | [v9 HF-style card](docs/v9_hf_dataset_card.md) | + +For linked methods, evaluation, and release cards, start with the +[SpaceBio-Bench card pack](docs/SPACEBIOBENCH_TRANSPARENCY_CARD_PACK.md). +For machine-readable release status, see +[release/release_manifest.json](release/release_manifest.json). + +## Benchmark Design + +| Layer | Design choice | +|---|---| +| Core split | Leave-One-Mission-Out (LOMO); mission is the independence unit | +| Primary label | Flight vs. ground control in public spaceflight transcriptomics | +| Main data source | NASA OSDR mouse spaceflight RNA-seq | +| Feature surfaces | Gene expression, Hallmark pathways, KEGG pathways, combined pathway features | +| Model tracks | Classical ML, gene-expression foundation models, text LLMs, graph/network baselines | +| Evaluation | AUROC, bootstrap confidence interval, permutation p-value, task-specific diagnostics | +| Leakage guard | Fold-specific variance filtering is computed on training missions only | + +## Scope At A Glance + +| Dimension | v1-v7 public benchmark surface | +|---|---| +| Tissues | 8: liver, gastrocnemius, kidney, thymus, skin, eye, lung, colon | +| Public OSDR source catalog | 24+ OSD accessions | +| Processed sample scope | 600+ binary/control samples across release layers | +| v4 multi-method grid | 8 tissues x 8 classifiers x 4 feature types = 256 evaluations | +| Model families | Classical ML, 4 gene-expression foundation models, 3 text LLMs | +| Public HF fold package | Selected reviewer-facing LOMO tasks with train/test matrices and metadata | + +## Headline Results + +The v7.1 canonical result source is +[docs/CANONICAL_RESULTS_V7_1.md](docs/CANONICAL_RESULTS_V7_1.md). + +| Result surface | Takeaway | +|---|---| +| Multi-method benchmark | PCA-LR is the strongest 8-tissue gene-level baseline in v4, with mean AUROC 0.776. | +| Best tissue rows | Thymus 0.948, colon 0.921, lung 0.901, kidney 0.829 across best method-feature combinations. | +| Cross-mission transfer | Thymus and gastrocnemius show the strongest mission-transfer signal; liver and kidney are harder. | +| Pathway features | Pathway representations rescue some weaker gene-level tissues, especially kidney and eye. | +| Foundation models | Tested gene-expression foundation models underperform tuned classical baselines on small-n bulk RNA-seq mission shift. | +| Held-out validation | Thymus RR-23 AUROC 0.905; skin RR-7 AUROC 0.885. | + +The original long-form README content is preserved at +[docs/README_LONGFORM_V7_1_ARCHIVE_2026_06_15.md](docs/README_LONGFORM_V7_1_ARCHIVE_2026_06_15.md). + +## Quick Start + +### Load A Public Fold From Hugging Face ```bash pip install -r requirements.txt huggingface_hub @@ -300,162 +93,100 @@ pip install -r requirements.txt huggingface_hub from huggingface_hub import hf_hub_download import pandas as pd -train_X = pd.read_csv( - hf_hub_download( - repo_id="jang1563/genelab-benchmark", - filename="A5_skin_lomo/fold_RR-7_test/train_X.csv", - repo_type="dataset", - ), - index_col=0, -) -train_y = pd.read_csv("tasks/A5_skin_lomo/fold_RR-7_test/train_y.csv", index_col=0) -print(f"Train: {train_X.shape}") # (72, 20110) +repo_id = "jang1563/genelab-benchmark" +fold = "A5_skin_lomo/fold_RR-7_test" + +def hf_csv(name): + return pd.read_csv( + hf_hub_download( + repo_id=repo_id, + filename=f"{fold}/{name}", + repo_type="dataset", + ), + index_col=0, + ) + +train_X = hf_csv("train_X.csv") +train_y = hf_csv("train_y.csv").iloc[:, 0] +test_X = hf_csv("test_X.csv") +test_y = hf_csv("test_y.csv").iloc[:, 0] + +print(f"Train: {train_X.shape}, Test: {test_X.shape}") ``` -### Option B -- Reproduce from OSDR raw data +Each public fold includes feature matrices, labels, sample metadata, +`fold_info.json`, and `selected_genes.txt`. -Requires R 4.2+ with Bioconductor. See [docs/r_dependencies.md](docs/r_dependencies.md). +### Validate The Public Release Manifest ```bash -# 1. Download raw data from NASA OSDR -python scripts/fetch_osdr.py --osd OSD-238 OSD-239 OSD-243 OSD-254 - -# 2. Normalize (DESeq2) -Rscript scripts/normalize_rr7_skin.R - -# 3. Quality filter + build all_missions -python scripts/quality_filter.py --tissue skin - -# 4. Generate LOMO folds -python scripts/generate_tasks.py --task A5 +make release-qa +make hpc-public-qa +python3 scripts/validate_release_manifest.py +python3 -m unittest tests/test_release_manifest.py ``` ---- - -## Evaluation Protocol - -All submissions are evaluated with: - -| Metric | Description | Go threshold | -|--------|-------------|-------------| -| Mean AUROC | Average AUROC across folds | > 0.700 | -| 95% CI lower | Bootstrap CI (N=2000) lower bound | > 0.600 | -| perm_p | Permutation p-value (N=1000, pseudocount) | < 0.050 | - -All three conditions must pass for a GO decision. - -### Submission Format - -Prepare a JSON file (see [docs/submission_format.md](docs/submission_format.md)): - -```json -{ - "task_id": "A5", - "model_name": "MyModel_v1", - "predictions": { - "fold_MHU-2_test": {"sample_id_1": 0.92, "sample_id_2": 0.07}, - "fold_RR-6_test": {"...": "..."}, - "fold_RR-7_test": {"...": "..."} - } -} -``` +Create a dry-run Hugging Face upload plan: ```bash -python scripts/evaluate_submission.py \ - --submission my_submission.json \ - --task A5 +make hf-upload-plan HF_TASK=A5 HF_UPLOAD_PLAN=/tmp/spacebiobench_hf_upload_plan_A5.json ``` ---- - -## Model Tracks - -| Track | Examples | Input Format | -|-------|---------|-------------| -| **Tier 1 -- Classical ML** | LR, RF, XGBoost, PCA-LR, SVM, LightGBM, TabNet | Tabular gene x sample | -| **Tier 2 -- Foundation Models** | Geneformer, scGPT, UCE, scFoundation | Gene rank order / embeddings | -| **Tier 3 -- Text LLMs** | GPT-4o, Claude, Llama 3 | Natural language gene list | - ---- - -## Data - -All data is derived from publicly available NASA OSDR datasets. +### Reproduce From OSDR Inputs -### v1 Core (6 tissues, 24 studies) +Raw-data reproduction requires R/Bioconductor and task-specific preprocessing. +See [docs/r_dependencies.md](docs/r_dependencies.md) and the scripts under +[scripts/](scripts/). -| Tissue | OSD Accession | Mission | n samples | -|--------|--------------|---------|-----------| -| Liver | OSD-48, 137, 245, 379, 242, 686 | RR-1, RR-3, RR-6, RR-8, RR-9, MHU-2 | 193 (v1) / 261 (v4 w/ BC/VC) | -| Gastrocnemius | OSD-101, 401, 326 | RR-1, RR-5, RR-9 | 32 | -| Kidney | OSD-102, 163, 253 | RR-1, RR-3, RR-7 | 118 | -| Thymus | OSD-289, 244, 421 | MHU-1, MHU-2, RR-6, RR-9 | 67 | -| Skin | OSD-238, 239, 243, 254 | MHU-2, RR-6, RR-7 | 102 | -| Eye | OSD-100, 194, 397 | RR-1, RR-3, OSD-397 | 37 | +## Repository Map -### v4 Additions (2 tissues) - -| Tissue | OSD Accession | Mission | n samples | -|--------|--------------|---------|-----------| -| Lung | Multiple OSD studies | Multiple missions | See v4 results | -| Colon | Multiple OSD studies | Multiple missions | See v4 results | - -Preprocessing: DESeq2 normalization (per-mission), log2(counts + 1), global low-expression filter (>=20% samples with count>1), top 75th percentile variance gene selection per fold (train missions only). - ---- - -## Design Decisions +```text +tasks/ Public v1 LOMO task inputs and selected fold packages +evaluation/ Historical v1 result JSON and summaries +v2/ ... v7/ Completed historical benchmark layers +v9/ Public-bulk alpha scaffold plus extension-lane drafts +docs/ Cards, canonical results, methods, plans, and release notes +release/ Machine-readable public release manifest +scripts/ Data, evaluation, upload, validation, and figure scripts +``` -Key methodological choices underpinning this benchmark: +## Key Documents -- **Feature encoding**: log2(DESeq2 normalized counts); LFC forbidden in Category A (label leakage) -- **LOMO splits**: Mission = independence unit; variance filter applied on training missions only (no test leakage) -- **Strain**: Track 2a = C57BL/6J only; Track 2b = all strains -- **Evaluation**: AUROC + bootstrap CI (N=2000) + permutation p (N=1000); GO requires all 3 AND conditions -- **Negative controls**: NC1 label permutation, NC2 housekeeping genes, NC3 cross-species gene set -- **v4 classifiers**: PCA-LR, ElasticNet-LR, RF, XGBoost, SVM-Linear, SVM-RBF, TabNet, LightGBM -- **v4 features**: Gene (log2-norm), Hallmark (ssGSEA), KEGG (ssGSEA), pathway-combined -- **Pathway analysis**: fGSEA group-level + gseapy ssGSEA sample-level -- **Text LLM track**: Gene list → natural language prompt → binary classification -- **Category B**: Transfer Pattern Summary (no single GO/NO-GO; use pairs ≥0.70 and perm_p<0.05 counts) +| Need | Document | +|---|---| +| Public result source | [docs/CANONICAL_RESULTS_V7_1.md](docs/CANONICAL_RESULTS_V7_1.md) | +| Methods, evaluation, and release cards | [docs/SPACEBIOBENCH_TRANSPARENCY_CARD_PACK.md](docs/SPACEBIOBENCH_TRANSPARENCY_CARD_PACK.md) | +| System scope | [docs/SPACEBIOBENCH_SYSTEM_CARD.md](docs/SPACEBIOBENCH_SYSTEM_CARD.md) | +| Evaluation interpretation | [docs/SPACEBIOBENCH_EVALUATION_CARD.md](docs/SPACEBIOBENCH_EVALUATION_CARD.md) | +| Release readiness | [docs/SPACEBIOBENCH_RELEASE_READINESS_CARD.md](docs/SPACEBIOBENCH_RELEASE_READINESS_CARD.md) | +| Hugging Face dataset card source | [docs/hf_dataset_card.md](docs/hf_dataset_card.md) | +| v9 metadata-alpha card source | [docs/v9_hf_dataset_card.md](docs/v9_hf_dataset_card.md) | +| GitHub/HF upgrade roadmap | [docs/GITHUB_HF_PREMIUM_UPGRADE_PLAN_2026_06_15.md](docs/GITHUB_HF_PREMIUM_UPGRADE_PLAN_2026_06_15.md) | +| Machine-readable release state | [release/release_manifest.json](release/release_manifest.json) | ---- +## Data And Release Notes -## Changelog +All source data is derived from publicly available NASA OSDR resources. Code is +MIT licensed. The processed public dataset package follows the license declared +in the Hugging Face dataset card; upstream OSDR datasets should be cited and +used under their individual terms. -| Version | Date | Changes | -|---------|------|---------| -| v7.1.2 | 2026-06-05 | Public-card, metadata, and evidence-visibility patch: reviewer fast path, transparency card pack, v9 metadata-alpha evidence subset, release archive gates, and synchronized citation metadata. No new benchmark result generation. | -| v7.0 | 2026-04-12 | Unified benchmark layer complete: scPRINT2 baseline, GNN/WGCNA graph comparisons across 6 tissues, portable checkpoint/result artifacts, and v7 methods/signal-hierarchy synthesis. | -| v6.0 | 2026-03-30 | Cross-species validation complete: gene conservation, pathway conservation, cross-species transfer, TF conservation, biomarker validation, drug target validation (6 result JSONs). | -| v5.0 | 2026-03-29 | Biological interpretation complete: immune deconvolution (mMCP-counter, 8 tissues), cross-organ signaling (OmniPath, 111 L–R pairs), metabolic flux (iMM1865 E-Flux + pFBA), drug targets (DGIdb + ChEMBL, 1,284 FDA interactions), consensus 20-gene biomarker panel (gastro AUROC 0.806). 5 integration figures. | -| v4.0 | 2026-03-28 | v4 complete: 256 evaluations (8 tissues × 8 methods × 4 features). PCA-LR best (AUROC 0.776). Friedman p=0.015. Ablation (569 evals), SHAP multi-method, Python WGCNA (6 tissues), module preservation, STRING PPI. 11 publication figures. | -| v3.0 | 2026-03-20 | Multi-species (E4 Drosophila KEGG), spatial Visium brain (NEGATIVE), RRRM-2 scRNA-seq (PBMC NK 0.845), UCE + scFoundation FM eval, 7×7 cross-tissue transfer, radiation analogs. 5 publication figures. | -| v2.0 | 2026-03-18 | RRRM-1 scRNA-seq 4 tissues (38K cells). E1 cross-species r=0.352. T1–T3 temporal dynamics. 3 publication figures. | -| v1.3 | 2026-03-13 | Tier 3 LLM zero-shot. scGPT mean 0.666. Held-out: thymus RR-23 (0.905), skin RR-7 (0.885). 4 main + 4 supplementary figures. | -| v1.0 | 2026-03-07 | Initial release: 6 tissues, Categories A–D, Geneformer, Cell 2020 validation, fGSEA, GSVA. | +Release labels are intentionally separated: ---- +- **v7.1**: canonical historical result surface and citation target. +- **v9 public bulk**: metadata-only public bulk alpha. ## Citation -*(Manuscript in preparation)* +Please cite the software using [CITATION.cff](CITATION.cff). GitHub renders the +same citation metadata in the repository citation panel. ```bibtex @dataset{kim2026genelab, - title = {GeneLab Benchmark: A Multi-Tissue Spaceflight Transcriptomics Benchmark for AI/ML Models}, - author = {Kim, JangKeun}, - year = {2026}, - url = {https://huggingface.co/datasets/jang1563/genelab-benchmark}, - note = {v7.1.2 documentation, public-card, metadata, and evidence-visibility patch over canonical v7.1 results; data freeze 2026-03-01} + title = {GeneLab Benchmark: A Multi-Tissue Spaceflight Transcriptomics Benchmark for AI/ML Models}, + author = {Kim, JangKeun}, + year = {2026}, + url = {https://huggingface.co/datasets/jang1563/genelab-benchmark}, + note = {v7.1.2 documentation, public-card, metadata, and evidence-visibility patch over canonical v7.1 results; data freeze 2026-03-01} } ``` - -Data source: NASA Open Science Data Repository (OSDR) -- [osdr.nasa.gov](https://osdr.nasa.gov/bio/repo/) - ---- - -## License - -Code: MIT License -Data: NASA OSDR public data (see individual dataset licenses at OSDR); processed benchmark feature matrices should follow the license declared in the public dataset card. diff --git a/docs/GITHUB_HF_PREMIUM_UPGRADE_PLAN_2026_06_15.md b/docs/GITHUB_HF_PREMIUM_UPGRADE_PLAN_2026_06_15.md new file mode 100644 index 0000000..ac68983 --- /dev/null +++ b/docs/GITHUB_HF_PREMIUM_UPGRADE_PLAN_2026_06_15.md @@ -0,0 +1,356 @@ +# GitHub and Hugging Face Premium Upgrade Plan + +Status: strategic execution plan +Date: 2026-06-15 +Scope: GitHub, Hugging Face, citation metadata, release manifests, and public +documentation for GeneLab Benchmark / SpaceBio-Bench. + +## Purpose + +This plan upgrades the public project surface so that both human readers and +machines can understand the benchmark without reconstructing context from many +separate documents. + +The goal is not simply a better README. The goal is a coherent public package: + +- GitHub explains the project, methods, status, and release history. +- Hugging Face distributes benchmark artifacts with clear dataset-card metadata. +- Machine-readable manifests define versions, tasks, files, checksums, schemas, + release status, and citation targets. +- Public-facing prose stays concise and confident, while detailed boundary and + reproducibility information lives in cards, schemas, and manifests. + +## Current Diagnosis + +The repository already has strong raw material: + +- `README.md` introduces the benchmark and summarizes v1-v7 results. +- `docs/hf_dataset_card.md` is a strong Hugging Face card for the public fold + package. +- `docs/v9_hf_dataset_card.md` clearly labels the v9 public bulk track as a + metadata-only alpha. +- `CITATION.cff` records the v7.1 software citation. +- `docs/SPACEBIOBENCH_TRANSPARENCY_CARD_PACK.md` maps the system, evaluation, + release-readiness, and claim-register documents. +- `spacebio_bench/schemas/` already contains task, run, and metric-profile + schemas. +- `spacebio_bench/datapackage.py` and `v9/datapackage.draft.json` begin the + machine-readable data-package spine. + +The main gap is not content volume. The main gap is a single public source of +truth. Today, README, Hugging Face card, citation metadata, v8 release notes, +and v9 alpha documents each carry part of the truth. A new reader or automated +agent has to infer which surface is current, which release lane is frozen, and +which artifact is authoritative. + +## North Star + +SpaceBio-Bench should feel like a premium benchmark platform: + +- A scientist can quickly understand the benchmark question, data sources, + task design, main results, and valid uses. +- A reviewer can trace every public claim to a release lane, result file, + method description, and validation status. +- A machine agent can parse the release manifest and discover tasks, schemas, + files, checksums, licenses, source accessions, and citation metadata. +- A contributor can run validation, rebuild public indices, and prepare a + release without reading the entire documentation history. + +## Public Status Model + +Use one controlled status vocabulary across README, Hugging Face cards, +release manifests, and citation metadata. + +| Lane | Public label | Meaning | Primary surfaces | +|---|---|---|---| +| v7.1 | canonical historical result surface | Public v1-v7 benchmark result and documentation surface | `README.md`, `docs/CANONICAL_RESULTS_V7_1.md`, `docs/hf_dataset_card.md`, `CITATION.cff` | +| v8 | incubating translational extension | Reproducibility hardening and artifact split in progress | `docs/V8_BETA_RELEASE_PLAN_2026_05_10.md`, `v8/release/` | +| v9 public bulk | metadata-only public bulk alpha | Task/source/provenance metadata and scaffold baselines are present; frozen payload release is not yet declared | `docs/v9_hf_dataset_card.md`, `v9/datapackage.draft.json`, `v9/task_manifest_index.csv` | +| v9 extension lanes | diagnostic or draft lanes | Single-cell, organoid, multispecies, and stressor extensions under design or diagnostic evaluation | `v9/README.md`, lane-specific reports | + +Public pages should not expose too much internal decision language. Detailed +release boundaries should remain available, but the landing experience should +read as a clean product surface: what this is, what is available, how to use it, +and where deeper validation lives. + +## Target Information Architecture + +### GitHub + +GitHub should be the canonical project operating system. + +Recommended top-level public structure: + +```text +README.md +CITATION.cff +LICENSE +CHANGELOG.md +CONTRIBUTING.md +docs/ + START_HERE.md + RELEASE_INDEX.md + cards/ + results/ + methods/ + releases/ + api/ +spacebio_bench/ + schemas/ +release/ + release_manifest.json + release_manifest.schema.json + checksums.sha256 +examples/ +scripts/ + release/ +``` + +The root README should become a compact public landing page: + +1. One-sentence benchmark identity. +2. Current public release table. +3. "Use the benchmark" quickstart. +4. "What is measured" task map. +5. "What the main results say" concise result panel. +6. Links to dataset card, transparency card pack, canonical results, v9 alpha, + and citation. + +Long result tables, historical detail, and lane-specific caveats should move +to linked docs. + +### Hugging Face + +Hugging Face should be the canonical dataset distribution surface. + +Recommended dataset package shape: + +```text +README.md +manifest.json +datapackage.json +checksums.sha256 +task_manifest_index.csv +task_data_index.csv +source_inventory.csv +examples/ +assets/ +v7_public_folds/ +v9_public_bulk_alpha/ +``` + +The HF `README.md` should keep a strong YAML metadata block because Hugging Face +uses dataset-card metadata for discoverability and display. The prose should be +shorter than GitHub docs and optimized for dataset users: + +1. What the package contains. +2. Which release lane it represents. +3. How to download/load it. +4. File layout. +5. Citation. +6. Links back to GitHub for full methods and transparency documentation. + +If v7 public folds and v9 metadata alpha continue to share one HF repository, +the lane separation must be obvious in both paths and metadata. A separate +HF dataset repo for v9 is cleaner if public users are likely to confuse v7 +benchmark folds with v9 alpha metadata. + +## Machine-Readable Spine + +Add a canonical `release/release_manifest.json` and validate it in CI. + +Minimum fields: + +```json +{ + "schema_version": "1.0.0", + "project": { + "name": "SpaceBio-Bench", + "legacy_name": "GeneLab Benchmark", + "repository": "https://github.com/jang1563/GeneLab_benchmark" + }, + "release_lanes": [], + "artifacts": [], + "schemas": [], + "huggingface": {}, + "citation": {}, + "checksums": {}, + "validation": {} +} +``` + +The manifest should connect: + +- release lane and status, +- Git commit and optional tag, +- Hugging Face repo id and revision, +- Zenodo or DOI target when available, +- artifact paths and SHA-256 hashes, +- task manifests and schemas, +- source accessions and licenses, +- citation version and preferred citation. + +This file should become the input for generated public surfaces: + +- README release table, +- HF card release table, +- release index, +- citation consistency check, +- upload dry-run report, +- post-upload remote validation. + +## Human-Facing Premium Assets + +The public project should have a small number of polished, reusable visual +assets: + +- benchmark task map, +- release-lane status map, +- artifact flow from OSDR source to task manifest to evaluation output, +- model-family comparison panel, +- HF package layout diagram. + +These should be stored under `docs/assets/` and reused in README, HF card, +slide decks, and manuscript support material. Each asset should have a source +script and visual QA record. + +## Automation Plan + +Add release commands that make premium quality repeatable: + +```bash +make release-qa +make build-public-manifest +make build-hf-package +make upload-hf-dry-run +make validate-hf-remote +``` + +Recommended checks: + +- JSON Schema validation for release, task, run, and metric manifests. +- README, HF card, and `CITATION.cff` version/status consistency. +- Broken-link check for public docs. +- SHA-256 verification for release artifacts. +- HF dry-run diff before upload. +- HF remote smoke download after upload. +- Status-language lint so alpha/draft surfaces do not accidentally claim a + frozen release. +- Large-file and local-cache guard before release commits. + +## 30/60/90 Day Execution Board + +### First 30 days: public coherence + +- Create `release/release_manifest.schema.json`. +- Create first `release/release_manifest.json` covering v7.1, v8, and v9 + public bulk alpha. +- Add a generated `docs/RELEASE_INDEX.md`. +- Rewrite root README into a compact landing page. +- Keep historical result detail in `docs/CANONICAL_RESULTS_V7_1.md`. +- Add a README/HF/CITATION consistency test. + +Deliverable: a new reader can understand the public state in under five +minutes, and an automated check can confirm that status labels agree. + +### Days 31-60: HF package hardening + +- Decide whether v9 alpha stays in the existing HF repo or moves to a separate + dataset repo. +- Generate HF card sections from the release manifest. +- Add `manifest.json`, `datapackage.json`, and `checksums.sha256` to the HF + package. +- Upgrade `scripts/upload_to_hf.py` into a release package builder with dry-run + diff and post-upload validation. +- Add one polished HF summary graphic generated from source data. + +Deliverable: HF becomes a deterministic, auditable distribution package rather +than a manually curated upload target. + +### Days 61-90: CI and release discipline + +- Add GitHub Actions for schema validation, docs consistency, and link checks. +- Add a release checklist template. +- Add contributor and issue templates. +- Add release-note generation from `release_manifest.json`. +- Record HF revision after every upload. +- Prepare Zenodo/DOI metadata if a frozen release is ready. + +Deliverable: public releases can be prepared from a clean checkout with a +repeatable validation path. + +### 3-6 months: platform maturity + +- Add RO-Crate or DataCite-compatible metadata for archival release. +- Add richer examples for classical ML, foundation-model adapters, and + submission validation. +- Add `docs/AI_AGENT_README.md` or `llms.txt` for machine-agent orientation. +- Publish stable API docs if `spacebio_bench` becomes an installable package. +- Add benchmark cards for each promoted task family. + +Deliverable: SpaceBio-Bench reads as a professional benchmark platform, not a +single project repository. + +## Decision Points + +### Public name + +Recommended approach: + +- Use "SpaceBio-Bench" as the forward-looking platform name. +- Preserve "GeneLab Benchmark" as the historical v1-v7 name and repository + alias. +- Make the transition explicit in README, HF card, and citation metadata. + +### HF repository split + +Option A: one HF repo. + +- Pros: one public dataset link, simpler historical continuity. +- Cons: v7 public folds and v9 metadata alpha can be confused. + +Option B: separate HF repos for stable/frozen and alpha/living tracks. + +- Pros: cleaner user mental model and safer release language. +- Cons: more metadata maintenance. + +Recommendation: keep the existing `jang1563/genelab-benchmark` for v7 public +folds and create a separate v9 alpha repo when public distribution expands +beyond metadata indices. + +### Package identity + +Decide whether the repo should remain documentation/scripts only or become an +installable Python package. If the latter, add `pyproject.toml`, console +entrypoints, and API docs. + +## Quality Bar + +A public update is premium quality only when all of the following are true: + +- A human can identify the current public release state from the first screen + of the README. +- The HF card says exactly what is packaged and what release lane it belongs to. +- `CITATION.cff` matches the release lane being cited. +- Every public artifact has a path, role, version, and checksum in a manifest. +- Every promoted task has a schema-valid task manifest. +- Every promoted result has a run manifest or equivalent provenance record. +- Public docs do not require internal project history to interpret. +- Release boundaries are clear without sounding apologetic or defensive. + +## Immediate Next Actions + +1. Create `release/release_manifest.schema.json`. +2. Create `release/release_manifest.json` with current v7.1, v8, and v9 public + lane status. +3. Add a small validator script for release-manifest schema and required public + files. +4. Draft the compact README structure without deleting existing result detail. +5. Draft the HF package layout decision: one repo now, separate v9 repo later. + +## External Platform References + +- Hugging Face Dataset Cards: https://huggingface.co/docs/hub/datasets-cards +- Hugging Face Model Cards: https://huggingface.co/docs/hub/model-cards +- GitHub README documentation: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-readmes +- GitHub CITATION files: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-citation-files diff --git a/docs/README_LONGFORM_V7_1_ARCHIVE_2026_06_15.md b/docs/README_LONGFORM_V7_1_ARCHIVE_2026_06_15.md new file mode 100644 index 0000000..a5fb658 --- /dev/null +++ b/docs/README_LONGFORM_V7_1_ARCHIVE_2026_06_15.md @@ -0,0 +1,431 @@ +# GeneLab Benchmark + +**A comprehensive benchmark for evaluating AI/ML and Foundation Models on NASA OSDR spaceflight transcriptomics data.** + +Version: v7.0 (2026-04-12) | Dataset freeze: 2026-03-01 +Status: **v1–v7 Complete** + +Maintainer / citation author: JangKeun Kim, Weill Cornell Medicine. + +Canonical v7.1 documentation source: [`docs/CANONICAL_RESULTS_V7_1.md`](docs/CANONICAL_RESULTS_V7_1.md) records the locked scope accounting, headline result table, and v8 boundary notes for public release text. + +Transparency card pack: [`docs/SPACEBIOBENCH_TRANSPARENCY_CARD_PACK.md`](docs/SPACEBIOBENCH_TRANSPARENCY_CARD_PACK.md) maps the system card, evaluation card, release readiness card, and claim register for v1-v7 results plus v8/v9 draft-surface boundaries. + +Portfolio brief: [`docs/SPACEBIOBENCH_PORTFOLIO_BRIEF.md`](docs/SPACEBIOBENCH_PORTFOLIO_BRIEF.md) summarizes the project contribution for portfolio and external-review contexts. + +[![Dataset on HuggingFace](https://img.shields.io/badge/HuggingFace-Dataset-yellow)](https://huggingface.co/datasets/jang1563/genelab-benchmark) +[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) + +--- + +## What Is This? + +GeneLab Benchmark provides standardized tasks for evaluating how well machine learning models — from classical baselines to gene expression foundation models (Geneformer, scGPT, UCE, scFoundation) and text-based LLMs (GPT-4o, Claude, Llama) — generalize **spaceflight transcriptomic signatures across missions**. + +**Core challenge**: Train a model on one spaceflight mission's RNA-seq data. Can it classify samples from a different mission it has never seen? + +**Data source**: [NASA Open Science Data Repository (OSDR)](https://osdr.nasa.gov/bio/repo/) — mouse multi-tissue bulk RNA-seq from ISS and ground control missions. + +### Benchmark Scope + +- **8 tissues**: Liver, Gastrocnemius, Kidney, Thymus, Skin, Eye, Lung, Colon +- **17+ ISS missions**: RR-1 through RR-9, MHU-1, MHU-2, and more +- **24+ verified OSD studies**, ~600+ samples (binary Flight/Ground, with BC/VC controls in v4) +- **25+ evaluation tasks** across 7 categories (A-D, J, NC, Validation) +- **v4**: 256 evaluations (8 tissues x 8 classifiers x 4 feature types) +- **Multi-species**: Mouse bulk, Drosophila KEGG, mouse scRNA-seq (RRRM-1/RRRM-2), spatial Visium +- **4 gene-expression foundation models + 3 text LLMs evaluated**: Geneformer, scGPT, UCE, scFoundation, plus GPT-4o/Claude/Llama + +--- + +## Key Features + +- **Leave-One-Mission-Out (LOMO)** cross-validation — mission = independence unit, preventing cross-mission data leakage +- **Category A**: Spaceflight detection per tissue (binary: Flight vs. Ground) +- **Category B**: Cross-mission transfer matrix (train on mission i, test on mission j) for all tissues +- **Category C**: Cross-tissue transfer (3 methods: gene, DEG, pathway) +- **Category D**: Condition/confounder prediction (mission, strain, hardware, gravity) +- **3-tier model evaluation**: Classical ML -> Gene Expression Foundation Models -> Text LLMs +- **Standardized submission format** with automatic AUROC/CI/p-value evaluation +- **Biological validation**: NES pathway conservation, Cell 2020 concordance, negative controls +- **v4 multi-method**: 8 classifiers (PCA-LR, ElasticNet-LR, RF, XGBoost, SVM-Linear, SVM-RBF, TabNet, LightGBM) +- **v4 multi-feature**: 4 feature types (gene, Hallmark, KEGG, pathway-combined) + +--- + +## Results Overview + +### v4 Phase 1 — Multi-Method Evaluation (256 evaluations) + +**Best overall classifier**: PCA-LR (gene mean AUROC 0.776), ElasticNet-LR 2nd (0.762) + +Canonical v7.1 result source: [`docs/CANONICAL_RESULTS_V7_1.md`](docs/CANONICAL_RESULTS_V7_1.md). Keep README, Hugging Face card, and paper outline synchronized with that table. + +| Tissue | Best AUROC | Best Method | Best Feature | Significant (p<0.05) | +|--------|-----------|-------------|-------------|---------------------| +| **Thymus** | **0.948*** | PCA-LR | KEGG | Yes | +| **Colon** | **0.921*** | PCA-LR | KEGG | Yes | +| **Lung** | **0.901*** | PCA-LR | Gene | Yes | +| **Kidney** | **0.829**** | ElasticNet-LR | Hallmark | Yes | +| **Skin** | 0.819 | PCA-LR | Gene | - | +| **Eye** | 0.823 | PCA-LR | Hallmark | - | +| **Liver** | 0.670 | PCA-LR | Gene | - | +| **Gastrocnemius** | 0.776 | PCA-LR | Gene | - | + +*p<0.05, **p<0.01. 40/256 evaluations significant at p<0.05; 6/8 tissues have at least 1 significant result. + +**Key v4 findings**: +- SVM-RBF worst on high-dimensional gene data (0.510 mean) +- TabNet disappointing (0.527 mean) — deep learning doesn't help small-n transcriptomics +- Pathway features improve kidney (0.584->0.829), thymus (0.908->0.948), eye (0.697->0.823) +- Gene features better for skin (0.819) and lung (0.901) +- v4 expanded controls: BC/VC included (e.g., liver 261 vs v1's 193 samples) + +### v1 Results — Spaceflight Detection (LOMO, 6 tissues) + +**Gene-level (primary)**: + +| Task | Tissue | Missions | Method | Mean AUROC | 95% CI lower | perm_p | Decision | +|------|--------|---------|--------|-----------|-------------|--------|----------| +| A4 | Thymus | 4 | PCA-LR | **0.923** | 0.878 | 0.037 | GO | +| A2 | Gastrocnemius | 3 | LR | **0.907** | 0.717 | 0.026 | GO | +| A5 | Skin | 3 | LR | **0.821** | 0.637 | 0.0023 | GO | +| A6 | Eye | 3 | LR | 0.811 | 0.470 | 0.063 | NO-GO | +| A1 | Liver | 6 | LR | 0.653 | 0.457 | 0.091 | NO-GO | +| A3 | Kidney | 3 | LR | 0.593 | 0.431 | 0.281 | NO-GO | + +**Pathway-level rescue**: Eye gene-level NO-GO -> GSVA Hallmark AUROC **0.915** (GO). Kidney 0.43->0.74. + +### Cross-Mission Transfer (Category B) + +| Tissue | Mean AUROC | 95% CI | AUROC>=0.70 | Tier | +|--------|-----------|--------|-----------|------| +| Thymus | **0.860** | [0.763, 0.953] | 9/12 | 1 | +| Gastrocnemius | **0.801** | [0.653, 0.944] | 4/6 | 1 | +| Skin | **0.772** | [0.691, 0.834] | 5/6 | 2 | +| Eye | 0.754 | [0.688, 0.838] | 5/6 | 2 | +| Liver | 0.577 | [0.492, 0.666] | 13/30 | 3 | +| Kidney | 0.555 | [0.397, 0.681] | 2/6 | 3 | + +### v5 — Biological Interpretation + +| Analysis | Key Finding | +|----------|------------| +| **Immune deconvolution** (mMCP-counter, 8 tissues) | Skin strongest: 6/14 cell types FDR<0.05. Kidney, thymus 2/14 each. Most tissues 0 significant | +| **Cross-organ signaling** (OmniPath, 111 L–R pairs) | 1 SHAP-active L–R pair. TF activity: thymus 240 sig, skin 241, kidney 177, liver 105 | +| **Metabolic flux** (iMM1865 E-Flux + pFBA) | Flight vs ground objectives differ in all 6 tissues (largest: thymus FLT 15,695 vs GC 14,696) | +| **Drug targets** (DGIdb + ChEMBL) | 271/834 spaceflight genes druggable; 1,284 FDA-approved drug–gene interactions | +| **Consensus biomarker panel** (20 genes) | Top genes: MUP22, Thrsp, Apoa1, NPAS2, PER2. AUROC: gastro 0.806, liver 0.754, eye 0.728 | + +### Foundation / LLM Comparison Snapshot + +| Model | Architecture | Mean AUROC (6 tissues) | vs PCA-LR | +|-------|------------|----------------------|-----------| +| **PCA-LR baseline** | Classical ML | **0.758** | — | +| scGPT | 12L Transformer, 33M human cells | 0.666 | -0.093 | +| scFoundation | 100M params, 50M cells | 0.635† (liver best, p<0.01) | Below baseline | +| UCE | 33L, 36M cells | 0.632† (thymus best, p=0.031) | Below baseline | +| Mouse-Geneformer | 6L BERT, 30M mouse cells | 0.476 | -0.283 | +| Text LLMs (3x) | GPT-4o, Claude, Llama 3 | 0.47-0.51 | Chance level | + +**FM verdict**: All foundation models underperform classical ML. The scGPT and Mouse-Geneformer rows report 6-tissue v1 means against the 0.758 PCA-LR baseline; the scFoundation and UCE rows show their best single-tissue v3 results, and their full cross-tissue means also remain below baseline. †Best single-tissue AUROC shown; cross-tissue means are lower. + +### Independent Held-Out Validation + +| Tissue | Mission | AUROC | 95% CI | p-value | Duration | n_test | +|--------|---------|-------|--------|---------|---------|--------| +| Thymus | RR-23 | **0.905** | [0.672, 1.000] | 0.005 | 30 days | 18 | +| Skin | RR-7 | **0.885** | [0.745, 0.986] | <0.001 | 75 days | 30 | + +--- + +## Key Scientific Findings + +### Pre-registered Hypotheses + +| Hypothesis | Statement | Verdict | Key Evidence | +|---|---|---|---| +| **H1** | Liver has the most consistent cross-mission transcriptome | **REFUTED** | Thymus (0.860) >> Liver (0.577). Thymus and Gastrocnemius = Tier 1. | +| **H2** | Transfer failure from biological diversity, not batch effects | **SUPPORTED** | NES conservation r=0.9 (5 tissues). D3 pathway F1=0.06 (batch-invariant). | +| **H3** | Pathway-level preserves spaceflight response better than gene-level | **CONDITIONALLY SUPPORTED** | Kidney rescue (0.43->0.74), Eye (0.79->0.92). But tissue-pair dependent. | + +### Novel Findings + +- **Tissue hierarchy**: Thymus > Gastrocnemius > Skin > Eye >> Liver > Kidney for spaceflight detection +- **Pathway rescue**: Pathway features rescue NO-GO tissues (kidney +0.31, eye +0.13) +- **FM failure mode**: scRNA-seq-pretrained FMs don't transfer to small-n bulk transcriptomics +- **Spaceflight amplifies aging**: OLD AUROC=0.945 vs YNG=0.679 (Delta=+0.266, RR-8 liver) +- **PBMC NK cells**: Strongest single-cell spaceflight signal (AUROC=0.845, RRRM-2) +- **Brain spatial**: Genuine negative — no detectable spaceflight signal (Visium AUROC=0.139) +- **Cross-species**: Drosophila-mouse NES correlations significant but negative (r=-0.19 to -0.59) + +--- + +## Version Structure + +| Version | Scope | Status | Location | +|---------|-------|--------|----------| +| **v1.0** | Mouse bulk RNA-seq, 6 tissues, 25+ tasks, 3 model tiers, 2 held-out validations | **Complete** | Project root | +| **v2.0** | Temporal dynamics (T1-T3), cross-species (E1-E3), single-cell (F1, F2 RRRM-1) | **Complete** | `v2/` | +| **v3.0** | Multi-species (E4), Spatial Visium (F3), RRRM-2 scRNA-seq (F5), UCE/scFoundation FM, radiation analogs, cross-tissue transfer 7×7 | **Complete** | `v3/` | +| **v4.0** | Multi-method benchmark: 8 tissues × 8 classifiers × 4 features (256 evals), ablation, SHAP, WGCNA, module preservation, STRING PPI | **Complete** | `v4/` | +| **v5.0** | Biological interpretation: immune deconvolution (mMCP-counter), metabolic flux (iMM1865 E-Flux), drug targets (DGIdb/ChEMBL), consensus 20-gene biomarker panel | **Complete** | `v5/` | +| **v6.0** | Cross-species validation: gene/pathway conservation, cross-species transfer, TF conservation, biomarker/drug target validation across species | **Complete** | `v6/` | +| **v7.0** | Unified/foundation-model benchmarking: scPRINT2, GNN/WGCNA graph baselines, cross-method synthesis, and signal hierarchy analysis | **Complete** | `v7/` | + +--- + +## Repository Structure + +``` +GeneLab_benchmark/ +├── README.md <- This file +├── DATA_CATALOG.md <- OSDR inventory (24+ studies) +├── CITATION.cff <- Citation metadata +│ +├── tasks/ <- Public task inputs (benchmark + sensitivity tasks) +│ ├── A1_liver_lomo/ <- 6 folds + 3 variants +│ ├── A2_gastrocnemius_lomo/ <- 3 folds +│ ├── A3_kidney_lomo/ <- 3 folds +│ ├── A4_thymus_lomo/ <- 4 folds + holdout +│ ├── A5_skin_lomo/ <- 3 folds +│ ├── A6_eye_lomo/ <- 3 folds +│ └── B1-B6_*_cross_mission/ <- N x (N-1) mission pairs per tissue +│ +├── scripts/ <- v1 pipeline scripts +│ ├── run_baselines.py <- Classical ML baseline runner +│ ├── evaluate_submission.py <- Submission evaluator (AUROC, CI, perm_p) +│ ├── generate_tasks.py <- LOMO split generator +│ └── ... <- See scripts/ for full list +│ +├── docs/ +│ ├── hf_dataset_card.md <- HuggingFace dataset documentation +│ ├── submission_format.md <- JSON submission specification +│ ├── text_llm_format.md <- Text LLM evaluation format +│ └── BIOLOGICAL_GROUND_TRUTH.md <- Validation reference (Cell 2020, SOMA 2024) +│ +├── evaluation/ <- Result JSON files (v1 baseline) +│ ├── RESULTS_SUMMARY.md <- Comprehensive results table (v1–v5) +│ └── *.json <- Per-task results +│ +├── v2/ <- Temporal dynamics, cross-species, scRNA-seq +│ ├── README.md +│ ├── scripts/ <- v2 analysis and runner scripts +│ ├── evaluation/ <- v2 results +│ └── figures/ <- v2 interactive HTML figures +│ +├── v3/ <- Multi-species, spatial Visium, FM evaluation +│ ├── README.md +│ ├── scripts/ <- v3 scripts (Python/Bash/R) +│ ├── evaluation/ <- v3 result summaries +│ └── figures/ <- v3 interactive HTML figures +│ +├── v4/ <- Multi-method benchmark + network biology +│ ├── scripts/ <- Classifiers, SHAP, WGCNA, and PPI analysis +│ ├── evaluation/ <- v4 result JSONs + SHAP/WGCNA outputs +│ ├── wgcna_outputs/ <- Per-tissue WGCNA module data +│ └── figures/html/ <- v4 interactive HTML figures +│ +├── v5/ <- Biological interpretation layer +│ ├── scripts/ <- Immune deconv, metabolic flux, drug targets +│ ├── evaluation/ <- Immune, TF, metabolic, and drug-target results +│ └── figures/html/ <- v5 interactive HTML figures +│ +├── v6/ <- Cross-species validation +│ ├── scripts/ <- Phase A-F Python scripts +│ └── evaluation/ <- v6 result summaries +│ +├── v7/ <- Unified/foundation-model benchmarking +│ ├── scripts/ <- HPC setup and launcher scripts +│ ├── unified/ <- scPRINT2, GNN/WGCNA, and synthesis drivers +│ ├── evaluation/ <- v7 result JSONs +│ └── figures/html/ <- v7 interactive HTML figures +│ +└── processed/ <- Intermediate analysis outputs + ├── A_detection/ <- Per-tissue LOMO data + ├── B_cross_mission/ <- Transfer matrices + ├── fgsea/ <- 60+ fGSEA results + └── pathway_scores/ <- 48 ssGSEA pathway score files +``` + +--- + +## Getting Started + +Self-contained public fold packages are hosted on HuggingFace. Each fold includes +`train_X.csv`, `test_X.csv`, labels, sample metadata, `fold_info.json`, and +`selected_genes.txt`. + +### Option A -- Load from HuggingFace (recommended) + +```bash +pip install -r requirements.txt huggingface_hub +``` + +```python +from huggingface_hub import hf_hub_download +import pandas as pd + +repo_id = "jang1563/genelab-benchmark" +fold = "A5_skin_lomo/fold_RR-7_test" + +def hf_csv(name): + return pd.read_csv( + hf_hub_download(repo_id=repo_id, filename=f"{fold}/{name}", repo_type="dataset"), + index_col=0, + ) + +train_X = hf_csv("train_X.csv") +train_y = hf_csv("train_y.csv").iloc[:, 0] +test_X = hf_csv("test_X.csv") +test_y = hf_csv("test_y.csv").iloc[:, 0] + +print(f"Train: {train_X.shape}, Test: {test_X.shape}") # (72, 20110), (30, 20110) +``` + +### Option B -- Reproduce from OSDR raw data + +Requires R 4.2+ with Bioconductor. See [docs/r_dependencies.md](docs/r_dependencies.md). + +```bash +# 1. Download raw data from NASA OSDR +python scripts/fetch_osdr.py --osd OSD-238 OSD-239 OSD-243 OSD-254 + +# 2. Normalize (DESeq2) +Rscript scripts/normalize_rr7_skin.R + +# 3. Quality filter + build all_missions +python scripts/quality_filter.py --tissue skin + +# 4. Generate LOMO folds +python scripts/generate_tasks.py --task A5 +``` + +--- + +## Evaluation Protocol + +All submissions are evaluated with: + +| Metric | Description | Go threshold | +|--------|-------------|-------------| +| Mean AUROC | Average AUROC across folds | > 0.700 | +| 95% CI lower | Bootstrap CI (N=2000) lower bound | > 0.600 | +| perm_p | Permutation p-value (N=1000, pseudocount) | < 0.050 | + +All three conditions must pass for a GO decision. + +### Submission Format + +Prepare a JSON file (see [docs/submission_format.md](docs/submission_format.md)): + +```json +{ + "task_id": "A5", + "model_name": "MyModel_v1", + "predictions": { + "fold_MHU-2_test": {"sample_id_1": 0.92, "sample_id_2": 0.07}, + "fold_RR-6_test": {"...": "..."}, + "fold_RR-7_test": {"...": "..."} + } +} +``` + +```bash +python scripts/evaluate_submission.py \ + --submission my_submission.json \ + --task A5 +``` + +--- + +## Model Tracks + +| Track | Examples | Input Format | +|-------|---------|-------------| +| **Tier 1 -- Classical ML** | LR, RF, XGBoost, PCA-LR, SVM, LightGBM, TabNet | Tabular gene x sample | +| **Tier 2 -- Foundation Models** | Geneformer, scGPT, UCE, scFoundation | Gene rank order / embeddings | +| **Tier 3 -- Text LLMs** | GPT-4o, Claude, Llama 3 | Natural language gene list | + +--- + +## Data + +All data is derived from publicly available NASA OSDR datasets. + +### v1 Core (6 tissues, 24 studies) + +| Tissue | OSD Accession | Mission | n samples | +|--------|--------------|---------|-----------| +| Liver | OSD-48, 137, 245, 379, 242, 686 | RR-1, RR-3, RR-6, RR-8, RR-9, MHU-2 | 193 (v1) / 261 (v4 w/ BC/VC) | +| Gastrocnemius | OSD-101, 401, 326 | RR-1, RR-5, RR-9 | 32 | +| Kidney | OSD-102, 163, 253 | RR-1, RR-3, RR-7 | 118 | +| Thymus | OSD-289, 244, 421 | MHU-1, MHU-2, RR-6, RR-9 | 67 | +| Skin | OSD-238, 239, 243, 254 | MHU-2, RR-6, RR-7 | 102 | +| Eye | OSD-100, 194, 397 | RR-1, RR-3, OSD-397 | 37 | + +### v4 Additions (2 tissues) + +| Tissue | OSD Accession | Mission | n samples | +|--------|--------------|---------|-----------| +| Lung | Multiple OSD studies | Multiple missions | See v4 results | +| Colon | Multiple OSD studies | Multiple missions | See v4 results | + +Preprocessing: DESeq2 normalization (per-mission), log2(counts + 1), global low-expression filter (>=20% samples with count>1), top 75th percentile variance gene selection per fold (train missions only). + +--- + +## Design Decisions + +Key methodological choices underpinning this benchmark: + +- **Feature encoding**: log2(DESeq2 normalized counts); LFC forbidden in Category A (label leakage) +- **LOMO splits**: Mission = independence unit; variance filter applied on training missions only (no test leakage) +- **Strain**: Track 2a = C57BL/6J only; Track 2b = all strains +- **Evaluation**: AUROC + bootstrap CI (N=2000) + permutation p (N=1000); GO requires all 3 AND conditions +- **Negative controls**: NC1 label permutation, NC2 housekeeping genes, NC3 cross-species gene set +- **v4 classifiers**: PCA-LR, ElasticNet-LR, RF, XGBoost, SVM-Linear, SVM-RBF, TabNet, LightGBM +- **v4 features**: Gene (log2-norm), Hallmark (ssGSEA), KEGG (ssGSEA), pathway-combined +- **Pathway analysis**: fGSEA group-level + gseapy ssGSEA sample-level +- **Text LLM track**: Gene list → natural language prompt → binary classification +- **Category B**: Transfer Pattern Summary (no single GO/NO-GO; use pairs ≥0.70 and perm_p<0.05 counts) + +--- + +## Changelog + +| Version | Date | Changes | +|---------|------|---------| +| v7.0 | 2026-04-12 | Unified benchmark layer complete: scPRINT2 baseline, GNN/WGCNA graph comparisons across 6 tissues, portable checkpoint/result artifacts, and v7 methods/signal-hierarchy synthesis. | +| v6.0 | 2026-03-30 | Cross-species validation complete: gene conservation, pathway conservation, cross-species transfer, TF conservation, biomarker validation, drug target validation (6 result JSONs). | +| v5.0 | 2026-03-29 | Biological interpretation complete: immune deconvolution (mMCP-counter, 8 tissues), cross-organ signaling (OmniPath, 111 L–R pairs), metabolic flux (iMM1865 E-Flux + pFBA), drug targets (DGIdb + ChEMBL, 1,284 FDA interactions), consensus 20-gene biomarker panel (gastro AUROC 0.806). 5 integration figures. | +| v4.0 | 2026-03-28 | v4 complete: 256 evaluations (8 tissues × 8 methods × 4 features). PCA-LR best (AUROC 0.776). Friedman p=0.015. Ablation (569 evals), SHAP multi-method, Python WGCNA (6 tissues), module preservation, STRING PPI. 11 publication figures. | +| v3.0 | 2026-03-20 | Multi-species (E4 Drosophila KEGG), spatial Visium brain (NEGATIVE), RRRM-2 scRNA-seq (PBMC NK 0.845), UCE + scFoundation FM eval, 7×7 cross-tissue transfer, radiation analogs. 5 publication figures. | +| v2.0 | 2026-03-18 | RRRM-1 scRNA-seq 4 tissues (38K cells). E1 cross-species r=0.352. T1–T3 temporal dynamics. 3 publication figures. | +| v1.3 | 2026-03-13 | Tier 3 LLM zero-shot. scGPT mean 0.666. Held-out: thymus RR-23 (0.905), skin RR-7 (0.885). 4 main + 4 supplementary figures. | +| v1.0 | 2026-03-07 | Initial release: 6 tissues, Categories A–D, Geneformer, Cell 2020 validation, fGSEA, GSVA. | + +--- + +## Citation + +*(Manuscript in preparation)* + +```bibtex +@dataset{kim2026genelab, + title = {GeneLab Benchmark: A Multi-Tissue Spaceflight Transcriptomics Benchmark for AI/ML Models}, + author = {Kim, JangKeun}, + year = {2026}, + url = {https://huggingface.co/datasets/jang1563/genelab-benchmark}, + note = {v7.0 with v7.1 documentation consistency patch; data freeze 2026-03-01} +} +``` + +Data source: NASA Open Science Data Repository (OSDR) -- [osdr.nasa.gov](https://osdr.nasa.gov/bio/repo/) + +--- + +## License + +Code: MIT License +Data: NASA OSDR public data (see individual dataset licenses at OSDR); processed benchmark feature matrices should follow the license declared in the public dataset card. diff --git a/docs/assets/hf_benchmark_summary.png b/docs/assets/hf_benchmark_summary.png index 1d0adcf..8103d45 100644 Binary files a/docs/assets/hf_benchmark_summary.png and b/docs/assets/hf_benchmark_summary.png differ diff --git a/docs/hf_dataset_card.md b/docs/hf_dataset_card.md index 9d9911a..ea676f0 100644 --- a/docs/hf_dataset_card.md +++ b/docs/hf_dataset_card.md @@ -6,315 +6,227 @@ tags: - genomics - transcriptomics - spaceflight + - space-biology - benchmarking - nasa-osdr - bulk-rna-seq - mouse - rna-seq + - mission-held-out - foundation-model - pathway-analysis - - single-cell - - spatial-transcriptomics + - spacebio-bench size_categories: - 100M -GeneLab Benchmark provides standardized train/test splits for evaluating how well machine learning models generalize **spaceflight transcriptomic signatures across ISS missions**. +Maintainer / citation author: JangKeun Kim, Weill Cornell Medicine. -**Core challenge**: Given RNA-seq from one spaceflight mission, can a classifier detect spaceflight vs. ground control in samples from a different mission it has never seen? +![SpaceBio-Bench benchmark at a glance](assets/hf_benchmark_summary.png) -**Data source**: NASA Open Science Data Repository ([OSDR](https://osdr.nasa.gov/bio/repo/)) -- mouse multi-tissue bulk RNA-seq from ISS rodent research missions (C57BL/6J strain). +## What Is In This Dataset -### Scope +This Hugging Face dataset contains self-contained public fold packages from the +GeneLab Benchmark v1-v7 surface. Each fold holds out one mission as the test set +and provides all files needed to train on the remaining missions and evaluate on +the held-out mission. -| Dimension | Coverage | +| Public package item | Description | |---|---| -| Tissues | 8 (Liver, Gastrocnemius, Kidney, Thymus, Skin, Eye, Lung, Colon) | -| Core ISS mission labels | 9 named labels in the public task package (RR-1, RR-3, RR-5, RR-6, RR-7, RR-8, RR-9, MHU-1, MHU-2) | -| OSD Studies | 24+ source accessions across release layers | -| Samples | 600+ binary/control samples across the processed release layers | -| Classifiers | 8 (PCA-LR, ElasticNet-LR, RF, XGBoost, SVM-Linear, SVM-RBF, TabNet, LightGBM) | -| Feature types | 4 (Gene, Hallmark pathways, KEGG pathways, Combined) | -| Foundation / Language Models | 4 gene-expression FMs + 3 text LLMs | - ---- - -The GitHub repository contains the full v1-v7 benchmark surface. This Hugging Face dataset card exposes the public feature-matrix package and reviewer-facing result summary; counts below separate full-release scope from specific analysis subsets. - -## Transparency and Release Boundary - -This Hugging Face card is the concise public card for the v1-v7 feature-matrix -package. Project-level transparency documentation is maintained in GitHub so the -system scope, evaluation protocol, release-readiness gates, and claim boundaries -can be reviewed together: - -- Transparency card pack: - https://github.com/jang1563/GeneLab_benchmark/blob/main/docs/SPACEBIOBENCH_TRANSPARENCY_CARD_PACK.md -- System card: - https://github.com/jang1563/GeneLab_benchmark/blob/main/docs/SPACEBIOBENCH_SYSTEM_CARD.md -- Evaluation card: - https://github.com/jang1563/GeneLab_benchmark/blob/main/docs/SPACEBIOBENCH_EVALUATION_CARD.md -- Release readiness card: - https://github.com/jang1563/GeneLab_benchmark/blob/main/docs/SPACEBIOBENCH_RELEASE_READINESS_CARD.md -- Claim register: - https://github.com/jang1563/GeneLab_benchmark/blob/main/docs/SPACEBIOBENCH_CLAIM_REGISTER.md - -For DOI-oriented citation, use the final GitHub release tag and archive DOI -recorded in the release archive manifest once minted. - -Current release wording: v1-v7 are the canonical historical benchmark and -reviewer-facing feature-matrix surface; v8 and v9 are documented as extension or -metadata-alpha surfaces and should not be described as frozen public benchmark -payload releases. +| `train_X.csv`, `test_X.csv` | Sample-by-gene expression matrices | +| `train_y.csv`, `test_y.csv` | Binary labels: `1 = Flight`, `0 = Ground` | +| `train_meta.csv`, `test_meta.csv` | Sample-level metadata used for fold auditing | +| `fold_info.json` | Held-out mission, train missions, sample counts, and provenance | +| `selected_genes.txt` | Fold-specific genes selected from training missions only | +| `task_info.json` | Task-level metadata and source summary | -## Dataset Structure +The web Dataset Viewer is disabled because these are high-dimensional +sample-by-gene matrices plus JSON artifacts. Use direct downloads for reliable +access. -This HuggingFace repository contains **feature matrices** (train_X.csv, test_X.csv) for 4 benchmark tasks that passed significance thresholds. Labels and metadata are in the [GitHub repository](https://github.com/jang1563/GeneLab_benchmark). +## Public Fold Layout -``` +```text genelab-benchmark/ -├── A2_gastrocnemius_lomo/ <- 3 missions, 32 samples +├── A2_gastrocnemius_lomo/ +│ ├── task_info.json │ ├── fold_RR-1_test/ -│ │ ├── train_X.csv <- Training features (samples x genes) -│ │ └── test_X.csv <- Test features │ ├── fold_RR-5_test/ │ └── fold_RR-9_test/ -│ -├── A4_thymus_lomo/ <- 4 missions, 67 samples +├── A4_thymus_lomo/ +│ ├── task_info.json │ ├── fold_MHU-1_test/ │ ├── fold_MHU-2_test/ │ ├── fold_RR-6_test/ -│ └── fold_RR-9_test/ -│ -├── A5_skin_lomo/ <- 3 missions, 102 samples +│ ├── fold_RR-9_test/ +│ └── fold_RR-23_holdout/ +├── A5_skin_lomo/ +│ ├── task_info.json │ ├── fold_MHU-2_test/ │ ├── fold_RR-6_test/ -│ └── fold_RR-7_test/ -│ -├── A6_eye_lomo/ <- 3 missions, 37 samples +│ ├── fold_RR-7_test/ +│ └── fold_RR-7_holdout/ +├── A6_eye_lomo/ +│ ├── task_info.json │ ├── fold_RR-1_test/ │ ├── fold_RR-3_test/ -│ └── fold_OSD-397_test/ <- stable public label for OSD-397 -│ -├── v4/evaluation/ <- Multi-method evaluation results (JSON) -├── v5/evaluation/ <- Systems biology analysis results -└── v6/evaluation/ <- Human translation analysis results +│ └── fold_OSD-397_test/ +├── v4/evaluation/ +├── v5/evaluation/ +└── v6/evaluation/ ``` -Each fold holds out one mission as the test set and trains on the remaining missions. This **Leave-One-Mission-Out (LOMO)** cross-validation evaluates true cross-mission generalization. - ---- - -## File Format - -### Feature matrix (train_X.csv, test_X.csv) - -- **Rows**: Sample IDs -- **Columns**: Ensembl mouse gene IDs (e.g., `ENSMUSG00000021969`) -- **Values**: Log2(DESeq2 size-factor normalized counts + 1) -- **Gene selection**: Top 75th percentile variance, computed on **training missions only** (no test leakage) -- **Typical shape**: ~20,000 genes per sample - -### Labels (in GitHub repo) - -| Value | Meaning | -|-------|---------| -| `1` | Flight (spaceflight / microgravity) | -| `0` | Ground Control (vivarium / ground control) | - ---- - -## Results +`fold_OSD-397_test` is the stable public label for the third A6 eye fold. -### Multi-Method Evaluation (256 evaluations: 8 tissues x 8 methods x 4 feature types) +## Scope -Best AUROC per tissue across all method-feature combinations, evaluated via LOMO cross-validation (6 tissues) or 5-fold stratified CV (Lung, Colon -- single-mission datasets): - -| Tissue | Best AUROC | Method | Feature | perm_p | Significant | -|--------|-----------|--------|---------|--------|-------------| -| **Thymus** | **0.948** | PCA-LR | KEGG | 0.031 | Yes | -| **Colon** | **0.921** | PCA-LR | KEGG | 0.033 | Yes | -| **Lung** | **0.901** | PCA-LR | Gene | <0.05 | Yes | -| **Kidney** | **0.829** | ElasticNet-LR | Hallmark | <0.01 | Yes | -| **Eye** | **0.823** | PCA-LR | Hallmark | -- | - | -| **Skin** | **0.819** | PCA-LR | Gene | -- | - | -| **Gastrocnemius** | **0.776** | PCA-LR | Gene | -- | - | -| **Liver** | **0.670** | PCA-LR | Gene | -- | - | - -Significance: best-row permutation p where available. Across all 256 configurations, 40 individual evaluations are significant at p<0.05 and 6/8 tissues have at least one significant configuration. - -PCA-LR on gene features provides a strong baseline (8-tissue mean AUROC = 0.776). - -### Foundation / Language Model Snapshot - -| Model | Best Single-Tissue AUROC | vs PCA-LR Baseline (0.776) | -|-------|-----------|-------------------| -| scGPT | 0.666 (6-tissue mean, v1) | Below baseline | -| scFoundation | 0.635 (liver, p<0.01) | Below baseline | -| UCE | 0.632 (thymus, p=0.031) | Below baseline | -| Mouse-Geneformer | 0.476 (6-tissue mean, v1) | Below baseline | -| Text LLMs (GPT-4o, Claude, Llama 3) | 0.47-0.51 | Chance level | - -scGPT and Mouse-Geneformer report 6-tissue v1 means; scFoundation and UCE rows show best single-tissue v3 results. All of these model families still underperform the classical PCA-LR benchmark surface. - -### Negative Controls - -- Permutation control: AUROC = 0.50 +/- 0.03 (label shuffling) -- Housekeeping gene control: AUROC = 0.49-0.55 (non-informative features) -- Held-out validation: Thymus RR-23 (0.905), Skin RR-7 (0.885) - ---- +| Dimension | Coverage | +|---|---| +| Full v1-v7 benchmark surface | 8 tissues | +| Public source catalog | 24+ NASA OSDR accessions | +| Processed sample scope | 600+ binary/control samples across release layers | +| v4 multi-method evaluation | 8 tissues x 8 classifiers x 4 feature types = 256 evaluations | +| Public HF fold package | 4 reviewer-facing LOMO tasks plus selected result artifacts | -## Downloading +The full GitHub benchmark also includes historical v2-v7 extensions for +temporal dynamics, cross-species analysis, single-cell and spatial pilots, +foundation-model comparisons, graph/network baselines, and biological +interpretation layers. This HF repository is optimized for processed dataset +access; GitHub is the complete methods, code, and release-documentation surface. -### Option A: Python API (recommended) +## Download Example ```python from huggingface_hub import hf_hub_download import pandas as pd -# Download one fold's feature matrix -train_X = pd.read_csv( - hf_hub_download( - repo_id="jang1563/genelab-benchmark", - filename="A5_skin_lomo/fold_RR-7_test/train_X.csv", - repo_type="dataset", - ), - index_col=0 -) -print(train_X.shape) # (72, 20110) +repo_id = "jang1563/genelab-benchmark" +fold = "A5_skin_lomo/fold_RR-7_test" + +def hf_csv(name): + return pd.read_csv( + hf_hub_download( + repo_id=repo_id, + filename=f"{fold}/{name}", + repo_type="dataset", + ), + index_col=0, + ) + +train_X = hf_csv("train_X.csv") +train_y = hf_csv("train_y.csv").iloc[:, 0] +test_X = hf_csv("test_X.csv") +test_y = hf_csv("test_y.csv").iloc[:, 0] +test_meta = hf_csv("test_meta.csv") + +print(train_X.shape, train_y.shape, test_X.shape, test_y.shape) ``` -### Option B: Download full task +Download a complete task: ```python from huggingface_hub import snapshot_download -# Download all A5 skin files snapshot_download( repo_id="jang1563/genelab-benchmark", repo_type="dataset", allow_patterns="A5_skin_lomo/**", - local_dir="./data/benchmark", + local_dir="./data/genelab-benchmark", ) ``` ---- +## File Contract -## Evaluation +| File | Contract | +|---|---| +| Feature matrices | Rows are sample IDs; columns are Ensembl mouse gene IDs | +| Expression values | Log2(DESeq2 size-factor normalized counts + 1) | +| Gene selection | Top 75th percentile variance, computed on training missions only | +| Labels | Binary Flight/Ground labels | +| Metadata | Sample and fold provenance for auditability | -Evaluate predictions using the included script ([GitHub](https://github.com/jang1563/GeneLab_benchmark)): +The fold design prevents test-mission leakage by applying variance filtering +inside each training split. -```python -# Prepare submission JSON -submission = { - "task_id": "A5", - "model_name": "MyModel_v1", - "predictions": { - "fold_MHU-2_test": {"sample_id_1": 0.92, "...": "..."}, - "fold_RR-6_test": {"...": "..."}, - "fold_RR-7_test": {"...": "..."} - } -} +## Evaluation Summary -# Run evaluation (requires cloning GitHub repo for labels) -# python scripts/evaluate_submission.py --submission my_submission.json --task A5 -``` +The canonical result source is `docs/CANONICAL_RESULTS_V7_1.md` in the GitHub +repository. -### Model Tracks +| Result surface | Takeaway | +|---|---| +| Multi-method benchmark | PCA-LR is the strongest 8-tissue gene-level baseline in v4, with mean AUROC 0.776. | +| Best tissue rows | Thymus 0.948, colon 0.921, lung 0.901, kidney 0.829 across best method-feature combinations. | +| Cross-mission transfer | Thymus and gastrocnemius show the strongest mission-transfer signal; liver and kidney are harder. | +| Pathway features | Pathway representations rescue some weaker gene-level tissues, especially kidney and eye. | +| Foundation models | Tested gene-expression foundation models underperform tuned classical baselines on small-n bulk RNA-seq mission shift. | +| Held-out validation | Thymus RR-23 AUROC 0.905; skin RR-7 AUROC 0.885. | -| Track | Examples | Input Format | -|-------|---------|-------------| -| **Classical ML** | PCA-LR, ElasticNet-LR, RF, XGBoost, SVM-Linear, SVM-RBF, TabNet, LightGBM | Tabular (gene x sample) | -| **Foundation Models** | Geneformer, scGPT, UCE, scFoundation | Gene rank order / embeddings | -| **Text LLMs** | GPT-4o, Claude, Llama 3 | Natural language gene list | +## Intended Use ---- +Use this dataset to: -## Source Data - -All data derived from publicly available NASA OSDR datasets: - -| OSD ID | Tissue | Mission | n (Flight + Ground) | -|--------|--------|---------|-----------| -| OSD-48 | Liver | RR-1 | 18 | -| OSD-137 | Liver | RR-3 | 20 | -| OSD-245 | Liver | RR-6 | 48 | -| OSD-379 | Liver | RR-8 | 40 | -| OSD-242 | Liver | RR-9 | 39 | -| OSD-686 | Liver | MHU-2 | 28 | -| OSD-101 | Gastrocnemius | RR-1 | 12 | -| OSD-401 | Gastrocnemius | RR-5 | 12 | -| OSD-326 | Gastrocnemius | RR-9 | 8 | -| OSD-102 | Kidney | RR-1 | 47 | -| OSD-163 | Kidney | RR-3 | 32 | -| OSD-253 | Kidney | RR-7 | 39 | -| OSD-289 | Thymus | MHU-2 | 12 | -| OSD-244 | Thymus | RR-6 | 35 | -| OSD-421 | Thymus | RR-9 | 20 | -| OSD-238 | Skin (dorsal) | MHU-2 | 18 | -| OSD-239 | Skin (femoral) | MHU-2 | 17 | -| OSD-243 | Skin | RR-6 | 37 | -| OSD-254 | Skin | RR-7 | 30 | -| OSD-100 | Eye | RR-1 | 12 | -| OSD-194 | Eye | RR-3 | 9 | -| OSD-397 | Eye | OSD-397 | 16 | -| OSD-248 | Lung | RR-6 | 39 | -| OSD-247 | Colon | RR-6 | 36 | - -Lung and Colon additionally include Basal Control samples treated as ground control (+19 and +18 respectively). +- evaluate spaceflight transcriptomics classifiers under mission-held-out shift; +- compare classical ML, foundation-model, and adapter methods on fixed folds; +- test preprocessing or feature representations without changing test missions; +- reproduce public benchmark summaries from the GitHub repository. ---- +For full methods and release status, use the GitHub documentation and release +manifest. -## Preprocessing +## Release Labels -1. DESeq2 size-factor normalization (per mission) -2. Log2(counts + 1) transformation -3. Low-expression filter (>=20% samples with count > 1) -4. Top 75th percentile variance gene selection **per fold, training missions only** (prevents test leakage) -5. Pathway scores: gseapy ssGSEA (MSigDB Hallmark, KEGG gene sets) +| Surface | Public label | +|---|---| +| v7.1 GeneLab Benchmark | Canonical historical result surface and citation target | +| v7.1.2 public-card patch | Documentation, metadata, and evidence-visibility patch over v7.1 results | +| v8 extension | Incubating translational extension | +| v9 public bulk | Metadata-only public bulk alpha | +| v9 extension lanes | Diagnostic or draft lanes | ---- +This HF dataset card describes the v7.1 public fold package with the v7.1.2 +public-card patch. The v9 public bulk metadata-alpha card is maintained +separately in the GitHub repository as +`docs/v9_hf_dataset_card.md`. ## Citation -*(Manuscript in preparation)* +Please cite the software and benchmark using the GitHub `CITATION.cff` metadata. ```bibtex @dataset{kim2026genelab, - title = {GeneLab Benchmark: A Multi-Tissue Spaceflight Transcriptomics - Benchmark for AI/ML Models}, - author = {Kim, JangKeun}, - year = {2026}, - url = {https://huggingface.co/datasets/jang1563/genelab-benchmark}, - note = {v7.1.2 documentation, public-card, metadata, and evidence-visibility patch over canonical v7.1 results; data freeze 2026-03-01} + title = {GeneLab Benchmark: A Multi-Tissue Spaceflight Transcriptomics Benchmark for AI/ML Models}, + author = {Kim, JangKeun}, + year = {2026}, + url = {https://huggingface.co/datasets/jang1563/genelab-benchmark}, + note = {v7.1.2 documentation, public-card, metadata, and evidence-visibility patch over canonical v7.1 results; data freeze 2026-03-01} } ``` -Data source: NASA Open Science Data Repository (OSDR) -- https://osdr.nasa.gov/bio/repo/ - ---- +Source data: NASA Open Science Data Repository (OSDR), . ## License -- Dataset: CC-BY-4.0 -- Code: MIT ([GitHub repository](https://github.com/jang1563/GeneLab_benchmark)) -- Source data: NASA OSDR public data; see individual OSDR dataset licenses. +- Processed dataset package: CC-BY-4.0 +- Code: MIT, in the GitHub repository +- Source data: NASA OSDR public data; follow individual source-dataset terms diff --git a/docs/hf_dataset_card_longform_v7_1_archive_2026_06_15.md b/docs/hf_dataset_card_longform_v7_1_archive_2026_06_15.md new file mode 100644 index 0000000..f47d9c2 --- /dev/null +++ b/docs/hf_dataset_card_longform_v7_1_archive_2026_06_15.md @@ -0,0 +1,325 @@ +--- +license: cc-by-4.0 +task_categories: + - tabular-classification +tags: + - genomics + - transcriptomics + - spaceflight + - benchmarking + - nasa-osdr + - bulk-rna-seq + - mouse + - rna-seq + - foundation-model + - pathway-analysis + - single-cell + - spatial-transcriptomics +size_categories: + - 100M=20% samples with count > 1) +4. Top 75th percentile variance gene selection **per fold, training missions only** (prevents test leakage) +5. Pathway scores: gseapy ssGSEA (MSigDB Hallmark, KEGG gene sets) + +--- + +## Citation + +*(Manuscript in preparation)* + +```bibtex +@dataset{kim2026genelab, + title = {GeneLab Benchmark: A Multi-Tissue Spaceflight Transcriptomics + Benchmark for AI/ML Models}, + author = {Kim, JangKeun}, + year = {2026}, + url = {https://huggingface.co/datasets/jang1563/genelab-benchmark}, + note = {v7.0 with v7.1 documentation consistency patch; data freeze 2026-03-01} +} +``` + +Data source: NASA Open Science Data Repository (OSDR) -- https://osdr.nasa.gov/bio/repo/ + +--- + +## License + +- Dataset: CC-BY-4.0 +- Code: MIT ([GitHub repository](https://github.com/jang1563/GeneLab_benchmark)) +- Source data: NASA OSDR public data; see individual OSDR dataset licenses. diff --git a/release/release_manifest.json b/release/release_manifest.json new file mode 100644 index 0000000..e4c8fc7 --- /dev/null +++ b/release/release_manifest.json @@ -0,0 +1,226 @@ +{ + "$schema": "release_manifest.schema.json", + "schema_version": "0.1.0", + "generated_at": "2026-06-15", + "project": { + "name": "SpaceBio-Bench", + "legacy_name": "GeneLab Benchmark", + "repository": "https://github.com/jang1563/GeneLab_benchmark", + "huggingface_dataset": "https://huggingface.co/datasets/jang1563/genelab-benchmark", + "maintainer": "JangKeun Kim, Weill Cornell Medicine" + }, + "source_control": { + "git_commit_at_manifest_creation": "24c5594165d5f662435ae00a9b09476d1905f30b", + "working_tree_status": "main_based_public_polish_branch" + }, + "release_lanes": [ + { + "lane_id": "v7.1", + "public_label": "canonical historical result surface", + "status": "canonical_historical_result_surface", + "version": "7.1.2", + "date_released": "2026-06-05", + "summary": "The v1-v7 canonical benchmark result surface plus the v7.1.2 documentation, public-card, metadata, and evidence-visibility patch. The patch does not introduce new benchmark result generation.", + "canonical_surfaces": [ + "README.md", + "docs/CANONICAL_RESULTS_V7_1.md", + "docs/hf_dataset_card.md", + "docs/SPACEBIOBENCH_TRANSPARENCY_CARD_PACK.md", + "CITATION.cff" + ], + "huggingface": { + "repo_id": "jang1563/genelab-benchmark", + "dataset_card_path": "docs/hf_dataset_card.md", + "revision_status": "remote_current_v7_1_2_public_card_patch" + }, + "citation": { + "cff_path": "CITATION.cff", + "citation_version": "7.1.2", + "status": "software_citation_available" + } + }, + { + "lane_id": "v9_public_bulk", + "public_label": "metadata-only public bulk alpha", + "status": "metadata_only_public_bulk_alpha", + "summary": "The v9 public bulk lane has metadata indices, source inventory, checksum audit, baseline summaries, and a draft data package; it is not a frozen payload release.", + "canonical_surfaces": [ + "docs/v9_hf_dataset_card.md", + "docs/V9_PUBLIC_BULK_ALPHA_CARD_DATAPACKAGE_BOUNDARY_UPDATE.md", + "v9/README.md", + "v9/datapackage.draft.json", + "v9/task_manifest_index.csv", + "v9/task_data_index.csv", + "v9/source_inventory.csv", + "v9/source_checksum_audit.csv" + ], + "huggingface": { + "repo_id": "recommended_separate_v9_repo_when_distribution_expands", + "dataset_card_path": "docs/v9_hf_dataset_card.md", + "revision_status": "not_uploaded_as_frozen_release" + }, + "citation": { + "status": "alpha_not_cited_as_frozen_release" + } + } + ], + "artifacts": [ + { + "artifact_id": "root_readme", + "lane_id": "v7.1", + "role": "github_landing_page_current", + "path": "README.md", + "kind": "markdown", + "checksum_status": "not_recorded" + }, + { + "artifact_id": "v7_hf_dataset_card", + "lane_id": "v7.1", + "role": "huggingface_dataset_card_source", + "path": "docs/hf_dataset_card.md", + "kind": "markdown", + "checksum_status": "not_recorded" + }, + { + "artifact_id": "v7_hf_dataset_card_longform_archive", + "lane_id": "v7.1", + "role": "preserved_longform_huggingface_dataset_card_before_distribution_focused_rewrite", + "path": "docs/hf_dataset_card_longform_v7_1_archive_2026_06_15.md", + "kind": "markdown", + "checksum_status": "not_recorded" + }, + { + "artifact_id": "v7_citation", + "lane_id": "v7.1", + "role": "software_citation", + "path": "CITATION.cff", + "kind": "cff", + "checksum_status": "not_recorded" + }, + { + "artifact_id": "transparency_card_pack", + "lane_id": "v7.1", + "role": "public_scope_and_release_boundary_map", + "path": "docs/SPACEBIOBENCH_TRANSPARENCY_CARD_PACK.md", + "kind": "markdown", + "checksum_status": "not_recorded" + }, + { + "artifact_id": "v9_hf_dataset_card", + "lane_id": "v9_public_bulk", + "role": "v9_public_bulk_alpha_dataset_card_source", + "path": "docs/v9_hf_dataset_card.md", + "kind": "markdown", + "checksum_status": "not_recorded" + }, + { + "artifact_id": "v9_datapackage_draft", + "lane_id": "v9_public_bulk", + "role": "draft_frictionless_data_package", + "path": "v9/datapackage.draft.json", + "kind": "json", + "checksum_status": "not_recorded" + }, + { + "artifact_id": "v9_task_manifest_index", + "lane_id": "v9_public_bulk", + "role": "task_manifest_index", + "path": "v9/task_manifest_index.csv", + "kind": "csv", + "checksum_status": "not_recorded" + }, + { + "artifact_id": "premium_upgrade_plan", + "lane_id": "v9_public_bulk", + "role": "github_hf_public_surface_upgrade_plan", + "path": "docs/GITHUB_HF_PREMIUM_UPGRADE_PLAN_2026_06_15.md", + "kind": "markdown", + "checksum_status": "not_recorded" + }, + { + "artifact_id": "release_makefile", + "lane_id": "v7.1", + "role": "public_release_helper_targets", + "path": "Makefile", + "kind": "makefile", + "checksum_status": "not_recorded" + }, + { + "artifact_id": "hf_upload_workflow", + "lane_id": "v7.1", + "role": "huggingface_upload_plan_and_smoke_validation", + "path": "scripts/upload_to_hf.py", + "kind": "python", + "checksum_status": "not_recorded" + }, + { + "artifact_id": "readme_longform_v7_1_archive", + "lane_id": "v7.1", + "role": "preserved_longform_root_readme_before_premium_landing_page", + "path": "docs/README_LONGFORM_V7_1_ARCHIVE_2026_06_15.md", + "kind": "markdown", + "checksum_status": "not_recorded" + } + ], + "schemas": [ + { + "schema_id": "release_manifest", + "path": "release/release_manifest.schema.json", + "applies_to": "release/release_manifest.json" + } + ], + "quality_gates": [ + { + "gate_id": "release_manifest_local_path_check", + "status": "implemented", + "description": "Validate that local canonical surfaces, artifacts, and schema paths referenced by this manifest exist." + }, + { + "gate_id": "readme_hf_citation_consistency", + "status": "implemented", + "description": "Check that README, Hugging Face cards, and CITATION.cff report compatible version and status labels." + }, + { + "gate_id": "hf_dry_run_diff", + "status": "implemented", + "description": "Generate a Hugging Face upload diff before modifying the remote dataset repository." + }, + { + "gate_id": "hf_remote_smoke_download", + "status": "implemented", + "description": "After upload, download representative files from the HF repo and validate manifest paths." + }, + { + "gate_id": "status_language_lint", + "status": "planned", + "description": "Flag public text that labels alpha or draft lanes as frozen releases." + }, + { + "gate_id": "make_release_qa", + "status": "implemented", + "description": "Run public release manifest, public-docs consistency, HF upload workflow, and Python compile checks through Makefile." + }, + { + "gate_id": "hpc_public_release_gate", + "status": "implemented", + "description": "Run the public-surface slice of the HPC release gate with scripts/hpc_release_validate.sh --public-only." + } + ], + "open_decisions": [ + { + "decision_id": "public_name_transition", + "question": "Should the public project name shift from GeneLab Benchmark to SpaceBio-Bench while preserving the legacy name for v1-v7?", + "recommended_default": "Use SpaceBio-Bench as the forward-looking platform name and GeneLab Benchmark as the historical v1-v7 alias." + }, + { + "decision_id": "hf_repo_split", + "question": "Should v9 public bulk alpha share the existing Hugging Face dataset repo or move to a separate repo?", + "recommended_default": "Keep the existing repo for v7 public folds and create a separate v9 repo when public distribution expands beyond metadata indices." + }, + { + "decision_id": "python_package_identity", + "question": "Should this repository become an installable Python package with public CLI entrypoints?", + "recommended_default": "Add packaging only after the release manifest and validation commands stabilize." + } + ] +} diff --git a/release/release_manifest.schema.json b/release/release_manifest.schema.json new file mode 100644 index 0000000..09a6601 --- /dev/null +++ b/release/release_manifest.schema.json @@ -0,0 +1,237 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://genelab-benchmark.local/release/release_manifest.schema.json", + "title": "SpaceBio-Bench public release manifest", + "type": "object", + "required": [ + "schema_version", + "generated_at", + "project", + "source_control", + "release_lanes", + "artifacts", + "schemas", + "quality_gates", + "open_decisions" + ], + "properties": { + "schema_version": { + "type": "string" + }, + "generated_at": { + "type": "string" + }, + "project": { + "type": "object", + "required": [ + "name", + "legacy_name", + "repository", + "huggingface_dataset" + ], + "properties": { + "name": { + "type": "string" + }, + "legacy_name": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "huggingface_dataset": { + "type": "string" + } + }, + "additionalProperties": true + }, + "source_control": { + "type": "object", + "required": [ + "git_commit_at_manifest_creation", + "working_tree_status" + ], + "properties": { + "git_commit_at_manifest_creation": { + "type": "string" + }, + "working_tree_status": { + "type": "string" + } + }, + "additionalProperties": true + }, + "release_lanes": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": [ + "lane_id", + "public_label", + "status", + "summary", + "canonical_surfaces" + ], + "properties": { + "lane_id": { + "type": "string" + }, + "public_label": { + "type": "string" + }, + "status": { + "enum": [ + "canonical_historical_result_surface", + "incubating_translational_extension", + "metadata_only_public_bulk_alpha", + "diagnostic_or_draft_lane" + ] + }, + "version": { + "type": "string" + }, + "date_released": { + "type": "string" + }, + "summary": { + "type": "string" + }, + "canonical_surfaces": { + "type": "array", + "items": { + "type": "string" + } + }, + "huggingface": { + "type": "object", + "additionalProperties": true + }, + "citation": { + "type": "object", + "additionalProperties": true + } + }, + "additionalProperties": true + } + }, + "artifacts": { + "type": "array", + "items": { + "type": "object", + "required": [ + "artifact_id", + "lane_id", + "role", + "path", + "kind", + "checksum_status" + ], + "properties": { + "artifact_id": { + "type": "string" + }, + "lane_id": { + "type": "string" + }, + "role": { + "type": "string" + }, + "path": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "checksum_status": { + "enum": [ + "not_recorded", + "recorded", + "external", + "pending" + ] + }, + "sha256": { + "type": "string" + } + }, + "additionalProperties": true + } + }, + "schemas": { + "type": "array", + "items": { + "type": "object", + "required": [ + "schema_id", + "path", + "applies_to" + ], + "properties": { + "schema_id": { + "type": "string" + }, + "path": { + "type": "string" + }, + "applies_to": { + "type": "string" + } + }, + "additionalProperties": true + } + }, + "quality_gates": { + "type": "array", + "items": { + "type": "object", + "required": [ + "gate_id", + "status", + "description" + ], + "properties": { + "gate_id": { + "type": "string" + }, + "status": { + "enum": [ + "planned", + "implemented", + "partial", + "blocked" + ] + }, + "description": { + "type": "string" + } + }, + "additionalProperties": true + } + }, + "open_decisions": { + "type": "array", + "items": { + "type": "object", + "required": [ + "decision_id", + "question", + "recommended_default" + ], + "properties": { + "decision_id": { + "type": "string" + }, + "question": { + "type": "string" + }, + "recommended_default": { + "type": "string" + } + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true +} diff --git a/scripts/generate_hf_visual_assets.py b/scripts/generate_hf_visual_assets.py index ae2dc63..017787d 100644 --- a/scripts/generate_hf_visual_assets.py +++ b/scripts/generate_hf_visual_assets.py @@ -21,18 +21,18 @@ OUT_PATH = OUT_DIR / "hf_benchmark_summary.png" TISSUE_RESULTS = [ - ("Thymus", 0.948, True), - ("Colon", 0.921, True), - ("Lung", 0.901, True), - ("Kidney", 0.829, True), - ("Eye", 0.823, False), - ("Skin", 0.819, False), - ("Gastrocnemius", 0.776, False), - ("Liver", 0.670, False), + ("Thymus", 0.948, "PCA-LR / KEGG", True), + ("Colon", 0.921, "PCA-LR / KEGG", True), + ("Lung", 0.901, "PCA-LR / Gene", True), + ("Kidney", 0.829, "ElasticNet-LR / Hallmark", True), + ("Eye", 0.823, "PCA-LR / Hallmark", False), + ("Skin", 0.819, "PCA-LR / Gene", False), + ("Gastrocnemius", 0.776, "PCA-LR / Gene", False), + ("Liver", 0.670, "PCA-LR / Gene", False), ] METRICS = [ - ("4", "public GO tasks"), + ("4", "public LOMO tasks"), ("15", "fold packages"), ("8", "tissues"), ("600+", "processed samples"), @@ -70,7 +70,7 @@ def draw_metric_cards(ax): transform=ax.transAxes, ha="left", va="center", - fontsize=24, + fontsize=28, fontweight="bold", color="#14202e", ) @@ -81,51 +81,50 @@ def draw_metric_cards(ax): transform=ax.transAxes, ha="left", va="center", - fontsize=10.5, + fontsize=11.5, color="#465465", ) def draw_right_panel(ax): - add_card(ax, (0.635, 0.095), 0.31, 0.49, radius=0.02, fc="#ffffff") + add_card(ax, (0.635, 0.055), 0.31, 0.53, radius=0.02, fc="#ffffff") ax.text( 0.665, 0.535, - "Review-ready package boundary", + "Self-contained fold package", transform=ax.transAxes, - fontsize=15, + fontsize=17.8, fontweight="bold", color="#14202e", ) bullets = [ - ("Direct HF folds", "features, labels, metadata, fold_info, selected_genes"), - ("Leakage guard", "gene selection is recomputed on training missions only"), - ("Held-out validation", "RR-23 thymus AUROC 0.905; RR-7 skin AUROC 0.885"), - ("Viewer disabled", "high-dimensional matrices are accessed via downloads"), - ("Stable A6 path", "fold_OSD-397_test replaces stale fold_TBD_test"), + ("Direct HF folds", "X/y matrices, metadata, fold_info, selected genes"), + ("Mission-held-out split", "held-out mission is excluded from feature selection"), + ("Public task families", "A2 muscle, A4 thymus, A5 skin, A6 eye"), + ("A6 eye holdout", "fold_OSD-397_test is the public third A6 fold"), ] - y = 0.485 + y = 0.475 for title, body in bullets: ax.text( 0.665, y, title, transform=ax.transAxes, - fontsize=11.5, + fontsize=13.8, fontweight="bold", color="#245b73", ) ax.text( 0.665, - y - 0.033, + y - 0.04, body, transform=ax.transAxes, - fontsize=10.2, + fontsize=12.5, color="#465465", wrap=True, ) - y -= 0.078 + y -= 0.105 def draw_figure(): @@ -137,46 +136,47 @@ def draw_figure(): ax.text( 0.055, 0.93, - "GeneLab Spaceflight Transcriptomics Benchmark", + "SpaceBio-Bench", transform=ax.transAxes, - fontsize=25, + fontsize=34, fontweight="bold", color="#101820", ) ax.text( 0.055, 0.89, - "Cross-mission detection of spaceflight transcriptomic signatures from public NASA OSDR mouse RNA-seq.", + "Mission-held-out transcriptomics benchmark for public NASA OSDR mouse RNA-seq.", transform=ax.transAxes, - fontsize=13.5, + fontsize=15.5, color="#465465", ) ax.text( 0.055, 0.842, - "Version: v7.1.2 | Dataset freeze: 2026-03-01 | Maintainer/citation: JangKeun Kim, Weill Cornell Medicine", + "v7.1.2 public fold package | Dataset freeze: 2026-03-01", transform=ax.transAxes, - fontsize=10.8, + fontsize=12.8, color="#687789", ) ax.text( 0.055, 0.812, - "Public fold package: self-contained features, labels, metadata, and provenance", + "Former public name: GeneLab Benchmark | features, labels, metadata, fold_info, selected genes", transform=ax.transAxes, - fontsize=10.8, + fontsize=12.8, color="#687789", ) draw_metric_cards(ax) - add_card(ax, (0.055, 0.095), 0.54, 0.49, radius=0.02, fc="#ffffff") + # Bar chart card. + add_card(ax, (0.055, 0.055), 0.54, 0.53, radius=0.02, fc="#ffffff") ax.text( 0.085, 0.535, "Best AUROC by tissue", transform=ax.transAxes, - fontsize=15, + fontsize=17, fontweight="bold", color="#14202e", ) @@ -185,50 +185,33 @@ def draw_figure(): 0.505, "Best method-feature row by tissue; detailed methods are tabulated below.", transform=ax.transAxes, - fontsize=10.2, + fontsize=11.3, color="#687789", ) - bx = fig.add_axes([0.115, 0.175, 0.405, 0.285]) + bx = fig.add_axes([0.115, 0.125, 0.405, 0.335]) tissues = [row[0] for row in reversed(TISSUE_RESULTS)] aurocs = [row[1] for row in reversed(TISSUE_RESULTS)] - significant = [row[2] for row in reversed(TISSUE_RESULTS)] + significant = [row[3] for row in reversed(TISSUE_RESULTS)] colors = ["#1b7f8a" if sig else "#8ea0b5" for sig in significant] bx.barh(range(len(tissues)), aurocs, color=colors, height=0.62) bx.axvline(0.776, color="#d97b34", linewidth=1.7, linestyle="--") - bx.text(0.781, 7.48, "PCA-LR mean 0.776", color="#a65f24", fontsize=9.5) + bx.text(0.781, 7.48, "PCA-LR mean 0.776", color="#a65f24", fontsize=10.8) bx.set_xlim(0.55, 1.0) bx.set_yticks(range(len(tissues))) - bx.set_yticklabels(tissues, fontsize=10.5) - bx.set_xlabel("AUROC", fontsize=10.5, color="#465465") - bx.tick_params(axis="x", labelsize=9.5, colors="#465465") + bx.set_yticklabels(tissues, fontsize=11.7) + bx.set_xlabel("AUROC", fontsize=11.6, color="#465465") + bx.tick_params(axis="x", labelsize=10.8, colors="#465465") bx.tick_params(axis="y", length=0, colors="#14202e") bx.grid(axis="x", color="#e3e8ef", linewidth=1) bx.set_axisbelow(True) for spine in bx.spines.values(): spine.set_visible(False) for idx, value in enumerate(aurocs): - bx.text(value + 0.008, idx, f"{value:.3f}", va="center", fontsize=9.8, color="#14202e") + bx.text(value + 0.008, idx, f"{value:.3f}", va="center", fontsize=11.2, color="#14202e") draw_right_panel(ax) - ax.text( - 0.055, - 0.055, - "Classical ML remains the strongest baseline in this release; foundation-model and text-LLM snapshots are included as calibration points.", - transform=ax.transAxes, - fontsize=10.8, - color="#465465", - ) - ax.text( - 0.055, - 0.027, - "Source data: NASA Open Science Data Repository (OSDR). Dataset license: CC-BY-4.0.", - transform=ax.transAxes, - fontsize=9.8, - color="#687789", - ) - OUT_DIR.mkdir(parents=True, exist_ok=True) fig.savefig(OUT_PATH, facecolor=fig.get_facecolor(), bbox_inches="tight", pad_inches=0.08) plt.close(fig) diff --git a/scripts/hpc_release_validate.sh b/scripts/hpc_release_validate.sh index 03eb28d..984f442 100644 --- a/scripts/hpc_release_validate.sh +++ b/scripts/hpc_release_validate.sh @@ -11,10 +11,19 @@ if [[ "$PYTHON_BIN" == "python3" && -x "$REPO_ROOT/.hpc-venv/bin/python" ]]; the PYTHON_BIN="$REPO_ROOT/.hpc-venv/bin/python" fi +RUN_V8_SUMMARY=0 +RUN_PUBLIC_ONLY=0 + for arg in "$@"; do case "$arg" in + --v8-summary) + RUN_V8_SUMMARY=1 + ;; + --public-only) + RUN_PUBLIC_ONLY=1 + ;; -h|--help) - echo "Usage: bash scripts/hpc_release_validate.sh" + echo "Usage: bash scripts/hpc_release_validate.sh [--public-only] [--v8-summary]" exit 0 ;; *) @@ -24,21 +33,39 @@ for arg in "$@"; do esac done -echo "[1/3] Python regression tests" +echo "[1/6] Python regression tests" "$PYTHON_BIN" -m unittest discover -s tests -p 'test_review_fixes.py' -echo "[2/3] Whitespace/conflict-marker diff check" +echo "[2/6] Public release manifest and docs validation" +"$PYTHON_BIN" scripts/validate_release_manifest.py +"$PYTHON_BIN" scripts/validate_public_docs_consistency.py +"$PYTHON_BIN" -m unittest \ + tests/test_release_manifest.py \ + tests/test_public_docs_consistency.py \ + tests/test_hf_upload_workflow.py +"$PYTHON_BIN" -m py_compile \ + scripts/upload_to_hf.py \ + scripts/validate_release_manifest.py \ + scripts/validate_public_docs_consistency.py + +if [[ "$RUN_PUBLIC_ONLY" -eq 1 ]]; then + echo "Public release manifest and docs validation OK" + exit 0 +fi + +echo "[3/6] Whitespace/conflict-marker diff check" git diff --check git diff --cached --check -echo "[3/3] Release hygiene check" +echo "[4/6] Release hygiene check" "$PYTHON_BIN" - <<'PY' +import json from pathlib import Path import subprocess import sys repo = Path.cwd() -forbidden = (".claude/", "__pycache__/") +forbidden = (".claude/", "v8/bridge/geo_cache/", "__pycache__/") max_bytes = 50 * 1024 * 1024 tracked = subprocess.check_output(["git", "ls-files"], text=True).splitlines() @@ -52,6 +79,29 @@ for rel in tracked: elif path.stat().st_size > max_bytes: offenders.append(f"{rel} ({path.stat().st_size} bytes)") +prov_dir = repo / "v8" / "provenance" / "runs" +if prov_dir.exists(): + for manifest in sorted(prov_dir.glob("*.json")): + text = manifest.read_text() + rel = manifest.relative_to(repo) + if "TBD_HPC_RUN" in text: + offenders.append(f"{rel} contains TBD_HPC_RUN") + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + offenders.append(f"{rel} is invalid JSON: {exc}") + continue + if data.get("status") == "draft": + offenders.append(f"{rel} remains draft") + if data.get("status") in {"hpc_validated", "release_candidate", "frozen"}: + if not data.get("generated_at"): + offenders.append(f"{rel} missing generated_at") + validation = data.get("validation", {}) + if validation.get("local_tests_run") is not False: + offenders.append(f"{rel} should record local_tests_run=false") + if validation.get("hpc_tests_run") is not True: + offenders.append(f"{rel} missing hpc_tests_run=true") + if offenders: print("Release hygiene failures:") for item in offenders: @@ -59,3 +109,21 @@ if offenders: sys.exit(1) print("Release hygiene OK") PY + +echo "[5/6] v8 provenance and beta metadata validation" +if [[ -f scripts/validate_v8_provenance.py ]]; then + "$PYTHON_BIN" scripts/validate_v8_provenance.py +else + echo "Skipping v8 provenance validation; scripts/validate_v8_provenance.py not present" +fi + +if [[ "$RUN_V8_SUMMARY" -eq 1 ]]; then + if [[ -f v8/RESULTS_SUMMARY.py ]]; then + echo "[6/6] Regenerate v8 summary" + "$PYTHON_BIN" v8/RESULTS_SUMMARY.py + else + echo "[6/6] Skipping v8 summary regeneration; v8/RESULTS_SUMMARY.py not present" + fi +else + echo "[6/6] Skipping v8 summary regeneration; pass --v8-summary to run it" +fi diff --git a/scripts/upload_to_hf.py b/scripts/upload_to_hf.py index 84efa48..7abaa6b 100644 --- a/scripts/upload_to_hf.py +++ b/scripts/upload_to_hf.py @@ -1,35 +1,33 @@ #!/usr/bin/env python3 -""" -upload_to_hf.py — Upload GeneLab benchmark feature matrices to HuggingFace Dataset - -Uploads train_X.csv and test_X.csv for GO tasks to a HuggingFace Dataset repo. -Labels, metadata, and fold structure (train_y.csv, test_y.csv, fold_info.json, etc.) -are kept in the GitHub repo. Feature matrices are hosted on HuggingFace due to size. - -HuggingFace repo: jang1563/genelab-benchmark +"""Upload SpaceBio-Bench / GeneLab Benchmark public fold packages to HF. -Usage: - python scripts/upload_to_hf.py --task A5 --dry-run - python scripts/upload_to_hf.py --task all - python scripts/upload_to_hf.py --task A2 A4 A5 - python scripts/upload_to_hf.py --card-only # Upload dataset card (README.md) only +The script is designed as a release workflow, not only a file copier: -Authentication: - export HF_TOKEN="hf_..." - python scripts/upload_to_hf.py --task A5 +- dry-run upload plans work without `huggingface_hub` installed; +- release manifests are included in the HF package by default; +- optional remote diffs show create/replace actions against the HF repo; +- optional smoke validation downloads small uploaded files and checks SHA-256. """ +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json import os import sys -import argparse +import tempfile +from dataclasses import dataclass from pathlib import Path +from typing import Any + BASE_DIR = Path(__file__).resolve().parent.parent TASKS_DIR = BASE_DIR / "tasks" - HF_REPO_ID = "jang1563/genelab-benchmark" -# GO tasks only (A1/A3 NO-GO excluded from primary benchmark) +# GO tasks only (A1/A3 NO-GO excluded from the primary public fold package). GO_TASKS = [ "A2_gastrocnemius_lomo", "A4_thymus_lomo", @@ -37,83 +35,399 @@ "A6_eye_lomo", ] -UPLOAD_FILES = ["train_X.csv", "test_X.csv"] +FOLD_PACKAGE_FILES = [ + "train_X.csv", + "test_X.csv", + "train_y.csv", + "test_y.csv", + "train_meta.csv", + "test_meta.csv", + "fold_info.json", + "selected_genes.txt", +] +TASK_ROOT_FILES = ["task_info.json"] +STALE_REMOTE_DIRS = [ + "A6_eye_lomo/fold_TBD_test", +] CARD_SRC = BASE_DIR / "docs" / "hf_dataset_card.md" CARD_ASSETS = [ - (BASE_DIR / "docs" / "assets" / "hf_benchmark_summary.png", "assets/hf_benchmark_summary.png"), + BASE_DIR / "docs" / "assets" / "hf_benchmark_summary.png", ] +RELEASE_METADATA_FILES = [ + (BASE_DIR / "release" / "release_manifest.json", "manifest.json"), + ( + BASE_DIR / "release" / "release_manifest.schema.json", + "metadata/release_manifest.schema.json", + ), +] +SMOKE_SMALL_FILENAMES = { + "README.md", + "manifest.json", + "metadata/release_manifest.schema.json", + "task_info.json", + "fold_info.json", + "selected_genes.txt", +} -def upload_card(api, repo_id: str, dry_run: bool = False) -> None: - """Upload docs/hf_dataset_card.md as README.md to the HF dataset repo.""" - if not CARD_SRC.exists(): - print(f" [SKIP] Dataset card not found: {CARD_SRC}") - return - tag = "[DRY-RUN] " if dry_run else "" - print(f" {tag}Upload: README.md (dataset card, {CARD_SRC.stat().st_size // 1024} KB)") - if not dry_run: +@dataclass(frozen=True) +class UploadEntry: + local_path: Path + repo_path: str + kind: str + + @property + def size_bytes(self) -> int: + return self.local_path.stat().st_size + + @property + def sha256(self) -> str: + return sha256_file(self.local_path) + + def as_plan_row(self, remote_files: set[str] | None = None) -> dict[str, Any]: + action = "upload" + if remote_files is not None: + action = "replace" if self.repo_path in remote_files else "create" + return { + "action": action, + "kind": self.kind, + "repo_path": self.repo_path, + "local_path": self.local_path.relative_to(BASE_DIR).as_posix(), + "size_bytes": self.size_bytes, + "sha256": self.sha256, + } + + +def utc_now() -> str: + return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z") + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def size_label(size_bytes: int) -> str: + size_mb = size_bytes / 1e6 + if size_mb >= 1: + return f"{size_mb:.1f} MB" + return f"{size_bytes / 1024:.1f} KB" + + +def resolve_tasks(values: list[str]) -> list[str]: + if "all" in values: + return GO_TASKS.copy() + + tasks: list[str] = [] + seen: set[str] = set() + for value in values: + if value in GO_TASKS: + matched = [value] + else: + matched = [name for name in GO_TASKS if name.startswith(value)] + if not matched: + print(f"[WARN] Unknown task: {value}. Available: {GO_TASKS}") + continue + for task in matched: + if task not in seen: + tasks.append(task) + seen.add(task) + return tasks + + +def collect_task_entries(task_name: str) -> tuple[list[UploadEntry], list[str]]: + task_dir = TASKS_DIR / task_name + warnings: list[str] = [] + entries: list[UploadEntry] = [] + if not task_dir.exists(): + warnings.append(f"{task_name}: directory not found") + return entries, warnings + + for filename in TASK_ROOT_FILES: + path = task_dir / filename + if path.exists(): + entries.append(UploadEntry(path, f"{task_name}/{filename}", "task_file")) + else: + warnings.append(f"{path.relative_to(BASE_DIR)}: not found") + + for fold_dir in sorted(task_dir.glob("fold_*")): + for filename in FOLD_PACKAGE_FILES: + path = fold_dir / filename + if path.exists(): + entries.append( + UploadEntry( + path, + f"{task_name}/{fold_dir.name}/{filename}", + "task_file", + ) + ) + else: + warnings.append(f"{path.relative_to(BASE_DIR)}: not found") + + return entries, warnings + + +def collect_card_entries() -> tuple[list[UploadEntry], list[str]]: + entries: list[UploadEntry] = [] + warnings: list[str] = [] + if CARD_SRC.exists(): + entries.append(UploadEntry(CARD_SRC, "README.md", "dataset_card")) + else: + warnings.append(f"{CARD_SRC.relative_to(BASE_DIR)}: not found") + + for asset in CARD_ASSETS: + if asset.exists(): + entries.append( + UploadEntry(asset, f"assets/{asset.name}", "dataset_card_asset") + ) + else: + warnings.append(f"{asset.relative_to(BASE_DIR)}: not found") + return entries, warnings + + +def collect_release_metadata_entries() -> tuple[list[UploadEntry], list[str]]: + entries: list[UploadEntry] = [] + warnings: list[str] = [] + for local_path, repo_path in RELEASE_METADATA_FILES: + if local_path.exists(): + entries.append(UploadEntry(local_path, repo_path, "release_metadata")) + else: + warnings.append(f"{local_path.relative_to(BASE_DIR)}: not found") + return entries, warnings + + +def collect_upload_entries( + tasks: list[str], + include_card: bool, + include_release_metadata: bool, +) -> tuple[list[UploadEntry], list[str]]: + entries: list[UploadEntry] = [] + warnings: list[str] = [] + + for task in tasks: + task_entries, task_warnings = collect_task_entries(task) + entries.extend(task_entries) + warnings.extend(task_warnings) + + if include_card: + card_entries, card_warnings = collect_card_entries() + entries.extend(card_entries) + warnings.extend(card_warnings) + + if include_release_metadata: + metadata_entries, metadata_warnings = collect_release_metadata_entries() + entries.extend(metadata_entries) + warnings.extend(metadata_warnings) + + return entries, warnings + + +def import_hf_api(): + try: + from huggingface_hub import HfApi + except ImportError: + print("[ERROR] huggingface_hub not installed.") + print(" Install: pip install huggingface_hub") + sys.exit(1) + return HfApi + + +def get_token(required: bool) -> str | None: + token = os.environ.get("HF_TOKEN") + if not token: + try: + from huggingface_hub import get_token as hf_get_token + + token = hf_get_token() + except Exception: + token = None + if required and not token: + print("[ERROR] No Hugging Face token found.") + print(" Option 1: export HF_TOKEN='hf_...'") + print(" Option 2: huggingface-cli login") + sys.exit(1) + return token + + +def list_remote_files(repo_id: str, token: str | None) -> set[str]: + HfApi = import_hf_api() + api = HfApi(token=token) + try: + return set(api.list_repo_files(repo_id=repo_id, repo_type="dataset")) + except Exception as exc: + print(f"[ERROR] Failed to list HF repo files for {repo_id}: {exc}") + sys.exit(1) + + +def print_plan( + entries: list[UploadEntry], + repo_id: str, + stale_remote_dirs: list[str], + remote_files: set[str] | None, + dry_run: bool, +) -> None: + mode = "DRY-RUN" if dry_run else "UPLOAD" + total_size = sum(entry.size_bytes for entry in entries) + print(f"=== {mode} PLAN → {repo_id} ===") + print(f"Files: {len(entries)} | Total local size: {size_label(total_size)}") + if stale_remote_dirs: + print(f"Stale remote dirs to prune: {len(stale_remote_dirs)}") + print() + + by_kind: dict[str, int] = {} + for entry in entries: + by_kind[entry.kind] = by_kind.get(entry.kind, 0) + 1 + for kind, count in sorted(by_kind.items()): + print(f" {kind}: {count}") + print() + + for entry in entries: + row = entry.as_plan_row(remote_files=remote_files) + print( + f" [{row['action']}] {entry.repo_path} " + f"({entry.kind}, {size_label(entry.size_bytes)}, sha256={entry.sha256[:12]}...)" + ) + for remote_dir in stale_remote_dirs: + print(f" [delete] {remote_dir}/") + + +def write_upload_plan( + path: Path, + repo_id: str, + entries: list[UploadEntry], + stale_remote_dirs: list[str], + remote_files: set[str] | None, +) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "generated_at": utc_now(), + "repo_id": repo_id, + "repo_type": "dataset", + "files": [entry.as_plan_row(remote_files=remote_files) for entry in entries], + "stale_remote_dirs": stale_remote_dirs, + } + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + print(f"\nWrote upload plan: {path}") + + +def upload_task_folder(api: Any, task_name: str, repo_id: str) -> int: + task_entries, _ = collect_task_entries(task_name) + if not task_entries: + print(f" [SKIP] {task_name}: no package files found") + return 0 + + allow_patterns = TASK_ROOT_FILES + [f"fold_*/{name}" for name in FOLD_PACKAGE_FILES] + api.upload_folder( + folder_path=str(TASKS_DIR / task_name), + path_in_repo=task_name, + repo_id=repo_id, + repo_type="dataset", + allow_patterns=allow_patterns, + commit_message=f"Upload {task_name} self-contained fold package", + ) + return len(task_entries) + + +def upload_file_entries(api: Any, entries: list[UploadEntry], repo_id: str) -> int: + uploaded = 0 + for entry in entries: api.upload_file( - path_or_fileobj=str(CARD_SRC), - path_in_repo="README.md", + path_or_fileobj=str(entry.local_path), + path_in_repo=entry.repo_path, repo_id=repo_id, repo_type="dataset", + commit_message=f"Upload {entry.repo_path}", ) + uploaded += 1 + print(f" Uploaded: {entry.repo_path}") + return uploaded -def upload_card_assets(api, repo_id: str, dry_run: bool = False) -> int: - """Upload static assets referenced by the Hugging Face dataset card.""" - n_uploaded = 0 - for src, hf_path in CARD_ASSETS: - if not src.exists(): - print(f" [SKIP] Card asset not found: {src}") - continue - size_kb = src.stat().st_size // 1024 - tag = "[DRY-RUN] " if dry_run else "" - print(f" {tag}Upload: {hf_path} (card asset, {size_kb} KB)") - if not dry_run: - api.upload_file( - path_or_fileobj=str(src), - path_in_repo=hf_path, +def prune_stale_remote_dirs(api: Any, repo_id: str, stale_remote_dirs: list[str]) -> None: + for remote_dir in stale_remote_dirs: + print(f" Delete stale remote folder: {remote_dir}/") + try: + api.delete_folder( + path_in_repo=remote_dir, repo_id=repo_id, repo_type="dataset", + commit_message=f"Remove stale {remote_dir}", ) - n_uploaded += 1 - return n_uploaded + except Exception as exc: + print(f" [WARN] Could not delete {remote_dir}/; it may already be absent: {exc}") -def upload_task(api, repo_id: str, task_name: str, dry_run: bool = False) -> int: - """Upload all fold feature matrices for one task. Returns file count.""" - task_dir = TASKS_DIR / task_name - if not task_dir.exists(): - print(f" [SKIP] {task_name}: directory not found") - return 0 +def choose_smoke_entries(entries: list[UploadEntry], limit: int) -> list[UploadEntry]: + candidates = [ + entry + for entry in entries + if entry.size_bytes <= 1_000_000 + and ( + entry.repo_path in SMOKE_SMALL_FILENAMES + or Path(entry.repo_path).name in SMOKE_SMALL_FILENAMES + or entry.kind in {"dataset_card", "dataset_card_asset", "release_metadata"} + ) + ] + priority = { + "release_metadata": 0, + "dataset_card": 1, + "dataset_card_asset": 2, + "task_file": 3, + } + candidates.sort(key=lambda entry: (priority.get(entry.kind, 9), entry.repo_path)) + return candidates[:limit] - n_uploaded = 0 - for fold_dir in sorted(task_dir.glob("fold_*")): - for fname in UPLOAD_FILES: - fpath = fold_dir / fname - if not fpath.exists(): - print(f" [SKIP] {fpath.relative_to(BASE_DIR)}: not found (gitignored?)") - continue - size_mb = fpath.stat().st_size / 1e6 - hf_path = f"{task_name}/{fold_dir.name}/{fname}" - tag = "[DRY-RUN] " if dry_run else "" - print(f" {tag}Upload: {hf_path} ({size_mb:.0f} MB)") - if not dry_run: - api.upload_file( - path_or_fileobj=str(fpath), - path_in_repo=hf_path, + +def validate_remote_files( + repo_id: str, + token: str | None, + entries: list[UploadEntry], + limit: int, +) -> None: + try: + from huggingface_hub import hf_hub_download + except ImportError: + print("[ERROR] huggingface_hub not installed.") + print(" Install: pip install huggingface_hub") + sys.exit(1) + + smoke_entries = choose_smoke_entries(entries, limit=limit) + if not smoke_entries: + print("[WARN] No small files selected for remote smoke validation.") + return + + print(f"\nRemote smoke validation → {repo_id}") + print("-" * 50) + with tempfile.TemporaryDirectory(prefix="spacebiobench_hf_smoke_") as tmpdir: + for entry in smoke_entries: + downloaded = Path( + hf_hub_download( repo_id=repo_id, + filename=entry.repo_path, repo_type="dataset", + token=token, + local_dir=tmpdir, + force_download=True, ) - n_uploaded += 1 - - return n_uploaded + ) + actual = sha256_file(downloaded) + expected = entry.sha256 + if actual != expected: + print( + f"[FAIL] {entry.repo_path}: sha256 mismatch " + f"expected {expected}, got {actual}", + file=sys.stderr, + ) + sys.exit(1) + print(f" [OK] {entry.repo_path} ({size_label(entry.size_bytes)})") -def main(): +def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( - description="Upload GeneLab benchmark feature matrices to HuggingFace Dataset" + description="Upload SpaceBio-Bench public fold packages to Hugging Face" ) parser.add_argument( "--task", @@ -127,117 +441,168 @@ def main(): parser.add_argument( "--dry-run", action="store_true", - help="Print files that would be uploaded without actually uploading", + help="Print the upload plan without changing the remote repo", + ) + parser.add_argument( + "--remote-diff", + action="store_true", + help="List remote files and label planned uploads as create/replace", + ) + parser.add_argument( + "--write-upload-plan", + type=Path, + help="Write a JSON upload plan with paths, sizes, and SHA-256 digests", ) parser.add_argument( "--repo", default=HF_REPO_ID, - help=f"HuggingFace dataset repo ID (default: {HF_REPO_ID})", + help=f"Hugging Face dataset repo ID (default: {HF_REPO_ID})", + ) + parser.add_argument( + "--private", + action="store_true", + help="Create the HF dataset repo as private if it does not already exist", ) parser.add_argument( "--card-only", action="store_true", - help="Upload only the dataset card (README.md) to HuggingFace, skip feature matrices", + help="Upload only the dataset card and card assets", + ) + parser.add_argument( + "--manifest-only", + action="store_true", + help="Upload only release manifest metadata files", ) parser.add_argument( "--skip-card", action="store_true", - help="Skip dataset card upload (default: card is uploaded with --task all)", + help="Skip dataset card upload during a full package upload", + ) + parser.add_argument( + "--skip-manifest", + action="store_true", + help="Skip release manifest metadata during a full package upload", + ) + parser.add_argument( + "--skip-prune", + action="store_true", + help="Skip deletion of known stale remote folders such as A6 fold_TBD_test", ) - args = parser.parse_args() - repo_id = args.repo + parser.add_argument( + "--validate-remote", + action="store_true", + help=( + "Download small planned files from HF and compare SHA-256; intended " + "for post-upload or already-synced remote checks" + ), + ) + parser.add_argument( + "--smoke-limit", + type=int, + default=12, + help="Maximum number of files to download during --validate-remote", + ) + args = parser.parse_args(argv) - if args.dry_run: - print("=== DRY-RUN MODE (no files will be uploaded) ===\n") + if args.card_only and args.manifest_only: + print( + "[ERROR] --card-only and --manifest-only are mutually exclusive.", + file=sys.stderr, + ) + return 1 - # Card-only mode: upload README.md and exit - if args.card_only: - try: - from huggingface_hub import HfApi - except ImportError: - print("[ERROR] huggingface_hub not installed. Install: pip install huggingface_hub") - sys.exit(1) - token = os.environ.get("HF_TOKEN") - if not token and not args.dry_run: - try: - from huggingface_hub import get_token - token = get_token() - except Exception: - pass - if not token and not args.dry_run: - print("[ERROR] No HuggingFace token found. export HF_TOKEN='hf_...' or huggingface-cli login") - sys.exit(1) - api = HfApi(token=token) if not args.dry_run else None - print(f"\nUploading dataset card -> {repo_id}") - print("-" * 50) - upload_card(api, repo_id, dry_run=args.dry_run) - upload_card_assets(api, repo_id, dry_run=args.dry_run) - return + tasks = [] if args.card_only or args.manifest_only else resolve_tasks(args.task) + if not tasks and not args.card_only and not args.manifest_only: + print("No valid tasks specified. Exiting.") + return 1 - # Resolve task list - if "all" in args.task: - tasks = GO_TASKS - else: - # Allow short names like "A5" as well as full names - tasks = [] - for t in args.task: - if t in GO_TASKS: - tasks.append(t) - else: - # Try prefix match - matched = [name for name in GO_TASKS if name.startswith(t)] - if matched: - tasks.extend(matched) - else: - print(f"[WARN] Unknown task: {t}. Available: {GO_TASKS}") - if not tasks: - print("No valid tasks specified. Exiting.") - sys.exit(1) - - # HuggingFace API (lazy import so script is usable without huggingface_hub installed) - try: - from huggingface_hub import HfApi - except ImportError: - print("[ERROR] huggingface_hub not installed.") - print(" Install: pip install huggingface_hub") - sys.exit(1) + include_card = args.card_only or (not args.manifest_only and not args.skip_card) + include_release_metadata = args.manifest_only or ( + not args.card_only and not args.skip_manifest + ) + stale_remote_dirs = ( + [] + if args.skip_prune or args.card_only or args.manifest_only + else STALE_REMOTE_DIRS + ) - token = os.environ.get("HF_TOKEN") - if not token and not args.dry_run: - # Fall back to huggingface-cli cached login - try: - from huggingface_hub import get_token - token = get_token() - except Exception: - pass - if not token and not args.dry_run: - print("[ERROR] No HuggingFace token found.") - print(" Option 1: export HF_TOKEN='hf_...'") - print(" Option 2: huggingface-cli login") - sys.exit(1) + entries, warnings = collect_upload_entries( + tasks=tasks, + include_card=include_card, + include_release_metadata=include_release_metadata, + ) + for warning in warnings: + print(f"[WARN] {warning}") + if not entries: + print("No files selected. Exiting.") + return 1 + + token = None + remote_files = None + if args.remote_diff: + token = get_token(required=False) + remote_files = list_remote_files(args.repo, token=token) + + print_plan( + entries=entries, + repo_id=args.repo, + stale_remote_dirs=stale_remote_dirs, + remote_files=remote_files, + dry_run=args.dry_run, + ) + + if args.write_upload_plan: + write_upload_plan( + path=args.write_upload_plan, + repo_id=args.repo, + entries=entries, + stale_remote_dirs=stale_remote_dirs, + remote_files=remote_files, + ) - api = HfApi(token=token) if not args.dry_run else None - # Ensure repo exists (create if needed) if not args.dry_run: - api.create_repo(repo_id=repo_id, repo_type="dataset", private=True, exist_ok=True) + HfApi = import_hf_api() + token = token or get_token(required=True) + api = HfApi(token=token) + api.create_repo( + repo_id=args.repo, + repo_type="dataset", + private=args.private, + exist_ok=True, + ) - total = 0 - for task in tasks: - print(f"\nUploading {task} -> {repo_id}") - print("-" * 50) - n = upload_task(api, repo_id, task, dry_run=args.dry_run) - total += n - print(f" -> {n} files") + if stale_remote_dirs: + print(f"\nPruning stale remote paths → {args.repo}") + print("-" * 50) + prune_stale_remote_dirs(api, args.repo, stale_remote_dirs=stale_remote_dirs) - # Upload dataset card with full uploads (unless skipped) - if not args.skip_card: - print(f"\nUploading dataset card -> {repo_id}") - print("-" * 50) - upload_card(api, repo_id, dry_run=args.dry_run) - upload_card_assets(api, repo_id, dry_run=args.dry_run) + total_task_files = 0 + for task in tasks: + print(f"\nUploading {task} → {args.repo}") + print("-" * 50) + count = upload_task_folder(api, task, args.repo) + total_task_files += count + print(f" → {count} task files") + + non_task_entries = [entry for entry in entries if entry.kind != "task_file"] + if non_task_entries: + print(f"\nUploading metadata/card files → {args.repo}") + print("-" * 50) + upload_file_entries(api, non_task_entries, args.repo) + + print(f"\nUploaded {len(entries)} planned files ({total_task_files} task files).") + + if args.validate_remote: + token = token or get_token(required=False) + validate_remote_files( + repo_id=args.repo, + token=token, + entries=entries, + limit=args.smoke_limit, + ) - print(f"\nTotal: {total} feature files {'(dry-run)' if args.dry_run else 'uploaded'}.") + return 0 if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/scripts/validate_public_docs_consistency.py b/scripts/validate_public_docs_consistency.py new file mode 100644 index 0000000..a1a3bcc --- /dev/null +++ b/scripts/validate_public_docs_consistency.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Validate public README, Hugging Face card, and citation consistency.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parents[1] +README = REPO_ROOT / "README.md" +HF_CARD = REPO_ROOT / "docs" / "hf_dataset_card.md" +CITATION = REPO_ROOT / "CITATION.cff" +RELEASE_MANIFEST = REPO_ROOT / "release" / "release_manifest.json" + + +def load_json(path: Path) -> Any: + return json.loads(path.read_text()) + + +def parse_cff_scalar(text: str, key: str) -> str | None: + pattern = re.compile(rf"^{re.escape(key)}:\s*\"?([^\"\n]+)\"?\s*$", re.MULTILINE) + match = pattern.search(text) + return match.group(1).strip() if match else None + + +def parse_front_matter(text: str) -> dict[str, str]: + if not text.startswith("---\n"): + return {} + end = text.find("\n---\n", 4) + if end == -1: + return {} + values: dict[str, str] = {} + for line in text[4:end].splitlines(): + if not line or line.startswith(" ") or line.lstrip().startswith("-"): + continue + if ":" not in line: + continue + key, value = line.split(":", 1) + values[key.strip()] = value.strip().strip('"') + return values + + +def require_contains(errors: list[str], label: str, text: str, expected: str) -> None: + if expected not in text: + errors.append(f"{label}: missing expected text {expected!r}") + + +def require_absent(errors: list[str], label: str, text: str, forbidden: str) -> None: + if forbidden in text: + errors.append(f"{label}: stale or forbidden text present {forbidden!r}") + + +def validate_public_docs() -> list[str]: + errors: list[str] = [] + manifest = load_json(RELEASE_MANIFEST) + readme = README.read_text() + hf_card = HF_CARD.read_text() + citation = CITATION.read_text() + hf_front_matter = parse_front_matter(hf_card) + + lanes = {lane["lane_id"]: lane for lane in manifest["release_lanes"]} + v7 = lanes.get("v7.1", {}) + v9 = lanes.get("v9_public_bulk", {}) + + citation_version = v7.get("citation", {}).get("citation_version") + if citation_version: + actual_version = parse_cff_scalar(citation, "version") + if actual_version != citation_version: + errors.append( + f"CITATION.cff: version {actual_version!r} does not match " + f"release manifest citation_version {citation_version!r}" + ) + + require_contains(errors, "README.md", readme, "# SpaceBio-Bench") + require_contains(errors, "README.md", readme, "Former public name: **GeneLab Benchmark**") + require_contains(errors, "README.md", readme, "v7.1 GeneLab Benchmark") + require_contains(errors, "README.md", readme, "release/release_manifest.json") + require_contains(errors, "README.md", readme, "https://huggingface.co/datasets/jang1563/genelab-benchmark") + require_absent(errors, "README.md", readme, "Version: v7.0 (2026-04-12)") + require_absent(errors, "README.md", readme, "Status: **v1–v7 Complete**") + + expected_pretty_name = "SpaceBio-Bench / GeneLab Benchmark v7.1.2 Public Fold Package" + if hf_front_matter.get("pretty_name") != expected_pretty_name: + errors.append( + "docs/hf_dataset_card.md: pretty_name does not match expected " + f"{expected_pretty_name!r}" + ) + if hf_front_matter.get("license") != "cc-by-4.0": + errors.append("docs/hf_dataset_card.md: expected license cc-by-4.0") + if hf_front_matter.get("viewer") != "false": + errors.append("docs/hf_dataset_card.md: expected viewer: false") + + require_contains(errors, "README.md", readme, "v7.1.2 public-card/metadata/evidence-visibility") + require_contains(errors, "docs/hf_dataset_card.md", hf_card, "Public status: **v7.1.2 public-card/metadata/evidence-visibility patch") + require_contains(errors, "docs/hf_dataset_card.md", hf_card, "Dataset freeze: **2026-03-01**") + require_contains(errors, "docs/hf_dataset_card.md", hf_card, "repo_id = \"jang1563/genelab-benchmark\"") + require_contains(errors, "docs/hf_dataset_card.md", hf_card, "docs/v9_hf_dataset_card.md") + + if v7.get("public_label"): + require_contains(errors, "README.md", readme.lower(), v7["public_label"].lower()) + require_contains(errors, "docs/hf_dataset_card.md", hf_card.lower(), v7["public_label"].lower()) + if v9.get("public_label"): + require_contains(errors, "README.md", readme.lower(), v9["public_label"].lower()) + require_contains(errors, "docs/hf_dataset_card.md", hf_card.lower(), v9["public_label"].lower()) + + return errors + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Validate public README, HF card, and citation consistency" + ) + parser.parse_args(argv) + + errors = validate_public_docs() + if errors: + print(f"[FAIL] public docs consistency: {len(errors)} issue(s)", file=sys.stderr) + for error in errors: + print(f" - {error}", file=sys.stderr) + return 1 + + print("[OK] public docs consistency") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_release_manifest.py b/scripts/validate_release_manifest.py new file mode 100644 index 0000000..78f2fd4 --- /dev/null +++ b/scripts/validate_release_manifest.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Validate the public release manifest. + +This checker intentionally uses only the Python standard library so it can run +inside lightweight release gates and clean checkouts. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_MANIFEST = REPO_ROOT / "release" / "release_manifest.json" +HEX40 = re.compile(r"^[0-9a-f]{40}$") +HEX64 = re.compile(r"^[0-9a-f]{64}$") +LANE_STATUSES = { + "canonical_historical_result_surface", + "incubating_translational_extension", + "metadata_only_public_bulk_alpha", + "diagnostic_or_draft_lane", +} +CHECKSUM_STATUSES = {"not_recorded", "recorded", "external", "pending"} +GATE_STATUSES = {"planned", "implemented", "partial", "blocked"} + + +def load_json(path: Path) -> Any: + try: + return json.loads(path.read_text()) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid JSON: {exc}") from exc + + +def repo_path(value: str) -> Path | None: + if not value or value.startswith(("http://", "https://")): + return None + if "*" in value: + return None + return REPO_ROOT / value + + +def require_mapping(errors: list[str], where: str, value: Any) -> bool: + if not isinstance(value, dict): + errors.append(f"{where}: expected object, got {type(value).__name__}") + return False + return True + + +def require_list(errors: list[str], where: str, value: Any) -> bool: + if not isinstance(value, list): + errors.append(f"{where}: expected array, got {type(value).__name__}") + return False + return True + + +def require_fields( + errors: list[str], + where: str, + mapping: dict[str, Any], + fields: tuple[str, ...], +) -> None: + for field in fields: + if field not in mapping: + errors.append(f"{where}: missing required key {field}") + + +def check_existing_path(errors: list[str], where: str, value: Any) -> None: + if not isinstance(value, str): + errors.append(f"{where}: expected string path") + return + path = repo_path(value) + if path is None: + return + if not path.exists(): + errors.append(f"{where}: path does not exist: {value}") + + +def validate_release_manifest(data: Any) -> list[str]: + errors: list[str] = [] + if not require_mapping(errors, "manifest", data): + return errors + + require_fields( + errors, + "manifest", + data, + ( + "schema_version", + "generated_at", + "project", + "source_control", + "release_lanes", + "artifacts", + "schemas", + "quality_gates", + "open_decisions", + ), + ) + + if data.get("schema_version") != "0.1.0": + errors.append("manifest.schema_version: expected 0.1.0") + + project = data.get("project") + if require_mapping(errors, "project", project): + require_fields( + errors, + "project", + project, + ("name", "legacy_name", "repository", "huggingface_dataset"), + ) + repository = project.get("repository") + if not isinstance(repository, str) or not repository.startswith("https://github.com/"): + errors.append("project.repository: expected GitHub URL") + hf_dataset = project.get("huggingface_dataset") + if not isinstance(hf_dataset, str) or "huggingface.co/datasets/" not in hf_dataset: + errors.append("project.huggingface_dataset: expected Hugging Face dataset URL") + + source_control = data.get("source_control") + if require_mapping(errors, "source_control", source_control): + commit = source_control.get("git_commit_at_manifest_creation") + if not isinstance(commit, str) or not HEX40.match(commit): + errors.append("source_control.git_commit_at_manifest_creation: expected 40-char git SHA") + + lanes = data.get("release_lanes") + lane_ids: set[str] = set() + if require_list(errors, "release_lanes", lanes): + for index, lane in enumerate(lanes): + where = f"release_lanes[{index}]" + if not require_mapping(errors, where, lane): + continue + require_fields( + errors, + where, + lane, + ("lane_id", "public_label", "status", "summary", "canonical_surfaces"), + ) + lane_id = lane.get("lane_id") + if not isinstance(lane_id, str) or not lane_id: + errors.append(f"{where}.lane_id: expected non-empty string") + elif lane_id in lane_ids: + errors.append(f"{where}.lane_id: duplicate lane id {lane_id}") + else: + lane_ids.add(lane_id) + if lane.get("status") not in LANE_STATUSES: + errors.append(f"{where}.status: unsupported status {lane.get('status')!r}") + surfaces = lane.get("canonical_surfaces") + if require_list(errors, f"{where}.canonical_surfaces", surfaces): + for surface_index, surface in enumerate(surfaces): + check_existing_path( + errors, + f"{where}.canonical_surfaces[{surface_index}]", + surface, + ) + + artifacts = data.get("artifacts") + artifact_ids: set[str] = set() + if require_list(errors, "artifacts", artifacts): + for index, artifact in enumerate(artifacts): + where = f"artifacts[{index}]" + if not require_mapping(errors, where, artifact): + continue + require_fields( + errors, + where, + artifact, + ("artifact_id", "lane_id", "role", "path", "kind", "checksum_status"), + ) + artifact_id = artifact.get("artifact_id") + if not isinstance(artifact_id, str) or not artifact_id: + errors.append(f"{where}.artifact_id: expected non-empty string") + elif artifact_id in artifact_ids: + errors.append(f"{where}.artifact_id: duplicate artifact id {artifact_id}") + else: + artifact_ids.add(artifact_id) + lane_id = artifact.get("lane_id") + if isinstance(lane_id, str) and lane_ids and lane_id not in lane_ids: + errors.append(f"{where}.lane_id: unknown lane id {lane_id}") + if artifact.get("checksum_status") not in CHECKSUM_STATUSES: + errors.append( + f"{where}.checksum_status: unsupported value " + f"{artifact.get('checksum_status')!r}" + ) + check_existing_path(errors, f"{where}.path", artifact.get("path")) + sha256 = artifact.get("sha256") + if sha256 is not None and (not isinstance(sha256, str) or not HEX64.match(sha256)): + errors.append(f"{where}.sha256: expected 64-char hex digest") + + schemas = data.get("schemas") + if require_list(errors, "schemas", schemas): + for index, schema in enumerate(schemas): + where = f"schemas[{index}]" + if not require_mapping(errors, where, schema): + continue + require_fields(errors, where, schema, ("schema_id", "path", "applies_to")) + check_existing_path(errors, f"{where}.path", schema.get("path")) + + gates = data.get("quality_gates") + if require_list(errors, "quality_gates", gates): + for index, gate in enumerate(gates): + where = f"quality_gates[{index}]" + if not require_mapping(errors, where, gate): + continue + require_fields(errors, where, gate, ("gate_id", "status", "description")) + if gate.get("status") not in GATE_STATUSES: + errors.append(f"{where}.status: unsupported value {gate.get('status')!r}") + + decisions = data.get("open_decisions") + if require_list(errors, "open_decisions", decisions): + for index, decision in enumerate(decisions): + where = f"open_decisions[{index}]" + if not require_mapping(errors, where, decision): + continue + require_fields( + errors, + where, + decision, + ("decision_id", "question", "recommended_default"), + ) + + return errors + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Validate release/release_manifest.json") + parser.add_argument( + "--manifest", + type=Path, + default=DEFAULT_MANIFEST, + help=f"Release manifest path (default: {DEFAULT_MANIFEST.relative_to(REPO_ROOT)})", + ) + args = parser.parse_args(argv) + + try: + data = load_json(args.manifest) + except ValueError as exc: + print(f"[FAIL] {args.manifest}: {exc}", file=sys.stderr) + return 1 + + errors = validate_release_manifest(data) + if errors: + print(f"[FAIL] {args.manifest}: {len(errors)} issue(s)", file=sys.stderr) + for error in errors: + print(f" - {error}", file=sys.stderr) + return 1 + + print(f"[OK] {args.manifest}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_hf_upload_workflow.py b/tests/test_hf_upload_workflow.py new file mode 100644 index 0000000..b6ba590 --- /dev/null +++ b/tests/test_hf_upload_workflow.py @@ -0,0 +1,70 @@ +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +UPLOAD_SCRIPT = REPO_ROOT / "scripts" / "upload_to_hf.py" + + +class HuggingFaceUploadWorkflowTests(unittest.TestCase): + def run_upload_plan(self, *args: str) -> tuple[subprocess.CompletedProcess[str], dict]: + with tempfile.TemporaryDirectory() as tmpdir: + plan_path = Path(tmpdir) / "upload_plan.json" + proc = subprocess.run( + [ + sys.executable, + str(UPLOAD_SCRIPT), + *args, + "--dry-run", + "--write-upload-plan", + str(plan_path), + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + payload = json.loads(plan_path.read_text()) if plan_path.exists() else {} + return proc, payload + + def test_card_only_dry_run_plan_has_card_files(self): + proc, payload = self.run_upload_plan("--card-only") + + self.assertEqual(proc.returncode, 0, proc.stderr) + repo_paths = {row["repo_path"] for row in payload["files"]} + self.assertIn("README.md", repo_paths) + self.assertIn("assets/hf_benchmark_summary.png", repo_paths) + self.assertNotIn("manifest.json", repo_paths) + + def test_manifest_only_dry_run_plan_has_release_metadata(self): + proc, payload = self.run_upload_plan("--manifest-only") + + self.assertEqual(proc.returncode, 0, proc.stderr) + repo_paths = {row["repo_path"] for row in payload["files"]} + self.assertEqual( + repo_paths, + {"manifest.json", "metadata/release_manifest.schema.json"}, + ) + + def test_task_dry_run_plan_includes_task_card_and_manifest(self): + proc, payload = self.run_upload_plan("--task", "A5", "--skip-prune") + + self.assertEqual(proc.returncode, 0, proc.stderr) + repo_paths = {row["repo_path"] for row in payload["files"]} + self.assertIn("A5_skin_lomo/task_info.json", repo_paths) + self.assertIn("A5_skin_lomo/fold_RR-7_test/fold_info.json", repo_paths) + self.assertIn("README.md", repo_paths) + self.assertIn("manifest.json", repo_paths) + + first = payload["files"][0] + self.assertIn("sha256", first) + self.assertIn("size_bytes", first) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_public_docs_consistency.py b/tests/test_public_docs_consistency.py new file mode 100644 index 0000000..6d34d36 --- /dev/null +++ b/tests/test_public_docs_consistency.py @@ -0,0 +1,29 @@ +import subprocess +import sys +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +class PublicDocsConsistencyTests(unittest.TestCase): + def test_public_docs_consistency_script_passes(self): + proc = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "validate_public_docs_consistency.py"), + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn("[OK]", proc.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_manifest.py b/tests/test_release_manifest.py new file mode 100644 index 0000000..7a15d18 --- /dev/null +++ b/tests/test_release_manifest.py @@ -0,0 +1,56 @@ +import json +import subprocess +import sys +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +class ReleaseManifestTests(unittest.TestCase): + def test_release_manifest_validator_passes_current_manifest(self): + proc = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "validate_release_manifest.py"), + "--manifest", + str(REPO_ROOT / "release" / "release_manifest.json"), + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn("[OK]", proc.stdout) + + def test_release_manifest_declares_public_release_lanes(self): + manifest = json.loads((REPO_ROOT / "release" / "release_manifest.json").read_text()) + lanes = {lane["lane_id"]: lane for lane in manifest["release_lanes"]} + + self.assertEqual( + lanes["v7.1"]["status"], + "canonical_historical_result_surface", + ) + self.assertEqual( + lanes["v9_public_bulk"]["status"], + "metadata_only_public_bulk_alpha", + ) + self.assertIn("docs/hf_dataset_card.md", lanes["v7.1"]["canonical_surfaces"]) + self.assertIn("docs/v9_hf_dataset_card.md", lanes["v9_public_bulk"]["canonical_surfaces"]) + + def test_release_manifest_artifacts_reference_known_lanes(self): + manifest = json.loads((REPO_ROOT / "release" / "release_manifest.json").read_text()) + lane_ids = {lane["lane_id"] for lane in manifest["release_lanes"]} + + for artifact in manifest["artifacts"]: + self.assertIn(artifact["lane_id"], lane_ids) + artifact_path = REPO_ROOT / artifact["path"] + self.assertTrue(artifact_path.exists(), artifact["path"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_review_fixes.py b/tests/test_review_fixes.py index cec7904..19bcb5b 100644 --- a/tests/test_review_fixes.py +++ b/tests/test_review_fixes.py @@ -14,6 +14,14 @@ class ReviewFixTests(unittest.TestCase): def read_repo_text(self, relative_path: str) -> str: return (REPO_ROOT / relative_path).read_text() + def skip_unless_paths_exist(self, *relative_paths: str) -> None: + missing = [rel for rel in relative_paths if not (REPO_ROOT / rel).exists()] + if missing: + self.skipTest( + "branch-specific files are not present in this checkout: " + + ", ".join(missing) + ) + def load_module(self, dotted_path: str): return importlib.import_module(dotted_path) @@ -159,7 +167,8 @@ def test_root_eye_scripts_use_stable_public_label(self): self.assertIn('list(mission = "MHU-1", dir = "MHU-2", glds = "GLDS-289", osd = "OSD-289")', run_fgsea) self.assertIn('list(mission = "OSD-397", dir = "TBD", glds = "GLDS-397", osd = "OSD-397")', run_fgsea) self.assertIn('"A6|A6_eye_lomo|RR-1 RR-3 OSD-397|3"', hpc_submit) - self.assertIn("| Eye | OSD-100, 194, 397 | RR-1, RR-3, OSD-397 | 37 |", root_readme) + self.assertIn("v7.1 GeneLab Benchmark", root_readme) + self.assertIn("docs/hf_dataset_card.md", root_readme) def test_eye_docs_use_stable_osd397_label(self): hf_card = self.read_repo_text("docs/hf_dataset_card.md") @@ -258,10 +267,10 @@ def test_fm_docs_use_current_scgpt_and_model_count_language(self): outline = self.read_repo_text("docs/PAPER_OUTLINE.md") results_summary = self.read_repo_text("evaluation/RESULTS_SUMMARY.md") - self.assertIn("4 gene-expression foundation models + 3 text LLMs evaluated", readme) - self.assertIn("| scGPT | 12L Transformer, 33M human cells | 0.666 | -0.093 |", readme) - self.assertIn("Foundation / Language Models | 4 gene-expression FMs + 3 text LLMs", hf_card) - self.assertIn("scGPT | 0.666 (6-tissue mean, v1)", hf_card) + self.assertIn("4 gene-expression foundation models, 3 text LLMs", readme) + self.assertIn("Tested gene-expression foundation models underperform", readme) + self.assertIn("foundation-model comparisons", hf_card) + self.assertIn("Tested gene-expression foundation models underperform", hf_card) self.assertIn("Classical ML wins 4/6 tissues vs scGPT and 6/6 tissues vs Geneformer.", paper) self.assertIn("| scGPT vs Baseline mean delta | -0.093 | 4/6 Baseline wins |", paper) self.assertIn("https://github.com/jang1563/GeneLab_benchmark", paper) @@ -276,7 +285,7 @@ def test_submission_docs_match_current_v1_task_surface_and_thresholds(self): readme = self.read_repo_text("README.md") submission = self.read_repo_text("docs/submission_format.md") - self.assertIn("| 95% CI lower | Bootstrap CI (N=2000) lower bound | > 0.600 |", readme) + self.assertIn("AUROC, bootstrap confidence interval, permutation p-value", readme) self.assertIn("| **Go/No-Go** | AUROC > 0.700 AND CI lower > 0.600 |", submission) self.assertIn("| `A6` | Eye | 3 | 37 | ~21k genes (log2 normalized) |", submission) self.assertIn("| `B6` | Eye | RR-1, RR-3, OSD-397 | 6 | `tasks/B6_eye_cross_mission/` |", submission) @@ -287,9 +296,9 @@ def test_submission_docs_match_current_v1_task_surface_and_thresholds(self): def test_root_readme_repository_map_avoids_stale_hard_coded_counts(self): readme = self.read_repo_text("README.md") - self.assertIn("tasks/ <- Public task inputs (benchmark + sensitivity tasks)", readme) - self.assertIn("v2 analysis and runner scripts", readme) - self.assertIn("v4 result JSONs + SHAP/WGCNA outputs", readme) + self.assertIn("tasks/ Public v1 LOMO task inputs and selected fold packages", readme) + self.assertIn("v2/ ... v7/ Completed historical benchmark layers", readme) + self.assertIn("release/ Machine-readable public release manifest", readme) self.assertNotIn("Public task inputs (17 directories)", readme) self.assertNotIn("v1 pipeline scripts (35 Python/R/shell)", readme) self.assertNotIn("19 Python scripts", readme) @@ -298,14 +307,10 @@ def test_root_readme_repository_map_avoids_stale_hard_coded_counts(self): def test_root_readme_includes_v7_in_status_structure_and_changelog(self): readme = self.read_repo_text("README.md") - self.assertIn("Version: v7.1.2 public-card/metadata/evidence-visibility patch (2026-06-05) | Canonical results: v7.1 | Dataset freeze: 2026-03-01", readme) - self.assertIn("Status: **v1-v7 Complete; public-review card pack ready**", readme) - self.assertIn("## Reviewer Fast Path", readme) - self.assertIn("docs/SPACEBIOBENCH_SYSTEM_CARD.md", readme) - self.assertIn("docs/SPACEBIOBENCH_CLAIM_REGISTER.md", readme) - self.assertIn("| **v7.0** | Unified/foundation-model benchmarking: scPRINT2, GNN/WGCNA graph baselines, cross-method synthesis, and signal hierarchy analysis | **Complete** | `v7/` |", readme) - self.assertIn("├── v7/ <- Unified/foundation-model benchmarking", readme) - self.assertIn("| v7.0 | 2026-04-12 | Unified benchmark layer complete:", readme) + self.assertIn("| v7.1 GeneLab Benchmark | Canonical historical result surface |", readme) + self.assertIn("v2/ ... v7/ Completed historical benchmark layers", readme) + self.assertIn("Former public name: **GeneLab Benchmark**", readme) + self.assertIn("release/release_manifest.json", readme) self.assertNotIn("v7 Graph Neural Networks In Progress", readme) def test_public_release_metadata_uses_v7_consistently(self): @@ -313,83 +318,22 @@ def test_public_release_metadata_uses_v7_consistently(self): hf_card = self.read_repo_text("docs/hf_dataset_card.md") citation = self.read_repo_text("CITATION.cff") - self.assertIn("Version: v7.1.2 public-card/metadata/evidence-visibility patch (2026-06-05) | Canonical results: v7.1 | Dataset freeze: 2026-03-01", readme) - self.assertIn("Canonical v7.1 documentation source:", readme) - self.assertIn("note = {v7.1.2 documentation, public-card, metadata, and evidence-visibility patch over canonical v7.1 results; data freeze 2026-03-01}", readme) - self.assertIn("Version: v7.1.2 public-card/metadata/evidence-visibility patch | Canonical results: v7.1 | Dataset freeze: 2026-03-01", hf_card) - self.assertIn("note = {v7.1.2 documentation, public-card, metadata, and evidence-visibility patch over canonical v7.1 results; data freeze 2026-03-01}", hf_card) - self.assertIn(" - 100M