diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc1b9dc..1dae6a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,8 +36,8 @@ jobs: version: "${{ matrix.NXF_VER }}" - name: Run pipeline with test data - # TODO nf-core: You can customise CI pipeline run tests as required - # For example: adding multiple test runs with different parameters - # Remember that you can parallelise this by using strategy.matrix + # The bundled assets/stub/ inputs are truncated placeholders, not real data, so this + # is a DAG/wiring check via each module's stub block rather than a data test. + # -entry OFFTARGET and -entry TRAIN are covered separately by offtarget-ci.yml. run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results + nextflow run ${GITHUB_WORKSPACE} -profile test,docker -stub-run --outdir ./results diff --git a/.github/workflows/offtarget-ci.yml b/.github/workflows/offtarget-ci.yml new file mode 100644 index 0000000..60d6eca --- /dev/null +++ b/.github/workflows/offtarget-ci.yml @@ -0,0 +1,148 @@ +name: offtarget CI +# Exercises the -entry OFFTARGET workflow that the default nf-core CI does not cover: +# 1. stub run — DAG wiring for all three modes (paired / wgs_only / ecs_only) +# 2. glue tests — the ECS<->WGS join logic (tests/test_offtarget_glue.py) +# 3. nf-test — REAL process bodies in the real container, on KB-sized fixtures +# +# (1) and (3) are complementary and neither replaces the other: a stub run only `touch`es +# outputs, so it proves the DAG connects but executes no line of bin/*.py; nf-test runs the +# actual command, so it catches a module passing a flag the script does not accept. +on: + push: + branches: + - dev + - "feat/offtarget-**" + pull_request: + paths: + - "workflows/offtarget.nf" + - "subworkflows/local/**" + - "modules/local/**" + - "bin/**" + - "tests/**" + - "conf/stub.config" + - "nf-test.config" + - "assets/offtarget_samplesheet_template.csv" + - ".github/workflows/offtarget-ci.yml" + +env: + NXF_ANSI_LOG: false + +concurrency: + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" + cancel-in-progress: true + +jobs: + stub: + name: OFFTARGET stub run (DAG wiring, all modes) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Nextflow + uses: nf-core/setup-nextflow@v1 + with: + # pin a stable release: the repo config's check_max() doesn't parse under the + # 26.x edge that "latest-everything" now resolves to. + version: "25.10.4" + + # stub blocks only `touch` their outputs, so no container/data is needed — this is a + # pure wiring check. Derive single-arm sheets from the template to hit every mode. + - name: Build per-mode samplesheets + run: | + SS=assets/offtarget_samplesheet_template.csv + head -1 "$SS" | tee ss_wgs.csv > ss_ecs.csv + grep -iE ',wgs,' "$SS" >> ss_wgs.csv + grep -iE ',ecs,' "$SS" >> ss_ecs.csv + + - name: Stub run — paired + run: nextflow run . -entry OFFTARGET -profile stub -stub-run --input "$SS" --outdir out_paired + env: + SS: assets/offtarget_samplesheet_template.csv + - name: Stub run — wgs_only + run: nextflow run . -entry OFFTARGET -profile stub -stub-run --input ss_wgs.csv --outdir out_wgs + - name: Stub run — ecs_only + run: nextflow run . -entry OFFTARGET -profile stub -stub-run --input ss_ecs.csv --outdir out_ecs + # ecs rows with a spacer but no target_file auto-generate the hotspot VCF (once per guide) + - name: Stub run — ecs target_file auto-wire from spacer + run: | + nextflow run . -entry OFFTARGET -profile stub -stub-run \ + --input assets/stub/offtarget_autowire.csv --fasta assets/stub/tiny.fa --outdir out_autowire + + hotspots-stub: + name: HOTSPOTS stub run (gRNA -> targets DAG wiring) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Nextflow + uses: nf-core/setup-nextflow@v1 + with: + version: "25.10.4" + + # stub blocks only `touch`/`printf` their outputs, so no container/genome is needed. + # Point --fasta at the tiny committed fixture (the real /storage2 reference is absent on CI). + - name: Stub run — Cas-OFFinder only (default) + run: | + nextflow run . -entry HOTSPOTS -profile stub -stub-run \ + --input assets/grna_samplesheet_template.csv \ + --fasta assets/stub/tiny.fa --outdir out_hotspots + - name: Stub run — with CRISPRme enabled + run: | + mkdir -p crisprme_idx + nextflow run . -entry HOTSPOTS -profile stub -stub-run \ + --input assets/grna_samplesheet_template.csv \ + --fasta assets/stub/tiny.fa --run_crisprme --crisprme_index_dir crisprme_idx \ + --outdir out_hotspots_cme + + glue-tests: + name: OFFTARGET glue-logic + combiner unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + # the glue scripts are pure-pandas; a clean checkout has no repo-root vendored dirs to shadow. + # scikit-learn is needed by bin/offtarget_metrics.py (PR-AUC / ROC-AUC); the + # off-target container pins 1.8.0, but here any recent version is fine since + # nothing unpickles a model in these tests. + - name: Install test deps + run: python -m pip install --quiet pandas matplotlib pytest openpyxl scikit-learn + + - name: Run glue + combiner tests + run: python -m pytest -q tests/test_offtarget_glue.py tests/test_combine_offtarget.py + + nf-test: + name: nf-test (real process bodies, real container) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Nextflow + uses: nf-core/setup-nextflow@v1 + with: + version: "25.10.4" + + # Installed straight from the vendor rather than via an action, so this step does + # not depend on a third-party action's input schema. The trailing argument pins + # the version; drop it to track latest (not recommended for a reproducible CI). + - name: Install nf-test + run: | + curl -fsSL https://code.askimed.com/install/nf-test | bash -s 0.9.5 + sudo mv nf-test /usr/local/bin/ + nf-test version + + # ghcr.io/dhslab/docker-scge-offtarget is publicly readable, so no login is needed. + # Pull it up front: a first-use pull inside a process makes a timeout look like a + # test failure, and the image is shared by every test here. + - name: Pre-pull the off-target container + run: docker pull ghcr.io/dhslab/docker-scge-offtarget:260710 + + # Fixtures are ~6 KB (tests/fixtures/), so this is a couple of minutes on a 2-CPU + # runner. `stub` supplies the params profile — it nulls the /storage2 DRAGEN paths + # that workflows/scge.nf validates at parse time — while `docker` supplies the + # engine. Neither implies -stub-run; the process bodies really execute. + - name: Run nf-test + run: nf-test test --profile stub,docker --verbose diff --git a/.gitignore b/.gitignore index 5533f03..c3a7f37 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,112 @@ +# ============================================================================= +# nf-core-scge — .gitignore +# Pattern-based. Root-anchored (leading /) entries are one-off local files that +# are intentionally kept out of git; delete or commit them deliberately. +# ============================================================================= + +# ---- Agent / tooling scratch ---- +# Claude Code keeps per-repo settings AND full worktree checkouts under .claude/ +# (each with its own .nextflow logs and work dirs) — hundreds of MB that must never +# be swept into a commit by `git add -A`. +.claude/ + +# ---- nf-test ---- +# nf-test writes a per-test scratch .nf file into the REPO ROOT (one per test id) and +# keeps its work dir in .nf-test/. Both are build artifacts of a test run, and there +# are ~100 of the scratch files after a full suite. +.nf-test/ +.nf-test-*.nf +.nf-test.log + +# ---- Nextflow runtime ---- .nextflow* work/ -data/ results/ -.DS_Store +results*/ +null/ +nxf-tmp.* + +# ---- Data & large / generated inputs ---- +data/ +logs/ testing/ testing* +mastersheets/ +test_output/ +test_results* +test_logs/ +test_mastersheets/ +test_null/ + +# ---- Python (vendored envs, caches, build metadata) ---- +__pycache__/ *.pyc -.dir_bash_history +*.dist-info/ +*.libs/ +.pytest_cache/ +# local test venv/cache created by run_offtarget_tests.sh +.offtarget_testenv*/ +.offtarget_testcache/ + +# ---- Scheduler / process logs ---- +*.log +*.err + +# ---- OS / editor ---- +.DS_Store +/.dir_bash_history + +# ---- Local-only files: not part of the pipeline ---- +/CLAUDE.md +/REPO_INVENTORY.md +/DEBUGGING_REPORT.md +/to_commit.md +/pipeline_explanation.txt +/Groovy vs. Shell in Nextflow.txt +/minimal.config +/flowchart.dot +/extracted_indels.json +/versions.yml +/build.sh +/nextflow +/-e +/s +/err.txt +/AAVS1site14_WTcas9-DNA.scge_report_ml.html +/AAVS1site14_WTcas9-DNA.scge_report_og.html +/analysis_samples.csv +/mastersheet.*.csv +/offtarget_samplesheet_aavs1.csv +/mock_dragen_dir/ +/aavs1_ml_testing/ +/training_data/ +/run.sh +/run_all.sh +/run_test.sh +/run_offtarget_aavs1.sh +/test_syntax.nf +/test_versions.nf + +# ---- Legacy CRISPR-ML scripts / models (deprecated; see REPO_INVENTORY.md) ---- +bin/*.bak +bin/build_from_curated_tsv.py +bin/build_training_from_tsv.py +bin/convert_targets.pl +bin/train_aavs1_v3.sh +bin/train_aavs1_v3_lsf.sh +bin/tsv_to_vcf.py +assets/NO_*.csv +assets/NO_*.tsv +assets/empty*.txt +assets/all_samples_mastersheet.csv +assets/ecs_samples_mastersheet.csv +assets/models/crispr_model_*.pkl +assets/models/crispr_model_*_cv_summary.csv +assets/stub/AAVS1_site14.targets.csv +assets/stub/aavs1site14_samplesheet.csv + +# ---- nf-core plugin cache ---- +subworkflows/nf-core/utils_nfschema_plugin/ + +# ---- Large committed-doc screenshots ---- +docs/images/Screenshot*.png diff --git a/README.md b/README.md index 9484164..87622ec 100644 --- a/README.md +++ b/README.md @@ -1,121 +1,214 @@
+