Move datasets to Zenodo with BiocFileCache caching

.Rbuildignore

@@ -1 +1,3 @@
-^\.github$
+^\.github$
+^_pkgdown\.yml$
+^CITATION\.cff$

.github/workflows/ci.yml

@@ -1,49 +1,20 @@
-name: pkgdown
+name: PR CMD check & build site
 
 on:
+  pull_request:
   push:
-    branches:
-      - devel
-      - 'RELEASE_*'
     paths:
       - 'DESCRIPTION'
       - '**.yml'
-      - 'vignettes/**'
-      - 'man/**'
-
-permissions:
-  contents: write
-  pages: write
-  id-token: write
+    branches:
+      - devel
+      - 'RELEASE_*'
 
 jobs:
-  pkgdown:
-    runs-on: ubuntu-latest
-    env:
-      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: r-lib/actions/setup-r@v2
-        with:
-          use-public-rspm: true
-
-      - uses: r-lib/actions/setup-pandoc@v2
-
-      - name: Install dependencies
-        run: |
-          install.packages(c("remotes", "pkgdown"))
-          remotes::install_local(dependencies = TRUE)
-        shell: Rscript {0}
-
-      - name: Build site
-        run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE)
-        shell: Rscript {0}
-
-      - name: Deploy to GitHub pages 🚀
-        if: github.event_name != 'pull_request'
-        uses: JamesIves/github-pages-deploy-action@v4
-        with:
-          clean: false
-          branch: gh-pages
-          folder: docs
+  ci:
+    uses: waldronlab/.github/.github/workflows/bioc-pr-cmdcheck-pkgdown.yml@devel
+    secrets: inherit
+    permissions:
+      contents: write
+      pages: write
+      id-token: write

DESCRIPTION

@@ -1,24 +1,31 @@
 Package: curatedCRCData
 Type: Package
 Title: Colorectal Cancer Gene Expression Analysis
-Version: 2.45.2
-Date: 2026-04-10
+Version: 2.45.3
+Date: 2026-10-03
 Authors@R: c(
     person("Princy", "Parsana", role = "aut"),
     person("Markus", "Riester", role = "aut"),
     person("Curtis", "Huttenhower", role = "aut"),
     person("Levi", "Waldron", email = "lwaldron.research@gmail.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-2725-0694")),
     person("National Cancer Institute", role = "fnd", comment = "5U24CA180996")
   )
-Description: The curatedCRC package provides relevant functions and data for gene expression analysis in patients with colorectal cancer.
+Description: Manually curated human colorectal cancer ExpressionSet objects
+    from 34 studies, with uniformly processed expression data and
+    standardized clinical metadata including survival outcomes. Datasets
+    are hosted on Zenodo and downloaded on first use, then cached locally
+    with BiocFileCache. Includes a configurable script for rule-based
+    creation of filtered dataset lists for meta-analysis.
 Depends: 
-    R (>= 4.0.0),
+    R (>= 4.6.0),
     Biobase
 Imports: 
-    BiocGenerics
+    BiocFileCache,
+    tools,
+    utils
 Suggests: 
     survival,
-    RUnit,
+    testthat (>= 3.0.0),
     metafor,
     genefilter,
     logging,
@@ -30,10 +37,11 @@ Suggests:
     rmarkdown
 VignetteBuilder: knitr
 License: Artistic-2.0
-Namespace: auto
 biocViews: 
     ExperimentData,
     CancerData,
     ExpressionData
 URL: https://github.com/waldronlab/curatedCRCData
 BugReports: https://github.com/waldronlab/curatedCRCData/issues
+BuildResaveData: no
+Config/testthat/edition: 3

NAMESPACE

@@ -1,11 +1,5 @@
-# Default NAMESPACE created by R
-# Remove the previous line if you edit this file
-
-# Export all names
-exportPattern(".")
-
-# Import all packages listed as Imports or Depends
-import(
-  Biobase,
-  BiocGenerics
-)
+export(curatedCRCData)
+import(Biobase)
+importFrom(BiocFileCache, BiocFileCache, bfcrpath, bfcquery, bfcremove)
+importFrom(utils, read.csv)
+importFrom(tools, md5sum, R_user_dir)

NEWS.md

@@ -1,3 +1,36 @@
+# curatedCRCData 2.46.0
+
+## SIGNIFICANT USER-VISIBLE CHANGES
+
+* The 34 datasets are no longer stored inside the package: they are hosted
+  on Zenodo and downloaded individually on first use, then cached locally
+  with `BiocFileCache`. This shrinks the installed package from ~400 MB to
+  a few MB.
+* New exported function `curatedCRCData()`: call it with no arguments to
+  list available datasets, with one dataset name to get an `ExpressionSet`,
+  or with several names to get a named list. `test = TRUE` loads small
+  offline subsets bundled with the package (used by examples, tests, and
+  the vignette).
+* Legacy `data(GSE39582_eset)` access still works (it downloads through the
+  same cache, deferred until the object is first used) but is deprecated
+  and will be removed in a future release.
+* `inst/extdata/createEsetList.R` now loads datasets through the getter; a
+  new `test.mode` option in the patientselection config files selects the
+  offline subsets.
+* The cache is package-specific (`tools::R_user_dir("curatedCRCData",
+  "cache")`); delete that directory to reclaim disk space. Downloads are
+  verified against md5 checksums recorded in
+  `inst/extdata/zenodo-manifest.csv`.
+
+## INTERNAL
+
+* Replaced the never-executed RUnit scaffolding with testthat tests.
+* CI restored to R CMD check + BiocCheck + pkgdown via the waldronlab
+  reusable workflow (previously reduced to pkgdown-only because checking
+  400 MB of data exhausted runner memory).
+* Removed `external_data_store.txt` and `Namespace: auto`; explicit
+  NAMESPACE exporting only `curatedCRCData()`.
+
 # curatedCRCData 2.43.1
 
 * Updated package maintainer to Levi Waldron (<lwaldron.research@gmail.com>).

R/getData.R

@@ -0,0 +1,103 @@
+## Data retrieval for curatedCRCData: datasets are hosted on Zenodo and
+## downloaded on first use into a package-specific BiocFileCache.
+##
+## NOTE: this file is intentionally duplicated (modulo package name) in
+## curatedOvarianData. Keep the two copies in sync.
+
+.pkgname <- "curatedCRCData"
+
+## session flags (e.g., show the data() deprecation note only once)
+.pkgenv <- new.env(parent = emptyenv())
+
+## Manifest mapping dataset names to Zenodo file URLs, md5 checksums, and
+## sizes. Regenerated by inst/scripts/make-data.R; the manifest is the ONLY
+## place download URLs live.
+.manifest <- function() {
+    fname <- system.file("extdata", "zenodo-manifest.csv",
+                         package = .pkgname, mustWork = TRUE)
+    utils::read.csv(fname, stringsAsFactors = FALSE)
+}
+
+.cache <- function() {
+    BiocFileCache::BiocFileCache(tools::R_user_dir(.pkgname, "cache"))
+}
+
+## Return the local path to one dataset's .rda, downloading into the cache
+## on first use. The md5 checksum is verified after a fresh download only;
+## on mismatch the corrupt entry is removed and the download retried once.
+.getDataFile <- function(name, retry = TRUE) {
+    meta <- .manifest()
+    i <- match(name, meta$dataset)
+    stopifnot(!is.na(i))
+    bfc <- .cache()
+    known <- BiocFileCache::bfcquery(bfc, meta$url[i], field = "fpath",
+                                     exact = TRUE)
+    fresh <- nrow(known) == 0L
+    if (fresh)
+        message("Downloading ", name, " (",
+                round(meta$size_bytes[i] / 2^20, 1), " MB) from Zenodo...")
+    path <- unname(BiocFileCache::bfcrpath(bfc, meta$url[i]))
+    if (fresh) {
+        md5 <- unname(tools::md5sum(path))
+        if (!identical(md5, meta$md5[i])) {
+            res <- BiocFileCache::bfcquery(bfc, meta$url[i], field = "fpath",
+                                           exact = TRUE)
+            if (nrow(res))
+                BiocFileCache::bfcremove(bfc, res$rid)
+            if (retry) {
+                message("Checksum mismatch; retrying the download once...")
+                return(.getDataFile(name, retry = FALSE))
+            }
+            stop("md5 checksum mismatch for '", name, "' downloaded from ",
+                 meta$url[i],
+                 "; the corrupt copy was removed from the cache.")
+        }
+    }
+    path
+}
+
+.loadOne <- function(name, test = FALSE) {
+    path <- if (test) {
+        system.file("extdata", "testdata", paste0(name, ".rda"),
+                    package = .pkgname, mustWork = TRUE)
+    } else {
+        .getDataFile(name)
+    }
+    env <- new.env(parent = emptyenv())
+    objname <- load(path, envir = env)
+    get(objname[1L], envir = env)
+}
+
+## Called by the data/*.R stubs so that legacy data(X) usage keeps working.
+## During this package's own R CMD check, resolve to the bundled test
+## fixture so checking needs no network access; checks of OTHER packages
+## (reverse dependencies) get the real data.
+.stubLoad <- function(name) {
+    if (identical(Sys.getenv("_R_CHECK_PACKAGE_NAME_"), .pkgname))
+        return(.loadOne(name, test = TRUE))
+    if (!isTRUE(.pkgenv$stub_noted)) {
+        .pkgenv$stub_noted <- TRUE
+        message("Note: loading ", .pkgname, " datasets via data() is ",
+                "deprecated and will be removed in a future release; use ",
+                .pkgname, "(\"", name, "\") instead. Datasets are now ",
+                "downloaded on first use and cached locally with ",
+                "BiocFileCache. (This note is shown once per session.)")
+    }
+    .loadOne(name)
+}
+
+#' @export
+curatedCRCData <- function(datasets, test = FALSE) {
+    meta <- .manifest()
+    if (missing(datasets))
+        return(meta$dataset)
+    stopifnot(is.character(datasets), !anyNA(datasets))
+    bad <- setdiff(datasets, meta$dataset)
+    if (length(bad))
+        stop("Unknown dataset(s): ", paste(bad, collapse = ", "),
+             ". Call curatedCRCData() with no arguments to list ",
+             "available datasets.")
+    res <- lapply(datasets, .loadOne, test = test)
+    names(res) <- datasets
+    if (length(res) == 1L) res[[1L]] else res
+}

_pkgdown.yml

@@ -0,0 +1,7 @@
+title: curatedCRCData
+url: https://waldronlab.github.io/curatedCRCData
+
+template:
+  bootstrap: 5
+  params:
+    bootswatch: flatly

data/GSE11237_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE11237_eset",
+    curatedCRCData:::.stubLoad("GSE11237_eset"),
+    assign.env = environment())

data/GSE12225.GPL3676_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE12225.GPL3676_eset",
+    curatedCRCData:::.stubLoad("GSE12225.GPL3676_eset"),
+    assign.env = environment())

data/GSE12945_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE12945_eset",
+    curatedCRCData:::.stubLoad("GSE12945_eset"),
+    assign.env = environment())

data/GSE13067_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE13067_eset",
+    curatedCRCData:::.stubLoad("GSE13067_eset"),
+    assign.env = environment())

data/GSE13294_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE13294_eset",
+    curatedCRCData:::.stubLoad("GSE13294_eset"),
+    assign.env = environment())

data/GSE14095_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE14095_eset",
+    curatedCRCData:::.stubLoad("GSE14095_eset"),
+    assign.env = environment())

data/GSE14333_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE14333_eset",
+    curatedCRCData:::.stubLoad("GSE14333_eset"),
+    assign.env = environment())

data/GSE16125.GPL5175_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE16125.GPL5175_eset",
+    curatedCRCData:::.stubLoad("GSE16125.GPL5175_eset"),
+    assign.env = environment())

data/GSE17536_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE17536_eset",
+    curatedCRCData:::.stubLoad("GSE17536_eset"),
+    assign.env = environment())