Move datasets to Zenodo, downloaded on demand and cached with BiocFileCache

DESCRIPTION

@@ -1,13 +1,34 @@
 Package: curatedBladderData
 Type: Package
 Title: Bladder Cancer Gene Expression Analysis
-Version: 1.19.2
-Date: 2019-03-28
+Version: 1.49.1
+Date: 2026-10-06
 Author: Markus Riester
 Maintainer: Markus Riester <markus@jimmy.harvard.edu>
-Description: The curatedBladderData package provides relevant functions and data for gene expression analysis in patients with bladder cancer.
-Depends: R (>= 2.10.0), affy
-Suggests: BiocStyle, survival, xtable, sva, genefilter, logging
+Description: Manually curated bladder cancer ExpressionSet objects from 10
+    studies (12 objects, since two studies contributed data on two platforms
+    each), with uniformly processed expression data and curated clinical
+    metadata. Datasets are hosted on Zenodo and downloaded on first use,
+    then cached locally with BiocFileCache. Includes a configurable script
+    for rule-based creation of filtered dataset lists for meta-analysis.
+Depends:
+    R (>= 4.0.0),
+    Biobase
+Imports:
+    BiocFileCache,
+    tools,
+    utils
+Suggests:
+    BiocStyle,
+    genefilter,
+    logging,
+    survival,
+    sva,
+    testthat (>= 3.0.0),
+    xtable
 License: Artistic-2.0
 URL: https://github.com/lima1/curatedBladderData
-biocViews: ExperimentData, CancerData, OvarianCancerData, MicroarrayData, ExpressionData
+BugReports: https://github.com/lima1/curatedBladderData/issues
+biocViews: ExperimentData, CancerData, MicroarrayData, ExpressionData, GEO
+BuildResaveData: no
+Config/testthat/edition: 3

NAMESPACE

@@ -1 +1,5 @@
-exportPattern("^[[:alpha:]]+")
+export(curatedBladderData)
+import(Biobase)
+importFrom(BiocFileCache, BiocFileCache, bfcrpath, bfcquery, bfcremove)
+importFrom(utils, read.csv)
+importFrom(tools, md5sum, R_user_dir)

NEWS.md

@@ -0,0 +1,39 @@
+# curatedBladderData 1.50.0
+
+## SIGNIFICANT USER-VISIBLE CHANGES
+
+* The 12 datasets are no longer stored inside the package. They are hosted
+  on Zenodo and downloaded individually on first use, then cached locally
+  with `BiocFileCache`. This shrinks the installed package from about
+  75 MB to a few MB.
+* New exported function `curatedBladderData()`: call it with no arguments
+  to list the available datasets, with one dataset name to get an
+  `ExpressionSet`, or with several names to get a named list. `test = TRUE`
+  loads small offline subsets bundled with the package, used by examples,
+  tests and the vignette.
+* Legacy `data(GSE89_eset)` access still works. It downloads through the
+  same cache, deferred until the object is first used, but is deprecated
+  and will be removed in a future release.
+* `inst/extdata/createEsetList.R` now loads datasets through the getter.
+  A new `test.mode` option in the patientselection config selects the
+  offline subsets.
+* The cache is package-specific, at
+  `tools::R_user_dir("curatedBladderData", "cache")`. Downloads are
+  verified against md5 checksums recorded in
+  `inst/extdata/zenodo-manifest.csv`.
+
+## BUG FIXES
+
+* `biocViews` listed `OvarianCancerData`; removed, and `GEO` added.
+* The package-level help page described ovarian cancer, listed the
+  curatedOvarianData author list, and named a different maintainer than
+  DESCRIPTION. Corrected to match DESCRIPTION.
+* Replaced the unused `affy` dependency with `Biobase`, which is where
+  `ExpressionSet`, `exprs()` and `pData()` actually come from.
+* `createEsetList.R` no longer errors when the filters exclude every
+  dataset (`which()` on an empty list).
+
+## INTERNAL
+
+* Added testthat tests and `inst/scripts/` documenting how the Zenodo
+  files, the offline fixtures and the data stubs are generated.

R/getData.R

@@ -0,0 +1,118 @@
+## Data retrieval for curatedBladderData: datasets are hosted on Zenodo and
+## downloaded on first use into a package-specific BiocFileCache.
+##
+## NOTE: this file is intentionally duplicated (modulo package name) in
+## curatedCRCData and curatedOvarianData. Keep the copies in sync.
+
+.pkgname <- "curatedBladderData"
+
+## session flags (e.g., show the data() deprecation note only once)
+.pkgenv <- new.env(parent = emptyenv())
+
+## Manifest mapping dataset names to Zenodo file URLs, md5 checksums, and
+## sizes. Regenerated by inst/scripts/make-data.R; the manifest is the ONLY
+## place download URLs live.
+.manifest <- function() {
+    fname <- system.file("extdata", "zenodo-manifest.csv",
+                         package = .pkgname, mustWork = TRUE)
+    utils::read.csv(fname, stringsAsFactors = FALSE)
+}
+
+.cache <- function() {
+    BiocFileCache::BiocFileCache(tools::R_user_dir(.pkgname, "cache"))
+}
+
+## Return the local path to one dataset's .rda, downloading into the cache
+## on first use. The md5 checksum is verified after a fresh download only;
+## on mismatch the corrupt entry is removed and the download retried once.
+.getDataFile <- function(name, retry = TRUE) {
+    meta <- .manifest()
+    i <- match(name, meta$dataset)
+    stopifnot(!is.na(i))
+    bfc <- .cache()
+    known <- BiocFileCache::bfcquery(bfc, meta$url[i], field = "fpath",
+                                     exact = TRUE)
+    fresh <- nrow(known) == 0L
+    if (fresh)
+        message("Downloading ", name, " (",
+                round(meta$size_bytes[i] / 2^20, 1), " MB) from Zenodo...")
+    path <- unname(BiocFileCache::bfcrpath(bfc, meta$url[i]))
+    if (!fresh && !isTRUE(file.size(path) == meta$size_bytes[i])) {
+        # cached file was truncated or modified after download: evict and
+        # re-download (cheap size check; full md5 runs after each download)
+        res <- BiocFileCache::bfcquery(bfc, meta$url[i], field = "fpath",
+                                       exact = TRUE)
+        if (nrow(res))
+            BiocFileCache::bfcremove(bfc, res$rid)
+        if (retry) {
+            message("Cached copy of ", name,
+                    " has the wrong size; re-downloading...")
+            return(.getDataFile(name, retry = FALSE))
+        }
+        stop("Cached file for '", name, "' has the wrong size and could ",
+             "not be refreshed.")
+    }
+    if (fresh) {
+        md5 <- unname(tools::md5sum(path))
+        if (!identical(md5, meta$md5[i])) {
+            res <- BiocFileCache::bfcquery(bfc, meta$url[i], field = "fpath",
+                                           exact = TRUE)
+            if (nrow(res))
+                BiocFileCache::bfcremove(bfc, res$rid)
+            if (retry) {
+                message("Checksum mismatch; retrying the download once...")
+                return(.getDataFile(name, retry = FALSE))
+            }
+            stop("md5 checksum mismatch for '", name, "' downloaded from ",
+                 meta$url[i],
+                 "; the corrupt copy was removed from the cache.")
+        }
+    }
+    path
+}
+
+.loadOne <- function(name, test = FALSE) {
+    path <- if (test) {
+        system.file("extdata", "testdata", paste0(name, ".rda"),
+                    package = .pkgname, mustWork = TRUE)
+    } else {
+        .getDataFile(name)
+    }
+    env <- new.env(parent = emptyenv())
+    objname <- load(path, envir = env)
+    get(objname[1L], envir = env)
+}
+
+## Called by the data/*.R stubs so that legacy data(X) usage keeps working.
+## During this package's own R CMD check, resolve to the bundled test
+## fixture so checking needs no network access; checks of OTHER packages
+## (reverse dependencies) get the real data.
+.stubLoad <- function(name) {
+    if (identical(Sys.getenv("_R_CHECK_PACKAGE_NAME_"), .pkgname))
+        return(.loadOne(name, test = TRUE))
+    if (!isTRUE(.pkgenv$stub_noted)) {
+        .pkgenv$stub_noted <- TRUE
+        message("Note: loading ", .pkgname, " datasets via data() is ",
+                "deprecated and will be removed in a future release; use ",
+                .pkgname, "(\"", name, "\") instead. Datasets are now ",
+                "downloaded on first use and cached locally with ",
+                "BiocFileCache. (This note is shown once per session.)")
+    }
+    .loadOne(name)
+}
+
+#' @export
+curatedBladderData <- function(datasets, test = FALSE) {
+    meta <- .manifest()
+    if (missing(datasets))
+        return(meta$dataset)
+    stopifnot(is.character(datasets), !anyNA(datasets))
+    bad <- setdiff(datasets, meta$dataset)
+    if (length(bad))
+        stop("Unknown dataset(s): ", paste(bad, collapse = ", "),
+             ". Call curatedBladderData() with no arguments to list ",
+             "available datasets.")
+    res <- lapply(datasets, .loadOne, test = test)
+    names(res) <- datasets
+    if (length(res) == 1L) res[[1L]] else res
+}

data/GSE13507_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE13507_eset",
+    curatedBladderData:::.stubLoad("GSE13507_eset"),
+    assign.env = environment())

data/GSE1827_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE1827_eset",
+    curatedBladderData:::.stubLoad("GSE1827_eset"),
+    assign.env = environment())

data/GSE19915.GPL3883_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE19915.GPL3883_eset",
+    curatedBladderData:::.stubLoad("GSE19915.GPL3883_eset"),
+    assign.env = environment())

data/GSE19915.GPL5186_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE19915.GPL5186_eset",
+    curatedBladderData:::.stubLoad("GSE19915.GPL5186_eset"),
+    assign.env = environment())

data/GSE31189_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE31189_eset",
+    curatedBladderData:::.stubLoad("GSE31189_eset"),
+    assign.env = environment())

data/GSE31684_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE31684_eset",
+    curatedBladderData:::.stubLoad("GSE31684_eset"),
+    assign.env = environment())

data/GSE32894_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE32894_eset",
+    curatedBladderData:::.stubLoad("GSE32894_eset"),
+    assign.env = environment())

data/GSE37317_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE37317_eset",
+    curatedBladderData:::.stubLoad("GSE37317_eset"),
+    assign.env = environment())

data/GSE5287_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE5287_eset",
+    curatedBladderData:::.stubLoad("GSE5287_eset"),
+    assign.env = environment())

data/GSE89_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("GSE89_eset",
+    curatedBladderData:::.stubLoad("GSE89_eset"),
+    assign.env = environment())

data/PMID17099711.GPL8300_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("PMID17099711.GPL8300_eset",
+    curatedBladderData:::.stubLoad("PMID17099711.GPL8300_eset"),
+    assign.env = environment())

data/PMID17099711.GPL91_eset.R

@@ -0,0 +1,3 @@
+delayedAssign("PMID17099711.GPL91_eset",
+    curatedBladderData:::.stubLoad("PMID17099711.GPL91_eset"),
+    assign.env = environment())

inst/extdata/createEsetList.R

@@ -62,7 +62,10 @@ intersectMany <- function(lst){
 }
 
 ##load the esets
-data(list=data(package=package.name)[[3]][,3])
+if (!exists("test.mode")) test.mode <- FALSE
+for (.nm in do.call(package.name, list()))
+    assign(.nm, do.call(package.name, list(datasets = .nm, test = test.mode)))
+rm(.nm)
 
 strEsets <- ls(pattern="^.*_eset$")
 
@@ -142,8 +145,8 @@ if(keep.common.only){
     })
 }
 
-ids.with.missing.data <- which(sapply(esets, function(X)
-sum(!complete.cases(exprs(X))) > 0))
+ids.with.missing.data <- which(vapply(esets, function(X)
+sum(!complete.cases(exprs(X))) > 0, logical(1)))
 loginfo(paste("Ids with missing data:", paste(names(ids.with.missing.data),
 collapse=", ")))
 

inst/extdata/patientselection_all.config

@@ -2,6 +2,11 @@
 ##settings for createEsetList.R
 ##-----------------------------
 
+
+#use the small offline test subsets bundled with the package instead of
+#downloading the full datasets from Zenodo? (for examples/vignettes/tests)
+test.mode <- FALSE
+
 package.name <- "curatedBladderData"
 
 #rescale each gene to z-score?

inst/extdata/zenodo-manifest.csv

@@ -0,0 +1,13 @@
+dataset,filename,url,md5,size_bytes
+GSE13507_eset,GSE13507_eset.rda,https://zenodo.org/records/23192472/files/GSE13507_eset.rda?download=1,9df8c8a2c77600fb3346d7a9d6b34b20,6289556
+GSE1827_eset,GSE1827_eset.rda,https://zenodo.org/records/23192472/files/GSE1827_eset.rda?download=1,3a195c532c1067c1204fd963583d4b2c,3157656
+GSE19915.GPL3883_eset,GSE19915.GPL3883_eset.rda,https://zenodo.org/records/23192472/files/GSE19915.GPL3883_eset.rda?download=1,6d433ad595361f1a893d0f3e07a0f4c3,5705184
+GSE19915.GPL5186_eset,GSE19915.GPL5186_eset.rda,https://zenodo.org/records/23192472/files/GSE19915.GPL5186_eset.rda?download=1,d78aa0d949148cd5a30868453896d058,7141496
+GSE31189_eset,GSE31189_eset.rda,https://zenodo.org/records/23192472/files/GSE31189_eset.rda?download=1,feb209ae4826e1691b7c4d39048b9635,8670296
+GSE31684_eset,GSE31684_eset.rda,https://zenodo.org/records/23192472/files/GSE31684_eset.rda?download=1,3deda3de9e45b169271f472d8c2c2e62,7548196
+GSE32894_eset,GSE32894_eset.rda,https://zenodo.org/records/23192472/files/GSE32894_eset.rda?download=1,43100e65e99bf474f00cb018500df411,14435224
+GSE37317_eset,GSE37317_eset.rda,https://zenodo.org/records/23192472/files/GSE37317_eset.rda?download=1,b0dd231adb053574f487f832e38ad5f5,1791700
+GSE5287_eset,GSE5287_eset.rda,https://zenodo.org/records/23192472/files/GSE5287_eset.rda?download=1,93666b747360690461639477e535ef70,2803624
+GSE89_eset,GSE89_eset.rda,https://zenodo.org/records/23192472/files/GSE89_eset.rda?download=1,93e550b942226b5958f7dad7abefcaa1,1542464
+PMID17099711.GPL8300_eset,PMID17099711.GPL8300_eset.rda,https://zenodo.org/records/23192472/files/PMID17099711.GPL8300_eset.rda?download=1,acd04f2130e55e221919f6491bf60e7b,1923304
+PMID17099711.GPL91_eset,PMID17099711.GPL91_eset.rda,https://zenodo.org/records/23192472/files/PMID17099711.GPL91_eset.rda?download=1,8c88b6bf8f2834efa022e5b3d7fd026a,1994320