diff --git a/.Rbuildignore b/.Rbuildignore index 62652f5..18eae90 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1 +1,3 @@ -^\.github$ \ No newline at end of file +^\.github$ +^_pkgdown\.yml$ +^CITATION\.cff$ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93585ca..2133e3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,49 +1,20 @@ -name: pkgdown +name: PR CMD check & build site on: + pull_request: push: - branches: - - devel - - 'RELEASE_*' paths: - 'DESCRIPTION' - '**.yml' - - 'vignettes/**' - - 'man/**' - -permissions: - contents: write - pages: write - id-token: write + branches: + - devel + - 'RELEASE_*' jobs: - pkgdown: - runs-on: ubuntu-latest - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@v4 - - - uses: r-lib/actions/setup-r@v2 - with: - use-public-rspm: true - - - uses: r-lib/actions/setup-pandoc@v2 - - - name: Install dependencies - run: | - install.packages(c("remotes", "pkgdown")) - remotes::install_local(dependencies = TRUE) - shell: Rscript {0} - - - name: Build site - run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE) - shell: Rscript {0} - - - name: Deploy to GitHub pages 🚀 - if: github.event_name != 'pull_request' - uses: JamesIves/github-pages-deploy-action@v4 - with: - clean: false - branch: gh-pages - folder: docs \ No newline at end of file + ci: + uses: waldronlab/.github/.github/workflows/bioc-pr-cmdcheck-pkgdown.yml@devel + secrets: inherit + permissions: + contents: write + pages: write + id-token: write diff --git a/DESCRIPTION b/DESCRIPTION index 949fff8..566b65b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Package: curatedCRCData Type: Package Title: Colorectal Cancer Gene Expression Analysis -Version: 2.45.2 -Date: 2026-04-10 +Version: 2.45.3 +Date: 2026-10-03 Authors@R: c( person("Princy", "Parsana", role = "aut"), person("Markus", "Riester", role = "aut"), @@ -10,15 +10,22 @@ Authors@R: c( person("Levi", "Waldron", email = "lwaldron.research@gmail.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-2725-0694")), person("National Cancer Institute", role = "fnd", comment = "5U24CA180996") ) -Description: The curatedCRC package provides relevant functions and data for gene expression analysis in patients with colorectal cancer. +Description: Manually curated human colorectal cancer ExpressionSet objects + from 34 studies, with uniformly processed expression data and + standardized clinical metadata including survival outcomes. Datasets + are hosted on Zenodo and downloaded on first use, then cached locally + with BiocFileCache. Includes a configurable script for rule-based + creation of filtered dataset lists for meta-analysis. Depends: - R (>= 4.0.0), + R (>= 4.6.0), Biobase Imports: - BiocGenerics + BiocFileCache, + tools, + utils Suggests: survival, - RUnit, + testthat (>= 3.0.0), metafor, genefilter, logging, @@ -30,10 +37,11 @@ Suggests: rmarkdown VignetteBuilder: knitr License: Artistic-2.0 -Namespace: auto biocViews: ExperimentData, CancerData, ExpressionData URL: https://github.com/waldronlab/curatedCRCData BugReports: https://github.com/waldronlab/curatedCRCData/issues +BuildResaveData: no +Config/testthat/edition: 3 diff --git a/NAMESPACE b/NAMESPACE index b6c9445..2ce1178 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,11 +1,5 @@ -# Default NAMESPACE created by R -# Remove the previous line if you edit this file - -# Export all names -exportPattern(".") - -# Import all packages listed as Imports or Depends -import( - Biobase, - BiocGenerics -) \ No newline at end of file +export(curatedCRCData) +import(Biobase) +importFrom(BiocFileCache, BiocFileCache, bfcrpath, bfcquery, bfcremove) +importFrom(utils, read.csv) +importFrom(tools, md5sum, R_user_dir) diff --git a/NEWS.md b/NEWS.md index 35b1777..bf00827 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,36 @@ +# curatedCRCData 2.46.0 + +## SIGNIFICANT USER-VISIBLE CHANGES + +* The 34 datasets are no longer stored inside the package: they are hosted + on Zenodo and downloaded individually on first use, then cached locally + with `BiocFileCache`. This shrinks the installed package from ~400 MB to + a few MB. +* New exported function `curatedCRCData()`: call it with no arguments to + list available datasets, with one dataset name to get an `ExpressionSet`, + or with several names to get a named list. `test = TRUE` loads small + offline subsets bundled with the package (used by examples, tests, and + the vignette). +* Legacy `data(GSE39582_eset)` access still works (it downloads through the + same cache, deferred until the object is first used) but is deprecated + and will be removed in a future release. +* `inst/extdata/createEsetList.R` now loads datasets through the getter; a + new `test.mode` option in the patientselection config files selects the + offline subsets. +* The cache is package-specific (`tools::R_user_dir("curatedCRCData", + "cache")`); delete that directory to reclaim disk space. Downloads are + verified against md5 checksums recorded in + `inst/extdata/zenodo-manifest.csv`. + +## INTERNAL + +* Replaced the never-executed RUnit scaffolding with testthat tests. +* CI restored to R CMD check + BiocCheck + pkgdown via the waldronlab + reusable workflow (previously reduced to pkgdown-only because checking + 400 MB of data exhausted runner memory). +* Removed `external_data_store.txt` and `Namespace: auto`; explicit + NAMESPACE exporting only `curatedCRCData()`. + # curatedCRCData 2.43.1 * Updated package maintainer to Levi Waldron (). diff --git a/R/getData.R b/R/getData.R new file mode 100644 index 0000000..ad8bd32 --- /dev/null +++ b/R/getData.R @@ -0,0 +1,103 @@ +## Data retrieval for curatedCRCData: datasets are hosted on Zenodo and +## downloaded on first use into a package-specific BiocFileCache. +## +## NOTE: this file is intentionally duplicated (modulo package name) in +## curatedOvarianData. Keep the two copies in sync. + +.pkgname <- "curatedCRCData" + +## session flags (e.g., show the data() deprecation note only once) +.pkgenv <- new.env(parent = emptyenv()) + +## Manifest mapping dataset names to Zenodo file URLs, md5 checksums, and +## sizes. Regenerated by inst/scripts/make-data.R; the manifest is the ONLY +## place download URLs live. +.manifest <- function() { + fname <- system.file("extdata", "zenodo-manifest.csv", + package = .pkgname, mustWork = TRUE) + utils::read.csv(fname, stringsAsFactors = FALSE) +} + +.cache <- function() { + BiocFileCache::BiocFileCache(tools::R_user_dir(.pkgname, "cache")) +} + +## Return the local path to one dataset's .rda, downloading into the cache +## on first use. The md5 checksum is verified after a fresh download only; +## on mismatch the corrupt entry is removed and the download retried once. +.getDataFile <- function(name, retry = TRUE) { + meta <- .manifest() + i <- match(name, meta$dataset) + stopifnot(!is.na(i)) + bfc <- .cache() + known <- BiocFileCache::bfcquery(bfc, meta$url[i], field = "fpath", + exact = TRUE) + fresh <- nrow(known) == 0L + if (fresh) + message("Downloading ", name, " (", + round(meta$size_bytes[i] / 2^20, 1), " MB) from Zenodo...") + path <- unname(BiocFileCache::bfcrpath(bfc, meta$url[i])) + if (fresh) { + md5 <- unname(tools::md5sum(path)) + if (!identical(md5, meta$md5[i])) { + res <- BiocFileCache::bfcquery(bfc, meta$url[i], field = "fpath", + exact = TRUE) + if (nrow(res)) + BiocFileCache::bfcremove(bfc, res$rid) + if (retry) { + message("Checksum mismatch; retrying the download once...") + return(.getDataFile(name, retry = FALSE)) + } + stop("md5 checksum mismatch for '", name, "' downloaded from ", + meta$url[i], + "; the corrupt copy was removed from the cache.") + } + } + path +} + +.loadOne <- function(name, test = FALSE) { + path <- if (test) { + system.file("extdata", "testdata", paste0(name, ".rda"), + package = .pkgname, mustWork = TRUE) + } else { + .getDataFile(name) + } + env <- new.env(parent = emptyenv()) + objname <- load(path, envir = env) + get(objname[1L], envir = env) +} + +## Called by the data/*.R stubs so that legacy data(X) usage keeps working. +## During this package's own R CMD check, resolve to the bundled test +## fixture so checking needs no network access; checks of OTHER packages +## (reverse dependencies) get the real data. +.stubLoad <- function(name) { + if (identical(Sys.getenv("_R_CHECK_PACKAGE_NAME_"), .pkgname)) + return(.loadOne(name, test = TRUE)) + if (!isTRUE(.pkgenv$stub_noted)) { + .pkgenv$stub_noted <- TRUE + message("Note: loading ", .pkgname, " datasets via data() is ", + "deprecated and will be removed in a future release; use ", + .pkgname, "(\"", name, "\") instead. Datasets are now ", + "downloaded on first use and cached locally with ", + "BiocFileCache. (This note is shown once per session.)") + } + .loadOne(name) +} + +#' @export +curatedCRCData <- function(datasets, test = FALSE) { + meta <- .manifest() + if (missing(datasets)) + return(meta$dataset) + stopifnot(is.character(datasets), !anyNA(datasets)) + bad <- setdiff(datasets, meta$dataset) + if (length(bad)) + stop("Unknown dataset(s): ", paste(bad, collapse = ", "), + ". Call curatedCRCData() with no arguments to list ", + "available datasets.") + res <- lapply(datasets, .loadOne, test = test) + names(res) <- datasets + if (length(res) == 1L) res[[1L]] else res +} diff --git a/_pkgdown.yml b/_pkgdown.yml new file mode 100644 index 0000000..23958f9 --- /dev/null +++ b/_pkgdown.yml @@ -0,0 +1,7 @@ +title: curatedCRCData +url: https://waldronlab.github.io/curatedCRCData + +template: + bootstrap: 5 + params: + bootswatch: flatly diff --git a/data/GSE11237_eset.R b/data/GSE11237_eset.R new file mode 100644 index 0000000..909ee81 --- /dev/null +++ b/data/GSE11237_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE11237_eset", + curatedCRCData:::.stubLoad("GSE11237_eset"), + assign.env = environment()) diff --git a/data/GSE11237_eset.rda b/data/GSE11237_eset.rda deleted file mode 100644 index a204061..0000000 Binary files a/data/GSE11237_eset.rda and /dev/null differ diff --git a/data/GSE12225.GPL3676_eset.R b/data/GSE12225.GPL3676_eset.R new file mode 100644 index 0000000..a447f06 --- /dev/null +++ b/data/GSE12225.GPL3676_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE12225.GPL3676_eset", + curatedCRCData:::.stubLoad("GSE12225.GPL3676_eset"), + assign.env = environment()) diff --git a/data/GSE12225.GPL3676_eset.rda b/data/GSE12225.GPL3676_eset.rda deleted file mode 100644 index d8f6211..0000000 Binary files a/data/GSE12225.GPL3676_eset.rda and /dev/null differ diff --git a/data/GSE12945_eset.R b/data/GSE12945_eset.R new file mode 100644 index 0000000..8d49ecf --- /dev/null +++ b/data/GSE12945_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE12945_eset", + curatedCRCData:::.stubLoad("GSE12945_eset"), + assign.env = environment()) diff --git a/data/GSE12945_eset.rda b/data/GSE12945_eset.rda deleted file mode 100644 index 8b80d5e..0000000 Binary files a/data/GSE12945_eset.rda and /dev/null differ diff --git a/data/GSE13067_eset.R b/data/GSE13067_eset.R new file mode 100644 index 0000000..3c11693 --- /dev/null +++ b/data/GSE13067_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE13067_eset", + curatedCRCData:::.stubLoad("GSE13067_eset"), + assign.env = environment()) diff --git a/data/GSE13067_eset.rda b/data/GSE13067_eset.rda deleted file mode 100644 index f23429c..0000000 Binary files a/data/GSE13067_eset.rda and /dev/null differ diff --git a/data/GSE13294_eset.R b/data/GSE13294_eset.R new file mode 100644 index 0000000..445ede9 --- /dev/null +++ b/data/GSE13294_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE13294_eset", + curatedCRCData:::.stubLoad("GSE13294_eset"), + assign.env = environment()) diff --git a/data/GSE13294_eset.rda b/data/GSE13294_eset.rda deleted file mode 100644 index 94214b9..0000000 Binary files a/data/GSE13294_eset.rda and /dev/null differ diff --git a/data/GSE14095_eset.R b/data/GSE14095_eset.R new file mode 100644 index 0000000..fdc8f5d --- /dev/null +++ b/data/GSE14095_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE14095_eset", + curatedCRCData:::.stubLoad("GSE14095_eset"), + assign.env = environment()) diff --git a/data/GSE14095_eset.rda b/data/GSE14095_eset.rda deleted file mode 100644 index 739ba73..0000000 Binary files a/data/GSE14095_eset.rda and /dev/null differ diff --git a/data/GSE14333_eset.R b/data/GSE14333_eset.R new file mode 100644 index 0000000..b4c5605 --- /dev/null +++ b/data/GSE14333_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE14333_eset", + curatedCRCData:::.stubLoad("GSE14333_eset"), + assign.env = environment()) diff --git a/data/GSE14333_eset.rda b/data/GSE14333_eset.rda deleted file mode 100644 index d4f3404..0000000 Binary files a/data/GSE14333_eset.rda and /dev/null differ diff --git a/data/GSE16125.GPL5175_eset.R b/data/GSE16125.GPL5175_eset.R new file mode 100644 index 0000000..7dbd626 --- /dev/null +++ b/data/GSE16125.GPL5175_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE16125.GPL5175_eset", + curatedCRCData:::.stubLoad("GSE16125.GPL5175_eset"), + assign.env = environment()) diff --git a/data/GSE16125.GPL5175_eset.rda b/data/GSE16125.GPL5175_eset.rda deleted file mode 100644 index 5895b0c..0000000 Binary files a/data/GSE16125.GPL5175_eset.rda and /dev/null differ diff --git a/data/GSE17536_eset.R b/data/GSE17536_eset.R new file mode 100644 index 0000000..4fa34cc --- /dev/null +++ b/data/GSE17536_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE17536_eset", + curatedCRCData:::.stubLoad("GSE17536_eset"), + assign.env = environment()) diff --git a/data/GSE17536_eset.rda b/data/GSE17536_eset.rda deleted file mode 100644 index c15e547..0000000 Binary files a/data/GSE17536_eset.rda and /dev/null differ diff --git a/data/GSE17537_eset.R b/data/GSE17537_eset.R new file mode 100644 index 0000000..aeacd9f --- /dev/null +++ b/data/GSE17537_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE17537_eset", + curatedCRCData:::.stubLoad("GSE17537_eset"), + assign.env = environment()) diff --git a/data/GSE17537_eset.rda b/data/GSE17537_eset.rda deleted file mode 100644 index a6f67fa..0000000 Binary files a/data/GSE17537_eset.rda and /dev/null differ diff --git a/data/GSE17538.GPL570_eset.R b/data/GSE17538.GPL570_eset.R new file mode 100644 index 0000000..77426be --- /dev/null +++ b/data/GSE17538.GPL570_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE17538.GPL570_eset", + curatedCRCData:::.stubLoad("GSE17538.GPL570_eset"), + assign.env = environment()) diff --git a/data/GSE17538.GPL570_eset.rda b/data/GSE17538.GPL570_eset.rda deleted file mode 100644 index b0c2cb0..0000000 Binary files a/data/GSE17538.GPL570_eset.rda and /dev/null differ diff --git a/data/GSE18105_eset.R b/data/GSE18105_eset.R new file mode 100644 index 0000000..29268be --- /dev/null +++ b/data/GSE18105_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE18105_eset", + curatedCRCData:::.stubLoad("GSE18105_eset"), + assign.env = environment()) diff --git a/data/GSE18105_eset.rda b/data/GSE18105_eset.rda deleted file mode 100644 index dee4c2a..0000000 Binary files a/data/GSE18105_eset.rda and /dev/null differ diff --git a/data/GSE2109_eset.R b/data/GSE2109_eset.R new file mode 100644 index 0000000..fb51c6d --- /dev/null +++ b/data/GSE2109_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE2109_eset", + curatedCRCData:::.stubLoad("GSE2109_eset"), + assign.env = environment()) diff --git a/data/GSE2109_eset.rda b/data/GSE2109_eset.rda deleted file mode 100644 index be33a31..0000000 Binary files a/data/GSE2109_eset.rda and /dev/null differ diff --git a/data/GSE21510_eset.R b/data/GSE21510_eset.R new file mode 100644 index 0000000..665978b --- /dev/null +++ b/data/GSE21510_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE21510_eset", + curatedCRCData:::.stubLoad("GSE21510_eset"), + assign.env = environment()) diff --git a/data/GSE21510_eset.rda b/data/GSE21510_eset.rda deleted file mode 100644 index bfa9484..0000000 Binary files a/data/GSE21510_eset.rda and /dev/null differ diff --git a/data/GSE21815_eset.R b/data/GSE21815_eset.R new file mode 100644 index 0000000..6865313 --- /dev/null +++ b/data/GSE21815_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE21815_eset", + curatedCRCData:::.stubLoad("GSE21815_eset"), + assign.env = environment()) diff --git a/data/GSE21815_eset.rda b/data/GSE21815_eset.rda deleted file mode 100644 index d5e8266..0000000 Binary files a/data/GSE21815_eset.rda and /dev/null differ diff --git a/data/GSE24549.GPL5175_eset.R b/data/GSE24549.GPL5175_eset.R new file mode 100644 index 0000000..cc6d12d --- /dev/null +++ b/data/GSE24549.GPL5175_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE24549.GPL5175_eset", + curatedCRCData:::.stubLoad("GSE24549.GPL5175_eset"), + assign.env = environment()) diff --git a/data/GSE24549.GPL5175_eset.rda b/data/GSE24549.GPL5175_eset.rda deleted file mode 100644 index 3e1363c..0000000 Binary files a/data/GSE24549.GPL5175_eset.rda and /dev/null differ diff --git a/data/GSE24550.GPL5175_eset.R b/data/GSE24550.GPL5175_eset.R new file mode 100644 index 0000000..8ae7abf --- /dev/null +++ b/data/GSE24550.GPL5175_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE24550.GPL5175_eset", + curatedCRCData:::.stubLoad("GSE24550.GPL5175_eset"), + assign.env = environment()) diff --git a/data/GSE24550.GPL5175_eset.rda b/data/GSE24550.GPL5175_eset.rda deleted file mode 100644 index b00ae22..0000000 Binary files a/data/GSE24550.GPL5175_eset.rda and /dev/null differ diff --git a/data/GSE2630_eset.R b/data/GSE2630_eset.R new file mode 100644 index 0000000..6907bed --- /dev/null +++ b/data/GSE2630_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE2630_eset", + curatedCRCData:::.stubLoad("GSE2630_eset"), + assign.env = environment()) diff --git a/data/GSE2630_eset.rda b/data/GSE2630_eset.rda deleted file mode 100644 index 8730ed2..0000000 Binary files a/data/GSE2630_eset.rda and /dev/null differ diff --git a/data/GSE26682.GPL570_eset.R b/data/GSE26682.GPL570_eset.R new file mode 100644 index 0000000..e49d1df --- /dev/null +++ b/data/GSE26682.GPL570_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE26682.GPL570_eset", + curatedCRCData:::.stubLoad("GSE26682.GPL570_eset"), + assign.env = environment()) diff --git a/data/GSE26682.GPL570_eset.rda b/data/GSE26682.GPL570_eset.rda deleted file mode 100644 index 68522b4..0000000 Binary files a/data/GSE26682.GPL570_eset.rda and /dev/null differ diff --git a/data/GSE26682.GPL96_eset.R b/data/GSE26682.GPL96_eset.R new file mode 100644 index 0000000..2b8b4f8 --- /dev/null +++ b/data/GSE26682.GPL96_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE26682.GPL96_eset", + curatedCRCData:::.stubLoad("GSE26682.GPL96_eset"), + assign.env = environment()) diff --git a/data/GSE26682.GPL96_eset.rda b/data/GSE26682.GPL96_eset.rda deleted file mode 100644 index a583637..0000000 Binary files a/data/GSE26682.GPL96_eset.rda and /dev/null differ diff --git a/data/GSE26906_eset.R b/data/GSE26906_eset.R new file mode 100644 index 0000000..d63f6ec --- /dev/null +++ b/data/GSE26906_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE26906_eset", + curatedCRCData:::.stubLoad("GSE26906_eset"), + assign.env = environment()) diff --git a/data/GSE26906_eset.rda b/data/GSE26906_eset.rda deleted file mode 100644 index a28e22d..0000000 Binary files a/data/GSE26906_eset.rda and /dev/null differ diff --git a/data/GSE27544_eset.R b/data/GSE27544_eset.R new file mode 100644 index 0000000..44cb3b8 --- /dev/null +++ b/data/GSE27544_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE27544_eset", + curatedCRCData:::.stubLoad("GSE27544_eset"), + assign.env = environment()) diff --git a/data/GSE27544_eset.rda b/data/GSE27544_eset.rda deleted file mode 100644 index c48f4c6..0000000 Binary files a/data/GSE27544_eset.rda and /dev/null differ diff --git a/data/GSE28702_eset.R b/data/GSE28702_eset.R new file mode 100644 index 0000000..6700c58 --- /dev/null +++ b/data/GSE28702_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE28702_eset", + curatedCRCData:::.stubLoad("GSE28702_eset"), + assign.env = environment()) diff --git a/data/GSE28702_eset.rda b/data/GSE28702_eset.rda deleted file mode 100644 index f00108a..0000000 Binary files a/data/GSE28702_eset.rda and /dev/null differ diff --git a/data/GSE3294_eset.R b/data/GSE3294_eset.R new file mode 100644 index 0000000..ab4f8b2 --- /dev/null +++ b/data/GSE3294_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE3294_eset", + curatedCRCData:::.stubLoad("GSE3294_eset"), + assign.env = environment()) diff --git a/data/GSE3294_eset.rda b/data/GSE3294_eset.rda deleted file mode 100644 index feb4285..0000000 Binary files a/data/GSE3294_eset.rda and /dev/null differ diff --git a/data/GSE33113_eset.R b/data/GSE33113_eset.R new file mode 100644 index 0000000..c344051 --- /dev/null +++ b/data/GSE33113_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE33113_eset", + curatedCRCData:::.stubLoad("GSE33113_eset"), + assign.env = environment()) diff --git a/data/GSE33113_eset.rda b/data/GSE33113_eset.rda deleted file mode 100644 index 14b2121..0000000 Binary files a/data/GSE33113_eset.rda and /dev/null differ diff --git a/data/GSE39582_eset.R b/data/GSE39582_eset.R new file mode 100644 index 0000000..d4dee3a --- /dev/null +++ b/data/GSE39582_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE39582_eset", + curatedCRCData:::.stubLoad("GSE39582_eset"), + assign.env = environment()) diff --git a/data/GSE39582_eset.rda b/data/GSE39582_eset.rda deleted file mode 100644 index 399b1af..0000000 Binary files a/data/GSE39582_eset.rda and /dev/null differ diff --git a/data/GSE3964_eset.R b/data/GSE3964_eset.R new file mode 100644 index 0000000..3530aa6 --- /dev/null +++ b/data/GSE3964_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE3964_eset", + curatedCRCData:::.stubLoad("GSE3964_eset"), + assign.env = environment()) diff --git a/data/GSE3964_eset.rda b/data/GSE3964_eset.rda deleted file mode 100644 index ae03eae..0000000 Binary files a/data/GSE3964_eset.rda and /dev/null differ diff --git a/data/GSE4045_eset.R b/data/GSE4045_eset.R new file mode 100644 index 0000000..59e7a77 --- /dev/null +++ b/data/GSE4045_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE4045_eset", + curatedCRCData:::.stubLoad("GSE4045_eset"), + assign.env = environment()) diff --git a/data/GSE4045_eset.rda b/data/GSE4045_eset.rda deleted file mode 100644 index e5c791f..0000000 Binary files a/data/GSE4045_eset.rda and /dev/null differ diff --git a/data/GSE4526_eset.R b/data/GSE4526_eset.R new file mode 100644 index 0000000..fe4e34d --- /dev/null +++ b/data/GSE4526_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE4526_eset", + curatedCRCData:::.stubLoad("GSE4526_eset"), + assign.env = environment()) diff --git a/data/GSE4526_eset.rda b/data/GSE4526_eset.rda deleted file mode 100644 index f96b94f..0000000 Binary files a/data/GSE4526_eset.rda and /dev/null differ diff --git a/data/GSE45270_eset.R b/data/GSE45270_eset.R new file mode 100644 index 0000000..f7e8e20 --- /dev/null +++ b/data/GSE45270_eset.R @@ -0,0 +1,3 @@ +delayedAssign("GSE45270_eset", + curatedCRCData:::.stubLoad("GSE45270_eset"), + assign.env = environment()) diff --git a/data/GSE45270_eset.rda b/data/GSE45270_eset.rda deleted file mode 100644 index ead6cac..0000000 Binary files a/data/GSE45270_eset.rda and /dev/null differ diff --git a/data/TCGA.COAD_eset.R b/data/TCGA.COAD_eset.R new file mode 100644 index 0000000..01966ae --- /dev/null +++ b/data/TCGA.COAD_eset.R @@ -0,0 +1,3 @@ +delayedAssign("TCGA.COAD_eset", + curatedCRCData:::.stubLoad("TCGA.COAD_eset"), + assign.env = environment()) diff --git a/data/TCGA.COAD_eset.rda b/data/TCGA.COAD_eset.rda deleted file mode 100644 index 9af1253..0000000 Binary files a/data/TCGA.COAD_eset.rda and /dev/null differ diff --git a/data/TCGA.READ_eset.R b/data/TCGA.READ_eset.R new file mode 100644 index 0000000..56c1441 --- /dev/null +++ b/data/TCGA.READ_eset.R @@ -0,0 +1,3 @@ +delayedAssign("TCGA.READ_eset", + curatedCRCData:::.stubLoad("TCGA.READ_eset"), + assign.env = environment()) diff --git a/data/TCGA.READ_eset.rda b/data/TCGA.READ_eset.rda deleted file mode 100644 index a7eb4b5..0000000 Binary files a/data/TCGA.READ_eset.rda and /dev/null differ diff --git a/data/TCGA.RNASeqV2.READ_eset.R b/data/TCGA.RNASeqV2.READ_eset.R new file mode 100644 index 0000000..79c833f --- /dev/null +++ b/data/TCGA.RNASeqV2.READ_eset.R @@ -0,0 +1,3 @@ +delayedAssign("TCGA.RNASeqV2.READ_eset", + curatedCRCData:::.stubLoad("TCGA.RNASeqV2.READ_eset"), + assign.env = environment()) diff --git a/data/TCGA.RNASeqV2.READ_eset.rda b/data/TCGA.RNASeqV2.READ_eset.rda deleted file mode 100644 index 26d389e..0000000 Binary files a/data/TCGA.RNASeqV2.READ_eset.rda and /dev/null differ diff --git a/data/TCGA.RNASeqV2_eset.R b/data/TCGA.RNASeqV2_eset.R new file mode 100644 index 0000000..97a3a14 --- /dev/null +++ b/data/TCGA.RNASeqV2_eset.R @@ -0,0 +1,3 @@ +delayedAssign("TCGA.RNASeqV2_eset", + curatedCRCData:::.stubLoad("TCGA.RNASeqV2_eset"), + assign.env = environment()) diff --git a/data/TCGA.RNASeqV2_eset.rda b/data/TCGA.RNASeqV2_eset.rda deleted file mode 100644 index 6305653..0000000 Binary files a/data/TCGA.RNASeqV2_eset.rda and /dev/null differ diff --git a/data/datalist b/data/datalist index 90bf526..0c8d987 100644 --- a/data/datalist +++ b/data/datalist @@ -30,5 +30,5 @@ GSE4526_eset GSE45270_eset TCGA.COAD_eset TCGA.READ_eset -TCGA.RNASeqV2.READ_eset TCGA.RNASeqV2_eset +TCGA.RNASeqV2.READ_eset diff --git a/external_data_store.txt b/external_data_store.txt deleted file mode 100644 index 0e0bceb..0000000 --- a/external_data_store.txt +++ /dev/null @@ -1,2 +0,0 @@ -data -inst/extdata diff --git a/inst/extdata/createEsetList.R b/inst/extdata/createEsetList.R index 6d0f0af..70d35b1 100644 --- a/inst/extdata/createEsetList.R +++ b/inst/extdata/createEsetList.R @@ -65,7 +65,10 @@ intersectMany <- function(lst){ } ##load the esets -data(list=data(package=package.name)[[3]][,3]) +if (!exists("test.mode")) test.mode <- FALSE +for (.nm in do.call(package.name, list())) + assign(.nm, do.call(package.name, list(datasets = .nm, test = test.mode))) +rm(.nm) strEsets <- ls(pattern="^.*_eset$") @@ -145,8 +148,8 @@ if(keep.common.only){ }) } -ids.with.missing.data <- which(sapply(esets, function(X) -sum(!complete.cases(exprs(X))) > 0)) +ids.with.missing.data <- which(vapply(esets, function(X) +sum(!complete.cases(exprs(X))) > 0, logical(1))) loginfo(paste("Ids with missing data:", paste(names(ids.with.missing.data), collapse=", "))) diff --git a/inst/extdata/patientselection.config b/inst/extdata/patientselection.config index 087ff2c..9085329 100644 --- a/inst/extdata/patientselection.config +++ b/inst/extdata/patientselection.config @@ -2,6 +2,10 @@ ##settings for createEsetList.R ##----------------------------- +#use the small offline test subsets bundled with the package instead of +#downloading the full datasets from Zenodo? (for examples/vignettes/tests) +test.mode <- FALSE + #remove retracted studies? remove.retracted <- FALSE diff --git a/inst/extdata/patientselection_all.config b/inst/extdata/patientselection_all.config index fed7a06..c06460e 100755 --- a/inst/extdata/patientselection_all.config +++ b/inst/extdata/patientselection_all.config @@ -2,6 +2,10 @@ ##settings for createEsetList.R ##----------------------------- +#use the small offline test subsets bundled with the package instead of +#downloading the full datasets from Zenodo? (for examples/vignettes/tests) +test.mode <- FALSE + #rescale each gene to z-score? rescale <- FALSE diff --git a/inst/extdata/testdata/GSE11237_eset.rda b/inst/extdata/testdata/GSE11237_eset.rda new file mode 100644 index 0000000..100a518 Binary files /dev/null and b/inst/extdata/testdata/GSE11237_eset.rda differ diff --git a/inst/extdata/testdata/GSE12225.GPL3676_eset.rda b/inst/extdata/testdata/GSE12225.GPL3676_eset.rda new file mode 100644 index 0000000..cebf165 Binary files /dev/null and b/inst/extdata/testdata/GSE12225.GPL3676_eset.rda differ diff --git a/inst/extdata/testdata/GSE12945_eset.rda b/inst/extdata/testdata/GSE12945_eset.rda new file mode 100644 index 0000000..ffeea06 Binary files /dev/null and b/inst/extdata/testdata/GSE12945_eset.rda differ diff --git a/inst/extdata/testdata/GSE13067_eset.rda b/inst/extdata/testdata/GSE13067_eset.rda new file mode 100644 index 0000000..0955086 Binary files /dev/null and b/inst/extdata/testdata/GSE13067_eset.rda differ diff --git a/inst/extdata/testdata/GSE13294_eset.rda b/inst/extdata/testdata/GSE13294_eset.rda new file mode 100644 index 0000000..50ebbca Binary files /dev/null and b/inst/extdata/testdata/GSE13294_eset.rda differ diff --git a/inst/extdata/testdata/GSE14095_eset.rda b/inst/extdata/testdata/GSE14095_eset.rda new file mode 100644 index 0000000..c2a8604 Binary files /dev/null and b/inst/extdata/testdata/GSE14095_eset.rda differ diff --git a/inst/extdata/testdata/GSE14333_eset.rda b/inst/extdata/testdata/GSE14333_eset.rda new file mode 100644 index 0000000..e534792 Binary files /dev/null and b/inst/extdata/testdata/GSE14333_eset.rda differ diff --git a/inst/extdata/testdata/GSE16125.GPL5175_eset.rda b/inst/extdata/testdata/GSE16125.GPL5175_eset.rda new file mode 100644 index 0000000..8c8057f Binary files /dev/null and b/inst/extdata/testdata/GSE16125.GPL5175_eset.rda differ diff --git a/inst/extdata/testdata/GSE17536_eset.rda b/inst/extdata/testdata/GSE17536_eset.rda new file mode 100644 index 0000000..0922bfb Binary files /dev/null and b/inst/extdata/testdata/GSE17536_eset.rda differ diff --git a/inst/extdata/testdata/GSE17537_eset.rda b/inst/extdata/testdata/GSE17537_eset.rda new file mode 100644 index 0000000..96690db Binary files /dev/null and b/inst/extdata/testdata/GSE17537_eset.rda differ diff --git a/inst/extdata/testdata/GSE17538.GPL570_eset.rda b/inst/extdata/testdata/GSE17538.GPL570_eset.rda new file mode 100644 index 0000000..52a7972 Binary files /dev/null and b/inst/extdata/testdata/GSE17538.GPL570_eset.rda differ diff --git a/inst/extdata/testdata/GSE18105_eset.rda b/inst/extdata/testdata/GSE18105_eset.rda new file mode 100644 index 0000000..13dce90 Binary files /dev/null and b/inst/extdata/testdata/GSE18105_eset.rda differ diff --git a/inst/extdata/testdata/GSE2109_eset.rda b/inst/extdata/testdata/GSE2109_eset.rda new file mode 100644 index 0000000..dd670bb Binary files /dev/null and b/inst/extdata/testdata/GSE2109_eset.rda differ diff --git a/inst/extdata/testdata/GSE21510_eset.rda b/inst/extdata/testdata/GSE21510_eset.rda new file mode 100644 index 0000000..5e5532d Binary files /dev/null and b/inst/extdata/testdata/GSE21510_eset.rda differ diff --git a/inst/extdata/testdata/GSE21815_eset.rda b/inst/extdata/testdata/GSE21815_eset.rda new file mode 100644 index 0000000..9647d2e Binary files /dev/null and b/inst/extdata/testdata/GSE21815_eset.rda differ diff --git a/inst/extdata/testdata/GSE24549.GPL5175_eset.rda b/inst/extdata/testdata/GSE24549.GPL5175_eset.rda new file mode 100644 index 0000000..3a3e775 Binary files /dev/null and b/inst/extdata/testdata/GSE24549.GPL5175_eset.rda differ diff --git a/inst/extdata/testdata/GSE24550.GPL5175_eset.rda b/inst/extdata/testdata/GSE24550.GPL5175_eset.rda new file mode 100644 index 0000000..84e253b Binary files /dev/null and b/inst/extdata/testdata/GSE24550.GPL5175_eset.rda differ diff --git a/inst/extdata/testdata/GSE2630_eset.rda b/inst/extdata/testdata/GSE2630_eset.rda new file mode 100644 index 0000000..d560b7d Binary files /dev/null and b/inst/extdata/testdata/GSE2630_eset.rda differ diff --git a/inst/extdata/testdata/GSE26682.GPL570_eset.rda b/inst/extdata/testdata/GSE26682.GPL570_eset.rda new file mode 100644 index 0000000..ad16193 Binary files /dev/null and b/inst/extdata/testdata/GSE26682.GPL570_eset.rda differ diff --git a/inst/extdata/testdata/GSE26682.GPL96_eset.rda b/inst/extdata/testdata/GSE26682.GPL96_eset.rda new file mode 100644 index 0000000..32096b0 Binary files /dev/null and b/inst/extdata/testdata/GSE26682.GPL96_eset.rda differ diff --git a/inst/extdata/testdata/GSE26906_eset.rda b/inst/extdata/testdata/GSE26906_eset.rda new file mode 100644 index 0000000..58d555b Binary files /dev/null and b/inst/extdata/testdata/GSE26906_eset.rda differ diff --git a/inst/extdata/testdata/GSE27544_eset.rda b/inst/extdata/testdata/GSE27544_eset.rda new file mode 100644 index 0000000..f20a7bb Binary files /dev/null and b/inst/extdata/testdata/GSE27544_eset.rda differ diff --git a/inst/extdata/testdata/GSE28702_eset.rda b/inst/extdata/testdata/GSE28702_eset.rda new file mode 100644 index 0000000..e392cdd Binary files /dev/null and b/inst/extdata/testdata/GSE28702_eset.rda differ diff --git a/inst/extdata/testdata/GSE3294_eset.rda b/inst/extdata/testdata/GSE3294_eset.rda new file mode 100644 index 0000000..d9cc980 Binary files /dev/null and b/inst/extdata/testdata/GSE3294_eset.rda differ diff --git a/inst/extdata/testdata/GSE33113_eset.rda b/inst/extdata/testdata/GSE33113_eset.rda new file mode 100644 index 0000000..2d8c006 Binary files /dev/null and b/inst/extdata/testdata/GSE33113_eset.rda differ diff --git a/inst/extdata/testdata/GSE39582_eset.rda b/inst/extdata/testdata/GSE39582_eset.rda new file mode 100644 index 0000000..f69e155 Binary files /dev/null and b/inst/extdata/testdata/GSE39582_eset.rda differ diff --git a/inst/extdata/testdata/GSE3964_eset.rda b/inst/extdata/testdata/GSE3964_eset.rda new file mode 100644 index 0000000..76634df Binary files /dev/null and b/inst/extdata/testdata/GSE3964_eset.rda differ diff --git a/inst/extdata/testdata/GSE4045_eset.rda b/inst/extdata/testdata/GSE4045_eset.rda new file mode 100644 index 0000000..c1e0423 Binary files /dev/null and b/inst/extdata/testdata/GSE4045_eset.rda differ diff --git a/inst/extdata/testdata/GSE4526_eset.rda b/inst/extdata/testdata/GSE4526_eset.rda new file mode 100644 index 0000000..0ba3fee Binary files /dev/null and b/inst/extdata/testdata/GSE4526_eset.rda differ diff --git a/inst/extdata/testdata/GSE45270_eset.rda b/inst/extdata/testdata/GSE45270_eset.rda new file mode 100644 index 0000000..c3cd524 Binary files /dev/null and b/inst/extdata/testdata/GSE45270_eset.rda differ diff --git a/inst/extdata/testdata/TCGA.COAD_eset.rda b/inst/extdata/testdata/TCGA.COAD_eset.rda new file mode 100644 index 0000000..bc1d297 Binary files /dev/null and b/inst/extdata/testdata/TCGA.COAD_eset.rda differ diff --git a/inst/extdata/testdata/TCGA.READ_eset.rda b/inst/extdata/testdata/TCGA.READ_eset.rda new file mode 100644 index 0000000..1a74d9e Binary files /dev/null and b/inst/extdata/testdata/TCGA.READ_eset.rda differ diff --git a/inst/extdata/testdata/TCGA.RNASeqV2.READ_eset.rda b/inst/extdata/testdata/TCGA.RNASeqV2.READ_eset.rda new file mode 100644 index 0000000..7ae667d Binary files /dev/null and b/inst/extdata/testdata/TCGA.RNASeqV2.READ_eset.rda differ diff --git a/inst/extdata/testdata/TCGA.RNASeqV2_eset.rda b/inst/extdata/testdata/TCGA.RNASeqV2_eset.rda new file mode 100644 index 0000000..a29610b Binary files /dev/null and b/inst/extdata/testdata/TCGA.RNASeqV2_eset.rda differ diff --git a/inst/extdata/zenodo-manifest.csv b/inst/extdata/zenodo-manifest.csv new file mode 100644 index 0000000..f61fa17 --- /dev/null +++ b/inst/extdata/zenodo-manifest.csv @@ -0,0 +1,35 @@ +dataset,filename,url,md5,size_bytes +GSE11237_eset,GSE11237_eset.rda,https://zenodo.org/records/23125734/files/GSE11237_eset.rda?download=1,307f4c572b663edaa8c65e620498d8e5,1462208 +GSE12225.GPL3676_eset,GSE12225.GPL3676_eset.rda,https://zenodo.org/records/23125734/files/GSE12225.GPL3676_eset.rda?download=1,ba5e2d45c597ae6e166fbfc7ce27ba44,1321808 +GSE12945_eset,GSE12945_eset.rda,https://zenodo.org/records/23125734/files/GSE12945_eset.rda?download=1,06747a29b28a2f3599c25d53cf1acbf5,5676968 +GSE13067_eset,GSE13067_eset.rda,https://zenodo.org/records/23125734/files/GSE13067_eset.rda?download=1,9ea4179c37ab2eea0f5779b9e5b9f8ef,10096264 +GSE13294_eset,GSE13294_eset.rda,https://zenodo.org/records/23125734/files/GSE13294_eset.rda?download=1,5919e1903d672ebb074ea98c5c773703,6862824 +GSE14095_eset,GSE14095_eset.rda,https://zenodo.org/records/23125734/files/GSE14095_eset.rda?download=1,902d4872308ee51e0764015fac2b062c,6909552 +GSE14333_eset,GSE14333_eset.rda,https://zenodo.org/records/23125734/files/GSE14333_eset.rda?download=1,ccc1735fb1d9f664afa77fc5d801d8b8,12431512 +GSE16125.GPL5175_eset,GSE16125.GPL5175_eset.rda,https://zenodo.org/records/23125734/files/GSE16125.GPL5175_eset.rda?download=1,aba783ba87e66bde68e5307a120c1a2e,1080260 +GSE17536_eset,GSE17536_eset.rda,https://zenodo.org/records/23125734/files/GSE17536_eset.rda?download=1,bbdaf3e0d0c3682e2daf9fe19d2a41ed,18625140 +GSE17537_eset,GSE17537_eset.rda,https://zenodo.org/records/23125734/files/GSE17537_eset.rda?download=1,0f47e4d4adf5815c0cb7f04e21fa9f25,7545656 +GSE17538.GPL570_eset,GSE17538.GPL570_eset.rda,https://zenodo.org/records/23125734/files/GSE17538.GPL570_eset.rda?download=1,659686dcd2a78d7726f8d21894384b65,23588252 +GSE18105_eset,GSE18105_eset.rda,https://zenodo.org/records/23125734/files/GSE18105_eset.rda?download=1,b7d2e879f1fef1ef8e0c0a184e09a52c,15109988 +GSE2109_eset,GSE2109_eset.rda,https://zenodo.org/records/23125734/files/GSE2109_eset.rda?download=1,9124df5c503a1e020b789a6c2cb3521f,18402064 +GSE21510_eset,GSE21510_eset.rda,https://zenodo.org/records/23125734/files/GSE21510_eset.rda?download=1,fa48020c10885777624d7007fb5f2896,20090392 +GSE21815_eset,GSE21815_eset.rda,https://zenodo.org/records/23125734/files/GSE21815_eset.rda?download=1,8739e1111a1604c4d733d58a270a241c,18108200 +GSE24549.GPL5175_eset,GSE24549.GPL5175_eset.rda,https://zenodo.org/records/23125734/files/GSE24549.GPL5175_eset.rda?download=1,b41298ddb66e3703352867ce8f2391e2,2379240 +GSE24550.GPL5175_eset,GSE24550.GPL5175_eset.rda,https://zenodo.org/records/23125734/files/GSE24550.GPL5175_eset.rda?download=1,3df5a16f0d90505439f92b92608aac58,2555172 +GSE2630_eset,GSE2630_eset.rda,https://zenodo.org/records/23125734/files/GSE2630_eset.rda?download=1,c650169bc58e13b6a1be326155c2a2a4,543940 +GSE26682.GPL570_eset,GSE26682.GPL570_eset.rda,https://zenodo.org/records/23125734/files/GSE26682.GPL570_eset.rda?download=1,4a552232901ae8036e2d390211ac66cd,3762880 +GSE26682.GPL96_eset,GSE26682.GPL96_eset.rda,https://zenodo.org/records/23125734/files/GSE26682.GPL96_eset.rda?download=1,440c10c04fa09fe56c650f5dcfad84fb,14050244 +GSE26906_eset,GSE26906_eset.rda,https://zenodo.org/records/23125734/files/GSE26906_eset.rda?download=1,3399ec7149ef6a76a7e6d77a21ba36bb,12271168 +GSE27544_eset,GSE27544_eset.rda,https://zenodo.org/records/23125734/files/GSE27544_eset.rda?download=1,ca61888b627394a753d6f93e6ea0eb08,1399976 +GSE28702_eset,GSE28702_eset.rda,https://zenodo.org/records/23125734/files/GSE28702_eset.rda?download=1,852f42b9d3ee28d3bc6076747df5aa38,11314752 +GSE3294_eset,GSE3294_eset.rda,https://zenodo.org/records/23125734/files/GSE3294_eset.rda?download=1,f8f62bce4fa9f8d8c27a8331defb118c,684632 +GSE33113_eset,GSE33113_eset.rda,https://zenodo.org/records/23125734/files/GSE33113_eset.rda?download=1,9814126b6fc740de2713e1fec4b15782,13066544 +GSE39582_eset,GSE39582_eset.rda,https://zenodo.org/records/23125734/files/GSE39582_eset.rda?download=1,7bdbbadfaa9d4ff88bdc791042287a9f,53940012 +GSE3964_eset,GSE3964_eset.rda,https://zenodo.org/records/23125734/files/GSE3964_eset.rda?download=1,0a73a4dda729f2dfccb26d658d6cbb8a,1308480 +GSE4045_eset,GSE4045_eset.rda,https://zenodo.org/records/23125734/files/GSE4045_eset.rda?download=1,55536e548b97689ad3fd7ab44fb33418,3432252 +GSE4526_eset,GSE4526_eset.rda,https://zenodo.org/records/23125734/files/GSE4526_eset.rda?download=1,194e39c56aa6e63eb926fd2d02c1b8e8,4499620 +GSE45270_eset,GSE45270_eset.rda,https://zenodo.org/records/23125734/files/GSE45270_eset.rda?download=1,0b1203cb8c724a0b5e8e33b63264ef94,1866416 +TCGA.COAD_eset,TCGA.COAD_eset.rda,https://zenodo.org/records/23125734/files/TCGA.COAD_eset.rda?download=1,4865921e88a3e140dc33535df3d9ff47,5730984 +TCGA.READ_eset,TCGA.READ_eset.rda,https://zenodo.org/records/23125734/files/TCGA.READ_eset.rda?download=1,d1a6688d116f0b5a4229d4d427efb8a5,2326880 +TCGA.RNASeqV2_eset,TCGA.RNASeqV2_eset.rda,https://zenodo.org/records/23125734/files/TCGA.RNASeqV2_eset.rda?download=1,982129545ba576976d54e63e2e9077a8,12831588 +TCGA.RNASeqV2.READ_eset,TCGA.RNASeqV2.READ_eset.rda,https://zenodo.org/records/23125734/files/TCGA.RNASeqV2.READ_eset.rda?download=1,b7e99d4b002ac703ca19cb508d91a5e8,525084 diff --git a/inst/scripts/data-integrity-report.Rmd b/inst/scripts/data-integrity-report.Rmd new file mode 100644 index 0000000..30fc9a8 --- /dev/null +++ b/inst/scripts/data-integrity-report.Rmd @@ -0,0 +1,91 @@ +--- +title: "Data integrity report: Zenodo offloading" +output: html_document +params: + package: "curatedCRCData" + upload_dir: "../../../curatedCRCData-zenodo-upload" +--- + + + +```{r setup, include=FALSE} +knitr::opts_chunk$set(echo = TRUE) +suppressPackageStartupMessages(library(Biobase)) +pkg <- params$package +stopifnot(requireNamespace(pkg)) +``` + +# 1. Uploaded objects vs pre-refactor package objects + +```{r compare-objects} +pre <- tempfile("pre-refactor-") +system2("git", c("worktree", "add", "--detach", pre, "pre-zenodo-refactor"), + stdout = FALSE) + +manifest <- read.csv(file.path("..", "extdata", "zenodo-manifest.csv")) +loadOne <- function(path) { + env <- new.env(); nm <- load(path, envir = env); get(nm[1], envir = env) +} +cmp <- vapply(manifest$dataset, function(nm) { + old <- loadOne(file.path(pre, "data", paste0(nm, ".rda"))) + new <- loadOne(file.path(params$upload_dir, paste0(nm, ".rda"))) + isTRUE(all.equal(exprs(old), exprs(new))) && + isTRUE(all.equal(pData(old), pData(new))) && + isTRUE(all.equal(fData(old), fData(new))) && + identical(annotation(old), annotation(new)) && + isTRUE(all.equal(experimentData(old), experimentData(new))) +}, logical(1)) +system2("git", c("worktree", "remove", "--force", pre)) +data.frame(dataset = manifest$dataset, identical = cmp) +stopifnot(all(cmp)) +``` + +# 2. Manifest md5s vs local uploads and Zenodo + +```{r compare-md5} +local_md5 <- tools::md5sum(file.path(params$upload_dir, manifest$filename)) +stopifnot(identical(unname(local_md5), manifest$md5)) + +record <- unique(sub(".*records/([0-9A-Za-z_]+)/.*", "\\1", manifest$url)) +stopifnot(length(record) == 1L) +api <- jsonlite::fromJSON(paste0("https://zenodo.org/api/records/", record)) +zen <- data.frame(filename = api$files$key, + md5 = sub("^md5:", "", api$files$checksum)) +merged <- merge(manifest[, c("filename", "md5")], zen, by = "filename", + suffixes = c(".manifest", ".zenodo")) +stopifnot(nrow(merged) == nrow(manifest), + identical(merged$md5.manifest, merged$md5.zenodo)) +merged +``` + +# 3. Download smoke test + +```{r smoke-test} +getter <- get(pkg, envir = asNamespace(pkg)) +smallest <- manifest$dataset[which.min(manifest$size_bytes)] +unlink(tools::R_user_dir(pkg, "cache"), recursive = TRUE) +eset <- getter(smallest) # downloads +stopifnot(is(eset, "ExpressionSet")) +eset2 <- getter(smallest) # must hit the cache silently +stopifnot(identical(dim(eset), dim(eset2))) +dim(eset) +``` + +```{r session-info} +sessionInfo() +``` diff --git a/inst/scripts/make-data.R b/inst/scripts/make-data.R new file mode 100644 index 0000000..aa6d536 --- /dev/null +++ b/inst/scripts/make-data.R @@ -0,0 +1,68 @@ +## Prepare the per-dataset .rda files uploaded to Zenodo, along with the +## download manifest, the data/ stub files, and data/datalist. +## +## The datasets themselves were produced by the curatedCRCData curation +## pipeline (see the package vignette and inst/extdata/template_crc.csv); +## this script re-serializes the .rda files that shipped inside the package +## through version 2.45.x (git tag: pre-zenodo-refactor) with xz compression +## for hosting on Zenodo. +## +## Run from the package root with the full data/*.rda files present: +## Rscript inst/scripts/make-data.R # before upload (placeholder URLs) +## Rscript inst/scripts/make-data.R # after publishing, to finalize URLs +## Already-compressed files in the output directory are left untouched, so +## re-running with the record ID only rewrites the manifest. Do NOT +## regenerate the .rda files after uploading (checksums must match Zenodo). +## +## NOTE: this script is intentionally duplicated in curatedOvarianData. + +pkg <- unname(read.dcf("DESCRIPTION")[, "Package"]) +record <- commandArgs(trailingOnly = TRUE) +record <- if (length(record)) record[1L] else "RECORD_ID" + +outdir <- file.path("..", paste0(pkg, "-zenodo-upload")) +stubdir <- file.path(outdir, "stubs") +dir.create(stubdir, recursive = TRUE, showWarnings = FALSE) + +## enumerate from data/ before the refactor, or from the already-compressed +## upload directory when re-running to finalize URLs +rdas <- sort(list.files("data", pattern = "\\.rda$")) +if (length(rdas) == 0L) + rdas <- sort(list.files(outdir, pattern = "\\.rda$")) +stopifnot(length(rdas) > 0L) + +rows <- lapply(rdas, function(f) { + nm <- sub("\\.rda$", "", f) + out <- file.path(outdir, f) + if (!file.exists(out)) { + env <- new.env(parent = emptyenv()) + obj <- load(file.path("data", f), envir = env) + stopifnot(identical(obj, nm)) + save(list = nm, envir = env, file = out, compress = "xz") + message("compressed ", f) + } + writeLines(sprintf( + 'delayedAssign("%s",\n %s:::.stubLoad("%s"),\n assign.env = environment())', + nm, pkg, nm), + file.path(stubdir, paste0(nm, ".R"))) + data.frame(dataset = nm, filename = f, + url = sprintf("https://zenodo.org/records/%s/files/%s?download=1", + record, f), + md5 = unname(tools::md5sum(out)), size_bytes = file.size(out)) +}) +manifest <- do.call(rbind, rows) + +write.csv(manifest, file.path(outdir, "zenodo-manifest.csv"), + row.names = FALSE, quote = FALSE) +writeLines(manifest$dataset, file.path(stubdir, "datalist")) +writeLines(sprintf("%s %12d %s", manifest$md5, manifest$size_bytes, + manifest$filename), + file.path(outdir, "upload-manifest.txt")) + +message(nrow(manifest), " datasets; total upload size ", + round(sum(manifest$size_bytes) / 2^20), " MB; written to ", outdir) +if (identical(record, "RECORD_ID")) + message("Upload the .rda files to a new Zenodo record, publish it, ", + "verify the md5s shown by Zenodo against upload-manifest.txt, ", + "then re-run this script with the record ID and copy ", + "zenodo-manifest.csv to inst/extdata/.") diff --git a/inst/scripts/make-test-data.R b/inst/scripts/make-test-data.R new file mode 100644 index 0000000..f0ebe97 --- /dev/null +++ b/inst/scripts/make-test-data.R @@ -0,0 +1,39 @@ +## Build the small offline test fixtures in inst/extdata/testdata/: one per +## dataset, subset to the first <=200 features and <=100 samples with full +## phenoData retained, so that examples, tests, vignettes, and R CMD check +## run without network access (see the test= argument of the getter and +## .stubLoad in R/getData.R). +## +## Run from the package root with the full data/*.rda files present +## (git tag: pre-zenodo-refactor): +## Rscript inst/scripts/make-test-data.R +## +## NOTE: this script is intentionally duplicated in curatedOvarianData. + +suppressPackageStartupMessages(library(Biobase)) + +dir.create("inst/extdata/testdata", recursive = TRUE, showWarnings = FALSE) +rdas <- sort(list.files("data", pattern = "\\.rda$")) +stopifnot(length(rdas) > 0L) + +for (f in rdas) { + nm <- sub("\\.rda$", "", f) + env <- new.env(parent = emptyenv()) + obj <- load(file.path("data", f), envir = env) + stopifnot(identical(obj, nm)) + eset <- get(nm, envir = env) + ## keep the first ~200 features plus genes used in the vignettes + ## (survival forest plots, probe-expansion examples) + fn <- featureNames(eset) + keep <- unique(c(intersect(c("CXCL12", "CXCR4"), fn), + head(grep("///", fn, value = TRUE), 5L), + head(fn, 200L))) + fx <- eset[keep, seq_len(min(100L, ncol(eset)))] + assign(nm, fx) + save(list = nm, file = file.path("inst/extdata/testdata", f), + compress = "xz") + message("fixture ", nm, ": ", nrow(fx), " x ", ncol(fx)) +} +message("total fixture size: ", + round(sum(file.size(list.files("inst/extdata/testdata", + full.names = TRUE))) / 2^10), " KB") diff --git a/inst/unitTests/test_checkCurated.R b/inst/unitTests/test_checkCurated.R deleted file mode 100644 index 4567d49..0000000 --- a/inst/unitTests/test_checkCurated.R +++ /dev/null @@ -1,105 +0,0 @@ -test_checkCurated <- function(){ - library(affy) - ##the template to use - ##template.file <- "../extdata/template_ov.csv" - template.file <- system.file("extdata/template_crc.csv", package = "curatedCRCData") - template <- read.csv(template.file, as.is=TRUE) - - ##get rid of the first row of template - this is sample_name - template <- template[-1, ] - - ##the list of datasets to check - dataset.names <- data(package="curatedCRCData")$results[, "Item"] - - ##Create a checklist for three checks in all files - checklist <- matrix(NA,ncol=2,nrow=length(dataset.names)) - colnames(checklist) <- c("column.names","variable.regex.checks") - rownames(checklist) <- dataset.names - - ##Make a list of curated metadata: - curated <- lapply(dataset.names, function(x){ - data(list=x, package="curatedCRCData") - pData(get(x)) - }) - names(curated) <- dataset.names - - - ##------------------------------------------------- - ##check that the column names match the template - ##------------------------------------------------- - for (i in 1:length(curated)){ - print(paste("Checking column names in",dataset.names[i])) - ##check column names - if(identical(all.equal(colnames(curated[[i]]),template$col.name),TRUE)){ - print("column names OK") - checklist[i,"column.names"] <- TRUE - }else{ - all.equal(colnames(curated[[i]]),template$col.name) - print("Column names should be: ") - print(template$col.name) - print("Column names are: ") - print(colnames(curated[[i]])) - checklist[i,"column.names"] <- FALSE - } - } - - ##------------------------------------------------- - ##construct the regexes from template$allowedvalues - ##------------------------------------------------- - regexes <- template$allowedvalues - regexes <- paste("^",regexes,"$",sep="") - regexes <- gsub("|","$|^",regexes,fixed=TRUE) - ##regexes[template$requiredness=="optional"] <- paste(regexes[template$requiredness=="optional"],"|^NA$",sep="") - regexes[grep("^*$",regexes,fixed=TRUE)] <- "." - names(regexes) <- template$col.name - regexes[grep("decimal",regexes)] <- "^[0-9]+\\.?[0-9]*$" - - ##------------------------------------------------- - ##Check the data entries in each column for regex - ## matching, uniqueness, and missingness - ##------------------------------------------------- - for (i in which(checklist[,"column.names"])){ - print(paste("Checking regexes in",dataset.names[i])) - ##check column names - column.OK <- rep(NA,ncol(curated[[i]])) - names(column.OK) <- colnames(curated[[i]]) - for(j in 1:ncol(curated[[i]])){ - doesmatch <- grep(regexes[j],curated[[i]][,j]) - if(template[j,"requiredness"]=="optional"){ - doesmatch <- c(doesmatch,which(is.na(curated[[i]][,j]))) - } - doesnotmatch <- 1:nrow(curated[[i]]) - doesnotmatch <- doesnotmatch[!doesnotmatch %in% doesmatch] - ## if field must be unique, add non-unique values to doesnotmatch - if(template[j,"uniqueness"]=="unique"){ - counts.table <- table(curated[[i]][,j]) - counts <- counts.table[match(curated[[i]][,j],names(counts.table))] #this counts the occurences - counts[is.na(counts)] <- 1 ##Consider NAs to be unique - nonunique <- which(counts>1) - doesnotmatch <- c(doesnotmatch,nonunique) - doesnotmatch <- unique(doesnotmatch) #don't duplicate fields which fail both tests - } - if(length(doesnotmatch)==0){ - column.OK[j] <- TRUE - }else{ - column.OK[j] <- FALSE - curated[[i]][doesnotmatch,j] <- paste("!!!",curated[[i]][doesnotmatch,j],"!!!",sep="") - } - } - if(all(column.OK)){ - print("all regex checks OK") - checklist[i,"variable.regex.checks"] <- TRUE - }else{ - print(paste("The following columns failed the regex check for ",dataset.names[i],":",sep="")) - print(names(column.OK)[!column.OK]) - print(curated[[i]][,!column.OK]) - checklist[i,"variable.regex.checks"] <- FALSE - } - } - - print(checklist) - - checkTrue( all(checklist[, "column.names"]) ) - checkTrue( all(checklist[, "variable.regex.checks"]) ) - -} diff --git a/inst/unitTests/test_checkCurated.Rout b/inst/unitTests/test_checkCurated.Rout deleted file mode 100644 index 5b3037e..0000000 --- a/inst/unitTests/test_checkCurated.Rout +++ /dev/null @@ -1,128 +0,0 @@ - -R version 3.0.1 (2013-05-16) -- "Good Sport" -Copyright (C) 2013 The R Foundation for Statistical Computing -Platform: x86_64-pc-linux-gnu (64-bit) - -R is free software and comes with ABSOLUTELY NO WARRANTY. -You are welcome to redistribute it under certain conditions. -Type 'license()' or 'licence()' for distribution details. - - Natural language support but running in an English locale - -R is a collaborative project with many contributors. -Type 'contributors()' for more information and -'citation()' on how to cite R or R packages in publications. - -Type 'demo()' for some demos, 'help()' for on-line help, or -'help.start()' for an HTML browser interface to help. -Type 'q()' to quit R. - -> test_checkCurated <- function(){ -+ library(affy) -+ ##the template to use -+ ##template.file <- "../extdata/template_ov.csv" -+ template.file <- system.file("extdata/template_crc.csv", package = "curatedCRCData") -+ template <- read.csv(template.file, as.is=TRUE) -+ -+ ##get rid of the first row of template - this is sample_name -+ template <- template[-1, ] -+ -+ ##the list of datasets to check -+ dataset.names <- data(package="curatedCRCData")$results[, "Item"] -+ -+ ##Create a checklist for three checks in all files -+ checklist <- matrix(NA,ncol=2,nrow=length(dataset.names)) -+ colnames(checklist) <- c("column.names","variable.regex.checks") -+ rownames(checklist) <- dataset.names -+ -+ ##Make a list of curated metadata: -+ curated <- lapply(dataset.names, function(x){ -+ data(list=x, package="curatedCRCData") -+ pData(get(x)) -+ }) -+ names(curated) <- dataset.names -+ -+ -+ ##------------------------------------------------- -+ ##check that the column names match the template -+ ##------------------------------------------------- -+ for (i in 1:length(curated)){ -+ print(paste("Checking column names in",dataset.names[i])) -+ ##check column names -+ if(identical(all.equal(colnames(curated[[i]]),template$col.name),TRUE)){ -+ print("column names OK") -+ checklist[i,"column.names"] <- TRUE -+ }else{ -+ all.equal(colnames(curated[[i]]),template$col.name) -+ print("Column names should be: ") -+ print(template$col.name) -+ print("Column names are: ") -+ print(colnames(curated[[i]])) -+ checklist[i,"column.names"] <- FALSE -+ } -+ } -+ -+ ##------------------------------------------------- -+ ##construct the regexes from template$allowedvalues -+ ##------------------------------------------------- -+ regexes <- template$allowedvalues -+ regexes <- paste("^",regexes,"$",sep="") -+ regexes <- gsub("|","$|^",regexes,fixed=TRUE) -+ ##regexes[template$requiredness=="optional"] <- paste(regexes[template$requiredness=="optional"],"|^NA$",sep="") -+ regexes[grep("^*$",regexes,fixed=TRUE)] <- "." -+ names(regexes) <- template$col.name -+ regexes[grep("decimal",regexes)] <- "^[0-9]+\\.?[0-9]*$" -+ -+ ##------------------------------------------------- -+ ##Check the data entries in each column for regex -+ ## matching, uniqueness, and missingness -+ ##------------------------------------------------- -+ for (i in which(checklist[,"column.names"])){ -+ print(paste("Checking regexes in",dataset.names[i])) -+ ##check column names -+ column.OK <- rep(NA,ncol(curated[[i]])) -+ names(column.OK) <- colnames(curated[[i]]) -+ for(j in 1:ncol(curated[[i]])){ -+ doesmatch <- grep(regexes[j],curated[[i]][,j]) -+ if(template[j,"requiredness"]=="optional"){ -+ doesmatch <- c(doesmatch,which(is.na(curated[[i]][,j]))) -+ } -+ doesnotmatch <- 1:nrow(curated[[i]]) -+ doesnotmatch <- doesnotmatch[!doesnotmatch %in% doesmatch] -+ ## if field must be unique, add non-unique values to doesnotmatch -+ if(template[j,"uniqueness"]=="unique"){ -+ counts.table <- table(curated[[i]][,j]) -+ counts <- counts.table[match(curated[[i]][,j],names(counts.table))] #this counts the occurences -+ counts[is.na(counts)] <- 1 ##Consider NAs to be unique -+ nonunique <- which(counts>1) -+ doesnotmatch <- c(doesnotmatch,nonunique) -+ doesnotmatch <- unique(doesnotmatch) #don't duplicate fields which fail both tests -+ } -+ if(length(doesnotmatch)==0){ -+ column.OK[j] <- TRUE -+ }else{ -+ column.OK[j] <- FALSE -+ curated[[i]][doesnotmatch,j] <- paste("!!!",curated[[i]][doesnotmatch,j],"!!!",sep="") -+ } -+ } -+ if(all(column.OK)){ -+ print("all regex checks OK") -+ checklist[i,"variable.regex.checks"] <- TRUE -+ }else{ -+ print(paste("The following columns failed the regex check for ",dataset.names[i],":",sep="")) -+ print(names(column.OK)[!column.OK]) -+ print(curated[[i]][,!column.OK]) -+ checklist[i,"variable.regex.checks"] <- FALSE -+ } -+ } -+ -+ print(checklist) -+ -+ checkTrue( all(checklist[, "column.names"]) ) -+ checkTrue( all(checklist[, "variable.regex.checks"]) ) -+ -+ } -> -> proc.time() - user system elapsed - 0.224 0.028 0.244 diff --git a/inst/unitTests/test_counts.R b/inst/unitTests/test_counts.R deleted file mode 100755 index 36e9457..0000000 --- a/inst/unitTests/test_counts.R +++ /dev/null @@ -1,25 +0,0 @@ -test_counts <- function() { - require(curatedCRCData) - require(affy) - strEsets <- data(package="curatedCRCData")[[3]][,3] - esets <- lapply(strEsets, function(x){ - data(list=x) - get(x) - }) - df <- data.frame(PMID=sapply(esets, pubMedIds), - ncol=sapply(esets,ncol), - nrow=sapply(esets,nrow), - nwithbatch=sapply(esets, function(x) sum(!is.na(x$batch))), - deceased=sapply(esets,function(X) - sum(X$vital_status=="deceased",na.rm=TRUE)), - stringsAsFactors = FALSE) - df = df[order(df$PMID, df$ncol, df$nrow),] - ## reference file was generated with the following commands on 1.2.0: - ## write.csv(df, file="../extdata/curatedOvarianData_counts.csv", quote=TRUE, row.names=FALSE) - ## dfref <- read.csv("../extdata/curatedOvarianData_counts.csv", as.is=TRUE) - dfref <- read.csv(system.file("extdata/curatedCRCData_counts.csv", package = "curatedCRCData"), as.is=TRUE) - sapply(1:ncol(df), function(i){ - df[df[, i] != dfref[ ,i], i] <- paste("!!!", df[df[, i] != dfref[ ,i], i], "!!!", sep="") - print(data.frame(i, df[, i], dfref[, i])) - checkEquals(as.character(df[,i]), as.character(dfref[,i]))}) -} diff --git a/man/curatedCRCData-package.Rd b/man/curatedCRCData-package.Rd index d020bc6..5d20384 100644 --- a/man/curatedCRCData-package.Rd +++ b/man/curatedCRCData-package.Rd @@ -1,32 +1,28 @@ \name{curatedCRCData-package} -\alias{curatedCRCData} +\alias{curatedCRCData-package} \docType{package} \title{Clinically Annotated Data for the CRC Cancer Transcriptome} \description{The curatedCRCData package provides manually curated - clinical data, uniformly processed expression data, and convenience - functions for gene expression analysis in patients with colorectal cancer. - + clinical data and uniformly processed expression data for gene + expression analysis in patients with colorectal cancer. + Note: For staging, the "summarystage" was curated as using information on the pathologic T, N, and M stages, such that "early" = T stage of Tis through T3 and N0 and M0 and "late" = T stage of 4 and/or any N>0 or M>0} \details{ -\tabular{ll}{ -Package: \tab curatedCRCData\cr -Type: \tab Package\cr -Version: \tab \Sexpr{packageDescription("curatedCRCData")$Version}\cr -Date: \tab \Sexpr{packageDescription("curatedCRCData")$Date}\cr -License: \tab Artistic-2.0\cr -Depends: \tab R (>= 2.10.0), affy\cr -} + Datasets are hosted on Zenodo and downloaded on first use with + \code{\link{curatedCRCData}()}, which caches them locally with + \pkg{BiocFileCache}. Legacy \code{data()} access still works but is + deprecated and downloads through the same mechanism. See the package + vignette for rule-based creation of a list of filtered datasets. } \author{ -Princy Parsana, Markus Riester, Levi Waldron +Princy Parsana, Markus Riester, Curtis Huttenhower, Levi Waldron -Department of Biostatistics and Computational Biology, Dana-Farber Cancer Institute, Harvard School of Public Health - -Maintainer: Princy Parsana +Maintainer: Levi Waldron } - \examples{ -##List all datasets: -data(package="curatedCRCData") -## +## List all datasets: +curatedCRCData() +## Load one dataset (test=TRUE uses a small offline subset): +eset <- curatedCRCData("GSE3294_eset", test = TRUE) } +\keyword{package} diff --git a/man/curatedCRCData.Rd b/man/curatedCRCData.Rd new file mode 100644 index 0000000..2f62ec1 --- /dev/null +++ b/man/curatedCRCData.Rd @@ -0,0 +1,40 @@ +\name{curatedCRCData} +\alias{curatedCRCData} +\title{Download curated colorectal cancer ExpressionSets} +\description{ + Datasets are hosted on Zenodo and downloaded individually on first use, + then cached locally with \pkg{BiocFileCache}; later calls load from the + cache without network access. Loading datasets with \code{data()} still + works but is deprecated in favor of this function. +} +\usage{ +curatedCRCData(datasets, test = FALSE) +} +\arguments{ + \item{datasets}{Character vector of dataset names (e.g. + \code{"GSE39582_eset"}). If missing, the names of all available + datasets are returned and nothing is downloaded.} + \item{test}{If \code{TRUE}, load small offline test subsets bundled + with the package instead of downloading the full datasets. Intended + for examples, tests, and vignettes; the subsets are not suitable + for analysis.} +} +\value{ + A single \code{\linkS4class{ExpressionSet}} if one dataset name is + given; a named list of \code{ExpressionSet}s if several; a character + vector of available dataset names if \code{datasets} is missing. +} +\details{ + Downloaded files are stored in a package-specific cache directory + (\code{tools::R_user_dir("curatedCRCData", "cache")}). To remove + cached files, delete that directory or manage it with + \pkg{BiocFileCache}. +} +\examples{ +curatedCRCData() # list available datasets +eset <- curatedCRCData("GSE3294_eset", test = TRUE) +dim(eset) +esets <- curatedCRCData(c("GSE2630_eset", "GSE3294_eset"), test = TRUE) +names(esets) +} +\keyword{utilities} diff --git a/tests/testthat.R b/tests/testthat.R new file mode 100644 index 0000000..e62ed5e --- /dev/null +++ b/tests/testthat.R @@ -0,0 +1,4 @@ +library(testthat) +library(curatedCRCData) + +test_check("curatedCRCData") diff --git a/tests/testthat/test-getData.R b/tests/testthat/test-getData.R new file mode 100644 index 0000000..0428fb8 --- /dev/null +++ b/tests/testthat/test-getData.R @@ -0,0 +1,77 @@ +# The 34 datasets this package has provided since its initial release; this +# list guards the Zenodo manifest against accidental row loss. +expected_datasets <- c( + "GSE11237_eset", "GSE12225.GPL3676_eset", "GSE12945_eset", + "GSE13067_eset", "GSE13294_eset", "GSE14095_eset", "GSE14333_eset", + "GSE16125.GPL5175_eset", "GSE17536_eset", "GSE17537_eset", + "GSE17538.GPL570_eset", "GSE18105_eset", "GSE2109_eset", + "GSE21510_eset", "GSE21815_eset", "GSE24549.GPL5175_eset", + "GSE24550.GPL5175_eset", "GSE2630_eset", "GSE26682.GPL570_eset", + "GSE26682.GPL96_eset", "GSE26906_eset", "GSE27544_eset", + "GSE28702_eset", "GSE3294_eset", "GSE33113_eset", "GSE39582_eset", + "GSE3964_eset", "GSE4045_eset", "GSE4526_eset", "GSE45270_eset", + "TCGA.COAD_eset", "TCGA.READ_eset", "TCGA.RNASeqV2.READ_eset", + "TCGA.RNASeqV2_eset" +) + +manifest <- read.csv(system.file("extdata", "zenodo-manifest.csv", + package = "curatedCRCData")) + +test_that("manifest is complete and well-formed", { + expect_setequal(manifest$dataset, expected_datasets) + expect_false(anyDuplicated(manifest$dataset) > 0) + expect_identical(manifest$filename, paste0(manifest$dataset, ".rda")) + expect_true(all(grepl("^[0-9a-f]{32}$", manifest$md5))) + expect_true(all(grepl( + "^https://zenodo\\.org/records/[0-9]+/files/.+\\?download=1$", + manifest$url))) + expect_true(all(manifest$size_bytes > 0)) +}) + +test_that("manifest, fixtures, and data/ stubs are in lock-step", { + fixtures <- sub("\\.rda$", "", list.files( + system.file("extdata", "testdata", package = "curatedCRCData"), + pattern = "\\.rda$")) + expect_setequal(fixtures, expected_datasets) + stubs <- data(package = "curatedCRCData")$results[, "Item"] + expect_setequal(stubs, expected_datasets) +}) + +test_that("no-argument call lists the datasets", { + expect_setequal(curatedCRCData(), expected_datasets) +}) + +test_that("getter returns ExpressionSets from offline fixtures", { + eset <- curatedCRCData("GSE3294_eset", test = TRUE) + expect_s4_class(eset, "ExpressionSet") + expect_gt(ncol(eset), 0) + esets <- curatedCRCData(c("GSE2630_eset", "GSE3294_eset"), test = TRUE) + expect_type(esets, "list") + expect_named(esets, c("GSE2630_eset", "GSE3294_eset")) + expect_s4_class(esets[[1]], "ExpressionSet") +}) + +test_that("unknown dataset names give an informative error", { + expect_error(curatedCRCData("NOT_A_DATASET"), "Unknown dataset") + expect_error(curatedCRCData("NOT_A_DATASET", test = TRUE), + "Unknown dataset") +}) + +test_that("data() stubs create a binding", { + e <- new.env() + data("GSE3294_eset", package = "curatedCRCData", envir = e) + expect_true(exists("GSE3294_eset", envir = e)) +}) + +test_that("full download works (opt-in; set RUN_FULL_DOWNLOAD_TESTS=1)", { + skip_on_bioc() + skip_if(!nzchar(Sys.getenv("RUN_FULL_DOWNLOAD_TESTS"))) + skip_if_offline("zenodo.org") + # smallest dataset in the collection + eset <- curatedCRCData("TCGA.RNASeqV2.READ_eset") + expect_s4_class(eset, "ExpressionSet") + # second call must hit the cache (no download message) + expect_silent(suppressMessages( + eset2 <- curatedCRCData("TCGA.RNASeqV2.READ_eset"))) + expect_identical(dim(eset), dim(eset2)) +}) diff --git a/vignettes/curatedCRCData_vignette.Rmd b/vignettes/curatedCRCData_vignette.Rmd index 8610ffd..5b48051 100644 --- a/vignettes/curatedCRCData_vignette.Rmd +++ b/vignettes/curatedCRCData_vignette.Rmd @@ -32,26 +32,47 @@ and clinical data formats. In this vignette, we give a short tour of the package and will show how to use it efficiently. +# Data access and caching + +As of version 2.46, the datasets are no longer stored inside the package: +they are hosted on [Zenodo](https://zenodo.org/) and downloaded +individually the first time they are requested, then cached locally with +`r BiocStyle::Biocpkg("BiocFileCache")` so later uses need no network +access. The cache lives in a package-specific directory, +`tools::R_user_dir("curatedCRCData", "cache")`; to reclaim disk space, +delete that directory or manage it with *BiocFileCache*. + +The legacy `data(TCGA.COAD_eset)` syntax still works (it downloads +through the same cache) but is deprecated and will be removed in a +future release; please use the `curatedCRCData()` function shown below. + +Throughout this vignette we set `test = TRUE` (and `test.mode <- TRUE`), +which loads small offline subsets of each dataset that are bundled with +the package, so that the vignette builds without downloading data. For +real analyses, omit these settings to work with the full datasets. + # Load data sets Loading a single dataset is very easy. First we load the package: ```{r example1, message=FALSE, warning=FALSE} library(curatedCRCData) ``` -To get a listing of all the datasets, use the `data` function: +To get a listing of all the datasets, call `curatedCRCData()` with no +arguments: ```{r example1tcgastep2} -data(package="curatedCRCData") +curatedCRCData() ``` -Now to load a single dataset, we use the `data` function again: +Now to load a single dataset (drop `test = TRUE` to download the full +dataset): ```{r example1_load_tcga} -data(TCGA.COAD_eset) +TCGA.COAD_eset <- curatedCRCData("TCGA.COAD_eset", test = TRUE) TCGA.COAD_eset ``` The datasets are provided as Bioconductor `ExpressionSet` objects and we refer to the Bioconductor documentation for users unfamiliar with this data -structure. +structure. # Load datasets based on rules For a meta-analysis, we typically want to filter datasets and patients to @@ -63,8 +84,9 @@ dataset. It is also possible to filter samples by annotation, for example to rem early stage and normal samples. ```{r example2loadstep1} -source(system.file("extdata", +source(system.file("extdata", "patientselection_all.config",package="curatedCRCData")) +test.mode <- TRUE # use the bundled offline subsets in this vignette ls() ``` @@ -193,7 +215,7 @@ a simple way to do it. For one dataset: ```{r simplygetdata, eval=FALSE} library(curatedCRCData) library(Biobase) -data(TCGA.COAD_eset) +TCGA.COAD_eset <- curatedCRCData("TCGA.COAD_eset") write.csv(exprs(TCGA.COAD_eset), file="TCGA.COAD_eset_exprs.csv") write.csv(pData(TCGA.COAD_eset), file="TCGA.COAD_eset_clindata.csv") ``` @@ -201,12 +223,12 @@ write.csv(pData(TCGA.COAD_eset), file="TCGA.COAD_eset_clindata.csv") Or for several datasets: ```{r simplyseveraldatasets, eval=FALSE} -data.to.fetch <- c("TCGA.COAD_eset", "GSE37317_eset") -for (onedata in data.to.fetch){ - print(paste("Fetching", onedata)) - data(list=onedata) - write.csv(exprs(get(onedata)), file=paste(onedata, "_exprs.csv", sep="")) - write.csv(pData(get(onedata)), file=paste(onedata, "_clindata.csv", sep="")) +data.to.fetch <- curatedCRCData(c("TCGA.COAD_eset", "GSE39582_eset")) +for (onedata in names(data.to.fetch)){ + write.csv(exprs(data.to.fetch[[onedata]]), + file=paste(onedata, "_exprs.csv", sep="")) + write.csv(pData(data.to.fetch[[onedata]]), + file=paste(onedata, "_clindata.csv", sep="")) } ```