diff --git a/DESCRIPTION b/DESCRIPTION index da8a8cb7..59410f74 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,14 +1,13 @@ Package: sccomp Type: Package Title: Differential Composition and Variability Analysis for Single-Cell Data -Version: 2.1.32 +Version: 2.1.33 Date: 2026-05-11 Authors@R: c(person("Stefano", "Mangiola", email = "stefano.mangiola@unimelb.edu.au", role = c("aut", "cre")), person("Alexandra J.", "Roth-Schulze", role = "aut"), person("Marie", "Trussart", role = "aut"), person("Enrique", "Zozaya-Valdés", role = "aut"), person("Mengyao", "Ma", role = "aut"), person("Zijie", "Gao", role = "aut"), person("Alan F.", "Rubin", role = "aut"), person("Terence P.", "Speed", role = "aut"), person("Heejung", "Shim", role = "aut"), person("Anthony T.", "Papenfuss", role = "aut")) Description: Comprehensive R package for differential composition and variability analysis in single-cell RNA sequencing, CyTOF, and microbiome data. Provides robust Bayesian modeling with outlier detection, random effects, and advanced statistical methods for cell type proportion analysis. Features include probabilistic outlier identification, mixed-effect modeling, differential variability testing, and comprehensive visualization tools. Perfect for cancer research, immunology, developmental biology, and single-cell genomics applications. License: GPL-3 Encoding: UTF-8 Roxygen: list(markdown = TRUE) -RoxygenNote: 7.3.3 Depends: R (>= 4.3.0), instantiate (>= 0.2.3) @@ -39,6 +38,7 @@ Imports: SingleCellExperiment, posterior Suggests: + mgcv, knitr, rmarkdown, BiocStyle, @@ -63,3 +63,4 @@ Config/testthat/edition: 3 Config/testthat/parallel: false Config/testthat/snapshot/parallel: false Config/testthat/snapshot/parallel/workers: 2 +Config/roxygen2/version: 8.0.0 diff --git a/R/sccomp_remove_outliers.R b/R/sccomp_remove_outliers.R index b9133be6..9b101747 100644 --- a/R/sccomp_remove_outliers.R +++ b/R/sccomp_remove_outliers.R @@ -196,8 +196,7 @@ sccomp_remove_outliers.sccomp_tbl = function(.estimate, # This is for the new data generation with selected factors to do adjustment data = - .estimate |> - attr("model_input") |> + data_for_model |> c( list( # Add subset of coefficients diff --git a/R/sccomp_replicate.R b/R/sccomp_replicate.R index fbc87b66..941e9b3f 100644 --- a/R/sccomp_replicate.R +++ b/R/sccomp_replicate.R @@ -135,6 +135,16 @@ sccomp_replicate.sccomp_tbl = function(fit, #' @param formula_variability Formula for the variability model #' @param new_data New data to generate predictions for. If NULL, uses the original data #' @param original_count_data Original count data from the model +#' @param smooth_results Optional named list of smooth-term metadata captured +#' at fit time. In the standard call path this is read from +#' `get_smooth_results(.data)` (an attribute on `model_input`, not Stan data). +#' Contains +#' `smooth_specs`, the `mgcv::smoothCon()` objects used to evaluate bases at +#' `new_data`; `smooth_re_objs`, the parallel `mgcv::smooth2random()` results +#' used to recover the fitted `Xf` / `Xr` parameterisation; and +#' `smooth_labels`, the original term labels used to name generated smooth +#' columns so they match the fit-time design. `NULL` when the composition +#' formula has no smooths. #' #' @return A list containing: #' - model_input: The prepared model input data @@ -156,7 +166,8 @@ prepare_replicate_data = function(X, original_formula_composition, formula_variability, new_data = NULL, - original_count_data) { + original_count_data, + smooth_results = NULL) { .sample = enquo(.sample) @@ -226,13 +237,17 @@ prepare_replicate_data = function(X, new_data = old_data |> bind_rows( new_data ) + # Smooth columns are evaluated separately via PredictMat below; keep the + # random-effect clauses so the later RE parsing still sees the same formula. + formula_composition = strip_smooth_terms(formula_composition) + new_X = new_data |> get_design_matrix( # Drop random intercept formula_composition |> + strip_random_effect_terms() |> as.character() |> - str_remove_all("\\+ ?\\(.+\\|.+\\)") |> paste(collapse="") |> as.formula(), !!.sample, @@ -242,6 +257,17 @@ prepare_replicate_data = function(X, # Remove columns that are not in the original design matrix .[,colnames(.) %in% colnames(X), drop=FALSE] + # Evaluate any smooth bases on the replicate rows and merge the resulting + # design pieces (unpenalised columns appended to `new_X`, one RE slot per + # penalised block) using the fit-time `smoothCon` / `smooth2random` objects. + smooth_design = build_smooth_replicate_design( + parametric_X = new_X, + new_data_tail = new_data |> tail(nrow_new_data), + smooth_results = smooth_results + ) + new_X = smooth_design$new_X + smooth_replicate_slots = smooth_design$smooth_replicate_slots + # Check that all effect combination were present when the model was fitted check_missing_parameters( new_X |> colnames(), @@ -263,8 +289,8 @@ prepare_replicate_data = function(X, get_design_matrix( # Drop random intercept formula_variability |> + strip_random_effect_terms() |> as.character() |> - str_remove_all("\\+ ?\\(.+\\|.+\\)") |> paste(collapse="") |> as.formula(), !!.sample, @@ -363,6 +389,25 @@ prepare_replicate_data = function(X, replicate_slots = map(seq_len(4L), build_replicate_slot) + # Append smooth-derived replicate slots (one per smooth term in the + # composition formula). They occupy whichever slots come after the + # explicit RE clauses, mirroring the slot ordering used at fit time. + if (length(smooth_replicate_slots) > 0) { + n_explicit_re = length(original_grouping_names) + n_smooth = length(smooth_replicate_slots) + n_used = n_explicit_re + n_smooth + if (n_used > 4L) { + stop(sprintf( + "sccomp says: the replicate model needs %d RE slot(s) but only 4 are available.", + n_used + )) + } + # Replace placeholder slots `[n_explicit_re + 1 .. n_used]` with smooths. + for (k in seq_len(n_smooth)) { + replicate_slots[[n_explicit_re + k]] = smooth_replicate_slots[[k]] + } + } + # setup default unknown_grouping variable for generated quantities unknown_grouping = rep(0L, 4L) @@ -464,7 +509,8 @@ replicate_data = function(.data, original_count_data = .data |> attr("count_data") |> - .subset(!!.sample) + .subset(!!.sample), + smooth_results = get_smooth_results(.data) ) # Original input diff --git a/R/smooths.R b/R/smooths.R new file mode 100644 index 00000000..75de99a9 --- /dev/null +++ b/R/smooths.R @@ -0,0 +1,520 @@ +# Smooth-term utilities for sccomp formulas (brms-style splines) +# +# This file implements the R-side machinery that turns smooth terms +# (`s()`, `t2()`) in a sccomp formula into design-matrix columns that the +# existing Stan model can consume without modification. +# +# The decomposition is the standard mgcv / brms one: +# * `Xf` — unpenalised null-space columns of the smooth (linear part). +# These are appended to the fixed-effect design matrix `X` and +# estimated as ordinary `beta` coefficients. +# * `Xr` — penalised "wiggly" basis columns. These occupy a sccomp +# random-effect slot with `n_factors = 1` and `n_groups = ncol(Xr)`, +# so the slot's per-cell-group SD plays the role of brms's `sds_*`. +# +# Built via: +# 1. `mgcv::smoothCon(s(...), data, absorb.cons = TRUE, +# diagonal.penalty = TRUE)[[1]]` — basis + penalty. +# 2. `mgcv::smooth2random(sm, vnames = "", type = 2)` — re-parameterise +# into `Xf` + standardised `rand$Xr` (so the wiggly coefficients have +# a `N(0, sds * I)` prior). +# +# At prediction time, `predict_smooth_at_newdata()` evaluates the basis at +# new covariate values via `mgcv::PredictMat()` and re-applies the same +# `trans.D` (diagonal rescaling) and `trans.U` (orthonormal rotation) +# captured at fit time, so the new columns line up with the columns the +# model was trained on. + +#' Detect smooth specials in a formula +#' +#' @param fm A one-sided formula. +#' @return TRUE if `fm` contains any `s(...)` or `t2(...)` term. +#' @keywords internal +#' @noRd +has_smooth_terms <- function(fm) { + if (is.null(fm)) return(FALSE) + trm <- stats::terms(fm, specials = c("s", "t2")) + length(unlist(attr(trm, "specials"))) > 0 +} + + +#' Strip smooth specials from a formula +#' +#' Returns a new formula with `s()` / `t2()` term labels removed. All other +#' terms, including random-effect clauses (`(... | g)`), are preserved. The +#' intercept setting (`+ 0` / `- 1`) is preserved. If nothing remains, returns +#' `~ 1` (or `~ 0`). +#' +#' @param fm A one-sided formula. +#' @return A one-sided formula without smooth term labels. +#' @keywords internal +#' @noRd +strip_smooth_terms <- function(fm) { + if (is.null(fm)) return(fm) + trm <- stats::terms(fm, specials = c("s", "t2")) + smooth_idx <- unlist(attr(trm, "specials")) + + vars <- attr(trm, "variables") + smooth_labels <- if (length(smooth_idx) > 0) { + vapply( + smooth_idx + 1L, + function(i) deparse(vars[[i]], width.cutoff = 500L), + character(1) + ) + } else { + character(0) + } + + all_labels <- attr(trm, "term.labels") + keep_labels <- setdiff(all_labels, smooth_labels) + keep_labels <- ifelse(grepl("\\|", keep_labels), paste0("(", keep_labels, ")"), keep_labels) + + has_intercept <- attr(trm, "intercept") == 1L + + if (length(keep_labels) == 0) { + return(if (has_intercept) ~ 1 else ~ 0) + } + + rhs <- paste(keep_labels, collapse = " + ") + if (!has_intercept) rhs <- paste0(rhs, " + 0") + stats::as.formula(paste("~", rhs), env = environment(fm)) +} + + +#' Strip random-effect clauses from a formula +#' +#' Returns a formula with `(... | g)` term labels removed, preserving all +#' ordinary fixed-effect terms and the intercept setting. This is used just +#' before calling `model.matrix()`, which cannot evaluate sccomp/brms-style +#' random-effect clauses. +#' +#' @param fm A one-sided formula. +#' @return A one-sided formula without random-effect term labels. +#' @keywords internal +#' @noRd +strip_random_effect_terms <- function(fm) { + if (is.null(fm)) return(fm) + trm <- stats::terms(fm) + + keep_labels <- attr(trm, "term.labels") + keep_labels <- keep_labels[!grepl("\\|", keep_labels)] + + has_intercept <- attr(trm, "intercept") == 1L + + if (length(keep_labels) == 0) { + return(if (has_intercept) ~ 1 else ~ 0) + } + + rhs <- paste(keep_labels, collapse = " + ") + if (!has_intercept) rhs <- paste0(rhs, " + 0") + stats::as.formula(paste("~", rhs), env = environment(fm)) +} + + +#' Parse smooth terms from a sccomp formula +#' +#' For each `s(...)` / `t2(...)` term, builds an mgcv smooth via +#' `smoothCon(..., absorb.cons = TRUE, diagonal.penalty = TRUE)` and the +#' brms-style re-parameterisation via `smooth2random(..., type = 2)`. Single- +#' penalty smooths (e.g. plain `s(x)`) yield one penalised block; multi- +#' penalty smooths (e.g. `t2(x, y)`, `s(x, z, bs = "fs")`) yield 2+ blocks, +#' one per penalty. Each block is mapped to its own sccomp random-effect +#' slot, with its own per-cell-group smoothing SD — matching the way brms +#' generates one `sds_*` parameter per penalty. +#' +#' @param fm A one-sided formula, possibly containing `s()` / `t2()` terms. +#' @param data A data frame containing every variable referenced inside the +#' smooth terms (in the same row order as the design matrices that will be +#' built downstream). +#' @return A list with components: +#' * `parametric_formula` — `fm` with smooth specials stripped. +#' * `smooth_labels` — character vector of original smooth labels +#' (one per `s()` / `t2()` term). +#' * `Xf_list` — list of `N × Ks_k` matrices (unpenalised cols), +#' one per smooth term. +#' * `Xr_list` — **flat** list of `N × k_b` matrices, one per +#' penalty block. A single-penalty smooth contributes 1 entry; a multi- +#' penalty smooth contributes 2+. Each consumes one RE slot. +#' * `Xr_to_smooth` — integer vector parallel to `Xr_list` mapping +#' each block to its parent smooth index. +#' * `Xr_slot_labels` — character vector parallel to `Xr_list` giving +#' each block its sccomp RE-slot label: just the smooth's label for +#' single-penalty smooths, or `