From b58f91028090ad9e6d1504b1daaf690e8c6ee80f Mon Sep 17 00:00:00 2001 From: jgabry Date: Fri, 24 Jul 2026 12:46:19 -0600 Subject: [PATCH 1/6] Allow lists and data frames for int-declared variables process_data() coerced variables declared as int with mode<-, which errors on lists and data frames. The list-to-array conversion happens later, in write_stan_json(), so the documented list convenience worked only for real variables. Extract the container conversions into convert_to_array(), shared by both functions, and apply them in process_data() before the length-1 and int-coercion checks. Document factor conversion, which was undocumented, and correct the claim that lists cannot be used for arrays with more than one dimension. A list contributes exactly one leading dimension, so array[K,L] vector[J] can be supplied as a list of K LxJ matrices; the real limitation is that nested lists are not supported. closes #817 --- NEWS.md | 3 ++ R/data.R | 79 ++++++++++++++++++++++++++------------ man/write_stan_json.Rd | 19 +++++++-- tests/testthat/test-data.R | 74 +++++++++++++++++++++++++++++++++++ tests/testthat/test-json.R | 43 +++++++++++++++++++++ 5 files changed, 189 insertions(+), 29 deletions(-) diff --git a/NEWS.md b/NEWS.md index 40f9cf387..d0700e252 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,8 @@ # cmdstanr (development version) +* Lists of matrices/vectors and data frames can now be supplied for variables +declared as `int` in the Stan program. Previously these worked only for `real` +variables and errored for `int` ones. (#817) * The `CMDSTANR_NO_VER_CHECK` R option and environment variable are deprecated as of CmdStanR 1.0.0; use the lowercase `cmdstanr_no_ver_check` forms instead. * `$cpp_options()` no longer includes a `STAN_VERSION` entry read from the model diff --git a/R/data.R b/R/data.R index cdd9c205f..34444b091 100644 --- a/R/data.R +++ b/R/data.R @@ -15,10 +15,18 @@ #' file: #' #' * `logical` -> `integer` (`TRUE` -> `1`, `FALSE` -> `0`) +#' * `factor` -> `integer` (the index of each value's level) #' * `data.frame` -> `matrix` (via [data.matrix()]) #' * `list` -> `array` #' * `table` -> `vector`, `matrix`, or `array` (depending on dimensions of table) #' +#' Factors are written as their level indices, which depend on the order of the +#' factor's levels (alphabetical by default) rather than on the values +#' themselves. For example, `factor(c(10, 9, 8))` is written as `[3, 2, 1]`, and +#' an unused level shifts the indices of the levels after it. If the original +#' values are what you want, convert them first, e.g. with +#' `as.numeric(as.character(x))`. +#' #' The `list` to `array` conversion is intended to make it easier to prepare #' the data for certain Stan declarations involving arrays: #' @@ -26,12 +34,15 @@ #' elements where each element is a vector of length `J` #' * `array[K] matrix[I,J] m ` can be constructed in \R as a list with `K` #' elements where each element is an `IxJ` matrix +#' * `array[K,I,J] int n ` can be constructed in \R as a list with `K` +#' elements where each element is an `IxJ` matrix of integers #' #' These can also be passed in from \R as arrays instead of lists but the list -#' option is provided for convenience. Unfortunately for arrays with more than -#' one dimension (e.g. `array[K,L] vector[J] v `) it is not possible to use an -#' \R list and an array must be used instead. For this example the array in \R -#' should have dimensions `KxLxJ`. +#' option is provided for convenience. A list always contributes exactly one +#' leading dimension, so `array[K,L] vector[J] v ` can be supplied either as a +#' list of `K` matrices each with dimensions `LxJ` or as a single \R array with +#' dimensions `KxLxJ`. Nested lists are not supported: every element of the list +#' must be a vector, matrix, or array. #' #' Because \R does not distinguish between a scalar and a vector of length 1, a #' length-1 vector like `c(42)` is written to JSON as a scalar (`42`) rather @@ -97,24 +108,11 @@ write_stan_json <- function(data, file, always_decimal = FALSE) { if (is.null(var)) { stop("Variable '", var_name, "' is NULL.", call. = FALSE) } - if (!(is.numeric(var) || is.factor(var) || is.logical(var) || - is.data.frame(var) || is.list(var))) { - stop("Variable '", var_name, "' is of invalid type.", call. = FALSE) - } + validate_data_type(var, var_name) if (anyNA(var)) { stop("Variable '", var_name, "' has NA values.", call. = FALSE) } - - if (is.table(var)) { - var <- unclass(var) - } else if (is.logical(var)) { - mode(var) <- "integer" - } else if (is.data.frame(var)) { - var <- data.matrix(var) - } else if (is.list(var)) { - var <- list_to_array(var, var_name) - } - data[[var_name]] <- var + data[[var_name]] <- convert_to_array(var, var_name) } # unboxing variables (N = 10 is stored as N : 10, not N: [10]) @@ -130,6 +128,39 @@ write_stan_json <- function(data, file, always_decimal = FALSE) { } +# Types accepted for a data variable and for each column of a data frame +is_valid_data_type <- function(x) { + is.numeric(x) || is.factor(x) || is.logical(x) +} + + +# Error if a variable is not one of the types accepted in a data list. Data +# frames and lists are accepted here and converted by convert_to_array(). +validate_data_type <- function(var, var_name) { + if (!is_valid_data_type(var) && !is.data.frame(var) && !is.list(var)) { + stop("Variable '", var_name, "' is of invalid type.", call. = FALSE) + } + invisible(NULL) +} + + +# Convert the R container types accepted in a data list to the atomic arrays +# CmdStan's JSON reader expects. Used by both write_stan_json() and +# process_data() so that the two paths agree. +convert_to_array <- function(var, var_name = NULL) { + if (is.table(var)) { + var <- unclass(var) + } else if (is.logical(var)) { + mode(var) <- "integer" + } else if (is.data.frame(var)) { + var <- data.matrix(var) + } else if (is.list(var)) { + var <- list_to_array(var, var_name) + } + var +} + + list_to_array <- function(x, name = NULL) { list_length <- length(x) if (list_length == 0) { @@ -158,12 +189,6 @@ list_to_array <- function(x, name = NULL) { #' @noRd #' @param data If not `NULL`, then either a path to a data file compatible with #' CmdStan, or a named list of \R objects to pass to [write_stan_json()]. -#' @param stan_file If not `NULL`, the path to the Stan model for which to -#' process the named list suppiled to the `data` argument. The Stan model -#' is used for checking whether the supplied named list has all the -#' required elements/Stan variables and to help differentiate between a -#' vector of length 1 and a scalar when genereting the JSON file. This -#' argument is ignored when a path to a data file is supplied for `data`. #' @param model_variables A list of all parameters with their types and #' number of dimensions. Typically the output of model$variables(). #' @return Path to data file. @@ -192,6 +217,10 @@ process_data <- function(data, model_variables = NULL) { if (is.null(data[[var_name]])) { stop("Variable '", var_name, "' is NULL.", call. = FALSE) } + validate_data_type(data[[var_name]], var_name) + # Convert lists and data frames to arrays before the checks below, + # which require an atomic object (#817) + data[[var_name]] <- convert_to_array(data[[var_name]], var_name) # distinguish between scalars and arrays/vectors of length 1 if (length(data[[var_name]]) == 1 && data_variables[[var_name]]$dimensions == 1) { diff --git a/man/write_stan_json.Rd b/man/write_stan_json.Rd index f041a106d..e2d5f75f2 100644 --- a/man/write_stan_json.Rd +++ b/man/write_stan_json.Rd @@ -27,11 +27,19 @@ Write data to a JSON file readable by CmdStan file: \itemize{ \item \code{logical} -> \code{integer} (\code{TRUE} -> \code{1}, \code{FALSE} -> \code{0}) +\item \code{factor} -> \code{integer} (the index of each value's level) \item \code{data.frame} -> \code{matrix} (via \code{\link[=data.matrix]{data.matrix()}}) \item \code{list} -> \code{array} \item \code{table} -> \code{vector}, \code{matrix}, or \code{array} (depending on dimensions of table) } +Factors are written as their level indices, which depend on the order of the +factor's levels (alphabetical by default) rather than on the values +themselves. For example, \code{factor(c(10, 9, 8))} is written as \verb{[3, 2, 1]}, and +an unused level shifts the indices of the levels after it. If the original +values are what you want, convert them first, e.g. with +\code{as.numeric(as.character(x))}. + The \code{list} to \code{array} conversion is intended to make it easier to prepare the data for certain Stan declarations involving arrays: \itemize{ @@ -39,13 +47,16 @@ the data for certain Stan declarations involving arrays: elements where each element is a vector of length \code{J} \item \verb{array[K] matrix[I,J] m } can be constructed in \R as a list with \code{K} elements where each element is an \code{IxJ} matrix +\item \verb{array[K,I,J] int n } can be constructed in \R as a list with \code{K} +elements where each element is an \code{IxJ} matrix of integers } These can also be passed in from \R as arrays instead of lists but the list -option is provided for convenience. Unfortunately for arrays with more than -one dimension (e.g. \verb{array[K,L] vector[J] v }) it is not possible to use an -\R list and an array must be used instead. For this example the array in \R -should have dimensions \code{KxLxJ}. +option is provided for convenience. A list always contributes exactly one +leading dimension, so \verb{array[K,L] vector[J] v } can be supplied either as a +list of \code{K} matrices each with dimensions \code{LxJ} or as a single \R array with +dimensions \code{KxLxJ}. Nested lists are not supported: every element of the list +must be a vector, matrix, or array. Because \R does not distinguish between a scalar and a vector of length 1, a length-1 vector like \code{c(42)} is written to JSON as a scalar (\code{42}) rather diff --git a/tests/testthat/test-data.R b/tests/testthat/test-data.R index 0e5bc60df..aa5cf6808 100644 --- a/tests/testthat/test-data.R +++ b/tests/testthat/test-data.R @@ -409,6 +409,80 @@ test_that("process_data warns on int coercion", { ) }) +test_that("process_data accepts lists of matrices/vectors for int variables", { + stan_file <- write_stan_file(" + data { + array[4,3,2] int x; + } + ") + mod <- cmdstan_model(stan_file, compile = FALSE) + model_variables <- mod$variables() + + a <- matrix(1:6, nrow = 3, ncol = 2) + arr <- array(dim = c(4, 3, 2)) + for (i in 1:4) arr[i, , ] <- a + + from_array <- readLines(process_data(list(x = arr), model_variables = model_variables)) + from_int_list <- readLines(process_data(list(x = list(a, a, a, a)), model_variables = model_variables)) + expect_identical(from_int_list, from_array) + + # a list of doubles must give the same result as a list of integers (#817) + storage.mode(a) <- "double" + from_dbl_list <- readLines(process_data(list(x = list(a, a, a, a)), model_variables = model_variables)) + expect_identical(from_dbl_list, from_array) + + # values are written as integers, not as decimals + expect_false(any(grepl(".", from_dbl_list, fixed = TRUE))) + + stan_file <- write_stan_file(" + data { + array[2,3] int x; + } + ") + mod <- cmdstan_model(stan_file, compile = FALSE) + test_file <- process_data(list(x = list(c(1, 2, 3), c(4, 5, 6))), model_variables = mod$variables()) + expect_equal( + jsonlite::read_json(test_file, simplifyVector = TRUE), + list(x = matrix(1:6, nrow = 2, ncol = 3, byrow = TRUE)) + ) +}) + +test_that("process_data accepts data frames for int variables", { + stan_file <- write_stan_file(" + data { + array[2,2] int x; + } + ") + mod <- cmdstan_model(stan_file, compile = FALSE) + model_variables <- mod$variables() + + df <- data.frame(a = c(1, 2), b = c(3, 4)) + from_df <- readLines(process_data(list(x = df), model_variables = model_variables)) + from_matrix <- readLines(process_data(list(x = data.matrix(df)), model_variables = model_variables)) + expect_identical(from_df, from_matrix) + expect_false(any(grepl(".", from_df, fixed = TRUE))) +}) + +test_that("process_data errors on invalid types", { + stan_file <- write_stan_file(" + data { + array[2,2] int x; + } + ") + mod <- cmdstan_model(stan_file, compile = FALSE) + model_variables <- mod$variables() + + expect_error( + process_data(list(x = c("v", "w")), model_variables = model_variables), + "Variable 'x' is of invalid type." + ) + # NAs inside a list are reported as NAs rather than as a coercion failure + expect_error( + process_data(list(x = list(c(1, NA), c(3, 4))), model_variables = model_variables), + "Variable 'x' has NA values" + ) +}) + test_that("Floating-point differences do not cause truncation towards 0", { stan_file <- write_stan_file(" data { diff --git a/tests/testthat/test-json.R b/tests/testthat/test-json.R index 0f7788f3d..229410821 100644 --- a/tests/testthat/test-json.R +++ b/tests/testthat/test-json.R @@ -143,6 +143,49 @@ test_that("write_stan_json() errors if vectors/matrices in same list are differe ) }) +test_that("a list contributes one leading dimension", { + # e.g. `array[K,L] vector[J] v` as a list of K matrices with dimensions LxJ + K <- 2; L <- 3; J <- 4 + arr <- array(1:(K * L * J), dim = c(K, L, J)) + lst <- lapply(seq_len(K), function(k) arr[k, , ]) + + temp_file_list <- tempfile() + temp_file_arr <- tempfile() + write_stan_json(list(v = lst), temp_file_list) + write_stan_json(list(v = arr), temp_file_arr) + expect_identical(readLines(temp_file_list), readLines(temp_file_arr)) + + # nested lists are not supported + expect_error( + write_stan_json(list(v = list(list(1:4, 5:8), list(9:12, 13:16))), tempfile()), + "All elements in list 'v' must be numeric!" + ) +}) + +test_that("factors are written as level indices", { + temp_file <- tempfile() + read_x <- function(file) jsonlite::read_json(file, simplifyVector = TRUE)$x + + # the level indices are written, not the values themselves + write_stan_json(list(x = factor(c(10, 9, 8))), temp_file) + expect_equal(read_x(temp_file), c(3L, 2L, 1L)) + + # the order of the levels determines the indices + write_stan_json(list(x = factor(c("foo", "bar"))), temp_file) + expect_equal(read_x(temp_file), c(2L, 1L)) + + write_stan_json(list(x = factor(c("foo", "bar"), levels = c("foo", "bar"))), temp_file) + expect_equal(read_x(temp_file), c(1L, 2L)) + + # an unused level shifts the indices of the levels after it + write_stan_json(list(x = factor(c("b", "c"), levels = c("a", "b", "c"))), temp_file) + expect_equal(read_x(temp_file), c(2L, 3L)) + + # factor columns of a data frame are converted the same way + write_stan_json(list(x = data.frame(a = factor(c(10, 9, 8)))), temp_file) + expect_equal(read_x(temp_file), matrix(c(3L, 2L, 1L), ncol = 1)) +}) + test_that("write_stan_json() errors if invalid types", { expect_error( write_stan_json(list(N = list("abc", "def")), file = "abc.txt"), From 894aafd455d2d806aa9c6ea1815efa685780047a Mon Sep 17 00:00:00 2001 From: jgabry Date: Fri, 24 Jul 2026 12:48:20 -0600 Subject: [PATCH 2/6] Error on data frame columns of invalid type data.matrix() silently coerced character columns to factor codes and date/time columns to numeric, so a value the data list would reject as a variable was accepted as a column. Apply the same type check per column. --- NEWS.md | 4 ++++ R/data.R | 11 ++++++++++- man/write_stan_json.Rd | 3 ++- tests/testthat/test-data.R | 4 ++++ tests/testthat/test-json.R | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index d0700e252..17059e5fd 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,10 @@ * Lists of matrices/vectors and data frames can now be supplied for variables declared as `int` in the Stan program. Previously these worked only for `real` variables and errored for `int` ones. (#817) +* Data frame columns that are not numeric, integer, logical, or +factor are now an error. Previously `data.matrix()` silently coerced them, so a +character column reached Stan as alphabetically ordered integer codes. Use +`factor()` explicitly if integer codes are what you want. (#817) * The `CMDSTANR_NO_VER_CHECK` R option and environment variable are deprecated as of CmdStanR 1.0.0; use the lowercase `cmdstanr_no_ver_check` forms instead. * `$cpp_options()` no longer includes a `STAN_VERSION` entry read from the model diff --git a/R/data.R b/R/data.R index 34444b091..2d0d05143 100644 --- a/R/data.R +++ b/R/data.R @@ -16,7 +16,8 @@ #' #' * `logical` -> `integer` (`TRUE` -> `1`, `FALSE` -> `0`) #' * `factor` -> `integer` (the index of each value's level) -#' * `data.frame` -> `matrix` (via [data.matrix()]) +#' * `data.frame` -> `matrix` (via [data.matrix()]); every column must be +#' numeric, integer, logical, or factor #' * `list` -> `array` #' * `table` -> `vector`, `matrix`, or `array` (depending on dimensions of table) #' @@ -153,6 +154,14 @@ convert_to_array <- function(var, var_name = NULL) { } else if (is.logical(var)) { mode(var) <- "integer" } else if (is.data.frame(var)) { + # data.matrix() silently coerces character columns to factor codes and + # date/time columns to their numeric representation, so apply the same + # type check used for the variables themselves (#817) + invalid <- !vapply(var, is_valid_data_type, logical(1)) + if (any(invalid)) { + stop("Variable '", var_name, "' has columns of invalid type: ", + paste(names(var)[invalid], collapse = ", "), ".", call. = FALSE) + } var <- data.matrix(var) } else if (is.list(var)) { var <- list_to_array(var, var_name) diff --git a/man/write_stan_json.Rd b/man/write_stan_json.Rd index e2d5f75f2..74bd3c1bb 100644 --- a/man/write_stan_json.Rd +++ b/man/write_stan_json.Rd @@ -28,7 +28,8 @@ file: \itemize{ \item \code{logical} -> \code{integer} (\code{TRUE} -> \code{1}, \code{FALSE} -> \code{0}) \item \code{factor} -> \code{integer} (the index of each value's level) -\item \code{data.frame} -> \code{matrix} (via \code{\link[=data.matrix]{data.matrix()}}) +\item \code{data.frame} -> \code{matrix} (via \code{\link[=data.matrix]{data.matrix()}}); every column must be +numeric, integer, logical, or factor \item \code{list} -> \code{array} \item \code{table} -> \code{vector}, \code{matrix}, or \code{array} (depending on dimensions of table) } diff --git a/tests/testthat/test-data.R b/tests/testthat/test-data.R index aa5cf6808..32bcf0038 100644 --- a/tests/testthat/test-data.R +++ b/tests/testthat/test-data.R @@ -472,6 +472,10 @@ test_that("process_data errors on invalid types", { mod <- cmdstan_model(stan_file, compile = FALSE) model_variables <- mod$variables() + expect_error( + process_data(list(x = data.frame(a = c(1, 2), b = c("v", "w"))), model_variables = model_variables), + "Variable 'x' has columns of invalid type: b." + ) expect_error( process_data(list(x = c("v", "w")), model_variables = model_variables), "Variable 'x' is of invalid type." diff --git a/tests/testthat/test-json.R b/tests/testthat/test-json.R index 229410821..7d10bb385 100644 --- a/tests/testthat/test-json.R +++ b/tests/testthat/test-json.R @@ -198,6 +198,40 @@ test_that("write_stan_json() errors if invalid types", { ) }) +test_that("write_stan_json() errors if data frame has columns of invalid type", { + # data.matrix() would silently coerce these instead of erroring + expect_error( + write_stan_json(list(N = data.frame(a = 1:2, b = c("x", "y"))), tempfile()), + "Variable 'N' has columns of invalid type: b." + ) + expect_error( + write_stan_json(list(N = data.frame(a = as.Date(c("2020-01-01", "2020-01-02")))), tempfile()), + "Variable 'N' has columns of invalid type: a." + ) + expect_error( + write_stan_json(list(N = data.frame(a = as.POSIXct("2020-01-01", tz = "UTC"))), tempfile()), + "Variable 'N' has columns of invalid type: a." + ) + expect_error( + write_stan_json(list(N = data.frame(a = c(1 + 2i, 3 + 4i))), tempfile()), + "Variable 'N' has columns of invalid type: a." + ) + + # all invalid columns are reported, not just the first + expect_error( + write_stan_json(list(N = data.frame(a = 1:2, b = c("x", "y"), c = c("v", "w"))), tempfile()), + "Variable 'N' has columns of invalid type: b, c." + ) + + # numeric, integer, logical and factor columns are still allowed + expect_no_error( + write_stan_json( + list(N = data.frame(a = c(1.5, 2.5), b = 1:2, c = c(TRUE, FALSE), d = factor(c("x", "y")))), + tempfile() + ) + ) +}) + test_that("write_stan_json() errors if bad names", { expect_error( write_stan_json(list(x = 1, y = 2, x = 3), file = tempfile()), From 2d9638fd5c5d66f6ff2eea045560a08aa2cc0266 Mon Sep 17 00:00:00 2001 From: jgabry Date: Fri, 24 Jul 2026 13:05:47 -0600 Subject: [PATCH 3/6] Handle logical and factor types consistently in data list_to_array() rejected logical elements even though logical variables and logical data frame columns are both accepted. Allow them, and apply the logical-to-integer conversion after the container conversions so a list of logicals is not written as JSON true/false. Factors are written as their level indices, which are only meaningful for variables declared as int. Error when one is supplied for any other type. --- NEWS.md | 3 +++ R/data.R | 20 ++++++++++++++++---- man/write_stan_json.Rd | 5 ++++- tests/testthat/test-data.R | 32 ++++++++++++++++++++++++++++++++ tests/testthat/test-json.R | 21 +++++++++++++++++++++ 5 files changed, 76 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index 17059e5fd..8c5dcd093 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,6 +7,9 @@ variables and errored for `int` ones. (#817) factor are now an error. Previously `data.matrix()` silently coerced them, so a character column reached Stan as alphabetically ordered integer codes. Use `factor()` explicitly if integer codes are what you want. (#817) +* Lists of logical vectors/matrices are now converted to integers like logical +variables are, instead of erroring. (#817) +* Supplying a factor for a variable not declared as `int` is now an error. (#817) * The `CMDSTANR_NO_VER_CHECK` R option and environment variable are deprecated as of CmdStanR 1.0.0; use the lowercase `cmdstanr_no_ver_check` forms instead. * `$cpp_options()` no longer includes a `STAN_VERSION` entry read from the model diff --git a/R/data.R b/R/data.R index 2d0d05143..7022d67eb 100644 --- a/R/data.R +++ b/R/data.R @@ -26,7 +26,10 @@ #' themselves. For example, `factor(c(10, 9, 8))` is written as `[3, 2, 1]`, and #' an unused level shifts the indices of the levels after it. If the original #' values are what you want, convert them first, e.g. with -#' `as.numeric(as.character(x))`. +#' `as.numeric(as.character(x))`. The fitting methods of a model compiled from a +#' Stan file error if a factor is supplied for a variable that is not declared +#' as `int`, but `write_stan_json()` has no declarations to check against and so +#' always converts. #' #' The `list` to `array` conversion is intended to make it easier to prepare #' the data for certain Stan declarations involving arrays: @@ -151,8 +154,6 @@ validate_data_type <- function(var, var_name) { convert_to_array <- function(var, var_name = NULL) { if (is.table(var)) { var <- unclass(var) - } else if (is.logical(var)) { - mode(var) <- "integer" } else if (is.data.frame(var)) { # data.matrix() silently coerces character columns to factor codes and # date/time columns to their numeric representation, so apply the same @@ -166,6 +167,10 @@ convert_to_array <- function(var, var_name = NULL) { } else if (is.list(var)) { var <- list_to_array(var, var_name) } + # after the conversions above so that lists of logicals are also converted + if (is.logical(var)) { + mode(var) <- "integer" + } var } @@ -182,7 +187,7 @@ list_to_array <- function(x, name = NULL) { if (!all_equal_dim) { stop("All matrices/vectors in list '", name, "' must be the same size!", call. = FALSE) } - all_numeric <- all(sapply(x, function(a) is.numeric(a))) + all_numeric <- all(sapply(x, function(a) is.numeric(a) || is.logical(a))) if (!all_numeric) { stop("All elements in list '", name, "' must be numeric!", call. = FALSE) } @@ -235,6 +240,13 @@ process_data <- function(data, model_variables = NULL) { && data_variables[[var_name]]$dimensions == 1) { data[[var_name]] <- array(data[[var_name]], dim = 1) } + # Factors are written as level indices, which are only meaningful for + # variables declared as int + if (data_variables[[var_name]]$type != "int" + && is.factor(data[[var_name]])) { + stop("A factor was supplied for '", var_name, "', which is declared as '", + data_variables[[var_name]]$type, "'.", call. = FALSE) + } # Make sure integer inputs are of integer type to avoid # generating a decimal point in write_stan_json if (data_variables[[var_name]]$type == "int" diff --git a/man/write_stan_json.Rd b/man/write_stan_json.Rd index 74bd3c1bb..3a1d36a87 100644 --- a/man/write_stan_json.Rd +++ b/man/write_stan_json.Rd @@ -39,7 +39,10 @@ factor's levels (alphabetical by default) rather than on the values themselves. For example, \code{factor(c(10, 9, 8))} is written as \verb{[3, 2, 1]}, and an unused level shifts the indices of the levels after it. If the original values are what you want, convert them first, e.g. with -\code{as.numeric(as.character(x))}. +\code{as.numeric(as.character(x))}. The fitting methods of a model compiled from a +Stan file error if a factor is supplied for a variable that is not declared +as \code{int}, but \code{write_stan_json()} has no declarations to check against and so +always converts. The \code{list} to \code{array} conversion is intended to make it easier to prepare the data for certain Stan declarations involving arrays: diff --git a/tests/testthat/test-data.R b/tests/testthat/test-data.R index 32bcf0038..baf55aaa0 100644 --- a/tests/testthat/test-data.R +++ b/tests/testthat/test-data.R @@ -487,6 +487,38 @@ test_that("process_data errors on invalid types", { ) }) +test_that("process_data errors on a factor for a non-int variable", { + stan_file <- write_stan_file(" + data { + int a; + array[2] int b; + real c; + vector[2] d; + } + ") + mod <- cmdstan_model(stan_file, compile = FALSE) + model_variables <- mod$variables() + data <- list(a = 1L, b = c(1L, 2L), c = 2.5, d = c(1, 2)) + + expect_error( + process_data(modifyList(data, list(c = factor("x"))), model_variables = model_variables), + "A factor was supplied for 'c', which is declared as 'real'." + ) + # vectors and matrices are also reported as 'real' + expect_error( + process_data(modifyList(data, list(d = factor(c("x", "y")))), model_variables = model_variables), + "A factor was supplied for 'd', which is declared as 'real'." + ) + + # factors are still allowed for int variables + expect_no_error( + process_data(modifyList(data, list(a = factor("x"))), model_variables = model_variables) + ) + expect_no_error( + process_data(modifyList(data, list(b = factor(c("x", "y")))), model_variables = model_variables) + ) +}) + test_that("Floating-point differences do not cause truncation towards 0", { stan_file <- write_stan_file(" data { diff --git a/tests/testthat/test-json.R b/tests/testthat/test-json.R index 7d10bb385..6c1ce5146 100644 --- a/tests/testthat/test-json.R +++ b/tests/testthat/test-json.R @@ -162,6 +162,27 @@ test_that("a list contributes one leading dimension", { ) }) +test_that("logical elements of a list are converted to integers", { + temp_file_list <- tempfile() + temp_file_arr <- tempfile() + matrices <- list( + matrix(c(TRUE, FALSE, TRUE, FALSE), nrow = 2), + matrix(c(FALSE, TRUE, FALSE, TRUE), nrow = 2) + ) + write_stan_json(list(x = matrices), temp_file_list) + write_stan_json(list(x = list_to_array(matrices)), temp_file_arr) + + # 0/1 rather than JSON true/false, matching a plain logical variable + expect_identical(readLines(temp_file_list), readLines(temp_file_arr)) + expect_false(any(grepl("true|false", readLines(temp_file_list)))) + + # factors are still not allowed as list elements + expect_error( + write_stan_json(list(x = list(factor("a"), factor("b"))), tempfile()), + "All elements in list 'x' must be numeric!" + ) +}) + test_that("factors are written as level indices", { temp_file <- tempfile() read_x <- function(file) jsonlite::read_json(file, simplifyVector = TRUE)$x From 7cbb6f405fc250721ca2362f2662c8e5671a63b2 Mon Sep 17 00:00:00 2001 From: jgabry Date: Fri, 24 Jul 2026 13:07:27 -0600 Subject: [PATCH 4/6] Update NEWS.md --- NEWS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 8c5dcd093..f52bb472f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -6,10 +6,10 @@ variables and errored for `int` ones. (#817) * Data frame columns that are not numeric, integer, logical, or factor are now an error. Previously `data.matrix()` silently coerced them, so a character column reached Stan as alphabetically ordered integer codes. Use -`factor()` explicitly if integer codes are what you want. (#817) +`factor()` explicitly if integer codes are what you want. (#1225) * Lists of logical vectors/matrices are now converted to integers like logical -variables are, instead of erroring. (#817) -* Supplying a factor for a variable not declared as `int` is now an error. (#817) +variables are, instead of erroring. (#1225) +* Supplying a factor for a variable not declared as `int` is now an error. (#1225) * The `CMDSTANR_NO_VER_CHECK` R option and environment variable are deprecated as of CmdStanR 1.0.0; use the lowercase `cmdstanr_no_ver_check` forms instead. * `$cpp_options()` no longer includes a `STAN_VERSION` entry read from the model From 014b0fda64bc93f7333b8b9c4c140ac510150975 Mon Sep 17 00:00:00 2001 From: jgabry Date: Fri, 24 Jul 2026 13:46:58 -0600 Subject: [PATCH 5/6] Fix factor handling for length-1 arrays and data frame columns array() drops the factor class, so the length-1 reshaping left a character array behind and factors failed for array[1] int. The factor check also ran after data.matrix(), so a factor column supplied for a real variable was silently converted to level codes. Move factor handling ahead of both conversions: convert to integer for int variables, and error for any other type, including data frames with factor columns. Describe level ordering in terms of levels() rather than "alphabetical", which is wrong for numeric input, and correct the list element error message now that logical elements are accepted. --- NEWS.md | 2 ++ R/data.R | 66 ++++++++++++++++++++++--------------- man/write_stan_json.Rd | 26 ++++++++++----- tests/testthat/test-data.R | 41 +++++++++++++++++++++++ tests/testthat/test-json.R | 6 ++-- tests/testthat/test-utils.R | 2 +- 6 files changed, 103 insertions(+), 40 deletions(-) diff --git a/NEWS.md b/NEWS.md index f52bb472f..1424c7094 100644 --- a/NEWS.md +++ b/NEWS.md @@ -10,6 +10,8 @@ character column reached Stan as alphabetically ordered integer codes. Use * Lists of logical vectors/matrices are now converted to integers like logical variables are, instead of erroring. (#1225) * Supplying a factor for a variable not declared as `int` is now an error. (#1225) +* Factors are now accepted for length-1 `int` arrays (e.g. `array[1] int x`), +which previously errored. (#1225) * The `CMDSTANR_NO_VER_CHECK` R option and environment variable are deprecated as of CmdStanR 1.0.0; use the lowercase `cmdstanr_no_ver_check` forms instead. * `$cpp_options()` no longer includes a `STAN_VERSION` entry read from the model diff --git a/R/data.R b/R/data.R index 7022d67eb..4765ce25f 100644 --- a/R/data.R +++ b/R/data.R @@ -21,16 +21,19 @@ #' * `list` -> `array` #' * `table` -> `vector`, `matrix`, or `array` (depending on dimensions of table) #' -#' Factors are written as their level indices, which depend on the order of the -#' factor's levels (alphabetical by default) rather than on the values -#' themselves. For example, `factor(c(10, 9, 8))` is written as `[3, 2, 1]`, and -#' an unused level shifts the indices of the levels after it. If the original -#' values are what you want, convert them first, e.g. with -#' `as.numeric(as.character(x))`. The fitting methods of a model compiled from a -#' Stan file error if a factor is supplied for a variable that is not declared -#' as `int`, but `write_stan_json()` has no declarations to check against and so -#' always converts. +#' ### Factor conversion +#' Factors are written as their level indices: the position of each value in +#' `levels(x)` rather than the value itself. The default levels are the sorted +#' unique values, so `factor(c(10, 9, 8))` has levels `8`, `9`, `10` and is +#' written as `[3, 2, 1]`, and an unused level shifts the indices of the levels +#' after it. If the original values are what you want, convert them first, e.g. +#' with `as.numeric(as.character(x))`. The fitting methods of a model compiled +#' from a Stan file error if a factor is supplied for a variable that is not +#' declared as `int`, but `write_stan_json()` has no declarations to check +#' against and so always converts. #' +#' +#' ### List to array conversion #' The `list` to `array` conversion is intended to make it easier to prepare #' the data for certain Stan declarations involving arrays: #' @@ -48,6 +51,7 @@ #' dimensions `KxLxJ`. Nested lists are not supported: every element of the list #' must be a vector, matrix, or array. #' +#' ### Scalar vs. length-1 vector #' Because \R does not distinguish between a scalar and a vector of length 1, a #' length-1 vector like `c(42)` is written to JSON as a scalar (`42`) rather #' than an array (`[42]`). If a Stan variable is declared as a vector or array @@ -138,6 +142,12 @@ is_valid_data_type <- function(x) { } +# TRUE for a factor, or a data frame with any factor column +has_factor <- function(x) { + is.factor(x) || (is.data.frame(x) && any(vapply(x, is.factor, logical(1)))) +} + + # Error if a variable is not one of the types accepted in a data list. Data # frames and lists are accepted here and converted by convert_to_array(). validate_data_type <- function(var, var_name) { @@ -189,7 +199,7 @@ list_to_array <- function(x, name = NULL) { } all_numeric <- all(sapply(x, function(a) is.numeric(a) || is.logical(a))) if (!all_numeric) { - stop("All elements in list '", name, "' must be numeric!", call. = FALSE) + stop("All elements in list '", name, "' must be numeric or logical!", call. = FALSE) } element_num_of_dim <- length(all_dims[[1]]) x <- unlist(x) @@ -232,6 +242,17 @@ process_data <- function(data, model_variables = NULL) { stop("Variable '", var_name, "' is NULL.", call. = FALSE) } validate_data_type(data[[var_name]], var_name) + # Factors are written as level indices, which are only meaningful for + # variables declared as int. Handle them before the conversions below, + # which replace factors with their codes and drop the factor class. + if (data_variables[[var_name]]$type == "int") { + if (is.factor(data[[var_name]])) { + data[[var_name]] <- as.integer(data[[var_name]]) + } + } else if (has_factor(data[[var_name]])) { + stop("A factor was supplied for '", var_name, "', which is declared as '", + data_variables[[var_name]]$type, "'.", call. = FALSE) + } # Convert lists and data frames to arrays before the checks below, # which require an atomic object (#817) data[[var_name]] <- convert_to_array(data[[var_name]], var_name) @@ -240,28 +261,19 @@ process_data <- function(data, model_variables = NULL) { && data_variables[[var_name]]$dimensions == 1) { data[[var_name]] <- array(data[[var_name]], dim = 1) } - # Factors are written as level indices, which are only meaningful for - # variables declared as int - if (data_variables[[var_name]]$type != "int" - && is.factor(data[[var_name]])) { - stop("A factor was supplied for '", var_name, "', which is declared as '", - data_variables[[var_name]]$type, "'.", call. = FALSE) - } # Make sure integer inputs are of integer type to avoid # generating a decimal point in write_stan_json if (data_variables[[var_name]]$type == "int" && !is.integer(data[[var_name]])) { - if (!is.factor(data[[var_name]])) { - if (!isTRUE(all(is_wholenumber(data[[var_name]])))) { - # Don't warn for NULL/NA, as different warnings are used for those - if (!isTRUE(anyNA(data[[var_name]]))) { - warning("A non-integer value was supplied for '", var_name, "'!", - " It will be truncated to an integer.", call. = FALSE) - } - } else { - # Round before setting mode to integer to avoid floating point errors - data[[var_name]] <- round(data[[var_name]]) + if (!isTRUE(all(is_wholenumber(data[[var_name]])))) { + # Don't warn for NULL/NA, as different warnings are used for those + if (!isTRUE(anyNA(data[[var_name]]))) { + warning("A non-integer value was supplied for '", var_name, "'!", + " It will be truncated to an integer.", call. = FALSE) } + } else { + # Round before setting mode to integer to avoid floating point errors + data[[var_name]] <- round(data[[var_name]]) } mode(data[[var_name]]) <- "integer" } diff --git a/man/write_stan_json.Rd b/man/write_stan_json.Rd index 3a1d36a87..a890aed42 100644 --- a/man/write_stan_json.Rd +++ b/man/write_stan_json.Rd @@ -33,16 +33,20 @@ numeric, integer, logical, or factor \item \code{list} -> \code{array} \item \code{table} -> \code{vector}, \code{matrix}, or \code{array} (depending on dimensions of table) } +\subsection{Factor conversion}{ -Factors are written as their level indices, which depend on the order of the -factor's levels (alphabetical by default) rather than on the values -themselves. For example, \code{factor(c(10, 9, 8))} is written as \verb{[3, 2, 1]}, and -an unused level shifts the indices of the levels after it. If the original -values are what you want, convert them first, e.g. with -\code{as.numeric(as.character(x))}. The fitting methods of a model compiled from a -Stan file error if a factor is supplied for a variable that is not declared -as \code{int}, but \code{write_stan_json()} has no declarations to check against and so -always converts. +Factors are written as their level indices: the position of each value in +\code{levels(x)} rather than the value itself. The default levels are the sorted +unique values, so \code{factor(c(10, 9, 8))} has levels \code{8}, \code{9}, \code{10} and is +written as \verb{[3, 2, 1]}, and an unused level shifts the indices of the levels +after it. If the original values are what you want, convert them first, e.g. +with \code{as.numeric(as.character(x))}. The fitting methods of a model compiled +from a Stan file error if a factor is supplied for a variable that is not +declared as \code{int}, but \code{write_stan_json()} has no declarations to check +against and so always converts. +} + +\subsection{List to array conversion}{ The \code{list} to \code{array} conversion is intended to make it easier to prepare the data for certain Stan declarations involving arrays: @@ -61,6 +65,9 @@ leading dimension, so \verb{array[K,L] vector[J] v } can be supplied either as a list of \code{K} matrices each with dimensions \code{LxJ} or as a single \R array with dimensions \code{KxLxJ}. Nested lists are not supported: every element of the list must be a vector, matrix, or array. +} + +\subsection{Scalar vs. length-1 vector}{ Because \R does not distinguish between a scalar and a vector of length 1, a length-1 vector like \code{c(42)} is written to JSON as a scalar (\code{42}) rather @@ -78,6 +85,7 @@ passing a data list to the fitting methods of a model compiled from a Stan file (e.g., \verb{$sample()}), CmdStanR uses the model's variable declarations to make this correction automatically. } +} \examples{ x <- matrix(rnorm(10), 5, 2) y <- rpois(nrow(x), lambda = 10) diff --git a/tests/testthat/test-data.R b/tests/testthat/test-data.R index baf55aaa0..f3a4ba783 100644 --- a/tests/testthat/test-data.R +++ b/tests/testthat/test-data.R @@ -510,6 +510,20 @@ test_that("process_data errors on a factor for a non-int variable", { "A factor was supplied for 'd', which is declared as 'real'." ) + # a factor column of a data frame is caught too, before data.matrix() + # converts it to codes + stan_file <- write_stan_file(" + data { + matrix[2,1] x; + } + ") + mod_matrix <- cmdstan_model(stan_file, compile = FALSE) + expect_error( + process_data(list(x = data.frame(a = factor(c("b", "a")))), + model_variables = mod_matrix$variables()), + "A factor was supplied for 'x', which is declared as 'real'." + ) + # factors are still allowed for int variables expect_no_error( process_data(modifyList(data, list(a = factor("x"))), model_variables = model_variables) @@ -519,6 +533,33 @@ test_that("process_data errors on a factor for a non-int variable", { ) }) +test_that("factors work for length-1 arrays", { + # array() drops the factor class, so the length-1 reshaping used to leave a + # character array behind for these + stan_file <- write_stan_file(" + data { + array[1] int a; + array[1] real b; + } + ") + mod <- cmdstan_model(stan_file, compile = FALSE) + model_variables <- mod$variables() + data <- list(a = 1L, b = 2.5) + + test_file <- process_data(modifyList(data, list(a = factor("x"))), + model_variables = model_variables) + expect_equal(jsonlite::read_json(test_file, simplifyVector = TRUE)$a, 1L) + + expect_error( + process_data(modifyList(data, list(b = factor("x"))), model_variables = model_variables), + "A factor was supplied for 'b', which is declared as 'real'." + ) + + # the length-1 reshaping still works for non-factors + test_file <- process_data(modifyList(data, list(a = 5)), model_variables = model_variables) + expect_equal(jsonlite::read_json(test_file, simplifyVector = TRUE)$a, 5L) +}) + test_that("Floating-point differences do not cause truncation towards 0", { stan_file <- write_stan_file(" data { diff --git a/tests/testthat/test-json.R b/tests/testthat/test-json.R index 6c1ce5146..834c93ec8 100644 --- a/tests/testthat/test-json.R +++ b/tests/testthat/test-json.R @@ -158,7 +158,7 @@ test_that("a list contributes one leading dimension", { # nested lists are not supported expect_error( write_stan_json(list(v = list(list(1:4, 5:8), list(9:12, 13:16))), tempfile()), - "All elements in list 'v' must be numeric!" + "All elements in list 'v' must be numeric or logical!" ) }) @@ -179,7 +179,7 @@ test_that("logical elements of a list are converted to integers", { # factors are still not allowed as list elements expect_error( write_stan_json(list(x = list(factor("a"), factor("b"))), tempfile()), - "All elements in list 'x' must be numeric!" + "All elements in list 'x' must be numeric or logical!" ) }) @@ -210,7 +210,7 @@ test_that("factors are written as level indices", { test_that("write_stan_json() errors if invalid types", { expect_error( write_stan_json(list(N = list("abc", "def")), file = "abc.txt"), - "All elements in list 'N' must be numeric!" + "All elements in list 'N' must be numeric or logical!" ) expect_error( diff --git a/tests/testthat/test-utils.R b/tests/testthat/test-utils.R index 5d4171f44..898433426 100644 --- a/tests/testthat/test-utils.R +++ b/tests/testthat/test-utils.R @@ -220,7 +220,7 @@ test_that("list_to_array works with empty list", { test_that("list_to_array fails for non-numeric values", { expect_error(list_to_array(list(k = "test"), name = "test-list"), - "All elements in list 'test-list' must be numeric!") + "All elements in list 'test-list' must be numeric or logical!") }) test_that("cmdstan_make_local() works", { From 4156dc9e681813b2749a762e49006e3a69885846 Mon Sep 17 00:00:00 2001 From: jgabry Date: Fri, 24 Jul 2026 15:21:17 -0600 Subject: [PATCH 6/6] Find NAs nested inside lists in write_stan_json() --- NEWS.md | 17 +++++++++-------- R/data.R | 4 +++- tests/testthat/test-data.R | 5 +++-- tests/testthat/test-json.R | 10 ++++++++++ 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/NEWS.md b/NEWS.md index 1424c7094..9ebafc794 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,16 +1,17 @@ # cmdstanr (development version) -* Lists of matrices/vectors and data frames can now be supplied for variables -declared as `int` in the Stan program. Previously these worked only for `real` +* Lists of matrices/vectors and data frames can now be supplied for variables +declared as `int` in the Stan program. Previously these worked only for `real` variables and errored for `int` ones. (#817) -* Data frame columns that are not numeric, integer, logical, or -factor are now an error. Previously `data.matrix()` silently coerced them, so a -character column reached Stan as alphabetically ordered integer codes. Use -`factor()` explicitly if integer codes are what you want. (#1225) -* Lists of logical vectors/matrices are now converted to integers like logical +* Data frame columns that are not numeric, integer, logical, or +factor are now an error. Previously `data.matrix()` silently coerced them, so a +character column reached Stan as alphabetically ordered integer codes. Convert +the column explicitly, e.g. with `as.integer()`, if integer codes are what you +want. (#1225) +* Lists of logical vectors/matrices are now converted to integers like logical variables are, instead of erroring. (#1225) * Supplying a factor for a variable not declared as `int` is now an error. (#1225) -* Factors are now accepted for length-1 `int` arrays (e.g. `array[1] int x`), +* Factors are now accepted for length-1 `int` arrays (e.g. `array[1] int x`), which previously errored. (#1225) * The `CMDSTANR_NO_VER_CHECK` R option and environment variable are deprecated as of CmdStanR 1.0.0; use the lowercase `cmdstanr_no_ver_check` forms instead. diff --git a/R/data.R b/R/data.R index 4765ce25f..bcf1d4571 100644 --- a/R/data.R +++ b/R/data.R @@ -117,10 +117,12 @@ write_stan_json <- function(data, file, always_decimal = FALSE) { stop("Variable '", var_name, "' is NULL.", call. = FALSE) } validate_data_type(var, var_name) + var <- convert_to_array(var, var_name) + # after the conversion, so that NAs nested inside a list are also found if (anyNA(var)) { stop("Variable '", var_name, "' has NA values.", call. = FALSE) } - data[[var_name]] <- convert_to_array(var, var_name) + data[[var_name]] <- var } # unboxing variables (N = 10 is stored as N : 10, not N: [10]) diff --git a/tests/testthat/test-data.R b/tests/testthat/test-data.R index f3a4ba783..bbdc19ab8 100644 --- a/tests/testthat/test-data.R +++ b/tests/testthat/test-data.R @@ -546,9 +546,10 @@ test_that("factors work for length-1 arrays", { model_variables <- mod$variables() data <- list(a = 1L, b = 2.5) + # read without simplification, which would make [1] indistinguishable from 1 test_file <- process_data(modifyList(data, list(a = factor("x"))), model_variables = model_variables) - expect_equal(jsonlite::read_json(test_file, simplifyVector = TRUE)$a, 1L) + expect_equal(jsonlite::read_json(test_file)$a, list(1L)) expect_error( process_data(modifyList(data, list(b = factor("x"))), model_variables = model_variables), @@ -557,7 +558,7 @@ test_that("factors work for length-1 arrays", { # the length-1 reshaping still works for non-factors test_file <- process_data(modifyList(data, list(a = 5)), model_variables = model_variables) - expect_equal(jsonlite::read_json(test_file, simplifyVector = TRUE)$a, 5L) + expect_equal(jsonlite::read_json(test_file)$a, list(5L)) }) test_that("Floating-point differences do not cause truncation towards 0", { diff --git a/tests/testthat/test-json.R b/tests/testthat/test-json.R index 834c93ec8..36527f54f 100644 --- a/tests/testthat/test-json.R +++ b/tests/testthat/test-json.R @@ -91,6 +91,16 @@ test_that("write_stan_json errors if NAs", { write_stan_json(list(x = list(1, NA)), tempfile()), "Variable 'x' has NA values" ) + # NAs nested inside list elements are found too, rather than being written + # to the JSON as the string "NA" + expect_error( + write_stan_json(list(x = list(c(1, NA), c(3, 4))), tempfile()), + "Variable 'x' has NA values" + ) + expect_error( + write_stan_json(list(x = list(matrix(c(1, NA, 3, 4), 2), matrix(1:4, 2))), tempfile()), + "Variable 'x' has NA values" + ) }) test_that("write_stan_json errors if NULL variables", {