Changes following rOpenSci's review

NAMESPACE

@@ -4,5 +4,6 @@ export(convert_bytes_to_array)
 export(parse_npy_datatype)
 export(read_npy)
 export(read_npz)
+importFrom(stats,setNames)
 importFrom(utils,unzip)
 useDynLib(grumpy, .registration = TRUE, .fixes = "C_")

R/read_npy.R

@@ -1,15 +1,36 @@
-#' Read a .npy file
+#' Read a `.npy` file
 #'
-#' @param file Path to the .npy file
+#' @param file Path to the `.npy` file
 #' @param ... Ignored. Reserved for future use.
 #'
-#' @returns An array containing the data from the .npy file
+#' @returns An array containing the data from the `.npy` file
 #'
 #' @export
 #'
 #' @examples
+#' # Array of integers. NumPy "<i4" dtype
 #' read_npy(
-#'   system.file("extdata", "test.npy", package = "grumpy")
+#'   system.file("extdata", "test_int32.npy", package = "grumpy")
+#' )
+#'
+#' # Array of logicals. NumPy "|b1" dtype
+#' read_npy(
+#'   system.file("extdata", "test_bool.npy", package = "grumpy")
+#' )
+#'
+#' # Array of numerics. NumPy "<f8" dtype
+#' read_npy(
+#'   system.file("extdata", "test_float64.npy", package = "grumpy")
+#' )
+#'
+#' # Array of strings. NumPy "<U" dtype
+#' read_npy(
+#'   system.file("extdata", "test_str_unicode.npy", package = "grumpy")
+#' )
+#'
+#' # "Array" of lists. Numpy structured dtype
+#' read_npy(
+#'   system.file("extdata", "test_structured.npy", package = "grumpy")
 #' )
 
 read_npy <- function(file, ...) {
@@ -66,7 +87,8 @@ read_npy <- function(file, ...) {
     stop(
       "This file contains a Python object array. ",
       "Reading .npy files with Python object arrays is not supported. ",
-      "A common reason for this is when the file is saved with `np.save(..., allow_pickle=True)`. ",
+      "A common reason for this is when the file is saved with ",
+      "`np.save(..., allow_pickle=True)`. ",
       call. = FALSE
     )
   }
@@ -106,24 +128,31 @@ parse_npy_descr <- function(bytes) {
 
 #' Parse a NumPy Array-protocol type strings
 #'
-#' @param descr A NumPy dtype description string, or a list of such strings fo
+#' @param descr A NumPy dtype description string, or a list of such strings for
 #'   structured dtypes
 #'
-#' @returns A list containing the parsed data type information, including the base
-#'   type, the number of bytes, and the endianness
+#' @returns A list containing the parsed data type information, including the
+#'   base type, the number of bytes, and the endianness
+#'
+#' @details
+#' If a `list` is passed to `descr`, each element can be of length 1, or of
+#' length 2 in which case the first element corresponds to the name of the field
+#' and the second to its dtype.
 #'
 #' @export
 #'
 #' @examples
 #' parse_npy_datatype(">i8")
 #' parse_npy_datatype("|b1")
+#' # A structured datatype where each element has 3 components, all integers,
+#' # named "r", "g" and "b".
 #' parse_npy_datatype(list(c("r", "<i8"), c("g", "<i8"), c("b", "<i8")))
 #'
 parse_npy_datatype <- function(descr) {
   if (is.list(descr)) {
     # structured data type
     types <- lapply(descr, function(field) {
-      parse_npy_datatype(field[[2]])
+      parse_npy_datatype(field[[2L]])
     })
     return(
       list(
@@ -171,16 +200,16 @@ parse_npy_datatype <- function(descr) {
 #' and endianness specified in the .npy file header.
 #'
 #' @param bytes A raw vector containing the bytes to convert
-#' @param what A character specifying the base type to convert to (e.g., `"float"`,
-#'   `"int"`, `"string"`, etc.)
+#' @param what A character specifying the base type to convert to (e.g.,
+#'   `"float"`, `"int"`, `"string"`, etc.)
 #' @param shape A numeric vector with desired shape of the output array
 #' @param size A numeric value with the number of bytes per element for the
 #'   specified type
 #' @param endian The endianness of the data (`"little"`, `"big"`, or `NA` for
 #'   single-byte types)
 #'
-#' @returns An R array containing the converted data, with the specified shape and
-#'   data type.
+#' @returns An R array containing the converted data, with the specified shape
+#    and data type.
 #'
 #' @export
 #'
@@ -189,7 +218,12 @@ parse_npy_datatype <- function(descr) {
 #' x
 #'
 #' y <- writeBin(c(x), raw()) |>
-#'   convert_bytes_to_array("int", shape = c(2L, 3L), size = 4L, endian = "little")
+#'   convert_bytes_to_array(
+#'      "int",
+#'      shape = c(2L, 3L),
+#'      size = 4L,
+#'      endian = "little"
+#'   )
 #' y
 #' dim(y)
 #' is.array(y)
@@ -212,7 +246,7 @@ convert_bytes_to_array <- function(bytes, what, shape, size, endian) {
         by = record_size,
         length.out = n_records
       )
-      idx <- rep(starts, each = size[[i]]) + seq_len(size[[i]]) - 1
+      idx <- rep(starts, each = size[[i]]) + seq_len(size[[i]]) - 1L
       res_fields[[i]] <- convert_bytes_to_array(
         bytes[idx],
         what = what[[i]],

R/read_npz.R

@@ -1,10 +1,11 @@
-#' Read a .npz file
+#' Read a `.npz` file
 #'
-#' @param file Path to the .npz file
+#' @param file Path to the `.npz` file
 #'
-#' @return A list of arrays containing the data from the .npz file
+#' @return A named list of arrays containing the data from the `.npz` file
 #'
 #' @importFrom utils unzip
+#' @importFrom stats setNames
 #'
 #' @export
 #'
@@ -24,5 +25,6 @@ read_npz <- function(file) {
     con <- unz(file, name, "rb")
     on.exit(close(con))
     read_npy(con)
-  })
+  }) |>
+    setNames(gsub("\\.npy$", "", files$Name))
 }

README.md

@@ -16,11 +16,17 @@ into R. It supports a wide range of data types and array shapes.
 
 As a file format generated by a Python package, `.npy` files are prime
 candidates for using the `{reticulate}` package to read them into R.
-However, this comes with downsides in terms of performance, flexibility,
-and robustness of the R package infrastructure. `{grumpy}`, on the other
-hand, is a pure R package with no dependency and is performant,
-flexible. Overall, it is designed to be used deep in the dependency
-graph of other packages.
+However, using a Python package in R comes with downsides in terms of: -
+performance, since the data needs to be copied across languages /
+sessions - flexibility, since reticulate makes opinionated choices to
+work out of the box for most use cases - robustness, since packages
+depending on reticulate need to risk breaking changes in the python
+packages they now depend on, or need to provide an environment via
+`{basilisk}`.
+
+`{grumpy}`, on the other hand, is a pure R package with no dependency
+and is performant, flexible. Overall, it is designed to be used deep in
+the dependency graph of other packages.
 
 For more details on the motivation and design principles underpinning
 `{grumpy}`, see the dedicated vignette:

README.qmd

@@ -26,7 +26,11 @@ knitr::opts_chunk$set(
 The `{grumpy}` R package provides a way to read NumPy's `.npy` files into R. It supports a wide range of data types and array shapes.
 
 As a file format generated by a Python package, `.npy` files are prime candidates for using the `{reticulate}` package to read them into R.
-However, this comes with downsides in terms of performance, flexibility, and robustness of the R package infrastructure. 
+However, using a Python package in R comes with downsides in terms of:
+- performance, since the data needs to be copied across languages / sessions
+- flexibility, since reticulate makes opinionated choices to work out of the box for most use cases
+- robustness, since packages depending on reticulate need to risk breaking changes in the python packages they now depend on, or need to provide an environment via `{basilisk}`.
+
 `{grumpy}`, on the other hand, is a pure R package with no dependency and is performant, flexible. Overall, it is designed to be used deep in the dependency graph of other packages.
 
 For more details on the motivation and design principles underpinning `{grumpy}`, see the dedicated vignette: `vignette("design", package = "grumpy")`.

inst/scripts/generate_test_data.py

@@ -58,7 +58,7 @@
 
 # Structured array
 dtype = np.dtype([("id", "int32"), ("value", "float64"), ("name", "U10")])
-data = np.array([(1, 3.14, "Alice"), (2, 2.71, "Bob"), (3, 1.62, "Charlie"), (4, 0.0, "Dave"), (5, -1.0, "Eve"), (6, 2.0, "Frank"), (7, 33.12, "Grace")], dtype=dtype)
+data = np.array([(1, 3.14, "Alice"), (2, 2.71, "Bob"), (3, 1.62, "Charlie"), (4, 0.0, "Dave"), (5, -1.0, "Eve"), (6, 2.0, "Frank"), (7, 33.12, "Grace"), (8, 13.9, "Hugo")], dtype=dtype).reshape((2, 4))
 np.save("inst/extdata/test_structured.npy", data)
 
 # NPZ archive (multiple arrays)

tests/testthat/test-read_npy.R

@@ -279,9 +279,10 @@ test_that("structured arrays work", {
           list(4L, 0.0, "Dave"),
           list(5L, -1.0, "Eve"),
           list(6L, 2.0, "Frank"),
-          list(7L, 33.12, "Grace")
+          list(7L, 33.12, "Grace"),
+          list(8L, 13.9, "Hugo")
         ),
-        dim = 7L
+        dim = c(2L, 4L)
       )
     )
 })

vignettes/beyond.qmd

@@ -19,20 +19,20 @@ library(Rarr)
 
 ## Introduction to Zarr
 
-If you are now understanding how the `.npy` file format works, the Zarr format is a natural extension of it.
+If you now understand how the `.npy` file format works, the Zarr format is a natural extension of it.
 Zarr is a chunked, compressed, N-dimensional array storage format. Each chunk of data corresponds to a small sub-array of the original array, and is stored in a separate file, **in a format nearly identical to `.npy`**. The chunk is then usually compressed with high-performance compression algorithms such as `zlib`, `blosc`, or `zstd`.
 
 One benefit of this chunked approach is that it allows for efficient reading and writing of large arrays, as only the necessary chunks need to be read or written, rather than the entire array.
 This is particularly useful for large datasets that do not fit into memory, as it allows for out-of-core processing of the data.
 It also provides strong benefits when some chunks are "empty" (e.g. filled with zeros, such as the white/black background of an image), as these chunks can be skipped during reading and writing, and thus reduce the overall storage requirements of the dataset.
 
 ```{r}
-x <- matrix(0, nrow = 1e4, ncol = 1e4)
-x[300:400, 700:800] <- seq_len(101 * 101)
+x <- matrix(0L, nrow = 1000L, ncol = 1000L)
+x[300L:400L, 700L:800L] <- seq_len(101L * 101L)
 
 f_zarr <- withr::local_tempfile(fileext = ".zarr")
 
-Rarr::write_zarr_array(x, f_zarr, chunk_dim = c(100, 100), compressor = NULL)
+Rarr::write_zarr_array(x, f_zarr, chunk_dim = c(100L, 100L), compressor = NULL)
 ```
 
 ## Storage size comparison
@@ -47,7 +47,7 @@ size <- list.files(f_zarr, full.names = TRUE, recursive = TRUE) |>
 size
 ```
 
-**Without compression**, the equivalent Zarr data is thus 320 kB on disk, so `round(8e8 / size)` times smaller than the `.npy` file. We could also use compression to further reduce the size of the Zarr file on disk, but this is out of scope for this vignette.
+**Without compression**, the equivalent Zarr data is thus 320 kB on disk, so `r round(8e8 / size)` times smaller than the `.npy` file. We could also use compression to further reduce the size of the Zarr file on disk, but this is out of scope for this vignette.
 
 ## Decoding speed comparison
 
@@ -65,7 +65,7 @@ np$save(f_npy, x)
 bm <- bench::mark(
   grumpy = read_npy(f_npy),
   zarr = read_zarr_array(f_zarr),
-  iterations = 50
+  iterations = 50L
 )
 bm
 summary(bm, relative = TRUE)