diff --git a/DESCRIPTION b/DESCRIPTION index c704a2f..cfebfb4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: grumpy Title: Read 'NumPy' '.npy' and '.npz' Files -Version: 0.1.1.9001 +Version: 0.1.1.9002 Authors@R: c( person("Hugo", "Gruson", , "hugo.gruson+R@normalesup.org", role = c("aut", "cre", "cph"), comment = c(ORCID = "0000-0002-4094-1476")), diff --git a/NEWS.md b/NEWS.md index 34f5af3..c5ba092 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,12 @@ # grumpy (development version) +## Significant new features + +* `read_npy()` gains a new `lazy` argument. When `lazy = TRUE`, raw bytes are + read lazily using ALTREP and mmap against the `.npy` file on disk, resulting + in much better speed & memory performance. This is following a feature + request from @btraven00 in #11. + # grumpy 0.1.1 * The package title and description have been revised based on CRAN feedback. diff --git a/R/read_npy.R b/R/read_npy.R index 4c2efce..da0dd35 100644 --- a/R/read_npy.R +++ b/R/read_npy.R @@ -1,6 +1,11 @@ #' Read a .npy file #' #' @param file Path to the .npy file +#' @param lazy If `TRUE`, and `file` is a path (not a connection), the data +#' payload is memory-mapped rather than read into memory upfront via +#' [readBin()]. This limits the number of copies in memory when the npy file +#' contains types native to R. Requires a POSIX platform; ignored (with a +#' warning) otherwise. Defaults to `FALSE`. #' @param ... Ignored. Reserved for future use. #' #' @returns An array containing the data from the .npy file @@ -11,8 +16,7 @@ #' read_npy( #' system.file("extdata", "test.npy", package = "grumpy") #' ) - -read_npy <- function(file, ...) { +read_npy <- function(file, lazy = FALSE, ...) { chkDots(...) if (is.character(file)) { @@ -22,6 +26,14 @@ read_npy <- function(file, ...) { con <- file(file, "rb") on.exit(close(con)) } else if (inherits(file, "connection")) { + if (lazy) { + warning( + "`lazy = TRUE` is only supported when `file` is a path; ", + "ignoring it for connections.", + call. = FALSE + ) + lazy <- FALSE + } con <- file } else { stop( @@ -73,7 +85,19 @@ read_npy <- function(file, ...) { # Read the data num_elements <- prod(header$shape) - bytes <- readBin(con, "raw", n = sum(num_elements * header$nbytes)) + n_bytes <- sum(num_elements * header$nbytes) + + if (lazy) { + # Offset of the payload is wherever the connection currently sits: + # 6 (magic) + 2 (version) + header_len_field_size (2 or 4) + header_len + payload_offset <- seek(con, origin = "current") + # When lazy = TRUE, we know `file` is a path, not a connection, so we can + # mmap it directly. + bytes <- mmap_raw(file, offset = payload_offset, length = n_bytes) + } else { + bytes <- readBin(con, "raw", n = n_bytes) + } + convert_bytes_to_array( bytes, what = header$base_type, @@ -83,6 +107,31 @@ read_npy <- function(file, ...) { ) } +#' Create an ALTREP raw vector backed by a memory-mapped file slice +#' +#' Internal helper used by [read_npy()] when `lazy = TRUE`. The returned +#' object behaves like an ordinary raw vector (as produced by [readBin()]), +#' but its bytes are only paged in from disk on first access, and no R-level +#' allocation happens for the full payload upfront. It can be passed to +#' [convert_bytes_to_array()] exactly like a materialized raw vector. +#' +#' @param path Path to the file to map. +#' @param offset Byte offset of the start of the desired slice. +#' @param length Number of bytes to expose. +#' +#' @returns An ALTREP raw vector of length `length`. +#' +#' @noRd +mmap_raw <- function(path, offset, length) { + .Call( + C_grumpy_make_mmap_raw, + path.expand(path), + as.double(offset), + as.double(length), + PACKAGE = "grumpy" + ) +} + parse_npy_descr <- function(bytes) { # TODO: If I understand correctly, fortranarray in python are still displayed # the same way as regular arrays, but with a different order in memory. diff --git a/man/read_npy.Rd b/man/read_npy.Rd index 6e7940b..0ec1a19 100644 --- a/man/read_npy.Rd +++ b/man/read_npy.Rd @@ -4,11 +4,17 @@ \alias{read_npy} \title{Read a .npy file} \usage{ -read_npy(file, ...) +read_npy(file, lazy = FALSE, ...) } \arguments{ \item{file}{Path to the .npy file} +\item{lazy}{If \code{TRUE}, and \code{file} is a path (not a connection), the data +payload is memory-mapped rather than read into memory upfront via +\code{\link[=readBin]{readBin()}}. This limits the number of copies in memory when the npy file +contains types native to R. Requires a POSIX platform; ignored (with a +warning) otherwise. Defaults to \code{FALSE}.} + \item{...}{Ignored. Reserved for future use.} } \value{ diff --git a/src/altrep_mmap.c b/src/altrep_mmap.c new file mode 100644 index 0000000..c7707c7 --- /dev/null +++ b/src/altrep_mmap.c @@ -0,0 +1,127 @@ +#include "altrep_mmap.h" + +#include +#include + +// FIXME: we only support POSIX mmap() for now. +// Windows has a different API (CreateFileMapping / MapViewOfFile) and would +// require a separate implementation. +#ifndef _WIN32 +#include +#include +#include +#include +#define GRUMPY_HAVE_MMAP 1 +#endif + +// ALTREP raw vector backed by a read-only mmap() of a slice of a file. +// +// Having a file-backed, lazy read of raw data is useful in the cases when +// we don't need to do anything transformation (e.g., int32). +// +// It could also be helpful in the future if we ever implement an `index` +// argument to `read_npy()` that allows reading a subset of the data. + +typedef struct { + void *addr; // mmap base address (page-aligned) + size_t map_len; // bytes mapped (offset rounded down + length) + size_t skip; // bytes to skip within the mapping to reach the logical start + R_xlen_t length; // logical length exposed to R +} grumpy_mmap_info; + +static R_altrep_class_t grumpy_mmap_raw_class; + +static void grumpy_mmap_xptr_finalize(SEXP xp) { + grumpy_mmap_info *info = (grumpy_mmap_info *) R_ExternalPtrAddr(xp); + if (info == NULL) return; +#ifdef GRUMPY_HAVE_MMAP + if (info->addr != NULL && info->addr != MAP_FAILED) { + munmap(info->addr, info->map_len); + } +#endif + free(info); + R_ClearExternalPtr(xp); +} + +static R_xlen_t grumpy_mmap_length(SEXP x) { + const grumpy_mmap_info *info = (grumpy_mmap_info *) R_ExternalPtrAddr(R_altrep_data1(x)); + return info->length; +} + +static void *grumpy_mmap_dataptr(SEXP x, Rboolean writeable) { + const grumpy_mmap_info *info = (grumpy_mmap_info *) R_ExternalPtrAddr(R_altrep_data1(x)); + return (char *) info->addr + info->skip; +} + +static const void *grumpy_mmap_dataptr_or_null(SEXP x) { + return grumpy_mmap_dataptr(x, FALSE); +} + +static Rbyte grumpy_mmap_elt(SEXP x, R_xlen_t i) { + const Rbyte *p = (const Rbyte *) grumpy_mmap_dataptr_or_null(x); + return p[i]; +} + +static R_xlen_t grumpy_mmap_get_region(SEXP x, R_xlen_t start, R_xlen_t n, Rbyte *buf) { + const Rbyte *p = (const Rbyte *) grumpy_mmap_dataptr_or_null(x); + R_xlen_t len = grumpy_mmap_length(x); + R_xlen_t ncopy = (start + n > len) ? (len - start) : n; + memcpy(buf, p + start, ncopy); + return ncopy; +} + +void grumpy_init_mmap_altrep(DllInfo *info) { + grumpy_mmap_raw_class = R_make_altraw_class("grumpy_mmap_raw", "grumpy", info); + + R_set_altrep_Length_method(grumpy_mmap_raw_class, grumpy_mmap_length); + R_set_altvec_Dataptr_method(grumpy_mmap_raw_class, grumpy_mmap_dataptr); + R_set_altvec_Dataptr_or_null_method(grumpy_mmap_raw_class, grumpy_mmap_dataptr_or_null); + R_set_altraw_Elt_method(grumpy_mmap_raw_class, grumpy_mmap_elt); + R_set_altraw_Get_region_method(grumpy_mmap_raw_class, grumpy_mmap_get_region); +} + +SEXP grumpy_make_mmap_raw(SEXP path_, SEXP offset_, SEXP length_) { +#ifndef GRUMPY_HAVE_MMAP + error("mmap-backed reading is not supported on this platform"); +#else + const char *path = CHAR(STRING_ELT(path_, 0)); + double offset = REAL(offset_)[0]; + double length = REAL(length_)[0]; + + if (offset < 0 || length < 0) + error("offset and length must be non-negative"); + + int fd = open(path, O_RDONLY); + if (fd < 0) + error("Cannot open file for mmap: %s", path); + + long page_size = sysconf(_SC_PAGE_SIZE); + size_t page_offset = ((size_t) offset) % page_size; + off_t map_start = (off_t) offset - (off_t) page_offset; + size_t map_len = (size_t) length + page_offset; + + void *addr = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fd, map_start); + // The fd is not needed once mapped. + close(fd); + + if (addr == MAP_FAILED) + error("mmap() failed for file: %s", path); + + grumpy_mmap_info *map_info = malloc(sizeof(grumpy_mmap_info)); + if (map_info == NULL) { + munmap(addr, map_len); + error("Failed to allocate mmap bookkeeping struct"); + } + map_info->addr = addr; + map_info->map_len = map_len; + map_info->skip = page_offset; + map_info->length = (R_xlen_t) length; + + SEXP xp = PROTECT(R_MakeExternalPtr(map_info, R_NilValue, R_NilValue)); + R_RegisterCFinalizerEx(xp, grumpy_mmap_xptr_finalize, TRUE); + SEXP ans = PROTECT(R_new_altrep(grumpy_mmap_raw_class, xp, R_NilValue)); + UNPROTECT(2); + + return ans; +#endif /* GRUMPY_HAVE_MMAP */ +} diff --git a/src/altrep_mmap.h b/src/altrep_mmap.h new file mode 100644 index 0000000..328aded --- /dev/null +++ b/src/altrep_mmap.h @@ -0,0 +1,13 @@ +#ifndef _GRUMPY_ALTREP_MMAP_H +#define _GRUMPY_ALTREP_MMAP_H + +#include "grumpy.h" + +// Create an ALTREP raw vector backed by a read-only mmap() of `path`, +// exposing `length` bytes starting at file offset `offset`. +SEXP grumpy_make_mmap_raw(SEXP path, SEXP offset, SEXP length); + +// Register the ALTREP class and its methods. Called from R_init_grumpy(). +void grumpy_init_mmap_altrep(DllInfo *info); + +#endif /* _GRUMPY_ALTREP_MMAP_H */ diff --git a/src/grumpy.c b/src/grumpy.c index d602205..c16758a 100644 --- a/src/grumpy.c +++ b/src/grumpy.c @@ -1,8 +1,10 @@ #include "grumpy.h" #include "type_conversion.h" +#include "altrep_mmap.h" static const R_CallMethodDef callMethods[] = { {"type_convert", (DL_FUNC) &type_convert, 5}, + {"grumpy_make_mmap_raw", (DL_FUNC) &grumpy_make_mmap_raw, 3}, {NULL, NULL, 0} }; @@ -11,4 +13,5 @@ void R_init_grumpy(DllInfo *info) R_registerRoutines(info, NULL, callMethods, NULL, NULL); R_useDynamicSymbols(info, FALSE); R_forceSymbols(info, TRUE); + grumpy_init_mmap_altrep(info); } diff --git a/src/type_conversion.c b/src/type_conversion.c index ab2c300..e7572c0 100644 --- a/src/type_conversion.c +++ b/src/type_conversion.c @@ -6,18 +6,22 @@ SEXP type_convert(SEXP input, SEXP what, SEXP _n_bytes, SEXP dims, SEXP _endian) const char *type = CHAR(STRING_ELT(what, 0)); SEXP result; + const int type_len = INTEGER(_n_bytes)[0]; + const R_xlen_t buf_len = xlength(input); + const void* raw_buffer = RAW_RO(input); + if (strcmp(type, "float") == 0) - result = type_convert_float(input, _n_bytes); + result = type_convert_float(raw_buffer, buf_len, type_len); else if (strcmp(type, "int") == 0) - result = type_convert_int(input, _n_bytes); + result = type_convert_int(raw_buffer, buf_len, type_len); else if (strcmp(type, "uint") == 0) - result = type_convert_uint(input, _n_bytes); + result = type_convert_uint(raw_buffer, buf_len, type_len); else if (strcmp(type, "bool") == 0) - result = type_convert_bool(input, _n_bytes); + result = type_convert_bool(raw_buffer, buf_len, type_len); else if (strcmp(type, "string") == 0) - result = type_convert_string(input, _n_bytes); + result = type_convert_string((const char*)raw_buffer, buf_len, type_len); else if (strcmp(type, "unicode") == 0) - result = type_convert_unicode(input, _n_bytes, _endian); + result = type_convert_unicode((const char*)raw_buffer, buf_len, type_len, _endian); else error("Unsupported data type: %s", type); @@ -30,32 +34,28 @@ SEXP type_convert(SEXP input, SEXP what, SEXP _n_bytes, SEXP dims, SEXP _endian) return result; } -SEXP type_convert_int(SEXP input, SEXP _n_bytes) { - - const int n_bytes = INTEGER(_n_bytes)[0]; - const R_xlen_t length = xlength(input); - const void* raw_buffer = RAW(input); +SEXP type_convert_int(const void* raw_buffer, R_xlen_t buf_len, int type_len) { int *p_data; SEXP data; - const R_xlen_t data_length = length / n_bytes; + const R_xlen_t data_length = buf_len / type_len; R_xlen_t i; // space for the converted output data = PROTECT(allocVector(INTSXP, data_length)); p_data = INTEGER(data); - if(n_bytes == 1) { + if(type_len == 1) { for (i = 0; i < data_length; i++) { p_data[i] = ((const int8_t *)raw_buffer)[i]; } - } else if(n_bytes == 2) { + } else if(type_len == 2) { for (i = 0; i < data_length; i++) { p_data[i] = ((const int16_t *)raw_buffer)[i]; } - } else if(n_bytes == 4) { - memcpy(p_data, raw_buffer, length); - } else if (n_bytes == 8) { + } else if(type_len == 4) { + memcpy(p_data, raw_buffer, buf_len); + } else if (type_len == 8) { // for now we convert to 32bit int and overflow values are NA_integer int bit64conversion = 0; if (bit64conversion == 0) { @@ -67,32 +67,28 @@ SEXP type_convert_int(SEXP input, SEXP _n_bytes) { return(data); } -SEXP type_convert_uint(SEXP input, SEXP _n_bytes) { - - const int n_bytes = INTEGER(_n_bytes)[0]; - const R_xlen_t length = xlength(input); - const void* raw_buffer = RAW(input); +SEXP type_convert_uint(const void* raw_buffer, R_xlen_t buf_len, int type_len) { int *p_data; SEXP data; - const R_xlen_t data_length = length / n_bytes; + const R_xlen_t data_length = buf_len / type_len; R_xlen_t i; // space for the converted output data = PROTECT(allocVector(INTSXP, data_length)); p_data = INTEGER(data); - if(n_bytes == 1) { + if(type_len == 1) { for (i = 0; i < data_length; i++) { p_data[i] = ((const uint8_t *)raw_buffer)[i]; } - } else if(n_bytes == 2) { + } else if(type_len == 2) { for (i = 0; i < data_length; i++) { p_data[i] = ((const uint16_t *)raw_buffer)[i]; } - } else if(n_bytes == 4) { + } else if(type_len == 4) { uint32_to_int32(raw_buffer, data_length, p_data); - } else if (n_bytes == 8) { + } else if (type_len == 8) { // for now we convert to 32bit int and overflow values are NA_integer int bit64conversion = 0; if (bit64conversion == 0) { @@ -104,53 +100,41 @@ SEXP type_convert_uint(SEXP input, SEXP _n_bytes) { return(data); } -SEXP type_convert_float(SEXP input, SEXP _n_bytes) { - - const int n_bytes = INTEGER(_n_bytes)[0]; - const R_xlen_t length = xlength(input); - const void* raw_buffer = RAW(input); +SEXP type_convert_float(const void* raw_buffer, R_xlen_t buf_len, int type_len) { - R_xlen_t data_length, i; double *p_data; SEXP data; + const R_xlen_t data_length = buf_len / type_len; + R_xlen_t i; - data_length = length / n_bytes; data = PROTECT(allocVector(REALSXP, data_length)); p_data = REAL(data); - if(n_bytes == 2) { - + if(type_len == 2) { const uint16_t *mock_buffer = (const uint16_t *)raw_buffer; for (i = 0; i < data_length; i++) { p_data[i] = (double)float16_to_float64(mock_buffer[i]); } - - } else if(n_bytes == 4) { - + } else if(type_len == 4) { for (i = 0; i < data_length; i++) { p_data[i] = (double)((const float *)raw_buffer)[i]; } - - } else if (n_bytes == 8) { - memcpy(p_data, raw_buffer, length); + } else if (type_len == 8) { + memcpy(p_data, raw_buffer, buf_len); } else { - error("%d byte floating point values are not currently supported\n", n_bytes); + error("%d byte floating point values are not currently supported\n", type_len); } UNPROTECT(1); return(data); } -SEXP type_convert_bool(SEXP input, SEXP _n_bytes) { - - const R_xlen_t length = xlength(input); - const void* raw_buffer = RAW(input); +SEXP type_convert_bool(const void* raw_buffer, R_xlen_t buf_len, int type_len) { int *p_data; - R_xlen_t i; SEXP data; - - const R_xlen_t data_length = length; + const R_xlen_t data_length = buf_len / type_len; + R_xlen_t i; data = PROTECT(allocVector(LGLSXP, data_length)); p_data = LOGICAL(data); @@ -163,23 +147,19 @@ SEXP type_convert_bool(SEXP input, SEXP _n_bytes) { return(data); } -SEXP type_convert_string(SEXP input, SEXP _n_bytes) { - - const int n_bytes = INTEGER(_n_bytes)[0]; - const R_xlen_t length = xlength(input); - const char* raw_buffer = (const char *)RAW(input); +SEXP type_convert_string(const char* raw_buffer, R_xlen_t buf_len, int type_len) { - const R_xlen_t data_length = length / n_bytes; + const R_xlen_t data_length = buf_len / type_len; R_xlen_t i; SEXP data; data = PROTECT(allocVector(STRSXP, data_length)); for (i = 0; i < data_length; i++) { - const char *field = raw_buffer + i * n_bytes; - // Check for R's NA_integer_ sentinel written by writeBin(NA_integer_, raw()). + const char *field = raw_buffer + i * type_len; + // Check for R's NA_integer_ sentinel written by writeBin(NA_integer_, RAW_RO)). // Using memcpy + NA_INTEGER comparison is endian-agnostic. - if (n_bytes >= 4) { + if (type_len >= 4) { int sentinel; memcpy(&sentinel, field, 4); if (sentinel == NA_INTEGER) { @@ -191,8 +171,8 @@ SEXP type_convert_string(SEXP input, SEXP _n_bytes) { // Read up to max length or NUL terminator. // We cannot do one without the other as strings may not be NUL terminated (truncated) and // mkCharLenCE complains about NUL characters in the string. - if (len > (size_t)n_bytes) - SET_STRING_ELT(data, i, mkCharLenCE(field, n_bytes, CE_NATIVE)); + if (len > (size_t)type_len) + SET_STRING_ELT(data, i, mkCharLenCE(field, type_len, CE_NATIVE)); else SET_STRING_ELT(data, i, mkCharCE(field, CE_NATIVE)); } @@ -201,16 +181,12 @@ SEXP type_convert_string(SEXP input, SEXP _n_bytes) { return(data); } -SEXP type_convert_unicode(SEXP input, SEXP _n_bytes, SEXP _endian) { +SEXP type_convert_unicode(const char* raw_buffer, R_xlen_t buf_len, int type_len, SEXP _endian) { // n_bytes is the total bytes per string element (num_codepoints * 4). // Bytes are passed as-is from the file; we select UTF-32LE or UTF-32BE // based on the file's endianness so no R-side byte-swapping is needed. - const size_t n_bytes = (size_t)INTEGER(_n_bytes)[0]; - const R_xlen_t length = xlength(input); - const char *raw_buffer = (const char *)RAW(input); - - const R_xlen_t data_length = length / n_bytes; + const R_xlen_t data_length = buf_len / type_len; R_xlen_t i; SEXP data; @@ -218,7 +194,7 @@ SEXP type_convert_unicode(SEXP input, SEXP _n_bytes, SEXP _endian) { const char *utf32_enc = (strcmp(endian, "big") == 0) ? "UTF-32BE" : "UTF-32LE"; // Worst case: 4 UTF-8 bytes per UTF-32 codepoint. - char *utf8_buf = (char *)R_alloc(n_bytes + 1, 1); + char *utf8_buf = (char *)R_alloc(type_len + 1, 1); void *cd = Riconv_open("UTF-8", utf32_enc); if (cd == (void *)-1) @@ -227,9 +203,9 @@ SEXP type_convert_unicode(SEXP input, SEXP _n_bytes, SEXP _endian) { data = PROTECT(allocVector(STRSXP, data_length)); for (i = 0; i < data_length; i++) { - const char *inbuf = raw_buffer + i * n_bytes; + const char *inbuf = raw_buffer + i * type_len; // Check for R's NA_integer_ sentinel. - if (n_bytes >= 4) { + if (type_len >= 4) { int sentinel; memcpy(&sentinel, inbuf, 4); if (sentinel == NA_INTEGER) { @@ -239,10 +215,10 @@ SEXP type_convert_unicode(SEXP input, SEXP _n_bytes, SEXP _endian) { } // Find the actual length: stop at the first null codepoint (4 zero bytes), - // or at n_bytes if there is no null terminator. + // or at type_len if there is no null terminator. // This allows us to handle both fixed-length and null-terminated strings. size_t field_bytes = 0; - while (field_bytes + 4 <= n_bytes) { + while (field_bytes + 4 <= type_len) { uint32_t cp; memcpy(&cp, inbuf + field_bytes, 4); if (cp == 0) break; @@ -250,7 +226,7 @@ SEXP type_convert_unicode(SEXP input, SEXP _n_bytes, SEXP _endian) { } size_t inbytesleft = field_bytes; - size_t outbytesleft = n_bytes; // safe upper bound + size_t outbytesleft = type_len; // safe upper bound char *outbuf = utf8_buf; Riconv(cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); diff --git a/src/type_conversion.h b/src/type_conversion.h index 882ed6d..cbf7dae 100644 --- a/src/type_conversion.h +++ b/src/type_conversion.h @@ -4,9 +4,9 @@ #include "float16_conversion.h" SEXP type_convert(SEXP input, SEXP what, SEXP _n_bytes, SEXP dims, SEXP _endian); -SEXP type_convert_int(SEXP input, SEXP _n_bytes); -SEXP type_convert_uint(SEXP input, SEXP _n_bytes); -SEXP type_convert_float(SEXP input, SEXP _n_bytes); -SEXP type_convert_bool(SEXP input, SEXP _n_bytes); -SEXP type_convert_string(SEXP input, SEXP _n_bytes); -SEXP type_convert_unicode(SEXP input, SEXP _n_bytes, SEXP _endian); +SEXP type_convert_int(const void* raw_buffer, R_xlen_t buf_len, int type_len); +SEXP type_convert_uint(const void* raw_buffer, R_xlen_t buf_len, int type_len); +SEXP type_convert_float(const void* raw_buffer, R_xlen_t buf_len, int type_len); +SEXP type_convert_bool(const void* raw_buffer, R_xlen_t buf_len, int type_len); +SEXP type_convert_string(const char* raw_buffer, R_xlen_t buf_len, int type_len); +SEXP type_convert_unicode(const char* raw_buffer, R_xlen_t buf_len, int type_len, SEXP _endian); diff --git a/tests/testthat/test-read_npy.R b/tests/testthat/test-read_npy.R index e4d8d75..bd16eb6 100644 --- a/tests/testthat/test-read_npy.R +++ b/tests/testthat/test-read_npy.R @@ -316,3 +316,15 @@ test_that("scalar or empty arrays work", { expect_no_condition() |> expect_length(0L) }) + +test_that("lazy raw reading gives identical results", { + skip_on_os("windows") + skip_on_os("emscripten") + + f <- system.file("extdata", "test.npy", package = "grumpy") + + expect_identical( + read_npy(f, lazy = TRUE), + read_npy(f, lazy = FALSE) + ) +}) diff --git a/vignettes/design.qmd b/vignettes/design.qmd index cf4b5c8..f8eb60d 100644 --- a/vignettes/design.qmd +++ b/vignettes/design.qmd @@ -16,7 +16,7 @@ Writing is out of scope. When working across multiple languages, one should pref ## Why not use `reticulate`? -- When reading `.npy` files with `{reticulate}`, at some point in time, two or three copies of the data are made in memory. This can be problematic for large files. With `{grumpy}`, two copies of the data are made in memory, with plans to make just one copy in the cases where the data type matches R native types. +- When reading `.npy` files with `{reticulate}`, at some point in time, two or three copies of the data are made in memory. This can be problematic for large files. With `{grumpy}`, one (for types matching R native types; i.e., `int32`, `float64`/`double`) or two copies of the data are made in memory, with plans to make just one copy in the cases where the data type matches R native types. - Reading data with `{reticulate}` requires a Python installation and additional python packages, which users in restricted environments may not have access to. `{grumpy}` is a pure R package with no external dependencies. This is especially important as we expect `{grumpy}` to be used deep in the dependency graph of other packages, and we want to minimize the number of dependencies. - A dedicated R package gives us more flexibility in how edge cases such as 64 bits integers are handled. `{reticulate}` automatically and silently converts 64 bits integers to double, which is a sensible default for many use cases. But we may want to have more control over this behavior, and `{grumpy}` will allow us to do that in the future. Another good example are structured data types (record arrays), which are returned as data.frames, not arrays, with `{reticulate}`. diff --git a/vignettes/grumpy.qmd b/vignettes/grumpy.qmd index b7f2600..8dfe677 100644 --- a/vignettes/grumpy.qmd +++ b/vignettes/grumpy.qmd @@ -29,6 +29,8 @@ Most users are expected to mostly want to use `grumpy::read_npy()` and `grumpy:: read_npy(system.file("extdata", "test_2d.npy", package = "grumpy")) ``` +If supported on your platform (POSIX-compatible platforms; i.e., Linux and macOS), `grumpy` can also memory-map the data payload of `.npy` files, resulting in reduced memory usage for large arrays. This is done by setting the `lazy` argument to `TRUE` when calling `read_npy()`. + ### Structured datatypes One notable example are structured datatypes, where each element of the array is a record with named fields.