Handle unicode endianness in C code

Author: Bisaloo

R/read_npy.R

@@ -164,21 +164,29 @@ parse_npy_data <- function(bytes, shape, datatype, signed, typesize, endian) {
   num_elements <- prod(shape)
 
   bytes <- readBin(bytes, "raw", n = num_elements * typesize)
-  # Unicode is UTF-32: each codepoint is 4 bytes, so swap per codepoint.
-  swap_unit <- if (datatype == "unicode") 4L else typesize
   # FIXME: optimize this
-  if (!is.na(endian) && endian != .Platform$endian) {
-    ind <- rep_len(rev(seq_len(swap_unit)), length(bytes)) +
-      (seq_along(bytes) - 1L) %/% swap_unit * swap_unit
+  if (datatype != "unicode" && !is.na(endian) && endian != .Platform$endian) {
+    ind <- rep_len(rev(seq_len(typesize)), length(bytes)) +
+      (seq_along(bytes) - 1L) %/% typesize * typesize
     bytes <- bytes[ind]
   }
 
-  data <- .Call(
-    paste0("type_convert_", datatype),
-    bytes,
-    typesize,
-    PACKAGE = "grumpy"
-  )
+  if (datatype == "unicode") {
+    data <- .Call(
+      "type_convert_unicode",
+      bytes,
+      typesize,
+      endian,
+      PACKAGE = "grumpy"
+    )
+  } else {
+    data <- .Call(
+      paste0("type_convert_", datatype),
+      bytes,
+      typesize,
+      PACKAGE = "grumpy"
+    )
+  }
 
   dim(data) <- shape
 

src/type_conversion.c

@@ -179,11 +179,11 @@ SEXP type_convert_string(SEXP input, SEXP _n_bytes) {
   return(data);
 }
 
-SEXP type_convert_unicode(SEXP input, SEXP _n_bytes) {
+SEXP type_convert_unicode(SEXP input, SEXP _n_bytes, SEXP _endian) {
 
   // n_bytes is the total bytes per string element (num_codepoints * 4).
-  // The raw bytes have already been endian-swapped per codepoint (4 bytes each)
-  // before being passed here, so the encoding is UTF-32LE.
+  // Bytes are passed as-is from the file; we select UTF-32LE or UTF-32BE
+  // based on the file's endianness so no R-side byte-swapping is needed.
   const size_t n_bytes = (size_t)INTEGER(_n_bytes)[0];
   const R_xlen_t length = xlength(input);
   const char *raw_buffer = (const char *)RAW(input);
@@ -192,10 +192,13 @@ SEXP type_convert_unicode(SEXP input, SEXP _n_bytes) {
   R_xlen_t i;
   SEXP data;
 
+  const char *endian = CHAR(STRING_ELT(_endian, 0));
+  const char *utf32_enc = (strcmp(endian, "big") == 0) ? "UTF-32BE" : "UTF-32LE";
+
   // Worst case: 4 UTF-8 bytes per UTF-32 codepoint.
   char *utf8_buf = (char *)R_alloc(n_bytes + 1, 1);
 
-  void *cd = Riconv_open("UTF-8", "UTF-32LE");
+  void *cd = Riconv_open("UTF-8", utf32_enc);
   if (cd == (void *)-1)
     error("Riconv_open failed: cannot convert UTF-32LE to UTF-8");
 

src/type_conversion.h

@@ -8,4 +8,4 @@ SEXP type_convert_uint(SEXP input, SEXP _n_bytes);
 SEXP type_convert_float(SEXP input, SEXP _n_bytes);
 SEXP type_convert_bool(SEXP input, SEXP _n_bytes);
 SEXP type_convert_string(SEXP input, SEXP _n_bytes);
-SEXP type_convert_unicode(SEXP input, SEXP _n_bytes);
+SEXP type_convert_unicode(SEXP input, SEXP _n_bytes, SEXP _endian);