Support Wasm Bulk Memory Operations

rts/Makefile

@@ -253,7 +253,8 @@ $(RTS_RUST_INCREMENTAL_DEBUG_WASM_A): $(RTS_DEPENDENCIES)
 #
 
 TEST_DEPENDENCIES=$(TOMMATH_WASM_A) $(TOMMATH_BINDINGS_RS)
-TEST_BUILD=cd motoko-rts-tests && cargo build --target=wasm32-wasi
+TEST_COMPILER_FLAGS=-C target-feature=+bulk-memory
+TEST_BUILD=cd motoko-rts-tests && RUSTFLAGS="${TEST_COMPILER_FLAGS}" cargo build --target=wasm32-wasi
 TEST_RUN=wasmtime --disable-cache motoko-rts-tests/target/wasm32-wasi/debug/motoko-rts-tests.wasm
 
 .PHONY: test
@@ -275,7 +276,6 @@ test-incremental: $(TEST_DEPENDENCIES)
 # These symbols from musl are used by the code generator directly
 EXPORTED_SYMBOLS=\
   __wasm_call_ctors \
-  memcpy \
   memcmp \
   tan \
   asin \

rts/motoko-rts-tests/src/main.rs

@@ -12,6 +12,7 @@ mod stream;
 mod text;
 mod utf8;
 
+use motoko_rts::mem_utils::{legacy_memcpy, legacy_memset};
 use motoko_rts::types::{read64, write64, Bytes};
 
 fn main() {
@@ -71,3 +72,13 @@ unsafe extern "C" fn print_ptr(ptr: usize, len: u32) {
     let str: &[u8] = core::slice::from_raw_parts(ptr as *const u8, len as usize);
     println!("[RTS] {}", String::from_utf8_lossy(str));
 }
+
+#[no_mangle]
+unsafe extern "C" fn memory_copy(to: usize, from: usize, length: Bytes<u32>) {
+    legacy_memcpy(to, from, length);
+}
+
+#[no_mangle]
+unsafe extern "C" fn memory_fill(to: usize, value: i32, length: Bytes<u32>) {
+    legacy_memset(to, value, length);
+}

rts/motoko-rts/src/lib.rs

@@ -29,7 +29,7 @@ pub mod gc;
 #[cfg(feature = "ic")]
 mod idl;
 pub mod leb128;
-mod mem_utils;
+pub mod mem_utils;
 pub mod memory;
 pub mod principal_id;
 mod static_checks;

rts/motoko-rts/src/mem_utils.rs

@@ -1,13 +1,31 @@
 use crate::types::{Bytes, Words};
 
+// Potentially optimized Wasm bulk memory operation, defined by the compiler backend.
+extern "C" {
+    fn memory_copy(to: usize, from: usize, n: Bytes<u32>);
+    fn memory_fill(to: usize, value: i32, n: Bytes<u32>);
+}
+
 pub(crate) unsafe fn memcpy_words(to: usize, from: usize, n: Words<u32>) {
-    libc::memcpy(to as *mut _, from as *const _, n.to_bytes().as_usize());
+    memory_copy(to, from, n.to_bytes());
 }
 
 pub(crate) unsafe fn memcpy_bytes(to: usize, from: usize, n: Bytes<u32>) {
-    libc::memcpy(to as *mut _, from as *const _, n.as_usize());
+    memory_copy(to, from, n);
 }
 
 pub(crate) unsafe fn memzero(to: usize, n: Words<u32>) {
-    libc::memset(to as *mut _, 0, n.to_bytes().as_usize());
+    memory_fill(to, 0, n.to_bytes());
+}
+
+// Legacy call used by `ic-ref` with missing Wasm bulk-memory operation support.
+#[no_mangle]
+pub unsafe extern "C" fn legacy_memcpy(to: usize, from: usize, n: Bytes<u32>) {
+    libc::memcpy(to as *mut _, from as *const _, n.as_usize());
+}
+
+// Legacy call used by `ic-ref` with missing Wasm bulk-memory operation support.
+#[no_mangle]
+pub unsafe extern "C" fn legacy_memset(dest: usize, c: i32, n: Bytes<u32>) {
+    libc::memset(dest as *mut _, c, n.as_usize());
 }

src/codegen/compile.ml

@@ -924,7 +924,8 @@ module RTS = struct
 
   (* The connection to the C and Rust parts of the RTS *)
   let system_imports env =
-    E.add_func_import env "rts" "memcpy" [I32Type; I32Type; I32Type] [I32Type]; (* standard libc memcpy *)
+    E.add_func_import env "rts" "legacy_memcpy" [I32Type; I32Type; I32Type] [];
+    E.add_func_import env "rts" "legacy_memset" [I32Type; I32Type; I32Type] [];
     E.add_func_import env "rts" "memcmp" [I32Type; I32Type; I32Type] [I32Type];
     E.add_func_import env "rts" "version" [] [I32Type];
     E.add_func_import env "rts" "parse_idl_header" [I32Type; I32Type; I32Type; I32Type; I32Type] [];
@@ -1179,7 +1180,17 @@ module Heap = struct
 
   (* Convenience functions related to memory *)
   (* Copying bytes (works on unskewed memory addresses) *)
-  let memcpy env = E.call_import env "rts" "memcpy" ^^ G.i Drop
+  let memcpy env = 
+    match E.mode env with
+    | Flags.ICMode | Flags.RefMode -> E.call_import env "rts" "legacy_memcpy"
+    | _ -> G.i MemoryCopy
+
+  (* Used for memory zero-initialization in the RTS *)
+  let memset env =
+    match E.mode env with
+    | Flags.ICMode | Flags.RefMode -> E.call_import env "rts" "legacy_memset"
+    | _ -> G.i MemoryFill
+
   (* Comparing bytes (works on unskewed memory addresses) *)
   let memcmp env = E.call_import env "rts" "memcmp"
 
@@ -5220,6 +5231,32 @@ end (* StableMemory *)
 
 module RTS_Exports = struct
   let system_exports env =
+    let memory_copy_fi = E.add_fun env "memory_copy" (
+      Func.of_body env ["to", I32Type; "from", I32Type; "byte_length", I32Type] [] (fun env ->
+        G.i (LocalGet (nr 0l)) ^^
+        G.i (LocalGet (nr 1l)) ^^
+        G.i (LocalGet (nr 2l)) ^^
+        Heap.memcpy env
+      )
+    ) in
+    E.add_export env (nr {
+      name = Lib.Utf8.decode "memory_copy";
+      edesc = nr (FuncExport (nr memory_copy_fi))
+    });
+
+    let memory_fill_fi = E.add_fun env "memory_fill" (
+      Func.of_body env ["to", I32Type; "value", I32Type; "byte_length", I32Type] [] (fun env ->
+        G.i (LocalGet (nr 0l)) ^^
+        G.i (LocalGet (nr 1l)) ^^
+        G.i (LocalGet (nr 2l)) ^^
+        Heap.memset env
+      )
+    ) in
+    E.add_export env (nr {
+      name = Lib.Utf8.decode "memory_fill";
+      edesc = nr (FuncExport (nr memory_fill_fi))
+    });
+
     let bigint_trap_fi = E.add_fun env "bigint_trap" (
       Func.of_body env [] [] (fun env ->
         E.trap_with env "bigint function error"

src/wasm-exts/ast.ml

@@ -5,7 +5,7 @@ reference implementation.
 Base revision: WebAssembly/spec@a7a1856.
 
 The changes are:
- * None for now
+ * Manual selective support for bulk-memory operations `memory_copy` and `memory_fill` (WebAssembly/spec@7fa2f20).
 
 The code is otherwise as untouched as possible, so that we can relatively
 easily apply diffs from the original code (possibly manually).
@@ -110,6 +110,10 @@ and instr' =
   | Store of storeop                  (* write memory at address *)
   | MemorySize                        (* size of linear memory *)
   | MemoryGrow                        (* grow linear memory *)
+  (* Manual extension for bulk memory operations *)
+  | MemoryFill                        (* fill memory range with value *)
+  | MemoryCopy                        (* copy memory ranges *)
+  (* End of manual extension *)
   | Const of literal                  (* constant *)
   | Test of testop                    (* numeric test *)
   | Compare of relop                  (* numeric comparison *)

src/wasm-exts/customModuleDecode.ml

@@ -4,6 +4,7 @@ reference implementation.
 
 The changes are:
  * Support for additional custom sections
+ * Manual selective support for bulk-memory operations `memory_copy` and `memory_fill` (WebAssembly/spec@7fa2f20).
 
 The code is otherwise as untouched as possible, so that we can relatively
 easily apply diffs from the original code (possibly manually).
@@ -220,6 +221,7 @@ let var s = vu32 s
 
 let op s = u8 s
 let end_ s = expect 0x0b s "END opcode expected"
+let zero s = expect 0x00 s "zero byte expected"
 
 let memop s =
   let align = vu32 s in
@@ -244,6 +246,10 @@ let math_prefix s =
   | 0x05 -> i64_trunc_sat_f32_u
   | 0x06 -> i64_trunc_sat_f64_s
   | 0x07 -> i64_trunc_sat_f64_u
+  (* Manual extension for specific bulk-memory operations *)
+  | 0x0a -> zero s; zero s; memory_copy
+  | 0x0b -> zero s; memory_fill
+  (* End of manual extension *)
   | b -> illegal s pos b
 
 let rec instr s =

src/wasm-exts/customModuleEncode.ml

@@ -5,6 +5,7 @@ reference implementation.
 The changes are:
  * Support for writing out a source map for the Code parts
  * Support for additional custom sections
+ * Manual selective support for bulk-memory operations `memory_copy` and `memory_fill` (WebAssembly/spec@7fa2f20).
 
 The code is otherwise as untouched as possible, so that we can relatively
 easily apply diffs from the original code (possibly manually).
@@ -471,6 +472,11 @@ let encode (em : extended_module) =
       | MemorySize -> op 0x3f; u8 0x00
       | MemoryGrow -> op 0x40; u8 0x00
 
+      (* Manual extension for bulk-memory operations *)
+      | MemoryFill -> op 0xfc; vu32 0x0bl; u8 0x00
+      | MemoryCopy -> op 0xfc; vu32 0x0al; u8 0x00; u8 0x00
+      (* End of manual extension *)
+
       | Const {it = I32 c; _} -> op 0x41; vs32 c
       | Const {it = I64 c; _} -> op 0x42; vs64 c
       | Const {it = F32 c; _} -> op 0x43; f32 c

src/wasm-exts/operators.ml

@@ -7,7 +7,7 @@ that it got basically replicated into the customModuleDecode.ml file.
 Base revision: WebAssembly/spec@a7a1856.
 
 The changes are:
- * None for now
+ * Manual selective support for bulk-memory operations `memory_copy` and `memory_fill` (WebAssembly/spec@7fa2f20).
 
 The code is otherwise as untouched as possible, so that we can relatively
 easily apply diffs from the original code (possibly manually).
@@ -230,3 +230,7 @@ let f64_reinterpret_i64 = Convert (F64 F64Op.ReinterpretInt)
 let memory_size = MemorySize
 let memory_grow = MemoryGrow
 
+(* Manual extension for specific bulk-memory operations *)
+let memory_fill = MemoryFill
+let memory_copy = MemoryCopy
+(* End of manual extension *)

test/bench/ok/bignum.drun-run-opt.ok

@@ -1,6 +1,6 @@
 ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a00000000000000000101
 ingress Completed: Reply: 0x4449444c0000
-debug.print: {cycles = 2_389_389; size = +59_652}
+debug.print: {cycles = 2_389_788; size = +59_652}
 ingress Completed: Reply: 0x4449444c0000
-debug.print: {cycles = 102_989_116; size = +1_817_872}
+debug.print: {cycles = 103_004_759; size = +1_817_872}
 ingress Completed: Reply: 0x4449444c0000

test/bench/ok/bignum.drun-run.ok

@@ -1,6 +1,6 @@
 ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a00000000000000000101
 ingress Completed: Reply: 0x4449444c0000
-debug.print: {cycles = 2_493_575; size = +59_652}
+debug.print: {cycles = 2_494_202; size = +59_652}
 ingress Completed: Reply: 0x4449444c0000
-debug.print: {cycles = 103_046_061; size = +1_817_872}
+debug.print: {cycles = 103_070_651; size = +1_817_872}
 ingress Completed: Reply: 0x4449444c0000

wasm-profiler/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2018"
 
 [dependencies]
-parity-wasm = { version = "0.42.2", features = ["std", "sign_ext"] }
+parity-wasm = { version = "0.42.2", features = ["std", "sign_ext", "bulk"] }
 structopt = "0.3"
 clap = "2.33"