fix(beam): decode UTF-8 codepoints in ToCharArray and related String functions

Author: MangelMaxime

src/Fable.Transforms/Beam/Replacements.fs

@@ -1166,8 +1166,10 @@ let private strings
                 Helper.LibCall(com, "fable_string", "last_index_of", t, [ c; sub; maxIdx ])
                 |> Some
         | _ -> None
-    // str.ToCharArray() → binary_to_list(Str), wrap as array ref
-    | "ToCharArray", Some c, [] -> emitExpr r t [ c ] "binary_to_list($0)" |> wrapArr com r t |> Some
+    // str.ToCharArray() → unicode:characters_to_list(Str), wrap as array ref.
+    // Must decode UTF-8 codepoints, not raw bytes — binary_to_list would split a
+    // multi-byte character into one bogus "char" per byte.
+    | "ToCharArray", Some c, [] -> emitExpr r t [ c ] "unicode:characters_to_list($0)" |> wrapArr com r t |> Some
     | "ToCharArray", Some c, [ start; len ] ->
         Helper.LibCall(com, "fable_string", "to_char_array", t, [ c; start; len ])
         |> wrapArr com r t

src/fable-library-beam/fable_string.erl

@@ -216,29 +216,29 @@ is_null_or_white_space(Str) ->
 %% String module functions (F# String.forall, String.exists, etc.)
 
 forall(Fn, Str) ->
-    Chars = binary_to_list(Str),
+    Chars = unicode:characters_to_list(Str),
     lists:all(Fn, Chars).
 
 exists(Fn, Str) ->
-    Chars = binary_to_list(Str),
+    Chars = unicode:characters_to_list(Str),
     lists:any(Fn, Chars).
 
 init(Count, Fn) ->
     Chars = lists:map(fun(I) -> (Fn)(I) end, lists:seq(0, Count - 1)),
     iolist_to_binary(Chars).
 
 collect(Fn, Str) ->
-    Chars = binary_to_list(Str),
+    Chars = unicode:characters_to_list(Str),
     Parts = lists:map(fun(C) -> (Fn)(C) end, Chars),
     iolist_to_binary(Parts).
 
 iter(Fn, Str) ->
-    Chars = binary_to_list(Str),
+    Chars = unicode:characters_to_list(Str),
     lists:foreach(Fn, Chars),
     ok.
 
 iteri(Fn, Str) ->
-    Chars = binary_to_list(Str),
+    Chars = unicode:characters_to_list(Str),
     iteri_loop(Fn, Chars, 0),
     ok.
 
@@ -249,17 +249,17 @@ iteri_loop(Fn, [C | Rest], Idx) ->
     iteri_loop(Fn, Rest, Idx + 1).
 
 map(Fn, Str) ->
-    Chars = binary_to_list(Str),
+    Chars = unicode:characters_to_list(Str),
     Mapped = lists:map(Fn, Chars),
     <<<<C/utf8>> || C <- Mapped>>.
 
 mapi(Fn, Str) ->
-    Chars = binary_to_list(Str),
+    Chars = unicode:characters_to_list(Str),
     {Mapped, _} = lists:mapfoldl(fun(C, I) -> {((Fn)(I))(C), I + 1} end, 0, Chars),
     <<<<C/utf8>> || C <- Mapped>>.
 
 filter(Fn, Str) ->
-    Chars = binary_to_list(Str),
+    Chars = unicode:characters_to_list(Str),
     Filtered = lists:filter(Fn, Chars),
     <<<<C/utf8>> || C <- Filtered>>.
 
@@ -285,8 +285,8 @@ index_of_any(Str, Chars, StartIdx) when is_reference(Chars) ->
     index_of_any(Str, get(Chars), StartIdx);
 index_of_any(Str, Chars, StartIdx) ->
     CharSet = Chars,
-    Bytes = binary_to_list(Str),
-    index_of_any_loop(Bytes, CharSet, 0, StartIdx).
+    Codepoints = unicode:characters_to_list(Str),
+    index_of_any_loop(Codepoints, CharSet, 0, StartIdx).
 
 index_of_any_loop([], _CharSet, _Idx, _StartIdx) ->
     -1;
@@ -322,26 +322,32 @@ contains(Str, Sub) ->
 trim_chars(Str, Chars) when is_integer(Chars) ->
     trim_chars(Str, [Chars]);
 trim_chars(Str, Chars) ->
-    iolist_to_binary(string:trim(binary_to_list(Str), both, Chars)).
+    iolist_to_binary(string:trim(Str, both, Chars)).
 
 trim_start_chars(Str, Chars) when is_integer(Chars) ->
     trim_start_chars(Str, [Chars]);
 trim_start_chars(Str, Chars) ->
-    iolist_to_binary(string:trim(binary_to_list(Str), leading, Chars)).
+    iolist_to_binary(string:trim(Str, leading, Chars)).
 
 trim_end_chars(Str, Chars) when is_integer(Chars) ->
     trim_end_chars(Str, [Chars]);
 trim_end_chars(Str, Chars) ->
-    iolist_to_binary(string:trim(binary_to_list(Str), trailing, Chars)).
+    iolist_to_binary(string:trim(Str, trailing, Chars)).
 
 %% ToCharArray
+%%
+%% An F# string is a UTF-8 binary, so splitting it into chars means walking Unicode
+%% codepoints, not raw bytes — `binary_to_list/1` would instead hand back one bogus
+%% "char" per byte of a multi-byte codepoint's UTF-8 encoding. Start/Len are char
+%% (codepoint) offsets, matching .NET's `ToCharArray(startIndex, length)`, so they're
+%% applied to the codepoint list rather than to the binary's byte offsets.
 
 to_char_array(Str) ->
-    binary_to_list(Str).
+    unicode:characters_to_list(Str).
 
 to_char_array(Str, Start, Len) ->
-    Part = binary:part(Str, Start, Len),
-    binary_to_list(Part).
+    Chars = unicode:characters_to_list(Str),
+    lists:sublist(Chars, Start + 1, Len).
 
 %% String comparison
 

tests/Beam/StringTests.fs

@@ -117,6 +117,10 @@ let ``test String.TrimEnd works`` () =
 let ``test String.Trim with chars works`` () =
     @"\\\abc///".Trim('\\','/') |> equal "abc"
 
+[<Fact>]
+let ``test String.Trim with chars works with non-ASCII characters`` () =
+    "café".Trim('é') |> equal "caf"
+
 [<Fact>]
 let ``test String.TrimStart with chars works`` () =
     "!!--abc   ".TrimStart('!','-') |> equal "abc   "
@@ -340,6 +344,11 @@ let ``test String.forall and exists work`` () =
     "a!a" |> String.forall (fun c -> c = '!') |> equal false
     "aaa" |> String.forall (fun c -> c = '!') |> equal false
 
+[<Fact>]
+let ``test String.forall works with non-ASCII characters`` () =
+    "café" |> String.forall (fun c -> c <> 'x') |> equal true
+    "café" |> String.exists (fun c -> c = 'é') |> equal true
+
 [<Fact>]
 let ``test String.init works`` () =
     String.init 3 (fun i -> "a") |> equal "aaa"
@@ -348,13 +357,23 @@ let ``test String.init works`` () =
 let ``test String.collect works`` () =
     "abc" |> String.collect (fun c -> "bcd") |> equal "bcdbcdbcd"
 
+[<Fact>]
+let ``test String.collect works with non-ASCII characters`` () =
+    "café" |> String.collect string |> equal "café"
+
 [<Fact>]
 let ``test String.iter works`` () =
     let res = ref ""
     "Hello world!"
     |> String.iter (fun c -> res.Value <- res.Value + c.ToString())
     equal "Hello world!" res.Value
 
+[<Fact>]
+let ``test String.iter works with non-ASCII characters`` () =
+    let res = ref ""
+    "café" |> String.iter (fun c -> res.Value <- res.Value + c.ToString())
+    equal "café" res.Value
+
 [<Fact>]
 let ``test String.iteri works`` () =
     let mutable res = ""
@@ -367,6 +386,11 @@ let ``test String.map works`` () =
     "Hello world!" |> String.map (fun c -> if c = 'H' then '_' else c)
     |> equal "_ello world!"
 
+[<Fact>]
+let ``test String.map works with non-ASCII characters`` () =
+    "café" |> String.map (fun c -> if c = 'é' then 'e' else c)
+    |> equal "cafe"
+
 [<Fact>]
 let ``test String.mapi works`` () =
     "Hello world!" |> String.mapi (fun i c -> if i = 1 || c = 'H' then '_' else c)
@@ -376,6 +400,10 @@ let ``test String.mapi works`` () =
 let ``test String.filter works`` () =
     String.filter (fun x -> x <> '.') "a.b.c" |> equal "abc"
 
+[<Fact>]
+let ``test String.filter works with non-ASCII characters`` () =
+    String.filter (fun c -> c <> 'é') "café" |> equal "caf"
+
 [<Fact>]
 let ``test String.filter works when predicate matches everything`` () =
     String.filter (fun x -> x <> '.') "abc" |> equal "abc"
@@ -433,6 +461,18 @@ let ``test String.ToCharArray with range works`` () =
     let arr = "abcd".ToCharArray(1, 2)
     arr |> equal [|'b';'c'|]
 
+[<Fact>]
+let ``test String.ToCharArray works with non-ASCII characters`` () =
+    let s = "café"
+    let arr = s.ToCharArray()
+    arr |> equal [|'c';'a';'f';'é'|]
+    let rebuilt = arr |> Array.map string |> String.concat ""
+    rebuilt |> equal s
+
+[<Fact>]
+let ``test String.ToCharArray with range works with non-ASCII characters`` () =
+    "café".ToCharArray(2, 2) |> equal [|'f';'é'|]
+
 // --- String.Equals ---
 
 [<Fact>]
@@ -723,6 +763,11 @@ let ``test String.IndexOfAny works`` () =
     "abcdbcebc".IndexOfAny([|'b'|], 2) |> equal 4
     "abcdbcebc".IndexOfAny([|'f';'e'|]) |> equal 6
 
+[<Fact>]
+let ``test String.IndexOfAny works with non-ASCII characters`` () =
+    "café".IndexOfAny([|'é'|]) |> equal 3
+    "café".IndexOfAny([|'f'|]) |> equal 2
+
 [<Fact>]
 let ``test String.Join with indices works`` () =
     String.Join("**", [|"a"; "b"; "c"; "d"|], 1, 2) |> equal "b**c"