fix: properly encode unicode codepoints outside BMP (#497)

Co-authored-by: edayot <pro.e.dayot@gmail.com>

Author: MineRobber9000

packages/bolt/examples/weird_char/beet.yaml

@@ -0,0 +1,11 @@
+
+data_pack:
+  load: .
+
+output: build
+
+require:
+  - bolt
+
+pipeline:
+  - mecha

packages/bolt/examples/weird_char/data/example/function/say.mcfunction

@@ -0,0 +1,24 @@
+say "🗡 "
+tellraw @a {text: "🗡"}
+
+
+recipe example:sharp_diamond {
+    "type":"minecraft:crafting_shapeless",
+    "ingredients":[["minecraft:diamond"]],
+    "result":{
+        "id":"minecraft:diamond",
+        "components":{"minecraft:item_name":"🗡"}
+    }
+}
+TEAM_1 = "duels.team_1"
+TEAM_2 = "duels.team_2"
+
+execute function ./load:
+    team add TEAM_1 "Duels Team 1"
+    team add TEAM_2 "Duels Team 2"
+    
+    for team in [TEAM_1, TEAM_2]:
+        team modify team friendlyFire false
+        team modify team prefix "🗡 "
+        team modify team seeFriendlyInvisibles false
+

packages/bolt/tests/snapshots/examples__build_weird_char__0.pack.md

@@ -0,0 +1,63 @@
+# Lectern snapshot
+
+## Data pack
+
+`@data_pack pack.mcmeta`
+
+```json
+{
+  "pack": {
+    "min_format": [
+      94,
+      1
+    ],
+    "max_format": [
+      94,
+      1
+    ],
+    "description": ""
+  }
+}
+```
+
+### example
+
+`@function example:say`
+
+```mcfunction
+say "🗡 "
+tellraw @a {text: "\U0001f5e1"}
+function example:load
+```
+
+`@function example:load`
+
+```mcfunction
+team add duels.team_1 "Duels Team 1"
+team add duels.team_2 "Duels Team 2"
+team modify duels.team_1 friendlyFire false
+team modify duels.team_1 prefix "\U0001f5e1 "
+team modify duels.team_1 seeFriendlyInvisibles false
+team modify duels.team_2 friendlyFire false
+team modify duels.team_2 prefix "\U0001f5e1 "
+team modify duels.team_2 seeFriendlyInvisibles false
+```
+
+`@recipe example:sharp_diamond`
+
+```json
+{
+  "type": "minecraft:crafting_shapeless",
+  "ingredients": [
+    [
+      "minecraft:diamond"
+    ]
+  ],
+  "result": {
+    "id": "minecraft:diamond",
+    "components": {
+      "minecraft:item_name": "\ud83d\udde1"
+    }
+  }
+}
+```

packages/mecha/src/mecha/utils.py

@@ -130,13 +130,18 @@ def handle_substitution(self, token: Token, match: "re.Match[str]") -> str:
             return chr(int(unicode_hex, 16))
         return super().handle_substitution(token, match)
 
+    def encode_non_bmp_char(self, codepoint: int) -> str:
+        return f"\\U{codepoint:08x}"
+
     def handle_quoting(self, value: str) -> str:
         value = super().handle_quoting(value)
 
         def escape_char(char: str) -> str:
             codepoint = ord(char)
             if codepoint < 128:
                 return char
+            if codepoint > 65535:
+                return self.encode_non_bmp_char(codepoint)
             return f"\\u{codepoint:04x}"
 
         return "".join(escape_char(c) for c in value)
@@ -156,6 +161,11 @@ class JsonQuoteHelper(QuoteHelperWithUnicode):
         }
     )
 
+    def encode_non_bmp_char(self, codepoint: int) -> str:
+        low_surrogate = 0xD800 + ((codepoint - 0x10000) >> 10)
+        high_surrogate = 0xDC00 + (codepoint & 0x3FF)
+        return f"\\u{low_surrogate:04x}\\u{high_surrogate:04x}"
+
 
 @dataclass
 class NbtQuoteHelper(QuoteHelperWithUnicode):