diff --git a/README.md b/README.md index 01c1746..6b0fc79 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,10 @@ python3 shared/tools/dump.py # process all missing versions python3 shared/tools/dump.py --force # re-dump even if dump.cs exists ``` -Requires `dotnet` 8.x on PATH (`apt install dotnet-sdk-8.0` on Ubuntu 24.04). +Requires `dotnet` 8.x on PATH. Il2CppDumper only needs the runtime, so +`dotnet-runtime-8.0` is enough (`ghcr.io/devcontainers/features/dotnet:2` with +`{"version": "none", "dotnetRuntimeVersions": "8.0"}` in a devcontainer, or +`apt install dotnet-runtime-8.0` on Ubuntu 24.04). ## Checking for RVA drift after a target update @@ -199,6 +202,41 @@ Any mismatch is reported with the expected vs. found prologue bytes and the dump line number. Create a new `recipes/v__.py` with the updated addresses and register it in `recipes/__init__.py`. +### Porting the SITES table automatically + +`tools/port_recipe.py` does the mechanical part of that: it looks every row's +label up in the new dump index, rewrites the RVA and prologue columns from the +new binary, and leaves row order, comments, and every other column untouched. + +```sh +PYTHONPATH=shared python3 -m tools.port_recipe \ + --base-recipe recipes/v1_0_2.py \ + --index assets/1.1.0/dump.cs.index.json \ + --ipa assets/1.1.0/AppName-1.1.0.ipa \ + --out recipes/v1_1_0.py +``` + +Rows whose label no longer resolves keep their old values, gain a +`# TODO(port_recipe): unresolved` marker, and make the tool exit non-zero — +those are the ones that need a human to look at the dump. Row order is never +changed: cave payloads are allocated in declaration order while the runtime +dispatcher indexes by hook id, so a reordered table mispoints every +orig-call trampoline. + +### Field offsets + +Prologue checks catch a moved method but not a moved *field* — a class that +gains a member above the one a hook reads shifts everything below it. +`tools/verify_offsets.py` diffs the field offsets of named types between two +dumps: + +```sh +PYTHONPATH=shared python3 -m tools.verify_offsets \ + --old assets/1.0.2/dump.cs --old-index assets/1.0.2/dump.cs.index.json \ + --new assets/1.1.0/dump.cs --new-index assets/1.1.0/dump.cs.index.json \ + --type SomeReply --type SomeStatus +``` + ## Development ```sh diff --git a/tests/test_port_recipe.py b/tests/test_port_recipe.py new file mode 100644 index 0000000..b2a44cf --- /dev/null +++ b/tests/test_port_recipe.py @@ -0,0 +1,136 @@ +"""Tests for ``tools.port_recipe``. + +The tool rewrites a recipe's SITES table against a new dump. Three +properties are load-bearing and easy to break: + + * row ORDER, comments, and every non-RVA column survive the rewrite + (cave payloads are allocated in declaration order, so a reordered + table silently mispoints every orig-call trampoline), + * an unresolved label is never guessed — the row keeps its old values, + gains a TODO marker, and the exit status is non-zero, and + * the RVA and prologue columns actually get the new values. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tools.port_recipe import PortError, find_sites_node, main, rewrite_row + +BASE_RECIPE = '''\ +"""Recipe for the old build.""" + +from recipes.common import CAVE_ENTRY + +BUILD = 12 + +# fmt: off +SITES = [ + # --- a section comment that must survive --- + (0x1000, "aaaaaaaa", "HOOK_A", CAVE_ENTRY, "Foo.Alpha"), + (0x2000, "bbbbbbbb", "HOOK_B", CAVE_ENTRY, "Foo.Beta"), + (0x3000, "cccccccc", "HOOK_C", CAVE_ENTRY, "Gone.Vanished"), +] +# fmt: on +''' + + +def _index(entries: dict[str, list[tuple[str, str]]]) -> dict: + """Build a minimal dump index: {type: [(method_name, rva_hex)]}.""" + return { + "images": [], + "types": [ + { + "name": name, + "namespace": "", + "kind": "class", + "tdi": i, + "line": 1, + "methods": [ + {"sig": f"public void {m}()", "rva": rva, "line": 2} + for m, rva in methods + ], + } + for i, (name, methods) in enumerate(entries.items()) + ], + } + + +@pytest.fixture +def workspace(tmp_path: Path) -> dict: + recipe = tmp_path / "v_old.py" + recipe.write_text(BASE_RECIPE) + + index = tmp_path / "dump.cs.index.json" + index.write_text( + json.dumps(_index({"Foo": [("Alpha", "0x9000"), ("Beta", "0xA000")]})) + ) + + # A stand-in Mach-O: the tool reads 4 bytes at each resolved RVA. + macho = tmp_path / "UnityFramework" + blob = bytearray(0xB000) + blob[0x9000:0x9004] = b"\x11\x22\x33\x44" + blob[0xA000:0xA004] = b"\x55\x66\x77\x88" + macho.write_bytes(bytes(blob)) + + return {"recipe": recipe, "index": index, "macho": macho, "out": tmp_path / "v_new.py"} + + +def _run(ws: dict, monkeypatch) -> int: + monkeypatch.setattr( + "sys.argv", + [ + "port_recipe", + "--base-recipe", str(ws["recipe"]), + "--index", str(ws["index"]), + "--macho", str(ws["macho"]), + "--out", str(ws["out"]), + ], + ) + return main() + + +def test_resolved_rows_get_new_rva_and_prologue(workspace, monkeypatch): + _run(workspace, monkeypatch) + out = workspace["out"].read_text() + assert '(0x9000, "11223344", "HOOK_A"' in out + assert '(0xA000, "55667788", "HOOK_B"' in out + + +def test_unresolved_row_is_marked_and_left_alone(workspace, monkeypatch): + rc = _run(workspace, monkeypatch) + out = workspace["out"].read_text() + assert rc == 1 + assert '(0x3000, "cccccccc", "HOOK_C"' in out + assert "TODO(port_recipe): unresolved" in out + + +def test_row_order_and_comments_survive(workspace, monkeypatch): + _run(workspace, monkeypatch) + out = workspace["out"].read_text() + assert "# --- a section comment that must survive ---" in out + assert "# fmt: off" in out and "# fmt: on" in out + assert out.index("HOOK_A") < out.index("HOOK_B") < out.index("HOOK_C") + assert 'BUILD = 12' in out + + +def test_missing_sites_table_is_an_error(tmp_path): + import ast + + module = ast.parse("PATCHES = []\n") + with pytest.raises(PortError): + find_sites_node(module) + + +def test_rewrite_row_replaces_only_the_first_two_columns(): + line = ' (0x5C3C29C, "fc6fbaa9", "HOOK_X", CAVE_ENTRY, "Foo.Bar"),\n' + got = rewrite_row(line, 0x5C3C29C, 0x6B58E3C, "deadbeef") + assert got == ' (0x6B58E3C, "deadbeef", "HOOK_X", CAVE_ENTRY, "Foo.Bar"),\n' + + +def test_rewrite_row_rejects_a_row_whose_rva_it_cannot_find(): + with pytest.raises(PortError): + rewrite_row(' (0x1, "aa", "H", CAVE_ENTRY, "F.B"),\n', 0xDEAD, 0x1, "bb") diff --git a/tests/test_verify_sites.py b/tests/test_verify_sites.py new file mode 100644 index 0000000..00a3966 --- /dev/null +++ b/tests/test_verify_sites.py @@ -0,0 +1,121 @@ +"""Tests for the label resolver in ``tools.verify_sites``. + +Recipe labels are the only durable anchor a hook site has: RVAs move on +every target build, so a label that resolves to the wrong overload silently +patches the wrong function. Google.Protobuf alone ships 15 ``MergeFrom`` +overloads, which is why labels may pin a parameter list. +""" + +from __future__ import annotations + +from tools.verify_sites import ( + find_method, + param_type, + split_label, + split_params, + types_by_name, +) + +INDEX = { + "types": [ + { + "name": "MessageExtensions", + "namespace": "Google.Protobuf", + "kind": "class", + "tdi": 1, + "line": 10, + "methods": [ + { + "sig": "public static void MergeFrom(IMessage message, byte[] data)", + "rva": "0x1000", + "line": 11, + }, + { + "sig": ( + "internal static void MergeFrom(IMessage message, " + "ReadOnlySequence data, bool discardUnknownFields, " + "ExtensionRegistry registry)" + ), + "rva": "0x2000", + "line": 12, + }, + ], + }, + { + "name": "TitleScene.d__10", + "namespace": "", + "kind": "struct", + "tdi": 2, + "line": 20, + "methods": [ + {"sig": "private void MoveNext()", "rva": "0x3000", "line": 21}, + ], + }, + { + "name": "Evaluator", + "namespace": "Game", + "kind": "class", + "tdi": 3, + "line": 30, + "methods": [ + { + "sig": "public void .ctor(string path, Settings settings)", + "rva": "0x4000", + "line": 31, + }, + ], + }, + ] +} + + +def _resolve(label: str): + by_name = types_by_name(INDEX) + type_name, method_name, params = split_label(label) + hit = find_method(by_name, type_name, method_name, param_types=params) + return None if hit is None else hit[1]["rva"] + + +def test_param_list_selects_one_overload(): + assert _resolve("MessageExtensions.MergeFrom(IMessage, byte[])") == "0x1000" + assert ( + _resolve( + "MessageExtensions.MergeFrom(IMessage, ReadOnlySequence, " + "bool, ExtensionRegistry)" + ) + == "0x2000" + ) + + +def test_label_without_params_matches_any_overload(): + assert _resolve("MessageExtensions.MergeFrom") == "0x1000" + + +def test_wrong_arity_does_not_resolve(): + assert _resolve("MessageExtensions.MergeFrom(IMessage)") is None + + +def test_namespace_qualified_type_resolves(): + assert _resolve("Google.Protobuf.MessageExtensions.MergeFrom(IMessage, byte[])") == "0x1000" + + +def test_nested_type_plus_notation_resolves(): + assert _resolve("TitleScene+d__10.MoveNext") == "0x3000" + + +def test_ctor_shorthand_resolves(): + assert _resolve("Evaluator.ctor") == "0x4000" + + +def test_split_params_respects_generic_commas(): + assert split_params("List> ranked, int topN") == ( + "List> ranked", + "int topN", + ) + + +def test_param_type_strips_names_modifiers_and_defaults(): + assert param_type("ref ParseContext input") == "ParseContext" + assert param_type("int maxLength = 180") == "int" + assert param_type("ExtensionRegistry") == "ExtensionRegistry" + assert param_type("List> ranked") == "List>" diff --git a/tools/caves.py b/tools/caves.py index cf29c6a..216c593 100644 --- a/tools/caves.py +++ b/tools/caves.py @@ -88,6 +88,21 @@ def apply_patches( # path can match both the site and the cave content byte-for-byte. cave_cursor = cave_start for site_off, expected, build_payload, label in cave_patches: + # A row with no site is a placeholder: the method it used to + # patch is gone from this build, but the cave slot it occupied + # must stay reserved because consumers address caves by index. + # Nothing is written — the payload isn't even built, since it + # would need a branch back to a site that doesn't exist. The + # slot is sized from `expected`, which the recipe fills with a + # zero block of one payload's length for exactly this purpose. + if site_off is None: + print( + f" RESERVE {label} " + f"(cave @ 0x{cave_cursor:X}, {len(expected)} B)" + ) + cave_cursor += len(expected) + continue + if len(expected) != 4: raise AssertionError(f"cave-patch site must be one 4B insn: {label}") diff --git a/tools/deploy.sh b/tools/deploy.sh new file mode 100755 index 0000000..70b6466 --- /dev/null +++ b/tools/deploy.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# Ship a patched IPA to an iOS device and install it through TrollStore. +# +# Two-step pipeline: +# +# 1. scp the IPA into the device's /tmp/. +# 2. Invoke trollstorehelper install force over SSH. +# 3. Relaunch the app so the just-installed build takes over the +# running instance (open → uiopen fallback → manual instruction). +# +# TrollStore vs TrollStore Lite: the trollstorehelper binary lives in +# different places per install flavour. Rootless JB devices with +# TrollStore keep it inside /var/jb/Applications/TrollStore*.app/, and +# TrollStore Lite specifically uses /var/jb/Applications/TrollStoreLite.app/. +# If neither is present, the binary can also live inside TrollStore.app's +# own container bundle under /var/containers/Bundle/Application//. +# When --helper is not supplied, the script SSH-finds it, preferring +# the /var/jb/Applications/TrollStore*.app/ entries so stale +# TrollStorePersistenceHelper.app leftovers from a prior JB session +# don't win. +# +# Remote staging path: /var/mobile/Documents/. trollstorehelper runs in +# a sandbox that CANNOT read /tmp/ — an IPA staged there is rejected +# with return code 166 ("IPA does not exist or is not accessible"), +# even though the file is physically present. Documents/ is one of +# the few locations both root (scp target) and mobile (trollstorehelper +# runtime) can access; the script chowns after scp so mobile can read. +# +# trollstorehelper commonly kicks SpringBoard mid-install, which yanks +# the SSH session and returns exit 255 even though the install itself +# succeeded. This script tolerates that specific exit — the caller +# should confirm the app is on the home screen. Any other non-zero +# exit is propagated so a genuine failure fails the target. +# +# Usage: +# shared/tools/deploy.sh \ +# --ipa \ +# --host \ +# --port \ +# --bundle-id \ +# [--user ] \ +# [--helper ] \ +# [--process-name ] + +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +Usage: deploy.sh --ipa IPA --host HOST --port PORT --bundle-id BID + [--user USER] [--helper PATH] [--process-name NAME] + + --ipa IPA Path to the patched .ipa on the local filesystem. + --host HOST Device host/IP reachable over SSH (e.g. host.docker.internal). + --port PORT SSH port (matches THEOS_DEVICE_PORT). + --bundle-id BID CFBundleIdentifier to relaunch after install + (e.g. com.neconome.shogi). + --user USER SSH username (defaults to root). + --helper PATH trollstorehelper path on the device. If omitted, the + script SSH-finds it under /var/jb/Applications and + /var/containers/Bundle/Application (TrollStore Lite). + --process-name NM Human-readable app name used in the manual-launch + fallback message. Defaults to the bundle id. +EOF + exit 64 +} + +IPA="" +HOST="" +PORT="" +BUNDLE_ID="" +SSH_USER="root" +HELPER="" +PROCESS_NAME="" + +while [ $# -gt 0 ]; do + case "$1" in + --ipa) IPA="$2"; shift 2;; + --host) HOST="$2"; shift 2;; + --port) PORT="$2"; shift 2;; + --bundle-id) BUNDLE_ID="$2"; shift 2;; + --user) SSH_USER="$2"; shift 2;; + --helper) HELPER="$2"; shift 2;; + --process-name) PROCESS_NAME="$2"; shift 2;; + -h|--help) usage;; + *) echo "error: unknown argument: $1" >&2; usage;; + esac +done + +if [ -z "$IPA" ] || [ -z "$HOST" ] || [ -z "$PORT" ] || [ -z "$BUNDLE_ID" ]; then + echo "error: --ipa, --host, --port, --bundle-id are all required" >&2 + usage +fi + +if [ ! -f "$IPA" ]; then + echo "error: IPA not found: $IPA" >&2 + exit 1 +fi + +if [ -z "$PROCESS_NAME" ]; then + PROCESS_NAME="$BUNDLE_ID" +fi + +IPA_NAME="$(basename "$IPA")" +# /var/mobile/Documents/ is the sandbox-visible staging area for +# trollstorehelper — /tmp/ is off-limits and yields error 166. +REMOTE_IPA="/var/mobile/Documents/$IPA_NAME" +# The target process name for `killall` before install. `open BID` after +# install would otherwise front-restore the old in-memory instance and +# the just-installed binary would never dyld_load — the operator sees +# "the tweak didn't take effect" symptoms. Killing first forces iOS to +# spawn a fresh process against the new .app on relaunch. +KILL_TARGET="$PROCESS_NAME" + +# --------------------------------------------------------------------------- +# Resolve trollstorehelper path (Lite / regular TrollStore). +# +# Discovery preference order: +# 1. /var/jb/Applications/TrollStore*.app/trollstorehelper — the live +# helper on JB-rootless. Prefer TrollStoreLite.app over generic +# TrollStore.app when both exist. Filter out +# TrollStorePersistenceHelper.app: on Lite it never gets installed, +# and a leftover directory from a previous non-Lite JB session +# often has a helper binary whose entitlements have been +# invalidated (invoking it SIGKILLs with exit 137 / ssh 255). +# 2. /var/containers/Bundle/Application//TrollStore*.app/... — +# fallback for setups without a rootless JB where the .app lives +# inside the App container tree. +# --------------------------------------------------------------------------- +if [ -z "$HELPER" ]; then + echo "==> discovering trollstorehelper on $HOST" + # Emit every candidate helper under known TrollStore-owning trees, + # drop the PersistenceHelper leftover (stale entitlements → SIGKILL), + # and let the sort rank TrollStoreLite ahead of plain TrollStore so + # a Lite install wins on a device that carries both. + HELPER=$(ssh -p "$PORT" "$SSH_USER@$HOST" \ + "find /var/jb/Applications /var/containers/Bundle/Application \ + -maxdepth 4 -type f -name trollstorehelper 2>/dev/null \ + | grep -v 'PersistenceHelper' \ + | awk 'BEGIN{FS=\"/\"} {for(i=1;i<=NF;i++) if(\$i ~ /^TrollStoreLite\\.app\$/){print \"0 \"\$0; next}} {print \"1 \"\$0}' \ + | sort -k1,1 \ + | awk '{print \$2}' \ + | head -n1") || true +fi + +if [ -z "$HELPER" ]; then + echo "error: trollstorehelper not found on device" >&2 + echo " pass --helper or install TrollStore / TrollStore Lite" >&2 + exit 1 +fi +echo "==> helper: $HELPER" + +# --------------------------------------------------------------------------- +# Ship + install. +# --------------------------------------------------------------------------- +REMOTE_DIR="$(dirname "$REMOTE_IPA")" +echo "==> scp $IPA_NAME -> $SSH_USER@$HOST:$REMOTE_DIR/" +scp -q -P "$PORT" "$IPA" "$SSH_USER@$HOST:$REMOTE_IPA" +# trollstorehelper runs as mobile; scp lands the file as root, so +# hand it over so the sandbox can actually read it. +ssh -p "$PORT" "$SSH_USER@$HOST" "chown mobile:mobile '$REMOTE_IPA' 2>/dev/null || true" + +echo "==> killall $KILL_TARGET (silent if not running)" +ssh -p "$PORT" "$SSH_USER@$HOST" "killall '$KILL_TARGET' 2>/dev/null || true" + +echo "==> trollstorehelper install force $REMOTE_IPA" +set +e +ssh -p "$PORT" "$SSH_USER@$HOST" "$HELPER install force $REMOTE_IPA" +rc=$? +set -e +if [ "$rc" -eq 255 ]; then + echo " (ssh exit 255 — trollstorehelper commonly restarts SpringBoard mid-install; continuing)" +elif [ "$rc" -ne 0 ]; then + echo "error: trollstorehelper exited $rc" >&2 + exit "$rc" +fi + +# --------------------------------------------------------------------------- +# Relaunch the just-installed app. +# --------------------------------------------------------------------------- +echo "==> launching $PROCESS_NAME ($BUNDLE_ID)" +ssh -p "$PORT" "$SSH_USER@$HOST" "sleep 1; \ + (open '$BUNDLE_ID' 2>/dev/null \ + || uiopen '$BUNDLE_ID://' 2>/dev/null \ + || echo 'no launcher tool; start $PROCESS_NAME manually')" diff --git a/tools/port_recipe.py b/tools/port_recipe.py new file mode 100644 index 0000000..2a2b45a --- /dev/null +++ b/tools/port_recipe.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Re-resolve a recipe's ``SITES`` table against a new target's dump index. + +When the target app ships a new build every RVA moves. This tool takes the +recipe you already trust, looks each row's label up in the new +``dump.cs.index.json``, and rewrites the RVA and prologue columns from the +new binary — leaving row order, comments, and every other column alone. + +Row order is load-bearing: cave payloads are allocated in declaration order +while the runtime dispatcher indexes by hook id, so the two only agree when +position equals id. Rows are therefore never reordered, dropped, or merged. + +Rows whose label does not resolve are emitted verbatim with a ``TODO`` +marker appended and reported on stderr; the exit status is non-zero so a +half-ported recipe can't be mistaken for a finished one. + +Usage: + python3 -m tools.port_recipe \\ + --base-recipe vendor/KIOU-Hook/recipes/v1_0_2.py \\ + --index assets/1.1.0/dump.cs.index.json \\ + --ipa assets/1.1.0/Kiou-1.1.0.ipa \\ + --out vendor/KIOU-Hook/recipes/v1_1_0.py + +Exit status: + 0 every row resolved + 1 one or more rows need manual attention + 2 bad inputs +""" + +from __future__ import annotations + +import argparse +import ast +import os +import sys + +from tools.verify_sites import ( + _read_ipa_macho_bytes, + _read_macho_bytes, + find_method, + load_dump_index, + split_label, + types_by_name, +) + +TODO_MARKER = "TODO(port_recipe): unresolved" + + +class PortError(Exception): + """Raised when the base recipe can't be parsed.""" + + +# --------------------------------------------------------------------------- +# Base-recipe parsing. +# +# The recipe is parsed, never imported: importing pulls in ``tools.encode`` +# and the ``recipes`` package, which forces a working sibling checkout just +# to read a table of integers. ``tools.check_recipes`` in KIOU-Hook takes +# the same approach for the same reason. +# --------------------------------------------------------------------------- + + +def find_sites_node(module: ast.Module) -> ast.List: + """Return the ``SITES = [...]`` list literal from a parsed recipe.""" + for node in module.body: + if not isinstance(node, ast.Assign): + continue + for target in node.targets: + if isinstance(target, ast.Name) and target.id == "SITES": + if not isinstance(node.value, ast.List): + raise PortError("SITES: expected a list literal") + return node.value + raise PortError("SITES: not found at module level") + + +def row_label(row: ast.Tuple) -> str: + """Return the label column (last element) of a SITES row.""" + if len(row.elts) < 5: + raise PortError(f"SITES row on line {row.lineno}: expected 5 columns") + last = row.elts[-1] + if not isinstance(last, ast.Constant) or not isinstance(last.value, str): + raise PortError(f"SITES row on line {row.lineno}: label is not a string") + return last.value + + +def iter_rows(sites: ast.List) -> list[ast.Tuple]: + rows = [] + for elt in sites.elts: + if not isinstance(elt, ast.Tuple): + raise PortError(f"SITES element on line {elt.lineno} is not a tuple") + rows.append(elt) + return rows + + +# --------------------------------------------------------------------------- +# Source rewriting. +# +# The rewrite is textual so that comments, blank lines, and the hand-tuned +# column alignment in the base recipe survive. Only the two leading columns +# of each row's source line are replaced. +# --------------------------------------------------------------------------- + + +def rewrite_row(source_line: str, old_rva: int, new_rva: int, prologue: str) -> str: + """Replace the RVA and prologue columns in one row's source text. + + The RVA is matched on its hex spelling as it appears in the source + (recipes write ``0x5C3C29C``), and the prologue is the first quoted + string after it. + """ + old_hex = f"0x{old_rva:X}" + head, sep, tail = source_line.partition(old_hex) + if not sep: + raise PortError(f"could not locate {old_hex} in row source: {source_line!r}") + quote = tail.find('"') + end = tail.find('"', quote + 1) + if quote < 0 or end < 0: + raise PortError(f"could not locate prologue column in row: {source_line!r}") + return f'{head}0x{new_rva:X}{tail[:quote]}"{prologue}"{tail[end + 1:]}' + + +def read_prologue(args: argparse.Namespace, offset: int) -> bytes: + if args.macho: + return _read_macho_bytes(args.macho, offset, 4) + return _read_ipa_macho_bytes(args.ipa, args.framework, offset, 4) + + +def port(args: argparse.Namespace) -> int: + with open(args.base_recipe, "r", encoding="utf-8") as fh: + source = fh.read() + lines = source.splitlines(keepends=True) + + module = ast.parse(source, filename=args.base_recipe) + rows = iter_rows(find_sites_node(module)) + + by_name = types_by_name(load_dump_index(args.index)) + + unresolved: list[tuple[int, str]] = [] + for row in rows: + label = row_label(row) + old_rva = ast.literal_eval(row.elts[0]) + type_name, method_name, param_types = split_label(label) + hit = find_method(by_name, type_name, method_name, param_types=param_types) + if hit is None: + unresolved.append((row.lineno, label)) + idx = row.lineno - 1 + lines[idx] = lines[idx].rstrip("\n") + f" # {TODO_MARKER}\n" + continue + new_rva = int(hit[1]["rva"], 0) + prologue = read_prologue(args, new_rva).hex() + idx = row.lineno - 1 + lines[idx] = rewrite_row(lines[idx], old_rva, new_rva, prologue) + + out = "".join(lines) + if args.out == "-": + sys.stdout.write(out) + else: + with open(args.out, "w", encoding="utf-8") as fh: + fh.write(out) + print(f"wrote {args.out}", file=sys.stderr) + + print( + f"{len(rows) - len(unresolved)} / {len(rows)} row(s) resolved", + file=sys.stderr, + ) + for lineno, label in unresolved: + print(f" UNRESOLVED line {lineno}: {label}", file=sys.stderr) + return 1 if unresolved else 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--base-recipe", + required=True, + help="Path to the recipe whose SITES table is being re-resolved.", + ) + parser.add_argument( + "--index", + required=True, + help="dump.cs.index.json for the NEW target build.", + ) + src = parser.add_mutually_exclusive_group(required=True) + src.add_argument("--macho", help="Path to the new target's Mach-O.") + src.add_argument("--ipa", help="Path to the new target's .ipa.") + parser.add_argument( + "--framework", + default="UnityFramework", + help="Basename of the Mach-O inside the .ipa (default: UnityFramework).", + ) + parser.add_argument( + "--out", + default="-", + help="Where to write the ported recipe ('-' for stdout, the default).", + ) + args = parser.parse_args() + + for path in (args.base_recipe, args.index, args.macho, args.ipa): + if path and not os.path.isfile(path): + print(f"error: not found: {path}", file=sys.stderr) + return 2 + + try: + return port(args) + except PortError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/verify_offsets.py b/tools/verify_offsets.py new file mode 100644 index 0000000..7f3ad21 --- /dev/null +++ b/tools/verify_offsets.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Diff il2cpp field offsets for named types between two ``dump.cs`` files. + +Prologue checks catch a moved method; nothing catches a moved *field*. A +class that gains one member above the one a hook reads silently shifts +every offset below it, and the hook then reads or writes the wrong bytes. + +The dump index carries only type and method rows, so this reads the field +declarations straight out of ``dump.cs``, using the index to find where +each type's body starts. + +Usage: + python3 -m tools.verify_offsets \\ + --old assets/1.0.2/dump.cs --old-index assets/1.0.2/dump.cs.index.json \\ + --new assets/1.1.0/dump.cs --new-index assets/1.1.0/dump.cs.index.json \\ + --type BeginnerSupportEvaluator --type ShogiMatchingPlayerStatus + +Exit status: + 0 every requested type has identical field offsets + 1 at least one type drifted (or is missing from one side) + 2 bad inputs +""" + +from __future__ import annotations + +import argparse +import itertools +import os +import re +import sys + +from tools.verify_sites import load_dump_index, types_by_name + +# `` private readonly int _analysisDepth; // 0x18`` +_RE_FIELD = re.compile(r"^\s+(?:\S.*?\s)?(\S+);\s*//\s*(0x[0-9A-Fa-f]+)\s*$") + +# How far past the type declaration to keep scanning for fields. Field +# declarations always precede the method block, so the first line that +# looks like a method or the closing brace ends the scan; this is only a +# backstop against a malformed dump. +_MAX_BODY_LINES = 400 + + +def read_fields(dump_path: str, line: int) -> list[tuple[str, str]]: + """Return ``[(field_name, offset_hex)]`` for the type declared at ``line``.""" + out: list[tuple[str, str]] = [] + with open(dump_path, "r", encoding="utf-8", errors="replace") as fh: + body = itertools.islice(fh, line, line + _MAX_BODY_LINES) + for raw in body: + if raw.startswith("}"): + break + m = _RE_FIELD.match(raw.rstrip("\n")) + if m: + out.append((m.group(1), m.group(2))) + elif "// RVA:" in raw: + break + return out + + +def locate(index: dict, type_name: str) -> int | None: + hits = types_by_name(index).get(type_name, []) + return hits[0]["line"] if hits else None + + +def diff_type(args: argparse.Namespace, type_name: str) -> bool: + """Print a side-by-side field diff. Returns True when they match.""" + old_line = locate(args._old_index, type_name) + new_line = locate(args._new_index, type_name) + if old_line is None or new_line is None: + missing = "old" if old_line is None else "new" + print(f" MISSING {type_name}: not present in the {missing} dump") + return False + + old = read_fields(args.old, old_line) + new = read_fields(args.new, new_line) + old_by_name = dict(old) + new_by_name = dict(new) + + if old == new: + print(f" SAME {type_name}: {len(old)} field(s) unchanged") + return True + + print(f" DRIFT {type_name}:") + for name, off in old: + if name not in new_by_name: + print(f" - {off:>6} {name} (removed)") + elif new_by_name[name] != off: + print(f" ~ {off:>6} -> {new_by_name[name]:>6} {name}") + for name, off in new: + if name not in old_by_name: + print(f" + {off:>6} {name} (added)") + return False + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--old", required=True, help="Baseline dump.cs.") + parser.add_argument("--old-index", required=True, help="Baseline dump.cs.index.json.") + parser.add_argument("--new", required=True, help="Target dump.cs.") + parser.add_argument("--new-index", required=True, help="Target dump.cs.index.json.") + parser.add_argument( + "--type", + action="append", + required=True, + dest="types", + help="Type name to compare (repeatable).", + ) + args = parser.parse_args() + + for path in (args.old, args.old_index, args.new, args.new_index): + if not os.path.isfile(path): + print(f"error: not found: {path}", file=sys.stderr) + return 2 + + args._old_index = load_dump_index(args.old_index) + args._new_index = load_dump_index(args.new_index) + + drifted = [t for t in args.types if not diff_type(args, t)] + print() + if drifted: + print(f"{len(drifted)} / {len(args.types)} type(s) need attention") + return 1 + print(f"all {len(args.types)} type(s) unchanged") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/verify_sites.py b/tools/verify_sites.py index 1685c60..0f386bf 100644 --- a/tools/verify_sites.py +++ b/tools/verify_sites.py @@ -46,13 +46,67 @@ # the nested boundary, which is the C# decoration convention; we treat # '+' and '.' equivalently when matching against the dump. # -# Splitting the LAST '.' off the label gives us (type_name, method_name) -# in every case we care about. +# A label may carry a parenthesised parameter-type list to pick one +# overload out of many ("MessageExtensions.MergeFrom(IMessage, +# ReadOnlySequence, bool, ExtensionRegistry)"). Without it, +# Google.Protobuf's 15 MergeFrom overloads are indistinguishable and the +# resolver would silently take whichever came first. # --------------------------------------------------------------------------- -def split_label(label: str) -> tuple[str | None, str]: - """Return ``(type_name, method_name)`` from a recipe label. +def split_params(text: str) -> tuple[str, ...]: + """Split a comma-separated C# parameter list, respecting generics. + + ``List> ranked, int topN`` has a comma + inside the generic argument list; splitting naively would produce + four bogus parameters instead of two. + """ + out: list[str] = [] + depth = 0 + current: list[str] = [] + for ch in text: + if ch in "<([": + depth += 1 + elif ch in ">)]": + depth -= 1 + if ch == "," and depth == 0: + out.append("".join(current).strip()) + current = [] + continue + current.append(ch) + tail = "".join(current).strip() + if tail: + out.append(tail) + return tuple(p for p in out if p) + + +def param_type(param: str) -> str: + """Reduce one declared parameter to its type. + + ``ref ParseContext input`` -> ``ParseContext``; + ``int maxLength = 180`` -> ``int``; + ``ExtensionRegistry`` -> ``ExtensionRegistry`` (unnamed). + """ + param = param.split("=", 1)[0].strip() + for modifier in ("ref ", "out ", "in ", "params ", "this "): + while param.startswith(modifier): + param = param[len(modifier):].lstrip() + # A declared name is whatever follows the last top-level space; a bare + # type has no space to split on. + depth = 0 + for i in range(len(param) - 1, -1, -1): + ch = param[i] + if ch in ">)]": + depth += 1 + elif ch in "<([": + depth -= 1 + elif ch == " " and depth == 0: + return param[:i].strip() + return param + + +def split_label(label: str) -> tuple[str | None, str, tuple[str, ...] | None]: + """Return ``(type_name, method_name, param_types)`` from a recipe label. ``+`` in the type segment is normalised to ``.`` so it lines up with the dump-index's nested-type spelling. @@ -65,14 +119,25 @@ def split_label(label: str) -> tuple[str | None, str]: Constructor entries are spelled either ``Foo.ctor`` (recipe) or ``.ctor`` (dump signature). We normalise the recipe form to ``.ctor`` so the dump match works without special-casing. + + ``param_types`` is ``None`` when the label carries no parenthesised + list, meaning "any overload". """ + head, sep, rest = label.partition("(") + params: tuple[str, ...] | None = None + if sep: + params = tuple( + param_type(p) for p in split_params(rest.rsplit(")", 1)[0]) + ) + label = head.strip() + if "." not in label: - return None, label + return None, label, params type_name, method_name = label.rsplit(".", 1) type_name = type_name.replace("+", ".") if method_name == "ctor": method_name = ".ctor" - return type_name, method_name + return type_name, method_name, params # --------------------------------------------------------------------------- @@ -86,11 +151,19 @@ def load_dump_index(path: str) -> dict: def types_by_name(index: dict) -> dict[str, list[dict]]: - """Bucket types by their bare ``name`` so we don't have to re-scan - the 20k+ row list per lookup.""" + """Bucket types by name so we don't have to re-scan the 20k+ row list + per lookup. + + Each type is filed under both its bare ``name`` and its + namespace-qualified spelling, so a recipe may write either + ``HeaderProvider`` or ``Project.Network.HeaderProvider``. + """ out: dict[str, list[dict]] = {} for t in index.get("types", []): out.setdefault(t["name"], []).append(t) + namespace = t.get("namespace", "") + if namespace: + out.setdefault(f"{namespace}.{t['name']}", []).append(t) return out @@ -114,17 +187,44 @@ def _sig_matches(sig: str, method_name: str) -> bool: return f" {method_name}(" in sig or f".{method_name}(" in sig +def sig_param_types(sig: str) -> tuple[str, ...]: + """Return the declared parameter types of a dump-index signature.""" + _head, sep, rest = sig.partition("(") + if not sep: + return () + return tuple(param_type(p) for p in split_params(rest.rsplit(")", 1)[0])) + + +def _params_match(sig: str, wanted: tuple[str, ...]) -> bool: + """True if ``sig``'s parameter types match ``wanted``. + + Comparison is on the trailing segment of each type so a recipe can + write ``ReadOnlySequence`` against a dump that spells it + ``pb::ReadOnlySequence``, and on arity first so an overload with + a different parameter count is rejected outright. + """ + actual = sig_param_types(sig) + if len(actual) != len(wanted): + return False + return all( + a == w or a.endswith(f".{w}") or a.endswith(f"::{w}") + for a, w in zip(actual, wanted, strict=True) + ) + + def find_method( by_name: dict[str, list[dict]], type_name: str | None, method_name: str, expected_rva: int | None = None, + param_types: tuple[str, ...] | None = None, ) -> tuple[dict, dict] | None: """Return ``(type_record, method_record)`` for the method matching ``method_name`` on the named type. - When the type has overloads (multiple sigs with the same method name), - and ``expected_rva`` is provided, the overload whose ``rva`` matches is + ``param_types`` selects one overload by its declared parameter types; + without it any overload matches. When the type has overloads and + ``expected_rva`` is provided, the overload whose ``rva`` matches is preferred over the first textual hit. This resolves ambiguity for methods like ``TryMakeMove`` that appear in both a single-arg and an out-arg variant. @@ -134,7 +234,10 @@ def find_method( def _pick_best(candidates_iter): first_hit = None for t, m in candidates_iter: - if not _sig_matches(m.get("sig", ""), method_name): + sig = m.get("sig", "") + if not _sig_matches(sig, method_name): + continue + if param_types is not None and not _params_match(sig, param_types): continue if expected_rva is not None and int(m.get("rva", "0x0"), 0) == expected_rva: return t, m # exact RVA match wins immediately @@ -244,13 +347,16 @@ def verify(args: argparse.Namespace) -> int: label = row[-1] total += 1 try: - type_name, method_name = split_label(label) + type_name, method_name, param_types = split_label(label) except ValueError as e: print(f" FAIL slot[{slot_index:>2}] {label!r}: {e}") fail += 1 continue - hit = find_method(by_name, type_name, method_name, expected_rva=site_off) + hit = find_method( + by_name, type_name, method_name, + expected_rva=site_off, param_types=param_types, + ) if hit is None: print( f" FAIL slot[{slot_index:>2}] {label!r}: "