From 825861b7f2c92425c33ee74b7ba2a846e9f9e389 Mon Sep 17 00:00:00 2001 From: klimovich008 Date: Mon, 9 Nov 2026 02:15:25 +0100 Subject: [PATCH 1/5] perf(lift): one page mark per straight-line group of stores through a base register A64_WRITE_TRACK marks the page of every non-SP store, two byte stores and the address arithmetic each. In a straight-line run (no label, branch, call, return, poll, barrier, atomic, exclusive, SVC or trap) the stores through one base register plus constants spanning at most 4 KB now share one A64_MARK of their byte span, placed after the group's last data store and before the base register changes or the run ends. The pages marked are the same, and each data store is still followed by a release store of its page's mark before any guest synchronisation. 43% fewer mark stores in the lifted game, about 24 MB less code. A64_MARK_EACH=1 at lift time keeps a mark per store. Co-Authored-By: Claude Opus 5.5 --- tests/test_mark_groups.py | 83 ++++++++++++++++++ third_party/a64-kit/lift/a64_lift.py | 122 +++++++++++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 tests/test_mark_groups.py diff --git a/tests/test_mark_groups.py b/tests/test_mark_groups.py new file mode 100644 index 00000000..8d837188 --- /dev/null +++ b/tests/test_mark_groups.py @@ -0,0 +1,83 @@ +"""a64_lift.mark_groups: the page marks of a straight-line run of stores through one base register shared by one A64_MARK +after the group's last store, and every case where a store must keep its own marks. + +python -I -B tests/test_mark_groups.py +""" +import os +import sys +import unittest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, "third_party", "a64-kit", "lift")) +import a64_lift # noqa: E402 + +HEAD = "void f_80000000(GuestContext *A64_NOESC c) {" + + +def run(*body): + return a64_lift.mark_groups("\n".join((HEAD,) + body + ("}",))).split("\n")[1:-1] + + +class MarkGroupsTests(unittest.TestCase): + def test_a_group_shares_one_mark_before_the_call(self): + out = run(" recomp_store64(c,c->x[19] + 8,c->x[1]);", + " recomp_store32(c,c->x[19] + 16,c->x[2]);", + " { uint64_t _b=c->x[19]; int64_t _o=24; recomp_stp64(c,(_b+_o),c->x[3],c->x[4]); }", + " c->x[0]=recomp_load32(c,c->x[20] + 0);", + " TAILCALL(f_80000100);") + self.assertEqual(out, [" recomp_store64_sp(c,c->x[19] + 8,c->x[1]);", + " recomp_store32_sp(c,c->x[19] + 16,c->x[2]);", + " { uint64_t _b=c->x[19]; int64_t _o=24; recomp_stp64_sp(c,(_b+_o),c->x[3],c->x[4]); }", + " c->x[0]=recomp_load32(c,c->x[20] + 0);", + " A64_MARK((c->x[19] + (uint64_t)(int64_t)(8)), 32);", # [8, 40): every byte the group wrote + " TAILCALL(f_80000100);"]) + + def test_a_write_of_the_base_marks_before_it(self): + out = run(" recomp_store64(c,c->x[8] + 0,c->x[1]);", + " recomp_store64(c,c->x[8] + 8,c->x[2]);", + " c->x[8]=c->x[8] + 16;", + " recomp_store64(c,c->x[8] + 0,c->x[3]);", + " a64_trap(c, 1);") + self.assertEqual(out[2], " A64_MARK((c->x[8] + (uint64_t)(int64_t)(0)), 16);") + self.assertEqual(out[3], " c->x[8]=c->x[8] + 16;") + self.assertEqual(out[4], " recomp_store64(c,c->x[8] + 0,c->x[3]);") # (alone after the write: its own marks) + + def test_every_run_end_marks_before_it(self): + for end in ("L_80000010:", "if (!c->z) { goto L_80000010; }", " recomp_barrier(c);", " a64_svc(c, 0x60);", + " a64_icall(c, c->x[16]);", " __atomic_fetch_add((uint32_t *)A64_PTR(c->x[2]), 1, __ATOMIC_SEQ_CST);", + " f_80001000(c);", " return;"): + out = run(" recomp_store32(c,c->x[21] + 0,c->x[1]);", " recomp_store32(c,c->x[21] + 4,c->x[2]);", end, + " recomp_store32(c,c->x[21] + 8,c->x[3]);") + self.assertEqual(out[2], " A64_MARK((c->x[21] + (uint64_t)(int64_t)(0)), 8);", end) + self.assertEqual(out[3], end) + + def test_stores_that_keep_their_own_marks(self): + cases = [ + # (a conditional store: it may not run) + (" if (c->z) recomp_store32(c,c->x[19] + 0,c->x[1]);", " recomp_store32(c,c->x[19] + 4,c->x[2]);"), + # (more than 4 KB apart: more than two pages) + (" recomp_store64(c,c->x[19] + 0,c->x[1]);", " recomp_store64(c,c->x[19] + 5000,c->x[2]);"), + # (a line that also writes a register) + (" recomp_store64(c,c->x[19] + 0,c->x[1]); c->x[1]=0;", " recomp_store64(c,c->x[19] + 8,c->x[2]); c->x[2]=0;"), + ] + for body in cases: + out = run(*body, " TAILCALL(f_80000100);") + self.assertFalse(any("A64_MARK" in l or "_sp(" in l for l in out), body) + + def test_bases_group_apart_and_sp_is_left_alone(self): + out = run(" recomp_store64(c,c->x[19] + 0,c->x[1]);", " recomp_store64(c,c->x[20] + 0,c->x[2]);", + " recomp_store64(c,c->x[19] + 8,c->x[3]);", " recomp_store64(c,c->x[20] + 8,c->x[4]);", + " recomp_store64(c,c->x[31] + 8,c->x[5]);", " TAILCALL(f_80000100);") + marks = [l for l in out if "A64_MARK" in l] + self.assertEqual(sorted(marks), [" A64_MARK((c->x[19] + (uint64_t)(int64_t)(0)), 16);", + " A64_MARK((c->x[20] + (uint64_t)(int64_t)(0)), 16);"]) + self.assertIn(" recomp_store64(c,c->x[31] + 8,c->x[5]);", out) + + def test_no_group_leaves_the_text_as_it_was(self): + body = (" recomp_store64(c,c->x[19] + 0,c->x[1]);", " TAILCALL(f_80000100);") + text = "\n".join((HEAD,) + body + ("}",)) + self.assertEqual(a64_lift.mark_groups(text), text) + + +if __name__ == "__main__": + unittest.main() diff --git a/third_party/a64-kit/lift/a64_lift.py b/third_party/a64-kit/lift/a64_lift.py index 1d3b7d50..85a34cd4 100644 --- a/third_party/a64-kit/lift/a64_lift.py +++ b/third_party/a64-kit/lift/a64_lift.py @@ -211,6 +211,126 @@ def stack_stores(text): return SP_STORE.sub(r"recomp_\1_sp(c,\2c->x[31]", text) +# Page marks per group (A2). In a straight-line run of a function's C (no label, branch, call, return, poll, barrier, +# atomic, exclusive, SVC or trap) the marked stores through the same base register plus a constant keep their data +# stores but share one A64_MARK(base + lo, hi - lo) for their byte span [lo, hi) (when it is at most 4 KB), placed +# after the group's last store and before the first later line that writes the base register or ends the run. The +# pages marked are the same: every store of the group lies in [base+lo, base+hi), at most two pages, which the stores +# at lo and ending at hi mark themselves. Each data store is still followed, in program order and before any guest +# synchronisation, by a release store of a mark of its page, so a collection that clears the mark sees the data +# (rt/a64.h A64_MARK). 43% fewer mark stores in OoT's lifted C. A64_MARK_EACH=1 at lift time: a mark per store. +MG_STORE = re.compile(r"recomp_(store8|store16|store32|store64|stp32|stp64)\(c,") +MG_SIZE = {"store8": 1, "store16": 2, "store32": 4, "store64": 8, "stp32": 8, "stp64": 16} +# address forms (the text right after "recomp_X(c,") +MG_ADDR = [ + (re.compile(r"c->x\[(\d+)\] \+ (-?\d+),"), "x"), # c->x[N] + IMM, + (re.compile(r"c->x\[(\d+)\],"), "x0"), # c->x[N], + (re.compile(r"\(c->x\[(\d+)\]\+(\d+)ULL\),"), "x"), # (c->x[N]+IMMULL), + (re.compile(r"\(_b\+_o\)\+(\d+),"), "bo+"), # (_b+_o)+IMM, + (re.compile(r"\(_b\+_o\),"), "bo"), # (_b+_o), + (re.compile(r"_b,"), "b"), # _b, + (re.compile(r"_a\+(\d+),"), "a+"), # _a+IMM, + (re.compile(r"_a,"), "a"), # _a, +] +MG_BDEF = re.compile(r"uint64_t _b=c->x\[(\d+)\]; int64_t _o=(-?\d+);") +MG_ADEF = re.compile(r"uint64_t _a=c->x\[(\d+)\];") +# a line that ends the straight-line run: control flow, calls, polls, any synchronisation or runtime call +MG_BOUNDARY = re.compile(r"^\s*L_[0-9a-f]+:|\bgoto\b|\breturn\b|TAILCALL|\bswitch\b|\bf_[0-9a-f]{8}\(|\bA64_[A-Z_]+\(|" + r"__atomic|__sync|recomp_barrier|recomp_sj_|recomp_(ldx|stx|ldax|stlx|ldxp|stxp|clrex|svc|cntpct|" + r"unhandled|ic_)|a64_(svc|trap|icall|ijump|poll|ret_mismatch)|^\s*\}\s*$") +# a line whose stores may not all run, or run more than once: its stores keep their own marks +MG_NOT_STRAIGHT = re.compile(r"\bif\b|\?|\bfor\b|\bwhile\b|\bswitch\b|\bcase\b") +# (any write of a guest register: =, compound assignments, ++/--, or its address taken (LDP's outputs)) +MG_REG_WRITE = re.compile(r"c->x\[(\d+)\]\s*(?:[-+*/%&|^]|<<|>>)?=(?!=)|&c->x\[(\d+)\]|(?:\+\+|--)\s*c->x\[(\d+)\]|" + r"c->x\[(\d+)\]\s*(?:\+\+|--)") +# (the register file as a whole: a line touching it so ends every group) +MG_REG_FILE = re.compile(r"c->x(?!\[)") + + +def mg_parse_store(line, m): + """(base register, offset, size) of the marked store at match m, or None""" + rest = line[m.end():] + size = MG_SIZE[m.group(1)] + for rx, kind in MG_ADDR: + a = rx.match(rest) + if not a: + continue + if kind == "x": + return int(a.group(1)), int(a.group(2)), size + if kind == "x0": + return int(a.group(1)), 0, size + if kind in ("bo", "bo+", "b"): + d = MG_BDEF.search(line[:m.start()]) + if not d: + return None + off = 0 if kind == "b" else int(d.group(2)) + (int(a.group(1)) if kind == "bo+" else 0) + return int(d.group(1)), off, size + if kind in ("a", "a+"): + d = MG_ADEF.search(line[:m.start()]) + if not d: + return None + return int(d.group(1)), (int(a.group(1)) if kind == "a+" else 0), size + return None + + +def mark_groups(text, stats=None): + lines = text.split("\n") + inserts = {} # line index -> mark lines to put before it + unmark = {} # line index -> set of match start offsets to turn into _sp + groups = {} # base register -> [(line index, match start, off, size)] + st = stats if stats is not None else {} + + def flush(reg, at): + g = groups.pop(reg, None) + if not g: + return + lo = min(o for _, _, o, _ in g) + hi = max(o + n for _, _, o, n in g) + st["stores"] = st.get("stores", 0) + len(g) + if len(g) < 2 or hi - lo > 4096: + return + for li, ms, _, _ in g: + unmark.setdefault(li, set()).add(ms) + inserts.setdefault(at, []).append(" A64_MARK((c->x[%d] + (uint64_t)(int64_t)(%d)), %d);" % (reg, lo, hi - lo)) + st["groups"] = st.get("groups", 0) + 1 + st["grouped_stores"] = st.get("grouped_stores", 0) + len(g) + + for i, line in enumerate(lines): + if line.startswith("void f_") or MG_BOUNDARY.search(line) or MG_REG_FILE.search(line): + for r in list(groups): + flush(r, i) + continue + writes = {int(next(g for g in m if g)) for m in MG_REG_WRITE.findall(line)} + stores = list(MG_STORE.finditer(line)) + if stores and not writes and not MG_NOT_STRAIGHT.search(line): + parsed = [(m, mg_parse_store(line, m)) for m in stores] + for m, p in parsed: + if p is None: + continue # (an address this pass does not follow: the store keeps its own marks) + reg, off, size = p + if reg == 31: + continue + groups.setdefault(reg, []).append((i, m.start(), off, size)) + continue + for r in writes: # (the base changes here: its group's mark goes before this line) + if r in groups: + flush(r, i) + for r in list(groups): + flush(r, len(lines)) + if not unmark: + return text + out = [] + for i, line in enumerate(lines): + out.extend(inserts.get(i, [])) + if i in unmark: + for ms in sorted(unmark[i], reverse=True): + m = MG_STORE.match(line, ms) + line = line[:m.start()] + "recomp_%s_sp(c," % m.group(1) + line[m.end():] + out.append(line) + out.extend(inserts.get(len(lines), [])) + return "\n".join(out) + + # Dead flags. Each flag-setting instruction stores all of N, Z, C and V in the context, where the compiler must keep # them: a callee or the caller could read them. The guest's ABI says otherwise (calls clobber NZCV, and nothing reads # flags across a return), so emit_function computes per function which flags are live after each instruction and @@ -1534,6 +1654,8 @@ def emit_function(self, e, texts): text = "\n".join(out) if frame: text = frame_stores(text) + if not os.environ.get("A64_MARK_EACH"): + text = mark_groups(text) return localize_regs(text, local, dirty) + "\n", self.refs def write(self, out_dir, chunk_bytes=CHUNK_BYTES): From 3fa63001afe0e4fafac6a1efc5980f2a35b4f32f Mon Sep 17 00:00:00 2001 From: klimovich008 Date: Mon, 9 Nov 2026 18:40:39 +0100 Subject: [PATCH 2/5] fix(lift): group page marks after the patches; a missing output-size edit stops the lift Review of #912. The groups changed the lifted text before nx2/oot_patches.py matched it exactly: OUT_EDITS[0] (nnMain's create arg) no longer matched, apply_output_size only warned, and the Output resolution was ignored. - Order: emit_function no longer groups. nx2/lift.py edit_c applies the patches, then a64_lift.group_marks over the written code_*.c files, last, once, with the lift's stats (reported). The kit's own write() keeps a mark per store; A64_MARK_EACH=1 still turns the groups off (in the lift key). - Required edits: on the supported build's whole standalone lift, every OUT_EDITS place must be patched, or found patched, exactly once, or the lift stops before the key is written. Other builds, partial lifts and the other patches warn as before. - Run ends, conservative: a line whose braces do not balance (no mark moves into or out of a multi-line block), STLR/LDAR in suyu's spelling, and any call-like token not on a short list of plain loads, stores and pure helpers (f_*, hos_* from the patches, A64_*, atomics, unknown helpers). - The pass is idempotent (its own marks are not run ends), takes the nearest _b/_a definition on a line, skips functions with local registers, and shares REG_WRITE (now with ++/--) with reg_writes. About 6x faster than before: functions with fewer than two stores skipped, lines with no open group skipped, cheap substring tests before each regex. - Lifted v2.0.6 C: mark stores 2,521,896 -> 1,454,508 (42.3% fewer; the first version 43.1%). Idempotent on all 80 files. - tests/test_mark_groups.py: the patches-then-groups sequence through lift.edit_c with the OUT_EDITS places (no game code), the old order as a failing control, the patched function compiled and run with and without groups (same memory, same pages), the review's address forms, the 4096 span, STLR/LDAR, blocks, host calls, A64_MARK_EACH, the lift key, stats, idempotence, and a random page-set check. nx2/test_leaf.py groups its functions as the lift does. Co-Authored-By: Claude Opus 5.5 --- nx2/lift.py | 12 +- nx2/oot_patches.py | 14 +- nx2/test_leaf.py | 3 +- tests/test_mark_groups.py | 408 +++++++++++++++++++++++++-- third_party/a64-kit/lift/a64_lift.py | 181 ++++++++---- 5 files changed, 533 insertions(+), 85 deletions(-) diff --git a/nx2/lift.py b/nx2/lift.py index c659cc8b..e3b65e7f 100644 --- a/nx2/lift.py +++ b/nx2/lift.py @@ -15,6 +15,7 @@ """ import argparse import collections +import glob import json import os import struct @@ -110,6 +111,15 @@ def coverage(L, img, out_dir): return total +def edit_c(gen, stats, supported, whole=True): + """The edits of the written C, in this order: the game's patches (nx2/oot_patches.py, which find the lift's exact text; the + supported build's whole standalone lift fails when an output-size edit finds no place), then, last, the page marks of each + group of stores shared (a64_lift.group_marks), which changes that text. stats: the lift's, for its report""" + import oot_patches + oot_patches.apply(gen, protect_hearts=supported, required=supported and whole) + a64_lift.group_marks(sorted(glob.glob(os.path.join(gen, "code_*.c"))), stats) + + def main(): kit.wait_for_quiet('the lift') p = argparse.ArgumentParser() @@ -180,7 +190,7 @@ def main(): counters=[] if a.no_polls else [poll.stats]) import oot_patches main_build = next((m["build_id"] for m in meta["modules"] if m["name"] == "main"), "") - oot_patches.apply(gen, protect_hearts=a.standalone and main_build == oot_patches.LIFE_BUILD_ID) + edit_c(gen, L.stats, a.standalone and main_build == oot_patches.LIFE_BUILD_ID, whole=not a.max_func) a64_rtgen.write(gen) if a.standalone: start, exit_ = standalone.write_image_c(gen, img, meta, a.exefs) diff --git a/nx2/oot_patches.py b/nx2/oot_patches.py index aec01252..d87bce63 100644 --- a/nx2/oot_patches.py +++ b/nx2/oot_patches.py @@ -153,7 +153,9 @@ def patch_text_speed(s): return s.replace('#include "a64.h"\n', '#include "a64.h"\n' + TEXT_DECL, 1) -def apply(gen, *, protect_hearts=False): +def apply(gen, *, protect_hearts=False, required=False): + """required: the lift of the supported build (LIFE_BUILD_ID), whole: a missing output-size edit stops it (apply_output_size). + The other patches only warn when their place is not found once.""" import oot_native_resolution # (HOS_SCENE_NATIVE=1: the scene's base scale 1.0; off by default since #849's output resolution) oot_native_resolution.apply(gen) done = {"dynamic resolution branch": 0, "text speed block": 0, "fixed path's scale": 0} @@ -206,7 +208,7 @@ def apply(gen, *, protect_hearts=False): for what, n in done.items(): if n != 1: print("oot_patches: WARNING: the %s was found %d times" % (what, n)) - apply_output_size(gen) + apply_output_size(gen, required=required) # Output size (host/nn_applet.c hos_output_size: the Insert menu's Output resolution, 2560 x 1440 as shipped): @@ -269,7 +271,10 @@ def apply(gen, *, protect_hearts=False): ] -def apply_output_size(gen): +def apply_output_size(gen, required=False): + """required (the supported build's whole lift): each edit must be made, or found made, exactly once, else SystemExit: without + one the Insert menu's Output resolution is ignored, or a 4K output runs out of the render targets' pool. Elsewhere (another + build, part of a lift, by hand) a warning""" found = [0] * len(OUT_EDITS) for path in sorted(glob.glob(os.path.join(gen, "code_*.c"))): with open(path, encoding="utf-8", newline="") as f: @@ -289,6 +294,9 @@ def apply_output_size(gen): f.write(s) print("oot_patches: output size in", os.path.basename(path)) if found != [1] * len(OUT_EDITS): + if required: + raise SystemExit("oot_patches: the output size places were found %s times, not once each (OUT_EDITS %s): the " + "lifted C no longer has the text they match" % (found, [i for i, n in enumerate(found) if n != 1])) print("oot_patches: WARNING: the output size places were found %s times" % found) diff --git a/nx2/test_leaf.py b/nx2/test_leaf.py index a060683b..db22f386 100644 --- a/nx2/test_leaf.py +++ b/nx2/test_leaf.py @@ -311,7 +311,8 @@ def build(tmp, P, roots, funcs, meta, img_size, addr64): if addr64: rt = rtgen.write(os.path.join(tmp, "rt64")) # (its a64.h shadows the vendored one) src = ['#include "a64.h"'] + ["void f_%08x(Ctx *A64_NOESC);" % e for e in sorted(funcs)] - src += [P.emitted[e][0] for e in sorted(funcs)] + body = "\n".join(P.emitted[e][0] for e in sorted(funcs)) + src.append(body if os.environ.get("A64_MARK_EACH") else a64_lift.mark_groups(body)) # (nx2/lift.py's last step) src.append(HARNESS % { "table": ",".join("f_%08x" % e for _, e, _ in roots), "regions": ",".join("{%#xULL,%#xULL}" % r for r in writable_regions(meta)), diff --git a/tests/test_mark_groups.py b/tests/test_mark_groups.py index 8d837188..6e4ae46f 100644 --- a/tests/test_mark_groups.py +++ b/tests/test_mark_groups.py @@ -1,21 +1,49 @@ """a64_lift.mark_groups: the page marks of a straight-line run of stores through one base register shared by one A64_MARK -after the group's last store, and every case where a store must keep its own marks. +after the group's last store, every case where a store must keep its own marks, and its place in the lift: after +nx2/oot_patches.py, whose exact-text edits a group would break (the supported build's lift fails when one is missing). -python -I -B tests/test_mark_groups.py +python -I -B tests/test_mark_groups.py (the compile-and-run case needs emsdk's clang: EMSDK, else it is skipped) """ +import contextlib +import glob +import importlib.util +import inspect +import io import os +import random +import re +import shutil +import subprocess import sys +import tempfile import unittest +from collections import Counter +from unittest import mock ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(ROOT, "third_party", "a64-kit", "lift")) +sys.path[:0] = [os.path.join(ROOT, "third_party", "a64-kit", d) for d in ("lift", "tools")] + [os.path.join(ROOT, "nx2")] import a64_lift # noqa: E402 +import oot_patches # noqa: E402 -HEAD = "void f_80000000(GuestContext *A64_NOESC c) {" +HEAD = "void f_80000000(Ctx *__restrict A64_NOESC c) {" -def run(*body): - return a64_lift.mark_groups("\n".join((HEAD,) + body + ("}",))).split("\n")[1:-1] +def mark(reg, lo, n): + return " A64_MARK((c->x[%d] + (uint64_t)(int64_t)(%d)), %d);" % (reg, lo, n) + + +def run(*body, stats=None): + return a64_lift.mark_groups("\n".join((HEAD,) + body + ("}",)), stats).split("\n")[1:-1] + + +def read(path): + with open(path, encoding="utf-8", newline="") as f: + return f.read() + + +def quiet(fn, *a, **k): + with contextlib.redirect_stdout(io.StringIO()): + return fn(*a, **k) class MarkGroupsTests(unittest.TestCase): @@ -29,54 +57,378 @@ def test_a_group_shares_one_mark_before_the_call(self): " recomp_store32_sp(c,c->x[19] + 16,c->x[2]);", " { uint64_t _b=c->x[19]; int64_t _o=24; recomp_stp64_sp(c,(_b+_o),c->x[3],c->x[4]); }", " c->x[0]=recomp_load32(c,c->x[20] + 0);", - " A64_MARK((c->x[19] + (uint64_t)(int64_t)(8)), 32);", # [8, 40): every byte the group wrote + mark(19, 8, 32), # [8, 40): every byte the group wrote " TAILCALL(f_80000100);"]) def test_a_write_of_the_base_marks_before_it(self): - out = run(" recomp_store64(c,c->x[8] + 0,c->x[1]);", - " recomp_store64(c,c->x[8] + 8,c->x[2]);", - " c->x[8]=c->x[8] + 16;", - " recomp_store64(c,c->x[8] + 0,c->x[3]);", - " a64_trap(c, 1);") - self.assertEqual(out[2], " A64_MARK((c->x[8] + (uint64_t)(int64_t)(0)), 16);") - self.assertEqual(out[3], " c->x[8]=c->x[8] + 16;") - self.assertEqual(out[4], " recomp_store64(c,c->x[8] + 0,c->x[3]);") # (alone after the write: its own marks) + for write in (" c->x[8]=c->x[8] + 16;", " c->x[8] += 16;", " c->x[8]++;", " --c->x[8];", + " { uint64_t _p0,_p1; recomp_ldp64(c,c->x[9] + 0,&c->x[8],&_p1); }"): + out = run(" recomp_store64(c,c->x[8] + 0,c->x[1]);", " recomp_store64(c,c->x[8] + 8,c->x[2]);", write, + " recomp_store64(c,c->x[8] + 0,c->x[3]);", " a64_trap(c, 1);") + self.assertEqual(out[2], mark(8, 0, 16), write) + self.assertEqual(out[3], write) + self.assertEqual(out[4], " recomp_store64(c,c->x[8] + 0,c->x[3]);", write) # (alone after it: its own marks) def test_every_run_end_marks_before_it(self): - for end in ("L_80000010:", "if (!c->z) { goto L_80000010; }", " recomp_barrier(c);", " a64_svc(c, 0x60);", - " a64_icall(c, c->x[16]);", " __atomic_fetch_add((uint32_t *)A64_PTR(c->x[2]), 1, __ATOMIC_SEQ_CST);", - " f_80001000(c);", " return;"): + for end in ("L_80000010:;", "L_80000010:; A64_POLL(c);", "if (!c->z) { goto L_80000010; }", " recomp_barrier();", + " a64_svc(c, 0x60);", " A64_ICALL(c, c->x[16]);", " A64_IJUMP(c, c->x[16]); return;", + " __atomic_fetch_add((uint32_t *)A64_PTR(c->x[2]), 1, __ATOMIC_SEQ_CST);", + " c->x[30] = 0x80000020ULL; f_80001000(c);", " return;", + # (store-release and load-acquire as suyu writes them, without nx2/insn.py: a plain store / load at + # the bare register) + " recomp_store32(c,c->x[3],c->x[4]);", " c->x[5] = recomp_load32(c,c->x[3]);", + # (host calls the patches put in, and any helper the pass does not know) + " c->x[8] = hos_output_size(0) | ((uint64_t)hos_output_size(1) << 32);", + " hos_oot_lift_player(c, c->x[19]);", " a64_new_helper(c);", " memcpy(A64_PTR(c->x[2]), &c->x[3], 8);", + # (the register file as a whole; a block opening or closing over several lines) + " memset(c->x, 0, sizeof c->x);", " {", " }", " if (c->z) {", " } else {"): out = run(" recomp_store32(c,c->x[21] + 0,c->x[1]);", " recomp_store32(c,c->x[21] + 4,c->x[2]);", end, " recomp_store32(c,c->x[21] + 8,c->x[3]);") - self.assertEqual(out[2], " A64_MARK((c->x[21] + (uint64_t)(int64_t)(0)), 8);", end) + self.assertEqual(out[2], mark(21, 0, 8), end) self.assertEqual(out[3], end) + def test_plain_lines_do_not_end_a_run(self): + for plain in (" c->x[0]=recomp_load64(c,c->x[20] + 8);", + " { uint64_t _p0,_p1; recomp_ldp64(c,(c->x[20]+16ULL),&_p0,&_p1); c->x[0]=_p0; c->x[1]=_p1; }", + " { uint64_t _v=0; memcpy(&_v,&c->vreg[3][0],8); a64_vset64(c->vreg[1],_v); }", + " { extern volatile uint64_t hos_wide_out; const uint64_t _s = hos_wide_out; if (_s) c->x[8] = _s; }", + " c->x[9] = UINT64_C(1) << 3;", " /* ic ivau: no self-modifying code */"): + out = run(" recomp_store32(c,c->x[21] + 0,c->x[1]);", plain, " recomp_store32(c,c->x[21] + 4,c->x[2]);", + " return;") + self.assertEqual(out[3], mark(21, 0, 8), plain) + def test_stores_that_keep_their_own_marks(self): cases = [ # (a conditional store: it may not run) (" if (c->z) recomp_store32(c,c->x[19] + 0,c->x[1]);", " recomp_store32(c,c->x[19] + 4,c->x[2]);"), + (" c->x[0] ? recomp_store32(c,c->x[19] + 0,c->x[1]) : (void)0;", " recomp_store32(c,c->x[19] + 4,c->x[2]);"), # (more than 4 KB apart: more than two pages) - (" recomp_store64(c,c->x[19] + 0,c->x[1]);", " recomp_store64(c,c->x[19] + 5000,c->x[2]);"), - # (a line that also writes a register) + (" recomp_store64(c,c->x[19] + 0,c->x[1]);", " recomp_store64(c,c->x[19] + 4089,c->x[2]);"), + # (a line that also writes a register; pre- and post-index writeback) (" recomp_store64(c,c->x[19] + 0,c->x[1]); c->x[1]=0;", " recomp_store64(c,c->x[19] + 8,c->x[2]); c->x[2]=0;"), + (" { uint64_t _b=c->x[19]; int64_t _o=16; recomp_store64(c,(_b+_o),c->x[1]); c->x[19]=_b+_o; }", + " { uint64_t _b=c->x[19]; int64_t _o=16; recomp_store64(c,_b,c->x[2]); c->x[19]=_b+_o; }"), + # (an address the pass does not follow: a register offset) + (" recomp_store64(c,c->x[19] + c->x[2],c->x[1]);", " recomp_store64(c,c->x[19] + 8 * c->x[3],c->x[2]);"), + # (the STLR form: it ends the run besides) + (" recomp_store32(c,c->x[19],c->x[1]);", " recomp_store32(c,c->x[19],c->x[2]);"), ] for body in cases: out = run(*body, " TAILCALL(f_80000100);") self.assertFalse(any("A64_MARK" in l or "_sp(" in l for l in out), body) + def test_address_forms(self): + q = " { { uint64_t _v=0; memcpy(&_v,&c->vreg[3][0],8); recomp_store64(c,(c->x[0]+%dULL),_v); } }" + cases = [ + # (Q / D stores at (c->x[N]+IMMULL)) + ((q % 16, q % 24), mark(0, 16, 16)), + # (STP of two Q registers: the second at (_b+_o)+16) + ((" { uint64_t _b=c->x[22]; int64_t _o=40; recomp_stp64(c,(_b+_o),c->vreg[2][0],c->vreg[2][1]); " + "recomp_stp64(c,(_b+_o)+16,c->vreg[3][0],c->vreg[3][1]); }",), mark(22, 40, 32)), + # (ST1 lanes at _a+IMM) + ((" { uint64_t _a=c->x[12]; { uint64_t _v=0; memcpy(&_v,(const uint8_t*)c->vreg[0]+8,4); recomp_store32(c,_a+0,_v); } " + "{ uint64_t _v=0; memcpy(&_v,(const uint8_t*)c->vreg[0]+12,4); recomp_store32(c,_a+4,_v); } }",), mark(12, 0, 8)), + # (STUR: negative offsets, both spellings) + ((" { uint64_t _b=c->x[8]; int64_t _o=-16; recomp_store64(c,(_b+_o),c->x[1]); }", + " recomp_store64(c,c->x[8] + -8,c->x[2]);"), mark(8, -16, 16)), + # (exactly 4096 bytes: two pages) + ((" recomp_store64(c,c->x[19] + 0,c->x[1]);", " recomp_store64(c,c->x[19] + 4088,c->x[2]);"), mark(19, 0, 4096)), + # (the nearest _b before a store is its own: x9's store stays alone, x7's two group) + ((" { uint64_t _b=c->x[9]; int64_t _o=0; recomp_store64(c,(_b+_o),c->x[1]); } " + "{ uint64_t _b=c->x[7]; int64_t _o=8; recomp_store64(c,(_b+_o),c->x[2]); }", + " recomp_store64(c,c->x[7] + 0,c->x[3]);"), mark(7, 0, 16)), + ] + for body, want in cases: + out = run(*body, " return;") + self.assertEqual([l for l in out if "A64_MARK" in l], [want], body) + self.assertEqual(out[-2], want, body) + left = re.findall(r"recomp_(?:store|stp)\d+\(c,[^,]*", "\n".join(out)) + self.assertEqual(left, ["recomp_store64(c,(_b+_o)"] if "c->x[9]" in body[0] else [], body) + + def test_the_pages_marked_are_the_same(self): + """random groups at random bases: the pages the marks left behind (and the group marks) mark are the pages a mark per + store marks""" + rng = random.Random(912) + store = re.compile(r"recomp_(store8|store16|store32|store64|stp32|stp64)(_sp)?\(c,c->x\[(\d+)\] \+ (-?\d+),") + own = re.compile(r"A64_MARK\(\(c->x\[(\d+)\] \+ \(uint64_t\)\(int64_t\)\((-?\d+)\)\), (\d+)\)") + + def pages(lines, base, each): + got = set() + for l in lines: + for m in store.finditer(l): + if each or not m.group(2): + a, n = base[int(m.group(3))] + int(m.group(4)), a64_lift.MG_SIZE[m.group(1)] + got |= {a >> 12, (a + n - 1) >> 12} + m = own.search(l) + if m: + a, n = base[int(m.group(1))] + int(m.group(2)), int(m.group(3)) + got |= {a >> 12, (a + n - 1) >> 12} + return got + grouped = 0 + for _ in range(3000): + body = [] + for _ in range(rng.randint(2, 9)): + kind = rng.choice(sorted(a64_lift.MG_SIZE)) + body.append(" recomp_%s(c,c->x[%d] + %d,c->x[1]%s);" % ( + kind, rng.choice((19, 20)), rng.randint(-300, 4400), ",c->x[2]" if kind.startswith("stp") else "")) + out = run(*body, " return;") + grouped += any("A64_MARK" in l for l in out) + for _ in range(4): + base = {19: rng.randrange(1 << 20, 1 << 30), 20: rng.choice((1 << 24, (1 << 24) - rng.randint(1, 4096)))} + self.assertEqual(pages(out, base, False), pages(body, base, True), body) + self.assertGreater(grouped, 1000) + + def test_multi_line_blocks_keep_their_marks_inside(self): + out = run(" recomp_store32(c,c->x[19] + 0,c->x[1]);", " recomp_store32(c,c->x[19] + 4,c->x[2]);", + " if (c->z) {", + " recomp_store32(c,c->x[19] + 8,c->x[3]);", " recomp_store32(c,c->x[19] + 12,c->x[3]);", + " }", + " recomp_store32(c,c->x[19] + 16,c->x[4]);", " return;") + self.assertEqual(out, [ + " recomp_store32_sp(c,c->x[19] + 0,c->x[1]);", " recomp_store32_sp(c,c->x[19] + 4,c->x[2]);", + mark(19, 0, 8), # (before the block: the block may not run) + " if (c->z) {", + " recomp_store32_sp(c,c->x[19] + 8,c->x[3]);", " recomp_store32_sp(c,c->x[19] + 12,c->x[3]);", + mark(19, 8, 8), # (inside it, with its stores) + " }", + " recomp_store32(c,c->x[19] + 16,c->x[4]);", " return;"]) + def test_bases_group_apart_and_sp_is_left_alone(self): out = run(" recomp_store64(c,c->x[19] + 0,c->x[1]);", " recomp_store64(c,c->x[20] + 0,c->x[2]);", " recomp_store64(c,c->x[19] + 8,c->x[3]);", " recomp_store64(c,c->x[20] + 8,c->x[4]);", - " recomp_store64(c,c->x[31] + 8,c->x[5]);", " TAILCALL(f_80000100);") - marks = [l for l in out if "A64_MARK" in l] - self.assertEqual(sorted(marks), [" A64_MARK((c->x[19] + (uint64_t)(int64_t)(0)), 16);", - " A64_MARK((c->x[20] + (uint64_t)(int64_t)(0)), 16);"]) + " recomp_store64(c,c->x[31] + 8,c->x[5]);", " recomp_store64(c,c->x[31] + 16,c->x[5]);", + " TAILCALL(f_80000100);") + self.assertEqual(sorted(l for l in out if "A64_MARK" in l), [mark(19, 0, 16), mark(20, 0, 16)]) self.assertIn(" recomp_store64(c,c->x[31] + 8,c->x[5]);", out) def test_no_group_leaves_the_text_as_it_was(self): - body = (" recomp_store64(c,c->x[19] + 0,c->x[1]);", " TAILCALL(f_80000100);") - text = "\n".join((HEAD,) + body + ("}",)) - self.assertEqual(a64_lift.mark_groups(text), text) + for body in ((" recomp_store64(c,c->x[19] + 0,c->x[1]);", " TAILCALL(f_80000100);"), + # (a function with locals, A64_LOCAL_REGS: left alone) + (" uint64_t x19_ = c->x[19];", " recomp_store64(c,x19_ + 0,c->x[1]);", + " recomp_store64(c,c->x[20] + 0,c->x[1]);", " recomp_store64(c,c->x[20] + 8,c->x[1]);")): + text = "\n".join((HEAD,) + body + ("}",)) + self.assertEqual(a64_lift.mark_groups(text), text) + + def test_twice_is_once(self): + """its own marks are not run ends: a group the first pass found too wide is not split by them on a second pass""" + text = "\n".join(('#include "a64.h"', "void f_80000000(Ctx *A64_NOESC);", HEAD, + " recomp_store64(c,c->x[19] + 0,c->x[1]);", " recomp_store64(c,c->x[19] + 8,c->x[1]);", + " recomp_store64(c,c->x[20] + 0,c->x[1]);", " recomp_store64(c,c->x[20] + 8,c->x[1]);", + " c->x[20] = 0;", " recomp_store64(c,c->x[19] + 5000,c->x[1]);", + " { uint64_t _b=c->x[22]; int64_t _o=40; recomp_stp64(c,(_b+_o),c->x[1],c->x[2]); }", + " recomp_store8(c,c->x[22] + 39,c->x[1]);", " return;", "}", + "void f_80000100(Ctx *__restrict A64_NOESC c) {", " recomp_store8(c,c->x[3] + 0,c->x[1]);", + " recomp_store8(c,c->x[3] + 1,c->x[1]);", "}", "")) + st = Counter() + once = a64_lift.mark_groups(text, st) + self.assertEqual(st["groups"], 3) # (x20, x22, and x3 of the second function; x19's is 5008 bytes) + self.assertEqual(a64_lift.mark_groups(once), once) + self.assertIn("void f_80000000(Ctx *A64_NOESC);", once.split("\n")) + + def test_mark_each_and_the_stats(self): + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "code_000.c") + text = "\n".join((HEAD, " recomp_store64(c,c->x[19] + 0,c->x[1]);", + " recomp_store64(c,c->x[19] + 8,c->x[1]);", " return;", "}", "")) + with open(path, "w", newline="") as f: + f.write(text) + st = Counter() + with mock.patch.dict(os.environ, {"A64_MARK_EACH": "1"}): + self.assertEqual(a64_lift.group_marks([path], st), 0) + self.assertEqual(read(path), text) + self.assertEqual(st["groups"], 0) + with mock.patch.dict(os.environ, {k: v for k, v in os.environ.items() if k != "A64_MARK_EACH"}, clear=True): + self.assertEqual(a64_lift.group_marks([path], st), 1) + self.assertEqual((st["stores"], st["groups"], st["grouped_stores"]), (2, 1, 2)) + self.assertIn(mark(19, 0, 16), read(path)) + + def test_the_lift_key_has_the_switch(self): + spec = importlib.util.spec_from_file_location("make_game_mg", os.path.join(ROOT, "nx2", "make_game.py")) + make_game = importlib.util.module_from_spec(spec) + spec.loader.exec_module(make_game) + with tempfile.TemporaryDirectory() as d: + exefs, work = os.path.join(d, "exefs"), os.path.join(d, "work") + os.makedirs(exefs) + os.makedirs(work) + with open(os.path.join(exefs, "main"), "wb") as f: + f.write(b"NSO0") + env = {k: v for k, v in os.environ.items() if not k.startswith(("A64_", "OOT_", "RX_"))} + with mock.patch.dict(os.environ, env, clear=True): + key = make_game.lift_key(exefs, work, {}) + with mock.patch.dict(os.environ, {"A64_MARK_EACH": "1"}): + self.assertNotEqual(make_game.lift_key(exefs, work, {}), key) + + def test_the_lift_groups_once_with_its_stats(self): + """emit_function leaves the marks as they are; nx2/lift.py's edit_c groups the written files, with the lift's stats, + before its report""" + self.assertNotIn("mark_groups", inspect.getsource(a64_lift.Lifter.emit_function)) + src = read(os.path.join(ROOT, "nx2", "lift.py")) + self.assertEqual(src.count("group_marks("), 1) + call = src.find("edit_c(gen, L.stats, ") + self.assertGreater(call, 0) + self.assertLess(call, src.find("L.report()")) + + +# The output-size edits' places (nx2/oot_patches.py OUT_EDITS) in small functions, no game code: [0] and [3] each followed by +# another store through the same base, so that a group taken before the patches would change the text they match. +GEN_000 = ('#include "a64.h"\n' + "void f_81222ce0(Ctx *__restrict A64_NOESC c) {\n" + " recomp_store64(c,c->x[21] + 4,c->x[1]);\n" + + oot_patches.OUT_EDITS[0][0] + "\n" + " recomp_store32(c,c->x[21] + 28,c->x[9]);\n" + + oot_patches.OUT_EDITS[3][0] + + " recomp_store32(c,c->x[19] + 3672,c->x[8]);\n" + " return;\n" + "}\n") +GEN_001 = ('#include "a64.h"\n' + "void f_80000000(Ctx *__restrict A64_NOESC c) {\n" + + "".join(old for old, _ in oot_patches.OUT_EDITS[1:3]) + " return;\n" + "".join( + old + " recomp_store32(c,c->x[19] + 0,c->x[1]);\n recomp_store32(c,c->x[19] + 4,c->x[1]);\n" + for old, _ in oot_patches.OUT_EDITS[4:]) + + " return;\n" + "}\n") + + +def make_gen(d): + for name, text in (("code_000.c", GEN_000), ("code_001.c", GEN_001)): + with open(os.path.join(d, name), "w", newline="") as f: + f.write(text) + return sorted(glob.glob(os.path.join(d, "code_*.c"))) + + +def ungroup(text): + """the text as it was before the groups: their marks gone, their stores marked again""" + text = "\n".join(l for l in text.split("\n") if not a64_lift.MG_OWN_MARK.match(l)) + return re.sub(r"recomp_(store8|store16|store32|store64|stp32|stp64)_sp\(c,", r"recomp_\1(c,", text) + + +def lift_driver(): + """nx2/lift.py as a module (capstone, which only its coverage count uses, stood in for when it is not installed)""" + try: + import capstone # noqa: F401 + except ImportError: + import types + sys.modules["capstone"] = types.ModuleType("capstone") + import lift + return lift + + +class OutputSizeOrderTests(unittest.TestCase): + def test_the_patches_then_the_groups(self): + """the lift's edits of its C (nx2/lift.py edit_c) for the supported build's whole lift: every output-size edit made + once, then the groups""" + with tempfile.TemporaryDirectory() as d: + files = make_gen(d) + st = Counter() + quiet(lift_driver().edit_c, d, st, True) + text = "".join(read(f) for f in files) + self.assertIn(oot_patches.OUT_DECL, text) + for old, new in oot_patches.OUT_EDITS: # (each made once, and only groups changed it since) + self.assertEqual(ungroup(text).count(new), 1, new) + # (the output size reaches the window's create arg: x21+20 is a store of x8 just after the patched lines, now in a + # group with x21+28, whose mark covers it) + f0 = read(files[0]) + self.assertIn(" c->x[8] = hos_output_size(0) | ((uint64_t)hos_output_size(1) << 32);\n", f0) + self.assertIn("{ uint64_t _b=c->x[21]; int64_t _o=20; recomp_store64_sp(c,(_b+_o),c->x[8]); }", f0) + self.assertIn(mark(21, 20, 12), f0) + self.assertIn(" recomp_store64(c,c->x[21] + 4,c->x[1]);", f0) # (the host call between: its own marks) + self.assertEqual(st["groups"], 5) # (x21 [20, 32) and the four pairs of code_001.c) + + def test_patches_made_before_are_found_once(self): + with tempfile.TemporaryDirectory() as d: + make_gen(d) + quiet(oot_patches.apply_output_size, d, required=True) + once = [read(f) for f in sorted(glob.glob(os.path.join(d, "code_*.c")))] + quiet(oot_patches.apply_output_size, d, required=True) + self.assertEqual([read(f) for f in sorted(glob.glob(os.path.join(d, "code_*.c")))], once) + + def test_groups_before_the_patches_fail_the_lift(self): + """the order before this change (sabotage control): the groups break OUT_EDITS[0] and [3], and the supported build's + lift stops instead of ignoring the Output resolution""" + with tempfile.TemporaryDirectory() as d: + files = make_gen(d) + a64_lift.group_marks(files) + with self.assertRaises(SystemExit) as e: + quiet(oot_patches.apply_output_size, d, required=True) + self.assertIn("OUT_EDITS [0, 3]", str(e.exception)) + with tempfile.TemporaryDirectory() as d: # (another build or part of a lift: a warning, as before) + files = make_gen(d) + a64_lift.group_marks(files) + out = io.StringIO() + with contextlib.redirect_stdout(out): + oot_patches.apply_output_size(d, required=False) + self.assertIn("WARNING: the output size places were found [0, 1, 1, 0, 1, 1, 1, 1] times", out.getvalue()) + + def test_a_missing_place_fails_once_required(self): + """the supported build's whole lift stops on a missing place; another build's, or part of a lift, warns as before""" + for supported, whole, fails in ((True, True, True), (False, True, False), (True, False, False)): + with tempfile.TemporaryDirectory() as d: + make_gen(d) + path = os.path.join(d, "code_001.c") + text = read(path).replace("L_836cea54:;\n", "L_836cea58:;\n") + with open(path, "w", newline="") as f: + f.write(text) + out = io.StringIO() + with contextlib.redirect_stdout(out): + if fails: + with self.assertRaises(SystemExit) as e: + lift_driver().edit_c(d, Counter(), supported, whole) + self.assertIn("OUT_EDITS [4]", str(e.exception)) + else: + lift_driver().edit_c(d, Counter(), supported, whole) + self.assertIn("WARNING: the output size places were found [1, 1, 1, 1, 0, 1, 1, 1] times", out.getvalue()) + + @unittest.skipUnless(os.path.isfile(os.path.join(os.environ.get("EMSDK", r"C:\emsdk"), "upstream", "bin", "clang.exe")) + or os.name != "nt" and shutil.which("clang"), "no clang") + def test_the_patched_function_runs_the_same_grouped(self): + """f_81222ce0's output size compiled from the patched C, without and with the groups: the same guest memory (the + output size at x21+20) and the same pages marked""" + import toolchain + cc = toolchain.need(toolchain.find("clang")) + harness = r''' +#include +#include +#include +uint8_t a64_test_mem[1 << 16]; +uint8_t a64_dirty[1u << 20]; +volatile uint64_t hos_wide_out; +uint32_t hos_output_size(int height) { return height ? 2160 : 3840; } +uint64_t hos_rt_pool_size(uint64_t g) { return g; } +#include "code_000.c" +int main(void) { + Ctx c; + memset(&c, 0, sizeof c); + c.x[1] = 0x1122334455667788ULL; c.x[9] = 0x99aabbccULL; c.x[21] = 0x2fec; c.x[19] = 0x4000; + f_81222ce0(&c); + for (unsigned i = 0; i < sizeof a64_test_mem; i++) if (a64_test_mem[i]) printf("m %x %02x\n", i, a64_test_mem[i]); + for (unsigned i = 0; i < 64; i++) if (a64_dirty[i]) printf("p %u\n", i); + return 0; +} +''' + outs = [] + with tempfile.TemporaryDirectory() as d: + make_gen(d) + quiet(oot_patches.apply, d, required=True) + for f in ("a64_suyu.h", "a64_fpx.h"): + open(os.path.join(d, f), "w").close() + with open(os.path.join(d, "harness.c"), "w") as f: + f.write(harness) + for grouped in (False, True): + if grouped: + self.assertEqual(a64_lift.group_marks([os.path.join(d, "code_000.c")]), 1) + exe = os.path.join(d, "h%d.exe" % grouped) + subprocess.run([cc] + toolchain.native_target() + toolchain.native_link() + + ["-O1", "-Wno-everything", "-DA64_WRITE_TRACK", "-DA64_HOST_BASE=((uintptr_t)a64_test_mem)", + "-I" + os.path.join(ROOT, "third_party", "a64-kit", "rt"), "-I" + d, + os.path.join(d, "harness.c"), "-o", exe], check=True) + outs.append(subprocess.run([exe], check=True, capture_output=True, text=True).stdout) + self.assertEqual(outs[0], outs[1]) + mem = {int(a, 16): int(v, 16) for k, a, v in (l.split() for l in outs[0].splitlines() if l[0] == "m")} + word = lambda a, n: sum(mem.get(a + i, 0) << (8 * i) for i in range(n)) + self.assertEqual(word(0x2fec + 20, 8), 3840 | 2160 << 32) # (the window's size, from hos_output_size) + self.assertEqual(word(0x4000 + 3668, 4), 0x45340000) # (2880.0f: 3/4 of the output's width) + self.assertEqual(word(0x4000 + 3672, 4), 0x44ca8000) # (1620.0f) + self.assertEqual([int(l.split()[1]) for l in outs[0].splitlines() if l[0] == "p"], [2, 3, 4]) if __name__ == "__main__": diff --git a/third_party/a64-kit/lift/a64_lift.py b/third_party/a64-kit/lift/a64_lift.py index 85a34cd4..3b448355 100644 --- a/third_party/a64-kit/lift/a64_lift.py +++ b/third_party/a64-kit/lift/a64_lift.py @@ -211,20 +211,27 @@ def stack_stores(text): return SP_STORE.sub(r"recomp_\1_sp(c,\2c->x[31]", text) -# Page marks per group (A2). In a straight-line run of a function's C (no label, branch, call, return, poll, barrier, -# atomic, exclusive, SVC or trap) the marked stores through the same base register plus a constant keep their data -# stores but share one A64_MARK(base + lo, hi - lo) for their byte span [lo, hi) (when it is at most 4 KB), placed -# after the group's last store and before the first later line that writes the base register or ends the run. The -# pages marked are the same: every store of the group lies in [base+lo, base+hi), at most two pages, which the stores -# at lo and ending at hi mark themselves. Each data store is still followed, in program order and before any guest -# synchronisation, by a release store of a mark of its page, so a collection that clears the mark sees the data -# (rt/a64.h A64_MARK). 43% fewer mark stores in OoT's lifted C. A64_MARK_EACH=1 at lift time: a mark per store. +# Page marks per group (A2). In a straight-line run of a function's C the marked stores through the same base register +# plus a constant keep their data stores but share one A64_MARK(base + lo, hi - lo) for their byte span [lo, hi) (when it +# is at most 4 KB), placed after the group's last store and before the first later line that writes the base register or +# ends the run. The pages marked are the same: every store of the group lies in [base+lo, base+hi), at most two pages, +# which the stores at lo and ending at hi mark themselves. Each data store is still followed, in program order and before +# any guest synchronisation, by a release store of a mark of its page, so a collection that clears the mark sees the data +# (rt/a64.h A64_MARK). +# It runs on the finished C files, last (nx2/lift.py: after nx2/oot_patches.py and every other edit of the lifted text, +# which match the lift's exact text), so it is not part of emit_function, and the kit's own write() leaves it out: a +# lift without nx2 keeps a mark per store unless it calls group_marks itself. A64_MARK_EACH=1 at lift time: a mark per +# store. Running it again changes nothing (its own marks are recognised, and are not run ends). +# A run ends at anything not shown to be straight-line code without synchronisation: a label, goto, return, a line +# whose braces do not balance (a multi-line block opens or closes: a mark never moves into or out of one), STLR / LDAR +# in suyu's spelling (a plain recomp_storeN(c,c->x[N],...) / recomp_loadN(c,c->x[N])), the register file as a whole, +# a local register (A64_LOCAL_REGS), and every call-like token but MG_PLAIN's: calls (f_*, TAILCALL, hos_* from the +# patches), polls, atomics, barriers, exclusives, SVCs, traps and any helper this list does not know. MG_STORE = re.compile(r"recomp_(store8|store16|store32|store64|stp32|stp64)\(c,") MG_SIZE = {"store8": 1, "store16": 2, "store32": 4, "store64": 8, "stp32": 8, "stp64": 16} # address forms (the text right after "recomp_X(c,") MG_ADDR = [ (re.compile(r"c->x\[(\d+)\] \+ (-?\d+),"), "x"), # c->x[N] + IMM, - (re.compile(r"c->x\[(\d+)\],"), "x0"), # c->x[N], (re.compile(r"\(c->x\[(\d+)\]\+(\d+)ULL\),"), "x"), # (c->x[N]+IMMULL), (re.compile(r"\(_b\+_o\)\+(\d+),"), "bo+"), # (_b+_o)+IMM, (re.compile(r"\(_b\+_o\),"), "bo"), # (_b+_o), @@ -234,17 +241,53 @@ def stack_stores(text): ] MG_BDEF = re.compile(r"uint64_t _b=c->x\[(\d+)\]; int64_t _o=(-?\d+);") MG_ADEF = re.compile(r"uint64_t _a=c->x\[(\d+)\];") -# a line that ends the straight-line run: control flow, calls, polls, any synchronisation or runtime call -MG_BOUNDARY = re.compile(r"^\s*L_[0-9a-f]+:|\bgoto\b|\breturn\b|TAILCALL|\bswitch\b|\bf_[0-9a-f]{8}\(|\bA64_[A-Z_]+\(|" - r"__atomic|__sync|recomp_barrier|recomp_sj_|recomp_(ldx|stx|ldax|stlx|ldxp|stxp|clrex|svc|cntpct|" - r"unhandled|ic_)|a64_(svc|trap|icall|ijump|poll|ret_mismatch)|^\s*\}\s*$") +# (call-like tokens that are plain: keywords, guest loads and stores without ordering (the stores are what is grouped), +# and helpers that only compute or move values between locals, registers and vector registers) +MG_PLAIN = re.compile(r"(?:if|for|while|sizeof|memcpy|memset|UINT64_C|INT64_C|fabsf|fabs|__builtin_add_overflow|" + r"recomp_(?:load|ldp|store|stp)(?:8|16|32|64)(?:_sp)?|" + r"recomp_(?:fixed_to_fp|fp_to_fixed|fp_to_int|fp_estimate|umulh|smulh|aes_sbox)|" + r"a64_(?:vset64|vset128|vset_lo|vst16|fcmp32|fcmp64|vfcmp)|a64_v?fpx_\w+)\Z") +MG_CALL = re.compile(r"\b([A-Za-z_]\w*)\s*\(") +_mg_plain = {} +# (the other run ends: a label, goto, return, STLR and LDAR, the register file as a whole) +MG_END = re.compile(r"^\s*L_[0-9a-f]+:|\bgoto\b|\breturn\b|recomp_(?:store|stp)\d+\(c,c->x\[\d+\],|" + r"recomp_load\d+\(c,c->x\[\d+\]\)|c->x(?!\[)") +MG_LOCAL = re.compile(r"\bx\d+_ = c->x\[") # (localize_regs's locals: such a function is left as it is) +MG_BRACE = re.compile(r"[{}]") # a line whose stores may not all run, or run more than once: its stores keep their own marks MG_NOT_STRAIGHT = re.compile(r"\bif\b|\?|\bfor\b|\bwhile\b|\bswitch\b|\bcase\b") -# (any write of a guest register: =, compound assignments, ++/--, or its address taken (LDP's outputs)) -MG_REG_WRITE = re.compile(r"c->x\[(\d+)\]\s*(?:[-+*/%&|^]|<<|>>)?=(?!=)|&c->x\[(\d+)\]|(?:\+\+|--)\s*c->x\[(\d+)\]|" - r"c->x\[(\d+)\]\s*(?:\+\+|--)") -# (the register file as a whole: a line touching it so ends every group) -MG_REG_FILE = re.compile(r"c->x(?!\[)") +MG_NOT_STRAIGHT_WORDS = ("if", "?", "for", "while", "switch", "case") +MG_MARK = " A64_MARK((c->x[%d] + (uint64_t)(int64_t)(%d)), %d);" +MG_OWN_MARK = re.compile(r" A64_MARK\(\(c->x\[\d+\] \+ \(uint64_t\)\(int64_t\)\(-?\d+\)\), \d+\);\Z") +MG_HEAD = re.compile(r"\nvoid f_[0-9a-f]+\(") # (a definition or declaration: its line begins after the newline) + + +def mg_balanced(line): + """the line's braces close what it opens, in order (a block on one line, or none)""" + if "{" not in line and "}" not in line: + return True + d = 0 + for b in MG_BRACE.findall(line): + d += 1 if b == "{" else -1 + if d < 0: + return False + return d == 0 + + +def mg_ends_run(line): + """the line ends a straight-line run (each test only on a line that has its text: this runs on most lines of the lift)""" + if ("{" in line or "}" in line) and not mg_balanced(line): + return True + if "(" in line: + for t in MG_CALL.findall(line): + p = _mg_plain.get(t) + if p is None: + p = _mg_plain[t] = MG_PLAIN.match(t) is not None + if not p: + return True + if "goto" in line or "return" in line or "L_" in line or "(c,c->x[" in line or line.count("c->x") != line.count("c->x["): + return MG_END.search(line) is not None + return False def mg_parse_store(line, m): @@ -257,28 +300,23 @@ def mg_parse_store(line, m): continue if kind == "x": return int(a.group(1)), int(a.group(2)), size - if kind == "x0": - return int(a.group(1)), 0, size + d = None # (the nearest definition before the store) + for d in (MG_BDEF if kind in ("bo", "bo+", "b") else MG_ADEF).finditer(line, 0, m.start()): + pass + if d is None: + return None if kind in ("bo", "bo+", "b"): - d = MG_BDEF.search(line[:m.start()]) - if not d: - return None off = 0 if kind == "b" else int(d.group(2)) + (int(a.group(1)) if kind == "bo+" else 0) return int(d.group(1)), off, size - if kind in ("a", "a+"): - d = MG_ADEF.search(line[:m.start()]) - if not d: - return None - return int(d.group(1)), (int(a.group(1)) if kind == "a+" else 0), size + return int(d.group(1)), (int(a.group(1)) if kind == "a+" else 0), size return None -def mark_groups(text, stats=None): - lines = text.split("\n") +def mark_groups_lines(lines, st): + """the lines of a function (or of any C text) with the page marks of each group shared""" inserts = {} # line index -> mark lines to put before it unmark = {} # line index -> set of match start offsets to turn into _sp groups = {} # base register -> [(line index, match start, off, size)] - st = stats if stats is not None else {} def flush(reg, at): g = groups.pop(reg, None) @@ -286,31 +324,33 @@ def flush(reg, at): return lo = min(o for _, _, o, _ in g) hi = max(o + n for _, _, o, n in g) - st["stores"] = st.get("stores", 0) + len(g) + st["stores"] += len(g) if len(g) < 2 or hi - lo > 4096: return for li, ms, _, _ in g: unmark.setdefault(li, set()).add(ms) - inserts.setdefault(at, []).append(" A64_MARK((c->x[%d] + (uint64_t)(int64_t)(%d)), %d);" % (reg, lo, hi - lo)) - st["groups"] = st.get("groups", 0) + 1 - st["grouped_stores"] = st.get("grouped_stores", 0) + len(g) + inserts.setdefault(at, []).append(MG_MARK % (reg, lo, hi - lo)) + st["groups"] += 1 + st["grouped_stores"] += len(g) for i, line in enumerate(lines): - if line.startswith("void f_") or MG_BOUNDARY.search(line) or MG_REG_FILE.search(line): + if not groups and "recomp_st" not in line: + continue # (nothing to end or to add to) + if line.startswith(" A64_MARK((") and MG_OWN_MARK.match(line): + continue + if mg_ends_run(line): for r in list(groups): flush(r, i) continue - writes = {int(next(g for g in m if g)) for m in MG_REG_WRITE.findall(line)} - stores = list(MG_STORE.finditer(line)) - if stores and not writes and not MG_NOT_STRAIGHT.search(line): - parsed = [(m, mg_parse_store(line, m)) for m in stores] - for m, p in parsed: - if p is None: - continue # (an address this pass does not follow: the store keeps its own marks) - reg, off, size = p - if reg == 31: - continue - groups.setdefault(reg, []).append((i, m.start(), off, size)) + writes = {int(a or b) for a, b in REG_WRITE.findall(line)} if ( + "=" in line or "&c->x" in line or "++" in line or "--" in line) else () + stores = list(MG_STORE.finditer(line)) if "recomp_st" in line else () + if stores and not writes and not (any(w in line for w in MG_NOT_STRAIGHT_WORDS) and MG_NOT_STRAIGHT.search(line)): + for m in stores: + p = mg_parse_store(line, m) + if p is None or p[0] == 31: + continue # (an address this pass does not follow, or SP: the store keeps what it has) + groups.setdefault(p[0], []).append((i, m.start(), p[1], p[2])) continue for r in writes: # (the base changes here: its group's mark goes before this line) if r in groups: @@ -318,7 +358,7 @@ def flush(reg, at): for r in list(groups): flush(r, len(lines)) if not unmark: - return text + return lines out = [] for i, line in enumerate(lines): out.extend(inserts.get(i, [])) @@ -328,7 +368,42 @@ def flush(reg, at): line = line[:m.start()] + "recomp_%s_sp(c," % m.group(1) + line[m.end():] out.append(line) out.extend(inserts.get(len(lines), [])) - return "\n".join(out) + return out + + +def mark_groups(text, stats=None): + """text (a C file of lifted functions, or one function) with the page marks of each group shared. stats: counts of + the stores looked at, the groups and their stores""" + st = stats if stats is not None else Counter() + for k in ("stores", "groups", "grouped_stores"): + st[k] = st.get(k, 0) + starts = [0] + [m.start() + 1 for m in MG_HEAD.finditer(text)] + starts.append(len(text)) + out = [] + for a, b in zip(starts, starts[1:]): + part = text[a:b] + if part.count("recomp_st") < 2 or len(MG_STORE.findall(part)) < 2 or ( + "_ = c->x[" in part and MG_LOCAL.search(part)): # (no group possible: most functions) + out.append(part) + continue + out.append("\n".join(mark_groups_lines(part.split("\n"), st))) + return "".join(out) + + +def group_marks(paths, stats=None): + """mark_groups over C files, in place (nx2/lift.py: the lift's last step). A64_MARK_EACH=1: none. The files changed""" + if os.environ.get("A64_MARK_EACH"): + return 0 + changed = 0 + for path in paths: + with open(path, encoding="utf-8", newline="") as f: + text = f.read() + new = mark_groups(text, stats) + if new != text: + with open(path, "w", encoding="utf-8", newline="") as f: + f.write(new) + changed += 1 + return changed # Dead flags. Each flag-setting instruction stores all of N, Z, C and V in the context, where the compiler must keep @@ -379,7 +454,8 @@ def drop_flag_stores(text, dead): SYNC_TAIL = "\x01T%d\x01" # (control leaving after the instruction: what it wrote too) SYNC_MARK = re.compile("\x01([CBRALT])(\\d+)\x01") REG_REF = re.compile(r"c->x\[(\d+)\]") -REG_WRITE = re.compile(r"(?:&c->x\[(\d+)\])|(?:c->x\[(\d+)\]\s*(?:[-+*/%&|^]|<<|>>)?=(?!=))") +# (a write of a guest register: its address taken (LDP's outputs), ++ / --, = and the compound assignments) +REG_WRITE = re.compile(r"(?:(?:&|\+\+\s*|--\s*)c->x\[(\d+)\])|(?:c->x\[(\d+)\]\s*(?:(?:[-+*/%&|^]|<<|>>)?=(?!=)|\+\+|--))") SCRATCH = 0xFE00 # x9-x15: dead at calls and returns _reg_writes = {} @@ -1654,8 +1730,6 @@ def emit_function(self, e, texts): text = "\n".join(out) if frame: text = frame_stores(text) - if not os.environ.get("A64_MARK_EACH"): - text = mark_groups(text) return localize_regs(text, local, dirty) + "\n", self.refs def write(self, out_dir, chunk_bytes=CHUNK_BYTES): @@ -1769,3 +1843,6 @@ def report(self): if self.stats["split_entries"]: print("split: %d extra entries in long functions; largest function now %d instructions" % ( self.stats["split_entries"], max(len(b) for b, _ in self.funcs.values()))) + if self.stats["stores"]: # (group_marks, when the lift ran it before this report) + print("page marks: %d groups share one mark for %d of %d stores through a base register" % ( + self.stats["groups"], self.stats["grouped_stores"], self.stats["stores"])) From 816a4728ca4f330183af6fefebefcf67a7c4625e Mon Sep 17 00:00:00 2001 From: klimovich008 Date: Mon, 9 Nov 2026 19:12:24 +0100 Subject: [PATCH 3/5] fix(lift): clear an earlier lift's hot pieces before the patches; count every made and raw output-size place Review of #912 (review229). - chunks.write removes an earlier lift's code_hot_*.c and code_hot.c.whole whether or not this lift writes a code_hot.c. With a hot list, the old pieces (patched already) stayed until make_game's split_hot, after the lift: nx2/oot_patches.py and the groups read them beside the new code_hot.c, an output-size place counted twice, and the required check failed a normal rebuild. split_hot makes this lift's pieces after it. - apply_output_size counts each made place and each raw place outside a made one (a new text can hold its old one), per file. A file with a place made twice, or made and raw, was counted once and accepted; now it is not once, and the supported build's lift stops. A file made once is accepted and left as it is. - tests/test_mark_groups.py: a rebuild over the first lift's patched hot pieces (chunks.write, edit_c, split_hot, twice) finds each place once; made once accepted; made twice, made and raw ([3], and [4] whose new text holds its old one), and raw in another file rejected. Co-Authored-By: Claude Opus 5.5 --- nx2/chunks.py | 15 ++++----- nx2/oot_patches.py | 10 +++--- tests/test_mark_groups.py | 64 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 11 deletions(-) diff --git a/nx2/chunks.py b/nx2/chunks.py index 14b0d6ad..86811bf2 100644 --- a/nx2/chunks.py +++ b/nx2/chunks.py @@ -200,13 +200,14 @@ def emit_function(e, texts): while os.path.exists(os.path.join(gen_dir, "code_%03d.c" % n)): # (fewer chunks than the last lift) os.remove(os.path.join(gen_dir, "code_%03d.c" % n)) n += 1 - # (a lift with no code_hot.c: no hot list, A64_HOT_CHUNK unset, or none of the listed functions in it. write() removes a - # code_hot.c, but split_hot.py has renamed it code_hot.c.whole and left code_hot_*.c, which build.py compiled again: the - # earlier lift's hot functions, each defined in a chunk too, linked twice) - if not any(os.path.basename(f) == "code_hot.c" for f in files): - for f in os.listdir(gen_dir): - if f == "code_hot.c.whole" or HOT_PIECE.fullmatch(f): - os.remove(os.path.join(gen_dir, f)) + # (an earlier lift's hot pieces go now, with or without a new code_hot.c, before the patches and the page-mark groups read + # code_*.c: split_hot.py makes this lift's after it. A lift with no code_hot.c (no hot list, A64_HOT_CHUNK unset, or none of the + # listed functions in it): split_hot.py had renamed the old one code_hot.c.whole and left code_hot_*.c, which build.py compiled + # again, the earlier lift's hot functions, each defined in a chunk too, linked twice. A lift with one: the old pieces, patched + # already, and the new whole file each held the output-size places, and nx2/oot_patches.py's count of them failed the lift) + for f in os.listdir(gen_dir): + if f == "code_hot.c.whole" or HOT_PIECE.fullmatch(f): + os.remove(os.path.join(gen_dir, f)) files += split_big(gen_dir, files) save(img_dir, starts or b.recorded) print("chunks: %d, boundaries %s" % (len(files), "kept" if starts else "from sizes (saved)")) diff --git a/nx2/oot_patches.py b/nx2/oot_patches.py index d87bce63..3a6f9cd0 100644 --- a/nx2/oot_patches.py +++ b/nx2/oot_patches.py @@ -281,10 +281,12 @@ def apply_output_size(gen, required=False): s = f.read() hit = False for i, (old, new) in enumerate(OUT_EDITS): - if new in s: # (applied already: a second run. Each new text holds its old one, which would be patched again) - found[i] += 1 - elif old in s: - found[i] += s.count(old) + # (every place: each made one, and each old text outside a made one; a new text can hold its old one. A file with + # both is counted for both, and its old ones are left: a second made place or a stray raw one is not once) + made = s.count(new) + raw = (s.replace(new, "\0") if made else s).count(old) + found[i] += made + raw + if raw and not made: s = s.replace(old, new) hit = True if hit: diff --git a/tests/test_mark_groups.py b/tests/test_mark_groups.py index 6e4ae46f..4ae6e44b 100644 --- a/tests/test_mark_groups.py +++ b/tests/test_mark_groups.py @@ -342,6 +342,70 @@ def test_patches_made_before_are_found_once(self): quiet(oot_patches.apply_output_size, d, required=True) self.assertEqual([read(f) for f in sorted(glob.glob(os.path.join(d, "code_*.c")))], once) + def test_each_place_counts(self): + """made places and raw ones are both counted, in one file or apart: made once is accepted; made twice, or made and raw, + is not (also for [4], whose new text holds its old one)""" + made = '#include "a64.h"\n' + oot_patches.OUT_DECL + "\n".join(new for _, new in oot_patches.OUT_EDITS) + "\n" + cases = [("made once", [made], None), + ("[3] made twice", [made + oot_patches.OUT_EDITS[3][1]], 3), + ("[3] made and raw", [made + oot_patches.OUT_EDITS[3][0]], 3), + ("[4] made twice", [made + oot_patches.OUT_EDITS[4][1]], 4), + ("[4] made and raw", [made + oot_patches.OUT_EDITS[4][0]], 4), + ("[0] raw in another file", [made, '#include "a64.h"\n' + oot_patches.OUT_EDITS[0][0] + "\n"], 0)] + for what, files, bad in cases: + with tempfile.TemporaryDirectory() as d: + for n, text in enumerate(files): + with open(os.path.join(d, "code_%03d.c" % n), "w", newline="") as f: + f.write(text) + before = [read(os.path.join(d, "code_%03d.c" % n)) for n in range(len(files))] + if bad is None: + quiet(oot_patches.apply_output_size, d, required=True) + self.assertEqual([read(os.path.join(d, "code_%03d.c" % n)) for n in range(len(files))], before, what) + continue + with self.assertRaises(SystemExit, msg=what) as e: + quiet(oot_patches.apply_output_size, d, required=True) + self.assertIn("OUT_EDITS [%d]" % bad, str(e.exception), what) + + def test_a_rebuild_over_an_earlier_lifts_hot_pieces(self): + """make_game's lift twice with a hot list (chunks.write, edit_c, then split_hot): the second lift's edits run while the + first one's patched code_hot_*.c are still there, and must find each place once in what this lift wrote""" + import chunks + import split_hot + bodies = {0x81222ce0: GEN_000.split("\n", 2)[2], 0x80000000: GEN_001.split("\n", 2)[2]} + bodies.update({0x80100000 + 0x40 * i: " c->x[1] = 1;\n return;\n}\n" for i in range(12)}) + + class Lift: + write = a64_lift.Lifter.write + _write_table = a64_lift.Lifter._write_table + fname = a64_lift.Lifter.fname + + def __init__(self): + self.funcs = {a: ({}, None) for a in bodies} + self.hooks = {} + self.stats, self.unhandled, self.ijump = Counter(), Counter(), Counter() + + def emit_function(self, e, texts): + return "void f_%08x(Ctx *__restrict A64_NOESC c) {\n" % e + bodies[e], set() + with tempfile.TemporaryDirectory() as work: + gen, img = os.path.join(work, "gen"), os.path.join(work, "image") + os.makedirs(img) + hot = os.path.join(work, "hot_funcs.txt") + with open(hot, "w") as f: + f.write("f_81222ce0\nf_80000000\nf_80100040\n") + env = {"A64_HOT_CHUNK": "1", "A64_HOT_FUNCS": hot} + with mock.patch.dict(os.environ, env), mock.patch.object(a64_lift.a64emit, "translate", lambda keys: ([], [])): + for build in (1, 2): + st = Counter() + quiet(chunks.write, Lift(), gen, img, rechunk=True, jobs=1) + quiet(lift_driver().edit_c, gen, st, True) # (SystemExit: a place counted twice) + quiet(split_hot.split, gen, 2) + names = sorted(glob.glob(os.path.join(gen, "code_*.c"))) + self.assertTrue(any("code_hot_" in n for n in names), build) + text = ungroup("".join(read(n) for n in names)) + for _, new in oot_patches.OUT_EDITS: + self.assertEqual(text.count(new), 1, (build, new)) + self.assertEqual(st["groups"], 5, build) + def test_groups_before_the_patches_fail_the_lift(self): """the order before this change (sabotage control): the groups break OUT_EDITS[0] and [3], and the supported build's lift stops instead of ignoring the Output resolution""" From b7ca9f4be722a597a7771e49def79ae44b0c9bde Mon Sep 17 00:00:00 2001 From: klimovich008 Date: Mon, 9 Nov 2026 20:18:15 +0100 Subject: [PATCH 4/5] test(lift): the mark-group tests run without lz4 installed nx2/lift.py imports nso.py, which imports lz4 for NSO decompression; the Python source checks have no lz4, so the three tests through lift.edit_c failed at import. They read no NSO: lz4 is stood in for like capstone. (test_output_resolution's hos_rt_pool_size assertion fails on main too.) Co-Authored-By: Claude Opus 5.5 --- tests/test_mark_groups.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/test_mark_groups.py b/tests/test_mark_groups.py index 4ae6e44b..a2608487 100644 --- a/tests/test_mark_groups.py +++ b/tests/test_mark_groups.py @@ -303,12 +303,15 @@ def ungroup(text): def lift_driver(): - """nx2/lift.py as a module (capstone, which only its coverage count uses, stood in for when it is not installed)""" - try: - import capstone # noqa: F401 - except ImportError: - import types - sys.modules["capstone"] = types.ModuleType("capstone") + """nx2/lift.py as a module. Stood in for when not installed (CI has neither): capstone, which only its coverage count uses, and + lz4, which nso.py imports for NSO decompression; these tests read no NSO""" + import types + for name in ("capstone", "lz4", "lz4.block"): + try: + __import__(name) + except ImportError: + sys.modules[name] = types.ModuleType(name) + sys.modules["lz4"].block = sys.modules["lz4.block"] import lift return lift From 1c5258f5d82c07edb673c6f47d041e958d68bb1c Mon Sep 17 00:00:00 2001 From: klimovich008 Date: Mon, 9 Nov 2026 20:44:41 +0100 Subject: [PATCH 5/5] fix(lift): edit_c in chunks.py, so its tests import without the lift's tools The order tests imported nx2/lift.py for edit_c, and lift.py imports the whole lifter at load time (capstone, nso.py's lz4, adr_entries' numpy): the CI runners have neither lz4 nor numpy, so the three tests of the patches-then-groups order errored out. edit_c (the patches, then the groups, over the written code_*.c) now lives beside chunks.write, which already writes those files and imports a64_lift; lift.py calls chunks.edit_c. The lz4/capstone stand-ins in the test go. Co-Authored-By: Claude Opus 5.5 --- nx2/chunks.py | 10 ++++++++++ nx2/lift.py | 9 +-------- tests/test_mark_groups.py | 30 ++++++++++-------------------- 3 files changed, 21 insertions(+), 28 deletions(-) diff --git a/nx2/chunks.py b/nx2/chunks.py index 86811bf2..71854506 100644 --- a/nx2/chunks.py +++ b/nx2/chunks.py @@ -16,6 +16,7 @@ """ import bisect import collections +import glob import multiprocessing import os import re @@ -292,3 +293,12 @@ def split_big(gen_dir, files, budget=None): os.remove(os.path.join(gen_dir, f)) write_costs(gen_dir, costs) return made + + +def edit_c(gen, stats, supported, whole=True): + """(nx2/lift.py, after the lift writes its C; here, beside the writing, so it imports without the lift's tools) The edits of the written C, in this order: the game's patches (nx2/oot_patches.py, which find the lift's exact text; the + supported build's whole standalone lift fails when an output-size edit finds no place), then, last, the page marks of each + group of stores shared (a64_lift.group_marks), which changes that text. stats: the lift's, for its report""" + import oot_patches + oot_patches.apply(gen, protect_hearts=supported, required=supported and whole) + a64_lift.group_marks(sorted(glob.glob(os.path.join(gen, "code_*.c"))), stats) diff --git a/nx2/lift.py b/nx2/lift.py index e3b65e7f..61f62057 100644 --- a/nx2/lift.py +++ b/nx2/lift.py @@ -111,13 +111,6 @@ def coverage(L, img, out_dir): return total -def edit_c(gen, stats, supported, whole=True): - """The edits of the written C, in this order: the game's patches (nx2/oot_patches.py, which find the lift's exact text; the - supported build's whole standalone lift fails when an output-size edit finds no place), then, last, the page marks of each - group of stores shared (a64_lift.group_marks), which changes that text. stats: the lift's, for its report""" - import oot_patches - oot_patches.apply(gen, protect_hearts=supported, required=supported and whole) - a64_lift.group_marks(sorted(glob.glob(os.path.join(gen, "code_*.c"))), stats) def main(): @@ -190,7 +183,7 @@ def main(): counters=[] if a.no_polls else [poll.stats]) import oot_patches main_build = next((m["build_id"] for m in meta["modules"] if m["name"] == "main"), "") - edit_c(gen, L.stats, a.standalone and main_build == oot_patches.LIFE_BUILD_ID, whole=not a.max_func) + chunks.edit_c(gen, L.stats, a.standalone and main_build == oot_patches.LIFE_BUILD_ID, whole=not a.max_func) a64_rtgen.write(gen) if a.standalone: start, exit_ = standalone.write_image_c(gen, img, meta, a.exefs) diff --git a/tests/test_mark_groups.py b/tests/test_mark_groups.py index a2608487..69f31e8b 100644 --- a/tests/test_mark_groups.py +++ b/tests/test_mark_groups.py @@ -24,6 +24,7 @@ sys.path[:0] = [os.path.join(ROOT, "third_party", "a64-kit", d) for d in ("lift", "tools")] + [os.path.join(ROOT, "nx2")] import a64_lift # noqa: E402 import oot_patches # noqa: E402 +import chunks # noqa: E402 HEAD = "void f_80000000(Ctx *__restrict A64_NOESC c) {" @@ -259,12 +260,13 @@ def test_the_lift_key_has_the_switch(self): self.assertNotEqual(make_game.lift_key(exefs, work, {}), key) def test_the_lift_groups_once_with_its_stats(self): - """emit_function leaves the marks as they are; nx2/lift.py's edit_c groups the written files, with the lift's stats, - before its report""" + """emit_function leaves the marks as they are; nx2/lift.py calls chunks.edit_c, which groups the written files once, with + the lift's stats, before its report""" self.assertNotIn("mark_groups", inspect.getsource(a64_lift.Lifter.emit_function)) src = read(os.path.join(ROOT, "nx2", "lift.py")) - self.assertEqual(src.count("group_marks("), 1) - call = src.find("edit_c(gen, L.stats, ") + self.assertEqual(src.count("group_marks(") + read(os.path.join(ROOT, "nx2", "chunks.py")).count("group_marks("), 1) + self.assertIn("group_marks(", inspect.getsource(chunks.edit_c)) + call = src.find("chunks.edit_c(gen, L.stats, ") self.assertGreater(call, 0) self.assertLess(call, src.find("L.report()")) @@ -302,18 +304,6 @@ def ungroup(text): return re.sub(r"recomp_(store8|store16|store32|store64|stp32|stp64)_sp\(c,", r"recomp_\1(c,", text) -def lift_driver(): - """nx2/lift.py as a module. Stood in for when not installed (CI has neither): capstone, which only its coverage count uses, and - lz4, which nso.py imports for NSO decompression; these tests read no NSO""" - import types - for name in ("capstone", "lz4", "lz4.block"): - try: - __import__(name) - except ImportError: - sys.modules[name] = types.ModuleType(name) - sys.modules["lz4"].block = sys.modules["lz4.block"] - import lift - return lift class OutputSizeOrderTests(unittest.TestCase): @@ -323,7 +313,7 @@ def test_the_patches_then_the_groups(self): with tempfile.TemporaryDirectory() as d: files = make_gen(d) st = Counter() - quiet(lift_driver().edit_c, d, st, True) + quiet(chunks.edit_c, d, st, True) text = "".join(read(f) for f in files) self.assertIn(oot_patches.OUT_DECL, text) for old, new in oot_patches.OUT_EDITS: # (each made once, and only groups changed it since) @@ -400,7 +390,7 @@ def emit_function(self, e, texts): for build in (1, 2): st = Counter() quiet(chunks.write, Lift(), gen, img, rechunk=True, jobs=1) - quiet(lift_driver().edit_c, gen, st, True) # (SystemExit: a place counted twice) + quiet(chunks.edit_c, gen, st, True) # (SystemExit: a place counted twice) quiet(split_hot.split, gen, 2) names = sorted(glob.glob(os.path.join(gen, "code_*.c"))) self.assertTrue(any("code_hot_" in n for n in names), build) @@ -439,10 +429,10 @@ def test_a_missing_place_fails_once_required(self): with contextlib.redirect_stdout(out): if fails: with self.assertRaises(SystemExit) as e: - lift_driver().edit_c(d, Counter(), supported, whole) + chunks.edit_c(d, Counter(), supported, whole) self.assertIn("OUT_EDITS [4]", str(e.exception)) else: - lift_driver().edit_c(d, Counter(), supported, whole) + chunks.edit_c(d, Counter(), supported, whole) self.assertIn("WARNING: the output size places were found [1, 1, 1, 1, 0, 1, 1, 1] times", out.getvalue()) @unittest.skipUnless(os.path.isfile(os.path.join(os.environ.get("EMSDK", r"C:\emsdk"), "upstream", "bin", "clang.exe"))