From 6de3bf1c614e2ae9ee2187b78a732280c53966a8 Mon Sep 17 00:00:00 2001 From: klimovich008 Date: Mon, 9 Nov 2026 02:24:11 +0100 Subject: [PATCH 1/3] perf(fpx): keep the hardware result for FMUL/FMLA S when the flags are already set The fma32-shaped wrappers (a64_fpx_mul32_fma32, a64_fpx_fma32_fma32) fell back to the exact soft-float body whenever the fast path declined: a subnormal operand under FZ, a tiny or infinite result. FPSR's cumulative flags are sticky, so when every flag the Arm op could raise is already set, one hardware FMA (or multiply) result is the Arm result: a subnormal operand under FZ flushed to a zero of its sign with IDC set, a tiny result with UFC set, an infinite result with OFC set, an exact cancellation. a64_fpx_sticky32 keeps those, in place of a64_fpx_fz_mul32 for these two shapes, and falls back otherwise. tests/test_fpx_sticky.py checks both wrappers as the generator emits them against recomp_fpe_fma32, value and FPSR: 116M checks, 0 mismatches. Co-Authored-By: Claude Opus 5.5 --- tests/fpx_sticky_test.c | 170 +++++++++++++++++++++++++++ tests/test_fpx_sticky.py | 150 +++++++++++++++++++++++ third_party/a64-kit/lift/a64_lift.py | 52 +++++++- 3 files changed, 371 insertions(+), 1 deletion(-) create mode 100644 tests/fpx_sticky_test.c create mode 100644 tests/test_fpx_sticky.py diff --git a/tests/fpx_sticky_test.c b/tests/fpx_sticky_test.c new file mode 100644 index 000000000..30c4ecffa --- /dev/null +++ b/tests/fpx_sticky_test.c @@ -0,0 +1,170 @@ +/* tests/test_fpx_sticky.py's driver: the generated FPX1 wrappers a64_fpx_mul32_fma32 (FMUL/FNMUL S) and + a64_fpx_fma32_fma32 (FMLA family S) against the exact body they fall back to (recomp_fpe_fma32), value and FPSR. + Built against the real a64.h with the a64_suyu.h / a64_suyu.c / a64_fpx.h the generators write. + + fpx_sticky_test random n random operand triples (both ops each) over random FPCR/FPSR states + fpx_sticky_test sweep every a with biased exponent <= 4 or >= 250 whose fraction is below 2^12, above + 2^23 - 2^12 or a multiple of stride, x a fixed set of b and addends, x FZ on/off, + x flags all set / IXC only + Exit 0: no mismatch and the sticky rescue decided cases of both ops. 1: a mismatch. 3: the CPU lacks FMA/AVX2. */ +#include "a64.h" +#include +#include +#include + +static uint64_t rs; +static uint64_t rnd(void) { rs ^= rs << 13; rs ^= rs >> 7; rs ^= rs << 17; return rs; } +static uint32_t r32(void) { return (uint32_t)(rnd() >> 16); } +static uint32_t rbelow(uint32_t n) { return (uint32_t)(((rnd() >> 32) * n) >> 32); } +static uint32_t mkf(uint32_t sign, int e, uint32_t frac) { return (sign << 31) | ((uint32_t)e << 23) | (frac & 0x7fffff); } +static float F(uint32_t x) { float f; memcpy(&f, &x, 4); return f; } +static uint32_t B(float f) { uint32_t x; memcpy(&x, &f, 4); return x; } + +/* ---- operand classes and states (fpstub's harness) ---- */ +static uint32_t sparse_frac(void) { /* fractions with few bits: exact products, ties, cancellations */ + switch (rbelow(4)) { + case 0: return 0; + case 1: return (uint32_t)1 << rbelow(23); + case 2: return r32() & 0x7ff000; + default: return r32() & 0x7fffff; + } +} +static uint32_t any_operand(void) { + const uint32_t s = r32() & 1; + switch (rbelow(16)) { + case 0: return s << 31; /* zero */ + case 1: return mkf(s, 0, r32() | 1); /* subnormal */ + case 2: return mkf(s, 0, rbelow(8) + 1); /* tiny subnormal */ + case 3: return mkf(s, 0, 0x7fffff - rbelow(8)); /* large subnormal */ + case 4: return mkf(s, 1 + rbelow(3), sparse_frac()); /* smallest normals */ + case 5: return mkf(s, 254 - rbelow(3), sparse_frac()); /* largest normals */ + case 6: return mkf(s, 255, 0); /* inf */ + case 7: return mkf(s, 255, (r32() | 1) & 0x7fffff); /* NaN, quiet or signalling */ + case 8: return mkf(s, 255, 0x400000 | (r32() & 0x3fffff)); /* quiet NaN */ + case 9: case 10: return mkf(s, 120 + rbelow(15), sparse_frac()); /* near 1 */ + default: return mkf(s, 1 + rbelow(254), r32()); /* any normal */ + } +} +static void tiny_pair(uint32_t *a, uint32_t *b) { /* a product near the bottom of the normal range, or the overflow edge */ + const int target = (rbelow(8) == 0) ? 255 - (int)rbelow(3) : (int)rbelow(60) - 30; /* biased product exponent */ + int ea = 1 + (int)rbelow(253); + int eb = target - ea + 127; + if (eb < 1 || eb > 254) { eb = 1 + (int)rbelow(254); ea = target - eb + 127; } + if (ea < 1) ea = rbelow(4) ? 1 : 0; + if (ea > 254) ea = 254; + *a = mkf(r32() & 1, ea, rbelow(2) ? sparse_frac() : r32()); + *b = mkf(r32() & 1, eb, rbelow(2) ? sparse_frac() : r32()); +} +static uint32_t addend_for(uint32_t a, uint32_t b) { + const double p = (double)F(a) * (double)F(b); + switch (rbelow(10)) { + case 0: return any_operand(); + case 1: return B((float)-p); /* near (often exact) cancellation */ + case 2: return B((float)-p) + (uint32_t)((int)rbelow(9) - 4); /* a few ulps off */ + case 3: return r32() & 0x80000000u; /* a zero */ + case 4: return B((float)p) ^ (rbelow(2) ? 0x80000000u : 0); + case 5: return mkf(r32() & 1, 0, r32()); /* subnormal */ + case 6: return mkf(r32() & 1, 1 + rbelow(4), r32()); /* near the smallest normal */ + default: return mkf(r32() & 1, 1 + rbelow(254), r32()); + } +} +static void gen_state(uint64_t *fpcr, uint64_t *fpsr) { + uint64_t cr = 0, sr = 0; + if (rbelow(2)) cr |= 1u << 24; /* FZ */ + if (rbelow(10) == 0) cr |= 1u << 25; /* DN (gate closed) */ + if (rbelow(10) == 0) cr |= (uint64_t)rbelow(4) << 22; /* RMode */ + if (rbelow(40) == 0) cr |= r32() & 0x07ff9f00u; /* anything else */ + if (rbelow(200) == 0) cr |= RECOMP_FPX_INHIBIT; + static const uint32_t bits[6] = {1, 2, 4, 8, 16, 128}; /* IOC DZC OFC UFC IXC IDC */ + for (int k = 0; k < 6; k++) + if (k == 4 ? rbelow(10) != 0 : rbelow(3) != 0) sr |= bits[k]; /* IXC 90%, the others 67% */ + if (rbelow(8) == 0) sr |= 1u << 27; /* QC */ + *fpcr = cr; + *fpsr = sr; +} + +/* ---- one check: both wrappers against the exact body ---- */ +static uint64_t checks, mismatches, rescued[2]; +static int report_left = 20; +static void check(uint64_t cr, uint64_t sr, uint32_t a, uint32_t b, uint32_t z, int mul) { + Ctx ref, got; + memset(&ref, 0, sizeof ref); + ref.fpcr = cr; ref.fpsr = sr; + got = ref; + const uint64_t r = recomp_fpe_fma32(&ref, a, b, z); + const uint64_t v = mul ? a64_fpx_mul32_fma32(&got, a, b, z) : a64_fpx_fma32_fma32(&got, a, b, z); + checks++; + { Ctx k = got; uint64_t t; k.fpsr = sr; /* (did the sticky rescue decide this one) */ + rescued[mul] += RECOMP_FPX_OPEN(&k) && !(mul ? recomp_fpx_mul32(a, b, RECOMP_FPX_FZ(&k), &t) + : recomp_fpx_fma32(a, b, z, RECOMP_FPX_FZ(&k), &t)) + && a64_fpx_sticky32(&k, a, b, z, mul, &t); } + if (v != r || got.fpsr != ref.fpsr) { + mismatches++; + if (report_left-- > 0) + printf("MISMATCH %s a=%08x b=%08x z=%08x fpcr=%llx fpsr=%llx: exact %08llx/%llx wrapper %08llx/%llx\n", + mul ? "FMUL" : "FMLA", a, b, z, (unsigned long long)cr, (unsigned long long)sr, (unsigned long long)r, + (unsigned long long)ref.fpsr, (unsigned long long)v, (unsigned long long)got.fpsr); + } +} +static void both(uint64_t cr, uint64_t sr, uint32_t a, uint32_t b, uint32_t z) { + check(cr, sr, a, b, (a ^ b) & 0x80000000u, 1); /* FMUL: the lifter passes a zero of the product's sign */ + check(cr, sr, a, b, z, 0); +} + +static void random_cases(uint64_t n, uint64_t seed) { + rs = seed * 0x9E3779B97F4A7C15ULL + 1; + for (uint64_t i = 0; i < n; i++) { + uint32_t a, b; uint64_t cr, sr; + if (rbelow(2)) tiny_pair(&a, &b); + else { a = any_operand(); b = any_operand(); } + const uint32_t z = addend_for(a, b); + gen_state(&cr, &sr); + both(cr, sr, a, b, z); + } +} + +static const uint32_t B_SET[] = { + 0x3f800000, /* 1 */ 0x3f000000, /* 0.5 */ 0x3fc00001, /* 1.5+ulp */ 0x00800000, /* min normal */ 0x7f7fffff, /* max */ + 0x80000001, /* -min subnormal */ 0x007fffff, /* max subnormal */ 0x7f800000, /* inf */ 0x00000000, /* +0 */ + 0x5f000000, /* 2^63 */ 0x3fffffff, /* ~2 */ 0x1f800000 /* 2^-64 */}; +static void sweep(uint32_t stride) { + for (unsigned bi = 0; bi < sizeof B_SET / sizeof B_SET[0]; bi++) { + const uint32_t b = B_SET[bi]; + for (uint32_t hi = 0; hi < 2 * 256; hi++) { /* sign and biased exponent of a */ + const uint32_t e = hi & 255; + if (!(e <= 4 || e >= 250)) continue; + for (uint32_t f = 0; f < (1u << 23); f++) { + if (f >= 4096 && f < (1u << 23) - 4096 && f % stride) continue; + const uint32_t a = (hi >> 8) << 31 | e << 23 | f; + const uint64_t cr = (f & 1) ? 0x01000000u : 0; /* FZ */ + const uint64_t sr = (f & 2) ? 0x9fu : 0x10u; /* every sticky bit set, or IXC only */ + uint32_t z; + switch ((f >> 2) & 3) { /* addend: cancellation, near the smallest normal, subnormal, zero */ + case 0: z = B(-(float)((double)F(a) * (double)F(b))); break; + case 1: z = 0x00800000u ^ (a & 0x80000000u); break; + case 2: z = 0x00000003u ^ (f << 31); break; + default: z = 0x80000000u; break; + } + both(cr, sr, a, b, z); + both(cr ^ 0x01000000u, sr ^ 0x8fu, a, b, z); /* the other FZ and flag state, same operands */ + } + } + } +} + +int main(int argc, char **argv) { + unsigned eax, ebx, ecx, edx; + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx) || !(ecx >> 12 & 1) || !__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx) || + !(ebx >> 5 & 1)) { + printf("this CPU has no FMA/AVX2 (x86-64-v3)\n"); + return 3; + } + if ((_mm_getcsr() & 0xffc0) != 0x1f80) { printf("MXCSR %x: not the FPX1 host contract\n", _mm_getcsr()); return 2; } + if (argc == 4 && !strcmp(argv[1], "random")) random_cases(strtoull(argv[2], 0, 0), strtoull(argv[3], 0, 0)); + else if (argc == 3 && !strcmp(argv[1], "sweep")) sweep((uint32_t)strtoul(argv[2], 0, 0)); + else { fprintf(stderr, "usage: random | sweep \n"); return 2; } + printf("%s: %llu checks, %llu mismatches, sticky rescue decided %llu FMUL / %llu FMLA\n", argv[1], + (unsigned long long)checks, (unsigned long long)mismatches, (unsigned long long)rescued[1], + (unsigned long long)rescued[0]); + return mismatches ? 1 : (rescued[0] && rescued[1]) ? 0 : 4; +} diff --git a/tests/test_fpx_sticky.py b/tests/test_fpx_sticky.py new file mode 100644 index 000000000..96858d470 --- /dev/null +++ b/tests/test_fpx_sticky.py @@ -0,0 +1,150 @@ +"""FPX1's sticky-flag rescue for single-precision FMUL/FNMUL/FMLA, game-free: bit-exact with the exact body. + +a64_lift.py's a64_fpx_mul32_fma32 and a64_fpx_fma32_fma32 (the cold wrappers lifted code calls when the inline fast path +cannot prove its result) try a64_fpx_sticky32 before the exact body recomp_fpe_fma32: when every FPSR flag the Arm op +could raise is already set, one hardware FMA (or multiply) is the Arm result. This builds what the real build builds, +a64_fpx.h from a64_lift.fpx_header() and a64_suyu.h / a64_suyu.c from a64_rtgen.py (through suyu's translator, +a64emit.cpp, built here at -O0 into a temporary folder), and compiles tests/fpx_sticky_test.c against the real rt/a64.h +at the build's FP flags (-march=x86-64-v3 -ffp-contract=off -O2). Each wrapper's value and FPSR must equal +recomp_fpe_fma32's for: + - 8M random operand triples (16M checks, seeded): zeros, subnormals, inf, NaN (quiet and signalling), the extreme + normals, products near the underflow and overflow edges, exact and near cancellations, under random FPCR (FZ, + DN, RMode, other bits, the kill switch) and FPSR states; + - an edge sweep (~100M checks): every a with biased exponent <= 4 or >= 250 whose fraction is below 2^12, above + 2^23 - 2^12 or a multiple of 97, against 12 fixed b and 4 addend kinds, each under FZ on and off with the flags + all set or IXC only. +Both runs must also see the rescue decide cases of both ops. About 15 s here (fpstub's harness, which this is cut down +from, saw no mismatch in 6.4e9 checks). + +It has teeth: with the helper's UFC test removed by hand (a64_fpx_sticky32's `if (!(s & 8u))` made `if (0)`, so a tiny +result is kept with UFC clear), both runs fail: 654k of the random checks and 5.4M of the sweep's mismatch (FPSR +without UFC). So does dropping its IDC test (a flushed subnormal operand with IDC clear): 406k and 7.1M. + + python -I -B tests/test_fpx_sticky.py [--cc clang] [--cxx clang++] + +Skipped (exit 0) on a host that is not x86-64 or has no C/C++ compiler (PATH, $CC/$CXX, or the build's own emsdk / +HOOKSHOT_MINGW toolchain), or on a CPU without x86-64-v3. +""" +import argparse +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +KIT = os.path.join(ROOT, "third_party", "a64-kit") +sys.path.insert(0, os.path.join(KIT, "lift")) +sys.path.insert(0, os.path.join(KIT, "tools")) +import toolchain # noqa: E402 + +RANDOM_TRIPLES, SEED, SWEEP_STRIDE = 8000000, 20261109, 97 +FLAGS = ["-march=x86-64-v3", "-ffp-contract=off", "-fno-strict-aliasing", "-O2"] # (nx2/build.py's lifted-code FP flags) + + +def skip(why): + print("fpx sticky: skipped (%s)" % why) + sys.exit(0) + + +def compiler(given, env, names, build_name): + """--cc/--cxx, else $CC/$CXX, else PATH, else the toolchain the build itself uses (emsdk, HOOKSHOT_MINGW)""" + for c in (given, os.environ.get(env)): + if c: + return shutil.which(c) or (c if os.path.isfile(c) else None) + for n in names: + if shutil.which(n): + return shutil.which(n) + c = toolchain.find(build_name, host=True) + return c if os.path.isfile(c) else None + + +def target_flags(cc, tmp): + """clang on Windows: the build's target and lld (as tools/toolchain.py, with its lld-link copy in tmp, not in tools/)""" + if os.name != "nt" or "clang" not in os.path.basename(cc).lower(): + return [] + flags = toolchain.native_target(host=True) + if toolchain.mingw(host=True): + return flags + toolchain.native_link(host=True) + here = os.path.dirname(cc) + link = shutil.which("lld-link", path=here) or shutil.which("lld-link") + if not link: + lld = shutil.which("lld", path=here) or shutil.which("lld") + if not lld: + return flags + link = os.path.join(tmp, "lld-link.exe") + if not os.path.exists(link): + shutil.copyfile(lld, link) + return flags + ["-fuse-ld=lld", "-B" + os.path.dirname(link), "-Wl,/nodefaultlib:oldnames"] + + +def run(cmd, timeout=600): + kw = {"creationflags": subprocess.BELOW_NORMAL_PRIORITY_CLASS | subprocess.CREATE_NO_WINDOW} if os.name == "nt" \ + else {"preexec_fn": lambda: os.nice(10)} + return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, errors="replace", + timeout=timeout, **kw) + + +def build(cmd, what): + r = run(cmd) + if r.returncode: + print(r.stdout) + raise SystemExit("fpx sticky: building %s failed" % what) + + +def generate(cxx, tmp, flags): + """a64_fpx.h, a64_suyu.h and a64_suyu.c, by the generators the lift runs (a64emit built into tmp)""" + dll = os.path.join(tmp, "a64emit.dll" if os.name == "nt" else "liba64emit.so") + build([cxx, "-std=c++20", "-O0", "-shared", "-Wno-everything", "-o", dll, os.path.join(KIT, "lift", "a64emit.cpp")] + + flags + ([] if os.name == "nt" else ["-fPIC"]), "a64emit") + import build_emit + build_emit.build = lambda force=False: dll # (not a64emit.dll beside the source) + import a64_lift + import a64_rtgen + import a64emit + try: + a64_rtgen.write(tmp) + finally: + if a64emit._lib is not None: # (unloaded, so the folder can go) + import _ctypes + (_ctypes.FreeLibrary if os.name == "nt" else _ctypes.dlclose)(a64emit._lib._handle) + a64emit._lib = None + with open(os.path.join(tmp, "a64_fpx.h"), "w", newline="\n") as f: + f.write(a64_lift.fpx_header({("mul32", "fma32", 2, 3), ("fma32", "fma32", 3, 3)})) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--cc") + p.add_argument("--cxx") + a = p.parse_args() + if platform.machine().lower() not in ("x86_64", "amd64"): + skip("not an x86-64 host: %s" % platform.machine()) + cc = compiler(a.cc, "CC", ("clang", "gcc", "cc"), "clang") + cxx = compiler(a.cxx, "CXX", ("clang++", "g++", "c++"), "clang++") + if not cc or not cxx: + skip("no C compiler" if not cc else "no C++ compiler") + t0 = time.time() + failed = [] + with tempfile.TemporaryDirectory(prefix="hookshot-fpx-sticky-") as tmp: + generate(cxx, tmp, target_flags(cxx, tmp)) + exe = os.path.join(tmp, "fpx_sticky_test.exe") + build([cc] + target_flags(cc, tmp) + FLAGS + ["-D_CRT_SECURE_NO_WARNINGS", "-I" + os.path.join(KIT, "rt"), + "-I" + tmp, os.path.join(ROOT, "tests", "fpx_sticky_test.c"), os.path.join(tmp, "a64_suyu.c"), "-o", exe], + "the driver") + for args in (["random", str(RANDOM_TRIPLES), str(SEED)], ["sweep", str(SWEEP_STRIDE)]): + r = run([exe] + args) + print(r.stdout.rstrip()) + if r.returncode == 3: + skip("this CPU is not x86-64-v3") + if r.returncode: + failed.append(args[0]) + if failed: + raise SystemExit("fpx sticky: FAIL (%s) in %.1f s" % (", ".join(failed), time.time() - t0)) + print("fpx sticky: pass in %.1f s" % (time.time() - t0)) + + +if __name__ == "__main__": + main() diff --git a/third_party/a64-kit/lift/a64_lift.py b/third_party/a64-kit/lift/a64_lift.py index b184db796..171110b02 100644 --- a/third_party/a64-kit/lift/a64_lift.py +++ b/third_party/a64-kit/lift/a64_lift.py @@ -630,6 +630,54 @@ def add(m): static inline int a64_fpx_fz_sub32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { (void)c; (void)a; (void)b; (void)v; return 0; } static inline int a64_fpx_fz_mul32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { (void)c; (void)a; (void)b; (void)v; return 0; } #endif +/* FMUL/FNMUL/FMLA-family S past the fast path (the wrappers' "sticky" rescue, in place of a64_fpx_fz_mul32): one hardware + fused multiply-add, kept when every FPSR bit the Arm op could raise is already set and the value is the Arm one. + FZ: a subnormal operand is flushed to a zero of its sign (IDC). NaN: never. Infinite: exact with an infinite operand, + else an overflow (OFC). A zero product or an exact cancellation (a*b == -z in binary64, where the product is exact): + exact. Otherwise a result at most the smallest normal may be tiny (UFC): FZ clear, the IEEE gradual-underflow result is + Arm's; FZ set, below the smallest normal the exact value is tiny and Arm returns a zero of its sign, and exactly the + smallest normal is ambiguous (falls back). Without a hardware FMA only the multiply is rescued. */ +#if RECOMP_FPX_HOST == 1 || (RECOMP_FPX_HOST == 2 && defined(__FMA__) && (defined(__GNUC__) || defined(__clang__))) +#define A64_FPX_FMA32 1 +#else +#define A64_FPX_FMA32 0 +#endif +static inline __attribute__((always_inline)) int a64_fpx_sticky32(const Ctx *c, uint64_t a, uint64_t b, uint64_t z, int mul, uint64_t *v) { +#if RECOMP_FPX_HOST + const uint64_t s = c->fpsr; + uint32_t x = (uint32_t)a, y = (uint32_t)b, w = (uint32_t)z, t; + if (RECOMP_FPX_FZ(c) && (RECOMP_FPX_SUBN32(x) || RECOMP_FPX_SUBN32(y) || (!mul && RECOMP_FPX_SUBN32(w)))) { + if (!(s & 0x80u)) return 0; + if (RECOMP_FPX_SUBN32(x)) x &= 0x80000000u; + if (RECOMP_FPX_SUBN32(y)) y &= 0x80000000u; + if (RECOMP_FPX_SUBN32(w)) w &= 0x80000000u; + } + if (mul) /* (w is the zero of the product's sign: x*y + w is x*y) */ + t = (uint32_t)recomp_fpx_fb(recomp_fpx_f(x) * recomp_fpx_f(y)); + else +#if A64_FPX_FMA32 + t = (uint32_t)recomp_fpx_fb(__builtin_fmaf(recomp_fpx_f(x), recomp_fpx_f(y), recomp_fpx_f(w))); +#else + return 0; +#endif + const uint32_t m = t & 0x7fffffffu; + if (m > 0x7f800000u) return 0; + if (m == 0x7f800000u) { + if (RECOMP_FPX_FIN32(x) && RECOMP_FPX_FIN32(y) && RECOMP_FPX_FIN32(w) && !(s & 4u)) return 0; + } else if (m <= 0x00800000u && !RECOMP_FPX_ZERO32(x) && !RECOMP_FPX_ZERO32(y)) { + if (!(s & 8u)) { /* UFC clear: only an exact cancellation (+0, no bit) is kept */ + if (mul || m != 0 || (double)recomp_fpx_f(x) * (double)recomp_fpx_f(y) != -(double)recomp_fpx_f(w)) return 0; + } else if (RECOMP_FPX_FZ(c)) { + if (m == 0x00800000u) return 0; + t &= 0x80000000u; + } + } + *v = t; + return 1; +#else + (void)c; (void)a; (void)b; (void)z; (void)mul; (void)v; return 0; +#endif +} """ @@ -673,7 +721,9 @@ def fpx_header(used): for fast, exact, nf, ne in sorted(used): params = ", ".join("uint64_t a%d" % i for i in range(ne)) fz = "" - if fast in ("add32", "sub32", "mul32"): + if (fast, exact) in (("mul32", "fma32"), ("fma32", "fma32")): + fz = "if (RECOMP_FPX_OPEN(c) && a64_fpx_sticky32(c, a0, a1, a2, %d, &v)) return v; " % (fast == "mul32") + elif fast in ("add32", "sub32", "mul32"): fz = "if (A64_FPX_FZ_OPEN(c) && a64_fpx_fz_%s(c, a0, a1, &v)) return v; " % fast out.append("A64_FPX_FN uint64_t a64_fpx_%s_%s(Ctx *c, %s) { uint64_t v; " "if (RECOMP_FPX_OPEN(c) && RECOMP_LIKELY(recomp_fpx_%s(%s, RECOMP_FPX_FZ(c), &v))) return v; " From 1ff4abe2efacfdd7de9ecb7ceb51f02f8cc57220 Mon Sep 17 00:00:00 2001 From: klimovich008 Date: Mon, 9 Nov 2026 18:26:09 +0100 Subject: [PATCH 2/3] test(fpx): count sticky rescues in the wrappers, cover x86-64-v2 and FIZ/AH/NEP, drop dead mul32 code Review of #917: - The generated wrappers call A64_FPX_STICKY_PROBE() (empty in the real build) when the sticky rescue keeps a result; the test counts there instead of calling a64_fpx_sticky32 itself, so a broken route (fpx_header's branch, the mul argument) fails it (exit 4). - The test builds and runs twice: -march=x86-64-v3 and x86-64-v2 (RECOMP_FPX_FMA64 off: FMUL rescued by a plain multiply, FMLA must decline every case). v2 runs without the AVX2 check. - gen_state sets FPCR bits 0-2 (alone and in the random mask); the sweep adds one state per a: partial flags, IXC clear, FIZ/AH/NEP. A rescue under a closed gate fails the test (the exact body models none of FIZ/AH/NEP, so values alone cannot tell). - A64_FPX_FMA32 is gone: the helper uses RECOMP_FPX_FMA64. - Dead code removed: a64_fpx_fz_mul32, a64_fpx_fz_result's mul param, "mul32" in the fz elif, A64_FPX_FZ_OPEN; FPX_FZ_HEADER's comment updated. - The wrapper tests RECOMP_FPX_OPEN once; a64_fpx_sticky32 rejects a NaN operand, and FMUL with OFC and UFC clear and nothing flushed, before computing (same rescue counts with and without these rejects). - FPX1's contract comment in arm64_to_c.h points to the outlined wrappers' rescues and says the shadow build does not reach them. tests/test_fpx_sticky.py: 2 x (16M random + 150M sweep) checks, 0 mismatches. Each sabotage in its docstring fails it. Co-Authored-By: Claude Opus 5.5 --- tests/fpx_sticky_test.c | 67 +++++++++++++++----- tests/test_fpx_sticky.py | 68 ++++++++++++-------- third_party/a64-kit/lift/a64_lift.py | 74 ++++++++++++---------- third_party/a64-kit/lift/suyu/arm64_to_c.h | 13 ++++ 4 files changed, 146 insertions(+), 76 deletions(-) diff --git a/tests/fpx_sticky_test.c b/tests/fpx_sticky_test.c index 30c4ecffa..c0868553c 100644 --- a/tests/fpx_sticky_test.c +++ b/tests/fpx_sticky_test.c @@ -1,16 +1,30 @@ /* tests/test_fpx_sticky.py's driver: the generated FPX1 wrappers a64_fpx_mul32_fma32 (FMUL/FNMUL S) and a64_fpx_fma32_fma32 (FMLA family S) against the exact body they fall back to (recomp_fpe_fma32), value and FPSR. - Built against the real a64.h with the a64_suyu.h / a64_suyu.c / a64_fpx.h the generators write. + Built against the real a64.h with the a64_suyu.h / a64_suyu.c / a64_fpx.h the generators write, at -march=x86-64-v3 + (-DEXPECT_FMA=1: a hardware FMA, both ops rescued) or -march=x86-64-v2 (-DEXPECT_FMA=0: only FMUL is; FMLA's rescue + must decline every case). The rescues are counted inside the generated wrappers (A64_FPX_STICKY_PROBE, empty in the + real build), and one under a closed gate (an FPCR control bit but FZ, or IXC clear) counts as a mismatch: the exact + body models none of FIZ/AH/NEP, so only the gate keeps the rescue away from them. fpx_sticky_test random n random operand triples (both ops each) over random FPCR/FPSR states fpx_sticky_test sweep every a with biased exponent <= 4 or >= 250 whose fraction is below 2^12, above 2^23 - 2^12 or a multiple of stride, x a fixed set of b and addends, x FZ on/off, - x flags all set / IXC only - Exit 0: no mismatch and the sticky rescue decided cases of both ops. 1: a mismatch. 3: the CPU lacks FMA/AVX2. */ + x flags all set / IXC only, and one more state per a (partial flags, IXC + clear, FIZ/AH/NEP) + Exit 0: no mismatch, the wrappers rescued FMUL cases, and FMLA ones exactly when EXPECT_FMA. 1: a mismatch. + 3: the CPU lacks this build's -march (x86-64-v3: FMA/AVX2; x86-64-v2: SSSE3/SSE4.2/POPCNT). 4: rescue counts wrong. */ +static unsigned long long g_sticky; /* the generated wrappers' rescues */ +#define A64_FPX_STICKY_PROBE() ((void)g_sticky++) #include "a64.h" #include #include #include +#if !defined(EXPECT_FMA) || RECOMP_FPX_HOST != 2 || RECOMP_FPX_FMA64 != EXPECT_FMA +#error "build with -DEXPECT_FMA=1 at x86-64-v3 or -DEXPECT_FMA=0 at x86-64-v2 (RECOMP_FPX_FMA64 must agree)" +#endif +/* The FPCR bits the rescue's soundness needs clear (the test's own copy of RECOMP_FPX_FPCR_MASK): every control bit + but FZ (FIZ AH NEP, trap enables, FZ16, RMode, DN, AHP) and the host's kill switch. */ +#define GATE_FPCR (UINT64_C(0x06C8FF07) | (UINT64_C(1) << 32)) static uint64_t rs; static uint64_t rnd(void) { rs ^= rs << 13; rs ^= rs >> 7; rs ^= rs << 17; return rs; } @@ -73,7 +87,8 @@ static void gen_state(uint64_t *fpcr, uint64_t *fpsr) { if (rbelow(2)) cr |= 1u << 24; /* FZ */ if (rbelow(10) == 0) cr |= 1u << 25; /* DN (gate closed) */ if (rbelow(10) == 0) cr |= (uint64_t)rbelow(4) << 22; /* RMode */ - if (rbelow(40) == 0) cr |= r32() & 0x07ff9f00u; /* anything else */ + if (rbelow(40) == 0) cr |= r32() & 0x07ff9f07u; /* anything else (FIZ, AH, NEP included) */ + if (rbelow(16) == 0) cr |= 1u << rbelow(3); /* FIZ, AH or NEP alone (the gate must stay closed) */ if (rbelow(200) == 0) cr |= RECOMP_FPX_INHIBIT; static const uint32_t bits[6] = {1, 2, 4, 8, 16, 128}; /* IOC DZC OFC UFC IXC IDC */ for (int k = 0; k < 6; k++) @@ -84,7 +99,7 @@ static void gen_state(uint64_t *fpcr, uint64_t *fpsr) { } /* ---- one check: both wrappers against the exact body ---- */ -static uint64_t checks, mismatches, rescued[2]; +static uint64_t checks, mismatches, rescued[2], closed_checks; static int report_left = 20; static void check(uint64_t cr, uint64_t sr, uint32_t a, uint32_t b, uint32_t z, int mul) { Ctx ref, got; @@ -92,13 +107,18 @@ static void check(uint64_t cr, uint64_t sr, uint32_t a, uint32_t b, uint32_t z, ref.fpcr = cr; ref.fpsr = sr; got = ref; const uint64_t r = recomp_fpe_fma32(&ref, a, b, z); + const unsigned long long before = g_sticky; const uint64_t v = mul ? a64_fpx_mul32_fma32(&got, a, b, z) : a64_fpx_fma32_fma32(&got, a, b, z); + const int kept = g_sticky != before, closed = (cr & GATE_FPCR) || !(sr & 16u); checks++; - { Ctx k = got; uint64_t t; k.fpsr = sr; /* (did the sticky rescue decide this one) */ - rescued[mul] += RECOMP_FPX_OPEN(&k) && !(mul ? recomp_fpx_mul32(a, b, RECOMP_FPX_FZ(&k), &t) - : recomp_fpx_fma32(a, b, z, RECOMP_FPX_FZ(&k), &t)) - && a64_fpx_sticky32(&k, a, b, z, mul, &t); } - if (v != r || got.fpsr != ref.fpsr) { + rescued[mul] += kept; + closed_checks += closed; + if (kept && closed) { + mismatches++; + if (report_left-- > 0) + printf("GATE %s a=%08x b=%08x z=%08x fpcr=%llx fpsr=%llx: rescued under a closed gate\n", mul ? "FMUL" : "FMLA", + a, b, z, (unsigned long long)cr, (unsigned long long)sr); + } else if (v != r || got.fpsr != ref.fpsr) { mismatches++; if (report_left-- > 0) printf("MISMATCH %s a=%08x b=%08x z=%08x fpcr=%llx fpsr=%llx: exact %08llx/%llx wrapper %08llx/%llx\n", @@ -127,6 +147,11 @@ static const uint32_t B_SET[] = { 0x3f800000, /* 1 */ 0x3f000000, /* 0.5 */ 0x3fc00001, /* 1.5+ulp */ 0x00800000, /* min normal */ 0x7f7fffff, /* max */ 0x80000001, /* -min subnormal */ 0x007fffff, /* max subnormal */ 0x7f800000, /* inf */ 0x00000000, /* +0 */ 0x5f000000, /* 2^63 */ 0x3fffffff, /* ~2 */ 0x1f800000 /* 2^-64 */}; +/* one more state per a: partial flags (each needed bit clear in turn), IXC clear, and FIZ / AH / NEP (gate closed) */ +static const uint32_t MORE[16][2] = { + {0x01000000u, 0x18u}, {0, 0x18u}, {0x01000000u, 0x90u}, {0x01000000u, 0x94u}, {0, 0x14u}, {0x01000000u, 0x98u}, + {0x01000000u, 0x1cu}, {0, 0x1cu}, {0x01000000u, 0x9bu}, {0x01000000u, 0x97u}, {0x01000000u, 0x1fu}, + {0x01000000u, 0x8fu}, {0x01000002u, 0x9fu}, {0x01000001u, 0x9fu}, {0x01000004u, 0x9fu}, {0x00000002u, 0x9fu}}; static void sweep(uint32_t stride) { for (unsigned bi = 0; bi < sizeof B_SET / sizeof B_SET[0]; bi++) { const uint32_t b = B_SET[bi]; @@ -147,6 +172,8 @@ static void sweep(uint32_t stride) { } both(cr, sr, a, b, z); both(cr ^ 0x01000000u, sr ^ 0x8fu, a, b, z); /* the other FZ and flag state, same operands */ + const uint32_t *m = MORE[((f >> 4) + hi + bi) & 15]; + both(m[0], m[1], a, b, z); } } } @@ -154,8 +181,11 @@ static void sweep(uint32_t stride) { int main(int argc, char **argv) { unsigned eax, ebx, ecx, edx; - if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx) || !(ecx >> 12 & 1) || !__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx) || - !(ebx >> 5 & 1)) { + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx) || !(ecx >> 9 & 1) || !(ecx >> 20 & 1) || !(ecx >> 23 & 1)) { + printf("this CPU has no SSSE3/SSE4.2/POPCNT (x86-64-v2)\n"); + return 3; + } + if (EXPECT_FMA && (!(ecx >> 12 & 1) || !__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx) || !(ebx >> 5 & 1))) { printf("this CPU has no FMA/AVX2 (x86-64-v3)\n"); return 3; } @@ -163,8 +193,13 @@ int main(int argc, char **argv) { if (argc == 4 && !strcmp(argv[1], "random")) random_cases(strtoull(argv[2], 0, 0), strtoull(argv[3], 0, 0)); else if (argc == 3 && !strcmp(argv[1], "sweep")) sweep((uint32_t)strtoul(argv[2], 0, 0)); else { fprintf(stderr, "usage: random | sweep \n"); return 2; } - printf("%s: %llu checks, %llu mismatches, sticky rescue decided %llu FMUL / %llu FMLA\n", argv[1], - (unsigned long long)checks, (unsigned long long)mismatches, (unsigned long long)rescued[1], - (unsigned long long)rescued[0]); - return mismatches ? 1 : (rescued[0] && rescued[1]) ? 0 : 4; + printf("%s (%s): %llu checks (%llu with the gate closed), %llu mismatches, the wrappers rescued %llu FMUL / %llu FMLA\n", + argv[1], EXPECT_FMA ? "x86-64-v3" : "x86-64-v2", (unsigned long long)checks, (unsigned long long)closed_checks, + (unsigned long long)mismatches, (unsigned long long)rescued[1], (unsigned long long)rescued[0]); + if (mismatches) return 1; + if (!rescued[1] || (EXPECT_FMA ? !rescued[0] : rescued[0] != 0)) { + printf("rescue counts wrong: want FMUL > 0 and FMLA %s\n", EXPECT_FMA ? "> 0" : "== 0 (no hardware FMA)"); + return 4; + } + return 0; } diff --git a/tests/test_fpx_sticky.py b/tests/test_fpx_sticky.py index 96858d470..c113ffe7a 100644 --- a/tests/test_fpx_sticky.py +++ b/tests/test_fpx_sticky.py @@ -5,25 +5,36 @@ could raise is already set, one hardware FMA (or multiply) is the Arm result. This builds what the real build builds, a64_fpx.h from a64_lift.fpx_header() and a64_suyu.h / a64_suyu.c from a64_rtgen.py (through suyu's translator, a64emit.cpp, built here at -O0 into a temporary folder), and compiles tests/fpx_sticky_test.c against the real rt/a64.h -at the build's FP flags (-march=x86-64-v3 -ffp-contract=off -O2). Each wrapper's value and FPSR must equal -recomp_fpe_fma32's for: +at the build's FP flags (-ffp-contract=off -O2) twice: at -march=x86-64-v3 (RECOMP_FPX_FMA64: both ops rescued) and at +-march=x86-64-v2 (what host_march() gives a CPU without AVX2: FMUL's rescue is a plain multiply and FMLA's must decline +every case). The v2 build runs on any x86-64-v2 CPU, whether or not it has AVX2. Each wrapper's value and FPSR must equal +recomp_fpe_fma32's for, in each build: - 8M random operand triples (16M checks, seeded): zeros, subnormals, inf, NaN (quiet and signalling), the extreme normals, products near the underflow and overflow edges, exact and near cancellations, under random FPCR (FZ, - DN, RMode, other bits, the kill switch) and FPSR states; - - an edge sweep (~100M checks): every a with biased exponent <= 4 or >= 250 whose fraction is below 2^12, above + DN, RMode, FIZ/AH/NEP alone or with other bits, the kill switch) and FPSR states; + - an edge sweep (~150M checks): every a with biased exponent <= 4 or >= 250 whose fraction is below 2^12, above 2^23 - 2^12 or a multiple of 97, against 12 fixed b and 4 addend kinds, each under FZ on and off with the flags - all set or IXC only. -Both runs must also see the rescue decide cases of both ops. About 15 s here (fpstub's harness, which this is cut down -from, saw no mismatch in 6.4e9 checks). - -It has teeth: with the helper's UFC test removed by hand (a64_fpx_sticky32's `if (!(s & 8u))` made `if (0)`, so a tiny -result is kept with UFC clear), both runs fail: 654k of the random checks and 5.4M of the sweep's mismatch (FPSR -without UFC). So does dropping its IDC test (a flushed subnormal operand with IDC clear): 406k and 7.1M. + all set or IXC only, and under one more state (by a): partial flags (UFC set and IDC clear, IDC set and UFC clear, + OFC only, every bit but one of IDC/UFC/OFC, ...), IXC clear, or FIZ/AH/NEP set. +The rescues are counted inside the generated wrappers (an A64_FPX_STICKY_PROBE() the driver defines; empty in the real +build): both builds must see FMUL rescues, v3 FMLA rescues and v2 none, and a rescue under a closed gate (an FPCR +control bit but FZ set, or IXC clear) fails. That last check is what guards FIZ/AH/NEP: the exact body models none of +them, so comparing values could not tell. About 35-50 s here. + +It has teeth (each by hand, then restored): + - the wrappers' route to the rescue removed (fpx_header's `(fast, exact) in (("mul32", "fma32"), ("fma32", "fma32"))` + made never true): both builds fail with no rescue counted (exit 4), though every value still matches; + - FMUL's wrapper passing mul=0 (an FMA with the zero addend, the same value with a hardware FMA): v2 fails (exit 4); + - FPCR bits 0-2 dropped from RECOMP_FPX_FPCR_MASK (0x06C8FF07 -> 0x06C8FF00): every value still matches, but 190k of + v3's random checks and 6.1M of its sweep's are rescues under FIZ/AH/NEP (v2: 99k and 2.6M); + - the helper's UFC test removed (a64_fpx_sticky32's `if (!(s & 8u))` made `if (0)`, so a tiny result is kept with + UFC clear): 486k and 3.5M mismatch at v3 (FPSR without UFC), 258k and 475k at v2; + - its IDC test dropped (a flushed subnormal operand with IDC clear): 383k and 10.0M at v3, 146k and 3.8M at v2. python -I -B tests/test_fpx_sticky.py [--cc clang] [--cxx clang++] Skipped (exit 0) on a host that is not x86-64 or has no C/C++ compiler (PATH, $CC/$CXX, or the build's own emsdk / -HOOKSHOT_MINGW toolchain), or on a CPU without x86-64-v3. +HOOKSHOT_MINGW toolchain), or on a CPU without x86-64-v2; the v3 build alone is skipped on a CPU without x86-64-v3. """ import argparse import os @@ -41,7 +52,8 @@ import toolchain # noqa: E402 RANDOM_TRIPLES, SEED, SWEEP_STRIDE = 8000000, 20261109, 97 -FLAGS = ["-march=x86-64-v3", "-ffp-contract=off", "-fno-strict-aliasing", "-O2"] # (nx2/build.py's lifted-code FP flags) +FLAGS = ["-ffp-contract=off", "-fno-strict-aliasing", "-O2"] # (nx2/build.py's lifted-code FP flags, with -march=) +BUILDS = (("x86-64-v3", 1), ("x86-64-v2", 0)) # (-march, EXPECT_FMA: whether RECOMP_FPX_FMA64 holds there) def skip(why): @@ -127,20 +139,26 @@ def main(): if not cc or not cxx: skip("no C compiler" if not cc else "no C++ compiler") t0 = time.time() - failed = [] + failed, ran = [], [] with tempfile.TemporaryDirectory(prefix="hookshot-fpx-sticky-") as tmp: generate(cxx, tmp, target_flags(cxx, tmp)) - exe = os.path.join(tmp, "fpx_sticky_test.exe") - build([cc] + target_flags(cc, tmp) + FLAGS + ["-D_CRT_SECURE_NO_WARNINGS", "-I" + os.path.join(KIT, "rt"), - "-I" + tmp, os.path.join(ROOT, "tests", "fpx_sticky_test.c"), os.path.join(tmp, "a64_suyu.c"), "-o", exe], - "the driver") - for args in (["random", str(RANDOM_TRIPLES), str(SEED)], ["sweep", str(SWEEP_STRIDE)]): - r = run([exe] + args) - print(r.stdout.rstrip()) - if r.returncode == 3: - skip("this CPU is not x86-64-v3") - if r.returncode: - failed.append(args[0]) + for march, fma in BUILDS: + exe = os.path.join(tmp, "fpx_sticky_test_%s.exe" % march) + build([cc] + target_flags(cc, tmp) + ["-march=" + march] + FLAGS + + ["-DEXPECT_FMA=%d" % fma, "-D_CRT_SECURE_NO_WARNINGS", "-I" + os.path.join(KIT, "rt"), "-I" + tmp, + os.path.join(ROOT, "tests", "fpx_sticky_test.c"), os.path.join(tmp, "a64_suyu.c"), "-o", exe], + "the driver at " + march) + for args in (["random", str(RANDOM_TRIPLES), str(SEED)], ["sweep", str(SWEEP_STRIDE)]): + r = run([exe] + args) + print(r.stdout.rstrip()) + if r.returncode == 3: + print("fpx sticky: %s skipped (this CPU is not %s)" % (march, march)) + break + ran.append(march) + if r.returncode: + failed.append("%s %s" % (march, args[0])) + if not ran: + skip("this CPU is not x86-64-v2") if failed: raise SystemExit("fpx sticky: FAIL (%s) in %.1f s" % (", ".join(failed), time.time() - t0)) print("fpx sticky: pass in %.1f s" % (time.time() - t0)) diff --git a/third_party/a64-kit/lift/a64_lift.py b/third_party/a64-kit/lift/a64_lift.py index 171110b02..c9d1d5cca 100644 --- a/third_party/a64-kit/lift/a64_lift.py +++ b/third_party/a64-kit/lift/a64_lift.py @@ -602,63 +602,66 @@ def add(m): # Results past the fast path's range once their flags are already raised (TOTK's threads: FPSR = 0x9f, every cumulative -# flag set; FPCR.FZ = 1). Overflow (OFC raised): an infinite result of finite operands is the native one, the -# round-to-nearest ±inf (~1M multiplies a second at the castle went through the exact body for it). Flush-to-zero (UFC -# raised): a result below the normal range is the native one flushed to a zero of its sign. Sums are exact there (a -# tiny sum of normal numbers is a subnormal exactly), products flush when the exact product is tiny, which the native -# result shows unless it rounded up to the smallest normal (then the exact body decides). No subnormal operand (those -# flush first). +# flag set; FPCR.FZ = 1). FADD/FSUB S (a64_fpx_fz_add32 / a64_fpx_fz_sub32, OFC or UFC raised): an infinite result of +# finite operands is the native one, the round-to-nearest ±inf (~1M multiplies a second at the castle went through the +# exact body for it); under FZ with UFC raised a result below the normal range is the native one flushed to a zero of its +# sign. Sums are exact there (a tiny sum of normal numbers is a subnormal exactly). No subnormal operand (those flush +# first). FMUL/FNMUL/FMLA-family S: a64_fpx_sticky32 below. FPX_FZ_HEADER = r""" -#define A64_FPX_FZ_OPEN(c) (RECOMP_FPX_OPEN(c) && ((c)->fpsr & 0xCu)) -static inline __attribute__((always_inline)) int a64_fpx_fz_result(const Ctx *c, uint64_t a, uint64_t b, uint64_t t, int mul, uint64_t *v) { +static inline __attribute__((always_inline)) int a64_fpx_fz_result(const Ctx *c, uint64_t a, uint64_t b, uint64_t t, uint64_t *v) { const uint32_t m = (uint32_t)t & 0x7fffffffu; if (m == 0x7f800000u) return (c->fpsr & 4u) && RECOMP_FPX_FIN32(a) && RECOMP_FPX_FIN32(b) ? (*v = t, 1) : 0; if (!RECOMP_FPX_FZ(c) || !(c->fpsr & 8u) || m > 0x7f800000u || RECOMP_FPX_SUBN32(a) || RECOMP_FPX_SUBN32(b)) return 0; - if (m >= 0x00800000u) return (mul && m == 0x00800000u) ? 0 : (*v = t, 1); + if (m >= 0x00800000u) return (*v = t, 1); *v = t & 0x80000000u; return 1; } #if RECOMP_FPX_HOST static inline __attribute__((always_inline)) int a64_fpx_fz_add32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { - return a64_fpx_fz_result(c, a, b, recomp_fpx_fb(recomp_fpx_f(a) + recomp_fpx_f(b)), 0, v); } + return a64_fpx_fz_result(c, a, b, recomp_fpx_fb(recomp_fpx_f(a) + recomp_fpx_f(b)), v); } static inline __attribute__((always_inline)) int a64_fpx_fz_sub32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { - return a64_fpx_fz_result(c, a, b, recomp_fpx_fb(recomp_fpx_f(a) - recomp_fpx_f(b)), 0, v); } -static inline __attribute__((always_inline)) int a64_fpx_fz_mul32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { - return a64_fpx_fz_result(c, a, b, recomp_fpx_fb(recomp_fpx_f(a) * recomp_fpx_f(b)), 1, v); } + return a64_fpx_fz_result(c, a, b, recomp_fpx_fb(recomp_fpx_f(a) - recomp_fpx_f(b)), v); } #else static inline int a64_fpx_fz_add32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { (void)c; (void)a; (void)b; (void)v; return 0; } static inline int a64_fpx_fz_sub32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { (void)c; (void)a; (void)b; (void)v; return 0; } -static inline int a64_fpx_fz_mul32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { (void)c; (void)a; (void)b; (void)v; return 0; } #endif -/* FMUL/FNMUL/FMLA-family S past the fast path (the wrappers' "sticky" rescue, in place of a64_fpx_fz_mul32): one hardware - fused multiply-add, kept when every FPSR bit the Arm op could raise is already set and the value is the Arm one. +/* FMUL/FNMUL/FMLA-family S past the fast path (the "sticky" rescue of the outlined wrappers a64_fpx_mul32_fma32 and + a64_fpx_fma32_fma32, inside their RECOMP_FPX_OPEN gate): one hardware multiply or fused multiply-add, kept when every + FPSR bit the Arm op could raise is already set and the value is the Arm one. FZ: a subnormal operand is flushed to a zero of its sign (IDC). NaN: never. Infinite: exact with an infinite operand, else an overflow (OFC). A zero product or an exact cancellation (a*b == -z in binary64, where the product is exact): exact. Otherwise a result at most the smallest normal may be tiny (UFC): FZ clear, the IEEE gradual-underflow result is Arm's; FZ set, below the smallest normal the exact value is tiny and Arm returns a zero of its sign, and exactly the - smallest normal is ambiguous (falls back). Without a hardware FMA only the multiply is rescued. */ -#if RECOMP_FPX_HOST == 1 || (RECOMP_FPX_HOST == 2 && defined(__FMA__) && (defined(__GNUC__) || defined(__clang__))) -#define A64_FPX_FMA32 1 -#else -#define A64_FPX_FMA32 0 + smallest normal is ambiguous (falls back). Rejected before computing: a NaN operand, a subnormal operand under FZ with + IDC clear, and FMUL of finite operands with neither OFC nor UFC set and nothing flushed (past the fast path, which keeps + a large product and a finite product of a zero, such a product is tiny or infinite). Without a hardware FMA + (RECOMP_FPX_FMA64; binary32 fmaf is the same instruction class) only the multiply is rescued. Returning 0 is always + sound: the wrapper then takes the exact body. */ +#ifndef A64_FPX_STICKY_PROBE +#define A64_FPX_STICKY_PROBE() ((void)0) /* (tests/test_fpx_sticky.py counts the rescues the wrappers keep here) */ #endif static inline __attribute__((always_inline)) int a64_fpx_sticky32(const Ctx *c, uint64_t a, uint64_t b, uint64_t z, int mul, uint64_t *v) { #if RECOMP_FPX_HOST const uint64_t s = c->fpsr; uint32_t x = (uint32_t)a, y = (uint32_t)b, w = (uint32_t)z, t; +#if !RECOMP_FPX_FMA64 + if (!mul) return 0; +#endif + if ((x & 0x7fffffffu) > 0x7f800000u || (y & 0x7fffffffu) > 0x7f800000u || (w & 0x7fffffffu) > 0x7f800000u) return 0; if (RECOMP_FPX_FZ(c) && (RECOMP_FPX_SUBN32(x) || RECOMP_FPX_SUBN32(y) || (!mul && RECOMP_FPX_SUBN32(w)))) { if (!(s & 0x80u)) return 0; if (RECOMP_FPX_SUBN32(x)) x &= 0x80000000u; if (RECOMP_FPX_SUBN32(y)) y &= 0x80000000u; if (RECOMP_FPX_SUBN32(w)) w &= 0x80000000u; + } else if (mul && !(s & 0xCu) && RECOMP_FPX_FIN32(x) && RECOMP_FPX_FIN32(y)) { + return 0; } - if (mul) /* (w is the zero of the product's sign: x*y + w is x*y) */ - t = (uint32_t)recomp_fpx_fb(recomp_fpx_f(x) * recomp_fpx_f(y)); - else -#if A64_FPX_FMA32 - t = (uint32_t)recomp_fpx_fb(__builtin_fmaf(recomp_fpx_f(x), recomp_fpx_f(y), recomp_fpx_f(w))); + /* (FMUL: w is the zero of the product's sign, so x*y + w is x*y) */ +#if RECOMP_FPX_FMA64 + t = (uint32_t)recomp_fpx_fb(mul ? recomp_fpx_f(x) * recomp_fpx_f(y) + : __builtin_fmaf(recomp_fpx_f(x), recomp_fpx_f(y), recomp_fpx_f(w))); #else - return 0; + t = (uint32_t)recomp_fpx_fb(recomp_fpx_f(x) * recomp_fpx_f(y)); #endif const uint32_t m = t & 0x7fffffffu; if (m > 0x7f800000u) return 0; @@ -720,16 +723,17 @@ def fpx_header(used): FPX_FZ_HEADER] for fast, exact, nf, ne in sorted(used): params = ", ".join("uint64_t a%d" % i for i in range(ne)) - fz = "" + # past the fast path, still inside its gate (tested once): the rescues above, else the exact body + rescue = "" if (fast, exact) in (("mul32", "fma32"), ("fma32", "fma32")): - fz = "if (RECOMP_FPX_OPEN(c) && a64_fpx_sticky32(c, a0, a1, a2, %d, &v)) return v; " % (fast == "mul32") - elif fast in ("add32", "sub32", "mul32"): - fz = "if (A64_FPX_FZ_OPEN(c) && a64_fpx_fz_%s(c, a0, a1, &v)) return v; " % fast + rescue = "if (a64_fpx_sticky32(c, a0, a1, a2, %d, &v)) { A64_FPX_STICKY_PROBE(); return v; } " % (fast == "mul32") + elif fast in ("add32", "sub32"): + rescue = "if ((c->fpsr & 0xCu) && a64_fpx_fz_%s(c, a0, a1, &v)) return v; " % fast out.append("A64_FPX_FN uint64_t a64_fpx_%s_%s(Ctx *c, %s) { uint64_t v; " - "if (RECOMP_FPX_OPEN(c) && RECOMP_LIKELY(recomp_fpx_%s(%s, RECOMP_FPX_FZ(c), &v))) return v; " - "%sreturn recomp_fpe_%s(c, %s); }" % (fast, exact, params, fast, - ", ".join("a%d" % i for i in range(nf)), fz, exact, - ", ".join("a%d" % i for i in range(ne)))) + "if (RECOMP_FPX_OPEN(c)) { if (RECOMP_LIKELY(recomp_fpx_%s(%s, RECOMP_FPX_FZ(c), &v))) return v; %s} " + "return recomp_fpe_%s(c, %s); }" % (fast, exact, params, fast, + ", ".join("a%d" % i for i in range(nf)), rescue, exact, + ", ".join("a%d" % i for i in range(ne)))) # (the hot functions' copy, see HOT_FPX: the fast path inline, everything else through the helper) out.append("static inline __attribute__((always_inline)) uint64_t a64_fpx_hot_%s_%s(Ctx *c, %s) { uint64_t v; " "if (RECOMP_FPX_OPEN(c) && RECOMP_LIKELY(recomp_fpx_%s(%s, RECOMP_FPX_FZ(c), &v))) return v; " diff --git a/third_party/a64-kit/lift/suyu/arm64_to_c.h b/third_party/a64-kit/lift/suyu/arm64_to_c.h index 1996ad9f3..147445fee 100644 --- a/third_party/a64-kit/lift/suyu/arm64_to_c.h +++ b/third_party/a64-kit/lift/suyu/arm64_to_c.h @@ -334,6 +334,8 @@ inline bool EmitGuardGen() { inline bool g_emit_fpx = false; // Instrumentation only, never timed: with g_emit_fpx, the exact body runs // after every fast path too, and recomp_fpx_shadow compares the two. +// It compares the inline form only: the outlined wrappers' rescues past the +// fast path (a64_lift.py's a64_fpx.h) never run in it. inline bool g_emit_fpx_shadow = false; // Hybrid exports translate the gated forms too. g_translate_all gates only SHL @@ -6535,6 +6537,17 @@ inline const char* FpxH() { the ARM result bit for bit. Anything else takes the unchanged exact body, which FPX1 sites call out of line (recomp_fpe_*, recomp_runtime.c). + Lifted code keeps more than the above: a64_lift.py outlines each guard + into a wrapper (a64_fpx_*, a64_fpx.h) that, past these fast paths and + inside the same gate, also keeps the native FADD/FSUB S result that + overflowed with OFC already set or, under FZ, is flushed with UFC already + set (a64_fpx_fz_add32/sub32), and the native + FMUL/FNMUL/FMLA-family S result, tiny, infinite or of flushed inputs + included, when every flag the op could raise (IDC, UFC, OFC) is already + set (a64_fpx_sticky32; checked game-free by Hookshot's + tests/test_fpx_sticky.py). The shadow build keeps the inline form above + and never reaches those wrappers, so it does not validate them. + Host contract, enforced by the host once it has negotiated FPX1 through recomp_image_fpx_v1: on x86-64 MXCSR round-to-nearest with DAZ and FTZ clear and every exception masked; on AArch64 FPCR 0. Bit 32 of fpcr is the host's From 0a2b38ea78041aed25609917eb2af67f43b767fa Mon Sep 17 00:00:00 2001 From: klimovich008 Date: Mon, 9 Nov 2026 18:56:11 +0100 Subject: [PATCH 3/3] test(fpx): run guest FMUL/FNMUL/FMADD/FMLA S words through the lifter's emit route tests/test_fpx_sticky.py gains a "lifted" case in both builds: four hand-encoded guest words (FMUL, FNMUL, FMADD, FMLA by element, S) go through a64emit.translate and a64_lift.op_text, the lifter's per-instruction pipeline (and hot_text, the hot functions' form), so the C calls the outlined wrapper exactly as lifted code does. Each runs against the same word translated with FPX off (the exact body inline) on 5 inputs that must be rescued and 7 that must fall back (IXC only, UFC/OFC/IDC clear, NaN, FZ|AH, a normal result): whole Ctx and the A64_FPX_STICKY_PROBE count must match. 96 runs a build, 0 mismatches; 40 rescues at x86-64-v3, 20 (FMUL/FNMUL only) at x86-64-v2. a64_lift.py: Lifter.write's per-instruction pipeline and the hot-function rewrite become op_text() and hot_text(), which the test calls (no change to what they emit). Sabotage: the emitter no longer outlining FMUL S (EmitFPMulAddValue's `FpxOutline()` made `FpxOutline() && !mul`) fails the lifted case in both builds (20 FMUL/FNMUL runs without a rescue) while the random and sweep runs, which call the wrappers by name, pass. Co-Authored-By: Claude Opus 5.5 --- tests/fpx_sticky_test.c | 59 +++++++++++++++++++++++++++- tests/test_fpx_sticky.py | 54 +++++++++++++++++++++---- third_party/a64-kit/lift/a64_lift.py | 15 ++++++- 3 files changed, 118 insertions(+), 10 deletions(-) diff --git a/tests/fpx_sticky_test.c b/tests/fpx_sticky_test.c index c0868553c..4bf7b65b9 100644 --- a/tests/fpx_sticky_test.c +++ b/tests/fpx_sticky_test.c @@ -6,6 +6,10 @@ real build), and one under a closed gate (an FPCR control bit but FZ, or IXC clear) counts as a mismatch: the exact body models none of FIZ/AH/NEP, so only the gate keeps the rescue away from them. + fpx_sticky_test lifted guest FMUL/FNMUL/FMADD/FMLA S words as the lifter emits them (fpx_lifted.h: the + outlined and the hot-function form) against the same words with FPX off, on + inputs that must be rescued and inputs that must fall back: whole Ctx and the + rescue count fpx_sticky_test random n random operand triples (both ops each) over random FPCR/FPSR states fpx_sticky_test sweep every a with biased exponent <= 4 or >= 250 whose fraction is below 2^12, above 2^23 - 2^12 or a multiple of stride, x a fixed set of b and addends, x FZ on/off, @@ -25,6 +29,7 @@ static unsigned long long g_sticky; /* the generated wrappers' rescues */ /* The FPCR bits the rescue's soundness needs clear (the test's own copy of RECOMP_FPX_FPCR_MASK): every control bit but FZ (FIZ AH NEP, trap enables, FZ16, RMode, DN, AHP) and the host's kill switch. */ #define GATE_FPCR (UINT64_C(0x06C8FF07) | (UINT64_C(1) << 32)) +#include "fpx_lifted.h" static uint64_t rs; static uint64_t rnd(void) { rs ^= rs << 13; rs ^= rs >> 7; rs ^= rs << 17; return rs; } @@ -179,6 +184,57 @@ static void sweep(uint32_t stride) { } } +/* ---- the lifted route: guest words through a64emit and a64_lift.op_text ---- */ +#define FZ 0x01000000u +static const struct { uint32_t a, b, z; uint32_t fpcr, fpsr; int rescue; const char *what; } LIFTED[] = { + {0x1c800000u, 0x1c800000u, 0, FZ, 0x9fu, 1, "tiny product, FZ, flags set: a zero"}, + {0x1c800000u, 0x9c800000u, 0x80000000u, FZ, 0x9fu, 1, "tiny negative product, FZ, flags set: -0"}, + {0x1c800000u, 0x1e000000u, 0, 0, 0x9fu, 1, "tiny product, no FZ, UFC set: the subnormal"}, + {0x71800000u, 0x71800000u, 0x3f800000u, FZ, 0x9fu, 1, "overflow, OFC set: inf"}, + {0x00000005u, 0x3fc00000u, 0x3f800000u, FZ, 0x9fu, 1, "subnormal operand, FZ, IDC set: flushed"}, + {0x1c800000u, 0x1c800000u, 0, FZ, 0x10u, 0, "tiny product, FZ, IXC only: falls back"}, + {0x1c800000u, 0x1e000000u, 0, 0, 0x97u, 0, "tiny product, no FZ, UFC clear: falls back"}, + {0x71800000u, 0x71800000u, 0x3f800000u, FZ, 0x9bu, 0, "overflow, OFC clear: falls back"}, + {0x00000005u, 0x3fc00000u, 0x3f800000u, FZ, 0x1fu, 0, "subnormal operand, FZ, IDC clear: falls back"}, + {0x7fc00001u, 0x3f800000u, 0x3f800000u, FZ, 0x9fu, 0, "NaN operand: falls back"}, + {0x1c800000u, 0x1c800000u, 0, FZ | 2u, 0x9fu, 0, "tiny product, FZ|AH (gate closed): falls back"}, + {0x3fc00000u, 0x40000000u, 0x3e800000u, FZ, 0x9fu, 0, "normal (the fast path keeps it)"}, +}; +static int lifted_cases(void) { + unsigned bad = 0, n = 0, kept = 0; + for (unsigned g = 0; g < sizeof GUEST / sizeof GUEST[0]; g++) + for (unsigned k = 0; k < sizeof LIFTED / sizeof LIFTED[0]; k++) + for (int hot = 0; hot < 2; hot++) { + Ctx ref, got; + memset(&ref, 0, sizeof ref); + for (int r = 0; r < 32; r++) { ref.vreg[r][0] = 0x0123456789abcdefULL * (r + 1); ref.vreg[r][1] = ~ref.vreg[r][0]; } + ref.vreg[1][0] = (ref.vreg[1][0] & ~0xffffffffULL) | LIFTED[k].a; + ref.vreg[2][0] = (ref.vreg[2][0] & ~0xffffffffULL) | LIFTED[k].b; + ref.vreg[3][0] = (ref.vreg[3][0] & ~0xffffffffULL) | LIFTED[k].z; /* FMADD's addend */ + ref.vreg[0][0] = (ref.vreg[0][0] & ~0xffffffffULL) | LIFTED[k].z; /* FMLA's accumulator */ + ref.fpcr = LIFTED[k].fpcr; + ref.fpsr = LIFTED[k].fpsr; + got = ref; + GUEST[g].exact(&ref); + const unsigned long long before = g_sticky; + (hot ? GUEST[g].hot : GUEST[g].lifted)(&got); + const unsigned long long rescued = g_sticky - before; + const unsigned want = LIFTED[k].rescue && (GUEST[g].mul || EXPECT_FMA); + n++; + kept += (unsigned)rescued; + if (rescued != want || memcmp(&ref, &got, sizeof ref)) { + bad++; + printf("LIFTED %s (%s), %s: rescued %llu, want %u; s0 exact %08x/fpsr %llx, lifted %08x/fpsr %llx%s\n", + GUEST[g].name, hot ? "hot" : "outlined", LIFTED[k].what, rescued, want, (uint32_t)ref.vreg[0][0], + (unsigned long long)ref.fpsr, (uint32_t)got.vreg[0][0], (unsigned long long)got.fpsr, + memcmp(&ref, &got, sizeof ref) ? " (Ctx differs)" : ""); + } + } + printf("lifted (%s): %u runs of %u guest words, %u mismatches, %u rescued in the wrappers\n", + EXPECT_FMA ? "x86-64-v3" : "x86-64-v2", n, (unsigned)(sizeof GUEST / sizeof GUEST[0]), bad, kept); + return bad ? 1 : 0; +} + int main(int argc, char **argv) { unsigned eax, ebx, ecx, edx; if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx) || !(ecx >> 9 & 1) || !(ecx >> 20 & 1) || !(ecx >> 23 & 1)) { @@ -190,9 +246,10 @@ int main(int argc, char **argv) { return 3; } if ((_mm_getcsr() & 0xffc0) != 0x1f80) { printf("MXCSR %x: not the FPX1 host contract\n", _mm_getcsr()); return 2; } + if (argc == 2 && !strcmp(argv[1], "lifted")) return lifted_cases(); if (argc == 4 && !strcmp(argv[1], "random")) random_cases(strtoull(argv[2], 0, 0), strtoull(argv[3], 0, 0)); else if (argc == 3 && !strcmp(argv[1], "sweep")) sweep((uint32_t)strtoul(argv[2], 0, 0)); - else { fprintf(stderr, "usage: random | sweep \n"); return 2; } + else { fprintf(stderr, "usage: lifted | random | sweep \n"); return 2; } printf("%s (%s): %llu checks (%llu with the gate closed), %llu mismatches, the wrappers rescued %llu FMUL / %llu FMLA\n", argv[1], EXPECT_FMA ? "x86-64-v3" : "x86-64-v2", (unsigned long long)checks, (unsigned long long)closed_checks, (unsigned long long)mismatches, (unsigned long long)rescued[1], (unsigned long long)rescued[0]); diff --git a/tests/test_fpx_sticky.py b/tests/test_fpx_sticky.py index c113ffe7a..5745df161 100644 --- a/tests/test_fpx_sticky.py +++ b/tests/test_fpx_sticky.py @@ -7,8 +7,12 @@ a64emit.cpp, built here at -O0 into a temporary folder), and compiles tests/fpx_sticky_test.c against the real rt/a64.h at the build's FP flags (-ffp-contract=off -O2) twice: at -march=x86-64-v3 (RECOMP_FPX_FMA64: both ops rescued) and at -march=x86-64-v2 (what host_march() gives a CPU without AVX2: FMUL's rescue is a plain multiply and FMLA's must decline -every case). The v2 build runs on any x86-64-v2 CPU, whether or not it has AVX2. Each wrapper's value and FPSR must equal -recomp_fpe_fma32's for, in each build: +every case). The v2 build runs on any x86-64-v2 CPU, whether or not it has AVX2. In each build: + - the lifted route: guest FMUL, FNMUL, FMADD and FMLA (by element) S words, hand-encoded, through a64emit.translate and + a64_lift.op_text (the lifter's per-instruction pipeline, so the C calls the outlined wrapper as lifted code does) and + hot_text (the hot functions' form), against the same words translated with FPX off (the exact body inline): the + whole Ctx and the rescue count must match, on 5 inputs that must be rescued and 7 that must not (96 runs); +and each wrapper's value and FPSR must equal recomp_fpe_fma32's for: - 8M random operand triples (16M checks, seeded): zeros, subnormals, inf, NaN (quiet and signalling), the extreme normals, products near the underflow and overflow edges, exact and near cancellations, under random FPCR (FZ, DN, RMode, FIZ/AH/NEP alone or with other bits, the kill switch) and FPSR states; @@ -22,8 +26,11 @@ them, so comparing values could not tell. About 35-50 s here. It has teeth (each by hand, then restored): + - the emitter no longer outlining FMUL/FNMUL S (arm64_to_c.h's EmitFPMulAddValue: `FpxOutline()` made + `FpxOutline() && !mul`, so the guest FMUL keeps suyu's inline guard): the lifted case fails in both builds (20 + FMUL/FNMUL runs with no rescue), while the random and sweep runs, which call the wrappers by name, still pass; - the wrappers' route to the rescue removed (fpx_header's `(fast, exact) in (("mul32", "fma32"), ("fma32", "fma32"))` - made never true): both builds fail with no rescue counted (exit 4), though every value still matches; + made never true): every run fails with no rescue counted, though every value still matches; - FMUL's wrapper passing mul=0 (an FMA with the zero addend, the same value with a hardware FMA): v2 fails (exit 4); - FPCR bits 0-2 dropped from RECOMP_FPX_FPCR_MASK (0x06C8FF07 -> 0x06C8FF00): every value still matches, but 190k of v3's random checks and 6.1M of its sweep's are rescues under FIZ/AH/NEP (v2: 99k and 2.6M); @@ -106,8 +113,37 @@ def build(cmd, what): raise SystemExit("fpx sticky: building %s failed" % what) +# Guest instructions for the lifted-route case (hand-written encodings): each runs as the lifter emits it, through +# a64emit.translate and a64_lift.op_text (and hot_text), against the same word translated with FPX off. +GUEST = (("FMUL s0, s1, s2", 0x1E220820, 1), ("FNMUL s0, s1, s2", 0x1E228820, 1), + ("FMADD s0, s1, s2, s3", 0x1F020C20, 0), ("FMLA s0, s1, v2.s[0]", 0x5F821020, 0)) # (name, word, FMUL) + + +def lifted_cases(a64emit, a64_lift, used): + """fpx_lifted.h: per guest word its lifted, hot and exact-only (FPX off) C as functions, and the GUEST table""" + words = [(0x1000, w) for _, w, _ in GUEST] + lifted, fl = a64emit.translate(words) + if any(fl): + raise SystemExit("fpx sticky: a64emit did not translate the guest words: %s" % fl) + lifted = [a64_lift.op_text(t, used) for t in lifted] + a64emit.lib().a64_config(1, 0) # (FPX off: the exact body inline, the reference) + try: + exact, _ = a64emit.translate(words) + finally: + a64emit.lib().a64_config(1, 1) + out = ["/* Generated by tests/test_fpx_sticky.py from guest words, as a64_lift.py lifts them. */"] + for i, (lt, ex) in enumerate(zip(lifted, exact)): + out.append("static void g_lifted%d(Ctx *__restrict c) { %s }" % (i, lt)) + out.append("static void g_hot%d(Ctx *__restrict c) { %s }" % (i, a64_lift.hot_text(lt))) + out.append("static void g_exact%d(Ctx *__restrict c) { %s }" % (i, ex)) + out.append("static const struct { const char *name; int mul; void (*lifted)(Ctx *), (*hot)(Ctx *), (*exact)(Ctx *); }" + " GUEST[] = {%s};" % ", ".join('{"%s", %d, g_lifted%d, g_hot%d, g_exact%d}' % (n, m, i, i, i) + for i, (n, _, m) in enumerate(GUEST))) + return "\n".join(out) + "\n" + + def generate(cxx, tmp, flags): - """a64_fpx.h, a64_suyu.h and a64_suyu.c, by the generators the lift runs (a64emit built into tmp)""" + """a64_fpx.h, a64_suyu.h, a64_suyu.c and fpx_lifted.h, by the generators the lift runs (a64emit built into tmp)""" dll = os.path.join(tmp, "a64emit.dll" if os.name == "nt" else "liba64emit.so") build([cxx, "-std=c++20", "-O0", "-shared", "-Wno-everything", "-o", dll, os.path.join(KIT, "lift", "a64emit.cpp")] + flags + ([] if os.name == "nt" else ["-fPIC"]), "a64emit") @@ -116,15 +152,19 @@ def generate(cxx, tmp, flags): import a64_lift import a64_rtgen import a64emit + used = set() try: a64_rtgen.write(tmp) + lifted = lifted_cases(a64emit, a64_lift, used) finally: if a64emit._lib is not None: # (unloaded, so the folder can go) import _ctypes (_ctypes.FreeLibrary if os.name == "nt" else _ctypes.dlclose)(a64emit._lib._handle) a64emit._lib = None - with open(os.path.join(tmp, "a64_fpx.h"), "w", newline="\n") as f: - f.write(a64_lift.fpx_header({("mul32", "fma32", 2, 3), ("fma32", "fma32", 3, 3)})) + with open(os.path.join(tmp, "fpx_lifted.h"), "w", newline="\n") as f: + f.write(lifted) + with open(os.path.join(tmp, "a64_fpx.h"), "w", newline="\n") as f: # (the guest words' shapes, and the two the + f.write(a64_lift.fpx_header(used | {("mul32", "fma32", 2, 3), ("fma32", "fma32", 3, 3)})) # runs call by name) def main(): @@ -148,7 +188,7 @@ def main(): ["-DEXPECT_FMA=%d" % fma, "-D_CRT_SECURE_NO_WARNINGS", "-I" + os.path.join(KIT, "rt"), "-I" + tmp, os.path.join(ROOT, "tests", "fpx_sticky_test.c"), os.path.join(tmp, "a64_suyu.c"), "-o", exe], "the driver at " + march) - for args in (["random", str(RANDOM_TRIPLES), str(SEED)], ["sweep", str(SWEEP_STRIDE)]): + for args in (["lifted"], ["random", str(RANDOM_TRIPLES), str(SEED)], ["sweep", str(SWEEP_STRIDE)]): r = run([exe] + args) print(r.stdout.rstrip()) if r.returncode == 3: diff --git a/third_party/a64-kit/lift/a64_lift.py b/third_party/a64-kit/lift/a64_lift.py index c9d1d5cca..1d3b7d503 100644 --- a/third_party/a64-kit/lift/a64_lift.py +++ b/third_party/a64-kit/lift/a64_lift.py @@ -691,6 +691,11 @@ def add(m): HOT_FPX = re.compile(r"\ba64_fpx_(?!hot_|fz_)(\w+)\(c,") +def hot_text(text): + """a hot function's text: its FPX helper calls go through the copies with the fast path inline""" + return HOT_FPX.sub(r"a64_fpx_hot_\1(c,", text) + + def hot_functions(): path = os.environ.get("A64_HOT_FUNCS") if not path: @@ -752,6 +757,12 @@ def fpx_header(used): return "\n".join(out) + "\n" +def op_text(text, fpx): + """one instruction's emitted C as lifted code has it: FPX guards outlined (their shapes added to fpx, which + fpx_header turns into a64_fpx.h), NEON FP vectorized, stores widened, flags packed""" + return stack_stores(pack_flags(widen_vstores(vectorize_fpx(outline_fpx(text, fpx))))) + + class Lifter: def __init__(self, img, comments=False): self.img = img @@ -1536,7 +1547,7 @@ def write(self, out_dir, chunk_bytes=CHUNK_BYTES): order = list(keys) tx, fl = a64emit.translate([keys[x] for x in order]) fpx = set() - texts = dict(zip(order, zip((stack_stores(pack_flags(widen_vstores(vectorize_fpx(outline_fpx(t, fpx))))) for t in tx), fl))) + texts = dict(zip(order, zip((op_text(t, fpx) for t in tx), fl))) del tx, fl, keys, order with open(os.path.join(out_dir, "a64_fpx.h"), "w", newline="\n") as f: f.write(fpx_header(fpx)) @@ -1570,7 +1581,7 @@ def flush(): for e in sorted(self.funcs): text, refs = self.emit_function(e, texts) if hot and self.fname(e) in hot: - text = HOT_FPX.sub(r"a64_fpx_hot_\1(c,", text) + text = hot_text(text) nhot += 1 if hot_chunk is not None: hot_chunk.append(text)