diff --git a/tests/fpx_sticky_test.c b/tests/fpx_sticky_test.c new file mode 100644 index 000000000..4bf7b65b9 --- /dev/null +++ b/tests/fpx_sticky_test.c @@ -0,0 +1,262 @@ +/* tests/test_fpx_sticky.py's driver: the generated FPX1 wrappers a64_fpx_mul32_fma32 (FMUL/FNMUL S) and + a64_fpx_fma32_fma32 (FMLA family S) against the exact body they fall back to (recomp_fpe_fma32), value and FPSR. + Built against the real a64.h with the a64_suyu.h / a64_suyu.c / a64_fpx.h the generators write, at -march=x86-64-v3 + (-DEXPECT_FMA=1: a hardware FMA, both ops rescued) or -march=x86-64-v2 (-DEXPECT_FMA=0: only FMUL is; FMLA's rescue + must decline every case). The rescues are counted inside the generated wrappers (A64_FPX_STICKY_PROBE, empty in the + real build), and one under a closed gate (an FPCR control bit but FZ, or IXC clear) counts as a mismatch: the exact + body models none of FIZ/AH/NEP, so only the gate keeps the rescue away from them. + + fpx_sticky_test lifted guest FMUL/FNMUL/FMADD/FMLA S words as the lifter emits them (fpx_lifted.h: the + outlined and the hot-function form) against the same words with FPX off, on + inputs that must be rescued and inputs that must fall back: whole Ctx and the + rescue count + fpx_sticky_test random n random operand triples (both ops each) over random FPCR/FPSR states + fpx_sticky_test sweep every a with biased exponent <= 4 or >= 250 whose fraction is below 2^12, above + 2^23 - 2^12 or a multiple of stride, x a fixed set of b and addends, x FZ on/off, + x flags all set / IXC only, and one more state per a (partial flags, IXC + clear, FIZ/AH/NEP) + Exit 0: no mismatch, the wrappers rescued FMUL cases, and FMLA ones exactly when EXPECT_FMA. 1: a mismatch. + 3: the CPU lacks this build's -march (x86-64-v3: FMA/AVX2; x86-64-v2: SSSE3/SSE4.2/POPCNT). 4: rescue counts wrong. */ +static unsigned long long g_sticky; /* the generated wrappers' rescues */ +#define A64_FPX_STICKY_PROBE() ((void)g_sticky++) +#include "a64.h" +#include +#include +#include +#if !defined(EXPECT_FMA) || RECOMP_FPX_HOST != 2 || RECOMP_FPX_FMA64 != EXPECT_FMA +#error "build with -DEXPECT_FMA=1 at x86-64-v3 or -DEXPECT_FMA=0 at x86-64-v2 (RECOMP_FPX_FMA64 must agree)" +#endif +/* The FPCR bits the rescue's soundness needs clear (the test's own copy of RECOMP_FPX_FPCR_MASK): every control bit + but FZ (FIZ AH NEP, trap enables, FZ16, RMode, DN, AHP) and the host's kill switch. */ +#define GATE_FPCR (UINT64_C(0x06C8FF07) | (UINT64_C(1) << 32)) +#include "fpx_lifted.h" + +static uint64_t rs; +static uint64_t rnd(void) { rs ^= rs << 13; rs ^= rs >> 7; rs ^= rs << 17; return rs; } +static uint32_t r32(void) { return (uint32_t)(rnd() >> 16); } +static uint32_t rbelow(uint32_t n) { return (uint32_t)(((rnd() >> 32) * n) >> 32); } +static uint32_t mkf(uint32_t sign, int e, uint32_t frac) { return (sign << 31) | ((uint32_t)e << 23) | (frac & 0x7fffff); } +static float F(uint32_t x) { float f; memcpy(&f, &x, 4); return f; } +static uint32_t B(float f) { uint32_t x; memcpy(&x, &f, 4); return x; } + +/* ---- operand classes and states (fpstub's harness) ---- */ +static uint32_t sparse_frac(void) { /* fractions with few bits: exact products, ties, cancellations */ + switch (rbelow(4)) { + case 0: return 0; + case 1: return (uint32_t)1 << rbelow(23); + case 2: return r32() & 0x7ff000; + default: return r32() & 0x7fffff; + } +} +static uint32_t any_operand(void) { + const uint32_t s = r32() & 1; + switch (rbelow(16)) { + case 0: return s << 31; /* zero */ + case 1: return mkf(s, 0, r32() | 1); /* subnormal */ + case 2: return mkf(s, 0, rbelow(8) + 1); /* tiny subnormal */ + case 3: return mkf(s, 0, 0x7fffff - rbelow(8)); /* large subnormal */ + case 4: return mkf(s, 1 + rbelow(3), sparse_frac()); /* smallest normals */ + case 5: return mkf(s, 254 - rbelow(3), sparse_frac()); /* largest normals */ + case 6: return mkf(s, 255, 0); /* inf */ + case 7: return mkf(s, 255, (r32() | 1) & 0x7fffff); /* NaN, quiet or signalling */ + case 8: return mkf(s, 255, 0x400000 | (r32() & 0x3fffff)); /* quiet NaN */ + case 9: case 10: return mkf(s, 120 + rbelow(15), sparse_frac()); /* near 1 */ + default: return mkf(s, 1 + rbelow(254), r32()); /* any normal */ + } +} +static void tiny_pair(uint32_t *a, uint32_t *b) { /* a product near the bottom of the normal range, or the overflow edge */ + const int target = (rbelow(8) == 0) ? 255 - (int)rbelow(3) : (int)rbelow(60) - 30; /* biased product exponent */ + int ea = 1 + (int)rbelow(253); + int eb = target - ea + 127; + if (eb < 1 || eb > 254) { eb = 1 + (int)rbelow(254); ea = target - eb + 127; } + if (ea < 1) ea = rbelow(4) ? 1 : 0; + if (ea > 254) ea = 254; + *a = mkf(r32() & 1, ea, rbelow(2) ? sparse_frac() : r32()); + *b = mkf(r32() & 1, eb, rbelow(2) ? sparse_frac() : r32()); +} +static uint32_t addend_for(uint32_t a, uint32_t b) { + const double p = (double)F(a) * (double)F(b); + switch (rbelow(10)) { + case 0: return any_operand(); + case 1: return B((float)-p); /* near (often exact) cancellation */ + case 2: return B((float)-p) + (uint32_t)((int)rbelow(9) - 4); /* a few ulps off */ + case 3: return r32() & 0x80000000u; /* a zero */ + case 4: return B((float)p) ^ (rbelow(2) ? 0x80000000u : 0); + case 5: return mkf(r32() & 1, 0, r32()); /* subnormal */ + case 6: return mkf(r32() & 1, 1 + rbelow(4), r32()); /* near the smallest normal */ + default: return mkf(r32() & 1, 1 + rbelow(254), r32()); + } +} +static void gen_state(uint64_t *fpcr, uint64_t *fpsr) { + uint64_t cr = 0, sr = 0; + if (rbelow(2)) cr |= 1u << 24; /* FZ */ + if (rbelow(10) == 0) cr |= 1u << 25; /* DN (gate closed) */ + if (rbelow(10) == 0) cr |= (uint64_t)rbelow(4) << 22; /* RMode */ + if (rbelow(40) == 0) cr |= r32() & 0x07ff9f07u; /* anything else (FIZ, AH, NEP included) */ + if (rbelow(16) == 0) cr |= 1u << rbelow(3); /* FIZ, AH or NEP alone (the gate must stay closed) */ + if (rbelow(200) == 0) cr |= RECOMP_FPX_INHIBIT; + static const uint32_t bits[6] = {1, 2, 4, 8, 16, 128}; /* IOC DZC OFC UFC IXC IDC */ + for (int k = 0; k < 6; k++) + if (k == 4 ? rbelow(10) != 0 : rbelow(3) != 0) sr |= bits[k]; /* IXC 90%, the others 67% */ + if (rbelow(8) == 0) sr |= 1u << 27; /* QC */ + *fpcr = cr; + *fpsr = sr; +} + +/* ---- one check: both wrappers against the exact body ---- */ +static uint64_t checks, mismatches, rescued[2], closed_checks; +static int report_left = 20; +static void check(uint64_t cr, uint64_t sr, uint32_t a, uint32_t b, uint32_t z, int mul) { + Ctx ref, got; + memset(&ref, 0, sizeof ref); + ref.fpcr = cr; ref.fpsr = sr; + got = ref; + const uint64_t r = recomp_fpe_fma32(&ref, a, b, z); + const unsigned long long before = g_sticky; + const uint64_t v = mul ? a64_fpx_mul32_fma32(&got, a, b, z) : a64_fpx_fma32_fma32(&got, a, b, z); + const int kept = g_sticky != before, closed = (cr & GATE_FPCR) || !(sr & 16u); + checks++; + rescued[mul] += kept; + closed_checks += closed; + if (kept && closed) { + mismatches++; + if (report_left-- > 0) + printf("GATE %s a=%08x b=%08x z=%08x fpcr=%llx fpsr=%llx: rescued under a closed gate\n", mul ? "FMUL" : "FMLA", + a, b, z, (unsigned long long)cr, (unsigned long long)sr); + } else if (v != r || got.fpsr != ref.fpsr) { + mismatches++; + if (report_left-- > 0) + printf("MISMATCH %s a=%08x b=%08x z=%08x fpcr=%llx fpsr=%llx: exact %08llx/%llx wrapper %08llx/%llx\n", + mul ? "FMUL" : "FMLA", a, b, z, (unsigned long long)cr, (unsigned long long)sr, (unsigned long long)r, + (unsigned long long)ref.fpsr, (unsigned long long)v, (unsigned long long)got.fpsr); + } +} +static void both(uint64_t cr, uint64_t sr, uint32_t a, uint32_t b, uint32_t z) { + check(cr, sr, a, b, (a ^ b) & 0x80000000u, 1); /* FMUL: the lifter passes a zero of the product's sign */ + check(cr, sr, a, b, z, 0); +} + +static void random_cases(uint64_t n, uint64_t seed) { + rs = seed * 0x9E3779B97F4A7C15ULL + 1; + for (uint64_t i = 0; i < n; i++) { + uint32_t a, b; uint64_t cr, sr; + if (rbelow(2)) tiny_pair(&a, &b); + else { a = any_operand(); b = any_operand(); } + const uint32_t z = addend_for(a, b); + gen_state(&cr, &sr); + both(cr, sr, a, b, z); + } +} + +static const uint32_t B_SET[] = { + 0x3f800000, /* 1 */ 0x3f000000, /* 0.5 */ 0x3fc00001, /* 1.5+ulp */ 0x00800000, /* min normal */ 0x7f7fffff, /* max */ + 0x80000001, /* -min subnormal */ 0x007fffff, /* max subnormal */ 0x7f800000, /* inf */ 0x00000000, /* +0 */ + 0x5f000000, /* 2^63 */ 0x3fffffff, /* ~2 */ 0x1f800000 /* 2^-64 */}; +/* one more state per a: partial flags (each needed bit clear in turn), IXC clear, and FIZ / AH / NEP (gate closed) */ +static const uint32_t MORE[16][2] = { + {0x01000000u, 0x18u}, {0, 0x18u}, {0x01000000u, 0x90u}, {0x01000000u, 0x94u}, {0, 0x14u}, {0x01000000u, 0x98u}, + {0x01000000u, 0x1cu}, {0, 0x1cu}, {0x01000000u, 0x9bu}, {0x01000000u, 0x97u}, {0x01000000u, 0x1fu}, + {0x01000000u, 0x8fu}, {0x01000002u, 0x9fu}, {0x01000001u, 0x9fu}, {0x01000004u, 0x9fu}, {0x00000002u, 0x9fu}}; +static void sweep(uint32_t stride) { + for (unsigned bi = 0; bi < sizeof B_SET / sizeof B_SET[0]; bi++) { + const uint32_t b = B_SET[bi]; + for (uint32_t hi = 0; hi < 2 * 256; hi++) { /* sign and biased exponent of a */ + const uint32_t e = hi & 255; + if (!(e <= 4 || e >= 250)) continue; + for (uint32_t f = 0; f < (1u << 23); f++) { + if (f >= 4096 && f < (1u << 23) - 4096 && f % stride) continue; + const uint32_t a = (hi >> 8) << 31 | e << 23 | f; + const uint64_t cr = (f & 1) ? 0x01000000u : 0; /* FZ */ + const uint64_t sr = (f & 2) ? 0x9fu : 0x10u; /* every sticky bit set, or IXC only */ + uint32_t z; + switch ((f >> 2) & 3) { /* addend: cancellation, near the smallest normal, subnormal, zero */ + case 0: z = B(-(float)((double)F(a) * (double)F(b))); break; + case 1: z = 0x00800000u ^ (a & 0x80000000u); break; + case 2: z = 0x00000003u ^ (f << 31); break; + default: z = 0x80000000u; break; + } + both(cr, sr, a, b, z); + both(cr ^ 0x01000000u, sr ^ 0x8fu, a, b, z); /* the other FZ and flag state, same operands */ + const uint32_t *m = MORE[((f >> 4) + hi + bi) & 15]; + both(m[0], m[1], a, b, z); + } + } + } +} + +/* ---- the lifted route: guest words through a64emit and a64_lift.op_text ---- */ +#define FZ 0x01000000u +static const struct { uint32_t a, b, z; uint32_t fpcr, fpsr; int rescue; const char *what; } LIFTED[] = { + {0x1c800000u, 0x1c800000u, 0, FZ, 0x9fu, 1, "tiny product, FZ, flags set: a zero"}, + {0x1c800000u, 0x9c800000u, 0x80000000u, FZ, 0x9fu, 1, "tiny negative product, FZ, flags set: -0"}, + {0x1c800000u, 0x1e000000u, 0, 0, 0x9fu, 1, "tiny product, no FZ, UFC set: the subnormal"}, + {0x71800000u, 0x71800000u, 0x3f800000u, FZ, 0x9fu, 1, "overflow, OFC set: inf"}, + {0x00000005u, 0x3fc00000u, 0x3f800000u, FZ, 0x9fu, 1, "subnormal operand, FZ, IDC set: flushed"}, + {0x1c800000u, 0x1c800000u, 0, FZ, 0x10u, 0, "tiny product, FZ, IXC only: falls back"}, + {0x1c800000u, 0x1e000000u, 0, 0, 0x97u, 0, "tiny product, no FZ, UFC clear: falls back"}, + {0x71800000u, 0x71800000u, 0x3f800000u, FZ, 0x9bu, 0, "overflow, OFC clear: falls back"}, + {0x00000005u, 0x3fc00000u, 0x3f800000u, FZ, 0x1fu, 0, "subnormal operand, FZ, IDC clear: falls back"}, + {0x7fc00001u, 0x3f800000u, 0x3f800000u, FZ, 0x9fu, 0, "NaN operand: falls back"}, + {0x1c800000u, 0x1c800000u, 0, FZ | 2u, 0x9fu, 0, "tiny product, FZ|AH (gate closed): falls back"}, + {0x3fc00000u, 0x40000000u, 0x3e800000u, FZ, 0x9fu, 0, "normal (the fast path keeps it)"}, +}; +static int lifted_cases(void) { + unsigned bad = 0, n = 0, kept = 0; + for (unsigned g = 0; g < sizeof GUEST / sizeof GUEST[0]; g++) + for (unsigned k = 0; k < sizeof LIFTED / sizeof LIFTED[0]; k++) + for (int hot = 0; hot < 2; hot++) { + Ctx ref, got; + memset(&ref, 0, sizeof ref); + for (int r = 0; r < 32; r++) { ref.vreg[r][0] = 0x0123456789abcdefULL * (r + 1); ref.vreg[r][1] = ~ref.vreg[r][0]; } + ref.vreg[1][0] = (ref.vreg[1][0] & ~0xffffffffULL) | LIFTED[k].a; + ref.vreg[2][0] = (ref.vreg[2][0] & ~0xffffffffULL) | LIFTED[k].b; + ref.vreg[3][0] = (ref.vreg[3][0] & ~0xffffffffULL) | LIFTED[k].z; /* FMADD's addend */ + ref.vreg[0][0] = (ref.vreg[0][0] & ~0xffffffffULL) | LIFTED[k].z; /* FMLA's accumulator */ + ref.fpcr = LIFTED[k].fpcr; + ref.fpsr = LIFTED[k].fpsr; + got = ref; + GUEST[g].exact(&ref); + const unsigned long long before = g_sticky; + (hot ? GUEST[g].hot : GUEST[g].lifted)(&got); + const unsigned long long rescued = g_sticky - before; + const unsigned want = LIFTED[k].rescue && (GUEST[g].mul || EXPECT_FMA); + n++; + kept += (unsigned)rescued; + if (rescued != want || memcmp(&ref, &got, sizeof ref)) { + bad++; + printf("LIFTED %s (%s), %s: rescued %llu, want %u; s0 exact %08x/fpsr %llx, lifted %08x/fpsr %llx%s\n", + GUEST[g].name, hot ? "hot" : "outlined", LIFTED[k].what, rescued, want, (uint32_t)ref.vreg[0][0], + (unsigned long long)ref.fpsr, (uint32_t)got.vreg[0][0], (unsigned long long)got.fpsr, + memcmp(&ref, &got, sizeof ref) ? " (Ctx differs)" : ""); + } + } + printf("lifted (%s): %u runs of %u guest words, %u mismatches, %u rescued in the wrappers\n", + EXPECT_FMA ? "x86-64-v3" : "x86-64-v2", n, (unsigned)(sizeof GUEST / sizeof GUEST[0]), bad, kept); + return bad ? 1 : 0; +} + +int main(int argc, char **argv) { + unsigned eax, ebx, ecx, edx; + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx) || !(ecx >> 9 & 1) || !(ecx >> 20 & 1) || !(ecx >> 23 & 1)) { + printf("this CPU has no SSSE3/SSE4.2/POPCNT (x86-64-v2)\n"); + return 3; + } + if (EXPECT_FMA && (!(ecx >> 12 & 1) || !__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx) || !(ebx >> 5 & 1))) { + printf("this CPU has no FMA/AVX2 (x86-64-v3)\n"); + return 3; + } + if ((_mm_getcsr() & 0xffc0) != 0x1f80) { printf("MXCSR %x: not the FPX1 host contract\n", _mm_getcsr()); return 2; } + if (argc == 2 && !strcmp(argv[1], "lifted")) return lifted_cases(); + if (argc == 4 && !strcmp(argv[1], "random")) random_cases(strtoull(argv[2], 0, 0), strtoull(argv[3], 0, 0)); + else if (argc == 3 && !strcmp(argv[1], "sweep")) sweep((uint32_t)strtoul(argv[2], 0, 0)); + else { fprintf(stderr, "usage: lifted | random | sweep \n"); return 2; } + printf("%s (%s): %llu checks (%llu with the gate closed), %llu mismatches, the wrappers rescued %llu FMUL / %llu FMLA\n", + argv[1], EXPECT_FMA ? "x86-64-v3" : "x86-64-v2", (unsigned long long)checks, (unsigned long long)closed_checks, + (unsigned long long)mismatches, (unsigned long long)rescued[1], (unsigned long long)rescued[0]); + if (mismatches) return 1; + if (!rescued[1] || (EXPECT_FMA ? !rescued[0] : rescued[0] != 0)) { + printf("rescue counts wrong: want FMUL > 0 and FMLA %s\n", EXPECT_FMA ? "> 0" : "== 0 (no hardware FMA)"); + return 4; + } + return 0; +} diff --git a/tests/test_fpx_sticky.py b/tests/test_fpx_sticky.py new file mode 100644 index 000000000..5745df161 --- /dev/null +++ b/tests/test_fpx_sticky.py @@ -0,0 +1,208 @@ +"""FPX1's sticky-flag rescue for single-precision FMUL/FNMUL/FMLA, game-free: bit-exact with the exact body. + +a64_lift.py's a64_fpx_mul32_fma32 and a64_fpx_fma32_fma32 (the cold wrappers lifted code calls when the inline fast path +cannot prove its result) try a64_fpx_sticky32 before the exact body recomp_fpe_fma32: when every FPSR flag the Arm op +could raise is already set, one hardware FMA (or multiply) is the Arm result. This builds what the real build builds, +a64_fpx.h from a64_lift.fpx_header() and a64_suyu.h / a64_suyu.c from a64_rtgen.py (through suyu's translator, +a64emit.cpp, built here at -O0 into a temporary folder), and compiles tests/fpx_sticky_test.c against the real rt/a64.h +at the build's FP flags (-ffp-contract=off -O2) twice: at -march=x86-64-v3 (RECOMP_FPX_FMA64: both ops rescued) and at +-march=x86-64-v2 (what host_march() gives a CPU without AVX2: FMUL's rescue is a plain multiply and FMLA's must decline +every case). The v2 build runs on any x86-64-v2 CPU, whether or not it has AVX2. In each build: + - the lifted route: guest FMUL, FNMUL, FMADD and FMLA (by element) S words, hand-encoded, through a64emit.translate and + a64_lift.op_text (the lifter's per-instruction pipeline, so the C calls the outlined wrapper as lifted code does) and + hot_text (the hot functions' form), against the same words translated with FPX off (the exact body inline): the + whole Ctx and the rescue count must match, on 5 inputs that must be rescued and 7 that must not (96 runs); +and each wrapper's value and FPSR must equal recomp_fpe_fma32's for: + - 8M random operand triples (16M checks, seeded): zeros, subnormals, inf, NaN (quiet and signalling), the extreme + normals, products near the underflow and overflow edges, exact and near cancellations, under random FPCR (FZ, + DN, RMode, FIZ/AH/NEP alone or with other bits, the kill switch) and FPSR states; + - an edge sweep (~150M checks): every a with biased exponent <= 4 or >= 250 whose fraction is below 2^12, above + 2^23 - 2^12 or a multiple of 97, against 12 fixed b and 4 addend kinds, each under FZ on and off with the flags + all set or IXC only, and under one more state (by a): partial flags (UFC set and IDC clear, IDC set and UFC clear, + OFC only, every bit but one of IDC/UFC/OFC, ...), IXC clear, or FIZ/AH/NEP set. +The rescues are counted inside the generated wrappers (an A64_FPX_STICKY_PROBE() the driver defines; empty in the real +build): both builds must see FMUL rescues, v3 FMLA rescues and v2 none, and a rescue under a closed gate (an FPCR +control bit but FZ set, or IXC clear) fails. That last check is what guards FIZ/AH/NEP: the exact body models none of +them, so comparing values could not tell. About 35-50 s here. + +It has teeth (each by hand, then restored): + - the emitter no longer outlining FMUL/FNMUL S (arm64_to_c.h's EmitFPMulAddValue: `FpxOutline()` made + `FpxOutline() && !mul`, so the guest FMUL keeps suyu's inline guard): the lifted case fails in both builds (20 + FMUL/FNMUL runs with no rescue), while the random and sweep runs, which call the wrappers by name, still pass; + - the wrappers' route to the rescue removed (fpx_header's `(fast, exact) in (("mul32", "fma32"), ("fma32", "fma32"))` + made never true): every run fails with no rescue counted, though every value still matches; + - FMUL's wrapper passing mul=0 (an FMA with the zero addend, the same value with a hardware FMA): v2 fails (exit 4); + - FPCR bits 0-2 dropped from RECOMP_FPX_FPCR_MASK (0x06C8FF07 -> 0x06C8FF00): every value still matches, but 190k of + v3's random checks and 6.1M of its sweep's are rescues under FIZ/AH/NEP (v2: 99k and 2.6M); + - the helper's UFC test removed (a64_fpx_sticky32's `if (!(s & 8u))` made `if (0)`, so a tiny result is kept with + UFC clear): 486k and 3.5M mismatch at v3 (FPSR without UFC), 258k and 475k at v2; + - its IDC test dropped (a flushed subnormal operand with IDC clear): 383k and 10.0M at v3, 146k and 3.8M at v2. + + python -I -B tests/test_fpx_sticky.py [--cc clang] [--cxx clang++] + +Skipped (exit 0) on a host that is not x86-64 or has no C/C++ compiler (PATH, $CC/$CXX, or the build's own emsdk / +HOOKSHOT_MINGW toolchain), or on a CPU without x86-64-v2; the v3 build alone is skipped on a CPU without x86-64-v3. +""" +import argparse +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +KIT = os.path.join(ROOT, "third_party", "a64-kit") +sys.path.insert(0, os.path.join(KIT, "lift")) +sys.path.insert(0, os.path.join(KIT, "tools")) +import toolchain # noqa: E402 + +RANDOM_TRIPLES, SEED, SWEEP_STRIDE = 8000000, 20261109, 97 +FLAGS = ["-ffp-contract=off", "-fno-strict-aliasing", "-O2"] # (nx2/build.py's lifted-code FP flags, with -march=) +BUILDS = (("x86-64-v3", 1), ("x86-64-v2", 0)) # (-march, EXPECT_FMA: whether RECOMP_FPX_FMA64 holds there) + + +def skip(why): + print("fpx sticky: skipped (%s)" % why) + sys.exit(0) + + +def compiler(given, env, names, build_name): + """--cc/--cxx, else $CC/$CXX, else PATH, else the toolchain the build itself uses (emsdk, HOOKSHOT_MINGW)""" + for c in (given, os.environ.get(env)): + if c: + return shutil.which(c) or (c if os.path.isfile(c) else None) + for n in names: + if shutil.which(n): + return shutil.which(n) + c = toolchain.find(build_name, host=True) + return c if os.path.isfile(c) else None + + +def target_flags(cc, tmp): + """clang on Windows: the build's target and lld (as tools/toolchain.py, with its lld-link copy in tmp, not in tools/)""" + if os.name != "nt" or "clang" not in os.path.basename(cc).lower(): + return [] + flags = toolchain.native_target(host=True) + if toolchain.mingw(host=True): + return flags + toolchain.native_link(host=True) + here = os.path.dirname(cc) + link = shutil.which("lld-link", path=here) or shutil.which("lld-link") + if not link: + lld = shutil.which("lld", path=here) or shutil.which("lld") + if not lld: + return flags + link = os.path.join(tmp, "lld-link.exe") + if not os.path.exists(link): + shutil.copyfile(lld, link) + return flags + ["-fuse-ld=lld", "-B" + os.path.dirname(link), "-Wl,/nodefaultlib:oldnames"] + + +def run(cmd, timeout=600): + kw = {"creationflags": subprocess.BELOW_NORMAL_PRIORITY_CLASS | subprocess.CREATE_NO_WINDOW} if os.name == "nt" \ + else {"preexec_fn": lambda: os.nice(10)} + return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, errors="replace", + timeout=timeout, **kw) + + +def build(cmd, what): + r = run(cmd) + if r.returncode: + print(r.stdout) + raise SystemExit("fpx sticky: building %s failed" % what) + + +# Guest instructions for the lifted-route case (hand-written encodings): each runs as the lifter emits it, through +# a64emit.translate and a64_lift.op_text (and hot_text), against the same word translated with FPX off. +GUEST = (("FMUL s0, s1, s2", 0x1E220820, 1), ("FNMUL s0, s1, s2", 0x1E228820, 1), + ("FMADD s0, s1, s2, s3", 0x1F020C20, 0), ("FMLA s0, s1, v2.s[0]", 0x5F821020, 0)) # (name, word, FMUL) + + +def lifted_cases(a64emit, a64_lift, used): + """fpx_lifted.h: per guest word its lifted, hot and exact-only (FPX off) C as functions, and the GUEST table""" + words = [(0x1000, w) for _, w, _ in GUEST] + lifted, fl = a64emit.translate(words) + if any(fl): + raise SystemExit("fpx sticky: a64emit did not translate the guest words: %s" % fl) + lifted = [a64_lift.op_text(t, used) for t in lifted] + a64emit.lib().a64_config(1, 0) # (FPX off: the exact body inline, the reference) + try: + exact, _ = a64emit.translate(words) + finally: + a64emit.lib().a64_config(1, 1) + out = ["/* Generated by tests/test_fpx_sticky.py from guest words, as a64_lift.py lifts them. */"] + for i, (lt, ex) in enumerate(zip(lifted, exact)): + out.append("static void g_lifted%d(Ctx *__restrict c) { %s }" % (i, lt)) + out.append("static void g_hot%d(Ctx *__restrict c) { %s }" % (i, a64_lift.hot_text(lt))) + out.append("static void g_exact%d(Ctx *__restrict c) { %s }" % (i, ex)) + out.append("static const struct { const char *name; int mul; void (*lifted)(Ctx *), (*hot)(Ctx *), (*exact)(Ctx *); }" + " GUEST[] = {%s};" % ", ".join('{"%s", %d, g_lifted%d, g_hot%d, g_exact%d}' % (n, m, i, i, i) + for i, (n, _, m) in enumerate(GUEST))) + return "\n".join(out) + "\n" + + +def generate(cxx, tmp, flags): + """a64_fpx.h, a64_suyu.h, a64_suyu.c and fpx_lifted.h, by the generators the lift runs (a64emit built into tmp)""" + dll = os.path.join(tmp, "a64emit.dll" if os.name == "nt" else "liba64emit.so") + build([cxx, "-std=c++20", "-O0", "-shared", "-Wno-everything", "-o", dll, os.path.join(KIT, "lift", "a64emit.cpp")] + + flags + ([] if os.name == "nt" else ["-fPIC"]), "a64emit") + import build_emit + build_emit.build = lambda force=False: dll # (not a64emit.dll beside the source) + import a64_lift + import a64_rtgen + import a64emit + used = set() + try: + a64_rtgen.write(tmp) + lifted = lifted_cases(a64emit, a64_lift, used) + finally: + if a64emit._lib is not None: # (unloaded, so the folder can go) + import _ctypes + (_ctypes.FreeLibrary if os.name == "nt" else _ctypes.dlclose)(a64emit._lib._handle) + a64emit._lib = None + with open(os.path.join(tmp, "fpx_lifted.h"), "w", newline="\n") as f: + f.write(lifted) + with open(os.path.join(tmp, "a64_fpx.h"), "w", newline="\n") as f: # (the guest words' shapes, and the two the + f.write(a64_lift.fpx_header(used | {("mul32", "fma32", 2, 3), ("fma32", "fma32", 3, 3)})) # runs call by name) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--cc") + p.add_argument("--cxx") + a = p.parse_args() + if platform.machine().lower() not in ("x86_64", "amd64"): + skip("not an x86-64 host: %s" % platform.machine()) + cc = compiler(a.cc, "CC", ("clang", "gcc", "cc"), "clang") + cxx = compiler(a.cxx, "CXX", ("clang++", "g++", "c++"), "clang++") + if not cc or not cxx: + skip("no C compiler" if not cc else "no C++ compiler") + t0 = time.time() + failed, ran = [], [] + with tempfile.TemporaryDirectory(prefix="hookshot-fpx-sticky-") as tmp: + generate(cxx, tmp, target_flags(cxx, tmp)) + for march, fma in BUILDS: + exe = os.path.join(tmp, "fpx_sticky_test_%s.exe" % march) + build([cc] + target_flags(cc, tmp) + ["-march=" + march] + FLAGS + + ["-DEXPECT_FMA=%d" % fma, "-D_CRT_SECURE_NO_WARNINGS", "-I" + os.path.join(KIT, "rt"), "-I" + tmp, + os.path.join(ROOT, "tests", "fpx_sticky_test.c"), os.path.join(tmp, "a64_suyu.c"), "-o", exe], + "the driver at " + march) + for args in (["lifted"], ["random", str(RANDOM_TRIPLES), str(SEED)], ["sweep", str(SWEEP_STRIDE)]): + r = run([exe] + args) + print(r.stdout.rstrip()) + if r.returncode == 3: + print("fpx sticky: %s skipped (this CPU is not %s)" % (march, march)) + break + ran.append(march) + if r.returncode: + failed.append("%s %s" % (march, args[0])) + if not ran: + skip("this CPU is not x86-64-v2") + if failed: + raise SystemExit("fpx sticky: FAIL (%s) in %.1f s" % (", ".join(failed), time.time() - t0)) + print("fpx sticky: pass in %.1f s" % (time.time() - t0)) + + +if __name__ == "__main__": + main() diff --git a/third_party/a64-kit/lift/a64_lift.py b/third_party/a64-kit/lift/a64_lift.py index b184db796..1d3b7d503 100644 --- a/third_party/a64-kit/lift/a64_lift.py +++ b/third_party/a64-kit/lift/a64_lift.py @@ -602,34 +602,85 @@ def add(m): # Results past the fast path's range once their flags are already raised (TOTK's threads: FPSR = 0x9f, every cumulative -# flag set; FPCR.FZ = 1). Overflow (OFC raised): an infinite result of finite operands is the native one, the -# round-to-nearest ±inf (~1M multiplies a second at the castle went through the exact body for it). Flush-to-zero (UFC -# raised): a result below the normal range is the native one flushed to a zero of its sign. Sums are exact there (a -# tiny sum of normal numbers is a subnormal exactly), products flush when the exact product is tiny, which the native -# result shows unless it rounded up to the smallest normal (then the exact body decides). No subnormal operand (those -# flush first). +# flag set; FPCR.FZ = 1). FADD/FSUB S (a64_fpx_fz_add32 / a64_fpx_fz_sub32, OFC or UFC raised): an infinite result of +# finite operands is the native one, the round-to-nearest ±inf (~1M multiplies a second at the castle went through the +# exact body for it); under FZ with UFC raised a result below the normal range is the native one flushed to a zero of its +# sign. Sums are exact there (a tiny sum of normal numbers is a subnormal exactly). No subnormal operand (those flush +# first). FMUL/FNMUL/FMLA-family S: a64_fpx_sticky32 below. FPX_FZ_HEADER = r""" -#define A64_FPX_FZ_OPEN(c) (RECOMP_FPX_OPEN(c) && ((c)->fpsr & 0xCu)) -static inline __attribute__((always_inline)) int a64_fpx_fz_result(const Ctx *c, uint64_t a, uint64_t b, uint64_t t, int mul, uint64_t *v) { +static inline __attribute__((always_inline)) int a64_fpx_fz_result(const Ctx *c, uint64_t a, uint64_t b, uint64_t t, uint64_t *v) { const uint32_t m = (uint32_t)t & 0x7fffffffu; if (m == 0x7f800000u) return (c->fpsr & 4u) && RECOMP_FPX_FIN32(a) && RECOMP_FPX_FIN32(b) ? (*v = t, 1) : 0; if (!RECOMP_FPX_FZ(c) || !(c->fpsr & 8u) || m > 0x7f800000u || RECOMP_FPX_SUBN32(a) || RECOMP_FPX_SUBN32(b)) return 0; - if (m >= 0x00800000u) return (mul && m == 0x00800000u) ? 0 : (*v = t, 1); + if (m >= 0x00800000u) return (*v = t, 1); *v = t & 0x80000000u; return 1; } #if RECOMP_FPX_HOST static inline __attribute__((always_inline)) int a64_fpx_fz_add32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { - return a64_fpx_fz_result(c, a, b, recomp_fpx_fb(recomp_fpx_f(a) + recomp_fpx_f(b)), 0, v); } + return a64_fpx_fz_result(c, a, b, recomp_fpx_fb(recomp_fpx_f(a) + recomp_fpx_f(b)), v); } static inline __attribute__((always_inline)) int a64_fpx_fz_sub32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { - return a64_fpx_fz_result(c, a, b, recomp_fpx_fb(recomp_fpx_f(a) - recomp_fpx_f(b)), 0, v); } -static inline __attribute__((always_inline)) int a64_fpx_fz_mul32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { - return a64_fpx_fz_result(c, a, b, recomp_fpx_fb(recomp_fpx_f(a) * recomp_fpx_f(b)), 1, v); } + return a64_fpx_fz_result(c, a, b, recomp_fpx_fb(recomp_fpx_f(a) - recomp_fpx_f(b)), v); } #else static inline int a64_fpx_fz_add32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { (void)c; (void)a; (void)b; (void)v; return 0; } static inline int a64_fpx_fz_sub32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { (void)c; (void)a; (void)b; (void)v; return 0; } -static inline int a64_fpx_fz_mul32(const Ctx *c, uint64_t a, uint64_t b, uint64_t *v) { (void)c; (void)a; (void)b; (void)v; return 0; } #endif +/* FMUL/FNMUL/FMLA-family S past the fast path (the "sticky" rescue of the outlined wrappers a64_fpx_mul32_fma32 and + a64_fpx_fma32_fma32, inside their RECOMP_FPX_OPEN gate): one hardware multiply or fused multiply-add, kept when every + FPSR bit the Arm op could raise is already set and the value is the Arm one. + FZ: a subnormal operand is flushed to a zero of its sign (IDC). NaN: never. Infinite: exact with an infinite operand, + else an overflow (OFC). A zero product or an exact cancellation (a*b == -z in binary64, where the product is exact): + exact. Otherwise a result at most the smallest normal may be tiny (UFC): FZ clear, the IEEE gradual-underflow result is + Arm's; FZ set, below the smallest normal the exact value is tiny and Arm returns a zero of its sign, and exactly the + smallest normal is ambiguous (falls back). Rejected before computing: a NaN operand, a subnormal operand under FZ with + IDC clear, and FMUL of finite operands with neither OFC nor UFC set and nothing flushed (past the fast path, which keeps + a large product and a finite product of a zero, such a product is tiny or infinite). Without a hardware FMA + (RECOMP_FPX_FMA64; binary32 fmaf is the same instruction class) only the multiply is rescued. Returning 0 is always + sound: the wrapper then takes the exact body. */ +#ifndef A64_FPX_STICKY_PROBE +#define A64_FPX_STICKY_PROBE() ((void)0) /* (tests/test_fpx_sticky.py counts the rescues the wrappers keep here) */ +#endif +static inline __attribute__((always_inline)) int a64_fpx_sticky32(const Ctx *c, uint64_t a, uint64_t b, uint64_t z, int mul, uint64_t *v) { +#if RECOMP_FPX_HOST + const uint64_t s = c->fpsr; + uint32_t x = (uint32_t)a, y = (uint32_t)b, w = (uint32_t)z, t; +#if !RECOMP_FPX_FMA64 + if (!mul) return 0; +#endif + if ((x & 0x7fffffffu) > 0x7f800000u || (y & 0x7fffffffu) > 0x7f800000u || (w & 0x7fffffffu) > 0x7f800000u) return 0; + if (RECOMP_FPX_FZ(c) && (RECOMP_FPX_SUBN32(x) || RECOMP_FPX_SUBN32(y) || (!mul && RECOMP_FPX_SUBN32(w)))) { + if (!(s & 0x80u)) return 0; + if (RECOMP_FPX_SUBN32(x)) x &= 0x80000000u; + if (RECOMP_FPX_SUBN32(y)) y &= 0x80000000u; + if (RECOMP_FPX_SUBN32(w)) w &= 0x80000000u; + } else if (mul && !(s & 0xCu) && RECOMP_FPX_FIN32(x) && RECOMP_FPX_FIN32(y)) { + return 0; + } + /* (FMUL: w is the zero of the product's sign, so x*y + w is x*y) */ +#if RECOMP_FPX_FMA64 + t = (uint32_t)recomp_fpx_fb(mul ? recomp_fpx_f(x) * recomp_fpx_f(y) + : __builtin_fmaf(recomp_fpx_f(x), recomp_fpx_f(y), recomp_fpx_f(w))); +#else + t = (uint32_t)recomp_fpx_fb(recomp_fpx_f(x) * recomp_fpx_f(y)); +#endif + const uint32_t m = t & 0x7fffffffu; + if (m > 0x7f800000u) return 0; + if (m == 0x7f800000u) { + if (RECOMP_FPX_FIN32(x) && RECOMP_FPX_FIN32(y) && RECOMP_FPX_FIN32(w) && !(s & 4u)) return 0; + } else if (m <= 0x00800000u && !RECOMP_FPX_ZERO32(x) && !RECOMP_FPX_ZERO32(y)) { + if (!(s & 8u)) { /* UFC clear: only an exact cancellation (+0, no bit) is kept */ + if (mul || m != 0 || (double)recomp_fpx_f(x) * (double)recomp_fpx_f(y) != -(double)recomp_fpx_f(w)) return 0; + } else if (RECOMP_FPX_FZ(c)) { + if (m == 0x00800000u) return 0; + t &= 0x80000000u; + } + } + *v = t; + return 1; +#else + (void)c; (void)a; (void)b; (void)z; (void)mul; (void)v; return 0; +#endif +} """ @@ -640,6 +691,11 @@ def add(m): HOT_FPX = re.compile(r"\ba64_fpx_(?!hot_|fz_)(\w+)\(c,") +def hot_text(text): + """a hot function's text: its FPX helper calls go through the copies with the fast path inline""" + return HOT_FPX.sub(r"a64_fpx_hot_\1(c,", text) + + def hot_functions(): path = os.environ.get("A64_HOT_FUNCS") if not path: @@ -672,14 +728,17 @@ def fpx_header(used): FPX_FZ_HEADER] for fast, exact, nf, ne in sorted(used): params = ", ".join("uint64_t a%d" % i for i in range(ne)) - fz = "" - if fast in ("add32", "sub32", "mul32"): - fz = "if (A64_FPX_FZ_OPEN(c) && a64_fpx_fz_%s(c, a0, a1, &v)) return v; " % fast + # past the fast path, still inside its gate (tested once): the rescues above, else the exact body + rescue = "" + if (fast, exact) in (("mul32", "fma32"), ("fma32", "fma32")): + rescue = "if (a64_fpx_sticky32(c, a0, a1, a2, %d, &v)) { A64_FPX_STICKY_PROBE(); return v; } " % (fast == "mul32") + elif fast in ("add32", "sub32"): + rescue = "if ((c->fpsr & 0xCu) && a64_fpx_fz_%s(c, a0, a1, &v)) return v; " % fast out.append("A64_FPX_FN uint64_t a64_fpx_%s_%s(Ctx *c, %s) { uint64_t v; " - "if (RECOMP_FPX_OPEN(c) && RECOMP_LIKELY(recomp_fpx_%s(%s, RECOMP_FPX_FZ(c), &v))) return v; " - "%sreturn recomp_fpe_%s(c, %s); }" % (fast, exact, params, fast, - ", ".join("a%d" % i for i in range(nf)), fz, exact, - ", ".join("a%d" % i for i in range(ne)))) + "if (RECOMP_FPX_OPEN(c)) { if (RECOMP_LIKELY(recomp_fpx_%s(%s, RECOMP_FPX_FZ(c), &v))) return v; %s} " + "return recomp_fpe_%s(c, %s); }" % (fast, exact, params, fast, + ", ".join("a%d" % i for i in range(nf)), rescue, exact, + ", ".join("a%d" % i for i in range(ne)))) # (the hot functions' copy, see HOT_FPX: the fast path inline, everything else through the helper) out.append("static inline __attribute__((always_inline)) uint64_t a64_fpx_hot_%s_%s(Ctx *c, %s) { uint64_t v; " "if (RECOMP_FPX_OPEN(c) && RECOMP_LIKELY(recomp_fpx_%s(%s, RECOMP_FPX_FZ(c), &v))) return v; " @@ -698,6 +757,12 @@ def fpx_header(used): return "\n".join(out) + "\n" +def op_text(text, fpx): + """one instruction's emitted C as lifted code has it: FPX guards outlined (their shapes added to fpx, which + fpx_header turns into a64_fpx.h), NEON FP vectorized, stores widened, flags packed""" + return stack_stores(pack_flags(widen_vstores(vectorize_fpx(outline_fpx(text, fpx))))) + + class Lifter: def __init__(self, img, comments=False): self.img = img @@ -1482,7 +1547,7 @@ def write(self, out_dir, chunk_bytes=CHUNK_BYTES): order = list(keys) tx, fl = a64emit.translate([keys[x] for x in order]) fpx = set() - texts = dict(zip(order, zip((stack_stores(pack_flags(widen_vstores(vectorize_fpx(outline_fpx(t, fpx))))) for t in tx), fl))) + texts = dict(zip(order, zip((op_text(t, fpx) for t in tx), fl))) del tx, fl, keys, order with open(os.path.join(out_dir, "a64_fpx.h"), "w", newline="\n") as f: f.write(fpx_header(fpx)) @@ -1516,7 +1581,7 @@ def flush(): for e in sorted(self.funcs): text, refs = self.emit_function(e, texts) if hot and self.fname(e) in hot: - text = HOT_FPX.sub(r"a64_fpx_hot_\1(c,", text) + text = hot_text(text) nhot += 1 if hot_chunk is not None: hot_chunk.append(text) diff --git a/third_party/a64-kit/lift/suyu/arm64_to_c.h b/third_party/a64-kit/lift/suyu/arm64_to_c.h index 1996ad9f3..147445fee 100644 --- a/third_party/a64-kit/lift/suyu/arm64_to_c.h +++ b/third_party/a64-kit/lift/suyu/arm64_to_c.h @@ -334,6 +334,8 @@ inline bool EmitGuardGen() { inline bool g_emit_fpx = false; // Instrumentation only, never timed: with g_emit_fpx, the exact body runs // after every fast path too, and recomp_fpx_shadow compares the two. +// It compares the inline form only: the outlined wrappers' rescues past the +// fast path (a64_lift.py's a64_fpx.h) never run in it. inline bool g_emit_fpx_shadow = false; // Hybrid exports translate the gated forms too. g_translate_all gates only SHL @@ -6535,6 +6537,17 @@ inline const char* FpxH() { the ARM result bit for bit. Anything else takes the unchanged exact body, which FPX1 sites call out of line (recomp_fpe_*, recomp_runtime.c). + Lifted code keeps more than the above: a64_lift.py outlines each guard + into a wrapper (a64_fpx_*, a64_fpx.h) that, past these fast paths and + inside the same gate, also keeps the native FADD/FSUB S result that + overflowed with OFC already set or, under FZ, is flushed with UFC already + set (a64_fpx_fz_add32/sub32), and the native + FMUL/FNMUL/FMLA-family S result, tiny, infinite or of flushed inputs + included, when every flag the op could raise (IDC, UFC, OFC) is already + set (a64_fpx_sticky32; checked game-free by Hookshot's + tests/test_fpx_sticky.py). The shadow build keeps the inline form above + and never reaches those wrappers, so it does not validate them. + Host contract, enforced by the host once it has negotiated FPX1 through recomp_image_fpx_v1: on x86-64 MXCSR round-to-nearest with DAZ and FTZ clear and every exception masked; on AArch64 FPCR 0. Bit 32 of fpcr is the host's