From fa0bfee65790828e68590d21e82cc7d1ece88398 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 11:54:58 +0200 Subject: [PATCH 001/156] Add corpus v2 implementation plan --- docs/CORPUS_V2_IMPLEMENTATION_PLAN.md | 236 ++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 docs/CORPUS_V2_IMPLEMENTATION_PLAN.md diff --git a/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md b/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md new file mode 100644 index 00000000..3db26e65 --- /dev/null +++ b/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md @@ -0,0 +1,236 @@ +# PS2 Optimization Corpus v2 integration plan + +This branch exists to integrate the PS2 Optimization Research Library v2 into the +runtime architecture without destabilising the known-good HDL installer branch. + +## Safety boundary + +- Branch: `perf/corpus-v2-integration` +- Baseline: `4b5aa8d85e86c9de570a2128b52d1eaa5b334844` +- Parent line: `feature/hdl-game-installer` +- No optimization is merged back until correctness tests, CI and the relevant + real-hardware A/B benchmark pass. +- PCSX2 remains useful for correctness and inspection, but timing/cache/DMA/ + FIFO/device claims require a real PlayStation 2. + +## Source-of-truth routing + +Always begin with `PS2_Optimization_Library_v2_MANIFEST.md`, then use +`PS2_PERFORMANCE_BIBLE.md` for the engineering workflow and the authoritative +subsystem corpus for each change. The project uses the following conflict order: + +```text +v2 +> more specialised current corpus +> current source/manual +> real-hardware reproduction +> integrator corpus +> emulator/reverse engineering +> historical forum/anecdote +``` + +Epistemic labels used in review notes and benchmark records: + +- **POTWIERDZONE**: manual, current source or real-hardware reproduction. +- **CURRENT IMPLEMENTATION**: behaviour of the pinned PS2SDK/toolchain source. +- **HISTORYCZNE**: old toolchains, stacks or community measurements. +- **INFERENCJA**: architectural conclusion not directly measured. +- **HIPOTEZA DO TESTU**: proposed change requiring a benchmark on real hardware. + +## Optimization order + +Every work item follows this order unless evidence justifies skipping a step: + +1. remove unnecessary work; +2. do work less often; +3. reduce data volume; +4. improve data layout and locality; +5. batch; +6. remove unnecessary copies and dynamic allocation; +7. add buffering and overlap; +8. use specialised hardware when the workload fits; +9. only then specialise a measured hot kernel. + +Compiler flags, VU/MMI, Scratchpad, larger buffers, 64-byte alignment and custom +low-level APIs are never accepted as universal optimizations. + +## Dataset contract + +Every major runtime dataset or streaming path should eventually document: + +```yaml +name: +producer: +consumers: +lifetime: +representation: +alignment: +transport: +batch_size: +deadline: +ownership_states: +copy_budget: +validation: +``` + +Alignment must state the actual domain: allocator, EE cache line, DMAC/SIF, +device sector/transfer unit, VIF/GIF packet, or another explicit contract. + +## Phase 0: measurement foundation + +Goal: establish evidence before altering architecture. + +- [ ] Record exact PS2SDK/toolchain/build flags automatically in benchmark logs. +- [ ] Record console SCPH/hardware revision, adapters, active IRX and workload. +- [ ] Add per-stage HDL fast-path timing for source I/O, prefetch wait, HDD, + SIF DMA and EE consumer work. +- [ ] Record useful bytes, DMA bytes, CPU-copy bytes and cache-maintenance bytes. +- [ ] Add p50/p95/p99/max reporting for I/O latency, not just average throughput. +- [ ] Keep hot-path logging binary/counter based; format only outside the path. +- [ ] Add R5900 performance-counter harness with companion non-instrumented run. +- [ ] Preserve linker map, symbol sizes and optimization audit in CI artifacts. + +Exit gate: measurements are reproducible on at least one real console and the +instrumented build has a documented overhead A/B against an uninstrumented build. + +## Phase 1: remove known unnecessary code/work + +- [ ] Replace the broad `draw2d` dependency used by the UI with the minimum GIF + primitives actually required, if the ELF A/B confirms removal of unused + arc/trigonometry/libm code. +- [ ] Audit formatted-I/O callsites and replace hot/control-only formatting with + bounded lightweight formatting where this materially reduces `.text`. +- [ ] Investigate current PS2SDK fileXio/newlib timestamp glue that pulls scanf/ + timezone machinery into the ELF. Treat any SDK change as a separate, + source-pinned compatibility patch. +- [ ] Remove source-level work duplicated across transaction stages when the + result can be safely retained under the same ownership/lifetime. + +Exit gate: same functional output, smaller ELF/hot code footprint, no regression +in hardware smoke tests. + +## Phase 2: I-cache and control-flow locality + +- [ ] Split `execute_transaction()` into state/stage handlers without changing + transaction semantics or recovery guarantees. +- [ ] Split other measured multi-kilobyte controller functions only where the + active path benefits from smaller working sets. +- [ ] Evaluate `-Os` for cold translation units and retain `-O2` for measured hot + paths; compare size and latency before adopting per-TU flags. +- [ ] Audit compiler-generated 64-bit divide/mod helpers and eliminate only cases + whose arithmetic contract proves a cheaper transformation correct. + +Exit gate: smaller active I-cache footprint plus equal correctness/error paths. + +## Phase 3: storage, APA and HDL dataflow + +- [ ] Describe APA catalogue, ISO source, HDL transaction and payload stream with + producer/consumer/lifetime/ownership contracts. +- [ ] Add a persistent compact HDL catalogue index with version, drive identity, + APA-chain validation and checksum; mismatch always falls back to full scan. +- [ ] Keep large sequential transfers and persistent descriptors; avoid repeated + small fileXio/RPC control-plane operations. +- [ ] Re-measure USB source, HDD target and verification independently. +- [ ] Tune chunk/batch size only with a sweep on the same device/workload. +- [ ] Audit sync/flush frequency against transaction durability requirements. + +Exit gate: lower non-hideable storage time without weakening journal or metadata +commit safety. + +## Phase 4: IOP/SIF service architecture + +- [ ] Measure queue/service/transport/completion latency separately. +- [ ] Maintain the IOP-local producer path where the final device consumer is on + the IOP; do not bounce payload through EE without a consumer requirement. +- [ ] Keep control metadata coarse-grained and bulk payload on DMA/data-plane paths. +- [ ] Express double buffering as explicit producer/consumer ownership states. +- [ ] Evaluate triple buffering only if telemetry shows producer/consumer jitter + that a third slot can actually hide within the IOP RAM budget. +- [ ] Add a static IOP RAM budget including IRX, staging buffers, fragment maps, + stacks and safety headroom. +- [ ] Sweep IOP worker priorities only after measuring service slack and stalls. + +Exit gate: higher overlap/lower p99 with no IOP starvation or device regressions. + +## Phase 5: allocators, copies and lifetime + +- [ ] Inventory dynamic allocation by lifetime class: permanent, menu/session, + transaction, streaming, temporary and IOP service. +- [ ] Replace allocation churn only where traces show jitter/fragmentation/copy + amplification. Candidate structures are arenas, pools and aligned rings. +- [ ] Record `copies_per_payload`, `bytes_touched_per_payload` and DMA bytes for + storage chunks and large recovery/forensic records. +- [ ] Audit every explicit `aligned(64)` and document the concrete consumer + contract; remove or change alignment only with evidence. + +Exit gate: lower allocation/copy cost and no lifetime/ownership regressions. + +## Phase 6: GS/frontend + +- [ ] Preserve the real-hardware-proven 640x224 native coordinate contract. +- [ ] Measure GIF packet size, submission count, waits and framebuffer/VRAM use. +- [ ] Remove redundant FINISH/waits only after proving ownership/completion. +- [ ] Keep UI data GS-ready and avoid runtime repacking where a persistent + representation is simpler. +- [ ] Re-run the existing multi-mode real-hardware video regression after every + renderer synchronization or VRAM-layout change. + +Exit gate: equal visual correctness and mode stability with smaller CPU/packet +cost or smaller code footprint. + +## Phase 7: specialised hot kernels + +Only after the earlier phases have moved the bottleneck: + +- [ ] Benchmark SHA-256 and other remaining hot kernels with R5900 counters. +- [ ] Compare portable C, compiler output and a simple specialised baseline. +- [ ] Evaluate MMI only for regular packed data that actually dominates CPU time. +- [ ] Evaluate Scratchpad only for a bounded explicit working set with proven + transfer/ownership benefit. +- [ ] Do not introduce VU/VIF/IPU merely because the hardware exists; require a + fitting regular workload and end-to-end win including transport/sync. + +Exit gate: real-hardware A/B win including p50/p95/p99/max and correctness hash. + +## Benchmark record + +Every accepted optimization benchmark should record at least: + +```yaml +console_scp: +hardware_revision: +network_adapter: +storage_adapter: +ps2sdk_commit: +toolchain: +active_irx: +build_flags: +workload: +direction: +buffering: +alignment: +sample_count: +units: +correctness_hash: +p50: +p95: +p99: +max: +deadline_misses: +``` + +## Merge policy + +Each material optimization lands as a small reviewable commit with: + +1. bottleneck and evidence; +2. authoritative corpus/current source; +3. performance hypothesis; +4. smallest meaningful change; +5. correctness/error handling retained; +6. measurement method; +7. alignment/lifetime/thread-context risks; +8. simpler A/B baseline for aggressive changes. + +After every major optimization, whole-system profiling is repeated because the +bottleneck is assumed to have moved. From 3d4b288a942673e346082ae5ffb1cbeacdbb522f Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 11:58:08 +0200 Subject: [PATCH 002/156] Instrument HDL fast path latency distribution --- src/hdl_tools/fast_io.inc | 194 +++++++++++++++++++++++++++++++++++++- 1 file changed, 191 insertions(+), 3 deletions(-) diff --git a/src/hdl_tools/fast_io.inc b/src/hdl_tools/fast_io.inc index 5df968b5..a0df3aed 100644 --- a/src/hdl_tools/fast_io.inc +++ b/src/hdl_tools/fast_io.inc @@ -15,12 +15,27 @@ * into IOP staging and performs one IOP->EE DMA, avoiding fileXio's extra * staging/copy layer. */ +#define HDL_FAST_LATENCY_BUCKETS 24u + +typedef struct { + uint32_t samples; + uint32_t maximum_us; + uint32_t buckets[HDL_FAST_LATENCY_BUCKETS]; +} hdl_fast_latency_stats_t; + static int hdl_fast_source_fd = -1; static uint64_t hdl_fast_source_position; static uint64_t hdl_fast_source_seek_value; static uint64_t hdl_fast_source_total; static uint64_t hdl_fast_copy_start_ticks; static uint64_t hdl_fast_copy_bytes; +static uint64_t hdl_fast_target_bytes; +static uint64_t hdl_fast_sif_dma_bytes; +static uint64_t hdl_fast_cache_maintenance_bytes; +static uint64_t hdl_fast_fallback_source_bytes; +static uint64_t hdl_fast_fallback_target_bytes; +static uint64_t hdl_fast_copy_consumer_start_ticks; +static uint64_t hdl_fast_target_consumer_start_ticks; static int hdl_fast_source_position_valid; static int hdl_fast_source_seek_pending; static int hdl_fast_pump_mode; @@ -29,6 +44,123 @@ static unsigned int hdl_fast_pump_ack_bytes; static int hdl_fast_path_logged; static int hdl_fast_stats_logged; static int hdl_fast_rate_logged; +static int hdl_fast_copy_profile_logged; +static int hdl_fast_target_profile_logged; +static hdl_fast_latency_stats_t hdl_fast_source_ioctl_latency; +static hdl_fast_latency_stats_t hdl_fast_pump_ioctl_latency; +static hdl_fast_latency_stats_t hdl_fast_target_ioctl_latency; +static hdl_fast_latency_stats_t hdl_fast_copy_consumer_latency; +static hdl_fast_latency_stats_t hdl_fast_target_consumer_latency; + +static void hdl_fast_latency_reset(hdl_fast_latency_stats_t *stats) +{ + memset(stats, 0, sizeof(*stats)); +} + +static uint32_t hdl_fast_elapsed_us(u64 start, u64 end) +{ + u32 seconds = 0; + u32 microseconds = 0; + + TimerBusClock2USec(end - start, &seconds, µseconds); + if (seconds > 4294u) + return 0xffffffffu; + return seconds * 1000000u + microseconds; +} + +static void hdl_fast_latency_record(hdl_fast_latency_stats_t *stats, + u64 start, u64 end) +{ + uint32_t usec; + uint32_t bound = 1u; + unsigned int bucket = 0u; + + if (start == 0 || end < start) + return; + usec = hdl_fast_elapsed_us(start, end); + while (bucket + 1u < HDL_FAST_LATENCY_BUCKETS && usec > bound) { + bound <<= 1; + bucket++; + } + stats->buckets[bucket]++; + stats->samples++; + if (usec > stats->maximum_us) + stats->maximum_us = usec; +} + +static uint32_t hdl_fast_latency_percentile(const hdl_fast_latency_stats_t *stats, + unsigned int percentile) +{ + uint32_t threshold; + uint32_t cumulative = 0; + unsigned int bucket; + + if (stats->samples == 0 || percentile == 0 || percentile > 100u) + return 0; + threshold = (stats->samples * percentile + 99u) / 100u; + for (bucket = 0; bucket < HDL_FAST_LATENCY_BUCKETS; bucket++) { + cumulative += stats->buckets[bucket]; + if (cumulative >= threshold) + return 1u << bucket; + } + return 1u << (HDL_FAST_LATENCY_BUCKETS - 1u); +} + +static void hdl_fast_latency_log(const char *name, + const hdl_fast_latency_stats_t *stats) +{ + if (stats->samples == 0) + return; + session_log_line( + "HDL perf %s samples=%u p50<=%uus p95<=%uus p99<=%uus max=%uus", + name, stats->samples, + hdl_fast_latency_percentile(stats, 50u), + hdl_fast_latency_percentile(stats, 95u), + hdl_fast_latency_percentile(stats, 99u), + stats->maximum_us); +} + +static void hdl_fast_record_ioctl_latency(int command, u64 start, u64 end) +{ + if (command == HDL_STREAM_IOCTL2_PUMP_TO_EE) + hdl_fast_latency_record(&hdl_fast_pump_ioctl_latency, start, end); + else if (command == HDL_STREAM_IOCTL2_SOURCE_TO_EE) + hdl_fast_latency_record(&hdl_fast_source_ioctl_latency, start, end); + else if (command == HDL_STREAM_IOCTL2_TARGET_TO_EE) + hdl_fast_latency_record(&hdl_fast_target_ioctl_latency, start, end); +} + +static void hdl_fast_log_copy_profile(void) +{ + if (hdl_fast_copy_profile_logged) + return; + hdl_fast_latency_log("pump-ioctl", &hdl_fast_pump_ioctl_latency); + hdl_fast_latency_log("source-ioctl", &hdl_fast_source_ioctl_latency); + hdl_fast_latency_log("copy-ee-consumer", &hdl_fast_copy_consumer_latency); + session_log_line( + "HDL perf copy traffic useful=%llu sif-dma=%llu ee-cache-maint=%llu fallback-source=%llu", + (unsigned long long)hdl_fast_copy_bytes, + (unsigned long long)hdl_fast_sif_dma_bytes, + (unsigned long long)hdl_fast_cache_maintenance_bytes, + (unsigned long long)hdl_fast_fallback_source_bytes); + hdl_fast_copy_profile_logged = 1; +} + +static void hdl_fast_log_target_profile(void) +{ + if (hdl_fast_target_profile_logged) + return; + hdl_fast_latency_log("target-ioctl", &hdl_fast_target_ioctl_latency); + hdl_fast_latency_log("verify-ee-consumer", &hdl_fast_target_consumer_latency); + session_log_line( + "HDL perf verify traffic target=%llu sif-dma-total=%llu ee-cache-maint-total=%llu fallback-target=%llu consumer-samples=%u final-chunk-excluded=1", + (unsigned long long)hdl_fast_target_bytes, + (unsigned long long)hdl_fast_sif_dma_bytes, + (unsigned long long)hdl_fast_cache_maintenance_bytes, + (unsigned long long)hdl_fast_fallback_target_bytes, + hdl_fast_target_consumer_latency.samples); + hdl_fast_target_profile_logged = 1; +} static void hdl_fast_io_reset(void) { @@ -38,6 +170,13 @@ static void hdl_fast_io_reset(void) hdl_fast_source_total = 0; hdl_fast_copy_start_ticks = 0; hdl_fast_copy_bytes = 0; + hdl_fast_target_bytes = 0; + hdl_fast_sif_dma_bytes = 0; + hdl_fast_cache_maintenance_bytes = 0; + hdl_fast_fallback_source_bytes = 0; + hdl_fast_fallback_target_bytes = 0; + hdl_fast_copy_consumer_start_ticks = 0; + hdl_fast_target_consumer_start_ticks = 0; hdl_fast_source_position_valid = 0; hdl_fast_source_seek_pending = 0; hdl_fast_pump_mode = 0; @@ -46,6 +185,13 @@ static void hdl_fast_io_reset(void) hdl_fast_path_logged = 0; hdl_fast_stats_logged = 0; hdl_fast_rate_logged = 0; + hdl_fast_copy_profile_logged = 0; + hdl_fast_target_profile_logged = 0; + hdl_fast_latency_reset(&hdl_fast_source_ioctl_latency); + hdl_fast_latency_reset(&hdl_fast_pump_ioctl_latency); + hdl_fast_latency_reset(&hdl_fast_target_ioctl_latency); + hdl_fast_latency_reset(&hdl_fast_copy_consumer_latency); + hdl_fast_latency_reset(&hdl_fast_target_consumer_latency); } static int hdl_fast_resolve_source_total(int source_fd) @@ -158,6 +304,8 @@ static s64 hdl_fast_fileXioLseek64(int fd, s64 offset, int whence) static int hdl_fast_dma_read(int command, int source_fd, void *buffer, unsigned int bytes) { + u64 call_start; + u64 call_end; int result; if (hdl_active_target_fd < 0 || buffer == NULL || bytes == 0 || @@ -167,8 +315,12 @@ static int hdl_fast_dma_read(int command, int source_fd, void *buffer, /* The IOP is about to DMA into cached EE memory. Write back any dirty line * first, then invalidate after the synchronous ioctl returns so the R5900 - * sees the new payload instead of a stale D-cache copy. */ + * sees the new payload instead of a stale D-cache copy. Count the touched + * bytes separately from useful payload so copy/coherency amplification is + * visible in corpus-v2 benchmark logs. */ SyncDCache(buffer, (unsigned char *)buffer + bytes); + hdl_fast_cache_maintenance_bytes += bytes; + call_start = GetTimerSystemTime(); if (command == HDL_STREAM_IOCTL2_TARGET_TO_EE) { hdl_stream_target_io_t request; @@ -192,8 +344,13 @@ static int hdl_fast_dma_read(int command, int source_fd, void *buffer, result = fileXioIoctl2(hdl_active_target_fd, command, &request, sizeof(request), NULL, 0); } - if (result >= 0) + call_end = GetTimerSystemTime(); + hdl_fast_record_ioctl_latency(command, call_start, call_end); + if (result >= 0) { InvalidDCache(buffer, (unsigned char *)buffer + bytes); + hdl_fast_cache_maintenance_bytes += bytes; + hdl_fast_sif_dma_bytes += bytes; + } return result; } @@ -205,13 +362,27 @@ static int hdl_fast_fileXioRead(int fd, void *buffer, int size) return fileXioRead(fd, buffer, size); if (fd == hdl_active_target_fd) { + if (hdl_fast_target_consumer_start_ticks != 0) { + hdl_fast_latency_record(&hdl_fast_target_consumer_latency, + hdl_fast_target_consumer_start_ticks, + GetTimerSystemTime()); + hdl_fast_target_consumer_start_ticks = 0; + } hdl_fast_pump_mode = 0; hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; result = hdl_fast_dma_read(HDL_STREAM_IOCTL2_TARGET_TO_EE, -1, buffer, (unsigned int)size); - if (result != INT_MIN) + if (result != INT_MIN) { + if (result > 0) { + hdl_fast_target_bytes += (unsigned int)result; + hdl_fast_target_consumer_start_ticks = GetTimerSystemTime(); + if (hdl_fast_source_total != 0 && + hdl_fast_target_bytes >= hdl_fast_source_total) + hdl_fast_log_target_profile(); + } return result; + } } else if (hdl_active_target_fd >= 0 && hdl_fast_source_position_valid && fd == hdl_fast_source_fd) { int command = hdl_fast_pump_mode @@ -230,6 +401,7 @@ static int hdl_fast_fileXioRead(int fd, void *buffer, int size) hdl_fast_pump_ack = 1; hdl_fast_pump_ack_bytes = (unsigned int)size; hdl_fast_copy_bytes += (unsigned int)size; + hdl_fast_copy_consumer_start_ticks = GetTimerSystemTime(); if (!hdl_fast_path_logged) { session_log_line( "HDL IOP fast pump active: double-buffered USB source -> ps2hdd + one IOP->EE SHA DMA, chunk=%u", @@ -239,6 +411,7 @@ static int hdl_fast_fileXioRead(int fd, void *buffer, int size) } else { hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; + hdl_fast_copy_consumer_start_ticks = 0; } return result; } @@ -257,12 +430,19 @@ static int hdl_fast_fileXioRead(int fd, void *buffer, int size) return seek < 0 ? (int)seek : HDL_INSTALL_COPY_FAILED; } result = fileXioRead(fd, buffer, size); + if (result > 0) { + if (fd == hdl_active_target_fd) + hdl_fast_fallback_target_bytes += (unsigned int)result; + else if (hdl_fast_source_position_valid && fd == hdl_fast_source_fd) + hdl_fast_fallback_source_bytes += (unsigned int)result; + } if (result > 0 && fd != hdl_active_target_fd && hdl_fast_source_position_valid && fd == hdl_fast_source_fd) hdl_fast_source_position += (unsigned int)result; hdl_fast_source_seek_pending = 0; hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; + hdl_fast_copy_consumer_start_ticks = 0; return result; } @@ -271,12 +451,19 @@ static int hdl_fast_fileXioWrite(int fd, const void *buffer, int size) if (hdl_fast_pump_ack) { if (fd == hdl_active_target_fd && size > 0 && (unsigned int)size == hdl_fast_pump_ack_bytes) { + if (hdl_fast_copy_consumer_start_ticks != 0) { + hdl_fast_latency_record(&hdl_fast_copy_consumer_latency, + hdl_fast_copy_consumer_start_ticks, + GetTimerSystemTime()); + hdl_fast_copy_consumer_start_ticks = 0; + } hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; if (hdl_fast_source_total != 0 && hdl_fast_source_position == hdl_fast_source_total) { hdl_fast_io_log_rate(); hdl_fast_io_log_stats(hdl_active_target_fd); + hdl_fast_log_copy_profile(); } return size; } @@ -284,6 +471,7 @@ static int hdl_fast_fileXioWrite(int fd, const void *buffer, int size) * mismatched acknowledgement would otherwise write the block twice. */ hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; + hdl_fast_copy_consumer_start_ticks = 0; return HDL_INSTALL_COPY_FAILED; } From 8ac93e0ab38afb229cc7249faf8fb314f5368240 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 12:01:00 +0200 Subject: [PATCH 003/156] Keep HDL profiling outside transaction I-cache working set --- src/hdl_tools/fast_io.inc | 43 ++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/hdl_tools/fast_io.inc b/src/hdl_tools/fast_io.inc index a0df3aed..c6fd9481 100644 --- a/src/hdl_tools/fast_io.inc +++ b/src/hdl_tools/fast_io.inc @@ -16,6 +16,8 @@ * staging/copy layer. */ #define HDL_FAST_LATENCY_BUCKETS 24u +#define HDL_FAST_NOINLINE __attribute__((noinline)) +#define HDL_FAST_COLD __attribute__((cold, noinline)) typedef struct { uint32_t samples; @@ -68,8 +70,8 @@ static uint32_t hdl_fast_elapsed_us(u64 start, u64 end) return seconds * 1000000u + microseconds; } -static void hdl_fast_latency_record(hdl_fast_latency_stats_t *stats, - u64 start, u64 end) +static HDL_FAST_NOINLINE void hdl_fast_latency_record( + hdl_fast_latency_stats_t *stats, u64 start, u64 end) { uint32_t usec; uint32_t bound = 1u; @@ -106,8 +108,8 @@ static uint32_t hdl_fast_latency_percentile(const hdl_fast_latency_stats_t *stat return 1u << (HDL_FAST_LATENCY_BUCKETS - 1u); } -static void hdl_fast_latency_log(const char *name, - const hdl_fast_latency_stats_t *stats) +static HDL_FAST_COLD void hdl_fast_latency_log( + const char *name, const hdl_fast_latency_stats_t *stats) { if (stats->samples == 0) return; @@ -120,7 +122,8 @@ static void hdl_fast_latency_log(const char *name, stats->maximum_us); } -static void hdl_fast_record_ioctl_latency(int command, u64 start, u64 end) +static HDL_FAST_NOINLINE void hdl_fast_record_ioctl_latency( + int command, u64 start, u64 end) { if (command == HDL_STREAM_IOCTL2_PUMP_TO_EE) hdl_fast_latency_record(&hdl_fast_pump_ioctl_latency, start, end); @@ -130,7 +133,7 @@ static void hdl_fast_record_ioctl_latency(int command, u64 start, u64 end) hdl_fast_latency_record(&hdl_fast_target_ioctl_latency, start, end); } -static void hdl_fast_log_copy_profile(void) +static HDL_FAST_COLD void hdl_fast_log_copy_profile(void) { if (hdl_fast_copy_profile_logged) return; @@ -146,7 +149,7 @@ static void hdl_fast_log_copy_profile(void) hdl_fast_copy_profile_logged = 1; } -static void hdl_fast_log_target_profile(void) +static HDL_FAST_COLD void hdl_fast_log_target_profile(void) { if (hdl_fast_target_profile_logged) return; @@ -162,7 +165,7 @@ static void hdl_fast_log_target_profile(void) hdl_fast_target_profile_logged = 1; } -static void hdl_fast_io_reset(void) +static HDL_FAST_NOINLINE void hdl_fast_io_reset(void) { hdl_fast_source_fd = -1; hdl_fast_source_position = 0; @@ -194,7 +197,7 @@ static void hdl_fast_io_reset(void) hdl_fast_latency_reset(&hdl_fast_target_consumer_latency); } -static int hdl_fast_resolve_source_total(int source_fd) +static HDL_FAST_NOINLINE int hdl_fast_resolve_source_total(int source_fd) { s64 end; s64 restore; @@ -219,7 +222,7 @@ static int hdl_fast_resolve_source_total(int source_fd) return 0; } -static void hdl_fast_io_log_stats(int target_fd) +static HDL_FAST_COLD void hdl_fast_io_log_stats(int target_fd) { hdl_stream_fast_stats_t stats; int result; @@ -242,7 +245,7 @@ static void hdl_fast_io_log_stats(int target_fd) hdl_fast_stats_logged = 1; } -static void hdl_fast_io_log_rate(void) +static HDL_FAST_COLD void hdl_fast_io_log_rate(void) { u64 elapsed; u32 seconds; @@ -277,7 +280,8 @@ static void hdl_fast_io_log_rate(void) hdl_fast_rate_logged = 1; } -static s64 hdl_fast_fileXioLseek64(int fd, s64 offset, int whence) +static HDL_FAST_NOINLINE s64 hdl_fast_fileXioLseek64(int fd, s64 offset, + int whence) { s64 result = fileXioLseek64(fd, offset, whence); @@ -301,8 +305,9 @@ static s64 hdl_fast_fileXioLseek64(int fd, s64 offset, int whence) return result; } -static int hdl_fast_dma_read(int command, int source_fd, void *buffer, - unsigned int bytes) +static HDL_FAST_NOINLINE int hdl_fast_dma_read(int command, int source_fd, + void *buffer, + unsigned int bytes) { u64 call_start; u64 call_end; @@ -354,7 +359,8 @@ static int hdl_fast_dma_read(int command, int source_fd, void *buffer, return result; } -static int hdl_fast_fileXioRead(int fd, void *buffer, int size) +static HDL_FAST_NOINLINE int hdl_fast_fileXioRead(int fd, void *buffer, + int size) { int result; @@ -446,7 +452,9 @@ static int hdl_fast_fileXioRead(int fd, void *buffer, int size) return result; } -static int hdl_fast_fileXioWrite(int fd, const void *buffer, int size) +static HDL_FAST_NOINLINE int hdl_fast_fileXioWrite(int fd, + const void *buffer, + int size) { if (hdl_fast_pump_ack) { if (fd == hdl_active_target_fd && size > 0 && @@ -478,3 +486,6 @@ static int hdl_fast_fileXioWrite(int fd, const void *buffer, int size) hdl_stream_tune_filexio_once(); return fileXioWrite(fd, buffer, size); } + +#undef HDL_FAST_COLD +#undef HDL_FAST_NOINLINE From 36a8aba35f85f0aeac48c8c58e8232e347d07442 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 13:02:28 +0200 Subject: [PATCH 004/156] Instrument HDL IOP transport stages --- include/hdl_stream_rpc.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/include/hdl_stream_rpc.h b/include/hdl_stream_rpc.h index 2d580990..c0f8dc53 100644 --- a/include/hdl_stream_rpc.h +++ b/include/hdl_stream_rpc.h @@ -54,6 +54,21 @@ typedef struct { #define HDL_STREAM_FAST_FLAG_DOUBLE_BUFFER 0x00000001u #define HDL_STREAM_FAST_FLAG_DIRECT_BDM 0x00000002u +#define HDL_STREAM_FAST_FLAG_IOP_TIMING 0x00000004u + +/* + * Corpus-v2 profiling uses logarithmic microsecond buckets on the IOP instead + * of formatting trace lines in the hot service path. 24 buckets cover 1 us + * through multi-second stalls while keeping the RPC snapshot compact enough + * for a single control-plane read after a bulk phase. + */ +#define HDL_STREAM_IOP_LATENCY_BUCKETS 24u + +typedef struct { + uint32_t samples; + uint32_t maximum_us; + uint32_t buckets[HDL_STREAM_IOP_LATENCY_BUCKETS]; +} hdl_stream_iop_latency_t; typedef struct { uint32_t flags; @@ -66,6 +81,21 @@ typedef struct { uint32_t pumped_sectors; uint32_t source_dma_chunks; uint32_t target_dma_chunks; + + /* Transport accounting is kept in 512-byte sectors to avoid cross-ABI + * uint64_t layout assumptions and to cover any practical PS2 workload. */ + uint32_t direct_source_sectors; + uint32_t fallback_source_sectors; + uint32_t hdd_write_sectors; + uint32_t hdd_read_sectors; + uint32_t sif_dma_sectors; + + hdl_stream_iop_latency_t direct_source_latency; + hdl_stream_iop_latency_t fallback_source_latency; + hdl_stream_iop_latency_t prefetch_wait_latency; + hdl_stream_iop_latency_t hdd_write_latency; + hdl_stream_iop_latency_t hdd_read_latency; + hdl_stream_iop_latency_t sif_dma_latency; } hdl_stream_fast_stats_t; /* From e3c4d818f22db6c7edfacafa7eff81df657238c9 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 13:02:47 +0200 Subject: [PATCH 005/156] Import IOP timing services for HDL profiler --- iop/hdl_stream/imports.lst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iop/hdl_stream/imports.lst b/iop/hdl_stream/imports.lst index ba724bc9..82a13d6f 100644 --- a/iop/hdl_stream/imports.lst +++ b/iop/hdl_stream/imports.lst @@ -38,6 +38,8 @@ I_CreateThread I_DeleteThread I_StartThread I_ExitThread +I_GetSystemTime +I_SysClock2USec thbase_IMPORTS_end thsemap_IMPORTS_start From e5fd732cd32067a8202e9f8fdb8eef81ae73fe31 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 13:05:01 +0200 Subject: [PATCH 006/156] Profile USB HDD and SIF stages on IOP --- iop/hdl_stream/hdl_stream.c | 98 ++++++++++++++++++++++++++++++++++--- 1 file changed, 91 insertions(+), 7 deletions(-) diff --git a/iop/hdl_stream/hdl_stream.c b/iop/hdl_stream/hdl_stream.c index e837e7c1..7ad20c2b 100644 --- a/iop/hdl_stream/hdl_stream.c +++ b/iop/hdl_stream/hdl_stream.c @@ -39,7 +39,7 @@ #define HDL_STREAM_PREFETCH_STACK 0x1000u #define HDL_STREAM_PREFETCH_PRIORITY 0x30u -IRX_ID("hdl_stream", 1, 4); +IRX_ID("hdl_stream", 1, 5); typedef struct { int source_fd; @@ -80,6 +80,46 @@ typedef struct { hdl_stream_fast_stats_t stats; } hdl_stream_file_t; +/* + * Phase-0 corpus-v2 profiling deliberately stores only compact histograms in + * the IOP hot path. Formatting/logging remains on the EE after a bulk phase. + * GetSystemTime/SysClock2USec are current ThreadMan services and the same clock + * conversion pattern is used by PS2SDK itself. Hardware A/B still decides + * whether this instrumentation remains enabled in performance builds. + */ +static uint32_t iop_elapsed_us(const iop_sys_clock_t *start, + const iop_sys_clock_t *end) +{ + iop_sys_clock_t diff; + uint32_t seconds = 0; + uint32_t usec = 0; + + diff.lo = end->lo - start->lo; + diff.hi = end->hi - start->hi - (start->lo > end->lo); + SysClock2USec(&diff, &seconds, &usec); + if (seconds > 4294u) + return 0xffffffffu; + return seconds * 1000000u + usec; +} + +static void iop_latency_record(hdl_stream_iop_latency_t *stats, + const iop_sys_clock_t *start, + const iop_sys_clock_t *end) +{ + uint32_t usec = iop_elapsed_us(start, end); + uint32_t bound = 1u; + unsigned int bucket = 0u; + + while (bucket + 1u < HDL_STREAM_IOP_LATENCY_BUCKETS && usec > bound) { + bound <<= 1; + bucket++; + } + stats->buckets[bucket]++; + stats->samples++; + if (usec > stats->maximum_us) + stats->maximum_us = usec; +} + static int stream_init(iomanX_iop_device_t *device) { (void)device; @@ -325,6 +365,9 @@ static int source_read_fallback(int source_fd, uint64_t offset, static int source_read_at(hdl_stream_file_t *stream, int source_fd, uint64_t offset, void *buffer, unsigned int bytes) { + iop_sys_clock_t start; + iop_sys_clock_t end; + /* * The stock usbmass BDM intentionally caps one SCSI request at 128 512-byte * sectors (64 KiB), and HDL_STREAM_IOP_STAGE_BYTES matches that exactly. @@ -336,19 +379,34 @@ static int source_read_at(hdl_stream_file_t *stream, int source_fd, (offset & (HDL_STREAM_USB_SECTOR_SIZE - 1u)) == 0 && (bytes & (HDL_STREAM_USB_SECTOR_SIZE - 1u)) == 0 && (bytes >> HDL_STREAM_USB_SECTOR_SHIFT) <= UINT16_MAX) { - int result = source_map_read( + int result; + + GetSystemTime(&start); + result = source_map_read( &stream->source, offset >> HDL_STREAM_USB_SECTOR_SHIFT, buffer, (uint16_t)(bytes >> HDL_STREAM_USB_SECTOR_SHIFT)); + GetSystemTime(&end); + iop_latency_record(&stream->stats.direct_source_latency, &start, &end); if (result >= 0) { stream->stats.direct_reads++; + stream->stats.direct_source_sectors += bytes >> 9; return (int)bytes; } source_map_disable(stream, source_fd); } stream->stats.fallback_reads++; - return source_read_fallback(source_fd, offset, buffer, bytes); + GetSystemTime(&start); + { + int result = source_read_fallback(source_fd, offset, buffer, bytes); + + GetSystemTime(&end); + iop_latency_record(&stream->stats.fallback_source_latency, &start, &end); + if (result > 0) + stream->stats.fallback_source_sectors += (unsigned int)result >> 9; + return result; + } } static void prefetch_worker(void *arg) @@ -438,11 +496,16 @@ static int prefetch_init(hdl_stream_file_t *stream) static int prefetch_wait(hdl_stream_file_t *stream) { + iop_sys_clock_t start; + iop_sys_clock_t end; int result; if (!stream->prefetch_active) return 0; + GetSystemTime(&start); result = WaitSema(stream->prefetch_done_sema); + GetSystemTime(&end); + iop_latency_record(&stream->stats.prefetch_wait_latency, &start, &end); if (result < 0) return result; stream->prefetch_active = 0; @@ -510,10 +573,12 @@ static void prefetch_shutdown(hdl_stream_file_t *stream) prefetch_delete_semas(stream); } -static int dma_to_ee(uint32_t ee_address, const void *source, - unsigned int bytes) +static int dma_to_ee(hdl_stream_file_t *stream, uint32_t ee_address, + const void *source, unsigned int bytes) { SifDmaTransfer_t transfer; + iop_sys_clock_t start; + iop_sys_clock_t end; int id; if (ee_address == 0 || bytes == 0 || (ee_address & 0x3fu) != 0 || @@ -523,10 +588,14 @@ static int dma_to_ee(uint32_t ee_address, const void *source, transfer.dest = (void *)(uintptr_t)ee_address; transfer.size = (int)bytes; transfer.attr = 0; + GetSystemTime(&start); id = sceSifSetDma(&transfer, 1); if (id <= 0) return -EIO; while (sceSifDmaStat(id) >= 0) {} + GetSystemTime(&end); + iop_latency_record(&stream->stats.sif_dma_latency, &start, &end); + stream->stats.sif_dma_sectors += bytes >> 9; return 0; } @@ -548,6 +617,7 @@ static int stream_open(iomanX_iop_file_t *file, const char *name, if (stream == NULL) return -ENOMEM; memset(stream, 0, sizeof(*stream)); + stream->stats.flags = HDL_STREAM_FAST_FLAG_IOP_TIMING; stream->source.source_fd = -1; stream->prefetch_thread = -1; stream->prefetch_request_sema = -1; @@ -685,6 +755,8 @@ static int stream_transfer(iomanX_iop_file_t *file, void *buffer, while (remaining > 0) { hddIoctl2Transfer_t transfer; + iop_sys_clock_t start; + iop_sys_clock_t end; uint64_t available; uint32_t part; uint32_t sector; @@ -703,8 +775,19 @@ static int stream_transfer(iomanX_iop_file_t *file, void *buffer, transfer.size = (uint32_t)chunk >> 9; transfer.mode = direction; transfer.buffer = cursor; + GetSystemTime(&start); result = iomanX_ioctl2(stream->hdd_fd, HIOCTRANSFER, &transfer, sizeof(transfer), NULL, 0); + GetSystemTime(&end); + if (direction == APA_IO_MODE_WRITE) { + iop_latency_record(&stream->stats.hdd_write_latency, &start, &end); + if (result >= 0) + stream->stats.hdd_write_sectors += transfer.size; + } else { + iop_latency_record(&stream->stats.hdd_read_latency, &start, &end); + if (result >= 0) + stream->stats.hdd_read_sectors += transfer.size; + } if (result < 0) return result; stream->position += (uint32_t)chunk; @@ -881,7 +964,7 @@ static int fast_source_to_ee(iomanX_iop_file_t *file, stream->stats.pumped_chunks++; stream->stats.pumped_sectors += request->bytes >> 9; } - result = dma_to_ee(request->ee_address, stage, request->bytes); + result = dma_to_ee(stream, request->ee_address, stage, request->bytes); if (result < 0) return result; stream->stats.source_dma_chunks++; @@ -904,7 +987,8 @@ static int fast_target_to_ee(iomanX_iop_file_t *file, APA_IO_MODE_READ); if (result != (int)request->bytes) return result < 0 ? result : -EIO; - result = dma_to_ee(request->ee_address, stream->stage[0], request->bytes); + result = dma_to_ee(stream, request->ee_address, stream->stage[0], + request->bytes); if (result < 0) return result; stream->stats.target_dma_chunks++; From eb27da9915ae90ee61179eeddb21d2f70e37186a Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 13:06:07 +0200 Subject: [PATCH 007/156] Report IOP stage latency distributions --- src/hdl_tools/fast_io.inc | 78 +++++++++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 12 deletions(-) diff --git a/src/hdl_tools/fast_io.inc b/src/hdl_tools/fast_io.inc index c6fd9481..bca28ed8 100644 --- a/src/hdl_tools/fast_io.inc +++ b/src/hdl_tools/fast_io.inc @@ -44,7 +44,8 @@ static int hdl_fast_pump_mode; static int hdl_fast_pump_ack; static unsigned int hdl_fast_pump_ack_bytes; static int hdl_fast_path_logged; -static int hdl_fast_stats_logged; +static int hdl_fast_copy_iop_stats_logged; +static int hdl_fast_verify_iop_stats_logged; static int hdl_fast_rate_logged; static int hdl_fast_copy_profile_logged; static int hdl_fast_target_profile_logged; @@ -108,6 +109,24 @@ static uint32_t hdl_fast_latency_percentile(const hdl_fast_latency_stats_t *stat return 1u << (HDL_FAST_LATENCY_BUCKETS - 1u); } +static uint32_t hdl_fast_iop_latency_percentile( + const hdl_stream_iop_latency_t *stats, unsigned int percentile) +{ + uint32_t threshold; + uint32_t cumulative = 0; + unsigned int bucket; + + if (stats->samples == 0 || percentile == 0 || percentile > 100u) + return 0; + threshold = (stats->samples * percentile + 99u) / 100u; + for (bucket = 0; bucket < HDL_STREAM_IOP_LATENCY_BUCKETS; bucket++) { + cumulative += stats->buckets[bucket]; + if (cumulative >= threshold) + return 1u << bucket; + } + return 1u << (HDL_STREAM_IOP_LATENCY_BUCKETS - 1u); +} + static HDL_FAST_COLD void hdl_fast_latency_log( const char *name, const hdl_fast_latency_stats_t *stats) { @@ -122,6 +141,20 @@ static HDL_FAST_COLD void hdl_fast_latency_log( stats->maximum_us); } +static HDL_FAST_COLD void hdl_fast_iop_latency_log( + const char *name, const hdl_stream_iop_latency_t *stats) +{ + if (stats->samples == 0) + return; + session_log_line( + "HDL IOP perf %s samples=%u p50<=%uus p95<=%uus p99<=%uus max=%uus", + name, stats->samples, + hdl_fast_iop_latency_percentile(stats, 50u), + hdl_fast_iop_latency_percentile(stats, 95u), + hdl_fast_iop_latency_percentile(stats, 99u), + stats->maximum_us); +} + static HDL_FAST_NOINLINE void hdl_fast_record_ioctl_latency( int command, u64 start, u64 end) { @@ -186,7 +219,8 @@ static HDL_FAST_NOINLINE void hdl_fast_io_reset(void) hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; hdl_fast_path_logged = 0; - hdl_fast_stats_logged = 0; + hdl_fast_copy_iop_stats_logged = 0; + hdl_fast_verify_iop_stats_logged = 0; hdl_fast_rate_logged = 0; hdl_fast_copy_profile_logged = 0; hdl_fast_target_profile_logged = 0; @@ -222,27 +256,45 @@ static HDL_FAST_NOINLINE int hdl_fast_resolve_source_total(int source_fd) return 0; } -static HDL_FAST_COLD void hdl_fast_io_log_stats(int target_fd) +static HDL_FAST_COLD void hdl_fast_io_log_stats(int target_fd, + int verify_phase) { hdl_stream_fast_stats_t stats; + const char *phase = verify_phase ? "verify-final" : "copy-final"; + int *logged = verify_phase ? &hdl_fast_verify_iop_stats_logged + : &hdl_fast_copy_iop_stats_logged; int result; - if (target_fd < 0 || hdl_fast_stats_logged) + if (target_fd < 0 || *logged) return; memset(&stats, 0, sizeof(stats)); result = fileXioIoctl2(target_fd, HDL_STREAM_IOCTL2_GET_FAST_STATS, NULL, 0, &stats, sizeof(stats)); if (result < 0) { - session_log_line("HDL fast I/O stats unavailable fd=%d result=%d", - target_fd, result); + session_log_line("HDL fast I/O stats unavailable phase=%s fd=%d result=%d", + phase, target_fd, result); return; } session_log_line( - "HDL fast I/O source stats flags=0x%08x fragments=%u direct=%u fallback=%u prefetch-hit=%u miss=%u pump=%u sectors=%u src-dma=%u", - stats.flags, stats.fragment_count, stats.direct_reads, + "HDL fast I/O snapshot phase=%s flags=0x%08x fragments=%u direct=%u fallback=%u prefetch-hit=%u miss=%u pump=%u sectors=%u src-dma=%u target-dma=%u", + phase, stats.flags, stats.fragment_count, stats.direct_reads, stats.fallback_reads, stats.prefetch_hits, stats.prefetch_misses, - stats.pumped_chunks, stats.pumped_sectors, stats.source_dma_chunks); - hdl_fast_stats_logged = 1; + stats.pumped_chunks, stats.pumped_sectors, stats.source_dma_chunks, + stats.target_dma_chunks); + session_log_line( + "HDL IOP traffic phase=%s direct-src-sectors=%u fallback-src-sectors=%u hdd-write-sectors=%u hdd-read-sectors=%u sif-dma-sectors=%u", + phase, stats.direct_source_sectors, stats.fallback_source_sectors, + stats.hdd_write_sectors, stats.hdd_read_sectors, + stats.sif_dma_sectors); + if ((stats.flags & HDL_STREAM_FAST_FLAG_IOP_TIMING) != 0) { + hdl_fast_iop_latency_log("usb-direct-read", &stats.direct_source_latency); + hdl_fast_iop_latency_log("source-fallback-read", &stats.fallback_source_latency); + hdl_fast_iop_latency_log("prefetch-consumer-wait", &stats.prefetch_wait_latency); + hdl_fast_iop_latency_log("hdd-write", &stats.hdd_write_latency); + hdl_fast_iop_latency_log("hdd-read", &stats.hdd_read_latency); + hdl_fast_iop_latency_log("sif-dma-completion", &stats.sif_dma_latency); + } + *logged = 1; } static HDL_FAST_COLD void hdl_fast_io_log_rate(void) @@ -384,8 +436,10 @@ static HDL_FAST_NOINLINE int hdl_fast_fileXioRead(int fd, void *buffer, hdl_fast_target_bytes += (unsigned int)result; hdl_fast_target_consumer_start_ticks = GetTimerSystemTime(); if (hdl_fast_source_total != 0 && - hdl_fast_target_bytes >= hdl_fast_source_total) + hdl_fast_target_bytes >= hdl_fast_source_total) { + hdl_fast_io_log_stats(hdl_active_target_fd, 1); hdl_fast_log_target_profile(); + } } return result; } @@ -470,7 +524,7 @@ static HDL_FAST_NOINLINE int hdl_fast_fileXioWrite(int fd, if (hdl_fast_source_total != 0 && hdl_fast_source_position == hdl_fast_source_total) { hdl_fast_io_log_rate(); - hdl_fast_io_log_stats(hdl_active_target_fd); + hdl_fast_io_log_stats(hdl_active_target_fd, 0); hdl_fast_log_copy_profile(); } return size; From 514700a5e971dd5f6e556613a37fc59864a07d22 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 13:12:32 +0200 Subject: [PATCH 008/156] Add R5900 performance counter API --- include/r5900_perf.h | 85 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 include/r5900_perf.h diff --git a/include/r5900_perf.h b/include/r5900_perf.h new file mode 100644 index 00000000..3123d357 --- /dev/null +++ b/include/r5900_perf.h @@ -0,0 +1,85 @@ +#ifndef PS2_HDD_BOOTSTRAP_MANAGER_R5900_PERF_H +#define PS2_HDD_BOOTSTRAP_MANAGER_R5900_PERF_H + +#include + +/* + * EE Core User's Manual v6.0, Table 7-1. + * + * PCR0 and PCR1 share the numeric event selector but not always the event + * meaning. Keep separate enums so callers cannot accidentally describe a PCR1 + * value using the PCR0 name. Event 16 deliberately means "no event" and is + * useful when a benchmark needs only one hardware counter. + */ +typedef enum { + R5900_PCR0_RESERVED = 0, + R5900_PCR0_PROCESSOR_CYCLE = 1, + R5900_PCR0_SINGLE_ISSUE = 2, + R5900_PCR0_BRANCH_ISSUED = 3, + R5900_PCR0_BTAC_MISS = 4, + R5900_PCR0_ITLB_MISS = 5, + R5900_PCR0_ICACHE_MISS = 6, + R5900_PCR0_DTLB_ACCESS = 7, + R5900_PCR0_NONBLOCKING_LOAD = 8, + R5900_PCR0_WBB_SINGLE_REQUEST = 9, + R5900_PCR0_WBB_BURST_REQUEST = 10, + R5900_PCR0_CPU_ADDRESS_BUS_BUSY = 11, + R5900_PCR0_INSTRUCTION_COMPLETED = 12, + R5900_PCR0_NON_BDS_INSTRUCTION_COMPLETED = 13, + R5900_PCR0_COP2_INSTRUCTION_COMPLETED = 14, + R5900_PCR0_LOAD_COMPLETED = 15, + R5900_PCR0_NO_EVENT = 16 +} r5900_pcr0_event_t; + +typedef enum { + R5900_PCR1_LOW_ORDER_BRANCH_ISSUED = 0, + R5900_PCR1_PROCESSOR_CYCLE = 1, + R5900_PCR1_DUAL_ISSUE = 2, + R5900_PCR1_BRANCH_MISPREDICTED = 3, + R5900_PCR1_TLB_MISS = 4, + R5900_PCR1_DTLB_MISS = 5, + R5900_PCR1_DCACHE_MISS = 6, + R5900_PCR1_WBB_SINGLE_UNAVAILABLE = 7, + R5900_PCR1_WBB_BURST_UNAVAILABLE = 8, + R5900_PCR1_WBB_BURST_ALMOST_FULL = 9, + R5900_PCR1_WBB_BURST_FULL = 10, + R5900_PCR1_CPU_DATA_BUS_BUSY = 11, + R5900_PCR1_INSTRUCTION_COMPLETED = 12, + R5900_PCR1_NON_BDS_INSTRUCTION_COMPLETED = 13, + R5900_PCR1_COP1_INSTRUCTION_COMPLETED = 14, + R5900_PCR1_STORE_COMPLETED = 15, + R5900_PCR1_NO_EVENT = 16 +} r5900_pcr1_event_t; + +typedef struct { + uint32_t previous_pccr; + uint32_t previous_pcr0; + uint32_t previous_pcr1; + uint32_t event0; + uint32_t event1; + int active; +} r5900_perf_scope_t; + +typedef struct { + uint32_t pcr0; + uint32_t pcr1; + int pcr0_overflow; + int pcr1_overflow; +} r5900_perf_result_t; + +/* + * Start both counters for normal application execution in user/supervisor/ + * kernel mode. Level-1 exception-handler work is deliberately excluded so an + * interrupt does not silently become part of the measured application region. + * Level-2 handlers are excluded by hardware. + * + * The scope preserves the previous counter state and r5900_perf_end() restores + * it. This makes the harness composable with debuggers or future project-wide + * profiling code instead of assuming ownership of COP0 Perf forever. + */ +int r5900_perf_begin(r5900_perf_scope_t *scope, + r5900_pcr0_event_t event0, + r5900_pcr1_event_t event1); +int r5900_perf_end(r5900_perf_scope_t *scope, r5900_perf_result_t *result); + +#endif From 7bc3d93afa3cb4d1e0de47b5bea24a65b0a2166f Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 13:12:59 +0200 Subject: [PATCH 009/156] Implement R5900 performance counter harness --- src/r5900_perf.c | 131 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 src/r5900_perf.c diff --git a/src/r5900_perf.c b/src/r5900_perf.c new file mode 100644 index 00000000..1e92f51a --- /dev/null +++ b/src/r5900_perf.c @@ -0,0 +1,131 @@ +#include "r5900_perf.h" + +#include + +#define R5900_PCCR_CTE (1u << 31) +#define R5900_PCCR_EVENT0_SHIFT 5u +#define R5900_PCCR_EVENT1_SHIFT 15u +#define R5900_PCCR_EVENT_MASK 0x1fu + +/* Count normal application execution in all three privilege modes while + * excluding Level-1 exception handlers. This is intentionally explicit rather + * than inheriting whatever mode bits a debugger happened to leave behind. */ +#define R5900_PCCR_APP_MODES0 ((1u << 2) | (1u << 3) | (1u << 4)) +#define R5900_PCCR_APP_MODES1 ((1u << 12) | (1u << 13) | (1u << 14)) +#define R5900_PCR_VALUE_MASK 0x7fffffffu +#define R5900_PCR_OVERFLOW (1u << 31) + +static inline void r5900_perf_sync(void) +{ + /* SYNC.P guarantees completion of preceding instructions before following + * instructions execute. The memory clobber also prevents GCC from moving + * ordinary memory operations across the benchmark boundary. */ + __asm__ __volatile__("sync.p" ::: "memory"); +} + +static inline uint32_t r5900_read_pccr(void) +{ + uint32_t value; + + __asm__ __volatile__("mfps %0, 0" : "=r"(value)); + return value; +} + +static inline uint32_t r5900_read_pcr0(void) +{ + uint32_t value; + + __asm__ __volatile__("mfpc %0, 0" : "=r"(value)); + return value; +} + +static inline uint32_t r5900_read_pcr1(void) +{ + uint32_t value; + + __asm__ __volatile__("mfpc %0, 1" : "=r"(value)); + return value; +} + +static inline void r5900_write_pccr(uint32_t value) +{ + __asm__ __volatile__("mtps %0, 0" :: "r"(value) : "memory"); +} + +static inline void r5900_write_pcr0(uint32_t value) +{ + __asm__ __volatile__("mtpc %0, 0" :: "r"(value)); +} + +static inline void r5900_write_pcr1(uint32_t value) +{ + __asm__ __volatile__("mtpc %0, 1" :: "r"(value)); +} + +static uint32_t r5900_perf_control(r5900_pcr0_event_t event0, + r5900_pcr1_event_t event1) +{ + return R5900_PCCR_CTE | R5900_PCCR_APP_MODES0 | R5900_PCCR_APP_MODES1 | + (((uint32_t)event0 & R5900_PCCR_EVENT_MASK) + << R5900_PCCR_EVENT0_SHIFT) | + (((uint32_t)event1 & R5900_PCCR_EVENT_MASK) + << R5900_PCCR_EVENT1_SHIFT); +} + +int r5900_perf_begin(r5900_perf_scope_t *scope, + r5900_pcr0_event_t event0, + r5900_pcr1_event_t event1) +{ + uint32_t control; + + if (scope == NULL || scope->active || event0 < 0 || event0 > 16 || + event1 < 0 || event1 > 16) + return -1; + + r5900_perf_sync(); + scope->previous_pccr = r5900_read_pccr(); + scope->previous_pcr0 = r5900_read_pcr0(); + scope->previous_pcr1 = r5900_read_pcr1(); + scope->event0 = (uint32_t)event0; + scope->event1 = (uint32_t)event1; + + /* Disable first so resetting PCR0/PCR1 cannot race an enabled counter. */ + r5900_write_pccr(scope->previous_pccr & ~R5900_PCCR_CTE); + r5900_perf_sync(); + r5900_write_pcr0(0); + r5900_write_pcr1(0); + control = r5900_perf_control(event0, event1); + r5900_write_pccr(control); + r5900_perf_sync(); + scope->active = 1; + return 0; +} + +int r5900_perf_end(r5900_perf_scope_t *scope, r5900_perf_result_t *result) +{ + uint32_t raw0; + uint32_t raw1; + + if (scope == NULL || result == NULL || !scope->active) + return -1; + + /* Stop at a serialized boundary, then snapshot before restoring any prior + * debugger/profiler state. */ + r5900_perf_sync(); + r5900_write_pccr(r5900_read_pccr() & ~R5900_PCCR_CTE); + r5900_perf_sync(); + raw0 = r5900_read_pcr0(); + raw1 = r5900_read_pcr1(); + + result->pcr0 = raw0 & R5900_PCR_VALUE_MASK; + result->pcr1 = raw1 & R5900_PCR_VALUE_MASK; + result->pcr0_overflow = (raw0 & R5900_PCR_OVERFLOW) != 0; + result->pcr1_overflow = (raw1 & R5900_PCR_OVERFLOW) != 0; + + r5900_write_pcr0(scope->previous_pcr0); + r5900_write_pcr1(scope->previous_pcr1); + r5900_write_pccr(scope->previous_pccr); + r5900_perf_sync(); + scope->active = 0; + return 0; +} From 9da8ad4509c57ea1c30678cf181f5bd23f30e70a Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 13:14:29 +0200 Subject: [PATCH 010/156] Build R5900 performance counter harness --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 43d66e90..420f9d50 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ EE_BIN = PS2_HDD_BOOTSTRAP_MANAGER.ELF EE_MAP = PS2_HDD_BOOTSTRAP_MANAGER.map -EE_OBJS = main.o manager_menu_ps2.o app_ui_ps2.o disk_status_ps2.o gs_ui_ps2.o gs_debug_compat_ps2.o app_error.o bootstrap_controller_ps2.o diagnostics_controller_ps2.o forensic_controller_ps2.o platform.o storage.o video_mode.o ui_layout.o ui_font.o spleen_font_data.o header_backup.o repair_snapshot.o forensic_snapshot.o rescue_image.o rescue_storage.o bootstrap_source.o bootstrap_signing.o apa.o apa_repair.o apa_forensic.o repair_health.o hdd_bounds.o hdd_read.o hdd_write.o hdd_repair_ps2.o hdd_forensic_repair_ps2.o repair_controller_ps2.o hdd_recovery_wrap.o bootstrap_transaction.o bootstrap_transaction_ps2.o boot_chain.o boot_chain_ps2.o boot_payload.o boot_payload_ps2.o boot_diagnostics_ps2.o boot_report.o boot_report_ps2.o boot_report_session.o session_log.o kelf.o sha256.o capsule_format.o mbr_compat.o hdl_iso.o hdl_partition.o hdl_transaction.o hdl_installer_ps2.o +EE_OBJS = main.o manager_menu_ps2.o app_ui_ps2.o disk_status_ps2.o gs_ui_ps2.o gs_debug_compat_ps2.o app_error.o bootstrap_controller_ps2.o diagnostics_controller_ps2.o forensic_controller_ps2.o platform.o storage.o video_mode.o ui_layout.o ui_font.o spleen_font_data.o header_backup.o repair_snapshot.o forensic_snapshot.o rescue_image.o rescue_storage.o bootstrap_source.o bootstrap_signing.o apa.o apa_repair.o apa_forensic.o repair_health.o hdd_bounds.o hdd_read.o hdd_write.o hdd_repair_ps2.o hdd_forensic_repair_ps2.o repair_controller_ps2.o hdd_recovery_wrap.o bootstrap_transaction.o bootstrap_transaction_ps2.o boot_chain.o boot_chain_ps2.o boot_payload.o boot_payload_ps2.o boot_diagnostics_ps2.o boot_report.o boot_report_ps2.o boot_report_session.o session_log.o kelf.o sha256.o capsule_format.o mbr_compat.o hdl_iso.o hdl_partition.o hdl_transaction.o hdl_installer_ps2.o r5900_perf.o EE_LIBS = -ldebug -ldraw -lgraph -lpacket -ldma -lm -lpad -lfileXio -lpatches -lpoweroff -lsecr -lkernel # LTO lets the R5900 compiler optimize across the deliberately small modules # while section GC still removes unused recovery/UI helpers from the final ELF. @@ -306,6 +306,9 @@ hdl_transaction.o: src/hdl_transaction.c hdl_installer_ps2.o: src/hdl_installer_ps2.c $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ +r5900_perf.o: src/r5900_perf.c + $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ + %_irx.c: $(PS2SDK)/bin/bin2c $(PS2SDK)/iop/irx/$*.irx $@ $*_irx From df31d6498adc0ca6c3c790a4f2207906d30605e9 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 13:15:01 +0200 Subject: [PATCH 011/156] Generate benchmark provenance manifest --- tools/build_benchmark_provenance.sh | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tools/build_benchmark_provenance.sh diff --git a/tools/build_benchmark_provenance.sh b/tools/build_benchmark_provenance.sh new file mode 100644 index 00000000..18ca829a --- /dev/null +++ b/tools/build_benchmark_provenance.sh @@ -0,0 +1,49 @@ +#!/bin/sh +set -eu + +OUT=${1:-BENCHMARK_PROVENANCE.yml} +CC=${EE_CC:-mips64r5900el-ps2-elf-gcc} +GIT_SHA=$(git rev-parse HEAD 2>/dev/null || printf 'unavailable') +GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || printf 'unavailable') +CC_TARGET=$($CC -dumpmachine 2>/dev/null || printf 'unavailable') +CC_VERSION=$($CC -dumpfullversion -dumpversion 2>/dev/null || printf 'unavailable') +PS2SDK_SHA=unavailable +PS2SDK_PATH_VALUE=${PS2SDK:-unavailable} + +if [ "${PS2SDK:-}" != "" ] && git -C "$PS2SDK" rev-parse HEAD >/dev/null 2>&1; then + PS2SDK_SHA=$(git -C "$PS2SDK" rev-parse HEAD) +fi + +cat > "$OUT" < Date: Tue, 25 Aug 2026 13:15:16 +0200 Subject: [PATCH 012/156] Archive corpus-v2 benchmark provenance --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36044c13..90f65dc2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,7 @@ jobs: docker run --rm -v "$PWD:/work" -w /work ps2dev/ps2dev:v2.0.0 sh -c 'apk add --no-cache make python3 >/dev/null && sh tools/r5900_toolchain_audit.sh GCC_R5900_TARGET.txt && + sh tools/build_benchmark_provenance.sh BENCHMARK_PROVENANCE.yml && python3 tools/corpus_v2_project_audit.py --output CORPUS_V2_PROJECT_AUDIT.txt && make clean && make && @@ -46,6 +47,7 @@ jobs: OPTIMIZATION_AUDIT.txt CORPUS_V2_PROJECT_AUDIT.txt GCC_R5900_TARGET.txt + BENCHMARK_PROVENANCE.yml HDDMAN.CFG LICENSE THIRD_PARTY_NOTICES.md From a3998f30d4c70819fc1de01d5112bf00b6e20e20 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 13:16:42 +0200 Subject: [PATCH 013/156] Add HDL hardware performance log parser --- tools/parse_hdl_perf.py | 189 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 tools/parse_hdl_perf.py diff --git a/tools/parse_hdl_perf.py b/tools/parse_hdl_perf.py new file mode 100644 index 00000000..c30b02d2 --- /dev/null +++ b/tools/parse_hdl_perf.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Parse corpus-v2 HDL performance records from HDDMAN.LOG. + +The PS2 runtime deliberately emits compact counters and formats them only at +phase boundaries. This host-side parser turns those records into stable JSON so +hardware A/B results can be compared without teaching the IOP about JSON, which +would be a fairly creative misuse of 2 MiB of RAM. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +PREFIX = r"(?:\[\d+\]\s+)?" + +RATE_RE = re.compile( + PREFIX + + r"HDL fast copy measured bytes=(\d+) usec=(\d+) rate=(\d+) KiB/s raw-usb11=(\d+)\.(\d+)%" +) +EE_LAT_RE = re.compile( + PREFIX + + r"HDL perf ([\w-]+) samples=(\d+) p50<=(\d+)us p95<=(\d+)us p99<=(\d+)us max=(\d+)us" +) +IOP_LAT_RE = re.compile( + PREFIX + + r"HDL IOP perf ([\w-]+) samples=(\d+) p50<=(\d+)us p95<=(\d+)us p99<=(\d+)us max=(\d+)us" +) +SNAPSHOT_RE = re.compile( + PREFIX + + r"HDL fast I/O snapshot phase=([\w-]+) flags=0x([0-9a-fA-F]+) fragments=(\d+) " + r"direct=(\d+) fallback=(\d+) prefetch-hit=(\d+) miss=(\d+) pump=(\d+) " + r"sectors=(\d+) src-dma=(\d+) target-dma=(\d+)" +) +TRAFFIC_RE = re.compile( + PREFIX + + r"HDL IOP traffic phase=([\w-]+) direct-src-sectors=(\d+) fallback-src-sectors=(\d+) " + r"hdd-write-sectors=(\d+) hdd-read-sectors=(\d+) sif-dma-sectors=(\d+)" +) +EE_COPY_TRAFFIC_RE = re.compile( + PREFIX + + r"HDL perf copy traffic useful=(\d+) sif-dma=(\d+) ee-cache-maint=(\d+) fallback-source=(\d+)" +) +EE_VERIFY_TRAFFIC_RE = re.compile( + PREFIX + + r"HDL perf verify traffic target=(\d+) sif-dma-total=(\d+) ee-cache-maint-total=(\d+) " + r"fallback-target=(\d+) consumer-samples=(\d+) final-chunk-excluded=(\d+)" +) + + +def _latency(match: re.Match[str]) -> dict[str, int]: + return { + "samples": int(match.group(2)), + "p50_upper_us": int(match.group(3)), + "p95_upper_us": int(match.group(4)), + "p99_upper_us": int(match.group(5)), + "max_us": int(match.group(6)), + } + + +def parse_log(text: str) -> dict[str, object]: + result: dict[str, object] = { + "copy_rate": None, + "ee_latency": {}, + "iop_latency": {}, + "snapshots": {}, + "iop_traffic": {}, + "ee_traffic": {}, + } + + for line in text.splitlines(): + match = RATE_RE.search(line) + if match: + result["copy_rate"] = { + "bytes": int(match.group(1)), + "usec": int(match.group(2)), + "kib_per_second": int(match.group(3)), + "raw_usb11_percent_tenths": int(match.group(4)) * 10 + + int(match.group(5)), + } + continue + + match = IOP_LAT_RE.search(line) + if match: + result["iop_latency"][match.group(1)] = _latency(match) # type: ignore[index] + continue + + match = EE_LAT_RE.search(line) + if match: + result["ee_latency"][match.group(1)] = _latency(match) # type: ignore[index] + continue + + match = SNAPSHOT_RE.search(line) + if match: + result["snapshots"][match.group(1)] = { # type: ignore[index] + "flags": int(match.group(2), 16), + "fragments": int(match.group(3)), + "direct_reads": int(match.group(4)), + "fallback_reads": int(match.group(5)), + "prefetch_hits": int(match.group(6)), + "prefetch_misses": int(match.group(7)), + "pumped_chunks": int(match.group(8)), + "pumped_sectors": int(match.group(9)), + "source_dma_chunks": int(match.group(10)), + "target_dma_chunks": int(match.group(11)), + } + continue + + match = TRAFFIC_RE.search(line) + if match: + result["iop_traffic"][match.group(1)] = { # type: ignore[index] + "direct_source_sectors": int(match.group(2)), + "fallback_source_sectors": int(match.group(3)), + "hdd_write_sectors": int(match.group(4)), + "hdd_read_sectors": int(match.group(5)), + "sif_dma_sectors": int(match.group(6)), + } + continue + + match = EE_COPY_TRAFFIC_RE.search(line) + if match: + result["ee_traffic"]["copy"] = { # type: ignore[index] + "useful_bytes": int(match.group(1)), + "sif_dma_bytes": int(match.group(2)), + "ee_cache_maintenance_bytes": int(match.group(3)), + "fallback_source_bytes": int(match.group(4)), + } + continue + + match = EE_VERIFY_TRAFFIC_RE.search(line) + if match: + result["ee_traffic"]["verify"] = { # type: ignore[index] + "target_bytes": int(match.group(1)), + "sif_dma_total_bytes": int(match.group(2)), + "ee_cache_maintenance_total_bytes": int(match.group(3)), + "fallback_target_bytes": int(match.group(4)), + "consumer_samples": int(match.group(5)), + "final_chunk_excluded": int(match.group(6)), + } + + return result + + +def selftest() -> None: + sample = """ +[0042] HDL fast copy measured bytes=1048576 usec=1000000 rate=1024 KiB/s raw-usb11=69.9% +[0043] HDL IOP perf usb-direct-read samples=16 p50<=65536us p95<=131072us p99<=131072us max=70000us +[0044] HDL perf pump-ioctl samples=16 p50<=65536us p95<=131072us p99<=131072us max=75000us +[0045] HDL fast I/O snapshot phase=copy-final flags=0x00000007 fragments=1 direct=16 fallback=0 prefetch-hit=15 miss=0 pump=16 sectors=2048 src-dma=16 target-dma=0 +[0046] HDL IOP traffic phase=copy-final direct-src-sectors=2048 fallback-src-sectors=0 hdd-write-sectors=2048 hdd-read-sectors=0 sif-dma-sectors=2048 +[0047] HDL perf copy traffic useful=1048576 sif-dma=1048576 ee-cache-maint=2097152 fallback-source=0 +""" + parsed = parse_log(sample) + assert parsed["copy_rate"]["kib_per_second"] == 1024 # type: ignore[index] + assert parsed["iop_latency"]["usb-direct-read"]["samples"] == 16 # type: ignore[index] + assert parsed["ee_latency"]["pump-ioctl"]["max_us"] == 75000 # type: ignore[index] + assert parsed["snapshots"]["copy-final"]["flags"] == 7 # type: ignore[index] + assert parsed["iop_traffic"]["copy-final"]["hdd_write_sectors"] == 2048 # type: ignore[index] + assert parsed["ee_traffic"]["copy"]["ee_cache_maintenance_bytes"] == 2097152 # type: ignore[index] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("log", nargs="?", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("parse_hdl_perf selftest: PASS") + return 0 + if args.log is None: + parser.error("log is required unless --selftest is used") + + parsed = parse_log(args.log.read_text(encoding="utf-8", errors="replace")) + rendered = json.dumps(parsed, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(rendered, encoding="utf-8") + else: + sys.stdout.write(rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From c5a7180d596f0ea1e04e94a8c07fac2135fa823c Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 13:16:57 +0200 Subject: [PATCH 014/156] Self-test HDL performance log parser --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90f65dc2..7675aeef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,8 @@ jobs: run: make test-host - name: Self-test guarded hardware fault injector run: python3 tools/hardware_fault_injector.py selftest + - name: Self-test HDL performance log parser + run: python3 tools/parse_hdl_perf.py --selftest ps2-build: runs-on: ubuntu-latest From 439fbc692ababd66bfd1d7483ced9d55c74860b6 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 13:21:37 +0200 Subject: [PATCH 015/156] Pin reproducible PS2SDK provenance --- tools/build_benchmark_provenance.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tools/build_benchmark_provenance.sh b/tools/build_benchmark_provenance.sh index 18ca829a..dc30e925 100644 --- a/tools/build_benchmark_provenance.sh +++ b/tools/build_benchmark_provenance.sh @@ -3,15 +3,23 @@ set -eu OUT=${1:-BENCHMARK_PROVENANCE.yml} CC=${EE_CC:-mips64r5900el-ps2-elf-gcc} -GIT_SHA=$(git rev-parse HEAD 2>/dev/null || printf 'unavailable') -GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || printf 'unavailable') +GIT_SHA=${PROJECT_GIT_SHA:-$(git rev-parse HEAD 2>/dev/null || printf 'unavailable')} +GIT_REF=${PROJECT_GIT_REF:-$(git rev-parse --abbrev-ref HEAD 2>/dev/null || printf 'unavailable')} CC_TARGET=$($CC -dumpmachine 2>/dev/null || printf 'unavailable') CC_VERSION=$($CC -dumpfullversion -dumpversion 2>/dev/null || printf 'unavailable') -PS2SDK_SHA=unavailable PS2SDK_PATH_VALUE=${PS2SDK:-unavailable} +PS2SDK_REF=${PS2SDK_SOURCE_REF:-unavailable} +PS2SDK_SHA=${PS2SDK_SOURCE_SHA:-unavailable} +PS2DEV_BUNDLE_REF=${PS2DEV_BUNDLE_REF:-unavailable} +# A development environment may preserve the ps2sdk .git directory. Prefer the +# exact installed checkout when available. Tagged ps2dev Docker images strip +# source metadata after installation, so CI passes the source ref/SHA that the +# image build scripts selected instead of pretending an absent .git means an +# unknown software stack. if [ "${PS2SDK:-}" != "" ] && git -C "$PS2SDK" rev-parse HEAD >/dev/null 2>&1; then PS2SDK_SHA=$(git -C "$PS2SDK" rev-parse HEAD) + PS2SDK_REF=$(git -C "$PS2SDK" describe --always --tags 2>/dev/null || printf '%s' "$PS2SDK_REF") fi cat > "$OUT" < "$OUT" < Date: Tue, 25 Aug 2026 13:22:02 +0200 Subject: [PATCH 016/156] Pass exact build provenance into PS2 container --- .github/workflows/ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7675aeef..204ae404 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,13 @@ jobs: - uses: actions/checkout@v4 - name: Build, audit and strip PS2 ELF with PS2DEV v2.0.0 run: >- - docker run --rm -v "$PWD:/work" -w /work ps2dev/ps2dev:v2.0.0 + docker run --rm + -e PROJECT_GIT_SHA="$GITHUB_SHA" + -e PROJECT_GIT_REF="$GITHUB_REF" + -e PS2DEV_BUNDLE_REF="v2.0.0" + -e PS2SDK_SOURCE_REF="v2.0.0" + -e PS2SDK_SOURCE_SHA="b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b" + -v "$PWD:/work" -w /work ps2dev/ps2dev:v2.0.0 sh -c 'apk add --no-cache make python3 >/dev/null && sh tools/r5900_toolchain_audit.sh GCC_R5900_TARGET.txt && sh tools/build_benchmark_provenance.sh BENCHMARK_PROVENANCE.yml && From fc8a2d314dd924c3d3ccb757d8f5ff1fbdaab120 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 13:23:33 +0200 Subject: [PATCH 017/156] Update corpus-v2 implementation progress --- docs/CORPUS_V2_IMPLEMENTATION_PLAN.md | 69 +++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md b/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md index 3db26e65..b7559a9d 100644 --- a/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md +++ b/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md @@ -80,15 +80,63 @@ device sector/transfer unit, VIF/GIF packet, or another explicit contract. Goal: establish evidence before altering architecture. -- [ ] Record exact PS2SDK/toolchain/build flags automatically in benchmark logs. -- [ ] Record console SCPH/hardware revision, adapters, active IRX and workload. -- [ ] Add per-stage HDL fast-path timing for source I/O, prefetch wait, HDD, - SIF DMA and EE consumer work. -- [ ] Record useful bytes, DMA bytes, CPU-copy bytes and cache-maintenance bytes. -- [ ] Add p50/p95/p99/max reporting for I/O latency, not just average throughput. -- [ ] Keep hot-path logging binary/counter based; format only outside the path. -- [ ] Add R5900 performance-counter harness with companion non-instrumented run. -- [ ] Preserve linker map, symbol sizes and optimization audit in CI artifacts. +- [x] Record project SHA/ref, pinned PS2SDK source ref/SHA, toolchain identity and + build flags automatically in `BENCHMARK_PROVENANCE.yml`. +- [ ] Record console SCPH/hardware revision, adapters, runtime active IRX and + workload during a real-hardware benchmark. Build CI intentionally leaves + those fields `UNRECORDED` rather than inferring them. +- [x] Add per-stage HDL fast-path timing for direct/fallback source I/O, + prefetch consumer wait, HDD read/write, SIF DMA and EE consumer work. +- [x] Record useful bytes, SIF DMA bytes, HDD/source sectors, fallback bytes and + EE cache-maintenance bytes. +- [x] Add p50/p95/p99/max reporting for I/O latency, not just average throughput. +- [x] Keep hot-path logging counter/histogram based; format only at phase exit. +- [x] Add an R5900 performance-counter harness using the EE Core Manual event + table and dedicated `mfpc/mtpc/mfps/mtps` instructions. The harness + preserves/restores prior counter state and has passed current-toolchain CI. +- [x] Preserve linker map, symbol sizes and optimization audit in CI artifacts. +- [x] Add a host parser that converts `HDDMAN.LOG` corpus-v2 performance records + to stable JSON and self-tests in CI. +- [ ] Add a same-source profiling-on/profiling-off build pair for authoritative + instrumentation-overhead A/B on real hardware. +- [ ] Exercise the R5900 counter harness in a bounded hardware benchmark and + measure empty-scope overhead before instrumenting application kernels. + +### Phase-0 implementation notes + +The first EE profiler pass increased `execute_transaction()` from the audited +6420 B baseline to 7492 B under LTO. This was rejected. Profiling helpers were +then isolated with selective `noinline` and report paths with `cold,noinline`. +The instrumented transaction body became 6176 B, smaller than baseline, while +retaining the counters. This proves only static footprint, not runtime overhead. + +The IOP path now records logarithmic microsecond latency histograms without +`printf` in the hot path. Current categories are: + +```text +usb-direct-read +source-fallback-read +prefetch-consumer-wait +hdd-write +hdd-read +sif-dma-completion +``` + +EE-side companion categories are: + +```text +pump-ioctl +source-ioctl +target-ioctl +copy-ee-consumer +verify-ee-consumer +``` + +The pinned `ps2dev/ps2dev:v2.0.0` image is tied to the `ps2dev` v2.0.0 build +bundle. Its tagged build passes `v2.0.0` to the PS2SDK build script, which checks +out PS2SDK `v2.0.0`; that annotated tag resolves to commit +`b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b`. CI records that exact source +provenance rather than substituting current PS2SDK master. Exit gate: measurements are reproducible on at least one real console and the instrumented build has a documented overhead A/B against an uninstrumented build. @@ -139,7 +187,8 @@ commit safety. ## Phase 4: IOP/SIF service architecture -- [ ] Measure queue/service/transport/completion latency separately. +- [x] Instrument queue-adjacent prefetch wait, service/device work and SIF + completion independently enough to locate the dominant 64 KiB stage. - [ ] Maintain the IOP-local producer path where the final device consumer is on the IOP; do not bounce payload through EE without a consumer requirement. - [ ] Keep control metadata coarse-grained and bulk payload on DMA/data-plane paths. From fe235ffce0c489201a7180fbf2fb7eede9772387 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 20:44:19 +0200 Subject: [PATCH 018/156] Phase 1: add minimal GS UI draw2d primitive subset --- src/gs_ui_draw_minimal_ps2.c | 135 +++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 src/gs_ui_draw_minimal_ps2.c diff --git a/src/gs_ui_draw_minimal_ps2.c b/src/gs_ui_draw_minimal_ps2.c new file mode 100644 index 00000000..b87c7f35 --- /dev/null +++ b/src/gs_ui_draw_minimal_ps2.c @@ -0,0 +1,135 @@ +/* + * Minimal libdraw-compatible 2D primitive subset for the manager UI. + * + * The application only needs filled/outlined rectangles, textured sprites and + * the draw2d blending toggle. Pulling PS2SDK's monolithic draw2d.o also pulls + * arc/rounded-rectangle code and therefore sinf/cosf plus their libm support. + * Keep the exact current draw2d packet semantics for the used primitives while + * leaving the rest of libdraw available for environment/texture helpers. + * + * Packet layout and coordinate biases follow current PS2SDK draw2d.c. This is + * intentionally a narrow compatibility shim, not a new renderer abstraction. + */ + +#include +#include +#include +#include + +#define GS_UI_DRAW_START_OFFSET 2047.5625f +#define GS_UI_DRAW_END_OFFSET 2048.5625f + +#define GS_UI_DRAW_RECT_OUT_NREG 8 +#define GS_UI_DRAW_RECT_OUT_REGLIST \ + (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ + ((u64)GIF_REG_XYZ2) << 8 | ((u64)GIF_REG_XYZ2) << 12 | \ + ((u64)GIF_REG_XYZ2) << 16 | ((u64)GIF_REG_XYZ2) << 20 | \ + ((u64)GIF_REG_XYZ2) << 24 | ((u64)GIF_REG_NOP) << 28) + +#define GS_UI_DRAW_SPRITE_NREG 4 +#define GS_UI_DRAW_SPRITE_REGLIST \ + (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ + ((u64)GIF_REG_XYZ2) << 8 | ((u64)GIF_REG_XYZ2) << 12) + +#define GS_UI_DRAW_TEX_NREG 6 +#define GS_UI_DRAW_TEX_REGLIST \ + (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ + ((u64)GIF_REG_UV) << 8 | ((u64)GIF_REG_XYZ2) << 12 | \ + ((u64)GIF_REG_UV) << 16 | ((u64)GIF_REG_XYZ2) << 20) + +static int gs_ui_draw_blending; + +void draw_enable_blending(void) +{ + gs_ui_draw_blending = 1; +} + +void draw_disable_blending(void) +{ + gs_ui_draw_blending = 0; +} + +qword_t *draw_rect_outline(qword_t *q, int context, rect_t *rect) +{ + int x0 = ftoi4(rect->v0.x + GS_UI_DRAW_START_OFFSET); + int y0 = ftoi4(rect->v0.y + GS_UI_DRAW_START_OFFSET); + int x1 = ftoi4(rect->v1.x + GS_UI_DRAW_END_OFFSET); + int y1 = ftoi4(rect->v1.y + GS_UI_DRAW_END_OFFSET); + + PACK_GIFTAG(q, + GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, + GS_UI_DRAW_RECT_OUT_NREG), + GS_UI_DRAW_RECT_OUT_REGLIST); + q++; + + q->dw[0] = GIF_SET_PRIM(PRIM_LINE_STRIP, 0, 0, 0, gs_ui_draw_blending, + 0, 0, context, 0); + q->dw[1] = rect->color.rgbaq; + q++; + + q->dw[0] = GIF_SET_XYZ(x0, y0, rect->v0.z); + q->dw[1] = GIF_SET_XYZ(x1, y0, rect->v0.z); + q++; + + q->dw[0] = GIF_SET_XYZ(x1, y1, rect->v0.z); + q->dw[1] = GIF_SET_XYZ(x0, y1, rect->v0.z); + q++; + + q->dw[0] = GIF_SET_XYZ(x0, y0, rect->v0.z); + q->dw[1] = 0; + q++; + + return q; +} + +qword_t *draw_rect_filled(qword_t *q, int context, rect_t *rect) +{ + PACK_GIFTAG(q, + GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, + GS_UI_DRAW_SPRITE_NREG), + GS_UI_DRAW_SPRITE_REGLIST); + q++; + + q->dw[0] = GIF_SET_PRIM(PRIM_SPRITE, 0, 0, 0, gs_ui_draw_blending, + 0, 0, context, 0); + q->dw[1] = rect->color.rgbaq; + q++; + + q->dw[0] = GIF_SET_XYZ(ftoi4(rect->v0.x + GS_UI_DRAW_START_OFFSET), + ftoi4(rect->v0.y + GS_UI_DRAW_START_OFFSET), + rect->v0.z); + q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v1.x + GS_UI_DRAW_END_OFFSET), + ftoi4(rect->v1.y + GS_UI_DRAW_END_OFFSET), + rect->v0.z); + q++; + + return q; +} + +qword_t *draw_rect_textured(qword_t *q, int context, texrect_t *rect) +{ + PACK_GIFTAG(q, + GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, + GS_UI_DRAW_TEX_NREG), + GS_UI_DRAW_TEX_REGLIST); + q++; + + q->dw[0] = GIF_SET_PRIM(PRIM_SPRITE, 0, DRAW_ENABLE, 0, + gs_ui_draw_blending, 0, PRIM_MAP_UV, context, 0); + q->dw[1] = rect->color.rgbaq; + q++; + + q->dw[0] = GIF_SET_UV(ftoi4(rect->t0.u), ftoi4(rect->t0.v)); + q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v0.x + GS_UI_DRAW_START_OFFSET), + ftoi4(rect->v0.y + GS_UI_DRAW_START_OFFSET), + rect->v0.z); + q++; + + q->dw[0] = GIF_SET_UV(ftoi4(rect->t1.u), ftoi4(rect->t1.v)); + q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v1.x + GS_UI_DRAW_END_OFFSET), + ftoi4(rect->v1.y + GS_UI_DRAW_END_OFFSET), + rect->v0.z); + q++; + + return q; +} From e8c81a309aa15535fe62c0c970e354ab9ba6a7ec Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 20:45:01 +0200 Subject: [PATCH 019/156] Phase 1: shadow unused draw2d archive with minimal UI primitives --- src/gs_debug_compat_ps2.c | 129 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/src/gs_debug_compat_ps2.c b/src/gs_debug_compat_ps2.c index bb60ff9c..22c65374 100644 --- a/src/gs_debug_compat_ps2.c +++ b/src/gs_debug_compat_ps2.c @@ -5,13 +5,47 @@ * libdebug is now allowed to provide init_scr() as the hardware-proven CRT / * read-circuit bootstrap. The linker wraps its drawing entry points, however, * so application text and clears still go exclusively through gs_ui_ps2. + * + * This translation unit also supplies the only draw2d primitives used by the + * frontend. Current PS2SDK keeps arcs, rounded rectangles and the basic sprite + * helpers in one draw2d.o archive member; referencing one basic primitive then + * drags sinf/cosf and their libm support into the EE ELF. The narrow subset + * below preserves current PS2SDK packet semantics for the operations we use so + * the monolithic archive member can remain unlinked. */ #include +#include +#include +#include +#include #include #include "gs_ui_ps2.h" +#define GS_UI_DRAW_START_OFFSET 2047.5625f +#define GS_UI_DRAW_END_OFFSET 2048.5625f + +#define GS_UI_DRAW_RECT_OUT_NREG 8 +#define GS_UI_DRAW_RECT_OUT_REGLIST \ + (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ + ((u64)GIF_REG_XYZ2) << 8 | ((u64)GIF_REG_XYZ2) << 12 | \ + ((u64)GIF_REG_XYZ2) << 16 | ((u64)GIF_REG_XYZ2) << 20 | \ + ((u64)GIF_REG_XYZ2) << 24 | ((u64)GIF_REG_NOP) << 28) + +#define GS_UI_DRAW_SPRITE_NREG 4 +#define GS_UI_DRAW_SPRITE_REGLIST \ + (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ + ((u64)GIF_REG_XYZ2) << 8 | ((u64)GIF_REG_XYZ2) << 12) + +#define GS_UI_DRAW_TEX_NREG 6 +#define GS_UI_DRAW_TEX_REGLIST \ + (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ + ((u64)GIF_REG_UV) << 8 | ((u64)GIF_REG_XYZ2) << 12 | \ + ((u64)GIF_REG_UV) << 16 | ((u64)GIF_REG_XYZ2) << 20) + +static int gs_ui_draw_blending; + void __wrap_scr_clear(void) { gs_ui_console_clear(); @@ -30,3 +64,98 @@ void __wrap_scr_printf(const char *format, ...) gs_ui_console_vprintf(format, arguments); va_end(arguments); } + +void draw_enable_blending(void) +{ + gs_ui_draw_blending = 1; +} + +void draw_disable_blending(void) +{ + gs_ui_draw_blending = 0; +} + +qword_t *draw_rect_outline(qword_t *q, int context, rect_t *rect) +{ + int x0 = ftoi4(rect->v0.x + GS_UI_DRAW_START_OFFSET); + int y0 = ftoi4(rect->v0.y + GS_UI_DRAW_START_OFFSET); + int x1 = ftoi4(rect->v1.x + GS_UI_DRAW_END_OFFSET); + int y1 = ftoi4(rect->v1.y + GS_UI_DRAW_END_OFFSET); + + PACK_GIFTAG(q, + GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, + GS_UI_DRAW_RECT_OUT_NREG), + GS_UI_DRAW_RECT_OUT_REGLIST); + q++; + + q->dw[0] = GIF_SET_PRIM(PRIM_LINE_STRIP, 0, 0, 0, gs_ui_draw_blending, + 0, 0, context, 0); + q->dw[1] = rect->color.rgbaq; + q++; + + q->dw[0] = GIF_SET_XYZ(x0, y0, rect->v0.z); + q->dw[1] = GIF_SET_XYZ(x1, y0, rect->v0.z); + q++; + + q->dw[0] = GIF_SET_XYZ(x1, y1, rect->v0.z); + q->dw[1] = GIF_SET_XYZ(x0, y1, rect->v0.z); + q++; + + q->dw[0] = GIF_SET_XYZ(x0, y0, rect->v0.z); + q->dw[1] = 0; + q++; + + return q; +} + +qword_t *draw_rect_filled(qword_t *q, int context, rect_t *rect) +{ + PACK_GIFTAG(q, + GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, + GS_UI_DRAW_SPRITE_NREG), + GS_UI_DRAW_SPRITE_REGLIST); + q++; + + q->dw[0] = GIF_SET_PRIM(PRIM_SPRITE, 0, 0, 0, gs_ui_draw_blending, + 0, 0, context, 0); + q->dw[1] = rect->color.rgbaq; + q++; + + q->dw[0] = GIF_SET_XYZ(ftoi4(rect->v0.x + GS_UI_DRAW_START_OFFSET), + ftoi4(rect->v0.y + GS_UI_DRAW_START_OFFSET), + rect->v0.z); + q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v1.x + GS_UI_DRAW_END_OFFSET), + ftoi4(rect->v1.y + GS_UI_DRAW_END_OFFSET), + rect->v0.z); + q++; + + return q; +} + +qword_t *draw_rect_textured(qword_t *q, int context, texrect_t *rect) +{ + PACK_GIFTAG(q, + GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, + GS_UI_DRAW_TEX_NREG), + GS_UI_DRAW_TEX_REGLIST); + q++; + + q->dw[0] = GIF_SET_PRIM(PRIM_SPRITE, 0, DRAW_ENABLE, 0, + gs_ui_draw_blending, 0, PRIM_MAP_UV, context, 0); + q->dw[1] = rect->color.rgbaq; + q++; + + q->dw[0] = GIF_SET_UV(ftoi4(rect->t0.u), ftoi4(rect->t0.v)); + q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v0.x + GS_UI_DRAW_START_OFFSET), + ftoi4(rect->v0.y + GS_UI_DRAW_START_OFFSET), + rect->v0.z); + q++; + + q->dw[0] = GIF_SET_UV(ftoi4(rect->t1.u), ftoi4(rect->t1.v)); + q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v1.x + GS_UI_DRAW_END_OFFSET), + ftoi4(rect->v1.y + GS_UI_DRAW_END_OFFSET), + rect->v0.z); + q++; + + return q; +} From 4a62a65ba4fb889e15c78f697b3ee1d9e3566414 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 20:45:20 +0200 Subject: [PATCH 020/156] Remove unused standalone minimal draw source --- src/gs_ui_draw_minimal_ps2.c | 135 ----------------------------------- 1 file changed, 135 deletions(-) delete mode 100644 src/gs_ui_draw_minimal_ps2.c diff --git a/src/gs_ui_draw_minimal_ps2.c b/src/gs_ui_draw_minimal_ps2.c deleted file mode 100644 index b87c7f35..00000000 --- a/src/gs_ui_draw_minimal_ps2.c +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Minimal libdraw-compatible 2D primitive subset for the manager UI. - * - * The application only needs filled/outlined rectangles, textured sprites and - * the draw2d blending toggle. Pulling PS2SDK's monolithic draw2d.o also pulls - * arc/rounded-rectangle code and therefore sinf/cosf plus their libm support. - * Keep the exact current draw2d packet semantics for the used primitives while - * leaving the rest of libdraw available for environment/texture helpers. - * - * Packet layout and coordinate biases follow current PS2SDK draw2d.c. This is - * intentionally a narrow compatibility shim, not a new renderer abstraction. - */ - -#include -#include -#include -#include - -#define GS_UI_DRAW_START_OFFSET 2047.5625f -#define GS_UI_DRAW_END_OFFSET 2048.5625f - -#define GS_UI_DRAW_RECT_OUT_NREG 8 -#define GS_UI_DRAW_RECT_OUT_REGLIST \ - (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ - ((u64)GIF_REG_XYZ2) << 8 | ((u64)GIF_REG_XYZ2) << 12 | \ - ((u64)GIF_REG_XYZ2) << 16 | ((u64)GIF_REG_XYZ2) << 20 | \ - ((u64)GIF_REG_XYZ2) << 24 | ((u64)GIF_REG_NOP) << 28) - -#define GS_UI_DRAW_SPRITE_NREG 4 -#define GS_UI_DRAW_SPRITE_REGLIST \ - (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ - ((u64)GIF_REG_XYZ2) << 8 | ((u64)GIF_REG_XYZ2) << 12) - -#define GS_UI_DRAW_TEX_NREG 6 -#define GS_UI_DRAW_TEX_REGLIST \ - (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ - ((u64)GIF_REG_UV) << 8 | ((u64)GIF_REG_XYZ2) << 12 | \ - ((u64)GIF_REG_UV) << 16 | ((u64)GIF_REG_XYZ2) << 20) - -static int gs_ui_draw_blending; - -void draw_enable_blending(void) -{ - gs_ui_draw_blending = 1; -} - -void draw_disable_blending(void) -{ - gs_ui_draw_blending = 0; -} - -qword_t *draw_rect_outline(qword_t *q, int context, rect_t *rect) -{ - int x0 = ftoi4(rect->v0.x + GS_UI_DRAW_START_OFFSET); - int y0 = ftoi4(rect->v0.y + GS_UI_DRAW_START_OFFSET); - int x1 = ftoi4(rect->v1.x + GS_UI_DRAW_END_OFFSET); - int y1 = ftoi4(rect->v1.y + GS_UI_DRAW_END_OFFSET); - - PACK_GIFTAG(q, - GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, - GS_UI_DRAW_RECT_OUT_NREG), - GS_UI_DRAW_RECT_OUT_REGLIST); - q++; - - q->dw[0] = GIF_SET_PRIM(PRIM_LINE_STRIP, 0, 0, 0, gs_ui_draw_blending, - 0, 0, context, 0); - q->dw[1] = rect->color.rgbaq; - q++; - - q->dw[0] = GIF_SET_XYZ(x0, y0, rect->v0.z); - q->dw[1] = GIF_SET_XYZ(x1, y0, rect->v0.z); - q++; - - q->dw[0] = GIF_SET_XYZ(x1, y1, rect->v0.z); - q->dw[1] = GIF_SET_XYZ(x0, y1, rect->v0.z); - q++; - - q->dw[0] = GIF_SET_XYZ(x0, y0, rect->v0.z); - q->dw[1] = 0; - q++; - - return q; -} - -qword_t *draw_rect_filled(qword_t *q, int context, rect_t *rect) -{ - PACK_GIFTAG(q, - GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, - GS_UI_DRAW_SPRITE_NREG), - GS_UI_DRAW_SPRITE_REGLIST); - q++; - - q->dw[0] = GIF_SET_PRIM(PRIM_SPRITE, 0, 0, 0, gs_ui_draw_blending, - 0, 0, context, 0); - q->dw[1] = rect->color.rgbaq; - q++; - - q->dw[0] = GIF_SET_XYZ(ftoi4(rect->v0.x + GS_UI_DRAW_START_OFFSET), - ftoi4(rect->v0.y + GS_UI_DRAW_START_OFFSET), - rect->v0.z); - q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v1.x + GS_UI_DRAW_END_OFFSET), - ftoi4(rect->v1.y + GS_UI_DRAW_END_OFFSET), - rect->v0.z); - q++; - - return q; -} - -qword_t *draw_rect_textured(qword_t *q, int context, texrect_t *rect) -{ - PACK_GIFTAG(q, - GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, - GS_UI_DRAW_TEX_NREG), - GS_UI_DRAW_TEX_REGLIST); - q++; - - q->dw[0] = GIF_SET_PRIM(PRIM_SPRITE, 0, DRAW_ENABLE, 0, - gs_ui_draw_blending, 0, PRIM_MAP_UV, context, 0); - q->dw[1] = rect->color.rgbaq; - q++; - - q->dw[0] = GIF_SET_UV(ftoi4(rect->t0.u), ftoi4(rect->t0.v)); - q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v0.x + GS_UI_DRAW_START_OFFSET), - ftoi4(rect->v0.y + GS_UI_DRAW_START_OFFSET), - rect->v0.z); - q++; - - q->dw[0] = GIF_SET_UV(ftoi4(rect->t1.u), ftoi4(rect->t1.v)); - q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v1.x + GS_UI_DRAW_END_OFFSET), - ftoi4(rect->v1.y + GS_UI_DRAW_END_OFFSET), - rect->v0.z); - q++; - - return q; -} From 44337560c0663f977477320a9b3a1deb0191f3b2 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 20:48:00 +0200 Subject: [PATCH 021/156] Restore GS debug compatibility after LTO archive experiment --- src/gs_debug_compat_ps2.c | 129 -------------------------------------- 1 file changed, 129 deletions(-) diff --git a/src/gs_debug_compat_ps2.c b/src/gs_debug_compat_ps2.c index 22c65374..bb60ff9c 100644 --- a/src/gs_debug_compat_ps2.c +++ b/src/gs_debug_compat_ps2.c @@ -5,47 +5,13 @@ * libdebug is now allowed to provide init_scr() as the hardware-proven CRT / * read-circuit bootstrap. The linker wraps its drawing entry points, however, * so application text and clears still go exclusively through gs_ui_ps2. - * - * This translation unit also supplies the only draw2d primitives used by the - * frontend. Current PS2SDK keeps arcs, rounded rectangles and the basic sprite - * helpers in one draw2d.o archive member; referencing one basic primitive then - * drags sinf/cosf and their libm support into the EE ELF. The narrow subset - * below preserves current PS2SDK packet semantics for the operations we use so - * the monolithic archive member can remain unlinked. */ #include -#include -#include -#include -#include #include #include "gs_ui_ps2.h" -#define GS_UI_DRAW_START_OFFSET 2047.5625f -#define GS_UI_DRAW_END_OFFSET 2048.5625f - -#define GS_UI_DRAW_RECT_OUT_NREG 8 -#define GS_UI_DRAW_RECT_OUT_REGLIST \ - (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ - ((u64)GIF_REG_XYZ2) << 8 | ((u64)GIF_REG_XYZ2) << 12 | \ - ((u64)GIF_REG_XYZ2) << 16 | ((u64)GIF_REG_XYZ2) << 20 | \ - ((u64)GIF_REG_XYZ2) << 24 | ((u64)GIF_REG_NOP) << 28) - -#define GS_UI_DRAW_SPRITE_NREG 4 -#define GS_UI_DRAW_SPRITE_REGLIST \ - (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ - ((u64)GIF_REG_XYZ2) << 8 | ((u64)GIF_REG_XYZ2) << 12) - -#define GS_UI_DRAW_TEX_NREG 6 -#define GS_UI_DRAW_TEX_REGLIST \ - (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ - ((u64)GIF_REG_UV) << 8 | ((u64)GIF_REG_XYZ2) << 12 | \ - ((u64)GIF_REG_UV) << 16 | ((u64)GIF_REG_XYZ2) << 20) - -static int gs_ui_draw_blending; - void __wrap_scr_clear(void) { gs_ui_console_clear(); @@ -64,98 +30,3 @@ void __wrap_scr_printf(const char *format, ...) gs_ui_console_vprintf(format, arguments); va_end(arguments); } - -void draw_enable_blending(void) -{ - gs_ui_draw_blending = 1; -} - -void draw_disable_blending(void) -{ - gs_ui_draw_blending = 0; -} - -qword_t *draw_rect_outline(qword_t *q, int context, rect_t *rect) -{ - int x0 = ftoi4(rect->v0.x + GS_UI_DRAW_START_OFFSET); - int y0 = ftoi4(rect->v0.y + GS_UI_DRAW_START_OFFSET); - int x1 = ftoi4(rect->v1.x + GS_UI_DRAW_END_OFFSET); - int y1 = ftoi4(rect->v1.y + GS_UI_DRAW_END_OFFSET); - - PACK_GIFTAG(q, - GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, - GS_UI_DRAW_RECT_OUT_NREG), - GS_UI_DRAW_RECT_OUT_REGLIST); - q++; - - q->dw[0] = GIF_SET_PRIM(PRIM_LINE_STRIP, 0, 0, 0, gs_ui_draw_blending, - 0, 0, context, 0); - q->dw[1] = rect->color.rgbaq; - q++; - - q->dw[0] = GIF_SET_XYZ(x0, y0, rect->v0.z); - q->dw[1] = GIF_SET_XYZ(x1, y0, rect->v0.z); - q++; - - q->dw[0] = GIF_SET_XYZ(x1, y1, rect->v0.z); - q->dw[1] = GIF_SET_XYZ(x0, y1, rect->v0.z); - q++; - - q->dw[0] = GIF_SET_XYZ(x0, y0, rect->v0.z); - q->dw[1] = 0; - q++; - - return q; -} - -qword_t *draw_rect_filled(qword_t *q, int context, rect_t *rect) -{ - PACK_GIFTAG(q, - GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, - GS_UI_DRAW_SPRITE_NREG), - GS_UI_DRAW_SPRITE_REGLIST); - q++; - - q->dw[0] = GIF_SET_PRIM(PRIM_SPRITE, 0, 0, 0, gs_ui_draw_blending, - 0, 0, context, 0); - q->dw[1] = rect->color.rgbaq; - q++; - - q->dw[0] = GIF_SET_XYZ(ftoi4(rect->v0.x + GS_UI_DRAW_START_OFFSET), - ftoi4(rect->v0.y + GS_UI_DRAW_START_OFFSET), - rect->v0.z); - q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v1.x + GS_UI_DRAW_END_OFFSET), - ftoi4(rect->v1.y + GS_UI_DRAW_END_OFFSET), - rect->v0.z); - q++; - - return q; -} - -qword_t *draw_rect_textured(qword_t *q, int context, texrect_t *rect) -{ - PACK_GIFTAG(q, - GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, - GS_UI_DRAW_TEX_NREG), - GS_UI_DRAW_TEX_REGLIST); - q++; - - q->dw[0] = GIF_SET_PRIM(PRIM_SPRITE, 0, DRAW_ENABLE, 0, - gs_ui_draw_blending, 0, PRIM_MAP_UV, context, 0); - q->dw[1] = rect->color.rgbaq; - q++; - - q->dw[0] = GIF_SET_UV(ftoi4(rect->t0.u), ftoi4(rect->t0.v)); - q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v0.x + GS_UI_DRAW_START_OFFSET), - ftoi4(rect->v0.y + GS_UI_DRAW_START_OFFSET), - rect->v0.z); - q++; - - q->dw[0] = GIF_SET_UV(ftoi4(rect->t1.u), ftoi4(rect->t1.v)); - q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v1.x + GS_UI_DRAW_END_OFFSET), - ftoi4(rect->v1.y + GS_UI_DRAW_END_OFFSET), - rect->v0.z); - q++; - - return q; -} From 4e01fadcfeb3c7457b6b21f0550d56b57cad973d Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 20:52:07 +0200 Subject: [PATCH 022/156] Phase 1: avoid heavyweight fileXio stat path for config existence --- src/storage.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/storage.c b/src/storage.c index 3d407e66..5f3a5856 100644 --- a/src/storage.c +++ b/src/storage.c @@ -292,13 +292,28 @@ int read_bounded_file(const char *path, unsigned int maximum_size, return 0; } -/* Return a boolean existence result without exposing driver-specific errors. */ +/* + * The app only needs a boolean answer for its small current configuration + * file. Do not call fileXioGetStat here: current PS2SDK's POSIX stat glue also + * converts three IOP timestamps through mktime(), which drags timezone/scanf + * machinery into this standalone ELF for data we never consume. + * + * Opening read-only preserves the useful contract for this call site: a + * readable current config exists, so a malformed current file must not be + * hidden by silently falling back to a legacy filename. The close is performed + * immediately and no file data is transferred. + */ int path_exists(const char *path) { - iox_stat_t status; + int fd; - memset(&status, 0, sizeof(status)); - return fileXioGetStat(path, &status) >= 0; + if (path == NULL || path[0] == '\0') + return 0; + fd = fileXioOpen(path, FIO_O_RDONLY, 0); + if (fd < 0) + return 0; + fileXioClose(fd); + return 1; } /* Load a small configuration file and always provide a trailing NUL byte. */ From 68d0ec213b025831813a951491d200e18242ed62 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 20:53:54 +0200 Subject: [PATCH 023/156] Phase 1: remove POSIX stat conversion from backup probe --- src/header_backup.c | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/header_backup.c b/src/header_backup.c index 2e8460ae..1ebd03ee 100644 --- a/src/header_backup.c +++ b/src/header_backup.c @@ -20,6 +20,31 @@ static void backup_path_for_slot(char *path, unsigned int capacity, slot == 0 ? "HDDMBR.BIN" : "HDDMBR2.BIN"); } +/* + * Probe only the property this policy actually needs: whether a readable file + * already occupies the backup slot and, if so, its byte size. fileXioGetStat + * would also convert atime/mtime/ctime through the current PS2SDK POSIX glue, + * pulling mktime/tzset/scanf machinery into the EE image even though backup + * policy never consumes timestamps or POSIX mode bits. + */ +static int backup_file_size(const char *path, int *size_out) +{ + int fd; + int size; + + if (path == NULL || size_out == NULL) + return -1; + fd = fileXioOpen(path, FIO_O_RDONLY, 0); + if (fd < 0) + return fd; + size = fileXioLseek(fd, 0, FIO_SEEK_END); + fileXioClose(fd); + if (size < 0) + return size; + *size_out = size; + return 0; +} + const char *header_backup_save( unsigned int storage, const unsigned char current_header[APA_HEADER_SIZE], @@ -40,13 +65,12 @@ const char *header_backup_save( } for (i = 0; i < HEADER_BACKUP_SLOT_COUNT; i++) { - iox_stat_t existing_stat; + int existing_size = 0; - memset(&existing_stat, 0, sizeof(existing_stat)); diagnostics->read_result[i] = - fileXioGetStat(diagnostics->path[i], &existing_stat); + backup_file_size(diagnostics->path[i], &existing_size); if (diagnostics->read_result[i] >= 0) { - if (existing_stat.size == APA_HEADER_SIZE && + if (existing_size == APA_HEADER_SIZE && read_exact_file(diagnostics->path[i], backup_scratch, APA_HEADER_SIZE) == 0 && is_standard_apa_header(backup_scratch) && From 69a242bf95ea03d0986b60219944bfd12c5d1c66 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 20:54:48 +0200 Subject: [PATCH 024/156] Phase 0: add real-hardware instrumentation A/B protocol --- docs/PHASE0_HARDWARE_AB_PROTOCOL.md | 123 ++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 docs/PHASE0_HARDWARE_AB_PROTOCOL.md diff --git a/docs/PHASE0_HARDWARE_AB_PROTOCOL.md b/docs/PHASE0_HARDWARE_AB_PROTOCOL.md new file mode 100644 index 00000000..e101ca12 --- /dev/null +++ b/docs/PHASE0_HARDWARE_AB_PROTOCOL.md @@ -0,0 +1,123 @@ +# Corpus v2 Phase 0 real-hardware A/B protocol + +Phase 0 exits only after the measurement build itself has been measured on a +real PlayStation 2. PCSX2 may be used for correctness/debugging but is not an +arbiter for EE cache, IOP scheduling, USB service latency, SIF DMA or DEV9 +throughput. + +## Compared builds + +### A: audited pre-instrumentation baseline + +- project commit: `4b5aa8d85e86c9de570a2128b52d1eaa5b334844` +- purpose: known-good HDL installer immediately before corpus-v2 measurement + instrumentation +- expected Phase-0 telemetry: absent + +### B: corpus-v2 measurement build + +Use the newest green commit on `perf/corpus-v2-integration` before accepting a +Phase-1 runtime optimization. The build must retain: + +- EE pump/source/target latency histograms; +- IOP direct-source/fallback/prefetch/HDD/SIF histograms; +- useful/DMA/cache-maintenance/fallback traffic counters; +- benchmark provenance artifact; +- linker/ELF audit artifacts. + +The project source, toolchain and HDL transaction semantics must otherwise stay +unchanged for the measurement comparison. Any Phase-1 change must be measured +separately and must not be folded into the Phase-0 overhead result. + +## Hardware provenance + +Record before each pair of runs: + +```yaml +console_scp: +hardware_revision: +romver: +storage_adapter: +usb_device: +ps2sdk_commit: +toolchain: +active_irx: +build_flags: +workload: +direction: +buffering: +alignment: +sample_count: +units: +correctness_hash: +``` + +The CI-generated `BENCHMARK_PROVENANCE.yml` supplies build-side fields. Hardware +fields remain explicit manual measurements rather than guessed metadata. + +## Workload contract + +Use the same: + +- console and adapters; +- HDD contents/layout before each timed run where practical; +- USB device, filesystem and USB port; +- ISO file and fragment layout; +- source direction and target operation; +- video mode and active background services; +- cold/warm policy. + +For HDL copy throughput, use one ISO large enough that startup/allocation noise +is negligible compared with the bulk copy phase. Do not compare different ISOs, +USB sticks or HDD layouts and call the result an instrumentation delta. + +## Measurements + +For each build record at least: + +- bulk copy wall time and useful KiB/s; +- p50, p95, p99 and max EE pump ioctl latency; +- p50, p95, p99 and max IOP direct-source latency; +- p50, p95, p99 and max prefetch consumer wait; +- p50, p95, p99 and max HDD write latency; +- p50, p95, p99 and max SIF DMA completion latency; +- prefetch hit/miss counts; +- fallback-source/fallback-target bytes; +- useful payload, SIF DMA and EE cache-maintenance bytes; +- final correctness result/hash; +- any cancellation, journal or metadata-commit failure. + +Run at least three complete comparable samples for an initial engineering +answer. More samples are required if p95/p99 or wall time is unstable. + +## Phase-0 acceptance + +Phase 0 may be marked hardware-complete only when: + +1. build A and build B both pass the same correctness workload; +2. B produces internally consistent stage counters and traffic accounting; +3. the measurement overhead of B is quantified rather than assumed negligible; +4. p50/p95/p99/max are retained, not replaced by an average; +5. the result includes console/toolchain/IRX/workload provenance; +6. R5900 counter calibration is checked on the real EE before counter-derived + optimization claims are made. + +If instrumentation materially changes throughput or tail latency, keep a +companion non-instrumented performance build and use the measurement build only +for diagnosis. + +## R5900 counter calibration + +Before using a counter event to justify code changes: + +1. run an empty serialized scope and record harness overhead; +2. run a deterministic integer loop whose instruction/cycle relationship is + intentionally simple; +3. repeat the scope several times and confirm stable monotonic results; +4. reject any run with PCR overflow; +5. measure one event pair at a time and do not compare events from different + workloads as if they were simultaneous; +6. preserve a timer-only companion result. + +Only after this calibration should cache miss, branch, single/dual issue or +instruction-completed counts be used as evidence for Phase 1/2 decisions. From 047c1a050f00fc549a0dd053109358992b9e5a19 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 20:57:33 +0200 Subject: [PATCH 025/156] Phase 1: revert stat-probe experiment after linker audit --- src/header_backup.c | 32 ++++---------------------------- src/storage.c | 23 ++++------------------- 2 files changed, 8 insertions(+), 47 deletions(-) diff --git a/src/header_backup.c b/src/header_backup.c index 1ebd03ee..2e8460ae 100644 --- a/src/header_backup.c +++ b/src/header_backup.c @@ -20,31 +20,6 @@ static void backup_path_for_slot(char *path, unsigned int capacity, slot == 0 ? "HDDMBR.BIN" : "HDDMBR2.BIN"); } -/* - * Probe only the property this policy actually needs: whether a readable file - * already occupies the backup slot and, if so, its byte size. fileXioGetStat - * would also convert atime/mtime/ctime through the current PS2SDK POSIX glue, - * pulling mktime/tzset/scanf machinery into the EE image even though backup - * policy never consumes timestamps or POSIX mode bits. - */ -static int backup_file_size(const char *path, int *size_out) -{ - int fd; - int size; - - if (path == NULL || size_out == NULL) - return -1; - fd = fileXioOpen(path, FIO_O_RDONLY, 0); - if (fd < 0) - return fd; - size = fileXioLseek(fd, 0, FIO_SEEK_END); - fileXioClose(fd); - if (size < 0) - return size; - *size_out = size; - return 0; -} - const char *header_backup_save( unsigned int storage, const unsigned char current_header[APA_HEADER_SIZE], @@ -65,12 +40,13 @@ const char *header_backup_save( } for (i = 0; i < HEADER_BACKUP_SLOT_COUNT; i++) { - int existing_size = 0; + iox_stat_t existing_stat; + memset(&existing_stat, 0, sizeof(existing_stat)); diagnostics->read_result[i] = - backup_file_size(diagnostics->path[i], &existing_size); + fileXioGetStat(diagnostics->path[i], &existing_stat); if (diagnostics->read_result[i] >= 0) { - if (existing_size == APA_HEADER_SIZE && + if (existing_stat.size == APA_HEADER_SIZE && read_exact_file(diagnostics->path[i], backup_scratch, APA_HEADER_SIZE) == 0 && is_standard_apa_header(backup_scratch) && diff --git a/src/storage.c b/src/storage.c index 5f3a5856..3d407e66 100644 --- a/src/storage.c +++ b/src/storage.c @@ -292,28 +292,13 @@ int read_bounded_file(const char *path, unsigned int maximum_size, return 0; } -/* - * The app only needs a boolean answer for its small current configuration - * file. Do not call fileXioGetStat here: current PS2SDK's POSIX stat glue also - * converts three IOP timestamps through mktime(), which drags timezone/scanf - * machinery into this standalone ELF for data we never consume. - * - * Opening read-only preserves the useful contract for this call site: a - * readable current config exists, so a malformed current file must not be - * hidden by silently falling back to a legacy filename. The close is performed - * immediately and no file data is transferred. - */ +/* Return a boolean existence result without exposing driver-specific errors. */ int path_exists(const char *path) { - int fd; + iox_stat_t status; - if (path == NULL || path[0] == '\0') - return 0; - fd = fileXioOpen(path, FIO_O_RDONLY, 0); - if (fd < 0) - return 0; - fileXioClose(fd); - return 1; + memset(&status, 0, sizeof(status)); + return fileXioGetStat(path, &status) >= 0; } /* Load a small configuration file and always provide a trailing NUL byte. */ From d0c6122e1533fc835d4fc0183677dcf7bee2276b Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 20:58:18 +0200 Subject: [PATCH 026/156] Phase 1: add non-LTO minimal UI draw primitives --- src/gs_ui_draw_minimal_ps2.c | 116 +++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 src/gs_ui_draw_minimal_ps2.c diff --git a/src/gs_ui_draw_minimal_ps2.c b/src/gs_ui_draw_minimal_ps2.c new file mode 100644 index 00000000..d1cf563b --- /dev/null +++ b/src/gs_ui_draw_minimal_ps2.c @@ -0,0 +1,116 @@ +/* + * Minimal libdraw-compatible 2D primitive subset for the manager frontend. + * + * Current PS2SDK places basic rectangles/textured sprites and the unrelated + * arc/rounded-rectangle code in one LTO archive member (draw2d.o). Referencing + * a basic primitive can therefore retain sinf/cosf and their libm support. + * This file preserves the current PS2SDK packet layout for the five draw2d + * symbols actually used by fhdb-bootstrap-manager. It is deliberately compiled + * as a normal non-LTO object so the linker can satisfy those symbols before it + * considers the monolithic libdraw archive member. + */ + +#include +#include +#include +#include + +#define UI_START_OFFSET 2047.5625f +#define UI_END_OFFSET 2048.5625f + +#define UI_RECT_OUT_NREG 8 +#define UI_RECT_OUT_REGLIST \ + (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ + ((u64)GIF_REG_XYZ2) << 8 | ((u64)GIF_REG_XYZ2) << 12 | \ + ((u64)GIF_REG_XYZ2) << 16 | ((u64)GIF_REG_XYZ2) << 20 | \ + ((u64)GIF_REG_XYZ2) << 24 | ((u64)GIF_REG_NOP) << 28) + +#define UI_SPRITE_NREG 4 +#define UI_SPRITE_REGLIST \ + (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ + ((u64)GIF_REG_XYZ2) << 8 | ((u64)GIF_REG_XYZ2) << 12) + +#define UI_TEX_NREG 6 +#define UI_TEX_REGLIST \ + (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ + ((u64)GIF_REG_UV) << 8 | ((u64)GIF_REG_XYZ2) << 12 | \ + ((u64)GIF_REG_UV) << 16 | ((u64)GIF_REG_XYZ2) << 20) + +static int ui_blending; + +void draw_enable_blending(void) +{ + ui_blending = 1; +} + +void draw_disable_blending(void) +{ + ui_blending = 0; +} + +qword_t *draw_rect_outline(qword_t *q, int context, rect_t *rect) +{ + int x0 = ftoi4(rect->v0.x + UI_START_OFFSET); + int y0 = ftoi4(rect->v0.y + UI_START_OFFSET); + int x1 = ftoi4(rect->v1.x + UI_END_OFFSET); + int y1 = ftoi4(rect->v1.y + UI_END_OFFSET); + + PACK_GIFTAG(q, + GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, + UI_RECT_OUT_NREG), + UI_RECT_OUT_REGLIST); + q++; + q->dw[0] = GIF_SET_PRIM(PRIM_LINE_STRIP, 0, 0, 0, ui_blending, + 0, 0, context, 0); + q->dw[1] = rect->color.rgbaq; + q++; + q->dw[0] = GIF_SET_XYZ(x0, y0, rect->v0.z); + q->dw[1] = GIF_SET_XYZ(x1, y0, rect->v0.z); + q++; + q->dw[0] = GIF_SET_XYZ(x1, y1, rect->v0.z); + q->dw[1] = GIF_SET_XYZ(x0, y1, rect->v0.z); + q++; + q->dw[0] = GIF_SET_XYZ(x0, y0, rect->v0.z); + q->dw[1] = 0; + q++; + return q; +} + +qword_t *draw_rect_filled(qword_t *q, int context, rect_t *rect) +{ + PACK_GIFTAG(q, + GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, UI_SPRITE_NREG), + UI_SPRITE_REGLIST); + q++; + q->dw[0] = GIF_SET_PRIM(PRIM_SPRITE, 0, 0, 0, ui_blending, + 0, 0, context, 0); + q->dw[1] = rect->color.rgbaq; + q++; + q->dw[0] = GIF_SET_XYZ(ftoi4(rect->v0.x + UI_START_OFFSET), + ftoi4(rect->v0.y + UI_START_OFFSET), rect->v0.z); + q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v1.x + UI_END_OFFSET), + ftoi4(rect->v1.y + UI_END_OFFSET), rect->v0.z); + q++; + return q; +} + +qword_t *draw_rect_textured(qword_t *q, int context, texrect_t *rect) +{ + PACK_GIFTAG(q, + GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, UI_TEX_NREG), + UI_TEX_REGLIST); + q++; + q->dw[0] = GIF_SET_PRIM(PRIM_SPRITE, 0, DRAW_ENABLE, 0, ui_blending, + 0, PRIM_MAP_UV, context, 0); + q->dw[1] = rect->color.rgbaq; + q++; + q->dw[0] = GIF_SET_UV(ftoi4(rect->t0.u), ftoi4(rect->t0.v)); + q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v0.x + UI_START_OFFSET), + ftoi4(rect->v0.y + UI_START_OFFSET), rect->v0.z); + q++; + q->dw[0] = GIF_SET_UV(ftoi4(rect->t1.u), ftoi4(rect->t1.v)); + q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v1.x + UI_END_OFFSET), + ftoi4(rect->v1.y + UI_END_OFFSET), rect->v0.z); + q++; + return q; +} From c997312fd98d9bd48e626729142d51276dae422f Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 20:58:38 +0200 Subject: [PATCH 027/156] Phase 1: resolve UI primitives before LTO libdraw archive --- GNUmakefile | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 GNUmakefile diff --git a/GNUmakefile b/GNUmakefile new file mode 100644 index 00000000..99fc18f2 --- /dev/null +++ b/GNUmakefile @@ -0,0 +1,18 @@ +# Corpus-v2 build overlay. +# +# GNU make prefers GNUmakefile over Makefile. Define the non-LTO UI primitive +# object as an override before loading the normal project build so the object is +# present in EE_OBJS while Makefile.eeglobal constructs the final link rule. +# The rest of the project's build remains in Makefile. + +override EE_OBJS += gs_ui_draw_minimal_ps2.o + +include Makefile + +# Keep this compatibility object out of LTO on purpose. With a normal object in +# the explicit link list, BFD ld can resolve the five draw2d symbols before it +# scans libdraw.a. The PS2SDK draw2d archive member is itself an LTO object and +# otherwise gets claimed by the plugin as a whole, retaining unrelated arc/trig +# code. This is a link-layout experiment, not a global no-LTO policy. +gs_ui_draw_minimal_ps2.o: src/gs_ui_draw_minimal_ps2.c + $(EE_CC) $(filter-out -flto,$(EE_CFLAGS)) $(EE_INCS) -c $< -o $@ From 91820c451540bb21dab5dff6f0d3bb1e11a293cb Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 20:59:59 +0200 Subject: [PATCH 028/156] Phase 1: fix build overlay object ordering --- GNUmakefile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 99fc18f2..fbed9046 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -1,14 +1,16 @@ # Corpus-v2 build overlay. # -# GNU make prefers GNUmakefile over Makefile. Define the non-LTO UI primitive -# object as an override before loading the normal project build so the object is -# present in EE_OBJS while Makefile.eeglobal constructs the final link rule. -# The rest of the project's build remains in Makefile. - -override EE_OBJS += gs_ui_draw_minimal_ps2.o +# GNU make prefers GNUmakefile over Makefile. Load the normal project build +# first, then append the one link-layout experiment object. Adding a prerequisite +# to the already-defined EE target keeps the normal dependency graph intact; +# Makefile.eeglobal expands EE_OBJS when it executes the link recipe, so the +# appended object is also present in the explicit link list before libdraw.a. include Makefile +EE_OBJS += gs_ui_draw_minimal_ps2.o +$(EE_BIN): gs_ui_draw_minimal_ps2.o + # Keep this compatibility object out of LTO on purpose. With a normal object in # the explicit link list, BFD ld can resolve the five draw2d symbols before it # scans libdraw.a. The PS2SDK draw2d archive member is itself an LTO object and From 496240e14f18f383d6b969e31a637737d073001a Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 21:01:41 +0200 Subject: [PATCH 029/156] Phase 1: link trimmed libdraw without monolithic draw2d member --- GNUmakefile | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index fbed9046..db097648 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -1,20 +1,24 @@ # Corpus-v2 build overlay. # -# GNU make prefers GNUmakefile over Makefile. Load the normal project build -# first, then append the one link-layout experiment object. Adding a prerequisite -# to the already-defined EE target keeps the normal dependency graph intact; -# Makefile.eeglobal expands EE_OBJS when it executes the link recipe, so the -# appended object is also present in the explicit link list before libdraw.a. +# GNU make prefers GNUmakefile over Makefile. Keep the production build rules in +# Makefile and layer only measured corpus experiments here, so reverting this +# branch never perturbs the known-good HDL development line. include Makefile EE_OBJS += gs_ui_draw_minimal_ps2.o -$(EE_BIN): gs_ui_draw_minimal_ps2.o +EE_LIBS := $(subst -ldraw,corpus_libdraw.a,$(EE_LIBS)) +$(EE_BIN): gs_ui_draw_minimal_ps2.o corpus_libdraw.a -# Keep this compatibility object out of LTO on purpose. With a normal object in -# the explicit link list, BFD ld can resolve the five draw2d symbols before it -# scans libdraw.a. The PS2SDK draw2d archive member is itself an LTO object and -# otherwise gets claimed by the plugin as a whole, retaining unrelated arc/trig -# code. This is a link-layout experiment, not a global no-LTO policy. +# Keep this compatibility object out of LTO on purpose. Current PS2SDK's +# draw2d.o is a fat-LTO archive member. The linker plugin claims that member as +# a unit even when these symbols are already provided by a normal object, which +# creates duplicate strong definitions and still retains unrelated trig code. +# Instead, link a byte-for-byte copy of libdraw with only draw2d.o removed, and +# provide the five draw2d entry points the manager actually uses here. gs_ui_draw_minimal_ps2.o: src/gs_ui_draw_minimal_ps2.c $(EE_CC) $(filter-out -flto,$(EE_CFLAGS)) $(EE_INCS) -c $< -o $@ + +corpus_libdraw.a: + cp $(PS2SDK)/ee/lib/libdraw.a $@ + $(EE_AR) d $@ draw2d.o From 0483bb6d9e2bfe6fc3e2931d9627ea21d7be38c7 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 21:04:57 +0200 Subject: [PATCH 030/156] Phase 1: keep draw_clear strip primitive in minimal UI subset --- src/gs_ui_draw_minimal_ps2.c | 49 +++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/src/gs_ui_draw_minimal_ps2.c b/src/gs_ui_draw_minimal_ps2.c index d1cf563b..70e10d94 100644 --- a/src/gs_ui_draw_minimal_ps2.c +++ b/src/gs_ui_draw_minimal_ps2.c @@ -4,10 +4,10 @@ * Current PS2SDK places basic rectangles/textured sprites and the unrelated * arc/rounded-rectangle code in one LTO archive member (draw2d.o). Referencing * a basic primitive can therefore retain sinf/cosf and their libm support. - * This file preserves the current PS2SDK packet layout for the five draw2d - * symbols actually used by fhdb-bootstrap-manager. It is deliberately compiled - * as a normal non-LTO object so the linker can satisfy those symbols before it - * considers the monolithic libdraw archive member. + * This file preserves the current PS2SDK packet layout for the draw2d symbols + * actually used by fhdb-bootstrap-manager and by retained PS2SDK draw.c helpers. + * It is deliberately compiled as a normal non-LTO object so the monolithic + * draw2d archive member can be omitted without copying its unrelated code. */ #include @@ -94,6 +94,47 @@ qword_t *draw_rect_filled(qword_t *q, int context, rect_t *rect) return q; } +/* draw_clear() in current PS2SDK draw.c uses the strip variant internally. + * Keep its exact 32-pixel strip stepping and packet layout, otherwise trimming + * draw2d.o would change framebuffer-clear semantics even though our own UI never + * calls this entry point directly. */ +qword_t *draw_rect_filled_strips(qword_t *q, int context, rect_t *rect) +{ + qword_t *giftag; + int x0 = ftoi4(rect->v0.x); + int y0 = ftoi4(rect->v0.y + UI_START_OFFSET); + int x1 = ftoi4(rect->v1.x); + int y1 = ftoi4(rect->v1.y + UI_END_OFFSET); + + PACK_GIFTAG(q, GIF_SET_TAG(2, 0, 0, 0, GIF_FLG_PACKED, 1), GIF_REG_AD); + q++; + PACK_GIFTAG(q, + GIF_SET_PRIM(PRIM_SPRITE, 0, 0, 0, ui_blending, + 0, 0, context, 0), + GIF_REG_PRIM); + q++; + PACK_GIFTAG(q, rect->color.rgbaq, GIF_REG_RGBAQ); + q++; + + giftag = q; + q++; + + while (x0 < x1) { + q->dw[0] = GIF_SET_XYZ(x0 + ftoi4(UI_START_OFFSET), y0, rect->v0.z); + x0 += 496; + if (x0 >= x1) + x0 = x1; + q->dw[1] = GIF_SET_XYZ(x0 + ftoi4(UI_END_OFFSET), y1, rect->v0.z); + x0 += 16; + q++; + } + + PACK_GIFTAG(giftag, + GIF_SET_TAG(q - giftag - 1, 0, 0, 0, GIF_FLG_REGLIST, 2), + DRAW_XYZ_REGLIST); + return q; +} + qword_t *draw_rect_textured(qword_t *q, int context, texrect_t *rect) { PACK_GIFTAG(q, From 4b6d14836d0aa5ddb5dae0a09ca63c69c078b4f3 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 21:16:07 +0200 Subject: [PATCH 031/156] Phase 1: revert draw2d shim after loaded-text regression --- GNUmakefile | 24 ------ src/gs_ui_draw_minimal_ps2.c | 157 ----------------------------------- 2 files changed, 181 deletions(-) delete mode 100644 GNUmakefile delete mode 100644 src/gs_ui_draw_minimal_ps2.c diff --git a/GNUmakefile b/GNUmakefile deleted file mode 100644 index db097648..00000000 --- a/GNUmakefile +++ /dev/null @@ -1,24 +0,0 @@ -# Corpus-v2 build overlay. -# -# GNU make prefers GNUmakefile over Makefile. Keep the production build rules in -# Makefile and layer only measured corpus experiments here, so reverting this -# branch never perturbs the known-good HDL development line. - -include Makefile - -EE_OBJS += gs_ui_draw_minimal_ps2.o -EE_LIBS := $(subst -ldraw,corpus_libdraw.a,$(EE_LIBS)) -$(EE_BIN): gs_ui_draw_minimal_ps2.o corpus_libdraw.a - -# Keep this compatibility object out of LTO on purpose. Current PS2SDK's -# draw2d.o is a fat-LTO archive member. The linker plugin claims that member as -# a unit even when these symbols are already provided by a normal object, which -# creates duplicate strong definitions and still retains unrelated trig code. -# Instead, link a byte-for-byte copy of libdraw with only draw2d.o removed, and -# provide the five draw2d entry points the manager actually uses here. -gs_ui_draw_minimal_ps2.o: src/gs_ui_draw_minimal_ps2.c - $(EE_CC) $(filter-out -flto,$(EE_CFLAGS)) $(EE_INCS) -c $< -o $@ - -corpus_libdraw.a: - cp $(PS2SDK)/ee/lib/libdraw.a $@ - $(EE_AR) d $@ draw2d.o diff --git a/src/gs_ui_draw_minimal_ps2.c b/src/gs_ui_draw_minimal_ps2.c deleted file mode 100644 index 70e10d94..00000000 --- a/src/gs_ui_draw_minimal_ps2.c +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Minimal libdraw-compatible 2D primitive subset for the manager frontend. - * - * Current PS2SDK places basic rectangles/textured sprites and the unrelated - * arc/rounded-rectangle code in one LTO archive member (draw2d.o). Referencing - * a basic primitive can therefore retain sinf/cosf and their libm support. - * This file preserves the current PS2SDK packet layout for the draw2d symbols - * actually used by fhdb-bootstrap-manager and by retained PS2SDK draw.c helpers. - * It is deliberately compiled as a normal non-LTO object so the monolithic - * draw2d archive member can be omitted without copying its unrelated code. - */ - -#include -#include -#include -#include - -#define UI_START_OFFSET 2047.5625f -#define UI_END_OFFSET 2048.5625f - -#define UI_RECT_OUT_NREG 8 -#define UI_RECT_OUT_REGLIST \ - (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ - ((u64)GIF_REG_XYZ2) << 8 | ((u64)GIF_REG_XYZ2) << 12 | \ - ((u64)GIF_REG_XYZ2) << 16 | ((u64)GIF_REG_XYZ2) << 20 | \ - ((u64)GIF_REG_XYZ2) << 24 | ((u64)GIF_REG_NOP) << 28) - -#define UI_SPRITE_NREG 4 -#define UI_SPRITE_REGLIST \ - (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ - ((u64)GIF_REG_XYZ2) << 8 | ((u64)GIF_REG_XYZ2) << 12) - -#define UI_TEX_NREG 6 -#define UI_TEX_REGLIST \ - (((u64)GIF_REG_PRIM) << 0 | ((u64)GIF_REG_RGBAQ) << 4 | \ - ((u64)GIF_REG_UV) << 8 | ((u64)GIF_REG_XYZ2) << 12 | \ - ((u64)GIF_REG_UV) << 16 | ((u64)GIF_REG_XYZ2) << 20) - -static int ui_blending; - -void draw_enable_blending(void) -{ - ui_blending = 1; -} - -void draw_disable_blending(void) -{ - ui_blending = 0; -} - -qword_t *draw_rect_outline(qword_t *q, int context, rect_t *rect) -{ - int x0 = ftoi4(rect->v0.x + UI_START_OFFSET); - int y0 = ftoi4(rect->v0.y + UI_START_OFFSET); - int x1 = ftoi4(rect->v1.x + UI_END_OFFSET); - int y1 = ftoi4(rect->v1.y + UI_END_OFFSET); - - PACK_GIFTAG(q, - GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, - UI_RECT_OUT_NREG), - UI_RECT_OUT_REGLIST); - q++; - q->dw[0] = GIF_SET_PRIM(PRIM_LINE_STRIP, 0, 0, 0, ui_blending, - 0, 0, context, 0); - q->dw[1] = rect->color.rgbaq; - q++; - q->dw[0] = GIF_SET_XYZ(x0, y0, rect->v0.z); - q->dw[1] = GIF_SET_XYZ(x1, y0, rect->v0.z); - q++; - q->dw[0] = GIF_SET_XYZ(x1, y1, rect->v0.z); - q->dw[1] = GIF_SET_XYZ(x0, y1, rect->v0.z); - q++; - q->dw[0] = GIF_SET_XYZ(x0, y0, rect->v0.z); - q->dw[1] = 0; - q++; - return q; -} - -qword_t *draw_rect_filled(qword_t *q, int context, rect_t *rect) -{ - PACK_GIFTAG(q, - GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, UI_SPRITE_NREG), - UI_SPRITE_REGLIST); - q++; - q->dw[0] = GIF_SET_PRIM(PRIM_SPRITE, 0, 0, 0, ui_blending, - 0, 0, context, 0); - q->dw[1] = rect->color.rgbaq; - q++; - q->dw[0] = GIF_SET_XYZ(ftoi4(rect->v0.x + UI_START_OFFSET), - ftoi4(rect->v0.y + UI_START_OFFSET), rect->v0.z); - q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v1.x + UI_END_OFFSET), - ftoi4(rect->v1.y + UI_END_OFFSET), rect->v0.z); - q++; - return q; -} - -/* draw_clear() in current PS2SDK draw.c uses the strip variant internally. - * Keep its exact 32-pixel strip stepping and packet layout, otherwise trimming - * draw2d.o would change framebuffer-clear semantics even though our own UI never - * calls this entry point directly. */ -qword_t *draw_rect_filled_strips(qword_t *q, int context, rect_t *rect) -{ - qword_t *giftag; - int x0 = ftoi4(rect->v0.x); - int y0 = ftoi4(rect->v0.y + UI_START_OFFSET); - int x1 = ftoi4(rect->v1.x); - int y1 = ftoi4(rect->v1.y + UI_END_OFFSET); - - PACK_GIFTAG(q, GIF_SET_TAG(2, 0, 0, 0, GIF_FLG_PACKED, 1), GIF_REG_AD); - q++; - PACK_GIFTAG(q, - GIF_SET_PRIM(PRIM_SPRITE, 0, 0, 0, ui_blending, - 0, 0, context, 0), - GIF_REG_PRIM); - q++; - PACK_GIFTAG(q, rect->color.rgbaq, GIF_REG_RGBAQ); - q++; - - giftag = q; - q++; - - while (x0 < x1) { - q->dw[0] = GIF_SET_XYZ(x0 + ftoi4(UI_START_OFFSET), y0, rect->v0.z); - x0 += 496; - if (x0 >= x1) - x0 = x1; - q->dw[1] = GIF_SET_XYZ(x0 + ftoi4(UI_END_OFFSET), y1, rect->v0.z); - x0 += 16; - q++; - } - - PACK_GIFTAG(giftag, - GIF_SET_TAG(q - giftag - 1, 0, 0, 0, GIF_FLG_REGLIST, 2), - DRAW_XYZ_REGLIST); - return q; -} - -qword_t *draw_rect_textured(qword_t *q, int context, texrect_t *rect) -{ - PACK_GIFTAG(q, - GIF_SET_TAG(1, 0, 0, 0, GIF_FLG_REGLIST, UI_TEX_NREG), - UI_TEX_REGLIST); - q++; - q->dw[0] = GIF_SET_PRIM(PRIM_SPRITE, 0, DRAW_ENABLE, 0, ui_blending, - 0, PRIM_MAP_UV, context, 0); - q->dw[1] = rect->color.rgbaq; - q++; - q->dw[0] = GIF_SET_UV(ftoi4(rect->t0.u), ftoi4(rect->t0.v)); - q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v0.x + UI_START_OFFSET), - ftoi4(rect->v0.y + UI_START_OFFSET), rect->v0.z); - q++; - q->dw[0] = GIF_SET_UV(ftoi4(rect->t1.u), ftoi4(rect->t1.v)); - q->dw[1] = GIF_SET_XYZ(ftoi4(rect->v1.x + UI_END_OFFSET), - ftoi4(rect->v1.y + UI_END_OFFSET), rect->v0.z); - q++; - return q; -} From 26db57091958ae77c725f1ec08a6467952cad9d7 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 21:23:56 +0200 Subject: [PATCH 032/156] Phase 1: bypass unused PS2SDK graph config archive root --- GNUmakefile | 13 +++++++ src/graph_config_link_policy_ps2.c | 59 ++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 GNUmakefile create mode 100644 src/graph_config_link_policy_ps2.c diff --git a/GNUmakefile b/GNUmakefile new file mode 100644 index 00000000..3689a42d --- /dev/null +++ b/GNUmakefile @@ -0,0 +1,13 @@ +# Corpus-v2 Phase-1 link experiment overlay. +# +# GNU make loads this before Makefile. The production build stays untouched; +# this branch only prepends one compatibility symbol before the PS2SDK archives +# so we can measure whether graph_config.o is unnecessary retained work. + +include Makefile + +EE_OBJS += graph_config_link_policy_ps2.o +$(EE_BIN): graph_config_link_policy_ps2.o + +graph_config_link_policy_ps2.o: src/graph_config_link_policy_ps2.c + $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ diff --git a/src/graph_config_link_policy_ps2.c b/src/graph_config_link_policy_ps2.c new file mode 100644 index 00000000..2f87f9b4 --- /dev/null +++ b/src/graph_config_link_policy_ps2.c @@ -0,0 +1,59 @@ +/* + * Phase-1 link policy for the pinned PS2SDK graph library. + * + * CURRENT IMPLEMENTATION (PS2SDK v2.0.0 / b12f8af): graph_mode.o carries a + * reference to graph_make_config(), so the linker extracts the monolithic + * graph_config.o archive member even though this application never calls the + * graph configuration-file API. That member also references fopen/fread/fwrite + * and formatted I/O, which can retain unrelated Newlib stdio code. + * + * The pinned graph_make_config() implementation overwrites the same output + * pointer for every field; its final observable output is therefore only + * ":" and it returns 0. Preserve that exact current behaviour here without + * general-purpose stdio. If the application ever starts using graph_get_config, + * graph_set_config, graph_load_config or graph_save_config, this policy must be + * re-audited against the then-current PS2SDK source rather than silently + * becoming an application API implementation. + */ + +#include + +static char *write_signed_decimal(char *out, int value) +{ + char digits[10]; + unsigned int magnitude; + unsigned int count = 0; + + if (value < 0) { + *out++ = '-'; + magnitude = 0u - (unsigned int)value; + } else { + magnitude = (unsigned int)value; + } + + do { + digits[count++] = (char)('0' + magnitude % 10u); + magnitude /= 10u; + } while (magnitude != 0u); + + while (count != 0u) + *out++ = digits[--count]; + return out; +} + +int graph_make_config(int mode, int interlace, int ffmd, int x, int y, + int flicker_filter, char *config) +{ + char *end; + + (void)mode; + (void)interlace; + (void)ffmd; + (void)x; + (void)flicker_filter; + + end = write_signed_decimal(config, y); + *end++ = ':'; + *end = '\0'; + return 0; +} From 57af905bfbe35f0dd13fef5ea79e73db444509a6 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 21:25:47 +0200 Subject: [PATCH 033/156] Phase 1: revert graph-config root experiment after zero-size delta --- GNUmakefile | 13 ------- src/graph_config_link_policy_ps2.c | 59 ------------------------------ 2 files changed, 72 deletions(-) delete mode 100644 GNUmakefile delete mode 100644 src/graph_config_link_policy_ps2.c diff --git a/GNUmakefile b/GNUmakefile deleted file mode 100644 index 3689a42d..00000000 --- a/GNUmakefile +++ /dev/null @@ -1,13 +0,0 @@ -# Corpus-v2 Phase-1 link experiment overlay. -# -# GNU make loads this before Makefile. The production build stays untouched; -# this branch only prepends one compatibility symbol before the PS2SDK archives -# so we can measure whether graph_config.o is unnecessary retained work. - -include Makefile - -EE_OBJS += graph_config_link_policy_ps2.o -$(EE_BIN): graph_config_link_policy_ps2.o - -graph_config_link_policy_ps2.o: src/graph_config_link_policy_ps2.c - $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ diff --git a/src/graph_config_link_policy_ps2.c b/src/graph_config_link_policy_ps2.c deleted file mode 100644 index 2f87f9b4..00000000 --- a/src/graph_config_link_policy_ps2.c +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Phase-1 link policy for the pinned PS2SDK graph library. - * - * CURRENT IMPLEMENTATION (PS2SDK v2.0.0 / b12f8af): graph_mode.o carries a - * reference to graph_make_config(), so the linker extracts the monolithic - * graph_config.o archive member even though this application never calls the - * graph configuration-file API. That member also references fopen/fread/fwrite - * and formatted I/O, which can retain unrelated Newlib stdio code. - * - * The pinned graph_make_config() implementation overwrites the same output - * pointer for every field; its final observable output is therefore only - * ":" and it returns 0. Preserve that exact current behaviour here without - * general-purpose stdio. If the application ever starts using graph_get_config, - * graph_set_config, graph_load_config or graph_save_config, this policy must be - * re-audited against the then-current PS2SDK source rather than silently - * becoming an application API implementation. - */ - -#include - -static char *write_signed_decimal(char *out, int value) -{ - char digits[10]; - unsigned int magnitude; - unsigned int count = 0; - - if (value < 0) { - *out++ = '-'; - magnitude = 0u - (unsigned int)value; - } else { - magnitude = (unsigned int)value; - } - - do { - digits[count++] = (char)('0' + magnitude % 10u); - magnitude /= 10u; - } while (magnitude != 0u); - - while (count != 0u) - *out++ = digits[--count]; - return out; -} - -int graph_make_config(int mode, int interlace, int ffmd, int x, int y, - int flicker_filter, char *config) -{ - char *end; - - (void)mode; - (void)interlace; - (void)ffmd; - (void)x; - (void)flicker_filter; - - end = write_signed_decimal(config, y); - *end++ = ':'; - *end = '\0'; - return 0; -} From bfb82c9464c1cf544195778f5246acd479ae62fd Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 21:26:43 +0200 Subject: [PATCH 034/156] Phase 1: isolate direct-fileXio app from unused POSIX fdman glue --- GNUmakefile | 13 +++++++++++++ src/filexio_fdman_policy_ps2.c | 29 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 GNUmakefile create mode 100644 src/filexio_fdman_policy_ps2.c diff --git a/GNUmakefile b/GNUmakefile new file mode 100644 index 00000000..33d7035c --- /dev/null +++ b/GNUmakefile @@ -0,0 +1,13 @@ +# Corpus-v2 Phase-1 fileXio/newlib link policy experiment. +# +# Keep the normal project Makefile unchanged. This PS2-only object is linked +# before libfileXio so its application-specific _ps2sdk_fileXio_init/deinit +# policy can be measured against the unmodified PS2SDK archive path. + +include Makefile + +EE_OBJS += filexio_fdman_policy_ps2.o +$(EE_BIN): filexio_fdman_policy_ps2.o + +filexio_fdman_policy_ps2.o: src/filexio_fdman_policy_ps2.c + $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ diff --git a/src/filexio_fdman_policy_ps2.c b/src/filexio_fdman_policy_ps2.c new file mode 100644 index 00000000..ebced5eb --- /dev/null +++ b/src/filexio_fdman_policy_ps2.c @@ -0,0 +1,29 @@ +/* + * Phase-1 application policy for PS2SDK fileXio/newlib integration. + * + * CURRENT IMPLEMENTATION (PS2SDK v2.0.0 / b12f8af): fileXioInit() calls the + * internal _ps2sdk_fileXio_init() hook. That hook switches Newlib's generic + * POSIX fd-manager path table to fileXio and, through the companion constructor, + * retains every POSIX adapter including __fileXioGetstatHelper(). The stat + * adapter converts three timestamps through mktime(), which in this pinned + * Newlib also retains timezone parsing and scanf machinery. + * + * fhdb-bootstrap-manager's storage contract is intentionally direct fileXio: + * application filesystem operations use fileXioOpen/Read/Write/Lseek/etc. The + * Newlib formatting used by the UI/log paths is memory/string formatting, not + * fopen/open/stat-based file access. Therefore this build does not need fileXio + * to replace Newlib's generic POSIX pathname backend. + * + * Keep these symbols tiny and strong so fileXioInit()/Exit() retain their RPC, + * semaphore and reset semantics while the optional POSIX adapter objects remain + * unreferenced. Any future application use of fopen/open/stat/opendir/FILE I/O + * must remove this policy or provide an explicitly reviewed equivalent path. + */ + +void _ps2sdk_fileXio_init(void) +{ +} + +void _ps2sdk_fileXio_deinit(void) +{ +} From 1516b4d716904a40524827566ca7899cbd7bd1b9 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 21:30:51 +0200 Subject: [PATCH 035/156] Phase 1: guard direct-fileXio policy against POSIX file calls --- tools/check_filexio_fdman_policy.py | 149 ++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 tools/check_filexio_fdman_policy.py diff --git a/tools/check_filexio_fdman_policy.py b/tools/check_filexio_fdman_policy.py new file mode 100644 index 00000000..d6fa1f85 --- /dev/null +++ b/tools/check_filexio_fdman_policy.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Reject EE runtime call sites that require PS2SDK's Newlib fileXio fd-manager. + +The corpus-v2 performance branch deliberately keeps application filesystem I/O on +fileXio's direct RPC API. A tiny application policy object therefore prevents +fileXioInit() from replacing Newlib's generic POSIX pathname backend. This saves +code only while src/include do not depend on fopen/open/stat/opendir-style file +access. + +This checker turns that runtime contract into a CI invariant. It scans C-family +project source after masking comments and string/character literals, so prose and +format strings do not create false positives. If a POSIX/stdio file call is +needed later, review the policy instead of adding an exemption by reflex. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +ROOTS = ("src", "include") +SUFFIXES = {".c", ".h", ".inc", ".s", ".S"} +FORBIDDEN = ( + "fopen", "freopen", "fclose", "fread", "fwrite", + "open", "close", "stat", "fstat", "lstat", + "opendir", "readdir", "closedir", + "remove", "rename", "mkdir", "rmdir", "chdir", "unlink", +) +CALL = re.compile(r"\b(" + "|".join(map(re.escape, FORBIDDEN)) + r")\s*\(") + + +def mask_non_code(text: str) -> str: + out = list(text) + state = "code" + i = 0 + while i < len(text): + ch = text[i] + nxt = text[i + 1] if i + 1 < len(text) else "" + if state == "code": + if ch == "/" and nxt == "/": + out[i] = out[i + 1] = " " + state = "line" + i += 1 + elif ch == "/" and nxt == "*": + out[i] = out[i + 1] = " " + state = "block" + i += 1 + elif ch == '"': + out[i] = " " + state = "string" + elif ch == "'": + out[i] = " " + state = "char" + elif state == "line": + if ch == "\n": + state = "code" + else: + out[i] = " " + elif state == "block": + if ch == "*" and nxt == "/": + out[i] = out[i + 1] = " " + state = "code" + i += 1 + elif ch != "\n": + out[i] = " " + elif state in ("string", "char"): + if ch == "\\": + out[i] = " " + if i + 1 < len(text): + if text[i + 1] != "\n": + out[i + 1] = " " + i += 1 + elif (state == "string" and ch == '"') or ( + state == "char" and ch == "'"): + out[i] = " " + state = "code" + elif ch != "\n": + out[i] = " " + i += 1 + return "".join(out) + + +def source_files(root: Path): + for dirname in ROOTS: + directory = root / dirname + if not directory.is_dir(): + continue + for path in sorted(directory.rglob("*")): + if path.is_file() and path.suffix in SUFFIXES: + yield path + + +def scan(root: Path): + findings = [] + for path in source_files(root): + text = path.read_text(encoding="utf-8", errors="replace") + masked = mask_non_code(text) + for match in CALL.finditer(masked): + line = masked.count("\n", 0, match.start()) + 1 + findings.append((str(path.relative_to(root)), line, match.group(1))) + return findings + + +def selftest() -> int: + sample = r''' + fileXioOpen("mass:/x", 1, 0); + snprintf(buf, sizeof(buf), "fopen(x)"); + /* stat(path); */ + // fwrite(data, 1, n, f); + int fopen_counter = 0; + fopen(path, "rb"); + stat(path, &st); + ''' + masked = mask_non_code(sample) + hits = [m.group(1) for m in CALL.finditer(masked)] + if hits != ["fopen", "stat"]: + print(f"selftest failed: {hits}", file=sys.stderr) + return 1 + print("fileXio fdman policy checker selftest: PASS") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", default=".") + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + if args.selftest: + return selftest() + + root = Path(args.root).resolve() + findings = scan(root) + if findings: + print("ERROR: direct-fileXio policy violated by POSIX/stdio file calls:", + file=sys.stderr) + for path, line, name in findings: + print(f" {path}:{line}: {name}()", file=sys.stderr) + print("Review src/filexio_fdman_policy_ps2.c before using these APIs.", + file=sys.stderr) + return 1 + + print("fileXio fdman policy: PASS (no POSIX/stdio file calls in src/include)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From bc077619ef972b274a55689de5fd035b6dff51f4 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 21:31:09 +0200 Subject: [PATCH 036/156] Phase 1: enforce direct-fileXio policy in CI --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 204ae404..5dce30fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,10 @@ jobs: run: python3 tools/hardware_fault_injector.py selftest - name: Self-test HDL performance log parser run: python3 tools/parse_hdl_perf.py --selftest + - name: Enforce direct-fileXio runtime policy + run: | + python3 tools/check_filexio_fdman_policy.py --selftest + python3 tools/check_filexio_fdman_policy.py ps2-build: runs-on: ubuntu-latest From 5cb0e200144b69cc2a9929b1ccc02e50e7d9fc46 Mon Sep 17 00:00:00 2001 From: Hifu Date: Tue, 25 Aug 2026 21:32:34 +0200 Subject: [PATCH 037/156] Phase 1: bypass unused timestamp conversion in legacy fio stat adapter --- src/filexio_fdman_policy_ps2.c | 65 +++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/src/filexio_fdman_policy_ps2.c b/src/filexio_fdman_policy_ps2.c index ebced5eb..0ea686f3 100644 --- a/src/filexio_fdman_policy_ps2.c +++ b/src/filexio_fdman_policy_ps2.c @@ -14,12 +14,23 @@ * fopen/open/stat-based file access. Therefore this build does not need fileXio * to replace Newlib's generic POSIX pathname backend. * - * Keep these symbols tiny and strong so fileXioInit()/Exit() retain their RPC, - * semaphore and reset semantics while the optional POSIX adapter objects remain - * unreferenced. Any future application use of fopen/open/stat/opendir/FILE I/O - * must remove this policy or provide an explicitly reviewed equivalent path. + * The startup libcglue backend still installs the older fio POSIX adapter for + * stdin/stdout/stderr. Its generic __fioGetstatHelper() performs the same costly + * mktime conversion even though this application never consumes POSIX file + * timestamps. Keep mode and size semantics for incidental library probes, but + * deliberately report zero timestamps. CI rejects application fopen/open/stat + * call sites, so a future consumer cannot silently depend on this reduced + * timestamp contract. */ +#include +#define NEWLIB_PORT_AWARE +#include +#include +#include +#include +#include + void _ps2sdk_fileXio_init(void) { } @@ -27,3 +38,49 @@ void _ps2sdk_fileXio_init(void) void _ps2sdk_fileXio_deinit(void) { } + +static mode_t fio_mode_to_posix(unsigned int mode) +{ + mode_t result = 0; + + if (mode & FIO_SO_IFREG) + result |= S_IFREG; + if (mode & FIO_SO_IFDIR) + result |= S_IFDIR; + if (mode & FIO_SO_IROTH) + result |= S_IRUSR | S_IRGRP | S_IROTH; + if (mode & FIO_SO_IWOTH) + result |= S_IWUSR | S_IWGRP | S_IWOTH; + if (mode & FIO_SO_IXOTH) + result |= S_IXUSR | S_IXGRP | S_IXOTH; + return result; +} + +int __fioGetstatHelper(const char *path, struct stat *status) +{ + io_stat_t iop_status; + + if (path == NULL || status == NULL) { + errno = EINVAL; + return -1; + } + + if (strncmp(path, "tty", 3) == 0 && path[3] >= '0' && path[3] <= '9' && + path[4] == ':') { + memset(status, 0, sizeof(*status)); + status->st_mode = S_IFCHR; + return 0; + } + + if (fioGetstat(path, &iop_status) < 0) { + errno = ENOENT; + return -1; + } + + memset(status, 0, sizeof(*status)); + status->st_mode = fio_mode_to_posix(iop_status.mode); + status->st_size = ((off_t)iop_status.hisize << 32) | (off_t)iop_status.size; + status->st_blksize = 16 * 1024; + status->st_blocks = status->st_size / 512; + return 0; +} From 425fc40f82a12e1b15a80c2d5edf249241ce973c Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 06:27:32 +0200 Subject: [PATCH 038/156] Phase 1: add formatter contract audit --- tools/printf_format_audit.py | 361 +++++++++++++++++++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 tools/printf_format_audit.py diff --git a/tools/printf_format_audit.py b/tools/printf_format_audit.py new file mode 100644 index 00000000..faa092e9 --- /dev/null +++ b/tools/printf_format_audit.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +"""Audit application printf-family format contracts. + +This is a Phase-1 evidence tool, not a formatter replacement. It scans the +runtime source tree, extracts direct calls to libc/debug/application formatting +APIs, classifies literal conversion specifiers and reports dynamic format +arguments that require manual call-graph review before an integer-only formatter +policy can be adopted. +""" + +from __future__ import annotations + +import argparse +import re +from collections import Counter +from dataclasses import dataclass +from pathlib import Path + +ROOTS = ("src", "include", "iop") +SUFFIXES = {".c", ".h", ".inc", ".S", ".s"} + +# value = zero-based index of the format argument +FORMAT_APIS = { + "printf": 0, + "vprintf": 0, + "fprintf": 1, + "vfprintf": 1, + "sprintf": 1, + "vsprintf": 1, + "snprintf": 2, + "vsnprintf": 2, + "iprintf": 0, + "viprintf": 0, + "fiprintf": 1, + "vfiprintf": 1, + "siprintf": 1, + "vsiprintf": 1, + "sniprintf": 2, + "vsniprintf": 2, + "scr_printf": 0, + "scr_vprintf": 0, + "gs_ui_console_printf": 0, + "gs_ui_console_vprintf": 0, + "session_log_line": 0, + "append_text": 3, +} + +FLOAT_CONVERSIONS = set("aAeEfFgG") +VALID_CONVERSIONS = set("diouxXfFeEgGaAcspn%") +IDENT = re.compile(r"[A-Za-z_]\w*") +STRING_TOKEN = re.compile(r'(?:u8|u|U|L)?"(?:\\.|[^"\\])*"', re.S) + + +@dataclass +class Site: + path: str + line: int + api: str + format_expr: str + literal: str | None + conversions: tuple[str, ...] + + +def source_files(root: Path): + for base in ROOTS: + directory = root / base + if not directory.is_dir(): + continue + for path in sorted(directory.rglob("*")): + if path.is_file() and path.suffix in SUFFIXES: + yield path + + +def mask_comments(text: str) -> str: + out = list(text) + i = 0 + state = "code" + while i < len(text): + ch = text[i] + nxt = text[i + 1] if i + 1 < len(text) else "" + if state == "code": + if ch == '"': + state = "string" + elif ch == "'": + state = "char" + elif ch == "/" and nxt == "/": + out[i] = out[i + 1] = " " + i += 1 + state = "line" + elif ch == "/" and nxt == "*": + out[i] = out[i + 1] = " " + i += 1 + state = "block" + elif state == "string": + if ch == "\\": + i += 1 + elif ch == '"': + state = "code" + elif state == "char": + if ch == "\\": + i += 1 + elif ch == "'": + state = "code" + elif state == "line": + if ch == "\n": + state = "code" + else: + out[i] = " " + elif state == "block": + if ch == "*" and nxt == "/": + out[i] = out[i + 1] = " " + i += 1 + state = "code" + elif ch != "\n": + out[i] = " " + i += 1 + return "".join(out) + + +def matching_paren(text: str, opening: int) -> int | None: + depth = 0 + state = "code" + i = opening + while i < len(text): + ch = text[i] + if state == "code": + if ch == '"': + state = "string" + elif ch == "'": + state = "char" + elif ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + return i + elif state == "string": + if ch == "\\": + i += 1 + elif ch == '"': + state = "code" + elif state == "char": + if ch == "\\": + i += 1 + elif ch == "'": + state = "code" + i += 1 + return None + + +def split_args(text: str) -> list[str]: + args: list[str] = [] + start = 0 + paren = bracket = brace = 0 + state = "code" + i = 0 + while i < len(text): + ch = text[i] + if state == "code": + if ch == '"': + state = "string" + elif ch == "'": + state = "char" + elif ch == "(": + paren += 1 + elif ch == ")": + paren -= 1 + elif ch == "[": + bracket += 1 + elif ch == "]": + bracket -= 1 + elif ch == "{": + brace += 1 + elif ch == "}": + brace -= 1 + elif ch == "," and paren == bracket == brace == 0: + args.append(text[start:i].strip()) + start = i + 1 + elif state == "string": + if ch == "\\": + i += 1 + elif ch == '"': + state = "code" + elif state == "char": + if ch == "\\": + i += 1 + elif ch == "'": + state = "code" + i += 1 + args.append(text[start:].strip()) + return args + + +def literal_string(expr: str) -> str | None: + pos = 0 + chunks: list[str] = [] + while pos < len(expr): + while pos < len(expr) and expr[pos].isspace(): + pos += 1 + match = STRING_TOKEN.match(expr, pos) + if not match: + return None + token = match.group(0) + quote = token.find('"') + body = token[quote + 1:-1] + # Percent signs and conversion letters are ASCII, so decoding C escapes + # is unnecessary for the contract audit. Preserve escaped percent text. + chunks.append(body) + pos = match.end() + return "".join(chunks) + + +def conversions(fmt: str) -> tuple[str, ...]: + result: list[str] = [] + i = 0 + while i < len(fmt): + if fmt[i] != "%": + i += 1 + continue + i += 1 + if i < len(fmt) and fmt[i] == "%": + i += 1 + continue + while i < len(fmt) and fmt[i] in "#0- +'": + i += 1 + if i < len(fmt) and fmt[i] == "*": + i += 1 + else: + while i < len(fmt) and fmt[i].isdigit(): + i += 1 + if i < len(fmt) and fmt[i] == ".": + i += 1 + if i < len(fmt) and fmt[i] == "*": + i += 1 + else: + while i < len(fmt) and fmt[i].isdigit(): + i += 1 + if fmt[i:i + 2] in ("hh", "ll"): + i += 2 + elif i < len(fmt) and fmt[i] in "hljztL": + i += 1 + if i >= len(fmt): + result.append("?") + break + conv = fmt[i] + result.append(conv if conv in VALID_CONVERSIONS else "?") + i += 1 + return tuple(result) + + +def scan_file(path: Path, root: Path) -> list[Site]: + raw = path.read_text(encoding="utf-8", errors="replace") + text = mask_comments(raw) + sites: list[Site] = [] + for match in IDENT.finditer(text): + api = match.group(0) + fmt_index = FORMAT_APIS.get(api) + if fmt_index is None: + continue + pos = match.end() + while pos < len(text) and text[pos].isspace(): + pos += 1 + if pos >= len(text) or text[pos] != "(": + continue + closing = matching_paren(text, pos) + if closing is None: + continue + args = split_args(text[pos + 1:closing]) + if fmt_index >= len(args): + continue + expr = args[fmt_index] + lit = literal_string(expr) + sites.append(Site( + str(path.relative_to(root)), + raw.count("\n", 0, match.start()) + 1, + api, + " ".join(expr.split()), + lit, + conversions(lit) if lit is not None else (), + )) + return sites + + +def selftest() -> None: + assert literal_string('"x=%08x"') == "x=%08x" + assert literal_string('"a" "b%llu"') == "ab%llu" + assert literal_string("format") is None + assert conversions("x=%08x %% %llu %s") == ("x", "u", "s") + assert conversions("%7.2f %.*g") == ("f", "g") + assert conversions("%zu %p %c") == ("u", "p", "c") + print("printf-format audit selftest: PASS") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", default=".") + parser.add_argument("--output", default="PRINTF_FORMAT_AUDIT.txt") + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + if args.selftest: + selftest() + return 0 + + root = Path(args.root).resolve() + sites = [site for path in source_files(root) for site in scan_file(path, root)] + api_counts = Counter(site.api for site in sites) + conv_counts = Counter(c for site in sites for c in site.conversions) + literal_sites = [site for site in sites if site.literal is not None] + dynamic_sites = [site for site in sites if site.literal is None] + float_sites = [site for site in literal_sites + if any(c in FLOAT_CONVERSIONS for c in site.conversions)] + malformed_sites = [site for site in literal_sites if "?" in site.conversions] + + lines = [ + "PS2 HDD Bootstrap Manager - printf-family format contract audit", + "", + "Epistemic status", + " CURRENT IMPLEMENTATION: direct source-level formatter call inventory.", + " INFERENCJA: integer-only policy is safe only after dynamic bridges are", + " traced to their callers and hardware correctness is tested.", + "", + f"formatter call sites: {len(sites)}", + f"literal format sites: {len(literal_sites)}", + f"dynamic format sites: {len(dynamic_sites)}", + f"literal floating-conversion sites: {len(float_sites)}", + f"malformed/unknown literal conversions: {len(malformed_sites)}", + "", + "API counts", + ] + lines += [f" {name:28s} {count}" for name, count in sorted(api_counts.items())] + lines += ["", "Literal conversion counts"] + lines += [f" %{name:3s} {count}" for name, count in sorted(conv_counts.items())] + + lines += ["", "Floating literal format sites"] + if float_sites: + for site in float_sites: + lines.append(f" {site.path}:{site.line}: {site.api}({site.format_expr})") + else: + lines.append(" none") + + lines += ["", "Dynamic format sites requiring caller review"] + if dynamic_sites: + for site in dynamic_sites: + lines.append(f" {site.path}:{site.line}: {site.api}({site.format_expr})") + else: + lines.append(" none") + + lines += ["", "Malformed/unknown literal conversion sites"] + if malformed_sites: + for site in malformed_sites: + lines.append(f" {site.path}:{site.line}: {site.api}({site.format_expr})") + else: + lines.append(" none") + + Path(args.output).write_text("\n".join(lines) + "\n", encoding="utf-8") + print("\n".join(lines[:12])) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 75c341eaa97d97b0dd75a3a268011fff6356d9c5 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 06:27:44 +0200 Subject: [PATCH 039/156] Phase 1: run formatter contract audit in CI --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5dce30fb..4941c7b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,8 @@ jobs: run: | python3 tools/check_filexio_fdman_policy.py --selftest python3 tools/check_filexio_fdman_policy.py + - name: Self-test formatter contract audit + run: python3 tools/printf_format_audit.py --selftest ps2-build: runs-on: ubuntu-latest @@ -42,6 +44,8 @@ jobs: sh tools/build_benchmark_provenance.sh BENCHMARK_PROVENANCE.yml && python3 tools/corpus_v2_project_audit.py --output CORPUS_V2_PROJECT_AUDIT.txt && + python3 tools/printf_format_audit.py + --output PRINTF_FORMAT_AUDIT.txt && make clean && make && python3 tools/optimization_audit.py --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF @@ -58,6 +62,7 @@ jobs: PS2_HDD_BOOTSTRAP_MANAGER.map OPTIMIZATION_AUDIT.txt CORPUS_V2_PROJECT_AUDIT.txt + PRINTF_FORMAT_AUDIT.txt GCC_R5900_TARGET.txt BENCHMARK_PROVENANCE.yml HDDMAN.CFG From c336066421ebdb529d064f7c7919f254c461cb23 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 06:31:40 +0200 Subject: [PATCH 040/156] Phase 1: bypass unused libcglue timezone bootstrap --- src/filexio_fdman_policy_ps2.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/filexio_fdman_policy_ps2.c b/src/filexio_fdman_policy_ps2.c index 0ea686f3..2326011f 100644 --- a/src/filexio_fdman_policy_ps2.c +++ b/src/filexio_fdman_policy_ps2.c @@ -21,6 +21,17 @@ * deliberately report zero timestamps. CI rejects application fopen/open/stat * call sites, so a future consumer cannot silently depend on this reduced * timestamp contract. + * + * CURRENT IMPLEMENTATION: PS2SDK's weak _libcglue_timezone_update() is invoked + * unconditionally from _libcglue_init(). Its default implementation reads the + * OSD timezone through POSIX open/read/close, builds a TZ string with sprintf() + * and calls setenv(). This application never calls localtime/mktime/strftime or + * PS2SDK's timezone/daylight setters, and none of its persistence formats expose + * libc local-time semantics. A strong no-op below therefore removes startup work + * and the associated formatting/environment dependency chain without changing + * the manager's declared storage or timing contracts. If local civil time is + * introduced later, this policy must be removed or replaced by an explicit, + * tested application time contract. */ #include @@ -39,6 +50,10 @@ void _ps2sdk_fileXio_deinit(void) { } +void _libcglue_timezone_update(void) +{ +} + static mode_t fio_mode_to_posix(unsigned int mode) { mode_t result = 0; From 2644d4f2b474d55ca15c04ae8b9185e6125dfad3 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 06:34:00 +0200 Subject: [PATCH 041/156] Phase 1: trace local variadic formatter bridges --- tools/printf_format_audit.py | 103 ++++++++++++++++++++++++----------- 1 file changed, 71 insertions(+), 32 deletions(-) diff --git a/tools/printf_format_audit.py b/tools/printf_format_audit.py index faa092e9..8c1a17ec 100644 --- a/tools/printf_format_audit.py +++ b/tools/printf_format_audit.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 """Audit application printf-family format contracts. -This is a Phase-1 evidence tool, not a formatter replacement. It scans the -runtime source tree, extracts direct calls to libc/debug/application formatting -APIs, classifies literal conversion specifiers and reports dynamic format -arguments that require manual call-graph review before an integer-only formatter -policy can be adopted. +This is a Phase-1 evidence tool, not a formatter replacement. It scans runtime +source, extracts direct calls to libc/debug/application formatting APIs and also +follows the small local variadic append bridges whose format argument is passed +through to vsnprintf(). The goal is to establish the actual conversion contract +before any integer-only Newlib path is considered. """ from __future__ import annotations @@ -16,8 +16,8 @@ from dataclasses import dataclass from pathlib import Path -ROOTS = ("src", "include", "iop") -SUFFIXES = {".c", ".h", ".inc", ".S", ".s"} +ROOTS = ("src", "iop") +SUFFIXES = {".c", ".inc", ".S", ".s"} # value = zero-based index of the format argument FORMAT_APIS = { @@ -42,7 +42,15 @@ "gs_ui_console_printf": 0, "gs_ui_console_vprintf": 0, "session_log_line": 0, - "append_text": 3, +} + +# Static local wrappers with different signatures but the same name are keyed by +# file. These are intentionally explicit so a new bridge appears as an unresolved +# dynamic vsnprintf site instead of being silently guessed by the audit. +FILE_FORMAT_APIS = { + ("src/session_log.c", "append_text"): 3, + ("src/boot_report.c", "report_append"): 3, + ("src/forensic_controller_ps2.c", "report_append"): 1, } FLOAT_CONVERSIONS = set("aAeEfFgG") @@ -58,6 +66,7 @@ class Site: api: str format_expr: str literal: str | None + fragment_text: str conversions: tuple[str, ...] @@ -191,6 +200,15 @@ def split_args(text: str) -> list[str]: return args +def string_token_bodies(expr: str) -> list[str]: + bodies: list[str] = [] + for token in STRING_TOKEN.finditer(expr): + raw = token.group(0) + quote = raw.find('"') + bodies.append(raw[quote + 1:-1]) + return bodies + + def literal_string(expr: str) -> str | None: pos = 0 chunks: list[str] = [] @@ -202,10 +220,7 @@ def literal_string(expr: str) -> str | None: return None token = match.group(0) quote = token.find('"') - body = token[quote + 1:-1] - # Percent signs and conversion letters are ASCII, so decoding C escapes - # is unnecessary for the contract audit. Preserve escaped percent text. - chunks.append(body) + chunks.append(token[quote + 1:-1]) pos = match.end() return "".join(chunks) @@ -251,10 +266,11 @@ def conversions(fmt: str) -> tuple[str, ...]: def scan_file(path: Path, root: Path) -> list[Site]: raw = path.read_text(encoding="utf-8", errors="replace") text = mask_comments(raw) + rel = str(path.relative_to(root)) sites: list[Site] = [] for match in IDENT.finditer(text): api = match.group(0) - fmt_index = FORMAT_APIS.get(api) + fmt_index = FILE_FORMAT_APIS.get((rel, api), FORMAT_APIS.get(api)) if fmt_index is None: continue pos = match.end() @@ -265,18 +281,30 @@ def scan_file(path: Path, root: Path) -> list[Site]: closing = matching_paren(text, pos) if closing is None: continue + + # Function definitions are not calls. Excluding them removes the most + # misleading kind of dynamic-format noise without trying to parse all C. + after = closing + 1 + while after < len(text) and text[after].isspace(): + after += 1 + if after < len(text) and text[after] == "{": + continue + args = split_args(text[pos + 1:closing]) if fmt_index >= len(args): continue expr = args[fmt_index] lit = literal_string(expr) + fragments = "".join(string_token_bodies(expr)) + conv_source = lit if lit is not None else fragments sites.append(Site( - str(path.relative_to(root)), + rel, raw.count("\n", 0, match.start()) + 1, api, " ".join(expr.split()), lit, - conversions(lit) if lit is not None else (), + fragments, + conversions(conv_source), )) return sites @@ -284,7 +312,8 @@ def scan_file(path: Path, root: Path) -> list[Site]: def selftest() -> None: assert literal_string('"x=%08x"') == "x=%08x" assert literal_string('"a" "b%llu"') == "ab%llu" - assert literal_string("format") is None + assert literal_string('APP_NAME " %s"') is None + assert string_token_bodies('APP_NAME " %s"') == [" %s"] assert conversions("x=%08x %% %llu %s") == ("x", "u", "s") assert conversions("%7.2f %.*g") == ("f", "g") assert conversions("%zu %p %c") == ("u", "p", "c") @@ -307,45 +336,55 @@ def main() -> int: conv_counts = Counter(c for site in sites for c in site.conversions) literal_sites = [site for site in sites if site.literal is not None] dynamic_sites = [site for site in sites if site.literal is None] - float_sites = [site for site in literal_sites + float_sites = [site for site in sites if any(c in FLOAT_CONVERSIONS for c in site.conversions)] - malformed_sites = [site for site in literal_sites if "?" in site.conversions] + malformed_sites = [site for site in sites if "?" in site.conversions] + opaque_dynamic = [site for site in dynamic_sites if not site.fragment_text] lines = [ "PS2 HDD Bootstrap Manager - printf-family format contract audit", "", "Epistemic status", - " CURRENT IMPLEMENTATION: direct source-level formatter call inventory.", - " INFERENCJA: integer-only policy is safe only after dynamic bridges are", - " traced to their callers and hardware correctness is tested.", + " CURRENT IMPLEMENTATION: source-level formatter and local bridge inventory.", + " INFERENCJA: integer-only policy is safe only after opaque dynamic bridges", + " are traced to callers and hardware correctness is tested.", "", f"formatter call sites: {len(sites)}", - f"literal format sites: {len(literal_sites)}", - f"dynamic format sites: {len(dynamic_sites)}", - f"literal floating-conversion sites: {len(float_sites)}", - f"malformed/unknown literal conversions: {len(malformed_sites)}", + f"fully literal format sites: {len(literal_sites)}", + f"dynamic/macro format sites: {len(dynamic_sites)}", + f"opaque dynamic format sites: {len(opaque_dynamic)}", + f"floating-conversion sites in visible string tokens: {len(float_sites)}", + f"malformed/unknown visible conversions: {len(malformed_sites)}", "", "API counts", ] lines += [f" {name:28s} {count}" for name, count in sorted(api_counts.items())] - lines += ["", "Literal conversion counts"] + lines += ["", "Visible conversion counts"] lines += [f" %{name:3s} {count}" for name, count in sorted(conv_counts.items())] - lines += ["", "Floating literal format sites"] + lines += ["", "Floating conversion sites"] if float_sites: for site in float_sites: lines.append(f" {site.path}:{site.line}: {site.api}({site.format_expr})") else: lines.append(" none") - lines += ["", "Dynamic format sites requiring caller review"] - if dynamic_sites: - for site in dynamic_sites: + lines += ["", "Opaque dynamic format sites requiring caller review"] + if opaque_dynamic: + for site in opaque_dynamic: + lines.append(f" {site.path}:{site.line}: {site.api}({site.format_expr})") + else: + lines.append(" none") + + lines += ["", "Macro/partially literal format sites"] + partial = [site for site in dynamic_sites if site.fragment_text] + if partial: + for site in partial: lines.append(f" {site.path}:{site.line}: {site.api}({site.format_expr})") else: lines.append(" none") - lines += ["", "Malformed/unknown literal conversion sites"] + lines += ["", "Malformed/unknown visible conversion sites"] if malformed_sites: for site in malformed_sites: lines.append(f" {site.path}:{site.line}: {site.api}({site.format_expr})") @@ -353,7 +392,7 @@ def main() -> int: lines.append(" none") Path(args.output).write_text("\n".join(lines) + "\n", encoding="utf-8") - print("\n".join(lines[:12])) + print("\n".join(lines[:14])) return 0 From 3267fa78a1b5e60fadf554a03d439698a2ce2c58 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 06:34:17 +0200 Subject: [PATCH 042/156] Phase 1: guard stateless timezone policy --- tools/check_libcglue_time_policy.py | 143 ++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 tools/check_libcglue_time_policy.py diff --git a/tools/check_libcglue_time_policy.py b/tools/check_libcglue_time_policy.py new file mode 100644 index 00000000..69e77c91 --- /dev/null +++ b/tools/check_libcglue_time_policy.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Guard the Phase-1 no-local-time libcglue startup policy. + +The application overrides PS2SDK's weak _libcglue_timezone_update() because the +manager has no local civil-time contract. This checker prevents future runtime +code from silently adding APIs that require that startup timezone state. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +ROOTS = ("src", "include") +SUFFIXES = {".c", ".h", ".inc", ".s", ".S"} +FORBIDDEN = ( + "localtime", "localtime_r", "mktime", "ctime", "ctime_r", "strftime", + "tzset", "_tzset_r", "_tzset_unlocked_r", + "ps2sdk_setTimezone", "ps2sdk_setDaylightSaving", +) +CALL = re.compile(r"\b(" + "|".join(map(re.escape, FORBIDDEN)) + r")\s*\(") + + +def mask_non_code(text: str) -> str: + out = list(text) + state = "code" + i = 0 + while i < len(text): + ch = text[i] + nxt = text[i + 1] if i + 1 < len(text) else "" + if state == "code": + if ch == "/" and nxt == "/": + out[i] = out[i + 1] = " " + state = "line" + i += 1 + elif ch == "/" and nxt == "*": + out[i] = out[i + 1] = " " + state = "block" + i += 1 + elif ch == '"': + out[i] = " " + state = "string" + elif ch == "'": + out[i] = " " + state = "char" + elif state == "line": + if ch == "\n": + state = "code" + else: + out[i] = " " + elif state == "block": + if ch == "*" and nxt == "/": + out[i] = out[i + 1] = " " + state = "code" + i += 1 + elif ch != "\n": + out[i] = " " + elif state in ("string", "char"): + if ch == "\\": + out[i] = " " + if i + 1 < len(text): + if text[i + 1] != "\n": + out[i + 1] = " " + i += 1 + elif (state == "string" and ch == '"') or ( + state == "char" and ch == "'"): + out[i] = " " + state = "code" + elif ch != "\n": + out[i] = " " + i += 1 + return "".join(out) + + +def source_files(root: Path): + for dirname in ROOTS: + directory = root / dirname + if not directory.is_dir(): + continue + for path in sorted(directory.rglob("*")): + if path.is_file() and path.suffix in SUFFIXES: + yield path + + +def scan(root: Path): + findings = [] + policy_file = root / "src" / "filexio_fdman_policy_ps2.c" + for path in source_files(root): + if path == policy_file: + continue + text = path.read_text(encoding="utf-8", errors="replace") + masked = mask_non_code(text) + for match in CALL.finditer(masked): + line = masked.count("\n", 0, match.start()) + 1 + findings.append((str(path.relative_to(root)), line, match.group(1))) + return findings + + +def selftest() -> int: + sample = r''' + GetTimerSystemTime(); + snprintf(buf, sizeof(buf), "localtime(x)"); + /* mktime(&tm); */ + // strftime(buf, n, fmt, &tm); + int localtime_counter = 0; + localtime(&now); + ps2sdk_setTimezone(60); + ''' + masked = mask_non_code(sample) + hits = [m.group(1) for m in CALL.finditer(masked)] + if hits != ["localtime", "ps2sdk_setTimezone"]: + print(f"selftest failed: {hits}", file=sys.stderr) + return 1 + print("libcglue time policy checker selftest: PASS") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", default=".") + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + if args.selftest: + return selftest() + + root = Path(args.root).resolve() + findings = scan(root) + if findings: + print("ERROR: no-local-time libcglue policy violated:", file=sys.stderr) + for path, line, name in findings: + print(f" {path}:{line}: {name}()", file=sys.stderr) + print("Review the _libcglue_timezone_update override before using these APIs.", + file=sys.stderr) + return 1 + + print("libcglue time policy: PASS (no local-time consumers in src/include)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 5f92915d2d85823b063e83494edb8574ca00e725 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 06:34:47 +0200 Subject: [PATCH 043/156] Phase 1: route bounded app formatting through integer Newlib core --- src/printf_policy_ps2.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 src/printf_policy_ps2.c diff --git a/src/printf_policy_ps2.c b/src/printf_policy_ps2.c new file mode 100644 index 00000000..188dd3d3 --- /dev/null +++ b/src/printf_policy_ps2.c @@ -0,0 +1,35 @@ +/* + * Phase-1 bounded formatting policy. + * + * CURRENT IMPLEMENTATION (PS2DEV Newlib ee-v4.6.0): vsniprintf() has the same + * bounded string-buffer contract as vsnprintf() but routes through + * _svfiprintf_r, the integer-only formatter, instead of _svfprintf_r. The + * latter retains dtoa/floating formatting support that this application does + * not use in its audited runtime format strings. + * + * The linker wraps only snprintf/vsnprintf references. The Phase-1 format audit + * is the companion correctness guard: introducing %a/%e/%f/%g formatting must + * first remove this policy or add an explicitly separate floating formatter. + */ + +#include +#include +#include + +int __wrap_vsnprintf(char *buffer, size_t capacity, + const char *format, va_list arguments) +{ + return vsniprintf(buffer, capacity, format, arguments); +} + +int __wrap_snprintf(char *buffer, size_t capacity, + const char *format, ...) +{ + va_list arguments; + int result; + + va_start(arguments, format); + result = vsniprintf(buffer, capacity, format, arguments); + va_end(arguments); + return result; +} From d758761cde7e8ffb91049412c6fe2e2a8fd0707d Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 06:35:01 +0200 Subject: [PATCH 044/156] Phase 1: link integer-only bounded formatting policy --- GNUmakefile | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 33d7035c..1f37a9cf 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -1,13 +1,17 @@ -# Corpus-v2 Phase-1 fileXio/newlib link policy experiment. +# Corpus-v2 Phase-1 application/runtime link policies. # -# Keep the normal project Makefile unchanged. This PS2-only object is linked -# before libfileXio so its application-specific _ps2sdk_fileXio_init/deinit -# policy can be measured against the unmodified PS2SDK archive path. +# Keep the normal project Makefile unchanged so every policy remains an explicit +# A/B layer. These small objects are linked before the PS2SDK/Newlib archives and +# replace only contracts that the Phase-1 source audits prove unused. include Makefile -EE_OBJS += filexio_fdman_policy_ps2.o -$(EE_BIN): filexio_fdman_policy_ps2.o +EE_OBJS += filexio_fdman_policy_ps2.o printf_policy_ps2.o +$(EE_BIN): filexio_fdman_policy_ps2.o printf_policy_ps2.o +EE_LDFLAGS += -Wl,--wrap=snprintf -Wl,--wrap=vsnprintf filexio_fdman_policy_ps2.o: src/filexio_fdman_policy_ps2.c $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ + +printf_policy_ps2.o: src/printf_policy_ps2.c + $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ From 8ac1448c2fd932bfddb8a53694bbe6721d2f7a0e Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 06:36:17 +0200 Subject: [PATCH 045/156] Phase 1: keep linker wrap shims outside LTO --- GNUmakefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/GNUmakefile b/GNUmakefile index 1f37a9cf..8cbe8180 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -13,5 +13,8 @@ EE_LDFLAGS += -Wl,--wrap=snprintf -Wl,--wrap=vsnprintf filexio_fdman_policy_ps2.o: src/filexio_fdman_policy_ps2.c $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ +# GNU ld --wrap rewrites references after the LTO plugin has performed its own +# reachability pass. Keep these two externally-named shims as ordinary object +# code so LTO cannot discard them before ld creates __wrap_* references. printf_policy_ps2.o: src/printf_policy_ps2.c - $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ + $(EE_CC) $(EE_CFLAGS) -fno-lto $(EE_INCS) -c $< -o $@ From 994b7ddad20306b4881af8543d3cd0e05117b158 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 06:38:29 +0200 Subject: [PATCH 046/156] Phase 1: enforce integer-only formatter contract --- tools/check_integer_format_policy.py | 112 +++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 tools/check_integer_format_policy.py diff --git a/tools/check_integer_format_policy.py b/tools/check_integer_format_policy.py new file mode 100644 index 00000000..d996033c --- /dev/null +++ b/tools/check_integer_format_policy.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Enforce the source contract required by printf_policy_ps2.c. + +All formatting that reaches the wrapped snprintf/vsnprintf path must remain +integer/string-only. A deliberately small set of opaque variadic pass-through +sites is allowed only because their callers are themselves covered by the format +audit. Any new opaque bridge forces review rather than silently inheriting this +policy. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from collections import Counter +from pathlib import Path + +import printf_format_audit as audit + +EXPECTED_OPAQUE = Counter({ + ("src/boot_report.c", "vsnprintf", "format"): 1, + ("src/forensic_controller_ps2.c", "vsnprintf", "format"): 1, + ("src/gs_debug_compat_ps2.c", "gs_ui_console_vprintf", "format"): 2, + ("src/gs_ui_ps2.c", "vsnprintf", "format"): 1, + ("src/gs_ui_ps2.c", "gs_ui_console_vprintf", "format"): 1, + ("src/printf_policy_ps2.c", "vsniprintf", "format"): 2, + ("src/session_log.c", "vsnprintf", "format"): 2, +}) + + +def app_name_is_safe(root: Path) -> bool: + header = (root / "include" / "app_identity.h").read_text( + encoding="utf-8", errors="replace") + match = re.search(r'^\s*#\s*define\s+APP_NAME\s+"((?:\\.|[^"\\])*)"\s*$', + header, re.M) + return bool(match) and "%" not in match.group(1) + + +def selftest() -> int: + if sum(EXPECTED_OPAQUE.values()) != 10: + print("selftest failed: opaque bridge baseline changed", file=sys.stderr) + return 1 + print("integer formatter policy checker selftest: PASS") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", default=".") + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + if args.selftest: + return selftest() + + root = Path(args.root).resolve() + sites = [site for path in audit.source_files(root) + for site in audit.scan_file(path, root)] + floats = [site for site in sites + if any(c in audit.FLOAT_CONVERSIONS for c in site.conversions)] + malformed = [site for site in sites if "?" in site.conversions] + opaque = Counter((site.path, site.api, site.format_expr) for site in sites + if site.literal is None and not site.fragment_text) + partial = [site for site in sites + if site.literal is None and site.fragment_text] + unexpected_partial = [site for site in partial + if not site.format_expr.startswith("APP_NAME ")] + + failed = False + if floats: + failed = True + print("ERROR: floating printf conversion under integer-only policy:", + file=sys.stderr) + for site in floats: + print(f" {site.path}:{site.line}: {site.api}({site.format_expr})", + file=sys.stderr) + if malformed: + failed = True + print("ERROR: malformed/unknown printf conversion:", file=sys.stderr) + for site in malformed: + print(f" {site.path}:{site.line}: {site.api}({site.format_expr})", + file=sys.stderr) + if opaque != EXPECTED_OPAQUE: + failed = True + print("ERROR: opaque formatter bridge set changed:", file=sys.stderr) + for key in sorted(set(opaque) | set(EXPECTED_OPAQUE)): + if opaque[key] != EXPECTED_OPAQUE[key]: + print(f" {key}: actual={opaque[key]} expected={EXPECTED_OPAQUE[key]}", + file=sys.stderr) + if unexpected_partial: + failed = True + print("ERROR: new macro/partially literal formatter expression:", + file=sys.stderr) + for site in unexpected_partial: + print(f" {site.path}:{site.line}: {site.api}({site.format_expr})", + file=sys.stderr) + if not app_name_is_safe(root): + failed = True + print("ERROR: APP_NAME must remain a percent-free string literal while it is used in format concatenation.", + file=sys.stderr) + + if failed: + print("Review src/printf_policy_ps2.c before changing the format contract.", + file=sys.stderr) + return 1 + + print(f"integer formatter policy: PASS ({len(sites)} audited sites, no floating conversions)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From a84df999ed2891ac39aca2b710cda3386381efe1 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 06:38:49 +0200 Subject: [PATCH 047/156] Phase 1: enforce libcglue and formatter policies in CI --- .github/workflows/ci.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4941c7b2..e21543ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,8 +23,15 @@ jobs: run: | python3 tools/check_filexio_fdman_policy.py --selftest python3 tools/check_filexio_fdman_policy.py - - name: Self-test formatter contract audit - run: python3 tools/printf_format_audit.py --selftest + - name: Enforce no-local-time libcglue policy + run: | + python3 tools/check_libcglue_time_policy.py --selftest + python3 tools/check_libcglue_time_policy.py + - name: Enforce integer-only formatter policy + run: | + python3 tools/printf_format_audit.py --selftest + python3 tools/check_integer_format_policy.py --selftest + python3 tools/check_integer_format_policy.py ps2-build: runs-on: ubuntu-latest From 40619e95b8a5591aa1511f1fedbab2687531a007 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 16:24:33 +0200 Subject: [PATCH 048/156] Phase 1: reuse committed HDL metadata for read-back --- src/hdl_tools/transaction.inc | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/hdl_tools/transaction.inc b/src/hdl_tools/transaction.inc index 75a03a4a..7cb1e926 100644 --- a/src/hdl_tools/transaction.inc +++ b/src/hdl_tools/transaction.inc @@ -459,6 +459,7 @@ static int execute_transaction(hdl_transaction_t *transaction) int target_fd = -1; int verified_this_run = 0; int source_digest_valid = 0; + int metadata_valid = 0; int result; source.fd = -1; @@ -591,6 +592,7 @@ static int execute_transaction(hdl_transaction_t *transaction) &metadata_options, metadata); if (result < 0) goto done; + metadata_valid = 1; disk_status_phase_at("Committing verified game metadata last", "Main partition attribute area / HDL metadata"); disk_status_io(DISK_STATUS_WRITE, @@ -613,18 +615,27 @@ static int execute_transaction(hdl_transaction_t *transaction) } } if (transaction->stage == HDL_TRANSACTION_STAGE_METADATA_COMMITTED) { - metadata_options.game_title = transaction->game_title; - metadata_options.startup = transaction->startup; - metadata_options.disc_type = transaction->disc_type; - metadata_options.layer1_start = transaction->layer1_start; - metadata_options.hdl_compat_flags = transaction->hdl_compat_flags; - metadata_options.opl_compat_flags = transaction->opl_compat_flags; - metadata_options.dma_type = transaction->dma_type; - metadata_options.dma_mode = transaction->dma_mode; - result = hdl_metadata_build(&plan, layout.starts, layout.count, - &metadata_options, metadata); - if (result < 0 || - !target_metadata_matches(transaction->target, metadata)) { + /* A stage-4 -> stage-5 transition in this invocation already built the + * exact canonical block that was committed. Keep using that same + * transaction-owned buffer for read-back comparison. Only a resumed + * stage-5 transaction needs to reconstruct it. */ + if (!metadata_valid) { + metadata_options.game_title = transaction->game_title; + metadata_options.startup = transaction->startup; + metadata_options.disc_type = transaction->disc_type; + metadata_options.layer1_start = transaction->layer1_start; + metadata_options.hdl_compat_flags = transaction->hdl_compat_flags; + metadata_options.opl_compat_flags = transaction->opl_compat_flags; + metadata_options.dma_type = transaction->dma_type; + metadata_options.dma_mode = transaction->dma_mode; + result = hdl_metadata_build(&plan, layout.starts, layout.count, + &metadata_options, metadata); + if (result < 0) { + result = HDL_INSTALL_METADATA_FAILED; + goto done; + } + } + if (!target_metadata_matches(transaction->target, metadata)) { result = HDL_INSTALL_METADATA_FAILED; goto done; } From dd1adfb7cbf6c42ac527c6b093dd94067167591a Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 20:06:56 +0200 Subject: [PATCH 049/156] Phase 0: add same-source HDL profiling A/B builds --- .github/workflows/ci.yml | 47 ++++++++++--- Makefile | 13 +++- docs/PHASE0_HARDWARE_AB_PROTOCOL.md | 105 +++++++++++++++++++++------- include/hdl_profile.h | 20 ++++++ iop/hdl_stream/Makefile | 8 ++- iop/hdl_stream/hdl_stream.c | 62 +++++++++++++++- src/hdl_tools/fast_io.inc | 76 +++++++++++++++----- tools/build_benchmark_provenance.sh | 12 +++- 8 files changed, 284 insertions(+), 59 deletions(-) create mode 100644 include/hdl_profile.h mode change 100644 => 100755 tools/build_benchmark_provenance.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e21543ff..f9cee274 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Build, audit and strip PS2 ELF with PS2DEV v2.0.0 + - name: Build PROFILE ON/OFF pair with PS2DEV v2.0.0 run: >- docker run --rm -e PROJECT_GIT_SHA="$GITHUB_SHA" @@ -48,18 +48,40 @@ jobs: -v "$PWD:/work" -w /work ps2dev/ps2dev:v2.0.0 sh -c 'apk add --no-cache make python3 >/dev/null && sh tools/r5900_toolchain_audit.sh GCC_R5900_TARGET.txt && - sh tools/build_benchmark_provenance.sh BENCHMARK_PROVENANCE.yml && python3 tools/corpus_v2_project_audit.py --output CORPUS_V2_PROJECT_AUDIT.txt && python3 tools/printf_format_audit.py --output PRINTF_FORMAT_AUDIT.txt && - make clean && make && + HDL_PROFILE=1 sh tools/build_benchmark_provenance.sh + BENCHMARK_PROVENANCE_PROFILE_ON.yml && + make clean && make HDL_PROFILE=1 && python3 tools/optimization_audit.py --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF - --output OPTIMIZATION_AUDIT.txt && - make release' - - name: Record artifact checksum - run: sha256sum PS2_HDD_BOOTSTRAP_MANAGER.ELF | tee PS2_HDD_BOOTSTRAP_MANAGER.ELF.sha256 + --output OPTIMIZATION_AUDIT_PROFILE_ON.txt && + make HDL_PROFILE=1 release && + cp PS2_HDD_BOOTSTRAP_MANAGER.ELF + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF && + cp PS2_HDD_BOOTSTRAP_MANAGER.map + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.map && + HDL_PROFILE=0 sh tools/build_benchmark_provenance.sh + BENCHMARK_PROVENANCE_PROFILE_OFF.yml && + make clean && make HDL_PROFILE=0 && + python3 tools/optimization_audit.py + --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF + --output OPTIMIZATION_AUDIT_PROFILE_OFF.txt && + make HDL_PROFILE=0 release && + cp PS2_HDD_BOOTSTRAP_MANAGER.ELF + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF && + cp PS2_HDD_BOOTSTRAP_MANAGER.map + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.map && + cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF + PS2_HDD_BOOTSTRAP_MANAGER.ELF && + cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.map + PS2_HDD_BOOTSTRAP_MANAGER.map' + - name: Record artifact checksums + run: | + sha256sum PS2_HDD_BOOTSTRAP_MANAGER.ELF | tee PS2_HDD_BOOTSTRAP_MANAGER.ELF.sha256 + sha256sum PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF | tee HDL_PROFILE_PAIR.sha256 - uses: actions/upload-artifact@v4 with: name: PS2-HDD-Bootstrap-Manager @@ -67,11 +89,18 @@ jobs: PS2_HDD_BOOTSTRAP_MANAGER.ELF PS2_HDD_BOOTSTRAP_MANAGER.ELF.sha256 PS2_HDD_BOOTSTRAP_MANAGER.map - OPTIMIZATION_AUDIT.txt + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.map + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.map + HDL_PROFILE_PAIR.sha256 + OPTIMIZATION_AUDIT_PROFILE_ON.txt + OPTIMIZATION_AUDIT_PROFILE_OFF.txt CORPUS_V2_PROJECT_AUDIT.txt PRINTF_FORMAT_AUDIT.txt GCC_R5900_TARGET.txt - BENCHMARK_PROVENANCE.yml + BENCHMARK_PROVENANCE_PROFILE_ON.yml + BENCHMARK_PROVENANCE_PROFILE_OFF.yml HDDMAN.CFG LICENSE THIRD_PARTY_NOTICES.md diff --git a/Makefile b/Makefile index 420f9d50..1a8c4f34 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,18 @@ EE_BIN = PS2_HDD_BOOTSTRAP_MANAGER.ELF EE_MAP = PS2_HDD_BOOTSTRAP_MANAGER.map EE_OBJS = main.o manager_menu_ps2.o app_ui_ps2.o disk_status_ps2.o gs_ui_ps2.o gs_debug_compat_ps2.o app_error.o bootstrap_controller_ps2.o diagnostics_controller_ps2.o forensic_controller_ps2.o platform.o storage.o video_mode.o ui_layout.o ui_font.o spleen_font_data.o header_backup.o repair_snapshot.o forensic_snapshot.o rescue_image.o rescue_storage.o bootstrap_source.o bootstrap_signing.o apa.o apa_repair.o apa_forensic.o repair_health.o hdd_bounds.o hdd_read.o hdd_write.o hdd_repair_ps2.o hdd_forensic_repair_ps2.o repair_controller_ps2.o hdd_recovery_wrap.o bootstrap_transaction.o bootstrap_transaction_ps2.o boot_chain.o boot_chain_ps2.o boot_payload.o boot_payload_ps2.o boot_diagnostics_ps2.o boot_report.o boot_report_ps2.o boot_report_session.o session_log.o kelf.o sha256.o capsule_format.o mbr_compat.o hdl_iso.o hdl_partition.o hdl_transaction.o hdl_installer_ps2.o r5900_perf.o EE_LIBS = -ldebug -ldraw -lgraph -lpacket -ldma -lm -lpad -lfileXio -lpatches -lpoweroff -lsecr -lkernel + +# Phase-0 profiler perturbation must be measurable with the same source and the +# same optimization flags. HDL_PROFILE=1 is the current instrumented default; +# HDL_PROFILE=0 compiles EE/IOP HDL telemetry out without changing dataflow. +HDL_PROFILE ?= 1 +ifeq ($(filter $(HDL_PROFILE),0 1),) +$(error HDL_PROFILE must be 0 or 1) +endif + # LTO lets the R5900 compiler optimize across the deliberately small modules # while section GC still removes unused recovery/UI helpers from the final ELF. -EE_CFLAGS = -O2 -flto -G0 -Wall -Wextra -Werror -std=gnu99 -fdata-sections -ffunction-sections -Iinclude +EE_CFLAGS = -O2 -flto -G0 -Wall -Wextra -Werror -std=gnu99 -fdata-sections -ffunction-sections -Iinclude -DHDL_PROFILE_ENABLED=$(HDL_PROFILE) # Keep a linker map for every build. The R5900 has a 16 KiB I-cache, so archive # provenance, section growth and final placement are performance data, not just # link-time trivia. This also lets CI explain why heavyweight Newlib routines @@ -319,7 +328,7 @@ ps2hdd_posix_irx.c: $(PS2SDK)/bin/bin2c $(PS2SDK)/iop/irx/ps2hdd-bdm.irx $@ ps2hdd_posix_irx hdl_stream.irx: - $(MAKE) -C iop/hdl_stream IOP_BIN=$(abspath $@) + $(MAKE) -C iop/hdl_stream IOP_BIN=$(abspath $@) HDL_PROFILE=$(HDL_PROFILE) hdl_stream_irx.c: hdl_stream.irx $(PS2SDK)/bin/bin2c $< $@ hdl_stream_irx diff --git a/docs/PHASE0_HARDWARE_AB_PROTOCOL.md b/docs/PHASE0_HARDWARE_AB_PROTOCOL.md index e101ca12..60367671 100644 --- a/docs/PHASE0_HARDWARE_AB_PROTOCOL.md +++ b/docs/PHASE0_HARDWARE_AB_PROTOCOL.md @@ -7,17 +7,40 @@ throughput. ## Compared builds -### A: audited pre-instrumentation baseline +The authoritative instrumentation-overhead comparison is a same-source pair +built from one green commit on `perf/corpus-v2-integration` with the same PS2DEV +container, PS2SDK source, optimization flags and runtime implementation. -- project commit: `4b5aa8d85e86c9de570a2128b52d1eaa5b334844` -- purpose: known-good HDL installer immediately before corpus-v2 measurement - instrumentation -- expected Phase-0 telemetry: absent +### A: PROFILE OFF -### B: corpus-v2 measurement build +Build the selected commit with: -Use the newest green commit on `perf/corpus-v2-integration` before accepting a -Phase-1 runtime optimization. The build must retain: +```text +HDL_PROFILE=0 +``` + +This compiles the Phase-0 EE/IOP HDL telemetry out while retaining the same: + +- direct-BDM source path; +- double-buffer prefetch worker and ownership flow; +- ps2hdd/HIOCTRANSFER target path; +- SIF DMA transfers; +- EE cache writeback/invalidation required for DMA correctness; +- SHA-256 verification; +- journal, flush, metadata commit and read-back semantics. + +`HDL_STREAM_IOCTL2_GET_FAST_STATS` remains ABI-compatible and returns a zeroed +record, but the timed hot-path accounting itself is absent. + +### B: PROFILE ON + +Build the exact same commit with: + +```text +HDL_PROFILE=1 +``` + +This is the diagnostic build and retains: - EE pump/source/target latency histograms; - IOP direct-source/fallback/prefetch/HDD/SIF histograms; @@ -25,9 +48,16 @@ Phase-1 runtime optimization. The build must retain: - benchmark provenance artifact; - linker/ELF audit artifacts. -The project source, toolchain and HDL transaction semantics must otherwise stay -unchanged for the measurement comparison. Any Phase-1 change must be measured -separately and must not be folded into the Phase-0 overhead result. +CI emits both ELFs, linker maps, optimization audits and provenance records from +one checkout and one toolchain image. The profile mode is written explicitly to +each provenance file. + +### Historical pre-instrumentation reference + +Commit `4b5aa8d85e86c9de570a2128b52d1eaa5b334844` remains useful as the original +known-good HDL installer baseline, but it is **not** the authoritative profiler +overhead A/B. Runtime and Phase-1 policy changes after that commit make such a +cross-commit timing delta confounded. ## Hardware provenance @@ -43,6 +73,7 @@ ps2sdk_commit: toolchain: active_irx: build_flags: +hdl_profile_enabled: workload: direction: buffering: @@ -52,8 +83,9 @@ units: correctness_hash: ``` -The CI-generated `BENCHMARK_PROVENANCE.yml` supplies build-side fields. Hardware -fields remain explicit manual measurements rather than guessed metadata. +The CI-generated `BENCHMARK_PROVENANCE_PROFILE_OFF.yml` and +`BENCHMARK_PROVENANCE_PROFILE_ON.yml` supply build-side fields. Hardware fields +remain explicit manual measurements rather than guessed metadata. ## Workload contract @@ -67,15 +99,28 @@ Use the same: - video mode and active background services; - cold/warm policy. +Prefer an interleaved order such as `OFF, ON, ON, OFF` or `ON, OFF, OFF, ON` +rather than running every sample of one build first. This reduces temperature, +device-state and session-order bias. Recreate or otherwise control the target +layout between destructive install samples so the compared workload stays +meaningfully equivalent. + For HDL copy throughput, use one ISO large enough that startup/allocation noise is negligible compared with the bulk copy phase. Do not compare different ISOs, USB sticks or HDD layouts and call the result an instrumentation delta. ## Measurements -For each build record at least: +For **both** builds record at least: +- total install wall time; - bulk copy wall time and useful KiB/s; +- payload verification wall time; +- final correctness result/hash; +- any cancellation, journal or metadata-commit failure. + +For PROFILE ON additionally retain: + - p50, p95, p99 and max EE pump ioctl latency; - p50, p95, p99 and max IOP direct-source latency; - p50, p95, p99 and max prefetch consumer wait; @@ -83,28 +128,36 @@ For each build record at least: - p50, p95, p99 and max SIF DMA completion latency; - prefetch hit/miss counts; - fallback-source/fallback-target bytes; -- useful payload, SIF DMA and EE cache-maintenance bytes; -- final correctness result/hash; -- any cancellation, journal or metadata-commit failure. +- useful payload, SIF DMA and EE cache-maintenance bytes. + +For the overhead result report PROFILE ON relative to PROFILE OFF for wall time +and throughput. Do not fabricate p95/p99 for PROFILE OFF from absent telemetry; +the point of the OFF build is to remove that instrumentation. -Run at least three complete comparable samples for an initial engineering -answer. More samples are required if p95/p99 or wall time is unstable. +Run at least four complete comparable samples, preferably two in each half of an +interleaved order, for an initial engineering answer. Add samples if wall time +or PROFILE ON tail latency is unstable. ## Phase-0 acceptance Phase 0 may be marked hardware-complete only when: -1. build A and build B both pass the same correctness workload; -2. B produces internally consistent stage counters and traffic accounting; -3. the measurement overhead of B is quantified rather than assumed negligible; -4. p50/p95/p99/max are retained, not replaced by an average; +1. PROFILE OFF and PROFILE ON are from the same project SHA and pass the same + correctness workload; +2. PROFILE ON produces internally consistent stage counters and traffic + accounting; +3. the measurement overhead of PROFILE ON is quantified rather than assumed + negligible; +4. PROFILE ON retains p50/p95/p99/max instead of replacing distributions with + an average; 5. the result includes console/toolchain/IRX/workload provenance; 6. R5900 counter calibration is checked on the real EE before counter-derived optimization claims are made. -If instrumentation materially changes throughput or tail latency, keep a -companion non-instrumented performance build and use the measurement build only -for diagnosis. +If instrumentation materially changes throughput or tail latency, keep +`HDL_PROFILE=0` as the performance build and use `HDL_PROFILE=1` only for +diagnosis. If the delta is negligible for the tested workload, that conclusion +still applies only to the recorded console/adapters/workload. ## R5900 counter calibration diff --git a/include/hdl_profile.h b/include/hdl_profile.h new file mode 100644 index 00000000..31dd02fd --- /dev/null +++ b/include/hdl_profile.h @@ -0,0 +1,20 @@ +#ifndef PS2_HDD_BOOTSTRAP_MANAGER_HDL_PROFILE_H +#define PS2_HDD_BOOTSTRAP_MANAGER_HDL_PROFILE_H + +/* + * Same-source Phase-0 instrumentation switch. + * + * PROFILE=1 is the diagnostic build used to collect EE/IOP latency and + * transport telemetry. PROFILE=0 compiles that telemetry out while preserving + * the exact HDL pump, prefetch, DMA, cache-coherency and durability paths so a + * real-console A/B measures profiler perturbation instead of another code path. + */ +#ifndef HDL_PROFILE_ENABLED +#define HDL_PROFILE_ENABLED 1 +#endif + +#if HDL_PROFILE_ENABLED != 0 && HDL_PROFILE_ENABLED != 1 +#error "HDL_PROFILE_ENABLED must be 0 or 1" +#endif + +#endif diff --git a/iop/hdl_stream/Makefile b/iop/hdl_stream/Makefile index b651b01a..3823993f 100644 --- a/iop/hdl_stream/Makefile +++ b/iop/hdl_stream/Makefile @@ -1,6 +1,11 @@ IOP_BIN ?= hdl_stream.irx IOP_OBJS = hdl_stream.o imports.o +HDL_PROFILE ?= 1 +ifeq ($(filter $(HDL_PROFILE),0 1),) +$(error HDL_PROFILE must be 0 or 1) +endif + # PS2SDK's generic IOP rules default to -Os. hdl_stream is not a resident-size # utility in our workload: it is the 37.5 MHz R3000A hot path that must keep # USB, DEV9 and SIF fed during multi-gigabyte ISO installs. Put -O2 after the @@ -8,7 +13,8 @@ IOP_OBJS = hdl_stream.o imports.o # minimum code size. We deliberately stop at -O2 because the IOP has a tiny # cache and -O3 loop growth can cost more than it saves on this core. IOP_CFLAGS += -O2 -DIOMANX_OLD_NAME_COMPATIBILITY=0 \ - -DIOMANX_OLD_NAME_ADDDELDRV=0 + -DIOMANX_OLD_NAME_ADDDELDRV=0 \ + -DHDL_PROFILE_ENABLED=$(HDL_PROFILE) IOP_WARNFLAGS = -Wall -Wextra -Werror IOP_INCS += -I../../include diff --git a/iop/hdl_stream/hdl_stream.c b/iop/hdl_stream/hdl_stream.c index 7ad20c2b..385a87d5 100644 --- a/iop/hdl_stream/hdl_stream.c +++ b/iop/hdl_stream/hdl_stream.c @@ -26,6 +26,7 @@ #include #include +#include "hdl_profile.h" #include "hdl_stream_rpc.h" #define HDL_STREAM_MAIN_SKIP 0x2000u @@ -77,9 +78,12 @@ typedef struct { uint32_t prefetch_stage_index; hdl_source_map_t source; +#if HDL_PROFILE_ENABLED hdl_stream_fast_stats_t stats; +#endif } hdl_stream_file_t; +#if HDL_PROFILE_ENABLED /* * Phase-0 corpus-v2 profiling deliberately stores only compact histograms in * the IOP hot path. Formatting/logging remains on the EE after a bulk phase. @@ -119,6 +123,7 @@ static void iop_latency_record(hdl_stream_iop_latency_t *stats, if (usec > stats->maximum_us) stats->maximum_us = usec; } +#endif static int stream_init(iomanX_iop_device_t *device) { @@ -185,7 +190,9 @@ static void source_map_disable(hdl_stream_file_t *stream, int source_fd) source_map_free_fragments(&stream->source); stream->source.source_fd = source_fd; stream->source.disabled = 1; +#if HDL_PROFILE_ENABLED stream->stats.flags &= ~HDL_STREAM_FAST_FLAG_DIRECT_BDM; +#endif } static int source_map_prepare(hdl_stream_file_t *stream, int source_fd) @@ -259,8 +266,10 @@ static int source_map_prepare(hdl_stream_file_t *stream, int source_fd) stream->source.fragment_count = (uint32_t)fragment_count; stream->source.cursor_fragment = 0; stream->source.cursor_base = 0; +#if HDL_PROFILE_ENABLED stream->stats.flags |= HDL_STREAM_FAST_FLAG_DIRECT_BDM; stream->stats.fragment_count = (uint32_t)fragment_count; +#endif return 0; } } @@ -365,8 +374,10 @@ static int source_read_fallback(int source_fd, uint64_t offset, static int source_read_at(hdl_stream_file_t *stream, int source_fd, uint64_t offset, void *buffer, unsigned int bytes) { +#if HDL_PROFILE_ENABLED iop_sys_clock_t start; iop_sys_clock_t end; +#endif /* * The stock usbmass BDM intentionally caps one SCSI request at 128 512-byte @@ -381,30 +392,39 @@ static int source_read_at(hdl_stream_file_t *stream, int source_fd, (bytes >> HDL_STREAM_USB_SECTOR_SHIFT) <= UINT16_MAX) { int result; +#if HDL_PROFILE_ENABLED GetSystemTime(&start); +#endif result = source_map_read( &stream->source, offset >> HDL_STREAM_USB_SECTOR_SHIFT, buffer, (uint16_t)(bytes >> HDL_STREAM_USB_SECTOR_SHIFT)); +#if HDL_PROFILE_ENABLED GetSystemTime(&end); iop_latency_record(&stream->stats.direct_source_latency, &start, &end); - +#endif if (result >= 0) { +#if HDL_PROFILE_ENABLED stream->stats.direct_reads++; stream->stats.direct_source_sectors += bytes >> 9; +#endif return (int)bytes; } source_map_disable(stream, source_fd); } +#if HDL_PROFILE_ENABLED stream->stats.fallback_reads++; GetSystemTime(&start); +#endif { int result = source_read_fallback(source_fd, offset, buffer, bytes); +#if HDL_PROFILE_ENABLED GetSystemTime(&end); iop_latency_record(&stream->stats.fallback_source_latency, &start, &end); if (result > 0) stream->stats.fallback_source_sectors += (unsigned int)result >> 9; +#endif return result; } } @@ -486,7 +506,9 @@ static int prefetch_init(hdl_stream_file_t *stream) goto fail; } +#if HDL_PROFILE_ENABLED stream->stats.flags |= HDL_STREAM_FAST_FLAG_DOUBLE_BUFFER; +#endif return 0; fail: @@ -496,16 +518,22 @@ static int prefetch_init(hdl_stream_file_t *stream) static int prefetch_wait(hdl_stream_file_t *stream) { +#if HDL_PROFILE_ENABLED iop_sys_clock_t start; iop_sys_clock_t end; +#endif int result; if (!stream->prefetch_active) return 0; +#if HDL_PROFILE_ENABLED GetSystemTime(&start); +#endif result = WaitSema(stream->prefetch_done_sema); +#if HDL_PROFILE_ENABLED GetSystemTime(&end); iop_latency_record(&stream->stats.prefetch_wait_latency, &start, &end); +#endif if (result < 0) return result; stream->prefetch_active = 0; @@ -526,12 +554,16 @@ static int prefetch_take(hdl_stream_file_t *stream, int source_fd, stream->prefetch_bytes == bytes; result = prefetch_wait(stream); if (!matches) { +#if HDL_PROFILE_ENABLED stream->stats.prefetch_misses++; +#endif return 0; } if (result != (int)bytes) return result < 0 ? result : -EIO; +#if HDL_PROFILE_ENABLED stream->stats.prefetch_hits++; +#endif *stage_index = stream->prefetch_stage_index; *stage = stream->stage[*stage_index]; return 1; @@ -577,8 +609,10 @@ static int dma_to_ee(hdl_stream_file_t *stream, uint32_t ee_address, const void *source, unsigned int bytes) { SifDmaTransfer_t transfer; +#if HDL_PROFILE_ENABLED iop_sys_clock_t start; iop_sys_clock_t end; +#endif int id; if (ee_address == 0 || bytes == 0 || (ee_address & 0x3fu) != 0 || @@ -588,14 +622,20 @@ static int dma_to_ee(hdl_stream_file_t *stream, uint32_t ee_address, transfer.dest = (void *)(uintptr_t)ee_address; transfer.size = (int)bytes; transfer.attr = 0; +#if HDL_PROFILE_ENABLED GetSystemTime(&start); +#endif id = sceSifSetDma(&transfer, 1); if (id <= 0) return -EIO; while (sceSifDmaStat(id) >= 0) {} +#if HDL_PROFILE_ENABLED GetSystemTime(&end); iop_latency_record(&stream->stats.sif_dma_latency, &start, &end); stream->stats.sif_dma_sectors += bytes >> 9; +#else + (void)stream; +#endif return 0; } @@ -617,7 +657,9 @@ static int stream_open(iomanX_iop_file_t *file, const char *name, if (stream == NULL) return -ENOMEM; memset(stream, 0, sizeof(*stream)); +#if HDL_PROFILE_ENABLED stream->stats.flags = HDL_STREAM_FAST_FLAG_IOP_TIMING; +#endif stream->source.source_fd = -1; stream->prefetch_thread = -1; stream->prefetch_request_sema = -1; @@ -755,8 +797,10 @@ static int stream_transfer(iomanX_iop_file_t *file, void *buffer, while (remaining > 0) { hddIoctl2Transfer_t transfer; +#if HDL_PROFILE_ENABLED iop_sys_clock_t start; iop_sys_clock_t end; +#endif uint64_t available; uint32_t part; uint32_t sector; @@ -775,9 +819,12 @@ static int stream_transfer(iomanX_iop_file_t *file, void *buffer, transfer.size = (uint32_t)chunk >> 9; transfer.mode = direction; transfer.buffer = cursor; +#if HDL_PROFILE_ENABLED GetSystemTime(&start); +#endif result = iomanX_ioctl2(stream->hdd_fd, HIOCTRANSFER, &transfer, sizeof(transfer), NULL, 0); +#if HDL_PROFILE_ENABLED GetSystemTime(&end); if (direction == APA_IO_MODE_WRITE) { iop_latency_record(&stream->stats.hdd_write_latency, &start, &end); @@ -788,6 +835,7 @@ static int stream_transfer(iomanX_iop_file_t *file, void *buffer, if (result >= 0) stream->stats.hdd_read_sectors += transfer.size; } +#endif if (result < 0) return result; stream->position += (uint32_t)chunk; @@ -961,13 +1009,17 @@ static int fast_source_to_ee(iomanX_iop_file_t *file, APA_IO_MODE_WRITE); if (result != (int)request->bytes) return result < 0 ? result : -EIO; +#if HDL_PROFILE_ENABLED stream->stats.pumped_chunks++; stream->stats.pumped_sectors += request->bytes >> 9; +#endif } result = dma_to_ee(stream, request->ee_address, stage, request->bytes); if (result < 0) return result; +#if HDL_PROFILE_ENABLED stream->stats.source_dma_chunks++; +#endif return (int)request->bytes; } @@ -991,7 +1043,9 @@ static int fast_target_to_ee(iomanX_iop_file_t *file, request->bytes); if (result < 0) return result; +#if HDL_PROFILE_ENABLED stream->stats.target_dma_chunks++; +#endif return (int)request->bytes; } @@ -1033,9 +1087,13 @@ static int stream_ioctl2(iomanX_iop_file_t *file, int command, if (command == HDL_STREAM_IOCTL2_TARGET_TO_EE) return fast_target_to_ee(file, argument, argument_length); if (command == HDL_STREAM_IOCTL2_GET_FAST_STATS) { - if (buffer == NULL || buffer_length < sizeof(stream->stats)) + if (buffer == NULL || buffer_length < sizeof(hdl_stream_fast_stats_t)) return -EINVAL; +#if HDL_PROFILE_ENABLED memcpy(buffer, &stream->stats, sizeof(stream->stats)); +#else + memset(buffer, 0, sizeof(hdl_stream_fast_stats_t)); +#endif return 0; } return -EINVAL; diff --git a/src/hdl_tools/fast_io.inc b/src/hdl_tools/fast_io.inc index bca28ed8..ef8fa180 100644 --- a/src/hdl_tools/fast_io.inc +++ b/src/hdl_tools/fast_io.inc @@ -1,4 +1,8 @@ +#include "hdl_profile.h" + +#if HDL_PROFILE_ENABLED #include +#endif /* * EE-side glue for the dev20 IOP payload pump. @@ -15,8 +19,20 @@ * into IOP staging and performs one IOP->EE DMA, avoiding fileXio's extra * staging/copy layer. */ -#define HDL_FAST_LATENCY_BUCKETS 24u #define HDL_FAST_NOINLINE __attribute__((noinline)) + +static int hdl_fast_source_fd = -1; +static uint64_t hdl_fast_source_position; +static uint64_t hdl_fast_source_seek_value; +static uint64_t hdl_fast_source_total; +static int hdl_fast_source_position_valid; +static int hdl_fast_source_seek_pending; +static int hdl_fast_pump_mode; +static int hdl_fast_pump_ack; +static unsigned int hdl_fast_pump_ack_bytes; + +#if HDL_PROFILE_ENABLED +#define HDL_FAST_LATENCY_BUCKETS 24u #define HDL_FAST_COLD __attribute__((cold, noinline)) typedef struct { @@ -25,10 +41,6 @@ typedef struct { uint32_t buckets[HDL_FAST_LATENCY_BUCKETS]; } hdl_fast_latency_stats_t; -static int hdl_fast_source_fd = -1; -static uint64_t hdl_fast_source_position; -static uint64_t hdl_fast_source_seek_value; -static uint64_t hdl_fast_source_total; static uint64_t hdl_fast_copy_start_ticks; static uint64_t hdl_fast_copy_bytes; static uint64_t hdl_fast_target_bytes; @@ -38,11 +50,6 @@ static uint64_t hdl_fast_fallback_source_bytes; static uint64_t hdl_fast_fallback_target_bytes; static uint64_t hdl_fast_copy_consumer_start_ticks; static uint64_t hdl_fast_target_consumer_start_ticks; -static int hdl_fast_source_position_valid; -static int hdl_fast_source_seek_pending; -static int hdl_fast_pump_mode; -static int hdl_fast_pump_ack; -static unsigned int hdl_fast_pump_ack_bytes; static int hdl_fast_path_logged; static int hdl_fast_copy_iop_stats_logged; static int hdl_fast_verify_iop_stats_logged; @@ -197,6 +204,7 @@ static HDL_FAST_COLD void hdl_fast_log_target_profile(void) hdl_fast_target_consumer_latency.samples); hdl_fast_target_profile_logged = 1; } +#endif static HDL_FAST_NOINLINE void hdl_fast_io_reset(void) { @@ -204,6 +212,12 @@ static HDL_FAST_NOINLINE void hdl_fast_io_reset(void) hdl_fast_source_position = 0; hdl_fast_source_seek_value = 0; hdl_fast_source_total = 0; + hdl_fast_source_position_valid = 0; + hdl_fast_source_seek_pending = 0; + hdl_fast_pump_mode = 0; + hdl_fast_pump_ack = 0; + hdl_fast_pump_ack_bytes = 0; +#if HDL_PROFILE_ENABLED hdl_fast_copy_start_ticks = 0; hdl_fast_copy_bytes = 0; hdl_fast_target_bytes = 0; @@ -213,11 +227,6 @@ static HDL_FAST_NOINLINE void hdl_fast_io_reset(void) hdl_fast_fallback_target_bytes = 0; hdl_fast_copy_consumer_start_ticks = 0; hdl_fast_target_consumer_start_ticks = 0; - hdl_fast_source_position_valid = 0; - hdl_fast_source_seek_pending = 0; - hdl_fast_pump_mode = 0; - hdl_fast_pump_ack = 0; - hdl_fast_pump_ack_bytes = 0; hdl_fast_path_logged = 0; hdl_fast_copy_iop_stats_logged = 0; hdl_fast_verify_iop_stats_logged = 0; @@ -229,6 +238,7 @@ static HDL_FAST_NOINLINE void hdl_fast_io_reset(void) hdl_fast_latency_reset(&hdl_fast_target_ioctl_latency); hdl_fast_latency_reset(&hdl_fast_copy_consumer_latency); hdl_fast_latency_reset(&hdl_fast_target_consumer_latency); +#endif } static HDL_FAST_NOINLINE int hdl_fast_resolve_source_total(int source_fd) @@ -256,6 +266,7 @@ static HDL_FAST_NOINLINE int hdl_fast_resolve_source_total(int source_fd) return 0; } +#if HDL_PROFILE_ENABLED static HDL_FAST_COLD void hdl_fast_io_log_stats(int target_fd, int verify_phase) { @@ -331,6 +342,7 @@ static HDL_FAST_COLD void hdl_fast_io_log_rate(void) (unsigned long long)(raw_usb_permille % 10u)); hdl_fast_rate_logged = 1; } +#endif static HDL_FAST_NOINLINE s64 hdl_fast_fileXioLseek64(int fd, s64 offset, int whence) @@ -361,8 +373,10 @@ static HDL_FAST_NOINLINE int hdl_fast_dma_read(int command, int source_fd, void *buffer, unsigned int bytes) { +#if HDL_PROFILE_ENABLED u64 call_start; u64 call_end; +#endif int result; if (hdl_active_target_fd < 0 || buffer == NULL || bytes == 0 || @@ -372,12 +386,12 @@ static HDL_FAST_NOINLINE int hdl_fast_dma_read(int command, int source_fd, /* The IOP is about to DMA into cached EE memory. Write back any dirty line * first, then invalidate after the synchronous ioctl returns so the R5900 - * sees the new payload instead of a stale D-cache copy. Count the touched - * bytes separately from useful payload so copy/coherency amplification is - * visible in corpus-v2 benchmark logs. */ + * sees the new payload instead of a stale D-cache copy. */ SyncDCache(buffer, (unsigned char *)buffer + bytes); +#if HDL_PROFILE_ENABLED hdl_fast_cache_maintenance_bytes += bytes; call_start = GetTimerSystemTime(); +#endif if (command == HDL_STREAM_IOCTL2_TARGET_TO_EE) { hdl_stream_target_io_t request; @@ -401,12 +415,16 @@ static HDL_FAST_NOINLINE int hdl_fast_dma_read(int command, int source_fd, result = fileXioIoctl2(hdl_active_target_fd, command, &request, sizeof(request), NULL, 0); } +#if HDL_PROFILE_ENABLED call_end = GetTimerSystemTime(); hdl_fast_record_ioctl_latency(command, call_start, call_end); +#endif if (result >= 0) { InvalidDCache(buffer, (unsigned char *)buffer + bytes); +#if HDL_PROFILE_ENABLED hdl_fast_cache_maintenance_bytes += bytes; hdl_fast_sif_dma_bytes += bytes; +#endif } return result; } @@ -420,18 +438,21 @@ static HDL_FAST_NOINLINE int hdl_fast_fileXioRead(int fd, void *buffer, return fileXioRead(fd, buffer, size); if (fd == hdl_active_target_fd) { +#if HDL_PROFILE_ENABLED if (hdl_fast_target_consumer_start_ticks != 0) { hdl_fast_latency_record(&hdl_fast_target_consumer_latency, hdl_fast_target_consumer_start_ticks, GetTimerSystemTime()); hdl_fast_target_consumer_start_ticks = 0; } +#endif hdl_fast_pump_mode = 0; hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; result = hdl_fast_dma_read(HDL_STREAM_IOCTL2_TARGET_TO_EE, -1, buffer, (unsigned int)size); if (result != INT_MIN) { +#if HDL_PROFILE_ENABLED if (result > 0) { hdl_fast_target_bytes += (unsigned int)result; hdl_fast_target_consumer_start_ticks = GetTimerSystemTime(); @@ -441,6 +462,7 @@ static HDL_FAST_NOINLINE int hdl_fast_fileXioRead(int fd, void *buffer, hdl_fast_log_target_profile(); } } +#endif return result; } } else if (hdl_active_target_fd >= 0 && @@ -449,9 +471,11 @@ static HDL_FAST_NOINLINE int hdl_fast_fileXioRead(int fd, void *buffer, ? HDL_STREAM_IOCTL2_PUMP_TO_EE : HDL_STREAM_IOCTL2_SOURCE_TO_EE; +#if HDL_PROFILE_ENABLED if (command == HDL_STREAM_IOCTL2_PUMP_TO_EE && hdl_fast_copy_start_ticks == 0) hdl_fast_copy_start_ticks = GetTimerSystemTime(); +#endif result = hdl_fast_dma_read(command, fd, buffer, (unsigned int)size); if (result != INT_MIN) { hdl_fast_source_seek_pending = 0; @@ -460,6 +484,7 @@ static HDL_FAST_NOINLINE int hdl_fast_fileXioRead(int fd, void *buffer, if (result == size && command == HDL_STREAM_IOCTL2_PUMP_TO_EE) { hdl_fast_pump_ack = 1; hdl_fast_pump_ack_bytes = (unsigned int)size; +#if HDL_PROFILE_ENABLED hdl_fast_copy_bytes += (unsigned int)size; hdl_fast_copy_consumer_start_ticks = GetTimerSystemTime(); if (!hdl_fast_path_logged) { @@ -468,10 +493,13 @@ static HDL_FAST_NOINLINE int hdl_fast_fileXioRead(int fd, void *buffer, (unsigned int)size); hdl_fast_path_logged = 1; } +#endif } else { hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; +#if HDL_PROFILE_ENABLED hdl_fast_copy_consumer_start_ticks = 0; +#endif } return result; } @@ -490,19 +518,23 @@ static HDL_FAST_NOINLINE int hdl_fast_fileXioRead(int fd, void *buffer, return seek < 0 ? (int)seek : HDL_INSTALL_COPY_FAILED; } result = fileXioRead(fd, buffer, size); +#if HDL_PROFILE_ENABLED if (result > 0) { if (fd == hdl_active_target_fd) hdl_fast_fallback_target_bytes += (unsigned int)result; else if (hdl_fast_source_position_valid && fd == hdl_fast_source_fd) hdl_fast_fallback_source_bytes += (unsigned int)result; } +#endif if (result > 0 && fd != hdl_active_target_fd && hdl_fast_source_position_valid && fd == hdl_fast_source_fd) hdl_fast_source_position += (unsigned int)result; hdl_fast_source_seek_pending = 0; hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; +#if HDL_PROFILE_ENABLED hdl_fast_copy_consumer_start_ticks = 0; +#endif return result; } @@ -513,27 +545,33 @@ static HDL_FAST_NOINLINE int hdl_fast_fileXioWrite(int fd, if (hdl_fast_pump_ack) { if (fd == hdl_active_target_fd && size > 0 && (unsigned int)size == hdl_fast_pump_ack_bytes) { +#if HDL_PROFILE_ENABLED if (hdl_fast_copy_consumer_start_ticks != 0) { hdl_fast_latency_record(&hdl_fast_copy_consumer_latency, hdl_fast_copy_consumer_start_ticks, GetTimerSystemTime()); hdl_fast_copy_consumer_start_ticks = 0; } +#endif hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; +#if HDL_PROFILE_ENABLED if (hdl_fast_source_total != 0 && hdl_fast_source_position == hdl_fast_source_total) { hdl_fast_io_log_rate(); hdl_fast_io_log_stats(hdl_active_target_fd, 0); hdl_fast_log_copy_profile(); } +#endif return size; } /* Never fall back after PUMP_TO_EE already advanced the target. A * mismatched acknowledgement would otherwise write the block twice. */ hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; +#if HDL_PROFILE_ENABLED hdl_fast_copy_consumer_start_ticks = 0; +#endif return HDL_INSTALL_COPY_FAILED; } @@ -541,5 +579,7 @@ static HDL_FAST_NOINLINE int hdl_fast_fileXioWrite(int fd, return fileXioWrite(fd, buffer, size); } +#if HDL_PROFILE_ENABLED #undef HDL_FAST_COLD +#endif #undef HDL_FAST_NOINLINE diff --git a/tools/build_benchmark_provenance.sh b/tools/build_benchmark_provenance.sh old mode 100644 new mode 100755 index dc30e925..d0f699fe --- a/tools/build_benchmark_provenance.sh +++ b/tools/build_benchmark_provenance.sh @@ -11,6 +11,15 @@ PS2SDK_PATH_VALUE=${PS2SDK:-unavailable} PS2SDK_REF=${PS2SDK_SOURCE_REF:-unavailable} PS2SDK_SHA=${PS2SDK_SOURCE_SHA:-unavailable} PS2DEV_BUNDLE_REF=${PS2DEV_BUNDLE_REF:-unavailable} +HDL_PROFILE_VALUE=${HDL_PROFILE:-1} + +case "$HDL_PROFILE_VALUE" in + 0|1) ;; + *) + printf 'HDL_PROFILE must be 0 or 1, got %s\n' "$HDL_PROFILE_VALUE" >&2 + exit 2 + ;; +esac # A development environment may preserve the ps2sdk .git directory. Prefer the # exact installed checkout when available. Tagged ps2dev Docker images strip @@ -41,7 +50,8 @@ ps2sdk_commit: "$PS2SDK_SHA" toolchain_target: "$CC_TARGET" toolchain_gcc: "$CC_VERSION" toolchain_container: "ps2dev/ps2dev:v2.0.0" -build_flags: "-O2 -flto -G0 -fdata-sections -ffunction-sections; ld --gc-sections" +hdl_profile_enabled: "$HDL_PROFILE_VALUE" +build_flags: "-O2 -flto -G0 -fdata-sections -ffunction-sections -DHDL_PROFILE_ENABLED=$HDL_PROFILE_VALUE; ld --gc-sections" active_irx: UNRECORDED embedded_irx: "iomanX fileXio secrman freesio2 freepad mcman mcserv secrsif poweroff bdm bdmfs_fatfs usbd usbmass_bd ps2dev9 ata_bd ps2fs ps2hdd-bdm hdl_stream" workload: UNRECORDED From 04cf59deaef1075620ad36d7d54752fbdd9f8a82 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 20:09:19 +0200 Subject: [PATCH 050/156] CI: record PR head SHA in benchmark provenance --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9cee274..1c6925d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,8 +40,8 @@ jobs: - name: Build PROFILE ON/OFF pair with PS2DEV v2.0.0 run: >- docker run --rm - -e PROJECT_GIT_SHA="$GITHUB_SHA" - -e PROJECT_GIT_REF="$GITHUB_REF" + -e PROJECT_GIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}" + -e PROJECT_GIT_REF="${{ github.head_ref || github.ref }}" -e PS2DEV_BUNDLE_REF="v2.0.0" -e PS2SDK_SOURCE_REF="v2.0.0" -e PS2SDK_SOURCE_SHA="b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b" From 2a8dc2533313aa5a3315ca89070c4f783b4261b1 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 20:14:39 +0200 Subject: [PATCH 051/156] Phase 0: add hardware profile A/B comparator --- .github/workflows/ci.yml | 2 + docs/CORPUS_V2_IMPLEMENTATION_PLAN.md | 56 ++++-- docs/PHASE0_HARDWARE_AB_PROTOCOL.md | 32 ++++ tools/compare_hdl_profile_ab.py | 241 ++++++++++++++++++++++++++ 4 files changed, 314 insertions(+), 17 deletions(-) create mode 100755 tools/compare_hdl_profile_ab.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c6925d8..4c909f95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,8 @@ jobs: run: python3 tools/hardware_fault_injector.py selftest - name: Self-test HDL performance log parser run: python3 tools/parse_hdl_perf.py --selftest + - name: Self-test HDL PROFILE A/B comparator + run: python3 tools/compare_hdl_profile_ab.py --selftest - name: Enforce direct-fileXio runtime policy run: | python3 tools/check_filexio_fdman_policy.py --selftest diff --git a/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md b/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md index b7559a9d..6c133180 100644 --- a/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md +++ b/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md @@ -81,7 +81,7 @@ device sector/transfer unit, VIF/GIF packet, or another explicit contract. Goal: establish evidence before altering architecture. - [x] Record project SHA/ref, pinned PS2SDK source ref/SHA, toolchain identity and - build flags automatically in `BENCHMARK_PROVENANCE.yml`. + build flags automatically in benchmark provenance artifacts. - [ ] Record console SCPH/hardware revision, adapters, runtime active IRX and workload during a real-hardware benchmark. Build CI intentionally leaves those fields `UNRECORDED` rather than inferring them. @@ -97,7 +97,7 @@ Goal: establish evidence before altering architecture. - [x] Preserve linker map, symbol sizes and optimization audit in CI artifacts. - [x] Add a host parser that converts `HDDMAN.LOG` corpus-v2 performance records to stable JSON and self-tests in CI. -- [ ] Add a same-source profiling-on/profiling-off build pair for authoritative +- [x] Add a same-source profiling-on/profiling-off build pair for authoritative instrumentation-overhead A/B on real hardware. - [ ] Exercise the R5900 counter harness in a bounded hardware benchmark and measure empty-scope overhead before instrumenting application kernels. @@ -107,11 +107,11 @@ Goal: establish evidence before altering architecture. The first EE profiler pass increased `execute_transaction()` from the audited 6420 B baseline to 7492 B under LTO. This was rejected. Profiling helpers were then isolated with selective `noinline` and report paths with `cold,noinline`. -The instrumented transaction body became 6176 B, smaller than baseline, while -retaining the counters. This proves only static footprint, not runtime overhead. +The instrumented transaction body later became 6156 B after Phase-1 work while +retaining the counters. Static footprint alone is not a runtime-overhead claim. -The IOP path now records logarithmic microsecond latency histograms without -`printf` in the hot path. Current categories are: +The IOP path records logarithmic microsecond latency histograms without `printf` +in the hot path. Current categories are: ```text usb-direct-read @@ -138,21 +138,43 @@ out PS2SDK `v2.0.0`; that annotated tag resolves to commit `b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b`. CI records that exact source provenance rather than substituting current PS2SDK master. +The Phase-0 same-source A/B switch is `HDL_PROFILE=1/0`. PROFILE OFF compiles +EE/IOP latency timers, histograms, traffic counters and phase-end telemetry out +while preserving pump/prefetch, SIF DMA, EE cache maintenance, SHA verification, +journal, flush and metadata durability paths. CI #660 produced both variants +from the same head and pinned toolchain: + +```text + PROFILE ON PROFILE OFF delta OFF vs ON +ELF 638388 634420 -3968 B +.text 233280 230440 -2840 B +named text 232780 229956 -2824 B +named functions 618 609 -9 +instructions 58246 57539 -707 +execute_transaction() 6156 6156 0 +``` + +These are static deltas only. The authoritative overhead result remains the +interleaved real-console wall-time comparison defined in +`docs/PHASE0_HARDWARE_AB_PROTOCOL.md`. + Exit gate: measurements are reproducible on at least one real console and the -instrumented build has a documented overhead A/B against an uninstrumented build. +instrumented build has a documented overhead A/B against its same-source +PROFILE-OFF companion. ## Phase 1: remove known unnecessary code/work -- [ ] Replace the broad `draw2d` dependency used by the UI with the minimum GIF - primitives actually required, if the ELF A/B confirms removal of unused - arc/trigonometry/libm code. -- [ ] Audit formatted-I/O callsites and replace hot/control-only formatting with - bounded lightweight formatting where this materially reduces `.text`. -- [ ] Investigate current PS2SDK fileXio/newlib timestamp glue that pulls scanf/ - timezone machinery into the ELF. Treat any SDK change as a separate, - source-pinned compatibility patch. -- [ ] Remove source-level work duplicated across transaction stages when the - result can be safely retained under the same ownership/lifetime. +- [x] Evaluate replacing the broad `draw2d` dependency with minimum GIF + primitives. The tested shim was rejected because final `.text` grew by + 480 B; the existing dependency remains the smaller measured result. +- [x] Audit formatted-I/O callsites and replace the unused floating formatter + dependency with the bounded integer-only formatting contract guarded by CI. +- [x] Audit current PS2SDK fileXio/Newlib timestamp and timezone glue. Keep direct + fileXio application semantics, remove unused local-time bootstrap work, and + guard the source contract in CI. +- [x] Remove source-level work duplicated across transaction stages where the + result can safely remain under the same ownership/lifetime. The normal + metadata-commit path now reuses its canonical 1024-byte block for read-back. Exit gate: same functional output, smaller ELF/hot code footprint, no regression in hardware smoke tests. diff --git a/docs/PHASE0_HARDWARE_AB_PROTOCOL.md b/docs/PHASE0_HARDWARE_AB_PROTOCOL.md index 60367671..fb69e097 100644 --- a/docs/PHASE0_HARDWARE_AB_PROTOCOL.md +++ b/docs/PHASE0_HARDWARE_AB_PROTOCOL.md @@ -138,6 +138,38 @@ Run at least four complete comparable samples, preferably two in each half of an interleaved order, for an initial engineering answer. Add samples if wall time or PROFILE ON tail latency is unstable. +## Host comparison record + +Store the comparable run timings in a JSON array. Every sample must include: + +```json +{ + "mode": "OFF", + "project_git_sha": "", + "workload_id": "", + "correctness_hash": "", + "source_bytes": 0, + "total_us": 0, + "copy_us": 0, + "verify_us": 0 +} +``` + +`copy_us` and `verify_us` are optional if the external measurement setup cannot +isolate those phases. `source_bytes` and `total_us` are mandatory. Compare the +record with: + +```text +python3 tools/compare_hdl_profile_ab.py samples.json --output profile-ab.json +``` + +The comparator refuses mixed project SHAs, workloads, correctness hashes or +source sizes and requires four samples of each mode by default. It reports +p50/p95/p99/max for available wall-time metrics plus signed PROFILE ON vs OFF +percent deltas. Throughput is derived from the recorded bytes and time. Rich EE +and IOP latency distributions remain sourced from `parse_hdl_perf.py` for +PROFILE ON only. + ## Phase-0 acceptance Phase 0 may be marked hardware-complete only when: diff --git a/tools/compare_hdl_profile_ab.py b/tools/compare_hdl_profile_ab.py new file mode 100755 index 00000000..1e338d13 --- /dev/null +++ b/tools/compare_hdl_profile_ab.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""Compare same-source HDL PROFILE=0/1 real-hardware samples. + +Input is a JSON array (or an object with a ``samples`` array). Each sample must +record ``mode`` (``OFF``/``ON``), ``project_git_sha``, ``workload_id``, +``correctness_hash``, ``source_bytes`` and ``total_us``. ``copy_us`` and +``verify_us`` are optional, but a metric is compared only when every accepted +sample in both modes provides it. + +The tool intentionally does not invent PROFILE=0 latency distributions from the +PROFILE=1 telemetry log. ``parse_hdl_perf.py`` remains the source for the rich +PROFILE=1 EE/IOP histograms; this script quantifies the profiler's end-to-end +perturbation. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Any + +REQUIRED = ( + "mode", + "project_git_sha", + "workload_id", + "correctness_hash", + "source_bytes", + "total_us", +) +OPTIONAL_TIMES = ("copy_us", "verify_us") + + +def _percentile(values: list[int], percentile: int) -> int: + if not values: + raise ValueError("percentile requires at least one value") + ordered = sorted(values) + rank = max(1, math.ceil(len(ordered) * percentile / 100.0)) + return ordered[rank - 1] + + +def _distribution(values: list[int]) -> dict[str, int]: + return { + "samples": len(values), + "p50": _percentile(values, 50), + "p95": _percentile(values, 95), + "p99": _percentile(values, 99), + "max": max(values), + } + + +def _delta_percent(on_value: int, off_value: int) -> float: + if off_value == 0: + raise ValueError("PROFILE OFF baseline metric cannot be zero") + return round((on_value - off_value) * 100.0 / off_value, 4) + + +def _throughput_kib_s(source_bytes: int, usec: int) -> int: + if source_bytes <= 0 or usec <= 0: + raise ValueError("source_bytes and time must be positive") + return (source_bytes * 1_000_000) // (usec * 1024) + + +def _normalize(raw: Any) -> list[dict[str, Any]]: + if isinstance(raw, dict): + raw = raw.get("samples") + if not isinstance(raw, list) or not raw: + raise ValueError("input must contain a non-empty samples array") + + samples: list[dict[str, Any]] = [] + for index, sample in enumerate(raw): + if not isinstance(sample, dict): + raise ValueError(f"sample {index}: expected object") + missing = [key for key in REQUIRED if key not in sample] + if missing: + raise ValueError(f"sample {index}: missing {', '.join(missing)}") + mode = str(sample["mode"]).upper() + if mode not in ("OFF", "ON"): + raise ValueError(f"sample {index}: mode must be OFF or ON") + normalized = dict(sample) + normalized["mode"] = mode + for key in ("source_bytes", "total_us", *OPTIONAL_TIMES): + if key in normalized: + value = normalized[key] + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise ValueError(f"sample {index}: {key} must be a positive integer") + for key in ("project_git_sha", "workload_id", "correctness_hash"): + if not isinstance(normalized[key], str) or not normalized[key]: + raise ValueError(f"sample {index}: {key} must be a non-empty string") + samples.append(normalized) + return samples + + +def compare_samples(samples: list[dict[str, Any]], min_samples: int = 4) -> dict[str, Any]: + groups = {mode: [sample for sample in samples if sample["mode"] == mode] + for mode in ("OFF", "ON")} + for mode, group in groups.items(): + if len(group) < min_samples: + raise ValueError( + f"PROFILE {mode} has {len(group)} samples; need at least {min_samples}" + ) + + shas = {sample["project_git_sha"] for sample in samples} + workloads = {sample["workload_id"] for sample in samples} + hashes = {sample["correctness_hash"] for sample in samples} + source_sizes = {sample["source_bytes"] for sample in samples} + if len(shas) != 1: + raise ValueError("samples do not share one project_git_sha") + if len(workloads) != 1: + raise ValueError("samples do not share one workload_id") + if len(hashes) != 1: + raise ValueError("correctness_hash differs across samples") + if len(source_sizes) != 1: + raise ValueError("source_bytes differs across samples") + + source_bytes = next(iter(source_sizes)) + result: dict[str, Any] = { + "project_git_sha": next(iter(shas)), + "workload_id": next(iter(workloads)), + "correctness_hash": next(iter(hashes)), + "source_bytes": source_bytes, + "sample_counts": {mode: len(groups[mode]) for mode in ("OFF", "ON")}, + "metrics": {}, + } + + metric_sources: dict[str, str] = {"total_us": "total_us"} + for key in OPTIONAL_TIMES: + if all(key in sample for sample in samples): + metric_sources[key] = key + + metrics: dict[str, Any] = result["metrics"] + for label, key in metric_sources.items(): + off_values = [int(sample[key]) for sample in groups["OFF"]] + on_values = [int(sample[key]) for sample in groups["ON"]] + off_dist = _distribution(off_values) + on_dist = _distribution(on_values) + metrics[label] = { + "off": off_dist, + "on": on_dist, + "on_vs_off_percent": { + percentile: _delta_percent(on_dist[percentile], off_dist[percentile]) + for percentile in ("p50", "p95", "p99", "max") + }, + } + + for timing_key, rate_label in ( + ("total_us", "end_to_end_kib_per_second"), + ("copy_us", "copy_kib_per_second"), + ): + if timing_key not in metric_sources: + continue + off_rates = [_throughput_kib_s(source_bytes, int(sample[timing_key])) + for sample in groups["OFF"]] + on_rates = [_throughput_kib_s(source_bytes, int(sample[timing_key])) + for sample in groups["ON"]] + off_dist = _distribution(off_rates) + on_dist = _distribution(on_rates) + metrics[rate_label] = { + "off": off_dist, + "on": on_dist, + "on_vs_off_percent": { + percentile: _delta_percent(on_dist[percentile], off_dist[percentile]) + for percentile in ("p50", "p95", "p99", "max") + }, + } + + return result + + +def selftest() -> None: + samples: list[dict[str, Any]] = [] + for mode, totals, copies, verifies in ( + ("OFF", [1000, 1010, 990, 1005], [700, 705, 695, 700], [200, 205, 195, 200]), + ("ON", [1020, 1030, 1010, 1025], [714, 719, 709, 714], [204, 209, 199, 204]), + ): + for total, copy, verify in zip(totals, copies, verifies): + samples.append({ + "mode": mode, + "project_git_sha": "abc123", + "workload_id": "fixture-iso-a", + "correctness_hash": "deadbeef", + "source_bytes": 1024 * 1024, + "total_us": total, + "copy_us": copy, + "verify_us": verify, + }) + result = compare_samples(samples) + assert result["sample_counts"] == {"OFF": 4, "ON": 4} + assert result["metrics"]["total_us"]["off"]["p50"] == 1000 + assert result["metrics"]["total_us"]["on"]["p50"] == 1020 + assert result["metrics"]["total_us"]["on_vs_off_percent"]["p50"] == 2.0 + assert result["metrics"]["copy_us"]["on_vs_off_percent"]["p50"] == 2.0 + assert result["metrics"]["end_to_end_kib_per_second"]["off"]["samples"] == 4 + + broken = [dict(sample) for sample in samples] + broken[-1]["project_git_sha"] = "different" + try: + compare_samples(broken) + except ValueError as error: + assert "project_git_sha" in str(error) + else: + raise AssertionError("mismatched SHA must fail") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("samples", nargs="?", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--min-samples", type=int, default=4) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("compare_hdl_profile_ab selftest: PASS") + return 0 + if args.samples is None: + parser.error("samples is required unless --selftest is used") + if args.min_samples < 1: + parser.error("--min-samples must be positive") + + try: + raw = json.loads(args.samples.read_text(encoding="utf-8")) + samples = _normalize(raw) + compared = compare_samples(samples, args.min_samples) + except (OSError, json.JSONDecodeError, ValueError) as error: + print(f"compare_hdl_profile_ab: {error}", file=sys.stderr) + return 2 + + rendered = json.dumps(compared, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(rendered, encoding="utf-8") + else: + sys.stdout.write(rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 16a6356031a5f16d866dae58c0c25c254eb09a59 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:18:19 +0200 Subject: [PATCH 052/156] Phase 0: isolate IOP objects by profiler mode --- iop/hdl_stream/Makefile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/iop/hdl_stream/Makefile b/iop/hdl_stream/Makefile index 3823993f..71e0fd1c 100644 --- a/iop/hdl_stream/Makefile +++ b/iop/hdl_stream/Makefile @@ -6,6 +6,12 @@ ifeq ($(filter $(HDL_PROFILE),0 1),) $(error HDL_PROFILE must be 0 or 1) endif +# Make the profiler mode part of the dependency graph. PS2SDK's generic IOP +# rules place objects under IOP_OBJS_DIR, so keeping PROFILE=0/1 in separate +# object trees prevents a stale object built with the other preprocessor mode +# from being silently reused when only HDL_PROFILE changes. +IOP_OBJS_DIR = obj/profile-$(HDL_PROFILE)/ + # PS2SDK's generic IOP rules default to -Os. hdl_stream is not a resident-size # utility in our workload: it is the 37.5 MHz R3000A hot path that must keep # USB, DEV9 and SIF fed during multi-gigabyte ISO installs. Put -O2 after the @@ -21,7 +27,8 @@ IOP_INCS += -I../../include all: $(IOP_BIN) clean: - rm -f -r $(IOP_OBJS_DIR) $(IOP_BIN) \ + rm -rf obj + rm -f $(IOP_BIN) \ $(IOP_BIN:.irx=.notiopmod.elf) \ $(IOP_BIN:.irx=.notiopmod.stripped.elf) From 61049fac54d05b4e383c947b9225a694be6ef0dc Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:18:41 +0200 Subject: [PATCH 053/156] Phase 0: validate distinct PROFILE IOP binaries --- .github/workflows/ci.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c909f95..f1149475 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,7 @@ jobs: HDL_PROFILE=1 sh tools/build_benchmark_provenance.sh BENCHMARK_PROVENANCE_PROFILE_ON.yml && make clean && make HDL_PROFILE=1 && + cp hdl_stream.irx HDL_STREAM_PROFILE_ON.irx && python3 tools/optimization_audit.py --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF --output OPTIMIZATION_AUDIT_PROFILE_ON.txt && @@ -68,6 +69,7 @@ jobs: HDL_PROFILE=0 sh tools/build_benchmark_provenance.sh BENCHMARK_PROVENANCE_PROFILE_OFF.yml && make clean && make HDL_PROFILE=0 && + cp hdl_stream.irx HDL_STREAM_PROFILE_OFF.irx && python3 tools/optimization_audit.py --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF --output OPTIMIZATION_AUDIT_PROFILE_OFF.txt && @@ -80,8 +82,14 @@ jobs: PS2_HDD_BOOTSTRAP_MANAGER.ELF && cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.map PS2_HDD_BOOTSTRAP_MANAGER.map' - - name: Record artifact checksums + - name: Validate PROFILE split and record checksums run: | + if cmp -s HDL_STREAM_PROFILE_ON.irx HDL_STREAM_PROFILE_OFF.irx; then + echo "::error::PROFILE ON/OFF embedded hdl_stream IRX files are byte-identical" + exit 1 + fi + sha256sum HDL_STREAM_PROFILE_ON.irx HDL_STREAM_PROFILE_OFF.irx | tee HDL_STREAM_PROFILE_PAIR.sha256 + wc -c HDL_STREAM_PROFILE_ON.irx HDL_STREAM_PROFILE_OFF.irx | tee HDL_STREAM_PROFILE_PAIR.sizes sha256sum PS2_HDD_BOOTSTRAP_MANAGER.ELF | tee PS2_HDD_BOOTSTRAP_MANAGER.ELF.sha256 sha256sum PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF | tee HDL_PROFILE_PAIR.sha256 - uses: actions/upload-artifact@v4 @@ -96,6 +104,10 @@ jobs: PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.map PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.map HDL_PROFILE_PAIR.sha256 + HDL_STREAM_PROFILE_ON.irx + HDL_STREAM_PROFILE_OFF.irx + HDL_STREAM_PROFILE_PAIR.sha256 + HDL_STREAM_PROFILE_PAIR.sizes OPTIMIZATION_AUDIT_PROFILE_ON.txt OPTIMIZATION_AUDIT_PROFILE_OFF.txt CORPUS_V2_PROJECT_AUDIT.txt From 16f96145b7d97f765b05611eea288335838dab71 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:20:25 +0200 Subject: [PATCH 054/156] Phase 0: report EE and IOP profile footprint --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1149475..c9fd0eb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,8 @@ jobs: PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF && cp PS2_HDD_BOOTSTRAP_MANAGER.map PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.map && + mipsel-none-elf-size HDL_STREAM_PROFILE_ON.irx + HDL_STREAM_PROFILE_OFF.irx > HDL_STREAM_PROFILE_PAIR.sections && cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF PS2_HDD_BOOTSTRAP_MANAGER.ELF && cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.map @@ -90,6 +92,11 @@ jobs: fi sha256sum HDL_STREAM_PROFILE_ON.irx HDL_STREAM_PROFILE_OFF.irx | tee HDL_STREAM_PROFILE_PAIR.sha256 wc -c HDL_STREAM_PROFILE_ON.irx HDL_STREAM_PROFILE_OFF.irx | tee HDL_STREAM_PROFILE_PAIR.sizes + cat HDL_STREAM_PROFILE_PAIR.sections + echo "--- PROFILE ON optimization audit ---" + cat OPTIMIZATION_AUDIT_PROFILE_ON.txt + echo "--- PROFILE OFF optimization audit ---" + cat OPTIMIZATION_AUDIT_PROFILE_OFF.txt sha256sum PS2_HDD_BOOTSTRAP_MANAGER.ELF | tee PS2_HDD_BOOTSTRAP_MANAGER.ELF.sha256 sha256sum PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF | tee HDL_PROFILE_PAIR.sha256 - uses: actions/upload-artifact@v4 @@ -108,6 +115,7 @@ jobs: HDL_STREAM_PROFILE_OFF.irx HDL_STREAM_PROFILE_PAIR.sha256 HDL_STREAM_PROFILE_PAIR.sizes + HDL_STREAM_PROFILE_PAIR.sections OPTIMIZATION_AUDIT_PROFILE_ON.txt OPTIMIZATION_AUDIT_PROFILE_OFF.txt CORPUS_V2_PROJECT_AUDIT.txt From 2e0f2cf269864ff693515c4689d73e3183ccdf21 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:21:34 +0200 Subject: [PATCH 055/156] Phase 0: bind benchmark provenance to built binaries --- tools/build_benchmark_provenance.sh | 33 ++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tools/build_benchmark_provenance.sh b/tools/build_benchmark_provenance.sh index d0f699fe..de50d2b2 100755 --- a/tools/build_benchmark_provenance.sh +++ b/tools/build_benchmark_provenance.sh @@ -12,6 +12,8 @@ PS2SDK_REF=${PS2SDK_SOURCE_REF:-unavailable} PS2SDK_SHA=${PS2SDK_SOURCE_SHA:-unavailable} PS2DEV_BUNDLE_REF=${PS2DEV_BUNDLE_REF:-unavailable} HDL_PROFILE_VALUE=${HDL_PROFILE:-1} +BENCHMARK_ELF_PATH=${BENCHMARK_ELF:-PS2_HDD_BOOTSTRAP_MANAGER.ELF} +HDL_STREAM_IRX_PATH=${HDL_STREAM_IRX:-hdl_stream.irx} case "$HDL_PROFILE_VALUE" in 0|1) ;; @@ -31,6 +33,29 @@ if [ "${PS2SDK:-}" != "" ] && git -C "$PS2SDK" rev-parse HEAD >/dev/null 2>&1; t PS2SDK_REF=$(git -C "$PS2SDK" describe --always --tags 2>/dev/null || printf '%s' "$PS2SDK_REF") fi +file_sha256() +{ + if [ -f "$1" ]; then + sha256sum "$1" | awk '{print $1}' + else + printf 'UNRECORDED' + fi +} + +file_bytes() +{ + if [ -f "$1" ]; then + wc -c < "$1" | tr -d ' ' + else + printf 'UNRECORDED' + fi +} + +BENCHMARK_ELF_SHA=$(file_sha256 "$BENCHMARK_ELF_PATH") +BENCHMARK_ELF_BYTES=$(file_bytes "$BENCHMARK_ELF_PATH") +HDL_STREAM_IRX_SHA=$(file_sha256 "$HDL_STREAM_IRX_PATH") +HDL_STREAM_IRX_BYTES=$(file_bytes "$HDL_STREAM_IRX_PATH") + cat > "$OUT" < Date: Wed, 26 Aug 2026 21:21:54 +0200 Subject: [PATCH 056/156] Phase 0: generate provenance from final profile binaries --- .github/workflows/ci.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9fd0eb2..01a86c52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,26 +54,26 @@ jobs: --output CORPUS_V2_PROJECT_AUDIT.txt && python3 tools/printf_format_audit.py --output PRINTF_FORMAT_AUDIT.txt && - HDL_PROFILE=1 sh tools/build_benchmark_provenance.sh - BENCHMARK_PROVENANCE_PROFILE_ON.yml && make clean && make HDL_PROFILE=1 && cp hdl_stream.irx HDL_STREAM_PROFILE_ON.irx && python3 tools/optimization_audit.py --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF --output OPTIMIZATION_AUDIT_PROFILE_ON.txt && make HDL_PROFILE=1 release && + HDL_PROFILE=1 sh tools/build_benchmark_provenance.sh + BENCHMARK_PROVENANCE_PROFILE_ON.yml && cp PS2_HDD_BOOTSTRAP_MANAGER.ELF PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF && cp PS2_HDD_BOOTSTRAP_MANAGER.map PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.map && - HDL_PROFILE=0 sh tools/build_benchmark_provenance.sh - BENCHMARK_PROVENANCE_PROFILE_OFF.yml && make clean && make HDL_PROFILE=0 && cp hdl_stream.irx HDL_STREAM_PROFILE_OFF.irx && python3 tools/optimization_audit.py --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF --output OPTIMIZATION_AUDIT_PROFILE_OFF.txt && make HDL_PROFILE=0 release && + HDL_PROFILE=0 sh tools/build_benchmark_provenance.sh + BENCHMARK_PROVENANCE_PROFILE_OFF.yml && cp PS2_HDD_BOOTSTRAP_MANAGER.ELF PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF && cp PS2_HDD_BOOTSTRAP_MANAGER.map @@ -93,6 +93,10 @@ jobs: sha256sum HDL_STREAM_PROFILE_ON.irx HDL_STREAM_PROFILE_OFF.irx | tee HDL_STREAM_PROFILE_PAIR.sha256 wc -c HDL_STREAM_PROFILE_ON.irx HDL_STREAM_PROFILE_OFF.irx | tee HDL_STREAM_PROFILE_PAIR.sizes cat HDL_STREAM_PROFILE_PAIR.sections + echo "--- PROFILE ON provenance ---" + cat BENCHMARK_PROVENANCE_PROFILE_ON.yml + echo "--- PROFILE OFF provenance ---" + cat BENCHMARK_PROVENANCE_PROFILE_OFF.yml echo "--- PROFILE ON optimization audit ---" cat OPTIMIZATION_AUDIT_PROFILE_ON.txt echo "--- PROFILE OFF optimization audit ---" From 1386fe1a3ddc6a4e3abb5bccc4510468a48f02e1 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:24:32 +0200 Subject: [PATCH 057/156] Phase 0: freeze validated hardware A/B pair --- docs/PHASE0_HARDWARE_AB_PROTOCOL.md | 87 +++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 16 deletions(-) diff --git a/docs/PHASE0_HARDWARE_AB_PROTOCOL.md b/docs/PHASE0_HARDWARE_AB_PROTOCOL.md index fb69e097..e38018bc 100644 --- a/docs/PHASE0_HARDWARE_AB_PROTOCOL.md +++ b/docs/PHASE0_HARDWARE_AB_PROTOCOL.md @@ -5,6 +5,53 @@ real PlayStation 2. PCSX2 may be used for correctness/debugging but is not an arbiter for EE cache, IOP scheduling, USB service latency, SIF DMA or DEV9 throughput. +## Frozen hardware-test pair + +Until a newer green pair is explicitly recorded here, use the artifacts from CI +run #666, project head: + +```text +project_git_sha 7875b14d837d6332f5edc37f1c12a55527d7dd87 +project_git_ref perf/corpus-v2-integration +ps2sdk_commit b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b +toolchain mips64r5900el-ps2-elf GCC 15.2.0 +container ps2dev/ps2dev:v2.0.0 +``` + +PROFILE ON: + +```text +ELF bytes 638388 +ELF SHA-256 964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7 +IRX bytes 9861 +IRX text 8595 +IRX data 144 +IRX SHA-256 8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db +``` + +PROFILE OFF: + +```text +ELF bytes 632884 +ELF SHA-256 4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916 +IRX bytes 8405 +IRX text 7139 +IRX data 144 +IRX SHA-256 f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 +``` + +The static footprint delta is evidence that the compiled-out path is real; it is +**not** a runtime speedup measurement. CI also enforces that the ON/OFF +`hdl_stream.irx` files are not byte-identical. + +An earlier pair through CI #661 is invalid for IOP profiler-overhead measurement: +PS2SDK's IOP rules place objects under `obj/`, while the old top-level clean did +not remove that directory. PROFILE OFF therefore reused the previously compiled +PROFILE ON `hdl_stream.o`. The corrected module Makefile makes the profile mode +part of the dependency graph with `obj/profile-0/` and `obj/profile-1/`, and its +own clean removes the entire object tree. Do not use the old OFF ELF/IRX as an +authoritative profiler-overhead sample. + ## Compared builds The authoritative instrumentation-overhead comparison is a same-source pair @@ -48,9 +95,10 @@ This is the diagnostic build and retains: - benchmark provenance artifact; - linker/ELF audit artifacts. -CI emits both ELFs, linker maps, optimization audits and provenance records from -one checkout and one toolchain image. The profile mode is written explicitly to -each provenance file. +CI emits both ELFs, both embedded `hdl_stream.irx` variants, linker maps, +optimization audits, binary hashes and provenance records from one checkout and +one toolchain image. The profile mode and the final ELF/IRX hashes are written +explicitly to each provenance file. ### Historical pre-instrumentation reference @@ -74,6 +122,8 @@ toolchain: active_irx: build_flags: hdl_profile_enabled: +benchmark_elf_sha256: +hdl_stream_irx_sha256: workload: direction: buffering: @@ -84,8 +134,12 @@ correctness_hash: ``` The CI-generated `BENCHMARK_PROVENANCE_PROFILE_OFF.yml` and -`BENCHMARK_PROVENANCE_PROFILE_ON.yml` supply build-side fields. Hardware fields -remain explicit manual measurements rather than guessed metadata. +`BENCHMARK_PROVENANCE_PROFILE_ON.yml` supply build-side fields, including exact +ELF/IRX hashes and sizes. Hardware fields remain explicit manual measurements +rather than guessed metadata. + +Before timing, verify the ELF SHA-256 on the medium used to launch each build. +A filename such as `PROFILE_ON.ELF` is not provenance. ## Workload contract @@ -134,9 +188,9 @@ For the overhead result report PROFILE ON relative to PROFILE OFF for wall time and throughput. Do not fabricate p95/p99 for PROFILE OFF from absent telemetry; the point of the OFF build is to remove that instrumentation. -Run at least four complete comparable samples, preferably two in each half of an -interleaved order, for an initial engineering answer. Add samples if wall time -or PROFILE ON tail latency is unstable. +Run at least four complete comparable samples per mode, preferably distributed +through an interleaved order. Add samples if wall time or PROFILE ON tail latency +is unstable. ## Host comparison record @@ -145,7 +199,7 @@ Store the comparable run timings in a JSON array. Every sample must include: ```json { "mode": "OFF", - "project_git_sha": "", + "project_git_sha": "7875b14d837d6332f5edc37f1c12a55527d7dd87", "workload_id": "", "correctness_hash": "", "source_bytes": 0, @@ -174,16 +228,17 @@ PROFILE ON only. Phase 0 may be marked hardware-complete only when: -1. PROFILE OFF and PROFILE ON are from the same project SHA and pass the same - correctness workload; -2. PROFILE ON produces internally consistent stage counters and traffic +1. PROFILE OFF and PROFILE ON are from the same project SHA and their recorded + ELF/IRX hashes match the selected CI pair; +2. both builds pass the same correctness workload; +3. PROFILE ON produces internally consistent stage counters and traffic accounting; -3. the measurement overhead of PROFILE ON is quantified rather than assumed +4. the measurement overhead of PROFILE ON is quantified rather than assumed negligible; -4. PROFILE ON retains p50/p95/p99/max instead of replacing distributions with +5. PROFILE ON retains p50/p95/p99/max instead of replacing distributions with an average; -5. the result includes console/toolchain/IRX/workload provenance; -6. R5900 counter calibration is checked on the real EE before counter-derived +6. the result includes console/toolchain/IRX/workload provenance; +7. R5900 counter calibration is checked on the real EE before counter-derived optimization claims are made. If instrumentation materially changes throughput or tail latency, keep From 165f6ae659cdf8886347efaf2b4cbe9a45650338 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:26:06 +0200 Subject: [PATCH 058/156] Phase 0: add hardware pair preflight verifier --- tools/phase0_profile_pair_preflight.py | 154 +++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 tools/phase0_profile_pair_preflight.py diff --git a/tools/phase0_profile_pair_preflight.py b/tools/phase0_profile_pair_preflight.py new file mode 100644 index 00000000..53d0eb6e --- /dev/null +++ b/tools/phase0_profile_pair_preflight.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Verify the frozen Phase-0 PROFILE ON/OFF ELF pair before hardware testing. + +This is deliberately a host-side guard. It does not instrument or modify the +benchmark binaries. The frozen pair is the CI #666 same-source build documented +in docs/PHASE0_HARDWARE_AB_PROTOCOL.md. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import tempfile +from pathlib import Path + +PROJECT_GIT_SHA = "7875b14d837d6332f5edc37f1c12a55527d7dd87" +PAIR = { + "ON": { + "bytes": 638388, + "sha256": "964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7", + }, + "OFF": { + "bytes": 632884, + "sha256": "4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916", + }, +} + +# Four samples per mode, distributed so neither build owns one contiguous block. +DEFAULT_ORDER = ("OFF", "ON", "ON", "OFF", "ON", "OFF", "OFF", "ON") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_blob(path: Path, expected_bytes: int, expected_sha256: str) -> tuple[int, str]: + size = path.stat().st_size + digest = _sha256(path) + if size != expected_bytes: + raise ValueError(f"{path}: size {size}, expected {expected_bytes}") + if digest != expected_sha256: + raise ValueError(f"{path}: sha256 {digest}, expected {expected_sha256}") + return size, digest + + +def sample_template() -> dict[str, object]: + samples: list[dict[str, object]] = [] + for index, mode in enumerate(DEFAULT_ORDER, start=1): + samples.append({ + "run": index, + "mode": mode, + "project_git_sha": PROJECT_GIT_SHA, + "workload_id": "FILL_ME", + "correctness_hash": "FILL_ME", + "source_bytes": 0, + "total_us": 0, + "copy_us": 0, + "verify_us": 0, + }) + return { + "note": ( + "Replace every FILL_ME/zero measurement before comparison. " + "Keep source_bytes and correctness_hash identical across comparable runs." + ), + "samples": samples, + } + + +def selftest() -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + payload = b"phase0-preflight-fixture\n" + fixture = root / "fixture.bin" + fixture.write_bytes(payload) + digest = hashlib.sha256(payload).hexdigest() + size, observed = verify_blob(fixture, len(payload), digest) + assert size == len(payload) + assert observed == digest + + try: + verify_blob(fixture, len(payload) + 1, digest) + except ValueError as error: + assert "size" in str(error) + else: + raise AssertionError("size mismatch must fail") + + try: + verify_blob(fixture, len(payload), "0" * 64) + except ValueError as error: + assert "sha256" in str(error) + else: + raise AssertionError("hash mismatch must fail") + + template = sample_template() + samples = template["samples"] + assert isinstance(samples, list) + assert len(samples) == 8 + assert sum(sample["mode"] == "ON" for sample in samples) == 4 + assert sum(sample["mode"] == "OFF" for sample in samples) == 4 + assert all(sample["project_git_sha"] == PROJECT_GIT_SHA for sample in samples) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--profile-on", + type=Path, + default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF"), + ) + parser.add_argument( + "--profile-off", + type=Path, + default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF"), + ) + parser.add_argument("--output-template", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("phase0_profile_pair_preflight selftest: PASS") + return 0 + + try: + on_size, on_hash = verify_blob( + args.profile_on, int(PAIR["ON"]["bytes"]), str(PAIR["ON"]["sha256"]) + ) + off_size, off_hash = verify_blob( + args.profile_off, int(PAIR["OFF"]["bytes"]), str(PAIR["OFF"]["sha256"]) + ) + except (OSError, ValueError) as error: + print(f"phase0_profile_pair_preflight: {error}", file=sys.stderr) + return 2 + + print(f"PROFILE ON PASS {on_size} B {on_hash}") + print(f"PROFILE OFF PASS {off_size} B {off_hash}") + print(f"project_git_sha {PROJECT_GIT_SHA}") + + if args.output_template: + rendered = json.dumps(sample_template(), indent=2) + "\n" + args.output_template.write_text(rendered, encoding="utf-8") + print(f"sample template {args.output_template}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 4b15ba5e5bdbc6e75a96dc49ee0fa455f6bf6dde Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:26:38 +0200 Subject: [PATCH 059/156] Phase 0: gate frozen pair with host preflight --- .github/workflows/ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01a86c52..4fec1572 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,8 @@ jobs: run: python3 tools/parse_hdl_perf.py --selftest - name: Self-test HDL PROFILE A/B comparator run: python3 tools/compare_hdl_profile_ab.py --selftest + - name: Self-test Phase-0 frozen pair preflight + run: python3 tools/phase0_profile_pair_preflight.py --selftest - name: Enforce direct-fileXio runtime policy run: | python3 tools/check_filexio_fdman_policy.py --selftest @@ -93,6 +95,10 @@ jobs: sha256sum HDL_STREAM_PROFILE_ON.irx HDL_STREAM_PROFILE_OFF.irx | tee HDL_STREAM_PROFILE_PAIR.sha256 wc -c HDL_STREAM_PROFILE_ON.irx HDL_STREAM_PROFILE_OFF.irx | tee HDL_STREAM_PROFILE_PAIR.sizes cat HDL_STREAM_PROFILE_PAIR.sections + python3 tools/phase0_profile_pair_preflight.py \ + --profile-on PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF \ + --profile-off PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF \ + --output-template PHASE0_AB_SAMPLES_TEMPLATE.json echo "--- PROFILE ON provenance ---" cat BENCHMARK_PROVENANCE_PROFILE_ON.yml echo "--- PROFILE OFF provenance ---" @@ -120,6 +126,7 @@ jobs: HDL_STREAM_PROFILE_PAIR.sha256 HDL_STREAM_PROFILE_PAIR.sizes HDL_STREAM_PROFILE_PAIR.sections + PHASE0_AB_SAMPLES_TEMPLATE.json OPTIMIZATION_AUDIT_PROFILE_ON.txt OPTIMIZATION_AUDIT_PROFILE_OFF.txt CORPUS_V2_PROJECT_AUDIT.txt From 48c43a67c3c5de55e0da8533ef70cc3c3d4e0890 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:27:27 +0200 Subject: [PATCH 060/156] Phase 0: document corrected frozen profile pair --- docs/CORPUS_V2_IMPLEMENTATION_PLAN.md | 47 ++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md b/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md index 6c133180..ec5dd4eb 100644 --- a/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md +++ b/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md @@ -99,6 +99,12 @@ Goal: establish evidence before altering architecture. to stable JSON and self-tests in CI. - [x] Add a same-source profiling-on/profiling-off build pair for authoritative instrumentation-overhead A/B on real hardware. +- [x] Isolate IOP objects by profile mode and reject byte-identical ON/OFF IRX + artifacts in CI. +- [x] Bind each benchmark provenance record to the final stripped ELF and + embedded `hdl_stream.irx` SHA-256 and byte size. +- [x] Add a host preflight verifier for the frozen hardware pair and emit an + interleaved eight-run sample template. - [ ] Exercise the R5900 counter harness in a bounded hardware benchmark and measure empty-scope overhead before instrumenting application kernels. @@ -141,22 +147,47 @@ provenance rather than substituting current PS2SDK master. The Phase-0 same-source A/B switch is `HDL_PROFILE=1/0`. PROFILE OFF compiles EE/IOP latency timers, histograms, traffic counters and phase-end telemetry out while preserving pump/prefetch, SIF DMA, EE cache maintenance, SHA verification, -journal, flush and metadata durability paths. CI #660 produced both variants -from the same head and pinned toolchain: +journal, flush and metadata durability paths. + +An early A/B pair through CI #661 was invalid for IOP profiler-overhead work. +PS2SDK's IOP build rules store objects in `obj/`, while the old top-level clean +did not remove that directory. PROFILE OFF therefore reused the PROFILE ON +`hdl_stream.o`, producing a byte-identical embedded IRX. The corrected module +build makes the profile mode part of the object path as `obj/profile-0/` and +`obj/profile-1/`, removes the object tree in the module clean target, and CI now +fails if the two IRX files compare equal. + +The frozen hardware pair is CI #666 at project commit +`7875b14d837d6332f5edc37f1c12a55527d7dd87` with PS2SDK +`b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b` and GCC 15.2.0: ```text PROFILE ON PROFILE OFF delta OFF vs ON -ELF 638388 634420 -3968 B -.text 233280 230440 -2840 B -named text 232780 229956 -2824 B -named functions 618 609 -9 -instructions 58246 57539 -707 +stripped ELF 638388 632884 -5504 B +EE named text 232780 229956 -2824 B +EE named functions 618 609 -9 +EE instructions 58246 57539 -707 execute_transaction() 6156 6156 0 +execute_transaction insn 1540 1540 0 +hdl_stream.irx file 9861 8405 -1456 B +hdl_stream.irx .text 8595 7139 -1456 B +hdl_stream.irx .data 144 144 0 +``` + +Frozen hashes: + +```text +PROFILE ON ELF 964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7 +PROFILE OFF ELF 4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916 +PROFILE ON IRX 8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db +PROFILE OFF IRX f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 ``` These are static deltas only. The authoritative overhead result remains the interleaved real-console wall-time comparison defined in -`docs/PHASE0_HARDWARE_AB_PROTOCOL.md`. +`docs/PHASE0_HARDWARE_AB_PROTOCOL.md`. Before a hardware run, +`tools/phase0_profile_pair_preflight.py` verifies that the selected ELF files +match the frozen pair and can emit the eight-run sample template. Exit gate: measurements are reproducible on at least one real console and the instrumented build has a documented overhead A/B against its same-source From 24c0fd85439add1129e41d920622d77ea835e628 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:30:06 +0200 Subject: [PATCH 061/156] Phase 0: separate artifact SHA from frozen binary identity --- tools/phase0_profile_pair_preflight.py | 45 ++++++++++++++++++++------ 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/tools/phase0_profile_pair_preflight.py b/tools/phase0_profile_pair_preflight.py index 53d0eb6e..29419cca 100644 --- a/tools/phase0_profile_pair_preflight.py +++ b/tools/phase0_profile_pair_preflight.py @@ -2,8 +2,10 @@ """Verify the frozen Phase-0 PROFILE ON/OFF ELF pair before hardware testing. This is deliberately a host-side guard. It does not instrument or modify the -benchmark binaries. The frozen pair is the CI #666 same-source build documented -in docs/PHASE0_HARDWARE_AB_PROTOCOL.md. +benchmark binaries. Frozen identity is the exact ON/OFF ELF hash pair from CI +#666; later documentation/host-tool commits may rebuild byte-identical ELFs from +a newer repository head, so the run record keeps that artifact's project SHA +separate from the binary identity. """ from __future__ import annotations @@ -15,15 +17,17 @@ import tempfile from pathlib import Path -PROJECT_GIT_SHA = "7875b14d837d6332f5edc37f1c12a55527d7dd87" +FROZEN_SOURCE_GIT_SHA = "7875b14d837d6332f5edc37f1c12a55527d7dd87" PAIR = { "ON": { "bytes": 638388, "sha256": "964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7", + "irx_sha256": "8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db", }, "OFF": { "bytes": 632884, "sha256": "4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916", + "irx_sha256": "f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751", }, } @@ -49,13 +53,18 @@ def verify_blob(path: Path, expected_bytes: int, expected_sha256: str) -> tuple[ return size, digest -def sample_template() -> dict[str, object]: +def sample_template(project_git_sha: str = FROZEN_SOURCE_GIT_SHA) -> dict[str, object]: + if not project_git_sha: + raise ValueError("project_git_sha must be non-empty") + samples: list[dict[str, object]] = [] for index, mode in enumerate(DEFAULT_ORDER, start=1): samples.append({ "run": index, "mode": mode, - "project_git_sha": PROJECT_GIT_SHA, + "project_git_sha": project_git_sha, + "benchmark_elf_sha256": PAIR[mode]["sha256"], + "hdl_stream_irx_sha256": PAIR[mode]["irx_sha256"], "workload_id": "FILL_ME", "correctness_hash": "FILL_ME", "source_bytes": 0, @@ -64,9 +73,11 @@ def sample_template() -> dict[str, object]: "verify_us": 0, }) return { + "frozen_binary_source_git_sha": FROZEN_SOURCE_GIT_SHA, "note": ( "Replace every FILL_ME/zero measurement before comparison. " - "Keep source_bytes and correctness_hash identical across comparable runs." + "Keep source_bytes and correctness_hash identical across comparable runs. " + "Do not edit the mode-specific ELF/IRX hashes." ), "samples": samples, } @@ -97,13 +108,18 @@ def selftest() -> None: else: raise AssertionError("hash mismatch must fail") - template = sample_template() + artifact_sha = "artifact-head-fixture" + template = sample_template(artifact_sha) samples = template["samples"] assert isinstance(samples, list) assert len(samples) == 8 assert sum(sample["mode"] == "ON" for sample in samples) == 4 assert sum(sample["mode"] == "OFF" for sample in samples) == 4 - assert all(sample["project_git_sha"] == PROJECT_GIT_SHA for sample in samples) + assert all(sample["project_git_sha"] == artifact_sha for sample in samples) + assert all(sample["benchmark_elf_sha256"] == PAIR[str(sample["mode"])]["sha256"] + for sample in samples) + assert all(sample["hdl_stream_irx_sha256"] == PAIR[str(sample["mode"])]["irx_sha256"] + for sample in samples) def main() -> int: @@ -118,6 +134,11 @@ def main() -> int: type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF"), ) + parser.add_argument( + "--project-git-sha", + default=FROZEN_SOURCE_GIT_SHA, + help="project SHA recorded by the CI artifact being tested", + ) parser.add_argument("--output-template", type=Path) parser.add_argument("--selftest", action="store_true") args = parser.parse_args() @@ -127,6 +148,9 @@ def main() -> int: print("phase0_profile_pair_preflight selftest: PASS") return 0 + if not args.project_git_sha: + parser.error("--project-git-sha must be non-empty") + try: on_size, on_hash = verify_blob( args.profile_on, int(PAIR["ON"]["bytes"]), str(PAIR["ON"]["sha256"]) @@ -140,10 +164,11 @@ def main() -> int: print(f"PROFILE ON PASS {on_size} B {on_hash}") print(f"PROFILE OFF PASS {off_size} B {off_hash}") - print(f"project_git_sha {PROJECT_GIT_SHA}") + print(f"artifact git SHA {args.project_git_sha}") + print(f"frozen source SHA {FROZEN_SOURCE_GIT_SHA}") if args.output_template: - rendered = json.dumps(sample_template(), indent=2) + "\n" + rendered = json.dumps(sample_template(args.project_git_sha), indent=2) + "\n" args.output_template.write_text(rendered, encoding="utf-8") print(f"sample template {args.output_template}") From bd7d1b835e35c686f627e3de1d70d9fea16cd04c Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:30:44 +0200 Subject: [PATCH 062/156] Phase 0: bind A/B samples to frozen binary hashes --- tools/compare_hdl_profile_ab.py | 61 +++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/tools/compare_hdl_profile_ab.py b/tools/compare_hdl_profile_ab.py index 1e338d13..db48dbc4 100755 --- a/tools/compare_hdl_profile_ab.py +++ b/tools/compare_hdl_profile_ab.py @@ -2,10 +2,10 @@ """Compare same-source HDL PROFILE=0/1 real-hardware samples. Input is a JSON array (or an object with a ``samples`` array). Each sample must -record ``mode`` (``OFF``/``ON``), ``project_git_sha``, ``workload_id``, -``correctness_hash``, ``source_bytes`` and ``total_us``. ``copy_us`` and -``verify_us`` are optional, but a metric is compared only when every accepted -sample in both modes provides it. +record ``mode`` (``OFF``/``ON``), ``project_git_sha``, the frozen mode-specific +ELF/IRX hashes, ``workload_id``, ``correctness_hash``, ``source_bytes`` and +``total_us``. ``copy_us`` and ``verify_us`` are optional, but a metric is +compared only when every accepted sample in both modes provides it. The tool intentionally does not invent PROFILE=0 latency distributions from the PROFILE=1 telemetry log. ``parse_hdl_perf.py`` remains the source for the rich @@ -22,9 +22,22 @@ from pathlib import Path from typing import Any +EXPECTED_BINARY_HASHES = { + "ON": { + "benchmark_elf_sha256": "964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7", + "hdl_stream_irx_sha256": "8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db", + }, + "OFF": { + "benchmark_elf_sha256": "4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916", + "hdl_stream_irx_sha256": "f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751", + }, +} + REQUIRED = ( "mode", "project_git_sha", + "benchmark_elf_sha256", + "hdl_stream_irx_sha256", "workload_id", "correctness_hash", "source_bytes", @@ -86,9 +99,23 @@ def _normalize(raw: Any) -> list[dict[str, Any]]: value = normalized[key] if not isinstance(value, int) or isinstance(value, bool) or value <= 0: raise ValueError(f"sample {index}: {key} must be a positive integer") - for key in ("project_git_sha", "workload_id", "correctness_hash"): + for key in ( + "project_git_sha", + "benchmark_elf_sha256", + "hdl_stream_irx_sha256", + "workload_id", + "correctness_hash", + ): if not isinstance(normalized[key], str) or not normalized[key]: raise ValueError(f"sample {index}: {key} must be a non-empty string") + + expected = EXPECTED_BINARY_HASHES[mode] + for key in ("benchmark_elf_sha256", "hdl_stream_irx_sha256"): + if normalized[key].lower() != expected[key]: + raise ValueError( + f"sample {index}: PROFILE {mode} {key} does not match frozen pair" + ) + normalized[key] = normalized[key].lower() samples.append(normalized) return samples @@ -118,6 +145,7 @@ def compare_samples(samples: list[dict[str, Any]], min_samples: int = 4) -> dict source_bytes = next(iter(source_sizes)) result: dict[str, Any] = { "project_git_sha": next(iter(shas)), + "binary_hashes": EXPECTED_BINARY_HASHES, "workload_id": next(iter(workloads)), "correctness_hash": next(iter(hashes)), "source_bytes": source_bytes, @@ -175,10 +203,13 @@ def selftest() -> None: ("OFF", [1000, 1010, 990, 1005], [700, 705, 695, 700], [200, 205, 195, 200]), ("ON", [1020, 1030, 1010, 1025], [714, 719, 709, 714], [204, 209, 199, 204]), ): + expected = EXPECTED_BINARY_HASHES[mode] for total, copy, verify in zip(totals, copies, verifies): samples.append({ "mode": mode, "project_git_sha": "abc123", + "benchmark_elf_sha256": expected["benchmark_elf_sha256"], + "hdl_stream_irx_sha256": expected["hdl_stream_irx_sha256"], "workload_id": "fixture-iso-a", "correctness_hash": "deadbeef", "source_bytes": 1024 * 1024, @@ -186,7 +217,8 @@ def selftest() -> None: "copy_us": copy, "verify_us": verify, }) - result = compare_samples(samples) + normalized = _normalize(samples) + result = compare_samples(normalized) assert result["sample_counts"] == {"OFF": 4, "ON": 4} assert result["metrics"]["total_us"]["off"]["p50"] == 1000 assert result["metrics"]["total_us"]["on"]["p50"] == 1020 @@ -194,14 +226,23 @@ def selftest() -> None: assert result["metrics"]["copy_us"]["on_vs_off_percent"]["p50"] == 2.0 assert result["metrics"]["end_to_end_kib_per_second"]["off"]["samples"] == 4 - broken = [dict(sample) for sample in samples] - broken[-1]["project_git_sha"] = "different" + broken_sha = [dict(sample) for sample in samples] + broken_sha[-1]["project_git_sha"] = "different" try: - compare_samples(broken) + compare_samples(_normalize(broken_sha)) except ValueError as error: assert "project_git_sha" in str(error) else: - raise AssertionError("mismatched SHA must fail") + raise AssertionError("mismatched project SHA must fail") + + broken_binary = [dict(sample) for sample in samples] + broken_binary[0]["benchmark_elf_sha256"] = "0" * 64 + try: + _normalize(broken_binary) + except ValueError as error: + assert "frozen pair" in str(error) + else: + raise AssertionError("mismatched binary hash must fail") def main() -> int: From 1bea3e42fb6aaf38df7ec3198280c9cc99629914 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:31:15 +0200 Subject: [PATCH 063/156] Phase 0: align sample provenance with artifact head --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fec1572..67ec7853 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,7 @@ jobs: python3 tools/phase0_profile_pair_preflight.py \ --profile-on PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF \ --profile-off PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF \ + --project-git-sha "${{ github.event.pull_request.head.sha || github.sha }}" \ --output-template PHASE0_AB_SAMPLES_TEMPLATE.json echo "--- PROFILE ON provenance ---" cat BENCHMARK_PROVENANCE_PROFILE_ON.yml From a26303ec2d140aaacda2700e615a4651363f89d1 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:32:05 +0200 Subject: [PATCH 064/156] Phase 0: distinguish artifact provenance from binary identity --- docs/PHASE0_HARDWARE_AB_PROTOCOL.md | 134 +++++++++++++++++++--------- 1 file changed, 91 insertions(+), 43 deletions(-) diff --git a/docs/PHASE0_HARDWARE_AB_PROTOCOL.md b/docs/PHASE0_HARDWARE_AB_PROTOCOL.md index e38018bc..af0d5792 100644 --- a/docs/PHASE0_HARDWARE_AB_PROTOCOL.md +++ b/docs/PHASE0_HARDWARE_AB_PROTOCOL.md @@ -7,15 +7,14 @@ throughput. ## Frozen hardware-test pair -Until a newer green pair is explicitly recorded here, use the artifacts from CI -run #666, project head: +The binary identity frozen by CI run #666 originates from project head: ```text -project_git_sha 7875b14d837d6332f5edc37f1c12a55527d7dd87 -project_git_ref perf/corpus-v2-integration -ps2sdk_commit b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b -toolchain mips64r5900el-ps2-elf GCC 15.2.0 -container ps2dev/ps2dev:v2.0.0 +frozen_source_git_sha 7875b14d837d6332f5edc37f1c12a55527d7dd87 +project_git_ref perf/corpus-v2-integration +ps2sdk_commit b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b +toolchain mips64r5900el-ps2-elf GCC 15.2.0 +container ps2dev/ps2dev:v2.0.0 ``` PROFILE ON: @@ -40,6 +39,18 @@ IRX data 144 IRX SHA-256 f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 ``` +The exact ELF/IRX hashes define the frozen binary pair. Later commits that touch +only documentation or host-side tooling may be rebuilt by CI into byte-identical +ELFs. In that case the CI provenance correctly records the newer artifact +`project_git_sha`, while the binary identity remains the hash pair above. A run +record must therefore preserve both: + +1. the `project_git_sha` from the artifact actually tested; +2. the mode-specific frozen ELF/IRX hashes. + +Do not rewrite the artifact SHA to `7875b14...` merely because the bytes match +the original CI #666 pair. + The static footprint delta is evidence that the compiled-out path is real; it is **not** a runtime speedup measurement. CI also enforces that the ON/OFF `hdl_stream.irx` files are not byte-identical. @@ -54,9 +65,10 @@ authoritative profiler-overhead sample. ## Compared builds -The authoritative instrumentation-overhead comparison is a same-source pair -built from one green commit on `perf/corpus-v2-integration` with the same PS2DEV -container, PS2SDK source, optimization flags and runtime implementation. +The authoritative instrumentation-overhead comparison is a same-runtime-source +pair built from one green artifact head with the same PS2DEV container, PS2SDK +source, optimization flags and runtime implementation. The frozen hashes above +are the final guard against accidentally testing a different binary. ### A: PROFILE OFF @@ -81,7 +93,7 @@ record, but the timed hot-path accounting itself is absent. ### B: PROFILE ON -Build the exact same commit with: +Build the exact same artifact head with: ```text HDL_PROFILE=1 @@ -107,6 +119,25 @@ known-good HDL installer baseline, but it is **not** the authoritative profiler overhead A/B. Runtime and Phase-1 policy changes after that commit make such a cross-commit timing delta confounded. +## Preflight before copying to the console + +Run the host-side guard on the exact two ELF files selected for the test: + +```text +python3 tools/phase0_profile_pair_preflight.py \ + --profile-on PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF \ + --profile-off PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF \ + --project-git-sha \ + --output-template PHASE0_AB_SAMPLES_TEMPLATE.json +``` + +The command fails if either ELF size or SHA-256 differs from the frozen pair. It +also creates the eight-run interleaved sample template with the artifact SHA and +mode-specific ELF/IRX hashes already filled in. Do not hand-edit those hashes. + +After copying the files to the launch medium, verify the ELF hashes there as +well. A filename such as `PROFILE_ON.ELF` is not provenance. + ## Hardware provenance Record before each pair of runs: @@ -122,6 +153,7 @@ toolchain: active_irx: build_flags: hdl_profile_enabled: +project_git_sha: benchmark_elf_sha256: hdl_stream_irx_sha256: workload: @@ -135,11 +167,8 @@ correctness_hash: The CI-generated `BENCHMARK_PROVENANCE_PROFILE_OFF.yml` and `BENCHMARK_PROVENANCE_PROFILE_ON.yml` supply build-side fields, including exact -ELF/IRX hashes and sizes. Hardware fields remain explicit manual measurements -rather than guessed metadata. - -Before timing, verify the ELF SHA-256 on the medium used to launch each build. -A filename such as `PROFILE_ON.ELF` is not provenance. +artifact project SHA, ELF/IRX hashes and sizes. Hardware fields remain explicit +manual measurements rather than guessed metadata. ## Workload contract @@ -153,11 +182,16 @@ Use the same: - video mode and active background services; - cold/warm policy. -Prefer an interleaved order such as `OFF, ON, ON, OFF` or `ON, OFF, OFF, ON` -rather than running every sample of one build first. This reduces temperature, -device-state and session-order bias. Recreate or otherwise control the target -layout between destructive install samples so the compared workload stays -meaningfully equivalent. +Use the generated eight-run order: + +```text +OFF, ON, ON, OFF, ON, OFF, OFF, ON +``` + +rather than running every sample of one build first. This distributes both modes +through the session and reduces temperature, device-state and session-order +bias. Recreate or otherwise control the target layout between destructive +install samples so the compared workload stays meaningfully equivalent. For HDL copy throughput, use one ISO large enough that startup/allocation noise is negligible compared with the bulk copy phase. Do not compare different ISOs, @@ -188,18 +222,21 @@ For the overhead result report PROFILE ON relative to PROFILE OFF for wall time and throughput. Do not fabricate p95/p99 for PROFILE OFF from absent telemetry; the point of the OFF build is to remove that instrumentation. -Run at least four complete comparable samples per mode, preferably distributed -through an interleaved order. Add samples if wall time or PROFILE ON tail latency -is unstable. +Run at least four complete comparable samples per mode. Add samples if wall time +or PROFILE ON tail latency is unstable. ## Host comparison record -Store the comparable run timings in a JSON array. Every sample must include: +Use the `PHASE0_AB_SAMPLES_TEMPLATE.json` emitted by preflight. Each sample has +this shape: ```json { + "run": 1, "mode": "OFF", - "project_git_sha": "7875b14d837d6332f5edc37f1c12a55527d7dd87", + "project_git_sha": "", + "benchmark_elf_sha256": "4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916", + "hdl_stream_irx_sha256": "f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751", "workload_id": "", "correctness_hash": "", "source_bytes": 0, @@ -210,35 +247,46 @@ Store the comparable run timings in a JSON array. Every sample must include: ``` `copy_us` and `verify_us` are optional if the external measurement setup cannot -isolate those phases. `source_bytes` and `total_us` are mandatory. Compare the -record with: +isolate those phases. `source_bytes` and `total_us` are mandatory. Replace every +zero placeholder before comparison, otherwise input validation rejects it. + +Compare the completed record with: ```text -python3 tools/compare_hdl_profile_ab.py samples.json --output profile-ab.json +python3 tools/compare_hdl_profile_ab.py PHASE0_AB_SAMPLES_TEMPLATE.json \ + --output profile-ab.json ``` -The comparator refuses mixed project SHAs, workloads, correctness hashes or -source sizes and requires four samples of each mode by default. It reports -p50/p95/p99/max for available wall-time metrics plus signed PROFILE ON vs OFF -percent deltas. Throughput is derived from the recorded bytes and time. Rich EE -and IOP latency distributions remain sourced from `parse_hdl_perf.py` for -PROFILE ON only. +The comparator refuses: + +- mixed artifact project SHAs; +- an ELF or IRX hash not belonging to the frozen mode-specific pair; +- mixed workloads; +- differing correctness hashes; +- differing source sizes; +- fewer than four samples of either mode by default. + +It reports p50/p95/p99/max for available wall-time metrics plus signed PROFILE +ON vs OFF percent deltas. Throughput is derived from the recorded bytes and +time. Rich EE and IOP latency distributions remain sourced from +`parse_hdl_perf.py` for PROFILE ON only. ## Phase-0 acceptance Phase 0 may be marked hardware-complete only when: -1. PROFILE OFF and PROFILE ON are from the same project SHA and their recorded - ELF/IRX hashes match the selected CI pair; -2. both builds pass the same correctness workload; -3. PROFILE ON produces internally consistent stage counters and traffic +1. PROFILE OFF and PROFILE ON samples share the artifact `project_git_sha` + recorded by the CI pair actually tested; +2. every sample carries the frozen mode-specific ELF/IRX hashes; +3. both builds pass the same correctness workload; +4. PROFILE ON produces internally consistent stage counters and traffic accounting; -4. the measurement overhead of PROFILE ON is quantified rather than assumed +5. the measurement overhead of PROFILE ON is quantified rather than assumed negligible; -5. PROFILE ON retains p50/p95/p99/max instead of replacing distributions with +6. PROFILE ON retains p50/p95/p99/max instead of replacing distributions with an average; -6. the result includes console/toolchain/IRX/workload provenance; -7. R5900 counter calibration is checked on the real EE before counter-derived +7. the result includes console/toolchain/IRX/workload provenance; +8. R5900 counter calibration is checked on the real EE before counter-derived optimization claims are made. If instrumentation materially changes throughput or tail latency, keep From bea7e853dd813b1660829fbc2a7faaeb9ac39049 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:35:14 +0200 Subject: [PATCH 065/156] Phase 0: add standalone R5900 counter calibration --- bench/r5900_calibration/main.c | 287 +++++++++++++++++++++++++++++++++ 1 file changed, 287 insertions(+) create mode 100644 bench/r5900_calibration/main.c diff --git a/bench/r5900_calibration/main.c b/bench/r5900_calibration/main.c new file mode 100644 index 00000000..d41737e7 --- /dev/null +++ b/bench/r5900_calibration/main.c @@ -0,0 +1,287 @@ +#include +#include +#include + +#include +#include + +#include "r5900_perf.h" + +#define CALIBRATION_SAMPLES 32u +#define CALIBRATION_ITERATIONS 100000u +#define CALIBRATION_BODY_INSNS_PER_ITERATION 4u + +typedef struct { + uint32_t cycles; + uint32_t instructions; + uint32_t timer_busclocks; + int counter_overflow; + int timer_overflow; +} calibration_sample_t; + +typedef struct { + uint32_t p50; + uint32_t p95; + uint32_t p99; + uint32_t max; +} calibration_distribution_t; + +static volatile uint32_t calibration_sink; + +/* + * Measurement reference kernel, not an application optimization. + * + * The loop body deliberately executes four fixed instructions per iteration: + * two ADDIU operations, BNEZ and its NOP delay slot. The one-time accumulator + * initialization and function return sit outside that repeated four-instruction + * body. Keeping this tiny body explicit makes the instruction-completed counter + * sanity check independent of GCC's loop transforms. + */ +static __attribute__((noinline)) uint32_t calibration_integer_loop(uint32_t iterations) +{ + uint32_t accumulator; + + __asm__ __volatile__( + "move %0, $zero\n" + "1:\n" + "addiu %0, %0, 1\n" + "addiu %1, %1, -1\n" + "bnez %1, 1b\n" + "nop\n" + : "=&r"(accumulator), "+r"(iterations) + : + : "memory"); + + return accumulator; +} + +static uint32_t timer_delta32(uint64_t start, uint64_t end, int *overflow) +{ + uint64_t delta = end - start; + + if (delta > UINT32_MAX) { + *overflow = 1; + return UINT32_MAX; + } + *overflow = 0; + return (uint32_t)delta; +} + +static int measure_empty(calibration_sample_t *sample) +{ + r5900_perf_scope_t scope; + r5900_perf_result_t result; + uint64_t timer_start; + uint64_t timer_end; + + memset(&scope, 0, sizeof(scope)); + memset(&result, 0, sizeof(result)); + memset(sample, 0, sizeof(*sample)); + + timer_start = GetTimerSystemTime(); + if (r5900_perf_begin(&scope, + R5900_PCR0_PROCESSOR_CYCLE, + R5900_PCR1_INSTRUCTION_COMPLETED) != 0) + return -1; + if (r5900_perf_end(&scope, &result) != 0) + return -1; + timer_end = GetTimerSystemTime(); + + sample->cycles = result.pcr0; + sample->instructions = result.pcr1; + sample->counter_overflow = result.pcr0_overflow || result.pcr1_overflow; + sample->timer_busclocks = timer_delta32(timer_start, timer_end, + &sample->timer_overflow); + return 0; +} + +static int measure_loop(calibration_sample_t *sample, uint32_t salt) +{ + r5900_perf_scope_t scope; + r5900_perf_result_t result; + uint64_t timer_start; + uint64_t timer_end; + uint32_t value; + + memset(&scope, 0, sizeof(scope)); + memset(&result, 0, sizeof(result)); + memset(sample, 0, sizeof(*sample)); + + timer_start = GetTimerSystemTime(); + if (r5900_perf_begin(&scope, + R5900_PCR0_PROCESSOR_CYCLE, + R5900_PCR1_INSTRUCTION_COMPLETED) != 0) + return -1; + + value = calibration_integer_loop(CALIBRATION_ITERATIONS); + + if (r5900_perf_end(&scope, &result) != 0) + return -1; + timer_end = GetTimerSystemTime(); + + /* Consume the result after the measured region. */ + calibration_sink ^= value + salt; + + sample->cycles = result.pcr0; + sample->instructions = result.pcr1; + sample->counter_overflow = result.pcr0_overflow || result.pcr1_overflow; + sample->timer_busclocks = timer_delta32(timer_start, timer_end, + &sample->timer_overflow); + return value == CALIBRATION_ITERATIONS ? 0 : -1; +} + +static void sort_u32(uint32_t *values, unsigned int count) +{ + unsigned int i; + + for (i = 1; i < count; ++i) { + uint32_t value = values[i]; + unsigned int j = i; + + while (j > 0 && values[j - 1] > value) { + values[j] = values[j - 1]; + --j; + } + values[j] = value; + } +} + +static uint32_t nearest_rank(const uint32_t *values, + unsigned int count, + unsigned int percentile) +{ + unsigned int rank = (count * percentile + 99u) / 100u; + + if (rank == 0) + rank = 1; + if (rank > count) + rank = count; + return values[rank - 1]; +} + +static calibration_distribution_t distribution(const calibration_sample_t *samples, + int field) +{ + uint32_t values[CALIBRATION_SAMPLES]; + calibration_distribution_t result; + unsigned int i; + + for (i = 0; i < CALIBRATION_SAMPLES; ++i) { + if (field == 0) + values[i] = samples[i].cycles; + else if (field == 1) + values[i] = samples[i].instructions; + else + values[i] = samples[i].timer_busclocks; + } + + sort_u32(values, CALIBRATION_SAMPLES); + result.p50 = nearest_rank(values, CALIBRATION_SAMPLES, 50); + result.p95 = nearest_rank(values, CALIBRATION_SAMPLES, 95); + result.p99 = nearest_rank(values, CALIBRATION_SAMPLES, 99); + result.max = values[CALIBRATION_SAMPLES - 1]; + return result; +} + +static uint32_t subtract_floor(uint32_t value, uint32_t overhead) +{ + return value > overhead ? value - overhead : 0; +} + +static void print_distribution(const char *name, + const calibration_distribution_t *value) +{ + scr_printf("%-11s p50=%u p95=%u p99=%u max=%u\n", + name, value->p50, value->p95, value->p99, value->max); +} + +int main(void) +{ + calibration_sample_t empty[CALIBRATION_SAMPLES]; + calibration_sample_t loop[CALIBRATION_SAMPLES]; + calibration_distribution_t empty_cycles; + calibration_distribution_t empty_instructions; + calibration_distribution_t empty_timer; + calibration_distribution_t loop_cycles; + calibration_distribution_t loop_instructions; + calibration_distribution_t loop_timer; + unsigned int counter_overflows = 0; + unsigned int timer_overflows = 0; + unsigned int failures = 0; + unsigned int i; + uint32_t expected_body_instructions = + CALIBRATION_ITERATIONS * CALIBRATION_BODY_INSNS_PER_ITERATION; + + init_scr(); + scr_printf("R5900 PERFORMANCE COUNTER CALIBRATION\n"); + scr_printf("No HDD/APA access. Standalone EE benchmark.\n\n"); + + /* Warm the reference kernel and its instruction footprint before sampling. */ + for (i = 0; i < 4; ++i) + calibration_sink ^= calibration_integer_loop(CALIBRATION_ITERATIONS); + + /* Pair samples and alternate order to reduce systematic first/second bias. */ + for (i = 0; i < CALIBRATION_SAMPLES; ++i) { + int empty_status; + int loop_status; + + if ((i & 1u) == 0) { + empty_status = measure_empty(&empty[i]); + loop_status = measure_loop(&loop[i], i); + } else { + loop_status = measure_loop(&loop[i], i); + empty_status = measure_empty(&empty[i]); + } + + if (empty_status != 0 || loop_status != 0) + ++failures; + if (empty[i].counter_overflow || loop[i].counter_overflow) + ++counter_overflows; + if (empty[i].timer_overflow || loop[i].timer_overflow) + ++timer_overflows; + } + + empty_cycles = distribution(empty, 0); + empty_instructions = distribution(empty, 1); + empty_timer = distribution(empty, 2); + loop_cycles = distribution(loop, 0); + loop_instructions = distribution(loop, 1); + loop_timer = distribution(loop, 2); + + scr_printf("samples=%u iterations=%u\n", + CALIBRATION_SAMPLES, CALIBRATION_ITERATIONS); + scr_printf("PCR0=processor cycles PCR1=instructions completed\n"); + scr_printf("timer=GetTimerSystemTime raw BUSCLK units\n\n"); + + scr_printf("EMPTY SCOPE\n"); + print_distribution("cycles", &empty_cycles); + print_distribution("instructions", &empty_instructions); + print_distribution("timer", &empty_timer); + + scr_printf("\nDETERMINISTIC LOOP\n"); + print_distribution("cycles", &loop_cycles); + print_distribution("instructions", &loop_instructions); + print_distribution("timer", &loop_timer); + + scr_printf("\nP50 LOOP - EMPTY\n"); + scr_printf("cycles=%u instructions=%u timer=%u\n", + subtract_floor(loop_cycles.p50, empty_cycles.p50), + subtract_floor(loop_instructions.p50, empty_instructions.p50), + subtract_floor(loop_timer.p50, empty_timer.p50)); + scr_printf("loop-body instruction floor=%u\n", expected_body_instructions); + scr_printf("overflows counter=%u timer=%u failures=%u\n", + counter_overflows, timer_overflows, failures); + scr_printf("sink=%08x\n", (unsigned int)calibration_sink); + + if (counter_overflows != 0 || timer_overflows != 0 || failures != 0 || + subtract_floor(loop_instructions.p50, empty_instructions.p50) < + expected_body_instructions) { + scr_printf("\nRESULT: INVALID - do not use counter claims\n"); + } else { + scr_printf("\nRESULT: CALIBRATION STRUCTURALLY VALID\n"); + scr_printf("Record screen + SCPH/revision/toolchain provenance.\n"); + } + + SleepThread(); + return 0; +} From 8507442beefbff01723ca20d144a633bb97ef2f0 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:35:23 +0200 Subject: [PATCH 066/156] Phase 0: add standalone calibration build target --- bench/r5900_calibration/Makefile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 bench/r5900_calibration/Makefile diff --git a/bench/r5900_calibration/Makefile b/bench/r5900_calibration/Makefile new file mode 100644 index 00000000..28afb921 --- /dev/null +++ b/bench/r5900_calibration/Makefile @@ -0,0 +1,22 @@ +EE_BIN = R5900_COUNTER_CALIBRATION.ELF +EE_OBJS = main.o r5900_perf.o +EE_LIBS = -ldebug -lkernel + +EE_CFLAGS = -O2 -G0 -Wall -Wextra -Werror -std=gnu99 -I../../include +EE_LDFLAGS = -Wl,-Map,R5900_COUNTER_CALIBRATION.map + +all: $(EE_BIN) + +main.o: main.c + $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ + +r5900_perf.o: ../../src/r5900_perf.c ../../include/r5900_perf.h + $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c ../../src/r5900_perf.c -o $@ + +clean: + rm -f $(EE_BIN) R5900_COUNTER_CALIBRATION.map $(EE_OBJS) + +include $(PS2SDK)/samples/Makefile.pref +include $(PS2SDK)/samples/Makefile.eeglobal + +.PHONY: all clean From 2ee4a96b5ad5fd42ce5d01f4ac853084807e10b2 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:36:00 +0200 Subject: [PATCH 067/156] Phase 0: build standalone R5900 calibration artifact --- .github/workflows/ci.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67ec7853..1297e9ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Build PROFILE ON/OFF pair with PS2DEV v2.0.0 + - name: Build PROFILE pair and calibration ELF with PS2DEV v2.0.0 run: >- docker run --rm -e PROJECT_GIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}" @@ -85,7 +85,16 @@ jobs: cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF PS2_HDD_BOOTSTRAP_MANAGER.ELF && cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.map - PS2_HDD_BOOTSTRAP_MANAGER.map' + PS2_HDD_BOOTSTRAP_MANAGER.map && + make -C bench/r5900_calibration clean all && + mips64r5900el-ps2-elf-size + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF + > bench/r5900_calibration/R5900_COUNTER_CALIBRATION.sections && + mips64r5900el-ps2-elf-objdump -dr + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF + > bench/r5900_calibration/R5900_COUNTER_CALIBRATION.disasm && + sha256sum bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF + > bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF.sha256' - name: Validate PROFILE split and record checksums run: | if cmp -s HDL_STREAM_PROFILE_ON.irx HDL_STREAM_PROFILE_OFF.irx; then @@ -100,6 +109,9 @@ jobs: --profile-off PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF \ --project-git-sha "${{ github.event.pull_request.head.sha || github.sha }}" \ --output-template PHASE0_AB_SAMPLES_TEMPLATE.json + echo "--- R5900 calibration ELF ---" + cat bench/r5900_calibration/R5900_COUNTER_CALIBRATION.sections + cat bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF.sha256 echo "--- PROFILE ON provenance ---" cat BENCHMARK_PROVENANCE_PROFILE_ON.yml echo "--- PROFILE OFF provenance ---" @@ -135,6 +147,11 @@ jobs: GCC_R5900_TARGET.txt BENCHMARK_PROVENANCE_PROFILE_ON.yml BENCHMARK_PROVENANCE_PROFILE_OFF.yml + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF.sha256 + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.map + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.sections + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.disasm HDDMAN.CFG LICENSE THIRD_PARTY_NOTICES.md From a4bff56c91cc8a3cfb678c5c79b2bd514c4bbcb7 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:38:26 +0200 Subject: [PATCH 068/156] Phase 0: fix R5900 calibration format ABI --- bench/r5900_calibration/main.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/bench/r5900_calibration/main.c b/bench/r5900_calibration/main.c index d41737e7..f81d47f5 100644 --- a/bench/r5900_calibration/main.c +++ b/bench/r5900_calibration/main.c @@ -191,8 +191,12 @@ static uint32_t subtract_floor(uint32_t value, uint32_t overhead) static void print_distribution(const char *name, const calibration_distribution_t *value) { - scr_printf("%-11s p50=%u p95=%u p99=%u max=%u\n", - name, value->p50, value->p95, value->p99, value->max); + scr_printf("%-11s p50=%lu p95=%lu p99=%lu max=%lu\n", + name, + (unsigned long)value->p50, + (unsigned long)value->p95, + (unsigned long)value->p99, + (unsigned long)value->max); } int main(void) @@ -264,11 +268,13 @@ int main(void) print_distribution("timer", &loop_timer); scr_printf("\nP50 LOOP - EMPTY\n"); - scr_printf("cycles=%u instructions=%u timer=%u\n", - subtract_floor(loop_cycles.p50, empty_cycles.p50), - subtract_floor(loop_instructions.p50, empty_instructions.p50), - subtract_floor(loop_timer.p50, empty_timer.p50)); - scr_printf("loop-body instruction floor=%u\n", expected_body_instructions); + scr_printf("cycles=%lu instructions=%lu timer=%lu\n", + (unsigned long)subtract_floor(loop_cycles.p50, empty_cycles.p50), + (unsigned long)subtract_floor(loop_instructions.p50, + empty_instructions.p50), + (unsigned long)subtract_floor(loop_timer.p50, empty_timer.p50)); + scr_printf("loop-body instruction floor=%lu\n", + (unsigned long)expected_body_instructions); scr_printf("overflows counter=%u timer=%u failures=%u\n", counter_overflows, timer_overflows, failures); scr_printf("sink=%08x\n", (unsigned int)calibration_sink); From bc4b1769823d716d3239d2f66ec255e033c10e1f Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:39:05 +0200 Subject: [PATCH 069/156] Phase 0: expose calibration reference kernel in CI --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1297e9ad..bc438bb5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,6 +112,9 @@ jobs: echo "--- R5900 calibration ELF ---" cat bench/r5900_calibration/R5900_COUNTER_CALIBRATION.sections cat bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF.sha256 + echo "--- calibration_integer_loop disassembly ---" + awk '/:/ {show=1; count=0} show {print; count++} show && count >= 12 {exit}' \ + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.disasm echo "--- PROFILE ON provenance ---" cat BENCHMARK_PROVENANCE_PROFILE_ON.yml echo "--- PROFILE OFF provenance ---" From 31dedfbd7ca7a118190fc37f86d74834b9079d16 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 21:41:15 +0200 Subject: [PATCH 070/156] Phase 0: keep calibration reference kernel uncloned --- bench/r5900_calibration/main.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bench/r5900_calibration/main.c b/bench/r5900_calibration/main.c index f81d47f5..9b801d83 100644 --- a/bench/r5900_calibration/main.c +++ b/bench/r5900_calibration/main.c @@ -35,9 +35,10 @@ static volatile uint32_t calibration_sink; * two ADDIU operations, BNEZ and its NOP delay slot. The one-time accumulator * initialization and function return sit outside that repeated four-instruction * body. Keeping this tiny body explicit makes the instruction-completed counter - * sanity check independent of GCC's loop transforms. + * sanity check independent of GCC's loop transforms. NOCLONE keeps a stable + * callable symbol so CI can validate the emitted reference sequence directly. */ -static __attribute__((noinline)) uint32_t calibration_integer_loop(uint32_t iterations) +static __attribute__((noinline, noclone)) uint32_t calibration_integer_loop(uint32_t iterations) { uint32_t accumulator; From a0d7f3caf7b0136a7edab2d7f6cf99f261b57bdd Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 22:21:15 +0200 Subject: [PATCH 071/156] ci: validate deterministic R5900 calibration loop --- tools/check_r5900_calibration_disasm.py | 211 ++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 tools/check_r5900_calibration_disasm.py diff --git a/tools/check_r5900_calibration_disasm.py b/tools/check_r5900_calibration_disasm.py new file mode 100644 index 00000000..e0f8127d --- /dev/null +++ b/tools/check_r5900_calibration_disasm.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Validate the deterministic R5900 counter-calibration loop in objdump output. + +This is a static build guard, not a timing claim. It verifies that the backward +branch in calibration_integer_loop repeats exactly four instructions: + + addiu , , +1 + addiu , , -1 + bnez , + nop + +Real-hardware performance-counter calibration is still required before PCR +results are treated as authoritative. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path + + +SYMBOL = "calibration_integer_loop" +SYMBOL_RE = re.compile(r"^\s*([0-9a-fA-F]+)\s+<([^>]+)>:\s*$") +INSN_RE = re.compile( + r"^\s*([0-9a-fA-F]+):\s+[0-9a-fA-F]+\s+([A-Za-z0-9_.]+)(?:\s+(.*?))?\s*$" +) + + +@dataclass(frozen=True) +class Instruction: + address: int + mnemonic: str + operands: str + + +def parse_function(text: str, symbol: str = SYMBOL) -> list[Instruction]: + inside = False + instructions: list[Instruction] = [] + + for line in text.splitlines(): + symbol_match = SYMBOL_RE.match(line) + if symbol_match: + if inside: + break + inside = symbol_match.group(2) == symbol + continue + + if not inside: + continue + + match = INSN_RE.match(line) + if match: + instructions.append( + Instruction( + address=int(match.group(1), 16), + mnemonic=match.group(2).lower(), + operands=(match.group(3) or "").strip(), + ) + ) + + if not instructions: + raise ValueError(f"symbol <{symbol}> not found or contains no instructions") + return instructions + + +def split_operands(operands: str) -> list[str]: + return [part.strip() for part in operands.split(",")] + + +def parse_integer(value: str) -> int: + value = value.strip() + if value.startswith("-"): + return -parse_integer(value[1:]) + if value.lower().startswith("0x"): + return int(value, 16) + return int(value, 10) + + +def validate_loop(instructions: list[Instruction]) -> str: + branches = [i for i, insn in enumerate(instructions) if insn.mnemonic == "bnez"] + if len(branches) != 1: + raise ValueError(f"expected exactly one bnez in <{SYMBOL}>, found {len(branches)}") + + branch_index = branches[0] + branch = instructions[branch_index] + branch_operands = split_operands(branch.operands) + if len(branch_operands) < 2: + raise ValueError("bnez operands are malformed") + + counter_register = branch_operands[0] + target_match = re.match(r"^([0-9a-fA-F]+)\b", branch_operands[1]) + if not target_match: + raise ValueError(f"cannot parse bnez target from: {branch.operands!r}") + target = int(target_match.group(1), 16) + + address_to_index = {insn.address: i for i, insn in enumerate(instructions)} + if target not in address_to_index: + raise ValueError(f"bnez target 0x{target:x} is outside <{SYMBOL}>") + target_index = address_to_index[target] + + if branch_index + 1 >= len(instructions): + raise ValueError("bnez has no visible delay-slot instruction") + + loop = instructions[target_index : branch_index + 2] + mnemonics = [insn.mnemonic for insn in loop] + expected = ["addiu", "addiu", "bnez", "nop"] + if mnemonics != expected: + raise ValueError( + "calibration loop body changed: expected " + + "/".join(expected) + + ", got " + + "/".join(mnemonics) + ) + + accumulator_ops = split_operands(loop[0].operands) + if len(accumulator_ops) != 3: + raise ValueError("first addiu operands are malformed") + if accumulator_ops[0] != accumulator_ops[1]: + raise ValueError("first addiu must update the accumulator in place") + try: + accumulator_step = parse_integer(accumulator_ops[2]) + except ValueError as exc: + raise ValueError("first addiu immediate is not an integer") from exc + if accumulator_step != 1: + raise ValueError(f"first addiu must increment by 1, got {accumulator_step}") + + counter_ops = split_operands(loop[1].operands) + if len(counter_ops) != 3: + raise ValueError("second addiu operands are malformed") + if counter_ops[0] != counter_register or counter_ops[1] != counter_register: + raise ValueError("second addiu must decrement the same register tested by bnez") + try: + counter_step = parse_integer(counter_ops[2]) + except ValueError as exc: + raise ValueError("second addiu immediate is not an integer") from exc + if counter_step != -1: + raise ValueError(f"second addiu must decrement by 1, got {counter_step}") + + return ( + f"PASS: <{SYMBOL}> repeats exactly addiu(+1)/addiu(-1)/bnez/nop " + f"from 0x{target:x} to branch 0x{branch.address:x}" + ) + + +def validate_text(text: str) -> str: + return validate_loop(parse_function(text)) + + +def selftest() -> None: + valid = """ +001015f8 : + 1015f8: 3c030001 lui v1,0x1 + 1015fc: 346386a0 ori v1,v1,0x86a0 + 101600: 00001025 move v0,zero + 101604: 24420001 addiu v0,v0,1 + 101608: 2463ffff addiu v1,v1,-1 + 10160c: 1460fffd bnez v1,101604 + 101610: 00000000 nop + 101614: 00000000 nop + 101618: 03e00008 jr ra + 10161c: 00000000 nop + +00101620 : +""" + validate_text(valid) + + invalid_cases = [ + valid.replace("101610: 00000000 nop", "101610: 24420001 addiu v0,v0,1"), + valid.replace("bnez v1,101604", "bnez v1,101608"), + valid.replace("addiu v1,v1,-1", "addiu v1,v1,-2"), + valid.replace("", ""), + ] + for index, bad in enumerate(invalid_cases, 1): + try: + validate_text(bad) + except ValueError: + continue + raise AssertionError(f"negative self-test {index} unexpectedly passed") + + print("R5900 calibration disassembly checker self-test: PASS") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--disasm", type=Path, help="objdump -dr output to validate") + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + + if args.disasm is None: + if args.selftest: + return 0 + parser.error("--disasm is required unless --selftest is used") + + try: + message = validate_text(args.disasm.read_text(encoding="utf-8")) + except (OSError, UnicodeError, ValueError) as exc: + print(f"R5900 calibration disassembly guard: FAIL: {exc}", file=sys.stderr) + return 1 + + print(f"R5900 calibration disassembly guard: {message}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 6a73b59856390c4be84805b65ebb0a15e19ad0fa Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 22:21:42 +0200 Subject: [PATCH 072/156] ci: enforce R5900 calibration loop shape --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc438bb5..a259184a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,8 @@ jobs: run: python3 tools/compare_hdl_profile_ab.py --selftest - name: Self-test Phase-0 frozen pair preflight run: python3 tools/phase0_profile_pair_preflight.py --selftest + - name: Self-test R5900 calibration disassembly guard + run: python3 tools/check_r5900_calibration_disasm.py --selftest - name: Enforce direct-fileXio runtime policy run: | python3 tools/check_filexio_fdman_policy.py --selftest @@ -109,6 +111,8 @@ jobs: --profile-off PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF \ --project-git-sha "${{ github.event.pull_request.head.sha || github.sha }}" \ --output-template PHASE0_AB_SAMPLES_TEMPLATE.json + python3 tools/check_r5900_calibration_disasm.py \ + --disasm bench/r5900_calibration/R5900_COUNTER_CALIBRATION.disasm echo "--- R5900 calibration ELF ---" cat bench/r5900_calibration/R5900_COUNTER_CALIBRATION.sections cat bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF.sha256 From 8a155c8a6bb4fb971890cde0040f6147e4797d94 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 22:29:28 +0200 Subject: [PATCH 073/156] perf: add portable HDL hash checkpoint format --- include/hdl_hash_checkpoint.h | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 include/hdl_hash_checkpoint.h diff --git a/include/hdl_hash_checkpoint.h b/include/hdl_hash_checkpoint.h new file mode 100644 index 00000000..d9e421b3 --- /dev/null +++ b/include/hdl_hash_checkpoint.h @@ -0,0 +1,36 @@ +#ifndef PS2_HDD_BOOTSTRAP_MANAGER_HDL_HASH_CHECKPOINT_H +#define PS2_HDD_BOOTSTRAP_MANAGER_HDL_HASH_CHECKPOINT_H + +#include "hdl_transaction.h" +#include "sha256.h" + +#define HDL_HASH_CHECKPOINT_RECORD_SIZE 256u + +enum { + HDL_HASH_CHECKPOINT_INVALID_ARGUMENT = -545, + HDL_HASH_CHECKPOINT_INVALID_RECORD = -546, + HDL_HASH_CHECKPOINT_HASH_MISMATCH = -547, + HDL_HASH_CHECKPOINT_TRANSACTION_MISMATCH = -548, + HDL_HASH_CHECKPOINT_CONTEXT_MISMATCH = -549 +}; + +/* + * Portable checkpoint for the streaming source SHA-256 state used by a + * resumable HDL copy. The record is deliberately separate from the stable + * 512-byte transaction journal so old journals remain readable. + * + * A checkpoint is an optimization hint, never transaction authority. Restore + * succeeds only when source size, completed byte count, source fingerprint and + * target ID match the already authenticated transaction journal. + */ +int hdl_hash_checkpoint_encode( + const hdl_transaction_t *transaction, + const sha256_context_t *context, + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]); + +int hdl_hash_checkpoint_restore( + const unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE], + const hdl_transaction_t *transaction, + sha256_context_t *context); + +#endif From 5151b2dd55ec60ddbd9dc5923fbe476f09d3a847 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 22:29:51 +0200 Subject: [PATCH 074/156] perf: implement HDL hash checkpoint codec --- src/hdl_hash_checkpoint.c | 174 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 src/hdl_hash_checkpoint.c diff --git a/src/hdl_hash_checkpoint.c b/src/hdl_hash_checkpoint.c new file mode 100644 index 00000000..4430f041 --- /dev/null +++ b/src/hdl_hash_checkpoint.c @@ -0,0 +1,174 @@ +#include +#include +#include + +#include "hdl_hash_checkpoint.h" + +#define HDL_HASH_CHECKPOINT_VERSION 1u +#define HDL_HASH_CHECKPOINT_HASH_OFFSET 224u +#define HDL_HASH_CHECKPOINT_TARGET_OFFSET 64u +#define HDL_HASH_CHECKPOINT_TARGET_BYTES HDL_TRANSACTION_TARGET_MAX +#define HDL_HASH_CHECKPOINT_STATE_OFFSET 100u +#define HDL_HASH_CHECKPOINT_TOTAL_OFFSET 132u +#define HDL_HASH_CHECKPOINT_BLOCK_USED_OFFSET 140u +#define HDL_HASH_CHECKPOINT_BLOCK_OFFSET 144u + +static const unsigned char checkpoint_magic[8] = { + 'H', 'D', 'L', 'H', 'A', 'S', 'H', 1 +}; + +static void write_le32(unsigned char *destination, uint32_t value) +{ + destination[0] = (unsigned char)value; + destination[1] = (unsigned char)(value >> 8); + destination[2] = (unsigned char)(value >> 16); + destination[3] = (unsigned char)(value >> 24); +} + +static void write_le64(unsigned char *destination, uint64_t value) +{ + write_le32(destination, (uint32_t)value); + write_le32(destination + 4, (uint32_t)(value >> 32)); +} + +static uint32_t read_le32(const unsigned char *source) +{ + return (uint32_t)source[0] | ((uint32_t)source[1] << 8) | + ((uint32_t)source[2] << 16) | ((uint32_t)source[3] << 24); +} + +static uint64_t read_le64(const unsigned char *source) +{ + return (uint64_t)read_le32(source) | + ((uint64_t)read_le32(source + 4) << 32); +} + +static size_t bounded_length(const char *text, size_t capacity) +{ + const char *end = memchr(text, '\0', capacity); + + return end == NULL ? capacity : (size_t)(end - text); +} + +static int checkpoint_progress(const hdl_transaction_t *transaction, + uint64_t *completed_bytes) +{ + if (transaction == NULL || completed_bytes == NULL || + transaction->source_bytes == 0 || + (transaction->source_bytes & 2047u) != 0 || + transaction->completed_sectors > transaction->total_sectors || + transaction->total_sectors != transaction->source_bytes / 2048u || + transaction->completed_sectors == 0 || + memchr(transaction->target, '\0', sizeof(transaction->target)) == NULL || + transaction->target[0] == '\0') + return 0; + + *completed_bytes = transaction->completed_sectors * 2048u; + return *completed_bytes <= transaction->source_bytes; +} + +int hdl_hash_checkpoint_encode( + const hdl_transaction_t *transaction, + const sha256_context_t *context, + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]) +{ + unsigned char digest[32]; + uint64_t completed_bytes; + size_t target_length; + unsigned int i; + + if (record == NULL || context == NULL || + !checkpoint_progress(transaction, &completed_bytes)) + return HDL_HASH_CHECKPOINT_INVALID_ARGUMENT; + if (context->total_bytes != completed_bytes || context->block_used > 63u || + context->block_used != (size_t)(completed_bytes & 63u)) + return HDL_HASH_CHECKPOINT_CONTEXT_MISMATCH; + + memset(record, 0, HDL_HASH_CHECKPOINT_RECORD_SIZE); + memcpy(record, checkpoint_magic, sizeof(checkpoint_magic)); + write_le32(record + 8, HDL_HASH_CHECKPOINT_VERSION); + write_le64(record + 16, transaction->source_bytes); + write_le64(record + 24, completed_bytes); + memcpy(record + 32, transaction->source_fingerprint, + sizeof(transaction->source_fingerprint)); + target_length = bounded_length(transaction->target, + sizeof(transaction->target)); + if (target_length >= HDL_HASH_CHECKPOINT_TARGET_BYTES) + return HDL_HASH_CHECKPOINT_INVALID_ARGUMENT; + memcpy(record + HDL_HASH_CHECKPOINT_TARGET_OFFSET, + transaction->target, target_length); + + for (i = 0; i < 8u; i++) + write_le32(record + HDL_HASH_CHECKPOINT_STATE_OFFSET + i * 4u, + context->state[i]); + write_le64(record + HDL_HASH_CHECKPOINT_TOTAL_OFFSET, + context->total_bytes); + write_le32(record + HDL_HASH_CHECKPOINT_BLOCK_USED_OFFSET, + (uint32_t)context->block_used); + if (context->block_used != 0) + memcpy(record + HDL_HASH_CHECKPOINT_BLOCK_OFFSET, + context->block, context->block_used); + + sha256_buffer(record, HDL_HASH_CHECKPOINT_HASH_OFFSET, digest); + memcpy(record + HDL_HASH_CHECKPOINT_HASH_OFFSET, digest, sizeof(digest)); + return 0; +} + +int hdl_hash_checkpoint_restore( + const unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE], + const hdl_transaction_t *transaction, + sha256_context_t *context) +{ + unsigned char digest[32]; + char target[HDL_TRANSACTION_TARGET_MAX]; + uint64_t completed_bytes; + uint64_t stored_source_bytes; + uint64_t stored_completed_bytes; + uint64_t stored_total_bytes; + uint32_t block_used; + unsigned int i; + + if (record == NULL || context == NULL || + !checkpoint_progress(transaction, &completed_bytes)) + return HDL_HASH_CHECKPOINT_INVALID_ARGUMENT; + if (memcmp(record, checkpoint_magic, sizeof(checkpoint_magic)) != 0 || + read_le32(record + 8) != HDL_HASH_CHECKPOINT_VERSION) + return HDL_HASH_CHECKPOINT_INVALID_RECORD; + + sha256_buffer(record, HDL_HASH_CHECKPOINT_HASH_OFFSET, digest); + if (memcmp(digest, record + HDL_HASH_CHECKPOINT_HASH_OFFSET, + sizeof(digest)) != 0) + return HDL_HASH_CHECKPOINT_HASH_MISMATCH; + + memset(target, 0, sizeof(target)); + memcpy(target, record + HDL_HASH_CHECKPOINT_TARGET_OFFSET, + sizeof(target)); + if (memchr(target, '\0', sizeof(target)) == NULL) + return HDL_HASH_CHECKPOINT_INVALID_RECORD; + + stored_source_bytes = read_le64(record + 16); + stored_completed_bytes = read_le64(record + 24); + if (stored_source_bytes != transaction->source_bytes || + stored_completed_bytes != completed_bytes || + memcmp(record + 32, transaction->source_fingerprint, + sizeof(transaction->source_fingerprint)) != 0 || + strcmp(target, transaction->target) != 0) + return HDL_HASH_CHECKPOINT_TRANSACTION_MISMATCH; + + stored_total_bytes = read_le64(record + HDL_HASH_CHECKPOINT_TOTAL_OFFSET); + block_used = read_le32(record + HDL_HASH_CHECKPOINT_BLOCK_USED_OFFSET); + if (stored_total_bytes != stored_completed_bytes || block_used > 63u || + block_used != (uint32_t)(stored_total_bytes & 63u)) + return HDL_HASH_CHECKPOINT_INVALID_RECORD; + + memset(context, 0, sizeof(*context)); + for (i = 0; i < 8u; i++) + context->state[i] = + read_le32(record + HDL_HASH_CHECKPOINT_STATE_OFFSET + i * 4u); + context->total_bytes = stored_total_bytes; + context->block_used = (size_t)block_used; + if (block_used != 0) + memcpy(context->block, record + HDL_HASH_CHECKPOINT_BLOCK_OFFSET, + block_used); + return 0; +} From 7ddddec83e4974ee80853bd711c008ea0284cb74 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 22:30:18 +0200 Subject: [PATCH 075/156] test: cover HDL hash checkpoint restore --- tests/test_hdl_hash_checkpoint.c | 167 +++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 tests/test_hdl_hash_checkpoint.c diff --git a/tests/test_hdl_hash_checkpoint.c b/tests/test_hdl_hash_checkpoint.c new file mode 100644 index 00000000..c2827a63 --- /dev/null +++ b/tests/test_hdl_hash_checkpoint.c @@ -0,0 +1,167 @@ +#include +#include +#include + +#include "hdl_hash_checkpoint.h" + +static hdl_transaction_t copying_transaction(void) +{ + hdl_transaction_t transaction; + unsigned int i; + + memset(&transaction, 0, sizeof(transaction)); + transaction.stage = HDL_TRANSACTION_STAGE_COPYING; + transaction.source_bytes = 8192; + transaction.total_sectors = 4; + transaction.completed_sectors = 2; + transaction.partition_count = 1; + strcpy(transaction.target, "PP.SLUS-12345.HDL.TEST"); + strcpy(transaction.startup, "SLUS_123.45"); + strcpy(transaction.source_path, "mass:/TEST.ISO"); + strcpy(transaction.game_title, "Test Game"); + transaction.disc_type = 0x14; + for (i = 0; i < sizeof(transaction.source_fingerprint); i++) + transaction.source_fingerprint[i] = (unsigned char)(0xa0u + i); + return transaction; +} + +static void fill_payload(unsigned char *payload, size_t size) +{ + size_t i; + + for (i = 0; i < size; i++) + payload[i] = (unsigned char)((i * 37u + i / 17u) & 0xffu); +} + +static void test_restore_continues_same_digest(void) +{ + hdl_transaction_t transaction = copying_transaction(); + unsigned char payload[8192]; + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + unsigned char expected[32]; + unsigned char resumed_digest[32]; + sha256_context_t original; + sha256_context_t resumed; + + fill_payload(payload, sizeof(payload)); + sha256_buffer(payload, sizeof(payload), expected); + + sha256_init(&original); + sha256_update(&original, payload, 4096); + assert(original.total_bytes == 4096); + assert(original.block_used == 0); + assert(hdl_hash_checkpoint_encode(&transaction, &original, record) == 0); + + memset(&resumed, 0xcc, sizeof(resumed)); + assert(hdl_hash_checkpoint_restore(record, &transaction, &resumed) == 0); + assert(resumed.total_bytes == original.total_bytes); + assert(resumed.block_used == original.block_used); + assert(memcmp(resumed.state, original.state, sizeof(original.state)) == 0); + + sha256_update(&resumed, payload + 4096, sizeof(payload) - 4096); + sha256_final(&resumed, resumed_digest); + assert(memcmp(expected, resumed_digest, sizeof(expected)) == 0); +} + +static void test_every_record_byte_is_authenticated(void) +{ + hdl_transaction_t transaction = copying_transaction(); + unsigned char payload[4096]; + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + unsigned char damaged[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + sha256_context_t hash; + sha256_context_t restored; + unsigned int i; + + fill_payload(payload, sizeof(payload)); + sha256_init(&hash); + sha256_update(&hash, payload, sizeof(payload)); + assert(hdl_hash_checkpoint_encode(&transaction, &hash, record) == 0); + + for (i = 0; i < sizeof(record); i++) { + memcpy(damaged, record, sizeof(damaged)); + damaged[i] ^= 0x01u; + assert(hdl_hash_checkpoint_restore(damaged, &transaction, &restored) < 0); + } +} + +static void test_transaction_identity_must_match(void) +{ + hdl_transaction_t transaction = copying_transaction(); + hdl_transaction_t wrong; + unsigned char payload[4096]; + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + sha256_context_t hash; + sha256_context_t restored; + + fill_payload(payload, sizeof(payload)); + sha256_init(&hash); + sha256_update(&hash, payload, sizeof(payload)); + assert(hdl_hash_checkpoint_encode(&transaction, &hash, record) == 0); + + wrong = transaction; + wrong.completed_sectors = 1; + assert(hdl_hash_checkpoint_restore(record, &wrong, &restored) == + HDL_HASH_CHECKPOINT_TRANSACTION_MISMATCH); + + wrong = transaction; + wrong.source_fingerprint[7] ^= 0x80u; + assert(hdl_hash_checkpoint_restore(record, &wrong, &restored) == + HDL_HASH_CHECKPOINT_TRANSACTION_MISMATCH); + + wrong = transaction; + strcpy(wrong.target, "PP.SLUS-12345.OTHER"); + assert(hdl_hash_checkpoint_restore(record, &wrong, &restored) == + HDL_HASH_CHECKPOINT_TRANSACTION_MISMATCH); +} + +static void test_context_progress_must_match_transaction(void) +{ + hdl_transaction_t transaction = copying_transaction(); + unsigned char payload[2048]; + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + sha256_context_t hash; + + fill_payload(payload, sizeof(payload)); + sha256_init(&hash); + sha256_update(&hash, payload, sizeof(payload)); + assert(hdl_hash_checkpoint_encode(&transaction, &hash, record) == + HDL_HASH_CHECKPOINT_CONTEXT_MISMATCH); + + transaction.completed_sectors = 0; + sha256_init(&hash); + assert(hdl_hash_checkpoint_encode(&transaction, &hash, record) == + HDL_HASH_CHECKPOINT_INVALID_ARGUMENT); +} + +static void test_full_payload_checkpoint_can_finalize(void) +{ + hdl_transaction_t transaction = copying_transaction(); + unsigned char payload[8192]; + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + unsigned char expected[32]; + unsigned char restored_digest[32]; + sha256_context_t hash; + sha256_context_t restored; + + transaction.completed_sectors = transaction.total_sectors; + fill_payload(payload, sizeof(payload)); + sha256_buffer(payload, sizeof(payload), expected); + sha256_init(&hash); + sha256_update(&hash, payload, sizeof(payload)); + assert(hdl_hash_checkpoint_encode(&transaction, &hash, record) == 0); + assert(hdl_hash_checkpoint_restore(record, &transaction, &restored) == 0); + sha256_final(&restored, restored_digest); + assert(memcmp(expected, restored_digest, sizeof(expected)) == 0); +} + +int main(void) +{ + test_restore_continues_same_digest(); + test_every_record_byte_is_authenticated(); + test_transaction_identity_must_match(); + test_context_progress_must_match_transaction(); + test_full_payload_checkpoint_can_finalize(); + puts("All HDL hash checkpoint tests passed."); + return 0; +} From 65511dd46c6b1b30629cb797980bd454ccf25f0c Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 22:32:21 +0200 Subject: [PATCH 076/156] perf: add isolated resume hash checkpoint experiment --- Makefile | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1a8c4f34..1a3e9b2b 100644 --- a/Makefile +++ b/Makefile @@ -11,9 +11,21 @@ ifeq ($(filter $(HDL_PROFILE),0 1),) $(error HDL_PROFILE must be 0 or 1) endif +# Phase-2 recovery experiment. Keep this disabled in the frozen Phase-0 pair: +# the experiment removes already-completed USB rehash work after interruption, +# but must earn its place with real-hardware A/B before becoming the default. +HDL_RESUME_HASH_CHECKPOINT ?= 0 +ifeq ($(filter $(HDL_RESUME_HASH_CHECKPOINT),0 1),) +$(error HDL_RESUME_HASH_CHECKPOINT must be 0 or 1) +endif + # LTO lets the R5900 compiler optimize across the deliberately small modules # while section GC still removes unused recovery/UI helpers from the final ELF. EE_CFLAGS = -O2 -flto -G0 -Wall -Wextra -Werror -std=gnu99 -fdata-sections -ffunction-sections -Iinclude -DHDL_PROFILE_ENABLED=$(HDL_PROFILE) +ifeq ($(HDL_RESUME_HASH_CHECKPOINT),1) +EE_CFLAGS += -DHDL_RESUME_HASH_CHECKPOINT_ENABLED=1 +EE_OBJS += hdl_hash_checkpoint.o +endif # Keep a linker map for every build. The R5900 has a 16 KiB I-cache, so archive # provenance, section growth and final placement are performance data, not just # link-time trivia. This also lets CI explain why heavyweight Newlib routines @@ -52,10 +64,11 @@ HOST_HDD_REPAIR_FIXTURE_TEST = tests/test_hdd_repair_fixtures HOST_HDL_ISO_TEST = tests/test_hdl_iso HOST_HDL_PARTITION_TEST = tests/test_hdl_partition HOST_HDL_TRANSACTION_TEST = tests/test_hdl_transaction +HOST_HDL_HASH_CHECKPOINT_TEST = tests/test_hdl_hash_checkpoint HOST_SPLEEN_GENERATED = tests/generated_spleen_font_data.c HOST_HDD_FIXTURE_DIR = tests/generated_hdds HOST_FORENSIC_FIXTURE_DIR = tests/generated_forensic_hdds -HOST_TESTS = $(HOST_APP_ERROR_TEST) $(HOST_FORMAT_TEST) $(HOST_APA_REPAIR_TEST) $(HOST_APA_FORENSIC_TEST) $(HOST_APA_FORENSIC_DORMANT_TEST) $(HOST_VIDEO_MODE_TEST) $(HOST_UI_LAYOUT_TEST) $(HOST_UI_FONT_TEST) $(HOST_FORENSIC_FIXTURE_TEST) $(HOST_BOOT_CHAIN_TEST) $(HOST_BOOT_PAYLOAD_TEST) $(HOST_BOOT_REPORT_TEST) $(HOST_KELF_TEST) $(HOST_BOOTSTRAP_TRANSACTION_TEST) $(HOST_RESCUE_IMAGE_TEST) $(HOST_HDD_FIXTURE_TEST) $(HOST_HDD_MUTATION_TEST) $(HOST_HDD_REPAIR_FIXTURE_TEST) $(HOST_HDL_ISO_TEST) $(HOST_HDL_PARTITION_TEST) $(HOST_HDL_TRANSACTION_TEST) +HOST_TESTS = $(HOST_APP_ERROR_TEST) $(HOST_FORMAT_TEST) $(HOST_APA_REPAIR_TEST) $(HOST_APA_FORENSIC_TEST) $(HOST_APA_FORENSIC_DORMANT_TEST) $(HOST_VIDEO_MODE_TEST) $(HOST_UI_LAYOUT_TEST) $(HOST_UI_FONT_TEST) $(HOST_FORENSIC_FIXTURE_TEST) $(HOST_BOOT_CHAIN_TEST) $(HOST_BOOT_PAYLOAD_TEST) $(HOST_BOOT_REPORT_TEST) $(HOST_KELF_TEST) $(HOST_BOOTSTRAP_TRANSACTION_TEST) $(HOST_RESCUE_IMAGE_TEST) $(HOST_HDD_FIXTURE_TEST) $(HOST_HDD_MUTATION_TEST) $(HOST_HDD_REPAIR_FIXTURE_TEST) $(HOST_HDL_ISO_TEST) $(HOST_HDL_PARTITION_TEST) $(HOST_HDL_TRANSACTION_TEST) $(HOST_HDL_HASH_CHECKPOINT_TEST) all: $(EE_BIN) @@ -149,9 +162,14 @@ test-host: tests/test_hdl_transaction.c src/hdl_transaction.c src/sha256.c \ -o $(HOST_HDL_TRANSACTION_TEST) ./$(HOST_HDL_TRANSACTION_TEST) + $(HOST_CC) -std=c99 -Wall -Wextra -Werror -Iinclude \ + tests/test_hdl_hash_checkpoint.c src/hdl_hash_checkpoint.c src/sha256.c \ + -o $(HOST_HDL_HASH_CHECKPOINT_TEST) + ./$(HOST_HDL_HASH_CHECKPOINT_TEST) clean: rm -f $(EE_BIN) $(EE_MAP) $(EE_OBJS) $(IRX_FILES:.irx=_irx.c) $(HOST_TESTS) + rm -f hdl_hash_checkpoint.o rm -f $(CUSTOM_IRX) hdl_stream_irx.c iop/hdl_stream/*.o \ iop/hdl_stream/*.elf iop/hdl_stream/*.irx rm -f ps2hdd_posix_irx.c @@ -312,6 +330,9 @@ hdl_partition.o: src/hdl_partition.c hdl_transaction.o: src/hdl_transaction.c $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ +hdl_hash_checkpoint.o: src/hdl_hash_checkpoint.c + $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ + hdl_installer_ps2.o: src/hdl_installer_ps2.c $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ From 9a88de9cf3a615ed1e30ede76277c0bbe48e5078 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 22:33:39 +0200 Subject: [PATCH 077/156] perf: add guarded resume hash sidecar I/O --- src/hdl_tools/source_ui.inc | 65 +++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/hdl_tools/source_ui.inc b/src/hdl_tools/source_ui.inc index 8742b99d..462b11ba 100644 --- a/src/hdl_tools/source_ui.inc +++ b/src/hdl_tools/source_ui.inc @@ -20,6 +20,9 @@ #include "disk_status_ps2.h" #include "gs_ui_ps2.h" #include "hdd_read.h" +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED +#include "hdl_hash_checkpoint.h" +#endif #include "hdl_installer_ps2.h" #include "hdl_iso.h" #include "hdl_partition.h" @@ -32,6 +35,10 @@ #define HDL_INSTALL_JOURNAL "mass:/HDLINSTALL.TXN" #define HDL_INSTALL_JOURNAL_NEW "mass:/HDLINSTALL.NEW" +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED +#define HDL_INSTALL_HASH_CHECKPOINT "mass:/HDLINSTALL.SHA" +#define HDL_INSTALL_HASH_CHECKPOINT_NEW "mass:/HDLINSTALL.SHN" +#endif #define HDL_BROWSER_PAGE_SIZE 8u #define HDL_INSTALL_IO_BYTES (64u * 1024u) #define HDL_INSTALL_JOURNAL_INTERVAL_SECTORS 16384u @@ -367,6 +374,57 @@ static int source_identity_matches(hdl_file_source_t *source, return 0; } +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED +static void hash_checkpoint_remove(void) +{ + (void)fileXioRemove(HDL_INSTALL_HASH_CHECKPOINT); + (void)fileXioRemove(HDL_INSTALL_HASH_CHECKPOINT_NEW); +} + +static int hash_checkpoint_save(const hdl_transaction_t *transaction, + const sha256_context_t *context) +{ + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + unsigned char verify[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + sha256_context_t restored; + int result; + + result = hdl_hash_checkpoint_encode(transaction, context, record); + if (result < 0) + return result; + result = write_whole_file(HDL_INSTALL_HASH_CHECKPOINT_NEW, + record, sizeof(record)); + if (result < 0) + return result; + result = read_exact_file(HDL_INSTALL_HASH_CHECKPOINT_NEW, + verify, sizeof(verify)); + if (result < 0 || memcmp(record, verify, sizeof(record)) != 0 || + hdl_hash_checkpoint_restore(verify, transaction, &restored) < 0) + return HDL_INSTALL_JOURNAL_INVALID; + (void)fileXioRemove(HDL_INSTALL_HASH_CHECKPOINT); + result = fileXioRename(HDL_INSTALL_HASH_CHECKPOINT_NEW, + HDL_INSTALL_HASH_CHECKPOINT); + return result < 0 ? result : 0; +} + +static int hash_checkpoint_load(const hdl_transaction_t *transaction, + sha256_context_t *context) +{ + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + int result; + + result = read_exact_file(HDL_INSTALL_HASH_CHECKPOINT, + record, sizeof(record)); + if (result == 0) + return hdl_hash_checkpoint_restore(record, transaction, context); + result = read_exact_file(HDL_INSTALL_HASH_CHECKPOINT_NEW, + record, sizeof(record)); + if (result < 0) + return result; + return hdl_hash_checkpoint_restore(record, transaction, context); +} +#endif + static int journal_save(const hdl_transaction_t *transaction) { unsigned char record[HDL_TRANSACTION_RECORD_SIZE]; @@ -374,6 +432,10 @@ static int journal_save(const hdl_transaction_t *transaction) hdl_transaction_t decoded; int result; +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + if (transaction != NULL && transaction->completed_sectors == 0) + hash_checkpoint_remove(); +#endif result = hdl_transaction_encode(transaction, record); if (result < 0) return result; @@ -407,6 +469,9 @@ static void journal_remove(void) { (void)fileXioRemove(HDL_INSTALL_JOURNAL); (void)fileXioRemove(HDL_INSTALL_JOURNAL_NEW); +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + hash_checkpoint_remove(); +#endif } static int target_path(const char *prefix, const char *target, From 93ee40bed1deaf6cbd1feb6f3d8b3d0018d51c28 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 22:35:00 +0200 Subject: [PATCH 078/156] perf: restore source SHA from durable copy checkpoints --- src/hdl_tools/transaction.inc | 131 ++++++++++++++++++++++++---------- 1 file changed, 93 insertions(+), 38 deletions(-) diff --git a/src/hdl_tools/transaction.inc b/src/hdl_tools/transaction.inc index 7cb1e926..ea68c9a7 100644 --- a/src/hdl_tools/transaction.inc +++ b/src/hdl_tools/transaction.inc @@ -258,11 +258,9 @@ static uint32_t physical_lba(const hdl_partition_plan_t *plan, /* * Hash a source ISO without involving the HDD. This is primarily used when a - * previously PAYLOAD_VERIFIED journal is resumed: the old journal format does - * not persist a full payload SHA-256, so one source pass is still required to - * reconstruct the expected digest. Fresh installs do not pay this second USB - * pass because copy_payload() hashes source bytes while they are already being - * transferred. + * previously PAYLOAD_VERIFIED journal is resumed and no matching optional + * SHA-state checkpoint exists. Fresh installs do not pay this second USB pass + * because copy_payload() hashes source bytes while they are already moving. */ static int hash_source_payload(const hdl_transaction_t *transaction, int source_fd, @@ -321,30 +319,52 @@ static int copy_payload(hdl_transaction_t *transaction, return HDL_INSTALL_MEMORY_FAILED; sha256_init(&source_hash); - /* Resume needs the SHA state for the already-copied prefix. Re-read only - * that prefix from USB, then continue hashing naturally while copying the - * remaining bytes. A fresh install starts at zero and skips this pass. */ + /* Resume needs the SHA state for the already-copied prefix. A validated + * checkpoint makes that prefix O(1) to restore. Missing/stale/corrupt + * checkpoints deliberately fall through to the historical full USB rehash, + * preserving the previous correctness contract. */ if (offset > 0) { - disk_status_phase_at("Rebuilding source digest for resumed copy", - "Already-copied prefix on mass:"); - if (fileXioLseek64(source_fd, 0, FIO_SEEK_SET) != 0) { - result = HDL_INSTALL_COPY_FAILED; - goto done; - } - while (hash_offset < offset) { - uint64_t remaining = offset - hash_offset; - unsigned int bytes = remaining > HDL_INSTALL_IO_BYTES ? - HDL_INSTALL_IO_BYTES : (unsigned int)remaining; - - result = read_exact_fd(source_fd, buffer, bytes); - if (result < 0) +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + int checkpoint_result = hash_checkpoint_load(transaction, &source_hash); + + if (checkpoint_result == 0) { + hash_offset = offset; + session_log_line( + "HDL restored source SHA checkpoint bytes=%llu; skipped prefix rehash", + (unsigned long long)offset); + } else { + session_log_line( + "HDL source SHA checkpoint unavailable result=%d; using safe prefix rehash", + checkpoint_result); +#endif + disk_status_phase_at("Rebuilding source digest for resumed copy", + "Already-copied prefix on mass:"); + if (fileXioLseek64(source_fd, 0, FIO_SEEK_SET) != 0) { + result = HDL_INSTALL_COPY_FAILED; goto done; - sha256_update(&source_hash, buffer, bytes); - hash_offset += bytes; - disk_status_io(DISK_STATUS_VERIFY, 0, 0, - (unsigned int)(hash_offset / 2048u), - (unsigned int)transaction->total_sectors); + } + while (hash_offset < offset) { + uint64_t remaining = offset - hash_offset; + unsigned int bytes = remaining > HDL_INSTALL_IO_BYTES ? + HDL_INSTALL_IO_BYTES : (unsigned int)remaining; + + result = read_exact_fd(source_fd, buffer, bytes); + if (result < 0) + goto done; + sha256_update(&source_hash, buffer, bytes); + hash_offset += bytes; + disk_status_io(DISK_STATUS_VERIFY, 0, 0, + (unsigned int)(hash_offset / 2048u), + (unsigned int)transaction->total_sectors); + } +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + checkpoint_result = hash_checkpoint_save(transaction, &source_hash); + if (checkpoint_result < 0) + session_log_line( + "HDL source SHA checkpoint refresh failed result=%d; resume remains safe", + checkpoint_result); } +#endif } disk_status_phase_at("Copying ISO payload from mass:", @@ -375,6 +395,15 @@ static int copy_payload(hdl_transaction_t *transaction, transaction->completed_sectors = offset / 2048u; if (transaction->completed_sectors >= next_journal || offset == transaction->source_bytes) { +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + int checkpoint_result = hash_checkpoint_save(transaction, &source_hash); + + if (checkpoint_result < 0) + session_log_line( + "HDL source SHA checkpoint save failed progress=%llu result=%d; journal remains authoritative", + (unsigned long long)transaction->completed_sectors, + checkpoint_result); +#endif result = journal_save(transaction); if (result < 0) goto done; @@ -382,6 +411,14 @@ static int copy_payload(hdl_transaction_t *transaction, HDL_INSTALL_JOURNAL_INTERVAL_SECTORS; } if (poll_for_press(&pressed) && (pressed & PAD_TRIANGLE)) { +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + int checkpoint_result = hash_checkpoint_save(transaction, &source_hash); + + if (checkpoint_result < 0) + session_log_line( + "HDL source SHA checkpoint save on cancel failed result=%d", + checkpoint_result); +#endif result = journal_save(transaction); if (result == 0) result = HDL_INSTALL_CANCELLED; @@ -395,11 +432,10 @@ done: } /* - * Verify the installed payload from HDD only. The source digest is either - * accumulated for free while fresh bytes are copied, or reconstructed by one - * source-only pass when resuming a PAYLOAD_VERIFIED journal. This removes the - * old second full USB read from the normal install path while preserving full - * end-to-end SHA-256 verification of every payload byte. + * Verify the installed payload from HDD only. The source digest is accumulated + * while fresh bytes are copied. On resume, an optional authenticated SHA-state + * checkpoint can reconstruct it without replaying already-consumed USB bytes; + * otherwise the historical source-only pass remains the safe baseline. */ static int verify_target_digest(const hdl_transaction_t *transaction, const hdl_partition_plan_t *plan, @@ -562,13 +598,32 @@ static int execute_transaction(hdl_transaction_t *transaction) } if (transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) { if (!verified_this_run) { - disk_status_phase_at("Hashing source for resumed verification", - "One source pass required by legacy journal format"); - result = hash_source_payload(transaction, source.fd, - source_payload_digest); - if (result < 0) - goto done; - source_digest_valid = 1; +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + sha256_context_t restored_source_hash; + int checkpoint_result = + hash_checkpoint_load(transaction, &restored_source_hash); + + if (checkpoint_result == 0) { + sha256_final(&restored_source_hash, source_payload_digest); + source_digest_valid = 1; + session_log_line( + "HDL restored complete source SHA checkpoint bytes=%llu; skipped full USB hash pass", + (unsigned long long)transaction->source_bytes); + } else { + session_log_line( + "HDL complete source SHA checkpoint unavailable result=%d; using safe full source hash", + checkpoint_result); +#endif + disk_status_phase_at("Hashing source for resumed verification", + "One source pass required without a matching hash checkpoint"); + result = hash_source_payload(transaction, source.fd, + source_payload_digest); + if (result < 0) + goto done; + source_digest_valid = 1; +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + } +#endif disk_status_phase_at("Re-verifying resumed payload on HDD", "HDD-only SHA-256 read-back against source digest"); result = verify_target_digest(transaction, &plan, &layout, From 37e9f426f8d0635cd9e3277415417a32a3ce7ef9 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 22:39:12 +0200 Subject: [PATCH 079/156] perf: isolate resume hash experiment from frozen baseline --- src/hdl_tools/source_ui.inc | 65 -- src/hdl_tools/source_ui_resume_hash.inc | 558 +++++++++++++++++ src/hdl_tools/transaction.inc | 131 ++-- src/hdl_tools/transaction_resume_hash.inc | 717 ++++++++++++++++++++++ 4 files changed, 1313 insertions(+), 158 deletions(-) create mode 100644 src/hdl_tools/source_ui_resume_hash.inc create mode 100644 src/hdl_tools/transaction_resume_hash.inc diff --git a/src/hdl_tools/source_ui.inc b/src/hdl_tools/source_ui.inc index 462b11ba..8742b99d 100644 --- a/src/hdl_tools/source_ui.inc +++ b/src/hdl_tools/source_ui.inc @@ -20,9 +20,6 @@ #include "disk_status_ps2.h" #include "gs_ui_ps2.h" #include "hdd_read.h" -#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED -#include "hdl_hash_checkpoint.h" -#endif #include "hdl_installer_ps2.h" #include "hdl_iso.h" #include "hdl_partition.h" @@ -35,10 +32,6 @@ #define HDL_INSTALL_JOURNAL "mass:/HDLINSTALL.TXN" #define HDL_INSTALL_JOURNAL_NEW "mass:/HDLINSTALL.NEW" -#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED -#define HDL_INSTALL_HASH_CHECKPOINT "mass:/HDLINSTALL.SHA" -#define HDL_INSTALL_HASH_CHECKPOINT_NEW "mass:/HDLINSTALL.SHN" -#endif #define HDL_BROWSER_PAGE_SIZE 8u #define HDL_INSTALL_IO_BYTES (64u * 1024u) #define HDL_INSTALL_JOURNAL_INTERVAL_SECTORS 16384u @@ -374,57 +367,6 @@ static int source_identity_matches(hdl_file_source_t *source, return 0; } -#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED -static void hash_checkpoint_remove(void) -{ - (void)fileXioRemove(HDL_INSTALL_HASH_CHECKPOINT); - (void)fileXioRemove(HDL_INSTALL_HASH_CHECKPOINT_NEW); -} - -static int hash_checkpoint_save(const hdl_transaction_t *transaction, - const sha256_context_t *context) -{ - unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; - unsigned char verify[HDL_HASH_CHECKPOINT_RECORD_SIZE]; - sha256_context_t restored; - int result; - - result = hdl_hash_checkpoint_encode(transaction, context, record); - if (result < 0) - return result; - result = write_whole_file(HDL_INSTALL_HASH_CHECKPOINT_NEW, - record, sizeof(record)); - if (result < 0) - return result; - result = read_exact_file(HDL_INSTALL_HASH_CHECKPOINT_NEW, - verify, sizeof(verify)); - if (result < 0 || memcmp(record, verify, sizeof(record)) != 0 || - hdl_hash_checkpoint_restore(verify, transaction, &restored) < 0) - return HDL_INSTALL_JOURNAL_INVALID; - (void)fileXioRemove(HDL_INSTALL_HASH_CHECKPOINT); - result = fileXioRename(HDL_INSTALL_HASH_CHECKPOINT_NEW, - HDL_INSTALL_HASH_CHECKPOINT); - return result < 0 ? result : 0; -} - -static int hash_checkpoint_load(const hdl_transaction_t *transaction, - sha256_context_t *context) -{ - unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; - int result; - - result = read_exact_file(HDL_INSTALL_HASH_CHECKPOINT, - record, sizeof(record)); - if (result == 0) - return hdl_hash_checkpoint_restore(record, transaction, context); - result = read_exact_file(HDL_INSTALL_HASH_CHECKPOINT_NEW, - record, sizeof(record)); - if (result < 0) - return result; - return hdl_hash_checkpoint_restore(record, transaction, context); -} -#endif - static int journal_save(const hdl_transaction_t *transaction) { unsigned char record[HDL_TRANSACTION_RECORD_SIZE]; @@ -432,10 +374,6 @@ static int journal_save(const hdl_transaction_t *transaction) hdl_transaction_t decoded; int result; -#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED - if (transaction != NULL && transaction->completed_sectors == 0) - hash_checkpoint_remove(); -#endif result = hdl_transaction_encode(transaction, record); if (result < 0) return result; @@ -469,9 +407,6 @@ static void journal_remove(void) { (void)fileXioRemove(HDL_INSTALL_JOURNAL); (void)fileXioRemove(HDL_INSTALL_JOURNAL_NEW); -#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED - hash_checkpoint_remove(); -#endif } static int target_path(const char *prefix, const char *target, diff --git a/src/hdl_tools/source_ui_resume_hash.inc b/src/hdl_tools/source_ui_resume_hash.inc new file mode 100644 index 00000000..462b11ba --- /dev/null +++ b/src/hdl_tools/source_ui_resume_hash.inc @@ -0,0 +1,558 @@ +/* Guarded on-console HDLoader installer and large-disk HDL catalogue. */ + +#include +#include +#include +#define NEWLIB_PORT_AWARE +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "apa.h" +#include "app_ui_ps2.h" +#include "disk_status_ps2.h" +#include "gs_ui_ps2.h" +#include "hdd_read.h" +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED +#include "hdl_hash_checkpoint.h" +#endif +#include "hdl_installer_ps2.h" +#include "hdl_iso.h" +#include "hdl_partition.h" +#include "hdl_stream_rpc.h" +#include "hdl_transaction.h" +#include "platform.h" +#include "session_log.h" +#include "sha256.h" +#include "storage.h" + +#define HDL_INSTALL_JOURNAL "mass:/HDLINSTALL.TXN" +#define HDL_INSTALL_JOURNAL_NEW "mass:/HDLINSTALL.NEW" +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED +#define HDL_INSTALL_HASH_CHECKPOINT "mass:/HDLINSTALL.SHA" +#define HDL_INSTALL_HASH_CHECKPOINT_NEW "mass:/HDLINSTALL.SHN" +#endif +#define HDL_BROWSER_PAGE_SIZE 8u +#define HDL_INSTALL_IO_BYTES (64u * 1024u) +#define HDL_INSTALL_JOURNAL_INTERVAL_SECTORS 16384u +#define HDL_APA_TYPE 0x1337u +#define HDL_APA_FLAG_SUB 0x0001u +#define HDL_METADATA_LBA_OFFSET 0x0800u +#define HDL_CHAIN_NODE_LIMIT 16384u +#define HDL_METADATA_NOT_LOADED INT_MIN + +enum { + HDL_INSTALL_CANCELLED = -560, + HDL_INSTALL_NO_IMAGES = -561, + HDL_INSTALL_JOURNAL_INVALID = -562, + HDL_INSTALL_DISK_UNSAFE = -563, + HDL_INSTALL_DISK_TOO_LARGE = -564, + HDL_INSTALL_NO_SPACE = -565, + HDL_INSTALL_TARGET_EXISTS = -566, + HDL_INSTALL_CREATE_FAILED = -567, + HDL_INSTALL_LAYOUT_MISMATCH = -568, + HDL_INSTALL_SOURCE_CHANGED = -569, + HDL_INSTALL_COPY_FAILED = -570, + HDL_INSTALL_VERIFY_FAILED = -571, + HDL_INSTALL_METADATA_FAILED = -572, + HDL_INSTALL_MEMORY_FAILED = -573, + HDL_INSTALL_NO_GAMES = -574, + HDL_INSTALL_TARGET_CHANGED = -575, + HDL_INSTALL_TARGET_BUSY = -576, + HDL_INSTALL_CHAIN_INVALID = -577 +}; + +typedef struct { + char name[128]; + char path[HDL_TRANSACTION_SOURCE_PATH_MAX]; + uint64_t bytes; +} hdl_image_entry_t; + +typedef struct { + int fd; + uint64_t bytes; +} hdl_file_source_t; + +typedef struct { + char id[HDL_PARTITION_ID_MAX]; + hdl_metadata_info_t metadata; + unsigned char metadata_sha256[32]; + uint64_t allocation_bytes; + uint32_t main_lba; + unsigned int apa_partition_count; + int metadata_state; +} hdl_game_entry_t; + +typedef struct { + hdl_game_entry_t *games; + unsigned int count; + unsigned int capacity; + unsigned int hdl_sub_count; + unsigned int expected_sub_count; + uint32_t total_sectors; + uint64_t used_sectors; + uint64_t free_sectors; +} hdl_catalog_t; + +static unsigned char admission_header[APA_HEADER_SIZE] + __attribute__((aligned(64))); + +static uint64_t stat_bytes(const iox_stat_t *stat) +{ + return ((uint64_t)stat->hisize << 32) | stat->size; +} + +static int iso_name(const char *name) +{ + size_t length = strlen(name); + + return length > 4 && name[length - 4] == '.' && + (name[length - 3] == 'i' || name[length - 3] == 'I') && + (name[length - 2] == 's' || name[length - 2] == 'S') && + (name[length - 1] == 'o' || name[length - 1] == 'O'); +} + +static int compare_images(const void *left_value, const void *right_value) +{ + const hdl_image_entry_t *left = left_value; + const hdl_image_entry_t *right = right_value; + + return strcmp(left->name, right->name); +} + +static int scan_mass_images(hdl_image_entry_t **images_out, + unsigned int *count_out) +{ + iox_dirent_t entry; + hdl_image_entry_t *images; + unsigned int capacity = 32u; + unsigned int count = 0; + int directory; + int result; + + if (images_out == NULL || count_out == NULL) + return HDL_INSTALL_NO_IMAGES; + *images_out = NULL; + *count_out = 0; + images = calloc(capacity, sizeof(*images)); + if (images == NULL) + return HDL_INSTALL_MEMORY_FAILED; + directory = fileXioDopen("mass:/"); + if (directory < 0) { + free(images); + return directory; + } + while ((result = fileXioDread(directory, &entry)) > 0) { + hdl_image_entry_t *image; + int written; + + if (!FIO_S_ISREG(entry.stat.mode) || !iso_name(entry.name)) + continue; + if (count == capacity) { + hdl_image_entry_t *grown; + unsigned int next = capacity < 1024u ? capacity * 2u + : capacity + 1024u; + + grown = realloc(images, next * sizeof(*images)); + if (grown == NULL) { + fileXioDclose(directory); + free(images); + return HDL_INSTALL_MEMORY_FAILED; + } + images = grown; + memset(images + capacity, 0, + (next - capacity) * sizeof(*images)); + capacity = next; + } + image = &images[count]; + snprintf(image->name, sizeof(image->name), "%s", entry.name); + written = snprintf(image->path, sizeof(image->path), + "mass:/%s", entry.name); + if (written < 0 || (unsigned int)written >= sizeof(image->path)) + continue; + image->bytes = stat_bytes(&entry.stat); + count++; + } + fileXioDclose(directory); + if (result < 0) { + free(images); + return result; + } + if (count == 0) { + free(images); + return HDL_INSTALL_NO_IMAGES; + } + qsort(images, count, sizeof(*images), compare_images); + *images_out = images; + *count_out = count; + return 0; +} + +static unsigned int page_count(unsigned int count) +{ + return (count + HDL_BROWSER_PAGE_SIZE - 1u) / HDL_BROWSER_PAGE_SIZE; +} + +static unsigned int page_move_selection(unsigned int selected, + unsigned int count, int direction) +{ + unsigned int pages = page_count(count); + unsigned int page = selected / HDL_BROWSER_PAGE_SIZE; + unsigned int row = selected % HDL_BROWSER_PAGE_SIZE; + unsigned int start; + unsigned int shown; + + if (pages <= 1u) + return selected; + if (direction < 0) + page = (page + pages - 1u) % pages; + else + page = (page + 1u) % pages; + start = page * HDL_BROWSER_PAGE_SIZE; + shown = count - start; + if (shown > HDL_BROWSER_PAGE_SIZE) + shown = HDL_BROWSER_PAGE_SIZE; + if (row >= shown) + row = shown - 1u; + return start + row; +} + +static int select_image(const hdl_image_entry_t *images, unsigned int count) +{ + unsigned int selected = 0; + + for (;;) { + const char *labels[HDL_BROWSER_PAGE_SIZE]; + const char *hints[HDL_BROWSER_PAGE_SIZE]; + unsigned char enabled[HDL_BROWSER_PAGE_SIZE]; + unsigned int page = selected / HDL_BROWSER_PAGE_SIZE; + unsigned int start = page * HDL_BROWSER_PAGE_SIZE; + unsigned int shown = count - start; + unsigned int local = selected - start; + unsigned int i; + char size_hint[HDL_BROWSER_PAGE_SIZE][48]; + char status[160]; + u32 pressed; + + if (shown > HDL_BROWSER_PAGE_SIZE) + shown = HDL_BROWSER_PAGE_SIZE; + for (i = 0; i < shown; i++) { + labels[i] = images[start + i].name; + snprintf(size_hint[i], sizeof(size_hint[i]), "%llu MiB", + (unsigned long long)(images[start + i].bytes / + 1024u / 1024u)); + hints[i] = size_hint[i]; + enabled[i] = 1u; + } + snprintf(status, sizeof(status), + "%u ISO%s | page %u/%u | %llu MiB | LEFT/RIGHT page", + count, count == 1 ? "" : "s", page + 1u, page_count(count), + (unsigned long long)(images[selected].bytes / + 1024u / 1024u)); + gs_ui_render_menu("HDL installer: choose ISO", status, + labels, hints, enabled, shown, local); + pressed = wait_for_press(); + if (pressed & PAD_UP) { + if (local == 0) + selected = start + shown - 1u; + else + selected--; + } + if (pressed & PAD_DOWN) { + if (local + 1u >= shown) + selected = start; + else + selected++; + } + if (pressed & PAD_LEFT) + selected = page_move_selection(selected, count, -1); + if (pressed & PAD_RIGHT) + selected = page_move_selection(selected, count, 1); + if (pressed & PAD_CROSS) + return (int)selected; + if (pressed & PAD_TRIANGLE) + return -1; + } +} + +static int source_read(void *context, uint64_t offset, + void *destination, size_t size) +{ + hdl_file_source_t *source = context; + unsigned char *cursor = destination; + size_t complete = 0; + + if (source == NULL || offset > source->bytes || + (uint64_t)size > source->bytes - offset || + fileXioLseek64(source->fd, (s64)offset, FIO_SEEK_SET) != (s64)offset) + return -1; + while (complete < size) { + int result = fileXioRead(source->fd, cursor + complete, + (int)(size - complete)); + + if (result <= 0) + return -1; + complete += (unsigned int)result; + } + return 0; +} + +static int open_source(const char *path, uint64_t expected, + hdl_file_source_t *source) +{ + iox_stat_t stat; + + if (fileXioGetStat(path, &stat) < 0 || stat_bytes(&stat) != expected) + return HDL_INSTALL_SOURCE_CHANGED; + source->fd = fileXioOpen(path, FIO_O_RDONLY, 0); + if (source->fd < 0) + return source->fd; + source->bytes = expected; + return 0; +} + +static int source_fingerprint(hdl_file_source_t *source, + unsigned char digest[32]) +{ + unsigned char *buffer; + unsigned char size_bytes[8]; + sha256_context_t hash; + uint64_t tail; + unsigned int amount; + unsigned int i; + int result = 0; + + buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; + sha256_init(&hash); + for (i = 0; i < sizeof(size_bytes); i++) + size_bytes[i] = (unsigned char)(source->bytes >> (i * 8u)); + sha256_update(&hash, size_bytes, sizeof(size_bytes)); + amount = source->bytes < HDL_INSTALL_IO_BYTES ? + (unsigned int)source->bytes : HDL_INSTALL_IO_BYTES; + if (source_read(source, 0, buffer, amount) < 0) { + result = HDL_INSTALL_SOURCE_CHANGED; + goto done; + } + sha256_update(&hash, buffer, amount); + tail = source->bytes > HDL_INSTALL_IO_BYTES ? + source->bytes - HDL_INSTALL_IO_BYTES : 0; + if (source_read(source, tail, buffer, amount) < 0) { + result = HDL_INSTALL_SOURCE_CHANGED; + goto done; + } + sha256_update(&hash, buffer, amount); + sha256_final(&hash, digest); +done: + free(buffer); + return result; +} + +static int source_identity_matches(hdl_file_source_t *source, + const hdl_transaction_t *transaction) +{ + hdl_iso_source_t iso_source; + hdl_iso_info_t info; + int result; + + iso_source.read = source_read; + iso_source.context = source; + iso_source.image_bytes = source->bytes; + result = hdl_iso_probe(&iso_source, &info); + if (result < 0 || info.requires_layer_break || + info.image_bytes != transaction->source_bytes || + strcmp(info.startup, transaction->startup) != 0) + return HDL_INSTALL_SOURCE_CHANGED; + return 0; +} + +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED +static void hash_checkpoint_remove(void) +{ + (void)fileXioRemove(HDL_INSTALL_HASH_CHECKPOINT); + (void)fileXioRemove(HDL_INSTALL_HASH_CHECKPOINT_NEW); +} + +static int hash_checkpoint_save(const hdl_transaction_t *transaction, + const sha256_context_t *context) +{ + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + unsigned char verify[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + sha256_context_t restored; + int result; + + result = hdl_hash_checkpoint_encode(transaction, context, record); + if (result < 0) + return result; + result = write_whole_file(HDL_INSTALL_HASH_CHECKPOINT_NEW, + record, sizeof(record)); + if (result < 0) + return result; + result = read_exact_file(HDL_INSTALL_HASH_CHECKPOINT_NEW, + verify, sizeof(verify)); + if (result < 0 || memcmp(record, verify, sizeof(record)) != 0 || + hdl_hash_checkpoint_restore(verify, transaction, &restored) < 0) + return HDL_INSTALL_JOURNAL_INVALID; + (void)fileXioRemove(HDL_INSTALL_HASH_CHECKPOINT); + result = fileXioRename(HDL_INSTALL_HASH_CHECKPOINT_NEW, + HDL_INSTALL_HASH_CHECKPOINT); + return result < 0 ? result : 0; +} + +static int hash_checkpoint_load(const hdl_transaction_t *transaction, + sha256_context_t *context) +{ + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + int result; + + result = read_exact_file(HDL_INSTALL_HASH_CHECKPOINT, + record, sizeof(record)); + if (result == 0) + return hdl_hash_checkpoint_restore(record, transaction, context); + result = read_exact_file(HDL_INSTALL_HASH_CHECKPOINT_NEW, + record, sizeof(record)); + if (result < 0) + return result; + return hdl_hash_checkpoint_restore(record, transaction, context); +} +#endif + +static int journal_save(const hdl_transaction_t *transaction) +{ + unsigned char record[HDL_TRANSACTION_RECORD_SIZE]; + unsigned char verify[HDL_TRANSACTION_RECORD_SIZE]; + hdl_transaction_t decoded; + int result; + +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + if (transaction != NULL && transaction->completed_sectors == 0) + hash_checkpoint_remove(); +#endif + result = hdl_transaction_encode(transaction, record); + if (result < 0) + return result; + result = write_whole_file(HDL_INSTALL_JOURNAL_NEW, record, sizeof(record)); + if (result < 0) + return result; + result = read_exact_file(HDL_INSTALL_JOURNAL_NEW, verify, sizeof(verify)); + if (result < 0 || memcmp(record, verify, sizeof(record)) != 0 || + hdl_transaction_decode(verify, &decoded) < 0) + return HDL_INSTALL_JOURNAL_INVALID; + (void)fileXioRemove(HDL_INSTALL_JOURNAL); + result = fileXioRename(HDL_INSTALL_JOURNAL_NEW, HDL_INSTALL_JOURNAL); + return result < 0 ? result : 0; +} + +static int journal_load(hdl_transaction_t *transaction) +{ + unsigned char record[HDL_TRANSACTION_RECORD_SIZE]; + int result = read_exact_file(HDL_INSTALL_JOURNAL, record, sizeof(record)); + + if (result == 0 && hdl_transaction_decode(record, transaction) == 0) + return 0; + result = read_exact_file(HDL_INSTALL_JOURNAL_NEW, + record, sizeof(record)); + if (result < 0) + return result; + return hdl_transaction_decode(record, transaction); +} + +static void journal_remove(void) +{ + (void)fileXioRemove(HDL_INSTALL_JOURNAL); + (void)fileXioRemove(HDL_INSTALL_JOURNAL_NEW); +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + hash_checkpoint_remove(); +#endif +} + +static int target_path(const char *prefix, const char *target, + char *destination, unsigned int capacity) +{ + int written = snprintf(destination, capacity, "%s%s", prefix, target); + + return written < 0 || (unsigned int)written >= capacity ? -1 : 0; +} + +static int read_target_metadata(const char *target, + unsigned char metadata[HDL_METADATA_SIZE]) +{ + char path[64]; + unsigned int complete = 0; + int fd; + int result; + + if (target_path("hdd0:", target, path, sizeof(path)) < 0) + return HDL_INSTALL_LAYOUT_MISMATCH; + fd = fileXioOpen(path, FIO_O_RDONLY, 0); + if (fd < 0) + return fd; + result = fileXioLseek(fd, 0x100000, FIO_SEEK_SET); + if (result != 0x100000) { + fileXioClose(fd); + return result < 0 ? result : HDL_INSTALL_METADATA_FAILED; + } + while (complete < HDL_METADATA_SIZE) { + result = fileXioRead(fd, metadata + complete, + HDL_METADATA_SIZE - complete); + if (result <= 0) { + fileXioClose(fd); + return result < 0 ? result : HDL_INSTALL_METADATA_FAILED; + } + complete += (unsigned int)result; + } + fileXioClose(fd); + return 0; +} + +static int target_has_metadata(const char *target) +{ + unsigned char metadata[HDL_METADATA_SIZE]; + int result = read_target_metadata(target, metadata); + + if (result < 0) + return result; + return metadata[0] == 0xed && metadata[1] == 0xfe && + metadata[2] == 0xad && metadata[3] == 0xde; +} + +static int target_metadata_matches(const char *target, + const unsigned char expected[HDL_METADATA_SIZE]) +{ + unsigned char actual[HDL_METADATA_SIZE]; + int result = read_target_metadata(target, actual); + + return result == 0 && memcmp(actual, expected, sizeof(actual)) == 0; +} + +static int target_exists(const char *target) +{ + iox_stat_t stat; + char path[64]; + + if (target_path("hdd0:", target, path, sizeof(path)) < 0) + return 1; + return fileXioGetStat(path, &stat) >= 0; +} + +static int remove_incomplete_target(const char *target) +{ + char path[64]; + int metadata_state = target_has_metadata(target); + + if (metadata_state > 0) + return HDL_INSTALL_TARGET_EXISTS; + if (metadata_state < 0) + return metadata_state; + if (target_path("hdd0:", target, path, sizeof(path)) < 0) + return HDL_INSTALL_LAYOUT_MISMATCH; + return fileXioRemove(path); +} diff --git a/src/hdl_tools/transaction.inc b/src/hdl_tools/transaction.inc index ea68c9a7..7cb1e926 100644 --- a/src/hdl_tools/transaction.inc +++ b/src/hdl_tools/transaction.inc @@ -258,9 +258,11 @@ static uint32_t physical_lba(const hdl_partition_plan_t *plan, /* * Hash a source ISO without involving the HDD. This is primarily used when a - * previously PAYLOAD_VERIFIED journal is resumed and no matching optional - * SHA-state checkpoint exists. Fresh installs do not pay this second USB pass - * because copy_payload() hashes source bytes while they are already moving. + * previously PAYLOAD_VERIFIED journal is resumed: the old journal format does + * not persist a full payload SHA-256, so one source pass is still required to + * reconstruct the expected digest. Fresh installs do not pay this second USB + * pass because copy_payload() hashes source bytes while they are already being + * transferred. */ static int hash_source_payload(const hdl_transaction_t *transaction, int source_fd, @@ -319,52 +321,30 @@ static int copy_payload(hdl_transaction_t *transaction, return HDL_INSTALL_MEMORY_FAILED; sha256_init(&source_hash); - /* Resume needs the SHA state for the already-copied prefix. A validated - * checkpoint makes that prefix O(1) to restore. Missing/stale/corrupt - * checkpoints deliberately fall through to the historical full USB rehash, - * preserving the previous correctness contract. */ + /* Resume needs the SHA state for the already-copied prefix. Re-read only + * that prefix from USB, then continue hashing naturally while copying the + * remaining bytes. A fresh install starts at zero and skips this pass. */ if (offset > 0) { -#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED - int checkpoint_result = hash_checkpoint_load(transaction, &source_hash); - - if (checkpoint_result == 0) { - hash_offset = offset; - session_log_line( - "HDL restored source SHA checkpoint bytes=%llu; skipped prefix rehash", - (unsigned long long)offset); - } else { - session_log_line( - "HDL source SHA checkpoint unavailable result=%d; using safe prefix rehash", - checkpoint_result); -#endif - disk_status_phase_at("Rebuilding source digest for resumed copy", - "Already-copied prefix on mass:"); - if (fileXioLseek64(source_fd, 0, FIO_SEEK_SET) != 0) { - result = HDL_INSTALL_COPY_FAILED; + disk_status_phase_at("Rebuilding source digest for resumed copy", + "Already-copied prefix on mass:"); + if (fileXioLseek64(source_fd, 0, FIO_SEEK_SET) != 0) { + result = HDL_INSTALL_COPY_FAILED; + goto done; + } + while (hash_offset < offset) { + uint64_t remaining = offset - hash_offset; + unsigned int bytes = remaining > HDL_INSTALL_IO_BYTES ? + HDL_INSTALL_IO_BYTES : (unsigned int)remaining; + + result = read_exact_fd(source_fd, buffer, bytes); + if (result < 0) goto done; - } - while (hash_offset < offset) { - uint64_t remaining = offset - hash_offset; - unsigned int bytes = remaining > HDL_INSTALL_IO_BYTES ? - HDL_INSTALL_IO_BYTES : (unsigned int)remaining; - - result = read_exact_fd(source_fd, buffer, bytes); - if (result < 0) - goto done; - sha256_update(&source_hash, buffer, bytes); - hash_offset += bytes; - disk_status_io(DISK_STATUS_VERIFY, 0, 0, - (unsigned int)(hash_offset / 2048u), - (unsigned int)transaction->total_sectors); - } -#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED - checkpoint_result = hash_checkpoint_save(transaction, &source_hash); - if (checkpoint_result < 0) - session_log_line( - "HDL source SHA checkpoint refresh failed result=%d; resume remains safe", - checkpoint_result); + sha256_update(&source_hash, buffer, bytes); + hash_offset += bytes; + disk_status_io(DISK_STATUS_VERIFY, 0, 0, + (unsigned int)(hash_offset / 2048u), + (unsigned int)transaction->total_sectors); } -#endif } disk_status_phase_at("Copying ISO payload from mass:", @@ -395,15 +375,6 @@ static int copy_payload(hdl_transaction_t *transaction, transaction->completed_sectors = offset / 2048u; if (transaction->completed_sectors >= next_journal || offset == transaction->source_bytes) { -#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED - int checkpoint_result = hash_checkpoint_save(transaction, &source_hash); - - if (checkpoint_result < 0) - session_log_line( - "HDL source SHA checkpoint save failed progress=%llu result=%d; journal remains authoritative", - (unsigned long long)transaction->completed_sectors, - checkpoint_result); -#endif result = journal_save(transaction); if (result < 0) goto done; @@ -411,14 +382,6 @@ static int copy_payload(hdl_transaction_t *transaction, HDL_INSTALL_JOURNAL_INTERVAL_SECTORS; } if (poll_for_press(&pressed) && (pressed & PAD_TRIANGLE)) { -#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED - int checkpoint_result = hash_checkpoint_save(transaction, &source_hash); - - if (checkpoint_result < 0) - session_log_line( - "HDL source SHA checkpoint save on cancel failed result=%d", - checkpoint_result); -#endif result = journal_save(transaction); if (result == 0) result = HDL_INSTALL_CANCELLED; @@ -432,10 +395,11 @@ done: } /* - * Verify the installed payload from HDD only. The source digest is accumulated - * while fresh bytes are copied. On resume, an optional authenticated SHA-state - * checkpoint can reconstruct it without replaying already-consumed USB bytes; - * otherwise the historical source-only pass remains the safe baseline. + * Verify the installed payload from HDD only. The source digest is either + * accumulated for free while fresh bytes are copied, or reconstructed by one + * source-only pass when resuming a PAYLOAD_VERIFIED journal. This removes the + * old second full USB read from the normal install path while preserving full + * end-to-end SHA-256 verification of every payload byte. */ static int verify_target_digest(const hdl_transaction_t *transaction, const hdl_partition_plan_t *plan, @@ -598,32 +562,13 @@ static int execute_transaction(hdl_transaction_t *transaction) } if (transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) { if (!verified_this_run) { -#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED - sha256_context_t restored_source_hash; - int checkpoint_result = - hash_checkpoint_load(transaction, &restored_source_hash); - - if (checkpoint_result == 0) { - sha256_final(&restored_source_hash, source_payload_digest); - source_digest_valid = 1; - session_log_line( - "HDL restored complete source SHA checkpoint bytes=%llu; skipped full USB hash pass", - (unsigned long long)transaction->source_bytes); - } else { - session_log_line( - "HDL complete source SHA checkpoint unavailable result=%d; using safe full source hash", - checkpoint_result); -#endif - disk_status_phase_at("Hashing source for resumed verification", - "One source pass required without a matching hash checkpoint"); - result = hash_source_payload(transaction, source.fd, - source_payload_digest); - if (result < 0) - goto done; - source_digest_valid = 1; -#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED - } -#endif + disk_status_phase_at("Hashing source for resumed verification", + "One source pass required by legacy journal format"); + result = hash_source_payload(transaction, source.fd, + source_payload_digest); + if (result < 0) + goto done; + source_digest_valid = 1; disk_status_phase_at("Re-verifying resumed payload on HDD", "HDD-only SHA-256 read-back against source digest"); result = verify_target_digest(transaction, &plan, &layout, diff --git a/src/hdl_tools/transaction_resume_hash.inc b/src/hdl_tools/transaction_resume_hash.inc new file mode 100644 index 00000000..ea68c9a7 --- /dev/null +++ b/src/hdl_tools/transaction_resume_hash.inc @@ -0,0 +1,717 @@ +static const char *allocation_name(uint64_t bytes) +{ + static const char *const names[] = {"128M", "256M", "512M", + "1G", "2G", "4G"}; + static const uint64_t sizes[] = { + 128ull << 20, 256ull << 20, 512ull << 20, + 1024ull << 20, 2048ull << 20, 4096ull << 20 + }; + unsigned int i; + + for (i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i++) { + if (bytes == sizes[i]) + return names[i]; + } + return NULL; +} + +static int journal_blocks_game_delete(const char *target) +{ + hdl_transaction_t transaction; + int result; + + if (!path_exists(HDL_INSTALL_JOURNAL) && + !path_exists(HDL_INSTALL_JOURNAL_NEW)) + return 0; + result = journal_load(&transaction); + if (result < 0) + return HDL_INSTALL_JOURNAL_INVALID; + if (transaction.stage != HDL_TRANSACTION_STAGE_COMPLETE && + strcmp(transaction.target, target) == 0) + return HDL_INSTALL_TARGET_BUSY; + return 0; +} + +static int verify_game_snapshot(const hdl_game_entry_t *game, + uint32_t total_sectors) +{ + unsigned char metadata[HDL_METADATA_SIZE] __attribute__((aligned(64))); + unsigned char digest[32]; + hdl_metadata_info_t parsed; + iox_stat_t stat; + char path[64]; + uint32_t lba; + int result; + + if (game->metadata_state != 0 || + target_path("hdd0:", game->id, path, sizeof(path)) < 0) + return HDL_INSTALL_TARGET_CHANGED; + result = fileXioGetStat(path, &stat); + if (result < 0 || stat.mode != HDL_APA_TYPE || + (stat.attr & HDL_APA_FLAG_SUB)) + return HDL_INSTALL_TARGET_CHANGED; + if (game->main_lba > UINT32_MAX - HDL_METADATA_LBA_OFFSET) + return HDL_INSTALL_TARGET_CHANGED; + lba = game->main_lba + HDL_METADATA_LBA_OFFSET; + if (lba >= total_sectors || total_sectors - lba < 2u || + hdd_read_raw_sectors(lba, 2, metadata) < 0 || + hdl_metadata_parse(metadata, &parsed) < 0 || + parsed.partition_count != game->apa_partition_count) + return HDL_INSTALL_TARGET_CHANGED; + sha256_buffer(metadata, sizeof(metadata), digest); + if (memcmp(digest, game->metadata_sha256, sizeof(digest)) != 0) + return HDL_INSTALL_TARGET_CHANGED; + return 0; +} + +static int delete_installed_game(const hdl_game_entry_t *game, + uint32_t total_sectors) +{ + uint32_t maximum; + uint32_t free_sectors; + char path[64]; + int result; + + /* Before the user arms deletion these are genuinely read-only safety + * checks, so keep the ordinary READ monitor semantics here. */ + result = recheck_disk(&maximum, &free_sectors); + if (result == 0) + result = journal_blocks_game_delete(game->id); + if (result == 0) + result = verify_game_snapshot(game, total_sectors); + if (result < 0) + return result; + (void)maximum; + (void)free_sectors; + + scr_clear(); + scr_printf("Delete installed HDL game\n\n"); + scr_printf("Game : %s\n", game->metadata.game_title); + scr_printf("Startup: %s\n", game->metadata.startup); + scr_printf("Target : %s\n", game->id); + scr_printf("Size : %llu MiB allocated\n\n", + (unsigned long long)(game->allocation_bytes / 1024u / 1024u)); + scr_printf("This removes the main partition and all of its subs.\n"); + scr_printf("There is no undo operation.\n\n"); + scr_printf("Hold L1+R1 and press SQUARE to delete.\n"); + scr_printf("TRIANGLE returns without changing the HDD.\n"); + if (!wait_for_chord(PAD_L1 | PAD_R1 | PAD_SQUARE)) + return HDL_INSTALL_CANCELLED; + + /* Once destructive confirmation has been accepted, validation reads are + * no longer an independent read-only operation: they are the preflight of + * an armed write path. Keep ACTION truthful, but never display the green + * read-only footer during this window. */ + disk_status_begin_at("HDL game removal", + "Write path armed; revalidating selected target", + "Selected HDL main/sub partition chain"); + disk_status_set_write_intent(1); + result = recheck_disk(&maximum, &free_sectors); + if (result == 0) + result = journal_blocks_game_delete(game->id); + if (result == 0) + result = verify_game_snapshot(game, total_sectors); + if (result < 0) { + disk_status_end(); + return result; + } + if (target_path("hdd0:", game->id, path, sizeof(path)) < 0) { + disk_status_end(); + return HDL_INSTALL_TARGET_CHANGED; + } + + disk_status_phase_at("Removing selected APA allocation", + "APA metadata and linked HDL sub-partitions"); + disk_status_io(DISK_STATUS_WRITE, game->main_lba, 2u, 0, 0); + result = fileXioRemove(path); + disk_status_end(); + session_log_line("HDL game delete target=%s startup=%s result=%d", + game->id, game->metadata.startup, result); + return result < 0 ? result : 0; +} + +static int create_partitions(const hdl_transaction_t *transaction, + const hdl_partition_plan_t *plan) +{ + char existing[64]; + char create[96]; + int fd; + unsigned int i; + + target_path("hdd0:", transaction->target, existing, sizeof(existing)); + fd = fileXioOpen(existing, FIO_O_RDONLY, 0); + if (fd >= 0) { + fileXioClose(fd); + return HDL_INSTALL_TARGET_EXISTS; + } + disk_status_phase_at("Creating HDL main partition", + "APA allocation metadata"); + disk_status_io(DISK_STATUS_WRITE, 0, 0, 0, 0); + snprintf(create, sizeof(create), "hdd0:%s,,,%s,HDL", + transaction->target, + allocation_name(plan->slices[0].allocation_bytes)); + fd = fileXioOpen(create, FIO_O_RDWR | FIO_O_CREAT, 0666); + if (fd < 0) + return fd; + for (i = 1; i < plan->count; i++) { + const char *size = allocation_name(plan->slices[i].allocation_bytes); + int result; + + disk_status_phase_at("Adding HDL sub-partition", + "APA allocation metadata"); + disk_status_io(DISK_STATUS_WRITE, 0, 0, i, plan->count); + result = fileXioIoctl2(fd, HIOCADDSUB, (void *)size, + strlen(size) + 1u, NULL, 0); + + if (result < 0) { + fileXioClose(fd); + (void)remove_incomplete_target(transaction->target); + return HDL_INSTALL_CREATE_FAILED; + } + } + disk_status_phase_at("Flushing APA allocation metadata", + "New HDL main/sub partition chain"); + disk_status_io(DISK_STATUS_FLUSH, 0, 0, plan->count, plan->count); + if (fileXioIoctl2(fd, HIOCFLUSH, NULL, 0, NULL, 0) < 0) { + fileXioClose(fd); + (void)remove_incomplete_target(transaction->target); + return HDL_INSTALL_CREATE_FAILED; + } + fileXioClose(fd); + return 0; +} + +static int open_target(const hdl_transaction_t *transaction, + const hdl_partition_plan_t *plan, + hdl_stream_layout_t *layout) +{ + char path[64]; + int fd; + unsigned int i; + + if (target_path("hdl0:", transaction->target, path, sizeof(path)) < 0) + return HDL_INSTALL_LAYOUT_MISMATCH; + fd = fileXioOpen(path, FIO_O_RDWR, 0); + if (fd < 0) + return fd; + if (fileXioIoctl2(fd, HDL_STREAM_IOCTL2_GET_LAYOUT, + NULL, 0, layout, sizeof(*layout)) < 0 || + layout->count != plan->count) { + fileXioClose(fd); + return HDL_INSTALL_LAYOUT_MISMATCH; + } + for (i = 0; i < plan->count; i++) { + if ((uint64_t)layout->lengths[i] * 512u != + plan->slices[i].allocation_bytes || layout->starts[i] == 0) { + fileXioClose(fd); + return HDL_INSTALL_LAYOUT_MISMATCH; + } + } + return fd; +} + +static int read_exact_fd(int fd, unsigned char *buffer, unsigned int bytes) +{ + unsigned int complete = 0; + + while (complete < bytes) { + int result = fileXioRead(fd, buffer + complete, bytes - complete); + + if (result <= 0) + return result < 0 ? result : HDL_INSTALL_COPY_FAILED; + complete += (unsigned int)result; + } + return 0; +} + +static int write_exact_fd(int fd, const unsigned char *buffer, + unsigned int bytes) +{ + unsigned int complete = 0; + + while (complete < bytes) { + int result = fileXioWrite(fd, buffer + complete, bytes - complete); + + if (result <= 0) + return result < 0 ? result : HDL_INSTALL_COPY_FAILED; + complete += (unsigned int)result; + } + return 0; +} + +static uint32_t physical_lba(const hdl_partition_plan_t *plan, + const hdl_stream_layout_t *layout, + uint64_t payload_offset) +{ + unsigned int i; + + for (i = 0; i < plan->count; i++) { + uint64_t begin = plan->slices[i].payload_offset; + uint64_t end = begin + plan->slices[i].payload_bytes; + + if (payload_offset >= begin && payload_offset < end) + return layout->starts[i] + (i == 0 ? 0x2000u : 0x0800u) + + (uint32_t)((payload_offset - begin) / 512u); + } + return 0; +} + +/* + * Hash a source ISO without involving the HDD. This is primarily used when a + * previously PAYLOAD_VERIFIED journal is resumed and no matching optional + * SHA-state checkpoint exists. Fresh installs do not pay this second USB pass + * because copy_payload() hashes source bytes while they are already moving. + */ +static int hash_source_payload(const hdl_transaction_t *transaction, + int source_fd, + unsigned char digest[32]) +{ + unsigned char *buffer; + sha256_context_t hash; + uint64_t offset = 0; + int result = 0; + + buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; + if (fileXioLseek64(source_fd, 0, FIO_SEEK_SET) != 0) { + result = HDL_INSTALL_VERIFY_FAILED; + goto done; + } + sha256_init(&hash); + while (offset < transaction->source_bytes) { + uint64_t remaining = transaction->source_bytes - offset; + unsigned int bytes = remaining > HDL_INSTALL_IO_BYTES ? + HDL_INSTALL_IO_BYTES : (unsigned int)remaining; + + if (read_exact_fd(source_fd, buffer, bytes) < 0) { + result = HDL_INSTALL_VERIFY_FAILED; + goto done; + } + sha256_update(&hash, buffer, bytes); + offset += bytes; + disk_status_io(DISK_STATUS_VERIFY, 0, 0, + (unsigned int)(offset / 2048u), + (unsigned int)transaction->total_sectors); + } + sha256_final(&hash, digest); +done: + free(buffer); + return result; +} + +static int copy_payload(hdl_transaction_t *transaction, + const hdl_partition_plan_t *plan, + const hdl_stream_layout_t *layout, + int source_fd, int target_fd, + unsigned char source_digest[32]) +{ + unsigned char *buffer; + sha256_context_t source_hash; + uint64_t offset = transaction->completed_sectors * 2048u; + uint64_t hash_offset = 0; + uint64_t next_journal = transaction->completed_sectors + + HDL_INSTALL_JOURNAL_INTERVAL_SECTORS; + int result = 0; + + buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; + sha256_init(&source_hash); + + /* Resume needs the SHA state for the already-copied prefix. A validated + * checkpoint makes that prefix O(1) to restore. Missing/stale/corrupt + * checkpoints deliberately fall through to the historical full USB rehash, + * preserving the previous correctness contract. */ + if (offset > 0) { +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + int checkpoint_result = hash_checkpoint_load(transaction, &source_hash); + + if (checkpoint_result == 0) { + hash_offset = offset; + session_log_line( + "HDL restored source SHA checkpoint bytes=%llu; skipped prefix rehash", + (unsigned long long)offset); + } else { + session_log_line( + "HDL source SHA checkpoint unavailable result=%d; using safe prefix rehash", + checkpoint_result); +#endif + disk_status_phase_at("Rebuilding source digest for resumed copy", + "Already-copied prefix on mass:"); + if (fileXioLseek64(source_fd, 0, FIO_SEEK_SET) != 0) { + result = HDL_INSTALL_COPY_FAILED; + goto done; + } + while (hash_offset < offset) { + uint64_t remaining = offset - hash_offset; + unsigned int bytes = remaining > HDL_INSTALL_IO_BYTES ? + HDL_INSTALL_IO_BYTES : (unsigned int)remaining; + + result = read_exact_fd(source_fd, buffer, bytes); + if (result < 0) + goto done; + sha256_update(&source_hash, buffer, bytes); + hash_offset += bytes; + disk_status_io(DISK_STATUS_VERIFY, 0, 0, + (unsigned int)(hash_offset / 2048u), + (unsigned int)transaction->total_sectors); + } +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + checkpoint_result = hash_checkpoint_save(transaction, &source_hash); + if (checkpoint_result < 0) + session_log_line( + "HDL source SHA checkpoint refresh failed result=%d; resume remains safe", + checkpoint_result); + } +#endif + } + + disk_status_phase_at("Copying ISO payload from mass:", + "HDL payload data area"); + if (fileXioLseek64(source_fd, (s64)offset, FIO_SEEK_SET) != (s64)offset || + fileXioLseek64(target_fd, (s64)offset, FIO_SEEK_SET) != (s64)offset) { + result = HDL_INSTALL_COPY_FAILED; + goto done; + } + while (offset < transaction->source_bytes) { + uint64_t remaining = transaction->source_bytes - offset; + unsigned int bytes = remaining > HDL_INSTALL_IO_BYTES ? + HDL_INSTALL_IO_BYTES : (unsigned int)remaining; + u32 pressed = 0; + + result = read_exact_fd(source_fd, buffer, bytes); + if (result < 0) + goto done; + sha256_update(&source_hash, buffer, bytes); + result = write_exact_fd(target_fd, buffer, bytes); + if (result < 0) + goto done; + offset += bytes; + disk_status_io(DISK_STATUS_WRITE, + physical_lba(plan, layout, offset - bytes), bytes / 512u, + (unsigned int)(offset / 2048u), + (unsigned int)transaction->total_sectors); + transaction->completed_sectors = offset / 2048u; + if (transaction->completed_sectors >= next_journal || + offset == transaction->source_bytes) { +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + int checkpoint_result = hash_checkpoint_save(transaction, &source_hash); + + if (checkpoint_result < 0) + session_log_line( + "HDL source SHA checkpoint save failed progress=%llu result=%d; journal remains authoritative", + (unsigned long long)transaction->completed_sectors, + checkpoint_result); +#endif + result = journal_save(transaction); + if (result < 0) + goto done; + next_journal = transaction->completed_sectors + + HDL_INSTALL_JOURNAL_INTERVAL_SECTORS; + } + if (poll_for_press(&pressed) && (pressed & PAD_TRIANGLE)) { +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + int checkpoint_result = hash_checkpoint_save(transaction, &source_hash); + + if (checkpoint_result < 0) + session_log_line( + "HDL source SHA checkpoint save on cancel failed result=%d", + checkpoint_result); +#endif + result = journal_save(transaction); + if (result == 0) + result = HDL_INSTALL_CANCELLED; + goto done; + } + } + sha256_final(&source_hash, source_digest); +done: + free(buffer); + return result; +} + +/* + * Verify the installed payload from HDD only. The source digest is accumulated + * while fresh bytes are copied. On resume, an optional authenticated SHA-state + * checkpoint can reconstruct it without replaying already-consumed USB bytes; + * otherwise the historical source-only pass remains the safe baseline. + */ +static int verify_target_digest(const hdl_transaction_t *transaction, + const hdl_partition_plan_t *plan, + const hdl_stream_layout_t *layout, + int target_fd, + const unsigned char expected_digest[32]) +{ + unsigned char *target_buffer; + unsigned char target_digest[32]; + sha256_context_t target_hash; + uint64_t offset = 0; + int result = 0; + + target_buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (target_buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; + if (fileXioLseek64(target_fd, 0, FIO_SEEK_SET) != 0) { + result = HDL_INSTALL_VERIFY_FAILED; + goto done; + } + sha256_init(&target_hash); + while (offset < transaction->source_bytes) { + uint64_t remaining = transaction->source_bytes - offset; + unsigned int bytes = remaining > HDL_INSTALL_IO_BYTES ? + HDL_INSTALL_IO_BYTES : (unsigned int)remaining; + + if (read_exact_fd(target_fd, target_buffer, bytes) < 0) { + result = HDL_INSTALL_VERIFY_FAILED; + goto done; + } + sha256_update(&target_hash, target_buffer, bytes); + disk_status_io(DISK_STATUS_VERIFY, + physical_lba(plan, layout, offset), bytes / 512u, + (unsigned int)(offset / 2048u), + (unsigned int)transaction->total_sectors); + offset += bytes; + } + sha256_final(&target_hash, target_digest); + if (memcmp(expected_digest, target_digest, sizeof(target_digest)) != 0) + result = HDL_INSTALL_VERIFY_FAILED; +done: + free(target_buffer); + return result; +} + +static int execute_transaction(hdl_transaction_t *transaction) +{ + hdl_partition_plan_t plan; + hdl_stream_layout_t layout; + hdl_metadata_options_t metadata_options; + unsigned char metadata[HDL_METADATA_SIZE]; + hdl_file_source_t source; + unsigned char fingerprint[32]; + unsigned char source_payload_digest[32]; + uint32_t maximum; + uint32_t free_sectors; + int target_fd = -1; + int verified_this_run = 0; + int source_digest_valid = 0; + int metadata_valid = 0; + int result; + + source.fd = -1; + source.bytes = transaction->source_bytes; + result = recheck_disk(&maximum, &free_sectors); + if (result < 0) + return result; + result = hdl_partition_plan(transaction->source_bytes, maximum, &plan); + if (result < 0) + return result; + if (transaction->partition_count != plan.count) + return HDL_INSTALL_LAYOUT_MISMATCH; + if (plan.allocation_bytes / 512u > free_sectors && + transaction->stage == HDL_TRANSACTION_STAGE_PLANNED) + return HDL_INSTALL_NO_SPACE; + + /* Once METADATA_COMMITTED is durable, payload and source identity have + * already passed full verification. Completion recovery only needs the + * target layout plus metadata read-back, so do not require the USB ISO to + * remain connected merely to move stage 5 -> COMPLETE. */ + if (transaction->stage < HDL_TRANSACTION_STAGE_METADATA_COMMITTED) { + result = open_source(transaction->source_path, + transaction->source_bytes, &source); + if (result < 0) + return result; + result = source_fingerprint(&source, fingerprint); + if (result == 0 && memcmp(fingerprint, transaction->source_fingerprint, + sizeof(fingerprint)) != 0) + result = HDL_INSTALL_SOURCE_CHANGED; + if (result == 0) + result = source_identity_matches(&source, transaction); + if (result < 0) { + fileXioClose(source.fd); + return result; + } + } + + pad_activity_begin(); + disk_status_begin_at("HDL game installation", + "Preparing guarded APA allocation", + "Standard HDL main/sub partitions"); + /* Reaching execute_transaction means the user has already authorized an + * install or resume. Validation may still read first, but the transaction + * owns an armed write path until it exits. */ + disk_status_set_write_intent(1); + if (transaction->stage == HDL_TRANSACTION_STAGE_PLANNED) { + result = create_partitions(transaction, &plan); + if (result < 0) + goto done; + result = hdl_transaction_set_stage( + transaction, HDL_TRANSACTION_STAGE_PARTITIONS_CREATED, 0); + if (result < 0 || journal_save(transaction) < 0) { + result = HDL_INSTALL_JOURNAL_INVALID; + goto done; + } + } + target_fd = open_target(transaction, &plan, &layout); + if (target_fd < 0) { + result = target_fd; + goto done; + } + if (transaction->stage == HDL_TRANSACTION_STAGE_PARTITIONS_CREATED) { + result = hdl_transaction_set_stage( + transaction, HDL_TRANSACTION_STAGE_COPYING, 0); + if (result < 0 || journal_save(transaction) < 0) { + result = HDL_INSTALL_JOURNAL_INVALID; + goto done; + } + } + if (transaction->stage == HDL_TRANSACTION_STAGE_COPYING) { + result = copy_payload(transaction, &plan, &layout, + source.fd, target_fd, source_payload_digest); + if (result < 0) + goto done; + source_digest_valid = 1; + disk_status_phase_at("Flushing copied HDL payload", + "HDL payload data area"); + disk_status_io(DISK_STATUS_FLUSH, + layout.starts[0], 0, transaction->total_sectors, + transaction->total_sectors); + if (fileXioIoctl2(target_fd, HDL_STREAM_IOCTL2_FLUSH, + NULL, 0, NULL, 0) < 0) { + result = HDL_INSTALL_COPY_FAILED; + goto done; + } + disk_status_phase_at("Verifying installed payload on HDD", + "HDD-only SHA-256 read-back against source digest"); + result = verify_target_digest(transaction, &plan, &layout, + target_fd, source_payload_digest); + if (result < 0) + goto done; + verified_this_run = 1; + result = hdl_transaction_set_stage( + transaction, HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED, + transaction->total_sectors); + if (result < 0 || journal_save(transaction) < 0) { + result = HDL_INSTALL_JOURNAL_INVALID; + goto done; + } + } + if (transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) { + if (!verified_this_run) { +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + sha256_context_t restored_source_hash; + int checkpoint_result = + hash_checkpoint_load(transaction, &restored_source_hash); + + if (checkpoint_result == 0) { + sha256_final(&restored_source_hash, source_payload_digest); + source_digest_valid = 1; + session_log_line( + "HDL restored complete source SHA checkpoint bytes=%llu; skipped full USB hash pass", + (unsigned long long)transaction->source_bytes); + } else { + session_log_line( + "HDL complete source SHA checkpoint unavailable result=%d; using safe full source hash", + checkpoint_result); +#endif + disk_status_phase_at("Hashing source for resumed verification", + "One source pass required without a matching hash checkpoint"); + result = hash_source_payload(transaction, source.fd, + source_payload_digest); + if (result < 0) + goto done; + source_digest_valid = 1; +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + } +#endif + disk_status_phase_at("Re-verifying resumed payload on HDD", + "HDD-only SHA-256 read-back against source digest"); + result = verify_target_digest(transaction, &plan, &layout, + target_fd, source_payload_digest); + if (result < 0) + goto done; + } + if (!source_digest_valid && !verified_this_run) { + result = HDL_INSTALL_VERIFY_FAILED; + goto done; + } + metadata_options.game_title = transaction->game_title; + metadata_options.startup = transaction->startup; + metadata_options.disc_type = transaction->disc_type; + metadata_options.layer1_start = transaction->layer1_start; + metadata_options.hdl_compat_flags = transaction->hdl_compat_flags; + metadata_options.opl_compat_flags = transaction->opl_compat_flags; + metadata_options.dma_type = transaction->dma_type; + metadata_options.dma_mode = transaction->dma_mode; + result = hdl_metadata_build(&plan, layout.starts, layout.count, + &metadata_options, metadata); + if (result < 0) + goto done; + metadata_valid = 1; + disk_status_phase_at("Committing verified game metadata last", + "Main partition attribute area / HDL metadata"); + disk_status_io(DISK_STATUS_WRITE, + layout.starts[0] + HDL_METADATA_LBA_OFFSET, + 2u, transaction->total_sectors, + transaction->total_sectors); + result = fileXioIoctl2(target_fd, + HDL_STREAM_IOCTL2_COMMIT_METADATA, + metadata, sizeof(metadata), NULL, 0); + if (result < 0) { + result = HDL_INSTALL_METADATA_FAILED; + goto done; + } + result = hdl_transaction_set_stage( + transaction, HDL_TRANSACTION_STAGE_METADATA_COMMITTED, + transaction->total_sectors); + if (result < 0 || journal_save(transaction) < 0) { + result = HDL_INSTALL_JOURNAL_INVALID; + goto done; + } + } + if (transaction->stage == HDL_TRANSACTION_STAGE_METADATA_COMMITTED) { + /* A stage-4 -> stage-5 transition in this invocation already built the + * exact canonical block that was committed. Keep using that same + * transaction-owned buffer for read-back comparison. Only a resumed + * stage-5 transaction needs to reconstruct it. */ + if (!metadata_valid) { + metadata_options.game_title = transaction->game_title; + metadata_options.startup = transaction->startup; + metadata_options.disc_type = transaction->disc_type; + metadata_options.layer1_start = transaction->layer1_start; + metadata_options.hdl_compat_flags = transaction->hdl_compat_flags; + metadata_options.opl_compat_flags = transaction->opl_compat_flags; + metadata_options.dma_type = transaction->dma_type; + metadata_options.dma_mode = transaction->dma_mode; + result = hdl_metadata_build(&plan, layout.starts, layout.count, + &metadata_options, metadata); + if (result < 0) { + result = HDL_INSTALL_METADATA_FAILED; + goto done; + } + } + if (!target_metadata_matches(transaction->target, metadata)) { + result = HDL_INSTALL_METADATA_FAILED; + goto done; + } + result = hdl_transaction_set_stage( + transaction, HDL_TRANSACTION_STAGE_COMPLETE, + transaction->total_sectors); + if (result < 0) + goto done; + journal_remove(); + } + result = 0; +done: + if (target_fd >= 0) + fileXioClose(target_fd); + if (source.fd >= 0) + fileXioClose(source.fd); + disk_status_end(); + pad_activity_end(); + session_log_line("HDL transaction target=%s stage=%u progress=%llu/%llu result=%d", + transaction->target, (unsigned int)transaction->stage, + (unsigned long long)transaction->completed_sectors, + (unsigned long long)transaction->total_sectors, result); + return result; +} From 68c93d33cded44acdaaeaf7d5e271630b090327f Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 22:40:00 +0200 Subject: [PATCH 080/156] perf: add isolated resume hash experiment build --- tools/build_resume_hash_experiment.sh | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 tools/build_resume_hash_experiment.sh diff --git a/tools/build_resume_hash_experiment.sh b/tools/build_resume_hash_experiment.sh new file mode 100644 index 00000000..ee8a97e3 --- /dev/null +++ b/tools/build_resume_hash_experiment.sh @@ -0,0 +1,40 @@ +#!/bin/sh +set -eu + +# Build the resume-hash experiment without letting its alternate fragments +# become the default source tree. The frozen PROFILE pair is built and validated +# separately before this script is run in CI. +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +BACKUP=$(mktemp -d) +SOURCE_UI="$ROOT/src/hdl_tools/source_ui.inc" +TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" + +restore_sources() { + if [ -f "$BACKUP/source_ui.inc" ]; then + cp "$BACKUP/source_ui.inc" "$SOURCE_UI" + fi + if [ -f "$BACKUP/transaction.inc" ]; then + cp "$BACKUP/transaction.inc" "$TRANSACTION" + fi + rm -rf "$BACKUP" +} +trap restore_sources EXIT HUP INT TERM + +cp "$SOURCE_UI" "$BACKUP/source_ui.inc" +cp "$TRANSACTION" "$BACKUP/transaction.inc" +cp "$ROOT/src/hdl_tools/source_ui_resume_hash.inc" "$SOURCE_UI" +cp "$ROOT/src/hdl_tools/transaction_resume_hash.inc" "$TRANSACTION" + +cd "$ROOT" +make clean +make HDL_PROFILE=0 HDL_RESUME_HASH_CHECKPOINT=1 +python3 tools/optimization_audit.py \ + --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF \ + --output OPTIMIZATION_AUDIT_RESUME_HASH.txt +make HDL_PROFILE=0 HDL_RESUME_HASH_CHECKPOINT=1 release +cp PS2_HDD_BOOTSTRAP_MANAGER.ELF PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF +cp PS2_HDD_BOOTSTRAP_MANAGER.map PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.map +sha256sum PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF \ + > PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.sha256 +wc -c PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF \ + > PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.size From ba9703af6c54850935bdc53ebd03d7f424960e87 Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 22:42:06 +0200 Subject: [PATCH 081/156] ci: build isolated HDL resume hash experiment --- .github/workflows/ci.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a259184a..3548d7f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,6 +129,32 @@ jobs: cat OPTIMIZATION_AUDIT_PROFILE_OFF.txt sha256sum PS2_HDD_BOOTSTRAP_MANAGER.ELF | tee PS2_HDD_BOOTSTRAP_MANAGER.ELF.sha256 sha256sum PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF | tee HDL_PROFILE_PAIR.sha256 + - name: Build isolated resume-hash experiment after frozen gate + run: >- + docker run --rm + -v "$PWD:/work" -w /work ps2dev/ps2dev:v2.0.0 + sh -c 'apk add --no-cache make python3 >/dev/null && + sh tools/build_resume_hash_experiment.sh && + mips64r5900el-ps2-elf-size + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF + > PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.sections && + cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF + PS2_HDD_BOOTSTRAP_MANAGER.ELF && + cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.map + PS2_HDD_BOOTSTRAP_MANAGER.map' + - name: Record resume-hash experiment identity + run: | + echo "--- resume-hash experiment ---" + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.sha256 + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.size + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.sections + echo "--- resume-hash optimization audit ---" + cat OPTIMIZATION_AUDIT_RESUME_HASH.txt + python3 tools/phase0_profile_pair_preflight.py \ + --profile-on PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF \ + --profile-off PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF \ + --project-git-sha "${{ github.event.pull_request.head.sha || github.sha }}" + sha256sum PS2_HDD_BOOTSTRAP_MANAGER.ELF | tee PS2_HDD_BOOTSTRAP_MANAGER.ELF.sha256 - uses: actions/upload-artifact@v4 with: name: PS2-HDD-Bootstrap-Manager @@ -159,6 +185,12 @@ jobs: bench/r5900_calibration/R5900_COUNTER_CALIBRATION.map bench/r5900_calibration/R5900_COUNTER_CALIBRATION.sections bench/r5900_calibration/R5900_COUNTER_CALIBRATION.disasm + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.sha256 + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.size + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.map + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.sections + OPTIMIZATION_AUDIT_RESUME_HASH.txt HDDMAN.CFG LICENSE THIRD_PARTY_NOTICES.md From 01b380f307368346e6921a8a5b9146ad5ce9776b Mon Sep 17 00:00:00 2001 From: Hifu Date: Wed, 26 Aug 2026 22:43:00 +0200 Subject: [PATCH 082/156] docs: define HDL resume hash checkpoint hardware gate --- docs/HDL_RESUME_HASH_BENCHMARK.md | 276 ++++++++++++++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 docs/HDL_RESUME_HASH_BENCHMARK.md diff --git a/docs/HDL_RESUME_HASH_BENCHMARK.md b/docs/HDL_RESUME_HASH_BENCHMARK.md new file mode 100644 index 00000000..867ddba9 --- /dev/null +++ b/docs/HDL_RESUME_HASH_BENCHMARK.md @@ -0,0 +1,276 @@ +# HDL resume-hash checkpoint hardware benchmark + +This document defines the real-PS2 gate for the optional +`HDL_RESUME_HASH_CHECKPOINT=1` experiment. + +The experiment is deliberately not the default build. It exists to test the +first optimization rule from the PS2 Optimization Research Library v2: remove +work before trying to make the same work faster. + +## Epistemic status + +**POTWIERDZONE** + +- the existing COPY resume path reconstructs the SHA-256 state by reading the + entire already-copied ISO prefix again; +- a resumed `PAYLOAD_VERIFIED` transaction without a persisted source digest + performs another complete source-ISO hash pass; +- the normal transaction journal remains authoritative; +- the optional checkpoint is a 256-byte authenticated sidecar bound to source + size, completed byte count, source fingerprint and target ID; +- a missing, corrupt or mismatching sidecar falls back to the old rehash path; +- host tests prove that a restored SHA state produces the same final digest as + an uninterrupted hash and that altered checkpoint records are rejected. + +**CURRENT IMPLEMENTATION** + +- transaction journal interval: 16384 ISO sectors = 32 MiB; +- experiment build: `HDL_PROFILE=0 HDL_RESUME_HASH_CHECKPOINT=1`; +- baseline performance build: frozen `HDL_PROFILE=0` ELF; +- the experiment writes/verifies/renames a 256-byte sidecar at durable copy + checkpoints and on an orderly cancel; +- the sidecar is deleted when a new zero-progress transaction supersedes it and + when a transaction reaches COMPLETE. + +**INFERENCJA** + +- removing an N-byte prefix replay should save approximately N bytes of USB + traffic on a matching COPY resume; +- restoring a full-payload checkpoint should remove one full source-ISO pass + before resumed HDD verification; +- the small sidecar operations may add measurable latency or jitter to an + uninterrupted install even though their byte volume is tiny. + +**HIPOTEZA DO TESTU** + +- recovery time improves materially on real PS2; +- the uninterrupted-copy regression from checkpoint maintenance is negligible; +- no adapter/USB implementation exposes a correctness or persistence corner + case not covered by host tests. + +No performance claim is accepted before this hardware gate passes. + +## Authoritative corpus rationale + +This test follows: + +- `PS2_PERFORMANCE_BIBLE.md`: remove work first; preserve ELF/map/toolchain, + correctness hash and benchmark provenance; distinguish instrumented and + release-like profiles; +- `PS2_HDD_APA_PFS_HDL_filesystem_optimization_research_corpus_v2.md`: keep HDL + raw/storage workloads separate, use deterministic checksum validation, and do + not infer throughput from interface headline rates; +- `PS2_Whole_System_Scheduling_research_corpus_v2.md`: treat USB, IOP, SIF and + storage as one producer/consumer path and report distribution/tail behavior, + not only an average; +- `PS2_USB_1_1_optimization_research_corpus.md`: USB Full-Speed scheduling has + millisecond-scale granularity, so redundant source traffic belongs on the + critical recovery path rather than being dismissed as free background work. + +## Frozen baseline identity + +Use the Phase-0 PROFILE OFF binary only after the preflight tool accepts it: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF +bytes 632884 +sha256 4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916 +``` + +The matching embedded PROFILE OFF `hdl_stream.irx` is: + +```text +bytes 8405 +sha256 f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 +``` + +Do not rebuild and silently call the result the same baseline. The frozen hash +is the identity. + +The experiment must come from the same CI artifact as the baseline and must +have its own recorded SHA-256 and section sizes. + +## Hardware record + +Record before the first measured run: + +```text +SCPH / hardware revision +ROMVER +HDD model or SSD model +network/HDD adapter model and revision +USB mass-storage device model +USB filesystem and relevant allocation state +PS2SDK commit +PS2DEV/toolchain version +active IRX set +baseline ELF SHA-256 +experiment ELF SHA-256 +source ISO SHA-256 +source ISO byte count +target HDD free-space/allocation state +``` + +Keep the same console, adapter, USB device, ISO, target disk, video mode and +launch method for the whole A/B block. + +## Correctness gate before timing + +For both binaries: + +1. start a fresh install; +2. cancel only through the normal guarded TRIANGLE path after at least one + durable journal checkpoint; +3. restart and choose resume; +4. complete payload verification and metadata commit; +5. confirm the final installed game is catalogued with the expected startup and + title; +6. preserve the transaction/session log; +7. verify the same final source/payload correctness hash in both variants. + +Any checksum mismatch, invalid target metadata, unexpected cleanup behavior or +resume refusal invalidates the performance sample. + +## Workload A: uninterrupted-install regression + +Purpose: measure the cost paid when recovery is never needed. + +Use one deterministic ISO large enough to cross many 32 MiB journal intervals. +Do not change the ISO between A and B. + +For each binary, perform at least eight interleaved fresh-install runs. Restore +an equivalent target state before each run. + +Recommended order: + +```text +OFF, EXP, EXP, OFF, EXP, OFF, OFF, EXP +``` + +Measure separately where the existing logs permit: + +```text +copy phase elapsed time +payload verification elapsed time +total transaction elapsed time +copy throughput +journal/checkpoint failures +correctness result +``` + +Report p50, p95, p99 and max. With only eight initial samples the tail +percentiles are coarse; treat them as a smoke gate and collect a larger sample +set if the result is near the acceptance boundary. + +Do not accept an improvement in recovery if uninterrupted installs acquire a +large or erratic regression. + +## Workload B: resumed COPY + +Purpose: measure removal of the already-copied-prefix replay. + +Choose three durable resume depths separated across the ISO. Prefer journal +boundaries near approximately 25%, 50% and 75% of source progress. Record the +exact `completed_sectors` from the journal rather than assuming the requested +percentage was hit. + +For each depth: + +1. perform a fresh copy to the selected durable checkpoint; +2. cancel normally; +3. preserve the journal and, for EXP, its checkpoint sidecar; +4. reboot/relaunch in the same way for each run; +5. start timing immediately before accepting resume; +6. stop the recovery-start metric when the first payload progress beyond the + stored `completed_sectors` is observed; +7. continue the install to completion for correctness. + +Derived redundant work for the legacy baseline is: + +```text +prefix_rehash_bytes = completed_sectors * 2048 +``` + +For a valid EXP checkpoint, expected skipped prefix bytes are the same number. +A session-log line must confirm checkpoint restore. If EXP falls back to safe +rehash, classify that run separately rather than pretending it was an optimized +sample. + +Report for each depth: + +```text +resume-to-first-new-progress p50/p95/p99/max +full resumed-transaction p50/p95/p99/max +completed_sectors +prefix_rehash_bytes baseline +checkpoint_restored_bytes experiment +fallback count +correctness failures +``` + +## Workload C: resumed PAYLOAD_VERIFIED + +Purpose: measure removal of the complete extra USB source-hash pass. + +Create a valid `PAYLOAD_VERIFIED` recovery point with the source ISO still +available. Preserve identical transaction state for baseline and experiment +runs as far as the format permits. + +Expected work difference: + +```text +baseline: one full source ISO SHA-256 pass, then HDD payload verification +EXP: restore final source SHA state, then HDD payload verification +``` + +The HDD verification remains required in both variants. Do not count its time as +work removed by this experiment. + +Report: + +```text +resume-to-HDD-verification-start p50/p95/p99/max +full recovery p50/p95/p99/max +source ISO bytes +checkpoint restore/fallback status +correctness failures +``` + +## Crash-window matrix + +The sidecar is not journal authority, so deliberately test these states before +promotion: + +```text +sidecar absent +sidecar checksum corrupt +sidecar from an earlier completed_sectors value +sidecar from another source fingerprint +sidecar from another target ID +valid .SHA primary +valid temporary .SHN with primary absent +``` + +Every invalid/mismatched case must fall back to the legacy rehash path and still +complete correctly. It must never advance transaction progress by itself. + +Use the existing guarded hardware fault-injection procedure only where its +safety preconditions are satisfied. Do not pull power during arbitrary APA +metadata writes merely to make the benchmark more exciting. The console has +suffered enough. + +## Acceptance rule + +Promote the checkpoint path only if all are true: + +1. zero correctness failures across the test matrix; +2. frozen baseline identity remains unchanged; +3. matching checkpoint resumes actually skip the expected redundant source + bytes; +4. COPY-resume and PAYLOAD_VERIFIED recovery show a clear real-hardware benefit; +5. uninterrupted-install p50/p95/p99/max do not show an unacceptable regression + or new long-tail spikes; +6. sidecar failure always degrades to the old safe rehash behavior. + +If the result is ambiguous, keep the flag off and collect more samples. A neat +architecture diagram is not a benchmark result. From fe3cb7925b427b9ab1f70f47a5a2f1d159ab08a6 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 20:28:17 +0200 Subject: [PATCH 083/156] bench: record resume-hash build provenance --- tools/build_benchmark_provenance.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tools/build_benchmark_provenance.sh b/tools/build_benchmark_provenance.sh index de50d2b2..a8429ff8 100755 --- a/tools/build_benchmark_provenance.sh +++ b/tools/build_benchmark_provenance.sh @@ -12,6 +12,7 @@ PS2SDK_REF=${PS2SDK_SOURCE_REF:-unavailable} PS2SDK_SHA=${PS2SDK_SOURCE_SHA:-unavailable} PS2DEV_BUNDLE_REF=${PS2DEV_BUNDLE_REF:-unavailable} HDL_PROFILE_VALUE=${HDL_PROFILE:-1} +HDL_RESUME_HASH_CHECKPOINT_VALUE=${HDL_RESUME_HASH_CHECKPOINT:-0} BENCHMARK_ELF_PATH=${BENCHMARK_ELF:-PS2_HDD_BOOTSTRAP_MANAGER.ELF} HDL_STREAM_IRX_PATH=${HDL_STREAM_IRX:-hdl_stream.irx} @@ -23,6 +24,21 @@ case "$HDL_PROFILE_VALUE" in ;; esac +case "$HDL_RESUME_HASH_CHECKPOINT_VALUE" in + 0|1) ;; + *) + printf 'HDL_RESUME_HASH_CHECKPOINT must be 0 or 1, got %s\n' \ + "$HDL_RESUME_HASH_CHECKPOINT_VALUE" >&2 + exit 2 + ;; +esac + +if [ "$HDL_RESUME_HASH_CHECKPOINT_VALUE" = "1" ]; then + RESUME_HASH_EE_FLAG=" -DHDL_RESUME_HASH_CHECKPOINT_ENABLED=1" +else + RESUME_HASH_EE_FLAG="" +fi + # A development environment may preserve the ps2sdk .git directory. Prefer the # exact installed checkout when available. Tagged ps2dev Docker images strip # source metadata after installation, so CI passes the source ref/SHA that the @@ -76,7 +92,8 @@ toolchain_target: "$CC_TARGET" toolchain_gcc: "$CC_VERSION" toolchain_container: "ps2dev/ps2dev:v2.0.0" hdl_profile_enabled: "$HDL_PROFILE_VALUE" -build_flags: "EE: -O2 -flto -G0 -fdata-sections -ffunction-sections -DHDL_PROFILE_ENABLED=$HDL_PROFILE_VALUE; ld --gc-sections; hdl_stream IOP: PS2SDK -Os baseline with appended -O2 -DHDL_PROFILE_ENABLED=$HDL_PROFILE_VALUE" +hdl_resume_hash_checkpoint_enabled: "$HDL_RESUME_HASH_CHECKPOINT_VALUE" +build_flags: "EE: -O2 -flto -G0 -fdata-sections -ffunction-sections -DHDL_PROFILE_ENABLED=$HDL_PROFILE_VALUE$RESUME_HASH_EE_FLAG; ld --gc-sections; hdl_stream IOP: PS2SDK -Os baseline with appended -O2 -DHDL_PROFILE_ENABLED=$HDL_PROFILE_VALUE" benchmark_elf: "$BENCHMARK_ELF_PATH" benchmark_elf_sha256: "$BENCHMARK_ELF_SHA" benchmark_elf_bytes: "$BENCHMARK_ELF_BYTES" From 00c49525263172cb2de51a0614715d63f6b0e2d0 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 20:28:38 +0200 Subject: [PATCH 084/156] bench: build matched resume-hash A/B variants --- tools/build_resume_hash_experiment.sh | 59 +++++++++++++++++++++------ 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/tools/build_resume_hash_experiment.sh b/tools/build_resume_hash_experiment.sh index ee8a97e3..b4fdfdb4 100644 --- a/tools/build_resume_hash_experiment.sh +++ b/tools/build_resume_hash_experiment.sh @@ -4,6 +4,10 @@ set -eu # Build the resume-hash experiment without letting its alternate fragments # become the default source tree. The frozen PROFILE pair is built and validated # separately before this script is run in CI. +# +# Build both profiler modes so hardware work can keep one variable per A/B: +# frozen PROFILE OFF vs resume-hash PROFILE OFF -> release-like timing +# frozen PROFILE ON vs resume-hash PROFILE ON -> identical telemetry ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) BACKUP=$(mktemp -d) SOURCE_UI="$ROOT/src/hdl_tools/source_ui.inc" @@ -25,16 +29,47 @@ cp "$TRANSACTION" "$BACKUP/transaction.inc" cp "$ROOT/src/hdl_tools/source_ui_resume_hash.inc" "$SOURCE_UI" cp "$ROOT/src/hdl_tools/transaction_resume_hash.inc" "$TRANSACTION" +build_variant() +{ + profile=$1 + label=$2 + elf="PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_${label}.ELF" + map="PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_${label}.map" + irx="HDL_STREAM_RESUME_HASH_PROFILE_${label}.irx" + audit="OPTIMIZATION_AUDIT_RESUME_HASH_PROFILE_${label}.txt" + provenance="BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_${label}.yml" + + make clean + make HDL_PROFILE="$profile" HDL_RESUME_HASH_CHECKPOINT=1 + cp hdl_stream.irx "$irx" + python3 tools/optimization_audit.py \ + --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF \ + --output "$audit" + make HDL_PROFILE="$profile" HDL_RESUME_HASH_CHECKPOINT=1 release + cp PS2_HDD_BOOTSTRAP_MANAGER.ELF "$elf" + cp PS2_HDD_BOOTSTRAP_MANAGER.map "$map" + sha256sum "$elf" > "$elf.sha256" + wc -c "$elf" > "$elf.size" + HDL_PROFILE="$profile" \ + HDL_RESUME_HASH_CHECKPOINT=1 \ + BENCHMARK_ELF="$elf" \ + HDL_STREAM_IRX="$irx" \ + sh tools/build_benchmark_provenance.sh "$provenance" +} + cd "$ROOT" -make clean -make HDL_PROFILE=0 HDL_RESUME_HASH_CHECKPOINT=1 -python3 tools/optimization_audit.py \ - --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF \ - --output OPTIMIZATION_AUDIT_RESUME_HASH.txt -make HDL_PROFILE=0 HDL_RESUME_HASH_CHECKPOINT=1 release -cp PS2_HDD_BOOTSTRAP_MANAGER.ELF PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF -cp PS2_HDD_BOOTSTRAP_MANAGER.map PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.map -sha256sum PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF \ - > PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.sha256 -wc -c PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF \ - > PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.size +build_variant 0 OFF +build_variant 1 ON + +# Preserve the original experiment artifact names as PROFILE OFF aliases while +# the benchmark documentation/tools migrate to the explicit pair names. +cp PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF \ + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF +cp PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.map \ + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.map +cp PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF.sha256 \ + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.sha256 +cp PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF.size \ + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.size +cp OPTIMIZATION_AUDIT_RESUME_HASH_PROFILE_OFF.txt \ + OPTIMIZATION_AUDIT_RESUME_HASH.txt From 5669b55f718409388bc1461d0b0b2c739895ca96 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 20:29:25 +0200 Subject: [PATCH 085/156] bench: add resume-hash A/B preflight --- tools/resume_hash_ab_preflight.py | 275 ++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 tools/resume_hash_ab_preflight.py diff --git a/tools/resume_hash_ab_preflight.py b/tools/resume_hash_ab_preflight.py new file mode 100644 index 00000000..afd1224b --- /dev/null +++ b/tools/resume_hash_ab_preflight.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""Bind the optional HDL resume-hash experiment to the frozen A/B baseline. + +The checkpoint experiment is EE-only. A valid CI artifact must therefore keep +both frozen baseline ELFs exact, build one checkpoint ELF for PROFILE OFF and +one for PROFILE ON, and keep each experiment's embedded hdl_stream IRX byte- +identical to the corresponding frozen profiler mode. + +The emitted identity and sample templates are the authoritative bridge between +CI artifacts and real-PS2 measurements. Experiment ELF hashes are intentionally +computed from the current artifact rather than hard-coded forever; any runtime +change therefore creates a new identity instead of silently inheriting an old +benchmark label. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import tempfile +from pathlib import Path +from typing import Any + +FROZEN_SOURCE_GIT_SHA = "7875b14d837d6332f5edc37f1c12a55527d7dd87" +FROZEN = { + "ON": { + "elf_bytes": 638388, + "elf_sha256": "964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7", + "irx_bytes": 9861, + "irx_sha256": "8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db", + }, + "OFF": { + "elf_bytes": 632884, + "elf_sha256": "4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916", + "irx_bytes": 8405, + "irx_sha256": "f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751", + }, +} +DEFAULT_ORDER = ("BASE", "EXP", "EXP", "BASE", "EXP", "BASE", "BASE", "EXP") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def inspect_blob(path: Path) -> dict[str, Any]: + return {"bytes": path.stat().st_size, "sha256": _sha256(path)} + + +def verify_frozen(path: Path, mode: str, kind: str) -> dict[str, Any]: + observed = inspect_blob(path) + expected_bytes = int(FROZEN[mode][f"{kind}_bytes"]) + expected_sha = str(FROZEN[mode][f"{kind}_sha256"]) + if observed["bytes"] != expected_bytes: + raise ValueError( + f"{path}: {mode} {kind} size {observed['bytes']}, expected {expected_bytes}" + ) + if observed["sha256"] != expected_sha: + raise ValueError( + f"{path}: {mode} {kind} sha256 {observed['sha256']}, expected {expected_sha}" + ) + return observed + + +def build_identity( + project_git_sha: str, + baseline: dict[str, dict[str, Any]], + experiment: dict[str, dict[str, Any]], +) -> dict[str, Any]: + if not project_git_sha: + raise ValueError("project_git_sha must be non-empty") + + profiles: dict[str, Any] = {} + for mode in ("OFF", "ON"): + base = baseline[mode] + exp = experiment[mode] + if exp["elf"]["sha256"] == base["elf"]["sha256"]: + raise ValueError(f"PROFILE {mode} experiment ELF is byte-identical to baseline") + if exp["irx"] != base["irx"]: + raise ValueError( + f"PROFILE {mode} experiment IRX differs from frozen baseline; " + "resume-hash must remain EE-only" + ) + profiles[mode] = { + "baseline": base, + "experiment": exp, + } + + return { + "project_git_sha": project_git_sha, + "frozen_baseline_source_git_sha": FROZEN_SOURCE_GIT_SHA, + "experiment": "HDL_RESUME_HASH_CHECKPOINT=1", + "profiles": profiles, + } + + +def sample_template(identity: dict[str, Any], profile: str) -> dict[str, Any]: + profile = profile.upper() + if profile not in ("OFF", "ON"): + raise ValueError("profile must be OFF or ON") + pair = identity["profiles"][profile] + project_git_sha = identity["project_git_sha"] + samples: list[dict[str, Any]] = [] + for run, mode in enumerate(DEFAULT_ORDER, start=1): + variant = pair["baseline" if mode == "BASE" else "experiment"] + samples.append({ + "run": run, + "profile": profile, + "mode": mode, + "project_git_sha": project_git_sha, + "benchmark_elf_sha256": variant["elf"]["sha256"], + "hdl_stream_irx_sha256": variant["irx"]["sha256"], + "workload_kind": "FILL_ME", + "workload_id": "FILL_ME", + "correctness_hash": "FILL_ME", + "source_bytes": 0, + "total_us": 0, + "copy_us": 0, + "verify_us": 0, + "resume_gate_us": 0, + "completed_sectors": 0, + "checkpoint_status": "FILL_ME", + "checkpoint_restored_bytes": 0, + }) + return { + "identity": { + "project_git_sha": project_git_sha, + "profile": profile, + "baseline_elf_sha256": pair["baseline"]["elf"]["sha256"], + "experiment_elf_sha256": pair["experiment"]["elf"]["sha256"], + "hdl_stream_irx_sha256": pair["baseline"]["irx"]["sha256"], + }, + "note": ( + "Replace every FILL_ME and zero measurement. For recovery workloads, " + "checkpoint_status is restored or fallback for EXP and not-applicable " + "for BASE. Keep one workload/depth per comparator input." + ), + "samples": samples, + } + + +def selftest() -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "fixture.bin" + payload = b"resume-hash-preflight\n" + path.write_bytes(payload) + observed = inspect_blob(path) + assert observed["bytes"] == len(payload) + assert observed["sha256"] == hashlib.sha256(payload).hexdigest() + + baseline: dict[str, dict[str, Any]] = {} + experiment: dict[str, dict[str, Any]] = {} + for mode in ("OFF", "ON"): + base_irx = {"bytes": 10 if mode == "OFF" else 11, "sha256": mode.lower() * 32} + baseline[mode] = { + "elf": {"bytes": 100, "sha256": ("a" if mode == "OFF" else "b") * 64}, + "irx": base_irx, + } + experiment[mode] = { + "elf": {"bytes": 110, "sha256": ("c" if mode == "OFF" else "d") * 64}, + "irx": dict(base_irx), + } + identity = build_identity("head-fixture", baseline, experiment) + template = sample_template(identity, "OFF") + assert len(template["samples"]) == 8 + assert sum(sample["mode"] == "BASE" for sample in template["samples"]) == 4 + assert sum(sample["mode"] == "EXP" for sample in template["samples"]) == 4 + + broken = {mode: {key: dict(value) if isinstance(value, dict) else value + for key, value in experiment[mode].items()} + for mode in experiment} + broken["OFF"]["irx"] = {"bytes": 99, "sha256": "e" * 64} + try: + build_identity("head-fixture", baseline, broken) + except ValueError as error: + assert "IRX differs" in str(error) + else: + raise AssertionError("IRX drift must fail") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--base-profile-on", type=Path, + default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF")) + parser.add_argument("--base-profile-off", type=Path, + default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF")) + parser.add_argument("--base-irx-on", type=Path, + default=Path("HDL_STREAM_PROFILE_ON.irx")) + parser.add_argument("--base-irx-off", type=Path, + default=Path("HDL_STREAM_PROFILE_OFF.irx")) + parser.add_argument("--exp-profile-on", type=Path, + default=Path("PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF")) + parser.add_argument("--exp-profile-off", type=Path, + default=Path("PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF")) + parser.add_argument("--exp-irx-on", type=Path, + default=Path("HDL_STREAM_RESUME_HASH_PROFILE_ON.irx")) + parser.add_argument("--exp-irx-off", type=Path, + default=Path("HDL_STREAM_RESUME_HASH_PROFILE_OFF.irx")) + parser.add_argument("--project-git-sha", default=FROZEN_SOURCE_GIT_SHA) + parser.add_argument("--identity-output", type=Path) + parser.add_argument("--profile-off-template", type=Path) + parser.add_argument("--profile-on-template", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("resume_hash_ab_preflight selftest: PASS") + return 0 + + try: + baseline = { + "OFF": { + "elf": verify_frozen(args.base_profile_off, "OFF", "elf"), + "irx": verify_frozen(args.base_irx_off, "OFF", "irx"), + }, + "ON": { + "elf": verify_frozen(args.base_profile_on, "ON", "elf"), + "irx": verify_frozen(args.base_irx_on, "ON", "irx"), + }, + } + experiment = { + "OFF": { + "elf": inspect_blob(args.exp_profile_off), + "irx": inspect_blob(args.exp_irx_off), + }, + "ON": { + "elf": inspect_blob(args.exp_profile_on), + "irx": inspect_blob(args.exp_irx_on), + }, + } + identity = build_identity(args.project_git_sha, baseline, experiment) + except (OSError, ValueError) as error: + print(f"resume_hash_ab_preflight: {error}", file=sys.stderr) + return 2 + + for profile in ("OFF", "ON"): + pair = identity["profiles"][profile] + print( + f"PROFILE {profile} BASE {pair['baseline']['elf']['bytes']} B " + f"{pair['baseline']['elf']['sha256']}" + ) + print( + f"PROFILE {profile} EXP {pair['experiment']['elf']['bytes']} B " + f"{pair['experiment']['elf']['sha256']}" + ) + print( + f"PROFILE {profile} IRX PASS {pair['experiment']['irx']['sha256']}" + ) + + if args.identity_output: + args.identity_output.write_text( + json.dumps(identity, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + if args.profile_off_template: + args.profile_off_template.write_text( + json.dumps(sample_template(identity, "OFF"), indent=2) + "\n", + encoding="utf-8", + ) + if args.profile_on_template: + args.profile_on_template.write_text( + json.dumps(sample_template(identity, "ON"), indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 7c5f69cae7bd3a8a884c46b773968a8a19c5280a Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 20:30:27 +0200 Subject: [PATCH 086/156] bench: add resume-hash A/B comparator --- tools/compare_hdl_resume_hash_ab.py | 442 ++++++++++++++++++++++++++++ 1 file changed, 442 insertions(+) create mode 100644 tools/compare_hdl_resume_hash_ab.py diff --git a/tools/compare_hdl_resume_hash_ab.py b/tools/compare_hdl_resume_hash_ab.py new file mode 100644 index 00000000..1f09a397 --- /dev/null +++ b/tools/compare_hdl_resume_hash_ab.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 +"""Compare real-PS2 HDL resume-hash checkpoint A/B samples. + +The identity JSON must come from ``resume_hash_ab_preflight.py`` in the same CI +artifact as the tested ELFs. This prevents a rebuilt or unrelated experiment +from inheriting another artifact's measurements. + +For recovery workloads, experiment runs that report ``checkpoint_status`` as +``fallback`` are counted but excluded from optimized timing distributions. A +safe fallback is a correctness success, not evidence that checkpoint restore +was slow. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Any + +REQUIRED = ( + "profile", + "mode", + "project_git_sha", + "benchmark_elf_sha256", + "hdl_stream_irx_sha256", + "workload_kind", + "workload_id", + "correctness_hash", + "source_bytes", + "total_us", + "checkpoint_status", + "checkpoint_restored_bytes", +) +OPTIONAL_TIMES = ("copy_us", "verify_us", "resume_gate_us") +WORKLOAD_KINDS = ("uninterrupted", "copy_resume", "payload_verified_resume") +CHECKPOINT_STATUSES = ("not-applicable", "restored", "fallback") + + +def _percentile(values: list[int], percentile: int) -> int: + if not values: + raise ValueError("percentile requires at least one value") + ordered = sorted(values) + rank = max(1, math.ceil(len(ordered) * percentile / 100.0)) + return ordered[rank - 1] + + +def _distribution(values: list[int]) -> dict[str, int]: + return { + "samples": len(values), + "p50": _percentile(values, 50), + "p95": _percentile(values, 95), + "p99": _percentile(values, 99), + "max": max(values), + } + + +def _delta_percent(exp_value: int, base_value: int) -> float: + if base_value == 0: + raise ValueError("baseline metric cannot be zero") + return round((exp_value - base_value) * 100.0 / base_value, 4) + + +def _throughput_kib_s(source_bytes: int, usec: int) -> int: + if source_bytes <= 0 or usec <= 0: + raise ValueError("source_bytes and time must be positive") + return (source_bytes * 1_000_000) // (usec * 1024) + + +def _load_identity(raw: Any) -> dict[str, Any]: + if not isinstance(raw, dict): + raise ValueError("identity must be an object") + if not isinstance(raw.get("project_git_sha"), str) or not raw["project_git_sha"]: + raise ValueError("identity project_git_sha must be non-empty") + profiles = raw.get("profiles") + if not isinstance(profiles, dict): + raise ValueError("identity profiles must be an object") + for profile in ("OFF", "ON"): + pair = profiles.get(profile) + if not isinstance(pair, dict): + raise ValueError(f"identity missing PROFILE {profile}") + for variant in ("baseline", "experiment"): + entry = pair.get(variant) + if not isinstance(entry, dict): + raise ValueError(f"identity PROFILE {profile} missing {variant}") + for kind in ("elf", "irx"): + blob = entry.get(kind) + if not isinstance(blob, dict): + raise ValueError( + f"identity PROFILE {profile} {variant} missing {kind}" + ) + if not isinstance(blob.get("sha256"), str) or not blob["sha256"]: + raise ValueError( + f"identity PROFILE {profile} {variant} {kind} sha256 invalid" + ) + return raw + + +def _positive_int(sample: dict[str, Any], key: str, index: int) -> None: + value = sample[key] + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise ValueError(f"sample {index}: {key} must be a positive integer") + + +def _nonnegative_int(sample: dict[str, Any], key: str, index: int) -> None: + value = sample[key] + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise ValueError(f"sample {index}: {key} must be a non-negative integer") + + +def _normalize(raw: Any, identity: dict[str, Any]) -> list[dict[str, Any]]: + if isinstance(raw, dict): + raw = raw.get("samples") + if not isinstance(raw, list) or not raw: + raise ValueError("input must contain a non-empty samples array") + + samples: list[dict[str, Any]] = [] + for index, sample in enumerate(raw): + if not isinstance(sample, dict): + raise ValueError(f"sample {index}: expected object") + missing = [key for key in REQUIRED if key not in sample] + if missing: + raise ValueError(f"sample {index}: missing {', '.join(missing)}") + + normalized = dict(sample) + profile = str(normalized["profile"]).upper() + mode = str(normalized["mode"]).upper() + workload_kind = str(normalized["workload_kind"]) + checkpoint_status = str(normalized["checkpoint_status"]) + if profile not in ("OFF", "ON"): + raise ValueError(f"sample {index}: profile must be OFF or ON") + if mode not in ("BASE", "EXP"): + raise ValueError(f"sample {index}: mode must be BASE or EXP") + if workload_kind not in WORKLOAD_KINDS: + raise ValueError( + f"sample {index}: workload_kind must be one of {', '.join(WORKLOAD_KINDS)}" + ) + if checkpoint_status not in CHECKPOINT_STATUSES: + raise ValueError( + f"sample {index}: invalid checkpoint_status {checkpoint_status}" + ) + if mode == "BASE" and checkpoint_status != "not-applicable": + raise ValueError( + f"sample {index}: BASE checkpoint_status must be not-applicable" + ) + if workload_kind == "uninterrupted" and checkpoint_status != "not-applicable": + raise ValueError( + f"sample {index}: uninterrupted runs cannot report checkpoint restore" + ) + if workload_kind != "uninterrupted" and mode == "EXP" and \ + checkpoint_status == "not-applicable": + raise ValueError( + f"sample {index}: recovery EXP must report restored or fallback" + ) + + normalized["profile"] = profile + normalized["mode"] = mode + normalized["workload_kind"] = workload_kind + normalized["checkpoint_status"] = checkpoint_status + + for key in ("source_bytes", "total_us"): + _positive_int(normalized, key, index) + for key in OPTIONAL_TIMES: + if key in normalized: + _positive_int(normalized, key, index) + for key in ("checkpoint_restored_bytes", "completed_sectors"): + if key in normalized: + _nonnegative_int(normalized, key, index) + + for key in ( + "project_git_sha", + "benchmark_elf_sha256", + "hdl_stream_irx_sha256", + "workload_id", + "correctness_hash", + ): + if not isinstance(normalized[key], str) or not normalized[key]: + raise ValueError(f"sample {index}: {key} must be a non-empty string") + + if normalized["project_git_sha"] != identity["project_git_sha"]: + raise ValueError(f"sample {index}: project_git_sha does not match identity") + pair = identity["profiles"][profile] + variant = pair["baseline" if mode == "BASE" else "experiment"] + if normalized["benchmark_elf_sha256"].lower() != variant["elf"]["sha256"]: + raise ValueError(f"sample {index}: ELF hash does not match identity") + if normalized["hdl_stream_irx_sha256"].lower() != variant["irx"]["sha256"]: + raise ValueError(f"sample {index}: IRX hash does not match identity") + normalized["benchmark_elf_sha256"] = normalized["benchmark_elf_sha256"].lower() + normalized["hdl_stream_irx_sha256"] = normalized["hdl_stream_irx_sha256"].lower() + samples.append(normalized) + return samples + + +def compare_samples( + samples: list[dict[str, Any]], identity: dict[str, Any], min_samples: int = 4 +) -> dict[str, Any]: + profiles = {sample["profile"] for sample in samples} + workload_kinds = {sample["workload_kind"] for sample in samples} + workloads = {sample["workload_id"] for sample in samples} + correctness_hashes = {sample["correctness_hash"] for sample in samples} + source_sizes = {sample["source_bytes"] for sample in samples} + project_shas = {sample["project_git_sha"] for sample in samples} + if len(profiles) != 1: + raise ValueError("compare one PROFILE mode at a time") + if len(workload_kinds) != 1: + raise ValueError("compare one workload_kind at a time") + if len(workloads) != 1: + raise ValueError("compare one workload_id/depth at a time") + if len(correctness_hashes) != 1: + raise ValueError("correctness_hash differs across samples") + if len(source_sizes) != 1: + raise ValueError("source_bytes differs across samples") + if project_shas != {identity["project_git_sha"]}: + raise ValueError("project_git_sha differs from identity") + + profile = next(iter(profiles)) + workload_kind = next(iter(workload_kinds)) + source_bytes = int(next(iter(source_sizes))) + groups = { + mode: [sample for sample in samples if sample["mode"] == mode] + for mode in ("BASE", "EXP") + } + if len(groups["BASE"]) < min_samples: + raise ValueError( + f"BASE has {len(groups['BASE'])} samples; need at least {min_samples}" + ) + + fallback_samples = [ + sample for sample in groups["EXP"] if sample["checkpoint_status"] == "fallback" + ] + if workload_kind == "uninterrupted": + accepted_exp = groups["EXP"] + else: + accepted_exp = [ + sample for sample in groups["EXP"] + if sample["checkpoint_status"] == "restored" + ] + if len(accepted_exp) < min_samples: + raise ValueError( + f"accepted EXP has {len(accepted_exp)} samples after fallback separation; " + f"need at least {min_samples}" + ) + + expected_restored_bytes = 0 + completed_sectors = None + if workload_kind == "copy_resume": + sectors = {sample.get("completed_sectors") for sample in samples} + if None in sectors or len(sectors) != 1: + raise ValueError("copy_resume samples must share one completed_sectors value") + completed_sectors = int(next(iter(sectors))) + if completed_sectors <= 0: + raise ValueError("copy_resume completed_sectors must be positive") + expected_restored_bytes = completed_sectors * 2048 + elif workload_kind == "payload_verified_resume": + expected_restored_bytes = source_bytes + + if expected_restored_bytes: + for sample in accepted_exp: + if int(sample["checkpoint_restored_bytes"]) != expected_restored_bytes: + raise ValueError( + "restored EXP sample does not report the expected skipped source bytes" + ) + for sample in fallback_samples: + if int(sample["checkpoint_restored_bytes"]) != 0: + raise ValueError("fallback EXP sample must report zero restored bytes") + + result: dict[str, Any] = { + "project_git_sha": identity["project_git_sha"], + "profile": profile, + "workload_kind": workload_kind, + "workload_id": next(iter(workloads)), + "correctness_hash": next(iter(correctness_hashes)), + "source_bytes": source_bytes, + "completed_sectors": completed_sectors, + "expected_checkpoint_restored_bytes": expected_restored_bytes, + "sample_counts": { + "base": len(groups["BASE"]), + "exp_total": len(groups["EXP"]), + "exp_accepted": len(accepted_exp), + "exp_fallback": len(fallback_samples), + }, + "binary_identity": identity["profiles"][profile], + "metrics": {}, + } + + compared = groups["BASE"] + accepted_exp + metric_keys = ["total_us"] + for key in OPTIONAL_TIMES: + if all(key in sample for sample in compared): + metric_keys.append(key) + + metrics: dict[str, Any] = result["metrics"] + for key in metric_keys: + base_values = [int(sample[key]) for sample in groups["BASE"]] + exp_values = [int(sample[key]) for sample in accepted_exp] + base_dist = _distribution(base_values) + exp_dist = _distribution(exp_values) + metrics[key] = { + "base": base_dist, + "experiment": exp_dist, + "experiment_vs_base_percent": { + percentile: _delta_percent(exp_dist[percentile], base_dist[percentile]) + for percentile in ("p50", "p95", "p99", "max") + }, + } + + if workload_kind == "uninterrupted" and "copy_us" in metric_keys: + base_rates = [ + _throughput_kib_s(source_bytes, int(sample["copy_us"])) + for sample in groups["BASE"] + ] + exp_rates = [ + _throughput_kib_s(source_bytes, int(sample["copy_us"])) + for sample in accepted_exp + ] + base_dist = _distribution(base_rates) + exp_dist = _distribution(exp_rates) + metrics["copy_kib_per_second"] = { + "base": base_dist, + "experiment": exp_dist, + "experiment_vs_base_percent": { + percentile: _delta_percent(exp_dist[percentile], base_dist[percentile]) + for percentile in ("p50", "p95", "p99", "max") + }, + } + + return result + + +def selftest() -> None: + identity = { + "project_git_sha": "head-fixture", + "profiles": { + "OFF": { + "baseline": { + "elf": {"sha256": "a" * 64, "bytes": 100}, + "irx": {"sha256": "b" * 64, "bytes": 10}, + }, + "experiment": { + "elf": {"sha256": "c" * 64, "bytes": 110}, + "irx": {"sha256": "b" * 64, "bytes": 10}, + }, + }, + "ON": { + "baseline": { + "elf": {"sha256": "d" * 64, "bytes": 120}, + "irx": {"sha256": "e" * 64, "bytes": 11}, + }, + "experiment": { + "elf": {"sha256": "f" * 64, "bytes": 130}, + "irx": {"sha256": "e" * 64, "bytes": 11}, + }, + }, + }, + } + identity = _load_identity(identity) + samples: list[dict[str, Any]] = [] + expected = 16384 * 2048 + for mode, totals in ( + ("BASE", [5000, 5100, 4900, 5050]), + ("EXP", [1000, 1100, 900, 1050]), + ): + variant = identity["profiles"]["ON"][ + "baseline" if mode == "BASE" else "experiment" + ] + for total in totals: + samples.append({ + "profile": "ON", + "mode": mode, + "project_git_sha": "head-fixture", + "benchmark_elf_sha256": variant["elf"]["sha256"], + "hdl_stream_irx_sha256": variant["irx"]["sha256"], + "workload_kind": "copy_resume", + "workload_id": "iso-a-depth-1", + "correctness_hash": "deadbeef", + "source_bytes": 1024 * 1024 * 1024, + "total_us": total, + "resume_gate_us": total // 2, + "completed_sectors": 16384, + "checkpoint_status": "not-applicable" if mode == "BASE" else "restored", + "checkpoint_restored_bytes": 0 if mode == "BASE" else expected, + }) + normalized = _normalize(samples, identity) + result = compare_samples(normalized, identity) + assert result["sample_counts"]["base"] == 4 + assert result["sample_counts"]["exp_accepted"] == 4 + assert result["expected_checkpoint_restored_bytes"] == expected + assert result["metrics"]["total_us"]["experiment"]["p50"] == 1000 + + fallback = [dict(sample) for sample in samples] + fallback[-1]["checkpoint_status"] = "fallback" + fallback[-1]["checkpoint_restored_bytes"] = 0 + try: + compare_samples(_normalize(fallback, identity), identity) + except ValueError as error: + assert "accepted EXP" in str(error) + else: + raise AssertionError("fallback must not count toward optimized minimum") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("identity", nargs="?", type=Path) + parser.add_argument("samples", nargs="?", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--min-samples", type=int, default=4) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("compare_hdl_resume_hash_ab selftest: PASS") + return 0 + if args.identity is None or args.samples is None: + parser.error("identity and samples are required unless --selftest is used") + if args.min_samples < 1: + parser.error("--min-samples must be positive") + + try: + identity = _load_identity( + json.loads(args.identity.read_text(encoding="utf-8")) + ) + samples = _normalize( + json.loads(args.samples.read_text(encoding="utf-8")), identity + ) + result = compare_samples(samples, identity, args.min_samples) + except (OSError, json.JSONDecodeError, ValueError) as error: + print(f"compare_hdl_resume_hash_ab: {error}", file=sys.stderr) + return 2 + + rendered = json.dumps(result, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(rendered, encoding="utf-8") + else: + sys.stdout.write(rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From fc820376fdce3356ef190333e350b679923bc0f6 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 20:31:09 +0200 Subject: [PATCH 087/156] bench: parse resume-hash recovery events --- tools/parse_hdl_perf.py | 96 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/tools/parse_hdl_perf.py b/tools/parse_hdl_perf.py index c30b02d2..12ab9b45 100644 --- a/tools/parse_hdl_perf.py +++ b/tools/parse_hdl_perf.py @@ -49,6 +49,30 @@ + r"HDL perf verify traffic target=(\d+) sif-dma-total=(\d+) ee-cache-maint-total=(\d+) " r"fallback-target=(\d+) consumer-samples=(\d+) final-chunk-excluded=(\d+)" ) +RESUME_PREFIX_RESTORE_RE = re.compile( + PREFIX + + r"HDL restored source SHA checkpoint bytes=(\d+); skipped prefix rehash" +) +RESUME_FULL_RESTORE_RE = re.compile( + PREFIX + + r"HDL restored complete source SHA checkpoint bytes=(\d+); skipped full USB hash pass" +) +RESUME_PREFIX_FALLBACK_RE = re.compile( + PREFIX + + r"HDL source SHA checkpoint unavailable result=(-?\d+); using safe prefix rehash" +) +RESUME_FULL_FALLBACK_RE = re.compile( + PREFIX + + r"HDL complete source SHA checkpoint unavailable result=(-?\d+); using safe full source hash" +) +CHECKPOINT_WRITE_FAILURE_RE = re.compile( + PREFIX + + r"HDL source SHA checkpoint (refresh|save(?: on cancel)?) failed(?: progress=(\d+))? result=(-?\d+)" +) +TRANSACTION_RE = re.compile( + PREFIX + + r"HDL transaction target=([^ ]+) stage=(\d+) progress=(\d+)/(\d+) result=(-?\d+)" +) def _latency(match: re.Match[str]) -> dict[str, int]: @@ -69,6 +93,14 @@ def parse_log(text: str) -> dict[str, object]: "snapshots": {}, "iop_traffic": {}, "ee_traffic": {}, + "resume_hash": { + "prefix_restores": [], + "full_restores": [], + "prefix_fallback_results": [], + "full_fallback_results": [], + "checkpoint_write_failures": [], + "transactions": [], + }, } for line in text.splitlines(): @@ -140,6 +172,58 @@ def parse_log(text: str) -> dict[str, object]: "consumer_samples": int(match.group(5)), "final_chunk_excluded": int(match.group(6)), } + continue + + match = RESUME_PREFIX_RESTORE_RE.search(line) + if match: + result["resume_hash"]["prefix_restores"].append( # type: ignore[index] + {"bytes": int(match.group(1))} + ) + continue + + match = RESUME_FULL_RESTORE_RE.search(line) + if match: + result["resume_hash"]["full_restores"].append( # type: ignore[index] + {"bytes": int(match.group(1))} + ) + continue + + match = RESUME_PREFIX_FALLBACK_RE.search(line) + if match: + result["resume_hash"]["prefix_fallback_results"].append( # type: ignore[index] + int(match.group(1)) + ) + continue + + match = RESUME_FULL_FALLBACK_RE.search(line) + if match: + result["resume_hash"]["full_fallback_results"].append( # type: ignore[index] + int(match.group(1)) + ) + continue + + match = CHECKPOINT_WRITE_FAILURE_RE.search(line) + if match: + result["resume_hash"]["checkpoint_write_failures"].append( # type: ignore[index] + { + "operation": match.group(1), + "progress": int(match.group(2)) if match.group(2) else None, + "result": int(match.group(3)), + } + ) + continue + + match = TRANSACTION_RE.search(line) + if match: + result["resume_hash"]["transactions"].append( # type: ignore[index] + { + "target": match.group(1), + "stage": int(match.group(2)), + "progress": int(match.group(3)), + "total": int(match.group(4)), + "result": int(match.group(5)), + } + ) return result @@ -152,6 +236,12 @@ def selftest() -> None: [0045] HDL fast I/O snapshot phase=copy-final flags=0x00000007 fragments=1 direct=16 fallback=0 prefetch-hit=15 miss=0 pump=16 sectors=2048 src-dma=16 target-dma=0 [0046] HDL IOP traffic phase=copy-final direct-src-sectors=2048 fallback-src-sectors=0 hdd-write-sectors=2048 hdd-read-sectors=0 sif-dma-sectors=2048 [0047] HDL perf copy traffic useful=1048576 sif-dma=1048576 ee-cache-maint=2097152 fallback-source=0 +[0048] HDL restored source SHA checkpoint bytes=33554432; skipped prefix rehash +[0049] HDL source SHA checkpoint unavailable result=-5; using safe prefix rehash +[0050] HDL restored complete source SHA checkpoint bytes=1073741824; skipped full USB hash pass +[0051] HDL complete source SHA checkpoint unavailable result=-7; using safe full source hash +[0052] HDL source SHA checkpoint save failed progress=32768 result=-12; journal remains authoritative +[0053] HDL transaction target=PP.TEST stage=6 progress=524288/524288 result=0 """ parsed = parse_log(sample) assert parsed["copy_rate"]["kib_per_second"] == 1024 # type: ignore[index] @@ -160,6 +250,12 @@ def selftest() -> None: assert parsed["snapshots"]["copy-final"]["flags"] == 7 # type: ignore[index] assert parsed["iop_traffic"]["copy-final"]["hdd_write_sectors"] == 2048 # type: ignore[index] assert parsed["ee_traffic"]["copy"]["ee_cache_maintenance_bytes"] == 2097152 # type: ignore[index] + assert parsed["resume_hash"]["prefix_restores"][0]["bytes"] == 33554432 # type: ignore[index] + assert parsed["resume_hash"]["prefix_fallback_results"] == [-5] # type: ignore[index] + assert parsed["resume_hash"]["full_restores"][0]["bytes"] == 1073741824 # type: ignore[index] + assert parsed["resume_hash"]["full_fallback_results"] == [-7] # type: ignore[index] + assert parsed["resume_hash"]["checkpoint_write_failures"][0]["progress"] == 32768 # type: ignore[index] + assert parsed["resume_hash"]["transactions"][0]["result"] == 0 # type: ignore[index] def main() -> int: From 8e56898177159244145a17bc9eec14609498f7a8 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 20:31:51 +0200 Subject: [PATCH 088/156] ci: bind resume-hash matched A/B artifacts --- .github/workflows/ci.yml | 66 ++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3548d7f5..877b46c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,10 @@ jobs: run: python3 tools/compare_hdl_profile_ab.py --selftest - name: Self-test Phase-0 frozen pair preflight run: python3 tools/phase0_profile_pair_preflight.py --selftest + - name: Self-test resume-hash A/B preflight + run: python3 tools/resume_hash_ab_preflight.py --selftest + - name: Self-test resume-hash A/B comparator + run: python3 tools/compare_hdl_resume_hash_ab.py --selftest - name: Self-test R5900 calibration disassembly guard run: python3 tools/check_r5900_calibration_disasm.py --selftest - name: Enforce direct-fileXio runtime policy @@ -132,24 +136,49 @@ jobs: - name: Build isolated resume-hash experiment after frozen gate run: >- docker run --rm + -e PROJECT_GIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}" + -e PROJECT_GIT_REF="${{ github.head_ref || github.ref }}" + -e PS2DEV_BUNDLE_REF="v2.0.0" + -e PS2SDK_SOURCE_REF="v2.0.0" + -e PS2SDK_SOURCE_SHA="b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b" -v "$PWD:/work" -w /work ps2dev/ps2dev:v2.0.0 sh -c 'apk add --no-cache make python3 >/dev/null && sh tools/build_resume_hash_experiment.sh && mips64r5900el-ps2-elf-size - PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF - > PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.sections && + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF + > PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.sections && + mips64r5900el-ps2-elf-size + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF + > PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.sections && + cp PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.sections + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.sections && cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF PS2_HDD_BOOTSTRAP_MANAGER.ELF && cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.map PS2_HDD_BOOTSTRAP_MANAGER.map' - - name: Record resume-hash experiment identity + - name: Record and validate resume-hash experiment identity run: | - echo "--- resume-hash experiment ---" - cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.sha256 - cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.size - cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.sections - echo "--- resume-hash optimization audit ---" - cat OPTIMIZATION_AUDIT_RESUME_HASH.txt + python3 tools/resume_hash_ab_preflight.py \ + --project-git-sha "${{ github.event.pull_request.head.sha || github.sha }}" \ + --identity-output RESUME_HASH_AB_IDENTITY.json \ + --profile-off-template RESUME_HASH_AB_PROFILE_OFF_TEMPLATE.json \ + --profile-on-template RESUME_HASH_AB_PROFILE_ON_TEMPLATE.json + echo "--- resume-hash PROFILE OFF ---" + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF.sha256 + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF.size + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.sections + cat BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_OFF.yml + echo "--- resume-hash PROFILE ON ---" + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF.sha256 + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF.size + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.sections + cat BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_ON.yml + echo "--- resume-hash identity ---" + cat RESUME_HASH_AB_IDENTITY.json + echo "--- resume-hash PROFILE OFF optimization audit ---" + cat OPTIMIZATION_AUDIT_RESUME_HASH_PROFILE_OFF.txt + echo "--- resume-hash PROFILE ON optimization audit ---" + cat OPTIMIZATION_AUDIT_RESUME_HASH_PROFILE_ON.txt python3 tools/phase0_profile_pair_preflight.py \ --profile-on PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF \ --profile-off PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF \ @@ -191,6 +220,25 @@ jobs: PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.map PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.sections OPTIMIZATION_AUDIT_RESUME_HASH.txt + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF.sha256 + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF.size + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.map + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.sections + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF.sha256 + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF.size + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.map + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.sections + HDL_STREAM_RESUME_HASH_PROFILE_OFF.irx + HDL_STREAM_RESUME_HASH_PROFILE_ON.irx + OPTIMIZATION_AUDIT_RESUME_HASH_PROFILE_OFF.txt + OPTIMIZATION_AUDIT_RESUME_HASH_PROFILE_ON.txt + BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_OFF.yml + BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_ON.yml + RESUME_HASH_AB_IDENTITY.json + RESUME_HASH_AB_PROFILE_OFF_TEMPLATE.json + RESUME_HASH_AB_PROFILE_ON_TEMPLATE.json HDDMAN.CFG LICENSE THIRD_PARTY_NOTICES.md From 9d17f6805927bbe6fba8417237ff2e72869e0452 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 20:35:09 +0200 Subject: [PATCH 089/156] ci: clean nested IOP state between resume-hash variants --- tools/build_resume_hash_experiment.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/build_resume_hash_experiment.sh b/tools/build_resume_hash_experiment.sh index b4fdfdb4..87a642f4 100644 --- a/tools/build_resume_hash_experiment.sh +++ b/tools/build_resume_hash_experiment.sh @@ -40,6 +40,14 @@ build_variant() provenance="BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_${label}.yml" make clean + # Root clean removes the public hdl_stream.irx but deliberately does not + # invoke the nested IOP Makefile. Its profile-specific objects and the + # absolute-path notiopmod intermediates can therefore survive and make a + # subsequent PROFILE variant reuse the wrong linked IRX. Clean the actual + # producer explicitly so OFF/ON experiment builds are independent. + make -C iop/hdl_stream clean \ + IOP_BIN="$ROOT/hdl_stream.irx" \ + HDL_PROFILE="$profile" make HDL_PROFILE="$profile" HDL_RESUME_HASH_CHECKPOINT=1 cp hdl_stream.irx "$irx" python3 tools/optimization_audit.py \ From c0c1294215c86299fe801fb62804eca884ed6eb9 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 20:39:29 +0200 Subject: [PATCH 090/156] docs: freeze matched resume-hash hardware gate --- docs/HDL_RESUME_HASH_BENCHMARK.md | 239 ++++++++++++++++++++++++++---- 1 file changed, 210 insertions(+), 29 deletions(-) diff --git a/docs/HDL_RESUME_HASH_BENCHMARK.md b/docs/HDL_RESUME_HASH_BENCHMARK.md index 867ddba9..de74de15 100644 --- a/docs/HDL_RESUME_HASH_BENCHMARK.md +++ b/docs/HDL_RESUME_HASH_BENCHMARK.md @@ -20,17 +20,31 @@ work before trying to make the same work faster. size, completed byte count, source fingerprint and target ID; - a missing, corrupt or mismatching sidecar falls back to the old rehash path; - host tests prove that a restored SHA state produces the same final digest as - an uninterrupted hash and that altered checkpoint records are rejected. + an uninterrupted hash and that altered checkpoint records are rejected; +- CI #699 builds matched PROFILE OFF and PROFILE ON experiment variants only + after the frozen Phase-0 pair has passed exact SHA-256 validation; +- in CI #699 the checkpoint experiment changes only the EE application image: + its PROFILE OFF `hdl_stream.irx` is byte-identical to the frozen PROFILE OFF + IRX and its PROFILE ON IRX is byte-identical to the frozen PROFILE ON IRX. **CURRENT IMPLEMENTATION** - transaction journal interval: 16384 ISO sectors = 32 MiB; -- experiment build: `HDL_PROFILE=0 HDL_RESUME_HASH_CHECKPOINT=1`; -- baseline performance build: frozen `HDL_PROFILE=0` ELF; -- the experiment writes/verifies/renames a 256-byte sidecar at durable copy - checkpoints and on an orderly cancel; +- release-like experiment: `HDL_PROFILE=0 HDL_RESUME_HASH_CHECKPOINT=1`; +- instrumented experiment: `HDL_PROFILE=1 HDL_RESUME_HASH_CHECKPOINT=1`; +- release-like baseline: frozen `HDL_PROFILE=0` ELF; +- instrumented baseline: frozen `HDL_PROFILE=1` ELF; +- the experiment writes, reads back, codec-verifies and renames a 256-byte + sidecar at existing 32 MiB transaction checkpoints and on an orderly cancel; +- checkpoint state is written before the corresponding transaction journal; - the sidecar is deleted when a new zero-progress transaction supersedes it and - when a transaction reaches COMPLETE. + when a transaction reaches COMPLETE; +- `tools/parse_hdl_perf.py` extracts checkpoint restore/fallback/write-failure + events and transaction results from `HDDMAN.LOG`; +- `tools/resume_hash_ab_preflight.py` binds every hardware sample template to + the exact baseline/experiment ELF and IRX identities from the CI artifact; +- `tools/compare_hdl_resume_hash_ab.py` separates safe fallback runs from valid + optimized restores and reports p50/p95/p99/max for the supplied timing data. **INFERENCJA** @@ -38,6 +52,10 @@ work before trying to make the same work faster. traffic on a matching COPY resume; - restoring a full-payload checkpoint should remove one full source-ISO pass before resumed HDD verification; +- because checkpoint state is bound to the journal progress, loss of power after + checkpoint replacement but before the newer journal becomes visible should + make the checkpoint stale relative to the older journal and therefore reject + it, falling back to source rehash; - the small sidecar operations may add measurable latency or jitter to an uninterrupted install even though their byte volume is tiny. @@ -45,10 +63,16 @@ work before trying to make the same work faster. - recovery time improves materially on real PS2; - the uninterrupted-copy regression from checkpoint maintenance is negligible; +- temp-file write/readback/rename plus the mass-storage stack provide adequate + persistence behavior across the tested orderly-cancel and guarded fault + windows on real hardware; - no adapter/USB implementation exposes a correctness or persistence corner case not covered by host tests. -No performance claim is accepted before this hardware gate passes. +The sidecar is therefore logically fail-safe in current source, but this document +does **not** call it physically durable across arbitrary power loss until the +real-hardware crash-window matrix demonstrates that property. No performance or +power-loss durability claim is accepted before this hardware gate passes. ## Authoritative corpus rationale @@ -58,8 +82,12 @@ This test follows: correctness hash and benchmark provenance; distinguish instrumented and release-like profiles; - `PS2_HDD_APA_PFS_HDL_filesystem_optimization_research_corpus_v2.md`: keep HDL - raw/storage workloads separate, use deterministic checksum validation, and do - not infer throughput from interface headline rates; + raw/storage workloads separate, eliminate repeated reads before attempting + lower-level acceleration, use deterministic checksum validation, and do not + infer throughput from interface headline rates; +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md`: reuse producer + state only when representation, lifetime and ownership remain valid for the + consumer; - `PS2_Whole_System_Scheduling_research_corpus_v2.md`: treat USB, IOP, SIF and storage as one producer/consumer path and report distribution/tail behavior, not only an average; @@ -67,9 +95,15 @@ This test follows: millisecond-scale granularity, so redundant source traffic belongs on the critical recovery path rather than being dismissed as free background work. -## Frozen baseline identity +## Frozen and experiment identities -Use the Phase-0 PROFILE OFF binary only after the preflight tool accepts it: +The authoritative matched A/B identity was emitted by CI #699 for project head +`9d17f6805927bbe6fba8417237ff2e72869e0452` in +`RESUME_HASH_AB_IDENTITY.json`. + +### PROFILE OFF: release-like pair + +Baseline: ```text PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF @@ -77,18 +111,90 @@ bytes 632884 sha256 4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916 ``` -The matching embedded PROFILE OFF `hdl_stream.irx` is: +Experiment: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF +bytes 636084 +sha256 d463064036b2f253462cf3fbc90b75db89da36979acfd88334b50e6eecd7ef11 +``` + +Both use the exact same PROFILE OFF `hdl_stream.irx`: ```text bytes 8405 sha256 f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 ``` -Do not rebuild and silently call the result the same baseline. The frozen hash -is the identity. +Static checkpoint delta against frozen PROFILE OFF: + +```text +stripped ELF +3200 B +EE named text +2512 B (229956 -> 232468) +EE named functions +4 (609 -> 613) +EE instructions +629 (57539 -> 58168) +execute_transaction() +288 B (6156 -> 6444) +execute_transaction insn +72 (1540 -> 1612) +``` + +### PROFILE ON: instrumented pair + +Baseline: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF +bytes 638388 +sha256 964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7 +``` + +Experiment: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF +bytes 641588 +sha256 849cb5fec6cce31b214210a09d8ffe81ce181a6790f0ef5fa28bb8f62ce89d62 +``` + +Both use the exact same PROFILE ON `hdl_stream.irx`: + +```text +bytes 9861 +sha256 8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db +``` + +Static checkpoint delta against frozen PROFILE ON: + +```text +stripped ELF +3200 B +EE named text +2512 B (232780 -> 235292) +EE named functions +4 (618 -> 622) +EE instructions +631 (58246 -> 58877) +execute_transaction() +288 B (6156 -> 6444) +execute_transaction insn +72 (1540 -> 1612) +``` + +The 2-instruction difference between OFF/ON checkpoint deltas is profiler-side +code generation outside `execute_transaction()`; the transaction body itself +has the same measured static delta in both pairs. -The experiment must come from the same CI artifact as the baseline and must -have its own recorded SHA-256 and section sizes. +Do not rebuild and silently call a result one of these binaries. Exact hash is +the identity. If experiment source changes, CI emits a new experiment identity +while the frozen baseline hashes remain fixed. + +## Why there are two A/B pairs + +Use PROFILE OFF for the acceptance timing result. It is the release-like pair +and does not pay the Phase-0 HDL profiler overhead. + +Use PROFILE ON for attribution. Both binaries in that pair contain the same IOP +latency/traffic profiler and the same PROFILE ON IRX, allowing the existing +`usb-direct-read`, `source-fallback-read`, `prefetch-consumer-wait`, `hdd-write`, +`hdd-read`, `sif-dma-completion`, `pump-ioctl`, `source-ioctl`, `target-ioctl`, +`copy-ee-consumer` and `verify-ee-consumer` categories to explain where time and +traffic changed. + +Never compare frozen PROFILE OFF directly against checkpoint PROFILE ON and call +that the checkpoint effect. That changes two variables at once. ## Hardware record @@ -104,23 +210,38 @@ USB filesystem and relevant allocation state PS2SDK commit PS2DEV/toolchain version active IRX set +PROFILE mode baseline ELF SHA-256 experiment ELF SHA-256 +hdl_stream.irx SHA-256 source ISO SHA-256 source ISO byte count target HDD free-space/allocation state +video mode / launch method ``` Keep the same console, adapter, USB device, ISO, target disk, video mode and launch method for the whole A/B block. +The CI artifact contains: + +```text +BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_OFF.yml +BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_ON.yml +RESUME_HASH_AB_IDENTITY.json +RESUME_HASH_AB_PROFILE_OFF_TEMPLATE.json +RESUME_HASH_AB_PROFILE_ON_TEMPLATE.json +``` + +Fill the hardware/runtime fields rather than inferring them from CI. + ## Correctness gate before timing For both binaries: 1. start a fresh install; 2. cancel only through the normal guarded TRIANGLE path after at least one - durable journal checkpoint; + transaction checkpoint; 3. restart and choose resume; 4. complete payload verification and metadata commit; 5. confirm the final installed game is catalogued with the expected startup and @@ -141,10 +262,10 @@ Do not change the ISO between A and B. For each binary, perform at least eight interleaved fresh-install runs. Restore an equivalent target state before each run. -Recommended order: +Recommended order, already emitted in the CI templates: ```text -OFF, EXP, EXP, OFF, EXP, OFF, OFF, EXP +BASE, EXP, EXP, BASE, EXP, BASE, BASE, EXP ``` Measure separately where the existing logs permit: @@ -154,7 +275,7 @@ copy phase elapsed time payload verification elapsed time total transaction elapsed time copy throughput -journal/checkpoint failures +checkpoint write failures correctness result ``` @@ -165,18 +286,23 @@ set if the result is near the acceptance boundary. Do not accept an improvement in recovery if uninterrupted installs acquire a large or erratic regression. +For release-like acceptance data, use +`RESUME_HASH_AB_PROFILE_OFF_TEMPLATE.json`. For profiler attribution, repeat the +selected workload with the PROFILE ON template rather than mixing modes in one +comparator input. + ## Workload B: resumed COPY Purpose: measure removal of the already-copied-prefix replay. -Choose three durable resume depths separated across the ISO. Prefer journal +Choose three persisted resume depths separated across the ISO. Prefer journal boundaries near approximately 25%, 50% and 75% of source progress. Record the exact `completed_sectors` from the journal rather than assuming the requested percentage was hit. For each depth: -1. perform a fresh copy to the selected durable checkpoint; +1. perform a fresh copy to the selected transaction checkpoint; 2. cancel normally; 3. preserve the journal and, for EXP, its checkpoint sidecar; 4. reboot/relaunch in the same way for each run; @@ -192,9 +318,10 @@ prefix_rehash_bytes = completed_sectors * 2048 ``` For a valid EXP checkpoint, expected skipped prefix bytes are the same number. -A session-log line must confirm checkpoint restore. If EXP falls back to safe -rehash, classify that run separately rather than pretending it was an optimized -sample. +`tools/parse_hdl_perf.py` must report a `prefix_restores` record with that byte +count. If EXP reports a prefix fallback, classify that run separately. +`tools/compare_hdl_resume_hash_ab.py` deliberately excludes fallback runs from +the optimized timing distribution while preserving their count. Report for each depth: @@ -226,6 +353,10 @@ EXP: restore final source SHA state, then HDD payload verification The HDD verification remains required in both variants. Do not count its time as work removed by this experiment. +For a valid optimized run, `parse_hdl_perf.py` must report a `full_restores` +record whose byte count equals `source_bytes`. Full-source fallback is kept as a +correctness-safe fallback sample, not as evidence for optimized latency. + Report: ```text @@ -236,19 +367,42 @@ checkpoint restore/fallback status correctness failures ``` -## Crash-window matrix +## Crash-window analysis and hardware matrix + +Current save order at a copy checkpoint is: + +```text +update transaction.completed_sectors in RAM +encode checkpoint for that exact progress +write HDLINSTALL.SHN +read back and verify exact bytes + checkpoint codec +remove previous HDLINSTALL.SHA +rename HDLINSTALL.SHN -> HDLINSTALL.SHA +encode/write/readback/verify/replace transaction journal +``` + +A checkpoint can never advance journal progress. Restore validates the sidecar +against the transaction loaded from the authoritative journal, including its +completed byte count, source fingerprint and target ID. + +**INFERENCJA:** if the new sidecar becomes visible but the newer journal does +not, the older journal progress makes the sidecar stale and restore is rejected. +If checkpoint replacement itself is lost/corrupt, the optimization is lost and +legacy rehash is the intended fallback. -The sidecar is not journal authority, so deliberately test these states before -promotion: +The actual mass-storage persistence semantics across reset/power loss are still +a hardware property. Test these states before promotion: ```text sidecar absent sidecar checksum corrupt sidecar from an earlier completed_sectors value +sidecar from a later completed_sectors value with older journal sidecar from another source fingerprint sidecar from another target ID valid .SHA primary valid temporary .SHN with primary absent +loss/reset around sidecar replacement where the guarded fault procedure permits ``` Every invalid/mismatched case must fall back to the legacy rehash path and still @@ -259,6 +413,29 @@ safety preconditions are satisfied. Do not pull power during arbitrary APA metadata writes merely to make the benchmark more exciting. The console has suffered enough. +## Host-side workflow + +After a hardware run, turn each preserved log into structured telemetry: + +```sh +python3 tools/parse_hdl_perf.py HDDMAN.LOG --output run.json +``` + +Fill one workload/depth per copy of the CI sample template. Do not combine COPY +resume at 25%, 50% and 75% in one comparator input. + +Then compare against the identity emitted with the tested artifact: + +```sh +python3 tools/compare_hdl_resume_hash_ab.py \ + RESUME_HASH_AB_IDENTITY.json samples.json \ + --output result.json +``` + +The comparator rejects wrong ELF/IRX hashes, mixed PROFILE modes, mixed workload +IDs, mismatching correctness hashes, invalid restored-byte counts and too few +accepted EXP samples after fallback separation. + ## Acceptance rule Promote the checkpoint path only if all are true: @@ -270,7 +447,11 @@ Promote the checkpoint path only if all are true: 4. COPY-resume and PAYLOAD_VERIFIED recovery show a clear real-hardware benefit; 5. uninterrupted-install p50/p95/p99/max do not show an unacceptable regression or new long-tail spikes; -6. sidecar failure always degrades to the old safe rehash behavior. +6. sidecar failure always degrades to the old safe rehash behavior; +7. PROFILE ON attribution agrees with the release-like PROFILE OFF result rather + than revealing an unrelated IOP/device change; +8. power/reset testing does not demonstrate a persistence behavior that can + violate the transaction correctness contract. If the result is ambiguous, keep the flag off and collect more samples. A neat architecture diagram is not a benchmark result. From 4266d451a7055942c8f8b5812df15c11442bcbc6 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 20:42:27 +0200 Subject: [PATCH 091/156] perf: materialize source-free stage4 resume experiment --- tools/materialize_resume_hash_transaction.py | 199 +++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 tools/materialize_resume_hash_transaction.py diff --git a/tools/materialize_resume_hash_transaction.py b/tools/materialize_resume_hash_transaction.py new file mode 100644 index 00000000..13e8d1a4 --- /dev/null +++ b/tools/materialize_resume_hash_transaction.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Materialize the isolated resume-hash transaction experiment. + +The frozen default transaction fragment must remain byte-for-byte untouched for +Phase-0 A/B. This build-time transform applies the next recovery-only cut to +its already-isolated resume-hash copy: a PAYLOAD_VERIFIED transaction with a +matching complete SHA checkpoint no longer re-opens the USB ISO merely to prove +identity before using that same checkpoint. + +If either expected source block drifts, generation fails instead of silently +building a partially transformed experiment. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +PRE_SOURCE_OLD = ''' /* Once METADATA_COMMITTED is durable, payload and source identity have + * already passed full verification. Completion recovery only needs the + * target layout plus metadata read-back, so do not require the USB ISO to + * remain connected merely to move stage 5 -> COMPLETE. */ + if (transaction->stage < HDL_TRANSACTION_STAGE_METADATA_COMMITTED) { + result = open_source(transaction->source_path, + transaction->source_bytes, &source); + if (result < 0) + return result; + result = source_fingerprint(&source, fingerprint); + if (result == 0 && memcmp(fingerprint, transaction->source_fingerprint, + sizeof(fingerprint)) != 0) + result = HDL_INSTALL_SOURCE_CHANGED; + if (result == 0) + result = source_identity_matches(&source, transaction); + if (result < 0) { + fileXioClose(source.fd); + return result; + } + } +''' + +PRE_SOURCE_NEW = ''' /* Once METADATA_COMMITTED is durable, payload and source identity have + * already passed full verification. Completion recovery only needs the + * target layout plus metadata read-back, so do not require the USB ISO to + * remain connected merely to move stage 5 -> COMPLETE. + * + * PAYLOAD_VERIFIED has the same opportunity when its authenticated + * checkpoint represents the complete source. The checkpoint is bound to + * this journal's source size, completed byte count, fingerprint and target + * ID; target SHA read-back below still proves the HDD payload matches the + * digest accumulated while the original source bytes were copied. */ +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + if (transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) { + sha256_context_t restored_source_hash; + int checkpoint_result = + hash_checkpoint_load(transaction, &restored_source_hash); + + if (checkpoint_result == 0) { + sha256_final(&restored_source_hash, source_payload_digest); + source_digest_valid = 1; + session_log_line( + "HDL restored complete source SHA checkpoint bytes=%llu; skipped full USB hash pass and source reopen", + (unsigned long long)transaction->source_bytes); + } else { + session_log_line( + "HDL complete source SHA checkpoint unavailable result=%d; using safe full source hash", + checkpoint_result); + } + } +#endif + if (transaction->stage < HDL_TRANSACTION_STAGE_METADATA_COMMITTED && + !(transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED && + source_digest_valid)) { + result = open_source(transaction->source_path, + transaction->source_bytes, &source); + if (result < 0) + return result; + result = source_fingerprint(&source, fingerprint); + if (result == 0 && memcmp(fingerprint, transaction->source_fingerprint, + sizeof(fingerprint)) != 0) + result = HDL_INSTALL_SOURCE_CHANGED; + if (result == 0) + result = source_identity_matches(&source, transaction); + if (result < 0) { + fileXioClose(source.fd); + return result; + } + } +''' + +STAGE4_OLD = ''' if (transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) { + if (!verified_this_run) { +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + sha256_context_t restored_source_hash; + int checkpoint_result = + hash_checkpoint_load(transaction, &restored_source_hash); + + if (checkpoint_result == 0) { + sha256_final(&restored_source_hash, source_payload_digest); + source_digest_valid = 1; + session_log_line( + "HDL restored complete source SHA checkpoint bytes=%llu; skipped full USB hash pass", + (unsigned long long)transaction->source_bytes); + } else { + session_log_line( + "HDL complete source SHA checkpoint unavailable result=%d; using safe full source hash", + checkpoint_result); +#endif + disk_status_phase_at("Hashing source for resumed verification", + "One source pass required without a matching hash checkpoint"); + result = hash_source_payload(transaction, source.fd, + source_payload_digest); + if (result < 0) + goto done; + source_digest_valid = 1; +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + } +#endif + disk_status_phase_at("Re-verifying resumed payload on HDD", + "HDD-only SHA-256 read-back against source digest"); + result = verify_target_digest(transaction, &plan, &layout, + target_fd, source_payload_digest); + if (result < 0) + goto done; + } +''' + +STAGE4_NEW = ''' if (transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) { + if (!verified_this_run) { + if (!source_digest_valid) { + disk_status_phase_at("Hashing source for resumed verification", + "One source pass required without a matching hash checkpoint"); + result = hash_source_payload(transaction, source.fd, + source_payload_digest); + if (result < 0) + goto done; + source_digest_valid = 1; + } + disk_status_phase_at("Re-verifying resumed payload on HDD", + "HDD-only SHA-256 read-back against source digest"); + result = verify_target_digest(transaction, &plan, &layout, + target_fd, source_payload_digest); + if (result < 0) + goto done; + } +''' + + +def materialize(text: str) -> str: + if text.count(PRE_SOURCE_OLD) != 1: + raise ValueError("expected exactly one pre-source validation block") + text = text.replace(PRE_SOURCE_OLD, PRE_SOURCE_NEW, 1) + if text.count(STAGE4_OLD) != 1: + raise ValueError("expected exactly one PAYLOAD_VERIFIED restore block") + text = text.replace(STAGE4_OLD, STAGE4_NEW, 1) + return text + + +def selftest() -> None: + fixture = "before\n" + PRE_SOURCE_OLD + "middle\n" + STAGE4_OLD + "after\n" + transformed = materialize(fixture) + assert PRE_SOURCE_OLD not in transformed + assert STAGE4_OLD not in transformed + assert transformed.count("skipped full USB hash pass and source reopen") == 1 + assert transformed.count("if (!source_digest_valid)") == 1 + try: + materialize("drifted input") + except ValueError: + pass + else: + raise AssertionError("source drift must fail materialization") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("source", nargs="?", type=Path) + parser.add_argument("output", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("materialize_resume_hash_transaction selftest: PASS") + return 0 + if args.source is None or args.output is None: + parser.error("source and output are required unless --selftest is used") + + try: + text = args.source.read_text(encoding="utf-8") + generated = materialize(text) + args.output.write_text(generated, encoding="utf-8") + except (OSError, ValueError) as error: + print(f"materialize_resume_hash_transaction: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 65fe739fa3aeb1ceea026aaa8588a8e9dbe6d567 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 20:42:47 +0200 Subject: [PATCH 092/156] perf: skip source reopen for verified checkpoint resume --- tools/build_resume_hash_experiment.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/build_resume_hash_experiment.sh b/tools/build_resume_hash_experiment.sh index 87a642f4..b5ae35b0 100644 --- a/tools/build_resume_hash_experiment.sh +++ b/tools/build_resume_hash_experiment.sh @@ -27,7 +27,8 @@ trap restore_sources EXIT HUP INT TERM cp "$SOURCE_UI" "$BACKUP/source_ui.inc" cp "$TRANSACTION" "$BACKUP/transaction.inc" cp "$ROOT/src/hdl_tools/source_ui_resume_hash.inc" "$SOURCE_UI" -cp "$ROOT/src/hdl_tools/transaction_resume_hash.inc" "$TRANSACTION" +python3 "$ROOT/tools/materialize_resume_hash_transaction.py" \ + "$ROOT/src/hdl_tools/transaction_resume_hash.inc" "$TRANSACTION" build_variant() { From d9100437a29eb133de50a900ddfdf64b93a4d1a2 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 20:44:09 +0200 Subject: [PATCH 093/156] docs: define HDL producer consumer ownership contracts --- docs/HDL_DATAFLOW_CONTRACTS.md | 287 +++++++++++++++++++++++++++++++++ 1 file changed, 287 insertions(+) create mode 100644 docs/HDL_DATAFLOW_CONTRACTS.md diff --git a/docs/HDL_DATAFLOW_CONTRACTS.md b/docs/HDL_DATAFLOW_CONTRACTS.md new file mode 100644 index 00000000..e6ff498c --- /dev/null +++ b/docs/HDL_DATAFLOW_CONTRACTS.md @@ -0,0 +1,287 @@ +# HDL dataflow and ownership contracts + +This document records the current producer/consumer/lifetime contract for the +HDL installer and catalogue paths before further storage/SIF optimization. +It follows the PS2 Optimization Research Library v2 rule that representation, +transport and ownership must be explicit before buffering, caching or reuse is +introduced. + +The contracts describe the current branch, not an idealized redesign. Anything +not proven by current source or the project corpus is labelled as inference or a +hardware hypothesis. + +## Epistemic labels + +- **POTWIERDZONE**: current source/manual/corpus contract or real-hardware fact. +- **CURRENT IMPLEMENTATION**: behavior of this branch and the pinned PS2SDK + stack. +- **INFERENCJA**: engineering conclusion not yet measured. +- **HIPOTEZA DO TESTU**: change/result that requires real-PS2 measurement. + +## Dataset: USB ISO source + +```yaml +name: usb_iso_source +producer: mass:/ filesystem backed by USB mass-storage / BDM +consumers: + - ISO9660 + SYSTEM.CNF probe + - source fingerprint + - payload copy + - legacy resume-prefix/full-source SHA reconstruction +lifetime: selected install or resumed transaction while source data is required +representation: raw ISO bytes +alignment: + file: no global alignment contract + fast_transfer: 64-byte aligned EE destination, 512-byte transfer multiple +transport: + fallback: fileXio read through mass: filesystem + fast_path: IOP BDM fragment map -> IOP stage buffer -> HDD + SIF DMA to EE +batch_size: + payload: 64 KiB + fingerprint: first 64 KiB + last 64 KiB plus encoded source size + journal_progress: 16384 ISO sectors = 32 MiB +ownership_states: + - filesystem/device owns source media + - prefetch worker owns alternate IOP stage while read is active + - main IOP path owns completed stage while writing HDD / submitting SIF + - EE owns destination after synchronous ioctl/SIF completion +validation: + - expected file size + - source fingerprint + - ISO probe/startup identity + - full SHA-256 accumulated during fresh copy or rebuilt on legacy resume +``` + +**POTWIERDZONE:** fresh copy already hashes bytes while they are transported, so +it does not perform a second full USB pass merely to produce the source digest. +The resume-hash experiment removes the remaining replay where lifetime permits. + +**CURRENT IMPLEMENTATION:** a valid complete stage-4 checkpoint is being tested +as sufficient producer state for resumed HDD verification. In that case the +current source ISO is no longer a consumer dependency; invalid/missing state +falls back to opening and validating the ISO exactly as before. + +## Dataset: transaction journal + +```yaml +name: hdl_transaction_journal +producer: EE transaction state machine +consumers: + - incomplete transaction UI + - resume path + - incomplete-target cleanup guard +lifetime: from PLANNED until COMPLETE/removal +representation: fixed 512-byte versioned/checksummed transaction record +alignment: ordinary filesystem record; no DMA alignment contract +transport: mass:/ small-file write/read/rename +batch_size: one record at semantic stage transitions and every 32 MiB of copy +ownership_states: + - in-memory transaction is mutable only by current EE transaction + - HDLINSTALL.NEW is replacement candidate + - HDLINSTALL.TXN is authoritative persisted record selected by loader +validation: + - encode/decode contract + - read-back byte equality + - checksum + stage/progress invariants +``` + +The journal is transaction authority. Performance hints must never advance its +progress or replace its recovery semantics. + +## Dataset: optional SHA-state sidecar + +```yaml +name: hdl_source_sha_checkpoint +producer: SHA-256 state accumulated while source bytes are already moving +consumers: + - resumed COPYING prefix reconstruction + - resumed PAYLOAD_VERIFIED source-digest reconstruction +lifetime: same transaction only; deleted at zero-progress replacement/COMPLETE +representation: 256-byte version-1 record + SHA-256 over record payload +alignment: ordinary filesystem record; no DMA alignment contract +transport: mass:/ temp write -> readback/codec verify -> rename +batch_size: same existing 32 MiB transaction checkpoint boundary + orderly cancel +ownership_states: + - in-memory SHA context belongs to current EE copy/hash operation + - HDLINSTALL.SHN is candidate replacement + - HDLINSTALL.SHA is preferred hint + - journal remains authority even when sidecar is newer +validation: + - source byte count + - exact completed byte count + - source fingerprint + - target ID + - SHA context total/block state + - record SHA-256 +fallback: any failure -> legacy source rehash +``` + +**INFERENCJA:** because restore validates exact journal progress, a sidecar that +becomes visible ahead of its matching journal should be rejected against the +older journal rather than advance progress. + +**HIPOTEZA DO TESTU:** physical persistence across reset/power loss depends on +the actual USB/FAT/device stack. The source contract proves fail-safe matching; +it does not by itself prove media-level durability. + +## Dataset: APA/HDL catalogue + +```yaml +name: hdl_catalogue_snapshot +producer: raw APA chain walker +consumers: + - installed-games browser + - game details + - guarded delete selection + - planning free-space/target-collision checks in scan-only mode +lifetime: one installed-games menu session; rebuilt after successful deletion +representation: + catalogue: growable EE array of hdl_game_entry_t + metadata: lazy 1024-byte HDL metadata read, parsed into entry + SHA-256 +alignment: + raw APA header: 64-byte aligned EE buffer for raw HDD transfer/cache contract + metadata: 64-byte aligned local buffer on raw read paths +transport: raw HDD sector reads + selected fileXio control queries +batch_size: + APA: one 1024-byte header (2 HDD sectors) per chain node + game_metadata: one 1024-byte block per game, lazy by visible browser page +ownership_states: + - catalogue array belongs to installed-games menu invocation + - each metadata_state transitions from NOT_LOADED to one cached result + - delete path treats snapshot only as selection evidence and revalidates live HDD +validation: + - APA magic/checksum/start/prev/next/bounds/type/sub-count + - total/used/free sector accounting + - HDL metadata parse + metadata SHA-256 + - destructive delete performs live target/journal/snapshot checks again +``` + +**POTWIERDZONE:** metadata is already loaded lazily by visible page and cached in +the session entry. A persistent catalogue index therefore cannot be justified by +"avoiding all metadata reads"; the current path does not read every game metadata +block up front. + +**INFERENCJA:** a persistent index is useful only if its validity can be checked +substantially cheaper than the raw APA work it replaces. If proving that no +external APA mutation occurred requires walking the whole chain anyway, the +index may simply move work around. Any persistent-index implementation must +first identify a trustworthy drive/APA generation signal or define a clearly +bounded weaker cache contract with full-scan fallback. + +## Dataset: HDL payload stream + +```yaml +name: hdl_payload_stream +producer: USB source or HDD target depending operation +consumers: + copy: + - ps2hdd target write on IOP + - EE SHA-256 consumer + verify: + - EE SHA-256 consumer of HDD read-back +lifetime: one open hdl0: stream descriptor / transaction phase +representation: sequential 64 KiB payload chunks +alignment: + IOP_stage_allocator: AllocSysMemory allocation plus manual 64-byte alignment + EE_DMA_destination: required 64-byte aligned + transfer_size: required multiple of 512 B; DMA helper requires 64 B multiple +transport: + source_fast: BDM read into IOP stage + target_write: IOP stage -> ps2hdd + EE_copy: SIF DMA once per chunk + fallback: stock fileXio path +batch_size: 64 KiB +ownership_states: + - stage[0]/stage[1] are IOP-owned buffers + - prefetch worker owns requested alternate stage until done semaphore + - main IOP path owns current completed stage during HDD/SIF consumption + - SIF DMA must complete before current ioctl returns + - EE destination becomes usable after ioctl return + D-cache invalidation +validation: + - layout query matches partition plan + - transfer bounds and sector multiples + - full target SHA-256 after required flush +``` + +The IOP tries to allocate two 64 KiB staging buffers (+ alignment slop). If that +fails it falls back to one stage, preserving correctness while losing prefetch. +The second stage is therefore an optimization state, not an admission contract. + +The producer schedule for COPY is already: + +```text +obtain current source stage +submit/schedule next source read into alternate stage +write current stage to HDD +DMA current stage to EE +return to EE +EE consumes/hash current destination +request next chunk +``` + +This is already `submit early` for USB prefetch. + +## Current bottleneck candidates after ownership audit + +### 1. Redundant source replay on recovery + +Status: **implemented as isolated experiment; hardware gate pending.** + +This is the highest-priority work-removal candidate because it can eliminate a +prefix or complete ISO pass rather than merely shorten a control operation. + +### 2. Source reopening at PAYLOAD_VERIFIED with a complete checkpoint + +Status: **implemented in the isolated resume-hash experiment; CI/hardware gate +pending.** + +A matching full checkpoint already provides the original producer digest. HDD +read-back remains the correctness consumer; current USB media is unnecessary +unless checkpoint restore fails. + +### 3. Persistent HDL catalogue index + +Status: **design not yet justified.** + +Do not implement merely because persistent caches sound fast. The current +catalogue already caches metadata per menu session and loads it lazily. First +prove a cheap invalidation/generation contract that detects external APA +mutation without replaying the same full chain walk. + +### 4. SIF DMA completion wait / second EE destination + +Status: **defer until real-hardware attribution.** + +Current IOP `dma_to_ee()` submits `sceSifSetDma()` and waits for completion before +returning the ioctl. Simply removing that wait is invalid because EE currently +uses a single destination buffer and hashes it immediately after the synchronous +ioctl returns. + +A real overlap experiment therefore requires a new ownership protocol, for +example two EE consumer buffers or a ring with explicit FREE/FILLING/READY/ +CONSUMING states. It is only justified if PROFILE ON hardware telemetry shows +`sif-dma-completion` is materially exposed on the critical path after USB/HDD +work is accounted for. + +### 5. Journal/sidecar small-file metadata churn + +Status: **measure before redesign.** + +The optional sidecar adds a 256-byte write/readback/replace transaction on the +same 32 MiB boundary as the existing 512-byte journal. Combining records or +changing journal ABI to save these operations is a larger correctness change +than the current evidence justifies. Release-like A/B must first show a +meaningful uninterrupted-install regression attributable to checkpoint +maintenance. + +## Next measurement decisions + +1. Finish the resume-hash matched A/B and correctness matrix on real PS2. +2. Use PROFILE ON only to attribute USB/HDD/SIF/EE time; use PROFILE OFF for the + release-like acceptance result. +3. If SIF completion is materially exposed, design explicit EE/IOP ownership + before adding another buffer. +4. If catalogue entry latency is materially user-visible, identify a cheap and + correct mutation-generation contract before implementing persistent cache. +5. Re-profile whole-system behavior after any accepted change because the + bottleneck may move. From 56e763208e8b19fc4d21060801dcc78c84ae74fc Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 20:45:34 +0200 Subject: [PATCH 094/156] bench: record source-free stage4 checkpoint restore --- tools/parse_hdl_perf.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/parse_hdl_perf.py b/tools/parse_hdl_perf.py index 12ab9b45..0bad8de0 100644 --- a/tools/parse_hdl_perf.py +++ b/tools/parse_hdl_perf.py @@ -55,7 +55,7 @@ ) RESUME_FULL_RESTORE_RE = re.compile( PREFIX - + r"HDL restored complete source SHA checkpoint bytes=(\d+); skipped full USB hash pass" + + r"HDL restored complete source SHA checkpoint bytes=(\d+); skipped full USB hash pass( and source reopen)?" ) RESUME_PREFIX_FALLBACK_RE = re.compile( PREFIX @@ -184,7 +184,10 @@ def parse_log(text: str) -> dict[str, object]: match = RESUME_FULL_RESTORE_RE.search(line) if match: result["resume_hash"]["full_restores"].append( # type: ignore[index] - {"bytes": int(match.group(1))} + { + "bytes": int(match.group(1)), + "source_reopen_skipped": match.group(2) is not None, + } ) continue @@ -238,7 +241,7 @@ def selftest() -> None: [0047] HDL perf copy traffic useful=1048576 sif-dma=1048576 ee-cache-maint=2097152 fallback-source=0 [0048] HDL restored source SHA checkpoint bytes=33554432; skipped prefix rehash [0049] HDL source SHA checkpoint unavailable result=-5; using safe prefix rehash -[0050] HDL restored complete source SHA checkpoint bytes=1073741824; skipped full USB hash pass +[0050] HDL restored complete source SHA checkpoint bytes=1073741824; skipped full USB hash pass and source reopen [0051] HDL complete source SHA checkpoint unavailable result=-7; using safe full source hash [0052] HDL source SHA checkpoint save failed progress=32768 result=-12; journal remains authoritative [0053] HDL transaction target=PP.TEST stage=6 progress=524288/524288 result=0 @@ -253,6 +256,7 @@ def selftest() -> None: assert parsed["resume_hash"]["prefix_restores"][0]["bytes"] == 33554432 # type: ignore[index] assert parsed["resume_hash"]["prefix_fallback_results"] == [-5] # type: ignore[index] assert parsed["resume_hash"]["full_restores"][0]["bytes"] == 1073741824 # type: ignore[index] + assert parsed["resume_hash"]["full_restores"][0]["source_reopen_skipped"] # type: ignore[index] assert parsed["resume_hash"]["full_fallback_results"] == [-7] # type: ignore[index] assert parsed["resume_hash"]["checkpoint_write_failures"][0]["progress"] == 32768 # type: ignore[index] assert parsed["resume_hash"]["transactions"][0]["result"] == 0 # type: ignore[index] From b9b2f5ffab45a535b24fec1fb0b044096216429c Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 20:46:56 +0200 Subject: [PATCH 095/156] bench: bind source-free stage4 evidence in samples --- tools/resume_hash_ab_preflight.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/resume_hash_ab_preflight.py b/tools/resume_hash_ab_preflight.py index afd1224b..4c7026b3 100644 --- a/tools/resume_hash_ab_preflight.py +++ b/tools/resume_hash_ab_preflight.py @@ -127,6 +127,7 @@ def sample_template(identity: dict[str, Any], profile: str) -> dict[str, Any]: "completed_sectors": 0, "checkpoint_status": "FILL_ME", "checkpoint_restored_bytes": 0, + "source_reopen_skipped": False, }) return { "identity": { @@ -139,7 +140,9 @@ def sample_template(identity: dict[str, Any], profile: str) -> dict[str, Any]: "note": ( "Replace every FILL_ME and zero measurement. For recovery workloads, " "checkpoint_status is restored or fallback for EXP and not-applicable " - "for BASE. Keep one workload/depth per comparator input." + "for BASE. source_reopen_skipped must be true only for a restored " + "PAYLOAD_VERIFIED EXP run whose log confirms the source-free path. " + "Keep one workload/depth per comparator input." ), "samples": samples, } @@ -171,6 +174,7 @@ def selftest() -> None: assert len(template["samples"]) == 8 assert sum(sample["mode"] == "BASE" for sample in template["samples"]) == 4 assert sum(sample["mode"] == "EXP" for sample in template["samples"]) == 4 + assert all(not sample["source_reopen_skipped"] for sample in template["samples"]) broken = {mode: {key: dict(value) if isinstance(value, dict) else value for key, value in experiment[mode].items()} From a43b073c32348e020c234fff64615c8c4cddc98d Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 20:47:31 +0200 Subject: [PATCH 096/156] bench: validate source-free stage4 sample evidence --- tools/validate_resume_hash_samples.py | 135 ++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 tools/validate_resume_hash_samples.py diff --git a/tools/validate_resume_hash_samples.py b/tools/validate_resume_hash_samples.py new file mode 100644 index 00000000..2c7608c6 --- /dev/null +++ b/tools/validate_resume_hash_samples.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Validate recovery-specific evidence before resume-hash A/B comparison. + +The generic comparator checks timing identity and restored byte counts. This +companion gate proves that PAYLOAD_VERIFIED experiment samples came from the +new source-free stage-4 path rather than an older checkpoint build that still +re-opened the ISO before restoring the same digest. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + + +def validate(raw: Any) -> int: + if isinstance(raw, dict): + raw = raw.get("samples") + if not isinstance(raw, list) or not raw: + raise ValueError("input must contain a non-empty samples array") + + payload_restores = 0 + for index, sample in enumerate(raw): + if not isinstance(sample, dict): + raise ValueError(f"sample {index}: expected object") + for key in ("mode", "workload_kind", "checkpoint_status", + "source_reopen_skipped"): + if key not in sample: + raise ValueError(f"sample {index}: missing {key}") + skipped = sample["source_reopen_skipped"] + if not isinstance(skipped, bool): + raise ValueError( + f"sample {index}: source_reopen_skipped must be boolean" + ) + mode = str(sample["mode"]).upper() + workload = str(sample["workload_kind"]) + status = str(sample["checkpoint_status"]) + + expected = ( + mode == "EXP" + and workload == "payload_verified_resume" + and status == "restored" + ) + if skipped != expected: + if expected: + raise ValueError( + f"sample {index}: restored PAYLOAD_VERIFIED EXP lacks " + "source-reopen-skip evidence" + ) + raise ValueError( + f"sample {index}: source_reopen_skipped is true outside a " + "restored PAYLOAD_VERIFIED EXP run" + ) + if expected: + payload_restores += 1 + return payload_restores + + +def selftest() -> None: + valid = { + "samples": [ + { + "mode": "BASE", + "workload_kind": "payload_verified_resume", + "checkpoint_status": "not-applicable", + "source_reopen_skipped": False, + }, + { + "mode": "EXP", + "workload_kind": "payload_verified_resume", + "checkpoint_status": "restored", + "source_reopen_skipped": True, + }, + { + "mode": "EXP", + "workload_kind": "payload_verified_resume", + "checkpoint_status": "fallback", + "source_reopen_skipped": False, + }, + { + "mode": "EXP", + "workload_kind": "copy_resume", + "checkpoint_status": "restored", + "source_reopen_skipped": False, + }, + ] + } + assert validate(valid) == 1 + + bad = json.loads(json.dumps(valid)) + bad["samples"][1]["source_reopen_skipped"] = False + try: + validate(bad) + except ValueError as error: + assert "lacks" in str(error) + else: + raise AssertionError("missing source-free evidence must fail") + + bad = json.loads(json.dumps(valid)) + bad["samples"][3]["source_reopen_skipped"] = True + try: + validate(bad) + except ValueError as error: + assert "outside" in str(error) + else: + raise AssertionError("false source-free evidence must fail") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("samples", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("validate_resume_hash_samples selftest: PASS") + return 0 + if args.samples is None: + parser.error("samples is required unless --selftest is used") + + try: + count = validate(json.loads(args.samples.read_text(encoding="utf-8"))) + except (OSError, json.JSONDecodeError, ValueError) as error: + print(f"validate_resume_hash_samples: {error}", file=sys.stderr) + return 2 + print(f"resume-hash sample evidence: PASS payload_source_free_restores={count}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 3edafadc5ad5b94f2bf42525a9982591835dbb85 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 21:11:34 +0200 Subject: [PATCH 097/156] Docs: freeze CI 706 resume-hash hardware gate --- docs/HDL_RESUME_HASH_BENCHMARK.md | 221 ++++++++++++++++++++---------- 1 file changed, 146 insertions(+), 75 deletions(-) diff --git a/docs/HDL_RESUME_HASH_BENCHMARK.md b/docs/HDL_RESUME_HASH_BENCHMARK.md index de74de15..fabd9427 100644 --- a/docs/HDL_RESUME_HASH_BENCHMARK.md +++ b/docs/HDL_RESUME_HASH_BENCHMARK.md @@ -3,56 +3,72 @@ This document defines the real-PS2 gate for the optional `HDL_RESUME_HASH_CHECKPOINT=1` experiment. -The experiment is deliberately not the default build. It exists to test the -first optimization rule from the PS2 Optimization Research Library v2: remove -work before trying to make the same work faster. +The experiment is deliberately not the default build. It tests the first rule +of the PS2 Optimization Research Library v2: remove work before trying to make +the same work faster. ## Epistemic status **POTWIERDZONE** -- the existing COPY resume path reconstructs the SHA-256 state by reading the - entire already-copied ISO prefix again; +- the legacy COPY resume path reconstructs the SHA-256 state by rereading the + entire already-copied ISO prefix; - a resumed `PAYLOAD_VERIFIED` transaction without a persisted source digest performs another complete source-ISO hash pass; - the normal transaction journal remains authoritative; - the optional checkpoint is a 256-byte authenticated sidecar bound to source - size, completed byte count, source fingerprint and target ID; + size, exact completed byte count, source fingerprint and target ID; - a missing, corrupt or mismatching sidecar falls back to the old rehash path; -- host tests prove that a restored SHA state produces the same final digest as - an uninterrupted hash and that altered checkpoint records are rejected; -- CI #699 builds matched PROFILE OFF and PROFILE ON experiment variants only - after the frozen Phase-0 pair has passed exact SHA-256 validation; -- in CI #699 the checkpoint experiment changes only the EE application image: +- host tests prove that restored SHA state produces the same final digest as an + uninterrupted hash and that altered checkpoint records are rejected; +- CI #706 builds matched PROFILE OFF and PROFILE ON experiment variants only + after the frozen Phase-0 pair passes exact SHA-256 validation; +- in CI #706 the checkpoint experiment changes only the EE application image: its PROFILE OFF `hdl_stream.irx` is byte-identical to the frozen PROFILE OFF - IRX and its PROFILE ON IRX is byte-identical to the frozen PROFILE ON IRX. + IRX and its PROFILE ON IRX is byte-identical to the frozen PROFILE ON IRX; +- for a resumed `PAYLOAD_VERIFIED` transaction, a valid complete checkpoint is + restored before the source ISO is opened; the experiment can therefore skip + source reopen, size check, first/tail fingerprint reads and ISO identity probe; +- HDD payload SHA-256 read-back remains mandatory before metadata commit. **CURRENT IMPLEMENTATION** +- project experiment identity: CI #706, head + `a43b073c32348e020c234fff64615c8c4cddc98d`; +- frozen baseline source identity remains CI #666 at + `7875b14d837d6332f5edc37f1c12a55527d7dd87`; - transaction journal interval: 16384 ISO sectors = 32 MiB; - release-like experiment: `HDL_PROFILE=0 HDL_RESUME_HASH_CHECKPOINT=1`; - instrumented experiment: `HDL_PROFILE=1 HDL_RESUME_HASH_CHECKPOINT=1`; - release-like baseline: frozen `HDL_PROFILE=0` ELF; - instrumented baseline: frozen `HDL_PROFILE=1` ELF; - the experiment writes, reads back, codec-verifies and renames a 256-byte - sidecar at existing 32 MiB transaction checkpoints and on an orderly cancel; + sidecar at existing 32 MiB transaction checkpoints and on orderly cancel; - checkpoint state is written before the corresponding transaction journal; - the sidecar is deleted when a new zero-progress transaction supersedes it and when a transaction reaches COMPLETE; -- `tools/parse_hdl_perf.py` extracts checkpoint restore/fallback/write-failure - events and transaction results from `HDDMAN.LOG`; +- `tools/parse_hdl_perf.py` extracts checkpoint restore/fallback/write-failure, + transaction results and the stage-4 source-reopen-skip event from `HDDMAN.LOG`; - `tools/resume_hash_ab_preflight.py` binds every hardware sample template to - the exact baseline/experiment ELF and IRX identities from the CI artifact; + exact baseline/experiment ELF and IRX identities from the CI artifact; - `tools/compare_hdl_resume_hash_ab.py` separates safe fallback runs from valid - optimized restores and reports p50/p95/p99/max for the supplied timing data. + optimized restores and reports p50/p95/p99/max for supplied timing data; +- an accepted `payload_verified_resume` optimized sample must also report + `source_reopen_skipped=true`, so an older checkpoint ELF cannot masquerade as + the current experiment merely because it restored the same SHA state; +- pinned PS2SDK v2.0.0 `bdmfs_fatfs` maps `mass:` close to FatFs `f_close()` + under its filesystem lock; the wrapper itself does not add a separate + block-device flush operation after close. **INFERENCJA** - removing an N-byte prefix replay should save approximately N bytes of USB traffic on a matching COPY resume; -- restoring a full-payload checkpoint should remove one full source-ISO pass - before resumed HDD verification; -- because checkpoint state is bound to the journal progress, loss of power after +- restoring a full-payload checkpoint should remove one complete source-ISO + pass before resumed HDD verification; +- restoring that checkpoint before source admission should additionally remove + source reopen, size/fingerprint and ISO-probe work from stage-4 recovery; +- because checkpoint state is bound to journal progress, loss of power after checkpoint replacement but before the newer journal becomes visible should make the checkpoint stale relative to the older journal and therefore reject it, falling back to source rehash; @@ -63,16 +79,21 @@ work before trying to make the same work faster. - recovery time improves materially on real PS2; - the uninterrupted-copy regression from checkpoint maintenance is negligible; -- temp-file write/readback/rename plus the mass-storage stack provide adequate - persistence behavior across the tested orderly-cancel and guarded fault +- stage-4 recovery with a valid full checkpoint remains correct with the source + USB device absent because the HDD read-back is compared against the digest + accumulated while the original source bytes were copied; +- temp-file write/readback/rename plus the tested mass-storage stack provide + adequate persistence behaviour across orderly-cancel and guarded fault windows on real hardware; - no adapter/USB implementation exposes a correctness or persistence corner case not covered by host tests. -The sidecar is therefore logically fail-safe in current source, but this document -does **not** call it physically durable across arbitrary power loss until the -real-hardware crash-window matrix demonstrates that property. No performance or -power-loss durability claim is accepted before this hardware gate passes. +The sidecar is logically fail-safe in current source, but this document does +**not** call it physically durable across arbitrary power loss until the +real-hardware crash-window matrix demonstrates that property. `f_close()` is +not treated as a magical PC-style `fsync()` contract without source or hardware +evidence. No performance or power-loss durability claim is accepted before the +hardware gate passes. ## Authoritative corpus rationale @@ -82,25 +103,34 @@ This test follows: correctness hash and benchmark provenance; distinguish instrumented and release-like profiles; - `PS2_HDD_APA_PFS_HDL_filesystem_optimization_research_corpus_v2.md`: keep HDL - raw/storage workloads separate, eliminate repeated reads before attempting - lower-level acceleration, use deterministic checksum validation, and do not - infer throughput from interface headline rates; + raw/storage workloads separate, eliminate repeated reads before lower-level + acceleration, use deterministic checksum validation, and do not infer + throughput from interface headline rates; - `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md`: reuse producer state only when representation, lifetime and ownership remain valid for the consumer; - `PS2_Whole_System_Scheduling_research_corpus_v2.md`: treat USB, IOP, SIF and - storage as one producer/consumer path and report distribution/tail behavior, + storage as one producer/consumer path and report distribution/tail behaviour, not only an average; - `PS2_USB_1_1_optimization_research_corpus.md`: USB Full-Speed scheduling has millisecond-scale granularity, so redundant source traffic belongs on the - critical recovery path rather than being dismissed as free background work. + critical recovery path rather than being dismissed as free background work; +- pinned PS2SDK source for `bdmfs_fatfs`: use the actual v2.0.0 filesystem close + path when reasoning about current implementation rather than generic POSIX + expectations. ## Frozen and experiment identities -The authoritative matched A/B identity was emitted by CI #699 for project head -`9d17f6805927bbe6fba8417237ff2e72869e0452` in +The authoritative matched A/B identity is emitted by CI #706 for project head +`a43b073c32348e020c234fff64615c8c4cddc98d` in `RESUME_HASH_AB_IDENTITY.json`. +Artifact ZIP digest: + +```text +sha256:3623a52466ca99408163b1bcfca3c4b02b3e5e2f3a173b1e18eaa91c288adc36 +``` + ### PROFILE OFF: release-like pair Baseline: @@ -115,8 +145,8 @@ Experiment: ```text PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF -bytes 636084 -sha256 d463064036b2f253462cf3fbc90b75db89da36979acfd88334b50e6eecd7ef11 +bytes 636340 +sha256 ab4dc7addd62e051a88a183dbe00fee8ba889a867549ee35c1781addfd60b0a5 ``` Both use the exact same PROFILE OFF `hdl_stream.irx`: @@ -126,15 +156,15 @@ bytes 8405 sha256 f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 ``` -Static checkpoint delta against frozen PROFILE OFF: +Static experiment delta against frozen PROFILE OFF: ```text -stripped ELF +3200 B -EE named text +2512 B (229956 -> 232468) +stripped ELF +3456 B +EE named text +2624 B (229956 -> 232580) EE named functions +4 (609 -> 613) -EE instructions +629 (57539 -> 58168) -execute_transaction() +288 B (6156 -> 6444) -execute_transaction insn +72 (1540 -> 1612) +EE instructions +657 (57539 -> 58196) +execute_transaction() +400 B (6156 -> 6556) +execute_transaction insn +100 (1540 -> 1640) ``` ### PROFILE ON: instrumented pair @@ -151,8 +181,8 @@ Experiment: ```text PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF -bytes 641588 -sha256 849cb5fec6cce31b214210a09d8ffe81ce181a6790f0ef5fa28bb8f62ce89d62 +bytes 641844 +sha256 0d27e82ba06f6a3d19d53e70427a397859ef2cead725c56bb8c25bc3458520dd ``` Both use the exact same PROFILE ON `hdl_stream.irx`: @@ -162,29 +192,33 @@ bytes 9861 sha256 8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db ``` -Static checkpoint delta against frozen PROFILE ON: +Static experiment delta against frozen PROFILE ON: ```text -stripped ELF +3200 B -EE named text +2512 B (232780 -> 235292) +stripped ELF +3456 B +EE named text +2640 B (232780 -> 235420) EE named functions +4 (618 -> 622) -EE instructions +631 (58246 -> 58877) -execute_transaction() +288 B (6156 -> 6444) -execute_transaction insn +72 (1540 -> 1612) +EE instructions +663 (58246 -> 58909) +execute_transaction() +416 B (6156 -> 6572) +execute_transaction insn +104 (1540 -> 1644) ``` -The 2-instruction difference between OFF/ON checkpoint deltas is profiler-side -code generation outside `execute_transaction()`; the transaction body itself -has the same measured static delta in both pairs. +The PROFILE ON/OFF static deltas differ slightly because profiler-side code +changes LTO placement/generation. The experiment remains EE-only because each +experiment IRX is byte-identical to the corresponding frozen baseline IRX. + +The stage-4 source-reopen cut added only 256 B of stripped ELF relative to the +previous valid CI #699 experiment, but the authoritative test identity is now +#706. Do not use #699 binaries for the source-reopen-skip claim. -Do not rebuild and silently call a result one of these binaries. Exact hash is -the identity. If experiment source changes, CI emits a new experiment identity -while the frozen baseline hashes remain fixed. +Do not rebuild and silently call a result one of these binaries. Exact SHA-256 +is the identity. If experiment source changes, CI emits a new experiment +identity while the frozen baseline hashes remain fixed. ## Why there are two A/B pairs Use PROFILE OFF for the acceptance timing result. It is the release-like pair -and does not pay the Phase-0 HDL profiler overhead. +and does not pay Phase-0 HDL profiler overhead. Use PROFILE ON for attribution. Both binaries in that pair contain the same IOP latency/traffic profiler and the same PROFILE ON IRX, allowing the existing @@ -233,7 +267,7 @@ RESUME_HASH_AB_PROFILE_OFF_TEMPLATE.json RESUME_HASH_AB_PROFILE_ON_TEMPLATE.json ``` -Fill the hardware/runtime fields rather than inferring them from CI. +Fill hardware/runtime fields rather than inferring them from CI. ## Correctness gate before timing @@ -249,8 +283,16 @@ For both binaries: 6. preserve the transaction/session log; 7. verify the same final source/payload correctness hash in both variants. -Any checksum mismatch, invalid target metadata, unexpected cleanup behavior or -resume refusal invalidates the performance sample. +For the CI #706 experiment add one stage-4 correctness case: + +8. create a valid `PAYLOAD_VERIFIED` recovery point, then remove the source USB + device before resuming EXP; a valid full checkpoint must allow HDD read-back, + metadata commit and COMPLETE without reopening the ISO. Run the equivalent + BASE case with source present because BASE has no persisted source digest. + +Any checksum mismatch, invalid target metadata, unexpected cleanup behaviour, +resume refusal, or stage-4 source-free recovery that skips mandatory HDD +verification invalidates the experiment. ## Workload A: uninterrupted-install regression @@ -337,25 +379,41 @@ correctness failures ## Workload C: resumed PAYLOAD_VERIFIED -Purpose: measure removal of the complete extra USB source-hash pass. +Purpose: measure removal of the complete extra USB source-hash pass and source +admission work when a complete checkpoint already represents the producer. -Create a valid `PAYLOAD_VERIFIED` recovery point with the source ISO still -available. Preserve identical transaction state for baseline and experiment -runs as far as the format permits. +Create a valid `PAYLOAD_VERIFIED` recovery point. For the ordinary matched A/B +run keep source state equivalent. Separately run the source-absent EXP +correctness case described above. Expected work difference: ```text -baseline: one full source ISO SHA-256 pass, then HDD payload verification -EXP: restore final source SHA state, then HDD payload verification +baseline: + open/validate source ISO + one full source ISO SHA-256 pass + HDD payload verification + +EXP with valid complete checkpoint: + restore final source SHA state before source admission + skip source reopen/size/fingerprint/ISO probe + HDD payload verification ``` The HDD verification remains required in both variants. Do not count its time as work removed by this experiment. -For a valid optimized run, `parse_hdl_perf.py` must report a `full_restores` -record whose byte count equals `source_bytes`. Full-source fallback is kept as a -correctness-safe fallback sample, not as evidence for optimized latency. +For a valid optimized run: + +- `parse_hdl_perf.py` must report a `full_restores` record whose byte count equals + `source_bytes`; +- it must also report the source-reopen-skip event; +- the sample template must set `source_reopen_skipped=true`; +- `tools/compare_hdl_resume_hash_ab.py` rejects an accepted stage-4 EXP restore + that does not carry that flag. + +Full-source fallback is kept as a correctness-safe fallback sample, not as +evidence for optimized latency. Report: @@ -364,6 +422,7 @@ resume-to-HDD-verification-start p50/p95/p99/max full recovery p50/p95/p99/max source ISO bytes checkpoint restore/fallback status +source_reopen_skipped correctness failures ``` @@ -382,9 +441,13 @@ encode/write/readback/verify/replace transaction journal ``` A checkpoint can never advance journal progress. Restore validates the sidecar -against the transaction loaded from the authoritative journal, including its +against the transaction loaded from the authoritative journal, including exact completed byte count, source fingerprint and target ID. +**CURRENT IMPLEMENTATION:** the pinned PS2SDK v2.0.0 `bdmfs_fatfs` close path +calls FatFs `f_close()` while holding the filesystem lock. No separate explicit +block-device flush is added by that wrapper after close. + **INFERENCJA:** if the new sidecar becomes visible but the newer journal does not, the older journal progress makes the sidecar stale and restore is rejected. If checkpoint replacement itself is lost/corrupt, the optimization is lost and @@ -403,6 +466,7 @@ sidecar from another target ID valid .SHA primary valid temporary .SHN with primary absent loss/reset around sidecar replacement where the guarded fault procedure permits +valid full stage-4 sidecar with source USB absent ``` Every invalid/mismatched case must fall back to the legacy rehash path and still @@ -424,6 +488,10 @@ python3 tools/parse_hdl_perf.py HDDMAN.LOG --output run.json Fill one workload/depth per copy of the CI sample template. Do not combine COPY resume at 25%, 50% and 75% in one comparator input. +For stage-4 optimized samples copy the parser result into the sample record so +`source_reopen_skipped` is true only when the actual CI #706 event is present. +Do not infer the flag from the filename. + Then compare against the identity emitted with the tested artifact: ```sh @@ -433,7 +501,8 @@ python3 tools/compare_hdl_resume_hash_ab.py \ ``` The comparator rejects wrong ELF/IRX hashes, mixed PROFILE modes, mixed workload -IDs, mismatching correctness hashes, invalid restored-byte counts and too few +IDs, mismatching correctness hashes, invalid restored-byte counts, missing +source-reopen-skip evidence for accepted stage-4 EXP samples, and too few accepted EXP samples after fallback separation. ## Acceptance rule @@ -444,13 +513,15 @@ Promote the checkpoint path only if all are true: 2. frozen baseline identity remains unchanged; 3. matching checkpoint resumes actually skip the expected redundant source bytes; -4. COPY-resume and PAYLOAD_VERIFIED recovery show a clear real-hardware benefit; -5. uninterrupted-install p50/p95/p99/max do not show an unacceptable regression +4. valid `PAYLOAD_VERIFIED` EXP recovery demonstrably skips source reopen and + still performs the complete HDD verification; +5. COPY-resume and PAYLOAD_VERIFIED recovery show a clear real-hardware benefit; +6. uninterrupted-install p50/p95/p99/max do not show an unacceptable regression or new long-tail spikes; -6. sidecar failure always degrades to the old safe rehash behavior; -7. PROFILE ON attribution agrees with the release-like PROFILE OFF result rather +7. sidecar failure always degrades to the old safe rehash behaviour; +8. PROFILE ON attribution agrees with the release-like PROFILE OFF result rather than revealing an unrelated IOP/device change; -8. power/reset testing does not demonstrate a persistence behavior that can +9. power/reset testing does not demonstrate persistence behaviour that can violate the transaction correctness contract. If the result is ambiguous, keep the flag off and collect more samples. A neat From 4e6fd176687e085fc42752c892d8bebf3a76f432 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 21:14:51 +0200 Subject: [PATCH 098/156] Docs: add HDL IOP RAM budget --- docs/HDL_IOP_RAM_BUDGET.md | 220 +++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 docs/HDL_IOP_RAM_BUDGET.md diff --git a/docs/HDL_IOP_RAM_BUDGET.md b/docs/HDL_IOP_RAM_BUDGET.md new file mode 100644 index 00000000..dc7333ca --- /dev/null +++ b/docs/HDL_IOP_RAM_BUDGET.md @@ -0,0 +1,220 @@ +# HDL IOP RAM budget + +This document records the memory budget of the custom `hdl_stream` service +before any further buffering or SIF overlap experiment. + +The goal is not to pretend that the IOP has a free 2 MiB heap. The PS2 IOP RAM +also contains the kernel/runtime, every active IRX, stacks, heaps, filesystem and +device state. This document therefore separates the custom service's known +incremental footprint from system-wide memory that still requires real runtime +inventory. + +## Source-of-truth routing + +- `PS2_IOP_SIF_optimization_research_corpus_v2.md`: IOP RAM/service/thread/SIF + architecture and the requirement to budget text/data/BSS, stacks, heap + allocations and persistent buffers; +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md`: buffering as + explicit producer/consumer ownership rather than an arbitrary count of slots; +- `PS2_Whole_System_Scheduling_research_corpus_v2.md`: new buffering is accepted + only if it hides measured exposed latency without creating another resource + bottleneck; +- pinned PS2SDK `b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b`: current API/data contracts; +- project CI #706: final `hdl_stream.irx` loaded section sizes for the matched + PROFILE pair. + +## Epistemic labels + +**POTWIERDZONE** + +- retail-class IOP RAM budget is about 2 MiB as routed by the project corpus; +- one `hdl_stream` open attempts a two-stage allocation first and falls back to + one stage if that allocation fails; +- each stage is 64 KiB; +- the two-stage allocation requests `2 * 64 KiB + 63` bytes and manually aligns + the first stage to 64 bytes; +- direct BDM source mapping permits at most 4096 fragments; +- pinned PS2SDK defines packed `bd_fragment_t` as one `u64 sector` plus one + `u32 count`, therefore 12 bytes per fragment; +- the dedicated prefetch worker stack is `0x1000` = 4096 bytes; +- the prefetch worker is created only when two stages are available; +- CI #706 reports final loaded `hdl_stream.irx` sections of 7139 B text + 144 B + data for PROFILE OFF and 8595 B text + 144 B data for PROFILE ON, with zero + reported BSS in those final IRX images. + +**CURRENT IMPLEMENTATION** + +- application-side fast-I/O state tracks one active HDL target descriptor; +- the custom IOP driver itself is not a hard single-open API, so the numbers + below describe the current installer workload, not an enforcement guarantee + against another future caller opening multiple streams; +- the direct fragment map is allocated lazily and freed when source mapping is + reset or the stream closes; +- PROFILE ON stores IOP latency/traffic statistics inside the stream object; +- semaphores/thread-control objects are allocated by ThreadMan and are not + counted as exact bytes here because the project has not measured their runtime + allocator cost on the pinned IOP image. + +**INFERENCJA** + +- adding a third 64 KiB stage before telemetry demonstrates a producer/consumer + gap would consume a material fraction of the service's existing incremental + budget for an unproven benefit; +- the relevant safety margin is system-wide free IOP memory after all active IRX, + stacks and driver buffers, not simply `2 MiB - hdl_stream`. + +**HIPOTEZA DO TESTU** + +- two stages are sufficient for the current USB/HDD/SIF pipeline on real + hardware; +- a third stage is useful only if PROFILE ON shows repeatable prefetch misses or + exposed producer jitter that the extra ownership slot can hide. + +## Known allocation budget + +### Final IRX image + +CI #706: + +```text +PROFILE OFF loaded sections + .text 7139 B + .data 144 B + .bss 0 B + total 7283 B + +PROFILE ON loaded sections + .text 8595 B + .data 144 B + .bss 0 B + total 8739 B +``` + +The on-disk IRX file sizes are larger because they also contain module/ELF +structure. For RAM budgeting use loaded sections, not the archive/file size. + +### Streaming stages + +```text +one-stage allocation + 65536 + 63 = 65599 B + +two-stage allocation + 131072 + 63 = 131135 B +``` + +The 63-byte slop exists solely to obtain the documented 64-byte-aligned stage +address. It is allocator/alignment overhead, not a third payload buffer. + +### Direct-BDM fragment map + +Pinned PS2SDK contract: + +```text +sizeof(bd_fragment_t) = 8 + 4 = 12 B (packed) +maximum fragments = 4096 +maximum fragment map = 49152 B +``` + +This is a worst-case project cap. Ordinary contiguous or lightly fragmented ISO +files consume less. + +### Prefetch worker stack + +```text +HDL_STREAM_PREFETCH_STACK = 0x1000 = 4096 B +``` + +This stack exists only on the two-stage path because one-stage fallback does not +create the prefetch worker. + +### Stream object and ThreadMan bookkeeping + +The exact IOP ABI `sizeof(hdl_stream_file_t)` is not emitted by current CI. +Manual field accounting puts it around 0.65 KiB without profiling and around +1.3 KiB with PROFILE ON, but this document deliberately does not promote those +manual layout estimates to POTWIERDZONE bytes. + +For planning, reserve **1536 B per current stream object** as a conservative +project accounting value until CI emits the compiler-observed size. This reserve +is an engineering budget, not a hardware fact. + +Thread and semaphore kernel-control allocations are listed as **UNMEASURED** and +must be added to the runtime inventory before claiming exact system free RAM. + +## Current incremental worst-case service envelope + +For one current installer stream, direct BDM available, two stages allocated: + +```text + PROFILE OFF PROFILE ON +IRX loaded sections 7283 B 8739 B +two-stage allocation 131135 B 131135 B +max fragment map 49152 B 49152 B +prefetch stack 4096 B 4096 B +stream-object planning reserve 1536 B 1536 B + ---------- ---------- +known/reserved subtotal 193202 B 194658 B +ThreadMan control objects UNMEASURED UNMEASURED +other active IRX/runtime NOT INCLUDED NOT INCLUDED +``` + +The subtotal is roughly 190 KiB. It is **not** a claim that only this amount of +IOP RAM is consumed by the whole application stack. + +For one-stage low-memory fallback, remove one 64 KiB stage and the dedicated +prefetch stack. Correctness remains available while overlap is reduced. + +## Buffer expansion gate + +Do not add triple buffering merely because 64 KiB appears small next to 2 MiB. +A third stage costs another 65536 B before any ownership metadata and can also +increase live working-set pressure in a memory already shared by USB, FATFS, +ps2hdd, DEV9 and other services. + +A third stage may be prototyped only if all of the following are true: + +1. PROFILE ON real-hardware data shows a repeatable exposed producer/consumer + stall, such as prefetch misses or wait tail latency that two slots cannot + hide; +2. the active IOP module/stack/heap inventory has been recorded for that test; +3. the experiment preserves a documented free-memory safety margin after the + extra stage is allocated; +4. the new slot has explicit ownership states and cannot be overwritten while + USB, HDD, SIF or EE consumption still owns it; +5. p50/p95/p99/max improve without device/audio/service regressions; +6. one-stage and two-stage fallback semantics remain correct. + +The numeric safety margin is intentionally not invented here. It must be based +on measured free IOP memory and peak transient demand on the actual active module +set. Until that measurement exists, the current two-stage design remains the +maximum accepted service allocation. + +## Required runtime inventory before Phase-4 buffer growth + +Record at least: + +```yaml +console_scp: +hardware_revision: +active_iop_modules: + - name: + text_bytes: + data_bytes: + bss_bytes: +owned_thread_stacks: +hdl_stream_profile_mode: +hdl_stream_stage_count: +hdl_stream_fragment_count: +free_iop_memory_before_stream: +free_iop_memory_after_stream_open: +free_iop_memory_at_peak_copy: +minimum_free_iop_memory_observed: +workload: +sample_count: +correctness_hash: +``` + +The project corpus specifically requires active module text/data/BSS and stack +sizes to be treated as performance data. This runtime inventory is therefore a +gate, not optional documentation polish. From c30c7f66b618cd2ecdb959c105127f0ab48fedb4 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 21:16:16 +0200 Subject: [PATCH 099/156] Docs: record compiler-observed HDL stream sizes --- docs/HDL_IOP_RAM_BUDGET.md | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/docs/HDL_IOP_RAM_BUDGET.md b/docs/HDL_IOP_RAM_BUDGET.md index dc7333ca..960750ad 100644 --- a/docs/HDL_IOP_RAM_BUDGET.md +++ b/docs/HDL_IOP_RAM_BUDGET.md @@ -20,8 +20,8 @@ inventory. only if it hides measured exposed latency without creating another resource bottleneck; - pinned PS2SDK `b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b`: current API/data contracts; -- project CI #706: final `hdl_stream.irx` loaded section sizes for the matched - PROFILE pair. +- project CI #706: final `hdl_stream.irx` loaded section sizes and compiler output + for the matched PROFILE pair. ## Epistemic labels @@ -40,7 +40,10 @@ inventory. - the prefetch worker is created only when two stages are available; - CI #706 reports final loaded `hdl_stream.irx` sections of 7139 B text + 144 B data for PROFILE OFF and 8595 B text + 144 B data for PROFILE ON, with zero - reported BSS in those final IRX images. + reported BSS in those final IRX images; +- CI #706 disassembly shows the compiler-generated `AllocSysMemory` size for + `hdl_stream_file_t` is `0x288` = 648 B in PROFILE OFF and `0x538` = 1336 B in + PROFILE ON. **CURRENT IMPLEMENTATION** @@ -130,16 +133,18 @@ create the prefetch worker. ### Stream object and ThreadMan bookkeeping -The exact IOP ABI `sizeof(hdl_stream_file_t)` is not emitted by current CI. -Manual field accounting puts it around 0.65 KiB without profiling and around -1.3 KiB with PROFILE ON, but this document deliberately does not promote those -manual layout estimates to POTWIERDZONE bytes. +The final CI #706 machine code exposes the exact size passed to IOP +`AllocSysMemory` for the stream object: -For planning, reserve **1536 B per current stream object** as a conservative -project accounting value until CI emits the compiler-observed size. This reserve -is an engineering budget, not a hardware fact. +```text +PROFILE OFF sizeof(hdl_stream_file_t) = 0x288 = 648 B +PROFILE ON sizeof(hdl_stream_file_t) = 0x538 = 1336 B +``` + +The PROFILE ON increase is expected because latency histograms and traffic +counters live inside the stream object. -Thread and semaphore kernel-control allocations are listed as **UNMEASURED** and +Thread and semaphore kernel-control allocations are still **UNMEASURED** and must be added to the runtime inventory before claiming exact system free RAM. ## Current incremental worst-case service envelope @@ -152,15 +157,15 @@ IRX loaded sections 7283 B 8739 B two-stage allocation 131135 B 131135 B max fragment map 49152 B 49152 B prefetch stack 4096 B 4096 B -stream-object planning reserve 1536 B 1536 B +stream object 648 B 1336 B ---------- ---------- -known/reserved subtotal 193202 B 194658 B +known subtotal 192314 B 194458 B ThreadMan control objects UNMEASURED UNMEASURED other active IRX/runtime NOT INCLUDED NOT INCLUDED ``` -The subtotal is roughly 190 KiB. It is **not** a claim that only this amount of -IOP RAM is consumed by the whole application stack. +The known subtotal is roughly 188-190 KiB. It is **not** a claim that only this +amount of IOP RAM is consumed by the whole application stack. For one-stage low-memory fallback, remove one 64 KiB stage and the dedicated prefetch stack. Correctness remains available while overlap is reduced. From 8c7191fda44fd2413cb0feea80ad4da9b4d06cd7 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 21:17:13 +0200 Subject: [PATCH 100/156] Docs: advance storage and IOP corpus plan --- docs/CORPUS_V2_IMPLEMENTATION_PLAN.md | 64 ++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md b/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md index ec5dd4eb..0e28f86f 100644 --- a/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md +++ b/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md @@ -221,19 +221,49 @@ in hardware smoke tests. - [ ] Audit compiler-generated 64-bit divide/mod helpers and eliminate only cases whose arithmetic contract proves a cheaper transformation correct. +Current note: the isolated resume-hash experiment grows `execute_transaction()` +to 6556 B in PROFILE OFF and 6572 B in PROFILE ON, so the I-cache concern remains +real. Phase 2 is nevertheless held behind the Phase-0/Phase-1 real-hardware gate +rather than treating static size as a timing result. + Exit gate: smaller active I-cache footprint plus equal correctness/error paths. ## Phase 3: storage, APA and HDL dataflow -- [ ] Describe APA catalogue, ISO source, HDL transaction and payload stream with - producer/consumer/lifetime/ownership contracts. -- [ ] Add a persistent compact HDL catalogue index with version, drive identity, - APA-chain validation and checksum; mismatch always falls back to full scan. +- [x] Describe APA catalogue, ISO source, HDL transaction, SHA checkpoint and + payload stream with producer/consumer/lifetime/ownership contracts in + `docs/HDL_DATAFLOW_CONTRACTS.md`. +- [x] Audit recovery producer lifetime and isolate the resume-hash checkpoint + experiment. A matching COPY checkpoint removes prefix replay; a matching + full `PAYLOAD_VERIFIED` checkpoint also removes source reopen/fingerprint/ + ISO-probe work while retaining full HDD SHA-256 read-back. +- [ ] Evaluate a persistent compact HDL catalogue index only after identifying a + cheap, trustworthy drive/APA mutation-generation signal. If validation + requires the same full chain walk, retain the current lazy session cache + instead of adding a cache that merely moves work around. - [ ] Keep large sequential transfers and persistent descriptors; avoid repeated - small fileXio/RPC control-plane operations. + small fileXio/RPC control-plane operations outside semantically required + journal/checkpoint boundaries. - [ ] Re-measure USB source, HDD target and verification independently. - [ ] Tune chunk/batch size only with a sweep on the same device/workload. -- [ ] Audit sync/flush frequency against transaction durability requirements. +- [x] Audit checkpoint/journal small-file frequency for the current recovery + experiment: the 256-byte sidecar is written at the existing 32 MiB journal + boundary plus orderly cancel, not per 64 KiB payload chunk. Broader HDD/ + filesystem flush-frequency changes remain measurement-gated. + +### Phase-3 implementation notes + +The current experiment identity is CI #706 at project commit +`a43b073c32348e020c234fff64615c8c4cddc98d`. The frozen baseline remains CI +#666. The matched experiment keeps `hdl_stream.irx` byte-identical to its +corresponding frozen PROFILE mode, so the resume-hash/source-lifetime experiment +is EE-only. + +`docs/HDL_RESUME_HASH_BENCHMARK.md` is the hardware correctness/performance gate. +Its PROFILE OFF pair is the release-like acceptance pair; PROFILE ON exists for +USB/HDD/SIF/EE attribution. The experiment remains **HIPOTEZA DO TESTU** until +real hardware validates recovery gain, uninterrupted-install regression and +crash-window behaviour. Exit gate: lower non-hideable storage time without weakening journal or metadata commit safety. @@ -242,16 +272,28 @@ commit safety. - [x] Instrument queue-adjacent prefetch wait, service/device work and SIF completion independently enough to locate the dominant 64 KiB stage. -- [ ] Maintain the IOP-local producer path where the final device consumer is on - the IOP; do not bounce payload through EE without a consumer requirement. +- [x] Maintain the IOP-local producer path where the final device consumer is on + the IOP. Current fast COPY reads into IOP staging, writes directly to + ps2hdd, and sends only the EE SHA consumer copy over SIF. - [ ] Keep control metadata coarse-grained and bulk payload on DMA/data-plane paths. -- [ ] Express double buffering as explicit producer/consumer ownership states. +- [ ] Express double buffering as explicit producer/consumer ownership states in + code only if the current semaphore/stage protocol needs further evolution; + the current ownership contract is documented in `HDL_DATAFLOW_CONTRACTS.md`. - [ ] Evaluate triple buffering only if telemetry shows producer/consumer jitter that a third slot can actually hide within the IOP RAM budget. -- [ ] Add a static IOP RAM budget including IRX, staging buffers, fragment maps, - stacks and safety headroom. +- [x] Add a static incremental IOP RAM budget for IRX sections, staging buffers, + fragment map, prefetch stack and compiler-observed stream-object size in + `docs/HDL_IOP_RAM_BUDGET.md`. System-wide free-memory headroom remains a + required real-runtime measurement before any buffer growth. +- [ ] Record active IOP module text/data/BSS, owned stacks and minimum free IOP + memory on real hardware before any triple-buffer/ring expansion. - [ ] Sweep IOP worker priorities only after measuring service slack and stalls. +Current known `hdl_stream` worst-case incremental subtotal for one active +installer stream is 192314 B in PROFILE OFF and 194458 B in PROFILE ON, excluding +ThreadMan control allocations and the rest of the active IOP runtime. Those +unknowns are deliberately not guessed. + Exit gate: higher overlap/lower p99 with no IOP starvation or device regressions. ## Phase 5: allocators, copies and lifetime From 117add763110d032a3827013a172ed52f013b1c3 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 21:19:42 +0200 Subject: [PATCH 101/156] Perf: record expected IOP module load order --- tools/build_benchmark_provenance.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/build_benchmark_provenance.sh b/tools/build_benchmark_provenance.sh index a8429ff8..dd964837 100755 --- a/tools/build_benchmark_provenance.sh +++ b/tools/build_benchmark_provenance.sh @@ -100,8 +100,13 @@ benchmark_elf_bytes: "$BENCHMARK_ELF_BYTES" hdl_stream_irx: "$HDL_STREAM_IRX_PATH" hdl_stream_irx_sha256: "$HDL_STREAM_IRX_SHA" hdl_stream_irx_bytes: "$HDL_STREAM_IRX_BYTES" +# active_irx is a real-runtime observation and remains deliberately unknown in CI. active_irx: UNRECORDED +# embedded_irx describes payloads compiled into the EE image, not runtime state. embedded_irx: "iomanX fileXio secrman freesio2 freepad mcman mcserv secrsif poweroff bdm bdmfs_fatfs usbd usbmass_bd ps2dev9 ata_bd ps2fs ps2hdd-bdm hdl_stream" +# Expected application-owned post-reset load sequence from src/platform.c. +# ROM/kernel modules outside this list are why active_irx still requires hardware. +app_irx_load_order: "iomanX fileXio secrman freesio2 freepad mcman mcserv secrsif poweroff bdm bdmfs_fatfs usbd ps2dev9 ata_bd ps2hdd-bdm hdl_stream ps2fs usbmass_bd" workload: UNRECORDED direction: UNRECORDED buffering: UNRECORDED @@ -114,4 +119,4 @@ p95: UNRECORDED p99: UNRECORDED max: UNRECORDED deadline_misses: UNRECORDED -EOF +EOF \ No newline at end of file From 367a8658fbc67f4abf0df7c50b0ca2d44a16cfd1 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 21:20:16 +0200 Subject: [PATCH 102/156] Perf: derive IOP load order from platform source --- tools/build_benchmark_provenance.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tools/build_benchmark_provenance.sh b/tools/build_benchmark_provenance.sh index dd964837..57095c35 100755 --- a/tools/build_benchmark_provenance.sh +++ b/tools/build_benchmark_provenance.sh @@ -1,6 +1,7 @@ #!/bin/sh set -eu +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) OUT=${1:-BENCHMARK_PROVENANCE.yml} CC=${EE_CC:-mips64r5900el-ps2-elf-gcc} GIT_SHA=${PROJECT_GIT_SHA:-$(git rev-parse HEAD 2>/dev/null || printf 'unavailable')} @@ -67,6 +68,20 @@ file_bytes() fi } +# Keep the build-side expected load sequence tied to the actual platform source. +# The capture intentionally records only exec_irx(_irx, ...) calls in +# load_modules(). ps2hdd_posix is the embedded C symbol for PS2SDK's ps2hdd-bdm. +APP_IRX_LOAD_ORDER=$( + sed -n 's/.*exec_irx(\([A-Za-z0-9_]*\)_irx,.*/\1/p' \ + "$ROOT/src/platform.c" | + tr '\n' ' ' | + sed -e 's/ $//' -e 's/ps2hdd_posix/ps2hdd-bdm/' +) +if [ -z "$APP_IRX_LOAD_ORDER" ]; then + printf 'could not derive IOP load order from src/platform.c\n' >&2 + exit 2 +fi + BENCHMARK_ELF_SHA=$(file_sha256 "$BENCHMARK_ELF_PATH") BENCHMARK_ELF_BYTES=$(file_bytes "$BENCHMARK_ELF_PATH") HDL_STREAM_IRX_SHA=$(file_sha256 "$HDL_STREAM_IRX_PATH") @@ -104,9 +119,9 @@ hdl_stream_irx_bytes: "$HDL_STREAM_IRX_BYTES" active_irx: UNRECORDED # embedded_irx describes payloads compiled into the EE image, not runtime state. embedded_irx: "iomanX fileXio secrman freesio2 freepad mcman mcserv secrsif poweroff bdm bdmfs_fatfs usbd usbmass_bd ps2dev9 ata_bd ps2fs ps2hdd-bdm hdl_stream" -# Expected application-owned post-reset load sequence from src/platform.c. +# Application-owned post-reset load sequence derived from src/platform.c. # ROM/kernel modules outside this list are why active_irx still requires hardware. -app_irx_load_order: "iomanX fileXio secrman freesio2 freepad mcman mcserv secrsif poweroff bdm bdmfs_fatfs usbd ps2dev9 ata_bd ps2hdd-bdm hdl_stream ps2fs usbmass_bd" +app_irx_load_order: "$APP_IRX_LOAD_ORDER" workload: UNRECORDED direction: UNRECORDED buffering: UNRECORDED From edb0ec35d55eec05f6a417d498dc368efa561b6a Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 21:27:54 +0200 Subject: [PATCH 103/156] Docs: audit explicit 64-byte alignment contracts --- docs/ALIGNMENT_CONTRACT_AUDIT.md | 216 +++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 docs/ALIGNMENT_CONTRACT_AUDIT.md diff --git a/docs/ALIGNMENT_CONTRACT_AUDIT.md b/docs/ALIGNMENT_CONTRACT_AUDIT.md new file mode 100644 index 00000000..e6a8431b --- /dev/null +++ b/docs/ALIGNMENT_CONTRACT_AUDIT.md @@ -0,0 +1,216 @@ +# Explicit alignment contract audit + +This document classifies every current `__attribute__((aligned(64)))` in the +project before Phase-5 alignment cleanup. It deliberately does not equate +"64-byte aligned" with "faster". Each site must identify the actual consumer +that requires an alignment domain. + +Heap `memalign(64, ...)` calls are audited separately because their ownership and +DMA/cache contracts differ from static/stack object placement. + +## Source-of-truth routing + +- `PS2_Memory_Allocators_optimization_research_corpus_v2.md`: allocator ABI, + cache-line and device/DMA alignment are separate contracts; avoid blanket + over-alignment; +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md`: representation + and final-consumer contract decide layout; +- `PS2_IOP_SIF_optimization_research_corpus_v2.md`: SIF/DMA buffers need explicit + ownership/cache/alignment contracts rather than generic alignment folklore; +- pinned PS2SDK `b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b`: current fileXio/libpad/draw + implementation used to decide API requirements; +- project CI #712 source audit: 25 explicit `aligned(64)` sites. + +## Current PS2SDK findings + +### fileXio devctl/ioctl2 does not require a 64-byte-aligned caller buffer + +Pinned EE `fileXioDevctl()`/`fileXioIoctl2()`: + +- copies input `arg` bytes into the library-owned RPC packet with `memcpy`; +- stores the caller output pointer only as the final destination; +- uses a library-owned return packet for the IOP->EE DMA; +- its EE callback then copies the returned bytes into the caller buffer with + ordinary `memcpy`. + +Pinned IOP fileXio server: + +- passes its own `rwbuf` payload to `iomanX_devctl()`/`iomanX_ioctl2()`; +- DMA-transfers the complete library-owned `fxio_ctl_return_pkt` to EE internal + callback storage, not directly into the application's destination. + +The server `rwbuf` itself is allocated with ordinary `AllocSysMemory()`. + +**CURRENT IMPLEMENTATION:** 64-byte alignment of application input/output +objects is not a fileXio devctl/ioctl2 API requirement. + +### fileXio read/write explicitly supports unaligned caller buffers + +Pinned `fileXioRead()` performs cache maintenance and uses an internal callback +packet for unaligned edge bytes. Pinned `fileXioWrite()` explicitly calculates a +leading non-64-byte fragment and copies up to 64 bytes into +`fxio_write_packet.unalignedData[]` before the bulk transfer. + +**CURRENT IMPLEMENTATION:** ordinary fileXio read/write callers are not required +to provide a 64-byte-aligned pointer. + +This does not mean every application buffer may be arbitrarily aligned. A buffer +can still have another consumer such as GIF DMAC or libpad. + +## Sites with demonstrated alignment contracts + +### `src/platform.c`: `pad_buffer[256]` + +```text +current alignment: 64 B +consumer: padPortOpen() +status: KEEP 64 B +``` + +Pinned `libpad.h` explicitly states that the new-libpad pad area must be a +256-byte region at a 64-byte-aligned address. This is a real API/device contract, +not an optimization preference. + +Classification: **POTWIERDZONE / API-DMA alignment**. + +### `src/gs_ui_ps2.c`: `font_atlas[]` + +```text +current alignment: 64 B +consumer: draw_texture_transfer() -> DMATAG_REF -> GIF DMAC +status: KEEP aligned, 64 B itself not demonstrated +minimum candidate: 16 B qword/DMAC alignment +``` + +Pinned `draw_texture_transfer()` places the source address directly into +`DMATAG_REF` blocks. The atlas is therefore a direct DMA source and must obey the +DMA packet/source contract. The project also calls `FlushCache(0)` after building +the atlas, so the current code does not depend on 64-byte start alignment for a +selective cache-line flush. + +Classification: **POTWIERDZONE direct DMA source; INFERENCJA that 16 B is the +sufficient project alignment pending a small A/B/correctness build.** + +Do not remove alignment entirely. A future cleanup may evaluate 64 -> 16, not +64 -> arbitrary. + +## Sites where 64 B is not supported by the current consumer contract + +The following objects are either: + +1. fileXio devctl/ioctl2 input/output; +2. ordinary fileXio read/write buffers; or +3. CPU-only scratch after such a transfer. + +Pinned fileXio already supports non-64-byte caller addresses for those paths. +No other direct DMA consumer was found for these objects. + +### Raw HDD transport / repair + +| File | Object/site | Current consumer | 64-B status | +| --- | --- | --- | --- | +| `src/hdd_read.c` | `read_transfer_buffer` | `fileXioDevctl(HDIOC_READSECTOR)` output | not required by fileXio | +| `src/hdd_write.c` | `write_packet` | `fileXioDevctl(HDIOC_WRITESECTOR)` input arg | not required by fileXio | +| `src/hdd_write.c` | `sector_verify_buffer` | raw-read devctl output + CPU compare | not required by fileXio | +| `src/hdd_write.c` | `header_verify_buffer` | raw-read devctl output + CPU parse | not required by fileXio | +| `src/hdd_repair_ps2.c` | `repair_packet` | write devctl input arg | not required by fileXio | +| `src/hdd_repair_ps2.c` | `repair_verify` | raw-read devctl output + compare | not required by fileXio | +| `src/hdd_recovery_wrap.c` | `recovery_header` | raw-read devctl output + CPU recovery | not required by fileXio | +| `src/hdd_forensic_repair_ps2.c` | `write_packet` | write devctl input arg | not required by fileXio | +| `src/hdd_forensic_repair_ps2.c` | `source_verify` | raw-read devctl output + compare | not required by fileXio | +| `src/hdd_forensic_repair_ps2.c` | `write_verify` | raw-read devctl output + compare | not required by fileXio | +| `src/hdd_forensic_repair_ps2.c` | `repaired_header` | CPU-built header, then copied into devctl arg | no DMA consumer | +| `src/main.c` | `header_buffer` | raw-read devctl output + CPU parse | not required by fileXio | + +Classification: **CURRENT IMPLEMENTATION supports ordinary alignment; candidate +cleanup after frozen hardware experiment.** + +### HDL installer/catalogue + +| File | Object/site | Current consumer | 64-B status | +| --- | --- | --- | --- | +| `src/hdl_installer_ps2.c` | `hdl_zero_metadata` | ordinary `fileXioWrite` | not required by fileXio | +| `src/hdl_installer_ps2.c` | local `verify[4]` | ordinary `fileXioRead` | not required by fileXio | +| `src/hdl_installer_ps2.c` | final `actual[1024]` | `fileXioIoctl2(READ_METADATA)` output | not required by fileXio | +| `src/hdl_tools/source_ui.inc` | `admission_header` | raw-read devctl output + APA parse | not required by fileXio | +| `src/hdl_tools/source_ui_resume_hash.inc` | `admission_header` | experiment equivalent of above | not required by fileXio | +| `src/hdl_tools/catalog.inc` | local APA `header[1024]` | raw-read devctl output + parse | not required by fileXio | +| `src/hdl_tools/catalog.inc` | local `metadata[1024]` | raw-read devctl output + parse/hash | not required by fileXio | +| `src/hdl_tools/transaction.inc` | snapshot `metadata[1024]` | raw-read devctl output + parse/hash | not required by fileXio | +| `src/hdl_tools/transaction_resume_hash.inc` | snapshot `metadata[1024]` | experiment equivalent of above | not required by fileXio | + +The normal and resume-hash `.inc` files are alternate build fragments, not two +simultaneously linked copies. Any cleanup must update both variants so an +experiment does not silently regain a stale layout assumption. + +Classification: **CURRENT IMPLEMENTATION supports ordinary alignment; candidate +cleanup after frozen hardware experiment.** + +### Storage snapshot/backup scratch + +| File | Object/site | Current consumer | 64-B status | +| --- | --- | --- | --- | +| `src/header_backup.c` | `backup_scratch[1024]` | `read_exact_file` + CPU validation | not required by fileXio | +| `src/repair_snapshot.c` | `snapshot_verify[1024]` | `read_exact_file` + `memcmp` | not required by fileXio | + +These are the cleanest first A/B candidates because they do not cross raw-HDD +or custom-stream APIs at all. + +Classification: **CURRENT IMPLEMENTATION supports ordinary alignment; highest +confidence cleanup candidates.** + +## Count summary + +Current explicit attributes: 25. + +```text +KEEP exact 64 B API contract + 1 pad_buffer + +KEEP aligned but evaluate 64 -> 16 + 1 font_atlas direct GIF-DMA source + +64 B not demonstrated by current consumer + 23 fileXio/raw-HDD/CPU scratch sites +``` + +This summary is not permission for one global search/replace. Object placement, +stack frames and LTO layout can change even when the API contract permits lower +alignment. Cleanup is therefore staged and measured. + +## Cleanup sequence + +1. Keep the CI #706 runtime experiment binary identity frozen for its hardware + gate. Do not mutate that binary merely to make this table prettier. +2. First post-gate A/B: remove 64-B alignment only from + `backup_scratch`/`snapshot_verify` and compare final ELF/map/BSS/symbol layout. +3. If useful or neutral with correctness preserved, clean the raw-HDD/fileXio + scratch group as one reviewed transport-contract change. +4. Separately test `font_atlas` at 16-B alignment with GS texture-upload and all + supported video-mode regressions. +5. Never change `pad_buffer` below 64 B while using the current libpad contract. +6. Audit `memalign(64)` calls independently before touching heap alignment. + +## Acceptance record for any alignment cleanup + +Record: + +```yaml +object: +producer: +consumer: +old_alignment: +new_alignment: +alignment_domain: +api_or_hardware_contract: +elf_delta_bytes: +text_delta_bytes: +bss_delta_bytes: +stack_frame_delta_if_applicable: +correctness_test: +real_hardware_required: +``` + +Removing an unnecessary alignment is an optimization only if it reduces memory, +stack/layout pressure or another measured cost. A source diff containing fewer +`aligned(64)` strings is not itself a performance result. From 733676cef01635212b3e3c08b2fbaa3071f39268 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 21:30:01 +0200 Subject: [PATCH 104/156] Docs: classify allocation lifetimes and reuse candidates --- docs/ALLOCATION_LIFETIME_AUDIT.md | 301 ++++++++++++++++++++++++++++++ 1 file changed, 301 insertions(+) create mode 100644 docs/ALLOCATION_LIFETIME_AUDIT.md diff --git a/docs/ALLOCATION_LIFETIME_AUDIT.md b/docs/ALLOCATION_LIFETIME_AUDIT.md new file mode 100644 index 00000000..b08ede1e --- /dev/null +++ b/docs/ALLOCATION_LIFETIME_AUDIT.md @@ -0,0 +1,301 @@ +# Allocation and lifetime audit + +This document classifies the major dynamic allocations by producer, consumer, +lifetime and ownership before Phase-5 allocator work. The goal is not to replace +`malloc()` because it exists. The project corpus requires allocation changes to +remove measured churn, copies, fragmentation risk or peak working-set pressure. + +## Source-of-truth routing + +- `PS2_Memory_Allocators_optimization_research_corpus_v2.md`: classify by + lifetime; alignment is a consumer contract; avoid per-item churn on hot paths; +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md`: producer, + consumer, lifetime, ownership and representation decide reuse; +- `PS2_PERFORMANCE_BIBLE.md`: remove work/copies/allocations before specialised + kernels, but only where the workload exposes the cost; +- project CI #712 source audit plus current branch source. + +## Epistemic labels + +- **POTWIERDZONE**: current source/current API contract. +- **CURRENT IMPLEMENTATION**: behaviour/layout of this branch/toolchain. +- **INFERENCJA**: likely optimization consequence, not yet hardware measured. +- **HIPOTEZA DO TESTU**: change requiring real-PS2 A/B before acceptance. + +## Highest-value candidate: one transaction-owned 64 KiB I/O workspace + +### Current allocation pattern + +The HDL transaction uses `HDL_INSTALL_IO_BYTES = 64 KiB` buffers in three +sequential helpers: + +```text +hash_source_payload() + memalign(64, 64 KiB) + source SHA reconstruction + free + +copy_payload() + memalign(64, 64 KiB) + source read / IOP pump DMA destination / EE SHA consumer + free + +verify_target_digest() + memalign(64, 64 KiB) + HDD -> EE DMA destination / target SHA consumer + free +``` + +A normal fresh install executes copy then target verification. A resumed +`PAYLOAD_VERIFIED` legacy/fallback path executes source hash then target +verification. The helpers do not own their buffers concurrently. + +### Alignment contract + +**POTWIERDZONE:** keep 64-byte alignment. `hdl_fast_dma_read()` explicitly rejects +an EE destination whose address is not 64-byte aligned. This is the custom SIF/ +cache transport contract, unlike the ordinary fileXio scratch buffers audited in +`ALIGNMENT_CONTRACT_AUDIT.md`. + +### Lifetime conclusion + +**POTWIERDZONE:** these three buffers have transaction-local, mutually exclusive +lifetimes. + +**INFERENCJA:** a single transaction-owned 64 KiB workspace can serve all three +helpers and remove repeated allocator calls without increasing peak payload +memory or changing SIF/HDD/source representation. + +Candidate ownership: + +```text +execute_transaction owns workspace + FREE/UNUSED before allocation + SOURCE_HASH while hash_source_payload consumes it + COPY_IO while copy_payload consumes it + TARGET_VERIFY while verify_target_digest consumes it + released once at transaction exit +``` + +No helper may retain the pointer after return. + +### Proposed post-gate A/B + +Baseline: current helper-local allocations. + +Experiment: + +1. allocate one `memalign(64, HDL_INSTALL_IO_BYTES)` workspace only for stages + that need source/copy/target hashing; +2. pass pointer + capacity to each helper; +3. remove helper-local alloc/free pairs; +4. preserve all fileXio/SIF/cache/journal/error semantics; +5. free exactly once on transaction exit. + +Measure: + +```text +allocator calls per transaction +peak EE heap delta +copy/verify p50/p95/p99/max +total transaction p50/p95/p99/max +execute_transaction/static text delta +correctness hash +``` + +Priority: **HIGH after the frozen resume-hash hardware gate**, because it changes +an active bulk transaction path but does not require a new representation. + +## USB ISO catalogue array + +Producer: `scan_mass_images()`. + +Current representation: + +```text +initial capacity: 32 hdl_image_entry_t +allocation: calloc + growth: doubling below 1024, then +1024 entries +consumer: ISO selection UI + selected path/size handoff +lifetime: one begin_new_install selection session +release: before destructive confirmation / execute_transaction +``` + +**POTWIERDZONE:** the array is freed after the selected ISO fields have been +copied into the transaction, before the long-running HDD transaction begins. + +**INFERENCJA:** this is a sensible variable-cardinality session allocation. Do +not replace it with a giant permanent table without measured directory-size or +allocation-jitter evidence. + +Potential improvement only if logs show large catalogues/realloc churn: + +- count/size directory entries first only if the second scan is cheaper than + growth for the real workload; +- or use a bounded chunked/session arena if large catalogues are common. + +Priority: **LOW until catalogue cardinality is measured.** + +## Installed HDL catalogue array + +Producer: raw APA chain walker. + +Current representation: + +```text +initial capacity: 64 entries on first growth + growth: capacity * 2 +consumer: installed-games menu/details/delete selection +lifetime: one menu session +metadata: loaded lazily by visible page and cached in each entry +release: leaving menu; rebuilt after successful deletion +``` + +**POTWIERDZONE:** `realloc` occurs only while discovering main HDL partitions; +metadata itself is not separately heap-allocated per game. + +**INFERENCJA:** the growable session array is appropriate unless very large HDL +catalogues demonstrate allocator/copy cost. A persistent index does not solve +this automatically because invalidation must still be cheaper than the APA walk. + +Priority: **LOW/MEDIUM depending measured game count and catalogue latency.** + +## Forensic HDDMETA snapshot + +Producer: `build_snapshot_image()`. + +Current peak state during save: + +```text +image = malloc(image_size) +verify = malloc(image_size) +write image +read entire file into verify +memcmp(verify, image, image_size) +free both +``` + +`image_size` scales with `patch_count` because every touched original 1024-byte +APA header is embedded in the safety record. + +**POTWIERDZONE:** two equal-size buffers are simultaneously live solely for +read-back verification. + +**INFERENCJA:** the second full-size allocation can be removed without weakening +verification by reading the saved file back in a bounded scratch window and +comparing each window to the still-owned canonical `image`. This retains exact +byte-for-byte verification rather than replacing it with an unchecked write. + +Alternative: compare a streamed read-back hash against a canonical image hash, +but exact chunk comparison is simpler and preserves the current error contract. + +Priority: **MEDIUM for peak-memory robustness, LOW for normal performance** +because forensic repair is an exceptional cold path. + +## Bootstrap payload (`MBR.XLF`) + +Producer: `load_payload_file()`. + +```text +allocation: malloc(file size), bounded by HDD_MAX_MBR_PAYLOAD_SIZE +consumer: KELF/layout validation and subsequent bootstrap write workflow +ownership: bootstrap_source_t.payload +lifetime: prepare -> caller operations -> bootstrap_source_release +``` + +**POTWIERDZONE:** this is not transient read scratch. The loaded representation +is itself consumed across multiple stages. + +**INFERENCJA:** retaining one owned payload buffer is correct. Replacing it with +chunked streaming would complicate KELF/layout consumers and should not be done +without evidence that payload peak memory is a problem. + +Priority: **KEEP unless memory measurements disagree.** + +## Raw active bootstrap payload read + +Producer: `hdd_read_payload_image()`. + +```text +allocation: malloc(total selected payload bytes) +producer scratch: fixed HDD_TRANSFER_BYTES temporary +consumer: caller receives payload_out +ownership transfer: function -> caller +``` + +**POTWIERDZONE:** the heap allocation is the returned dataset, not helper-local +scratch. It cannot be removed without changing the API/consumer representation. + +Priority: **KEEP; redesign only with an explicit streaming consumer.** + +## Boot-chain text/config files + +Current source allocates bounded text buffers for complete small configuration +files and frees them at the end of the corresponding probe/parse operation. + +**INFERENCJA:** these are cold startup/configuration allocations. Replacing them +with custom pools is lower value than transaction/storage work unless startup +profiling identifies allocator cost or fragmentation. + +Priority: **LOW.** + +## Rescue/forensic/general storage allocations + +The source audit identifies additional allocations in `rescue_storage.c`, +`forensic_snapshot.c`, `bootstrap_source.c`, `storage.c` and boot tooling. They +are mostly operation/session-owned bounded records rather than per-64-KiB hot +loop allocations. + +Rule for subsequent review: + +```text +if allocation happens once per user operation: + measure peak bytes and failure behaviour first +if allocation happens at each bulk phase boundary: + consider lifetime reuse +if allocation happens inside a chunk/item loop: + treat as immediate review trigger +``` + +The current HDL fast 64-KiB loop does **not** allocate per chunk. Its allocation +churn is per phase, which is why one transaction workspace is the appropriate +first allocator experiment rather than an arena rewrite. + +## Phase-5 priority order + +1. **After hardware gate:** A/B one transaction-owned 64 KiB aligned workspace. +2. Record EE heap before/after transaction and at major phase boundaries if a + current safe heap query is available without materially perturbing the path. +3. If exceptional recovery memory matters, replace forensic full-size read-back + duplicate with bounded exact chunk comparison. +4. Measure ISO/game catalogue cardinality before changing their growth strategy. +5. Leave payload-owning allocations intact until a consumer can operate on a + different representation. +6. Do not introduce custom pools/arenas merely to reduce the count of `malloc` + strings in source. + +## Acceptance record for allocator changes + +```yaml +allocation_site: +producer: +consumer: +lifetime: +bytes: +alignment: +ownership_before: +ownership_after: +alloc_calls_before: +alloc_calls_after: +peak_heap_before: +peak_heap_after: +p50: +p95: +p99: +max: +correctness_hash: +error_path_test: +``` + +Allocator optimization is accepted only if it removes a real cost or reduces a +meaningful peak-memory risk while preserving ownership/error semantics. From eb0e180e6f1cfd35fee2fd82b69dc52477a4fe2e Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 21:31:27 +0200 Subject: [PATCH 105/156] Perf: add complete allocation inventory tool --- tools/allocation_inventory.py | 159 ++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 tools/allocation_inventory.py diff --git a/tools/allocation_inventory.py b/tools/allocation_inventory.py new file mode 100644 index 00000000..9602ef55 --- /dev/null +++ b/tools/allocation_inventory.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Emit a complete runtime allocation/free inventory for corpus-v2 review. + +This is deliberately a source inventory, not an allocator-performance claim. +The output keeps every malloc/calloc/realloc/memalign/free occurrence instead of +the sample-limited evidence section in the broader project audit, and associates +it with the containing C function when the existing source parser can resolve +one. +""" + +from __future__ import annotations + +import argparse +import collections +import json +import re +import tempfile +from pathlib import Path +from typing import Any + +from corpus_v2_project_audit import ( + RUNTIME_ROOTS, + SOURCE_SUFFIXES, + functions_in, + read_text, + source_files, +) + +ALLOC_RE = re.compile(r"\b(malloc|calloc|realloc|memalign|free)\s*\(") +ALLOCATORS = {"malloc", "calloc", "realloc", "memalign"} + + +def _function_index(root: Path, files: list[Path]) -> dict[str, list[Any]]: + return { + str(path.relative_to(root)): functions_in(path, root) + for path in files + if path.suffix in {".c", ".inc"} + } + + +def _scope_for(functions: list[Any], lineno: int) -> str: + for fn in functions: + if fn.start_line <= lineno <= fn.end_line: + return fn.name + return "" + + +def collect(root: Path) -> dict[str, Any]: + files = source_files(root, RUNTIME_ROOTS, SOURCE_SUFFIXES) + by_path = _function_index(root, files) + events: list[dict[str, Any]] = [] + operation_counts: collections.Counter[str] = collections.Counter() + file_counts: collections.Counter[str] = collections.Counter() + function_counts: collections.Counter[str] = collections.Counter() + + for path in files: + rel = str(path.relative_to(root)) + functions = by_path.get(rel, []) + for lineno, line in enumerate(read_text(path).splitlines(), 1): + code = line.split("//", 1)[0] + for match in ALLOC_RE.finditer(code): + operation = match.group(1) + scope = _scope_for(functions, lineno) + event = { + "path": rel, + "line": lineno, + "function": scope, + "operation": operation, + "kind": "allocate" if operation in ALLOCATORS else "free", + "source": line.strip(), + } + events.append(event) + operation_counts[operation] += 1 + file_counts[rel] += 1 + function_counts[f"{rel}:{scope}"] += 1 + + events.sort(key=lambda item: (item["path"], item["line"], item["operation"])) + return { + "epistemic_status": "CURRENT IMPLEMENTATION: static source inventory", + "runtime_roots": list(RUNTIME_ROOTS), + "total_events": len(events), + "allocation_events": sum( + count for operation, count in operation_counts.items() + if operation in ALLOCATORS + ), + "free_events": operation_counts["free"], + "operation_counts": dict(sorted(operation_counts.items())), + "file_counts": dict(sorted(file_counts.items())), + "function_counts": dict(sorted(function_counts.items())), + "events": events, + } + + +def selftest() -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "src").mkdir() + (root / "include").mkdir() + (root / "iop").mkdir() + (root / "src" / "fixture.c").write_text( + """ +#include +static void *phase(unsigned int n) +{ + void *a = malloc(n); + void *b = memalign(64, n); + a = realloc(a, n + 1); + free(b); + free(a); + return 0; +} +""".lstrip(), + encoding="utf-8", + ) + result = collect(root) + assert result["total_events"] == 5 + assert result["allocation_events"] == 3 + assert result["free_events"] == 2 + assert result["operation_counts"] == { + "free": 2, + "malloc": 1, + "memalign": 1, + "realloc": 1, + } + assert all( + event["function"] == "phase" for event in result["events"] + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(".")) + parser.add_argument("--output", type=Path, + default=Path("ALLOCATION_INVENTORY.json")) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("allocation_inventory selftest: PASS") + return 0 + + root = args.root.resolve() + result = collect(root) + args.output.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print( + "allocation inventory: " + f"{result['allocation_events']} allocate events, " + f"{result['free_events']} free events, " + f"{result['total_events']} total" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 2e3f294434922e199e5434fa1fd32412879fa019 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 21:32:15 +0200 Subject: [PATCH 106/156] CI: archive complete runtime allocation inventory --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 877b46c9..e8ab50c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,8 @@ jobs: run: python3 tools/compare_hdl_resume_hash_ab.py --selftest - name: Self-test R5900 calibration disassembly guard run: python3 tools/check_r5900_calibration_disasm.py --selftest + - name: Self-test allocation inventory + run: python3 tools/allocation_inventory.py --selftest - name: Enforce direct-fileXio runtime policy run: | python3 tools/check_filexio_fdman_policy.py --selftest @@ -60,6 +62,8 @@ jobs: sh tools/r5900_toolchain_audit.sh GCC_R5900_TARGET.txt && python3 tools/corpus_v2_project_audit.py --output CORPUS_V2_PROJECT_AUDIT.txt && + python3 tools/allocation_inventory.py + --output ALLOCATION_INVENTORY.json && python3 tools/printf_format_audit.py --output PRINTF_FORMAT_AUDIT.txt && make clean && make HDL_PROFILE=1 && @@ -205,6 +209,7 @@ jobs: OPTIMIZATION_AUDIT_PROFILE_ON.txt OPTIMIZATION_AUDIT_PROFILE_OFF.txt CORPUS_V2_PROJECT_AUDIT.txt + ALLOCATION_INVENTORY.json PRINTF_FORMAT_AUDIT.txt GCC_R5900_TARGET.txt BENCHMARK_PROVENANCE_PROFILE_ON.yml From 821056b0d378cdbeea338af1078e0b84143ec2ec Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 21:33:52 +0200 Subject: [PATCH 107/156] Perf: guard audited explicit alignment set --- tools/check_alignment_policy.py | 133 ++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 tools/check_alignment_policy.py diff --git a/tools/check_alignment_policy.py b/tools/check_alignment_policy.py new file mode 100644 index 00000000..687c43b8 --- /dev/null +++ b/tools/check_alignment_policy.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Guard the reviewed explicit 64-byte alignment set. + +The policy is intentionally conservative. Existing aligned(64) sites are not all +endorsed: docs/ALIGNMENT_CONTRACT_AUDIT.md classifies most as cleanup candidates. +This guard merely prevents new blanket 64-byte attributes from appearing without +an explicit review/update, while separately enforcing the one current libpad +buffer whose 64-byte contract is mandatory. +""" + +from __future__ import annotations + +import argparse +import collections +import re +import tempfile +from pathlib import Path + +from corpus_v2_project_audit import RUNTIME_ROOTS, SOURCE_SUFFIXES, read_text, source_files + +ALIGN64_RE = re.compile( + r"__attribute__\s*\(\(\s*aligned\s*\(\s*64\s*\)\s*\)\)" +) + +EXPECTED_BY_FILE = { + "src/gs_ui_ps2.c": 1, + "src/hdd_forensic_repair_ps2.c": 4, + "src/hdd_read.c": 1, + "src/hdd_recovery_wrap.c": 1, + "src/hdd_repair_ps2.c": 2, + "src/hdd_write.c": 3, + "src/hdl_installer_ps2.c": 3, + "src/hdl_tools/catalog.inc": 2, + "src/hdl_tools/source_ui.inc": 1, + "src/hdl_tools/source_ui_resume_hash.inc": 1, + "src/hdl_tools/transaction.inc": 1, + "src/hdl_tools/transaction_resume_hash.inc": 1, + "src/header_backup.c": 1, + "src/main.c": 1, + "src/platform.c": 1, + "src/repair_snapshot.c": 1, +} + +PAD_DECL_RE = re.compile( + r"static\s+unsigned\s+char\s+pad_buffer\s*\[\s*256\s*\]" + r"\s*__attribute__\s*\(\(\s*aligned\s*\(\s*64\s*\)\s*\)\)\s*;", + re.S, +) + + +def collect(root: Path) -> dict[str, int]: + counts: collections.Counter[str] = collections.Counter() + for path in source_files(root, RUNTIME_ROOTS, SOURCE_SUFFIXES): + rel = str(path.relative_to(root)) + text = read_text(path) + # The declaration itself can span lines. Strip comments only to avoid a + # documentation example becoming a policy hit. + text = re.sub(r"/\*.*?\*/", "", text, flags=re.S) + text = re.sub(r"//[^\n]*", "", text) + count = len(ALIGN64_RE.findall(text)) + if count: + counts[rel] = count + return dict(sorted(counts.items())) + + +def check(root: Path) -> list[str]: + errors: list[str] = [] + observed = collect(root) + expected = dict(sorted(EXPECTED_BY_FILE.items())) + if observed != expected: + all_paths = sorted(set(observed) | set(expected)) + for path in all_paths: + got = observed.get(path, 0) + want = expected.get(path, 0) + if got != want: + errors.append( + f"{path}: aligned(64) count {got}, audited count {want}; " + "review docs/ALIGNMENT_CONTRACT_AUDIT.md and update policy intentionally" + ) + + platform = root / "src" / "platform.c" + if not platform.is_file() or not PAD_DECL_RE.search(read_text(platform)): + errors.append( + "src/platform.c: pad_buffer must remain a 256-byte, 64-byte-aligned " + "libpad area under the current PS2SDK contract" + ) + return errors + + +def selftest() -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name in RUNTIME_ROOTS: + (root / name).mkdir(parents=True, exist_ok=True) + # Unit-test collection independently from the repository-specific + # EXPECTED_BY_FILE table. + (root / "src" / "a.c").write_text( + "static unsigned char x[8] __attribute__((aligned(64)));\n", + encoding="utf-8", + ) + (root / "src" / "b.c").write_text( + "/* __attribute__((aligned(64))) */\n" + "static unsigned char y[8];\n", + encoding="utf-8", + ) + assert collect(root) == {"src/a.c": 1} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(".")) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("check_alignment_policy selftest: PASS") + return 0 + + errors = check(args.root.resolve()) + if errors: + for error in errors: + print(f"alignment policy: {error}") + return 2 + print( + "alignment policy: PASS " + f"({sum(EXPECTED_BY_FILE.values())} audited aligned(64) sites)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 64c3030fac6d336976f9e5f18f6b6efcad9d7b8e Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 21:34:36 +0200 Subject: [PATCH 108/156] CI: enforce reviewed explicit alignment policy --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8ab50c6..0c6f4b5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,10 @@ jobs: run: python3 tools/check_r5900_calibration_disasm.py --selftest - name: Self-test allocation inventory run: python3 tools/allocation_inventory.py --selftest + - name: Enforce reviewed explicit alignment policy + run: | + python3 tools/check_alignment_policy.py --selftest + python3 tools/check_alignment_policy.py - name: Enforce direct-fileXio runtime policy run: | python3 tools/check_filexio_fdman_policy.py --selftest From d860bf332fbd72e8a9e2b3a8bd4d3eff716d6b5f Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:01:51 +0200 Subject: [PATCH 109/156] Phase 5: add transaction workspace materializer --- tools/materialize_transaction_workspace.py | 317 +++++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 tools/materialize_transaction_workspace.py diff --git a/tools/materialize_transaction_workspace.py b/tools/materialize_transaction_workspace.py new file mode 100644 index 00000000..cbd9565f --- /dev/null +++ b/tools/materialize_transaction_workspace.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +"""Materialize the isolated HDL transaction-workspace experiment. + +The default source tree keeps the historical per-phase memalign/free policy. +This tool rewrites a temporary copy of transaction.inc so one transaction owns +one 64 KiB / 64-byte-aligned EE workspace across source/copy/HDD-verify phases. + +The rewrite is deliberately exact and fails closed if the audited source shape +moves. It is used only by the experiment build; it does not promote the policy +to the default runtime. +""" + +from __future__ import annotations + +import argparse +import tempfile +from pathlib import Path + +MARKER = "HDL transaction workspace experiment: one owner, borrowed by phase helpers" + + +class MaterializeError(RuntimeError): + pass + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise MaterializeError(f"{label}: expected one match, found {count}") + return text.replace(old, new, 1) + + +def function_span(text: str, name: str) -> tuple[int, int]: + start_token = f"static int {name}(" + start = text.find(start_token) + if start < 0: + raise MaterializeError(f"missing function {name}") + next_start = text.find("\nstatic int ", start + len(start_token)) + if next_start < 0: + raise MaterializeError(f"cannot bound function {name}") + return start, next_start + + +def edit_function(text: str, name: str, edits: list[tuple[str, str, str]]) -> str: + start, end = function_span(text, name) + body = text[start:end] + for old, new, label in edits: + body = replace_once(body, old, new, f"{name}: {label}") + return text[:start] + body + text[end:] + + +def materialize(text: str) -> str: + if MARKER in text: + raise MaterializeError("source already contains transaction workspace experiment") + + text = edit_function( + text, + "hash_source_payload", + [ + ( + " unsigned char digest[32])\n", + " unsigned char digest[32],\n" + " unsigned char *workspace)\n", + "workspace parameter", + ), + ( + " unsigned char *buffer;\n", + " unsigned char *buffer = workspace;\n", + "borrow workspace", + ), + ( + " buffer = memalign(64, HDL_INSTALL_IO_BYTES);\n" + " if (buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + " if (buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + "remove phase allocation", + ), + ( + "done:\n free(buffer);\n return result;\n", + "done:\n return result;\n", + "remove phase free", + ), + ], + ) + + text = edit_function( + text, + "copy_payload", + [ + ( + " unsigned char source_digest[32])\n", + " unsigned char source_digest[32],\n" + " unsigned char *workspace)\n", + "workspace parameter", + ), + ( + " unsigned char *buffer;\n", + " unsigned char *buffer = workspace;\n", + "borrow workspace", + ), + ( + " buffer = memalign(64, HDL_INSTALL_IO_BYTES);\n" + " if (buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + " if (buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + "remove phase allocation", + ), + ( + "done:\n free(buffer);\n return result;\n", + "done:\n return result;\n", + "remove phase free", + ), + ], + ) + + text = edit_function( + text, + "verify_target_digest", + [ + ( + " const unsigned char expected_digest[32])\n", + " const unsigned char expected_digest[32],\n" + " unsigned char *workspace)\n", + "workspace parameter", + ), + ( + " unsigned char *target_buffer;\n", + " unsigned char *target_buffer = workspace;\n", + "borrow workspace", + ), + ( + " target_buffer = memalign(64, HDL_INSTALL_IO_BYTES);\n" + " if (target_buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + " if (target_buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + "remove phase allocation", + ), + ( + "done:\n free(target_buffer);\n return result;\n", + "done:\n return result;\n", + "remove phase free", + ), + ], + ) + + start, end = function_span(text, "execute_transaction") + body = text[start:end] + body = replace_once( + body, + " unsigned char source_payload_digest[32];\n", + " unsigned char source_payload_digest[32];\n" + f" /* {MARKER}. */\n" + " unsigned char *workspace = NULL;\n", + "execute_transaction: workspace owner", + ) + body = replace_once( + body, + " if (transaction->stage == HDL_TRANSACTION_STAGE_COPYING) {\n" + " result = copy_payload(transaction, &plan, &layout,\n" + " source.fd, target_fd, source_payload_digest);\n", + " if (transaction->stage == HDL_TRANSACTION_STAGE_COPYING ||\n" + " transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) {\n" + " workspace = memalign(64, HDL_INSTALL_IO_BYTES);\n" + " if (workspace == NULL) {\n" + " result = HDL_INSTALL_MEMORY_FAILED;\n" + " goto done;\n" + " }\n" + " }\n" + " if (transaction->stage == HDL_TRANSACTION_STAGE_COPYING) {\n" + " result = copy_payload(transaction, &plan, &layout,\n" + " source.fd, target_fd, source_payload_digest,\n" + " workspace);\n", + "execute_transaction: allocate once before phase work", + ) + body = replace_once( + body, + " result = verify_target_digest(transaction, &plan, &layout,\n" + " target_fd, source_payload_digest);\n", + " result = verify_target_digest(transaction, &plan, &layout,\n" + " target_fd, source_payload_digest,\n" + " workspace);\n", + "execute_transaction: copy-path verify workspace", + ) + body = replace_once( + body, + " result = hash_source_payload(transaction, source.fd,\n" + " source_payload_digest);\n", + " result = hash_source_payload(transaction, source.fd,\n" + " source_payload_digest, workspace);\n", + "execute_transaction: resumed source hash workspace", + ) + body = replace_once( + body, + " result = verify_target_digest(transaction, &plan, &layout,\n" + " target_fd, source_payload_digest);\n", + " result = verify_target_digest(transaction, &plan, &layout,\n" + " target_fd, source_payload_digest,\n" + " workspace);\n", + "execute_transaction: resumed verify workspace", + ) + body = replace_once( + body, + "done:\n if (target_fd >= 0)\n", + "done:\n free(workspace);\n if (target_fd >= 0)\n", + "execute_transaction: release owner workspace", + ) + text = text[:start] + body + text[end:] + + if text.count("memalign(64, HDL_INSTALL_IO_BYTES)") != 1: + raise MaterializeError("materialized transaction must contain exactly one 64 KiB memalign") + if text.count("free(workspace);") != 1: + raise MaterializeError("materialized transaction must contain exactly one workspace free") + for token in ("free(buffer);", "free(target_buffer);"): + if token in text: + raise MaterializeError(f"phase-local free survived: {token}") + return text + + +def selftest() -> None: + fixture = r'''static int hash_source_payload(const hdl_transaction_t *transaction, + int source_fd, + unsigned char digest[32]) +{ + unsigned char *buffer; + buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; +done: + free(buffer); + return result; +} + +static int copy_payload(hdl_transaction_t *transaction, + const hdl_partition_plan_t *plan, + const hdl_stream_layout_t *layout, + int source_fd, int target_fd, + unsigned char source_digest[32]) +{ + unsigned char *buffer; + buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; +done: + free(buffer); + return result; +} + +static int verify_target_digest(const hdl_transaction_t *transaction, + const hdl_partition_plan_t *plan, + const hdl_stream_layout_t *layout, + int target_fd, + const unsigned char expected_digest[32]) +{ + unsigned char *target_buffer; + target_buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (target_buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; +done: + free(target_buffer); + return result; +} + +static int execute_transaction(hdl_transaction_t *transaction) +{ + unsigned char source_payload_digest[32]; + if (transaction->stage == HDL_TRANSACTION_STAGE_COPYING) { + result = copy_payload(transaction, &plan, &layout, + source.fd, target_fd, source_payload_digest); + result = verify_target_digest(transaction, &plan, &layout, + target_fd, source_payload_digest); + } + if (transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) { + result = hash_source_payload(transaction, source.fd, + source_payload_digest); + result = verify_target_digest(transaction, &plan, &layout, + target_fd, source_payload_digest); + } +done: + if (target_fd >= 0) + fileXioClose(target_fd); + return result; +} + +static int sentinel(void) { return 0; } +''' + out = materialize(fixture) + assert MARKER in out + assert out.count("memalign(64, HDL_INSTALL_IO_BYTES)") == 1 + assert out.count("free(workspace);") == 1 + assert "source_payload_digest, workspace" in out + assert "source_payload_digest,\n workspace" in out + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("input", nargs="?", type=Path) + parser.add_argument("output", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + return 0 + if args.input is None or args.output is None: + parser.error("input and output are required unless --selftest is used") + + source = args.input.read_text(encoding="utf-8") + result = materialize(source) + args.output.write_text(result, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 343d1b2d570c5ee4a3c11ed527a53c7246dac2ef Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:02:51 +0200 Subject: [PATCH 110/156] Phase 5: add transaction workspace experiment build --- .../build_transaction_workspace_experiment.sh | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tools/build_transaction_workspace_experiment.sh diff --git a/tools/build_transaction_workspace_experiment.sh b/tools/build_transaction_workspace_experiment.sh new file mode 100644 index 00000000..1d1cb358 --- /dev/null +++ b/tools/build_transaction_workspace_experiment.sh @@ -0,0 +1,64 @@ +#!/bin/sh +set -eu + +# Build the isolated Phase-5 transaction-workspace experiment without changing +# the default runtime source. The frozen PROFILE pair remains the baseline. +# +# The experiment changes only EE ownership of the existing 64 KiB / 64-byte +# aligned transaction I/O buffer. IOP code and transport are expected to remain +# byte-identical to the corresponding frozen PROFILE variant. +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +BACKUP=$(mktemp -d) +TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" + +restore_sources() { + if [ -f "$BACKUP/transaction.inc" ]; then + cp "$BACKUP/transaction.inc" "$TRANSACTION" + fi + rm -rf "$BACKUP" +} +trap restore_sources EXIT HUP INT TERM + +cp "$TRANSACTION" "$BACKUP/transaction.inc" +python3 "$ROOT/tools/materialize_transaction_workspace.py" \ + "$TRANSACTION" "$TRANSACTION" + +build_variant() +{ + profile=$1 + label=$2 + elf="PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_${label}.ELF" + map="PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_${label}.map" + irx="HDL_STREAM_TX_WORKSPACE_PROFILE_${label}.irx" + audit="OPTIMIZATION_AUDIT_TX_WORKSPACE_PROFILE_${label}.txt" + provenance="BENCHMARK_PROVENANCE_TX_WORKSPACE_PROFILE_${label}.yml" + + make clean + make -C iop/hdl_stream clean \ + IOP_BIN="$ROOT/hdl_stream.irx" \ + HDL_PROFILE="$profile" + make HDL_PROFILE="$profile" + cp hdl_stream.irx "$irx" + python3 tools/optimization_audit.py \ + --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF \ + --output "$audit" + make HDL_PROFILE="$profile" release + cp PS2_HDD_BOOTSTRAP_MANAGER.ELF "$elf" + cp PS2_HDD_BOOTSTRAP_MANAGER.map "$map" + sha256sum "$elf" > "$elf.sha256" + wc -c "$elf" | awk '{print $1}' > "$elf.size" + HDL_PROFILE="$profile" \ + BENCHMARK_ELF="$elf" \ + HDL_STREAM_IRX="$irx" \ + sh tools/build_benchmark_provenance.sh "$provenance" + cat >> "$provenance" < Date: Fri, 4 Sep 2026 22:04:12 +0200 Subject: [PATCH 111/156] Phase 5: bind transaction workspace A/B identity --- tools/transaction_workspace_ab_preflight.py | 167 ++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 tools/transaction_workspace_ab_preflight.py diff --git a/tools/transaction_workspace_ab_preflight.py b/tools/transaction_workspace_ab_preflight.py new file mode 100644 index 00000000..027d895c --- /dev/null +++ b/tools/transaction_workspace_ab_preflight.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Validate and bind the isolated HDL transaction-workspace A/B pair.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +FROZEN = { + "OFF": { + "elf_sha256": "4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916", + "elf_bytes": 632884, + "irx_sha256": "f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751", + "irx_bytes": 8405, + }, + "ON": { + "elf_sha256": "964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7", + "elf_bytes": 638388, + "irx_sha256": "8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db", + "irx_bytes": 9861, + }, +} + +ORDER = ["BASE", "EXP", "EXP", "BASE", "EXP", "BASE", "BASE", "EXP"] + + +def info(path: Path) -> dict[str, object]: + data = path.read_bytes() + return { + "path": path.name, + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } + + +def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) -> None: + expected = FROZEN[label] + if elf["sha256"] != expected["elf_sha256"] or elf["bytes"] != expected["elf_bytes"]: + raise SystemExit(f"{label} baseline ELF is not the frozen Phase-0 binary") + if irx["sha256"] != expected["irx_sha256"] or irx["bytes"] != expected["irx_bytes"]: + raise SystemExit(f"{label} baseline IRX is not the frozen Phase-0 binary") + + +def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) -> dict: + return { + "experiment": "hdl-transaction-workspace-v1", + "profile": profile, + "workload": "successful-hdl-transaction-copy-and-verify", + "expected_source_change": { + "workspace_bytes": 65536, + "workspace_alignment": 64, + "baseline_phase_local_memalign_free_pairs": 2, + "experiment_transaction_owned_memalign_free_pairs": 1, + "pair_reduction": 1, + "transport_change": False, + "iop_binary_change": False, + }, + "baseline": baseline, + "experiment_binary": experiment, + "hdl_stream_irx": irx, + "hardware": { + "console_scp": "UNRECORDED", + "hardware_revision": "UNRECORDED", + "romver": "UNRECORDED", + "storage_adapter": "UNRECORDED", + "hdd_model": "UNRECORDED", + "usb_device": "UNRECORDED", + "active_irx": "UNRECORDED", + }, + "runs": [ + { + "index": i + 1, + "variant": variant, + "transaction_elapsed_us": None, + "copy_elapsed_us": None, + "verify_elapsed_us": None, + "correctness_hash": None, + "result": None, + } + for i, variant in enumerate(ORDER) + ], + "report": { + "transaction_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, + "copy_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, + "verify_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, + "correctness_failures": None, + }, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--project-git-sha", required=True) + parser.add_argument("--baseline-off", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF")) + parser.add_argument("--baseline-on", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF")) + parser.add_argument("--baseline-irx-off", type=Path, default=Path("HDL_STREAM_PROFILE_OFF.irx")) + parser.add_argument("--baseline-irx-on", type=Path, default=Path("HDL_STREAM_PROFILE_ON.irx")) + parser.add_argument("--experiment-off", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF")) + parser.add_argument("--experiment-on", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF")) + parser.add_argument("--experiment-irx-off", type=Path, default=Path("HDL_STREAM_TX_WORKSPACE_PROFILE_OFF.irx")) + parser.add_argument("--experiment-irx-on", type=Path, default=Path("HDL_STREAM_TX_WORKSPACE_PROFILE_ON.irx")) + parser.add_argument("--identity-output", type=Path, default=Path("TRANSACTION_WORKSPACE_AB_IDENTITY.json")) + parser.add_argument("--profile-off-template", type=Path, default=Path("TRANSACTION_WORKSPACE_AB_PROFILE_OFF_TEMPLATE.json")) + parser.add_argument("--profile-on-template", type=Path, default=Path("TRANSACTION_WORKSPACE_AB_PROFILE_ON_TEMPLATE.json")) + args = parser.parse_args() + + baseline_off = info(args.baseline_off) + baseline_on = info(args.baseline_on) + baseline_irx_off = info(args.baseline_irx_off) + baseline_irx_on = info(args.baseline_irx_on) + experiment_off = info(args.experiment_off) + experiment_on = info(args.experiment_on) + experiment_irx_off = info(args.experiment_irx_off) + experiment_irx_on = info(args.experiment_irx_on) + + validate_frozen("OFF", baseline_off, baseline_irx_off) + validate_frozen("ON", baseline_on, baseline_irx_on) + + if experiment_irx_off["sha256"] != baseline_irx_off["sha256"]: + raise SystemExit("PROFILE OFF workspace experiment changed hdl_stream.irx") + if experiment_irx_on["sha256"] != baseline_irx_on["sha256"]: + raise SystemExit("PROFILE ON workspace experiment changed hdl_stream.irx") + if experiment_off["sha256"] == baseline_off["sha256"]: + raise SystemExit("PROFILE OFF workspace experiment did not change the EE ELF") + if experiment_on["sha256"] == baseline_on["sha256"]: + raise SystemExit("PROFILE ON workspace experiment did not change the EE ELF") + + identity = { + "experiment": "hdl-transaction-workspace-v1", + "project_git_sha": args.project_git_sha, + "frozen_phase0_commit": "7875b14d837d6332f5edc37f1c12a55527d7dd87", + "ps2sdk_commit": "b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b", + "toolchain": "mips64r5900el-ps2-elf GCC 15.2.0", + "workspace": { + "bytes": 65536, + "alignment": 64, + "owner": "execute_transaction", + "borrowers": ["copy_payload", "hash_source_payload", "verify_target_digest"], + "expected_removed_general_heap_pairs_per_successful_path": 1, + }, + "PROFILE_OFF": { + "baseline_elf": baseline_off, + "experiment_elf": experiment_off, + "hdl_stream_irx": baseline_irx_off, + }, + "PROFILE_ON": { + "baseline_elf": baseline_on, + "experiment_elf": experiment_on, + "hdl_stream_irx": baseline_irx_on, + }, + } + args.identity_output.write_text(json.dumps(identity, indent=2, sort_keys=True) + "\n", encoding="utf-8") + args.profile_off_template.write_text( + json.dumps(sample_template("OFF", baseline_off, experiment_off, baseline_irx_off), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + args.profile_on_template.write_text( + json.dumps(sample_template("ON", baseline_on, experiment_on, baseline_irx_on), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 46391b184c54c68f06acdc967385cb4fadd9b158 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:05:41 +0200 Subject: [PATCH 112/156] Phase 5: build and bind transaction workspace experiment --- .github/workflows/ci.yml | 62 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c6f4b5c..9dc68007 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,8 @@ jobs: run: python3 tools/check_r5900_calibration_disasm.py --selftest - name: Self-test allocation inventory run: python3 tools/allocation_inventory.py --selftest + - name: Self-test transaction workspace materializer + run: python3 tools/materialize_transaction_workspace.py --selftest - name: Enforce reviewed explicit alignment policy run: | python3 tools/check_alignment_policy.py --selftest @@ -141,6 +143,47 @@ jobs: cat OPTIMIZATION_AUDIT_PROFILE_OFF.txt sha256sum PS2_HDD_BOOTSTRAP_MANAGER.ELF | tee PS2_HDD_BOOTSTRAP_MANAGER.ELF.sha256 sha256sum PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF | tee HDL_PROFILE_PAIR.sha256 + - name: Build isolated transaction-workspace experiment after frozen gate + run: >- + docker run --rm + -e PROJECT_GIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}" + -e PROJECT_GIT_REF="${{ github.head_ref || github.ref }}" + -e PS2DEV_BUNDLE_REF="v2.0.0" + -e PS2SDK_SOURCE_REF="v2.0.0" + -e PS2SDK_SOURCE_SHA="b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b" + -v "$PWD:/work" -w /work ps2dev/ps2dev:v2.0.0 + sh -c 'apk add --no-cache make python3 >/dev/null && + sh tools/build_transaction_workspace_experiment.sh && + mips64r5900el-ps2-elf-size + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF + > PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.sections && + mips64r5900el-ps2-elf-size + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF + > PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.sections' + - name: Record and validate transaction-workspace experiment identity + run: | + python3 tools/transaction_workspace_ab_preflight.py \ + --project-git-sha "${{ github.event.pull_request.head.sha || github.sha }}" + echo "--- transaction workspace PROFILE OFF ---" + cat PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF.sha256 + cat PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF.size + cat PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.sections + cat BENCHMARK_PROVENANCE_TX_WORKSPACE_PROFILE_OFF.yml + echo "--- transaction workspace PROFILE ON ---" + cat PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF.sha256 + cat PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF.size + cat PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.sections + cat BENCHMARK_PROVENANCE_TX_WORKSPACE_PROFILE_ON.yml + echo "--- transaction workspace identity ---" + cat TRANSACTION_WORKSPACE_AB_IDENTITY.json + echo "--- transaction workspace PROFILE OFF optimization audit ---" + cat OPTIMIZATION_AUDIT_TX_WORKSPACE_PROFILE_OFF.txt + echo "--- transaction workspace PROFILE ON optimization audit ---" + cat OPTIMIZATION_AUDIT_TX_WORKSPACE_PROFILE_ON.txt + python3 tools/phase0_profile_pair_preflight.py \ + --profile-on PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF \ + --profile-off PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF \ + --project-git-sha "${{ github.event.pull_request.head.sha || github.sha }}" - name: Build isolated resume-hash experiment after frozen gate run: >- docker run --rm @@ -223,6 +266,25 @@ jobs: bench/r5900_calibration/R5900_COUNTER_CALIBRATION.map bench/r5900_calibration/R5900_COUNTER_CALIBRATION.sections bench/r5900_calibration/R5900_COUNTER_CALIBRATION.disasm + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF.sha256 + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF.size + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.map + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.sections + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF.sha256 + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF.size + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.map + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.sections + HDL_STREAM_TX_WORKSPACE_PROFILE_OFF.irx + HDL_STREAM_TX_WORKSPACE_PROFILE_ON.irx + OPTIMIZATION_AUDIT_TX_WORKSPACE_PROFILE_OFF.txt + OPTIMIZATION_AUDIT_TX_WORKSPACE_PROFILE_ON.txt + BENCHMARK_PROVENANCE_TX_WORKSPACE_PROFILE_OFF.yml + BENCHMARK_PROVENANCE_TX_WORKSPACE_PROFILE_ON.yml + TRANSACTION_WORKSPACE_AB_IDENTITY.json + TRANSACTION_WORKSPACE_AB_PROFILE_OFF_TEMPLATE.json + TRANSACTION_WORKSPACE_AB_PROFILE_ON_TEMPLATE.json PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.sha256 PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.size From 36198e360e97885b4ce4c701dac85092255b49e2 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:07:04 +0200 Subject: [PATCH 113/156] Phase 5: fix transaction workspace self-test --- tools/materialize_transaction_workspace.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/materialize_transaction_workspace.py b/tools/materialize_transaction_workspace.py index cbd9565f..69bbdf29 100644 --- a/tools/materialize_transaction_workspace.py +++ b/tools/materialize_transaction_workspace.py @@ -13,7 +13,6 @@ from __future__ import annotations import argparse -import tempfile from pathlib import Path MARKER = "HDL transaction workspace experiment: one owner, borrowed by phase helpers" @@ -290,8 +289,9 @@ def selftest() -> None: assert MARKER in out assert out.count("memalign(64, HDL_INSTALL_IO_BYTES)") == 1 assert out.count("free(workspace);") == 1 - assert "source_payload_digest, workspace" in out - assert "source_payload_digest,\n workspace" in out + assert "source_payload_digest,\n workspace" in out + assert "source_payload_digest, workspace" in out or \ + "source_payload_digest,\n workspace" in out def main() -> int: From 5c2b4d34a7cee05f330be0f6de2d44552d46a136 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:10:03 +0200 Subject: [PATCH 114/156] Phase 5: handle final transaction function in materializer --- tools/materialize_transaction_workspace.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tools/materialize_transaction_workspace.py b/tools/materialize_transaction_workspace.py index 69bbdf29..22cb857e 100644 --- a/tools/materialize_transaction_workspace.py +++ b/tools/materialize_transaction_workspace.py @@ -35,8 +35,10 @@ def function_span(text: str, name: str) -> tuple[int, int]: if start < 0: raise MaterializeError(f"missing function {name}") next_start = text.find("\nstatic int ", start + len(start_token)) + # execute_transaction() is intentionally the final function in the real + # include. EOF is therefore a valid exact bound, not a parser failure. if next_start < 0: - raise MaterializeError(f"cannot bound function {name}") + next_start = len(text) return start, next_start @@ -218,8 +220,9 @@ def materialize(text: str) -> str: return text -def selftest() -> None: - fixture = r'''static int hash_source_payload(const hdl_transaction_t *transaction, +def fixture(final_sentinel: bool) -> str: + tail = "\nstatic int sentinel(void) { return 0; }\n" if final_sentinel else "\n" + return r'''static int hash_source_payload(const hdl_transaction_t *transaction, int source_fd, unsigned char digest[32]) { @@ -282,10 +285,10 @@ def selftest() -> None: fileXioClose(target_fd); return result; } +''' + tail + -static int sentinel(void) { return 0; } -''' - out = materialize(fixture) +def assert_materialized(out: str) -> None: assert MARKER in out assert out.count("memalign(64, HDL_INSTALL_IO_BYTES)") == 1 assert out.count("free(workspace);") == 1 @@ -294,6 +297,13 @@ def selftest() -> None: "source_payload_digest,\n workspace" in out +def selftest() -> None: + # Cover both a function followed by another static function and the real + # transaction.inc shape where execute_transaction() ends at EOF. + assert_materialized(materialize(fixture(True))) + assert_materialized(materialize(fixture(False))) + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("input", nargs="?", type=Path) From e7e01c8081dc166acfdef5c09a4dab4e317b2219 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:15:43 +0200 Subject: [PATCH 115/156] Phase 5: document transaction workspace hardware gate --- docs/HDL_TRANSACTION_WORKSPACE_BENCHMARK.md | 282 ++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 docs/HDL_TRANSACTION_WORKSPACE_BENCHMARK.md diff --git a/docs/HDL_TRANSACTION_WORKSPACE_BENCHMARK.md b/docs/HDL_TRANSACTION_WORKSPACE_BENCHMARK.md new file mode 100644 index 00000000..16e08034 --- /dev/null +++ b/docs/HDL_TRANSACTION_WORKSPACE_BENCHMARK.md @@ -0,0 +1,282 @@ +# HDL transaction workspace hardware benchmark + +This document defines the real-PS2 acceptance gate for the isolated Phase-5 +transaction-workspace experiment. + +The experiment does not change the default runtime source. CI materializes a +temporary `transaction.inc` in which one 64 KiB / 64-byte-aligned EE buffer is +owned by `execute_transaction()` and borrowed sequentially by source-hash, +copy and HDD-verification helpers. + +## Source-of-truth routing + +- `PS2_Optimization_Library_v2_MANIFEST.md` +- `PS2_PERFORMANCE_BIBLE.md` +- `PS2_Memory_Allocators_optimization_research_corpus_v2.md` +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md` +- `PS2_Whole_System_Scheduling_research_corpus_v2.md` +- `PS2_IOP_SIF_optimization_research_corpus_v2.md` +- pinned PS2SDK `b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b` + +The relevant corpus rule is lifetime ownership, not a blanket preference for a +custom allocator. The 64-byte alignment remains because the current HDL fast +path explicitly requires it for the EE destination used by SIF DMA. + +## Epistemic status + +**POTWIERDZONE** + +- the frozen baseline allocates one 64 KiB aligned helper buffer for + `copy_payload()` and then a second, non-overlapping 64 KiB aligned helper + buffer for `verify_target_digest()` on a successful fresh COPY path; +- a resumed `PAYLOAD_VERIFIED` legacy/fallback path similarly allocates one + buffer for `hash_source_payload()` and a later one for target verification; +- these helper buffers are not live concurrently; +- `hdl_fast_dma_read()` requires the EE destination to be 64-byte aligned; +- CI #724 materializes one transaction-owned aligned buffer and passes it to the + three helpers without changing the IOP source, pump, SIF DMA, cache, + journal, flush, metadata or durability paths; +- CI #724 proves the experiment IRX is byte-identical to the corresponding + frozen PROFILE IRX. + +**CURRENT IMPLEMENTATION** + +- workspace bytes: 65536; +- workspace alignment: 64; +- owner: `execute_transaction()`; +- borrowers: `copy_payload()`, `hash_source_payload()`, + `verify_target_digest()`; +- successful fresh COPY+verify changes two phase-local `memalign/free` pairs to + one transaction-owned pair; +- successful resumed stage-4 hash+verify changes two phase-local pairs to one; +- allocation failure remains `HDL_INSTALL_MEMORY_FAILED`; +- all helper error returns still converge on transaction cleanup; +- peak payload workspace size remains one 64 KiB buffer in both baseline and + experiment. + +**INFERENCJA** + +- removing one general-heap allocation/free pair per successful bulk phase pair + should reduce allocator churn and may reduce small latency/jitter at the + copy->verify boundary; +- because the allocation count is per phase rather than per 64 KiB chunk, the + wall-time effect may be below storage noise; +- the static code reduction is valuable evidence that ownership centralization + did not trade allocator churn for I-cache growth, but it is not a runtime + speedup measurement. + +**HIPOTEZA DO TESTU** + +- real-PS2 transaction latency or tail jitter improves measurably, or at minimum + does not regress while correctness remains identical; +- retaining the workspace until transaction cleanup does not create a harmful + EE heap-pressure interaction in the tested transaction path. + +## Frozen baseline and CI #724 experiment identity + +Frozen Phase-0 source point: + +```text +7875b14d837d6332f5edc37f1c12a55527d7dd87 +``` + +CI #724 experiment/materializer source point: + +```text +5c2b4d34a7cee05f330be0f6de2d44552d46a136 +``` + +PS2SDK: + +```text +b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b +``` + +Toolchain: + +```text +mips64r5900el-ps2-elf GCC 15.2.0 +ps2dev/ps2dev:v2.0.0 +``` + +### PROFILE OFF, release-like acceptance pair + +Baseline: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF +bytes 632884 +sha256 4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916 +``` + +Experiment: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF +bytes 632756 +sha256 23bbf6dfc28eb87bc7d484875a8940b9309eb5e3994d9c922388c9a0249415c6 +``` + +Both use: + +```text +hdl_stream.irx +bytes 8405 +sha256 f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 +``` + +Static delta: + +```text +stripped ELF -128 B +.text -192 B (230440 -> 230248) +EE named text -192 B (229956 -> 229764) +EE named functions 0 (609 -> 609) +EE instructions -48 (57539 -> 57491) +execute_transaction() -148 B (6156 -> 6008) +execute_transaction insn -38 (1540 -> 1502) +``` + +### PROFILE ON, attribution pair + +Baseline: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF +bytes 638388 +sha256 964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7 +``` + +Experiment: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF +bytes 638260 +sha256 09185cd6a21bbb9990d0b7f8cfe70fa80b4e9ba01a00e9648cb9fa70d9b3d693 +``` + +Both use: + +```text +hdl_stream.irx +bytes 9861 +sha256 8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db +``` + +Static delta: + +```text +stripped ELF -128 B +.text -224 B (233280 -> 233056) +EE named text -220 B (232780 -> 232560) +EE named functions 0 (618 -> 618) +EE instructions -56 (58246 -> 58190) +execute_transaction() -148 B (6156 -> 6008) +execute_transaction insn -38 (1540 -> 1502) +``` + +The PROFILE OFF/ON difference outside `execute_transaction()` is compiler/LTO +layout around profiler-enabled code. The transaction function itself has the +same static reduction in both variants. + +CI #724 artifact digest: + +```text +sha256:cddf67a31e55e50e23bd6f98ce30739c3a93fff37e3a87d8096c8f9c77416c17 +``` + +Exact binary identity is mandatory. Do not rebuild later and call the result +this A/B pair merely because the source looks equivalent. + +## Correctness gate + +Before timing, both baseline and experiment must complete the same functional +matrix: + +1. fresh install from zero progress; +2. ordinary guarded cancel during COPYING; +3. resume from COPYING; +4. resume from persisted PAYLOAD_VERIFIED; +5. mandatory HDD SHA-256 read-back succeeds; +6. metadata commit + read-back succeeds; +7. final game metadata/startup/title are identical; +8. transaction journal reaches COMPLETE/removal exactly as baseline; +9. memory-allocation failure injection, where practical in host/test scaffolding, + still maps to `HDL_INSTALL_MEMORY_FAILED` without leaking a target/source FD. + +Any correctness mismatch rejects the experiment before performance data is +considered. + +## Hardware workload + +Use the PROFILE OFF pair for acceptance and PROFILE ON only for attribution. +Keep console, HDD, adapter, USB source, ISO, video mode and launch method fixed. + +Use at least eight interleaved successful fresh transactions with an equivalent +target state restored between runs: + +```text +BASE, EXP, EXP, BASE, EXP, BASE, BASE, EXP +``` + +The CI artifact contains: + +```text +TRANSACTION_WORKSPACE_AB_IDENTITY.json +TRANSACTION_WORKSPACE_AB_PROFILE_OFF_TEMPLATE.json +TRANSACTION_WORKSPACE_AB_PROFILE_ON_TEMPLATE.json +``` + +Record: + +```yaml +console_scp: +hardware_revision: +romver: +storage_adapter: +hdd_model: +usb_device: +active_irx: +source_iso_sha256: +source_iso_bytes: +profile_mode: +baseline_elf_sha256: +experiment_elf_sha256: +hdl_stream_irx_sha256: +sample_count: +correctness_hash: +``` + +For each run record at least: + +```text +copy elapsed us +verify elapsed us +total transaction elapsed us +result +correctness hash +``` + +Report p50, p95, p99 and max. Eight runs are a smoke gate; if the delta is near +noise, collect more samples instead of manufacturing certainty from two decimal +places. + +## Acceptance rule + +Promote the ownership change only if all are true: + +1. zero correctness regressions in the functional matrix; +2. exact frozen baseline and experiment identities match this document; +3. experiment IRX remains byte-identical to baseline for each PROFILE mode; +4. peak payload workspace does not exceed the baseline one-buffer 64 KiB peak; +5. real hardware shows no meaningful p95/p99/max regression; +6. any claimed latency/jitter improvement is repeatable rather than a single-run + difference; +7. no new allocation failure/error-path leak appears; +8. after promotion, whole-system profiling is repeated because the bottleneck + may move. + +If timing is indistinguishable but correctness is equal and the smaller code / +clearer ownership are considered sufficient maintenance benefits, that may +justify a code-quality promotion. In that case document it explicitly as a +static/lifetime improvement, not as a measured performance speedup. From be281f9597e3436b8746b4bf229a3e896fc0a30a Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:16:39 +0200 Subject: [PATCH 116/156] Phase 5: record materialized allocation inventory --- tools/build_transaction_workspace_experiment.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/build_transaction_workspace_experiment.sh b/tools/build_transaction_workspace_experiment.sh index 1d1cb358..75bdc930 100644 --- a/tools/build_transaction_workspace_experiment.sh +++ b/tools/build_transaction_workspace_experiment.sh @@ -23,6 +23,13 @@ cp "$TRANSACTION" "$BACKUP/transaction.inc" python3 "$ROOT/tools/materialize_transaction_workspace.py" \ "$TRANSACTION" "$TRANSACTION" +# Record the full source-level allocation inventory while the experiment source +# is materialized. This proves the ownership rewrite actually removed the two +# phase-local memalign/free pairs rather than relying on a comment or filename. +python3 "$ROOT/tools/allocation_inventory.py" \ + --root "$ROOT" \ + --output "$ROOT/ALLOCATION_INVENTORY_TX_WORKSPACE.json" + build_variant() { profile=$1 From 371613f67f6783327e1971711e445d9cb5a37350 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:19:09 +0200 Subject: [PATCH 117/156] Phase 5: add source-admission workspace experiment --- tools/materialize_transaction_workspace_v2.py | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 tools/materialize_transaction_workspace_v2.py diff --git a/tools/materialize_transaction_workspace_v2.py b/tools/materialize_transaction_workspace_v2.py new file mode 100644 index 00000000..dcb17e85 --- /dev/null +++ b/tools/materialize_transaction_workspace_v2.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Materialize Phase-5 transaction workspace v2. + +V1 centralizes the mutually-exclusive COPY/source-hash/target-verify buffers. +V2 additionally lets the execute_transaction source fingerprint borrow the same +workspace before any destructive HDD work. The pre-confirmation UI fingerprint +keeps its original helper-owned allocation, so no 64 KiB buffer is retained +while waiting for the user. + +This remains an isolated build experiment. Default runtime sources are not +modified permanently. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from materialize_transaction_workspace import ( + MARKER as V1_MARKER, + MaterializeError, + function_span, + materialize as materialize_v1, + replace_once, +) + +V2_MARKER = "HDL transaction workspace v2: source admission borrows transaction workspace" + + +def materialize_source(text: str) -> str: + if V2_MARKER in text: + raise MaterializeError("source already contains transaction workspace v2") + + start, end = function_span(text, "source_fingerprint") + body = text[start:end] + body = replace_once( + body, + "static int source_fingerprint(hdl_file_source_t *source,\n" + " unsigned char digest[32])\n", + "static int source_fingerprint_with_workspace(\n" + " hdl_file_source_t *source, unsigned char digest[32],\n" + " unsigned char *workspace)\n", + "source fingerprint borrowed signature", + ) + body = replace_once( + body, + " unsigned char *buffer;\n", + f" /* {V2_MARKER}. */\n" + " unsigned char *buffer = workspace;\n", + "source fingerprint borrowed buffer", + ) + body = replace_once( + body, + " buffer = memalign(64, HDL_INSTALL_IO_BYTES);\n" + " if (buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + " if (buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + "remove source fingerprint allocation", + ) + body = replace_once( + body, + "done:\n free(buffer);\n return result;\n", + "done:\n return result;\n", + "remove source fingerprint free", + ) + + wrapper = ''' +static int source_fingerprint(hdl_file_source_t *source, + unsigned char digest[32]) +{ + unsigned char *buffer = memalign(64, HDL_INSTALL_IO_BYTES); + int result; + + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; + result = source_fingerprint_with_workspace(source, digest, buffer); + free(buffer); + return result; +} +''' + text = text[:start] + body + wrapper + text[end:] + return text + + +def materialize_transaction(text: str) -> str: + text = materialize_v1(text) + start, end = function_span(text, "execute_transaction") + body = text[start:end] + + # V1 allocates only after target open. V2 moves that single allocation to + # source admission so fingerprint, copy/hash and target verify share it. + v1_allocation = ''' if (transaction->stage == HDL_TRANSACTION_STAGE_COPYING || + transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) { + workspace = memalign(64, HDL_INSTALL_IO_BYTES); + if (workspace == NULL) { + result = HDL_INSTALL_MEMORY_FAILED; + goto done; + } + } +''' + body = replace_once( + body, + v1_allocation, + "", + "remove v1 late workspace allocation", + ) + + body = replace_once( + body, + " if (transaction->stage < HDL_TRANSACTION_STAGE_METADATA_COMMITTED) {\n" + " result = open_source(transaction->source_path,\n", + " if (transaction->stage < HDL_TRANSACTION_STAGE_METADATA_COMMITTED) {\n" + " workspace = memalign(64, HDL_INSTALL_IO_BYTES);\n" + " if (workspace == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n" + " result = open_source(transaction->source_path,\n", + "allocate workspace at source admission", + ) + body = replace_once( + body, + " if (result < 0)\n" + " return result;\n" + " result = source_fingerprint(&source, fingerprint);\n", + " if (result < 0) {\n" + " free(workspace);\n" + " return result;\n" + " }\n" + " result = source_fingerprint_with_workspace(\n" + " &source, fingerprint, workspace);\n", + "borrow workspace for transaction fingerprint", + ) + body = replace_once( + body, + " if (result < 0) {\n" + " fileXioClose(source.fd);\n" + " return result;\n" + " }\n" + " }\n", + " if (result < 0) {\n" + " fileXioClose(source.fd);\n" + " free(workspace);\n" + " return result;\n" + " }\n" + " }\n", + "release workspace on source-validation failure", + ) + + text = text[:start] + body + text[end:] + if text.count("memalign(64, HDL_INSTALL_IO_BYTES)") != 1: + raise MaterializeError("v2 transaction must contain exactly one 64 KiB memalign") + if "source_fingerprint_with_workspace(" not in text: + raise MaterializeError("v2 transaction did not borrow source fingerprint workspace") + return text + + +def selftest() -> None: + source = r'''static int source_fingerprint(hdl_file_source_t *source, + unsigned char digest[32]) +{ + unsigned char *buffer; + int result = 0; + buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; +done: + free(buffer); + return result; +} + +static int source_identity_matches(void) +{ + return 0; +} +''' + transformed = materialize_source(source) + assert V2_MARKER in transformed + assert transformed.count("memalign(64, HDL_INSTALL_IO_BYTES)") == 1 + assert "source_fingerprint_with_workspace" in transformed + assert "result = source_fingerprint_with_workspace(source, digest, buffer);" in transformed + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("transaction", nargs="?", type=Path) + parser.add_argument("source", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + return 0 + if args.transaction is None or args.source is None: + parser.error("transaction and source are required unless --selftest is used") + + source_text = materialize_source(args.source.read_text(encoding="utf-8")) + transaction_text = materialize_transaction( + args.transaction.read_text(encoding="utf-8") + ) + args.source.write_text(source_text, encoding="utf-8") + args.transaction.write_text(transaction_text, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 9507804bc9b1abdd953b5ec9a371c753d7c96814 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:23:00 +0200 Subject: [PATCH 118/156] Phase 5: add transaction workspace v2 build --- ...ild_transaction_workspace_v2_experiment.sh | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tools/build_transaction_workspace_v2_experiment.sh diff --git a/tools/build_transaction_workspace_v2_experiment.sh b/tools/build_transaction_workspace_v2_experiment.sh new file mode 100644 index 00000000..563b55d8 --- /dev/null +++ b/tools/build_transaction_workspace_v2_experiment.sh @@ -0,0 +1,74 @@ +#!/bin/sh +set -eu + +# Build the isolated Phase-5 transaction-workspace v2 experiment without +# changing default runtime sources. V2 extends v1 ownership backwards through +# execute_transaction() source admission so its fingerprint, COPY/source-hash +# and HDD verify phases borrow one 64 KiB / 64-byte-aligned EE workspace. +# The pre-confirmation UI fingerprint remains helper-owned and short-lived. +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +BACKUP=$(mktemp -d) +TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" +SOURCE_UI="$ROOT/src/hdl_tools/source_ui.inc" + +restore_sources() { + if [ -f "$BACKUP/transaction.inc" ]; then + cp "$BACKUP/transaction.inc" "$TRANSACTION" + fi + if [ -f "$BACKUP/source_ui.inc" ]; then + cp "$BACKUP/source_ui.inc" "$SOURCE_UI" + fi + rm -rf "$BACKUP" +} +trap restore_sources EXIT HUP INT TERM + +cp "$TRANSACTION" "$BACKUP/transaction.inc" +cp "$SOURCE_UI" "$BACKUP/source_ui.inc" +python3 "$ROOT/tools/materialize_transaction_workspace_v2.py" \ + "$TRANSACTION" "$SOURCE_UI" + +python3 "$ROOT/tools/allocation_inventory.py" \ + --root "$ROOT" \ + --output "$ROOT/ALLOCATION_INVENTORY_TX_WORKSPACE_V2.json" + +build_variant() +{ + profile=$1 + label=$2 + elf="PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_V2_PROFILE_${label}.ELF" + map="PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_V2_PROFILE_${label}.map" + irx="HDL_STREAM_TX_WORKSPACE_V2_PROFILE_${label}.irx" + audit="OPTIMIZATION_AUDIT_TX_WORKSPACE_V2_PROFILE_${label}.txt" + provenance="BENCHMARK_PROVENANCE_TX_WORKSPACE_V2_PROFILE_${label}.yml" + + make clean + make -C iop/hdl_stream clean \ + IOP_BIN="$ROOT/hdl_stream.irx" \ + HDL_PROFILE="$profile" + make HDL_PROFILE="$profile" + cp hdl_stream.irx "$irx" + python3 tools/optimization_audit.py \ + --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF \ + --output "$audit" + make HDL_PROFILE="$profile" release + cp PS2_HDD_BOOTSTRAP_MANAGER.ELF "$elf" + cp PS2_HDD_BOOTSTRAP_MANAGER.map "$map" + sha256sum "$elf" > "$elf.sha256" + wc -c "$elf" | awk '{print $1}' > "$elf.size" + HDL_PROFILE="$profile" \ + BENCHMARK_ELF="$elf" \ + HDL_STREAM_IRX="$irx" \ + sh tools/build_benchmark_provenance.sh "$provenance" + cat >> "$provenance" < Date: Fri, 4 Sep 2026 22:23:35 +0200 Subject: [PATCH 119/156] Phase 5: bind transaction workspace v2 A/B identity --- .../transaction_workspace_v2_ab_preflight.py | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 tools/transaction_workspace_v2_ab_preflight.py diff --git a/tools/transaction_workspace_v2_ab_preflight.py b/tools/transaction_workspace_v2_ab_preflight.py new file mode 100644 index 00000000..91bb784e --- /dev/null +++ b/tools/transaction_workspace_v2_ab_preflight.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Validate and bind the isolated HDL transaction-workspace v2 A/B pair.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +FROZEN = { + "OFF": { + "elf_sha256": "4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916", + "elf_bytes": 632884, + "irx_sha256": "f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751", + "irx_bytes": 8405, + }, + "ON": { + "elf_sha256": "964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7", + "elf_bytes": 638388, + "irx_sha256": "8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db", + "irx_bytes": 9861, + }, +} + +ORDER = ["BASE", "EXP", "EXP", "BASE", "EXP", "BASE", "BASE", "EXP"] + + +def info(path: Path) -> dict[str, object]: + data = path.read_bytes() + return { + "path": path.name, + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } + + +def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) -> None: + expected = FROZEN[label] + if elf["sha256"] != expected["elf_sha256"] or elf["bytes"] != expected["elf_bytes"]: + raise SystemExit(f"{label} baseline ELF is not the frozen Phase-0 binary") + if irx["sha256"] != expected["irx_sha256"] or irx["bytes"] != expected["irx_bytes"]: + raise SystemExit(f"{label} baseline IRX is not the frozen Phase-0 binary") + + +def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) -> dict: + return { + "experiment": "hdl-transaction-workspace-v2", + "profile": profile, + "workload": "successful-hdl-transaction-source-admission-copy-and-verify", + "expected_source_change": { + "workspace_bytes": 65536, + "workspace_alignment": 64, + "baseline_transaction_memalign_free_pairs": 3, + "experiment_transaction_owned_memalign_free_pairs": 1, + "pair_reduction": 2, + "preconfirmation_fingerprint_unchanged": True, + "transport_change": False, + "iop_binary_change": False, + }, + "baseline": baseline, + "experiment_binary": experiment, + "hdl_stream_irx": irx, + "hardware": { + "console_scp": "UNRECORDED", + "hardware_revision": "UNRECORDED", + "romver": "UNRECORDED", + "storage_adapter": "UNRECORDED", + "hdd_model": "UNRECORDED", + "usb_device": "UNRECORDED", + "active_irx": "UNRECORDED", + }, + "runs": [ + { + "index": i + 1, + "variant": variant, + "transaction_elapsed_us": None, + "source_admission_elapsed_us": None, + "copy_elapsed_us": None, + "verify_elapsed_us": None, + "correctness_hash": None, + "result": None, + } + for i, variant in enumerate(ORDER) + ], + "report": { + "transaction_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, + "source_admission_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, + "copy_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, + "verify_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, + "correctness_failures": None, + }, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--project-git-sha", required=True) + parser.add_argument("--baseline-off", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF")) + parser.add_argument("--baseline-on", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF")) + parser.add_argument("--baseline-irx-off", type=Path, default=Path("HDL_STREAM_PROFILE_OFF.irx")) + parser.add_argument("--baseline-irx-on", type=Path, default=Path("HDL_STREAM_PROFILE_ON.irx")) + parser.add_argument("--experiment-off", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_V2_PROFILE_OFF.ELF")) + parser.add_argument("--experiment-on", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_V2_PROFILE_ON.ELF")) + parser.add_argument("--experiment-irx-off", type=Path, default=Path("HDL_STREAM_TX_WORKSPACE_V2_PROFILE_OFF.irx")) + parser.add_argument("--experiment-irx-on", type=Path, default=Path("HDL_STREAM_TX_WORKSPACE_V2_PROFILE_ON.irx")) + parser.add_argument("--identity-output", type=Path, default=Path("TRANSACTION_WORKSPACE_V2_AB_IDENTITY.json")) + parser.add_argument("--profile-off-template", type=Path, default=Path("TRANSACTION_WORKSPACE_V2_AB_PROFILE_OFF_TEMPLATE.json")) + parser.add_argument("--profile-on-template", type=Path, default=Path("TRANSACTION_WORKSPACE_V2_AB_PROFILE_ON_TEMPLATE.json")) + args = parser.parse_args() + + baseline_off = info(args.baseline_off) + baseline_on = info(args.baseline_on) + baseline_irx_off = info(args.baseline_irx_off) + baseline_irx_on = info(args.baseline_irx_on) + experiment_off = info(args.experiment_off) + experiment_on = info(args.experiment_on) + experiment_irx_off = info(args.experiment_irx_off) + experiment_irx_on = info(args.experiment_irx_on) + + validate_frozen("OFF", baseline_off, baseline_irx_off) + validate_frozen("ON", baseline_on, baseline_irx_on) + + if experiment_irx_off["sha256"] != baseline_irx_off["sha256"]: + raise SystemExit("PROFILE OFF workspace-v2 experiment changed hdl_stream.irx") + if experiment_irx_on["sha256"] != baseline_irx_on["sha256"]: + raise SystemExit("PROFILE ON workspace-v2 experiment changed hdl_stream.irx") + if experiment_off["sha256"] == baseline_off["sha256"]: + raise SystemExit("PROFILE OFF workspace-v2 experiment did not change the EE ELF") + if experiment_on["sha256"] == baseline_on["sha256"]: + raise SystemExit("PROFILE ON workspace-v2 experiment did not change the EE ELF") + + identity = { + "experiment": "hdl-transaction-workspace-v2", + "project_git_sha": args.project_git_sha, + "frozen_phase0_commit": "7875b14d837d6332f5edc37f1c12a55527d7dd87", + "ps2sdk_commit": "b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b", + "toolchain": "mips64r5900el-ps2-elf GCC 15.2.0", + "workspace": { + "bytes": 65536, + "alignment": 64, + "owner": "execute_transaction", + "borrowers": [ + "source_fingerprint_with_workspace", + "copy_payload", + "hash_source_payload", + "verify_target_digest", + ], + "preconfirmation_source_fingerprint": "helper-owned allocation unchanged", + "expected_removed_general_heap_pairs_per_transaction": 2, + }, + "PROFILE_OFF": { + "baseline_elf": baseline_off, + "experiment_elf": experiment_off, + "hdl_stream_irx": baseline_irx_off, + }, + "PROFILE_ON": { + "baseline_elf": baseline_on, + "experiment_elf": experiment_on, + "hdl_stream_irx": baseline_irx_on, + }, + } + args.identity_output.write_text(json.dumps(identity, indent=2, sort_keys=True) + "\n", encoding="utf-8") + args.profile_off_template.write_text( + json.dumps(sample_template("OFF", baseline_off, experiment_off, baseline_irx_off), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + args.profile_on_template.write_text( + json.dumps(sample_template("ON", baseline_on, experiment_on, baseline_irx_on), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 9bd2af3350dd51cbe8330d21f1cda9fc553d53fb Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:24:33 +0200 Subject: [PATCH 120/156] Phase 5: advance transaction workspace experiment to v2 --- .../build_transaction_workspace_experiment.sh | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tools/build_transaction_workspace_experiment.sh b/tools/build_transaction_workspace_experiment.sh index 75bdc930..2df3341d 100644 --- a/tools/build_transaction_workspace_experiment.sh +++ b/tools/build_transaction_workspace_experiment.sh @@ -1,31 +1,36 @@ #!/bin/sh set -eu -# Build the isolated Phase-5 transaction-workspace experiment without changing -# the default runtime source. The frozen PROFILE pair remains the baseline. -# -# The experiment changes only EE ownership of the existing 64 KiB / 64-byte -# aligned transaction I/O buffer. IOP code and transport are expected to remain -# byte-identical to the corresponding frozen PROFILE variant. +# Build the active isolated Phase-5 transaction-workspace experiment without +# changing default runtime sources. V1 is frozen at CI #724. Active v2 extends +# ownership backwards through execute_transaction() source admission so its +# fingerprint, COPY/source-hash and HDD verify phases borrow one 64 KiB / +# 64-byte-aligned EE workspace. The pre-confirmation UI fingerprint remains +# helper-owned and short-lived. ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) BACKUP=$(mktemp -d) TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" +SOURCE_UI="$ROOT/src/hdl_tools/source_ui.inc" restore_sources() { if [ -f "$BACKUP/transaction.inc" ]; then cp "$BACKUP/transaction.inc" "$TRANSACTION" fi + if [ -f "$BACKUP/source_ui.inc" ]; then + cp "$BACKUP/source_ui.inc" "$SOURCE_UI" + fi rm -rf "$BACKUP" } trap restore_sources EXIT HUP INT TERM cp "$TRANSACTION" "$BACKUP/transaction.inc" -python3 "$ROOT/tools/materialize_transaction_workspace.py" \ - "$TRANSACTION" "$TRANSACTION" +cp "$SOURCE_UI" "$BACKUP/source_ui.inc" +python3 "$ROOT/tools/materialize_transaction_workspace_v2.py" \ + "$TRANSACTION" "$SOURCE_UI" -# Record the full source-level allocation inventory while the experiment source -# is materialized. This proves the ownership rewrite actually removed the two -# phase-local memalign/free pairs rather than relying on a comment or filename. +# Record the full source-level allocation inventory while v2 is materialized. +# The transaction should now have one 64 KiB owner instead of separate source +# admission, bulk-copy/source-hash and target-verify allocation lifetimes. python3 "$ROOT/tools/allocation_inventory.py" \ --root "$ROOT" \ --output "$ROOT/ALLOCATION_INVENTORY_TX_WORKSPACE.json" @@ -60,9 +65,12 @@ build_variant() sh tools/build_benchmark_provenance.sh "$provenance" cat >> "$provenance" < Date: Fri, 4 Sep 2026 22:25:14 +0200 Subject: [PATCH 121/156] Phase 5: advance transaction workspace A/B identity to v2 --- tools/transaction_workspace_ab_preflight.py | 39 ++++++++++++++------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/tools/transaction_workspace_ab_preflight.py b/tools/transaction_workspace_ab_preflight.py index 027d895c..dd8503c1 100644 --- a/tools/transaction_workspace_ab_preflight.py +++ b/tools/transaction_workspace_ab_preflight.py @@ -1,5 +1,10 @@ #!/usr/bin/env python3 -"""Validate and bind the isolated HDL transaction-workspace A/B pair.""" +"""Validate and bind the active HDL transaction-workspace v2 A/B pair. + +V1 is frozen at CI #724. Active v2 extends the single transaction-owned 64 KiB +workspace backwards through execute_transaction() source admission while keeping +the pre-confirmation source fingerprint helper-owned and short-lived. +""" from __future__ import annotations @@ -45,15 +50,16 @@ def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) -> dict: return { - "experiment": "hdl-transaction-workspace-v1", + "experiment": "hdl-transaction-workspace-v2", "profile": profile, - "workload": "successful-hdl-transaction-copy-and-verify", + "workload": "successful-hdl-transaction-source-admission-copy-and-verify", "expected_source_change": { "workspace_bytes": 65536, "workspace_alignment": 64, - "baseline_phase_local_memalign_free_pairs": 2, + "baseline_transaction_memalign_free_pairs": 3, "experiment_transaction_owned_memalign_free_pairs": 1, - "pair_reduction": 1, + "pair_reduction": 2, + "preconfirmation_fingerprint_unchanged": True, "transport_change": False, "iop_binary_change": False, }, @@ -74,6 +80,7 @@ def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) - "index": i + 1, "variant": variant, "transaction_elapsed_us": None, + "source_admission_elapsed_us": None, "copy_elapsed_us": None, "verify_elapsed_us": None, "correctness_hash": None, @@ -83,6 +90,7 @@ def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) - ], "report": { "transaction_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, + "source_admission_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, "copy_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, "verify_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, "correctness_failures": None, @@ -119,16 +127,17 @@ def main() -> int: validate_frozen("ON", baseline_on, baseline_irx_on) if experiment_irx_off["sha256"] != baseline_irx_off["sha256"]: - raise SystemExit("PROFILE OFF workspace experiment changed hdl_stream.irx") + raise SystemExit("PROFILE OFF workspace-v2 experiment changed hdl_stream.irx") if experiment_irx_on["sha256"] != baseline_irx_on["sha256"]: - raise SystemExit("PROFILE ON workspace experiment changed hdl_stream.irx") + raise SystemExit("PROFILE ON workspace-v2 experiment changed hdl_stream.irx") if experiment_off["sha256"] == baseline_off["sha256"]: - raise SystemExit("PROFILE OFF workspace experiment did not change the EE ELF") + raise SystemExit("PROFILE OFF workspace-v2 experiment did not change the EE ELF") if experiment_on["sha256"] == baseline_on["sha256"]: - raise SystemExit("PROFILE ON workspace experiment did not change the EE ELF") + raise SystemExit("PROFILE ON workspace-v2 experiment did not change the EE ELF") identity = { - "experiment": "hdl-transaction-workspace-v1", + "experiment": "hdl-transaction-workspace-v2", + "v1_frozen_ci": 724, "project_git_sha": args.project_git_sha, "frozen_phase0_commit": "7875b14d837d6332f5edc37f1c12a55527d7dd87", "ps2sdk_commit": "b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b", @@ -137,8 +146,14 @@ def main() -> int: "bytes": 65536, "alignment": 64, "owner": "execute_transaction", - "borrowers": ["copy_payload", "hash_source_payload", "verify_target_digest"], - "expected_removed_general_heap_pairs_per_successful_path": 1, + "borrowers": [ + "source_fingerprint_with_workspace", + "copy_payload", + "hash_source_payload", + "verify_target_digest", + ], + "preconfirmation_source_fingerprint": "helper-owned allocation unchanged", + "expected_removed_general_heap_pairs_per_transaction": 2, }, "PROFILE_OFF": { "baseline_elf": baseline_off, From 1e684b22ef5901d2d6f9d2517b7bd206be27a4bb Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:25:41 +0200 Subject: [PATCH 122/156] Phase 5: remove redundant workspace v2 build wrapper --- ...ild_transaction_workspace_v2_experiment.sh | 74 ------------------- 1 file changed, 74 deletions(-) delete mode 100644 tools/build_transaction_workspace_v2_experiment.sh diff --git a/tools/build_transaction_workspace_v2_experiment.sh b/tools/build_transaction_workspace_v2_experiment.sh deleted file mode 100644 index 563b55d8..00000000 --- a/tools/build_transaction_workspace_v2_experiment.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/sh -set -eu - -# Build the isolated Phase-5 transaction-workspace v2 experiment without -# changing default runtime sources. V2 extends v1 ownership backwards through -# execute_transaction() source admission so its fingerprint, COPY/source-hash -# and HDD verify phases borrow one 64 KiB / 64-byte-aligned EE workspace. -# The pre-confirmation UI fingerprint remains helper-owned and short-lived. -ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) -BACKUP=$(mktemp -d) -TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" -SOURCE_UI="$ROOT/src/hdl_tools/source_ui.inc" - -restore_sources() { - if [ -f "$BACKUP/transaction.inc" ]; then - cp "$BACKUP/transaction.inc" "$TRANSACTION" - fi - if [ -f "$BACKUP/source_ui.inc" ]; then - cp "$BACKUP/source_ui.inc" "$SOURCE_UI" - fi - rm -rf "$BACKUP" -} -trap restore_sources EXIT HUP INT TERM - -cp "$TRANSACTION" "$BACKUP/transaction.inc" -cp "$SOURCE_UI" "$BACKUP/source_ui.inc" -python3 "$ROOT/tools/materialize_transaction_workspace_v2.py" \ - "$TRANSACTION" "$SOURCE_UI" - -python3 "$ROOT/tools/allocation_inventory.py" \ - --root "$ROOT" \ - --output "$ROOT/ALLOCATION_INVENTORY_TX_WORKSPACE_V2.json" - -build_variant() -{ - profile=$1 - label=$2 - elf="PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_V2_PROFILE_${label}.ELF" - map="PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_V2_PROFILE_${label}.map" - irx="HDL_STREAM_TX_WORKSPACE_V2_PROFILE_${label}.irx" - audit="OPTIMIZATION_AUDIT_TX_WORKSPACE_V2_PROFILE_${label}.txt" - provenance="BENCHMARK_PROVENANCE_TX_WORKSPACE_V2_PROFILE_${label}.yml" - - make clean - make -C iop/hdl_stream clean \ - IOP_BIN="$ROOT/hdl_stream.irx" \ - HDL_PROFILE="$profile" - make HDL_PROFILE="$profile" - cp hdl_stream.irx "$irx" - python3 tools/optimization_audit.py \ - --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF \ - --output "$audit" - make HDL_PROFILE="$profile" release - cp PS2_HDD_BOOTSTRAP_MANAGER.ELF "$elf" - cp PS2_HDD_BOOTSTRAP_MANAGER.map "$map" - sha256sum "$elf" > "$elf.sha256" - wc -c "$elf" | awk '{print $1}' > "$elf.size" - HDL_PROFILE="$profile" \ - BENCHMARK_ELF="$elf" \ - HDL_STREAM_IRX="$irx" \ - sh tools/build_benchmark_provenance.sh "$provenance" - cat >> "$provenance" < Date: Fri, 4 Sep 2026 22:25:48 +0200 Subject: [PATCH 123/156] Phase 5: remove redundant workspace v2 preflight wrapper --- .../transaction_workspace_v2_ab_preflight.py | 176 ------------------ 1 file changed, 176 deletions(-) delete mode 100644 tools/transaction_workspace_v2_ab_preflight.py diff --git a/tools/transaction_workspace_v2_ab_preflight.py b/tools/transaction_workspace_v2_ab_preflight.py deleted file mode 100644 index 91bb784e..00000000 --- a/tools/transaction_workspace_v2_ab_preflight.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env python3 -"""Validate and bind the isolated HDL transaction-workspace v2 A/B pair.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -from pathlib import Path - -FROZEN = { - "OFF": { - "elf_sha256": "4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916", - "elf_bytes": 632884, - "irx_sha256": "f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751", - "irx_bytes": 8405, - }, - "ON": { - "elf_sha256": "964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7", - "elf_bytes": 638388, - "irx_sha256": "8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db", - "irx_bytes": 9861, - }, -} - -ORDER = ["BASE", "EXP", "EXP", "BASE", "EXP", "BASE", "BASE", "EXP"] - - -def info(path: Path) -> dict[str, object]: - data = path.read_bytes() - return { - "path": path.name, - "bytes": len(data), - "sha256": hashlib.sha256(data).hexdigest(), - } - - -def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) -> None: - expected = FROZEN[label] - if elf["sha256"] != expected["elf_sha256"] or elf["bytes"] != expected["elf_bytes"]: - raise SystemExit(f"{label} baseline ELF is not the frozen Phase-0 binary") - if irx["sha256"] != expected["irx_sha256"] or irx["bytes"] != expected["irx_bytes"]: - raise SystemExit(f"{label} baseline IRX is not the frozen Phase-0 binary") - - -def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) -> dict: - return { - "experiment": "hdl-transaction-workspace-v2", - "profile": profile, - "workload": "successful-hdl-transaction-source-admission-copy-and-verify", - "expected_source_change": { - "workspace_bytes": 65536, - "workspace_alignment": 64, - "baseline_transaction_memalign_free_pairs": 3, - "experiment_transaction_owned_memalign_free_pairs": 1, - "pair_reduction": 2, - "preconfirmation_fingerprint_unchanged": True, - "transport_change": False, - "iop_binary_change": False, - }, - "baseline": baseline, - "experiment_binary": experiment, - "hdl_stream_irx": irx, - "hardware": { - "console_scp": "UNRECORDED", - "hardware_revision": "UNRECORDED", - "romver": "UNRECORDED", - "storage_adapter": "UNRECORDED", - "hdd_model": "UNRECORDED", - "usb_device": "UNRECORDED", - "active_irx": "UNRECORDED", - }, - "runs": [ - { - "index": i + 1, - "variant": variant, - "transaction_elapsed_us": None, - "source_admission_elapsed_us": None, - "copy_elapsed_us": None, - "verify_elapsed_us": None, - "correctness_hash": None, - "result": None, - } - for i, variant in enumerate(ORDER) - ], - "report": { - "transaction_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, - "source_admission_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, - "copy_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, - "verify_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, - "correctness_failures": None, - }, - } - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--project-git-sha", required=True) - parser.add_argument("--baseline-off", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF")) - parser.add_argument("--baseline-on", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF")) - parser.add_argument("--baseline-irx-off", type=Path, default=Path("HDL_STREAM_PROFILE_OFF.irx")) - parser.add_argument("--baseline-irx-on", type=Path, default=Path("HDL_STREAM_PROFILE_ON.irx")) - parser.add_argument("--experiment-off", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_V2_PROFILE_OFF.ELF")) - parser.add_argument("--experiment-on", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_V2_PROFILE_ON.ELF")) - parser.add_argument("--experiment-irx-off", type=Path, default=Path("HDL_STREAM_TX_WORKSPACE_V2_PROFILE_OFF.irx")) - parser.add_argument("--experiment-irx-on", type=Path, default=Path("HDL_STREAM_TX_WORKSPACE_V2_PROFILE_ON.irx")) - parser.add_argument("--identity-output", type=Path, default=Path("TRANSACTION_WORKSPACE_V2_AB_IDENTITY.json")) - parser.add_argument("--profile-off-template", type=Path, default=Path("TRANSACTION_WORKSPACE_V2_AB_PROFILE_OFF_TEMPLATE.json")) - parser.add_argument("--profile-on-template", type=Path, default=Path("TRANSACTION_WORKSPACE_V2_AB_PROFILE_ON_TEMPLATE.json")) - args = parser.parse_args() - - baseline_off = info(args.baseline_off) - baseline_on = info(args.baseline_on) - baseline_irx_off = info(args.baseline_irx_off) - baseline_irx_on = info(args.baseline_irx_on) - experiment_off = info(args.experiment_off) - experiment_on = info(args.experiment_on) - experiment_irx_off = info(args.experiment_irx_off) - experiment_irx_on = info(args.experiment_irx_on) - - validate_frozen("OFF", baseline_off, baseline_irx_off) - validate_frozen("ON", baseline_on, baseline_irx_on) - - if experiment_irx_off["sha256"] != baseline_irx_off["sha256"]: - raise SystemExit("PROFILE OFF workspace-v2 experiment changed hdl_stream.irx") - if experiment_irx_on["sha256"] != baseline_irx_on["sha256"]: - raise SystemExit("PROFILE ON workspace-v2 experiment changed hdl_stream.irx") - if experiment_off["sha256"] == baseline_off["sha256"]: - raise SystemExit("PROFILE OFF workspace-v2 experiment did not change the EE ELF") - if experiment_on["sha256"] == baseline_on["sha256"]: - raise SystemExit("PROFILE ON workspace-v2 experiment did not change the EE ELF") - - identity = { - "experiment": "hdl-transaction-workspace-v2", - "project_git_sha": args.project_git_sha, - "frozen_phase0_commit": "7875b14d837d6332f5edc37f1c12a55527d7dd87", - "ps2sdk_commit": "b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b", - "toolchain": "mips64r5900el-ps2-elf GCC 15.2.0", - "workspace": { - "bytes": 65536, - "alignment": 64, - "owner": "execute_transaction", - "borrowers": [ - "source_fingerprint_with_workspace", - "copy_payload", - "hash_source_payload", - "verify_target_digest", - ], - "preconfirmation_source_fingerprint": "helper-owned allocation unchanged", - "expected_removed_general_heap_pairs_per_transaction": 2, - }, - "PROFILE_OFF": { - "baseline_elf": baseline_off, - "experiment_elf": experiment_off, - "hdl_stream_irx": baseline_irx_off, - }, - "PROFILE_ON": { - "baseline_elf": baseline_on, - "experiment_elf": experiment_on, - "hdl_stream_irx": baseline_irx_on, - }, - } - args.identity_output.write_text(json.dumps(identity, indent=2, sort_keys=True) + "\n", encoding="utf-8") - args.profile_off_template.write_text( - json.dumps(sample_template("OFF", baseline_off, experiment_off, baseline_irx_off), indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - args.profile_on_template.write_text( - json.dumps(sample_template("ON", baseline_on, experiment_on, baseline_irx_on), indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From 97f2ec542a0810fac3efc19bb1d082fcb93c3d20 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:28:40 +0200 Subject: [PATCH 124/156] Phase 5: keep workspace v1 as active experiment --- .../build_transaction_workspace_experiment.sh | 31 +++++++------------ 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/tools/build_transaction_workspace_experiment.sh b/tools/build_transaction_workspace_experiment.sh index 2df3341d..614df471 100644 --- a/tools/build_transaction_workspace_experiment.sh +++ b/tools/build_transaction_workspace_experiment.sh @@ -1,36 +1,28 @@ #!/bin/sh set -eu -# Build the active isolated Phase-5 transaction-workspace experiment without -# changing default runtime sources. V1 is frozen at CI #724. Active v2 extends -# ownership backwards through execute_transaction() source admission so its -# fingerprint, COPY/source-hash and HDD verify phases borrow one 64 KiB / -# 64-byte-aligned EE workspace. The pre-confirmation UI fingerprint remains -# helper-owned and short-lived. +# Build the isolated Phase-5 transaction-workspace v1 experiment without +# changing default runtime source. CI #724 is the frozen v1 point. +# +# One 64 KiB / 64-byte-aligned EE workspace is owned by execute_transaction() +# across COPY/source-hash and HDD verification. Source admission keeps its +# shorter helper-local lifetime after v2 showed static/controller growth. ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) BACKUP=$(mktemp -d) TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" -SOURCE_UI="$ROOT/src/hdl_tools/source_ui.inc" restore_sources() { if [ -f "$BACKUP/transaction.inc" ]; then cp "$BACKUP/transaction.inc" "$TRANSACTION" fi - if [ -f "$BACKUP/source_ui.inc" ]; then - cp "$BACKUP/source_ui.inc" "$SOURCE_UI" - fi rm -rf "$BACKUP" } trap restore_sources EXIT HUP INT TERM cp "$TRANSACTION" "$BACKUP/transaction.inc" -cp "$SOURCE_UI" "$BACKUP/source_ui.inc" -python3 "$ROOT/tools/materialize_transaction_workspace_v2.py" \ - "$TRANSACTION" "$SOURCE_UI" +python3 "$ROOT/tools/materialize_transaction_workspace.py" \ + "$TRANSACTION" "$TRANSACTION" -# Record the full source-level allocation inventory while v2 is materialized. -# The transaction should now have one 64 KiB owner instead of separate source -# admission, bulk-copy/source-hash and target-verify allocation lifetimes. python3 "$ROOT/tools/allocation_inventory.py" \ --root "$ROOT" \ --output "$ROOT/ALLOCATION_INVENTORY_TX_WORKSPACE.json" @@ -65,12 +57,11 @@ build_variant() sh tools/build_benchmark_provenance.sh "$provenance" cat >> "$provenance" < Date: Fri, 4 Sep 2026 22:29:23 +0200 Subject: [PATCH 125/156] Phase 5: restore workspace v1 A/B identity after v2 rejection --- tools/transaction_workspace_ab_preflight.py | 43 +++++++++------------ 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/tools/transaction_workspace_ab_preflight.py b/tools/transaction_workspace_ab_preflight.py index dd8503c1..8659b4d2 100644 --- a/tools/transaction_workspace_ab_preflight.py +++ b/tools/transaction_workspace_ab_preflight.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -"""Validate and bind the active HDL transaction-workspace v2 A/B pair. +"""Validate and bind the active HDL transaction-workspace v1 A/B pair. -V1 is frozen at CI #724. Active v2 extends the single transaction-owned 64 KiB -workspace backwards through execute_transaction() source admission while keeping -the pre-confirmation source fingerprint helper-owned and short-lived. +V1 is frozen at CI #724. V2 source-admission reuse was built at CI #733 but is +not the active experiment because it increased execute_transaction/static text +for only one additional per-transaction alloc/free removal. """ from __future__ import annotations @@ -50,16 +50,15 @@ def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) -> dict: return { - "experiment": "hdl-transaction-workspace-v2", + "experiment": "hdl-transaction-workspace-v1", "profile": profile, - "workload": "successful-hdl-transaction-source-admission-copy-and-verify", + "workload": "successful-hdl-transaction-copy-and-verify", "expected_source_change": { "workspace_bytes": 65536, "workspace_alignment": 64, - "baseline_transaction_memalign_free_pairs": 3, + "baseline_phase_local_memalign_free_pairs": 2, "experiment_transaction_owned_memalign_free_pairs": 1, - "pair_reduction": 2, - "preconfirmation_fingerprint_unchanged": True, + "pair_reduction": 1, "transport_change": False, "iop_binary_change": False, }, @@ -80,7 +79,6 @@ def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) - "index": i + 1, "variant": variant, "transaction_elapsed_us": None, - "source_admission_elapsed_us": None, "copy_elapsed_us": None, "verify_elapsed_us": None, "correctness_hash": None, @@ -90,7 +88,6 @@ def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) - ], "report": { "transaction_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, - "source_admission_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, "copy_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, "verify_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, "correctness_failures": None, @@ -127,33 +124,29 @@ def main() -> int: validate_frozen("ON", baseline_on, baseline_irx_on) if experiment_irx_off["sha256"] != baseline_irx_off["sha256"]: - raise SystemExit("PROFILE OFF workspace-v2 experiment changed hdl_stream.irx") + raise SystemExit("PROFILE OFF workspace experiment changed hdl_stream.irx") if experiment_irx_on["sha256"] != baseline_irx_on["sha256"]: - raise SystemExit("PROFILE ON workspace-v2 experiment changed hdl_stream.irx") + raise SystemExit("PROFILE ON workspace experiment changed hdl_stream.irx") if experiment_off["sha256"] == baseline_off["sha256"]: - raise SystemExit("PROFILE OFF workspace-v2 experiment did not change the EE ELF") + raise SystemExit("PROFILE OFF workspace experiment did not change the EE ELF") if experiment_on["sha256"] == baseline_on["sha256"]: - raise SystemExit("PROFILE ON workspace-v2 experiment did not change the EE ELF") + raise SystemExit("PROFILE ON workspace experiment did not change the EE ELF") identity = { - "experiment": "hdl-transaction-workspace-v2", - "v1_frozen_ci": 724, + "experiment": "hdl-transaction-workspace-v1", "project_git_sha": args.project_git_sha, "frozen_phase0_commit": "7875b14d837d6332f5edc37f1c12a55527d7dd87", + "v1_frozen_ci": 724, + "v2_rejected_ci": 733, "ps2sdk_commit": "b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b", "toolchain": "mips64r5900el-ps2-elf GCC 15.2.0", "workspace": { "bytes": 65536, "alignment": 64, "owner": "execute_transaction", - "borrowers": [ - "source_fingerprint_with_workspace", - "copy_payload", - "hash_source_payload", - "verify_target_digest", - ], - "preconfirmation_source_fingerprint": "helper-owned allocation unchanged", - "expected_removed_general_heap_pairs_per_transaction": 2, + "borrowers": ["copy_payload", "hash_source_payload", "verify_target_digest"], + "source_admission": "helper-local allocation retained", + "expected_removed_general_heap_pairs_per_successful_path": 1, }, "PROFILE_OFF": { "baseline_elf": baseline_off, From f0287d88a303ba359052aac35338e51ade0cb2f3 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:31:29 +0200 Subject: [PATCH 126/156] Phase 5: document workspace v2 rejection --- .../HDL_TRANSACTION_WORKSPACE_V2_REJECTION.md | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 docs/HDL_TRANSACTION_WORKSPACE_V2_REJECTION.md diff --git a/docs/HDL_TRANSACTION_WORKSPACE_V2_REJECTION.md b/docs/HDL_TRANSACTION_WORKSPACE_V2_REJECTION.md new file mode 100644 index 00000000..abd55aae --- /dev/null +++ b/docs/HDL_TRANSACTION_WORKSPACE_V2_REJECTION.md @@ -0,0 +1,144 @@ +# HDL transaction workspace v2 experiment rejection + +This record preserves the second Phase-5 allocator/lifetime experiment so it is +not rediscovered later as an apparently new optimization. + +## Source-of-truth routing + +- `PS2_Optimization_Library_v2_MANIFEST.md` +- `PS2_PERFORMANCE_BIBLE.md` +- `PS2_Memory_Allocators_optimization_research_corpus_v2.md` +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md` +- `docs/HDL_TRANSACTION_WORKSPACE_BENCHMARK.md` + +## Hypothesis + +Workspace v1, frozen at CI #724, changes the mutually-exclusive bulk helpers: + +```text +copy_payload / hash_source_payload -> verify_target_digest +``` + +from two separate 64 KiB `memalign(64)` lifetimes to one transaction-owned +workspace. + +V2 tested extending that ownership backwards through `execute_transaction()` +source admission so the transaction's `source_fingerprint()` also borrowed the +same 64 KiB buffer. The pre-confirmation UI fingerprint deliberately retained +its short helper-owned allocation. + +Expected transaction allocation policy: + +```text +baseline transaction + source fingerprint alloc/free + copy or source-hash alloc/free + HDD verify alloc/free + +v1 + source fingerprint alloc/free + transaction workspace alloc + copy or source-hash + HDD verify + transaction workspace free + +v2 + transaction workspace alloc + source fingerprint + partition/open path + copy or source-hash + HDD verify + transaction workspace free +``` + +## Epistemic status + +**POTWIERDZONE** + +- current pinned fileXio source does not require the source-fingerprint caller + buffer to be 64-byte aligned; +- v2 preserves the existing 64-byte transaction workspace alignment required by + the custom EE/SIF fast path later in COPY/verify; +- v2 does not change the IOP source or transport; +- CI #733 completed host tests, EE/IOP builds, frozen identity checks, + resume-hash build and artifact upload successfully; +- both v2 experiment IRX files are byte-identical to the corresponding frozen + Phase-0 IRX files. + +**CURRENT IMPLEMENTATION / CI #733** + +Source point: + +```text +98988fcdca78dd392f95aa40d5b61157ac0bea27 +``` + +Artifact digest: + +```text +sha256:8b69850212fe9928d16efb3d4d7610a3b3c2576e4917ff25cd306380ebbcdbe8 +``` + +PROFILE OFF v2: + +```text +ELF bytes 632756 +ELF sha256 b8865fe6a95a3d0e7d54fb0b72519a68f080eb234f6b6167ad55467c3faed86f +named text 229804 B +instructions 57500 +execute_transaction 6032 B / 1508 instructions +IRX sha256 f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 +``` + +PROFILE ON v2: + +```text +ELF bytes 638260 +ELF sha256 c6865cace130952befa06c2c06fa43485973f08ca14847c322578c5847f61d66 +named text 232600 B +instructions 58201 +execute_transaction 6032 B / 1508 instructions +IRX sha256 8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db +``` + +## V2 versus v1 #724 + +```text + v1 #724 v2 #733 v2 delta +PROFILE OFF ELF 632756 632756 0 B +PROFILE OFF named text 229764 229804 +40 B +PROFILE OFF instructions 57491 57500 +9 +PROFILE ON ELF 638260 638260 0 B +PROFILE ON named text 232560 232600 +40 B +PROFILE ON instructions 58190 58201 +11 +execute_transaction 6008 6032 +24 B +execute_transaction instructions 1502 1508 +6 +removed alloc/free pairs vs baseline 1 2 +1 +``` + +## Decision + +**REJECTED AS THE ACTIVE STATIC OPTIMIZATION.** + +The additional source-admission reuse removes one more general-heap alloc/free +pair per transaction, but that pair is not inside the 64 KiB chunk loop. V2 also: + +- lengthens the 64 KiB workspace lifetime across source admission and partition + creation/opening; +- increases `execute_transaction()` by 24 bytes / 6 instructions compared with + v1; +- increases whole EE named text by 40 bytes; +- gives no final stripped-ELF size reduction beyond v1. + +The corpus prioritizes shortest correct lifetime and measured cost rather than +minimum textual allocator-call count. With no real-hardware evidence that one +additional per-transaction `memalign/free` is material, v1 has the better static +tradeoff. + +V2 remains **HIPOTEZA DO TESTU** only if future allocator profiling shows the +source-admission allocation itself contributes measurable latency/jitter or heap +fragmentation. Until then CI should materialize v1 for the active workspace A/B. + +This is not evidence that v2 is slower on hardware. It is evidence that the +available static data do not justify paying its longer lifetime and larger hot +controller solely to remove one unmeasured allocation pair. From 05335f91c43de59d5b4502d60fcd289e7e7b9dc5 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:33:04 +0200 Subject: [PATCH 127/156] Phase 5: add source fingerprint heap-alignment experiment --- .../materialize_source_fingerprint_malloc.py | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tools/materialize_source_fingerprint_malloc.py diff --git a/tools/materialize_source_fingerprint_malloc.py b/tools/materialize_source_fingerprint_malloc.py new file mode 100644 index 00000000..091983b0 --- /dev/null +++ b/tools/materialize_source_fingerprint_malloc.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Materialize the isolated source-fingerprint malloc experiment. + +Pinned PS2SDK fileXioRead accepts unaligned EE caller buffers, so 64-byte heap +alignment is not a correctness/API requirement for source_fingerprint(). This +experiment replaces only that helper's memalign(64, 64 KiB) with malloc(64 KiB). +It does not touch the custom hdl0: SIF/DMA buffers, whose 64-byte contract stays +in force. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +MARKER = "HDL source fingerprint ordinary-heap experiment" + + +class MaterializeError(RuntimeError): + pass + + +def function_span(text: str, name: str) -> tuple[int, int]: + token = f"static int {name}(" + start = text.find(token) + if start < 0: + raise MaterializeError(f"missing function {name}") + next_start = text.find("\nstatic int ", start + len(token)) + return start, len(text) if next_start < 0 else next_start + + +def transform(text: str) -> str: + if MARKER in text: + raise MaterializeError("source already contains fingerprint malloc experiment") + start, end = function_span(text, "source_fingerprint") + body = text[start:end] + old = " buffer = memalign(64, HDL_INSTALL_IO_BYTES);\n" + if body.count(old) != 1: + raise MaterializeError( + f"source_fingerprint: expected one memalign site, found {body.count(old)}" + ) + body = body.replace( + old, + f" /* {MARKER}. */\n" + " buffer = malloc(HDL_INSTALL_IO_BYTES);\n", + 1, + ) + if "memalign(64, HDL_INSTALL_IO_BYTES)" in body: + raise MaterializeError("source_fingerprint memalign survived transform") + return text[:start] + body + text[end:] + + +def selftest() -> None: + fixture = r'''static int source_fingerprint(hdl_file_source_t *source, + unsigned char digest[32]) +{ + unsigned char *buffer; + buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; + free(buffer); + return 0; +} + +static int sentinel(void) { return 0; } +''' + out = transform(fixture) + assert MARKER in out + assert "buffer = malloc(HDL_INSTALL_IO_BYTES);" in out + assert "memalign(64, HDL_INSTALL_IO_BYTES)" not in out + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("files", nargs="*", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + if args.selftest: + selftest() + return 0 + if not args.files: + parser.error("at least one source file is required unless --selftest is used") + for path in args.files: + path.write_text(transform(path.read_text(encoding="utf-8")), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 84d7f3f6868fa9f63fcc1ceecfe69716a95eb847 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:33:27 +0200 Subject: [PATCH 128/156] Phase 5: test source fingerprint malloc on workspace v1 --- .../build_transaction_workspace_experiment.sh | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tools/build_transaction_workspace_experiment.sh b/tools/build_transaction_workspace_experiment.sh index 614df471..79102f63 100644 --- a/tools/build_transaction_workspace_experiment.sh +++ b/tools/build_transaction_workspace_experiment.sh @@ -1,27 +1,38 @@ #!/bin/sh set -eu -# Build the isolated Phase-5 transaction-workspace v1 experiment without -# changing default runtime source. CI #724 is the frozen v1 point. +# Build the active isolated Phase-5 incremental experiment without changing +# default runtime sources. CI #724 remains the frozen workspace-v1 point. # -# One 64 KiB / 64-byte-aligned EE workspace is owned by execute_transaction() -# across COPY/source-hash and HDD verification. Source admission keeps its -# shorter helper-local lifetime after v2 showed static/controller growth. +# Active experiment: +# 1. materialize workspace v1 for COPY/source-hash + HDD verify; +# 2. replace source_fingerprint()'s helper-local memalign(64, 64 KiB) with +# ordinary malloc(64 KiB), because pinned fileXioRead does not require a +# 64-byte caller address. +# +# The custom hdl0: SIF/DMA transaction workspace remains 64-byte aligned. ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) BACKUP=$(mktemp -d) TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" +SOURCE_UI="$ROOT/src/hdl_tools/source_ui.inc" restore_sources() { if [ -f "$BACKUP/transaction.inc" ]; then cp "$BACKUP/transaction.inc" "$TRANSACTION" fi + if [ -f "$BACKUP/source_ui.inc" ]; then + cp "$BACKUP/source_ui.inc" "$SOURCE_UI" + fi rm -rf "$BACKUP" } trap restore_sources EXIT HUP INT TERM cp "$TRANSACTION" "$BACKUP/transaction.inc" +cp "$SOURCE_UI" "$BACKUP/source_ui.inc" python3 "$ROOT/tools/materialize_transaction_workspace.py" \ "$TRANSACTION" "$TRANSACTION" +python3 "$ROOT/tools/materialize_source_fingerprint_malloc.py" \ + "$SOURCE_UI" python3 "$ROOT/tools/allocation_inventory.py" \ --root "$ROOT" \ @@ -61,7 +72,8 @@ hdl_transaction_workspace_version: "1" hdl_transaction_workspace_materializer: "tools/materialize_transaction_workspace.py" hdl_transaction_workspace_bytes: "65536" hdl_transaction_workspace_alignment: "64" -hdl_transaction_workspace_source_admission: "helper-local" +hdl_source_fingerprint_heap_experiment: "malloc" +hdl_source_fingerprint_alignment_requirement: "ordinary-fileXio-no-64B-contract" EOF } From 503ac0274b3cde18814cc7e1d170b9f656fb2615 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:34:18 +0200 Subject: [PATCH 129/156] Phase 5: bind fingerprint-malloc incremental A/B --- tools/transaction_workspace_ab_preflight.py | 69 ++++++++++++++------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/tools/transaction_workspace_ab_preflight.py b/tools/transaction_workspace_ab_preflight.py index 8659b4d2..b2fcd1e7 100644 --- a/tools/transaction_workspace_ab_preflight.py +++ b/tools/transaction_workspace_ab_preflight.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -"""Validate and bind the active HDL transaction-workspace v1 A/B pair. +"""Validate and bind the active Phase-5 workspace-v1 + fingerprint-malloc A/B. -V1 is frozen at CI #724. V2 source-admission reuse was built at CI #733 but is -not the active experiment because it increased execute_transaction/static text -for only one additional per-transaction alloc/free removal. +Frozen Phase-0 remains the correctness/identity baseline. Workspace v1 itself is +frozen at CI #724, so the new experiment can also be compared incrementally +against those exact v1 ELF hashes after CI produces the new artifacts. """ from __future__ import annotations @@ -28,6 +28,25 @@ }, } +WORKSPACE_V1 = { + "OFF": { + "elf_sha256": "23bbf6dfc28eb87bc7d484875a8940b9309eb5e3994d9c922388c9a0249415c6", + "elf_bytes": 632756, + "named_text": 229764, + "instructions": 57491, + "execute_transaction_bytes": 6008, + "execute_transaction_instructions": 1502, + }, + "ON": { + "elf_sha256": "09185cd6a21bbb9990d0b7f8cfe70fa80b4e9ba01a00e9648cb9fa70d9b3d693", + "elf_bytes": 638260, + "named_text": 232560, + "instructions": 58190, + "execute_transaction_bytes": 6008, + "execute_transaction_instructions": 1502, + }, +} + ORDER = ["BASE", "EXP", "EXP", "BASE", "EXP", "BASE", "BASE", "EXP"] @@ -50,19 +69,23 @@ def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) -> dict: return { - "experiment": "hdl-transaction-workspace-v1", + "experiment": "hdl-workspace-v1-source-fingerprint-malloc", "profile": profile, "workload": "successful-hdl-transaction-copy-and-verify", "expected_source_change": { + "transaction_workspace_version": 1, "workspace_bytes": 65536, "workspace_alignment": 64, - "baseline_phase_local_memalign_free_pairs": 2, - "experiment_transaction_owned_memalign_free_pairs": 1, - "pair_reduction": 1, + "source_fingerprint_allocator_before": "memalign(64,65536)", + "source_fingerprint_allocator_after": "malloc(65536)", + "source_fingerprint_reads": 2, + "source_fingerprint_filexio_alignment_required": False, + "custom_hdl_sif_dma_alignment_changed": False, "transport_change": False, "iop_binary_change": False, }, "baseline": baseline, + "workspace_v1_reference": WORKSPACE_V1[profile], "experiment_binary": experiment, "hdl_stream_irx": irx, "hardware": { @@ -78,6 +101,7 @@ def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) - { "index": i + 1, "variant": variant, + "source_fingerprint_elapsed_us": None, "transaction_elapsed_us": None, "copy_elapsed_us": None, "verify_elapsed_us": None, @@ -87,6 +111,7 @@ def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) - for i, variant in enumerate(ORDER) ], "report": { + "source_fingerprint_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, "transaction_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, "copy_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, "verify_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, @@ -124,29 +149,29 @@ def main() -> int: validate_frozen("ON", baseline_on, baseline_irx_on) if experiment_irx_off["sha256"] != baseline_irx_off["sha256"]: - raise SystemExit("PROFILE OFF workspace experiment changed hdl_stream.irx") + raise SystemExit("PROFILE OFF incremental experiment changed hdl_stream.irx") if experiment_irx_on["sha256"] != baseline_irx_on["sha256"]: - raise SystemExit("PROFILE ON workspace experiment changed hdl_stream.irx") + raise SystemExit("PROFILE ON incremental experiment changed hdl_stream.irx") if experiment_off["sha256"] == baseline_off["sha256"]: - raise SystemExit("PROFILE OFF workspace experiment did not change the EE ELF") + raise SystemExit("PROFILE OFF incremental experiment did not change the EE ELF") if experiment_on["sha256"] == baseline_on["sha256"]: - raise SystemExit("PROFILE ON workspace experiment did not change the EE ELF") + raise SystemExit("PROFILE ON incremental experiment did not change the EE ELF") identity = { - "experiment": "hdl-transaction-workspace-v1", + "experiment": "hdl-workspace-v1-source-fingerprint-malloc", "project_git_sha": args.project_git_sha, "frozen_phase0_commit": "7875b14d837d6332f5edc37f1c12a55527d7dd87", - "v1_frozen_ci": 724, - "v2_rejected_ci": 733, + "workspace_v1_frozen_ci": 724, + "workspace_v1_reference": WORKSPACE_V1, + "workspace_v2_rejected_ci": 733, "ps2sdk_commit": "b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b", "toolchain": "mips64r5900el-ps2-elf GCC 15.2.0", - "workspace": { - "bytes": 65536, - "alignment": 64, - "owner": "execute_transaction", - "borrowers": ["copy_payload", "hash_source_payload", "verify_target_digest"], - "source_admission": "helper-local allocation retained", - "expected_removed_general_heap_pairs_per_successful_path": 1, + "change": { + "function": "source_fingerprint", + "allocator_before": "memalign(64,65536)", + "allocator_after": "malloc(65536)", + "consumer": "ordinary pinned fileXioRead + EE SHA-256", + "custom_sif_dma_buffer_changed": False, }, "PROFILE_OFF": { "baseline_elf": baseline_off, From fe0d28222aa8bdbebdde2ee17fb156baa7203968 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:35:31 +0200 Subject: [PATCH 130/156] Phase 5: add storage scratch alignment experiment --- ...alize_storage_scratch_natural_alignment.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 tools/materialize_storage_scratch_natural_alignment.py diff --git a/tools/materialize_storage_scratch_natural_alignment.py b/tools/materialize_storage_scratch_natural_alignment.py new file mode 100644 index 00000000..631f82b4 --- /dev/null +++ b/tools/materialize_storage_scratch_natural_alignment.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Materialize the first explicit-alignment cleanup experiment. + +Only two 1024-byte static buffers are changed: +- header_backup.c::backup_scratch +- repair_snapshot.c::snapshot_verify + +Both are consumed by ordinary fileXio-backed reads and CPU parsing/memcmp. Pinned +PS2SDK does not require 64-byte caller alignment for these paths. This tool does +not touch pad, GIF-DMA, hdl0: SIF/DMA, or raw custom device buffers. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +MARKER = "natural-alignment storage scratch experiment" + +SITES = { + "header_backup.c": ( + "static unsigned char backup_scratch[APA_HEADER_SIZE]\n" + " __attribute__((aligned(64)));\n", + "static unsigned char backup_scratch[APA_HEADER_SIZE];\n" + f"/* {MARKER}: ordinary fileXio + CPU consumer. */\n", + ), + "repair_snapshot.c": ( + "static unsigned char snapshot_verify[APA_HEADER_SIZE]\n" + " __attribute__((aligned(64)));\n", + "static unsigned char snapshot_verify[APA_HEADER_SIZE];\n" + f"/* {MARKER}: ordinary fileXio + CPU consumer. */\n", + ), +} + + +class MaterializeError(RuntimeError): + pass + + +def transform(path: Path, text: str) -> str: + key = path.name + if key not in SITES: + raise MaterializeError(f"unsupported file {path}") + if MARKER in text: + raise MaterializeError(f"{path}: experiment already materialized") + old, new = SITES[key] + count = text.count(old) + if count != 1: + raise MaterializeError(f"{path}: expected one alignment site, found {count}") + out = text.replace(old, new, 1) + if old in out: + raise MaterializeError(f"{path}: old alignment survived") + return out + + +def selftest() -> None: + for name, (old, _) in SITES.items(): + path = Path(name) + out = transform(path, old + "static int sentinel;\n") + assert MARKER in out + assert "aligned(64)" not in out + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("files", nargs="*", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + if args.selftest: + selftest() + return 0 + if not args.files: + parser.error("source files are required unless --selftest is used") + for path in args.files: + path.write_text(transform(path, path.read_text(encoding="utf-8")), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 16444498434d7923b474680f78a7e6873ae37c5d Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:38:47 +0200 Subject: [PATCH 131/156] Phase 5: document source fingerprint heap alignment experiment --- ...SOURCE_FINGERPRINT_ALIGNMENT_EXPERIMENT.md | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 docs/HDL_SOURCE_FINGERPRINT_ALIGNMENT_EXPERIMENT.md diff --git a/docs/HDL_SOURCE_FINGERPRINT_ALIGNMENT_EXPERIMENT.md b/docs/HDL_SOURCE_FINGERPRINT_ALIGNMENT_EXPERIMENT.md new file mode 100644 index 00000000..ee78d048 --- /dev/null +++ b/docs/HDL_SOURCE_FINGERPRINT_ALIGNMENT_EXPERIMENT.md @@ -0,0 +1,147 @@ +# HDL source fingerprint heap-alignment experiment + +This record binds the incremental Phase-5 experiment that keeps transaction +workspace v1 from CI #724 but replaces only `source_fingerprint()`'s +`memalign(64, 65536)` with ordinary `malloc(65536)`. + +## Source-of-truth routing + +- `PS2_Optimization_Library_v2_MANIFEST.md` +- `PS2_PERFORMANCE_BIBLE.md` +- `PS2_Memory_Allocators_optimization_research_corpus_v2.md` +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md` +- `PS2_PS2SDK_optimization_research_corpus_v2.md` +- pinned PS2SDK commit `b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b` +- `docs/ALIGNMENT_CONTRACT_AUDIT.md` + +## Contract + +**CURRENT IMPLEMENTATION:** pinned PS2SDK `fileXioRead()` accepts an arbitrary EE +caller buffer, performs cache writeback for the supplied range, and handles edge +bytes through its normal read/RPC residual path. The application pointer is not +required by the fileXio API to start on a 64-byte cache-line boundary. + +`source_fingerprint()` is consumed by ordinary source `fileXioRead()` plus EE +SHA-256. It is not the custom `hdl0:` fast SIF/DMA destination. + +Therefore: + +```text +source_fingerprint heap alignment + 64 B: not a correctness/API requirement + +transaction COPY/target-verify workspace + 64 B: KEEP, custom hdl_fast_dma_read() contract +``` + +This is deliberately not a global `memalign(64) -> malloc` policy. + +## Hypothesis + +**HIPOTEZA DO TESTU:** removing one unnecessary aligned allocation may reduce a +small amount of allocator work without changing source bytes, hashing or +ownership lifetime. + +Counter-hypothesis: an arbitrary `malloc()` address may make the two 64 KiB +fingerprint reads use fileXio's unaligned edge handling, so real hardware can be +neutral or worse even though the API permits it. + +## CI #739 identity + +Source point: + +```text +503ac0274b3cde18814cc7e1d170b9f656fb2615 +``` + +Artifact digest: + +```text +sha256:b958018ee0e7cc5ab184f430794d592ce430342c87faeaaaf1993e21ad6f1ff0 +``` + +Frozen workspace-v1 reference remains CI #724. + +### PROFILE OFF + +Workspace-v1 #724: + +```text +ELF 632756 B +sha256 23bbf6dfc28eb87bc7d484875a8940b9309eb5e3994d9c922388c9a0249415c6 +named text 229764 B +instructions 57491 +execute_transaction 6008 B / 1502 instructions +``` + +Fingerprint-malloc #739: + +```text +ELF 632756 B +sha256 97e2a802952ae6f3b46c9fa0148359db8f8b69e22923f8105378f094de59c28b +named text 229756 B +instructions 57488 +execute_transaction 6008 B / 1502 instructions +``` + +Incremental static delta: + +```text +ELF 0 B +named text -8 B +instructions -3 +execute_transaction 0 B / 0 instructions +``` + +### PROFILE ON + +Workspace-v1 #724: + +```text +ELF 638260 B +sha256 09185cd6a21bbb9990d0b7f8cfe70fa80b4e9ba01a00e9648cb9fa70d9b3d693 +named text 232560 B +instructions 58190 +execute_transaction 6008 B / 1502 instructions +``` + +Fingerprint-malloc #739: + +```text +ELF 638132 B +sha256 c8da50fe5147c3a24dc2f26d4ab910660bac615bf8e26f48e0bff3a2483f578b +named text 232552 B +instructions 58189 +execute_transaction 6008 B / 1502 instructions +``` + +Incremental static delta: + +```text +ELF -128 B +named text -8 B +instructions -1 +execute_transaction 0 B / 0 instructions +``` + +Both experiment IRX files remain byte-identical to the frozen Phase-0 pair: + +```text +PROFILE OFF f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 +PROFILE ON 8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db +``` + +## Decision + +**KEEP AS A HARDWARE HYPOTHESIS, NOT A MEASURED SPEEDUP.** + +Unlike workspace v2, this change does not lengthen a 64 KiB lifetime or grow the +hot transaction controller. Static code is neutral-to-smaller. That is enough to +keep the experiment available for real-console A/B, but not enough to promote it +to the default runtime. + +Acceptance requires the same source ISO and interleaved baseline/experiment +runs, with source-fingerprint and total transaction p50/p95/p99/max plus +correctness hash. If unaligned fileXio edge handling causes a repeatable tail or +wall-time regression, retain `memalign(64)` despite the absence of an API +requirement. From 86f105d86400353a2828b7feb916c8646c21bfcd Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:39:23 +0200 Subject: [PATCH 132/156] Phase 5: test natural alignment for storage scratch --- .../build_transaction_workspace_experiment.sh | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/tools/build_transaction_workspace_experiment.sh b/tools/build_transaction_workspace_experiment.sh index 79102f63..fa7e5ccf 100644 --- a/tools/build_transaction_workspace_experiment.sh +++ b/tools/build_transaction_workspace_experiment.sh @@ -2,37 +2,45 @@ set -eu # Build the active isolated Phase-5 incremental experiment without changing -# default runtime sources. CI #724 remains the frozen workspace-v1 point. +# default runtime sources. CI #724 remains frozen workspace v1 and CI #739 is +# the frozen workspace-v1 + source-fingerprint-malloc reference. # -# Active experiment: -# 1. materialize workspace v1 for COPY/source-hash + HDD verify; -# 2. replace source_fingerprint()'s helper-local memalign(64, 64 KiB) with -# ordinary malloc(64 KiB), because pinned fileXioRead does not require a -# 64-byte caller address. -# -# The custom hdl0: SIF/DMA transaction workspace remains 64-byte aligned. +# Active experiment additionally removes explicit 64-byte alignment from two +# 1024-byte storage scratch buffers whose only consumers are ordinary fileXio +# reads plus EE CPU parse/memcmp. No direct DMA, libpad or hdl0: buffer changes. ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) BACKUP=$(mktemp -d) TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" SOURCE_UI="$ROOT/src/hdl_tools/source_ui.inc" +HEADER_BACKUP="$ROOT/src/header_backup.c" +REPAIR_SNAPSHOT="$ROOT/src/repair_snapshot.c" restore_sources() { - if [ -f "$BACKUP/transaction.inc" ]; then - cp "$BACKUP/transaction.inc" "$TRANSACTION" - fi - if [ -f "$BACKUP/source_ui.inc" ]; then - cp "$BACKUP/source_ui.inc" "$SOURCE_UI" - fi + for name in transaction.inc source_ui.inc header_backup.c repair_snapshot.c; do + if [ -f "$BACKUP/$name" ]; then + case "$name" in + transaction.inc) cp "$BACKUP/$name" "$TRANSACTION" ;; + source_ui.inc) cp "$BACKUP/$name" "$SOURCE_UI" ;; + header_backup.c) cp "$BACKUP/$name" "$HEADER_BACKUP" ;; + repair_snapshot.c) cp "$BACKUP/$name" "$REPAIR_SNAPSHOT" ;; + esac + fi + done rm -rf "$BACKUP" } trap restore_sources EXIT HUP INT TERM cp "$TRANSACTION" "$BACKUP/transaction.inc" cp "$SOURCE_UI" "$BACKUP/source_ui.inc" +cp "$HEADER_BACKUP" "$BACKUP/header_backup.c" +cp "$REPAIR_SNAPSHOT" "$BACKUP/repair_snapshot.c" + python3 "$ROOT/tools/materialize_transaction_workspace.py" \ "$TRANSACTION" "$TRANSACTION" python3 "$ROOT/tools/materialize_source_fingerprint_malloc.py" \ "$SOURCE_UI" +python3 "$ROOT/tools/materialize_storage_scratch_natural_alignment.py" \ + "$HEADER_BACKUP" "$REPAIR_SNAPSHOT" python3 "$ROOT/tools/allocation_inventory.py" \ --root "$ROOT" \ @@ -74,6 +82,8 @@ hdl_transaction_workspace_bytes: "65536" hdl_transaction_workspace_alignment: "64" hdl_source_fingerprint_heap_experiment: "malloc" hdl_source_fingerprint_alignment_requirement: "ordinary-fileXio-no-64B-contract" +storage_scratch_alignment_experiment: "natural" +storage_scratch_alignment_sites: "header_backup.backup_scratch repair_snapshot.snapshot_verify" EOF } From 902f873dac54813c11f6056b7f0667f2283e7bbe Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:40:27 +0200 Subject: [PATCH 133/156] Phase 5: bind storage scratch alignment incremental A/B --- tools/transaction_workspace_ab_preflight.py | 120 +++++++++++--------- 1 file changed, 64 insertions(+), 56 deletions(-) diff --git a/tools/transaction_workspace_ab_preflight.py b/tools/transaction_workspace_ab_preflight.py index b2fcd1e7..30bca277 100644 --- a/tools/transaction_workspace_ab_preflight.py +++ b/tools/transaction_workspace_ab_preflight.py @@ -1,9 +1,13 @@ #!/usr/bin/env python3 -"""Validate and bind the active Phase-5 workspace-v1 + fingerprint-malloc A/B. +"""Validate and bind the active Phase-5 incremental A/B pair. -Frozen Phase-0 remains the correctness/identity baseline. Workspace v1 itself is -frozen at CI #724, so the new experiment can also be compared incrementally -against those exact v1 ELF hashes after CI produces the new artifacts. +Frozen references: +- Phase-0 baseline: CI #666 +- transaction workspace v1: CI #724 +- workspace v1 + source-fingerprint malloc: CI #739 + +The active experiment adds natural alignment for two 1024-byte storage scratch +buffers that are consumed only by ordinary fileXio reads and EE CPU code. """ from __future__ import annotations @@ -47,16 +51,31 @@ }, } +FINGERPRINT_MALLOC = { + "OFF": { + "elf_sha256": "97e2a802952ae6f3b46c9fa0148359db8f8b69e22923f8105378f094de59c28b", + "elf_bytes": 632756, + "named_text": 229756, + "instructions": 57488, + "execute_transaction_bytes": 6008, + "execute_transaction_instructions": 1502, + }, + "ON": { + "elf_sha256": "c8da50fe5147c3a24dc2f26d4ab910660bac615bf8e26f48e0bff3a2483f578b", + "elf_bytes": 638132, + "named_text": 232552, + "instructions": 58189, + "execute_transaction_bytes": 6008, + "execute_transaction_instructions": 1502, + }, +} + ORDER = ["BASE", "EXP", "EXP", "BASE", "EXP", "BASE", "BASE", "EXP"] def info(path: Path) -> dict[str, object]: data = path.read_bytes() - return { - "path": path.name, - "bytes": len(data), - "sha256": hashlib.sha256(data).hexdigest(), - } + return {"path": path.name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()} def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) -> None: @@ -69,23 +88,28 @@ def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) -> dict: return { - "experiment": "hdl-workspace-v1-source-fingerprint-malloc", + "experiment": "hdl-workspace-v1-fingerprint-malloc-storage-scratch-natural", "profile": profile, - "workload": "successful-hdl-transaction-copy-and-verify", + "workload": "functional-storage-snapshot-plus-successful-hdl-transaction", "expected_source_change": { "transaction_workspace_version": 1, - "workspace_bytes": 65536, - "workspace_alignment": 64, - "source_fingerprint_allocator_before": "memalign(64,65536)", - "source_fingerprint_allocator_after": "malloc(65536)", - "source_fingerprint_reads": 2, - "source_fingerprint_filexio_alignment_required": False, + "source_fingerprint_allocator": "malloc(65536)", + "storage_scratch_sites": [ + "header_backup.c::backup_scratch", + "repair_snapshot.c::snapshot_verify", + ], + "storage_scratch_bytes_each": 1024, + "storage_scratch_old_alignment": 64, + "storage_scratch_new_alignment": "natural", + "storage_scratch_consumer": "ordinary fileXio read + EE CPU parse/memcmp", "custom_hdl_sif_dma_alignment_changed": False, - "transport_change": False, + "pad_alignment_changed": False, + "gif_dma_alignment_changed": False, "iop_binary_change": False, }, "baseline": baseline, "workspace_v1_reference": WORKSPACE_V1[profile], + "fingerprint_malloc_reference": FINGERPRINT_MALLOC[profile], "experiment_binary": experiment, "hdl_stream_irx": irx, "hardware": { @@ -101,20 +125,17 @@ def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) - { "index": i + 1, "variant": variant, + "snapshot_save_result": None, + "snapshot_readback_match": None, + "header_backup_result": None, "source_fingerprint_elapsed_us": None, "transaction_elapsed_us": None, - "copy_elapsed_us": None, - "verify_elapsed_us": None, "correctness_hash": None, - "result": None, } for i, variant in enumerate(ORDER) ], "report": { - "source_fingerprint_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, "transaction_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, - "copy_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, - "verify_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, "correctness_failures": None, }, } @@ -147,52 +168,39 @@ def main() -> int: validate_frozen("OFF", baseline_off, baseline_irx_off) validate_frozen("ON", baseline_on, baseline_irx_on) - if experiment_irx_off["sha256"] != baseline_irx_off["sha256"]: - raise SystemExit("PROFILE OFF incremental experiment changed hdl_stream.irx") + raise SystemExit("PROFILE OFF alignment experiment changed hdl_stream.irx") if experiment_irx_on["sha256"] != baseline_irx_on["sha256"]: - raise SystemExit("PROFILE ON incremental experiment changed hdl_stream.irx") - if experiment_off["sha256"] == baseline_off["sha256"]: - raise SystemExit("PROFILE OFF incremental experiment did not change the EE ELF") - if experiment_on["sha256"] == baseline_on["sha256"]: - raise SystemExit("PROFILE ON incremental experiment did not change the EE ELF") + raise SystemExit("PROFILE ON alignment experiment changed hdl_stream.irx") + if experiment_off["sha256"] == FINGERPRINT_MALLOC["OFF"]["elf_sha256"]: + raise SystemExit("PROFILE OFF storage-scratch experiment did not change the EE ELF") + if experiment_on["sha256"] == FINGERPRINT_MALLOC["ON"]["elf_sha256"]: + raise SystemExit("PROFILE ON storage-scratch experiment did not change the EE ELF") identity = { - "experiment": "hdl-workspace-v1-source-fingerprint-malloc", + "experiment": "hdl-workspace-v1-fingerprint-malloc-storage-scratch-natural", "project_git_sha": args.project_git_sha, "frozen_phase0_commit": "7875b14d837d6332f5edc37f1c12a55527d7dd87", "workspace_v1_frozen_ci": 724, - "workspace_v1_reference": WORKSPACE_V1, + "fingerprint_malloc_frozen_ci": 739, "workspace_v2_rejected_ci": 733, + "workspace_v1_reference": WORKSPACE_V1, + "fingerprint_malloc_reference": FINGERPRINT_MALLOC, "ps2sdk_commit": "b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b", "toolchain": "mips64r5900el-ps2-elf GCC 15.2.0", "change": { - "function": "source_fingerprint", - "allocator_before": "memalign(64,65536)", - "allocator_after": "malloc(65536)", - "consumer": "ordinary pinned fileXioRead + EE SHA-256", - "custom_sif_dma_buffer_changed": False, - }, - "PROFILE_OFF": { - "baseline_elf": baseline_off, - "experiment_elf": experiment_off, - "hdl_stream_irx": baseline_irx_off, - }, - "PROFILE_ON": { - "baseline_elf": baseline_on, - "experiment_elf": experiment_on, - "hdl_stream_irx": baseline_irx_on, + "sites": ["header_backup.c::backup_scratch", "repair_snapshot.c::snapshot_verify"], + "old_alignment": 64, + "new_alignment": "natural", + "consumer": "ordinary pinned fileXio read + EE CPU parse/memcmp", + "direct_dma_consumer": False, }, + "PROFILE_OFF": {"baseline_elf": baseline_off, "experiment_elf": experiment_off, "hdl_stream_irx": baseline_irx_off}, + "PROFILE_ON": {"baseline_elf": baseline_on, "experiment_elf": experiment_on, "hdl_stream_irx": baseline_irx_on}, } args.identity_output.write_text(json.dumps(identity, indent=2, sort_keys=True) + "\n", encoding="utf-8") - args.profile_off_template.write_text( - json.dumps(sample_template("OFF", baseline_off, experiment_off, baseline_irx_off), indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - args.profile_on_template.write_text( - json.dumps(sample_template("ON", baseline_on, experiment_on, baseline_irx_on), indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) + args.profile_off_template.write_text(json.dumps(sample_template("OFF", baseline_off, experiment_off, baseline_irx_off), indent=2, sort_keys=True) + "\n", encoding="utf-8") + args.profile_on_template.write_text(json.dumps(sample_template("ON", baseline_on, experiment_on, baseline_irx_on), indent=2, sort_keys=True) + "\n", encoding="utf-8") return 0 From cd2f5605c0d69ef9c33d3b5277b4e439220410d1 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:43:18 +0200 Subject: [PATCH 134/156] Phase 5: keep fingerprint malloc after no-op scratch alignment --- .../build_transaction_workspace_experiment.sh | 40 ++++++++----------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/tools/build_transaction_workspace_experiment.sh b/tools/build_transaction_workspace_experiment.sh index fa7e5ccf..3c474659 100644 --- a/tools/build_transaction_workspace_experiment.sh +++ b/tools/build_transaction_workspace_experiment.sh @@ -2,45 +2,39 @@ set -eu # Build the active isolated Phase-5 incremental experiment without changing -# default runtime sources. CI #724 remains frozen workspace v1 and CI #739 is -# the frozen workspace-v1 + source-fingerprint-malloc reference. +# default runtime sources. CI #724 remains the frozen workspace-v1 point. # -# Active experiment additionally removes explicit 64-byte alignment from two -# 1024-byte storage scratch buffers whose only consumers are ordinary fileXio -# reads plus EE CPU parse/memcmp. No direct DMA, libpad or hdl0: buffer changes. +# Active experiment: +# 1. materialize workspace v1 for COPY/source-hash + HDD verify; +# 2. replace source_fingerprint()'s helper-local memalign(64, 64 KiB) with +# ordinary malloc(64 KiB), because pinned fileXioRead does not require a +# 64-byte caller address. +# +# CI #743 tested natural alignment for two 1 KiB storage scratch buffers and +# found zero section/BSS/text/instruction benefit, so those alignments remain. +# The custom hdl0: SIF/DMA transaction workspace remains 64-byte aligned. ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) BACKUP=$(mktemp -d) TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" SOURCE_UI="$ROOT/src/hdl_tools/source_ui.inc" -HEADER_BACKUP="$ROOT/src/header_backup.c" -REPAIR_SNAPSHOT="$ROOT/src/repair_snapshot.c" restore_sources() { - for name in transaction.inc source_ui.inc header_backup.c repair_snapshot.c; do - if [ -f "$BACKUP/$name" ]; then - case "$name" in - transaction.inc) cp "$BACKUP/$name" "$TRANSACTION" ;; - source_ui.inc) cp "$BACKUP/$name" "$SOURCE_UI" ;; - header_backup.c) cp "$BACKUP/$name" "$HEADER_BACKUP" ;; - repair_snapshot.c) cp "$BACKUP/$name" "$REPAIR_SNAPSHOT" ;; - esac - fi - done + if [ -f "$BACKUP/transaction.inc" ]; then + cp "$BACKUP/transaction.inc" "$TRANSACTION" + fi + if [ -f "$BACKUP/source_ui.inc" ]; then + cp "$BACKUP/source_ui.inc" "$SOURCE_UI" + fi rm -rf "$BACKUP" } trap restore_sources EXIT HUP INT TERM cp "$TRANSACTION" "$BACKUP/transaction.inc" cp "$SOURCE_UI" "$BACKUP/source_ui.inc" -cp "$HEADER_BACKUP" "$BACKUP/header_backup.c" -cp "$REPAIR_SNAPSHOT" "$BACKUP/repair_snapshot.c" - python3 "$ROOT/tools/materialize_transaction_workspace.py" \ "$TRANSACTION" "$TRANSACTION" python3 "$ROOT/tools/materialize_source_fingerprint_malloc.py" \ "$SOURCE_UI" -python3 "$ROOT/tools/materialize_storage_scratch_natural_alignment.py" \ - "$HEADER_BACKUP" "$REPAIR_SNAPSHOT" python3 "$ROOT/tools/allocation_inventory.py" \ --root "$ROOT" \ @@ -82,8 +76,6 @@ hdl_transaction_workspace_bytes: "65536" hdl_transaction_workspace_alignment: "64" hdl_source_fingerprint_heap_experiment: "malloc" hdl_source_fingerprint_alignment_requirement: "ordinary-fileXio-no-64B-contract" -storage_scratch_alignment_experiment: "natural" -storage_scratch_alignment_sites: "header_backup.backup_scratch repair_snapshot.snapshot_verify" EOF } From de7ada2f3d9810ff40e05bd35a31abc85b8ac0c2 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:43:47 +0200 Subject: [PATCH 135/156] Phase 5: restore fingerprint malloc A/B after scratch no-op --- tools/transaction_workspace_ab_preflight.py | 124 ++++++++++---------- 1 file changed, 59 insertions(+), 65 deletions(-) diff --git a/tools/transaction_workspace_ab_preflight.py b/tools/transaction_workspace_ab_preflight.py index 30bca277..d8fa9755 100644 --- a/tools/transaction_workspace_ab_preflight.py +++ b/tools/transaction_workspace_ab_preflight.py @@ -1,13 +1,10 @@ #!/usr/bin/env python3 -"""Validate and bind the active Phase-5 incremental A/B pair. +"""Validate and bind the active Phase-5 workspace-v1 + fingerprint-malloc A/B. -Frozen references: -- Phase-0 baseline: CI #666 -- transaction workspace v1: CI #724 -- workspace v1 + source-fingerprint malloc: CI #739 - -The active experiment adds natural alignment for two 1024-byte storage scratch -buffers that are consumed only by ordinary fileXio reads and EE CPU code. +Frozen Phase-0 remains the correctness/identity baseline. Workspace v1 itself is +frozen at CI #724. CI #743 tested natural alignment for two 1 KiB storage +scratch buffers and found no section/BSS/text/instruction benefit, so that +layout-only experiment is not part of the active pair. """ from __future__ import annotations @@ -51,34 +48,19 @@ }, } -FINGERPRINT_MALLOC = { - "OFF": { - "elf_sha256": "97e2a802952ae6f3b46c9fa0148359db8f8b69e22923f8105378f094de59c28b", - "elf_bytes": 632756, - "named_text": 229756, - "instructions": 57488, - "execute_transaction_bytes": 6008, - "execute_transaction_instructions": 1502, - }, - "ON": { - "elf_sha256": "c8da50fe5147c3a24dc2f26d4ab910660bac615bf8e26f48e0bff3a2483f578b", - "elf_bytes": 638132, - "named_text": 232552, - "instructions": 58189, - "execute_transaction_bytes": 6008, - "execute_transaction_instructions": 1502, - }, -} - ORDER = ["BASE", "EXP", "EXP", "BASE", "EXP", "BASE", "BASE", "EXP"] def info(path: Path) -> dict[str, object]: data = path.read_bytes() - return {"path": path.name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()} + return { + "path": path.name, + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } -def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) -> None: +def validate_frozen(label: str, elf: dict[str,object], irx: dict[str,object]) -> None: expected = FROZEN[label] if elf["sha256"] != expected["elf_sha256"] or elf["bytes"] != expected["elf_bytes"]: raise SystemExit(f"{label} baseline ELF is not the frozen Phase-0 binary") @@ -88,28 +70,23 @@ def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) -> dict: return { - "experiment": "hdl-workspace-v1-fingerprint-malloc-storage-scratch-natural", + "experiment": "hdl-workspace-v1-source-fingerprint-malloc", "profile": profile, - "workload": "functional-storage-snapshot-plus-successful-hdl-transaction", + "workload": "successful-hdl-transaction-copy-and-verify", "expected_source_change": { "transaction_workspace_version": 1, - "source_fingerprint_allocator": "malloc(65536)", - "storage_scratch_sites": [ - "header_backup.c::backup_scratch", - "repair_snapshot.c::snapshot_verify", - ], - "storage_scratch_bytes_each": 1024, - "storage_scratch_old_alignment": 64, - "storage_scratch_new_alignment": "natural", - "storage_scratch_consumer": "ordinary fileXio read + EE CPU parse/memcmp", + "workspace_bytes": 65536, + "workspace_alignment": 64, + "source_fingerprint_allocator_before": "memalign(64,65536)", + "source_fingerprint_allocator_after": "malloc(65536)", + "source_fingerprint_reads": 2, + "source_fingerprint_filexio_alignment_required": False, "custom_hdl_sif_dma_alignment_changed": False, - "pad_alignment_changed": False, - "gif_dma_alignment_changed": False, + "transport_change": False, "iop_binary_change": False, }, "baseline": baseline, "workspace_v1_reference": WORKSPACE_V1[profile], - "fingerprint_malloc_reference": FINGERPRINT_MALLOC[profile], "experiment_binary": experiment, "hdl_stream_irx": irx, "hardware": { @@ -125,17 +102,20 @@ def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) - { "index": i + 1, "variant": variant, - "snapshot_save_result": None, - "snapshot_readback_match": None, - "header_backup_result": None, "source_fingerprint_elapsed_us": None, "transaction_elapsed_us": None, + "copy_elapsed_us": None, + "verify_elapsed_us": None, "correctness_hash": None, + "result": None, } for i, variant in enumerate(ORDER) ], "report": { + "source_fingerprint_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, "transaction_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, + "copy_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, + "verify_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, "correctness_failures": None, }, } @@ -168,39 +148,53 @@ def main() -> int: validate_frozen("OFF", baseline_off, baseline_irx_off) validate_frozen("ON", baseline_on, baseline_irx_on) + if experiment_irx_off["sha256"] != baseline_irx_off["sha256"]: - raise SystemExit("PROFILE OFF alignment experiment changed hdl_stream.irx") + raise SystemExit("PROFILE OFF incremental experiment changed hdl_stream.irx") if experiment_irx_on["sha256"] != baseline_irx_on["sha256"]: - raise SystemExit("PROFILE ON alignment experiment changed hdl_stream.irx") - if experiment_off["sha256"] == FINGERPRINT_MALLOC["OFF"]["elf_sha256"]: - raise SystemExit("PROFILE OFF storage-scratch experiment did not change the EE ELF") - if experiment_on["sha256"] == FINGERPRINT_MALLOC["ON"]["elf_sha256"]: - raise SystemExit("PROFILE ON storage-scratch experiment did not change the EE ELF") + raise SystemExit("PROFILE ON incremental experiment changed hdl_stream.irx") + if experiment_off["sha256"] == baseline_off["sha256"]: + raise SystemExit("PROFILE OFF incremental experiment did not change the EE ELF") + if experiment_on["sha256"] == baseline_on["sha256"]: + raise SystemExit("PROFILE ON incremental experiment did not change the EE ELF") identity = { - "experiment": "hdl-workspace-v1-fingerprint-malloc-storage-scratch-natural", + "experiment": "hdl-workspace-v1-source-fingerprint-malloc", "project_git_sha": args.project_git_sha, "frozen_phase0_commit": "7875b14d837d6332f5edc37f1c12a55527d7dd87", "workspace_v1_frozen_ci": 724, - "fingerprint_malloc_frozen_ci": 739, - "workspace_v2_rejected_ci": 733, "workspace_v1_reference": WORKSPACE_V1, - "fingerprint_malloc_reference": FINGERPRINT_MALLOC, + "workspace_v2_rejected_ci": 733, + "storage_scratch_natural_rejected_ci": 743, "ps2sdk_commit": "b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b", "toolchain": "mips64r5900el-ps2-elf GCC 15.2.0", "change": { - "sites": ["header_backup.c::backup_scratch", "repair_snapshot.c::snapshot_verify"], - "old_alignment": 64, - "new_alignment": "natural", - "consumer": "ordinary pinned fileXio read + EE CPU parse/memcmp", - "direct_dma_consumer": False, + "function": "source_fingerprint", + "allocator_before": "memalign(64,65536)", + "allocator_after": "malloc(65536)", + "consumer": "ordinary pinned fileXioRead + EE SHA-256", + "custom_sif_dma_buffer_changed": False, + }, + "PROFILE_OFF": { + "baseline_elf": baseline_off, + "experiment_elf": experiment_off, + "hdl_stream_irx": baseline_irx_off, + }, + "PROFILE_ON": { + "baseline_elf": baseline_on, + "experiment_elf": experiment_on, + "hdl_stream_irx": baseline_irx_on, }, - "PROFILE_OFF": {"baseline_elf": baseline_off, "experiment_elf": experiment_off, "hdl_stream_irx": baseline_irx_off}, - "PROFILE_ON": {"baseline_elf": baseline_on, "experiment_elf": experiment_on, "hdl_stream_irx": baseline_irx_on}, } args.identity_output.write_text(json.dumps(identity, indent=2, sort_keys=True) + "\n", encoding="utf-8") - args.profile_off_template.write_text(json.dumps(sample_template("OFF", baseline_off, experiment_off, baseline_irx_off), indent=2, sort_keys=True) + "\n", encoding="utf-8") - args.profile_on_template.write_text(json.dumps(sample_template("ON", baseline_on, experiment_on, baseline_irx_on), indent=2, sort_keys=True) + "\n", encoding="utf-8") + args.profile_off_template.write_text( + json.dumps(sample_template("OFF", baseline_off, experiment_off, baseline_irx_off), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + args.profile_on_template.write_text( + json.dumps(sample_template("ON", baseline_on, experiment_on, baseline_irx_on), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) return 0 From 5554e6aed983f04f54eb278b68df4ddb0841ca91 Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:44:16 +0200 Subject: [PATCH 136/156] Phase 5: document storage scratch alignment no-op --- docs/STORAGE_SCRATCH_ALIGNMENT_REJECTION.md | 77 +++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 docs/STORAGE_SCRATCH_ALIGNMENT_REJECTION.md diff --git a/docs/STORAGE_SCRATCH_ALIGNMENT_REJECTION.md b/docs/STORAGE_SCRATCH_ALIGNMENT_REJECTION.md new file mode 100644 index 00000000..53967fdc --- /dev/null +++ b/docs/STORAGE_SCRATCH_ALIGNMENT_REJECTION.md @@ -0,0 +1,77 @@ +# Storage scratch 64-byte alignment experiment rejection + +CI #743 tested removal of explicit 64-byte alignment from the two highest- +confidence file-only scratch buffers identified by `ALIGNMENT_CONTRACT_AUDIT.md`: + +```text +src/header_backup.c::backup_scratch[1024] +src/repair_snapshot.c::snapshot_verify[1024] +``` + +## Contract + +**CURRENT IMPLEMENTATION:** pinned PS2SDK ordinary fileXio read paths do not +require these application buffers to start on a 64-byte boundary. Neither object +is a direct GIF/DMAC, libpad or custom hdl0: SIF/DMA buffer. + +Removing the attributes was therefore correctness-compatible at the documented +API-contract level and worth testing as a layout experiment. + +## CI #743 + +Source point: + +```text +902f873dac54813c11f6056b7f0667f2283e7bbe +``` + +Artifact digest: + +```text +sha256:e6010f1bba48dc2e96ade33652ce2e15cefa9f6b54f246d3da89404484a84e14 +``` + +Reference was workspace-v1 + source-fingerprint-malloc from CI #739. + +### PROFILE OFF + +```text + #739 #743 delta +ELF bytes 632756 632756 0 +size text 286437 286437 0 +size data 345332 345332 0 +size bss 3012344 3012344 0 +named text 229756 229756 0 +instructions 57488 57488 0 +execute_transaction 6008 6008 0 +``` + +### PROFILE ON + +```text + #739 #743 delta +ELF bytes 638132 638132 0 +size text 290381 290381 0 +size data 346804 346804 0 +size bss 3012984 3012984 0 +named text 232552 232552 0 +instructions 58189 58189 0 +execute_transaction 6008 6008 0 +``` + +The ELF SHA changed because object/symbol placement changed, but every measured +section/code metric above remained identical. The hdl_stream IRX pair also +remained byte-identical to the frozen Phase-0 pair. + +## Decision + +**REJECTED AS AN OPTIMIZATION / RETAIN CURRENT ALIGNMENT.** + +The current source alignment is not required by fileXio, but removing it produced +no reduction in BSS, text, final ELF size or instruction count. Keeping the +existing layout avoids an otherwise pointless address-placement change in a +recovery-oriented code path. + +This result is intentionally recorded so the same source-cleanup-only change is +not later rediscovered and promoted as a performance optimization merely because +there are fewer `aligned(64)` strings in the source. From a1826551cf6331501ffdc4c6b422588f8a1e528f Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:45:47 +0200 Subject: [PATCH 137/156] Phase 5: add bounded forensic snapshot verify experiment --- ...ialize_forensic_snapshot_bounded_verify.py | 203 ++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 tools/materialize_forensic_snapshot_bounded_verify.py diff --git a/tools/materialize_forensic_snapshot_bounded_verify.py b/tools/materialize_forensic_snapshot_bounded_verify.py new file mode 100644 index 00000000..55f3211a --- /dev/null +++ b/tools/materialize_forensic_snapshot_bounded_verify.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Materialize the isolated bounded HDDMETA read-back experiment. + +The baseline forensic snapshot keeps two complete images live: the canonical +serialized HDDMETA image and an equally large read-back buffer used only for +byte-for-byte verification. This experiment keeps the canonical image but bounds +the read-back scratch to 64 KiB and compares every returned chunk exactly. + +No on-disk format, hash, slot, overwrite, error, or repair policy changes. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +MARKER = "bounded forensic snapshot read-back experiment" +CHUNK_DEFINE = "#define SNAPSHOT_VERIFY_CHUNK_BYTES (64u * 1024u)" + + +class MaterializeError(RuntimeError): + pass + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise MaterializeError(f"{label}: expected one match, found {count}") + return text.replace(old, new, 1) + + +def materialize(text: str) -> str: + if MARKER in text: + raise MaterializeError("forensic bounded verify already materialized") + + text = replace_once( + text, + "#define SNAPSHOT_TRAILER_BYTES 32u\n", + "#define SNAPSHOT_TRAILER_BYTES 32u\n" + f"{CHUNK_DEFINE}\n", + "chunk constant", + ) + + anchor = "int forensic_snapshot_save(unsigned int storage,\n" + helper = r'''/* bounded forensic snapshot read-back experiment. + * Keep exact byte-for-byte verification but do not duplicate the complete + * variable-size HDDMETA image merely to read it back. */ +static int snapshot_file_matches_bounded(const char *path, + const unsigned char *expected, + unsigned int size, + unsigned char *scratch, + unsigned int scratch_size) +{ + unsigned int offset = 0; + int fd; + int result; + + if (path == NULL || expected == NULL || scratch == NULL || + scratch_size == 0) + return -1; + fd = fileXioOpen(path, FIO_O_RDONLY, 0); + if (fd < 0) + return fd; + result = fileXioLseek(fd, 0, FIO_SEEK_END); + if (result < 0 || (unsigned int)result != size || + fileXioLseek(fd, 0, FIO_SEEK_SET) < 0) { + fileXioClose(fd); + return -1; + } + + while (offset < size) { + unsigned int bytes = size - offset; + unsigned int complete = 0; + + if (bytes > scratch_size) + bytes = scratch_size; + while (complete < bytes) { + result = fileXioRead(fd, scratch + complete, + (int)(bytes - complete)); + if (result <= 0) { + fileXioClose(fd); + return result < 0 ? result : -1; + } + complete += (unsigned int)result; + } + if (memcmp(scratch, expected + offset, bytes) != 0) { + fileXioClose(fd); + return 0; + } + offset += bytes; + } + + fileXioClose(fd); + return 1; +} + +''' + text = replace_once(text, anchor, helper + anchor, "bounded compare helper") + + text = replace_once( + text, + " unsigned int image_size = 0;\n" + " unsigned int slot;\n", + " unsigned int image_size = 0;\n" + " unsigned int verify_size = 0;\n" + " unsigned int slot;\n", + "verify size state", + ) + + text = replace_once( + text, + " verify = malloc(image_size);\n" + " if (verify == NULL) {\n", + " verify_size = image_size < SNAPSHOT_VERIFY_CHUNK_BYTES\n" + " ? image_size : SNAPSHOT_VERIFY_CHUNK_BYTES;\n" + " verify = malloc(verify_size);\n" + " if (verify == NULL) {\n", + "bounded verify allocation", + ) + + text = replace_once( + text, + " if (stat.size == image_size &&\n" + " read_exact_file(path, verify, (int)image_size) == 0 &&\n" + " memcmp(verify, image, image_size) == 0) {\n", + " if (stat.size == image_size &&\n" + " snapshot_file_matches_bounded(path, image, image_size,\n" + " verify, verify_size) == 1) {\n", + "existing snapshot compare", + ) + + text = replace_once( + text, + " if (read_exact_file(path, verify, (int)image_size) < 0 ||\n" + " memcmp(verify, image, image_size) != 0) {\n", + " if (snapshot_file_matches_bounded(path, image, image_size,\n" + " verify, verify_size) != 1) {\n", + "new snapshot compare", + ) + + if text.count("verify = malloc(image_size);") != 0: + raise MaterializeError("full-size verify allocation survived") + if text.count("snapshot_file_matches_bounded(") != 3: + raise MaterializeError("unexpected bounded compare helper/call count") + return text + + +def selftest() -> None: + fixture = r'''#define SNAPSHOT_TRAILER_BYTES 32u + +int forensic_snapshot_save(unsigned int storage, + const void *result, + const void *plan, + char path_out[64]) +{ + unsigned char *image = NULL; + unsigned char *verify = NULL; + unsigned int image_size = 0; + unsigned int slot; + verify = malloc(image_size); + if (verify == NULL) { + return -1; + } + if (stat.size == image_size && + read_exact_file(path, verify, (int)image_size) == 0 && + memcmp(verify, image, image_size) == 0) { + return 0; + } + if (read_exact_file(path, verify, (int)image_size) < 0 || + memcmp(verify, image, image_size) != 0) { + return -2; + } + free(verify); + free(image); + return 0; +} +''' + out = materialize(fixture) + assert MARKER in out + assert CHUNK_DEFINE in out + assert "verify = malloc(verify_size);" in out + assert "verify = malloc(image_size);" not in out + assert out.count("snapshot_file_matches_bounded(") == 3 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("source", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + if args.selftest: + selftest() + return 0 + if args.source is None: + parser.error("source is required unless --selftest is used") + args.source.write_text( + materialize(args.source.read_text(encoding="utf-8")), encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 6f3ef162fe4cf6689b0a4a3438ae9e94733f399d Mon Sep 17 00:00:00 2001 From: Hifu Date: Fri, 4 Sep 2026 22:46:08 +0200 Subject: [PATCH 138/156] Phase 5: test bounded forensic snapshot readback --- .../build_transaction_workspace_experiment.sh | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/tools/build_transaction_workspace_experiment.sh b/tools/build_transaction_workspace_experiment.sh index 3c474659..79856f12 100644 --- a/tools/build_transaction_workspace_experiment.sh +++ b/tools/build_transaction_workspace_experiment.sh @@ -2,21 +2,17 @@ set -eu # Build the active isolated Phase-5 incremental experiment without changing -# default runtime sources. CI #724 remains the frozen workspace-v1 point. +# default runtime sources. Frozen references: +# workspace v1 CI #724 +# workspace v1 + fingerprint malloc CI #739 # -# Active experiment: -# 1. materialize workspace v1 for COPY/source-hash + HDD verify; -# 2. replace source_fingerprint()'s helper-local memalign(64, 64 KiB) with -# ordinary malloc(64 KiB), because pinned fileXioRead does not require a -# 64-byte caller address. -# -# CI #743 tested natural alignment for two 1 KiB storage scratch buffers and -# found zero section/BSS/text/instruction benefit, so those alignments remain. -# The custom hdl0: SIF/DMA transaction workspace remains 64-byte aligned. +# Active experiment additionally bounds forensic HDDMETA read-back verification +# scratch to 64 KiB while preserving exact byte-for-byte comparison. ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) BACKUP=$(mktemp -d) TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" SOURCE_UI="$ROOT/src/hdl_tools/source_ui.inc" +FORENSIC_SNAPSHOT="$ROOT/src/forensic_snapshot.c" restore_sources() { if [ -f "$BACKUP/transaction.inc" ]; then @@ -25,16 +21,23 @@ restore_sources() { if [ -f "$BACKUP/source_ui.inc" ]; then cp "$BACKUP/source_ui.inc" "$SOURCE_UI" fi + if [ -f "$BACKUP/forensic_snapshot.c" ]; then + cp "$BACKUP/forensic_snapshot.c" "$FORENSIC_SNAPSHOT" + fi rm -rf "$BACKUP" } trap restore_sources EXIT HUP INT TERM cp "$TRANSACTION" "$BACKUP/transaction.inc" cp "$SOURCE_UI" "$BACKUP/source_ui.inc" +cp "$FORENSIC_SNAPSHOT" "$BACKUP/forensic_snapshot.c" + python3 "$ROOT/tools/materialize_transaction_workspace.py" \ "$TRANSACTION" "$TRANSACTION" python3 "$ROOT/tools/materialize_source_fingerprint_malloc.py" \ "$SOURCE_UI" +python3 "$ROOT/tools/materialize_forensic_snapshot_bounded_verify.py" \ + "$FORENSIC_SNAPSHOT" python3 "$ROOT/tools/allocation_inventory.py" \ --root "$ROOT" \ @@ -71,11 +74,12 @@ build_variant() cat >> "$provenance" < Date: Fri, 4 Sep 2026 22:46:43 +0200 Subject: [PATCH 139/156] Phase 5: bind bounded forensic snapshot experiment --- tools/transaction_workspace_ab_preflight.py | 141 ++++++++++---------- 1 file changed, 67 insertions(+), 74 deletions(-) diff --git a/tools/transaction_workspace_ab_preflight.py b/tools/transaction_workspace_ab_preflight.py index d8fa9755..d466e266 100644 --- a/tools/transaction_workspace_ab_preflight.py +++ b/tools/transaction_workspace_ab_preflight.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 -"""Validate and bind the active Phase-5 workspace-v1 + fingerprint-malloc A/B. +"""Validate and bind the active Phase-5 bounded HDDMETA verification A/B. -Frozen Phase-0 remains the correctness/identity baseline. Workspace v1 itself is -frozen at CI #724. CI #743 tested natural alignment for two 1 KiB storage -scratch buffers and found no section/BSS/text/instruction benefit, so that -layout-only experiment is not part of the active pair. +Frozen references: +- Phase-0 baseline: CI #666 +- transaction workspace v1: CI #724 +- workspace v1 + source-fingerprint malloc: CI #739 + +The active incremental change bounds forensic snapshot read-back scratch to +64 KiB while retaining exact byte-for-byte comparison. """ from __future__ import annotations @@ -29,38 +32,40 @@ }, } -WORKSPACE_V1 = { +FINGERPRINT_MALLOC = { "OFF": { - "elf_sha256": "23bbf6dfc28eb87bc7d484875a8940b9309eb5e3994d9c922388c9a0249415c6", + "elf_sha256": "97e2a802952ae6f3b46c9fa0148359db8f8b69e22923f8105378f094de59c28b", "elf_bytes": 632756, - "named_text": 229764, - "instructions": 57491, + "named_text": 229756, + "instructions": 57488, "execute_transaction_bytes": 6008, "execute_transaction_instructions": 1502, }, "ON": { - "elf_sha256": "09185cd6a21bbb9990d0b7f8cfe70fa80b4e9ba01a00e9648cb9fa70d9b3d693", - "elf_bytes": 638260, - "named_text": 232560, - "instructions": 58190, + "elf_sha256": "c8da50fe5147c3a24dc2f26d4ab910660bac615bf8e26f48e0bff3a2483f578b", + "elf_bytes": 638132, + "named_text": 232552, + "instructions": 58189, "execute_transaction_bytes": 6008, "execute_transaction_instructions": 1502, }, } +MAX_PATCHES = 2048 +SNAPSHOT_ENTRY_BYTES = 4 + 32 + 1024 +SNAPSHOT_MAX_BYTES = 64 + MAX_PATCHES * SNAPSHOT_ENTRY_BYTES + 32 +VERIFY_CHUNK_BYTES = 64 * 1024 +BASELINE_MAX_PEAK_BYTES = SNAPSHOT_MAX_BYTES * 2 +EXPERIMENT_MAX_PEAK_BYTES = SNAPSHOT_MAX_BYTES + VERIFY_CHUNK_BYTES ORDER = ["BASE", "EXP", "EXP", "BASE", "EXP", "BASE", "BASE", "EXP"] def info(path: Path) -> dict[str, object]: data = path.read_bytes() - return { - "path": path.name, - "bytes": len(data), - "sha256": hashlib.sha256(data).hexdigest(), - } + return {"path": path.name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()} -def validate_frozen(label: str, elf: dict[str,object], irx: dict[str,object]) -> None: +def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) -> None: expected = FROZEN[label] if elf["sha256"] != expected["elf_sha256"] or elf["bytes"] != expected["elf_bytes"]: raise SystemExit(f"{label} baseline ELF is not the frozen Phase-0 binary") @@ -70,23 +75,23 @@ def validate_frozen(label: str, elf: dict[str,object], irx: dict[str,object]) -> def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) -> dict: return { - "experiment": "hdl-workspace-v1-source-fingerprint-malloc", + "experiment": "forensic-snapshot-bounded-readback", "profile": profile, - "workload": "successful-hdl-transaction-copy-and-verify", + "workload": "forensic-HDDMETA-save-existing-slot-and-new-slot-readback", + "incremental_reference": FINGERPRINT_MALLOC[profile], "expected_source_change": { - "transaction_workspace_version": 1, - "workspace_bytes": 65536, - "workspace_alignment": 64, - "source_fingerprint_allocator_before": "memalign(64,65536)", - "source_fingerprint_allocator_after": "malloc(65536)", - "source_fingerprint_reads": 2, - "source_fingerprint_filexio_alignment_required": False, - "custom_hdl_sif_dma_alignment_changed": False, - "transport_change": False, + "snapshot_format_changed": False, + "exact_byte_compare_preserved": True, + "max_patch_count": MAX_PATCHES, + "max_snapshot_bytes": SNAPSHOT_MAX_BYTES, + "baseline_verify_allocation_bytes_at_max": SNAPSHOT_MAX_BYTES, + "experiment_verify_allocation_bytes_at_max": VERIFY_CHUNK_BYTES, + "baseline_image_plus_verify_peak_bytes_at_max": BASELINE_MAX_PEAK_BYTES, + "experiment_image_plus_verify_peak_bytes_at_max": EXPERIMENT_MAX_PEAK_BYTES, + "peak_reduction_bytes_at_max": BASELINE_MAX_PEAK_BYTES - EXPERIMENT_MAX_PEAK_BYTES, "iop_binary_change": False, }, "baseline": baseline, - "workspace_v1_reference": WORKSPACE_V1[profile], "experiment_binary": experiment, "hdl_stream_irx": irx, "hardware": { @@ -94,28 +99,24 @@ def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) - "hardware_revision": "UNRECORDED", "romver": "UNRECORDED", "storage_adapter": "UNRECORDED", - "hdd_model": "UNRECORDED", - "usb_device": "UNRECORDED", "active_irx": "UNRECORDED", }, "runs": [ { "index": i + 1, "variant": variant, - "source_fingerprint_elapsed_us": None, - "transaction_elapsed_us": None, - "copy_elapsed_us": None, - "verify_elapsed_us": None, + "patch_count": None, + "snapshot_bytes": None, + "existing_slot_match": None, + "new_slot_write_readback_match": None, + "elapsed_us": None, "correctness_hash": None, "result": None, } for i, variant in enumerate(ORDER) ], "report": { - "source_fingerprint_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, - "transaction_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, - "copy_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, - "verify_elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, + "elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, "correctness_failures": None, }, } @@ -148,53 +149,45 @@ def main() -> int: validate_frozen("OFF", baseline_off, baseline_irx_off) validate_frozen("ON", baseline_on, baseline_irx_on) - if experiment_irx_off["sha256"] != baseline_irx_off["sha256"]: - raise SystemExit("PROFILE OFF incremental experiment changed hdl_stream.irx") + raise SystemExit("PROFILE OFF forensic experiment changed hdl_stream.irx") if experiment_irx_on["sha256"] != baseline_irx_on["sha256"]: - raise SystemExit("PROFILE ON incremental experiment changed hdl_stream.irx") - if experiment_off["sha256"] == baseline_off["sha256"]: - raise SystemExit("PROFILE OFF incremental experiment did not change the EE ELF") - if experiment_on["sha256"] == baseline_on["sha256"]: - raise SystemExit("PROFILE ON incremental experiment did not change the EE ELF") + raise SystemExit("PROFILE ON forensic experiment changed hdl_stream.irx") + if experiment_off["sha256"] == FINGERPRINT_MALLOC["OFF"]["elf_sha256"]: + raise SystemExit("PROFILE OFF forensic experiment did not change the EE ELF") + if experiment_on["sha256"] == FINGERPRINT_MALLOC["ON"]["elf_sha256"]: + raise SystemExit("PROFILE ON forensic experiment did not change the EE ELF") identity = { - "experiment": "hdl-workspace-v1-source-fingerprint-malloc", + "experiment": "forensic-snapshot-bounded-readback", "project_git_sha": args.project_git_sha, "frozen_phase0_commit": "7875b14d837d6332f5edc37f1c12a55527d7dd87", "workspace_v1_frozen_ci": 724, - "workspace_v1_reference": WORKSPACE_V1, - "workspace_v2_rejected_ci": 733, + "fingerprint_malloc_frozen_ci": 739, "storage_scratch_natural_rejected_ci": 743, + "incremental_reference": FINGERPRINT_MALLOC, "ps2sdk_commit": "b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b", "toolchain": "mips64r5900el-ps2-elf GCC 15.2.0", - "change": { - "function": "source_fingerprint", - "allocator_before": "memalign(64,65536)", - "allocator_after": "malloc(65536)", - "consumer": "ordinary pinned fileXioRead + EE SHA-256", - "custom_sif_dma_buffer_changed": False, - }, - "PROFILE_OFF": { - "baseline_elf": baseline_off, - "experiment_elf": experiment_off, - "hdl_stream_irx": baseline_irx_off, + "memory_model": { + "max_patch_count": MAX_PATCHES, + "max_snapshot_bytes": SNAPSHOT_MAX_BYTES, + "verify_chunk_bytes": VERIFY_CHUNK_BYTES, + "baseline_image_plus_verify_peak_bytes": BASELINE_MAX_PEAK_BYTES, + "experiment_image_plus_verify_peak_bytes": EXPERIMENT_MAX_PEAK_BYTES, + "peak_reduction_bytes": BASELINE_MAX_PEAK_BYTES - EXPERIMENT_MAX_PEAK_BYTES, }, - "PROFILE_ON": { - "baseline_elf": baseline_on, - "experiment_elf": experiment_on, - "hdl_stream_irx": baseline_irx_on, + "correctness_contract": { + "on_disk_format_changed": False, + "slot_policy_changed": False, + "overwrite_policy_changed": False, + "exact_byte_compare_preserved": True, }, + "PROFILE_OFF": {"baseline_elf": baseline_off, "experiment_elf": experiment_off, "hdl_stream_irx": baseline_irx_off}, + "PROFILE_ON": {"baseline_elf": baseline_on, "experiment_elf": experiment_on, "hdl_stream_irx": baseline_irx_on}, } args.identity_output.write_text(json.dumps(identity, indent=2, sort_keys=True) + "\n", encoding="utf-8") - args.profile_off_template.write_text( - json.dumps(sample_template("OFF", baseline_off, experiment_off, baseline_irx_off), indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - args.profile_on_template.write_text( - json.dumps(sample_template("ON", baseline_on, experiment_on, baseline_irx_on), indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) + args.profile_off_template.write_text(json.dumps(sample_template("OFF", baseline_off, experiment_off, baseline_irx_off), indent=2, sort_keys=True) + "\n", encoding="utf-8") + args.profile_on_template.write_text(json.dumps(sample_template("ON", baseline_on, experiment_on, baseline_irx_on), indent=2, sort_keys=True) + "\n", encoding="utf-8") return 0 From d366f8140f3be661a0c55cb7fab5f73d198013f1 Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:05:22 +0200 Subject: [PATCH 140/156] Phase 5: simplify bounded HDDMETA exact readback --- ...ialize_forensic_snapshot_bounded_verify.py | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/tools/materialize_forensic_snapshot_bounded_verify.py b/tools/materialize_forensic_snapshot_bounded_verify.py index 55f3211a..b0e0e907 100644 --- a/tools/materialize_forensic_snapshot_bounded_verify.py +++ b/tools/materialize_forensic_snapshot_bounded_verify.py @@ -1,10 +1,15 @@ #!/usr/bin/env python3 -"""Materialize the isolated bounded HDDMETA read-back experiment. +"""Materialize the isolated bounded HDDMETA read-back v2 experiment. The baseline forensic snapshot keeps two complete images live: the canonical serialized HDDMETA image and an equally large read-back buffer used only for -byte-for-byte verification. This experiment keeps the canonical image but bounds -the read-back scratch to 64 KiB and compares every returned chunk exactly. +byte-for-byte verification. Bounded v1 (CI #749) reduced that second allocation +to 64 KiB but used two fileXioLseek RPCs to prove the file size before reading. + +V2 keeps the same bounded memory model and exact byte comparison while removing +those seek RPCs. It reads exactly the expected byte count and then requires one +additional one-byte read to return EOF, which still rejects both truncation and +trailing data. No on-disk format, hash, slot, overwrite, error, or repair policy changes. """ @@ -14,7 +19,7 @@ import argparse from pathlib import Path -MARKER = "bounded forensic snapshot read-back experiment" +MARKER = "bounded forensic snapshot read-back v2 experiment" CHUNK_DEFINE = "#define SNAPSHOT_VERIFY_CHUNK_BYTES (64u * 1024u)" @@ -31,7 +36,7 @@ def replace_once(text: str, old: str, new: str, label: str) -> str: def materialize(text: str) -> str: if MARKER in text: - raise MaterializeError("forensic bounded verify already materialized") + raise MaterializeError("forensic bounded verify v2 already materialized") text = replace_once( text, @@ -42,9 +47,10 @@ def materialize(text: str) -> str: ) anchor = "int forensic_snapshot_save(unsigned int storage,\n" - helper = r'''/* bounded forensic snapshot read-back experiment. - * Keep exact byte-for-byte verification but do not duplicate the complete - * variable-size HDDMETA image merely to read it back. */ + helper = r'''/* bounded forensic snapshot read-back v2 experiment. + * Keep exact byte-for-byte verification without duplicating the complete + * variable-size HDDMETA image. After the expected bytes are consumed, one + * final read must report EOF so trailing data is rejected without seek RPCs. */ static int snapshot_file_matches_bounded(const char *path, const unsigned char *expected, unsigned int size, @@ -61,12 +67,6 @@ def materialize(text: str) -> str: fd = fileXioOpen(path, FIO_O_RDONLY, 0); if (fd < 0) return fd; - result = fileXioLseek(fd, 0, FIO_SEEK_END); - if (result < 0 || (unsigned int)result != size || - fileXioLseek(fd, 0, FIO_SEEK_SET) < 0) { - fileXioClose(fd); - return -1; - } while (offset < size) { unsigned int bytes = size - offset; @@ -90,8 +90,11 @@ def materialize(text: str) -> str: offset += bytes; } + result = fileXioRead(fd, scratch, 1); fileXioClose(fd); - return 1; + if (result < 0) + return result; + return result == 0 ? 1 : 0; } ''' @@ -142,6 +145,8 @@ def materialize(text: str) -> str: raise MaterializeError("full-size verify allocation survived") if text.count("snapshot_file_matches_bounded(") != 3: raise MaterializeError("unexpected bounded compare helper/call count") + if "fileXioLseek(fd" in text[text.index(MARKER):text.index(anchor)]: + raise MaterializeError("seek-based size check survived in bounded helper") return text @@ -181,6 +186,9 @@ def selftest() -> None: assert "verify = malloc(verify_size);" in out assert "verify = malloc(image_size);" not in out assert out.count("snapshot_file_matches_bounded(") == 3 + helper = out[out.index(MARKER):out.index("int forensic_snapshot_save")] + assert "fileXioLseek" not in helper + assert "fileXioRead(fd, scratch, 1)" in helper def main() -> int: From c1430109ebe359f56fb1fceacc6b71f06f8722a0 Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:05:35 +0200 Subject: [PATCH 141/156] Phase 5: build bounded HDDMETA readback v2 --- tools/build_transaction_workspace_experiment.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/build_transaction_workspace_experiment.sh b/tools/build_transaction_workspace_experiment.sh index 79856f12..f06fb387 100644 --- a/tools/build_transaction_workspace_experiment.sh +++ b/tools/build_transaction_workspace_experiment.sh @@ -5,9 +5,12 @@ set -eu # default runtime sources. Frozen references: # workspace v1 CI #724 # workspace v1 + fingerprint malloc CI #739 +# bounded HDDMETA read-back v1 CI #749 # -# Active experiment additionally bounds forensic HDDMETA read-back verification -# scratch to 64 KiB while preserving exact byte-for-byte comparison. +# Active bounded-v2 experiment keeps the 64 KiB exact read-back scratch but +# removes the two seek RPCs used by v1. It reads exactly the expected bytes and +# then requires one extra byte read to report EOF, preserving truncation and +# trailing-data detection. ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) BACKUP=$(mktemp -d) TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" @@ -78,8 +81,11 @@ hdl_transaction_workspace_bytes: "65536" hdl_transaction_workspace_alignment: "64" hdl_source_fingerprint_heap_experiment: "malloc" forensic_snapshot_bounded_verify_enabled: "1" +forensic_snapshot_bounded_verify_version: "2" forensic_snapshot_verify_chunk_bytes: "65536" forensic_snapshot_verify_policy: "exact-byte-compare" +forensic_snapshot_size_check: "exact-bytes-plus-eof-read" +forensic_snapshot_seek_rpcs_per_verify: "0" EOF } From c219907436fd84869fd95c0a209247ccd4ce7d2f Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:06:04 +0200 Subject: [PATCH 142/156] Phase 5: bind bounded HDDMETA readback v2 A/B --- tools/transaction_workspace_ab_preflight.py | 64 ++++++++++++++++----- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/tools/transaction_workspace_ab_preflight.py b/tools/transaction_workspace_ab_preflight.py index d466e266..e1956ced 100644 --- a/tools/transaction_workspace_ab_preflight.py +++ b/tools/transaction_workspace_ab_preflight.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 -"""Validate and bind the active Phase-5 bounded HDDMETA verification A/B. +"""Validate and bind the active Phase-5 bounded HDDMETA verification v2 A/B. Frozen references: - Phase-0 baseline: CI #666 - transaction workspace v1: CI #724 - workspace v1 + source-fingerprint malloc: CI #739 +- bounded HDDMETA read-back v1: CI #749 -The active incremental change bounds forensic snapshot read-back scratch to -64 KiB while retaining exact byte-for-byte comparison. +V2 keeps the same 64 KiB exact byte-comparison memory model as v1 but replaces +v1's two fileXioLseek size-check RPCs with one final one-byte read that must +report EOF after all expected bytes have been consumed. """ from __future__ import annotations @@ -51,6 +53,27 @@ }, } +BOUNDED_V1 = { + "OFF": { + "elf_sha256": "64bcc758c9a848cb8c2a85284a47e9217693d725004b6a016813f825339a2775", + "elf_bytes": 633012, + "section_text": 286789, + "named_text": 230112, + "instructions": 57579, + "execute_transaction_bytes": 6008, + "execute_transaction_instructions": 1502, + }, + "ON": { + "elf_sha256": "ad13f89be93575fe63cbb762893e2c4f7ee102b37ffd73794ef91bf224988af7", + "elf_bytes": 638516, + "section_text": 290749, + "named_text": 232908, + "instructions": 58278, + "execute_transaction_bytes": 6008, + "execute_transaction_instructions": 1502, + }, +} + MAX_PATCHES = 2048 SNAPSHOT_ENTRY_BYTES = 4 + 32 + 1024 SNAPSHOT_MAX_BYTES = 64 + MAX_PATCHES * SNAPSHOT_ENTRY_BYTES + 32 @@ -75,20 +98,22 @@ def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) -> dict: return { - "experiment": "forensic-snapshot-bounded-readback", + "experiment": "forensic-snapshot-bounded-readback-v2", "profile": profile, "workload": "forensic-HDDMETA-save-existing-slot-and-new-slot-readback", - "incremental_reference": FINGERPRINT_MALLOC[profile], + "bounded_v1_reference": BOUNDED_V1[profile], "expected_source_change": { "snapshot_format_changed": False, "exact_byte_compare_preserved": True, "max_patch_count": MAX_PATCHES, "max_snapshot_bytes": SNAPSHOT_MAX_BYTES, - "baseline_verify_allocation_bytes_at_max": SNAPSHOT_MAX_BYTES, - "experiment_verify_allocation_bytes_at_max": VERIFY_CHUNK_BYTES, + "verify_chunk_bytes": VERIFY_CHUNK_BYTES, "baseline_image_plus_verify_peak_bytes_at_max": BASELINE_MAX_PEAK_BYTES, "experiment_image_plus_verify_peak_bytes_at_max": EXPERIMENT_MAX_PEAK_BYTES, "peak_reduction_bytes_at_max": BASELINE_MAX_PEAK_BYTES - EXPERIMENT_MAX_PEAK_BYTES, + "bounded_v1_seek_rpcs_per_verify": 2, + "bounded_v2_seek_rpcs_per_verify": 0, + "bounded_v2_final_eof_reads_per_verify": 1, "iop_binary_change": False, }, "baseline": baseline, @@ -150,22 +175,23 @@ def main() -> int: validate_frozen("OFF", baseline_off, baseline_irx_off) validate_frozen("ON", baseline_on, baseline_irx_on) if experiment_irx_off["sha256"] != baseline_irx_off["sha256"]: - raise SystemExit("PROFILE OFF forensic experiment changed hdl_stream.irx") + raise SystemExit("PROFILE OFF forensic v2 experiment changed hdl_stream.irx") if experiment_irx_on["sha256"] != baseline_irx_on["sha256"]: - raise SystemExit("PROFILE ON forensic experiment changed hdl_stream.irx") - if experiment_off["sha256"] == FINGERPRINT_MALLOC["OFF"]["elf_sha256"]: - raise SystemExit("PROFILE OFF forensic experiment did not change the EE ELF") - if experiment_on["sha256"] == FINGERPRINT_MALLOC["ON"]["elf_sha256"]: - raise SystemExit("PROFILE ON forensic experiment did not change the EE ELF") + raise SystemExit("PROFILE ON forensic v2 experiment changed hdl_stream.irx") + if experiment_off["sha256"] == BOUNDED_V1["OFF"]["elf_sha256"]: + raise SystemExit("PROFILE OFF bounded v2 experiment did not change the EE ELF from bounded v1") + if experiment_on["sha256"] == BOUNDED_V1["ON"]["elf_sha256"]: + raise SystemExit("PROFILE ON bounded v2 experiment did not change the EE ELF from bounded v1") identity = { - "experiment": "forensic-snapshot-bounded-readback", + "experiment": "forensic-snapshot-bounded-readback-v2", "project_git_sha": args.project_git_sha, "frozen_phase0_commit": "7875b14d837d6332f5edc37f1c12a55527d7dd87", "workspace_v1_frozen_ci": 724, "fingerprint_malloc_frozen_ci": 739, "storage_scratch_natural_rejected_ci": 743, - "incremental_reference": FINGERPRINT_MALLOC, + "bounded_readback_v1_frozen_ci": 749, + "bounded_v1_reference": BOUNDED_V1, "ps2sdk_commit": "b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b", "toolchain": "mips64r5900el-ps2-elf GCC 15.2.0", "memory_model": { @@ -176,10 +202,18 @@ def main() -> int: "experiment_image_plus_verify_peak_bytes": EXPERIMENT_MAX_PEAK_BYTES, "peak_reduction_bytes": BASELINE_MAX_PEAK_BYTES - EXPERIMENT_MAX_PEAK_BYTES, }, + "io_model": { + "bounded_v1_seek_rpcs_per_verify": 2, + "bounded_v2_seek_rpcs_per_verify": 0, + "bounded_v2_final_eof_reads_per_verify": 1, + "short_read_handling_preserved": True, + }, "correctness_contract": { "on_disk_format_changed": False, "slot_policy_changed": False, "overwrite_policy_changed": False, + "truncation_detection_preserved": True, + "trailing_data_detection_preserved": True, "exact_byte_compare_preserved": True, }, "PROFILE_OFF": {"baseline_elf": baseline_off, "experiment_elf": experiment_off, "hdl_stream_irx": baseline_irx_off}, From a38c1513b8ef743fcdcdb4a205cf32ad94f64f9f Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:09:15 +0200 Subject: [PATCH 143/156] Phase 5: include IOP SysMem in allocation inventory --- tools/allocation_inventory.py | 80 ++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/tools/allocation_inventory.py b/tools/allocation_inventory.py index 9602ef55..bc6770da 100644 --- a/tools/allocation_inventory.py +++ b/tools/allocation_inventory.py @@ -1,11 +1,15 @@ #!/usr/bin/env python3 -"""Emit a complete runtime allocation/free inventory for corpus-v2 review. +"""Emit a complete direct runtime allocation/free inventory for corpus-v2 review. This is deliberately a source inventory, not an allocator-performance claim. -The output keeps every malloc/calloc/realloc/memalign/free occurrence instead of -the sample-limited evidence section in the broader project audit, and associates -it with the containing C function when the existing source parser can resolve -one. +The output keeps every direct EE libc malloc/calloc/realloc/memalign/free call +and every direct IOP SysMem AllocSysMemory/FreeSysMemory call instead of the +sample-limited evidence section in the broader project audit. Events are bound +to the containing C function when the existing source parser can resolve one. + +ThreadMan objects (CreateThread/CreateSema/etc.) are intentionally not reported +as heap allocations here: their implementation-owned memory cost belongs in the +runtime IOP-resource budget and must not be guessed from an API call count. """ from __future__ import annotations @@ -26,8 +30,17 @@ source_files, ) -ALLOC_RE = re.compile(r"\b(malloc|calloc|realloc|memalign|free)\s*\(") -ALLOCATORS = {"malloc", "calloc", "realloc", "memalign"} +ALLOC_RE = re.compile( + r"\b(malloc|calloc|realloc|memalign|free|AllocSysMemory|FreeSysMemory)\s*\(" +) +ALLOCATORS = {"malloc", "calloc", "realloc", "memalign", "AllocSysMemory"} +FREES = {"free", "FreeSysMemory"} + + +def _domain(operation: str) -> str: + if operation in {"AllocSysMemory", "FreeSysMemory"}: + return "IOP SysMem" + return "EE/libc heap" def _function_index(root: Path, files: list[Path]) -> dict[str, list[Any]]: @@ -50,6 +63,7 @@ def collect(root: Path) -> dict[str, Any]: by_path = _function_index(root, files) events: list[dict[str, Any]] = [] operation_counts: collections.Counter[str] = collections.Counter() + domain_counts: collections.Counter[str] = collections.Counter() file_counts: collections.Counter[str] = collections.Counter() function_counts: collections.Counter[str] = collections.Counter() @@ -61,30 +75,43 @@ def collect(root: Path) -> dict[str, Any]: for match in ALLOC_RE.finditer(code): operation = match.group(1) scope = _scope_for(functions, lineno) + domain = _domain(operation) event = { "path": rel, "line": lineno, "function": scope, "operation": operation, "kind": "allocate" if operation in ALLOCATORS else "free", + "memory_domain": domain, "source": line.strip(), } events.append(event) operation_counts[operation] += 1 + domain_counts[domain] += 1 file_counts[rel] += 1 function_counts[f"{rel}:{scope}"] += 1 events.sort(key=lambda item: (item["path"], item["line"], item["operation"])) return { - "epistemic_status": "CURRENT IMPLEMENTATION: static source inventory", + "epistemic_status": "CURRENT IMPLEMENTATION: static direct allocator-call inventory", + "coverage_notes": [ + "EE libc heap calls: malloc/calloc/realloc/memalign/free", + "IOP SysMem calls: AllocSysMemory/FreeSysMemory", + "ThreadMan object backing memory is implementation-owned and not inferred from CreateThread/CreateSema call counts", + "allocator-internal allocations not visible as direct source calls are outside this static inventory", + ], "runtime_roots": list(RUNTIME_ROOTS), "total_events": len(events), "allocation_events": sum( count for operation, count in operation_counts.items() if operation in ALLOCATORS ), - "free_events": operation_counts["free"], + "free_events": sum( + count for operation, count in operation_counts.items() + if operation in FREES + ), "operation_counts": dict(sorted(operation_counts.items())), + "memory_domain_event_counts": dict(sorted(domain_counts.items())), "file_counts": dict(sorted(file_counts.items())), "function_counts": dict(sorted(function_counts.items())), "events": events, @@ -109,21 +136,40 @@ def selftest() -> None: free(a); return 0; } +""".lstrip(), + encoding="utf-8", + ) + (root / "iop" / "fixture.c").write_text( + """ +static void iop_phase(unsigned int n) +{ + void *p = AllocSysMemory(0, n, 0); + if (p != 0) + FreeSysMemory(p); +} """.lstrip(), encoding="utf-8", ) result = collect(root) - assert result["total_events"] == 5 - assert result["allocation_events"] == 3 - assert result["free_events"] == 2 + assert result["total_events"] == 7 + assert result["allocation_events"] == 4 + assert result["free_events"] == 3 assert result["operation_counts"] == { + "AllocSysMemory": 1, + "FreeSysMemory": 1, "free": 2, "malloc": 1, "memalign": 1, "realloc": 1, } - assert all( - event["function"] == "phase" for event in result["events"] + assert result["memory_domain_event_counts"] == { + "EE/libc heap": 5, + "IOP SysMem": 2, + } + assert any( + event["function"] == "iop_phase" and + event["memory_domain"] == "IOP SysMem" + for event in result["events"] ) @@ -150,7 +196,11 @@ def main() -> int: "allocation inventory: " f"{result['allocation_events']} allocate events, " f"{result['free_events']} free events, " - f"{result['total_events']} total" + f"{result['total_events']} total; " + + ", ".join( + f"{domain}={count}" + for domain, count in result["memory_domain_event_counts"].items() + ) ) return 0 From 12652325d8070b2cd3a593e4a6419d9ffa16ea25 Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:11:09 +0200 Subject: [PATCH 144/156] Phase 5: extend allocation lifetime audit to IOP SysMem --- docs/ALLOCATION_LIFETIME_AUDIT.md | 424 +++++++++++++++++------------- 1 file changed, 240 insertions(+), 184 deletions(-) diff --git a/docs/ALLOCATION_LIFETIME_AUDIT.md b/docs/ALLOCATION_LIFETIME_AUDIT.md index b08ede1e..54c063b9 100644 --- a/docs/ALLOCATION_LIFETIME_AUDIT.md +++ b/docs/ALLOCATION_LIFETIME_AUDIT.md @@ -1,19 +1,24 @@ # Allocation and lifetime audit This document classifies the major dynamic allocations by producer, consumer, -lifetime and ownership before Phase-5 allocator work. The goal is not to replace -`malloc()` because it exists. The project corpus requires allocation changes to -remove measured churn, copies, fragmentation risk or peak working-set pressure. +lifetime, memory domain and ownership before further Phase-5 allocator work. The +goal is not to replace `malloc()` because it exists. The project corpus requires +allocation changes to remove measured churn, copies, fragmentation risk or peak +working-set pressure. ## Source-of-truth routing -- `PS2_Memory_Allocators_optimization_research_corpus_v2.md`: classify by - lifetime; alignment is a consumer contract; avoid per-item churn on hot paths; -- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md`: producer, - consumer, lifetime, ownership and representation decide reuse; -- `PS2_PERFORMANCE_BIBLE.md`: remove work/copies/allocations before specialised - kernels, but only where the workload exposes the cost; -- project CI #712 source audit plus current branch source. +- `PS2_Optimization_Library_v2_MANIFEST.md` +- `PS2_PERFORMANCE_BIBLE.md` +- `PS2_Memory_Allocators_optimization_research_corpus_v2.md` +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md` +- `PS2_Whole_System_Scheduling_research_corpus_v2.md` +- `PS2_IOP_SIF_optimization_research_corpus_v2.md` +- pinned PS2SDK `b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b` + +The corpus rule applied here is: classify producer, consumer, lifetime and the +actual alignment domain before choosing an allocator. A pool/arena is not +implicitly better than the current heap merely because it sounds console-like. ## Epistemic labels @@ -22,91 +27,55 @@ remove measured churn, copies, fragmentation risk or peak working-set pressure. - **INFERENCJA**: likely optimization consequence, not yet hardware measured. - **HIPOTEZA DO TESTU**: change requiring real-PS2 A/B before acceptance. -## Highest-value candidate: one transaction-owned 64 KiB I/O workspace - -### Current allocation pattern - -The HDL transaction uses `HDL_INSTALL_IO_BYTES = 64 KiB` buffers in three -sequential helpers: +`tools/allocation_inventory.py` is a static direct-call inventory. It now covers: ```text -hash_source_payload() - memalign(64, 64 KiB) - source SHA reconstruction +EE/libc heap + malloc + calloc + realloc + memalign free -copy_payload() - memalign(64, 64 KiB) - source read / IOP pump DMA destination / EE SHA consumer - free - -verify_target_digest() - memalign(64, 64 KiB) - HDD -> EE DMA destination / target SHA consumer - free +IOP SysMem + AllocSysMemory + FreeSysMemory ``` -A normal fresh install executes copy then target verification. A resumed -`PAYLOAD_VERIFIED` legacy/fallback path executes source hash then target -verification. The helpers do not own their buffers concurrently. - -### Alignment contract - -**POTWIERDZONE:** keep 64-byte alignment. `hdl_fast_dma_read()` explicitly rejects -an EE destination whose address is not 64-byte aligned. This is the custom SIF/ -cache transport contract, unlike the ordinary fileXio scratch buffers audited in -`ALIGNMENT_CONTRACT_AUDIT.md`. - -### Lifetime conclusion +It deliberately does not guess ThreadMan implementation-owned backing memory +from `CreateThread`/`CreateSema` call counts. Those costs remain part of the +runtime IOP-memory gate. -**POTWIERDZONE:** these three buffers have transaction-local, mutually exclusive -lifetimes. - -**INFERENCJA:** a single transaction-owned 64 KiB workspace can serve all three -helpers and remove repeated allocator calls without increasing peak payload -memory or changing SIF/HDD/source representation. - -Candidate ownership: +## Lifetime classes ```text -execute_transaction owns workspace - FREE/UNUSED before allocation - SOURCE_HASH while hash_source_payload consumes it - COPY_IO while copy_payload consumes it - TARGET_VERIFY while verify_target_digest consumes it - released once at transaction exit +permanent process lifetime +menu/session one UI/session/catalogue lifetime +action one bounded user operation +transaction one HDL install/recovery transaction +stream one open streaming service instance +phase-temporary one scan/hash/copy/verify sub-phase ``` -No helper may retain the pointer after return. +These are review labels derived from current ownership structure, not allocator +latency measurements. -### Proposed post-gate A/B +# EE/libc heap -Baseline: current helper-local allocations. +## Boot-chain and generic bounded-file helpers -Experiment: +| Site | Lifetime | Producer / consumer | Current action | +| --- | --- | --- | --- | +| `boot_chain_ps2.c:read_skip_hdd_setting` | phase-temporary | small config reader -> parser | keep | +| `boot_chain_ps2.c:scan_sysconf_partition` | phase-temporary | config reader -> parser | keep | +| `bootstrap_source.c:load_payload_file` | action | file -> bootstrap payload validation/write flow | keep unless payload peak is measured problematic | +| `storage.c:read_bounded_file` | caller-owned action | bounded file -> caller | keep generic helper | +| `hdd_read.c:hdd_read_payload_image` | action | HDD payload -> recovery consumer | returned dataset, not disposable scratch | -1. allocate one `memalign(64, HDL_INSTALL_IO_BYTES)` workspace only for stages - that need source/copy/target hashing; -2. pass pointer + capacity to each helper; -3. remove helper-local alloc/free pairs; -4. preserve all fileXio/SIF/cache/journal/error semantics; -5. free exactly once on transaction exit. +These are ordinary CPU/fileXio data. None has a demonstrated need for a custom +allocator today. -Measure: - -```text -allocator calls per transaction -peak EE heap delta -copy/verify p50/p95/p99/max -total transaction p50/p95/p99/max -execute_transaction/static text delta -correctness hash -``` - -Priority: **HIGH after the frozen resume-hash hardware gate**, because it changes -an active bulk transaction path but does not require a new representation. - -## USB ISO catalogue array +## USB ISO selection array Producer: `scan_mass_images()`. @@ -115,26 +84,23 @@ Current representation: ```text initial capacity: 32 hdl_image_entry_t allocation: calloc - growth: doubling below 1024, then +1024 entries +growth: doubling below 1024, then +1024 entries consumer: ISO selection UI + selected path/size handoff -lifetime: one begin_new_install selection session -release: before destructive confirmation / execute_transaction +lifetime: one begin_new_install source-selection session +release: before the long-running destructive transaction ``` -**POTWIERDZONE:** the array is freed after the selected ISO fields have been -copied into the transaction, before the long-running HDD transaction begins. - -**INFERENCJA:** this is a sensible variable-cardinality session allocation. Do -not replace it with a giant permanent table without measured directory-size or -allocation-jitter evidence. +The resume-hash `.inc` contains the alternate build equivalent; both fragments +are not linked simultaneously. -Potential improvement only if logs show large catalogues/realloc churn: +**POTWIERDZONE:** the array is function/session owned and released after the +selected source identity is copied out. -- count/size directory entries first only if the second scan is cheaper than - growth for the real workload; -- or use a bounded chunked/session arena if large catalogues are common. +**INFERENCJA:** this is an appropriate variable-cardinality session allocation. +Do not replace it with a giant permanent table without catalogue cardinality or +allocator-jitter evidence. -Priority: **LOW until catalogue cardinality is measured.** +Priority: **LOW until large source catalogues are measured.** ## Installed HDL catalogue array @@ -143,159 +109,249 @@ Producer: raw APA chain walker. Current representation: ```text -initial capacity: 64 entries on first growth - growth: capacity * 2 +first capacity: 64 entries +growth: capacity * 2 consumer: installed-games menu/details/delete selection -lifetime: one menu session -metadata: loaded lazily by visible page and cached in each entry -release: leaving menu; rebuilt after successful deletion +lifetime: one catalogue/menu session +metadata: lazy per visible page, stored inside each entry +release: catalog_free() ``` **POTWIERDZONE:** `realloc` occurs only while discovering main HDL partitions; metadata itself is not separately heap-allocated per game. -**INFERENCJA:** the growable session array is appropriate unless very large HDL -catalogues demonstrate allocator/copy cost. A persistent index does not solve -this automatically because invalidation must still be cheaper than the APA walk. +**INFERENCJA:** geometric growth already avoids per-entry allocation churn. A +pool or persistent index is not justified until game-count/catalogue latency is +measured and invalidation can be made cheaper than the APA walk. + +Priority: **LOW/MEDIUM depending real catalogue size.** + +## HDL transaction / streaming workspaces + +Default source contains separate 64 KiB helper allocations for: + +```text +source_fingerprint() +hash_source_payload() +copy_payload() +verify_target_digest() +``` + +The three transaction helpers use a 64-byte-aligned EE destination because the +custom `hdl0:` SIF/DMA path explicitly requires that destination contract. That +alignment must not be confused with ordinary fileXio caller alignment. + +### CI #724: transaction workspace v1 -Priority: **LOW/MEDIUM depending measured game count and catalogue latency.** +**POTWIERDZONE:** COPY/source-hash and target-verify workspaces have mutually +exclusive lifetimes. + +The isolated v1 experiment gives ownership to `execute_transaction()` and lends +one 64 KiB / 64-byte-aligned workspace sequentially to those helpers. It removes +one general-heap allocation/free pair on the successful bulk path without +increasing peak workspace. + +### CI #733: workspace v2, rejected/held + +Extending the same workspace backwards into source admission removes another +allocation pair but grows the transaction control path and lengthens workspace +lifetime. One extra allocation per transaction is too small a prize to accept +that trade without real hardware evidence. + +### CI #739: source fingerprint `memalign` -> `malloc` + +Pinned `fileXioRead()` accepts ordinary caller alignment, so the standalone +source-fingerprint buffer has no demonstrated 64-byte API requirement. + +Static result versus workspace v1 is slightly smaller and does not grow +`execute_transaction()`. This remains **HIPOTEZA DO TESTU** because fileXio's +unaligned-edge handling can cost time even when the API contract permits it. + +The custom SIF/DMA transaction workspace remains 64-byte aligned. ## Forensic HDDMETA snapshot Producer: `build_snapshot_image()`. -Current peak state during save: +Baseline save keeps two variable-size allocations live: + +```text +canonical image = 64 + patch_count * (4 + 32 + 1024) + 32 +verify buffer = canonical image size +``` + +At the current maximum `patch_count = 2048`: + +```text +canonical image 2,170,976 B +baseline full verify 2,170,976 B +baseline pair peak 4,341,952 B +``` + +### CI #749: bounded read-back v1 + +The first isolated bounded experiment keeps the canonical image but replaces the +second full-size allocation with at most 64 KiB and compares every returned +chunk byte-for-byte. It preserves format, slot selection, non-overwrite policy, +truncation detection and full read-back verification. ```text -image = malloc(image_size) -verify = malloc(image_size) -write image -read entire file into verify -memcmp(verify, image, image_size) -free both +canonical image 2,170,976 B +bounded verify 65,536 B +experiment pair peak 2,236,512 B +peak reduction 2,105,440 B ``` -`image_size` scales with `patch_count` because every touched original 1024-byte -APA header is embedded in the safety record. +### CI #752: bounded read-back v2 -**POTWIERDZONE:** two equal-size buffers are simultaneously live solely for -read-back verification. +V1 used two `fileXioLseek()` RPCs inside each exact-compare call to prove file +size. V2 instead: -**INFERENCJA:** the second full-size allocation can be removed without weakening -verification by reading the saved file back in a bounded scratch window and -comparing each window to the still-owned canonical `image`. This retains exact -byte-for-byte verification rather than replacing it with an unchecked write. +1. reads exactly the expected bytes with short-read handling; +2. compares every chunk exactly; +3. performs one final one-byte read which must return EOF. -Alternative: compare a streamed read-back hash against a canonical image hash, -but exact chunk comparison is simpler and preserves the current error contract. +This still rejects truncation and trailing data while removing both seek RPCs. +Compared with CI #749, CI #752 reduces `.text` by 32 B, named text by 36 B and +eight R5900 instructions in both PROFILE modes. `execute_transaction()` and the +IOP binary remain unchanged. -Priority: **MEDIUM for peak-memory robustness, LOW for normal performance** -because forensic repair is an exceptional cold path. +**POTWIERDZONE:** maximum pair peak is reduced by 2,105,440 B relative to the +original full-image verify representation. -## Bootstrap payload (`MBR.XLF`) +**HIPOTEZA DO TESTU:** real PS2 must still verify recovery correctness and +fileXio latency. This is primarily a peak-working-set optimization, not a hot +transaction speedup claim. -Producer: `load_payload_file()`. +The next architectural candidate is eliminating the full canonical image with a +streaming APAMETA1 serializer. That would alter producer lifetime and write +structure and therefore requires an isolated experiment plus stronger reference +serialization tests before hardware use. + +# IOP SysMem + +The custom `hdl_stream` service has three direct SysMem ownership classes. + +## Stream object + +Current source: ```text -allocation: malloc(file size), bounded by HDD_MAX_MBR_PAYLOAD_SIZE -consumer: KELF/layout validation and subsequent bootstrap write workflow -ownership: bootstrap_source_t.payload -lifetime: prepare -> caller operations -> bootstrap_source_release +AllocSysMemory(ALLOC_FIRST, sizeof(*stream), NULL) ``` -**POTWIERDZONE:** this is not transient read scratch. The loaded representation -is itself consumed across multiple stages. +Lifetime: one open `hdl0:` stream. -**INFERENCJA:** retaining one owned payload buffer is correct. Replacing it with -chunked streaming would complicate KELF/layout consumers and should not be done -without evidence that payload peak memory is a problem. +Owner: `stream_open()` -> `stream_close()`. -Priority: **KEEP unless memory measurements disagree.** +The object stores partition geometry, source-map state, staging ownership, +prefetch handles and optional PROFILE counters. Pooling a single stream object +has no demonstrated benefit. -## Raw active bootstrap payload read +## Staging allocation -Producer: `hdd_read_payload_image()`. +Preferred admission: ```text -allocation: malloc(total selected payload bytes) -producer scratch: fixed HDD_TRANSFER_BYTES temporary -consumer: caller receives payload_out -ownership transfer: function -> caller +AllocSysMemory(ALLOC_FIRST, + 2 * HDL_STREAM_IOP_STAGE_BYTES + 63, + NULL) ``` -**POTWIERDZONE:** the heap allocation is the returned dataset, not helper-local -scratch. It cannot be removed without changing the API/consumer representation. +Low-memory fallback: -Priority: **KEEP; redesign only with an explicit streaming consumer.** +```text +AllocSysMemory(ALLOC_FIRST, + HDL_STREAM_IOP_STAGE_BYTES + 63, + NULL) +``` -## Boot-chain text/config files +The two calls are mutually exclusive outcomes. The allocation is manually +rounded to a 64-byte stage address and remains owned for the stream lifetime. -Current source allocates bounded text buffers for complete small configuration -files and frees them at the end of the corresponding probe/parse operation. +**POTWIERDZONE:** failure to obtain double buffering falls back to one stage; it +does not fail stream admission if one stage still fits. -**INFERENCJA:** these are cold startup/configuration allocations. Replacing them -with custom pools is lower value than transaction/storage work unless startup -profiling identifies allocator cost or fragmentation. +This is an explicit resilience/ownership policy. Do not replace it with a larger +ring until real IOP memory and stall telemetry justify more buffering. -Priority: **LOW.** +## Direct-BDM USB fragment map -## Rescue/forensic/general storage allocations +Current source allocates: -The source audit identifies additional allocations in `rescue_storage.c`, -`forensic_snapshot.c`, `bootstrap_source.c`, `storage.c` and boot tooling. They -are mostly operation/session-owned bounded records rather than per-64-KiB hot -loop allocations. +```text +fragment_count * sizeof(bd_fragment_t) +``` -Rule for subsequent review: +with `AllocSysMemory`, up to the current 4096-fragment hard limit. + +Lifetime: one usable direct-BDM source mapping. It is released when source +mapping is reset/disabled or when the stream closes. + +The static IOP budget records the current worst-case map as 49,152 B. + +This is producer representation state, not generic scratch. Any compaction must +preserve fragmented-file correctness and the sequential cursor behaviour that +avoids rescanning thousands of fragments for every 64 KiB block. + +## IOP memory outside the direct SysMem inventory + +ThreadMan owns backing memory for resources including: ```text -if allocation happens once per user operation: - measure peak bytes and failure behaviour first -if allocation happens at each bulk phase boundary: - consider lifetime reuse -if allocation happens inside a chunk/item loop: - treat as immediate review trigger +prefetch worker thread +request semaphore +done semaphore +stopped semaphore ``` -The current HDL fast 64-KiB loop does **not** allocate per chunk. Its allocation -churn is per phase, which is why one transaction workspace is the appropriate -first allocator experiment rather than an arena rewrite. +The worker requests a 4096-byte stack, but ThreadMan control-object overhead is +not inferred here. `docs/HDL_IOP_RAM_BUDGET.md` correctly leaves that overhead +unmeasured until real hardware records active modules and minimum free IOP RAM. -## Phase-5 priority order +# Phase-5 priority order -1. **After hardware gate:** A/B one transaction-owned 64 KiB aligned workspace. -2. Record EE heap before/after transaction and at major phase boundaries if a - current safe heap query is available without materially perturbing the path. -3. If exceptional recovery memory matters, replace forensic full-size read-back - duplicate with bounded exact chunk comparison. -4. Measure ISO/game catalogue cardinality before changing their growth strategy. -5. Leave payload-owning allocations intact until a consumer can operate on a +1. Keep **CI #752 bounded HDDMETA v2** as the current peak-memory candidate. +2. Hardware A/B transaction workspace v1 and source-fingerprint allocator policy. +3. Build a reference serializer test before attempting streaming canonical + HDDMETA generation. +4. Measure ISO/game catalogue cardinality before changing growth strategy. +5. Leave payload-owning allocations intact until their consumers can use a different representation. -6. Do not introduce custom pools/arenas merely to reduce the count of `malloc` - strings in source. +6. Do not introduce global pools/arenas or global 64-byte heap alignment merely + to reduce allocator call counts in source. +7. Do not expand IOP staging before runtime free-memory and latency attribution + are available. -## Acceptance record for allocator changes +## Acceptance record for allocator/lifetime changes ```yaml allocation_site: +memory_domain: producer: consumer: -lifetime: -bytes: -alignment: +lifetime_before: +lifetime_after: +bytes_requested: +alignment_before: +alignment_after: ownership_before: ownership_after: alloc_calls_before: alloc_calls_after: -peak_heap_before: -peak_heap_after: +peak_bytes_before: +peak_bytes_after: +elf_text_delta: +correctness_hash: +console_scp: +hardware_revision: +active_irx: +sample_count: p50: p95: p99: max: -correctness_hash: -error_path_test: ``` -Allocator optimization is accepted only if it removes a real cost or reduces a -meaningful peak-memory risk while preserving ownership/error semantics. +Static source inventory can prove call structure, ownership and requested-size +bounds. It cannot prove fragmentation, allocator latency or whole-system speedup. From ab204d822d8c7f99940d7d3e46dbf91af9b03980 Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:13:38 +0200 Subject: [PATCH 145/156] Phase 5: classify metadata commit IOP verify lifetime --- docs/ALLOCATION_LIFETIME_AUDIT.md | 46 +++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/docs/ALLOCATION_LIFETIME_AUDIT.md b/docs/ALLOCATION_LIFETIME_AUDIT.md index 54c063b9..ce9c7183 100644 --- a/docs/ALLOCATION_LIFETIME_AUDIT.md +++ b/docs/ALLOCATION_LIFETIME_AUDIT.md @@ -46,6 +46,11 @@ It deliberately does not guess ThreadMan implementation-owned backing memory from `CreateThread`/`CreateSema` call counts. Those costs remain part of the runtime IOP-memory gate. +CI #753 records 25 direct allocation calls and 72 direct frees (97 events total), +including 22 IOP SysMem events that the earlier libc-only scanner could not see. +This is a direct-call source inventory, not a claim that allocator-internal or +ThreadMan-owned memory has been measured. + ## Lifetime classes ```text @@ -230,7 +235,7 @@ serialization tests before hardware use. # IOP SysMem -The custom `hdl_stream` service has three direct SysMem ownership classes. +The custom `hdl_stream` service has four direct SysMem ownership classes. ## Stream object @@ -294,6 +299,39 @@ This is producer representation state, not generic scratch. Any compaction must preserve fragmented-file correctness and the sequential cursor behaviour that avoids rescanning thousands of fragments for every 64 KiB block. +## Metadata-commit read-back buffer + +`commit_metadata()` allocates one fixed-size SysMem buffer: + +```text +verify = AllocSysMemory(ALLOC_FIRST, HDL_STREAM_METADATA_SIZE, NULL) +``` + +Lifetime: only the metadata commit call. + +Producer/consumer chain: + +```text +canonical EE metadata + -> IOP write at HDL_STREAM_METADATA_OFFSET + -> HIOCFLUSH durability barrier + -> read_metadata() into IOP verify + -> memcmp(canonical, verify) + -> FreeSysMemory(verify) +``` + +**POTWIERDZONE:** the buffer participates in mandatory post-flush read-back +verification. It is not retained across transactions and is not allocated in a +payload chunk loop. + +**CURRENT IMPLEMENTATION:** the allocation is only `HDL_STREAM_METADATA_SIZE` +(1024 B in the current HDL metadata contract). + +Priority: **KEEP**. Reusing staging memory or removing this allocation would +couple a tiny cold allocation to the much more important stream/durability +ownership path for at most 1 KiB peak savings. No measured evidence justifies +that risk. + ## IOP memory outside the direct SysMem inventory ThreadMan owns backing memory for resources including: @@ -318,9 +356,11 @@ unmeasured until real hardware records active modules and minimum free IOP RAM. 4. Measure ISO/game catalogue cardinality before changing growth strategy. 5. Leave payload-owning allocations intact until their consumers can use a different representation. -6. Do not introduce global pools/arenas or global 64-byte heap alignment merely +6. Keep the fixed 1 KiB metadata-commit read-back allocation until evidence says + otherwise; correctness/durability value dominates its tiny peak cost. +7. Do not introduce global pools/arenas or global 64-byte heap alignment merely to reduce allocator call counts in source. -7. Do not expand IOP staging before runtime free-memory and latency attribution +8. Do not expand IOP staging before runtime free-memory and latency attribution are available. ## Acceptance record for allocator/lifetime changes From cb23838c905523a83172d5acfc4ce07ace455375 Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:14:42 +0200 Subject: [PATCH 146/156] Phase 5: add independent APAMETA1 reference vector --- tools/forensic_snapshot_reference.py | 170 +++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 tools/forensic_snapshot_reference.py diff --git a/tools/forensic_snapshot_reference.py b/tools/forensic_snapshot_reference.py new file mode 100644 index 00000000..406e4996 --- /dev/null +++ b/tools/forensic_snapshot_reference.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Independent APAMETA1 reference serializer/vector for Phase-5 work. + +This tool intentionally does not import or materialize the runtime C serializer. +It encodes the documented version-1 snapshot contract directly and pins a small +deterministic vector. Future streaming implementations must reproduce the same +bytes before they are eligible for PS2 hardware testing. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import struct +from pathlib import Path + +MAGIC = b"APAMETA1" +VERSION = 1 +HEADER_BYTES = 64 +APA_HEADER_BYTES = 1024 +ENTRY_BYTES = 4 + 32 + APA_HEADER_BYTES +TRAILER_BYTES = 32 + + +def le32(value: int) -> bytes: + return struct.pack(" int: + return (old_next ^ new_next).bit_count() + (old_prev ^ new_prev).bit_count() + + +def serialize(total_sectors: int, map_index: int, confidence: int, + corroborated_count: int, speculative_count: int, + entries: list[dict[str, object]]) -> bytes: + if not entries: + raise ValueError("at least one APAMETA1 entry is required") + + one_or_two = sum( + patch_bit_distance( + int(entry["old_next"]), int(entry["old_prev"]), + int(entry["new_next"]), int(entry["new_prev"]), + ) in (1, 2) + for entry in entries + ) + + image = bytearray(HEADER_BYTES) + image[0:8] = MAGIC + image[8:12] = le32(VERSION) + image[12:16] = le32(total_sectors) + image[16:20] = le32(map_index) + image[20:24] = le32(confidence) + image[24:28] = le32(len(entries)) + image[28:32] = le32(corroborated_count) + image[32:36] = le32(speculative_count) + image[36:40] = le32(one_or_two) + + for entry in entries: + header = bytes(entry["header"]) + if len(header) != APA_HEADER_BYTES: + raise ValueError("APA header must be exactly 1024 bytes") + image += le32(int(entry["lba"])) + image += hashlib.sha256(header).digest() + image += header + + image += hashlib.sha256(image).digest() + return bytes(image) + + +def reference_entries() -> list[dict[str, object]]: + return [ + { + "lba": 0x1000, + "header": bytes(i & 0xFF for i in range(APA_HEADER_BYTES)), + "old_next": 0x100, + "old_prev": 0x80, + "new_next": 0x101, + "new_prev": 0x80, + }, + { + "lba": 0x2000, + "header": bytes((255 - i) & 0xFF for i in range(APA_HEADER_BYTES)), + "old_next": 0x200, + "old_prev": 0x40, + "new_next": 0x202, + "new_prev": 0x41, + }, + ] + + +def build_reference() -> bytes: + return serialize( + total_sectors=0x12345678, + map_index=1, + confidence=88, + corroborated_count=1, + speculative_count=1, + entries=reference_entries(), + ) + + +def selftest() -> dict[str, object]: + image = build_reference() + entry0 = reference_entries()[0] + entry1 = reference_entries()[1] + header0_sha = hashlib.sha256(bytes(entry0["header"])).hexdigest() + header1_sha = hashlib.sha256(bytes(entry1["header"])).hexdigest() + trailer = image[-TRAILER_BYTES:].hex() + image_sha = hashlib.sha256(image).hexdigest() + + expected = { + "bytes": 2216, + "header0_sha256": "785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9", + "header1_sha256": "3af6dbef8362452d2b45ad97deb9e43180fb90aac309860e26e123860cce62a7", + "trailer_sha256": "2654254d3abccac8459893f15c96b9f3e186fdf2abf423bdbb2150997431773b", + "image_sha256": "601ba74fc619738dac19baa2a6cb53054b67803e00b1fccb6bf89c69ef4bab6f", + } + + assert len(image) == expected["bytes"] + assert image[0:8] == MAGIC + assert struct.unpack_from(" int: + parser = argparse.ArgumentParser() + parser.add_argument("--selftest", action="store_true") + parser.add_argument("--json", type=Path, + help="write reference metadata JSON after validation") + parser.add_argument("--binary", type=Path, + help="write the exact reference APAMETA1 image") + args = parser.parse_args() + + result = selftest() + if args.json is not None: + args.json.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", + encoding="utf-8") + if args.binary is not None: + args.binary.write_bytes(build_reference()) + if args.selftest or (args.json is None and args.binary is None): + print("forensic snapshot reference selftest: PASS") + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From d2d3a9e5b2c30483d35aacde9eef94bbeb4e7890 Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:15:00 +0200 Subject: [PATCH 147/156] Phase 5: gate bounded HDDMETA with reference vector --- tools/build_transaction_workspace_experiment.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/build_transaction_workspace_experiment.sh b/tools/build_transaction_workspace_experiment.sh index f06fb387..7f415e14 100644 --- a/tools/build_transaction_workspace_experiment.sh +++ b/tools/build_transaction_workspace_experiment.sh @@ -10,7 +10,9 @@ set -eu # Active bounded-v2 experiment keeps the 64 KiB exact read-back scratch but # removes the two seek RPCs used by v1. It reads exactly the expected bytes and # then requires one extra byte read to report EOF, preserving truncation and -# trailing-data detection. +# trailing-data detection. An independent APAMETA1 reference vector is validated +# before materializing the runtime experiment so future streaming work has a +# byte-exact oracle that does not share the runtime implementation. ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) BACKUP=$(mktemp -d) TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" @@ -31,6 +33,9 @@ restore_sources() { } trap restore_sources EXIT HUP INT TERM +cd "$ROOT" +python3 tools/forensic_snapshot_reference.py --selftest + cp "$TRANSACTION" "$BACKUP/transaction.inc" cp "$SOURCE_UI" "$BACKUP/source_ui.inc" cp "$FORENSIC_SNAPSHOT" "$BACKUP/forensic_snapshot.c" @@ -86,9 +91,10 @@ forensic_snapshot_verify_chunk_bytes: "65536" forensic_snapshot_verify_policy: "exact-byte-compare" forensic_snapshot_size_check: "exact-bytes-plus-eof-read" forensic_snapshot_seek_rpcs_per_verify: "0" +forensic_snapshot_reference_format: "APAMETA1" +forensic_snapshot_reference_image_sha256: "601ba74fc619738dac19baa2a6cb53054b67803e00b1fccb6bf89c69ef4bab6f" EOF } -cd "$ROOT" build_variant 0 OFF build_variant 1 ON From 1a00b2bc5319098a022a206feb613be1fdfe1297 Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:18:41 +0200 Subject: [PATCH 148/156] Phase 5: add streaming APAMETA1 experiment --- ...materialize_forensic_snapshot_streaming.py | 491 ++++++++++++++++++ 1 file changed, 491 insertions(+) create mode 100644 tools/materialize_forensic_snapshot_streaming.py diff --git a/tools/materialize_forensic_snapshot_streaming.py b/tools/materialize_forensic_snapshot_streaming.py new file mode 100644 index 00000000..77df719d --- /dev/null +++ b/tools/materialize_forensic_snapshot_streaming.py @@ -0,0 +1,491 @@ +#!/usr/bin/env python3 +"""Materialize the isolated streaming APAMETA1 snapshot experiment. + +This supersedes the full canonical-image + bounded-readback representation only +inside the experiment build. It preserves APAMETA1 bytes, per-header SHA-256, +trailer SHA-256, slot/non-overwrite policy and exact read-back comparison. + +Workspace layout at maximum patch count: + expected/write chunk 64 KiB + actual/read-back chunk 64 KiB + cached entry digests patch_count * 32 B (<= 64 KiB) + +One allocation therefore replaces the full 2.17 MiB canonical image plus verify +scratch. Default runtime source is restored after the experiment build. +""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +MARKER = "streaming APAMETA1 snapshot experiment" + + +class MaterializeError(RuntimeError): + pass + + +def function_span(text: str, name: str) -> tuple[int, int]: + match = re.search(rf"(?:^|\n)(?:static\s+)?(?:[\w\s\*]+?)\b{re.escape(name)}\s*\([^;]*?\)\s*\{{", text) + if match is None: + raise MaterializeError(f"function {name} not found") + start = match.start() + if start < len(text) and text[start] == "\n": + start += 1 + brace = text.find("{", match.start()) + depth = 0 + i = brace + while i < len(text): + if text[i] == "{": + depth += 1 + elif text[i] == "}": + depth -= 1 + if depth == 0: + end = i + 1 + if end < len(text) and text[end] == "\n": + end += 1 + return start, end + i += 1 + raise MaterializeError(f"function {name} has unmatched braces") + + +def replace_function(text: str, name: str, replacement: str) -> str: + start, end = function_span(text, name) + return text[:start] + replacement.rstrip() + "\n" + text[end:] + + +STREAM_HELPERS = r'''/* streaming APAMETA1 snapshot experiment. + * Keep the on-disk representation and exact read-back contract while bounding + * the producer/consumer workspace instead of owning a complete serialized image. */ +#define SNAPSHOT_STREAM_CHUNK_BYTES (64u * 1024u) +#define SNAPSHOT_DIGEST_BYTES 32u + +static int snapshot_prepare_stream(const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + unsigned char *digests, + unsigned int *image_size_out, + unsigned int *one_or_two_out) +{ + unsigned int one_or_two = 0; + unsigned int i; + + if (plan->patch_count == 0 || plan->patch_count > APA_FORENSIC_MAX_PATCHES) + return fail_snapshot(FORENSIC_SNAPSHOT_INVALID_ARGUMENT, + "validate forensic patch count"); + if (plan->patch_count > + (0xffffffffu - SNAPSHOT_HEADER_BYTES - SNAPSHOT_TRAILER_BYTES) / + SNAPSHOT_ENTRY_BYTES) + return fail_snapshot(FORENSIC_SNAPSHOT_INVALID_ARGUMENT, + "validate HDDMETA image size"); + + for (i = 0; i < plan->patch_count; i++) { + const apa_forensic_patch_t *patch = &plan->patches[i]; + const apa_forensic_node_t *node; + unsigned int distance; + + if (patch->node_index >= result->node_count) + return fail_snapshot(FORENSIC_SNAPSHOT_INVALID_ARGUMENT, + "resolve patch node for HDDMETA"); + node = &result->nodes[patch->node_index]; + if (node->lba != patch->lba) + return fail_snapshot(FORENSIC_SNAPSHOT_INVALID_ARGUMENT, + "match patch LBA to scanned node"); + sha256_buffer(node->header, APA_HEADER_SIZE, + digests + i * SNAPSHOT_DIGEST_BYTES); + distance = apa_forensic_patch_bit_distance(patch); + if (distance == 1u || distance == 2u) + one_or_two++; + } + + *image_size_out = SNAPSHOT_HEADER_BYTES + + plan->patch_count * SNAPSHOT_ENTRY_BYTES + + SNAPSHOT_TRAILER_BYTES; + *one_or_two_out = one_or_two; + return 0; +} + +static void snapshot_build_header(const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + unsigned int one_or_two, + unsigned char header[SNAPSHOT_HEADER_BYTES]) +{ + memset(header, 0, SNAPSHOT_HEADER_BYTES); + memcpy(header, snapshot_magic, sizeof(snapshot_magic)); + write_le32_snapshot(header + 8, FORENSIC_SNAPSHOT_VERSION); + write_le32_snapshot(header + 12, result->total_sectors); + write_le32_snapshot(header + 16, plan->map_index); + write_le32_snapshot(header + 20, plan->confidence); + write_le32_snapshot(header + 24, plan->patch_count); + write_le32_snapshot(header + 28, plan->corroborated_count); + write_le32_snapshot(header + 32, plan->speculative_count); + write_le32_snapshot(header + 36, one_or_two); +} + +static void snapshot_build_entry(const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + const unsigned char *digests, + unsigned int index, + unsigned char entry[SNAPSHOT_ENTRY_BYTES]) +{ + const apa_forensic_patch_t *patch = &plan->patches[index]; + const apa_forensic_node_t *node = &result->nodes[patch->node_index]; + + write_le32_snapshot(entry, patch->lba); + memcpy(entry + 4, digests + index * SNAPSHOT_DIGEST_BYTES, + SNAPSHOT_DIGEST_BYTES); + memcpy(entry + 36, node->header, APA_HEADER_SIZE); +} + +static void snapshot_compute_trailer(const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + const unsigned char *digests, + unsigned int one_or_two, + unsigned char trailer[SNAPSHOT_TRAILER_BYTES]) +{ + sha256_context_t context; + unsigned char header[SNAPSHOT_HEADER_BYTES]; + unsigned char lba[4]; + unsigned int i; + + snapshot_build_header(result, plan, one_or_two, header); + sha256_init(&context); + sha256_update(&context, header, sizeof(header)); + for (i = 0; i < plan->patch_count; i++) { + const apa_forensic_patch_t *patch = &plan->patches[i]; + const apa_forensic_node_t *node = &result->nodes[patch->node_index]; + + write_le32_snapshot(lba, patch->lba); + sha256_update(&context, lba, sizeof(lba)); + sha256_update(&context, digests + i * SNAPSHOT_DIGEST_BYTES, + SNAPSHOT_DIGEST_BYTES); + sha256_update(&context, node->header, APA_HEADER_SIZE); + } + sha256_final(&context, trailer); +} + +static void snapshot_fill_range(const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + const unsigned char *digests, + unsigned int one_or_two, + const unsigned char trailer[SNAPSHOT_TRAILER_BYTES], + unsigned int offset, + unsigned char *destination, + unsigned int bytes) +{ + unsigned char header[SNAPSHOT_HEADER_BYTES]; + unsigned char entry[SNAPSHOT_ENTRY_BYTES]; + unsigned int trailer_offset = SNAPSHOT_HEADER_BYTES + + plan->patch_count * SNAPSHOT_ENTRY_BYTES; + unsigned int done = 0; + + snapshot_build_header(result, plan, one_or_two, header); + while (done < bytes) { + unsigned int position = offset + done; + unsigned int take; + + if (position < SNAPSHOT_HEADER_BYTES) { + unsigned int inner = position; + take = SNAPSHOT_HEADER_BYTES - inner; + if (take > bytes - done) + take = bytes - done; + memcpy(destination + done, header + inner, take); + } else if (position < trailer_offset) { + unsigned int relative = position - SNAPSHOT_HEADER_BYTES; + unsigned int index = relative / SNAPSHOT_ENTRY_BYTES; + unsigned int inner = relative - index * SNAPSHOT_ENTRY_BYTES; + + snapshot_build_entry(result, plan, digests, index, entry); + take = SNAPSHOT_ENTRY_BYTES - inner; + if (take > bytes - done) + take = bytes - done; + memcpy(destination + done, entry + inner, take); + } else { + unsigned int inner = position - trailer_offset; + take = SNAPSHOT_TRAILER_BYTES - inner; + if (take > bytes - done) + take = bytes - done; + memcpy(destination + done, trailer + inner, take); + } + done += take; + } +} + +static int snapshot_write_exact(int fd, const unsigned char *data, + unsigned int size) +{ + unsigned int done = 0; + + while (done < size) { + int result = fileXioWrite(fd, data + done, (int)(size - done)); + if (result <= 0) + return result < 0 ? result : -1; + done += (unsigned int)result; + } + return 0; +} + +static int snapshot_read_exact(int fd, unsigned char *data, unsigned int size) +{ + unsigned int done = 0; + + while (done < size) { + int result = fileXioRead(fd, data + done, (int)(size - done)); + if (result <= 0) + return result < 0 ? result : -1; + done += (unsigned int)result; + } + return 0; +} + +static int snapshot_write_streamed(const char *path, + const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + const unsigned char *digests, + unsigned int one_or_two, + const unsigned char trailer[SNAPSHOT_TRAILER_BYTES], + unsigned char *chunk, + unsigned int image_size) +{ + unsigned int offset = 0; + int fd = fileXioOpen(path, FIO_O_WRONLY | FIO_O_CREAT | FIO_O_TRUNC, 0666); + + if (fd < 0) + return fd; + while (offset < image_size) { + unsigned int bytes = image_size - offset; + int result; + + if (bytes > SNAPSHOT_STREAM_CHUNK_BYTES) + bytes = SNAPSHOT_STREAM_CHUNK_BYTES; + snapshot_fill_range(result, plan, digests, one_or_two, trailer, + offset, chunk, bytes); + result = snapshot_write_exact(fd, chunk, bytes); + if (result < 0) { + fileXioClose(fd); + return result; + } + offset += bytes; + } + return fileXioClose(fd) < 0 ? -1 : 0; +} + +static int snapshot_matches_streamed(const char *path, + const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + const unsigned char *digests, + unsigned int one_or_two, + const unsigned char trailer[SNAPSHOT_TRAILER_BYTES], + unsigned char *expected, + unsigned char *actual, + unsigned int image_size) +{ + unsigned int offset = 0; + int fd = fileXioOpen(path, FIO_O_RDONLY, 0); + + if (fd < 0) + return fd; + while (offset < image_size) { + unsigned int bytes = image_size - offset; + int result_code; + + if (bytes > SNAPSHOT_STREAM_CHUNK_BYTES) + bytes = SNAPSHOT_STREAM_CHUNK_BYTES; + snapshot_fill_range(result, plan, digests, one_or_two, trailer, + offset, expected, bytes); + result_code = snapshot_read_exact(fd, actual, bytes); + if (result_code < 0) { + fileXioClose(fd); + return result_code; + } + if (memcmp(expected, actual, bytes) != 0) { + fileXioClose(fd); + return 0; + } + offset += bytes; + } + { + int extra = fileXioRead(fd, actual, 1); + fileXioClose(fd); + if (extra < 0) + return extra; + return extra == 0 ? 1 : 0; + } +} +''' + +STREAM_SAVE = r'''int forensic_snapshot_save(unsigned int storage, + const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + char path_out[FORENSIC_SNAPSHOT_PATH_SIZE]) +{ + unsigned char trailer[SNAPSHOT_TRAILER_BYTES]; + unsigned char *workspace = NULL; + unsigned char *expected; + unsigned char *actual; + unsigned char *digests; + unsigned int image_size = 0; + unsigned int one_or_two = 0; + unsigned int digest_bytes; + unsigned int workspace_bytes; + unsigned int slot; + int result_code; + + if (storage >= STORAGE_TARGET_COUNT || result == NULL || plan == NULL || + path_out == NULL) + return fail_snapshot(FORENSIC_SNAPSHOT_INVALID_ARGUMENT, + "validate HDDMETA snapshot arguments"); + if (plan->patch_count == 0 || plan->patch_count > APA_FORENSIC_MAX_PATCHES) + return fail_snapshot(FORENSIC_SNAPSHOT_INVALID_ARGUMENT, + "validate forensic patch count"); + + digest_bytes = plan->patch_count * SNAPSHOT_DIGEST_BYTES; + workspace_bytes = SNAPSHOT_STREAM_CHUNK_BYTES * 2u + digest_bytes; + workspace = malloc(workspace_bytes); + if (workspace == NULL) + return fail_snapshot(FORENSIC_SNAPSHOT_ALLOC_FAILED, + "allocate streaming HDDMETA workspace"); + expected = workspace; + actual = expected + SNAPSHOT_STREAM_CHUNK_BYTES; + digests = actual + SNAPSHOT_STREAM_CHUNK_BYTES; + + disk_status_begin_at("Forensic repair safety snapshot", + "Streaming HDDMETA from every header the plan may touch", + "Forensic evidence / bounded APAMETA1 workspace"); + disk_status_io(DISK_STATUS_SCAN, 0, 0, 1, 3); + result_code = snapshot_prepare_stream(result, plan, digests, &image_size, + &one_or_two); + if (result_code < 0) { + free(workspace); + disk_status_end(); + return result_code; + } + snapshot_compute_trailer(result, plan, digests, one_or_two, trailer); + + for (slot = 0; slot < FORENSIC_SNAPSHOT_SLOT_COUNT; slot++) { + char path[FORENSIC_SNAPSHOT_PATH_SIZE]; + iox_stat_t stat; + int stat_result; + + storage_path(path, sizeof(path), storage, snapshot_names[slot]); + disk_status_phase_at("Selecting non-overwriting HDDMETA slot", path); + disk_status_io(DISK_STATUS_SCAN, 0, 0, 1, 3); + memset(&stat, 0, sizeof(stat)); + stat_result = fileXioGetStat(path, &stat); + if (stat_result >= 0) { + if (stat.size == image_size && + snapshot_matches_streamed(path, result, plan, digests, + one_or_two, trailer, expected, actual, + image_size) == 1) { + strncpy(path_out, path, FORENSIC_SNAPSHOT_PATH_SIZE - 1u); + path_out[FORENSIC_SNAPSHOT_PATH_SIZE - 1u] = '\0'; + free(workspace); + disk_status_end(); + return 0; + } + continue; + } + + disk_status_phase_at("Writing complete pre-repair HDDMETA snapshot", path); + disk_status_io(DISK_STATUS_SCAN, 0, 0, 2, 3); + result_code = snapshot_write_streamed(path, result, plan, digests, + one_or_two, trailer, expected, + image_size); + if (result_code < 0) { + free(workspace); + disk_status_end(); + return fail_snapshot(FORENSIC_SNAPSHOT_WRITE_FAILED, + "write HDDMETA snapshot"); + } + disk_status_phase_at("Reading HDDMETA back and comparing every byte", path); + disk_status_io(DISK_STATUS_VERIFY, 0, 0, 3, 3); + if (snapshot_matches_streamed(path, result, plan, digests, one_or_two, + trailer, expected, actual, image_size) != 1) { + free(workspace); + disk_status_end(); + return fail_snapshot(FORENSIC_SNAPSHOT_VERIFY_FAILED, + "read back/compare HDDMETA snapshot"); + } + + strncpy(path_out, path, FORENSIC_SNAPSHOT_PATH_SIZE - 1u); + path_out[FORENSIC_SNAPSHOT_PATH_SIZE - 1u] = '\0'; + free(workspace); + disk_status_end(); + return 0; + } + + free(workspace); + disk_status_end(); + return fail_snapshot(FORENSIC_SNAPSHOT_NO_SLOT, + "select non-overwriting HDDMETA slot"); +} +''' + + +def materialize(text: str) -> str: + if MARKER in text: + raise MaterializeError("streaming snapshot already materialized") + start, end = function_span(text, "build_snapshot_image") + text = text[:start] + f"/* {MARKER}. */\n" + STREAM_HELPERS + "\n" + text[end:] + text = replace_function(text, "forensic_snapshot_save", STREAM_SAVE) + + if "build_snapshot_image(" in text: + raise MaterializeError("full-image builder survived streaming transform") + if "malloc(image_size)" in text: + raise MaterializeError("full-size snapshot allocation survived") + if text.count("workspace = malloc(workspace_bytes);") != 1: + raise MaterializeError("streaming workspace allocation missing") + if text.count("snapshot_matches_streamed(") != 3: + raise MaterializeError("unexpected streamed compare helper/call count") + return text + + +def selftest() -> None: + fixture = r'''#define SNAPSHOT_HEADER_BYTES 64u +#define SNAPSHOT_ENTRY_BYTES (4u + 32u + APA_HEADER_SIZE) +#define SNAPSHOT_TRAILER_BYTES 32u +static int build_snapshot_image(const void *result, const void *plan, + unsigned char **image_out, + unsigned int *size_out) +{ + unsigned char *image = malloc(1024); + free(image); + return 0; +} +int forensic_snapshot_save(unsigned int storage, const void *result, + const void *plan, char path_out[64]) +{ + unsigned char *image = malloc(1024); + free(image); + return 0; +} +''' + out = materialize(fixture) + assert MARKER in out + assert "build_snapshot_image(" not in out + assert "workspace = malloc(workspace_bytes);" in out + assert "snapshot_compute_trailer" in out + assert "snapshot_write_streamed" in out + assert "snapshot_matches_streamed" in out + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("source", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("forensic snapshot streaming materializer selftest: PASS") + return 0 + if args.source is None: + parser.error("source is required unless --selftest is used") + args.source.write_text(materialize(args.source.read_text(encoding="utf-8")), + encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 21446188f10f779021c4d50dc9a671623a7621a8 Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:19:29 +0200 Subject: [PATCH 149/156] Phase 5: add active streaming APAMETA1 materializer v2 --- ...erialize_forensic_snapshot_streaming_v2.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 tools/materialize_forensic_snapshot_streaming_v2.py diff --git a/tools/materialize_forensic_snapshot_streaming_v2.py b/tools/materialize_forensic_snapshot_streaming_v2.py new file mode 100644 index 00000000..a972a8ef --- /dev/null +++ b/tools/materialize_forensic_snapshot_streaming_v2.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Active wrapper for the isolated streaming APAMETA1 materializer. + +The first prototype used the identifier `result` both for the forensic-result +parameter and for the local write return code in generated `snapshot_write_streamed`. +That prototype is retained as review history; this wrapper materializes it and +renames only the conflicting local state before the generated C is compiled. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import materialize_forensic_snapshot_streaming as v1 + +MARKER = "streaming APAMETA1 snapshot experiment v2 scope fix" + + +def materialize(text: str) -> str: + out = v1.materialize(text) + if MARKER in out: + raise v1.MaterializeError("streaming v2 already materialized") + + start, end = v1.function_span(out, "snapshot_write_streamed") + body = out[start:end] + replacements = [ + (" int result;\n", " int write_result;\n"), + (" result = snapshot_write_exact(fd, chunk, bytes);\n", + " write_result = snapshot_write_exact(fd, chunk, bytes);\n"), + (" if (result < 0) {\n", " if (write_result < 0) {\n"), + (" return result;\n", " return write_result;\n"), + ] + for old, new in replacements: + count = body.count(old) + if count != 1: + raise v1.MaterializeError( + f"streaming v2 scope fix expected one {old!r}, found {count}" + ) + body = body.replace(old, new, 1) + + body = body.replace( + "{\n unsigned int offset = 0;\n", + "{\n /* " + MARKER + ". */\n unsigned int offset = 0;\n", + 1, + ) + out = out[:start] + body + out[end:] + + fixed_start, fixed_end = v1.function_span(out, "snapshot_write_streamed") + fixed = out[fixed_start:fixed_end] + if " int result;\n" in fixed: + raise v1.MaterializeError("conflicting write result identifier survived") + if "int write_result;" not in fixed: + raise v1.MaterializeError("write_result fix missing") + return out + + +def selftest() -> None: + fixture = r'''#define SNAPSHOT_HEADER_BYTES 64u +#define SNAPSHOT_ENTRY_BYTES (4u + 32u + APA_HEADER_SIZE) +#define SNAPSHOT_TRAILER_BYTES 32u +static int build_snapshot_image(const void *result, const void *plan, + unsigned char **image_out, + unsigned int *size_out) +{ + unsigned char *image = malloc(1024); + free(image); + return 0; +} +int forensic_snapshot_save(unsigned int storage, const void *result, + const void *plan, char path_out[64]) +{ + unsigned char *image = malloc(1024); + free(image); + return 0; +} +''' + out = materialize(fixture) + assert MARKER in out + start, end = v1.function_span(out, "snapshot_write_streamed") + body = out[start:end] + assert "int write_result;" in body + assert "write_result = snapshot_write_exact" in body + assert "return write_result;" in body + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("source", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("forensic snapshot streaming v2 materializer selftest: PASS") + return 0 + if args.source is None: + parser.error("source is required unless --selftest is used") + args.source.write_text(materialize(args.source.read_text(encoding="utf-8")), + encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 00637fd7fa771b910b1ac4b872c1b9d780da3df8 Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:20:04 +0200 Subject: [PATCH 150/156] Phase 5: preserve snapshot close semantics in streaming v2 --- .../materialize_forensic_snapshot_streaming_v2.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tools/materialize_forensic_snapshot_streaming_v2.py b/tools/materialize_forensic_snapshot_streaming_v2.py index a972a8ef..92693f86 100644 --- a/tools/materialize_forensic_snapshot_streaming_v2.py +++ b/tools/materialize_forensic_snapshot_streaming_v2.py @@ -3,8 +3,10 @@ The first prototype used the identifier `result` both for the forensic-result parameter and for the local write return code in generated `snapshot_write_streamed`. -That prototype is retained as review history; this wrapper materializes it and -renames only the conflicting local state before the generated C is compiled. +It also made a successful write depend on `fileXioClose()` returning success, +while the current `write_whole_file()` contract ignores close's return value and +relies on mandatory read-back verification. This wrapper fixes only those two +prototype differences before the generated C is compiled. """ from __future__ import annotations @@ -14,7 +16,7 @@ import materialize_forensic_snapshot_streaming as v1 -MARKER = "streaming APAMETA1 snapshot experiment v2 scope fix" +MARKER = "streaming APAMETA1 snapshot experiment v2 scope/close fix" def materialize(text: str) -> str: @@ -30,12 +32,14 @@ def materialize(text: str) -> str: " write_result = snapshot_write_exact(fd, chunk, bytes);\n"), (" if (result < 0) {\n", " if (write_result < 0) {\n"), (" return result;\n", " return write_result;\n"), + (" return fileXioClose(fd) < 0 ? -1 : 0;\n", + " fileXioClose(fd);\n return 0;\n"), ] for old, new in replacements: count = body.count(old) if count != 1: raise v1.MaterializeError( - f"streaming v2 scope fix expected one {old!r}, found {count}" + f"streaming v2 fix expected one {old!r}, found {count}" ) body = body.replace(old, new, 1) @@ -52,6 +56,8 @@ def materialize(text: str) -> str: raise v1.MaterializeError("conflicting write result identifier survived") if "int write_result;" not in fixed: raise v1.MaterializeError("write_result fix missing") + if "return fileXioClose(fd) < 0" in fixed: + raise v1.MaterializeError("close-return semantic change survived") return out @@ -82,6 +88,7 @@ def selftest() -> None: assert "int write_result;" in body assert "write_result = snapshot_write_exact" in body assert "return write_result;" in body + assert "fileXioClose(fd);\n return 0;" in body def main() -> int: From 80ad5df534d51782d263214dc3bb05ded29df5e6 Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:20:24 +0200 Subject: [PATCH 151/156] Phase 5: build streaming APAMETA1 experiment --- .../build_transaction_workspace_experiment.sh | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tools/build_transaction_workspace_experiment.sh b/tools/build_transaction_workspace_experiment.sh index 7f415e14..0b935855 100644 --- a/tools/build_transaction_workspace_experiment.sh +++ b/tools/build_transaction_workspace_experiment.sh @@ -6,13 +6,11 @@ set -eu # workspace v1 CI #724 # workspace v1 + fingerprint malloc CI #739 # bounded HDDMETA read-back v1 CI #749 +# bounded HDDMETA read-back v2 CI #752 / identical runtime CI #757 # -# Active bounded-v2 experiment keeps the 64 KiB exact read-back scratch but -# removes the two seek RPCs used by v1. It reads exactly the expected bytes and -# then requires one extra byte read to report EOF, preserving truncation and -# trailing-data detection. An independent APAMETA1 reference vector is validated -# before materializing the runtime experiment so future streaming work has a -# byte-exact oracle that does not share the runtime implementation. +# Active experiment replaces the full canonical APAMETA1 image with one bounded +# streaming workspace while preserving exact serialized bytes and exact read-back. +# The independent APAMETA1 reference vector is validated before materialization. ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) BACKUP=$(mktemp -d) TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" @@ -35,6 +33,7 @@ trap restore_sources EXIT HUP INT TERM cd "$ROOT" python3 tools/forensic_snapshot_reference.py --selftest +python3 tools/materialize_forensic_snapshot_streaming_v2.py --selftest cp "$TRANSACTION" "$BACKUP/transaction.inc" cp "$SOURCE_UI" "$BACKUP/source_ui.inc" @@ -44,7 +43,7 @@ python3 "$ROOT/tools/materialize_transaction_workspace.py" \ "$TRANSACTION" "$TRANSACTION" python3 "$ROOT/tools/materialize_source_fingerprint_malloc.py" \ "$SOURCE_UI" -python3 "$ROOT/tools/materialize_forensic_snapshot_bounded_verify.py" \ +python3 "$ROOT/tools/materialize_forensic_snapshot_streaming_v2.py" \ "$FORENSIC_SNAPSHOT" python3 "$ROOT/tools/allocation_inventory.py" \ @@ -85,12 +84,14 @@ hdl_transaction_workspace_version: "1" hdl_transaction_workspace_bytes: "65536" hdl_transaction_workspace_alignment: "64" hdl_source_fingerprint_heap_experiment: "malloc" -forensic_snapshot_bounded_verify_enabled: "1" -forensic_snapshot_bounded_verify_version: "2" -forensic_snapshot_verify_chunk_bytes: "65536" +forensic_snapshot_streaming_enabled: "1" +forensic_snapshot_streaming_version: "2" +forensic_snapshot_stream_chunk_bytes: "65536" +forensic_snapshot_actual_chunk_bytes: "65536" +forensic_snapshot_digest_cache_bytes_max: "65536" +forensic_snapshot_workspace_bytes_max: "196608" forensic_snapshot_verify_policy: "exact-byte-compare" forensic_snapshot_size_check: "exact-bytes-plus-eof-read" -forensic_snapshot_seek_rpcs_per_verify: "0" forensic_snapshot_reference_format: "APAMETA1" forensic_snapshot_reference_image_sha256: "601ba74fc619738dac19baa2a6cb53054b67803e00b1fccb6bf89c69ef4bab6f" EOF From 8e1e0974a5a787e2cd24aca30b0fe70fa2c3deb2 Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:21:00 +0200 Subject: [PATCH 152/156] Phase 5: bind streaming APAMETA1 A/B --- tools/transaction_workspace_ab_preflight.py | 116 +++++++++----------- 1 file changed, 54 insertions(+), 62 deletions(-) diff --git a/tools/transaction_workspace_ab_preflight.py b/tools/transaction_workspace_ab_preflight.py index e1956ced..a055d7af 100644 --- a/tools/transaction_workspace_ab_preflight.py +++ b/tools/transaction_workspace_ab_preflight.py @@ -1,15 +1,16 @@ #!/usr/bin/env python3 -"""Validate and bind the active Phase-5 bounded HDDMETA verification v2 A/B. +"""Validate and bind the active Phase-5 streaming APAMETA1 A/B. Frozen references: - Phase-0 baseline: CI #666 - transaction workspace v1: CI #724 - workspace v1 + source-fingerprint malloc: CI #739 - bounded HDDMETA read-back v1: CI #749 +- bounded HDDMETA read-back v2: CI #752 (runtime identity re-proved by CI #757) -V2 keeps the same 64 KiB exact byte-comparison memory model as v1 but replaces -v1's two fileXioLseek size-check RPCs with one final one-byte read that must -report EOF after all expected bytes have been consumed. +The active experiment removes the complete canonical snapshot allocation and +uses one bounded workspace for serialized write chunks, exact read-back chunks +and cached per-entry SHA-256 digests. """ from __future__ import annotations @@ -34,41 +35,22 @@ }, } -FINGERPRINT_MALLOC = { +BOUNDED_V2 = { "OFF": { - "elf_sha256": "97e2a802952ae6f3b46c9fa0148359db8f8b69e22923f8105378f094de59c28b", - "elf_bytes": 632756, - "named_text": 229756, - "instructions": 57488, - "execute_transaction_bytes": 6008, - "execute_transaction_instructions": 1502, - }, - "ON": { - "elf_sha256": "c8da50fe5147c3a24dc2f26d4ab910660bac615bf8e26f48e0bff3a2483f578b", - "elf_bytes": 638132, - "named_text": 232552, - "instructions": 58189, - "execute_transaction_bytes": 6008, - "execute_transaction_instructions": 1502, - }, -} - -BOUNDED_V1 = { - "OFF": { - "elf_sha256": "64bcc758c9a848cb8c2a85284a47e9217693d725004b6a016813f825339a2775", + "elf_sha256": "ecd99a7aee199039146cfa8275d2ecbe360b9b486bb290adb3bd30d86ae10a54", "elf_bytes": 633012, - "section_text": 286789, - "named_text": 230112, - "instructions": 57579, + "section_text": 286757, + "named_text": 230076, + "instructions": 57571, "execute_transaction_bytes": 6008, "execute_transaction_instructions": 1502, }, "ON": { - "elf_sha256": "ad13f89be93575fe63cbb762893e2c4f7ee102b37ffd73794ef91bf224988af7", + "elf_sha256": "3ace7ea8730dc7dd56fe6bea078b2aeedc1e7735c5bcc831e7d7b883f65bdd2f", "elf_bytes": 638516, - "section_text": 290749, - "named_text": 232908, - "instructions": 58278, + "section_text": 290717, + "named_text": 232872, + "instructions": 58270, "execute_transaction_bytes": 6008, "execute_transaction_instructions": 1502, }, @@ -77,9 +59,11 @@ MAX_PATCHES = 2048 SNAPSHOT_ENTRY_BYTES = 4 + 32 + 1024 SNAPSHOT_MAX_BYTES = 64 + MAX_PATCHES * SNAPSHOT_ENTRY_BYTES + 32 -VERIFY_CHUNK_BYTES = 64 * 1024 -BASELINE_MAX_PEAK_BYTES = SNAPSHOT_MAX_BYTES * 2 -EXPERIMENT_MAX_PEAK_BYTES = SNAPSHOT_MAX_BYTES + VERIFY_CHUNK_BYTES +STREAM_CHUNK_BYTES = 64 * 1024 +DIGEST_CACHE_BYTES_MAX = MAX_PATCHES * 32 +STREAM_WORKSPACE_BYTES_MAX = STREAM_CHUNK_BYTES * 2 + DIGEST_CACHE_BYTES_MAX +ORIGINAL_PAIR_PEAK_BYTES = SNAPSHOT_MAX_BYTES * 2 +BOUNDED_V2_PEAK_BYTES = SNAPSHOT_MAX_BYTES + STREAM_CHUNK_BYTES ORDER = ["BASE", "EXP", "EXP", "BASE", "EXP", "BASE", "BASE", "EXP"] @@ -98,22 +82,24 @@ def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) -> dict: return { - "experiment": "forensic-snapshot-bounded-readback-v2", + "experiment": "forensic-snapshot-streaming-apameta1-v2", "profile": profile, "workload": "forensic-HDDMETA-save-existing-slot-and-new-slot-readback", - "bounded_v1_reference": BOUNDED_V1[profile], + "bounded_v2_reference": BOUNDED_V2[profile], "expected_source_change": { "snapshot_format_changed": False, "exact_byte_compare_preserved": True, + "per_header_sha256_preserved": True, + "trailer_sha256_preserved": True, "max_patch_count": MAX_PATCHES, "max_snapshot_bytes": SNAPSHOT_MAX_BYTES, - "verify_chunk_bytes": VERIFY_CHUNK_BYTES, - "baseline_image_plus_verify_peak_bytes_at_max": BASELINE_MAX_PEAK_BYTES, - "experiment_image_plus_verify_peak_bytes_at_max": EXPERIMENT_MAX_PEAK_BYTES, - "peak_reduction_bytes_at_max": BASELINE_MAX_PEAK_BYTES - EXPERIMENT_MAX_PEAK_BYTES, - "bounded_v1_seek_rpcs_per_verify": 2, - "bounded_v2_seek_rpcs_per_verify": 0, - "bounded_v2_final_eof_reads_per_verify": 1, + "stream_chunk_bytes": STREAM_CHUNK_BYTES, + "digest_cache_bytes_max": DIGEST_CACHE_BYTES_MAX, + "stream_workspace_bytes_max": STREAM_WORKSPACE_BYTES_MAX, + "bounded_v2_peak_bytes_at_max": BOUNDED_V2_PEAK_BYTES, + "stream_peak_bytes_at_max": STREAM_WORKSPACE_BYTES_MAX, + "peak_reduction_vs_bounded_v2": BOUNDED_V2_PEAK_BYTES - STREAM_WORKSPACE_BYTES_MAX, + "peak_reduction_vs_original_full_pair": ORIGINAL_PAIR_PEAK_BYTES - STREAM_WORKSPACE_BYTES_MAX, "iop_binary_change": False, }, "baseline": baseline, @@ -175,43 +161,49 @@ def main() -> int: validate_frozen("OFF", baseline_off, baseline_irx_off) validate_frozen("ON", baseline_on, baseline_irx_on) if experiment_irx_off["sha256"] != baseline_irx_off["sha256"]: - raise SystemExit("PROFILE OFF forensic v2 experiment changed hdl_stream.irx") + raise SystemExit("PROFILE OFF streaming experiment changed hdl_stream.irx") if experiment_irx_on["sha256"] != baseline_irx_on["sha256"]: - raise SystemExit("PROFILE ON forensic v2 experiment changed hdl_stream.irx") - if experiment_off["sha256"] == BOUNDED_V1["OFF"]["elf_sha256"]: - raise SystemExit("PROFILE OFF bounded v2 experiment did not change the EE ELF from bounded v1") - if experiment_on["sha256"] == BOUNDED_V1["ON"]["elf_sha256"]: - raise SystemExit("PROFILE ON bounded v2 experiment did not change the EE ELF from bounded v1") + raise SystemExit("PROFILE ON streaming experiment changed hdl_stream.irx") + if experiment_off["sha256"] == BOUNDED_V2["OFF"]["elf_sha256"]: + raise SystemExit("PROFILE OFF streaming experiment did not change the EE ELF from bounded v2") + if experiment_on["sha256"] == BOUNDED_V2["ON"]["elf_sha256"]: + raise SystemExit("PROFILE ON streaming experiment did not change the EE ELF from bounded v2") identity = { - "experiment": "forensic-snapshot-bounded-readback-v2", + "experiment": "forensic-snapshot-streaming-apameta1-v2", "project_git_sha": args.project_git_sha, "frozen_phase0_commit": "7875b14d837d6332f5edc37f1c12a55527d7dd87", "workspace_v1_frozen_ci": 724, "fingerprint_malloc_frozen_ci": 739, "storage_scratch_natural_rejected_ci": 743, "bounded_readback_v1_frozen_ci": 749, - "bounded_v1_reference": BOUNDED_V1, + "bounded_readback_v2_frozen_ci": 752, + "bounded_readback_v2_identity_reproved_ci": 757, + "bounded_v2_reference": BOUNDED_V2, + "reference_vector": { + "format": "APAMETA1", + "bytes": 2216, + "image_sha256": "601ba74fc619738dac19baa2a6cb53054b67803e00b1fccb6bf89c69ef4bab6f", + }, "ps2sdk_commit": "b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b", "toolchain": "mips64r5900el-ps2-elf GCC 15.2.0", "memory_model": { "max_patch_count": MAX_PATCHES, "max_snapshot_bytes": SNAPSHOT_MAX_BYTES, - "verify_chunk_bytes": VERIFY_CHUNK_BYTES, - "baseline_image_plus_verify_peak_bytes": BASELINE_MAX_PEAK_BYTES, - "experiment_image_plus_verify_peak_bytes": EXPERIMENT_MAX_PEAK_BYTES, - "peak_reduction_bytes": BASELINE_MAX_PEAK_BYTES - EXPERIMENT_MAX_PEAK_BYTES, - }, - "io_model": { - "bounded_v1_seek_rpcs_per_verify": 2, - "bounded_v2_seek_rpcs_per_verify": 0, - "bounded_v2_final_eof_reads_per_verify": 1, - "short_read_handling_preserved": True, + "stream_chunk_bytes": STREAM_CHUNK_BYTES, + "digest_cache_bytes_max": DIGEST_CACHE_BYTES_MAX, + "stream_workspace_bytes_max": STREAM_WORKSPACE_BYTES_MAX, + "original_full_pair_peak_bytes": ORIGINAL_PAIR_PEAK_BYTES, + "bounded_v2_peak_bytes": BOUNDED_V2_PEAK_BYTES, + "peak_reduction_vs_bounded_v2": BOUNDED_V2_PEAK_BYTES - STREAM_WORKSPACE_BYTES_MAX, + "peak_reduction_vs_original_full_pair": ORIGINAL_PAIR_PEAK_BYTES - STREAM_WORKSPACE_BYTES_MAX, }, "correctness_contract": { "on_disk_format_changed": False, "slot_policy_changed": False, "overwrite_policy_changed": False, + "per_header_sha256_preserved": True, + "trailer_sha256_preserved": True, "truncation_detection_preserved": True, "trailing_data_detection_preserved": True, "exact_byte_compare_preserved": True, From fff3b514951cf6a2ea5bc043ba12759666bbf509 Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:23:59 +0200 Subject: [PATCH 153/156] Phase 5: isolate streaming snapshot from repair screen --- ...erialize_forensic_snapshot_streaming_v3.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 tools/materialize_forensic_snapshot_streaming_v3.py diff --git a/tools/materialize_forensic_snapshot_streaming_v3.py b/tools/materialize_forensic_snapshot_streaming_v3.py new file mode 100644 index 00000000..e4211cf0 --- /dev/null +++ b/tools/materialize_forensic_snapshot_streaming_v3.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Streaming APAMETA1 v3: keep the cold serializer out of repair_plan_screen. + +V2 proved the bounded memory model but LTO inlined part of the expanded forensic +snapshot path into the repair UI controller. V3 changes only code placement: +`forensic_snapshot_save()` is explicitly cold and noinline. Data representation, +workspace ownership, fileXio sequencing and verification semantics are identical +to streaming v2. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import materialize_forensic_snapshot_streaming_v2 as v2 + +MARKER = "streaming APAMETA1 v3 cold noinline boundary" + + +def materialize(text: str) -> str: + out = v2.materialize(text) + if MARKER in out: + raise v2.v1.MaterializeError("streaming v3 already materialized") + + start, end = v2.v1.function_span(out, "forensic_snapshot_save") + body = out[start:end] + old = "int forensic_snapshot_save(unsigned int storage,\n" + new = ( + "/* " + MARKER + ". */\n" + "__attribute__((noinline, cold))\n" + "int forensic_snapshot_save(unsigned int storage,\n" + ) + if body.count(old) != 1: + raise v2.v1.MaterializeError("forensic_snapshot_save signature mismatch") + body = body.replace(old, new, 1) + out = out[:start] + body + out[end:] + + fixed_start, fixed_end = v2.v1.function_span(out, "forensic_snapshot_save") + fixed = out[fixed_start:fixed_end] + if "__attribute__((noinline, cold))" not in fixed: + raise v2.v1.MaterializeError("cold/noinline boundary missing") + return out + + +def selftest() -> None: + fixture = r'''#define SNAPSHOT_HEADER_BYTES 64u +#define SNAPSHOT_ENTRY_BYTES (4u + 32u + APA_HEADER_SIZE) +#define SNAPSHOT_TRAILER_BYTES 32u +static int build_snapshot_image(const void *result, const void *plan, + unsigned char **image_out, + unsigned int *size_out) +{ + unsigned char *image = malloc(1024); + free(image); + return 0; +} +int forensic_snapshot_save(unsigned int storage, const void *result, + const void *plan, char path_out[64]) +{ + unsigned char *image = malloc(1024); + free(image); + return 0; +} +''' + out = materialize(fixture) + assert MARKER in out + start, end = v2.v1.function_span(out, "forensic_snapshot_save") + body = out[start:end] + assert "__attribute__((noinline, cold))" in body + assert "workspace = malloc(workspace_bytes);" in body + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("source", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("forensic snapshot streaming v3 materializer selftest: PASS") + return 0 + if args.source is None: + parser.error("source is required unless --selftest is used") + args.source.write_text(materialize(args.source.read_text(encoding="utf-8")), + encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From ab77c9fccb978a265a6010f83fc663e8e2259d6d Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:24:21 +0200 Subject: [PATCH 154/156] Phase 5: build cold-isolated streaming APAMETA1 v3 --- tools/build_transaction_workspace_experiment.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/build_transaction_workspace_experiment.sh b/tools/build_transaction_workspace_experiment.sh index 0b935855..7a75299b 100644 --- a/tools/build_transaction_workspace_experiment.sh +++ b/tools/build_transaction_workspace_experiment.sh @@ -7,10 +7,11 @@ set -eu # workspace v1 + fingerprint malloc CI #739 # bounded HDDMETA read-back v1 CI #749 # bounded HDDMETA read-back v2 CI #752 / identical runtime CI #757 +# streaming APAMETA1 v2 CI #762 # -# Active experiment replaces the full canonical APAMETA1 image with one bounded -# streaming workspace while preserving exact serialized bytes and exact read-back. -# The independent APAMETA1 reference vector is validated before materialization. +# Active v3 keeps streaming-v2 dataflow/memory semantics but forces +# forensic_snapshot_save() to stay cold and noinline so LTO cannot inflate the +# repair-plan UI controller with snapshot serialization machinery. ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) BACKUP=$(mktemp -d) TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" @@ -33,7 +34,7 @@ trap restore_sources EXIT HUP INT TERM cd "$ROOT" python3 tools/forensic_snapshot_reference.py --selftest -python3 tools/materialize_forensic_snapshot_streaming_v2.py --selftest +python3 tools/materialize_forensic_snapshot_streaming_v3.py --selftest cp "$TRANSACTION" "$BACKUP/transaction.inc" cp "$SOURCE_UI" "$BACKUP/source_ui.inc" @@ -43,7 +44,7 @@ python3 "$ROOT/tools/materialize_transaction_workspace.py" \ "$TRANSACTION" "$TRANSACTION" python3 "$ROOT/tools/materialize_source_fingerprint_malloc.py" \ "$SOURCE_UI" -python3 "$ROOT/tools/materialize_forensic_snapshot_streaming_v2.py" \ +python3 "$ROOT/tools/materialize_forensic_snapshot_streaming_v3.py" \ "$FORENSIC_SNAPSHOT" python3 "$ROOT/tools/allocation_inventory.py" \ @@ -85,7 +86,8 @@ hdl_transaction_workspace_bytes: "65536" hdl_transaction_workspace_alignment: "64" hdl_source_fingerprint_heap_experiment: "malloc" forensic_snapshot_streaming_enabled: "1" -forensic_snapshot_streaming_version: "2" +forensic_snapshot_streaming_version: "3" +forensic_snapshot_cold_noinline: "1" forensic_snapshot_stream_chunk_bytes: "65536" forensic_snapshot_actual_chunk_bytes: "65536" forensic_snapshot_digest_cache_bytes_max: "65536" From bf22db0d2da3b786d88379521d0708733c6878d3 Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:24:52 +0200 Subject: [PATCH 155/156] Phase 5: bind cold-isolated streaming APAMETA1 v3 --- tools/transaction_workspace_ab_preflight.py | 126 ++++++++------------ 1 file changed, 52 insertions(+), 74 deletions(-) diff --git a/tools/transaction_workspace_ab_preflight.py b/tools/transaction_workspace_ab_preflight.py index a055d7af..cc28e480 100644 --- a/tools/transaction_workspace_ab_preflight.py +++ b/tools/transaction_workspace_ab_preflight.py @@ -1,16 +1,10 @@ #!/usr/bin/env python3 -"""Validate and bind the active Phase-5 streaming APAMETA1 A/B. - -Frozen references: -- Phase-0 baseline: CI #666 -- transaction workspace v1: CI #724 -- workspace v1 + source-fingerprint malloc: CI #739 -- bounded HDDMETA read-back v1: CI #749 -- bounded HDDMETA read-back v2: CI #752 (runtime identity re-proved by CI #757) - -The active experiment removes the complete canonical snapshot allocation and -uses one bounded workspace for serialized write chunks, exact read-back chunks -and cached per-entry SHA-256 digests. +"""Bind the active cold-isolated streaming APAMETA1 v3 experiment. + +V3 changes only code placement relative to streaming v2 (CI #762): the exported +forensic snapshot save boundary is `cold,noinline` so LTO cannot inflate the +repair-plan UI controller. Streaming representation, fileXio sequencing, +workspace ownership and exact read-back semantics remain unchanged. """ from __future__ import annotations @@ -35,39 +29,42 @@ }, } -BOUNDED_V2 = { +STREAM_V2 = { "OFF": { - "elf_sha256": "ecd99a7aee199039146cfa8275d2ecbe360b9b486bb290adb3bd30d86ae10a54", - "elf_bytes": 633012, - "section_text": 286757, - "named_text": 230076, - "instructions": 57571, + "elf_sha256": "161b68d578e4ac0cbe1a37b34259d630f22822f43973c8462beb169fd494e223", + "elf_bytes": 634420, + "section_text": 288077, + "named_text": 231436, + "instructions": 57911, + "repair_plan_screen_bytes": 6916, + "repair_plan_screen_instructions": 1730, "execute_transaction_bytes": 6008, "execute_transaction_instructions": 1502, }, "ON": { - "elf_sha256": "3ace7ea8730dc7dd56fe6bea078b2aeedc1e7735c5bcc831e7d7b883f65bdd2f", - "elf_bytes": 638516, - "section_text": 290717, - "named_text": 232872, - "instructions": 58270, + "elf_sha256": "08e1694c9fafef6db0f09fc1e696baed3bb5518c675914ca48515e9a8ba10898", + "elf_bytes": 639924, + "section_text": 292037, + "named_text": 234232, + "instructions": 58610, + "repair_plan_screen_bytes": 6916, + "repair_plan_screen_instructions": 1730, "execute_transaction_bytes": 6008, "execute_transaction_instructions": 1502, }, } MAX_PATCHES = 2048 -SNAPSHOT_ENTRY_BYTES = 4 + 32 + 1024 -SNAPSHOT_MAX_BYTES = 64 + MAX_PATCHES * SNAPSHOT_ENTRY_BYTES + 32 +SNAPSHOT_MAX_BYTES = 64 + MAX_PATCHES * (4 + 32 + 1024) + 32 STREAM_CHUNK_BYTES = 64 * 1024 DIGEST_CACHE_BYTES_MAX = MAX_PATCHES * 32 STREAM_WORKSPACE_BYTES_MAX = STREAM_CHUNK_BYTES * 2 + DIGEST_CACHE_BYTES_MAX -ORIGINAL_PAIR_PEAK_BYTES = SNAPSHOT_MAX_BYTES * 2 BOUNDED_V2_PEAK_BYTES = SNAPSHOT_MAX_BYTES + STREAM_CHUNK_BYTES +ORIGINAL_PAIR_PEAK_BYTES = SNAPSHOT_MAX_BYTES * 2 ORDER = ["BASE", "EXP", "EXP", "BASE", "EXP", "BASE", "BASE", "EXP"] -def info(path: Path) -> dict[str, object]: +def file_info(path: Path) -> dict[str, object]: data = path.read_bytes() return {"path": path.name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()} @@ -77,27 +74,21 @@ def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) if elf["sha256"] != expected["elf_sha256"] or elf["bytes"] != expected["elf_bytes"]: raise SystemExit(f"{label} baseline ELF is not the frozen Phase-0 binary") if irx["sha256"] != expected["irx_sha256"] or irx["bytes"] != expected["irx_bytes"]: - raise SystemExit(f"{label} baseline IRX is not the frozen Phase-0 binary") + raise SystemExit(f"{label} baseline IRX is not frozen Phase-0 identity") def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) -> dict: return { - "experiment": "forensic-snapshot-streaming-apameta1-v2", + "experiment": "forensic-snapshot-streaming-apameta1-v3-cold", "profile": profile, "workload": "forensic-HDDMETA-save-existing-slot-and-new-slot-readback", - "bounded_v2_reference": BOUNDED_V2[profile], + "stream_v2_reference": STREAM_V2[profile], "expected_source_change": { + "dataflow_changed_from_stream_v2": False, + "cold_noinline_boundary_added": True, "snapshot_format_changed": False, "exact_byte_compare_preserved": True, - "per_header_sha256_preserved": True, - "trailer_sha256_preserved": True, - "max_patch_count": MAX_PATCHES, - "max_snapshot_bytes": SNAPSHOT_MAX_BYTES, - "stream_chunk_bytes": STREAM_CHUNK_BYTES, - "digest_cache_bytes_max": DIGEST_CACHE_BYTES_MAX, "stream_workspace_bytes_max": STREAM_WORKSPACE_BYTES_MAX, - "bounded_v2_peak_bytes_at_max": BOUNDED_V2_PEAK_BYTES, - "stream_peak_bytes_at_max": STREAM_WORKSPACE_BYTES_MAX, "peak_reduction_vs_bounded_v2": BOUNDED_V2_PEAK_BYTES - STREAM_WORKSPACE_BYTES_MAX, "peak_reduction_vs_original_full_pair": ORIGINAL_PAIR_PEAK_BYTES - STREAM_WORKSPACE_BYTES_MAX, "iop_binary_change": False, @@ -118,8 +109,6 @@ def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) - "variant": variant, "patch_count": None, "snapshot_bytes": None, - "existing_slot_match": None, - "new_slot_write_readback_match": None, "elapsed_us": None, "correctness_hash": None, "result": None, @@ -149,55 +138,44 @@ def main() -> int: parser.add_argument("--profile-on-template", type=Path, default=Path("TRANSACTION_WORKSPACE_AB_PROFILE_ON_TEMPLATE.json")) args = parser.parse_args() - baseline_off = info(args.baseline_off) - baseline_on = info(args.baseline_on) - baseline_irx_off = info(args.baseline_irx_off) - baseline_irx_on = info(args.baseline_irx_on) - experiment_off = info(args.experiment_off) - experiment_on = info(args.experiment_on) - experiment_irx_off = info(args.experiment_irx_off) - experiment_irx_on = info(args.experiment_irx_on) - - validate_frozen("OFF", baseline_off, baseline_irx_off) - validate_frozen("ON", baseline_on, baseline_irx_on) - if experiment_irx_off["sha256"] != baseline_irx_off["sha256"]: - raise SystemExit("PROFILE OFF streaming experiment changed hdl_stream.irx") - if experiment_irx_on["sha256"] != baseline_irx_on["sha256"]: - raise SystemExit("PROFILE ON streaming experiment changed hdl_stream.irx") - if experiment_off["sha256"] == BOUNDED_V2["OFF"]["elf_sha256"]: - raise SystemExit("PROFILE OFF streaming experiment did not change the EE ELF from bounded v2") - if experiment_on["sha256"] == BOUNDED_V2["ON"]["elf_sha256"]: - raise SystemExit("PROFILE ON streaming experiment did not change the EE ELF from bounded v2") + baseline = {"OFF": file_info(args.baseline_off), "ON": file_info(args.baseline_on)} + irx = {"OFF": file_info(args.baseline_irx_off), "ON": file_info(args.baseline_irx_on)} + experiment = {"OFF": file_info(args.experiment_off), "ON": file_info(args.experiment_on)} + experiment_irx = {"OFF": file_info(args.experiment_irx_off), "ON": file_info(args.experiment_irx_on)} + + for label in ("OFF", "ON"): + validate_frozen(label, baseline[label], irx[label]) + if experiment_irx[label]["sha256"] != irx[label]["sha256"]: + raise SystemExit(f"PROFILE {label} streaming v3 changed hdl_stream.irx") + if experiment[label]["sha256"] == STREAM_V2[label]["elf_sha256"]: + raise SystemExit(f"PROFILE {label} cold boundary did not change EE ELF from streaming v2") identity = { - "experiment": "forensic-snapshot-streaming-apameta1-v2", + "experiment": "forensic-snapshot-streaming-apameta1-v3-cold", "project_git_sha": args.project_git_sha, "frozen_phase0_commit": "7875b14d837d6332f5edc37f1c12a55527d7dd87", - "workspace_v1_frozen_ci": 724, - "fingerprint_malloc_frozen_ci": 739, - "storage_scratch_natural_rejected_ci": 743, - "bounded_readback_v1_frozen_ci": 749, "bounded_readback_v2_frozen_ci": 752, - "bounded_readback_v2_identity_reproved_ci": 757, - "bounded_v2_reference": BOUNDED_V2, + "streaming_v2_frozen_ci": 762, + "stream_v2_reference": STREAM_V2, "reference_vector": { "format": "APAMETA1", "bytes": 2216, "image_sha256": "601ba74fc619738dac19baa2a6cb53054b67803e00b1fccb6bf89c69ef4bab6f", }, - "ps2sdk_commit": "b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b", - "toolchain": "mips64r5900el-ps2-elf GCC 15.2.0", "memory_model": { - "max_patch_count": MAX_PATCHES, "max_snapshot_bytes": SNAPSHOT_MAX_BYTES, "stream_chunk_bytes": STREAM_CHUNK_BYTES, "digest_cache_bytes_max": DIGEST_CACHE_BYTES_MAX, "stream_workspace_bytes_max": STREAM_WORKSPACE_BYTES_MAX, - "original_full_pair_peak_bytes": ORIGINAL_PAIR_PEAK_BYTES, "bounded_v2_peak_bytes": BOUNDED_V2_PEAK_BYTES, + "original_full_pair_peak_bytes": ORIGINAL_PAIR_PEAK_BYTES, "peak_reduction_vs_bounded_v2": BOUNDED_V2_PEAK_BYTES - STREAM_WORKSPACE_BYTES_MAX, "peak_reduction_vs_original_full_pair": ORIGINAL_PAIR_PEAK_BYTES - STREAM_WORKSPACE_BYTES_MAX, }, + "code_placement": { + "forensic_snapshot_save": "cold,noinline", + "reason": "prevent streaming serializer growth from inflating repair_plan_screen through LTO", + }, "correctness_contract": { "on_disk_format_changed": False, "slot_policy_changed": False, @@ -208,12 +186,12 @@ def main() -> int: "trailing_data_detection_preserved": True, "exact_byte_compare_preserved": True, }, - "PROFILE_OFF": {"baseline_elf": baseline_off, "experiment_elf": experiment_off, "hdl_stream_irx": baseline_irx_off}, - "PROFILE_ON": {"baseline_elf": baseline_on, "experiment_elf": experiment_on, "hdl_stream_irx": baseline_irx_on}, + "PROFILE_OFF": {"baseline_elf": baseline["OFF"], "experiment_elf": experiment["OFF"], "hdl_stream_irx": irx["OFF"]}, + "PROFILE_ON": {"baseline_elf": baseline["ON"], "experiment_elf": experiment["ON"], "hdl_stream_irx": irx["ON"]}, } args.identity_output.write_text(json.dumps(identity, indent=2, sort_keys=True) + "\n", encoding="utf-8") - args.profile_off_template.write_text(json.dumps(sample_template("OFF", baseline_off, experiment_off, baseline_irx_off), indent=2, sort_keys=True) + "\n", encoding="utf-8") - args.profile_on_template.write_text(json.dumps(sample_template("ON", baseline_on, experiment_on, baseline_irx_on), indent=2, sort_keys=True) + "\n", encoding="utf-8") + args.profile_off_template.write_text(json.dumps(sample_template("OFF", baseline["OFF"], experiment["OFF"], irx["OFF"]), indent=2, sort_keys=True) + "\n", encoding="utf-8") + args.profile_on_template.write_text(json.dumps(sample_template("ON", baseline["ON"], experiment["ON"], irx["ON"]), indent=2, sort_keys=True) + "\n", encoding="utf-8") return 0 From a334b8131b63dd97817c4ac05d97ae5a96116936 Mon Sep 17 00:00:00 2001 From: Hifu Date: Sat, 5 Sep 2026 11:27:31 +0200 Subject: [PATCH 156/156] Phase 5: fix streaming v3 attribute selftest --- ...erialize_forensic_snapshot_streaming_v3.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tools/materialize_forensic_snapshot_streaming_v3.py b/tools/materialize_forensic_snapshot_streaming_v3.py index e4211cf0..26a5c88c 100644 --- a/tools/materialize_forensic_snapshot_streaming_v3.py +++ b/tools/materialize_forensic_snapshot_streaming_v3.py @@ -16,6 +16,10 @@ import materialize_forensic_snapshot_streaming_v2 as v2 MARKER = "streaming APAMETA1 v3 cold noinline boundary" +ATTRIBUTE_SIGNATURE = ( + "__attribute__((noinline, cold))\n" + "int forensic_snapshot_save(" +) def materialize(text: str) -> str: @@ -36,10 +40,12 @@ def materialize(text: str) -> str: body = body.replace(old, new, 1) out = out[:start] + body + out[end:] - fixed_start, fixed_end = v2.v1.function_span(out, "forensic_snapshot_save") - fixed = out[fixed_start:fixed_end] - if "__attribute__((noinline, cold))" not in fixed: - raise v2.v1.MaterializeError("cold/noinline boundary missing") + # function_span() intentionally starts at the function declaration. Once the + # attribute is added on the preceding line it is therefore outside a second + # span lookup. Validate the generated source directly instead of treating + # that parser behaviour as a missing attribute. + if out.count(ATTRIBUTE_SIGNATURE) != 1: + raise v2.v1.MaterializeError("cold/noinline boundary missing or duplicated") return out @@ -65,10 +71,8 @@ def selftest() -> None: ''' out = materialize(fixture) assert MARKER in out - start, end = v2.v1.function_span(out, "forensic_snapshot_save") - body = out[start:end] - assert "__attribute__((noinline, cold))" in body - assert "workspace = malloc(workspace_bytes);" in body + assert out.count(ATTRIBUTE_SIGNATURE) == 1 + assert "workspace = malloc(workspace_bytes);" in out def main() -> int: