diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36044c13..9dc68007 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,25 +17,224 @@ jobs: run: make test-host - name: Self-test guarded hardware fault injector run: python3 tools/hardware_fault_injector.py selftest + - name: Self-test HDL performance log parser + run: python3 tools/parse_hdl_perf.py --selftest + - name: Self-test HDL PROFILE A/B comparator + run: python3 tools/compare_hdl_profile_ab.py --selftest + - name: Self-test Phase-0 frozen pair preflight + run: python3 tools/phase0_profile_pair_preflight.py --selftest + - name: Self-test resume-hash A/B preflight + run: python3 tools/resume_hash_ab_preflight.py --selftest + - name: Self-test resume-hash A/B comparator + run: python3 tools/compare_hdl_resume_hash_ab.py --selftest + - name: Self-test R5900 calibration disassembly guard + run: python3 tools/check_r5900_calibration_disasm.py --selftest + - name: Self-test allocation inventory + run: python3 tools/allocation_inventory.py --selftest + - name: Self-test transaction workspace materializer + run: python3 tools/materialize_transaction_workspace.py --selftest + - name: Enforce reviewed explicit alignment policy + run: | + python3 tools/check_alignment_policy.py --selftest + python3 tools/check_alignment_policy.py + - name: Enforce direct-fileXio runtime policy + run: | + python3 tools/check_filexio_fdman_policy.py --selftest + python3 tools/check_filexio_fdman_policy.py + - name: Enforce no-local-time libcglue policy + run: | + python3 tools/check_libcglue_time_policy.py --selftest + python3 tools/check_libcglue_time_policy.py + - name: Enforce integer-only formatter policy + run: | + python3 tools/printf_format_audit.py --selftest + python3 tools/check_integer_format_policy.py --selftest + python3 tools/check_integer_format_policy.py ps2-build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Build, audit and strip PS2 ELF with PS2DEV v2.0.0 + - name: Build PROFILE pair and calibration ELF with PS2DEV v2.0.0 run: >- - docker run --rm -v "$PWD:/work" -w /work ps2dev/ps2dev:v2.0.0 + docker run --rm + -e PROJECT_GIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}" + -e PROJECT_GIT_REF="${{ github.head_ref || github.ref }}" + -e PS2DEV_BUNDLE_REF="v2.0.0" + -e PS2SDK_SOURCE_REF="v2.0.0" + -e PS2SDK_SOURCE_SHA="b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b" + -v "$PWD:/work" -w /work ps2dev/ps2dev:v2.0.0 sh -c 'apk add --no-cache make python3 >/dev/null && sh tools/r5900_toolchain_audit.sh GCC_R5900_TARGET.txt && python3 tools/corpus_v2_project_audit.py --output CORPUS_V2_PROJECT_AUDIT.txt && - make clean && make && + python3 tools/allocation_inventory.py + --output ALLOCATION_INVENTORY.json && + python3 tools/printf_format_audit.py + --output PRINTF_FORMAT_AUDIT.txt && + make clean && make HDL_PROFILE=1 && + cp hdl_stream.irx HDL_STREAM_PROFILE_ON.irx && python3 tools/optimization_audit.py --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF - --output OPTIMIZATION_AUDIT.txt && - make release' - - name: Record artifact checksum - run: sha256sum PS2_HDD_BOOTSTRAP_MANAGER.ELF | tee PS2_HDD_BOOTSTRAP_MANAGER.ELF.sha256 + --output OPTIMIZATION_AUDIT_PROFILE_ON.txt && + make HDL_PROFILE=1 release && + HDL_PROFILE=1 sh tools/build_benchmark_provenance.sh + BENCHMARK_PROVENANCE_PROFILE_ON.yml && + cp PS2_HDD_BOOTSTRAP_MANAGER.ELF + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF && + cp PS2_HDD_BOOTSTRAP_MANAGER.map + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.map && + make clean && make HDL_PROFILE=0 && + cp hdl_stream.irx HDL_STREAM_PROFILE_OFF.irx && + python3 tools/optimization_audit.py + --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF + --output OPTIMIZATION_AUDIT_PROFILE_OFF.txt && + make HDL_PROFILE=0 release && + HDL_PROFILE=0 sh tools/build_benchmark_provenance.sh + BENCHMARK_PROVENANCE_PROFILE_OFF.yml && + cp PS2_HDD_BOOTSTRAP_MANAGER.ELF + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF && + cp PS2_HDD_BOOTSTRAP_MANAGER.map + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.map && + mipsel-none-elf-size HDL_STREAM_PROFILE_ON.irx + HDL_STREAM_PROFILE_OFF.irx > HDL_STREAM_PROFILE_PAIR.sections && + cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF + PS2_HDD_BOOTSTRAP_MANAGER.ELF && + cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.map + PS2_HDD_BOOTSTRAP_MANAGER.map && + make -C bench/r5900_calibration clean all && + mips64r5900el-ps2-elf-size + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF + > bench/r5900_calibration/R5900_COUNTER_CALIBRATION.sections && + mips64r5900el-ps2-elf-objdump -dr + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF + > bench/r5900_calibration/R5900_COUNTER_CALIBRATION.disasm && + sha256sum bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF + > bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF.sha256' + - name: Validate PROFILE split and record checksums + run: | + if cmp -s HDL_STREAM_PROFILE_ON.irx HDL_STREAM_PROFILE_OFF.irx; then + echo "::error::PROFILE ON/OFF embedded hdl_stream IRX files are byte-identical" + exit 1 + fi + sha256sum HDL_STREAM_PROFILE_ON.irx HDL_STREAM_PROFILE_OFF.irx | tee HDL_STREAM_PROFILE_PAIR.sha256 + wc -c HDL_STREAM_PROFILE_ON.irx HDL_STREAM_PROFILE_OFF.irx | tee HDL_STREAM_PROFILE_PAIR.sizes + cat HDL_STREAM_PROFILE_PAIR.sections + python3 tools/phase0_profile_pair_preflight.py \ + --profile-on PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF \ + --profile-off PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF \ + --project-git-sha "${{ github.event.pull_request.head.sha || github.sha }}" \ + --output-template PHASE0_AB_SAMPLES_TEMPLATE.json + python3 tools/check_r5900_calibration_disasm.py \ + --disasm bench/r5900_calibration/R5900_COUNTER_CALIBRATION.disasm + echo "--- R5900 calibration ELF ---" + cat bench/r5900_calibration/R5900_COUNTER_CALIBRATION.sections + cat bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF.sha256 + echo "--- calibration_integer_loop disassembly ---" + awk '/:/ {show=1; count=0} show {print; count++} show && count >= 12 {exit}' \ + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.disasm + echo "--- PROFILE ON provenance ---" + cat BENCHMARK_PROVENANCE_PROFILE_ON.yml + echo "--- PROFILE OFF provenance ---" + cat BENCHMARK_PROVENANCE_PROFILE_OFF.yml + echo "--- PROFILE ON optimization audit ---" + cat OPTIMIZATION_AUDIT_PROFILE_ON.txt + echo "--- PROFILE OFF optimization audit ---" + cat OPTIMIZATION_AUDIT_PROFILE_OFF.txt + sha256sum PS2_HDD_BOOTSTRAP_MANAGER.ELF | tee PS2_HDD_BOOTSTRAP_MANAGER.ELF.sha256 + sha256sum PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF | tee HDL_PROFILE_PAIR.sha256 + - name: Build isolated transaction-workspace experiment after frozen gate + run: >- + docker run --rm + -e PROJECT_GIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}" + -e PROJECT_GIT_REF="${{ github.head_ref || github.ref }}" + -e PS2DEV_BUNDLE_REF="v2.0.0" + -e PS2SDK_SOURCE_REF="v2.0.0" + -e PS2SDK_SOURCE_SHA="b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b" + -v "$PWD:/work" -w /work ps2dev/ps2dev:v2.0.0 + sh -c 'apk add --no-cache make python3 >/dev/null && + sh tools/build_transaction_workspace_experiment.sh && + mips64r5900el-ps2-elf-size + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF + > PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.sections && + mips64r5900el-ps2-elf-size + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF + > PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.sections' + - name: Record and validate transaction-workspace experiment identity + run: | + python3 tools/transaction_workspace_ab_preflight.py \ + --project-git-sha "${{ github.event.pull_request.head.sha || github.sha }}" + echo "--- transaction workspace PROFILE OFF ---" + cat PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF.sha256 + cat PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF.size + cat PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.sections + cat BENCHMARK_PROVENANCE_TX_WORKSPACE_PROFILE_OFF.yml + echo "--- transaction workspace PROFILE ON ---" + cat PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF.sha256 + cat PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF.size + cat PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.sections + cat BENCHMARK_PROVENANCE_TX_WORKSPACE_PROFILE_ON.yml + echo "--- transaction workspace identity ---" + cat TRANSACTION_WORKSPACE_AB_IDENTITY.json + echo "--- transaction workspace PROFILE OFF optimization audit ---" + cat OPTIMIZATION_AUDIT_TX_WORKSPACE_PROFILE_OFF.txt + echo "--- transaction workspace PROFILE ON optimization audit ---" + cat OPTIMIZATION_AUDIT_TX_WORKSPACE_PROFILE_ON.txt + python3 tools/phase0_profile_pair_preflight.py \ + --profile-on PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF \ + --profile-off PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF \ + --project-git-sha "${{ github.event.pull_request.head.sha || github.sha }}" + - name: Build isolated resume-hash experiment after frozen gate + run: >- + docker run --rm + -e PROJECT_GIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}" + -e PROJECT_GIT_REF="${{ github.head_ref || github.ref }}" + -e PS2DEV_BUNDLE_REF="v2.0.0" + -e PS2SDK_SOURCE_REF="v2.0.0" + -e PS2SDK_SOURCE_SHA="b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b" + -v "$PWD:/work" -w /work ps2dev/ps2dev:v2.0.0 + sh -c 'apk add --no-cache make python3 >/dev/null && + sh tools/build_resume_hash_experiment.sh && + mips64r5900el-ps2-elf-size + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF + > PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.sections && + mips64r5900el-ps2-elf-size + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF + > PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.sections && + cp PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.sections + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.sections && + cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF + PS2_HDD_BOOTSTRAP_MANAGER.ELF && + cp PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.map + PS2_HDD_BOOTSTRAP_MANAGER.map' + - name: Record and validate resume-hash experiment identity + run: | + python3 tools/resume_hash_ab_preflight.py \ + --project-git-sha "${{ github.event.pull_request.head.sha || github.sha }}" \ + --identity-output RESUME_HASH_AB_IDENTITY.json \ + --profile-off-template RESUME_HASH_AB_PROFILE_OFF_TEMPLATE.json \ + --profile-on-template RESUME_HASH_AB_PROFILE_ON_TEMPLATE.json + echo "--- resume-hash PROFILE OFF ---" + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF.sha256 + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF.size + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.sections + cat BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_OFF.yml + echo "--- resume-hash PROFILE ON ---" + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF.sha256 + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF.size + cat PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.sections + cat BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_ON.yml + echo "--- resume-hash identity ---" + cat RESUME_HASH_AB_IDENTITY.json + echo "--- resume-hash PROFILE OFF optimization audit ---" + cat OPTIMIZATION_AUDIT_RESUME_HASH_PROFILE_OFF.txt + echo "--- resume-hash PROFILE ON optimization audit ---" + cat OPTIMIZATION_AUDIT_RESUME_HASH_PROFILE_ON.txt + python3 tools/phase0_profile_pair_preflight.py \ + --profile-on PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF \ + --profile-off PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF \ + --project-git-sha "${{ github.event.pull_request.head.sha || github.sha }}" + sha256sum PS2_HDD_BOOTSTRAP_MANAGER.ELF | tee PS2_HDD_BOOTSTRAP_MANAGER.ELF.sha256 - uses: actions/upload-artifact@v4 with: name: PS2-HDD-Bootstrap-Manager @@ -43,9 +242,74 @@ jobs: PS2_HDD_BOOTSTRAP_MANAGER.ELF PS2_HDD_BOOTSTRAP_MANAGER.ELF.sha256 PS2_HDD_BOOTSTRAP_MANAGER.map - OPTIMIZATION_AUDIT.txt + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.map + PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.map + HDL_PROFILE_PAIR.sha256 + HDL_STREAM_PROFILE_ON.irx + HDL_STREAM_PROFILE_OFF.irx + HDL_STREAM_PROFILE_PAIR.sha256 + HDL_STREAM_PROFILE_PAIR.sizes + HDL_STREAM_PROFILE_PAIR.sections + PHASE0_AB_SAMPLES_TEMPLATE.json + OPTIMIZATION_AUDIT_PROFILE_ON.txt + OPTIMIZATION_AUDIT_PROFILE_OFF.txt CORPUS_V2_PROJECT_AUDIT.txt + ALLOCATION_INVENTORY.json + PRINTF_FORMAT_AUDIT.txt GCC_R5900_TARGET.txt + BENCHMARK_PROVENANCE_PROFILE_ON.yml + BENCHMARK_PROVENANCE_PROFILE_OFF.yml + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.ELF.sha256 + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.map + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.sections + bench/r5900_calibration/R5900_COUNTER_CALIBRATION.disasm + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF.sha256 + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF.size + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.map + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.sections + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF.sha256 + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF.size + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.map + PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.sections + HDL_STREAM_TX_WORKSPACE_PROFILE_OFF.irx + HDL_STREAM_TX_WORKSPACE_PROFILE_ON.irx + OPTIMIZATION_AUDIT_TX_WORKSPACE_PROFILE_OFF.txt + OPTIMIZATION_AUDIT_TX_WORKSPACE_PROFILE_ON.txt + BENCHMARK_PROVENANCE_TX_WORKSPACE_PROFILE_OFF.yml + BENCHMARK_PROVENANCE_TX_WORKSPACE_PROFILE_ON.yml + TRANSACTION_WORKSPACE_AB_IDENTITY.json + TRANSACTION_WORKSPACE_AB_PROFILE_OFF_TEMPLATE.json + TRANSACTION_WORKSPACE_AB_PROFILE_ON_TEMPLATE.json + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.sha256 + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.size + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.map + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.sections + OPTIMIZATION_AUDIT_RESUME_HASH.txt + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF.sha256 + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF.size + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.map + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.sections + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF.sha256 + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF.size + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.map + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.sections + HDL_STREAM_RESUME_HASH_PROFILE_OFF.irx + HDL_STREAM_RESUME_HASH_PROFILE_ON.irx + OPTIMIZATION_AUDIT_RESUME_HASH_PROFILE_OFF.txt + OPTIMIZATION_AUDIT_RESUME_HASH_PROFILE_ON.txt + BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_OFF.yml + BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_ON.yml + RESUME_HASH_AB_IDENTITY.json + RESUME_HASH_AB_PROFILE_OFF_TEMPLATE.json + RESUME_HASH_AB_PROFILE_ON_TEMPLATE.json HDDMAN.CFG LICENSE THIRD_PARTY_NOTICES.md diff --git a/GNUmakefile b/GNUmakefile new file mode 100644 index 00000000..8cbe8180 --- /dev/null +++ b/GNUmakefile @@ -0,0 +1,20 @@ +# Corpus-v2 Phase-1 application/runtime link policies. +# +# Keep the normal project Makefile unchanged so every policy remains an explicit +# A/B layer. These small objects are linked before the PS2SDK/Newlib archives and +# replace only contracts that the Phase-1 source audits prove unused. + +include Makefile + +EE_OBJS += filexio_fdman_policy_ps2.o printf_policy_ps2.o +$(EE_BIN): filexio_fdman_policy_ps2.o printf_policy_ps2.o +EE_LDFLAGS += -Wl,--wrap=snprintf -Wl,--wrap=vsnprintf + +filexio_fdman_policy_ps2.o: src/filexio_fdman_policy_ps2.c + $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ + +# GNU ld --wrap rewrites references after the LTO plugin has performed its own +# reachability pass. Keep these two externally-named shims as ordinary object +# code so LTO cannot discard them before ld creates __wrap_* references. +printf_policy_ps2.o: src/printf_policy_ps2.c + $(EE_CC) $(EE_CFLAGS) -fno-lto $(EE_INCS) -c $< -o $@ diff --git a/Makefile b/Makefile index 43d66e90..1a3e9b2b 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,31 @@ EE_BIN = PS2_HDD_BOOTSTRAP_MANAGER.ELF EE_MAP = PS2_HDD_BOOTSTRAP_MANAGER.map -EE_OBJS = main.o manager_menu_ps2.o app_ui_ps2.o disk_status_ps2.o gs_ui_ps2.o gs_debug_compat_ps2.o app_error.o bootstrap_controller_ps2.o diagnostics_controller_ps2.o forensic_controller_ps2.o platform.o storage.o video_mode.o ui_layout.o ui_font.o spleen_font_data.o header_backup.o repair_snapshot.o forensic_snapshot.o rescue_image.o rescue_storage.o bootstrap_source.o bootstrap_signing.o apa.o apa_repair.o apa_forensic.o repair_health.o hdd_bounds.o hdd_read.o hdd_write.o hdd_repair_ps2.o hdd_forensic_repair_ps2.o repair_controller_ps2.o hdd_recovery_wrap.o bootstrap_transaction.o bootstrap_transaction_ps2.o boot_chain.o boot_chain_ps2.o boot_payload.o boot_payload_ps2.o boot_diagnostics_ps2.o boot_report.o boot_report_ps2.o boot_report_session.o session_log.o kelf.o sha256.o capsule_format.o mbr_compat.o hdl_iso.o hdl_partition.o hdl_transaction.o hdl_installer_ps2.o +EE_OBJS = main.o manager_menu_ps2.o app_ui_ps2.o disk_status_ps2.o gs_ui_ps2.o gs_debug_compat_ps2.o app_error.o bootstrap_controller_ps2.o diagnostics_controller_ps2.o forensic_controller_ps2.o platform.o storage.o video_mode.o ui_layout.o ui_font.o spleen_font_data.o header_backup.o repair_snapshot.o forensic_snapshot.o rescue_image.o rescue_storage.o bootstrap_source.o bootstrap_signing.o apa.o apa_repair.o apa_forensic.o repair_health.o hdd_bounds.o hdd_read.o hdd_write.o hdd_repair_ps2.o hdd_forensic_repair_ps2.o repair_controller_ps2.o hdd_recovery_wrap.o bootstrap_transaction.o bootstrap_transaction_ps2.o boot_chain.o boot_chain_ps2.o boot_payload.o boot_payload_ps2.o boot_diagnostics_ps2.o boot_report.o boot_report_ps2.o boot_report_session.o session_log.o kelf.o sha256.o capsule_format.o mbr_compat.o hdl_iso.o hdl_partition.o hdl_transaction.o hdl_installer_ps2.o r5900_perf.o EE_LIBS = -ldebug -ldraw -lgraph -lpacket -ldma -lm -lpad -lfileXio -lpatches -lpoweroff -lsecr -lkernel + +# Phase-0 profiler perturbation must be measurable with the same source and the +# same optimization flags. HDL_PROFILE=1 is the current instrumented default; +# HDL_PROFILE=0 compiles EE/IOP HDL telemetry out without changing dataflow. +HDL_PROFILE ?= 1 +ifeq ($(filter $(HDL_PROFILE),0 1),) +$(error HDL_PROFILE must be 0 or 1) +endif + +# Phase-2 recovery experiment. Keep this disabled in the frozen Phase-0 pair: +# the experiment removes already-completed USB rehash work after interruption, +# but must earn its place with real-hardware A/B before becoming the default. +HDL_RESUME_HASH_CHECKPOINT ?= 0 +ifeq ($(filter $(HDL_RESUME_HASH_CHECKPOINT),0 1),) +$(error HDL_RESUME_HASH_CHECKPOINT must be 0 or 1) +endif + # LTO lets the R5900 compiler optimize across the deliberately small modules # while section GC still removes unused recovery/UI helpers from the final ELF. -EE_CFLAGS = -O2 -flto -G0 -Wall -Wextra -Werror -std=gnu99 -fdata-sections -ffunction-sections -Iinclude +EE_CFLAGS = -O2 -flto -G0 -Wall -Wextra -Werror -std=gnu99 -fdata-sections -ffunction-sections -Iinclude -DHDL_PROFILE_ENABLED=$(HDL_PROFILE) +ifeq ($(HDL_RESUME_HASH_CHECKPOINT),1) +EE_CFLAGS += -DHDL_RESUME_HASH_CHECKPOINT_ENABLED=1 +EE_OBJS += hdl_hash_checkpoint.o +endif # Keep a linker map for every build. The R5900 has a 16 KiB I-cache, so archive # provenance, section growth and final placement are performance data, not just # link-time trivia. This also lets CI explain why heavyweight Newlib routines @@ -43,10 +64,11 @@ HOST_HDD_REPAIR_FIXTURE_TEST = tests/test_hdd_repair_fixtures HOST_HDL_ISO_TEST = tests/test_hdl_iso HOST_HDL_PARTITION_TEST = tests/test_hdl_partition HOST_HDL_TRANSACTION_TEST = tests/test_hdl_transaction +HOST_HDL_HASH_CHECKPOINT_TEST = tests/test_hdl_hash_checkpoint HOST_SPLEEN_GENERATED = tests/generated_spleen_font_data.c HOST_HDD_FIXTURE_DIR = tests/generated_hdds HOST_FORENSIC_FIXTURE_DIR = tests/generated_forensic_hdds -HOST_TESTS = $(HOST_APP_ERROR_TEST) $(HOST_FORMAT_TEST) $(HOST_APA_REPAIR_TEST) $(HOST_APA_FORENSIC_TEST) $(HOST_APA_FORENSIC_DORMANT_TEST) $(HOST_VIDEO_MODE_TEST) $(HOST_UI_LAYOUT_TEST) $(HOST_UI_FONT_TEST) $(HOST_FORENSIC_FIXTURE_TEST) $(HOST_BOOT_CHAIN_TEST) $(HOST_BOOT_PAYLOAD_TEST) $(HOST_BOOT_REPORT_TEST) $(HOST_KELF_TEST) $(HOST_BOOTSTRAP_TRANSACTION_TEST) $(HOST_RESCUE_IMAGE_TEST) $(HOST_HDD_FIXTURE_TEST) $(HOST_HDD_MUTATION_TEST) $(HOST_HDD_REPAIR_FIXTURE_TEST) $(HOST_HDL_ISO_TEST) $(HOST_HDL_PARTITION_TEST) $(HOST_HDL_TRANSACTION_TEST) +HOST_TESTS = $(HOST_APP_ERROR_TEST) $(HOST_FORMAT_TEST) $(HOST_APA_REPAIR_TEST) $(HOST_APA_FORENSIC_TEST) $(HOST_APA_FORENSIC_DORMANT_TEST) $(HOST_VIDEO_MODE_TEST) $(HOST_UI_LAYOUT_TEST) $(HOST_UI_FONT_TEST) $(HOST_FORENSIC_FIXTURE_TEST) $(HOST_BOOT_CHAIN_TEST) $(HOST_BOOT_PAYLOAD_TEST) $(HOST_BOOT_REPORT_TEST) $(HOST_KELF_TEST) $(HOST_BOOTSTRAP_TRANSACTION_TEST) $(HOST_RESCUE_IMAGE_TEST) $(HOST_HDD_FIXTURE_TEST) $(HOST_HDD_MUTATION_TEST) $(HOST_HDD_REPAIR_FIXTURE_TEST) $(HOST_HDL_ISO_TEST) $(HOST_HDL_PARTITION_TEST) $(HOST_HDL_TRANSACTION_TEST) $(HOST_HDL_HASH_CHECKPOINT_TEST) all: $(EE_BIN) @@ -140,9 +162,14 @@ test-host: tests/test_hdl_transaction.c src/hdl_transaction.c src/sha256.c \ -o $(HOST_HDL_TRANSACTION_TEST) ./$(HOST_HDL_TRANSACTION_TEST) + $(HOST_CC) -std=c99 -Wall -Wextra -Werror -Iinclude \ + tests/test_hdl_hash_checkpoint.c src/hdl_hash_checkpoint.c src/sha256.c \ + -o $(HOST_HDL_HASH_CHECKPOINT_TEST) + ./$(HOST_HDL_HASH_CHECKPOINT_TEST) clean: rm -f $(EE_BIN) $(EE_MAP) $(EE_OBJS) $(IRX_FILES:.irx=_irx.c) $(HOST_TESTS) + rm -f hdl_hash_checkpoint.o rm -f $(CUSTOM_IRX) hdl_stream_irx.c iop/hdl_stream/*.o \ iop/hdl_stream/*.elf iop/hdl_stream/*.irx rm -f ps2hdd_posix_irx.c @@ -303,9 +330,15 @@ hdl_partition.o: src/hdl_partition.c hdl_transaction.o: src/hdl_transaction.c $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ +hdl_hash_checkpoint.o: src/hdl_hash_checkpoint.c + $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ + hdl_installer_ps2.o: src/hdl_installer_ps2.c $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ +r5900_perf.o: src/r5900_perf.c + $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ + %_irx.c: $(PS2SDK)/bin/bin2c $(PS2SDK)/iop/irx/$*.irx $@ $*_irx @@ -316,7 +349,7 @@ ps2hdd_posix_irx.c: $(PS2SDK)/bin/bin2c $(PS2SDK)/iop/irx/ps2hdd-bdm.irx $@ ps2hdd_posix_irx hdl_stream.irx: - $(MAKE) -C iop/hdl_stream IOP_BIN=$(abspath $@) + $(MAKE) -C iop/hdl_stream IOP_BIN=$(abspath $@) HDL_PROFILE=$(HDL_PROFILE) hdl_stream_irx.c: hdl_stream.irx $(PS2SDK)/bin/bin2c $< $@ hdl_stream_irx diff --git a/bench/r5900_calibration/Makefile b/bench/r5900_calibration/Makefile new file mode 100644 index 00000000..28afb921 --- /dev/null +++ b/bench/r5900_calibration/Makefile @@ -0,0 +1,22 @@ +EE_BIN = R5900_COUNTER_CALIBRATION.ELF +EE_OBJS = main.o r5900_perf.o +EE_LIBS = -ldebug -lkernel + +EE_CFLAGS = -O2 -G0 -Wall -Wextra -Werror -std=gnu99 -I../../include +EE_LDFLAGS = -Wl,-Map,R5900_COUNTER_CALIBRATION.map + +all: $(EE_BIN) + +main.o: main.c + $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@ + +r5900_perf.o: ../../src/r5900_perf.c ../../include/r5900_perf.h + $(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c ../../src/r5900_perf.c -o $@ + +clean: + rm -f $(EE_BIN) R5900_COUNTER_CALIBRATION.map $(EE_OBJS) + +include $(PS2SDK)/samples/Makefile.pref +include $(PS2SDK)/samples/Makefile.eeglobal + +.PHONY: all clean diff --git a/bench/r5900_calibration/main.c b/bench/r5900_calibration/main.c new file mode 100644 index 00000000..9b801d83 --- /dev/null +++ b/bench/r5900_calibration/main.c @@ -0,0 +1,294 @@ +#include +#include +#include + +#include +#include + +#include "r5900_perf.h" + +#define CALIBRATION_SAMPLES 32u +#define CALIBRATION_ITERATIONS 100000u +#define CALIBRATION_BODY_INSNS_PER_ITERATION 4u + +typedef struct { + uint32_t cycles; + uint32_t instructions; + uint32_t timer_busclocks; + int counter_overflow; + int timer_overflow; +} calibration_sample_t; + +typedef struct { + uint32_t p50; + uint32_t p95; + uint32_t p99; + uint32_t max; +} calibration_distribution_t; + +static volatile uint32_t calibration_sink; + +/* + * Measurement reference kernel, not an application optimization. + * + * The loop body deliberately executes four fixed instructions per iteration: + * two ADDIU operations, BNEZ and its NOP delay slot. The one-time accumulator + * initialization and function return sit outside that repeated four-instruction + * body. Keeping this tiny body explicit makes the instruction-completed counter + * sanity check independent of GCC's loop transforms. NOCLONE keeps a stable + * callable symbol so CI can validate the emitted reference sequence directly. + */ +static __attribute__((noinline, noclone)) uint32_t calibration_integer_loop(uint32_t iterations) +{ + uint32_t accumulator; + + __asm__ __volatile__( + "move %0, $zero\n" + "1:\n" + "addiu %0, %0, 1\n" + "addiu %1, %1, -1\n" + "bnez %1, 1b\n" + "nop\n" + : "=&r"(accumulator), "+r"(iterations) + : + : "memory"); + + return accumulator; +} + +static uint32_t timer_delta32(uint64_t start, uint64_t end, int *overflow) +{ + uint64_t delta = end - start; + + if (delta > UINT32_MAX) { + *overflow = 1; + return UINT32_MAX; + } + *overflow = 0; + return (uint32_t)delta; +} + +static int measure_empty(calibration_sample_t *sample) +{ + r5900_perf_scope_t scope; + r5900_perf_result_t result; + uint64_t timer_start; + uint64_t timer_end; + + memset(&scope, 0, sizeof(scope)); + memset(&result, 0, sizeof(result)); + memset(sample, 0, sizeof(*sample)); + + timer_start = GetTimerSystemTime(); + if (r5900_perf_begin(&scope, + R5900_PCR0_PROCESSOR_CYCLE, + R5900_PCR1_INSTRUCTION_COMPLETED) != 0) + return -1; + if (r5900_perf_end(&scope, &result) != 0) + return -1; + timer_end = GetTimerSystemTime(); + + sample->cycles = result.pcr0; + sample->instructions = result.pcr1; + sample->counter_overflow = result.pcr0_overflow || result.pcr1_overflow; + sample->timer_busclocks = timer_delta32(timer_start, timer_end, + &sample->timer_overflow); + return 0; +} + +static int measure_loop(calibration_sample_t *sample, uint32_t salt) +{ + r5900_perf_scope_t scope; + r5900_perf_result_t result; + uint64_t timer_start; + uint64_t timer_end; + uint32_t value; + + memset(&scope, 0, sizeof(scope)); + memset(&result, 0, sizeof(result)); + memset(sample, 0, sizeof(*sample)); + + timer_start = GetTimerSystemTime(); + if (r5900_perf_begin(&scope, + R5900_PCR0_PROCESSOR_CYCLE, + R5900_PCR1_INSTRUCTION_COMPLETED) != 0) + return -1; + + value = calibration_integer_loop(CALIBRATION_ITERATIONS); + + if (r5900_perf_end(&scope, &result) != 0) + return -1; + timer_end = GetTimerSystemTime(); + + /* Consume the result after the measured region. */ + calibration_sink ^= value + salt; + + sample->cycles = result.pcr0; + sample->instructions = result.pcr1; + sample->counter_overflow = result.pcr0_overflow || result.pcr1_overflow; + sample->timer_busclocks = timer_delta32(timer_start, timer_end, + &sample->timer_overflow); + return value == CALIBRATION_ITERATIONS ? 0 : -1; +} + +static void sort_u32(uint32_t *values, unsigned int count) +{ + unsigned int i; + + for (i = 1; i < count; ++i) { + uint32_t value = values[i]; + unsigned int j = i; + + while (j > 0 && values[j - 1] > value) { + values[j] = values[j - 1]; + --j; + } + values[j] = value; + } +} + +static uint32_t nearest_rank(const uint32_t *values, + unsigned int count, + unsigned int percentile) +{ + unsigned int rank = (count * percentile + 99u) / 100u; + + if (rank == 0) + rank = 1; + if (rank > count) + rank = count; + return values[rank - 1]; +} + +static calibration_distribution_t distribution(const calibration_sample_t *samples, + int field) +{ + uint32_t values[CALIBRATION_SAMPLES]; + calibration_distribution_t result; + unsigned int i; + + for (i = 0; i < CALIBRATION_SAMPLES; ++i) { + if (field == 0) + values[i] = samples[i].cycles; + else if (field == 1) + values[i] = samples[i].instructions; + else + values[i] = samples[i].timer_busclocks; + } + + sort_u32(values, CALIBRATION_SAMPLES); + result.p50 = nearest_rank(values, CALIBRATION_SAMPLES, 50); + result.p95 = nearest_rank(values, CALIBRATION_SAMPLES, 95); + result.p99 = nearest_rank(values, CALIBRATION_SAMPLES, 99); + result.max = values[CALIBRATION_SAMPLES - 1]; + return result; +} + +static uint32_t subtract_floor(uint32_t value, uint32_t overhead) +{ + return value > overhead ? value - overhead : 0; +} + +static void print_distribution(const char *name, + const calibration_distribution_t *value) +{ + scr_printf("%-11s p50=%lu p95=%lu p99=%lu max=%lu\n", + name, + (unsigned long)value->p50, + (unsigned long)value->p95, + (unsigned long)value->p99, + (unsigned long)value->max); +} + +int main(void) +{ + calibration_sample_t empty[CALIBRATION_SAMPLES]; + calibration_sample_t loop[CALIBRATION_SAMPLES]; + calibration_distribution_t empty_cycles; + calibration_distribution_t empty_instructions; + calibration_distribution_t empty_timer; + calibration_distribution_t loop_cycles; + calibration_distribution_t loop_instructions; + calibration_distribution_t loop_timer; + unsigned int counter_overflows = 0; + unsigned int timer_overflows = 0; + unsigned int failures = 0; + unsigned int i; + uint32_t expected_body_instructions = + CALIBRATION_ITERATIONS * CALIBRATION_BODY_INSNS_PER_ITERATION; + + init_scr(); + scr_printf("R5900 PERFORMANCE COUNTER CALIBRATION\n"); + scr_printf("No HDD/APA access. Standalone EE benchmark.\n\n"); + + /* Warm the reference kernel and its instruction footprint before sampling. */ + for (i = 0; i < 4; ++i) + calibration_sink ^= calibration_integer_loop(CALIBRATION_ITERATIONS); + + /* Pair samples and alternate order to reduce systematic first/second bias. */ + for (i = 0; i < CALIBRATION_SAMPLES; ++i) { + int empty_status; + int loop_status; + + if ((i & 1u) == 0) { + empty_status = measure_empty(&empty[i]); + loop_status = measure_loop(&loop[i], i); + } else { + loop_status = measure_loop(&loop[i], i); + empty_status = measure_empty(&empty[i]); + } + + if (empty_status != 0 || loop_status != 0) + ++failures; + if (empty[i].counter_overflow || loop[i].counter_overflow) + ++counter_overflows; + if (empty[i].timer_overflow || loop[i].timer_overflow) + ++timer_overflows; + } + + empty_cycles = distribution(empty, 0); + empty_instructions = distribution(empty, 1); + empty_timer = distribution(empty, 2); + loop_cycles = distribution(loop, 0); + loop_instructions = distribution(loop, 1); + loop_timer = distribution(loop, 2); + + scr_printf("samples=%u iterations=%u\n", + CALIBRATION_SAMPLES, CALIBRATION_ITERATIONS); + scr_printf("PCR0=processor cycles PCR1=instructions completed\n"); + scr_printf("timer=GetTimerSystemTime raw BUSCLK units\n\n"); + + scr_printf("EMPTY SCOPE\n"); + print_distribution("cycles", &empty_cycles); + print_distribution("instructions", &empty_instructions); + print_distribution("timer", &empty_timer); + + scr_printf("\nDETERMINISTIC LOOP\n"); + print_distribution("cycles", &loop_cycles); + print_distribution("instructions", &loop_instructions); + print_distribution("timer", &loop_timer); + + scr_printf("\nP50 LOOP - EMPTY\n"); + scr_printf("cycles=%lu instructions=%lu timer=%lu\n", + (unsigned long)subtract_floor(loop_cycles.p50, empty_cycles.p50), + (unsigned long)subtract_floor(loop_instructions.p50, + empty_instructions.p50), + (unsigned long)subtract_floor(loop_timer.p50, empty_timer.p50)); + scr_printf("loop-body instruction floor=%lu\n", + (unsigned long)expected_body_instructions); + scr_printf("overflows counter=%u timer=%u failures=%u\n", + counter_overflows, timer_overflows, failures); + scr_printf("sink=%08x\n", (unsigned int)calibration_sink); + + if (counter_overflows != 0 || timer_overflows != 0 || failures != 0 || + subtract_floor(loop_instructions.p50, empty_instructions.p50) < + expected_body_instructions) { + scr_printf("\nRESULT: INVALID - do not use counter claims\n"); + } else { + scr_printf("\nRESULT: CALIBRATION STRUCTURALLY VALID\n"); + scr_printf("Record screen + SCPH/revision/toolchain provenance.\n"); + } + + SleepThread(); + return 0; +} diff --git a/docs/ALIGNMENT_CONTRACT_AUDIT.md b/docs/ALIGNMENT_CONTRACT_AUDIT.md new file mode 100644 index 00000000..e6a8431b --- /dev/null +++ b/docs/ALIGNMENT_CONTRACT_AUDIT.md @@ -0,0 +1,216 @@ +# Explicit alignment contract audit + +This document classifies every current `__attribute__((aligned(64)))` in the +project before Phase-5 alignment cleanup. It deliberately does not equate +"64-byte aligned" with "faster". Each site must identify the actual consumer +that requires an alignment domain. + +Heap `memalign(64, ...)` calls are audited separately because their ownership and +DMA/cache contracts differ from static/stack object placement. + +## Source-of-truth routing + +- `PS2_Memory_Allocators_optimization_research_corpus_v2.md`: allocator ABI, + cache-line and device/DMA alignment are separate contracts; avoid blanket + over-alignment; +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md`: representation + and final-consumer contract decide layout; +- `PS2_IOP_SIF_optimization_research_corpus_v2.md`: SIF/DMA buffers need explicit + ownership/cache/alignment contracts rather than generic alignment folklore; +- pinned PS2SDK `b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b`: current fileXio/libpad/draw + implementation used to decide API requirements; +- project CI #712 source audit: 25 explicit `aligned(64)` sites. + +## Current PS2SDK findings + +### fileXio devctl/ioctl2 does not require a 64-byte-aligned caller buffer + +Pinned EE `fileXioDevctl()`/`fileXioIoctl2()`: + +- copies input `arg` bytes into the library-owned RPC packet with `memcpy`; +- stores the caller output pointer only as the final destination; +- uses a library-owned return packet for the IOP->EE DMA; +- its EE callback then copies the returned bytes into the caller buffer with + ordinary `memcpy`. + +Pinned IOP fileXio server: + +- passes its own `rwbuf` payload to `iomanX_devctl()`/`iomanX_ioctl2()`; +- DMA-transfers the complete library-owned `fxio_ctl_return_pkt` to EE internal + callback storage, not directly into the application's destination. + +The server `rwbuf` itself is allocated with ordinary `AllocSysMemory()`. + +**CURRENT IMPLEMENTATION:** 64-byte alignment of application input/output +objects is not a fileXio devctl/ioctl2 API requirement. + +### fileXio read/write explicitly supports unaligned caller buffers + +Pinned `fileXioRead()` performs cache maintenance and uses an internal callback +packet for unaligned edge bytes. Pinned `fileXioWrite()` explicitly calculates a +leading non-64-byte fragment and copies up to 64 bytes into +`fxio_write_packet.unalignedData[]` before the bulk transfer. + +**CURRENT IMPLEMENTATION:** ordinary fileXio read/write callers are not required +to provide a 64-byte-aligned pointer. + +This does not mean every application buffer may be arbitrarily aligned. A buffer +can still have another consumer such as GIF DMAC or libpad. + +## Sites with demonstrated alignment contracts + +### `src/platform.c`: `pad_buffer[256]` + +```text +current alignment: 64 B +consumer: padPortOpen() +status: KEEP 64 B +``` + +Pinned `libpad.h` explicitly states that the new-libpad pad area must be a +256-byte region at a 64-byte-aligned address. This is a real API/device contract, +not an optimization preference. + +Classification: **POTWIERDZONE / API-DMA alignment**. + +### `src/gs_ui_ps2.c`: `font_atlas[]` + +```text +current alignment: 64 B +consumer: draw_texture_transfer() -> DMATAG_REF -> GIF DMAC +status: KEEP aligned, 64 B itself not demonstrated +minimum candidate: 16 B qword/DMAC alignment +``` + +Pinned `draw_texture_transfer()` places the source address directly into +`DMATAG_REF` blocks. The atlas is therefore a direct DMA source and must obey the +DMA packet/source contract. The project also calls `FlushCache(0)` after building +the atlas, so the current code does not depend on 64-byte start alignment for a +selective cache-line flush. + +Classification: **POTWIERDZONE direct DMA source; INFERENCJA that 16 B is the +sufficient project alignment pending a small A/B/correctness build.** + +Do not remove alignment entirely. A future cleanup may evaluate 64 -> 16, not +64 -> arbitrary. + +## Sites where 64 B is not supported by the current consumer contract + +The following objects are either: + +1. fileXio devctl/ioctl2 input/output; +2. ordinary fileXio read/write buffers; or +3. CPU-only scratch after such a transfer. + +Pinned fileXio already supports non-64-byte caller addresses for those paths. +No other direct DMA consumer was found for these objects. + +### Raw HDD transport / repair + +| File | Object/site | Current consumer | 64-B status | +| --- | --- | --- | --- | +| `src/hdd_read.c` | `read_transfer_buffer` | `fileXioDevctl(HDIOC_READSECTOR)` output | not required by fileXio | +| `src/hdd_write.c` | `write_packet` | `fileXioDevctl(HDIOC_WRITESECTOR)` input arg | not required by fileXio | +| `src/hdd_write.c` | `sector_verify_buffer` | raw-read devctl output + CPU compare | not required by fileXio | +| `src/hdd_write.c` | `header_verify_buffer` | raw-read devctl output + CPU parse | not required by fileXio | +| `src/hdd_repair_ps2.c` | `repair_packet` | write devctl input arg | not required by fileXio | +| `src/hdd_repair_ps2.c` | `repair_verify` | raw-read devctl output + compare | not required by fileXio | +| `src/hdd_recovery_wrap.c` | `recovery_header` | raw-read devctl output + CPU recovery | not required by fileXio | +| `src/hdd_forensic_repair_ps2.c` | `write_packet` | write devctl input arg | not required by fileXio | +| `src/hdd_forensic_repair_ps2.c` | `source_verify` | raw-read devctl output + compare | not required by fileXio | +| `src/hdd_forensic_repair_ps2.c` | `write_verify` | raw-read devctl output + compare | not required by fileXio | +| `src/hdd_forensic_repair_ps2.c` | `repaired_header` | CPU-built header, then copied into devctl arg | no DMA consumer | +| `src/main.c` | `header_buffer` | raw-read devctl output + CPU parse | not required by fileXio | + +Classification: **CURRENT IMPLEMENTATION supports ordinary alignment; candidate +cleanup after frozen hardware experiment.** + +### HDL installer/catalogue + +| File | Object/site | Current consumer | 64-B status | +| --- | --- | --- | --- | +| `src/hdl_installer_ps2.c` | `hdl_zero_metadata` | ordinary `fileXioWrite` | not required by fileXio | +| `src/hdl_installer_ps2.c` | local `verify[4]` | ordinary `fileXioRead` | not required by fileXio | +| `src/hdl_installer_ps2.c` | final `actual[1024]` | `fileXioIoctl2(READ_METADATA)` output | not required by fileXio | +| `src/hdl_tools/source_ui.inc` | `admission_header` | raw-read devctl output + APA parse | not required by fileXio | +| `src/hdl_tools/source_ui_resume_hash.inc` | `admission_header` | experiment equivalent of above | not required by fileXio | +| `src/hdl_tools/catalog.inc` | local APA `header[1024]` | raw-read devctl output + parse | not required by fileXio | +| `src/hdl_tools/catalog.inc` | local `metadata[1024]` | raw-read devctl output + parse/hash | not required by fileXio | +| `src/hdl_tools/transaction.inc` | snapshot `metadata[1024]` | raw-read devctl output + parse/hash | not required by fileXio | +| `src/hdl_tools/transaction_resume_hash.inc` | snapshot `metadata[1024]` | experiment equivalent of above | not required by fileXio | + +The normal and resume-hash `.inc` files are alternate build fragments, not two +simultaneously linked copies. Any cleanup must update both variants so an +experiment does not silently regain a stale layout assumption. + +Classification: **CURRENT IMPLEMENTATION supports ordinary alignment; candidate +cleanup after frozen hardware experiment.** + +### Storage snapshot/backup scratch + +| File | Object/site | Current consumer | 64-B status | +| --- | --- | --- | --- | +| `src/header_backup.c` | `backup_scratch[1024]` | `read_exact_file` + CPU validation | not required by fileXio | +| `src/repair_snapshot.c` | `snapshot_verify[1024]` | `read_exact_file` + `memcmp` | not required by fileXio | + +These are the cleanest first A/B candidates because they do not cross raw-HDD +or custom-stream APIs at all. + +Classification: **CURRENT IMPLEMENTATION supports ordinary alignment; highest +confidence cleanup candidates.** + +## Count summary + +Current explicit attributes: 25. + +```text +KEEP exact 64 B API contract + 1 pad_buffer + +KEEP aligned but evaluate 64 -> 16 + 1 font_atlas direct GIF-DMA source + +64 B not demonstrated by current consumer + 23 fileXio/raw-HDD/CPU scratch sites +``` + +This summary is not permission for one global search/replace. Object placement, +stack frames and LTO layout can change even when the API contract permits lower +alignment. Cleanup is therefore staged and measured. + +## Cleanup sequence + +1. Keep the CI #706 runtime experiment binary identity frozen for its hardware + gate. Do not mutate that binary merely to make this table prettier. +2. First post-gate A/B: remove 64-B alignment only from + `backup_scratch`/`snapshot_verify` and compare final ELF/map/BSS/symbol layout. +3. If useful or neutral with correctness preserved, clean the raw-HDD/fileXio + scratch group as one reviewed transport-contract change. +4. Separately test `font_atlas` at 16-B alignment with GS texture-upload and all + supported video-mode regressions. +5. Never change `pad_buffer` below 64 B while using the current libpad contract. +6. Audit `memalign(64)` calls independently before touching heap alignment. + +## Acceptance record for any alignment cleanup + +Record: + +```yaml +object: +producer: +consumer: +old_alignment: +new_alignment: +alignment_domain: +api_or_hardware_contract: +elf_delta_bytes: +text_delta_bytes: +bss_delta_bytes: +stack_frame_delta_if_applicable: +correctness_test: +real_hardware_required: +``` + +Removing an unnecessary alignment is an optimization only if it reduces memory, +stack/layout pressure or another measured cost. A source diff containing fewer +`aligned(64)` strings is not itself a performance result. diff --git a/docs/ALLOCATION_LIFETIME_AUDIT.md b/docs/ALLOCATION_LIFETIME_AUDIT.md new file mode 100644 index 00000000..ce9c7183 --- /dev/null +++ b/docs/ALLOCATION_LIFETIME_AUDIT.md @@ -0,0 +1,397 @@ +# Allocation and lifetime audit + +This document classifies the major dynamic allocations by producer, consumer, +lifetime, memory domain and ownership before further Phase-5 allocator work. The +goal is not to replace `malloc()` because it exists. The project corpus requires +allocation changes to remove measured churn, copies, fragmentation risk or peak +working-set pressure. + +## Source-of-truth routing + +- `PS2_Optimization_Library_v2_MANIFEST.md` +- `PS2_PERFORMANCE_BIBLE.md` +- `PS2_Memory_Allocators_optimization_research_corpus_v2.md` +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md` +- `PS2_Whole_System_Scheduling_research_corpus_v2.md` +- `PS2_IOP_SIF_optimization_research_corpus_v2.md` +- pinned PS2SDK `b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b` + +The corpus rule applied here is: classify producer, consumer, lifetime and the +actual alignment domain before choosing an allocator. A pool/arena is not +implicitly better than the current heap merely because it sounds console-like. + +## Epistemic labels + +- **POTWIERDZONE**: current source/current API contract. +- **CURRENT IMPLEMENTATION**: behaviour/layout of this branch/toolchain. +- **INFERENCJA**: likely optimization consequence, not yet hardware measured. +- **HIPOTEZA DO TESTU**: change requiring real-PS2 A/B before acceptance. + +`tools/allocation_inventory.py` is a static direct-call inventory. It now covers: + +```text +EE/libc heap + malloc + calloc + realloc + memalign + free + +IOP SysMem + AllocSysMemory + FreeSysMemory +``` + +It deliberately does not guess ThreadMan implementation-owned backing memory +from `CreateThread`/`CreateSema` call counts. Those costs remain part of the +runtime IOP-memory gate. + +CI #753 records 25 direct allocation calls and 72 direct frees (97 events total), +including 22 IOP SysMem events that the earlier libc-only scanner could not see. +This is a direct-call source inventory, not a claim that allocator-internal or +ThreadMan-owned memory has been measured. + +## Lifetime classes + +```text +permanent process lifetime +menu/session one UI/session/catalogue lifetime +action one bounded user operation +transaction one HDL install/recovery transaction +stream one open streaming service instance +phase-temporary one scan/hash/copy/verify sub-phase +``` + +These are review labels derived from current ownership structure, not allocator +latency measurements. + +# EE/libc heap + +## Boot-chain and generic bounded-file helpers + +| Site | Lifetime | Producer / consumer | Current action | +| --- | --- | --- | --- | +| `boot_chain_ps2.c:read_skip_hdd_setting` | phase-temporary | small config reader -> parser | keep | +| `boot_chain_ps2.c:scan_sysconf_partition` | phase-temporary | config reader -> parser | keep | +| `bootstrap_source.c:load_payload_file` | action | file -> bootstrap payload validation/write flow | keep unless payload peak is measured problematic | +| `storage.c:read_bounded_file` | caller-owned action | bounded file -> caller | keep generic helper | +| `hdd_read.c:hdd_read_payload_image` | action | HDD payload -> recovery consumer | returned dataset, not disposable scratch | + +These are ordinary CPU/fileXio data. None has a demonstrated need for a custom +allocator today. + +## USB ISO selection array + +Producer: `scan_mass_images()`. + +Current representation: + +```text +initial capacity: 32 hdl_image_entry_t +allocation: calloc +growth: doubling below 1024, then +1024 entries +consumer: ISO selection UI + selected path/size handoff +lifetime: one begin_new_install source-selection session +release: before the long-running destructive transaction +``` + +The resume-hash `.inc` contains the alternate build equivalent; both fragments +are not linked simultaneously. + +**POTWIERDZONE:** the array is function/session owned and released after the +selected source identity is copied out. + +**INFERENCJA:** this is an appropriate variable-cardinality session allocation. +Do not replace it with a giant permanent table without catalogue cardinality or +allocator-jitter evidence. + +Priority: **LOW until large source catalogues are measured.** + +## Installed HDL catalogue array + +Producer: raw APA chain walker. + +Current representation: + +```text +first capacity: 64 entries +growth: capacity * 2 +consumer: installed-games menu/details/delete selection +lifetime: one catalogue/menu session +metadata: lazy per visible page, stored inside each entry +release: catalog_free() +``` + +**POTWIERDZONE:** `realloc` occurs only while discovering main HDL partitions; +metadata itself is not separately heap-allocated per game. + +**INFERENCJA:** geometric growth already avoids per-entry allocation churn. A +pool or persistent index is not justified until game-count/catalogue latency is +measured and invalidation can be made cheaper than the APA walk. + +Priority: **LOW/MEDIUM depending real catalogue size.** + +## HDL transaction / streaming workspaces + +Default source contains separate 64 KiB helper allocations for: + +```text +source_fingerprint() +hash_source_payload() +copy_payload() +verify_target_digest() +``` + +The three transaction helpers use a 64-byte-aligned EE destination because the +custom `hdl0:` SIF/DMA path explicitly requires that destination contract. That +alignment must not be confused with ordinary fileXio caller alignment. + +### CI #724: transaction workspace v1 + +**POTWIERDZONE:** COPY/source-hash and target-verify workspaces have mutually +exclusive lifetimes. + +The isolated v1 experiment gives ownership to `execute_transaction()` and lends +one 64 KiB / 64-byte-aligned workspace sequentially to those helpers. It removes +one general-heap allocation/free pair on the successful bulk path without +increasing peak workspace. + +### CI #733: workspace v2, rejected/held + +Extending the same workspace backwards into source admission removes another +allocation pair but grows the transaction control path and lengthens workspace +lifetime. One extra allocation per transaction is too small a prize to accept +that trade without real hardware evidence. + +### CI #739: source fingerprint `memalign` -> `malloc` + +Pinned `fileXioRead()` accepts ordinary caller alignment, so the standalone +source-fingerprint buffer has no demonstrated 64-byte API requirement. + +Static result versus workspace v1 is slightly smaller and does not grow +`execute_transaction()`. This remains **HIPOTEZA DO TESTU** because fileXio's +unaligned-edge handling can cost time even when the API contract permits it. + +The custom SIF/DMA transaction workspace remains 64-byte aligned. + +## Forensic HDDMETA snapshot + +Producer: `build_snapshot_image()`. + +Baseline save keeps two variable-size allocations live: + +```text +canonical image = 64 + patch_count * (4 + 32 + 1024) + 32 +verify buffer = canonical image size +``` + +At the current maximum `patch_count = 2048`: + +```text +canonical image 2,170,976 B +baseline full verify 2,170,976 B +baseline pair peak 4,341,952 B +``` + +### CI #749: bounded read-back v1 + +The first isolated bounded experiment keeps the canonical image but replaces the +second full-size allocation with at most 64 KiB and compares every returned +chunk byte-for-byte. It preserves format, slot selection, non-overwrite policy, +truncation detection and full read-back verification. + +```text +canonical image 2,170,976 B +bounded verify 65,536 B +experiment pair peak 2,236,512 B +peak reduction 2,105,440 B +``` + +### CI #752: bounded read-back v2 + +V1 used two `fileXioLseek()` RPCs inside each exact-compare call to prove file +size. V2 instead: + +1. reads exactly the expected bytes with short-read handling; +2. compares every chunk exactly; +3. performs one final one-byte read which must return EOF. + +This still rejects truncation and trailing data while removing both seek RPCs. +Compared with CI #749, CI #752 reduces `.text` by 32 B, named text by 36 B and +eight R5900 instructions in both PROFILE modes. `execute_transaction()` and the +IOP binary remain unchanged. + +**POTWIERDZONE:** maximum pair peak is reduced by 2,105,440 B relative to the +original full-image verify representation. + +**HIPOTEZA DO TESTU:** real PS2 must still verify recovery correctness and +fileXio latency. This is primarily a peak-working-set optimization, not a hot +transaction speedup claim. + +The next architectural candidate is eliminating the full canonical image with a +streaming APAMETA1 serializer. That would alter producer lifetime and write +structure and therefore requires an isolated experiment plus stronger reference +serialization tests before hardware use. + +# IOP SysMem + +The custom `hdl_stream` service has four direct SysMem ownership classes. + +## Stream object + +Current source: + +```text +AllocSysMemory(ALLOC_FIRST, sizeof(*stream), NULL) +``` + +Lifetime: one open `hdl0:` stream. + +Owner: `stream_open()` -> `stream_close()`. + +The object stores partition geometry, source-map state, staging ownership, +prefetch handles and optional PROFILE counters. Pooling a single stream object +has no demonstrated benefit. + +## Staging allocation + +Preferred admission: + +```text +AllocSysMemory(ALLOC_FIRST, + 2 * HDL_STREAM_IOP_STAGE_BYTES + 63, + NULL) +``` + +Low-memory fallback: + +```text +AllocSysMemory(ALLOC_FIRST, + HDL_STREAM_IOP_STAGE_BYTES + 63, + NULL) +``` + +The two calls are mutually exclusive outcomes. The allocation is manually +rounded to a 64-byte stage address and remains owned for the stream lifetime. + +**POTWIERDZONE:** failure to obtain double buffering falls back to one stage; it +does not fail stream admission if one stage still fits. + +This is an explicit resilience/ownership policy. Do not replace it with a larger +ring until real IOP memory and stall telemetry justify more buffering. + +## Direct-BDM USB fragment map + +Current source allocates: + +```text +fragment_count * sizeof(bd_fragment_t) +``` + +with `AllocSysMemory`, up to the current 4096-fragment hard limit. + +Lifetime: one usable direct-BDM source mapping. It is released when source +mapping is reset/disabled or when the stream closes. + +The static IOP budget records the current worst-case map as 49,152 B. + +This is producer representation state, not generic scratch. Any compaction must +preserve fragmented-file correctness and the sequential cursor behaviour that +avoids rescanning thousands of fragments for every 64 KiB block. + +## Metadata-commit read-back buffer + +`commit_metadata()` allocates one fixed-size SysMem buffer: + +```text +verify = AllocSysMemory(ALLOC_FIRST, HDL_STREAM_METADATA_SIZE, NULL) +``` + +Lifetime: only the metadata commit call. + +Producer/consumer chain: + +```text +canonical EE metadata + -> IOP write at HDL_STREAM_METADATA_OFFSET + -> HIOCFLUSH durability barrier + -> read_metadata() into IOP verify + -> memcmp(canonical, verify) + -> FreeSysMemory(verify) +``` + +**POTWIERDZONE:** the buffer participates in mandatory post-flush read-back +verification. It is not retained across transactions and is not allocated in a +payload chunk loop. + +**CURRENT IMPLEMENTATION:** the allocation is only `HDL_STREAM_METADATA_SIZE` +(1024 B in the current HDL metadata contract). + +Priority: **KEEP**. Reusing staging memory or removing this allocation would +couple a tiny cold allocation to the much more important stream/durability +ownership path for at most 1 KiB peak savings. No measured evidence justifies +that risk. + +## IOP memory outside the direct SysMem inventory + +ThreadMan owns backing memory for resources including: + +```text +prefetch worker thread +request semaphore +done semaphore +stopped semaphore +``` + +The worker requests a 4096-byte stack, but ThreadMan control-object overhead is +not inferred here. `docs/HDL_IOP_RAM_BUDGET.md` correctly leaves that overhead +unmeasured until real hardware records active modules and minimum free IOP RAM. + +# Phase-5 priority order + +1. Keep **CI #752 bounded HDDMETA v2** as the current peak-memory candidate. +2. Hardware A/B transaction workspace v1 and source-fingerprint allocator policy. +3. Build a reference serializer test before attempting streaming canonical + HDDMETA generation. +4. Measure ISO/game catalogue cardinality before changing growth strategy. +5. Leave payload-owning allocations intact until their consumers can use a + different representation. +6. Keep the fixed 1 KiB metadata-commit read-back allocation until evidence says + otherwise; correctness/durability value dominates its tiny peak cost. +7. Do not introduce global pools/arenas or global 64-byte heap alignment merely + to reduce allocator call counts in source. +8. Do not expand IOP staging before runtime free-memory and latency attribution + are available. + +## Acceptance record for allocator/lifetime changes + +```yaml +allocation_site: +memory_domain: +producer: +consumer: +lifetime_before: +lifetime_after: +bytes_requested: +alignment_before: +alignment_after: +ownership_before: +ownership_after: +alloc_calls_before: +alloc_calls_after: +peak_bytes_before: +peak_bytes_after: +elf_text_delta: +correctness_hash: +console_scp: +hardware_revision: +active_irx: +sample_count: +p50: +p95: +p99: +max: +``` + +Static source inventory can prove call structure, ownership and requested-size +bounds. It cannot prove fragmentation, allocator latency or whole-system speedup. diff --git a/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md b/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md new file mode 100644 index 00000000..0e28f86f --- /dev/null +++ b/docs/CORPUS_V2_IMPLEMENTATION_PLAN.md @@ -0,0 +1,380 @@ +# PS2 Optimization Corpus v2 integration plan + +This branch exists to integrate the PS2 Optimization Research Library v2 into the +runtime architecture without destabilising the known-good HDL installer branch. + +## Safety boundary + +- Branch: `perf/corpus-v2-integration` +- Baseline: `4b5aa8d85e86c9de570a2128b52d1eaa5b334844` +- Parent line: `feature/hdl-game-installer` +- No optimization is merged back until correctness tests, CI and the relevant + real-hardware A/B benchmark pass. +- PCSX2 remains useful for correctness and inspection, but timing/cache/DMA/ + FIFO/device claims require a real PlayStation 2. + +## Source-of-truth routing + +Always begin with `PS2_Optimization_Library_v2_MANIFEST.md`, then use +`PS2_PERFORMANCE_BIBLE.md` for the engineering workflow and the authoritative +subsystem corpus for each change. The project uses the following conflict order: + +```text +v2 +> more specialised current corpus +> current source/manual +> real-hardware reproduction +> integrator corpus +> emulator/reverse engineering +> historical forum/anecdote +``` + +Epistemic labels used in review notes and benchmark records: + +- **POTWIERDZONE**: manual, current source or real-hardware reproduction. +- **CURRENT IMPLEMENTATION**: behaviour of the pinned PS2SDK/toolchain source. +- **HISTORYCZNE**: old toolchains, stacks or community measurements. +- **INFERENCJA**: architectural conclusion not directly measured. +- **HIPOTEZA DO TESTU**: proposed change requiring a benchmark on real hardware. + +## Optimization order + +Every work item follows this order unless evidence justifies skipping a step: + +1. remove unnecessary work; +2. do work less often; +3. reduce data volume; +4. improve data layout and locality; +5. batch; +6. remove unnecessary copies and dynamic allocation; +7. add buffering and overlap; +8. use specialised hardware when the workload fits; +9. only then specialise a measured hot kernel. + +Compiler flags, VU/MMI, Scratchpad, larger buffers, 64-byte alignment and custom +low-level APIs are never accepted as universal optimizations. + +## Dataset contract + +Every major runtime dataset or streaming path should eventually document: + +```yaml +name: +producer: +consumers: +lifetime: +representation: +alignment: +transport: +batch_size: +deadline: +ownership_states: +copy_budget: +validation: +``` + +Alignment must state the actual domain: allocator, EE cache line, DMAC/SIF, +device sector/transfer unit, VIF/GIF packet, or another explicit contract. + +## Phase 0: measurement foundation + +Goal: establish evidence before altering architecture. + +- [x] Record project SHA/ref, pinned PS2SDK source ref/SHA, toolchain identity and + build flags automatically in benchmark provenance artifacts. +- [ ] Record console SCPH/hardware revision, adapters, runtime active IRX and + workload during a real-hardware benchmark. Build CI intentionally leaves + those fields `UNRECORDED` rather than inferring them. +- [x] Add per-stage HDL fast-path timing for direct/fallback source I/O, + prefetch consumer wait, HDD read/write, SIF DMA and EE consumer work. +- [x] Record useful bytes, SIF DMA bytes, HDD/source sectors, fallback bytes and + EE cache-maintenance bytes. +- [x] Add p50/p95/p99/max reporting for I/O latency, not just average throughput. +- [x] Keep hot-path logging counter/histogram based; format only at phase exit. +- [x] Add an R5900 performance-counter harness using the EE Core Manual event + table and dedicated `mfpc/mtpc/mfps/mtps` instructions. The harness + preserves/restores prior counter state and has passed current-toolchain CI. +- [x] Preserve linker map, symbol sizes and optimization audit in CI artifacts. +- [x] Add a host parser that converts `HDDMAN.LOG` corpus-v2 performance records + to stable JSON and self-tests in CI. +- [x] Add a same-source profiling-on/profiling-off build pair for authoritative + instrumentation-overhead A/B on real hardware. +- [x] Isolate IOP objects by profile mode and reject byte-identical ON/OFF IRX + artifacts in CI. +- [x] Bind each benchmark provenance record to the final stripped ELF and + embedded `hdl_stream.irx` SHA-256 and byte size. +- [x] Add a host preflight verifier for the frozen hardware pair and emit an + interleaved eight-run sample template. +- [ ] Exercise the R5900 counter harness in a bounded hardware benchmark and + measure empty-scope overhead before instrumenting application kernels. + +### Phase-0 implementation notes + +The first EE profiler pass increased `execute_transaction()` from the audited +6420 B baseline to 7492 B under LTO. This was rejected. Profiling helpers were +then isolated with selective `noinline` and report paths with `cold,noinline`. +The instrumented transaction body later became 6156 B after Phase-1 work while +retaining the counters. Static footprint alone is not a runtime-overhead claim. + +The IOP path records logarithmic microsecond latency histograms without `printf` +in the hot path. Current categories are: + +```text +usb-direct-read +source-fallback-read +prefetch-consumer-wait +hdd-write +hdd-read +sif-dma-completion +``` + +EE-side companion categories are: + +```text +pump-ioctl +source-ioctl +target-ioctl +copy-ee-consumer +verify-ee-consumer +``` + +The pinned `ps2dev/ps2dev:v2.0.0` image is tied to the `ps2dev` v2.0.0 build +bundle. Its tagged build passes `v2.0.0` to the PS2SDK build script, which checks +out PS2SDK `v2.0.0`; that annotated tag resolves to commit +`b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b`. CI records that exact source +provenance rather than substituting current PS2SDK master. + +The Phase-0 same-source A/B switch is `HDL_PROFILE=1/0`. PROFILE OFF compiles +EE/IOP latency timers, histograms, traffic counters and phase-end telemetry out +while preserving pump/prefetch, SIF DMA, EE cache maintenance, SHA verification, +journal, flush and metadata durability paths. + +An early A/B pair through CI #661 was invalid for IOP profiler-overhead work. +PS2SDK's IOP build rules store objects in `obj/`, while the old top-level clean +did not remove that directory. PROFILE OFF therefore reused the PROFILE ON +`hdl_stream.o`, producing a byte-identical embedded IRX. The corrected module +build makes the profile mode part of the object path as `obj/profile-0/` and +`obj/profile-1/`, removes the object tree in the module clean target, and CI now +fails if the two IRX files compare equal. + +The frozen hardware pair is CI #666 at project commit +`7875b14d837d6332f5edc37f1c12a55527d7dd87` with PS2SDK +`b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b` and GCC 15.2.0: + +```text + PROFILE ON PROFILE OFF delta OFF vs ON +stripped ELF 638388 632884 -5504 B +EE named text 232780 229956 -2824 B +EE named functions 618 609 -9 +EE instructions 58246 57539 -707 +execute_transaction() 6156 6156 0 +execute_transaction insn 1540 1540 0 +hdl_stream.irx file 9861 8405 -1456 B +hdl_stream.irx .text 8595 7139 -1456 B +hdl_stream.irx .data 144 144 0 +``` + +Frozen hashes: + +```text +PROFILE ON ELF 964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7 +PROFILE OFF ELF 4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916 +PROFILE ON IRX 8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db +PROFILE OFF IRX f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 +``` + +These are static deltas only. The authoritative overhead result remains the +interleaved real-console wall-time comparison defined in +`docs/PHASE0_HARDWARE_AB_PROTOCOL.md`. Before a hardware run, +`tools/phase0_profile_pair_preflight.py` verifies that the selected ELF files +match the frozen pair and can emit the eight-run sample template. + +Exit gate: measurements are reproducible on at least one real console and the +instrumented build has a documented overhead A/B against its same-source +PROFILE-OFF companion. + +## Phase 1: remove known unnecessary code/work + +- [x] Evaluate replacing the broad `draw2d` dependency with minimum GIF + primitives. The tested shim was rejected because final `.text` grew by + 480 B; the existing dependency remains the smaller measured result. +- [x] Audit formatted-I/O callsites and replace the unused floating formatter + dependency with the bounded integer-only formatting contract guarded by CI. +- [x] Audit current PS2SDK fileXio/Newlib timestamp and timezone glue. Keep direct + fileXio application semantics, remove unused local-time bootstrap work, and + guard the source contract in CI. +- [x] Remove source-level work duplicated across transaction stages where the + result can safely remain under the same ownership/lifetime. The normal + metadata-commit path now reuses its canonical 1024-byte block for read-back. + +Exit gate: same functional output, smaller ELF/hot code footprint, no regression +in hardware smoke tests. + +## Phase 2: I-cache and control-flow locality + +- [ ] Split `execute_transaction()` into state/stage handlers without changing + transaction semantics or recovery guarantees. +- [ ] Split other measured multi-kilobyte controller functions only where the + active path benefits from smaller working sets. +- [ ] Evaluate `-Os` for cold translation units and retain `-O2` for measured hot + paths; compare size and latency before adopting per-TU flags. +- [ ] Audit compiler-generated 64-bit divide/mod helpers and eliminate only cases + whose arithmetic contract proves a cheaper transformation correct. + +Current note: the isolated resume-hash experiment grows `execute_transaction()` +to 6556 B in PROFILE OFF and 6572 B in PROFILE ON, so the I-cache concern remains +real. Phase 2 is nevertheless held behind the Phase-0/Phase-1 real-hardware gate +rather than treating static size as a timing result. + +Exit gate: smaller active I-cache footprint plus equal correctness/error paths. + +## Phase 3: storage, APA and HDL dataflow + +- [x] Describe APA catalogue, ISO source, HDL transaction, SHA checkpoint and + payload stream with producer/consumer/lifetime/ownership contracts in + `docs/HDL_DATAFLOW_CONTRACTS.md`. +- [x] Audit recovery producer lifetime and isolate the resume-hash checkpoint + experiment. A matching COPY checkpoint removes prefix replay; a matching + full `PAYLOAD_VERIFIED` checkpoint also removes source reopen/fingerprint/ + ISO-probe work while retaining full HDD SHA-256 read-back. +- [ ] Evaluate a persistent compact HDL catalogue index only after identifying a + cheap, trustworthy drive/APA mutation-generation signal. If validation + requires the same full chain walk, retain the current lazy session cache + instead of adding a cache that merely moves work around. +- [ ] Keep large sequential transfers and persistent descriptors; avoid repeated + small fileXio/RPC control-plane operations outside semantically required + journal/checkpoint boundaries. +- [ ] Re-measure USB source, HDD target and verification independently. +- [ ] Tune chunk/batch size only with a sweep on the same device/workload. +- [x] Audit checkpoint/journal small-file frequency for the current recovery + experiment: the 256-byte sidecar is written at the existing 32 MiB journal + boundary plus orderly cancel, not per 64 KiB payload chunk. Broader HDD/ + filesystem flush-frequency changes remain measurement-gated. + +### Phase-3 implementation notes + +The current experiment identity is CI #706 at project commit +`a43b073c32348e020c234fff64615c8c4cddc98d`. The frozen baseline remains CI +#666. The matched experiment keeps `hdl_stream.irx` byte-identical to its +corresponding frozen PROFILE mode, so the resume-hash/source-lifetime experiment +is EE-only. + +`docs/HDL_RESUME_HASH_BENCHMARK.md` is the hardware correctness/performance gate. +Its PROFILE OFF pair is the release-like acceptance pair; PROFILE ON exists for +USB/HDD/SIF/EE attribution. The experiment remains **HIPOTEZA DO TESTU** until +real hardware validates recovery gain, uninterrupted-install regression and +crash-window behaviour. + +Exit gate: lower non-hideable storage time without weakening journal or metadata +commit safety. + +## Phase 4: IOP/SIF service architecture + +- [x] Instrument queue-adjacent prefetch wait, service/device work and SIF + completion independently enough to locate the dominant 64 KiB stage. +- [x] Maintain the IOP-local producer path where the final device consumer is on + the IOP. Current fast COPY reads into IOP staging, writes directly to + ps2hdd, and sends only the EE SHA consumer copy over SIF. +- [ ] Keep control metadata coarse-grained and bulk payload on DMA/data-plane paths. +- [ ] Express double buffering as explicit producer/consumer ownership states in + code only if the current semaphore/stage protocol needs further evolution; + the current ownership contract is documented in `HDL_DATAFLOW_CONTRACTS.md`. +- [ ] Evaluate triple buffering only if telemetry shows producer/consumer jitter + that a third slot can actually hide within the IOP RAM budget. +- [x] Add a static incremental IOP RAM budget for IRX sections, staging buffers, + fragment map, prefetch stack and compiler-observed stream-object size in + `docs/HDL_IOP_RAM_BUDGET.md`. System-wide free-memory headroom remains a + required real-runtime measurement before any buffer growth. +- [ ] Record active IOP module text/data/BSS, owned stacks and minimum free IOP + memory on real hardware before any triple-buffer/ring expansion. +- [ ] Sweep IOP worker priorities only after measuring service slack and stalls. + +Current known `hdl_stream` worst-case incremental subtotal for one active +installer stream is 192314 B in PROFILE OFF and 194458 B in PROFILE ON, excluding +ThreadMan control allocations and the rest of the active IOP runtime. Those +unknowns are deliberately not guessed. + +Exit gate: higher overlap/lower p99 with no IOP starvation or device regressions. + +## Phase 5: allocators, copies and lifetime + +- [ ] Inventory dynamic allocation by lifetime class: permanent, menu/session, + transaction, streaming, temporary and IOP service. +- [ ] Replace allocation churn only where traces show jitter/fragmentation/copy + amplification. Candidate structures are arenas, pools and aligned rings. +- [ ] Record `copies_per_payload`, `bytes_touched_per_payload` and DMA bytes for + storage chunks and large recovery/forensic records. +- [ ] Audit every explicit `aligned(64)` and document the concrete consumer + contract; remove or change alignment only with evidence. + +Exit gate: lower allocation/copy cost and no lifetime/ownership regressions. + +## Phase 6: GS/frontend + +- [ ] Preserve the real-hardware-proven 640x224 native coordinate contract. +- [ ] Measure GIF packet size, submission count, waits and framebuffer/VRAM use. +- [ ] Remove redundant FINISH/waits only after proving ownership/completion. +- [ ] Keep UI data GS-ready and avoid runtime repacking where a persistent + representation is simpler. +- [ ] Re-run the existing multi-mode real-hardware video regression after every + renderer synchronization or VRAM-layout change. + +Exit gate: equal visual correctness and mode stability with smaller CPU/packet +cost or smaller code footprint. + +## Phase 7: specialised hot kernels + +Only after the earlier phases have moved the bottleneck: + +- [ ] Benchmark SHA-256 and other remaining hot kernels with R5900 counters. +- [ ] Compare portable C, compiler output and a simple specialised baseline. +- [ ] Evaluate MMI only for regular packed data that actually dominates CPU time. +- [ ] Evaluate Scratchpad only for a bounded explicit working set with proven + transfer/ownership benefit. +- [ ] Do not introduce VU/VIF/IPU merely because the hardware exists; require a + fitting regular workload and end-to-end win including transport/sync. + +Exit gate: real-hardware A/B win including p50/p95/p99/max and correctness hash. + +## Benchmark record + +Every accepted optimization benchmark should record at least: + +```yaml +console_scp: +hardware_revision: +network_adapter: +storage_adapter: +ps2sdk_commit: +toolchain: +active_irx: +build_flags: +workload: +direction: +buffering: +alignment: +sample_count: +units: +correctness_hash: +p50: +p95: +p99: +max: +deadline_misses: +``` + +## Merge policy + +Each material optimization lands as a small reviewable commit with: + +1. bottleneck and evidence; +2. authoritative corpus/current source; +3. performance hypothesis; +4. smallest meaningful change; +5. correctness/error handling retained; +6. measurement method; +7. alignment/lifetime/thread-context risks; +8. simpler A/B baseline for aggressive changes. + +After every major optimization, whole-system profiling is repeated because the +bottleneck is assumed to have moved. diff --git a/docs/HDL_DATAFLOW_CONTRACTS.md b/docs/HDL_DATAFLOW_CONTRACTS.md new file mode 100644 index 00000000..e6ff498c --- /dev/null +++ b/docs/HDL_DATAFLOW_CONTRACTS.md @@ -0,0 +1,287 @@ +# HDL dataflow and ownership contracts + +This document records the current producer/consumer/lifetime contract for the +HDL installer and catalogue paths before further storage/SIF optimization. +It follows the PS2 Optimization Research Library v2 rule that representation, +transport and ownership must be explicit before buffering, caching or reuse is +introduced. + +The contracts describe the current branch, not an idealized redesign. Anything +not proven by current source or the project corpus is labelled as inference or a +hardware hypothesis. + +## Epistemic labels + +- **POTWIERDZONE**: current source/manual/corpus contract or real-hardware fact. +- **CURRENT IMPLEMENTATION**: behavior of this branch and the pinned PS2SDK + stack. +- **INFERENCJA**: engineering conclusion not yet measured. +- **HIPOTEZA DO TESTU**: change/result that requires real-PS2 measurement. + +## Dataset: USB ISO source + +```yaml +name: usb_iso_source +producer: mass:/ filesystem backed by USB mass-storage / BDM +consumers: + - ISO9660 + SYSTEM.CNF probe + - source fingerprint + - payload copy + - legacy resume-prefix/full-source SHA reconstruction +lifetime: selected install or resumed transaction while source data is required +representation: raw ISO bytes +alignment: + file: no global alignment contract + fast_transfer: 64-byte aligned EE destination, 512-byte transfer multiple +transport: + fallback: fileXio read through mass: filesystem + fast_path: IOP BDM fragment map -> IOP stage buffer -> HDD + SIF DMA to EE +batch_size: + payload: 64 KiB + fingerprint: first 64 KiB + last 64 KiB plus encoded source size + journal_progress: 16384 ISO sectors = 32 MiB +ownership_states: + - filesystem/device owns source media + - prefetch worker owns alternate IOP stage while read is active + - main IOP path owns completed stage while writing HDD / submitting SIF + - EE owns destination after synchronous ioctl/SIF completion +validation: + - expected file size + - source fingerprint + - ISO probe/startup identity + - full SHA-256 accumulated during fresh copy or rebuilt on legacy resume +``` + +**POTWIERDZONE:** fresh copy already hashes bytes while they are transported, so +it does not perform a second full USB pass merely to produce the source digest. +The resume-hash experiment removes the remaining replay where lifetime permits. + +**CURRENT IMPLEMENTATION:** a valid complete stage-4 checkpoint is being tested +as sufficient producer state for resumed HDD verification. In that case the +current source ISO is no longer a consumer dependency; invalid/missing state +falls back to opening and validating the ISO exactly as before. + +## Dataset: transaction journal + +```yaml +name: hdl_transaction_journal +producer: EE transaction state machine +consumers: + - incomplete transaction UI + - resume path + - incomplete-target cleanup guard +lifetime: from PLANNED until COMPLETE/removal +representation: fixed 512-byte versioned/checksummed transaction record +alignment: ordinary filesystem record; no DMA alignment contract +transport: mass:/ small-file write/read/rename +batch_size: one record at semantic stage transitions and every 32 MiB of copy +ownership_states: + - in-memory transaction is mutable only by current EE transaction + - HDLINSTALL.NEW is replacement candidate + - HDLINSTALL.TXN is authoritative persisted record selected by loader +validation: + - encode/decode contract + - read-back byte equality + - checksum + stage/progress invariants +``` + +The journal is transaction authority. Performance hints must never advance its +progress or replace its recovery semantics. + +## Dataset: optional SHA-state sidecar + +```yaml +name: hdl_source_sha_checkpoint +producer: SHA-256 state accumulated while source bytes are already moving +consumers: + - resumed COPYING prefix reconstruction + - resumed PAYLOAD_VERIFIED source-digest reconstruction +lifetime: same transaction only; deleted at zero-progress replacement/COMPLETE +representation: 256-byte version-1 record + SHA-256 over record payload +alignment: ordinary filesystem record; no DMA alignment contract +transport: mass:/ temp write -> readback/codec verify -> rename +batch_size: same existing 32 MiB transaction checkpoint boundary + orderly cancel +ownership_states: + - in-memory SHA context belongs to current EE copy/hash operation + - HDLINSTALL.SHN is candidate replacement + - HDLINSTALL.SHA is preferred hint + - journal remains authority even when sidecar is newer +validation: + - source byte count + - exact completed byte count + - source fingerprint + - target ID + - SHA context total/block state + - record SHA-256 +fallback: any failure -> legacy source rehash +``` + +**INFERENCJA:** because restore validates exact journal progress, a sidecar that +becomes visible ahead of its matching journal should be rejected against the +older journal rather than advance progress. + +**HIPOTEZA DO TESTU:** physical persistence across reset/power loss depends on +the actual USB/FAT/device stack. The source contract proves fail-safe matching; +it does not by itself prove media-level durability. + +## Dataset: APA/HDL catalogue + +```yaml +name: hdl_catalogue_snapshot +producer: raw APA chain walker +consumers: + - installed-games browser + - game details + - guarded delete selection + - planning free-space/target-collision checks in scan-only mode +lifetime: one installed-games menu session; rebuilt after successful deletion +representation: + catalogue: growable EE array of hdl_game_entry_t + metadata: lazy 1024-byte HDL metadata read, parsed into entry + SHA-256 +alignment: + raw APA header: 64-byte aligned EE buffer for raw HDD transfer/cache contract + metadata: 64-byte aligned local buffer on raw read paths +transport: raw HDD sector reads + selected fileXio control queries +batch_size: + APA: one 1024-byte header (2 HDD sectors) per chain node + game_metadata: one 1024-byte block per game, lazy by visible browser page +ownership_states: + - catalogue array belongs to installed-games menu invocation + - each metadata_state transitions from NOT_LOADED to one cached result + - delete path treats snapshot only as selection evidence and revalidates live HDD +validation: + - APA magic/checksum/start/prev/next/bounds/type/sub-count + - total/used/free sector accounting + - HDL metadata parse + metadata SHA-256 + - destructive delete performs live target/journal/snapshot checks again +``` + +**POTWIERDZONE:** metadata is already loaded lazily by visible page and cached in +the session entry. A persistent catalogue index therefore cannot be justified by +"avoiding all metadata reads"; the current path does not read every game metadata +block up front. + +**INFERENCJA:** a persistent index is useful only if its validity can be checked +substantially cheaper than the raw APA work it replaces. If proving that no +external APA mutation occurred requires walking the whole chain anyway, the +index may simply move work around. Any persistent-index implementation must +first identify a trustworthy drive/APA generation signal or define a clearly +bounded weaker cache contract with full-scan fallback. + +## Dataset: HDL payload stream + +```yaml +name: hdl_payload_stream +producer: USB source or HDD target depending operation +consumers: + copy: + - ps2hdd target write on IOP + - EE SHA-256 consumer + verify: + - EE SHA-256 consumer of HDD read-back +lifetime: one open hdl0: stream descriptor / transaction phase +representation: sequential 64 KiB payload chunks +alignment: + IOP_stage_allocator: AllocSysMemory allocation plus manual 64-byte alignment + EE_DMA_destination: required 64-byte aligned + transfer_size: required multiple of 512 B; DMA helper requires 64 B multiple +transport: + source_fast: BDM read into IOP stage + target_write: IOP stage -> ps2hdd + EE_copy: SIF DMA once per chunk + fallback: stock fileXio path +batch_size: 64 KiB +ownership_states: + - stage[0]/stage[1] are IOP-owned buffers + - prefetch worker owns requested alternate stage until done semaphore + - main IOP path owns current completed stage during HDD/SIF consumption + - SIF DMA must complete before current ioctl returns + - EE destination becomes usable after ioctl return + D-cache invalidation +validation: + - layout query matches partition plan + - transfer bounds and sector multiples + - full target SHA-256 after required flush +``` + +The IOP tries to allocate two 64 KiB staging buffers (+ alignment slop). If that +fails it falls back to one stage, preserving correctness while losing prefetch. +The second stage is therefore an optimization state, not an admission contract. + +The producer schedule for COPY is already: + +```text +obtain current source stage +submit/schedule next source read into alternate stage +write current stage to HDD +DMA current stage to EE +return to EE +EE consumes/hash current destination +request next chunk +``` + +This is already `submit early` for USB prefetch. + +## Current bottleneck candidates after ownership audit + +### 1. Redundant source replay on recovery + +Status: **implemented as isolated experiment; hardware gate pending.** + +This is the highest-priority work-removal candidate because it can eliminate a +prefix or complete ISO pass rather than merely shorten a control operation. + +### 2. Source reopening at PAYLOAD_VERIFIED with a complete checkpoint + +Status: **implemented in the isolated resume-hash experiment; CI/hardware gate +pending.** + +A matching full checkpoint already provides the original producer digest. HDD +read-back remains the correctness consumer; current USB media is unnecessary +unless checkpoint restore fails. + +### 3. Persistent HDL catalogue index + +Status: **design not yet justified.** + +Do not implement merely because persistent caches sound fast. The current +catalogue already caches metadata per menu session and loads it lazily. First +prove a cheap invalidation/generation contract that detects external APA +mutation without replaying the same full chain walk. + +### 4. SIF DMA completion wait / second EE destination + +Status: **defer until real-hardware attribution.** + +Current IOP `dma_to_ee()` submits `sceSifSetDma()` and waits for completion before +returning the ioctl. Simply removing that wait is invalid because EE currently +uses a single destination buffer and hashes it immediately after the synchronous +ioctl returns. + +A real overlap experiment therefore requires a new ownership protocol, for +example two EE consumer buffers or a ring with explicit FREE/FILLING/READY/ +CONSUMING states. It is only justified if PROFILE ON hardware telemetry shows +`sif-dma-completion` is materially exposed on the critical path after USB/HDD +work is accounted for. + +### 5. Journal/sidecar small-file metadata churn + +Status: **measure before redesign.** + +The optional sidecar adds a 256-byte write/readback/replace transaction on the +same 32 MiB boundary as the existing 512-byte journal. Combining records or +changing journal ABI to save these operations is a larger correctness change +than the current evidence justifies. Release-like A/B must first show a +meaningful uninterrupted-install regression attributable to checkpoint +maintenance. + +## Next measurement decisions + +1. Finish the resume-hash matched A/B and correctness matrix on real PS2. +2. Use PROFILE ON only to attribute USB/HDD/SIF/EE time; use PROFILE OFF for the + release-like acceptance result. +3. If SIF completion is materially exposed, design explicit EE/IOP ownership + before adding another buffer. +4. If catalogue entry latency is materially user-visible, identify a cheap and + correct mutation-generation contract before implementing persistent cache. +5. Re-profile whole-system behavior after any accepted change because the + bottleneck may move. diff --git a/docs/HDL_IOP_RAM_BUDGET.md b/docs/HDL_IOP_RAM_BUDGET.md new file mode 100644 index 00000000..960750ad --- /dev/null +++ b/docs/HDL_IOP_RAM_BUDGET.md @@ -0,0 +1,225 @@ +# HDL IOP RAM budget + +This document records the memory budget of the custom `hdl_stream` service +before any further buffering or SIF overlap experiment. + +The goal is not to pretend that the IOP has a free 2 MiB heap. The PS2 IOP RAM +also contains the kernel/runtime, every active IRX, stacks, heaps, filesystem and +device state. This document therefore separates the custom service's known +incremental footprint from system-wide memory that still requires real runtime +inventory. + +## Source-of-truth routing + +- `PS2_IOP_SIF_optimization_research_corpus_v2.md`: IOP RAM/service/thread/SIF + architecture and the requirement to budget text/data/BSS, stacks, heap + allocations and persistent buffers; +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md`: buffering as + explicit producer/consumer ownership rather than an arbitrary count of slots; +- `PS2_Whole_System_Scheduling_research_corpus_v2.md`: new buffering is accepted + only if it hides measured exposed latency without creating another resource + bottleneck; +- pinned PS2SDK `b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b`: current API/data contracts; +- project CI #706: final `hdl_stream.irx` loaded section sizes and compiler output + for the matched PROFILE pair. + +## Epistemic labels + +**POTWIERDZONE** + +- retail-class IOP RAM budget is about 2 MiB as routed by the project corpus; +- one `hdl_stream` open attempts a two-stage allocation first and falls back to + one stage if that allocation fails; +- each stage is 64 KiB; +- the two-stage allocation requests `2 * 64 KiB + 63` bytes and manually aligns + the first stage to 64 bytes; +- direct BDM source mapping permits at most 4096 fragments; +- pinned PS2SDK defines packed `bd_fragment_t` as one `u64 sector` plus one + `u32 count`, therefore 12 bytes per fragment; +- the dedicated prefetch worker stack is `0x1000` = 4096 bytes; +- the prefetch worker is created only when two stages are available; +- CI #706 reports final loaded `hdl_stream.irx` sections of 7139 B text + 144 B + data for PROFILE OFF and 8595 B text + 144 B data for PROFILE ON, with zero + reported BSS in those final IRX images; +- CI #706 disassembly shows the compiler-generated `AllocSysMemory` size for + `hdl_stream_file_t` is `0x288` = 648 B in PROFILE OFF and `0x538` = 1336 B in + PROFILE ON. + +**CURRENT IMPLEMENTATION** + +- application-side fast-I/O state tracks one active HDL target descriptor; +- the custom IOP driver itself is not a hard single-open API, so the numbers + below describe the current installer workload, not an enforcement guarantee + against another future caller opening multiple streams; +- the direct fragment map is allocated lazily and freed when source mapping is + reset or the stream closes; +- PROFILE ON stores IOP latency/traffic statistics inside the stream object; +- semaphores/thread-control objects are allocated by ThreadMan and are not + counted as exact bytes here because the project has not measured their runtime + allocator cost on the pinned IOP image. + +**INFERENCJA** + +- adding a third 64 KiB stage before telemetry demonstrates a producer/consumer + gap would consume a material fraction of the service's existing incremental + budget for an unproven benefit; +- the relevant safety margin is system-wide free IOP memory after all active IRX, + stacks and driver buffers, not simply `2 MiB - hdl_stream`. + +**HIPOTEZA DO TESTU** + +- two stages are sufficient for the current USB/HDD/SIF pipeline on real + hardware; +- a third stage is useful only if PROFILE ON shows repeatable prefetch misses or + exposed producer jitter that the extra ownership slot can hide. + +## Known allocation budget + +### Final IRX image + +CI #706: + +```text +PROFILE OFF loaded sections + .text 7139 B + .data 144 B + .bss 0 B + total 7283 B + +PROFILE ON loaded sections + .text 8595 B + .data 144 B + .bss 0 B + total 8739 B +``` + +The on-disk IRX file sizes are larger because they also contain module/ELF +structure. For RAM budgeting use loaded sections, not the archive/file size. + +### Streaming stages + +```text +one-stage allocation + 65536 + 63 = 65599 B + +two-stage allocation + 131072 + 63 = 131135 B +``` + +The 63-byte slop exists solely to obtain the documented 64-byte-aligned stage +address. It is allocator/alignment overhead, not a third payload buffer. + +### Direct-BDM fragment map + +Pinned PS2SDK contract: + +```text +sizeof(bd_fragment_t) = 8 + 4 = 12 B (packed) +maximum fragments = 4096 +maximum fragment map = 49152 B +``` + +This is a worst-case project cap. Ordinary contiguous or lightly fragmented ISO +files consume less. + +### Prefetch worker stack + +```text +HDL_STREAM_PREFETCH_STACK = 0x1000 = 4096 B +``` + +This stack exists only on the two-stage path because one-stage fallback does not +create the prefetch worker. + +### Stream object and ThreadMan bookkeeping + +The final CI #706 machine code exposes the exact size passed to IOP +`AllocSysMemory` for the stream object: + +```text +PROFILE OFF sizeof(hdl_stream_file_t) = 0x288 = 648 B +PROFILE ON sizeof(hdl_stream_file_t) = 0x538 = 1336 B +``` + +The PROFILE ON increase is expected because latency histograms and traffic +counters live inside the stream object. + +Thread and semaphore kernel-control allocations are still **UNMEASURED** and +must be added to the runtime inventory before claiming exact system free RAM. + +## Current incremental worst-case service envelope + +For one current installer stream, direct BDM available, two stages allocated: + +```text + PROFILE OFF PROFILE ON +IRX loaded sections 7283 B 8739 B +two-stage allocation 131135 B 131135 B +max fragment map 49152 B 49152 B +prefetch stack 4096 B 4096 B +stream object 648 B 1336 B + ---------- ---------- +known subtotal 192314 B 194458 B +ThreadMan control objects UNMEASURED UNMEASURED +other active IRX/runtime NOT INCLUDED NOT INCLUDED +``` + +The known subtotal is roughly 188-190 KiB. It is **not** a claim that only this +amount of IOP RAM is consumed by the whole application stack. + +For one-stage low-memory fallback, remove one 64 KiB stage and the dedicated +prefetch stack. Correctness remains available while overlap is reduced. + +## Buffer expansion gate + +Do not add triple buffering merely because 64 KiB appears small next to 2 MiB. +A third stage costs another 65536 B before any ownership metadata and can also +increase live working-set pressure in a memory already shared by USB, FATFS, +ps2hdd, DEV9 and other services. + +A third stage may be prototyped only if all of the following are true: + +1. PROFILE ON real-hardware data shows a repeatable exposed producer/consumer + stall, such as prefetch misses or wait tail latency that two slots cannot + hide; +2. the active IOP module/stack/heap inventory has been recorded for that test; +3. the experiment preserves a documented free-memory safety margin after the + extra stage is allocated; +4. the new slot has explicit ownership states and cannot be overwritten while + USB, HDD, SIF or EE consumption still owns it; +5. p50/p95/p99/max improve without device/audio/service regressions; +6. one-stage and two-stage fallback semantics remain correct. + +The numeric safety margin is intentionally not invented here. It must be based +on measured free IOP memory and peak transient demand on the actual active module +set. Until that measurement exists, the current two-stage design remains the +maximum accepted service allocation. + +## Required runtime inventory before Phase-4 buffer growth + +Record at least: + +```yaml +console_scp: +hardware_revision: +active_iop_modules: + - name: + text_bytes: + data_bytes: + bss_bytes: +owned_thread_stacks: +hdl_stream_profile_mode: +hdl_stream_stage_count: +hdl_stream_fragment_count: +free_iop_memory_before_stream: +free_iop_memory_after_stream_open: +free_iop_memory_at_peak_copy: +minimum_free_iop_memory_observed: +workload: +sample_count: +correctness_hash: +``` + +The project corpus specifically requires active module text/data/BSS and stack +sizes to be treated as performance data. This runtime inventory is therefore a +gate, not optional documentation polish. diff --git a/docs/HDL_RESUME_HASH_BENCHMARK.md b/docs/HDL_RESUME_HASH_BENCHMARK.md new file mode 100644 index 00000000..fabd9427 --- /dev/null +++ b/docs/HDL_RESUME_HASH_BENCHMARK.md @@ -0,0 +1,528 @@ +# HDL resume-hash checkpoint hardware benchmark + +This document defines the real-PS2 gate for the optional +`HDL_RESUME_HASH_CHECKPOINT=1` experiment. + +The experiment is deliberately not the default build. It tests the first rule +of the PS2 Optimization Research Library v2: remove work before trying to make +the same work faster. + +## Epistemic status + +**POTWIERDZONE** + +- the legacy COPY resume path reconstructs the SHA-256 state by rereading the + entire already-copied ISO prefix; +- a resumed `PAYLOAD_VERIFIED` transaction without a persisted source digest + performs another complete source-ISO hash pass; +- the normal transaction journal remains authoritative; +- the optional checkpoint is a 256-byte authenticated sidecar bound to source + size, exact completed byte count, source fingerprint and target ID; +- a missing, corrupt or mismatching sidecar falls back to the old rehash path; +- host tests prove that restored SHA state produces the same final digest as an + uninterrupted hash and that altered checkpoint records are rejected; +- CI #706 builds matched PROFILE OFF and PROFILE ON experiment variants only + after the frozen Phase-0 pair passes exact SHA-256 validation; +- in CI #706 the checkpoint experiment changes only the EE application image: + its PROFILE OFF `hdl_stream.irx` is byte-identical to the frozen PROFILE OFF + IRX and its PROFILE ON IRX is byte-identical to the frozen PROFILE ON IRX; +- for a resumed `PAYLOAD_VERIFIED` transaction, a valid complete checkpoint is + restored before the source ISO is opened; the experiment can therefore skip + source reopen, size check, first/tail fingerprint reads and ISO identity probe; +- HDD payload SHA-256 read-back remains mandatory before metadata commit. + +**CURRENT IMPLEMENTATION** + +- project experiment identity: CI #706, head + `a43b073c32348e020c234fff64615c8c4cddc98d`; +- frozen baseline source identity remains CI #666 at + `7875b14d837d6332f5edc37f1c12a55527d7dd87`; +- transaction journal interval: 16384 ISO sectors = 32 MiB; +- release-like experiment: `HDL_PROFILE=0 HDL_RESUME_HASH_CHECKPOINT=1`; +- instrumented experiment: `HDL_PROFILE=1 HDL_RESUME_HASH_CHECKPOINT=1`; +- release-like baseline: frozen `HDL_PROFILE=0` ELF; +- instrumented baseline: frozen `HDL_PROFILE=1` ELF; +- the experiment writes, reads back, codec-verifies and renames a 256-byte + sidecar at existing 32 MiB transaction checkpoints and on orderly cancel; +- checkpoint state is written before the corresponding transaction journal; +- the sidecar is deleted when a new zero-progress transaction supersedes it and + when a transaction reaches COMPLETE; +- `tools/parse_hdl_perf.py` extracts checkpoint restore/fallback/write-failure, + transaction results and the stage-4 source-reopen-skip event from `HDDMAN.LOG`; +- `tools/resume_hash_ab_preflight.py` binds every hardware sample template to + exact baseline/experiment ELF and IRX identities from the CI artifact; +- `tools/compare_hdl_resume_hash_ab.py` separates safe fallback runs from valid + optimized restores and reports p50/p95/p99/max for supplied timing data; +- an accepted `payload_verified_resume` optimized sample must also report + `source_reopen_skipped=true`, so an older checkpoint ELF cannot masquerade as + the current experiment merely because it restored the same SHA state; +- pinned PS2SDK v2.0.0 `bdmfs_fatfs` maps `mass:` close to FatFs `f_close()` + under its filesystem lock; the wrapper itself does not add a separate + block-device flush operation after close. + +**INFERENCJA** + +- removing an N-byte prefix replay should save approximately N bytes of USB + traffic on a matching COPY resume; +- restoring a full-payload checkpoint should remove one complete source-ISO + pass before resumed HDD verification; +- restoring that checkpoint before source admission should additionally remove + source reopen, size/fingerprint and ISO-probe work from stage-4 recovery; +- because checkpoint state is bound to journal progress, loss of power after + checkpoint replacement but before the newer journal becomes visible should + make the checkpoint stale relative to the older journal and therefore reject + it, falling back to source rehash; +- the small sidecar operations may add measurable latency or jitter to an + uninterrupted install even though their byte volume is tiny. + +**HIPOTEZA DO TESTU** + +- recovery time improves materially on real PS2; +- the uninterrupted-copy regression from checkpoint maintenance is negligible; +- stage-4 recovery with a valid full checkpoint remains correct with the source + USB device absent because the HDD read-back is compared against the digest + accumulated while the original source bytes were copied; +- temp-file write/readback/rename plus the tested mass-storage stack provide + adequate persistence behaviour across orderly-cancel and guarded fault + windows on real hardware; +- no adapter/USB implementation exposes a correctness or persistence corner + case not covered by host tests. + +The sidecar is logically fail-safe in current source, but this document does +**not** call it physically durable across arbitrary power loss until the +real-hardware crash-window matrix demonstrates that property. `f_close()` is +not treated as a magical PC-style `fsync()` contract without source or hardware +evidence. No performance or power-loss durability claim is accepted before the +hardware gate passes. + +## Authoritative corpus rationale + +This test follows: + +- `PS2_PERFORMANCE_BIBLE.md`: remove work first; preserve ELF/map/toolchain, + correctness hash and benchmark provenance; distinguish instrumented and + release-like profiles; +- `PS2_HDD_APA_PFS_HDL_filesystem_optimization_research_corpus_v2.md`: keep HDL + raw/storage workloads separate, eliminate repeated reads before lower-level + acceleration, use deterministic checksum validation, and do not infer + throughput from interface headline rates; +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md`: reuse producer + state only when representation, lifetime and ownership remain valid for the + consumer; +- `PS2_Whole_System_Scheduling_research_corpus_v2.md`: treat USB, IOP, SIF and + storage as one producer/consumer path and report distribution/tail behaviour, + not only an average; +- `PS2_USB_1_1_optimization_research_corpus.md`: USB Full-Speed scheduling has + millisecond-scale granularity, so redundant source traffic belongs on the + critical recovery path rather than being dismissed as free background work; +- pinned PS2SDK source for `bdmfs_fatfs`: use the actual v2.0.0 filesystem close + path when reasoning about current implementation rather than generic POSIX + expectations. + +## Frozen and experiment identities + +The authoritative matched A/B identity is emitted by CI #706 for project head +`a43b073c32348e020c234fff64615c8c4cddc98d` in +`RESUME_HASH_AB_IDENTITY.json`. + +Artifact ZIP digest: + +```text +sha256:3623a52466ca99408163b1bcfca3c4b02b3e5e2f3a173b1e18eaa91c288adc36 +``` + +### PROFILE OFF: release-like pair + +Baseline: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF +bytes 632884 +sha256 4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916 +``` + +Experiment: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF +bytes 636340 +sha256 ab4dc7addd62e051a88a183dbe00fee8ba889a867549ee35c1781addfd60b0a5 +``` + +Both use the exact same PROFILE OFF `hdl_stream.irx`: + +```text +bytes 8405 +sha256 f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 +``` + +Static experiment delta against frozen PROFILE OFF: + +```text +stripped ELF +3456 B +EE named text +2624 B (229956 -> 232580) +EE named functions +4 (609 -> 613) +EE instructions +657 (57539 -> 58196) +execute_transaction() +400 B (6156 -> 6556) +execute_transaction insn +100 (1540 -> 1640) +``` + +### PROFILE ON: instrumented pair + +Baseline: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF +bytes 638388 +sha256 964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7 +``` + +Experiment: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF +bytes 641844 +sha256 0d27e82ba06f6a3d19d53e70427a397859ef2cead725c56bb8c25bc3458520dd +``` + +Both use the exact same PROFILE ON `hdl_stream.irx`: + +```text +bytes 9861 +sha256 8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db +``` + +Static experiment delta against frozen PROFILE ON: + +```text +stripped ELF +3456 B +EE named text +2640 B (232780 -> 235420) +EE named functions +4 (618 -> 622) +EE instructions +663 (58246 -> 58909) +execute_transaction() +416 B (6156 -> 6572) +execute_transaction insn +104 (1540 -> 1644) +``` + +The PROFILE ON/OFF static deltas differ slightly because profiler-side code +changes LTO placement/generation. The experiment remains EE-only because each +experiment IRX is byte-identical to the corresponding frozen baseline IRX. + +The stage-4 source-reopen cut added only 256 B of stripped ELF relative to the +previous valid CI #699 experiment, but the authoritative test identity is now +#706. Do not use #699 binaries for the source-reopen-skip claim. + +Do not rebuild and silently call a result one of these binaries. Exact SHA-256 +is the identity. If experiment source changes, CI emits a new experiment +identity while the frozen baseline hashes remain fixed. + +## Why there are two A/B pairs + +Use PROFILE OFF for the acceptance timing result. It is the release-like pair +and does not pay Phase-0 HDL profiler overhead. + +Use PROFILE ON for attribution. Both binaries in that pair contain the same IOP +latency/traffic profiler and the same PROFILE ON IRX, allowing the existing +`usb-direct-read`, `source-fallback-read`, `prefetch-consumer-wait`, `hdd-write`, +`hdd-read`, `sif-dma-completion`, `pump-ioctl`, `source-ioctl`, `target-ioctl`, +`copy-ee-consumer` and `verify-ee-consumer` categories to explain where time and +traffic changed. + +Never compare frozen PROFILE OFF directly against checkpoint PROFILE ON and call +that the checkpoint effect. That changes two variables at once. + +## Hardware record + +Record before the first measured run: + +```text +SCPH / hardware revision +ROMVER +HDD model or SSD model +network/HDD adapter model and revision +USB mass-storage device model +USB filesystem and relevant allocation state +PS2SDK commit +PS2DEV/toolchain version +active IRX set +PROFILE mode +baseline ELF SHA-256 +experiment ELF SHA-256 +hdl_stream.irx SHA-256 +source ISO SHA-256 +source ISO byte count +target HDD free-space/allocation state +video mode / launch method +``` + +Keep the same console, adapter, USB device, ISO, target disk, video mode and +launch method for the whole A/B block. + +The CI artifact contains: + +```text +BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_OFF.yml +BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_ON.yml +RESUME_HASH_AB_IDENTITY.json +RESUME_HASH_AB_PROFILE_OFF_TEMPLATE.json +RESUME_HASH_AB_PROFILE_ON_TEMPLATE.json +``` + +Fill hardware/runtime fields rather than inferring them from CI. + +## Correctness gate before timing + +For both binaries: + +1. start a fresh install; +2. cancel only through the normal guarded TRIANGLE path after at least one + transaction checkpoint; +3. restart and choose resume; +4. complete payload verification and metadata commit; +5. confirm the final installed game is catalogued with the expected startup and + title; +6. preserve the transaction/session log; +7. verify the same final source/payload correctness hash in both variants. + +For the CI #706 experiment add one stage-4 correctness case: + +8. create a valid `PAYLOAD_VERIFIED` recovery point, then remove the source USB + device before resuming EXP; a valid full checkpoint must allow HDD read-back, + metadata commit and COMPLETE without reopening the ISO. Run the equivalent + BASE case with source present because BASE has no persisted source digest. + +Any checksum mismatch, invalid target metadata, unexpected cleanup behaviour, +resume refusal, or stage-4 source-free recovery that skips mandatory HDD +verification invalidates the experiment. + +## Workload A: uninterrupted-install regression + +Purpose: measure the cost paid when recovery is never needed. + +Use one deterministic ISO large enough to cross many 32 MiB journal intervals. +Do not change the ISO between A and B. + +For each binary, perform at least eight interleaved fresh-install runs. Restore +an equivalent target state before each run. + +Recommended order, already emitted in the CI templates: + +```text +BASE, EXP, EXP, BASE, EXP, BASE, BASE, EXP +``` + +Measure separately where the existing logs permit: + +```text +copy phase elapsed time +payload verification elapsed time +total transaction elapsed time +copy throughput +checkpoint write failures +correctness result +``` + +Report p50, p95, p99 and max. With only eight initial samples the tail +percentiles are coarse; treat them as a smoke gate and collect a larger sample +set if the result is near the acceptance boundary. + +Do not accept an improvement in recovery if uninterrupted installs acquire a +large or erratic regression. + +For release-like acceptance data, use +`RESUME_HASH_AB_PROFILE_OFF_TEMPLATE.json`. For profiler attribution, repeat the +selected workload with the PROFILE ON template rather than mixing modes in one +comparator input. + +## Workload B: resumed COPY + +Purpose: measure removal of the already-copied-prefix replay. + +Choose three persisted resume depths separated across the ISO. Prefer journal +boundaries near approximately 25%, 50% and 75% of source progress. Record the +exact `completed_sectors` from the journal rather than assuming the requested +percentage was hit. + +For each depth: + +1. perform a fresh copy to the selected transaction checkpoint; +2. cancel normally; +3. preserve the journal and, for EXP, its checkpoint sidecar; +4. reboot/relaunch in the same way for each run; +5. start timing immediately before accepting resume; +6. stop the recovery-start metric when the first payload progress beyond the + stored `completed_sectors` is observed; +7. continue the install to completion for correctness. + +Derived redundant work for the legacy baseline is: + +```text +prefix_rehash_bytes = completed_sectors * 2048 +``` + +For a valid EXP checkpoint, expected skipped prefix bytes are the same number. +`tools/parse_hdl_perf.py` must report a `prefix_restores` record with that byte +count. If EXP reports a prefix fallback, classify that run separately. +`tools/compare_hdl_resume_hash_ab.py` deliberately excludes fallback runs from +the optimized timing distribution while preserving their count. + +Report for each depth: + +```text +resume-to-first-new-progress p50/p95/p99/max +full resumed-transaction p50/p95/p99/max +completed_sectors +prefix_rehash_bytes baseline +checkpoint_restored_bytes experiment +fallback count +correctness failures +``` + +## Workload C: resumed PAYLOAD_VERIFIED + +Purpose: measure removal of the complete extra USB source-hash pass and source +admission work when a complete checkpoint already represents the producer. + +Create a valid `PAYLOAD_VERIFIED` recovery point. For the ordinary matched A/B +run keep source state equivalent. Separately run the source-absent EXP +correctness case described above. + +Expected work difference: + +```text +baseline: + open/validate source ISO + one full source ISO SHA-256 pass + HDD payload verification + +EXP with valid complete checkpoint: + restore final source SHA state before source admission + skip source reopen/size/fingerprint/ISO probe + HDD payload verification +``` + +The HDD verification remains required in both variants. Do not count its time as +work removed by this experiment. + +For a valid optimized run: + +- `parse_hdl_perf.py` must report a `full_restores` record whose byte count equals + `source_bytes`; +- it must also report the source-reopen-skip event; +- the sample template must set `source_reopen_skipped=true`; +- `tools/compare_hdl_resume_hash_ab.py` rejects an accepted stage-4 EXP restore + that does not carry that flag. + +Full-source fallback is kept as a correctness-safe fallback sample, not as +evidence for optimized latency. + +Report: + +```text +resume-to-HDD-verification-start p50/p95/p99/max +full recovery p50/p95/p99/max +source ISO bytes +checkpoint restore/fallback status +source_reopen_skipped +correctness failures +``` + +## Crash-window analysis and hardware matrix + +Current save order at a copy checkpoint is: + +```text +update transaction.completed_sectors in RAM +encode checkpoint for that exact progress +write HDLINSTALL.SHN +read back and verify exact bytes + checkpoint codec +remove previous HDLINSTALL.SHA +rename HDLINSTALL.SHN -> HDLINSTALL.SHA +encode/write/readback/verify/replace transaction journal +``` + +A checkpoint can never advance journal progress. Restore validates the sidecar +against the transaction loaded from the authoritative journal, including exact +completed byte count, source fingerprint and target ID. + +**CURRENT IMPLEMENTATION:** the pinned PS2SDK v2.0.0 `bdmfs_fatfs` close path +calls FatFs `f_close()` while holding the filesystem lock. No separate explicit +block-device flush is added by that wrapper after close. + +**INFERENCJA:** if the new sidecar becomes visible but the newer journal does +not, the older journal progress makes the sidecar stale and restore is rejected. +If checkpoint replacement itself is lost/corrupt, the optimization is lost and +legacy rehash is the intended fallback. + +The actual mass-storage persistence semantics across reset/power loss are still +a hardware property. Test these states before promotion: + +```text +sidecar absent +sidecar checksum corrupt +sidecar from an earlier completed_sectors value +sidecar from a later completed_sectors value with older journal +sidecar from another source fingerprint +sidecar from another target ID +valid .SHA primary +valid temporary .SHN with primary absent +loss/reset around sidecar replacement where the guarded fault procedure permits +valid full stage-4 sidecar with source USB absent +``` + +Every invalid/mismatched case must fall back to the legacy rehash path and still +complete correctly. It must never advance transaction progress by itself. + +Use the existing guarded hardware fault-injection procedure only where its +safety preconditions are satisfied. Do not pull power during arbitrary APA +metadata writes merely to make the benchmark more exciting. The console has +suffered enough. + +## Host-side workflow + +After a hardware run, turn each preserved log into structured telemetry: + +```sh +python3 tools/parse_hdl_perf.py HDDMAN.LOG --output run.json +``` + +Fill one workload/depth per copy of the CI sample template. Do not combine COPY +resume at 25%, 50% and 75% in one comparator input. + +For stage-4 optimized samples copy the parser result into the sample record so +`source_reopen_skipped` is true only when the actual CI #706 event is present. +Do not infer the flag from the filename. + +Then compare against the identity emitted with the tested artifact: + +```sh +python3 tools/compare_hdl_resume_hash_ab.py \ + RESUME_HASH_AB_IDENTITY.json samples.json \ + --output result.json +``` + +The comparator rejects wrong ELF/IRX hashes, mixed PROFILE modes, mixed workload +IDs, mismatching correctness hashes, invalid restored-byte counts, missing +source-reopen-skip evidence for accepted stage-4 EXP samples, and too few +accepted EXP samples after fallback separation. + +## Acceptance rule + +Promote the checkpoint path only if all are true: + +1. zero correctness failures across the test matrix; +2. frozen baseline identity remains unchanged; +3. matching checkpoint resumes actually skip the expected redundant source + bytes; +4. valid `PAYLOAD_VERIFIED` EXP recovery demonstrably skips source reopen and + still performs the complete HDD verification; +5. COPY-resume and PAYLOAD_VERIFIED recovery show a clear real-hardware benefit; +6. uninterrupted-install p50/p95/p99/max do not show an unacceptable regression + or new long-tail spikes; +7. sidecar failure always degrades to the old safe rehash behaviour; +8. PROFILE ON attribution agrees with the release-like PROFILE OFF result rather + than revealing an unrelated IOP/device change; +9. power/reset testing does not demonstrate persistence behaviour that can + violate the transaction correctness contract. + +If the result is ambiguous, keep the flag off and collect more samples. A neat +architecture diagram is not a benchmark result. diff --git a/docs/HDL_SOURCE_FINGERPRINT_ALIGNMENT_EXPERIMENT.md b/docs/HDL_SOURCE_FINGERPRINT_ALIGNMENT_EXPERIMENT.md new file mode 100644 index 00000000..ee78d048 --- /dev/null +++ b/docs/HDL_SOURCE_FINGERPRINT_ALIGNMENT_EXPERIMENT.md @@ -0,0 +1,147 @@ +# HDL source fingerprint heap-alignment experiment + +This record binds the incremental Phase-5 experiment that keeps transaction +workspace v1 from CI #724 but replaces only `source_fingerprint()`'s +`memalign(64, 65536)` with ordinary `malloc(65536)`. + +## Source-of-truth routing + +- `PS2_Optimization_Library_v2_MANIFEST.md` +- `PS2_PERFORMANCE_BIBLE.md` +- `PS2_Memory_Allocators_optimization_research_corpus_v2.md` +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md` +- `PS2_PS2SDK_optimization_research_corpus_v2.md` +- pinned PS2SDK commit `b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b` +- `docs/ALIGNMENT_CONTRACT_AUDIT.md` + +## Contract + +**CURRENT IMPLEMENTATION:** pinned PS2SDK `fileXioRead()` accepts an arbitrary EE +caller buffer, performs cache writeback for the supplied range, and handles edge +bytes through its normal read/RPC residual path. The application pointer is not +required by the fileXio API to start on a 64-byte cache-line boundary. + +`source_fingerprint()` is consumed by ordinary source `fileXioRead()` plus EE +SHA-256. It is not the custom `hdl0:` fast SIF/DMA destination. + +Therefore: + +```text +source_fingerprint heap alignment + 64 B: not a correctness/API requirement + +transaction COPY/target-verify workspace + 64 B: KEEP, custom hdl_fast_dma_read() contract +``` + +This is deliberately not a global `memalign(64) -> malloc` policy. + +## Hypothesis + +**HIPOTEZA DO TESTU:** removing one unnecessary aligned allocation may reduce a +small amount of allocator work without changing source bytes, hashing or +ownership lifetime. + +Counter-hypothesis: an arbitrary `malloc()` address may make the two 64 KiB +fingerprint reads use fileXio's unaligned edge handling, so real hardware can be +neutral or worse even though the API permits it. + +## CI #739 identity + +Source point: + +```text +503ac0274b3cde18814cc7e1d170b9f656fb2615 +``` + +Artifact digest: + +```text +sha256:b958018ee0e7cc5ab184f430794d592ce430342c87faeaaaf1993e21ad6f1ff0 +``` + +Frozen workspace-v1 reference remains CI #724. + +### PROFILE OFF + +Workspace-v1 #724: + +```text +ELF 632756 B +sha256 23bbf6dfc28eb87bc7d484875a8940b9309eb5e3994d9c922388c9a0249415c6 +named text 229764 B +instructions 57491 +execute_transaction 6008 B / 1502 instructions +``` + +Fingerprint-malloc #739: + +```text +ELF 632756 B +sha256 97e2a802952ae6f3b46c9fa0148359db8f8b69e22923f8105378f094de59c28b +named text 229756 B +instructions 57488 +execute_transaction 6008 B / 1502 instructions +``` + +Incremental static delta: + +```text +ELF 0 B +named text -8 B +instructions -3 +execute_transaction 0 B / 0 instructions +``` + +### PROFILE ON + +Workspace-v1 #724: + +```text +ELF 638260 B +sha256 09185cd6a21bbb9990d0b7f8cfe70fa80b4e9ba01a00e9648cb9fa70d9b3d693 +named text 232560 B +instructions 58190 +execute_transaction 6008 B / 1502 instructions +``` + +Fingerprint-malloc #739: + +```text +ELF 638132 B +sha256 c8da50fe5147c3a24dc2f26d4ab910660bac615bf8e26f48e0bff3a2483f578b +named text 232552 B +instructions 58189 +execute_transaction 6008 B / 1502 instructions +``` + +Incremental static delta: + +```text +ELF -128 B +named text -8 B +instructions -1 +execute_transaction 0 B / 0 instructions +``` + +Both experiment IRX files remain byte-identical to the frozen Phase-0 pair: + +```text +PROFILE OFF f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 +PROFILE ON 8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db +``` + +## Decision + +**KEEP AS A HARDWARE HYPOTHESIS, NOT A MEASURED SPEEDUP.** + +Unlike workspace v2, this change does not lengthen a 64 KiB lifetime or grow the +hot transaction controller. Static code is neutral-to-smaller. That is enough to +keep the experiment available for real-console A/B, but not enough to promote it +to the default runtime. + +Acceptance requires the same source ISO and interleaved baseline/experiment +runs, with source-fingerprint and total transaction p50/p95/p99/max plus +correctness hash. If unaligned fileXio edge handling causes a repeatable tail or +wall-time regression, retain `memalign(64)` despite the absence of an API +requirement. diff --git a/docs/HDL_TRANSACTION_WORKSPACE_BENCHMARK.md b/docs/HDL_TRANSACTION_WORKSPACE_BENCHMARK.md new file mode 100644 index 00000000..16e08034 --- /dev/null +++ b/docs/HDL_TRANSACTION_WORKSPACE_BENCHMARK.md @@ -0,0 +1,282 @@ +# HDL transaction workspace hardware benchmark + +This document defines the real-PS2 acceptance gate for the isolated Phase-5 +transaction-workspace experiment. + +The experiment does not change the default runtime source. CI materializes a +temporary `transaction.inc` in which one 64 KiB / 64-byte-aligned EE buffer is +owned by `execute_transaction()` and borrowed sequentially by source-hash, +copy and HDD-verification helpers. + +## Source-of-truth routing + +- `PS2_Optimization_Library_v2_MANIFEST.md` +- `PS2_PERFORMANCE_BIBLE.md` +- `PS2_Memory_Allocators_optimization_research_corpus_v2.md` +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md` +- `PS2_Whole_System_Scheduling_research_corpus_v2.md` +- `PS2_IOP_SIF_optimization_research_corpus_v2.md` +- pinned PS2SDK `b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b` + +The relevant corpus rule is lifetime ownership, not a blanket preference for a +custom allocator. The 64-byte alignment remains because the current HDL fast +path explicitly requires it for the EE destination used by SIF DMA. + +## Epistemic status + +**POTWIERDZONE** + +- the frozen baseline allocates one 64 KiB aligned helper buffer for + `copy_payload()` and then a second, non-overlapping 64 KiB aligned helper + buffer for `verify_target_digest()` on a successful fresh COPY path; +- a resumed `PAYLOAD_VERIFIED` legacy/fallback path similarly allocates one + buffer for `hash_source_payload()` and a later one for target verification; +- these helper buffers are not live concurrently; +- `hdl_fast_dma_read()` requires the EE destination to be 64-byte aligned; +- CI #724 materializes one transaction-owned aligned buffer and passes it to the + three helpers without changing the IOP source, pump, SIF DMA, cache, + journal, flush, metadata or durability paths; +- CI #724 proves the experiment IRX is byte-identical to the corresponding + frozen PROFILE IRX. + +**CURRENT IMPLEMENTATION** + +- workspace bytes: 65536; +- workspace alignment: 64; +- owner: `execute_transaction()`; +- borrowers: `copy_payload()`, `hash_source_payload()`, + `verify_target_digest()`; +- successful fresh COPY+verify changes two phase-local `memalign/free` pairs to + one transaction-owned pair; +- successful resumed stage-4 hash+verify changes two phase-local pairs to one; +- allocation failure remains `HDL_INSTALL_MEMORY_FAILED`; +- all helper error returns still converge on transaction cleanup; +- peak payload workspace size remains one 64 KiB buffer in both baseline and + experiment. + +**INFERENCJA** + +- removing one general-heap allocation/free pair per successful bulk phase pair + should reduce allocator churn and may reduce small latency/jitter at the + copy->verify boundary; +- because the allocation count is per phase rather than per 64 KiB chunk, the + wall-time effect may be below storage noise; +- the static code reduction is valuable evidence that ownership centralization + did not trade allocator churn for I-cache growth, but it is not a runtime + speedup measurement. + +**HIPOTEZA DO TESTU** + +- real-PS2 transaction latency or tail jitter improves measurably, or at minimum + does not regress while correctness remains identical; +- retaining the workspace until transaction cleanup does not create a harmful + EE heap-pressure interaction in the tested transaction path. + +## Frozen baseline and CI #724 experiment identity + +Frozen Phase-0 source point: + +```text +7875b14d837d6332f5edc37f1c12a55527d7dd87 +``` + +CI #724 experiment/materializer source point: + +```text +5c2b4d34a7cee05f330be0f6de2d44552d46a136 +``` + +PS2SDK: + +```text +b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b +``` + +Toolchain: + +```text +mips64r5900el-ps2-elf GCC 15.2.0 +ps2dev/ps2dev:v2.0.0 +``` + +### PROFILE OFF, release-like acceptance pair + +Baseline: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF +bytes 632884 +sha256 4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916 +``` + +Experiment: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF +bytes 632756 +sha256 23bbf6dfc28eb87bc7d484875a8940b9309eb5e3994d9c922388c9a0249415c6 +``` + +Both use: + +```text +hdl_stream.irx +bytes 8405 +sha256 f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 +``` + +Static delta: + +```text +stripped ELF -128 B +.text -192 B (230440 -> 230248) +EE named text -192 B (229956 -> 229764) +EE named functions 0 (609 -> 609) +EE instructions -48 (57539 -> 57491) +execute_transaction() -148 B (6156 -> 6008) +execute_transaction insn -38 (1540 -> 1502) +``` + +### PROFILE ON, attribution pair + +Baseline: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF +bytes 638388 +sha256 964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7 +``` + +Experiment: + +```text +PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF +bytes 638260 +sha256 09185cd6a21bbb9990d0b7f8cfe70fa80b4e9ba01a00e9648cb9fa70d9b3d693 +``` + +Both use: + +```text +hdl_stream.irx +bytes 9861 +sha256 8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db +``` + +Static delta: + +```text +stripped ELF -128 B +.text -224 B (233280 -> 233056) +EE named text -220 B (232780 -> 232560) +EE named functions 0 (618 -> 618) +EE instructions -56 (58246 -> 58190) +execute_transaction() -148 B (6156 -> 6008) +execute_transaction insn -38 (1540 -> 1502) +``` + +The PROFILE OFF/ON difference outside `execute_transaction()` is compiler/LTO +layout around profiler-enabled code. The transaction function itself has the +same static reduction in both variants. + +CI #724 artifact digest: + +```text +sha256:cddf67a31e55e50e23bd6f98ce30739c3a93fff37e3a87d8096c8f9c77416c17 +``` + +Exact binary identity is mandatory. Do not rebuild later and call the result +this A/B pair merely because the source looks equivalent. + +## Correctness gate + +Before timing, both baseline and experiment must complete the same functional +matrix: + +1. fresh install from zero progress; +2. ordinary guarded cancel during COPYING; +3. resume from COPYING; +4. resume from persisted PAYLOAD_VERIFIED; +5. mandatory HDD SHA-256 read-back succeeds; +6. metadata commit + read-back succeeds; +7. final game metadata/startup/title are identical; +8. transaction journal reaches COMPLETE/removal exactly as baseline; +9. memory-allocation failure injection, where practical in host/test scaffolding, + still maps to `HDL_INSTALL_MEMORY_FAILED` without leaking a target/source FD. + +Any correctness mismatch rejects the experiment before performance data is +considered. + +## Hardware workload + +Use the PROFILE OFF pair for acceptance and PROFILE ON only for attribution. +Keep console, HDD, adapter, USB source, ISO, video mode and launch method fixed. + +Use at least eight interleaved successful fresh transactions with an equivalent +target state restored between runs: + +```text +BASE, EXP, EXP, BASE, EXP, BASE, BASE, EXP +``` + +The CI artifact contains: + +```text +TRANSACTION_WORKSPACE_AB_IDENTITY.json +TRANSACTION_WORKSPACE_AB_PROFILE_OFF_TEMPLATE.json +TRANSACTION_WORKSPACE_AB_PROFILE_ON_TEMPLATE.json +``` + +Record: + +```yaml +console_scp: +hardware_revision: +romver: +storage_adapter: +hdd_model: +usb_device: +active_irx: +source_iso_sha256: +source_iso_bytes: +profile_mode: +baseline_elf_sha256: +experiment_elf_sha256: +hdl_stream_irx_sha256: +sample_count: +correctness_hash: +``` + +For each run record at least: + +```text +copy elapsed us +verify elapsed us +total transaction elapsed us +result +correctness hash +``` + +Report p50, p95, p99 and max. Eight runs are a smoke gate; if the delta is near +noise, collect more samples instead of manufacturing certainty from two decimal +places. + +## Acceptance rule + +Promote the ownership change only if all are true: + +1. zero correctness regressions in the functional matrix; +2. exact frozen baseline and experiment identities match this document; +3. experiment IRX remains byte-identical to baseline for each PROFILE mode; +4. peak payload workspace does not exceed the baseline one-buffer 64 KiB peak; +5. real hardware shows no meaningful p95/p99/max regression; +6. any claimed latency/jitter improvement is repeatable rather than a single-run + difference; +7. no new allocation failure/error-path leak appears; +8. after promotion, whole-system profiling is repeated because the bottleneck + may move. + +If timing is indistinguishable but correctness is equal and the smaller code / +clearer ownership are considered sufficient maintenance benefits, that may +justify a code-quality promotion. In that case document it explicitly as a +static/lifetime improvement, not as a measured performance speedup. diff --git a/docs/HDL_TRANSACTION_WORKSPACE_V2_REJECTION.md b/docs/HDL_TRANSACTION_WORKSPACE_V2_REJECTION.md new file mode 100644 index 00000000..abd55aae --- /dev/null +++ b/docs/HDL_TRANSACTION_WORKSPACE_V2_REJECTION.md @@ -0,0 +1,144 @@ +# HDL transaction workspace v2 experiment rejection + +This record preserves the second Phase-5 allocator/lifetime experiment so it is +not rediscovered later as an apparently new optimization. + +## Source-of-truth routing + +- `PS2_Optimization_Library_v2_MANIFEST.md` +- `PS2_PERFORMANCE_BIBLE.md` +- `PS2_Memory_Allocators_optimization_research_corpus_v2.md` +- `PS2_Data_Oriented_Design_optimization_research_corpus_v2.md` +- `docs/HDL_TRANSACTION_WORKSPACE_BENCHMARK.md` + +## Hypothesis + +Workspace v1, frozen at CI #724, changes the mutually-exclusive bulk helpers: + +```text +copy_payload / hash_source_payload -> verify_target_digest +``` + +from two separate 64 KiB `memalign(64)` lifetimes to one transaction-owned +workspace. + +V2 tested extending that ownership backwards through `execute_transaction()` +source admission so the transaction's `source_fingerprint()` also borrowed the +same 64 KiB buffer. The pre-confirmation UI fingerprint deliberately retained +its short helper-owned allocation. + +Expected transaction allocation policy: + +```text +baseline transaction + source fingerprint alloc/free + copy or source-hash alloc/free + HDD verify alloc/free + +v1 + source fingerprint alloc/free + transaction workspace alloc + copy or source-hash + HDD verify + transaction workspace free + +v2 + transaction workspace alloc + source fingerprint + partition/open path + copy or source-hash + HDD verify + transaction workspace free +``` + +## Epistemic status + +**POTWIERDZONE** + +- current pinned fileXio source does not require the source-fingerprint caller + buffer to be 64-byte aligned; +- v2 preserves the existing 64-byte transaction workspace alignment required by + the custom EE/SIF fast path later in COPY/verify; +- v2 does not change the IOP source or transport; +- CI #733 completed host tests, EE/IOP builds, frozen identity checks, + resume-hash build and artifact upload successfully; +- both v2 experiment IRX files are byte-identical to the corresponding frozen + Phase-0 IRX files. + +**CURRENT IMPLEMENTATION / CI #733** + +Source point: + +```text +98988fcdca78dd392f95aa40d5b61157ac0bea27 +``` + +Artifact digest: + +```text +sha256:8b69850212fe9928d16efb3d4d7610a3b3c2576e4917ff25cd306380ebbcdbe8 +``` + +PROFILE OFF v2: + +```text +ELF bytes 632756 +ELF sha256 b8865fe6a95a3d0e7d54fb0b72519a68f080eb234f6b6167ad55467c3faed86f +named text 229804 B +instructions 57500 +execute_transaction 6032 B / 1508 instructions +IRX sha256 f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 +``` + +PROFILE ON v2: + +```text +ELF bytes 638260 +ELF sha256 c6865cace130952befa06c2c06fa43485973f08ca14847c322578c5847f61d66 +named text 232600 B +instructions 58201 +execute_transaction 6032 B / 1508 instructions +IRX sha256 8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db +``` + +## V2 versus v1 #724 + +```text + v1 #724 v2 #733 v2 delta +PROFILE OFF ELF 632756 632756 0 B +PROFILE OFF named text 229764 229804 +40 B +PROFILE OFF instructions 57491 57500 +9 +PROFILE ON ELF 638260 638260 0 B +PROFILE ON named text 232560 232600 +40 B +PROFILE ON instructions 58190 58201 +11 +execute_transaction 6008 6032 +24 B +execute_transaction instructions 1502 1508 +6 +removed alloc/free pairs vs baseline 1 2 +1 +``` + +## Decision + +**REJECTED AS THE ACTIVE STATIC OPTIMIZATION.** + +The additional source-admission reuse removes one more general-heap alloc/free +pair per transaction, but that pair is not inside the 64 KiB chunk loop. V2 also: + +- lengthens the 64 KiB workspace lifetime across source admission and partition + creation/opening; +- increases `execute_transaction()` by 24 bytes / 6 instructions compared with + v1; +- increases whole EE named text by 40 bytes; +- gives no final stripped-ELF size reduction beyond v1. + +The corpus prioritizes shortest correct lifetime and measured cost rather than +minimum textual allocator-call count. With no real-hardware evidence that one +additional per-transaction `memalign/free` is material, v1 has the better static +tradeoff. + +V2 remains **HIPOTEZA DO TESTU** only if future allocator profiling shows the +source-admission allocation itself contributes measurable latency/jitter or heap +fragmentation. Until then CI should materialize v1 for the active workspace A/B. + +This is not evidence that v2 is slower on hardware. It is evidence that the +available static data do not justify paying its longer lifetime and larger hot +controller solely to remove one unmeasured allocation pair. diff --git a/docs/PHASE0_HARDWARE_AB_PROTOCOL.md b/docs/PHASE0_HARDWARE_AB_PROTOCOL.md new file mode 100644 index 00000000..af0d5792 --- /dev/null +++ b/docs/PHASE0_HARDWARE_AB_PROTOCOL.md @@ -0,0 +1,311 @@ +# Corpus v2 Phase 0 real-hardware A/B protocol + +Phase 0 exits only after the measurement build itself has been measured on a +real PlayStation 2. PCSX2 may be used for correctness/debugging but is not an +arbiter for EE cache, IOP scheduling, USB service latency, SIF DMA or DEV9 +throughput. + +## Frozen hardware-test pair + +The binary identity frozen by CI run #666 originates from project head: + +```text +frozen_source_git_sha 7875b14d837d6332f5edc37f1c12a55527d7dd87 +project_git_ref perf/corpus-v2-integration +ps2sdk_commit b12f8af37bd42ec13b1bafb7ab6e7bdcfb4b683b +toolchain mips64r5900el-ps2-elf GCC 15.2.0 +container ps2dev/ps2dev:v2.0.0 +``` + +PROFILE ON: + +```text +ELF bytes 638388 +ELF SHA-256 964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7 +IRX bytes 9861 +IRX text 8595 +IRX data 144 +IRX SHA-256 8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db +``` + +PROFILE OFF: + +```text +ELF bytes 632884 +ELF SHA-256 4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916 +IRX bytes 8405 +IRX text 7139 +IRX data 144 +IRX SHA-256 f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751 +``` + +The exact ELF/IRX hashes define the frozen binary pair. Later commits that touch +only documentation or host-side tooling may be rebuilt by CI into byte-identical +ELFs. In that case the CI provenance correctly records the newer artifact +`project_git_sha`, while the binary identity remains the hash pair above. A run +record must therefore preserve both: + +1. the `project_git_sha` from the artifact actually tested; +2. the mode-specific frozen ELF/IRX hashes. + +Do not rewrite the artifact SHA to `7875b14...` merely because the bytes match +the original CI #666 pair. + +The static footprint delta is evidence that the compiled-out path is real; it is +**not** a runtime speedup measurement. CI also enforces that the ON/OFF +`hdl_stream.irx` files are not byte-identical. + +An earlier pair through CI #661 is invalid for IOP profiler-overhead measurement: +PS2SDK's IOP rules place objects under `obj/`, while the old top-level clean did +not remove that directory. PROFILE OFF therefore reused the previously compiled +PROFILE ON `hdl_stream.o`. The corrected module Makefile makes the profile mode +part of the dependency graph with `obj/profile-0/` and `obj/profile-1/`, and its +own clean removes the entire object tree. Do not use the old OFF ELF/IRX as an +authoritative profiler-overhead sample. + +## Compared builds + +The authoritative instrumentation-overhead comparison is a same-runtime-source +pair built from one green artifact head with the same PS2DEV container, PS2SDK +source, optimization flags and runtime implementation. The frozen hashes above +are the final guard against accidentally testing a different binary. + +### A: PROFILE OFF + +Build the selected commit with: + +```text +HDL_PROFILE=0 +``` + +This compiles the Phase-0 EE/IOP HDL telemetry out while retaining the same: + +- direct-BDM source path; +- double-buffer prefetch worker and ownership flow; +- ps2hdd/HIOCTRANSFER target path; +- SIF DMA transfers; +- EE cache writeback/invalidation required for DMA correctness; +- SHA-256 verification; +- journal, flush, metadata commit and read-back semantics. + +`HDL_STREAM_IOCTL2_GET_FAST_STATS` remains ABI-compatible and returns a zeroed +record, but the timed hot-path accounting itself is absent. + +### B: PROFILE ON + +Build the exact same artifact head with: + +```text +HDL_PROFILE=1 +``` + +This is the diagnostic build and retains: + +- EE pump/source/target latency histograms; +- IOP direct-source/fallback/prefetch/HDD/SIF histograms; +- useful/DMA/cache-maintenance/fallback traffic counters; +- benchmark provenance artifact; +- linker/ELF audit artifacts. + +CI emits both ELFs, both embedded `hdl_stream.irx` variants, linker maps, +optimization audits, binary hashes and provenance records from one checkout and +one toolchain image. The profile mode and the final ELF/IRX hashes are written +explicitly to each provenance file. + +### Historical pre-instrumentation reference + +Commit `4b5aa8d85e86c9de570a2128b52d1eaa5b334844` remains useful as the original +known-good HDL installer baseline, but it is **not** the authoritative profiler +overhead A/B. Runtime and Phase-1 policy changes after that commit make such a +cross-commit timing delta confounded. + +## Preflight before copying to the console + +Run the host-side guard on the exact two ELF files selected for the test: + +```text +python3 tools/phase0_profile_pair_preflight.py \ + --profile-on PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF \ + --profile-off PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF \ + --project-git-sha \ + --output-template PHASE0_AB_SAMPLES_TEMPLATE.json +``` + +The command fails if either ELF size or SHA-256 differs from the frozen pair. It +also creates the eight-run interleaved sample template with the artifact SHA and +mode-specific ELF/IRX hashes already filled in. Do not hand-edit those hashes. + +After copying the files to the launch medium, verify the ELF hashes there as +well. A filename such as `PROFILE_ON.ELF` is not provenance. + +## Hardware provenance + +Record before each pair of runs: + +```yaml +console_scp: +hardware_revision: +romver: +storage_adapter: +usb_device: +ps2sdk_commit: +toolchain: +active_irx: +build_flags: +hdl_profile_enabled: +project_git_sha: +benchmark_elf_sha256: +hdl_stream_irx_sha256: +workload: +direction: +buffering: +alignment: +sample_count: +units: +correctness_hash: +``` + +The CI-generated `BENCHMARK_PROVENANCE_PROFILE_OFF.yml` and +`BENCHMARK_PROVENANCE_PROFILE_ON.yml` supply build-side fields, including exact +artifact project SHA, ELF/IRX hashes and sizes. Hardware fields remain explicit +manual measurements rather than guessed metadata. + +## Workload contract + +Use the same: + +- console and adapters; +- HDD contents/layout before each timed run where practical; +- USB device, filesystem and USB port; +- ISO file and fragment layout; +- source direction and target operation; +- video mode and active background services; +- cold/warm policy. + +Use the generated eight-run order: + +```text +OFF, ON, ON, OFF, ON, OFF, OFF, ON +``` + +rather than running every sample of one build first. This distributes both modes +through the session and reduces temperature, device-state and session-order +bias. Recreate or otherwise control the target layout between destructive +install samples so the compared workload stays meaningfully equivalent. + +For HDL copy throughput, use one ISO large enough that startup/allocation noise +is negligible compared with the bulk copy phase. Do not compare different ISOs, +USB sticks or HDD layouts and call the result an instrumentation delta. + +## Measurements + +For **both** builds record at least: + +- total install wall time; +- bulk copy wall time and useful KiB/s; +- payload verification wall time; +- final correctness result/hash; +- any cancellation, journal or metadata-commit failure. + +For PROFILE ON additionally retain: + +- p50, p95, p99 and max EE pump ioctl latency; +- p50, p95, p99 and max IOP direct-source latency; +- p50, p95, p99 and max prefetch consumer wait; +- p50, p95, p99 and max HDD write latency; +- p50, p95, p99 and max SIF DMA completion latency; +- prefetch hit/miss counts; +- fallback-source/fallback-target bytes; +- useful payload, SIF DMA and EE cache-maintenance bytes. + +For the overhead result report PROFILE ON relative to PROFILE OFF for wall time +and throughput. Do not fabricate p95/p99 for PROFILE OFF from absent telemetry; +the point of the OFF build is to remove that instrumentation. + +Run at least four complete comparable samples per mode. Add samples if wall time +or PROFILE ON tail latency is unstable. + +## Host comparison record + +Use the `PHASE0_AB_SAMPLES_TEMPLATE.json` emitted by preflight. Each sample has +this shape: + +```json +{ + "run": 1, + "mode": "OFF", + "project_git_sha": "", + "benchmark_elf_sha256": "4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916", + "hdl_stream_irx_sha256": "f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751", + "workload_id": "", + "correctness_hash": "", + "source_bytes": 0, + "total_us": 0, + "copy_us": 0, + "verify_us": 0 +} +``` + +`copy_us` and `verify_us` are optional if the external measurement setup cannot +isolate those phases. `source_bytes` and `total_us` are mandatory. Replace every +zero placeholder before comparison, otherwise input validation rejects it. + +Compare the completed record with: + +```text +python3 tools/compare_hdl_profile_ab.py PHASE0_AB_SAMPLES_TEMPLATE.json \ + --output profile-ab.json +``` + +The comparator refuses: + +- mixed artifact project SHAs; +- an ELF or IRX hash not belonging to the frozen mode-specific pair; +- mixed workloads; +- differing correctness hashes; +- differing source sizes; +- fewer than four samples of either mode by default. + +It reports p50/p95/p99/max for available wall-time metrics plus signed PROFILE +ON vs OFF percent deltas. Throughput is derived from the recorded bytes and +time. Rich EE and IOP latency distributions remain sourced from +`parse_hdl_perf.py` for PROFILE ON only. + +## Phase-0 acceptance + +Phase 0 may be marked hardware-complete only when: + +1. PROFILE OFF and PROFILE ON samples share the artifact `project_git_sha` + recorded by the CI pair actually tested; +2. every sample carries the frozen mode-specific ELF/IRX hashes; +3. both builds pass the same correctness workload; +4. PROFILE ON produces internally consistent stage counters and traffic + accounting; +5. the measurement overhead of PROFILE ON is quantified rather than assumed + negligible; +6. PROFILE ON retains p50/p95/p99/max instead of replacing distributions with + an average; +7. the result includes console/toolchain/IRX/workload provenance; +8. R5900 counter calibration is checked on the real EE before counter-derived + optimization claims are made. + +If instrumentation materially changes throughput or tail latency, keep +`HDL_PROFILE=0` as the performance build and use `HDL_PROFILE=1` only for +diagnosis. If the delta is negligible for the tested workload, that conclusion +still applies only to the recorded console/adapters/workload. + +## R5900 counter calibration + +Before using a counter event to justify code changes: + +1. run an empty serialized scope and record harness overhead; +2. run a deterministic integer loop whose instruction/cycle relationship is + intentionally simple; +3. repeat the scope several times and confirm stable monotonic results; +4. reject any run with PCR overflow; +5. measure one event pair at a time and do not compare events from different + workloads as if they were simultaneous; +6. preserve a timer-only companion result. + +Only after this calibration should cache miss, branch, single/dual issue or +instruction-completed counts be used as evidence for Phase 1/2 decisions. diff --git a/docs/STORAGE_SCRATCH_ALIGNMENT_REJECTION.md b/docs/STORAGE_SCRATCH_ALIGNMENT_REJECTION.md new file mode 100644 index 00000000..53967fdc --- /dev/null +++ b/docs/STORAGE_SCRATCH_ALIGNMENT_REJECTION.md @@ -0,0 +1,77 @@ +# Storage scratch 64-byte alignment experiment rejection + +CI #743 tested removal of explicit 64-byte alignment from the two highest- +confidence file-only scratch buffers identified by `ALIGNMENT_CONTRACT_AUDIT.md`: + +```text +src/header_backup.c::backup_scratch[1024] +src/repair_snapshot.c::snapshot_verify[1024] +``` + +## Contract + +**CURRENT IMPLEMENTATION:** pinned PS2SDK ordinary fileXio read paths do not +require these application buffers to start on a 64-byte boundary. Neither object +is a direct GIF/DMAC, libpad or custom hdl0: SIF/DMA buffer. + +Removing the attributes was therefore correctness-compatible at the documented +API-contract level and worth testing as a layout experiment. + +## CI #743 + +Source point: + +```text +902f873dac54813c11f6056b7f0667f2283e7bbe +``` + +Artifact digest: + +```text +sha256:e6010f1bba48dc2e96ade33652ce2e15cefa9f6b54f246d3da89404484a84e14 +``` + +Reference was workspace-v1 + source-fingerprint-malloc from CI #739. + +### PROFILE OFF + +```text + #739 #743 delta +ELF bytes 632756 632756 0 +size text 286437 286437 0 +size data 345332 345332 0 +size bss 3012344 3012344 0 +named text 229756 229756 0 +instructions 57488 57488 0 +execute_transaction 6008 6008 0 +``` + +### PROFILE ON + +```text + #739 #743 delta +ELF bytes 638132 638132 0 +size text 290381 290381 0 +size data 346804 346804 0 +size bss 3012984 3012984 0 +named text 232552 232552 0 +instructions 58189 58189 0 +execute_transaction 6008 6008 0 +``` + +The ELF SHA changed because object/symbol placement changed, but every measured +section/code metric above remained identical. The hdl_stream IRX pair also +remained byte-identical to the frozen Phase-0 pair. + +## Decision + +**REJECTED AS AN OPTIMIZATION / RETAIN CURRENT ALIGNMENT.** + +The current source alignment is not required by fileXio, but removing it produced +no reduction in BSS, text, final ELF size or instruction count. Keeping the +existing layout avoids an otherwise pointless address-placement change in a +recovery-oriented code path. + +This result is intentionally recorded so the same source-cleanup-only change is +not later rediscovered and promoted as a performance optimization merely because +there are fewer `aligned(64)` strings in the source. diff --git a/include/hdl_hash_checkpoint.h b/include/hdl_hash_checkpoint.h new file mode 100644 index 00000000..d9e421b3 --- /dev/null +++ b/include/hdl_hash_checkpoint.h @@ -0,0 +1,36 @@ +#ifndef PS2_HDD_BOOTSTRAP_MANAGER_HDL_HASH_CHECKPOINT_H +#define PS2_HDD_BOOTSTRAP_MANAGER_HDL_HASH_CHECKPOINT_H + +#include "hdl_transaction.h" +#include "sha256.h" + +#define HDL_HASH_CHECKPOINT_RECORD_SIZE 256u + +enum { + HDL_HASH_CHECKPOINT_INVALID_ARGUMENT = -545, + HDL_HASH_CHECKPOINT_INVALID_RECORD = -546, + HDL_HASH_CHECKPOINT_HASH_MISMATCH = -547, + HDL_HASH_CHECKPOINT_TRANSACTION_MISMATCH = -548, + HDL_HASH_CHECKPOINT_CONTEXT_MISMATCH = -549 +}; + +/* + * Portable checkpoint for the streaming source SHA-256 state used by a + * resumable HDL copy. The record is deliberately separate from the stable + * 512-byte transaction journal so old journals remain readable. + * + * A checkpoint is an optimization hint, never transaction authority. Restore + * succeeds only when source size, completed byte count, source fingerprint and + * target ID match the already authenticated transaction journal. + */ +int hdl_hash_checkpoint_encode( + const hdl_transaction_t *transaction, + const sha256_context_t *context, + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]); + +int hdl_hash_checkpoint_restore( + const unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE], + const hdl_transaction_t *transaction, + sha256_context_t *context); + +#endif diff --git a/include/hdl_profile.h b/include/hdl_profile.h new file mode 100644 index 00000000..31dd02fd --- /dev/null +++ b/include/hdl_profile.h @@ -0,0 +1,20 @@ +#ifndef PS2_HDD_BOOTSTRAP_MANAGER_HDL_PROFILE_H +#define PS2_HDD_BOOTSTRAP_MANAGER_HDL_PROFILE_H + +/* + * Same-source Phase-0 instrumentation switch. + * + * PROFILE=1 is the diagnostic build used to collect EE/IOP latency and + * transport telemetry. PROFILE=0 compiles that telemetry out while preserving + * the exact HDL pump, prefetch, DMA, cache-coherency and durability paths so a + * real-console A/B measures profiler perturbation instead of another code path. + */ +#ifndef HDL_PROFILE_ENABLED +#define HDL_PROFILE_ENABLED 1 +#endif + +#if HDL_PROFILE_ENABLED != 0 && HDL_PROFILE_ENABLED != 1 +#error "HDL_PROFILE_ENABLED must be 0 or 1" +#endif + +#endif diff --git a/include/hdl_stream_rpc.h b/include/hdl_stream_rpc.h index 2d580990..c0f8dc53 100644 --- a/include/hdl_stream_rpc.h +++ b/include/hdl_stream_rpc.h @@ -54,6 +54,21 @@ typedef struct { #define HDL_STREAM_FAST_FLAG_DOUBLE_BUFFER 0x00000001u #define HDL_STREAM_FAST_FLAG_DIRECT_BDM 0x00000002u +#define HDL_STREAM_FAST_FLAG_IOP_TIMING 0x00000004u + +/* + * Corpus-v2 profiling uses logarithmic microsecond buckets on the IOP instead + * of formatting trace lines in the hot service path. 24 buckets cover 1 us + * through multi-second stalls while keeping the RPC snapshot compact enough + * for a single control-plane read after a bulk phase. + */ +#define HDL_STREAM_IOP_LATENCY_BUCKETS 24u + +typedef struct { + uint32_t samples; + uint32_t maximum_us; + uint32_t buckets[HDL_STREAM_IOP_LATENCY_BUCKETS]; +} hdl_stream_iop_latency_t; typedef struct { uint32_t flags; @@ -66,6 +81,21 @@ typedef struct { uint32_t pumped_sectors; uint32_t source_dma_chunks; uint32_t target_dma_chunks; + + /* Transport accounting is kept in 512-byte sectors to avoid cross-ABI + * uint64_t layout assumptions and to cover any practical PS2 workload. */ + uint32_t direct_source_sectors; + uint32_t fallback_source_sectors; + uint32_t hdd_write_sectors; + uint32_t hdd_read_sectors; + uint32_t sif_dma_sectors; + + hdl_stream_iop_latency_t direct_source_latency; + hdl_stream_iop_latency_t fallback_source_latency; + hdl_stream_iop_latency_t prefetch_wait_latency; + hdl_stream_iop_latency_t hdd_write_latency; + hdl_stream_iop_latency_t hdd_read_latency; + hdl_stream_iop_latency_t sif_dma_latency; } hdl_stream_fast_stats_t; /* diff --git a/include/r5900_perf.h b/include/r5900_perf.h new file mode 100644 index 00000000..3123d357 --- /dev/null +++ b/include/r5900_perf.h @@ -0,0 +1,85 @@ +#ifndef PS2_HDD_BOOTSTRAP_MANAGER_R5900_PERF_H +#define PS2_HDD_BOOTSTRAP_MANAGER_R5900_PERF_H + +#include + +/* + * EE Core User's Manual v6.0, Table 7-1. + * + * PCR0 and PCR1 share the numeric event selector but not always the event + * meaning. Keep separate enums so callers cannot accidentally describe a PCR1 + * value using the PCR0 name. Event 16 deliberately means "no event" and is + * useful when a benchmark needs only one hardware counter. + */ +typedef enum { + R5900_PCR0_RESERVED = 0, + R5900_PCR0_PROCESSOR_CYCLE = 1, + R5900_PCR0_SINGLE_ISSUE = 2, + R5900_PCR0_BRANCH_ISSUED = 3, + R5900_PCR0_BTAC_MISS = 4, + R5900_PCR0_ITLB_MISS = 5, + R5900_PCR0_ICACHE_MISS = 6, + R5900_PCR0_DTLB_ACCESS = 7, + R5900_PCR0_NONBLOCKING_LOAD = 8, + R5900_PCR0_WBB_SINGLE_REQUEST = 9, + R5900_PCR0_WBB_BURST_REQUEST = 10, + R5900_PCR0_CPU_ADDRESS_BUS_BUSY = 11, + R5900_PCR0_INSTRUCTION_COMPLETED = 12, + R5900_PCR0_NON_BDS_INSTRUCTION_COMPLETED = 13, + R5900_PCR0_COP2_INSTRUCTION_COMPLETED = 14, + R5900_PCR0_LOAD_COMPLETED = 15, + R5900_PCR0_NO_EVENT = 16 +} r5900_pcr0_event_t; + +typedef enum { + R5900_PCR1_LOW_ORDER_BRANCH_ISSUED = 0, + R5900_PCR1_PROCESSOR_CYCLE = 1, + R5900_PCR1_DUAL_ISSUE = 2, + R5900_PCR1_BRANCH_MISPREDICTED = 3, + R5900_PCR1_TLB_MISS = 4, + R5900_PCR1_DTLB_MISS = 5, + R5900_PCR1_DCACHE_MISS = 6, + R5900_PCR1_WBB_SINGLE_UNAVAILABLE = 7, + R5900_PCR1_WBB_BURST_UNAVAILABLE = 8, + R5900_PCR1_WBB_BURST_ALMOST_FULL = 9, + R5900_PCR1_WBB_BURST_FULL = 10, + R5900_PCR1_CPU_DATA_BUS_BUSY = 11, + R5900_PCR1_INSTRUCTION_COMPLETED = 12, + R5900_PCR1_NON_BDS_INSTRUCTION_COMPLETED = 13, + R5900_PCR1_COP1_INSTRUCTION_COMPLETED = 14, + R5900_PCR1_STORE_COMPLETED = 15, + R5900_PCR1_NO_EVENT = 16 +} r5900_pcr1_event_t; + +typedef struct { + uint32_t previous_pccr; + uint32_t previous_pcr0; + uint32_t previous_pcr1; + uint32_t event0; + uint32_t event1; + int active; +} r5900_perf_scope_t; + +typedef struct { + uint32_t pcr0; + uint32_t pcr1; + int pcr0_overflow; + int pcr1_overflow; +} r5900_perf_result_t; + +/* + * Start both counters for normal application execution in user/supervisor/ + * kernel mode. Level-1 exception-handler work is deliberately excluded so an + * interrupt does not silently become part of the measured application region. + * Level-2 handlers are excluded by hardware. + * + * The scope preserves the previous counter state and r5900_perf_end() restores + * it. This makes the harness composable with debuggers or future project-wide + * profiling code instead of assuming ownership of COP0 Perf forever. + */ +int r5900_perf_begin(r5900_perf_scope_t *scope, + r5900_pcr0_event_t event0, + r5900_pcr1_event_t event1); +int r5900_perf_end(r5900_perf_scope_t *scope, r5900_perf_result_t *result); + +#endif diff --git a/iop/hdl_stream/Makefile b/iop/hdl_stream/Makefile index b651b01a..71e0fd1c 100644 --- a/iop/hdl_stream/Makefile +++ b/iop/hdl_stream/Makefile @@ -1,6 +1,17 @@ IOP_BIN ?= hdl_stream.irx IOP_OBJS = hdl_stream.o imports.o +HDL_PROFILE ?= 1 +ifeq ($(filter $(HDL_PROFILE),0 1),) +$(error HDL_PROFILE must be 0 or 1) +endif + +# Make the profiler mode part of the dependency graph. PS2SDK's generic IOP +# rules place objects under IOP_OBJS_DIR, so keeping PROFILE=0/1 in separate +# object trees prevents a stale object built with the other preprocessor mode +# from being silently reused when only HDL_PROFILE changes. +IOP_OBJS_DIR = obj/profile-$(HDL_PROFILE)/ + # PS2SDK's generic IOP rules default to -Os. hdl_stream is not a resident-size # utility in our workload: it is the 37.5 MHz R3000A hot path that must keep # USB, DEV9 and SIF fed during multi-gigabyte ISO installs. Put -O2 after the @@ -8,14 +19,16 @@ IOP_OBJS = hdl_stream.o imports.o # minimum code size. We deliberately stop at -O2 because the IOP has a tiny # cache and -O3 loop growth can cost more than it saves on this core. IOP_CFLAGS += -O2 -DIOMANX_OLD_NAME_COMPATIBILITY=0 \ - -DIOMANX_OLD_NAME_ADDDELDRV=0 + -DIOMANX_OLD_NAME_ADDDELDRV=0 \ + -DHDL_PROFILE_ENABLED=$(HDL_PROFILE) IOP_WARNFLAGS = -Wall -Wextra -Werror IOP_INCS += -I../../include all: $(IOP_BIN) clean: - rm -f -r $(IOP_OBJS_DIR) $(IOP_BIN) \ + rm -rf obj + rm -f $(IOP_BIN) \ $(IOP_BIN:.irx=.notiopmod.elf) \ $(IOP_BIN:.irx=.notiopmod.stripped.elf) diff --git a/iop/hdl_stream/hdl_stream.c b/iop/hdl_stream/hdl_stream.c index e837e7c1..385a87d5 100644 --- a/iop/hdl_stream/hdl_stream.c +++ b/iop/hdl_stream/hdl_stream.c @@ -26,6 +26,7 @@ #include #include +#include "hdl_profile.h" #include "hdl_stream_rpc.h" #define HDL_STREAM_MAIN_SKIP 0x2000u @@ -39,7 +40,7 @@ #define HDL_STREAM_PREFETCH_STACK 0x1000u #define HDL_STREAM_PREFETCH_PRIORITY 0x30u -IRX_ID("hdl_stream", 1, 4); +IRX_ID("hdl_stream", 1, 5); typedef struct { int source_fd; @@ -77,9 +78,53 @@ typedef struct { uint32_t prefetch_stage_index; hdl_source_map_t source; +#if HDL_PROFILE_ENABLED hdl_stream_fast_stats_t stats; +#endif } hdl_stream_file_t; +#if HDL_PROFILE_ENABLED +/* + * Phase-0 corpus-v2 profiling deliberately stores only compact histograms in + * the IOP hot path. Formatting/logging remains on the EE after a bulk phase. + * GetSystemTime/SysClock2USec are current ThreadMan services and the same clock + * conversion pattern is used by PS2SDK itself. Hardware A/B still decides + * whether this instrumentation remains enabled in performance builds. + */ +static uint32_t iop_elapsed_us(const iop_sys_clock_t *start, + const iop_sys_clock_t *end) +{ + iop_sys_clock_t diff; + uint32_t seconds = 0; + uint32_t usec = 0; + + diff.lo = end->lo - start->lo; + diff.hi = end->hi - start->hi - (start->lo > end->lo); + SysClock2USec(&diff, &seconds, &usec); + if (seconds > 4294u) + return 0xffffffffu; + return seconds * 1000000u + usec; +} + +static void iop_latency_record(hdl_stream_iop_latency_t *stats, + const iop_sys_clock_t *start, + const iop_sys_clock_t *end) +{ + uint32_t usec = iop_elapsed_us(start, end); + uint32_t bound = 1u; + unsigned int bucket = 0u; + + while (bucket + 1u < HDL_STREAM_IOP_LATENCY_BUCKETS && usec > bound) { + bound <<= 1; + bucket++; + } + stats->buckets[bucket]++; + stats->samples++; + if (usec > stats->maximum_us) + stats->maximum_us = usec; +} +#endif + static int stream_init(iomanX_iop_device_t *device) { (void)device; @@ -145,7 +190,9 @@ static void source_map_disable(hdl_stream_file_t *stream, int source_fd) source_map_free_fragments(&stream->source); stream->source.source_fd = source_fd; stream->source.disabled = 1; +#if HDL_PROFILE_ENABLED stream->stats.flags &= ~HDL_STREAM_FAST_FLAG_DIRECT_BDM; +#endif } static int source_map_prepare(hdl_stream_file_t *stream, int source_fd) @@ -219,8 +266,10 @@ static int source_map_prepare(hdl_stream_file_t *stream, int source_fd) stream->source.fragment_count = (uint32_t)fragment_count; stream->source.cursor_fragment = 0; stream->source.cursor_base = 0; +#if HDL_PROFILE_ENABLED stream->stats.flags |= HDL_STREAM_FAST_FLAG_DIRECT_BDM; stream->stats.fragment_count = (uint32_t)fragment_count; +#endif return 0; } } @@ -325,6 +374,11 @@ static int source_read_fallback(int source_fd, uint64_t offset, static int source_read_at(hdl_stream_file_t *stream, int source_fd, uint64_t offset, void *buffer, unsigned int bytes) { +#if HDL_PROFILE_ENABLED + iop_sys_clock_t start; + iop_sys_clock_t end; +#endif + /* * The stock usbmass BDM intentionally caps one SCSI request at 128 512-byte * sectors (64 KiB), and HDL_STREAM_IOP_STAGE_BYTES matches that exactly. @@ -336,19 +390,43 @@ static int source_read_at(hdl_stream_file_t *stream, int source_fd, (offset & (HDL_STREAM_USB_SECTOR_SIZE - 1u)) == 0 && (bytes & (HDL_STREAM_USB_SECTOR_SIZE - 1u)) == 0 && (bytes >> HDL_STREAM_USB_SECTOR_SHIFT) <= UINT16_MAX) { - int result = source_map_read( + int result; + +#if HDL_PROFILE_ENABLED + GetSystemTime(&start); +#endif + result = source_map_read( &stream->source, offset >> HDL_STREAM_USB_SECTOR_SHIFT, buffer, (uint16_t)(bytes >> HDL_STREAM_USB_SECTOR_SHIFT)); - +#if HDL_PROFILE_ENABLED + GetSystemTime(&end); + iop_latency_record(&stream->stats.direct_source_latency, &start, &end); +#endif if (result >= 0) { +#if HDL_PROFILE_ENABLED stream->stats.direct_reads++; + stream->stats.direct_source_sectors += bytes >> 9; +#endif return (int)bytes; } source_map_disable(stream, source_fd); } +#if HDL_PROFILE_ENABLED stream->stats.fallback_reads++; - return source_read_fallback(source_fd, offset, buffer, bytes); + GetSystemTime(&start); +#endif + { + int result = source_read_fallback(source_fd, offset, buffer, bytes); + +#if HDL_PROFILE_ENABLED + GetSystemTime(&end); + iop_latency_record(&stream->stats.fallback_source_latency, &start, &end); + if (result > 0) + stream->stats.fallback_source_sectors += (unsigned int)result >> 9; +#endif + return result; + } } static void prefetch_worker(void *arg) @@ -428,7 +506,9 @@ static int prefetch_init(hdl_stream_file_t *stream) goto fail; } +#if HDL_PROFILE_ENABLED stream->stats.flags |= HDL_STREAM_FAST_FLAG_DOUBLE_BUFFER; +#endif return 0; fail: @@ -438,11 +518,22 @@ static int prefetch_init(hdl_stream_file_t *stream) static int prefetch_wait(hdl_stream_file_t *stream) { +#if HDL_PROFILE_ENABLED + iop_sys_clock_t start; + iop_sys_clock_t end; +#endif int result; if (!stream->prefetch_active) return 0; +#if HDL_PROFILE_ENABLED + GetSystemTime(&start); +#endif result = WaitSema(stream->prefetch_done_sema); +#if HDL_PROFILE_ENABLED + GetSystemTime(&end); + iop_latency_record(&stream->stats.prefetch_wait_latency, &start, &end); +#endif if (result < 0) return result; stream->prefetch_active = 0; @@ -463,12 +554,16 @@ static int prefetch_take(hdl_stream_file_t *stream, int source_fd, stream->prefetch_bytes == bytes; result = prefetch_wait(stream); if (!matches) { +#if HDL_PROFILE_ENABLED stream->stats.prefetch_misses++; +#endif return 0; } if (result != (int)bytes) return result < 0 ? result : -EIO; +#if HDL_PROFILE_ENABLED stream->stats.prefetch_hits++; +#endif *stage_index = stream->prefetch_stage_index; *stage = stream->stage[*stage_index]; return 1; @@ -510,10 +605,14 @@ static void prefetch_shutdown(hdl_stream_file_t *stream) prefetch_delete_semas(stream); } -static int dma_to_ee(uint32_t ee_address, const void *source, - unsigned int bytes) +static int dma_to_ee(hdl_stream_file_t *stream, uint32_t ee_address, + const void *source, unsigned int bytes) { SifDmaTransfer_t transfer; +#if HDL_PROFILE_ENABLED + iop_sys_clock_t start; + iop_sys_clock_t end; +#endif int id; if (ee_address == 0 || bytes == 0 || (ee_address & 0x3fu) != 0 || @@ -523,10 +622,20 @@ static int dma_to_ee(uint32_t ee_address, const void *source, transfer.dest = (void *)(uintptr_t)ee_address; transfer.size = (int)bytes; transfer.attr = 0; +#if HDL_PROFILE_ENABLED + GetSystemTime(&start); +#endif id = sceSifSetDma(&transfer, 1); if (id <= 0) return -EIO; while (sceSifDmaStat(id) >= 0) {} +#if HDL_PROFILE_ENABLED + GetSystemTime(&end); + iop_latency_record(&stream->stats.sif_dma_latency, &start, &end); + stream->stats.sif_dma_sectors += bytes >> 9; +#else + (void)stream; +#endif return 0; } @@ -548,6 +657,9 @@ static int stream_open(iomanX_iop_file_t *file, const char *name, if (stream == NULL) return -ENOMEM; memset(stream, 0, sizeof(*stream)); +#if HDL_PROFILE_ENABLED + stream->stats.flags = HDL_STREAM_FAST_FLAG_IOP_TIMING; +#endif stream->source.source_fd = -1; stream->prefetch_thread = -1; stream->prefetch_request_sema = -1; @@ -685,6 +797,10 @@ static int stream_transfer(iomanX_iop_file_t *file, void *buffer, while (remaining > 0) { hddIoctl2Transfer_t transfer; +#if HDL_PROFILE_ENABLED + iop_sys_clock_t start; + iop_sys_clock_t end; +#endif uint64_t available; uint32_t part; uint32_t sector; @@ -703,8 +819,23 @@ static int stream_transfer(iomanX_iop_file_t *file, void *buffer, transfer.size = (uint32_t)chunk >> 9; transfer.mode = direction; transfer.buffer = cursor; +#if HDL_PROFILE_ENABLED + GetSystemTime(&start); +#endif result = iomanX_ioctl2(stream->hdd_fd, HIOCTRANSFER, &transfer, sizeof(transfer), NULL, 0); +#if HDL_PROFILE_ENABLED + GetSystemTime(&end); + if (direction == APA_IO_MODE_WRITE) { + iop_latency_record(&stream->stats.hdd_write_latency, &start, &end); + if (result >= 0) + stream->stats.hdd_write_sectors += transfer.size; + } else { + iop_latency_record(&stream->stats.hdd_read_latency, &start, &end); + if (result >= 0) + stream->stats.hdd_read_sectors += transfer.size; + } +#endif if (result < 0) return result; stream->position += (uint32_t)chunk; @@ -878,13 +1009,17 @@ static int fast_source_to_ee(iomanX_iop_file_t *file, APA_IO_MODE_WRITE); if (result != (int)request->bytes) return result < 0 ? result : -EIO; +#if HDL_PROFILE_ENABLED stream->stats.pumped_chunks++; stream->stats.pumped_sectors += request->bytes >> 9; +#endif } - result = dma_to_ee(request->ee_address, stage, request->bytes); + result = dma_to_ee(stream, request->ee_address, stage, request->bytes); if (result < 0) return result; +#if HDL_PROFILE_ENABLED stream->stats.source_dma_chunks++; +#endif return (int)request->bytes; } @@ -904,10 +1039,13 @@ static int fast_target_to_ee(iomanX_iop_file_t *file, APA_IO_MODE_READ); if (result != (int)request->bytes) return result < 0 ? result : -EIO; - result = dma_to_ee(request->ee_address, stream->stage[0], request->bytes); + result = dma_to_ee(stream, request->ee_address, stream->stage[0], + request->bytes); if (result < 0) return result; +#if HDL_PROFILE_ENABLED stream->stats.target_dma_chunks++; +#endif return (int)request->bytes; } @@ -949,9 +1087,13 @@ static int stream_ioctl2(iomanX_iop_file_t *file, int command, if (command == HDL_STREAM_IOCTL2_TARGET_TO_EE) return fast_target_to_ee(file, argument, argument_length); if (command == HDL_STREAM_IOCTL2_GET_FAST_STATS) { - if (buffer == NULL || buffer_length < sizeof(stream->stats)) + if (buffer == NULL || buffer_length < sizeof(hdl_stream_fast_stats_t)) return -EINVAL; +#if HDL_PROFILE_ENABLED memcpy(buffer, &stream->stats, sizeof(stream->stats)); +#else + memset(buffer, 0, sizeof(hdl_stream_fast_stats_t)); +#endif return 0; } return -EINVAL; diff --git a/iop/hdl_stream/imports.lst b/iop/hdl_stream/imports.lst index ba724bc9..82a13d6f 100644 --- a/iop/hdl_stream/imports.lst +++ b/iop/hdl_stream/imports.lst @@ -38,6 +38,8 @@ I_CreateThread I_DeleteThread I_StartThread I_ExitThread +I_GetSystemTime +I_SysClock2USec thbase_IMPORTS_end thsemap_IMPORTS_start diff --git a/src/filexio_fdman_policy_ps2.c b/src/filexio_fdman_policy_ps2.c new file mode 100644 index 00000000..2326011f --- /dev/null +++ b/src/filexio_fdman_policy_ps2.c @@ -0,0 +1,101 @@ +/* + * Phase-1 application policy for PS2SDK fileXio/newlib integration. + * + * CURRENT IMPLEMENTATION (PS2SDK v2.0.0 / b12f8af): fileXioInit() calls the + * internal _ps2sdk_fileXio_init() hook. That hook switches Newlib's generic + * POSIX fd-manager path table to fileXio and, through the companion constructor, + * retains every POSIX adapter including __fileXioGetstatHelper(). The stat + * adapter converts three timestamps through mktime(), which in this pinned + * Newlib also retains timezone parsing and scanf machinery. + * + * fhdb-bootstrap-manager's storage contract is intentionally direct fileXio: + * application filesystem operations use fileXioOpen/Read/Write/Lseek/etc. The + * Newlib formatting used by the UI/log paths is memory/string formatting, not + * fopen/open/stat-based file access. Therefore this build does not need fileXio + * to replace Newlib's generic POSIX pathname backend. + * + * The startup libcglue backend still installs the older fio POSIX adapter for + * stdin/stdout/stderr. Its generic __fioGetstatHelper() performs the same costly + * mktime conversion even though this application never consumes POSIX file + * timestamps. Keep mode and size semantics for incidental library probes, but + * deliberately report zero timestamps. CI rejects application fopen/open/stat + * call sites, so a future consumer cannot silently depend on this reduced + * timestamp contract. + * + * CURRENT IMPLEMENTATION: PS2SDK's weak _libcglue_timezone_update() is invoked + * unconditionally from _libcglue_init(). Its default implementation reads the + * OSD timezone through POSIX open/read/close, builds a TZ string with sprintf() + * and calls setenv(). This application never calls localtime/mktime/strftime or + * PS2SDK's timezone/daylight setters, and none of its persistence formats expose + * libc local-time semantics. A strong no-op below therefore removes startup work + * and the associated formatting/environment dependency chain without changing + * the manager's declared storage or timing contracts. If local civil time is + * introduced later, this policy must be removed or replaced by an explicit, + * tested application time contract. + */ + +#include +#define NEWLIB_PORT_AWARE +#include +#include +#include +#include +#include + +void _ps2sdk_fileXio_init(void) +{ +} + +void _ps2sdk_fileXio_deinit(void) +{ +} + +void _libcglue_timezone_update(void) +{ +} + +static mode_t fio_mode_to_posix(unsigned int mode) +{ + mode_t result = 0; + + if (mode & FIO_SO_IFREG) + result |= S_IFREG; + if (mode & FIO_SO_IFDIR) + result |= S_IFDIR; + if (mode & FIO_SO_IROTH) + result |= S_IRUSR | S_IRGRP | S_IROTH; + if (mode & FIO_SO_IWOTH) + result |= S_IWUSR | S_IWGRP | S_IWOTH; + if (mode & FIO_SO_IXOTH) + result |= S_IXUSR | S_IXGRP | S_IXOTH; + return result; +} + +int __fioGetstatHelper(const char *path, struct stat *status) +{ + io_stat_t iop_status; + + if (path == NULL || status == NULL) { + errno = EINVAL; + return -1; + } + + if (strncmp(path, "tty", 3) == 0 && path[3] >= '0' && path[3] <= '9' && + path[4] == ':') { + memset(status, 0, sizeof(*status)); + status->st_mode = S_IFCHR; + return 0; + } + + if (fioGetstat(path, &iop_status) < 0) { + errno = ENOENT; + return -1; + } + + memset(status, 0, sizeof(*status)); + status->st_mode = fio_mode_to_posix(iop_status.mode); + status->st_size = ((off_t)iop_status.hisize << 32) | (off_t)iop_status.size; + status->st_blksize = 16 * 1024; + status->st_blocks = status->st_size / 512; + return 0; +} diff --git a/src/hdl_hash_checkpoint.c b/src/hdl_hash_checkpoint.c new file mode 100644 index 00000000..4430f041 --- /dev/null +++ b/src/hdl_hash_checkpoint.c @@ -0,0 +1,174 @@ +#include +#include +#include + +#include "hdl_hash_checkpoint.h" + +#define HDL_HASH_CHECKPOINT_VERSION 1u +#define HDL_HASH_CHECKPOINT_HASH_OFFSET 224u +#define HDL_HASH_CHECKPOINT_TARGET_OFFSET 64u +#define HDL_HASH_CHECKPOINT_TARGET_BYTES HDL_TRANSACTION_TARGET_MAX +#define HDL_HASH_CHECKPOINT_STATE_OFFSET 100u +#define HDL_HASH_CHECKPOINT_TOTAL_OFFSET 132u +#define HDL_HASH_CHECKPOINT_BLOCK_USED_OFFSET 140u +#define HDL_HASH_CHECKPOINT_BLOCK_OFFSET 144u + +static const unsigned char checkpoint_magic[8] = { + 'H', 'D', 'L', 'H', 'A', 'S', 'H', 1 +}; + +static void write_le32(unsigned char *destination, uint32_t value) +{ + destination[0] = (unsigned char)value; + destination[1] = (unsigned char)(value >> 8); + destination[2] = (unsigned char)(value >> 16); + destination[3] = (unsigned char)(value >> 24); +} + +static void write_le64(unsigned char *destination, uint64_t value) +{ + write_le32(destination, (uint32_t)value); + write_le32(destination + 4, (uint32_t)(value >> 32)); +} + +static uint32_t read_le32(const unsigned char *source) +{ + return (uint32_t)source[0] | ((uint32_t)source[1] << 8) | + ((uint32_t)source[2] << 16) | ((uint32_t)source[3] << 24); +} + +static uint64_t read_le64(const unsigned char *source) +{ + return (uint64_t)read_le32(source) | + ((uint64_t)read_le32(source + 4) << 32); +} + +static size_t bounded_length(const char *text, size_t capacity) +{ + const char *end = memchr(text, '\0', capacity); + + return end == NULL ? capacity : (size_t)(end - text); +} + +static int checkpoint_progress(const hdl_transaction_t *transaction, + uint64_t *completed_bytes) +{ + if (transaction == NULL || completed_bytes == NULL || + transaction->source_bytes == 0 || + (transaction->source_bytes & 2047u) != 0 || + transaction->completed_sectors > transaction->total_sectors || + transaction->total_sectors != transaction->source_bytes / 2048u || + transaction->completed_sectors == 0 || + memchr(transaction->target, '\0', sizeof(transaction->target)) == NULL || + transaction->target[0] == '\0') + return 0; + + *completed_bytes = transaction->completed_sectors * 2048u; + return *completed_bytes <= transaction->source_bytes; +} + +int hdl_hash_checkpoint_encode( + const hdl_transaction_t *transaction, + const sha256_context_t *context, + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]) +{ + unsigned char digest[32]; + uint64_t completed_bytes; + size_t target_length; + unsigned int i; + + if (record == NULL || context == NULL || + !checkpoint_progress(transaction, &completed_bytes)) + return HDL_HASH_CHECKPOINT_INVALID_ARGUMENT; + if (context->total_bytes != completed_bytes || context->block_used > 63u || + context->block_used != (size_t)(completed_bytes & 63u)) + return HDL_HASH_CHECKPOINT_CONTEXT_MISMATCH; + + memset(record, 0, HDL_HASH_CHECKPOINT_RECORD_SIZE); + memcpy(record, checkpoint_magic, sizeof(checkpoint_magic)); + write_le32(record + 8, HDL_HASH_CHECKPOINT_VERSION); + write_le64(record + 16, transaction->source_bytes); + write_le64(record + 24, completed_bytes); + memcpy(record + 32, transaction->source_fingerprint, + sizeof(transaction->source_fingerprint)); + target_length = bounded_length(transaction->target, + sizeof(transaction->target)); + if (target_length >= HDL_HASH_CHECKPOINT_TARGET_BYTES) + return HDL_HASH_CHECKPOINT_INVALID_ARGUMENT; + memcpy(record + HDL_HASH_CHECKPOINT_TARGET_OFFSET, + transaction->target, target_length); + + for (i = 0; i < 8u; i++) + write_le32(record + HDL_HASH_CHECKPOINT_STATE_OFFSET + i * 4u, + context->state[i]); + write_le64(record + HDL_HASH_CHECKPOINT_TOTAL_OFFSET, + context->total_bytes); + write_le32(record + HDL_HASH_CHECKPOINT_BLOCK_USED_OFFSET, + (uint32_t)context->block_used); + if (context->block_used != 0) + memcpy(record + HDL_HASH_CHECKPOINT_BLOCK_OFFSET, + context->block, context->block_used); + + sha256_buffer(record, HDL_HASH_CHECKPOINT_HASH_OFFSET, digest); + memcpy(record + HDL_HASH_CHECKPOINT_HASH_OFFSET, digest, sizeof(digest)); + return 0; +} + +int hdl_hash_checkpoint_restore( + const unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE], + const hdl_transaction_t *transaction, + sha256_context_t *context) +{ + unsigned char digest[32]; + char target[HDL_TRANSACTION_TARGET_MAX]; + uint64_t completed_bytes; + uint64_t stored_source_bytes; + uint64_t stored_completed_bytes; + uint64_t stored_total_bytes; + uint32_t block_used; + unsigned int i; + + if (record == NULL || context == NULL || + !checkpoint_progress(transaction, &completed_bytes)) + return HDL_HASH_CHECKPOINT_INVALID_ARGUMENT; + if (memcmp(record, checkpoint_magic, sizeof(checkpoint_magic)) != 0 || + read_le32(record + 8) != HDL_HASH_CHECKPOINT_VERSION) + return HDL_HASH_CHECKPOINT_INVALID_RECORD; + + sha256_buffer(record, HDL_HASH_CHECKPOINT_HASH_OFFSET, digest); + if (memcmp(digest, record + HDL_HASH_CHECKPOINT_HASH_OFFSET, + sizeof(digest)) != 0) + return HDL_HASH_CHECKPOINT_HASH_MISMATCH; + + memset(target, 0, sizeof(target)); + memcpy(target, record + HDL_HASH_CHECKPOINT_TARGET_OFFSET, + sizeof(target)); + if (memchr(target, '\0', sizeof(target)) == NULL) + return HDL_HASH_CHECKPOINT_INVALID_RECORD; + + stored_source_bytes = read_le64(record + 16); + stored_completed_bytes = read_le64(record + 24); + if (stored_source_bytes != transaction->source_bytes || + stored_completed_bytes != completed_bytes || + memcmp(record + 32, transaction->source_fingerprint, + sizeof(transaction->source_fingerprint)) != 0 || + strcmp(target, transaction->target) != 0) + return HDL_HASH_CHECKPOINT_TRANSACTION_MISMATCH; + + stored_total_bytes = read_le64(record + HDL_HASH_CHECKPOINT_TOTAL_OFFSET); + block_used = read_le32(record + HDL_HASH_CHECKPOINT_BLOCK_USED_OFFSET); + if (stored_total_bytes != stored_completed_bytes || block_used > 63u || + block_used != (uint32_t)(stored_total_bytes & 63u)) + return HDL_HASH_CHECKPOINT_INVALID_RECORD; + + memset(context, 0, sizeof(*context)); + for (i = 0; i < 8u; i++) + context->state[i] = + read_le32(record + HDL_HASH_CHECKPOINT_STATE_OFFSET + i * 4u); + context->total_bytes = stored_total_bytes; + context->block_used = (size_t)block_used; + if (block_used != 0) + memcpy(context->block, record + HDL_HASH_CHECKPOINT_BLOCK_OFFSET, + block_used); + return 0; +} diff --git a/src/hdl_tools/fast_io.inc b/src/hdl_tools/fast_io.inc index 5df968b5..ef8fa180 100644 --- a/src/hdl_tools/fast_io.inc +++ b/src/hdl_tools/fast_io.inc @@ -1,4 +1,8 @@ +#include "hdl_profile.h" + +#if HDL_PROFILE_ENABLED #include +#endif /* * EE-side glue for the dev20 IOP payload pump. @@ -15,40 +19,229 @@ * into IOP staging and performs one IOP->EE DMA, avoiding fileXio's extra * staging/copy layer. */ +#define HDL_FAST_NOINLINE __attribute__((noinline)) + static int hdl_fast_source_fd = -1; static uint64_t hdl_fast_source_position; static uint64_t hdl_fast_source_seek_value; static uint64_t hdl_fast_source_total; -static uint64_t hdl_fast_copy_start_ticks; -static uint64_t hdl_fast_copy_bytes; static int hdl_fast_source_position_valid; static int hdl_fast_source_seek_pending; static int hdl_fast_pump_mode; static int hdl_fast_pump_ack; static unsigned int hdl_fast_pump_ack_bytes; + +#if HDL_PROFILE_ENABLED +#define HDL_FAST_LATENCY_BUCKETS 24u +#define HDL_FAST_COLD __attribute__((cold, noinline)) + +typedef struct { + uint32_t samples; + uint32_t maximum_us; + uint32_t buckets[HDL_FAST_LATENCY_BUCKETS]; +} hdl_fast_latency_stats_t; + +static uint64_t hdl_fast_copy_start_ticks; +static uint64_t hdl_fast_copy_bytes; +static uint64_t hdl_fast_target_bytes; +static uint64_t hdl_fast_sif_dma_bytes; +static uint64_t hdl_fast_cache_maintenance_bytes; +static uint64_t hdl_fast_fallback_source_bytes; +static uint64_t hdl_fast_fallback_target_bytes; +static uint64_t hdl_fast_copy_consumer_start_ticks; +static uint64_t hdl_fast_target_consumer_start_ticks; static int hdl_fast_path_logged; -static int hdl_fast_stats_logged; +static int hdl_fast_copy_iop_stats_logged; +static int hdl_fast_verify_iop_stats_logged; static int hdl_fast_rate_logged; +static int hdl_fast_copy_profile_logged; +static int hdl_fast_target_profile_logged; +static hdl_fast_latency_stats_t hdl_fast_source_ioctl_latency; +static hdl_fast_latency_stats_t hdl_fast_pump_ioctl_latency; +static hdl_fast_latency_stats_t hdl_fast_target_ioctl_latency; +static hdl_fast_latency_stats_t hdl_fast_copy_consumer_latency; +static hdl_fast_latency_stats_t hdl_fast_target_consumer_latency; -static void hdl_fast_io_reset(void) +static void hdl_fast_latency_reset(hdl_fast_latency_stats_t *stats) +{ + memset(stats, 0, sizeof(*stats)); +} + +static uint32_t hdl_fast_elapsed_us(u64 start, u64 end) +{ + u32 seconds = 0; + u32 microseconds = 0; + + TimerBusClock2USec(end - start, &seconds, µseconds); + if (seconds > 4294u) + return 0xffffffffu; + return seconds * 1000000u + microseconds; +} + +static HDL_FAST_NOINLINE void hdl_fast_latency_record( + hdl_fast_latency_stats_t *stats, u64 start, u64 end) +{ + uint32_t usec; + uint32_t bound = 1u; + unsigned int bucket = 0u; + + if (start == 0 || end < start) + return; + usec = hdl_fast_elapsed_us(start, end); + while (bucket + 1u < HDL_FAST_LATENCY_BUCKETS && usec > bound) { + bound <<= 1; + bucket++; + } + stats->buckets[bucket]++; + stats->samples++; + if (usec > stats->maximum_us) + stats->maximum_us = usec; +} + +static uint32_t hdl_fast_latency_percentile(const hdl_fast_latency_stats_t *stats, + unsigned int percentile) +{ + uint32_t threshold; + uint32_t cumulative = 0; + unsigned int bucket; + + if (stats->samples == 0 || percentile == 0 || percentile > 100u) + return 0; + threshold = (stats->samples * percentile + 99u) / 100u; + for (bucket = 0; bucket < HDL_FAST_LATENCY_BUCKETS; bucket++) { + cumulative += stats->buckets[bucket]; + if (cumulative >= threshold) + return 1u << bucket; + } + return 1u << (HDL_FAST_LATENCY_BUCKETS - 1u); +} + +static uint32_t hdl_fast_iop_latency_percentile( + const hdl_stream_iop_latency_t *stats, unsigned int percentile) +{ + uint32_t threshold; + uint32_t cumulative = 0; + unsigned int bucket; + + if (stats->samples == 0 || percentile == 0 || percentile > 100u) + return 0; + threshold = (stats->samples * percentile + 99u) / 100u; + for (bucket = 0; bucket < HDL_STREAM_IOP_LATENCY_BUCKETS; bucket++) { + cumulative += stats->buckets[bucket]; + if (cumulative >= threshold) + return 1u << bucket; + } + return 1u << (HDL_STREAM_IOP_LATENCY_BUCKETS - 1u); +} + +static HDL_FAST_COLD void hdl_fast_latency_log( + const char *name, const hdl_fast_latency_stats_t *stats) +{ + if (stats->samples == 0) + return; + session_log_line( + "HDL perf %s samples=%u p50<=%uus p95<=%uus p99<=%uus max=%uus", + name, stats->samples, + hdl_fast_latency_percentile(stats, 50u), + hdl_fast_latency_percentile(stats, 95u), + hdl_fast_latency_percentile(stats, 99u), + stats->maximum_us); +} + +static HDL_FAST_COLD void hdl_fast_iop_latency_log( + const char *name, const hdl_stream_iop_latency_t *stats) +{ + if (stats->samples == 0) + return; + session_log_line( + "HDL IOP perf %s samples=%u p50<=%uus p95<=%uus p99<=%uus max=%uus", + name, stats->samples, + hdl_fast_iop_latency_percentile(stats, 50u), + hdl_fast_iop_latency_percentile(stats, 95u), + hdl_fast_iop_latency_percentile(stats, 99u), + stats->maximum_us); +} + +static HDL_FAST_NOINLINE void hdl_fast_record_ioctl_latency( + int command, u64 start, u64 end) +{ + if (command == HDL_STREAM_IOCTL2_PUMP_TO_EE) + hdl_fast_latency_record(&hdl_fast_pump_ioctl_latency, start, end); + else if (command == HDL_STREAM_IOCTL2_SOURCE_TO_EE) + hdl_fast_latency_record(&hdl_fast_source_ioctl_latency, start, end); + else if (command == HDL_STREAM_IOCTL2_TARGET_TO_EE) + hdl_fast_latency_record(&hdl_fast_target_ioctl_latency, start, end); +} + +static HDL_FAST_COLD void hdl_fast_log_copy_profile(void) +{ + if (hdl_fast_copy_profile_logged) + return; + hdl_fast_latency_log("pump-ioctl", &hdl_fast_pump_ioctl_latency); + hdl_fast_latency_log("source-ioctl", &hdl_fast_source_ioctl_latency); + hdl_fast_latency_log("copy-ee-consumer", &hdl_fast_copy_consumer_latency); + session_log_line( + "HDL perf copy traffic useful=%llu sif-dma=%llu ee-cache-maint=%llu fallback-source=%llu", + (unsigned long long)hdl_fast_copy_bytes, + (unsigned long long)hdl_fast_sif_dma_bytes, + (unsigned long long)hdl_fast_cache_maintenance_bytes, + (unsigned long long)hdl_fast_fallback_source_bytes); + hdl_fast_copy_profile_logged = 1; +} + +static HDL_FAST_COLD void hdl_fast_log_target_profile(void) +{ + if (hdl_fast_target_profile_logged) + return; + hdl_fast_latency_log("target-ioctl", &hdl_fast_target_ioctl_latency); + hdl_fast_latency_log("verify-ee-consumer", &hdl_fast_target_consumer_latency); + session_log_line( + "HDL perf verify traffic target=%llu sif-dma-total=%llu ee-cache-maint-total=%llu fallback-target=%llu consumer-samples=%u final-chunk-excluded=1", + (unsigned long long)hdl_fast_target_bytes, + (unsigned long long)hdl_fast_sif_dma_bytes, + (unsigned long long)hdl_fast_cache_maintenance_bytes, + (unsigned long long)hdl_fast_fallback_target_bytes, + hdl_fast_target_consumer_latency.samples); + hdl_fast_target_profile_logged = 1; +} +#endif + +static HDL_FAST_NOINLINE void hdl_fast_io_reset(void) { hdl_fast_source_fd = -1; hdl_fast_source_position = 0; hdl_fast_source_seek_value = 0; hdl_fast_source_total = 0; - hdl_fast_copy_start_ticks = 0; - hdl_fast_copy_bytes = 0; hdl_fast_source_position_valid = 0; hdl_fast_source_seek_pending = 0; hdl_fast_pump_mode = 0; hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; +#if HDL_PROFILE_ENABLED + hdl_fast_copy_start_ticks = 0; + hdl_fast_copy_bytes = 0; + hdl_fast_target_bytes = 0; + hdl_fast_sif_dma_bytes = 0; + hdl_fast_cache_maintenance_bytes = 0; + hdl_fast_fallback_source_bytes = 0; + hdl_fast_fallback_target_bytes = 0; + hdl_fast_copy_consumer_start_ticks = 0; + hdl_fast_target_consumer_start_ticks = 0; hdl_fast_path_logged = 0; - hdl_fast_stats_logged = 0; + hdl_fast_copy_iop_stats_logged = 0; + hdl_fast_verify_iop_stats_logged = 0; hdl_fast_rate_logged = 0; + hdl_fast_copy_profile_logged = 0; + hdl_fast_target_profile_logged = 0; + hdl_fast_latency_reset(&hdl_fast_source_ioctl_latency); + hdl_fast_latency_reset(&hdl_fast_pump_ioctl_latency); + hdl_fast_latency_reset(&hdl_fast_target_ioctl_latency); + hdl_fast_latency_reset(&hdl_fast_copy_consumer_latency); + hdl_fast_latency_reset(&hdl_fast_target_consumer_latency); +#endif } -static int hdl_fast_resolve_source_total(int source_fd) +static HDL_FAST_NOINLINE int hdl_fast_resolve_source_total(int source_fd) { s64 end; s64 restore; @@ -73,30 +266,49 @@ static int hdl_fast_resolve_source_total(int source_fd) return 0; } -static void hdl_fast_io_log_stats(int target_fd) +#if HDL_PROFILE_ENABLED +static HDL_FAST_COLD void hdl_fast_io_log_stats(int target_fd, + int verify_phase) { hdl_stream_fast_stats_t stats; + const char *phase = verify_phase ? "verify-final" : "copy-final"; + int *logged = verify_phase ? &hdl_fast_verify_iop_stats_logged + : &hdl_fast_copy_iop_stats_logged; int result; - if (target_fd < 0 || hdl_fast_stats_logged) + if (target_fd < 0 || *logged) return; memset(&stats, 0, sizeof(stats)); result = fileXioIoctl2(target_fd, HDL_STREAM_IOCTL2_GET_FAST_STATS, NULL, 0, &stats, sizeof(stats)); if (result < 0) { - session_log_line("HDL fast I/O stats unavailable fd=%d result=%d", - target_fd, result); + session_log_line("HDL fast I/O stats unavailable phase=%s fd=%d result=%d", + phase, target_fd, result); return; } session_log_line( - "HDL fast I/O source stats flags=0x%08x fragments=%u direct=%u fallback=%u prefetch-hit=%u miss=%u pump=%u sectors=%u src-dma=%u", - stats.flags, stats.fragment_count, stats.direct_reads, + "HDL fast I/O snapshot phase=%s flags=0x%08x fragments=%u direct=%u fallback=%u prefetch-hit=%u miss=%u pump=%u sectors=%u src-dma=%u target-dma=%u", + phase, stats.flags, stats.fragment_count, stats.direct_reads, stats.fallback_reads, stats.prefetch_hits, stats.prefetch_misses, - stats.pumped_chunks, stats.pumped_sectors, stats.source_dma_chunks); - hdl_fast_stats_logged = 1; + stats.pumped_chunks, stats.pumped_sectors, stats.source_dma_chunks, + stats.target_dma_chunks); + session_log_line( + "HDL IOP traffic phase=%s direct-src-sectors=%u fallback-src-sectors=%u hdd-write-sectors=%u hdd-read-sectors=%u sif-dma-sectors=%u", + phase, stats.direct_source_sectors, stats.fallback_source_sectors, + stats.hdd_write_sectors, stats.hdd_read_sectors, + stats.sif_dma_sectors); + if ((stats.flags & HDL_STREAM_FAST_FLAG_IOP_TIMING) != 0) { + hdl_fast_iop_latency_log("usb-direct-read", &stats.direct_source_latency); + hdl_fast_iop_latency_log("source-fallback-read", &stats.fallback_source_latency); + hdl_fast_iop_latency_log("prefetch-consumer-wait", &stats.prefetch_wait_latency); + hdl_fast_iop_latency_log("hdd-write", &stats.hdd_write_latency); + hdl_fast_iop_latency_log("hdd-read", &stats.hdd_read_latency); + hdl_fast_iop_latency_log("sif-dma-completion", &stats.sif_dma_latency); + } + *logged = 1; } -static void hdl_fast_io_log_rate(void) +static HDL_FAST_COLD void hdl_fast_io_log_rate(void) { u64 elapsed; u32 seconds; @@ -130,8 +342,10 @@ static void hdl_fast_io_log_rate(void) (unsigned long long)(raw_usb_permille % 10u)); hdl_fast_rate_logged = 1; } +#endif -static s64 hdl_fast_fileXioLseek64(int fd, s64 offset, int whence) +static HDL_FAST_NOINLINE s64 hdl_fast_fileXioLseek64(int fd, s64 offset, + int whence) { s64 result = fileXioLseek64(fd, offset, whence); @@ -155,9 +369,14 @@ static s64 hdl_fast_fileXioLseek64(int fd, s64 offset, int whence) return result; } -static int hdl_fast_dma_read(int command, int source_fd, void *buffer, - unsigned int bytes) +static HDL_FAST_NOINLINE int hdl_fast_dma_read(int command, int source_fd, + void *buffer, + unsigned int bytes) { +#if HDL_PROFILE_ENABLED + u64 call_start; + u64 call_end; +#endif int result; if (hdl_active_target_fd < 0 || buffer == NULL || bytes == 0 || @@ -169,6 +388,10 @@ static int hdl_fast_dma_read(int command, int source_fd, void *buffer, * first, then invalidate after the synchronous ioctl returns so the R5900 * sees the new payload instead of a stale D-cache copy. */ SyncDCache(buffer, (unsigned char *)buffer + bytes); +#if HDL_PROFILE_ENABLED + hdl_fast_cache_maintenance_bytes += bytes; + call_start = GetTimerSystemTime(); +#endif if (command == HDL_STREAM_IOCTL2_TARGET_TO_EE) { hdl_stream_target_io_t request; @@ -192,12 +415,22 @@ static int hdl_fast_dma_read(int command, int source_fd, void *buffer, result = fileXioIoctl2(hdl_active_target_fd, command, &request, sizeof(request), NULL, 0); } - if (result >= 0) +#if HDL_PROFILE_ENABLED + call_end = GetTimerSystemTime(); + hdl_fast_record_ioctl_latency(command, call_start, call_end); +#endif + if (result >= 0) { InvalidDCache(buffer, (unsigned char *)buffer + bytes); +#if HDL_PROFILE_ENABLED + hdl_fast_cache_maintenance_bytes += bytes; + hdl_fast_sif_dma_bytes += bytes; +#endif + } return result; } -static int hdl_fast_fileXioRead(int fd, void *buffer, int size) +static HDL_FAST_NOINLINE int hdl_fast_fileXioRead(int fd, void *buffer, + int size) { int result; @@ -205,22 +438,44 @@ static int hdl_fast_fileXioRead(int fd, void *buffer, int size) return fileXioRead(fd, buffer, size); if (fd == hdl_active_target_fd) { +#if HDL_PROFILE_ENABLED + if (hdl_fast_target_consumer_start_ticks != 0) { + hdl_fast_latency_record(&hdl_fast_target_consumer_latency, + hdl_fast_target_consumer_start_ticks, + GetTimerSystemTime()); + hdl_fast_target_consumer_start_ticks = 0; + } +#endif hdl_fast_pump_mode = 0; hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; result = hdl_fast_dma_read(HDL_STREAM_IOCTL2_TARGET_TO_EE, -1, buffer, (unsigned int)size); - if (result != INT_MIN) + if (result != INT_MIN) { +#if HDL_PROFILE_ENABLED + if (result > 0) { + hdl_fast_target_bytes += (unsigned int)result; + hdl_fast_target_consumer_start_ticks = GetTimerSystemTime(); + if (hdl_fast_source_total != 0 && + hdl_fast_target_bytes >= hdl_fast_source_total) { + hdl_fast_io_log_stats(hdl_active_target_fd, 1); + hdl_fast_log_target_profile(); + } + } +#endif return result; + } } else if (hdl_active_target_fd >= 0 && hdl_fast_source_position_valid && fd == hdl_fast_source_fd) { int command = hdl_fast_pump_mode ? HDL_STREAM_IOCTL2_PUMP_TO_EE : HDL_STREAM_IOCTL2_SOURCE_TO_EE; +#if HDL_PROFILE_ENABLED if (command == HDL_STREAM_IOCTL2_PUMP_TO_EE && hdl_fast_copy_start_ticks == 0) hdl_fast_copy_start_ticks = GetTimerSystemTime(); +#endif result = hdl_fast_dma_read(command, fd, buffer, (unsigned int)size); if (result != INT_MIN) { hdl_fast_source_seek_pending = 0; @@ -229,16 +484,22 @@ static int hdl_fast_fileXioRead(int fd, void *buffer, int size) if (result == size && command == HDL_STREAM_IOCTL2_PUMP_TO_EE) { hdl_fast_pump_ack = 1; hdl_fast_pump_ack_bytes = (unsigned int)size; +#if HDL_PROFILE_ENABLED hdl_fast_copy_bytes += (unsigned int)size; + hdl_fast_copy_consumer_start_ticks = GetTimerSystemTime(); if (!hdl_fast_path_logged) { session_log_line( "HDL IOP fast pump active: double-buffered USB source -> ps2hdd + one IOP->EE SHA DMA, chunk=%u", (unsigned int)size); hdl_fast_path_logged = 1; } +#endif } else { hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; +#if HDL_PROFILE_ENABLED + hdl_fast_copy_consumer_start_ticks = 0; +#endif } return result; } @@ -257,36 +518,68 @@ static int hdl_fast_fileXioRead(int fd, void *buffer, int size) return seek < 0 ? (int)seek : HDL_INSTALL_COPY_FAILED; } result = fileXioRead(fd, buffer, size); +#if HDL_PROFILE_ENABLED + if (result > 0) { + if (fd == hdl_active_target_fd) + hdl_fast_fallback_target_bytes += (unsigned int)result; + else if (hdl_fast_source_position_valid && fd == hdl_fast_source_fd) + hdl_fast_fallback_source_bytes += (unsigned int)result; + } +#endif if (result > 0 && fd != hdl_active_target_fd && hdl_fast_source_position_valid && fd == hdl_fast_source_fd) hdl_fast_source_position += (unsigned int)result; hdl_fast_source_seek_pending = 0; hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; +#if HDL_PROFILE_ENABLED + hdl_fast_copy_consumer_start_ticks = 0; +#endif return result; } -static int hdl_fast_fileXioWrite(int fd, const void *buffer, int size) +static HDL_FAST_NOINLINE int hdl_fast_fileXioWrite(int fd, + const void *buffer, + int size) { if (hdl_fast_pump_ack) { if (fd == hdl_active_target_fd && size > 0 && (unsigned int)size == hdl_fast_pump_ack_bytes) { +#if HDL_PROFILE_ENABLED + if (hdl_fast_copy_consumer_start_ticks != 0) { + hdl_fast_latency_record(&hdl_fast_copy_consumer_latency, + hdl_fast_copy_consumer_start_ticks, + GetTimerSystemTime()); + hdl_fast_copy_consumer_start_ticks = 0; + } +#endif hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; +#if HDL_PROFILE_ENABLED if (hdl_fast_source_total != 0 && hdl_fast_source_position == hdl_fast_source_total) { hdl_fast_io_log_rate(); - hdl_fast_io_log_stats(hdl_active_target_fd); + hdl_fast_io_log_stats(hdl_active_target_fd, 0); + hdl_fast_log_copy_profile(); } +#endif return size; } /* Never fall back after PUMP_TO_EE already advanced the target. A * mismatched acknowledgement would otherwise write the block twice. */ hdl_fast_pump_ack = 0; hdl_fast_pump_ack_bytes = 0; +#if HDL_PROFILE_ENABLED + hdl_fast_copy_consumer_start_ticks = 0; +#endif return HDL_INSTALL_COPY_FAILED; } hdl_stream_tune_filexio_once(); return fileXioWrite(fd, buffer, size); } + +#if HDL_PROFILE_ENABLED +#undef HDL_FAST_COLD +#endif +#undef HDL_FAST_NOINLINE diff --git a/src/hdl_tools/source_ui_resume_hash.inc b/src/hdl_tools/source_ui_resume_hash.inc new file mode 100644 index 00000000..462b11ba --- /dev/null +++ b/src/hdl_tools/source_ui_resume_hash.inc @@ -0,0 +1,558 @@ +/* Guarded on-console HDLoader installer and large-disk HDL catalogue. */ + +#include +#include +#include +#define NEWLIB_PORT_AWARE +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "apa.h" +#include "app_ui_ps2.h" +#include "disk_status_ps2.h" +#include "gs_ui_ps2.h" +#include "hdd_read.h" +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED +#include "hdl_hash_checkpoint.h" +#endif +#include "hdl_installer_ps2.h" +#include "hdl_iso.h" +#include "hdl_partition.h" +#include "hdl_stream_rpc.h" +#include "hdl_transaction.h" +#include "platform.h" +#include "session_log.h" +#include "sha256.h" +#include "storage.h" + +#define HDL_INSTALL_JOURNAL "mass:/HDLINSTALL.TXN" +#define HDL_INSTALL_JOURNAL_NEW "mass:/HDLINSTALL.NEW" +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED +#define HDL_INSTALL_HASH_CHECKPOINT "mass:/HDLINSTALL.SHA" +#define HDL_INSTALL_HASH_CHECKPOINT_NEW "mass:/HDLINSTALL.SHN" +#endif +#define HDL_BROWSER_PAGE_SIZE 8u +#define HDL_INSTALL_IO_BYTES (64u * 1024u) +#define HDL_INSTALL_JOURNAL_INTERVAL_SECTORS 16384u +#define HDL_APA_TYPE 0x1337u +#define HDL_APA_FLAG_SUB 0x0001u +#define HDL_METADATA_LBA_OFFSET 0x0800u +#define HDL_CHAIN_NODE_LIMIT 16384u +#define HDL_METADATA_NOT_LOADED INT_MIN + +enum { + HDL_INSTALL_CANCELLED = -560, + HDL_INSTALL_NO_IMAGES = -561, + HDL_INSTALL_JOURNAL_INVALID = -562, + HDL_INSTALL_DISK_UNSAFE = -563, + HDL_INSTALL_DISK_TOO_LARGE = -564, + HDL_INSTALL_NO_SPACE = -565, + HDL_INSTALL_TARGET_EXISTS = -566, + HDL_INSTALL_CREATE_FAILED = -567, + HDL_INSTALL_LAYOUT_MISMATCH = -568, + HDL_INSTALL_SOURCE_CHANGED = -569, + HDL_INSTALL_COPY_FAILED = -570, + HDL_INSTALL_VERIFY_FAILED = -571, + HDL_INSTALL_METADATA_FAILED = -572, + HDL_INSTALL_MEMORY_FAILED = -573, + HDL_INSTALL_NO_GAMES = -574, + HDL_INSTALL_TARGET_CHANGED = -575, + HDL_INSTALL_TARGET_BUSY = -576, + HDL_INSTALL_CHAIN_INVALID = -577 +}; + +typedef struct { + char name[128]; + char path[HDL_TRANSACTION_SOURCE_PATH_MAX]; + uint64_t bytes; +} hdl_image_entry_t; + +typedef struct { + int fd; + uint64_t bytes; +} hdl_file_source_t; + +typedef struct { + char id[HDL_PARTITION_ID_MAX]; + hdl_metadata_info_t metadata; + unsigned char metadata_sha256[32]; + uint64_t allocation_bytes; + uint32_t main_lba; + unsigned int apa_partition_count; + int metadata_state; +} hdl_game_entry_t; + +typedef struct { + hdl_game_entry_t *games; + unsigned int count; + unsigned int capacity; + unsigned int hdl_sub_count; + unsigned int expected_sub_count; + uint32_t total_sectors; + uint64_t used_sectors; + uint64_t free_sectors; +} hdl_catalog_t; + +static unsigned char admission_header[APA_HEADER_SIZE] + __attribute__((aligned(64))); + +static uint64_t stat_bytes(const iox_stat_t *stat) +{ + return ((uint64_t)stat->hisize << 32) | stat->size; +} + +static int iso_name(const char *name) +{ + size_t length = strlen(name); + + return length > 4 && name[length - 4] == '.' && + (name[length - 3] == 'i' || name[length - 3] == 'I') && + (name[length - 2] == 's' || name[length - 2] == 'S') && + (name[length - 1] == 'o' || name[length - 1] == 'O'); +} + +static int compare_images(const void *left_value, const void *right_value) +{ + const hdl_image_entry_t *left = left_value; + const hdl_image_entry_t *right = right_value; + + return strcmp(left->name, right->name); +} + +static int scan_mass_images(hdl_image_entry_t **images_out, + unsigned int *count_out) +{ + iox_dirent_t entry; + hdl_image_entry_t *images; + unsigned int capacity = 32u; + unsigned int count = 0; + int directory; + int result; + + if (images_out == NULL || count_out == NULL) + return HDL_INSTALL_NO_IMAGES; + *images_out = NULL; + *count_out = 0; + images = calloc(capacity, sizeof(*images)); + if (images == NULL) + return HDL_INSTALL_MEMORY_FAILED; + directory = fileXioDopen("mass:/"); + if (directory < 0) { + free(images); + return directory; + } + while ((result = fileXioDread(directory, &entry)) > 0) { + hdl_image_entry_t *image; + int written; + + if (!FIO_S_ISREG(entry.stat.mode) || !iso_name(entry.name)) + continue; + if (count == capacity) { + hdl_image_entry_t *grown; + unsigned int next = capacity < 1024u ? capacity * 2u + : capacity + 1024u; + + grown = realloc(images, next * sizeof(*images)); + if (grown == NULL) { + fileXioDclose(directory); + free(images); + return HDL_INSTALL_MEMORY_FAILED; + } + images = grown; + memset(images + capacity, 0, + (next - capacity) * sizeof(*images)); + capacity = next; + } + image = &images[count]; + snprintf(image->name, sizeof(image->name), "%s", entry.name); + written = snprintf(image->path, sizeof(image->path), + "mass:/%s", entry.name); + if (written < 0 || (unsigned int)written >= sizeof(image->path)) + continue; + image->bytes = stat_bytes(&entry.stat); + count++; + } + fileXioDclose(directory); + if (result < 0) { + free(images); + return result; + } + if (count == 0) { + free(images); + return HDL_INSTALL_NO_IMAGES; + } + qsort(images, count, sizeof(*images), compare_images); + *images_out = images; + *count_out = count; + return 0; +} + +static unsigned int page_count(unsigned int count) +{ + return (count + HDL_BROWSER_PAGE_SIZE - 1u) / HDL_BROWSER_PAGE_SIZE; +} + +static unsigned int page_move_selection(unsigned int selected, + unsigned int count, int direction) +{ + unsigned int pages = page_count(count); + unsigned int page = selected / HDL_BROWSER_PAGE_SIZE; + unsigned int row = selected % HDL_BROWSER_PAGE_SIZE; + unsigned int start; + unsigned int shown; + + if (pages <= 1u) + return selected; + if (direction < 0) + page = (page + pages - 1u) % pages; + else + page = (page + 1u) % pages; + start = page * HDL_BROWSER_PAGE_SIZE; + shown = count - start; + if (shown > HDL_BROWSER_PAGE_SIZE) + shown = HDL_BROWSER_PAGE_SIZE; + if (row >= shown) + row = shown - 1u; + return start + row; +} + +static int select_image(const hdl_image_entry_t *images, unsigned int count) +{ + unsigned int selected = 0; + + for (;;) { + const char *labels[HDL_BROWSER_PAGE_SIZE]; + const char *hints[HDL_BROWSER_PAGE_SIZE]; + unsigned char enabled[HDL_BROWSER_PAGE_SIZE]; + unsigned int page = selected / HDL_BROWSER_PAGE_SIZE; + unsigned int start = page * HDL_BROWSER_PAGE_SIZE; + unsigned int shown = count - start; + unsigned int local = selected - start; + unsigned int i; + char size_hint[HDL_BROWSER_PAGE_SIZE][48]; + char status[160]; + u32 pressed; + + if (shown > HDL_BROWSER_PAGE_SIZE) + shown = HDL_BROWSER_PAGE_SIZE; + for (i = 0; i < shown; i++) { + labels[i] = images[start + i].name; + snprintf(size_hint[i], sizeof(size_hint[i]), "%llu MiB", + (unsigned long long)(images[start + i].bytes / + 1024u / 1024u)); + hints[i] = size_hint[i]; + enabled[i] = 1u; + } + snprintf(status, sizeof(status), + "%u ISO%s | page %u/%u | %llu MiB | LEFT/RIGHT page", + count, count == 1 ? "" : "s", page + 1u, page_count(count), + (unsigned long long)(images[selected].bytes / + 1024u / 1024u)); + gs_ui_render_menu("HDL installer: choose ISO", status, + labels, hints, enabled, shown, local); + pressed = wait_for_press(); + if (pressed & PAD_UP) { + if (local == 0) + selected = start + shown - 1u; + else + selected--; + } + if (pressed & PAD_DOWN) { + if (local + 1u >= shown) + selected = start; + else + selected++; + } + if (pressed & PAD_LEFT) + selected = page_move_selection(selected, count, -1); + if (pressed & PAD_RIGHT) + selected = page_move_selection(selected, count, 1); + if (pressed & PAD_CROSS) + return (int)selected; + if (pressed & PAD_TRIANGLE) + return -1; + } +} + +static int source_read(void *context, uint64_t offset, + void *destination, size_t size) +{ + hdl_file_source_t *source = context; + unsigned char *cursor = destination; + size_t complete = 0; + + if (source == NULL || offset > source->bytes || + (uint64_t)size > source->bytes - offset || + fileXioLseek64(source->fd, (s64)offset, FIO_SEEK_SET) != (s64)offset) + return -1; + while (complete < size) { + int result = fileXioRead(source->fd, cursor + complete, + (int)(size - complete)); + + if (result <= 0) + return -1; + complete += (unsigned int)result; + } + return 0; +} + +static int open_source(const char *path, uint64_t expected, + hdl_file_source_t *source) +{ + iox_stat_t stat; + + if (fileXioGetStat(path, &stat) < 0 || stat_bytes(&stat) != expected) + return HDL_INSTALL_SOURCE_CHANGED; + source->fd = fileXioOpen(path, FIO_O_RDONLY, 0); + if (source->fd < 0) + return source->fd; + source->bytes = expected; + return 0; +} + +static int source_fingerprint(hdl_file_source_t *source, + unsigned char digest[32]) +{ + unsigned char *buffer; + unsigned char size_bytes[8]; + sha256_context_t hash; + uint64_t tail; + unsigned int amount; + unsigned int i; + int result = 0; + + buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; + sha256_init(&hash); + for (i = 0; i < sizeof(size_bytes); i++) + size_bytes[i] = (unsigned char)(source->bytes >> (i * 8u)); + sha256_update(&hash, size_bytes, sizeof(size_bytes)); + amount = source->bytes < HDL_INSTALL_IO_BYTES ? + (unsigned int)source->bytes : HDL_INSTALL_IO_BYTES; + if (source_read(source, 0, buffer, amount) < 0) { + result = HDL_INSTALL_SOURCE_CHANGED; + goto done; + } + sha256_update(&hash, buffer, amount); + tail = source->bytes > HDL_INSTALL_IO_BYTES ? + source->bytes - HDL_INSTALL_IO_BYTES : 0; + if (source_read(source, tail, buffer, amount) < 0) { + result = HDL_INSTALL_SOURCE_CHANGED; + goto done; + } + sha256_update(&hash, buffer, amount); + sha256_final(&hash, digest); +done: + free(buffer); + return result; +} + +static int source_identity_matches(hdl_file_source_t *source, + const hdl_transaction_t *transaction) +{ + hdl_iso_source_t iso_source; + hdl_iso_info_t info; + int result; + + iso_source.read = source_read; + iso_source.context = source; + iso_source.image_bytes = source->bytes; + result = hdl_iso_probe(&iso_source, &info); + if (result < 0 || info.requires_layer_break || + info.image_bytes != transaction->source_bytes || + strcmp(info.startup, transaction->startup) != 0) + return HDL_INSTALL_SOURCE_CHANGED; + return 0; +} + +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED +static void hash_checkpoint_remove(void) +{ + (void)fileXioRemove(HDL_INSTALL_HASH_CHECKPOINT); + (void)fileXioRemove(HDL_INSTALL_HASH_CHECKPOINT_NEW); +} + +static int hash_checkpoint_save(const hdl_transaction_t *transaction, + const sha256_context_t *context) +{ + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + unsigned char verify[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + sha256_context_t restored; + int result; + + result = hdl_hash_checkpoint_encode(transaction, context, record); + if (result < 0) + return result; + result = write_whole_file(HDL_INSTALL_HASH_CHECKPOINT_NEW, + record, sizeof(record)); + if (result < 0) + return result; + result = read_exact_file(HDL_INSTALL_HASH_CHECKPOINT_NEW, + verify, sizeof(verify)); + if (result < 0 || memcmp(record, verify, sizeof(record)) != 0 || + hdl_hash_checkpoint_restore(verify, transaction, &restored) < 0) + return HDL_INSTALL_JOURNAL_INVALID; + (void)fileXioRemove(HDL_INSTALL_HASH_CHECKPOINT); + result = fileXioRename(HDL_INSTALL_HASH_CHECKPOINT_NEW, + HDL_INSTALL_HASH_CHECKPOINT); + return result < 0 ? result : 0; +} + +static int hash_checkpoint_load(const hdl_transaction_t *transaction, + sha256_context_t *context) +{ + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + int result; + + result = read_exact_file(HDL_INSTALL_HASH_CHECKPOINT, + record, sizeof(record)); + if (result == 0) + return hdl_hash_checkpoint_restore(record, transaction, context); + result = read_exact_file(HDL_INSTALL_HASH_CHECKPOINT_NEW, + record, sizeof(record)); + if (result < 0) + return result; + return hdl_hash_checkpoint_restore(record, transaction, context); +} +#endif + +static int journal_save(const hdl_transaction_t *transaction) +{ + unsigned char record[HDL_TRANSACTION_RECORD_SIZE]; + unsigned char verify[HDL_TRANSACTION_RECORD_SIZE]; + hdl_transaction_t decoded; + int result; + +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + if (transaction != NULL && transaction->completed_sectors == 0) + hash_checkpoint_remove(); +#endif + result = hdl_transaction_encode(transaction, record); + if (result < 0) + return result; + result = write_whole_file(HDL_INSTALL_JOURNAL_NEW, record, sizeof(record)); + if (result < 0) + return result; + result = read_exact_file(HDL_INSTALL_JOURNAL_NEW, verify, sizeof(verify)); + if (result < 0 || memcmp(record, verify, sizeof(record)) != 0 || + hdl_transaction_decode(verify, &decoded) < 0) + return HDL_INSTALL_JOURNAL_INVALID; + (void)fileXioRemove(HDL_INSTALL_JOURNAL); + result = fileXioRename(HDL_INSTALL_JOURNAL_NEW, HDL_INSTALL_JOURNAL); + return result < 0 ? result : 0; +} + +static int journal_load(hdl_transaction_t *transaction) +{ + unsigned char record[HDL_TRANSACTION_RECORD_SIZE]; + int result = read_exact_file(HDL_INSTALL_JOURNAL, record, sizeof(record)); + + if (result == 0 && hdl_transaction_decode(record, transaction) == 0) + return 0; + result = read_exact_file(HDL_INSTALL_JOURNAL_NEW, + record, sizeof(record)); + if (result < 0) + return result; + return hdl_transaction_decode(record, transaction); +} + +static void journal_remove(void) +{ + (void)fileXioRemove(HDL_INSTALL_JOURNAL); + (void)fileXioRemove(HDL_INSTALL_JOURNAL_NEW); +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + hash_checkpoint_remove(); +#endif +} + +static int target_path(const char *prefix, const char *target, + char *destination, unsigned int capacity) +{ + int written = snprintf(destination, capacity, "%s%s", prefix, target); + + return written < 0 || (unsigned int)written >= capacity ? -1 : 0; +} + +static int read_target_metadata(const char *target, + unsigned char metadata[HDL_METADATA_SIZE]) +{ + char path[64]; + unsigned int complete = 0; + int fd; + int result; + + if (target_path("hdd0:", target, path, sizeof(path)) < 0) + return HDL_INSTALL_LAYOUT_MISMATCH; + fd = fileXioOpen(path, FIO_O_RDONLY, 0); + if (fd < 0) + return fd; + result = fileXioLseek(fd, 0x100000, FIO_SEEK_SET); + if (result != 0x100000) { + fileXioClose(fd); + return result < 0 ? result : HDL_INSTALL_METADATA_FAILED; + } + while (complete < HDL_METADATA_SIZE) { + result = fileXioRead(fd, metadata + complete, + HDL_METADATA_SIZE - complete); + if (result <= 0) { + fileXioClose(fd); + return result < 0 ? result : HDL_INSTALL_METADATA_FAILED; + } + complete += (unsigned int)result; + } + fileXioClose(fd); + return 0; +} + +static int target_has_metadata(const char *target) +{ + unsigned char metadata[HDL_METADATA_SIZE]; + int result = read_target_metadata(target, metadata); + + if (result < 0) + return result; + return metadata[0] == 0xed && metadata[1] == 0xfe && + metadata[2] == 0xad && metadata[3] == 0xde; +} + +static int target_metadata_matches(const char *target, + const unsigned char expected[HDL_METADATA_SIZE]) +{ + unsigned char actual[HDL_METADATA_SIZE]; + int result = read_target_metadata(target, actual); + + return result == 0 && memcmp(actual, expected, sizeof(actual)) == 0; +} + +static int target_exists(const char *target) +{ + iox_stat_t stat; + char path[64]; + + if (target_path("hdd0:", target, path, sizeof(path)) < 0) + return 1; + return fileXioGetStat(path, &stat) >= 0; +} + +static int remove_incomplete_target(const char *target) +{ + char path[64]; + int metadata_state = target_has_metadata(target); + + if (metadata_state > 0) + return HDL_INSTALL_TARGET_EXISTS; + if (metadata_state < 0) + return metadata_state; + if (target_path("hdd0:", target, path, sizeof(path)) < 0) + return HDL_INSTALL_LAYOUT_MISMATCH; + return fileXioRemove(path); +} diff --git a/src/hdl_tools/transaction.inc b/src/hdl_tools/transaction.inc index 75a03a4a..7cb1e926 100644 --- a/src/hdl_tools/transaction.inc +++ b/src/hdl_tools/transaction.inc @@ -459,6 +459,7 @@ static int execute_transaction(hdl_transaction_t *transaction) int target_fd = -1; int verified_this_run = 0; int source_digest_valid = 0; + int metadata_valid = 0; int result; source.fd = -1; @@ -591,6 +592,7 @@ static int execute_transaction(hdl_transaction_t *transaction) &metadata_options, metadata); if (result < 0) goto done; + metadata_valid = 1; disk_status_phase_at("Committing verified game metadata last", "Main partition attribute area / HDL metadata"); disk_status_io(DISK_STATUS_WRITE, @@ -613,18 +615,27 @@ static int execute_transaction(hdl_transaction_t *transaction) } } if (transaction->stage == HDL_TRANSACTION_STAGE_METADATA_COMMITTED) { - metadata_options.game_title = transaction->game_title; - metadata_options.startup = transaction->startup; - metadata_options.disc_type = transaction->disc_type; - metadata_options.layer1_start = transaction->layer1_start; - metadata_options.hdl_compat_flags = transaction->hdl_compat_flags; - metadata_options.opl_compat_flags = transaction->opl_compat_flags; - metadata_options.dma_type = transaction->dma_type; - metadata_options.dma_mode = transaction->dma_mode; - result = hdl_metadata_build(&plan, layout.starts, layout.count, - &metadata_options, metadata); - if (result < 0 || - !target_metadata_matches(transaction->target, metadata)) { + /* A stage-4 -> stage-5 transition in this invocation already built the + * exact canonical block that was committed. Keep using that same + * transaction-owned buffer for read-back comparison. Only a resumed + * stage-5 transaction needs to reconstruct it. */ + if (!metadata_valid) { + metadata_options.game_title = transaction->game_title; + metadata_options.startup = transaction->startup; + metadata_options.disc_type = transaction->disc_type; + metadata_options.layer1_start = transaction->layer1_start; + metadata_options.hdl_compat_flags = transaction->hdl_compat_flags; + metadata_options.opl_compat_flags = transaction->opl_compat_flags; + metadata_options.dma_type = transaction->dma_type; + metadata_options.dma_mode = transaction->dma_mode; + result = hdl_metadata_build(&plan, layout.starts, layout.count, + &metadata_options, metadata); + if (result < 0) { + result = HDL_INSTALL_METADATA_FAILED; + goto done; + } + } + if (!target_metadata_matches(transaction->target, metadata)) { result = HDL_INSTALL_METADATA_FAILED; goto done; } diff --git a/src/hdl_tools/transaction_resume_hash.inc b/src/hdl_tools/transaction_resume_hash.inc new file mode 100644 index 00000000..ea68c9a7 --- /dev/null +++ b/src/hdl_tools/transaction_resume_hash.inc @@ -0,0 +1,717 @@ +static const char *allocation_name(uint64_t bytes) +{ + static const char *const names[] = {"128M", "256M", "512M", + "1G", "2G", "4G"}; + static const uint64_t sizes[] = { + 128ull << 20, 256ull << 20, 512ull << 20, + 1024ull << 20, 2048ull << 20, 4096ull << 20 + }; + unsigned int i; + + for (i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i++) { + if (bytes == sizes[i]) + return names[i]; + } + return NULL; +} + +static int journal_blocks_game_delete(const char *target) +{ + hdl_transaction_t transaction; + int result; + + if (!path_exists(HDL_INSTALL_JOURNAL) && + !path_exists(HDL_INSTALL_JOURNAL_NEW)) + return 0; + result = journal_load(&transaction); + if (result < 0) + return HDL_INSTALL_JOURNAL_INVALID; + if (transaction.stage != HDL_TRANSACTION_STAGE_COMPLETE && + strcmp(transaction.target, target) == 0) + return HDL_INSTALL_TARGET_BUSY; + return 0; +} + +static int verify_game_snapshot(const hdl_game_entry_t *game, + uint32_t total_sectors) +{ + unsigned char metadata[HDL_METADATA_SIZE] __attribute__((aligned(64))); + unsigned char digest[32]; + hdl_metadata_info_t parsed; + iox_stat_t stat; + char path[64]; + uint32_t lba; + int result; + + if (game->metadata_state != 0 || + target_path("hdd0:", game->id, path, sizeof(path)) < 0) + return HDL_INSTALL_TARGET_CHANGED; + result = fileXioGetStat(path, &stat); + if (result < 0 || stat.mode != HDL_APA_TYPE || + (stat.attr & HDL_APA_FLAG_SUB)) + return HDL_INSTALL_TARGET_CHANGED; + if (game->main_lba > UINT32_MAX - HDL_METADATA_LBA_OFFSET) + return HDL_INSTALL_TARGET_CHANGED; + lba = game->main_lba + HDL_METADATA_LBA_OFFSET; + if (lba >= total_sectors || total_sectors - lba < 2u || + hdd_read_raw_sectors(lba, 2, metadata) < 0 || + hdl_metadata_parse(metadata, &parsed) < 0 || + parsed.partition_count != game->apa_partition_count) + return HDL_INSTALL_TARGET_CHANGED; + sha256_buffer(metadata, sizeof(metadata), digest); + if (memcmp(digest, game->metadata_sha256, sizeof(digest)) != 0) + return HDL_INSTALL_TARGET_CHANGED; + return 0; +} + +static int delete_installed_game(const hdl_game_entry_t *game, + uint32_t total_sectors) +{ + uint32_t maximum; + uint32_t free_sectors; + char path[64]; + int result; + + /* Before the user arms deletion these are genuinely read-only safety + * checks, so keep the ordinary READ monitor semantics here. */ + result = recheck_disk(&maximum, &free_sectors); + if (result == 0) + result = journal_blocks_game_delete(game->id); + if (result == 0) + result = verify_game_snapshot(game, total_sectors); + if (result < 0) + return result; + (void)maximum; + (void)free_sectors; + + scr_clear(); + scr_printf("Delete installed HDL game\n\n"); + scr_printf("Game : %s\n", game->metadata.game_title); + scr_printf("Startup: %s\n", game->metadata.startup); + scr_printf("Target : %s\n", game->id); + scr_printf("Size : %llu MiB allocated\n\n", + (unsigned long long)(game->allocation_bytes / 1024u / 1024u)); + scr_printf("This removes the main partition and all of its subs.\n"); + scr_printf("There is no undo operation.\n\n"); + scr_printf("Hold L1+R1 and press SQUARE to delete.\n"); + scr_printf("TRIANGLE returns without changing the HDD.\n"); + if (!wait_for_chord(PAD_L1 | PAD_R1 | PAD_SQUARE)) + return HDL_INSTALL_CANCELLED; + + /* Once destructive confirmation has been accepted, validation reads are + * no longer an independent read-only operation: they are the preflight of + * an armed write path. Keep ACTION truthful, but never display the green + * read-only footer during this window. */ + disk_status_begin_at("HDL game removal", + "Write path armed; revalidating selected target", + "Selected HDL main/sub partition chain"); + disk_status_set_write_intent(1); + result = recheck_disk(&maximum, &free_sectors); + if (result == 0) + result = journal_blocks_game_delete(game->id); + if (result == 0) + result = verify_game_snapshot(game, total_sectors); + if (result < 0) { + disk_status_end(); + return result; + } + if (target_path("hdd0:", game->id, path, sizeof(path)) < 0) { + disk_status_end(); + return HDL_INSTALL_TARGET_CHANGED; + } + + disk_status_phase_at("Removing selected APA allocation", + "APA metadata and linked HDL sub-partitions"); + disk_status_io(DISK_STATUS_WRITE, game->main_lba, 2u, 0, 0); + result = fileXioRemove(path); + disk_status_end(); + session_log_line("HDL game delete target=%s startup=%s result=%d", + game->id, game->metadata.startup, result); + return result < 0 ? result : 0; +} + +static int create_partitions(const hdl_transaction_t *transaction, + const hdl_partition_plan_t *plan) +{ + char existing[64]; + char create[96]; + int fd; + unsigned int i; + + target_path("hdd0:", transaction->target, existing, sizeof(existing)); + fd = fileXioOpen(existing, FIO_O_RDONLY, 0); + if (fd >= 0) { + fileXioClose(fd); + return HDL_INSTALL_TARGET_EXISTS; + } + disk_status_phase_at("Creating HDL main partition", + "APA allocation metadata"); + disk_status_io(DISK_STATUS_WRITE, 0, 0, 0, 0); + snprintf(create, sizeof(create), "hdd0:%s,,,%s,HDL", + transaction->target, + allocation_name(plan->slices[0].allocation_bytes)); + fd = fileXioOpen(create, FIO_O_RDWR | FIO_O_CREAT, 0666); + if (fd < 0) + return fd; + for (i = 1; i < plan->count; i++) { + const char *size = allocation_name(plan->slices[i].allocation_bytes); + int result; + + disk_status_phase_at("Adding HDL sub-partition", + "APA allocation metadata"); + disk_status_io(DISK_STATUS_WRITE, 0, 0, i, plan->count); + result = fileXioIoctl2(fd, HIOCADDSUB, (void *)size, + strlen(size) + 1u, NULL, 0); + + if (result < 0) { + fileXioClose(fd); + (void)remove_incomplete_target(transaction->target); + return HDL_INSTALL_CREATE_FAILED; + } + } + disk_status_phase_at("Flushing APA allocation metadata", + "New HDL main/sub partition chain"); + disk_status_io(DISK_STATUS_FLUSH, 0, 0, plan->count, plan->count); + if (fileXioIoctl2(fd, HIOCFLUSH, NULL, 0, NULL, 0) < 0) { + fileXioClose(fd); + (void)remove_incomplete_target(transaction->target); + return HDL_INSTALL_CREATE_FAILED; + } + fileXioClose(fd); + return 0; +} + +static int open_target(const hdl_transaction_t *transaction, + const hdl_partition_plan_t *plan, + hdl_stream_layout_t *layout) +{ + char path[64]; + int fd; + unsigned int i; + + if (target_path("hdl0:", transaction->target, path, sizeof(path)) < 0) + return HDL_INSTALL_LAYOUT_MISMATCH; + fd = fileXioOpen(path, FIO_O_RDWR, 0); + if (fd < 0) + return fd; + if (fileXioIoctl2(fd, HDL_STREAM_IOCTL2_GET_LAYOUT, + NULL, 0, layout, sizeof(*layout)) < 0 || + layout->count != plan->count) { + fileXioClose(fd); + return HDL_INSTALL_LAYOUT_MISMATCH; + } + for (i = 0; i < plan->count; i++) { + if ((uint64_t)layout->lengths[i] * 512u != + plan->slices[i].allocation_bytes || layout->starts[i] == 0) { + fileXioClose(fd); + return HDL_INSTALL_LAYOUT_MISMATCH; + } + } + return fd; +} + +static int read_exact_fd(int fd, unsigned char *buffer, unsigned int bytes) +{ + unsigned int complete = 0; + + while (complete < bytes) { + int result = fileXioRead(fd, buffer + complete, bytes - complete); + + if (result <= 0) + return result < 0 ? result : HDL_INSTALL_COPY_FAILED; + complete += (unsigned int)result; + } + return 0; +} + +static int write_exact_fd(int fd, const unsigned char *buffer, + unsigned int bytes) +{ + unsigned int complete = 0; + + while (complete < bytes) { + int result = fileXioWrite(fd, buffer + complete, bytes - complete); + + if (result <= 0) + return result < 0 ? result : HDL_INSTALL_COPY_FAILED; + complete += (unsigned int)result; + } + return 0; +} + +static uint32_t physical_lba(const hdl_partition_plan_t *plan, + const hdl_stream_layout_t *layout, + uint64_t payload_offset) +{ + unsigned int i; + + for (i = 0; i < plan->count; i++) { + uint64_t begin = plan->slices[i].payload_offset; + uint64_t end = begin + plan->slices[i].payload_bytes; + + if (payload_offset >= begin && payload_offset < end) + return layout->starts[i] + (i == 0 ? 0x2000u : 0x0800u) + + (uint32_t)((payload_offset - begin) / 512u); + } + return 0; +} + +/* + * Hash a source ISO without involving the HDD. This is primarily used when a + * previously PAYLOAD_VERIFIED journal is resumed and no matching optional + * SHA-state checkpoint exists. Fresh installs do not pay this second USB pass + * because copy_payload() hashes source bytes while they are already moving. + */ +static int hash_source_payload(const hdl_transaction_t *transaction, + int source_fd, + unsigned char digest[32]) +{ + unsigned char *buffer; + sha256_context_t hash; + uint64_t offset = 0; + int result = 0; + + buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; + if (fileXioLseek64(source_fd, 0, FIO_SEEK_SET) != 0) { + result = HDL_INSTALL_VERIFY_FAILED; + goto done; + } + sha256_init(&hash); + while (offset < transaction->source_bytes) { + uint64_t remaining = transaction->source_bytes - offset; + unsigned int bytes = remaining > HDL_INSTALL_IO_BYTES ? + HDL_INSTALL_IO_BYTES : (unsigned int)remaining; + + if (read_exact_fd(source_fd, buffer, bytes) < 0) { + result = HDL_INSTALL_VERIFY_FAILED; + goto done; + } + sha256_update(&hash, buffer, bytes); + offset += bytes; + disk_status_io(DISK_STATUS_VERIFY, 0, 0, + (unsigned int)(offset / 2048u), + (unsigned int)transaction->total_sectors); + } + sha256_final(&hash, digest); +done: + free(buffer); + return result; +} + +static int copy_payload(hdl_transaction_t *transaction, + const hdl_partition_plan_t *plan, + const hdl_stream_layout_t *layout, + int source_fd, int target_fd, + unsigned char source_digest[32]) +{ + unsigned char *buffer; + sha256_context_t source_hash; + uint64_t offset = transaction->completed_sectors * 2048u; + uint64_t hash_offset = 0; + uint64_t next_journal = transaction->completed_sectors + + HDL_INSTALL_JOURNAL_INTERVAL_SECTORS; + int result = 0; + + buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; + sha256_init(&source_hash); + + /* Resume needs the SHA state for the already-copied prefix. A validated + * checkpoint makes that prefix O(1) to restore. Missing/stale/corrupt + * checkpoints deliberately fall through to the historical full USB rehash, + * preserving the previous correctness contract. */ + if (offset > 0) { +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + int checkpoint_result = hash_checkpoint_load(transaction, &source_hash); + + if (checkpoint_result == 0) { + hash_offset = offset; + session_log_line( + "HDL restored source SHA checkpoint bytes=%llu; skipped prefix rehash", + (unsigned long long)offset); + } else { + session_log_line( + "HDL source SHA checkpoint unavailable result=%d; using safe prefix rehash", + checkpoint_result); +#endif + disk_status_phase_at("Rebuilding source digest for resumed copy", + "Already-copied prefix on mass:"); + if (fileXioLseek64(source_fd, 0, FIO_SEEK_SET) != 0) { + result = HDL_INSTALL_COPY_FAILED; + goto done; + } + while (hash_offset < offset) { + uint64_t remaining = offset - hash_offset; + unsigned int bytes = remaining > HDL_INSTALL_IO_BYTES ? + HDL_INSTALL_IO_BYTES : (unsigned int)remaining; + + result = read_exact_fd(source_fd, buffer, bytes); + if (result < 0) + goto done; + sha256_update(&source_hash, buffer, bytes); + hash_offset += bytes; + disk_status_io(DISK_STATUS_VERIFY, 0, 0, + (unsigned int)(hash_offset / 2048u), + (unsigned int)transaction->total_sectors); + } +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + checkpoint_result = hash_checkpoint_save(transaction, &source_hash); + if (checkpoint_result < 0) + session_log_line( + "HDL source SHA checkpoint refresh failed result=%d; resume remains safe", + checkpoint_result); + } +#endif + } + + disk_status_phase_at("Copying ISO payload from mass:", + "HDL payload data area"); + if (fileXioLseek64(source_fd, (s64)offset, FIO_SEEK_SET) != (s64)offset || + fileXioLseek64(target_fd, (s64)offset, FIO_SEEK_SET) != (s64)offset) { + result = HDL_INSTALL_COPY_FAILED; + goto done; + } + while (offset < transaction->source_bytes) { + uint64_t remaining = transaction->source_bytes - offset; + unsigned int bytes = remaining > HDL_INSTALL_IO_BYTES ? + HDL_INSTALL_IO_BYTES : (unsigned int)remaining; + u32 pressed = 0; + + result = read_exact_fd(source_fd, buffer, bytes); + if (result < 0) + goto done; + sha256_update(&source_hash, buffer, bytes); + result = write_exact_fd(target_fd, buffer, bytes); + if (result < 0) + goto done; + offset += bytes; + disk_status_io(DISK_STATUS_WRITE, + physical_lba(plan, layout, offset - bytes), bytes / 512u, + (unsigned int)(offset / 2048u), + (unsigned int)transaction->total_sectors); + transaction->completed_sectors = offset / 2048u; + if (transaction->completed_sectors >= next_journal || + offset == transaction->source_bytes) { +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + int checkpoint_result = hash_checkpoint_save(transaction, &source_hash); + + if (checkpoint_result < 0) + session_log_line( + "HDL source SHA checkpoint save failed progress=%llu result=%d; journal remains authoritative", + (unsigned long long)transaction->completed_sectors, + checkpoint_result); +#endif + result = journal_save(transaction); + if (result < 0) + goto done; + next_journal = transaction->completed_sectors + + HDL_INSTALL_JOURNAL_INTERVAL_SECTORS; + } + if (poll_for_press(&pressed) && (pressed & PAD_TRIANGLE)) { +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + int checkpoint_result = hash_checkpoint_save(transaction, &source_hash); + + if (checkpoint_result < 0) + session_log_line( + "HDL source SHA checkpoint save on cancel failed result=%d", + checkpoint_result); +#endif + result = journal_save(transaction); + if (result == 0) + result = HDL_INSTALL_CANCELLED; + goto done; + } + } + sha256_final(&source_hash, source_digest); +done: + free(buffer); + return result; +} + +/* + * Verify the installed payload from HDD only. The source digest is accumulated + * while fresh bytes are copied. On resume, an optional authenticated SHA-state + * checkpoint can reconstruct it without replaying already-consumed USB bytes; + * otherwise the historical source-only pass remains the safe baseline. + */ +static int verify_target_digest(const hdl_transaction_t *transaction, + const hdl_partition_plan_t *plan, + const hdl_stream_layout_t *layout, + int target_fd, + const unsigned char expected_digest[32]) +{ + unsigned char *target_buffer; + unsigned char target_digest[32]; + sha256_context_t target_hash; + uint64_t offset = 0; + int result = 0; + + target_buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (target_buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; + if (fileXioLseek64(target_fd, 0, FIO_SEEK_SET) != 0) { + result = HDL_INSTALL_VERIFY_FAILED; + goto done; + } + sha256_init(&target_hash); + while (offset < transaction->source_bytes) { + uint64_t remaining = transaction->source_bytes - offset; + unsigned int bytes = remaining > HDL_INSTALL_IO_BYTES ? + HDL_INSTALL_IO_BYTES : (unsigned int)remaining; + + if (read_exact_fd(target_fd, target_buffer, bytes) < 0) { + result = HDL_INSTALL_VERIFY_FAILED; + goto done; + } + sha256_update(&target_hash, target_buffer, bytes); + disk_status_io(DISK_STATUS_VERIFY, + physical_lba(plan, layout, offset), bytes / 512u, + (unsigned int)(offset / 2048u), + (unsigned int)transaction->total_sectors); + offset += bytes; + } + sha256_final(&target_hash, target_digest); + if (memcmp(expected_digest, target_digest, sizeof(target_digest)) != 0) + result = HDL_INSTALL_VERIFY_FAILED; +done: + free(target_buffer); + return result; +} + +static int execute_transaction(hdl_transaction_t *transaction) +{ + hdl_partition_plan_t plan; + hdl_stream_layout_t layout; + hdl_metadata_options_t metadata_options; + unsigned char metadata[HDL_METADATA_SIZE]; + hdl_file_source_t source; + unsigned char fingerprint[32]; + unsigned char source_payload_digest[32]; + uint32_t maximum; + uint32_t free_sectors; + int target_fd = -1; + int verified_this_run = 0; + int source_digest_valid = 0; + int metadata_valid = 0; + int result; + + source.fd = -1; + source.bytes = transaction->source_bytes; + result = recheck_disk(&maximum, &free_sectors); + if (result < 0) + return result; + result = hdl_partition_plan(transaction->source_bytes, maximum, &plan); + if (result < 0) + return result; + if (transaction->partition_count != plan.count) + return HDL_INSTALL_LAYOUT_MISMATCH; + if (plan.allocation_bytes / 512u > free_sectors && + transaction->stage == HDL_TRANSACTION_STAGE_PLANNED) + return HDL_INSTALL_NO_SPACE; + + /* Once METADATA_COMMITTED is durable, payload and source identity have + * already passed full verification. Completion recovery only needs the + * target layout plus metadata read-back, so do not require the USB ISO to + * remain connected merely to move stage 5 -> COMPLETE. */ + if (transaction->stage < HDL_TRANSACTION_STAGE_METADATA_COMMITTED) { + result = open_source(transaction->source_path, + transaction->source_bytes, &source); + if (result < 0) + return result; + result = source_fingerprint(&source, fingerprint); + if (result == 0 && memcmp(fingerprint, transaction->source_fingerprint, + sizeof(fingerprint)) != 0) + result = HDL_INSTALL_SOURCE_CHANGED; + if (result == 0) + result = source_identity_matches(&source, transaction); + if (result < 0) { + fileXioClose(source.fd); + return result; + } + } + + pad_activity_begin(); + disk_status_begin_at("HDL game installation", + "Preparing guarded APA allocation", + "Standard HDL main/sub partitions"); + /* Reaching execute_transaction means the user has already authorized an + * install or resume. Validation may still read first, but the transaction + * owns an armed write path until it exits. */ + disk_status_set_write_intent(1); + if (transaction->stage == HDL_TRANSACTION_STAGE_PLANNED) { + result = create_partitions(transaction, &plan); + if (result < 0) + goto done; + result = hdl_transaction_set_stage( + transaction, HDL_TRANSACTION_STAGE_PARTITIONS_CREATED, 0); + if (result < 0 || journal_save(transaction) < 0) { + result = HDL_INSTALL_JOURNAL_INVALID; + goto done; + } + } + target_fd = open_target(transaction, &plan, &layout); + if (target_fd < 0) { + result = target_fd; + goto done; + } + if (transaction->stage == HDL_TRANSACTION_STAGE_PARTITIONS_CREATED) { + result = hdl_transaction_set_stage( + transaction, HDL_TRANSACTION_STAGE_COPYING, 0); + if (result < 0 || journal_save(transaction) < 0) { + result = HDL_INSTALL_JOURNAL_INVALID; + goto done; + } + } + if (transaction->stage == HDL_TRANSACTION_STAGE_COPYING) { + result = copy_payload(transaction, &plan, &layout, + source.fd, target_fd, source_payload_digest); + if (result < 0) + goto done; + source_digest_valid = 1; + disk_status_phase_at("Flushing copied HDL payload", + "HDL payload data area"); + disk_status_io(DISK_STATUS_FLUSH, + layout.starts[0], 0, transaction->total_sectors, + transaction->total_sectors); + if (fileXioIoctl2(target_fd, HDL_STREAM_IOCTL2_FLUSH, + NULL, 0, NULL, 0) < 0) { + result = HDL_INSTALL_COPY_FAILED; + goto done; + } + disk_status_phase_at("Verifying installed payload on HDD", + "HDD-only SHA-256 read-back against source digest"); + result = verify_target_digest(transaction, &plan, &layout, + target_fd, source_payload_digest); + if (result < 0) + goto done; + verified_this_run = 1; + result = hdl_transaction_set_stage( + transaction, HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED, + transaction->total_sectors); + if (result < 0 || journal_save(transaction) < 0) { + result = HDL_INSTALL_JOURNAL_INVALID; + goto done; + } + } + if (transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) { + if (!verified_this_run) { +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + sha256_context_t restored_source_hash; + int checkpoint_result = + hash_checkpoint_load(transaction, &restored_source_hash); + + if (checkpoint_result == 0) { + sha256_final(&restored_source_hash, source_payload_digest); + source_digest_valid = 1; + session_log_line( + "HDL restored complete source SHA checkpoint bytes=%llu; skipped full USB hash pass", + (unsigned long long)transaction->source_bytes); + } else { + session_log_line( + "HDL complete source SHA checkpoint unavailable result=%d; using safe full source hash", + checkpoint_result); +#endif + disk_status_phase_at("Hashing source for resumed verification", + "One source pass required without a matching hash checkpoint"); + result = hash_source_payload(transaction, source.fd, + source_payload_digest); + if (result < 0) + goto done; + source_digest_valid = 1; +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + } +#endif + disk_status_phase_at("Re-verifying resumed payload on HDD", + "HDD-only SHA-256 read-back against source digest"); + result = verify_target_digest(transaction, &plan, &layout, + target_fd, source_payload_digest); + if (result < 0) + goto done; + } + if (!source_digest_valid && !verified_this_run) { + result = HDL_INSTALL_VERIFY_FAILED; + goto done; + } + metadata_options.game_title = transaction->game_title; + metadata_options.startup = transaction->startup; + metadata_options.disc_type = transaction->disc_type; + metadata_options.layer1_start = transaction->layer1_start; + metadata_options.hdl_compat_flags = transaction->hdl_compat_flags; + metadata_options.opl_compat_flags = transaction->opl_compat_flags; + metadata_options.dma_type = transaction->dma_type; + metadata_options.dma_mode = transaction->dma_mode; + result = hdl_metadata_build(&plan, layout.starts, layout.count, + &metadata_options, metadata); + if (result < 0) + goto done; + metadata_valid = 1; + disk_status_phase_at("Committing verified game metadata last", + "Main partition attribute area / HDL metadata"); + disk_status_io(DISK_STATUS_WRITE, + layout.starts[0] + HDL_METADATA_LBA_OFFSET, + 2u, transaction->total_sectors, + transaction->total_sectors); + result = fileXioIoctl2(target_fd, + HDL_STREAM_IOCTL2_COMMIT_METADATA, + metadata, sizeof(metadata), NULL, 0); + if (result < 0) { + result = HDL_INSTALL_METADATA_FAILED; + goto done; + } + result = hdl_transaction_set_stage( + transaction, HDL_TRANSACTION_STAGE_METADATA_COMMITTED, + transaction->total_sectors); + if (result < 0 || journal_save(transaction) < 0) { + result = HDL_INSTALL_JOURNAL_INVALID; + goto done; + } + } + if (transaction->stage == HDL_TRANSACTION_STAGE_METADATA_COMMITTED) { + /* A stage-4 -> stage-5 transition in this invocation already built the + * exact canonical block that was committed. Keep using that same + * transaction-owned buffer for read-back comparison. Only a resumed + * stage-5 transaction needs to reconstruct it. */ + if (!metadata_valid) { + metadata_options.game_title = transaction->game_title; + metadata_options.startup = transaction->startup; + metadata_options.disc_type = transaction->disc_type; + metadata_options.layer1_start = transaction->layer1_start; + metadata_options.hdl_compat_flags = transaction->hdl_compat_flags; + metadata_options.opl_compat_flags = transaction->opl_compat_flags; + metadata_options.dma_type = transaction->dma_type; + metadata_options.dma_mode = transaction->dma_mode; + result = hdl_metadata_build(&plan, layout.starts, layout.count, + &metadata_options, metadata); + if (result < 0) { + result = HDL_INSTALL_METADATA_FAILED; + goto done; + } + } + if (!target_metadata_matches(transaction->target, metadata)) { + result = HDL_INSTALL_METADATA_FAILED; + goto done; + } + result = hdl_transaction_set_stage( + transaction, HDL_TRANSACTION_STAGE_COMPLETE, + transaction->total_sectors); + if (result < 0) + goto done; + journal_remove(); + } + result = 0; +done: + if (target_fd >= 0) + fileXioClose(target_fd); + if (source.fd >= 0) + fileXioClose(source.fd); + disk_status_end(); + pad_activity_end(); + session_log_line("HDL transaction target=%s stage=%u progress=%llu/%llu result=%d", + transaction->target, (unsigned int)transaction->stage, + (unsigned long long)transaction->completed_sectors, + (unsigned long long)transaction->total_sectors, result); + return result; +} diff --git a/src/printf_policy_ps2.c b/src/printf_policy_ps2.c new file mode 100644 index 00000000..188dd3d3 --- /dev/null +++ b/src/printf_policy_ps2.c @@ -0,0 +1,35 @@ +/* + * Phase-1 bounded formatting policy. + * + * CURRENT IMPLEMENTATION (PS2DEV Newlib ee-v4.6.0): vsniprintf() has the same + * bounded string-buffer contract as vsnprintf() but routes through + * _svfiprintf_r, the integer-only formatter, instead of _svfprintf_r. The + * latter retains dtoa/floating formatting support that this application does + * not use in its audited runtime format strings. + * + * The linker wraps only snprintf/vsnprintf references. The Phase-1 format audit + * is the companion correctness guard: introducing %a/%e/%f/%g formatting must + * first remove this policy or add an explicitly separate floating formatter. + */ + +#include +#include +#include + +int __wrap_vsnprintf(char *buffer, size_t capacity, + const char *format, va_list arguments) +{ + return vsniprintf(buffer, capacity, format, arguments); +} + +int __wrap_snprintf(char *buffer, size_t capacity, + const char *format, ...) +{ + va_list arguments; + int result; + + va_start(arguments, format); + result = vsniprintf(buffer, capacity, format, arguments); + va_end(arguments); + return result; +} diff --git a/src/r5900_perf.c b/src/r5900_perf.c new file mode 100644 index 00000000..1e92f51a --- /dev/null +++ b/src/r5900_perf.c @@ -0,0 +1,131 @@ +#include "r5900_perf.h" + +#include + +#define R5900_PCCR_CTE (1u << 31) +#define R5900_PCCR_EVENT0_SHIFT 5u +#define R5900_PCCR_EVENT1_SHIFT 15u +#define R5900_PCCR_EVENT_MASK 0x1fu + +/* Count normal application execution in all three privilege modes while + * excluding Level-1 exception handlers. This is intentionally explicit rather + * than inheriting whatever mode bits a debugger happened to leave behind. */ +#define R5900_PCCR_APP_MODES0 ((1u << 2) | (1u << 3) | (1u << 4)) +#define R5900_PCCR_APP_MODES1 ((1u << 12) | (1u << 13) | (1u << 14)) +#define R5900_PCR_VALUE_MASK 0x7fffffffu +#define R5900_PCR_OVERFLOW (1u << 31) + +static inline void r5900_perf_sync(void) +{ + /* SYNC.P guarantees completion of preceding instructions before following + * instructions execute. The memory clobber also prevents GCC from moving + * ordinary memory operations across the benchmark boundary. */ + __asm__ __volatile__("sync.p" ::: "memory"); +} + +static inline uint32_t r5900_read_pccr(void) +{ + uint32_t value; + + __asm__ __volatile__("mfps %0, 0" : "=r"(value)); + return value; +} + +static inline uint32_t r5900_read_pcr0(void) +{ + uint32_t value; + + __asm__ __volatile__("mfpc %0, 0" : "=r"(value)); + return value; +} + +static inline uint32_t r5900_read_pcr1(void) +{ + uint32_t value; + + __asm__ __volatile__("mfpc %0, 1" : "=r"(value)); + return value; +} + +static inline void r5900_write_pccr(uint32_t value) +{ + __asm__ __volatile__("mtps %0, 0" :: "r"(value) : "memory"); +} + +static inline void r5900_write_pcr0(uint32_t value) +{ + __asm__ __volatile__("mtpc %0, 0" :: "r"(value)); +} + +static inline void r5900_write_pcr1(uint32_t value) +{ + __asm__ __volatile__("mtpc %0, 1" :: "r"(value)); +} + +static uint32_t r5900_perf_control(r5900_pcr0_event_t event0, + r5900_pcr1_event_t event1) +{ + return R5900_PCCR_CTE | R5900_PCCR_APP_MODES0 | R5900_PCCR_APP_MODES1 | + (((uint32_t)event0 & R5900_PCCR_EVENT_MASK) + << R5900_PCCR_EVENT0_SHIFT) | + (((uint32_t)event1 & R5900_PCCR_EVENT_MASK) + << R5900_PCCR_EVENT1_SHIFT); +} + +int r5900_perf_begin(r5900_perf_scope_t *scope, + r5900_pcr0_event_t event0, + r5900_pcr1_event_t event1) +{ + uint32_t control; + + if (scope == NULL || scope->active || event0 < 0 || event0 > 16 || + event1 < 0 || event1 > 16) + return -1; + + r5900_perf_sync(); + scope->previous_pccr = r5900_read_pccr(); + scope->previous_pcr0 = r5900_read_pcr0(); + scope->previous_pcr1 = r5900_read_pcr1(); + scope->event0 = (uint32_t)event0; + scope->event1 = (uint32_t)event1; + + /* Disable first so resetting PCR0/PCR1 cannot race an enabled counter. */ + r5900_write_pccr(scope->previous_pccr & ~R5900_PCCR_CTE); + r5900_perf_sync(); + r5900_write_pcr0(0); + r5900_write_pcr1(0); + control = r5900_perf_control(event0, event1); + r5900_write_pccr(control); + r5900_perf_sync(); + scope->active = 1; + return 0; +} + +int r5900_perf_end(r5900_perf_scope_t *scope, r5900_perf_result_t *result) +{ + uint32_t raw0; + uint32_t raw1; + + if (scope == NULL || result == NULL || !scope->active) + return -1; + + /* Stop at a serialized boundary, then snapshot before restoring any prior + * debugger/profiler state. */ + r5900_perf_sync(); + r5900_write_pccr(r5900_read_pccr() & ~R5900_PCCR_CTE); + r5900_perf_sync(); + raw0 = r5900_read_pcr0(); + raw1 = r5900_read_pcr1(); + + result->pcr0 = raw0 & R5900_PCR_VALUE_MASK; + result->pcr1 = raw1 & R5900_PCR_VALUE_MASK; + result->pcr0_overflow = (raw0 & R5900_PCR_OVERFLOW) != 0; + result->pcr1_overflow = (raw1 & R5900_PCR_OVERFLOW) != 0; + + r5900_write_pcr0(scope->previous_pcr0); + r5900_write_pcr1(scope->previous_pcr1); + r5900_write_pccr(scope->previous_pccr); + r5900_perf_sync(); + scope->active = 0; + return 0; +} diff --git a/tests/test_hdl_hash_checkpoint.c b/tests/test_hdl_hash_checkpoint.c new file mode 100644 index 00000000..c2827a63 --- /dev/null +++ b/tests/test_hdl_hash_checkpoint.c @@ -0,0 +1,167 @@ +#include +#include +#include + +#include "hdl_hash_checkpoint.h" + +static hdl_transaction_t copying_transaction(void) +{ + hdl_transaction_t transaction; + unsigned int i; + + memset(&transaction, 0, sizeof(transaction)); + transaction.stage = HDL_TRANSACTION_STAGE_COPYING; + transaction.source_bytes = 8192; + transaction.total_sectors = 4; + transaction.completed_sectors = 2; + transaction.partition_count = 1; + strcpy(transaction.target, "PP.SLUS-12345.HDL.TEST"); + strcpy(transaction.startup, "SLUS_123.45"); + strcpy(transaction.source_path, "mass:/TEST.ISO"); + strcpy(transaction.game_title, "Test Game"); + transaction.disc_type = 0x14; + for (i = 0; i < sizeof(transaction.source_fingerprint); i++) + transaction.source_fingerprint[i] = (unsigned char)(0xa0u + i); + return transaction; +} + +static void fill_payload(unsigned char *payload, size_t size) +{ + size_t i; + + for (i = 0; i < size; i++) + payload[i] = (unsigned char)((i * 37u + i / 17u) & 0xffu); +} + +static void test_restore_continues_same_digest(void) +{ + hdl_transaction_t transaction = copying_transaction(); + unsigned char payload[8192]; + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + unsigned char expected[32]; + unsigned char resumed_digest[32]; + sha256_context_t original; + sha256_context_t resumed; + + fill_payload(payload, sizeof(payload)); + sha256_buffer(payload, sizeof(payload), expected); + + sha256_init(&original); + sha256_update(&original, payload, 4096); + assert(original.total_bytes == 4096); + assert(original.block_used == 0); + assert(hdl_hash_checkpoint_encode(&transaction, &original, record) == 0); + + memset(&resumed, 0xcc, sizeof(resumed)); + assert(hdl_hash_checkpoint_restore(record, &transaction, &resumed) == 0); + assert(resumed.total_bytes == original.total_bytes); + assert(resumed.block_used == original.block_used); + assert(memcmp(resumed.state, original.state, sizeof(original.state)) == 0); + + sha256_update(&resumed, payload + 4096, sizeof(payload) - 4096); + sha256_final(&resumed, resumed_digest); + assert(memcmp(expected, resumed_digest, sizeof(expected)) == 0); +} + +static void test_every_record_byte_is_authenticated(void) +{ + hdl_transaction_t transaction = copying_transaction(); + unsigned char payload[4096]; + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + unsigned char damaged[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + sha256_context_t hash; + sha256_context_t restored; + unsigned int i; + + fill_payload(payload, sizeof(payload)); + sha256_init(&hash); + sha256_update(&hash, payload, sizeof(payload)); + assert(hdl_hash_checkpoint_encode(&transaction, &hash, record) == 0); + + for (i = 0; i < sizeof(record); i++) { + memcpy(damaged, record, sizeof(damaged)); + damaged[i] ^= 0x01u; + assert(hdl_hash_checkpoint_restore(damaged, &transaction, &restored) < 0); + } +} + +static void test_transaction_identity_must_match(void) +{ + hdl_transaction_t transaction = copying_transaction(); + hdl_transaction_t wrong; + unsigned char payload[4096]; + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + sha256_context_t hash; + sha256_context_t restored; + + fill_payload(payload, sizeof(payload)); + sha256_init(&hash); + sha256_update(&hash, payload, sizeof(payload)); + assert(hdl_hash_checkpoint_encode(&transaction, &hash, record) == 0); + + wrong = transaction; + wrong.completed_sectors = 1; + assert(hdl_hash_checkpoint_restore(record, &wrong, &restored) == + HDL_HASH_CHECKPOINT_TRANSACTION_MISMATCH); + + wrong = transaction; + wrong.source_fingerprint[7] ^= 0x80u; + assert(hdl_hash_checkpoint_restore(record, &wrong, &restored) == + HDL_HASH_CHECKPOINT_TRANSACTION_MISMATCH); + + wrong = transaction; + strcpy(wrong.target, "PP.SLUS-12345.OTHER"); + assert(hdl_hash_checkpoint_restore(record, &wrong, &restored) == + HDL_HASH_CHECKPOINT_TRANSACTION_MISMATCH); +} + +static void test_context_progress_must_match_transaction(void) +{ + hdl_transaction_t transaction = copying_transaction(); + unsigned char payload[2048]; + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + sha256_context_t hash; + + fill_payload(payload, sizeof(payload)); + sha256_init(&hash); + sha256_update(&hash, payload, sizeof(payload)); + assert(hdl_hash_checkpoint_encode(&transaction, &hash, record) == + HDL_HASH_CHECKPOINT_CONTEXT_MISMATCH); + + transaction.completed_sectors = 0; + sha256_init(&hash); + assert(hdl_hash_checkpoint_encode(&transaction, &hash, record) == + HDL_HASH_CHECKPOINT_INVALID_ARGUMENT); +} + +static void test_full_payload_checkpoint_can_finalize(void) +{ + hdl_transaction_t transaction = copying_transaction(); + unsigned char payload[8192]; + unsigned char record[HDL_HASH_CHECKPOINT_RECORD_SIZE]; + unsigned char expected[32]; + unsigned char restored_digest[32]; + sha256_context_t hash; + sha256_context_t restored; + + transaction.completed_sectors = transaction.total_sectors; + fill_payload(payload, sizeof(payload)); + sha256_buffer(payload, sizeof(payload), expected); + sha256_init(&hash); + sha256_update(&hash, payload, sizeof(payload)); + assert(hdl_hash_checkpoint_encode(&transaction, &hash, record) == 0); + assert(hdl_hash_checkpoint_restore(record, &transaction, &restored) == 0); + sha256_final(&restored, restored_digest); + assert(memcmp(expected, restored_digest, sizeof(expected)) == 0); +} + +int main(void) +{ + test_restore_continues_same_digest(); + test_every_record_byte_is_authenticated(); + test_transaction_identity_must_match(); + test_context_progress_must_match_transaction(); + test_full_payload_checkpoint_can_finalize(); + puts("All HDL hash checkpoint tests passed."); + return 0; +} diff --git a/tools/allocation_inventory.py b/tools/allocation_inventory.py new file mode 100644 index 00000000..bc6770da --- /dev/null +++ b/tools/allocation_inventory.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Emit a complete direct runtime allocation/free inventory for corpus-v2 review. + +This is deliberately a source inventory, not an allocator-performance claim. +The output keeps every direct EE libc malloc/calloc/realloc/memalign/free call +and every direct IOP SysMem AllocSysMemory/FreeSysMemory call instead of the +sample-limited evidence section in the broader project audit. Events are bound +to the containing C function when the existing source parser can resolve one. + +ThreadMan objects (CreateThread/CreateSema/etc.) are intentionally not reported +as heap allocations here: their implementation-owned memory cost belongs in the +runtime IOP-resource budget and must not be guessed from an API call count. +""" + +from __future__ import annotations + +import argparse +import collections +import json +import re +import tempfile +from pathlib import Path +from typing import Any + +from corpus_v2_project_audit import ( + RUNTIME_ROOTS, + SOURCE_SUFFIXES, + functions_in, + read_text, + source_files, +) + +ALLOC_RE = re.compile( + r"\b(malloc|calloc|realloc|memalign|free|AllocSysMemory|FreeSysMemory)\s*\(" +) +ALLOCATORS = {"malloc", "calloc", "realloc", "memalign", "AllocSysMemory"} +FREES = {"free", "FreeSysMemory"} + + +def _domain(operation: str) -> str: + if operation in {"AllocSysMemory", "FreeSysMemory"}: + return "IOP SysMem" + return "EE/libc heap" + + +def _function_index(root: Path, files: list[Path]) -> dict[str, list[Any]]: + return { + str(path.relative_to(root)): functions_in(path, root) + for path in files + if path.suffix in {".c", ".inc"} + } + + +def _scope_for(functions: list[Any], lineno: int) -> str: + for fn in functions: + if fn.start_line <= lineno <= fn.end_line: + return fn.name + return "" + + +def collect(root: Path) -> dict[str, Any]: + files = source_files(root, RUNTIME_ROOTS, SOURCE_SUFFIXES) + by_path = _function_index(root, files) + events: list[dict[str, Any]] = [] + operation_counts: collections.Counter[str] = collections.Counter() + domain_counts: collections.Counter[str] = collections.Counter() + file_counts: collections.Counter[str] = collections.Counter() + function_counts: collections.Counter[str] = collections.Counter() + + for path in files: + rel = str(path.relative_to(root)) + functions = by_path.get(rel, []) + for lineno, line in enumerate(read_text(path).splitlines(), 1): + code = line.split("//", 1)[0] + for match in ALLOC_RE.finditer(code): + operation = match.group(1) + scope = _scope_for(functions, lineno) + domain = _domain(operation) + event = { + "path": rel, + "line": lineno, + "function": scope, + "operation": operation, + "kind": "allocate" if operation in ALLOCATORS else "free", + "memory_domain": domain, + "source": line.strip(), + } + events.append(event) + operation_counts[operation] += 1 + domain_counts[domain] += 1 + file_counts[rel] += 1 + function_counts[f"{rel}:{scope}"] += 1 + + events.sort(key=lambda item: (item["path"], item["line"], item["operation"])) + return { + "epistemic_status": "CURRENT IMPLEMENTATION: static direct allocator-call inventory", + "coverage_notes": [ + "EE libc heap calls: malloc/calloc/realloc/memalign/free", + "IOP SysMem calls: AllocSysMemory/FreeSysMemory", + "ThreadMan object backing memory is implementation-owned and not inferred from CreateThread/CreateSema call counts", + "allocator-internal allocations not visible as direct source calls are outside this static inventory", + ], + "runtime_roots": list(RUNTIME_ROOTS), + "total_events": len(events), + "allocation_events": sum( + count for operation, count in operation_counts.items() + if operation in ALLOCATORS + ), + "free_events": sum( + count for operation, count in operation_counts.items() + if operation in FREES + ), + "operation_counts": dict(sorted(operation_counts.items())), + "memory_domain_event_counts": dict(sorted(domain_counts.items())), + "file_counts": dict(sorted(file_counts.items())), + "function_counts": dict(sorted(function_counts.items())), + "events": events, + } + + +def selftest() -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "src").mkdir() + (root / "include").mkdir() + (root / "iop").mkdir() + (root / "src" / "fixture.c").write_text( + """ +#include +static void *phase(unsigned int n) +{ + void *a = malloc(n); + void *b = memalign(64, n); + a = realloc(a, n + 1); + free(b); + free(a); + return 0; +} +""".lstrip(), + encoding="utf-8", + ) + (root / "iop" / "fixture.c").write_text( + """ +static void iop_phase(unsigned int n) +{ + void *p = AllocSysMemory(0, n, 0); + if (p != 0) + FreeSysMemory(p); +} +""".lstrip(), + encoding="utf-8", + ) + result = collect(root) + assert result["total_events"] == 7 + assert result["allocation_events"] == 4 + assert result["free_events"] == 3 + assert result["operation_counts"] == { + "AllocSysMemory": 1, + "FreeSysMemory": 1, + "free": 2, + "malloc": 1, + "memalign": 1, + "realloc": 1, + } + assert result["memory_domain_event_counts"] == { + "EE/libc heap": 5, + "IOP SysMem": 2, + } + assert any( + event["function"] == "iop_phase" and + event["memory_domain"] == "IOP SysMem" + for event in result["events"] + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(".")) + parser.add_argument("--output", type=Path, + default=Path("ALLOCATION_INVENTORY.json")) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("allocation_inventory selftest: PASS") + return 0 + + root = args.root.resolve() + result = collect(root) + args.output.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print( + "allocation inventory: " + f"{result['allocation_events']} allocate events, " + f"{result['free_events']} free events, " + f"{result['total_events']} total; " + + ", ".join( + f"{domain}={count}" + for domain, count in result["memory_domain_event_counts"].items() + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/build_benchmark_provenance.sh b/tools/build_benchmark_provenance.sh new file mode 100755 index 00000000..57095c35 --- /dev/null +++ b/tools/build_benchmark_provenance.sh @@ -0,0 +1,137 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +OUT=${1:-BENCHMARK_PROVENANCE.yml} +CC=${EE_CC:-mips64r5900el-ps2-elf-gcc} +GIT_SHA=${PROJECT_GIT_SHA:-$(git rev-parse HEAD 2>/dev/null || printf 'unavailable')} +GIT_REF=${PROJECT_GIT_REF:-$(git rev-parse --abbrev-ref HEAD 2>/dev/null || printf 'unavailable')} +CC_TARGET=$($CC -dumpmachine 2>/dev/null || printf 'unavailable') +CC_VERSION=$($CC -dumpfullversion -dumpversion 2>/dev/null || printf 'unavailable') +PS2SDK_PATH_VALUE=${PS2SDK:-unavailable} +PS2SDK_REF=${PS2SDK_SOURCE_REF:-unavailable} +PS2SDK_SHA=${PS2SDK_SOURCE_SHA:-unavailable} +PS2DEV_BUNDLE_REF=${PS2DEV_BUNDLE_REF:-unavailable} +HDL_PROFILE_VALUE=${HDL_PROFILE:-1} +HDL_RESUME_HASH_CHECKPOINT_VALUE=${HDL_RESUME_HASH_CHECKPOINT:-0} +BENCHMARK_ELF_PATH=${BENCHMARK_ELF:-PS2_HDD_BOOTSTRAP_MANAGER.ELF} +HDL_STREAM_IRX_PATH=${HDL_STREAM_IRX:-hdl_stream.irx} + +case "$HDL_PROFILE_VALUE" in + 0|1) ;; + *) + printf 'HDL_PROFILE must be 0 or 1, got %s\n' "$HDL_PROFILE_VALUE" >&2 + exit 2 + ;; +esac + +case "$HDL_RESUME_HASH_CHECKPOINT_VALUE" in + 0|1) ;; + *) + printf 'HDL_RESUME_HASH_CHECKPOINT must be 0 or 1, got %s\n' \ + "$HDL_RESUME_HASH_CHECKPOINT_VALUE" >&2 + exit 2 + ;; +esac + +if [ "$HDL_RESUME_HASH_CHECKPOINT_VALUE" = "1" ]; then + RESUME_HASH_EE_FLAG=" -DHDL_RESUME_HASH_CHECKPOINT_ENABLED=1" +else + RESUME_HASH_EE_FLAG="" +fi + +# A development environment may preserve the ps2sdk .git directory. Prefer the +# exact installed checkout when available. Tagged ps2dev Docker images strip +# source metadata after installation, so CI passes the source ref/SHA that the +# image build scripts selected instead of pretending an absent .git means an +# unknown software stack. +if [ "${PS2SDK:-}" != "" ] && git -C "$PS2SDK" rev-parse HEAD >/dev/null 2>&1; then + PS2SDK_SHA=$(git -C "$PS2SDK" rev-parse HEAD) + PS2SDK_REF=$(git -C "$PS2SDK" describe --always --tags 2>/dev/null || printf '%s' "$PS2SDK_REF") +fi + +file_sha256() +{ + if [ -f "$1" ]; then + sha256sum "$1" | awk '{print $1}' + else + printf 'UNRECORDED' + fi +} + +file_bytes() +{ + if [ -f "$1" ]; then + wc -c < "$1" | tr -d ' ' + else + printf 'UNRECORDED' + fi +} + +# Keep the build-side expected load sequence tied to the actual platform source. +# The capture intentionally records only exec_irx(_irx, ...) calls in +# load_modules(). ps2hdd_posix is the embedded C symbol for PS2SDK's ps2hdd-bdm. +APP_IRX_LOAD_ORDER=$( + sed -n 's/.*exec_irx(\([A-Za-z0-9_]*\)_irx,.*/\1/p' \ + "$ROOT/src/platform.c" | + tr '\n' ' ' | + sed -e 's/ $//' -e 's/ps2hdd_posix/ps2hdd-bdm/' +) +if [ -z "$APP_IRX_LOAD_ORDER" ]; then + printf 'could not derive IOP load order from src/platform.c\n' >&2 + exit 2 +fi + +BENCHMARK_ELF_SHA=$(file_sha256 "$BENCHMARK_ELF_PATH") +BENCHMARK_ELF_BYTES=$(file_bytes "$BENCHMARK_ELF_PATH") +HDL_STREAM_IRX_SHA=$(file_sha256 "$HDL_STREAM_IRX_PATH") +HDL_STREAM_IRX_BYTES=$(file_bytes "$HDL_STREAM_IRX_PATH") + +cat > "$OUT" < release-like timing +# frozen PROFILE ON vs resume-hash PROFILE ON -> identical telemetry +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +BACKUP=$(mktemp -d) +SOURCE_UI="$ROOT/src/hdl_tools/source_ui.inc" +TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" + +restore_sources() { + if [ -f "$BACKUP/source_ui.inc" ]; then + cp "$BACKUP/source_ui.inc" "$SOURCE_UI" + fi + if [ -f "$BACKUP/transaction.inc" ]; then + cp "$BACKUP/transaction.inc" "$TRANSACTION" + fi + rm -rf "$BACKUP" +} +trap restore_sources EXIT HUP INT TERM + +cp "$SOURCE_UI" "$BACKUP/source_ui.inc" +cp "$TRANSACTION" "$BACKUP/transaction.inc" +cp "$ROOT/src/hdl_tools/source_ui_resume_hash.inc" "$SOURCE_UI" +python3 "$ROOT/tools/materialize_resume_hash_transaction.py" \ + "$ROOT/src/hdl_tools/transaction_resume_hash.inc" "$TRANSACTION" + +build_variant() +{ + profile=$1 + label=$2 + elf="PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_${label}.ELF" + map="PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_${label}.map" + irx="HDL_STREAM_RESUME_HASH_PROFILE_${label}.irx" + audit="OPTIMIZATION_AUDIT_RESUME_HASH_PROFILE_${label}.txt" + provenance="BENCHMARK_PROVENANCE_RESUME_HASH_PROFILE_${label}.yml" + + make clean + # Root clean removes the public hdl_stream.irx but deliberately does not + # invoke the nested IOP Makefile. Its profile-specific objects and the + # absolute-path notiopmod intermediates can therefore survive and make a + # subsequent PROFILE variant reuse the wrong linked IRX. Clean the actual + # producer explicitly so OFF/ON experiment builds are independent. + make -C iop/hdl_stream clean \ + IOP_BIN="$ROOT/hdl_stream.irx" \ + HDL_PROFILE="$profile" + make HDL_PROFILE="$profile" HDL_RESUME_HASH_CHECKPOINT=1 + cp hdl_stream.irx "$irx" + python3 tools/optimization_audit.py \ + --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF \ + --output "$audit" + make HDL_PROFILE="$profile" HDL_RESUME_HASH_CHECKPOINT=1 release + cp PS2_HDD_BOOTSTRAP_MANAGER.ELF "$elf" + cp PS2_HDD_BOOTSTRAP_MANAGER.map "$map" + sha256sum "$elf" > "$elf.sha256" + wc -c "$elf" > "$elf.size" + HDL_PROFILE="$profile" \ + HDL_RESUME_HASH_CHECKPOINT=1 \ + BENCHMARK_ELF="$elf" \ + HDL_STREAM_IRX="$irx" \ + sh tools/build_benchmark_provenance.sh "$provenance" +} + +cd "$ROOT" +build_variant 0 OFF +build_variant 1 ON + +# Preserve the original experiment artifact names as PROFILE OFF aliases while +# the benchmark documentation/tools migrate to the explicit pair names. +cp PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF \ + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF +cp PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.map \ + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.map +cp PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF.sha256 \ + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.sha256 +cp PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF.size \ + PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH.ELF.size +cp OPTIMIZATION_AUDIT_RESUME_HASH_PROFILE_OFF.txt \ + OPTIMIZATION_AUDIT_RESUME_HASH.txt diff --git a/tools/build_transaction_workspace_experiment.sh b/tools/build_transaction_workspace_experiment.sh new file mode 100644 index 00000000..7a75299b --- /dev/null +++ b/tools/build_transaction_workspace_experiment.sh @@ -0,0 +1,103 @@ +#!/bin/sh +set -eu + +# Build the active isolated Phase-5 incremental experiment without changing +# default runtime sources. Frozen references: +# workspace v1 CI #724 +# workspace v1 + fingerprint malloc CI #739 +# bounded HDDMETA read-back v1 CI #749 +# bounded HDDMETA read-back v2 CI #752 / identical runtime CI #757 +# streaming APAMETA1 v2 CI #762 +# +# Active v3 keeps streaming-v2 dataflow/memory semantics but forces +# forensic_snapshot_save() to stay cold and noinline so LTO cannot inflate the +# repair-plan UI controller with snapshot serialization machinery. +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +BACKUP=$(mktemp -d) +TRANSACTION="$ROOT/src/hdl_tools/transaction.inc" +SOURCE_UI="$ROOT/src/hdl_tools/source_ui.inc" +FORENSIC_SNAPSHOT="$ROOT/src/forensic_snapshot.c" + +restore_sources() { + if [ -f "$BACKUP/transaction.inc" ]; then + cp "$BACKUP/transaction.inc" "$TRANSACTION" + fi + if [ -f "$BACKUP/source_ui.inc" ]; then + cp "$BACKUP/source_ui.inc" "$SOURCE_UI" + fi + if [ -f "$BACKUP/forensic_snapshot.c" ]; then + cp "$BACKUP/forensic_snapshot.c" "$FORENSIC_SNAPSHOT" + fi + rm -rf "$BACKUP" +} +trap restore_sources EXIT HUP INT TERM + +cd "$ROOT" +python3 tools/forensic_snapshot_reference.py --selftest +python3 tools/materialize_forensic_snapshot_streaming_v3.py --selftest + +cp "$TRANSACTION" "$BACKUP/transaction.inc" +cp "$SOURCE_UI" "$BACKUP/source_ui.inc" +cp "$FORENSIC_SNAPSHOT" "$BACKUP/forensic_snapshot.c" + +python3 "$ROOT/tools/materialize_transaction_workspace.py" \ + "$TRANSACTION" "$TRANSACTION" +python3 "$ROOT/tools/materialize_source_fingerprint_malloc.py" \ + "$SOURCE_UI" +python3 "$ROOT/tools/materialize_forensic_snapshot_streaming_v3.py" \ + "$FORENSIC_SNAPSHOT" + +python3 "$ROOT/tools/allocation_inventory.py" \ + --root "$ROOT" \ + --output "$ROOT/ALLOCATION_INVENTORY_TX_WORKSPACE.json" + +build_variant() +{ + profile=$1 + label=$2 + elf="PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_${label}.ELF" + map="PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_${label}.map" + irx="HDL_STREAM_TX_WORKSPACE_PROFILE_${label}.irx" + audit="OPTIMIZATION_AUDIT_TX_WORKSPACE_PROFILE_${label}.txt" + provenance="BENCHMARK_PROVENANCE_TX_WORKSPACE_PROFILE_${label}.yml" + + make clean + make -C iop/hdl_stream clean \ + IOP_BIN="$ROOT/hdl_stream.irx" \ + HDL_PROFILE="$profile" + make HDL_PROFILE="$profile" + cp hdl_stream.irx "$irx" + python3 tools/optimization_audit.py \ + --elf PS2_HDD_BOOTSTRAP_MANAGER.ELF \ + --output "$audit" + make HDL_PROFILE="$profile" release + cp PS2_HDD_BOOTSTRAP_MANAGER.ELF "$elf" + cp PS2_HDD_BOOTSTRAP_MANAGER.map "$map" + sha256sum "$elf" > "$elf.sha256" + wc -c "$elf" | awk '{print $1}' > "$elf.size" + HDL_PROFILE="$profile" \ + BENCHMARK_ELF="$elf" \ + HDL_STREAM_IRX="$irx" \ + sh tools/build_benchmark_provenance.sh "$provenance" + cat >> "$provenance" < dict[str, int]: + counts: collections.Counter[str] = collections.Counter() + for path in source_files(root, RUNTIME_ROOTS, SOURCE_SUFFIXES): + rel = str(path.relative_to(root)) + text = read_text(path) + # The declaration itself can span lines. Strip comments only to avoid a + # documentation example becoming a policy hit. + text = re.sub(r"/\*.*?\*/", "", text, flags=re.S) + text = re.sub(r"//[^\n]*", "", text) + count = len(ALIGN64_RE.findall(text)) + if count: + counts[rel] = count + return dict(sorted(counts.items())) + + +def check(root: Path) -> list[str]: + errors: list[str] = [] + observed = collect(root) + expected = dict(sorted(EXPECTED_BY_FILE.items())) + if observed != expected: + all_paths = sorted(set(observed) | set(expected)) + for path in all_paths: + got = observed.get(path, 0) + want = expected.get(path, 0) + if got != want: + errors.append( + f"{path}: aligned(64) count {got}, audited count {want}; " + "review docs/ALIGNMENT_CONTRACT_AUDIT.md and update policy intentionally" + ) + + platform = root / "src" / "platform.c" + if not platform.is_file() or not PAD_DECL_RE.search(read_text(platform)): + errors.append( + "src/platform.c: pad_buffer must remain a 256-byte, 64-byte-aligned " + "libpad area under the current PS2SDK contract" + ) + return errors + + +def selftest() -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name in RUNTIME_ROOTS: + (root / name).mkdir(parents=True, exist_ok=True) + # Unit-test collection independently from the repository-specific + # EXPECTED_BY_FILE table. + (root / "src" / "a.c").write_text( + "static unsigned char x[8] __attribute__((aligned(64)));\n", + encoding="utf-8", + ) + (root / "src" / "b.c").write_text( + "/* __attribute__((aligned(64))) */\n" + "static unsigned char y[8];\n", + encoding="utf-8", + ) + assert collect(root) == {"src/a.c": 1} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(".")) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("check_alignment_policy selftest: PASS") + return 0 + + errors = check(args.root.resolve()) + if errors: + for error in errors: + print(f"alignment policy: {error}") + return 2 + print( + "alignment policy: PASS " + f"({sum(EXPECTED_BY_FILE.values())} audited aligned(64) sites)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/check_filexio_fdman_policy.py b/tools/check_filexio_fdman_policy.py new file mode 100644 index 00000000..d6fa1f85 --- /dev/null +++ b/tools/check_filexio_fdman_policy.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Reject EE runtime call sites that require PS2SDK's Newlib fileXio fd-manager. + +The corpus-v2 performance branch deliberately keeps application filesystem I/O on +fileXio's direct RPC API. A tiny application policy object therefore prevents +fileXioInit() from replacing Newlib's generic POSIX pathname backend. This saves +code only while src/include do not depend on fopen/open/stat/opendir-style file +access. + +This checker turns that runtime contract into a CI invariant. It scans C-family +project source after masking comments and string/character literals, so prose and +format strings do not create false positives. If a POSIX/stdio file call is +needed later, review the policy instead of adding an exemption by reflex. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +ROOTS = ("src", "include") +SUFFIXES = {".c", ".h", ".inc", ".s", ".S"} +FORBIDDEN = ( + "fopen", "freopen", "fclose", "fread", "fwrite", + "open", "close", "stat", "fstat", "lstat", + "opendir", "readdir", "closedir", + "remove", "rename", "mkdir", "rmdir", "chdir", "unlink", +) +CALL = re.compile(r"\b(" + "|".join(map(re.escape, FORBIDDEN)) + r")\s*\(") + + +def mask_non_code(text: str) -> str: + out = list(text) + state = "code" + i = 0 + while i < len(text): + ch = text[i] + nxt = text[i + 1] if i + 1 < len(text) else "" + if state == "code": + if ch == "/" and nxt == "/": + out[i] = out[i + 1] = " " + state = "line" + i += 1 + elif ch == "/" and nxt == "*": + out[i] = out[i + 1] = " " + state = "block" + i += 1 + elif ch == '"': + out[i] = " " + state = "string" + elif ch == "'": + out[i] = " " + state = "char" + elif state == "line": + if ch == "\n": + state = "code" + else: + out[i] = " " + elif state == "block": + if ch == "*" and nxt == "/": + out[i] = out[i + 1] = " " + state = "code" + i += 1 + elif ch != "\n": + out[i] = " " + elif state in ("string", "char"): + if ch == "\\": + out[i] = " " + if i + 1 < len(text): + if text[i + 1] != "\n": + out[i + 1] = " " + i += 1 + elif (state == "string" and ch == '"') or ( + state == "char" and ch == "'"): + out[i] = " " + state = "code" + elif ch != "\n": + out[i] = " " + i += 1 + return "".join(out) + + +def source_files(root: Path): + for dirname in ROOTS: + directory = root / dirname + if not directory.is_dir(): + continue + for path in sorted(directory.rglob("*")): + if path.is_file() and path.suffix in SUFFIXES: + yield path + + +def scan(root: Path): + findings = [] + for path in source_files(root): + text = path.read_text(encoding="utf-8", errors="replace") + masked = mask_non_code(text) + for match in CALL.finditer(masked): + line = masked.count("\n", 0, match.start()) + 1 + findings.append((str(path.relative_to(root)), line, match.group(1))) + return findings + + +def selftest() -> int: + sample = r''' + fileXioOpen("mass:/x", 1, 0); + snprintf(buf, sizeof(buf), "fopen(x)"); + /* stat(path); */ + // fwrite(data, 1, n, f); + int fopen_counter = 0; + fopen(path, "rb"); + stat(path, &st); + ''' + masked = mask_non_code(sample) + hits = [m.group(1) for m in CALL.finditer(masked)] + if hits != ["fopen", "stat"]: + print(f"selftest failed: {hits}", file=sys.stderr) + return 1 + print("fileXio fdman policy checker selftest: PASS") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", default=".") + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + if args.selftest: + return selftest() + + root = Path(args.root).resolve() + findings = scan(root) + if findings: + print("ERROR: direct-fileXio policy violated by POSIX/stdio file calls:", + file=sys.stderr) + for path, line, name in findings: + print(f" {path}:{line}: {name}()", file=sys.stderr) + print("Review src/filexio_fdman_policy_ps2.c before using these APIs.", + file=sys.stderr) + return 1 + + print("fileXio fdman policy: PASS (no POSIX/stdio file calls in src/include)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/check_integer_format_policy.py b/tools/check_integer_format_policy.py new file mode 100644 index 00000000..d996033c --- /dev/null +++ b/tools/check_integer_format_policy.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Enforce the source contract required by printf_policy_ps2.c. + +All formatting that reaches the wrapped snprintf/vsnprintf path must remain +integer/string-only. A deliberately small set of opaque variadic pass-through +sites is allowed only because their callers are themselves covered by the format +audit. Any new opaque bridge forces review rather than silently inheriting this +policy. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from collections import Counter +from pathlib import Path + +import printf_format_audit as audit + +EXPECTED_OPAQUE = Counter({ + ("src/boot_report.c", "vsnprintf", "format"): 1, + ("src/forensic_controller_ps2.c", "vsnprintf", "format"): 1, + ("src/gs_debug_compat_ps2.c", "gs_ui_console_vprintf", "format"): 2, + ("src/gs_ui_ps2.c", "vsnprintf", "format"): 1, + ("src/gs_ui_ps2.c", "gs_ui_console_vprintf", "format"): 1, + ("src/printf_policy_ps2.c", "vsniprintf", "format"): 2, + ("src/session_log.c", "vsnprintf", "format"): 2, +}) + + +def app_name_is_safe(root: Path) -> bool: + header = (root / "include" / "app_identity.h").read_text( + encoding="utf-8", errors="replace") + match = re.search(r'^\s*#\s*define\s+APP_NAME\s+"((?:\\.|[^"\\])*)"\s*$', + header, re.M) + return bool(match) and "%" not in match.group(1) + + +def selftest() -> int: + if sum(EXPECTED_OPAQUE.values()) != 10: + print("selftest failed: opaque bridge baseline changed", file=sys.stderr) + return 1 + print("integer formatter policy checker selftest: PASS") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", default=".") + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + if args.selftest: + return selftest() + + root = Path(args.root).resolve() + sites = [site for path in audit.source_files(root) + for site in audit.scan_file(path, root)] + floats = [site for site in sites + if any(c in audit.FLOAT_CONVERSIONS for c in site.conversions)] + malformed = [site for site in sites if "?" in site.conversions] + opaque = Counter((site.path, site.api, site.format_expr) for site in sites + if site.literal is None and not site.fragment_text) + partial = [site for site in sites + if site.literal is None and site.fragment_text] + unexpected_partial = [site for site in partial + if not site.format_expr.startswith("APP_NAME ")] + + failed = False + if floats: + failed = True + print("ERROR: floating printf conversion under integer-only policy:", + file=sys.stderr) + for site in floats: + print(f" {site.path}:{site.line}: {site.api}({site.format_expr})", + file=sys.stderr) + if malformed: + failed = True + print("ERROR: malformed/unknown printf conversion:", file=sys.stderr) + for site in malformed: + print(f" {site.path}:{site.line}: {site.api}({site.format_expr})", + file=sys.stderr) + if opaque != EXPECTED_OPAQUE: + failed = True + print("ERROR: opaque formatter bridge set changed:", file=sys.stderr) + for key in sorted(set(opaque) | set(EXPECTED_OPAQUE)): + if opaque[key] != EXPECTED_OPAQUE[key]: + print(f" {key}: actual={opaque[key]} expected={EXPECTED_OPAQUE[key]}", + file=sys.stderr) + if unexpected_partial: + failed = True + print("ERROR: new macro/partially literal formatter expression:", + file=sys.stderr) + for site in unexpected_partial: + print(f" {site.path}:{site.line}: {site.api}({site.format_expr})", + file=sys.stderr) + if not app_name_is_safe(root): + failed = True + print("ERROR: APP_NAME must remain a percent-free string literal while it is used in format concatenation.", + file=sys.stderr) + + if failed: + print("Review src/printf_policy_ps2.c before changing the format contract.", + file=sys.stderr) + return 1 + + print(f"integer formatter policy: PASS ({len(sites)} audited sites, no floating conversions)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/check_libcglue_time_policy.py b/tools/check_libcglue_time_policy.py new file mode 100644 index 00000000..69e77c91 --- /dev/null +++ b/tools/check_libcglue_time_policy.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Guard the Phase-1 no-local-time libcglue startup policy. + +The application overrides PS2SDK's weak _libcglue_timezone_update() because the +manager has no local civil-time contract. This checker prevents future runtime +code from silently adding APIs that require that startup timezone state. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +ROOTS = ("src", "include") +SUFFIXES = {".c", ".h", ".inc", ".s", ".S"} +FORBIDDEN = ( + "localtime", "localtime_r", "mktime", "ctime", "ctime_r", "strftime", + "tzset", "_tzset_r", "_tzset_unlocked_r", + "ps2sdk_setTimezone", "ps2sdk_setDaylightSaving", +) +CALL = re.compile(r"\b(" + "|".join(map(re.escape, FORBIDDEN)) + r")\s*\(") + + +def mask_non_code(text: str) -> str: + out = list(text) + state = "code" + i = 0 + while i < len(text): + ch = text[i] + nxt = text[i + 1] if i + 1 < len(text) else "" + if state == "code": + if ch == "/" and nxt == "/": + out[i] = out[i + 1] = " " + state = "line" + i += 1 + elif ch == "/" and nxt == "*": + out[i] = out[i + 1] = " " + state = "block" + i += 1 + elif ch == '"': + out[i] = " " + state = "string" + elif ch == "'": + out[i] = " " + state = "char" + elif state == "line": + if ch == "\n": + state = "code" + else: + out[i] = " " + elif state == "block": + if ch == "*" and nxt == "/": + out[i] = out[i + 1] = " " + state = "code" + i += 1 + elif ch != "\n": + out[i] = " " + elif state in ("string", "char"): + if ch == "\\": + out[i] = " " + if i + 1 < len(text): + if text[i + 1] != "\n": + out[i + 1] = " " + i += 1 + elif (state == "string" and ch == '"') or ( + state == "char" and ch == "'"): + out[i] = " " + state = "code" + elif ch != "\n": + out[i] = " " + i += 1 + return "".join(out) + + +def source_files(root: Path): + for dirname in ROOTS: + directory = root / dirname + if not directory.is_dir(): + continue + for path in sorted(directory.rglob("*")): + if path.is_file() and path.suffix in SUFFIXES: + yield path + + +def scan(root: Path): + findings = [] + policy_file = root / "src" / "filexio_fdman_policy_ps2.c" + for path in source_files(root): + if path == policy_file: + continue + text = path.read_text(encoding="utf-8", errors="replace") + masked = mask_non_code(text) + for match in CALL.finditer(masked): + line = masked.count("\n", 0, match.start()) + 1 + findings.append((str(path.relative_to(root)), line, match.group(1))) + return findings + + +def selftest() -> int: + sample = r''' + GetTimerSystemTime(); + snprintf(buf, sizeof(buf), "localtime(x)"); + /* mktime(&tm); */ + // strftime(buf, n, fmt, &tm); + int localtime_counter = 0; + localtime(&now); + ps2sdk_setTimezone(60); + ''' + masked = mask_non_code(sample) + hits = [m.group(1) for m in CALL.finditer(masked)] + if hits != ["localtime", "ps2sdk_setTimezone"]: + print(f"selftest failed: {hits}", file=sys.stderr) + return 1 + print("libcglue time policy checker selftest: PASS") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", default=".") + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + if args.selftest: + return selftest() + + root = Path(args.root).resolve() + findings = scan(root) + if findings: + print("ERROR: no-local-time libcglue policy violated:", file=sys.stderr) + for path, line, name in findings: + print(f" {path}:{line}: {name}()", file=sys.stderr) + print("Review the _libcglue_timezone_update override before using these APIs.", + file=sys.stderr) + return 1 + + print("libcglue time policy: PASS (no local-time consumers in src/include)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/check_r5900_calibration_disasm.py b/tools/check_r5900_calibration_disasm.py new file mode 100644 index 00000000..e0f8127d --- /dev/null +++ b/tools/check_r5900_calibration_disasm.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Validate the deterministic R5900 counter-calibration loop in objdump output. + +This is a static build guard, not a timing claim. It verifies that the backward +branch in calibration_integer_loop repeats exactly four instructions: + + addiu , , +1 + addiu , , -1 + bnez , + nop + +Real-hardware performance-counter calibration is still required before PCR +results are treated as authoritative. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path + + +SYMBOL = "calibration_integer_loop" +SYMBOL_RE = re.compile(r"^\s*([0-9a-fA-F]+)\s+<([^>]+)>:\s*$") +INSN_RE = re.compile( + r"^\s*([0-9a-fA-F]+):\s+[0-9a-fA-F]+\s+([A-Za-z0-9_.]+)(?:\s+(.*?))?\s*$" +) + + +@dataclass(frozen=True) +class Instruction: + address: int + mnemonic: str + operands: str + + +def parse_function(text: str, symbol: str = SYMBOL) -> list[Instruction]: + inside = False + instructions: list[Instruction] = [] + + for line in text.splitlines(): + symbol_match = SYMBOL_RE.match(line) + if symbol_match: + if inside: + break + inside = symbol_match.group(2) == symbol + continue + + if not inside: + continue + + match = INSN_RE.match(line) + if match: + instructions.append( + Instruction( + address=int(match.group(1), 16), + mnemonic=match.group(2).lower(), + operands=(match.group(3) or "").strip(), + ) + ) + + if not instructions: + raise ValueError(f"symbol <{symbol}> not found or contains no instructions") + return instructions + + +def split_operands(operands: str) -> list[str]: + return [part.strip() for part in operands.split(",")] + + +def parse_integer(value: str) -> int: + value = value.strip() + if value.startswith("-"): + return -parse_integer(value[1:]) + if value.lower().startswith("0x"): + return int(value, 16) + return int(value, 10) + + +def validate_loop(instructions: list[Instruction]) -> str: + branches = [i for i, insn in enumerate(instructions) if insn.mnemonic == "bnez"] + if len(branches) != 1: + raise ValueError(f"expected exactly one bnez in <{SYMBOL}>, found {len(branches)}") + + branch_index = branches[0] + branch = instructions[branch_index] + branch_operands = split_operands(branch.operands) + if len(branch_operands) < 2: + raise ValueError("bnez operands are malformed") + + counter_register = branch_operands[0] + target_match = re.match(r"^([0-9a-fA-F]+)\b", branch_operands[1]) + if not target_match: + raise ValueError(f"cannot parse bnez target from: {branch.operands!r}") + target = int(target_match.group(1), 16) + + address_to_index = {insn.address: i for i, insn in enumerate(instructions)} + if target not in address_to_index: + raise ValueError(f"bnez target 0x{target:x} is outside <{SYMBOL}>") + target_index = address_to_index[target] + + if branch_index + 1 >= len(instructions): + raise ValueError("bnez has no visible delay-slot instruction") + + loop = instructions[target_index : branch_index + 2] + mnemonics = [insn.mnemonic for insn in loop] + expected = ["addiu", "addiu", "bnez", "nop"] + if mnemonics != expected: + raise ValueError( + "calibration loop body changed: expected " + + "/".join(expected) + + ", got " + + "/".join(mnemonics) + ) + + accumulator_ops = split_operands(loop[0].operands) + if len(accumulator_ops) != 3: + raise ValueError("first addiu operands are malformed") + if accumulator_ops[0] != accumulator_ops[1]: + raise ValueError("first addiu must update the accumulator in place") + try: + accumulator_step = parse_integer(accumulator_ops[2]) + except ValueError as exc: + raise ValueError("first addiu immediate is not an integer") from exc + if accumulator_step != 1: + raise ValueError(f"first addiu must increment by 1, got {accumulator_step}") + + counter_ops = split_operands(loop[1].operands) + if len(counter_ops) != 3: + raise ValueError("second addiu operands are malformed") + if counter_ops[0] != counter_register or counter_ops[1] != counter_register: + raise ValueError("second addiu must decrement the same register tested by bnez") + try: + counter_step = parse_integer(counter_ops[2]) + except ValueError as exc: + raise ValueError("second addiu immediate is not an integer") from exc + if counter_step != -1: + raise ValueError(f"second addiu must decrement by 1, got {counter_step}") + + return ( + f"PASS: <{SYMBOL}> repeats exactly addiu(+1)/addiu(-1)/bnez/nop " + f"from 0x{target:x} to branch 0x{branch.address:x}" + ) + + +def validate_text(text: str) -> str: + return validate_loop(parse_function(text)) + + +def selftest() -> None: + valid = """ +001015f8 : + 1015f8: 3c030001 lui v1,0x1 + 1015fc: 346386a0 ori v1,v1,0x86a0 + 101600: 00001025 move v0,zero + 101604: 24420001 addiu v0,v0,1 + 101608: 2463ffff addiu v1,v1,-1 + 10160c: 1460fffd bnez v1,101604 + 101610: 00000000 nop + 101614: 00000000 nop + 101618: 03e00008 jr ra + 10161c: 00000000 nop + +00101620 : +""" + validate_text(valid) + + invalid_cases = [ + valid.replace("101610: 00000000 nop", "101610: 24420001 addiu v0,v0,1"), + valid.replace("bnez v1,101604", "bnez v1,101608"), + valid.replace("addiu v1,v1,-1", "addiu v1,v1,-2"), + valid.replace("", ""), + ] + for index, bad in enumerate(invalid_cases, 1): + try: + validate_text(bad) + except ValueError: + continue + raise AssertionError(f"negative self-test {index} unexpectedly passed") + + print("R5900 calibration disassembly checker self-test: PASS") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--disasm", type=Path, help="objdump -dr output to validate") + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + + if args.disasm is None: + if args.selftest: + return 0 + parser.error("--disasm is required unless --selftest is used") + + try: + message = validate_text(args.disasm.read_text(encoding="utf-8")) + except (OSError, UnicodeError, ValueError) as exc: + print(f"R5900 calibration disassembly guard: FAIL: {exc}", file=sys.stderr) + return 1 + + print(f"R5900 calibration disassembly guard: {message}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/compare_hdl_profile_ab.py b/tools/compare_hdl_profile_ab.py new file mode 100755 index 00000000..db48dbc4 --- /dev/null +++ b/tools/compare_hdl_profile_ab.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +"""Compare same-source HDL PROFILE=0/1 real-hardware samples. + +Input is a JSON array (or an object with a ``samples`` array). Each sample must +record ``mode`` (``OFF``/``ON``), ``project_git_sha``, the frozen mode-specific +ELF/IRX hashes, ``workload_id``, ``correctness_hash``, ``source_bytes`` and +``total_us``. ``copy_us`` and ``verify_us`` are optional, but a metric is +compared only when every accepted sample in both modes provides it. + +The tool intentionally does not invent PROFILE=0 latency distributions from the +PROFILE=1 telemetry log. ``parse_hdl_perf.py`` remains the source for the rich +PROFILE=1 EE/IOP histograms; this script quantifies the profiler's end-to-end +perturbation. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Any + +EXPECTED_BINARY_HASHES = { + "ON": { + "benchmark_elf_sha256": "964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7", + "hdl_stream_irx_sha256": "8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db", + }, + "OFF": { + "benchmark_elf_sha256": "4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916", + "hdl_stream_irx_sha256": "f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751", + }, +} + +REQUIRED = ( + "mode", + "project_git_sha", + "benchmark_elf_sha256", + "hdl_stream_irx_sha256", + "workload_id", + "correctness_hash", + "source_bytes", + "total_us", +) +OPTIONAL_TIMES = ("copy_us", "verify_us") + + +def _percentile(values: list[int], percentile: int) -> int: + if not values: + raise ValueError("percentile requires at least one value") + ordered = sorted(values) + rank = max(1, math.ceil(len(ordered) * percentile / 100.0)) + return ordered[rank - 1] + + +def _distribution(values: list[int]) -> dict[str, int]: + return { + "samples": len(values), + "p50": _percentile(values, 50), + "p95": _percentile(values, 95), + "p99": _percentile(values, 99), + "max": max(values), + } + + +def _delta_percent(on_value: int, off_value: int) -> float: + if off_value == 0: + raise ValueError("PROFILE OFF baseline metric cannot be zero") + return round((on_value - off_value) * 100.0 / off_value, 4) + + +def _throughput_kib_s(source_bytes: int, usec: int) -> int: + if source_bytes <= 0 or usec <= 0: + raise ValueError("source_bytes and time must be positive") + return (source_bytes * 1_000_000) // (usec * 1024) + + +def _normalize(raw: Any) -> list[dict[str, Any]]: + if isinstance(raw, dict): + raw = raw.get("samples") + if not isinstance(raw, list) or not raw: + raise ValueError("input must contain a non-empty samples array") + + samples: list[dict[str, Any]] = [] + for index, sample in enumerate(raw): + if not isinstance(sample, dict): + raise ValueError(f"sample {index}: expected object") + missing = [key for key in REQUIRED if key not in sample] + if missing: + raise ValueError(f"sample {index}: missing {', '.join(missing)}") + mode = str(sample["mode"]).upper() + if mode not in ("OFF", "ON"): + raise ValueError(f"sample {index}: mode must be OFF or ON") + normalized = dict(sample) + normalized["mode"] = mode + for key in ("source_bytes", "total_us", *OPTIONAL_TIMES): + if key in normalized: + value = normalized[key] + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise ValueError(f"sample {index}: {key} must be a positive integer") + for key in ( + "project_git_sha", + "benchmark_elf_sha256", + "hdl_stream_irx_sha256", + "workload_id", + "correctness_hash", + ): + if not isinstance(normalized[key], str) or not normalized[key]: + raise ValueError(f"sample {index}: {key} must be a non-empty string") + + expected = EXPECTED_BINARY_HASHES[mode] + for key in ("benchmark_elf_sha256", "hdl_stream_irx_sha256"): + if normalized[key].lower() != expected[key]: + raise ValueError( + f"sample {index}: PROFILE {mode} {key} does not match frozen pair" + ) + normalized[key] = normalized[key].lower() + samples.append(normalized) + return samples + + +def compare_samples(samples: list[dict[str, Any]], min_samples: int = 4) -> dict[str, Any]: + groups = {mode: [sample for sample in samples if sample["mode"] == mode] + for mode in ("OFF", "ON")} + for mode, group in groups.items(): + if len(group) < min_samples: + raise ValueError( + f"PROFILE {mode} has {len(group)} samples; need at least {min_samples}" + ) + + shas = {sample["project_git_sha"] for sample in samples} + workloads = {sample["workload_id"] for sample in samples} + hashes = {sample["correctness_hash"] for sample in samples} + source_sizes = {sample["source_bytes"] for sample in samples} + if len(shas) != 1: + raise ValueError("samples do not share one project_git_sha") + if len(workloads) != 1: + raise ValueError("samples do not share one workload_id") + if len(hashes) != 1: + raise ValueError("correctness_hash differs across samples") + if len(source_sizes) != 1: + raise ValueError("source_bytes differs across samples") + + source_bytes = next(iter(source_sizes)) + result: dict[str, Any] = { + "project_git_sha": next(iter(shas)), + "binary_hashes": EXPECTED_BINARY_HASHES, + "workload_id": next(iter(workloads)), + "correctness_hash": next(iter(hashes)), + "source_bytes": source_bytes, + "sample_counts": {mode: len(groups[mode]) for mode in ("OFF", "ON")}, + "metrics": {}, + } + + metric_sources: dict[str, str] = {"total_us": "total_us"} + for key in OPTIONAL_TIMES: + if all(key in sample for sample in samples): + metric_sources[key] = key + + metrics: dict[str, Any] = result["metrics"] + for label, key in metric_sources.items(): + off_values = [int(sample[key]) for sample in groups["OFF"]] + on_values = [int(sample[key]) for sample in groups["ON"]] + off_dist = _distribution(off_values) + on_dist = _distribution(on_values) + metrics[label] = { + "off": off_dist, + "on": on_dist, + "on_vs_off_percent": { + percentile: _delta_percent(on_dist[percentile], off_dist[percentile]) + for percentile in ("p50", "p95", "p99", "max") + }, + } + + for timing_key, rate_label in ( + ("total_us", "end_to_end_kib_per_second"), + ("copy_us", "copy_kib_per_second"), + ): + if timing_key not in metric_sources: + continue + off_rates = [_throughput_kib_s(source_bytes, int(sample[timing_key])) + for sample in groups["OFF"]] + on_rates = [_throughput_kib_s(source_bytes, int(sample[timing_key])) + for sample in groups["ON"]] + off_dist = _distribution(off_rates) + on_dist = _distribution(on_rates) + metrics[rate_label] = { + "off": off_dist, + "on": on_dist, + "on_vs_off_percent": { + percentile: _delta_percent(on_dist[percentile], off_dist[percentile]) + for percentile in ("p50", "p95", "p99", "max") + }, + } + + return result + + +def selftest() -> None: + samples: list[dict[str, Any]] = [] + for mode, totals, copies, verifies in ( + ("OFF", [1000, 1010, 990, 1005], [700, 705, 695, 700], [200, 205, 195, 200]), + ("ON", [1020, 1030, 1010, 1025], [714, 719, 709, 714], [204, 209, 199, 204]), + ): + expected = EXPECTED_BINARY_HASHES[mode] + for total, copy, verify in zip(totals, copies, verifies): + samples.append({ + "mode": mode, + "project_git_sha": "abc123", + "benchmark_elf_sha256": expected["benchmark_elf_sha256"], + "hdl_stream_irx_sha256": expected["hdl_stream_irx_sha256"], + "workload_id": "fixture-iso-a", + "correctness_hash": "deadbeef", + "source_bytes": 1024 * 1024, + "total_us": total, + "copy_us": copy, + "verify_us": verify, + }) + normalized = _normalize(samples) + result = compare_samples(normalized) + assert result["sample_counts"] == {"OFF": 4, "ON": 4} + assert result["metrics"]["total_us"]["off"]["p50"] == 1000 + assert result["metrics"]["total_us"]["on"]["p50"] == 1020 + assert result["metrics"]["total_us"]["on_vs_off_percent"]["p50"] == 2.0 + assert result["metrics"]["copy_us"]["on_vs_off_percent"]["p50"] == 2.0 + assert result["metrics"]["end_to_end_kib_per_second"]["off"]["samples"] == 4 + + broken_sha = [dict(sample) for sample in samples] + broken_sha[-1]["project_git_sha"] = "different" + try: + compare_samples(_normalize(broken_sha)) + except ValueError as error: + assert "project_git_sha" in str(error) + else: + raise AssertionError("mismatched project SHA must fail") + + broken_binary = [dict(sample) for sample in samples] + broken_binary[0]["benchmark_elf_sha256"] = "0" * 64 + try: + _normalize(broken_binary) + except ValueError as error: + assert "frozen pair" in str(error) + else: + raise AssertionError("mismatched binary hash must fail") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("samples", nargs="?", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--min-samples", type=int, default=4) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("compare_hdl_profile_ab selftest: PASS") + return 0 + if args.samples is None: + parser.error("samples is required unless --selftest is used") + if args.min_samples < 1: + parser.error("--min-samples must be positive") + + try: + raw = json.loads(args.samples.read_text(encoding="utf-8")) + samples = _normalize(raw) + compared = compare_samples(samples, args.min_samples) + except (OSError, json.JSONDecodeError, ValueError) as error: + print(f"compare_hdl_profile_ab: {error}", file=sys.stderr) + return 2 + + rendered = json.dumps(compared, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(rendered, encoding="utf-8") + else: + sys.stdout.write(rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/compare_hdl_resume_hash_ab.py b/tools/compare_hdl_resume_hash_ab.py new file mode 100644 index 00000000..1f09a397 --- /dev/null +++ b/tools/compare_hdl_resume_hash_ab.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 +"""Compare real-PS2 HDL resume-hash checkpoint A/B samples. + +The identity JSON must come from ``resume_hash_ab_preflight.py`` in the same CI +artifact as the tested ELFs. This prevents a rebuilt or unrelated experiment +from inheriting another artifact's measurements. + +For recovery workloads, experiment runs that report ``checkpoint_status`` as +``fallback`` are counted but excluded from optimized timing distributions. A +safe fallback is a correctness success, not evidence that checkpoint restore +was slow. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Any + +REQUIRED = ( + "profile", + "mode", + "project_git_sha", + "benchmark_elf_sha256", + "hdl_stream_irx_sha256", + "workload_kind", + "workload_id", + "correctness_hash", + "source_bytes", + "total_us", + "checkpoint_status", + "checkpoint_restored_bytes", +) +OPTIONAL_TIMES = ("copy_us", "verify_us", "resume_gate_us") +WORKLOAD_KINDS = ("uninterrupted", "copy_resume", "payload_verified_resume") +CHECKPOINT_STATUSES = ("not-applicable", "restored", "fallback") + + +def _percentile(values: list[int], percentile: int) -> int: + if not values: + raise ValueError("percentile requires at least one value") + ordered = sorted(values) + rank = max(1, math.ceil(len(ordered) * percentile / 100.0)) + return ordered[rank - 1] + + +def _distribution(values: list[int]) -> dict[str, int]: + return { + "samples": len(values), + "p50": _percentile(values, 50), + "p95": _percentile(values, 95), + "p99": _percentile(values, 99), + "max": max(values), + } + + +def _delta_percent(exp_value: int, base_value: int) -> float: + if base_value == 0: + raise ValueError("baseline metric cannot be zero") + return round((exp_value - base_value) * 100.0 / base_value, 4) + + +def _throughput_kib_s(source_bytes: int, usec: int) -> int: + if source_bytes <= 0 or usec <= 0: + raise ValueError("source_bytes and time must be positive") + return (source_bytes * 1_000_000) // (usec * 1024) + + +def _load_identity(raw: Any) -> dict[str, Any]: + if not isinstance(raw, dict): + raise ValueError("identity must be an object") + if not isinstance(raw.get("project_git_sha"), str) or not raw["project_git_sha"]: + raise ValueError("identity project_git_sha must be non-empty") + profiles = raw.get("profiles") + if not isinstance(profiles, dict): + raise ValueError("identity profiles must be an object") + for profile in ("OFF", "ON"): + pair = profiles.get(profile) + if not isinstance(pair, dict): + raise ValueError(f"identity missing PROFILE {profile}") + for variant in ("baseline", "experiment"): + entry = pair.get(variant) + if not isinstance(entry, dict): + raise ValueError(f"identity PROFILE {profile} missing {variant}") + for kind in ("elf", "irx"): + blob = entry.get(kind) + if not isinstance(blob, dict): + raise ValueError( + f"identity PROFILE {profile} {variant} missing {kind}" + ) + if not isinstance(blob.get("sha256"), str) or not blob["sha256"]: + raise ValueError( + f"identity PROFILE {profile} {variant} {kind} sha256 invalid" + ) + return raw + + +def _positive_int(sample: dict[str, Any], key: str, index: int) -> None: + value = sample[key] + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise ValueError(f"sample {index}: {key} must be a positive integer") + + +def _nonnegative_int(sample: dict[str, Any], key: str, index: int) -> None: + value = sample[key] + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise ValueError(f"sample {index}: {key} must be a non-negative integer") + + +def _normalize(raw: Any, identity: dict[str, Any]) -> list[dict[str, Any]]: + if isinstance(raw, dict): + raw = raw.get("samples") + if not isinstance(raw, list) or not raw: + raise ValueError("input must contain a non-empty samples array") + + samples: list[dict[str, Any]] = [] + for index, sample in enumerate(raw): + if not isinstance(sample, dict): + raise ValueError(f"sample {index}: expected object") + missing = [key for key in REQUIRED if key not in sample] + if missing: + raise ValueError(f"sample {index}: missing {', '.join(missing)}") + + normalized = dict(sample) + profile = str(normalized["profile"]).upper() + mode = str(normalized["mode"]).upper() + workload_kind = str(normalized["workload_kind"]) + checkpoint_status = str(normalized["checkpoint_status"]) + if profile not in ("OFF", "ON"): + raise ValueError(f"sample {index}: profile must be OFF or ON") + if mode not in ("BASE", "EXP"): + raise ValueError(f"sample {index}: mode must be BASE or EXP") + if workload_kind not in WORKLOAD_KINDS: + raise ValueError( + f"sample {index}: workload_kind must be one of {', '.join(WORKLOAD_KINDS)}" + ) + if checkpoint_status not in CHECKPOINT_STATUSES: + raise ValueError( + f"sample {index}: invalid checkpoint_status {checkpoint_status}" + ) + if mode == "BASE" and checkpoint_status != "not-applicable": + raise ValueError( + f"sample {index}: BASE checkpoint_status must be not-applicable" + ) + if workload_kind == "uninterrupted" and checkpoint_status != "not-applicable": + raise ValueError( + f"sample {index}: uninterrupted runs cannot report checkpoint restore" + ) + if workload_kind != "uninterrupted" and mode == "EXP" and \ + checkpoint_status == "not-applicable": + raise ValueError( + f"sample {index}: recovery EXP must report restored or fallback" + ) + + normalized["profile"] = profile + normalized["mode"] = mode + normalized["workload_kind"] = workload_kind + normalized["checkpoint_status"] = checkpoint_status + + for key in ("source_bytes", "total_us"): + _positive_int(normalized, key, index) + for key in OPTIONAL_TIMES: + if key in normalized: + _positive_int(normalized, key, index) + for key in ("checkpoint_restored_bytes", "completed_sectors"): + if key in normalized: + _nonnegative_int(normalized, key, index) + + for key in ( + "project_git_sha", + "benchmark_elf_sha256", + "hdl_stream_irx_sha256", + "workload_id", + "correctness_hash", + ): + if not isinstance(normalized[key], str) or not normalized[key]: + raise ValueError(f"sample {index}: {key} must be a non-empty string") + + if normalized["project_git_sha"] != identity["project_git_sha"]: + raise ValueError(f"sample {index}: project_git_sha does not match identity") + pair = identity["profiles"][profile] + variant = pair["baseline" if mode == "BASE" else "experiment"] + if normalized["benchmark_elf_sha256"].lower() != variant["elf"]["sha256"]: + raise ValueError(f"sample {index}: ELF hash does not match identity") + if normalized["hdl_stream_irx_sha256"].lower() != variant["irx"]["sha256"]: + raise ValueError(f"sample {index}: IRX hash does not match identity") + normalized["benchmark_elf_sha256"] = normalized["benchmark_elf_sha256"].lower() + normalized["hdl_stream_irx_sha256"] = normalized["hdl_stream_irx_sha256"].lower() + samples.append(normalized) + return samples + + +def compare_samples( + samples: list[dict[str, Any]], identity: dict[str, Any], min_samples: int = 4 +) -> dict[str, Any]: + profiles = {sample["profile"] for sample in samples} + workload_kinds = {sample["workload_kind"] for sample in samples} + workloads = {sample["workload_id"] for sample in samples} + correctness_hashes = {sample["correctness_hash"] for sample in samples} + source_sizes = {sample["source_bytes"] for sample in samples} + project_shas = {sample["project_git_sha"] for sample in samples} + if len(profiles) != 1: + raise ValueError("compare one PROFILE mode at a time") + if len(workload_kinds) != 1: + raise ValueError("compare one workload_kind at a time") + if len(workloads) != 1: + raise ValueError("compare one workload_id/depth at a time") + if len(correctness_hashes) != 1: + raise ValueError("correctness_hash differs across samples") + if len(source_sizes) != 1: + raise ValueError("source_bytes differs across samples") + if project_shas != {identity["project_git_sha"]}: + raise ValueError("project_git_sha differs from identity") + + profile = next(iter(profiles)) + workload_kind = next(iter(workload_kinds)) + source_bytes = int(next(iter(source_sizes))) + groups = { + mode: [sample for sample in samples if sample["mode"] == mode] + for mode in ("BASE", "EXP") + } + if len(groups["BASE"]) < min_samples: + raise ValueError( + f"BASE has {len(groups['BASE'])} samples; need at least {min_samples}" + ) + + fallback_samples = [ + sample for sample in groups["EXP"] if sample["checkpoint_status"] == "fallback" + ] + if workload_kind == "uninterrupted": + accepted_exp = groups["EXP"] + else: + accepted_exp = [ + sample for sample in groups["EXP"] + if sample["checkpoint_status"] == "restored" + ] + if len(accepted_exp) < min_samples: + raise ValueError( + f"accepted EXP has {len(accepted_exp)} samples after fallback separation; " + f"need at least {min_samples}" + ) + + expected_restored_bytes = 0 + completed_sectors = None + if workload_kind == "copy_resume": + sectors = {sample.get("completed_sectors") for sample in samples} + if None in sectors or len(sectors) != 1: + raise ValueError("copy_resume samples must share one completed_sectors value") + completed_sectors = int(next(iter(sectors))) + if completed_sectors <= 0: + raise ValueError("copy_resume completed_sectors must be positive") + expected_restored_bytes = completed_sectors * 2048 + elif workload_kind == "payload_verified_resume": + expected_restored_bytes = source_bytes + + if expected_restored_bytes: + for sample in accepted_exp: + if int(sample["checkpoint_restored_bytes"]) != expected_restored_bytes: + raise ValueError( + "restored EXP sample does not report the expected skipped source bytes" + ) + for sample in fallback_samples: + if int(sample["checkpoint_restored_bytes"]) != 0: + raise ValueError("fallback EXP sample must report zero restored bytes") + + result: dict[str, Any] = { + "project_git_sha": identity["project_git_sha"], + "profile": profile, + "workload_kind": workload_kind, + "workload_id": next(iter(workloads)), + "correctness_hash": next(iter(correctness_hashes)), + "source_bytes": source_bytes, + "completed_sectors": completed_sectors, + "expected_checkpoint_restored_bytes": expected_restored_bytes, + "sample_counts": { + "base": len(groups["BASE"]), + "exp_total": len(groups["EXP"]), + "exp_accepted": len(accepted_exp), + "exp_fallback": len(fallback_samples), + }, + "binary_identity": identity["profiles"][profile], + "metrics": {}, + } + + compared = groups["BASE"] + accepted_exp + metric_keys = ["total_us"] + for key in OPTIONAL_TIMES: + if all(key in sample for sample in compared): + metric_keys.append(key) + + metrics: dict[str, Any] = result["metrics"] + for key in metric_keys: + base_values = [int(sample[key]) for sample in groups["BASE"]] + exp_values = [int(sample[key]) for sample in accepted_exp] + base_dist = _distribution(base_values) + exp_dist = _distribution(exp_values) + metrics[key] = { + "base": base_dist, + "experiment": exp_dist, + "experiment_vs_base_percent": { + percentile: _delta_percent(exp_dist[percentile], base_dist[percentile]) + for percentile in ("p50", "p95", "p99", "max") + }, + } + + if workload_kind == "uninterrupted" and "copy_us" in metric_keys: + base_rates = [ + _throughput_kib_s(source_bytes, int(sample["copy_us"])) + for sample in groups["BASE"] + ] + exp_rates = [ + _throughput_kib_s(source_bytes, int(sample["copy_us"])) + for sample in accepted_exp + ] + base_dist = _distribution(base_rates) + exp_dist = _distribution(exp_rates) + metrics["copy_kib_per_second"] = { + "base": base_dist, + "experiment": exp_dist, + "experiment_vs_base_percent": { + percentile: _delta_percent(exp_dist[percentile], base_dist[percentile]) + for percentile in ("p50", "p95", "p99", "max") + }, + } + + return result + + +def selftest() -> None: + identity = { + "project_git_sha": "head-fixture", + "profiles": { + "OFF": { + "baseline": { + "elf": {"sha256": "a" * 64, "bytes": 100}, + "irx": {"sha256": "b" * 64, "bytes": 10}, + }, + "experiment": { + "elf": {"sha256": "c" * 64, "bytes": 110}, + "irx": {"sha256": "b" * 64, "bytes": 10}, + }, + }, + "ON": { + "baseline": { + "elf": {"sha256": "d" * 64, "bytes": 120}, + "irx": {"sha256": "e" * 64, "bytes": 11}, + }, + "experiment": { + "elf": {"sha256": "f" * 64, "bytes": 130}, + "irx": {"sha256": "e" * 64, "bytes": 11}, + }, + }, + }, + } + identity = _load_identity(identity) + samples: list[dict[str, Any]] = [] + expected = 16384 * 2048 + for mode, totals in ( + ("BASE", [5000, 5100, 4900, 5050]), + ("EXP", [1000, 1100, 900, 1050]), + ): + variant = identity["profiles"]["ON"][ + "baseline" if mode == "BASE" else "experiment" + ] + for total in totals: + samples.append({ + "profile": "ON", + "mode": mode, + "project_git_sha": "head-fixture", + "benchmark_elf_sha256": variant["elf"]["sha256"], + "hdl_stream_irx_sha256": variant["irx"]["sha256"], + "workload_kind": "copy_resume", + "workload_id": "iso-a-depth-1", + "correctness_hash": "deadbeef", + "source_bytes": 1024 * 1024 * 1024, + "total_us": total, + "resume_gate_us": total // 2, + "completed_sectors": 16384, + "checkpoint_status": "not-applicable" if mode == "BASE" else "restored", + "checkpoint_restored_bytes": 0 if mode == "BASE" else expected, + }) + normalized = _normalize(samples, identity) + result = compare_samples(normalized, identity) + assert result["sample_counts"]["base"] == 4 + assert result["sample_counts"]["exp_accepted"] == 4 + assert result["expected_checkpoint_restored_bytes"] == expected + assert result["metrics"]["total_us"]["experiment"]["p50"] == 1000 + + fallback = [dict(sample) for sample in samples] + fallback[-1]["checkpoint_status"] = "fallback" + fallback[-1]["checkpoint_restored_bytes"] = 0 + try: + compare_samples(_normalize(fallback, identity), identity) + except ValueError as error: + assert "accepted EXP" in str(error) + else: + raise AssertionError("fallback must not count toward optimized minimum") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("identity", nargs="?", type=Path) + parser.add_argument("samples", nargs="?", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--min-samples", type=int, default=4) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("compare_hdl_resume_hash_ab selftest: PASS") + return 0 + if args.identity is None or args.samples is None: + parser.error("identity and samples are required unless --selftest is used") + if args.min_samples < 1: + parser.error("--min-samples must be positive") + + try: + identity = _load_identity( + json.loads(args.identity.read_text(encoding="utf-8")) + ) + samples = _normalize( + json.loads(args.samples.read_text(encoding="utf-8")), identity + ) + result = compare_samples(samples, identity, args.min_samples) + except (OSError, json.JSONDecodeError, ValueError) as error: + print(f"compare_hdl_resume_hash_ab: {error}", file=sys.stderr) + return 2 + + rendered = json.dumps(result, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(rendered, encoding="utf-8") + else: + sys.stdout.write(rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/forensic_snapshot_reference.py b/tools/forensic_snapshot_reference.py new file mode 100644 index 00000000..406e4996 --- /dev/null +++ b/tools/forensic_snapshot_reference.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Independent APAMETA1 reference serializer/vector for Phase-5 work. + +This tool intentionally does not import or materialize the runtime C serializer. +It encodes the documented version-1 snapshot contract directly and pins a small +deterministic vector. Future streaming implementations must reproduce the same +bytes before they are eligible for PS2 hardware testing. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import struct +from pathlib import Path + +MAGIC = b"APAMETA1" +VERSION = 1 +HEADER_BYTES = 64 +APA_HEADER_BYTES = 1024 +ENTRY_BYTES = 4 + 32 + APA_HEADER_BYTES +TRAILER_BYTES = 32 + + +def le32(value: int) -> bytes: + return struct.pack(" int: + return (old_next ^ new_next).bit_count() + (old_prev ^ new_prev).bit_count() + + +def serialize(total_sectors: int, map_index: int, confidence: int, + corroborated_count: int, speculative_count: int, + entries: list[dict[str, object]]) -> bytes: + if not entries: + raise ValueError("at least one APAMETA1 entry is required") + + one_or_two = sum( + patch_bit_distance( + int(entry["old_next"]), int(entry["old_prev"]), + int(entry["new_next"]), int(entry["new_prev"]), + ) in (1, 2) + for entry in entries + ) + + image = bytearray(HEADER_BYTES) + image[0:8] = MAGIC + image[8:12] = le32(VERSION) + image[12:16] = le32(total_sectors) + image[16:20] = le32(map_index) + image[20:24] = le32(confidence) + image[24:28] = le32(len(entries)) + image[28:32] = le32(corroborated_count) + image[32:36] = le32(speculative_count) + image[36:40] = le32(one_or_two) + + for entry in entries: + header = bytes(entry["header"]) + if len(header) != APA_HEADER_BYTES: + raise ValueError("APA header must be exactly 1024 bytes") + image += le32(int(entry["lba"])) + image += hashlib.sha256(header).digest() + image += header + + image += hashlib.sha256(image).digest() + return bytes(image) + + +def reference_entries() -> list[dict[str, object]]: + return [ + { + "lba": 0x1000, + "header": bytes(i & 0xFF for i in range(APA_HEADER_BYTES)), + "old_next": 0x100, + "old_prev": 0x80, + "new_next": 0x101, + "new_prev": 0x80, + }, + { + "lba": 0x2000, + "header": bytes((255 - i) & 0xFF for i in range(APA_HEADER_BYTES)), + "old_next": 0x200, + "old_prev": 0x40, + "new_next": 0x202, + "new_prev": 0x41, + }, + ] + + +def build_reference() -> bytes: + return serialize( + total_sectors=0x12345678, + map_index=1, + confidence=88, + corroborated_count=1, + speculative_count=1, + entries=reference_entries(), + ) + + +def selftest() -> dict[str, object]: + image = build_reference() + entry0 = reference_entries()[0] + entry1 = reference_entries()[1] + header0_sha = hashlib.sha256(bytes(entry0["header"])).hexdigest() + header1_sha = hashlib.sha256(bytes(entry1["header"])).hexdigest() + trailer = image[-TRAILER_BYTES:].hex() + image_sha = hashlib.sha256(image).hexdigest() + + expected = { + "bytes": 2216, + "header0_sha256": "785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9", + "header1_sha256": "3af6dbef8362452d2b45ad97deb9e43180fb90aac309860e26e123860cce62a7", + "trailer_sha256": "2654254d3abccac8459893f15c96b9f3e186fdf2abf423bdbb2150997431773b", + "image_sha256": "601ba74fc619738dac19baa2a6cb53054b67803e00b1fccb6bf89c69ef4bab6f", + } + + assert len(image) == expected["bytes"] + assert image[0:8] == MAGIC + assert struct.unpack_from(" int: + parser = argparse.ArgumentParser() + parser.add_argument("--selftest", action="store_true") + parser.add_argument("--json", type=Path, + help="write reference metadata JSON after validation") + parser.add_argument("--binary", type=Path, + help="write the exact reference APAMETA1 image") + args = parser.parse_args() + + result = selftest() + if args.json is not None: + args.json.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", + encoding="utf-8") + if args.binary is not None: + args.binary.write_bytes(build_reference()) + if args.selftest or (args.json is None and args.binary is None): + print("forensic snapshot reference selftest: PASS") + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/materialize_forensic_snapshot_bounded_verify.py b/tools/materialize_forensic_snapshot_bounded_verify.py new file mode 100644 index 00000000..b0e0e907 --- /dev/null +++ b/tools/materialize_forensic_snapshot_bounded_verify.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Materialize the isolated bounded HDDMETA read-back v2 experiment. + +The baseline forensic snapshot keeps two complete images live: the canonical +serialized HDDMETA image and an equally large read-back buffer used only for +byte-for-byte verification. Bounded v1 (CI #749) reduced that second allocation +to 64 KiB but used two fileXioLseek RPCs to prove the file size before reading. + +V2 keeps the same bounded memory model and exact byte comparison while removing +those seek RPCs. It reads exactly the expected byte count and then requires one +additional one-byte read to return EOF, which still rejects both truncation and +trailing data. + +No on-disk format, hash, slot, overwrite, error, or repair policy changes. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +MARKER = "bounded forensic snapshot read-back v2 experiment" +CHUNK_DEFINE = "#define SNAPSHOT_VERIFY_CHUNK_BYTES (64u * 1024u)" + + +class MaterializeError(RuntimeError): + pass + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise MaterializeError(f"{label}: expected one match, found {count}") + return text.replace(old, new, 1) + + +def materialize(text: str) -> str: + if MARKER in text: + raise MaterializeError("forensic bounded verify v2 already materialized") + + text = replace_once( + text, + "#define SNAPSHOT_TRAILER_BYTES 32u\n", + "#define SNAPSHOT_TRAILER_BYTES 32u\n" + f"{CHUNK_DEFINE}\n", + "chunk constant", + ) + + anchor = "int forensic_snapshot_save(unsigned int storage,\n" + helper = r'''/* bounded forensic snapshot read-back v2 experiment. + * Keep exact byte-for-byte verification without duplicating the complete + * variable-size HDDMETA image. After the expected bytes are consumed, one + * final read must report EOF so trailing data is rejected without seek RPCs. */ +static int snapshot_file_matches_bounded(const char *path, + const unsigned char *expected, + unsigned int size, + unsigned char *scratch, + unsigned int scratch_size) +{ + unsigned int offset = 0; + int fd; + int result; + + if (path == NULL || expected == NULL || scratch == NULL || + scratch_size == 0) + return -1; + fd = fileXioOpen(path, FIO_O_RDONLY, 0); + if (fd < 0) + return fd; + + while (offset < size) { + unsigned int bytes = size - offset; + unsigned int complete = 0; + + if (bytes > scratch_size) + bytes = scratch_size; + while (complete < bytes) { + result = fileXioRead(fd, scratch + complete, + (int)(bytes - complete)); + if (result <= 0) { + fileXioClose(fd); + return result < 0 ? result : -1; + } + complete += (unsigned int)result; + } + if (memcmp(scratch, expected + offset, bytes) != 0) { + fileXioClose(fd); + return 0; + } + offset += bytes; + } + + result = fileXioRead(fd, scratch, 1); + fileXioClose(fd); + if (result < 0) + return result; + return result == 0 ? 1 : 0; +} + +''' + text = replace_once(text, anchor, helper + anchor, "bounded compare helper") + + text = replace_once( + text, + " unsigned int image_size = 0;\n" + " unsigned int slot;\n", + " unsigned int image_size = 0;\n" + " unsigned int verify_size = 0;\n" + " unsigned int slot;\n", + "verify size state", + ) + + text = replace_once( + text, + " verify = malloc(image_size);\n" + " if (verify == NULL) {\n", + " verify_size = image_size < SNAPSHOT_VERIFY_CHUNK_BYTES\n" + " ? image_size : SNAPSHOT_VERIFY_CHUNK_BYTES;\n" + " verify = malloc(verify_size);\n" + " if (verify == NULL) {\n", + "bounded verify allocation", + ) + + text = replace_once( + text, + " if (stat.size == image_size &&\n" + " read_exact_file(path, verify, (int)image_size) == 0 &&\n" + " memcmp(verify, image, image_size) == 0) {\n", + " if (stat.size == image_size &&\n" + " snapshot_file_matches_bounded(path, image, image_size,\n" + " verify, verify_size) == 1) {\n", + "existing snapshot compare", + ) + + text = replace_once( + text, + " if (read_exact_file(path, verify, (int)image_size) < 0 ||\n" + " memcmp(verify, image, image_size) != 0) {\n", + " if (snapshot_file_matches_bounded(path, image, image_size,\n" + " verify, verify_size) != 1) {\n", + "new snapshot compare", + ) + + if text.count("verify = malloc(image_size);") != 0: + raise MaterializeError("full-size verify allocation survived") + if text.count("snapshot_file_matches_bounded(") != 3: + raise MaterializeError("unexpected bounded compare helper/call count") + if "fileXioLseek(fd" in text[text.index(MARKER):text.index(anchor)]: + raise MaterializeError("seek-based size check survived in bounded helper") + return text + + +def selftest() -> None: + fixture = r'''#define SNAPSHOT_TRAILER_BYTES 32u + +int forensic_snapshot_save(unsigned int storage, + const void *result, + const void *plan, + char path_out[64]) +{ + unsigned char *image = NULL; + unsigned char *verify = NULL; + unsigned int image_size = 0; + unsigned int slot; + verify = malloc(image_size); + if (verify == NULL) { + return -1; + } + if (stat.size == image_size && + read_exact_file(path, verify, (int)image_size) == 0 && + memcmp(verify, image, image_size) == 0) { + return 0; + } + if (read_exact_file(path, verify, (int)image_size) < 0 || + memcmp(verify, image, image_size) != 0) { + return -2; + } + free(verify); + free(image); + return 0; +} +''' + out = materialize(fixture) + assert MARKER in out + assert CHUNK_DEFINE in out + assert "verify = malloc(verify_size);" in out + assert "verify = malloc(image_size);" not in out + assert out.count("snapshot_file_matches_bounded(") == 3 + helper = out[out.index(MARKER):out.index("int forensic_snapshot_save")] + assert "fileXioLseek" not in helper + assert "fileXioRead(fd, scratch, 1)" in helper + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("source", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + if args.selftest: + selftest() + return 0 + if args.source is None: + parser.error("source is required unless --selftest is used") + args.source.write_text( + materialize(args.source.read_text(encoding="utf-8")), encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/materialize_forensic_snapshot_streaming.py b/tools/materialize_forensic_snapshot_streaming.py new file mode 100644 index 00000000..77df719d --- /dev/null +++ b/tools/materialize_forensic_snapshot_streaming.py @@ -0,0 +1,491 @@ +#!/usr/bin/env python3 +"""Materialize the isolated streaming APAMETA1 snapshot experiment. + +This supersedes the full canonical-image + bounded-readback representation only +inside the experiment build. It preserves APAMETA1 bytes, per-header SHA-256, +trailer SHA-256, slot/non-overwrite policy and exact read-back comparison. + +Workspace layout at maximum patch count: + expected/write chunk 64 KiB + actual/read-back chunk 64 KiB + cached entry digests patch_count * 32 B (<= 64 KiB) + +One allocation therefore replaces the full 2.17 MiB canonical image plus verify +scratch. Default runtime source is restored after the experiment build. +""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +MARKER = "streaming APAMETA1 snapshot experiment" + + +class MaterializeError(RuntimeError): + pass + + +def function_span(text: str, name: str) -> tuple[int, int]: + match = re.search(rf"(?:^|\n)(?:static\s+)?(?:[\w\s\*]+?)\b{re.escape(name)}\s*\([^;]*?\)\s*\{{", text) + if match is None: + raise MaterializeError(f"function {name} not found") + start = match.start() + if start < len(text) and text[start] == "\n": + start += 1 + brace = text.find("{", match.start()) + depth = 0 + i = brace + while i < len(text): + if text[i] == "{": + depth += 1 + elif text[i] == "}": + depth -= 1 + if depth == 0: + end = i + 1 + if end < len(text) and text[end] == "\n": + end += 1 + return start, end + i += 1 + raise MaterializeError(f"function {name} has unmatched braces") + + +def replace_function(text: str, name: str, replacement: str) -> str: + start, end = function_span(text, name) + return text[:start] + replacement.rstrip() + "\n" + text[end:] + + +STREAM_HELPERS = r'''/* streaming APAMETA1 snapshot experiment. + * Keep the on-disk representation and exact read-back contract while bounding + * the producer/consumer workspace instead of owning a complete serialized image. */ +#define SNAPSHOT_STREAM_CHUNK_BYTES (64u * 1024u) +#define SNAPSHOT_DIGEST_BYTES 32u + +static int snapshot_prepare_stream(const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + unsigned char *digests, + unsigned int *image_size_out, + unsigned int *one_or_two_out) +{ + unsigned int one_or_two = 0; + unsigned int i; + + if (plan->patch_count == 0 || plan->patch_count > APA_FORENSIC_MAX_PATCHES) + return fail_snapshot(FORENSIC_SNAPSHOT_INVALID_ARGUMENT, + "validate forensic patch count"); + if (plan->patch_count > + (0xffffffffu - SNAPSHOT_HEADER_BYTES - SNAPSHOT_TRAILER_BYTES) / + SNAPSHOT_ENTRY_BYTES) + return fail_snapshot(FORENSIC_SNAPSHOT_INVALID_ARGUMENT, + "validate HDDMETA image size"); + + for (i = 0; i < plan->patch_count; i++) { + const apa_forensic_patch_t *patch = &plan->patches[i]; + const apa_forensic_node_t *node; + unsigned int distance; + + if (patch->node_index >= result->node_count) + return fail_snapshot(FORENSIC_SNAPSHOT_INVALID_ARGUMENT, + "resolve patch node for HDDMETA"); + node = &result->nodes[patch->node_index]; + if (node->lba != patch->lba) + return fail_snapshot(FORENSIC_SNAPSHOT_INVALID_ARGUMENT, + "match patch LBA to scanned node"); + sha256_buffer(node->header, APA_HEADER_SIZE, + digests + i * SNAPSHOT_DIGEST_BYTES); + distance = apa_forensic_patch_bit_distance(patch); + if (distance == 1u || distance == 2u) + one_or_two++; + } + + *image_size_out = SNAPSHOT_HEADER_BYTES + + plan->patch_count * SNAPSHOT_ENTRY_BYTES + + SNAPSHOT_TRAILER_BYTES; + *one_or_two_out = one_or_two; + return 0; +} + +static void snapshot_build_header(const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + unsigned int one_or_two, + unsigned char header[SNAPSHOT_HEADER_BYTES]) +{ + memset(header, 0, SNAPSHOT_HEADER_BYTES); + memcpy(header, snapshot_magic, sizeof(snapshot_magic)); + write_le32_snapshot(header + 8, FORENSIC_SNAPSHOT_VERSION); + write_le32_snapshot(header + 12, result->total_sectors); + write_le32_snapshot(header + 16, plan->map_index); + write_le32_snapshot(header + 20, plan->confidence); + write_le32_snapshot(header + 24, plan->patch_count); + write_le32_snapshot(header + 28, plan->corroborated_count); + write_le32_snapshot(header + 32, plan->speculative_count); + write_le32_snapshot(header + 36, one_or_two); +} + +static void snapshot_build_entry(const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + const unsigned char *digests, + unsigned int index, + unsigned char entry[SNAPSHOT_ENTRY_BYTES]) +{ + const apa_forensic_patch_t *patch = &plan->patches[index]; + const apa_forensic_node_t *node = &result->nodes[patch->node_index]; + + write_le32_snapshot(entry, patch->lba); + memcpy(entry + 4, digests + index * SNAPSHOT_DIGEST_BYTES, + SNAPSHOT_DIGEST_BYTES); + memcpy(entry + 36, node->header, APA_HEADER_SIZE); +} + +static void snapshot_compute_trailer(const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + const unsigned char *digests, + unsigned int one_or_two, + unsigned char trailer[SNAPSHOT_TRAILER_BYTES]) +{ + sha256_context_t context; + unsigned char header[SNAPSHOT_HEADER_BYTES]; + unsigned char lba[4]; + unsigned int i; + + snapshot_build_header(result, plan, one_or_two, header); + sha256_init(&context); + sha256_update(&context, header, sizeof(header)); + for (i = 0; i < plan->patch_count; i++) { + const apa_forensic_patch_t *patch = &plan->patches[i]; + const apa_forensic_node_t *node = &result->nodes[patch->node_index]; + + write_le32_snapshot(lba, patch->lba); + sha256_update(&context, lba, sizeof(lba)); + sha256_update(&context, digests + i * SNAPSHOT_DIGEST_BYTES, + SNAPSHOT_DIGEST_BYTES); + sha256_update(&context, node->header, APA_HEADER_SIZE); + } + sha256_final(&context, trailer); +} + +static void snapshot_fill_range(const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + const unsigned char *digests, + unsigned int one_or_two, + const unsigned char trailer[SNAPSHOT_TRAILER_BYTES], + unsigned int offset, + unsigned char *destination, + unsigned int bytes) +{ + unsigned char header[SNAPSHOT_HEADER_BYTES]; + unsigned char entry[SNAPSHOT_ENTRY_BYTES]; + unsigned int trailer_offset = SNAPSHOT_HEADER_BYTES + + plan->patch_count * SNAPSHOT_ENTRY_BYTES; + unsigned int done = 0; + + snapshot_build_header(result, plan, one_or_two, header); + while (done < bytes) { + unsigned int position = offset + done; + unsigned int take; + + if (position < SNAPSHOT_HEADER_BYTES) { + unsigned int inner = position; + take = SNAPSHOT_HEADER_BYTES - inner; + if (take > bytes - done) + take = bytes - done; + memcpy(destination + done, header + inner, take); + } else if (position < trailer_offset) { + unsigned int relative = position - SNAPSHOT_HEADER_BYTES; + unsigned int index = relative / SNAPSHOT_ENTRY_BYTES; + unsigned int inner = relative - index * SNAPSHOT_ENTRY_BYTES; + + snapshot_build_entry(result, plan, digests, index, entry); + take = SNAPSHOT_ENTRY_BYTES - inner; + if (take > bytes - done) + take = bytes - done; + memcpy(destination + done, entry + inner, take); + } else { + unsigned int inner = position - trailer_offset; + take = SNAPSHOT_TRAILER_BYTES - inner; + if (take > bytes - done) + take = bytes - done; + memcpy(destination + done, trailer + inner, take); + } + done += take; + } +} + +static int snapshot_write_exact(int fd, const unsigned char *data, + unsigned int size) +{ + unsigned int done = 0; + + while (done < size) { + int result = fileXioWrite(fd, data + done, (int)(size - done)); + if (result <= 0) + return result < 0 ? result : -1; + done += (unsigned int)result; + } + return 0; +} + +static int snapshot_read_exact(int fd, unsigned char *data, unsigned int size) +{ + unsigned int done = 0; + + while (done < size) { + int result = fileXioRead(fd, data + done, (int)(size - done)); + if (result <= 0) + return result < 0 ? result : -1; + done += (unsigned int)result; + } + return 0; +} + +static int snapshot_write_streamed(const char *path, + const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + const unsigned char *digests, + unsigned int one_or_two, + const unsigned char trailer[SNAPSHOT_TRAILER_BYTES], + unsigned char *chunk, + unsigned int image_size) +{ + unsigned int offset = 0; + int fd = fileXioOpen(path, FIO_O_WRONLY | FIO_O_CREAT | FIO_O_TRUNC, 0666); + + if (fd < 0) + return fd; + while (offset < image_size) { + unsigned int bytes = image_size - offset; + int result; + + if (bytes > SNAPSHOT_STREAM_CHUNK_BYTES) + bytes = SNAPSHOT_STREAM_CHUNK_BYTES; + snapshot_fill_range(result, plan, digests, one_or_two, trailer, + offset, chunk, bytes); + result = snapshot_write_exact(fd, chunk, bytes); + if (result < 0) { + fileXioClose(fd); + return result; + } + offset += bytes; + } + return fileXioClose(fd) < 0 ? -1 : 0; +} + +static int snapshot_matches_streamed(const char *path, + const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + const unsigned char *digests, + unsigned int one_or_two, + const unsigned char trailer[SNAPSHOT_TRAILER_BYTES], + unsigned char *expected, + unsigned char *actual, + unsigned int image_size) +{ + unsigned int offset = 0; + int fd = fileXioOpen(path, FIO_O_RDONLY, 0); + + if (fd < 0) + return fd; + while (offset < image_size) { + unsigned int bytes = image_size - offset; + int result_code; + + if (bytes > SNAPSHOT_STREAM_CHUNK_BYTES) + bytes = SNAPSHOT_STREAM_CHUNK_BYTES; + snapshot_fill_range(result, plan, digests, one_or_two, trailer, + offset, expected, bytes); + result_code = snapshot_read_exact(fd, actual, bytes); + if (result_code < 0) { + fileXioClose(fd); + return result_code; + } + if (memcmp(expected, actual, bytes) != 0) { + fileXioClose(fd); + return 0; + } + offset += bytes; + } + { + int extra = fileXioRead(fd, actual, 1); + fileXioClose(fd); + if (extra < 0) + return extra; + return extra == 0 ? 1 : 0; + } +} +''' + +STREAM_SAVE = r'''int forensic_snapshot_save(unsigned int storage, + const apa_forensic_result_t *result, + const apa_forensic_repair_plan_t *plan, + char path_out[FORENSIC_SNAPSHOT_PATH_SIZE]) +{ + unsigned char trailer[SNAPSHOT_TRAILER_BYTES]; + unsigned char *workspace = NULL; + unsigned char *expected; + unsigned char *actual; + unsigned char *digests; + unsigned int image_size = 0; + unsigned int one_or_two = 0; + unsigned int digest_bytes; + unsigned int workspace_bytes; + unsigned int slot; + int result_code; + + if (storage >= STORAGE_TARGET_COUNT || result == NULL || plan == NULL || + path_out == NULL) + return fail_snapshot(FORENSIC_SNAPSHOT_INVALID_ARGUMENT, + "validate HDDMETA snapshot arguments"); + if (plan->patch_count == 0 || plan->patch_count > APA_FORENSIC_MAX_PATCHES) + return fail_snapshot(FORENSIC_SNAPSHOT_INVALID_ARGUMENT, + "validate forensic patch count"); + + digest_bytes = plan->patch_count * SNAPSHOT_DIGEST_BYTES; + workspace_bytes = SNAPSHOT_STREAM_CHUNK_BYTES * 2u + digest_bytes; + workspace = malloc(workspace_bytes); + if (workspace == NULL) + return fail_snapshot(FORENSIC_SNAPSHOT_ALLOC_FAILED, + "allocate streaming HDDMETA workspace"); + expected = workspace; + actual = expected + SNAPSHOT_STREAM_CHUNK_BYTES; + digests = actual + SNAPSHOT_STREAM_CHUNK_BYTES; + + disk_status_begin_at("Forensic repair safety snapshot", + "Streaming HDDMETA from every header the plan may touch", + "Forensic evidence / bounded APAMETA1 workspace"); + disk_status_io(DISK_STATUS_SCAN, 0, 0, 1, 3); + result_code = snapshot_prepare_stream(result, plan, digests, &image_size, + &one_or_two); + if (result_code < 0) { + free(workspace); + disk_status_end(); + return result_code; + } + snapshot_compute_trailer(result, plan, digests, one_or_two, trailer); + + for (slot = 0; slot < FORENSIC_SNAPSHOT_SLOT_COUNT; slot++) { + char path[FORENSIC_SNAPSHOT_PATH_SIZE]; + iox_stat_t stat; + int stat_result; + + storage_path(path, sizeof(path), storage, snapshot_names[slot]); + disk_status_phase_at("Selecting non-overwriting HDDMETA slot", path); + disk_status_io(DISK_STATUS_SCAN, 0, 0, 1, 3); + memset(&stat, 0, sizeof(stat)); + stat_result = fileXioGetStat(path, &stat); + if (stat_result >= 0) { + if (stat.size == image_size && + snapshot_matches_streamed(path, result, plan, digests, + one_or_two, trailer, expected, actual, + image_size) == 1) { + strncpy(path_out, path, FORENSIC_SNAPSHOT_PATH_SIZE - 1u); + path_out[FORENSIC_SNAPSHOT_PATH_SIZE - 1u] = '\0'; + free(workspace); + disk_status_end(); + return 0; + } + continue; + } + + disk_status_phase_at("Writing complete pre-repair HDDMETA snapshot", path); + disk_status_io(DISK_STATUS_SCAN, 0, 0, 2, 3); + result_code = snapshot_write_streamed(path, result, plan, digests, + one_or_two, trailer, expected, + image_size); + if (result_code < 0) { + free(workspace); + disk_status_end(); + return fail_snapshot(FORENSIC_SNAPSHOT_WRITE_FAILED, + "write HDDMETA snapshot"); + } + disk_status_phase_at("Reading HDDMETA back and comparing every byte", path); + disk_status_io(DISK_STATUS_VERIFY, 0, 0, 3, 3); + if (snapshot_matches_streamed(path, result, plan, digests, one_or_two, + trailer, expected, actual, image_size) != 1) { + free(workspace); + disk_status_end(); + return fail_snapshot(FORENSIC_SNAPSHOT_VERIFY_FAILED, + "read back/compare HDDMETA snapshot"); + } + + strncpy(path_out, path, FORENSIC_SNAPSHOT_PATH_SIZE - 1u); + path_out[FORENSIC_SNAPSHOT_PATH_SIZE - 1u] = '\0'; + free(workspace); + disk_status_end(); + return 0; + } + + free(workspace); + disk_status_end(); + return fail_snapshot(FORENSIC_SNAPSHOT_NO_SLOT, + "select non-overwriting HDDMETA slot"); +} +''' + + +def materialize(text: str) -> str: + if MARKER in text: + raise MaterializeError("streaming snapshot already materialized") + start, end = function_span(text, "build_snapshot_image") + text = text[:start] + f"/* {MARKER}. */\n" + STREAM_HELPERS + "\n" + text[end:] + text = replace_function(text, "forensic_snapshot_save", STREAM_SAVE) + + if "build_snapshot_image(" in text: + raise MaterializeError("full-image builder survived streaming transform") + if "malloc(image_size)" in text: + raise MaterializeError("full-size snapshot allocation survived") + if text.count("workspace = malloc(workspace_bytes);") != 1: + raise MaterializeError("streaming workspace allocation missing") + if text.count("snapshot_matches_streamed(") != 3: + raise MaterializeError("unexpected streamed compare helper/call count") + return text + + +def selftest() -> None: + fixture = r'''#define SNAPSHOT_HEADER_BYTES 64u +#define SNAPSHOT_ENTRY_BYTES (4u + 32u + APA_HEADER_SIZE) +#define SNAPSHOT_TRAILER_BYTES 32u +static int build_snapshot_image(const void *result, const void *plan, + unsigned char **image_out, + unsigned int *size_out) +{ + unsigned char *image = malloc(1024); + free(image); + return 0; +} +int forensic_snapshot_save(unsigned int storage, const void *result, + const void *plan, char path_out[64]) +{ + unsigned char *image = malloc(1024); + free(image); + return 0; +} +''' + out = materialize(fixture) + assert MARKER in out + assert "build_snapshot_image(" not in out + assert "workspace = malloc(workspace_bytes);" in out + assert "snapshot_compute_trailer" in out + assert "snapshot_write_streamed" in out + assert "snapshot_matches_streamed" in out + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("source", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("forensic snapshot streaming materializer selftest: PASS") + return 0 + if args.source is None: + parser.error("source is required unless --selftest is used") + args.source.write_text(materialize(args.source.read_text(encoding="utf-8")), + encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/materialize_forensic_snapshot_streaming_v2.py b/tools/materialize_forensic_snapshot_streaming_v2.py new file mode 100644 index 00000000..92693f86 --- /dev/null +++ b/tools/materialize_forensic_snapshot_streaming_v2.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Active wrapper for the isolated streaming APAMETA1 materializer. + +The first prototype used the identifier `result` both for the forensic-result +parameter and for the local write return code in generated `snapshot_write_streamed`. +It also made a successful write depend on `fileXioClose()` returning success, +while the current `write_whole_file()` contract ignores close's return value and +relies on mandatory read-back verification. This wrapper fixes only those two +prototype differences before the generated C is compiled. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import materialize_forensic_snapshot_streaming as v1 + +MARKER = "streaming APAMETA1 snapshot experiment v2 scope/close fix" + + +def materialize(text: str) -> str: + out = v1.materialize(text) + if MARKER in out: + raise v1.MaterializeError("streaming v2 already materialized") + + start, end = v1.function_span(out, "snapshot_write_streamed") + body = out[start:end] + replacements = [ + (" int result;\n", " int write_result;\n"), + (" result = snapshot_write_exact(fd, chunk, bytes);\n", + " write_result = snapshot_write_exact(fd, chunk, bytes);\n"), + (" if (result < 0) {\n", " if (write_result < 0) {\n"), + (" return result;\n", " return write_result;\n"), + (" return fileXioClose(fd) < 0 ? -1 : 0;\n", + " fileXioClose(fd);\n return 0;\n"), + ] + for old, new in replacements: + count = body.count(old) + if count != 1: + raise v1.MaterializeError( + f"streaming v2 fix expected one {old!r}, found {count}" + ) + body = body.replace(old, new, 1) + + body = body.replace( + "{\n unsigned int offset = 0;\n", + "{\n /* " + MARKER + ". */\n unsigned int offset = 0;\n", + 1, + ) + out = out[:start] + body + out[end:] + + fixed_start, fixed_end = v1.function_span(out, "snapshot_write_streamed") + fixed = out[fixed_start:fixed_end] + if " int result;\n" in fixed: + raise v1.MaterializeError("conflicting write result identifier survived") + if "int write_result;" not in fixed: + raise v1.MaterializeError("write_result fix missing") + if "return fileXioClose(fd) < 0" in fixed: + raise v1.MaterializeError("close-return semantic change survived") + return out + + +def selftest() -> None: + fixture = r'''#define SNAPSHOT_HEADER_BYTES 64u +#define SNAPSHOT_ENTRY_BYTES (4u + 32u + APA_HEADER_SIZE) +#define SNAPSHOT_TRAILER_BYTES 32u +static int build_snapshot_image(const void *result, const void *plan, + unsigned char **image_out, + unsigned int *size_out) +{ + unsigned char *image = malloc(1024); + free(image); + return 0; +} +int forensic_snapshot_save(unsigned int storage, const void *result, + const void *plan, char path_out[64]) +{ + unsigned char *image = malloc(1024); + free(image); + return 0; +} +''' + out = materialize(fixture) + assert MARKER in out + start, end = v1.function_span(out, "snapshot_write_streamed") + body = out[start:end] + assert "int write_result;" in body + assert "write_result = snapshot_write_exact" in body + assert "return write_result;" in body + assert "fileXioClose(fd);\n return 0;" in body + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("source", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("forensic snapshot streaming v2 materializer selftest: PASS") + return 0 + if args.source is None: + parser.error("source is required unless --selftest is used") + args.source.write_text(materialize(args.source.read_text(encoding="utf-8")), + encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/materialize_forensic_snapshot_streaming_v3.py b/tools/materialize_forensic_snapshot_streaming_v3.py new file mode 100644 index 00000000..26a5c88c --- /dev/null +++ b/tools/materialize_forensic_snapshot_streaming_v3.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Streaming APAMETA1 v3: keep the cold serializer out of repair_plan_screen. + +V2 proved the bounded memory model but LTO inlined part of the expanded forensic +snapshot path into the repair UI controller. V3 changes only code placement: +`forensic_snapshot_save()` is explicitly cold and noinline. Data representation, +workspace ownership, fileXio sequencing and verification semantics are identical +to streaming v2. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import materialize_forensic_snapshot_streaming_v2 as v2 + +MARKER = "streaming APAMETA1 v3 cold noinline boundary" +ATTRIBUTE_SIGNATURE = ( + "__attribute__((noinline, cold))\n" + "int forensic_snapshot_save(" +) + + +def materialize(text: str) -> str: + out = v2.materialize(text) + if MARKER in out: + raise v2.v1.MaterializeError("streaming v3 already materialized") + + start, end = v2.v1.function_span(out, "forensic_snapshot_save") + body = out[start:end] + old = "int forensic_snapshot_save(unsigned int storage,\n" + new = ( + "/* " + MARKER + ". */\n" + "__attribute__((noinline, cold))\n" + "int forensic_snapshot_save(unsigned int storage,\n" + ) + if body.count(old) != 1: + raise v2.v1.MaterializeError("forensic_snapshot_save signature mismatch") + body = body.replace(old, new, 1) + out = out[:start] + body + out[end:] + + # function_span() intentionally starts at the function declaration. Once the + # attribute is added on the preceding line it is therefore outside a second + # span lookup. Validate the generated source directly instead of treating + # that parser behaviour as a missing attribute. + if out.count(ATTRIBUTE_SIGNATURE) != 1: + raise v2.v1.MaterializeError("cold/noinline boundary missing or duplicated") + return out + + +def selftest() -> None: + fixture = r'''#define SNAPSHOT_HEADER_BYTES 64u +#define SNAPSHOT_ENTRY_BYTES (4u + 32u + APA_HEADER_SIZE) +#define SNAPSHOT_TRAILER_BYTES 32u +static int build_snapshot_image(const void *result, const void *plan, + unsigned char **image_out, + unsigned int *size_out) +{ + unsigned char *image = malloc(1024); + free(image); + return 0; +} +int forensic_snapshot_save(unsigned int storage, const void *result, + const void *plan, char path_out[64]) +{ + unsigned char *image = malloc(1024); + free(image); + return 0; +} +''' + out = materialize(fixture) + assert MARKER in out + assert out.count(ATTRIBUTE_SIGNATURE) == 1 + assert "workspace = malloc(workspace_bytes);" in out + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("source", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("forensic snapshot streaming v3 materializer selftest: PASS") + return 0 + if args.source is None: + parser.error("source is required unless --selftest is used") + args.source.write_text(materialize(args.source.read_text(encoding="utf-8")), + encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/materialize_resume_hash_transaction.py b/tools/materialize_resume_hash_transaction.py new file mode 100644 index 00000000..13e8d1a4 --- /dev/null +++ b/tools/materialize_resume_hash_transaction.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Materialize the isolated resume-hash transaction experiment. + +The frozen default transaction fragment must remain byte-for-byte untouched for +Phase-0 A/B. This build-time transform applies the next recovery-only cut to +its already-isolated resume-hash copy: a PAYLOAD_VERIFIED transaction with a +matching complete SHA checkpoint no longer re-opens the USB ISO merely to prove +identity before using that same checkpoint. + +If either expected source block drifts, generation fails instead of silently +building a partially transformed experiment. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +PRE_SOURCE_OLD = ''' /* Once METADATA_COMMITTED is durable, payload and source identity have + * already passed full verification. Completion recovery only needs the + * target layout plus metadata read-back, so do not require the USB ISO to + * remain connected merely to move stage 5 -> COMPLETE. */ + if (transaction->stage < HDL_TRANSACTION_STAGE_METADATA_COMMITTED) { + result = open_source(transaction->source_path, + transaction->source_bytes, &source); + if (result < 0) + return result; + result = source_fingerprint(&source, fingerprint); + if (result == 0 && memcmp(fingerprint, transaction->source_fingerprint, + sizeof(fingerprint)) != 0) + result = HDL_INSTALL_SOURCE_CHANGED; + if (result == 0) + result = source_identity_matches(&source, transaction); + if (result < 0) { + fileXioClose(source.fd); + return result; + } + } +''' + +PRE_SOURCE_NEW = ''' /* Once METADATA_COMMITTED is durable, payload and source identity have + * already passed full verification. Completion recovery only needs the + * target layout plus metadata read-back, so do not require the USB ISO to + * remain connected merely to move stage 5 -> COMPLETE. + * + * PAYLOAD_VERIFIED has the same opportunity when its authenticated + * checkpoint represents the complete source. The checkpoint is bound to + * this journal's source size, completed byte count, fingerprint and target + * ID; target SHA read-back below still proves the HDD payload matches the + * digest accumulated while the original source bytes were copied. */ +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + if (transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) { + sha256_context_t restored_source_hash; + int checkpoint_result = + hash_checkpoint_load(transaction, &restored_source_hash); + + if (checkpoint_result == 0) { + sha256_final(&restored_source_hash, source_payload_digest); + source_digest_valid = 1; + session_log_line( + "HDL restored complete source SHA checkpoint bytes=%llu; skipped full USB hash pass and source reopen", + (unsigned long long)transaction->source_bytes); + } else { + session_log_line( + "HDL complete source SHA checkpoint unavailable result=%d; using safe full source hash", + checkpoint_result); + } + } +#endif + if (transaction->stage < HDL_TRANSACTION_STAGE_METADATA_COMMITTED && + !(transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED && + source_digest_valid)) { + result = open_source(transaction->source_path, + transaction->source_bytes, &source); + if (result < 0) + return result; + result = source_fingerprint(&source, fingerprint); + if (result == 0 && memcmp(fingerprint, transaction->source_fingerprint, + sizeof(fingerprint)) != 0) + result = HDL_INSTALL_SOURCE_CHANGED; + if (result == 0) + result = source_identity_matches(&source, transaction); + if (result < 0) { + fileXioClose(source.fd); + return result; + } + } +''' + +STAGE4_OLD = ''' if (transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) { + if (!verified_this_run) { +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + sha256_context_t restored_source_hash; + int checkpoint_result = + hash_checkpoint_load(transaction, &restored_source_hash); + + if (checkpoint_result == 0) { + sha256_final(&restored_source_hash, source_payload_digest); + source_digest_valid = 1; + session_log_line( + "HDL restored complete source SHA checkpoint bytes=%llu; skipped full USB hash pass", + (unsigned long long)transaction->source_bytes); + } else { + session_log_line( + "HDL complete source SHA checkpoint unavailable result=%d; using safe full source hash", + checkpoint_result); +#endif + disk_status_phase_at("Hashing source for resumed verification", + "One source pass required without a matching hash checkpoint"); + result = hash_source_payload(transaction, source.fd, + source_payload_digest); + if (result < 0) + goto done; + source_digest_valid = 1; +#if defined(HDL_RESUME_HASH_CHECKPOINT_ENABLED) && HDL_RESUME_HASH_CHECKPOINT_ENABLED + } +#endif + disk_status_phase_at("Re-verifying resumed payload on HDD", + "HDD-only SHA-256 read-back against source digest"); + result = verify_target_digest(transaction, &plan, &layout, + target_fd, source_payload_digest); + if (result < 0) + goto done; + } +''' + +STAGE4_NEW = ''' if (transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) { + if (!verified_this_run) { + if (!source_digest_valid) { + disk_status_phase_at("Hashing source for resumed verification", + "One source pass required without a matching hash checkpoint"); + result = hash_source_payload(transaction, source.fd, + source_payload_digest); + if (result < 0) + goto done; + source_digest_valid = 1; + } + disk_status_phase_at("Re-verifying resumed payload on HDD", + "HDD-only SHA-256 read-back against source digest"); + result = verify_target_digest(transaction, &plan, &layout, + target_fd, source_payload_digest); + if (result < 0) + goto done; + } +''' + + +def materialize(text: str) -> str: + if text.count(PRE_SOURCE_OLD) != 1: + raise ValueError("expected exactly one pre-source validation block") + text = text.replace(PRE_SOURCE_OLD, PRE_SOURCE_NEW, 1) + if text.count(STAGE4_OLD) != 1: + raise ValueError("expected exactly one PAYLOAD_VERIFIED restore block") + text = text.replace(STAGE4_OLD, STAGE4_NEW, 1) + return text + + +def selftest() -> None: + fixture = "before\n" + PRE_SOURCE_OLD + "middle\n" + STAGE4_OLD + "after\n" + transformed = materialize(fixture) + assert PRE_SOURCE_OLD not in transformed + assert STAGE4_OLD not in transformed + assert transformed.count("skipped full USB hash pass and source reopen") == 1 + assert transformed.count("if (!source_digest_valid)") == 1 + try: + materialize("drifted input") + except ValueError: + pass + else: + raise AssertionError("source drift must fail materialization") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("source", nargs="?", type=Path) + parser.add_argument("output", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("materialize_resume_hash_transaction selftest: PASS") + return 0 + if args.source is None or args.output is None: + parser.error("source and output are required unless --selftest is used") + + try: + text = args.source.read_text(encoding="utf-8") + generated = materialize(text) + args.output.write_text(generated, encoding="utf-8") + except (OSError, ValueError) as error: + print(f"materialize_resume_hash_transaction: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/materialize_source_fingerprint_malloc.py b/tools/materialize_source_fingerprint_malloc.py new file mode 100644 index 00000000..091983b0 --- /dev/null +++ b/tools/materialize_source_fingerprint_malloc.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Materialize the isolated source-fingerprint malloc experiment. + +Pinned PS2SDK fileXioRead accepts unaligned EE caller buffers, so 64-byte heap +alignment is not a correctness/API requirement for source_fingerprint(). This +experiment replaces only that helper's memalign(64, 64 KiB) with malloc(64 KiB). +It does not touch the custom hdl0: SIF/DMA buffers, whose 64-byte contract stays +in force. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +MARKER = "HDL source fingerprint ordinary-heap experiment" + + +class MaterializeError(RuntimeError): + pass + + +def function_span(text: str, name: str) -> tuple[int, int]: + token = f"static int {name}(" + start = text.find(token) + if start < 0: + raise MaterializeError(f"missing function {name}") + next_start = text.find("\nstatic int ", start + len(token)) + return start, len(text) if next_start < 0 else next_start + + +def transform(text: str) -> str: + if MARKER in text: + raise MaterializeError("source already contains fingerprint malloc experiment") + start, end = function_span(text, "source_fingerprint") + body = text[start:end] + old = " buffer = memalign(64, HDL_INSTALL_IO_BYTES);\n" + if body.count(old) != 1: + raise MaterializeError( + f"source_fingerprint: expected one memalign site, found {body.count(old)}" + ) + body = body.replace( + old, + f" /* {MARKER}. */\n" + " buffer = malloc(HDL_INSTALL_IO_BYTES);\n", + 1, + ) + if "memalign(64, HDL_INSTALL_IO_BYTES)" in body: + raise MaterializeError("source_fingerprint memalign survived transform") + return text[:start] + body + text[end:] + + +def selftest() -> None: + fixture = r'''static int source_fingerprint(hdl_file_source_t *source, + unsigned char digest[32]) +{ + unsigned char *buffer; + buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; + free(buffer); + return 0; +} + +static int sentinel(void) { return 0; } +''' + out = transform(fixture) + assert MARKER in out + assert "buffer = malloc(HDL_INSTALL_IO_BYTES);" in out + assert "memalign(64, HDL_INSTALL_IO_BYTES)" not in out + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("files", nargs="*", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + if args.selftest: + selftest() + return 0 + if not args.files: + parser.error("at least one source file is required unless --selftest is used") + for path in args.files: + path.write_text(transform(path.read_text(encoding="utf-8")), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/materialize_storage_scratch_natural_alignment.py b/tools/materialize_storage_scratch_natural_alignment.py new file mode 100644 index 00000000..631f82b4 --- /dev/null +++ b/tools/materialize_storage_scratch_natural_alignment.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Materialize the first explicit-alignment cleanup experiment. + +Only two 1024-byte static buffers are changed: +- header_backup.c::backup_scratch +- repair_snapshot.c::snapshot_verify + +Both are consumed by ordinary fileXio-backed reads and CPU parsing/memcmp. Pinned +PS2SDK does not require 64-byte caller alignment for these paths. This tool does +not touch pad, GIF-DMA, hdl0: SIF/DMA, or raw custom device buffers. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +MARKER = "natural-alignment storage scratch experiment" + +SITES = { + "header_backup.c": ( + "static unsigned char backup_scratch[APA_HEADER_SIZE]\n" + " __attribute__((aligned(64)));\n", + "static unsigned char backup_scratch[APA_HEADER_SIZE];\n" + f"/* {MARKER}: ordinary fileXio + CPU consumer. */\n", + ), + "repair_snapshot.c": ( + "static unsigned char snapshot_verify[APA_HEADER_SIZE]\n" + " __attribute__((aligned(64)));\n", + "static unsigned char snapshot_verify[APA_HEADER_SIZE];\n" + f"/* {MARKER}: ordinary fileXio + CPU consumer. */\n", + ), +} + + +class MaterializeError(RuntimeError): + pass + + +def transform(path: Path, text: str) -> str: + key = path.name + if key not in SITES: + raise MaterializeError(f"unsupported file {path}") + if MARKER in text: + raise MaterializeError(f"{path}: experiment already materialized") + old, new = SITES[key] + count = text.count(old) + if count != 1: + raise MaterializeError(f"{path}: expected one alignment site, found {count}") + out = text.replace(old, new, 1) + if old in out: + raise MaterializeError(f"{path}: old alignment survived") + return out + + +def selftest() -> None: + for name, (old, _) in SITES.items(): + path = Path(name) + out = transform(path, old + "static int sentinel;\n") + assert MARKER in out + assert "aligned(64)" not in out + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("files", nargs="*", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + if args.selftest: + selftest() + return 0 + if not args.files: + parser.error("source files are required unless --selftest is used") + for path in args.files: + path.write_text(transform(path, path.read_text(encoding="utf-8")), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/materialize_transaction_workspace.py b/tools/materialize_transaction_workspace.py new file mode 100644 index 00000000..22cb857e --- /dev/null +++ b/tools/materialize_transaction_workspace.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +"""Materialize the isolated HDL transaction-workspace experiment. + +The default source tree keeps the historical per-phase memalign/free policy. +This tool rewrites a temporary copy of transaction.inc so one transaction owns +one 64 KiB / 64-byte-aligned EE workspace across source/copy/HDD-verify phases. + +The rewrite is deliberately exact and fails closed if the audited source shape +moves. It is used only by the experiment build; it does not promote the policy +to the default runtime. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +MARKER = "HDL transaction workspace experiment: one owner, borrowed by phase helpers" + + +class MaterializeError(RuntimeError): + pass + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise MaterializeError(f"{label}: expected one match, found {count}") + return text.replace(old, new, 1) + + +def function_span(text: str, name: str) -> tuple[int, int]: + start_token = f"static int {name}(" + start = text.find(start_token) + if start < 0: + raise MaterializeError(f"missing function {name}") + next_start = text.find("\nstatic int ", start + len(start_token)) + # execute_transaction() is intentionally the final function in the real + # include. EOF is therefore a valid exact bound, not a parser failure. + if next_start < 0: + next_start = len(text) + return start, next_start + + +def edit_function(text: str, name: str, edits: list[tuple[str, str, str]]) -> str: + start, end = function_span(text, name) + body = text[start:end] + for old, new, label in edits: + body = replace_once(body, old, new, f"{name}: {label}") + return text[:start] + body + text[end:] + + +def materialize(text: str) -> str: + if MARKER in text: + raise MaterializeError("source already contains transaction workspace experiment") + + text = edit_function( + text, + "hash_source_payload", + [ + ( + " unsigned char digest[32])\n", + " unsigned char digest[32],\n" + " unsigned char *workspace)\n", + "workspace parameter", + ), + ( + " unsigned char *buffer;\n", + " unsigned char *buffer = workspace;\n", + "borrow workspace", + ), + ( + " buffer = memalign(64, HDL_INSTALL_IO_BYTES);\n" + " if (buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + " if (buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + "remove phase allocation", + ), + ( + "done:\n free(buffer);\n return result;\n", + "done:\n return result;\n", + "remove phase free", + ), + ], + ) + + text = edit_function( + text, + "copy_payload", + [ + ( + " unsigned char source_digest[32])\n", + " unsigned char source_digest[32],\n" + " unsigned char *workspace)\n", + "workspace parameter", + ), + ( + " unsigned char *buffer;\n", + " unsigned char *buffer = workspace;\n", + "borrow workspace", + ), + ( + " buffer = memalign(64, HDL_INSTALL_IO_BYTES);\n" + " if (buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + " if (buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + "remove phase allocation", + ), + ( + "done:\n free(buffer);\n return result;\n", + "done:\n return result;\n", + "remove phase free", + ), + ], + ) + + text = edit_function( + text, + "verify_target_digest", + [ + ( + " const unsigned char expected_digest[32])\n", + " const unsigned char expected_digest[32],\n" + " unsigned char *workspace)\n", + "workspace parameter", + ), + ( + " unsigned char *target_buffer;\n", + " unsigned char *target_buffer = workspace;\n", + "borrow workspace", + ), + ( + " target_buffer = memalign(64, HDL_INSTALL_IO_BYTES);\n" + " if (target_buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + " if (target_buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + "remove phase allocation", + ), + ( + "done:\n free(target_buffer);\n return result;\n", + "done:\n return result;\n", + "remove phase free", + ), + ], + ) + + start, end = function_span(text, "execute_transaction") + body = text[start:end] + body = replace_once( + body, + " unsigned char source_payload_digest[32];\n", + " unsigned char source_payload_digest[32];\n" + f" /* {MARKER}. */\n" + " unsigned char *workspace = NULL;\n", + "execute_transaction: workspace owner", + ) + body = replace_once( + body, + " if (transaction->stage == HDL_TRANSACTION_STAGE_COPYING) {\n" + " result = copy_payload(transaction, &plan, &layout,\n" + " source.fd, target_fd, source_payload_digest);\n", + " if (transaction->stage == HDL_TRANSACTION_STAGE_COPYING ||\n" + " transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) {\n" + " workspace = memalign(64, HDL_INSTALL_IO_BYTES);\n" + " if (workspace == NULL) {\n" + " result = HDL_INSTALL_MEMORY_FAILED;\n" + " goto done;\n" + " }\n" + " }\n" + " if (transaction->stage == HDL_TRANSACTION_STAGE_COPYING) {\n" + " result = copy_payload(transaction, &plan, &layout,\n" + " source.fd, target_fd, source_payload_digest,\n" + " workspace);\n", + "execute_transaction: allocate once before phase work", + ) + body = replace_once( + body, + " result = verify_target_digest(transaction, &plan, &layout,\n" + " target_fd, source_payload_digest);\n", + " result = verify_target_digest(transaction, &plan, &layout,\n" + " target_fd, source_payload_digest,\n" + " workspace);\n", + "execute_transaction: copy-path verify workspace", + ) + body = replace_once( + body, + " result = hash_source_payload(transaction, source.fd,\n" + " source_payload_digest);\n", + " result = hash_source_payload(transaction, source.fd,\n" + " source_payload_digest, workspace);\n", + "execute_transaction: resumed source hash workspace", + ) + body = replace_once( + body, + " result = verify_target_digest(transaction, &plan, &layout,\n" + " target_fd, source_payload_digest);\n", + " result = verify_target_digest(transaction, &plan, &layout,\n" + " target_fd, source_payload_digest,\n" + " workspace);\n", + "execute_transaction: resumed verify workspace", + ) + body = replace_once( + body, + "done:\n if (target_fd >= 0)\n", + "done:\n free(workspace);\n if (target_fd >= 0)\n", + "execute_transaction: release owner workspace", + ) + text = text[:start] + body + text[end:] + + if text.count("memalign(64, HDL_INSTALL_IO_BYTES)") != 1: + raise MaterializeError("materialized transaction must contain exactly one 64 KiB memalign") + if text.count("free(workspace);") != 1: + raise MaterializeError("materialized transaction must contain exactly one workspace free") + for token in ("free(buffer);", "free(target_buffer);"): + if token in text: + raise MaterializeError(f"phase-local free survived: {token}") + return text + + +def fixture(final_sentinel: bool) -> str: + tail = "\nstatic int sentinel(void) { return 0; }\n" if final_sentinel else "\n" + return r'''static int hash_source_payload(const hdl_transaction_t *transaction, + int source_fd, + unsigned char digest[32]) +{ + unsigned char *buffer; + buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; +done: + free(buffer); + return result; +} + +static int copy_payload(hdl_transaction_t *transaction, + const hdl_partition_plan_t *plan, + const hdl_stream_layout_t *layout, + int source_fd, int target_fd, + unsigned char source_digest[32]) +{ + unsigned char *buffer; + buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; +done: + free(buffer); + return result; +} + +static int verify_target_digest(const hdl_transaction_t *transaction, + const hdl_partition_plan_t *plan, + const hdl_stream_layout_t *layout, + int target_fd, + const unsigned char expected_digest[32]) +{ + unsigned char *target_buffer; + target_buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (target_buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; +done: + free(target_buffer); + return result; +} + +static int execute_transaction(hdl_transaction_t *transaction) +{ + unsigned char source_payload_digest[32]; + if (transaction->stage == HDL_TRANSACTION_STAGE_COPYING) { + result = copy_payload(transaction, &plan, &layout, + source.fd, target_fd, source_payload_digest); + result = verify_target_digest(transaction, &plan, &layout, + target_fd, source_payload_digest); + } + if (transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) { + result = hash_source_payload(transaction, source.fd, + source_payload_digest); + result = verify_target_digest(transaction, &plan, &layout, + target_fd, source_payload_digest); + } +done: + if (target_fd >= 0) + fileXioClose(target_fd); + return result; +} +''' + tail + + +def assert_materialized(out: str) -> None: + assert MARKER in out + assert out.count("memalign(64, HDL_INSTALL_IO_BYTES)") == 1 + assert out.count("free(workspace);") == 1 + assert "source_payload_digest,\n workspace" in out + assert "source_payload_digest, workspace" in out or \ + "source_payload_digest,\n workspace" in out + + +def selftest() -> None: + # Cover both a function followed by another static function and the real + # transaction.inc shape where execute_transaction() ends at EOF. + assert_materialized(materialize(fixture(True))) + assert_materialized(materialize(fixture(False))) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("input", nargs="?", type=Path) + parser.add_argument("output", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + return 0 + if args.input is None or args.output is None: + parser.error("input and output are required unless --selftest is used") + + source = args.input.read_text(encoding="utf-8") + result = materialize(source) + args.output.write_text(result, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/materialize_transaction_workspace_v2.py b/tools/materialize_transaction_workspace_v2.py new file mode 100644 index 00000000..dcb17e85 --- /dev/null +++ b/tools/materialize_transaction_workspace_v2.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Materialize Phase-5 transaction workspace v2. + +V1 centralizes the mutually-exclusive COPY/source-hash/target-verify buffers. +V2 additionally lets the execute_transaction source fingerprint borrow the same +workspace before any destructive HDD work. The pre-confirmation UI fingerprint +keeps its original helper-owned allocation, so no 64 KiB buffer is retained +while waiting for the user. + +This remains an isolated build experiment. Default runtime sources are not +modified permanently. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from materialize_transaction_workspace import ( + MARKER as V1_MARKER, + MaterializeError, + function_span, + materialize as materialize_v1, + replace_once, +) + +V2_MARKER = "HDL transaction workspace v2: source admission borrows transaction workspace" + + +def materialize_source(text: str) -> str: + if V2_MARKER in text: + raise MaterializeError("source already contains transaction workspace v2") + + start, end = function_span(text, "source_fingerprint") + body = text[start:end] + body = replace_once( + body, + "static int source_fingerprint(hdl_file_source_t *source,\n" + " unsigned char digest[32])\n", + "static int source_fingerprint_with_workspace(\n" + " hdl_file_source_t *source, unsigned char digest[32],\n" + " unsigned char *workspace)\n", + "source fingerprint borrowed signature", + ) + body = replace_once( + body, + " unsigned char *buffer;\n", + f" /* {V2_MARKER}. */\n" + " unsigned char *buffer = workspace;\n", + "source fingerprint borrowed buffer", + ) + body = replace_once( + body, + " buffer = memalign(64, HDL_INSTALL_IO_BYTES);\n" + " if (buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + " if (buffer == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n", + "remove source fingerprint allocation", + ) + body = replace_once( + body, + "done:\n free(buffer);\n return result;\n", + "done:\n return result;\n", + "remove source fingerprint free", + ) + + wrapper = ''' +static int source_fingerprint(hdl_file_source_t *source, + unsigned char digest[32]) +{ + unsigned char *buffer = memalign(64, HDL_INSTALL_IO_BYTES); + int result; + + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; + result = source_fingerprint_with_workspace(source, digest, buffer); + free(buffer); + return result; +} +''' + text = text[:start] + body + wrapper + text[end:] + return text + + +def materialize_transaction(text: str) -> str: + text = materialize_v1(text) + start, end = function_span(text, "execute_transaction") + body = text[start:end] + + # V1 allocates only after target open. V2 moves that single allocation to + # source admission so fingerprint, copy/hash and target verify share it. + v1_allocation = ''' if (transaction->stage == HDL_TRANSACTION_STAGE_COPYING || + transaction->stage == HDL_TRANSACTION_STAGE_PAYLOAD_VERIFIED) { + workspace = memalign(64, HDL_INSTALL_IO_BYTES); + if (workspace == NULL) { + result = HDL_INSTALL_MEMORY_FAILED; + goto done; + } + } +''' + body = replace_once( + body, + v1_allocation, + "", + "remove v1 late workspace allocation", + ) + + body = replace_once( + body, + " if (transaction->stage < HDL_TRANSACTION_STAGE_METADATA_COMMITTED) {\n" + " result = open_source(transaction->source_path,\n", + " if (transaction->stage < HDL_TRANSACTION_STAGE_METADATA_COMMITTED) {\n" + " workspace = memalign(64, HDL_INSTALL_IO_BYTES);\n" + " if (workspace == NULL)\n" + " return HDL_INSTALL_MEMORY_FAILED;\n" + " result = open_source(transaction->source_path,\n", + "allocate workspace at source admission", + ) + body = replace_once( + body, + " if (result < 0)\n" + " return result;\n" + " result = source_fingerprint(&source, fingerprint);\n", + " if (result < 0) {\n" + " free(workspace);\n" + " return result;\n" + " }\n" + " result = source_fingerprint_with_workspace(\n" + " &source, fingerprint, workspace);\n", + "borrow workspace for transaction fingerprint", + ) + body = replace_once( + body, + " if (result < 0) {\n" + " fileXioClose(source.fd);\n" + " return result;\n" + " }\n" + " }\n", + " if (result < 0) {\n" + " fileXioClose(source.fd);\n" + " free(workspace);\n" + " return result;\n" + " }\n" + " }\n", + "release workspace on source-validation failure", + ) + + text = text[:start] + body + text[end:] + if text.count("memalign(64, HDL_INSTALL_IO_BYTES)") != 1: + raise MaterializeError("v2 transaction must contain exactly one 64 KiB memalign") + if "source_fingerprint_with_workspace(" not in text: + raise MaterializeError("v2 transaction did not borrow source fingerprint workspace") + return text + + +def selftest() -> None: + source = r'''static int source_fingerprint(hdl_file_source_t *source, + unsigned char digest[32]) +{ + unsigned char *buffer; + int result = 0; + buffer = memalign(64, HDL_INSTALL_IO_BYTES); + if (buffer == NULL) + return HDL_INSTALL_MEMORY_FAILED; +done: + free(buffer); + return result; +} + +static int source_identity_matches(void) +{ + return 0; +} +''' + transformed = materialize_source(source) + assert V2_MARKER in transformed + assert transformed.count("memalign(64, HDL_INSTALL_IO_BYTES)") == 1 + assert "source_fingerprint_with_workspace" in transformed + assert "result = source_fingerprint_with_workspace(source, digest, buffer);" in transformed + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("transaction", nargs="?", type=Path) + parser.add_argument("source", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + return 0 + if args.transaction is None or args.source is None: + parser.error("transaction and source are required unless --selftest is used") + + source_text = materialize_source(args.source.read_text(encoding="utf-8")) + transaction_text = materialize_transaction( + args.transaction.read_text(encoding="utf-8") + ) + args.source.write_text(source_text, encoding="utf-8") + args.transaction.write_text(transaction_text, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/parse_hdl_perf.py b/tools/parse_hdl_perf.py new file mode 100644 index 00000000..0bad8de0 --- /dev/null +++ b/tools/parse_hdl_perf.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +"""Parse corpus-v2 HDL performance records from HDDMAN.LOG. + +The PS2 runtime deliberately emits compact counters and formats them only at +phase boundaries. This host-side parser turns those records into stable JSON so +hardware A/B results can be compared without teaching the IOP about JSON, which +would be a fairly creative misuse of 2 MiB of RAM. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +PREFIX = r"(?:\[\d+\]\s+)?" + +RATE_RE = re.compile( + PREFIX + + r"HDL fast copy measured bytes=(\d+) usec=(\d+) rate=(\d+) KiB/s raw-usb11=(\d+)\.(\d+)%" +) +EE_LAT_RE = re.compile( + PREFIX + + r"HDL perf ([\w-]+) samples=(\d+) p50<=(\d+)us p95<=(\d+)us p99<=(\d+)us max=(\d+)us" +) +IOP_LAT_RE = re.compile( + PREFIX + + r"HDL IOP perf ([\w-]+) samples=(\d+) p50<=(\d+)us p95<=(\d+)us p99<=(\d+)us max=(\d+)us" +) +SNAPSHOT_RE = re.compile( + PREFIX + + r"HDL fast I/O snapshot phase=([\w-]+) flags=0x([0-9a-fA-F]+) fragments=(\d+) " + r"direct=(\d+) fallback=(\d+) prefetch-hit=(\d+) miss=(\d+) pump=(\d+) " + r"sectors=(\d+) src-dma=(\d+) target-dma=(\d+)" +) +TRAFFIC_RE = re.compile( + PREFIX + + r"HDL IOP traffic phase=([\w-]+) direct-src-sectors=(\d+) fallback-src-sectors=(\d+) " + r"hdd-write-sectors=(\d+) hdd-read-sectors=(\d+) sif-dma-sectors=(\d+)" +) +EE_COPY_TRAFFIC_RE = re.compile( + PREFIX + + r"HDL perf copy traffic useful=(\d+) sif-dma=(\d+) ee-cache-maint=(\d+) fallback-source=(\d+)" +) +EE_VERIFY_TRAFFIC_RE = re.compile( + PREFIX + + r"HDL perf verify traffic target=(\d+) sif-dma-total=(\d+) ee-cache-maint-total=(\d+) " + r"fallback-target=(\d+) consumer-samples=(\d+) final-chunk-excluded=(\d+)" +) +RESUME_PREFIX_RESTORE_RE = re.compile( + PREFIX + + r"HDL restored source SHA checkpoint bytes=(\d+); skipped prefix rehash" +) +RESUME_FULL_RESTORE_RE = re.compile( + PREFIX + + r"HDL restored complete source SHA checkpoint bytes=(\d+); skipped full USB hash pass( and source reopen)?" +) +RESUME_PREFIX_FALLBACK_RE = re.compile( + PREFIX + + r"HDL source SHA checkpoint unavailable result=(-?\d+); using safe prefix rehash" +) +RESUME_FULL_FALLBACK_RE = re.compile( + PREFIX + + r"HDL complete source SHA checkpoint unavailable result=(-?\d+); using safe full source hash" +) +CHECKPOINT_WRITE_FAILURE_RE = re.compile( + PREFIX + + r"HDL source SHA checkpoint (refresh|save(?: on cancel)?) failed(?: progress=(\d+))? result=(-?\d+)" +) +TRANSACTION_RE = re.compile( + PREFIX + + r"HDL transaction target=([^ ]+) stage=(\d+) progress=(\d+)/(\d+) result=(-?\d+)" +) + + +def _latency(match: re.Match[str]) -> dict[str, int]: + return { + "samples": int(match.group(2)), + "p50_upper_us": int(match.group(3)), + "p95_upper_us": int(match.group(4)), + "p99_upper_us": int(match.group(5)), + "max_us": int(match.group(6)), + } + + +def parse_log(text: str) -> dict[str, object]: + result: dict[str, object] = { + "copy_rate": None, + "ee_latency": {}, + "iop_latency": {}, + "snapshots": {}, + "iop_traffic": {}, + "ee_traffic": {}, + "resume_hash": { + "prefix_restores": [], + "full_restores": [], + "prefix_fallback_results": [], + "full_fallback_results": [], + "checkpoint_write_failures": [], + "transactions": [], + }, + } + + for line in text.splitlines(): + match = RATE_RE.search(line) + if match: + result["copy_rate"] = { + "bytes": int(match.group(1)), + "usec": int(match.group(2)), + "kib_per_second": int(match.group(3)), + "raw_usb11_percent_tenths": int(match.group(4)) * 10 + + int(match.group(5)), + } + continue + + match = IOP_LAT_RE.search(line) + if match: + result["iop_latency"][match.group(1)] = _latency(match) # type: ignore[index] + continue + + match = EE_LAT_RE.search(line) + if match: + result["ee_latency"][match.group(1)] = _latency(match) # type: ignore[index] + continue + + match = SNAPSHOT_RE.search(line) + if match: + result["snapshots"][match.group(1)] = { # type: ignore[index] + "flags": int(match.group(2), 16), + "fragments": int(match.group(3)), + "direct_reads": int(match.group(4)), + "fallback_reads": int(match.group(5)), + "prefetch_hits": int(match.group(6)), + "prefetch_misses": int(match.group(7)), + "pumped_chunks": int(match.group(8)), + "pumped_sectors": int(match.group(9)), + "source_dma_chunks": int(match.group(10)), + "target_dma_chunks": int(match.group(11)), + } + continue + + match = TRAFFIC_RE.search(line) + if match: + result["iop_traffic"][match.group(1)] = { # type: ignore[index] + "direct_source_sectors": int(match.group(2)), + "fallback_source_sectors": int(match.group(3)), + "hdd_write_sectors": int(match.group(4)), + "hdd_read_sectors": int(match.group(5)), + "sif_dma_sectors": int(match.group(6)), + } + continue + + match = EE_COPY_TRAFFIC_RE.search(line) + if match: + result["ee_traffic"]["copy"] = { # type: ignore[index] + "useful_bytes": int(match.group(1)), + "sif_dma_bytes": int(match.group(2)), + "ee_cache_maintenance_bytes": int(match.group(3)), + "fallback_source_bytes": int(match.group(4)), + } + continue + + match = EE_VERIFY_TRAFFIC_RE.search(line) + if match: + result["ee_traffic"]["verify"] = { # type: ignore[index] + "target_bytes": int(match.group(1)), + "sif_dma_total_bytes": int(match.group(2)), + "ee_cache_maintenance_total_bytes": int(match.group(3)), + "fallback_target_bytes": int(match.group(4)), + "consumer_samples": int(match.group(5)), + "final_chunk_excluded": int(match.group(6)), + } + continue + + match = RESUME_PREFIX_RESTORE_RE.search(line) + if match: + result["resume_hash"]["prefix_restores"].append( # type: ignore[index] + {"bytes": int(match.group(1))} + ) + continue + + match = RESUME_FULL_RESTORE_RE.search(line) + if match: + result["resume_hash"]["full_restores"].append( # type: ignore[index] + { + "bytes": int(match.group(1)), + "source_reopen_skipped": match.group(2) is not None, + } + ) + continue + + match = RESUME_PREFIX_FALLBACK_RE.search(line) + if match: + result["resume_hash"]["prefix_fallback_results"].append( # type: ignore[index] + int(match.group(1)) + ) + continue + + match = RESUME_FULL_FALLBACK_RE.search(line) + if match: + result["resume_hash"]["full_fallback_results"].append( # type: ignore[index] + int(match.group(1)) + ) + continue + + match = CHECKPOINT_WRITE_FAILURE_RE.search(line) + if match: + result["resume_hash"]["checkpoint_write_failures"].append( # type: ignore[index] + { + "operation": match.group(1), + "progress": int(match.group(2)) if match.group(2) else None, + "result": int(match.group(3)), + } + ) + continue + + match = TRANSACTION_RE.search(line) + if match: + result["resume_hash"]["transactions"].append( # type: ignore[index] + { + "target": match.group(1), + "stage": int(match.group(2)), + "progress": int(match.group(3)), + "total": int(match.group(4)), + "result": int(match.group(5)), + } + ) + + return result + + +def selftest() -> None: + sample = """ +[0042] HDL fast copy measured bytes=1048576 usec=1000000 rate=1024 KiB/s raw-usb11=69.9% +[0043] HDL IOP perf usb-direct-read samples=16 p50<=65536us p95<=131072us p99<=131072us max=70000us +[0044] HDL perf pump-ioctl samples=16 p50<=65536us p95<=131072us p99<=131072us max=75000us +[0045] HDL fast I/O snapshot phase=copy-final flags=0x00000007 fragments=1 direct=16 fallback=0 prefetch-hit=15 miss=0 pump=16 sectors=2048 src-dma=16 target-dma=0 +[0046] HDL IOP traffic phase=copy-final direct-src-sectors=2048 fallback-src-sectors=0 hdd-write-sectors=2048 hdd-read-sectors=0 sif-dma-sectors=2048 +[0047] HDL perf copy traffic useful=1048576 sif-dma=1048576 ee-cache-maint=2097152 fallback-source=0 +[0048] HDL restored source SHA checkpoint bytes=33554432; skipped prefix rehash +[0049] HDL source SHA checkpoint unavailable result=-5; using safe prefix rehash +[0050] HDL restored complete source SHA checkpoint bytes=1073741824; skipped full USB hash pass and source reopen +[0051] HDL complete source SHA checkpoint unavailable result=-7; using safe full source hash +[0052] HDL source SHA checkpoint save failed progress=32768 result=-12; journal remains authoritative +[0053] HDL transaction target=PP.TEST stage=6 progress=524288/524288 result=0 +""" + parsed = parse_log(sample) + assert parsed["copy_rate"]["kib_per_second"] == 1024 # type: ignore[index] + assert parsed["iop_latency"]["usb-direct-read"]["samples"] == 16 # type: ignore[index] + assert parsed["ee_latency"]["pump-ioctl"]["max_us"] == 75000 # type: ignore[index] + assert parsed["snapshots"]["copy-final"]["flags"] == 7 # type: ignore[index] + assert parsed["iop_traffic"]["copy-final"]["hdd_write_sectors"] == 2048 # type: ignore[index] + assert parsed["ee_traffic"]["copy"]["ee_cache_maintenance_bytes"] == 2097152 # type: ignore[index] + assert parsed["resume_hash"]["prefix_restores"][0]["bytes"] == 33554432 # type: ignore[index] + assert parsed["resume_hash"]["prefix_fallback_results"] == [-5] # type: ignore[index] + assert parsed["resume_hash"]["full_restores"][0]["bytes"] == 1073741824 # type: ignore[index] + assert parsed["resume_hash"]["full_restores"][0]["source_reopen_skipped"] # type: ignore[index] + assert parsed["resume_hash"]["full_fallback_results"] == [-7] # type: ignore[index] + assert parsed["resume_hash"]["checkpoint_write_failures"][0]["progress"] == 32768 # type: ignore[index] + assert parsed["resume_hash"]["transactions"][0]["result"] == 0 # type: ignore[index] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("log", nargs="?", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("parse_hdl_perf selftest: PASS") + return 0 + if args.log is None: + parser.error("log is required unless --selftest is used") + + parsed = parse_log(args.log.read_text(encoding="utf-8", errors="replace")) + rendered = json.dumps(parsed, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(rendered, encoding="utf-8") + else: + sys.stdout.write(rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/phase0_profile_pair_preflight.py b/tools/phase0_profile_pair_preflight.py new file mode 100644 index 00000000..29419cca --- /dev/null +++ b/tools/phase0_profile_pair_preflight.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Verify the frozen Phase-0 PROFILE ON/OFF ELF pair before hardware testing. + +This is deliberately a host-side guard. It does not instrument or modify the +benchmark binaries. Frozen identity is the exact ON/OFF ELF hash pair from CI +#666; later documentation/host-tool commits may rebuild byte-identical ELFs from +a newer repository head, so the run record keeps that artifact's project SHA +separate from the binary identity. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import tempfile +from pathlib import Path + +FROZEN_SOURCE_GIT_SHA = "7875b14d837d6332f5edc37f1c12a55527d7dd87" +PAIR = { + "ON": { + "bytes": 638388, + "sha256": "964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7", + "irx_sha256": "8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db", + }, + "OFF": { + "bytes": 632884, + "sha256": "4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916", + "irx_sha256": "f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751", + }, +} + +# Four samples per mode, distributed so neither build owns one contiguous block. +DEFAULT_ORDER = ("OFF", "ON", "ON", "OFF", "ON", "OFF", "OFF", "ON") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_blob(path: Path, expected_bytes: int, expected_sha256: str) -> tuple[int, str]: + size = path.stat().st_size + digest = _sha256(path) + if size != expected_bytes: + raise ValueError(f"{path}: size {size}, expected {expected_bytes}") + if digest != expected_sha256: + raise ValueError(f"{path}: sha256 {digest}, expected {expected_sha256}") + return size, digest + + +def sample_template(project_git_sha: str = FROZEN_SOURCE_GIT_SHA) -> dict[str, object]: + if not project_git_sha: + raise ValueError("project_git_sha must be non-empty") + + samples: list[dict[str, object]] = [] + for index, mode in enumerate(DEFAULT_ORDER, start=1): + samples.append({ + "run": index, + "mode": mode, + "project_git_sha": project_git_sha, + "benchmark_elf_sha256": PAIR[mode]["sha256"], + "hdl_stream_irx_sha256": PAIR[mode]["irx_sha256"], + "workload_id": "FILL_ME", + "correctness_hash": "FILL_ME", + "source_bytes": 0, + "total_us": 0, + "copy_us": 0, + "verify_us": 0, + }) + return { + "frozen_binary_source_git_sha": FROZEN_SOURCE_GIT_SHA, + "note": ( + "Replace every FILL_ME/zero measurement before comparison. " + "Keep source_bytes and correctness_hash identical across comparable runs. " + "Do not edit the mode-specific ELF/IRX hashes." + ), + "samples": samples, + } + + +def selftest() -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + payload = b"phase0-preflight-fixture\n" + fixture = root / "fixture.bin" + fixture.write_bytes(payload) + digest = hashlib.sha256(payload).hexdigest() + size, observed = verify_blob(fixture, len(payload), digest) + assert size == len(payload) + assert observed == digest + + try: + verify_blob(fixture, len(payload) + 1, digest) + except ValueError as error: + assert "size" in str(error) + else: + raise AssertionError("size mismatch must fail") + + try: + verify_blob(fixture, len(payload), "0" * 64) + except ValueError as error: + assert "sha256" in str(error) + else: + raise AssertionError("hash mismatch must fail") + + artifact_sha = "artifact-head-fixture" + template = sample_template(artifact_sha) + samples = template["samples"] + assert isinstance(samples, list) + assert len(samples) == 8 + assert sum(sample["mode"] == "ON" for sample in samples) == 4 + assert sum(sample["mode"] == "OFF" for sample in samples) == 4 + assert all(sample["project_git_sha"] == artifact_sha for sample in samples) + assert all(sample["benchmark_elf_sha256"] == PAIR[str(sample["mode"])]["sha256"] + for sample in samples) + assert all(sample["hdl_stream_irx_sha256"] == PAIR[str(sample["mode"])]["irx_sha256"] + for sample in samples) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--profile-on", + type=Path, + default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF"), + ) + parser.add_argument( + "--profile-off", + type=Path, + default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF"), + ) + parser.add_argument( + "--project-git-sha", + default=FROZEN_SOURCE_GIT_SHA, + help="project SHA recorded by the CI artifact being tested", + ) + parser.add_argument("--output-template", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("phase0_profile_pair_preflight selftest: PASS") + return 0 + + if not args.project_git_sha: + parser.error("--project-git-sha must be non-empty") + + try: + on_size, on_hash = verify_blob( + args.profile_on, int(PAIR["ON"]["bytes"]), str(PAIR["ON"]["sha256"]) + ) + off_size, off_hash = verify_blob( + args.profile_off, int(PAIR["OFF"]["bytes"]), str(PAIR["OFF"]["sha256"]) + ) + except (OSError, ValueError) as error: + print(f"phase0_profile_pair_preflight: {error}", file=sys.stderr) + return 2 + + print(f"PROFILE ON PASS {on_size} B {on_hash}") + print(f"PROFILE OFF PASS {off_size} B {off_hash}") + print(f"artifact git SHA {args.project_git_sha}") + print(f"frozen source SHA {FROZEN_SOURCE_GIT_SHA}") + + if args.output_template: + rendered = json.dumps(sample_template(args.project_git_sha), indent=2) + "\n" + args.output_template.write_text(rendered, encoding="utf-8") + print(f"sample template {args.output_template}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/printf_format_audit.py b/tools/printf_format_audit.py new file mode 100644 index 00000000..8c1a17ec --- /dev/null +++ b/tools/printf_format_audit.py @@ -0,0 +1,400 @@ +#!/usr/bin/env python3 +"""Audit application printf-family format contracts. + +This is a Phase-1 evidence tool, not a formatter replacement. It scans runtime +source, extracts direct calls to libc/debug/application formatting APIs and also +follows the small local variadic append bridges whose format argument is passed +through to vsnprintf(). The goal is to establish the actual conversion contract +before any integer-only Newlib path is considered. +""" + +from __future__ import annotations + +import argparse +import re +from collections import Counter +from dataclasses import dataclass +from pathlib import Path + +ROOTS = ("src", "iop") +SUFFIXES = {".c", ".inc", ".S", ".s"} + +# value = zero-based index of the format argument +FORMAT_APIS = { + "printf": 0, + "vprintf": 0, + "fprintf": 1, + "vfprintf": 1, + "sprintf": 1, + "vsprintf": 1, + "snprintf": 2, + "vsnprintf": 2, + "iprintf": 0, + "viprintf": 0, + "fiprintf": 1, + "vfiprintf": 1, + "siprintf": 1, + "vsiprintf": 1, + "sniprintf": 2, + "vsniprintf": 2, + "scr_printf": 0, + "scr_vprintf": 0, + "gs_ui_console_printf": 0, + "gs_ui_console_vprintf": 0, + "session_log_line": 0, +} + +# Static local wrappers with different signatures but the same name are keyed by +# file. These are intentionally explicit so a new bridge appears as an unresolved +# dynamic vsnprintf site instead of being silently guessed by the audit. +FILE_FORMAT_APIS = { + ("src/session_log.c", "append_text"): 3, + ("src/boot_report.c", "report_append"): 3, + ("src/forensic_controller_ps2.c", "report_append"): 1, +} + +FLOAT_CONVERSIONS = set("aAeEfFgG") +VALID_CONVERSIONS = set("diouxXfFeEgGaAcspn%") +IDENT = re.compile(r"[A-Za-z_]\w*") +STRING_TOKEN = re.compile(r'(?:u8|u|U|L)?"(?:\\.|[^"\\])*"', re.S) + + +@dataclass +class Site: + path: str + line: int + api: str + format_expr: str + literal: str | None + fragment_text: str + conversions: tuple[str, ...] + + +def source_files(root: Path): + for base in ROOTS: + directory = root / base + if not directory.is_dir(): + continue + for path in sorted(directory.rglob("*")): + if path.is_file() and path.suffix in SUFFIXES: + yield path + + +def mask_comments(text: str) -> str: + out = list(text) + i = 0 + state = "code" + while i < len(text): + ch = text[i] + nxt = text[i + 1] if i + 1 < len(text) else "" + if state == "code": + if ch == '"': + state = "string" + elif ch == "'": + state = "char" + elif ch == "/" and nxt == "/": + out[i] = out[i + 1] = " " + i += 1 + state = "line" + elif ch == "/" and nxt == "*": + out[i] = out[i + 1] = " " + i += 1 + state = "block" + elif state == "string": + if ch == "\\": + i += 1 + elif ch == '"': + state = "code" + elif state == "char": + if ch == "\\": + i += 1 + elif ch == "'": + state = "code" + elif state == "line": + if ch == "\n": + state = "code" + else: + out[i] = " " + elif state == "block": + if ch == "*" and nxt == "/": + out[i] = out[i + 1] = " " + i += 1 + state = "code" + elif ch != "\n": + out[i] = " " + i += 1 + return "".join(out) + + +def matching_paren(text: str, opening: int) -> int | None: + depth = 0 + state = "code" + i = opening + while i < len(text): + ch = text[i] + if state == "code": + if ch == '"': + state = "string" + elif ch == "'": + state = "char" + elif ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + return i + elif state == "string": + if ch == "\\": + i += 1 + elif ch == '"': + state = "code" + elif state == "char": + if ch == "\\": + i += 1 + elif ch == "'": + state = "code" + i += 1 + return None + + +def split_args(text: str) -> list[str]: + args: list[str] = [] + start = 0 + paren = bracket = brace = 0 + state = "code" + i = 0 + while i < len(text): + ch = text[i] + if state == "code": + if ch == '"': + state = "string" + elif ch == "'": + state = "char" + elif ch == "(": + paren += 1 + elif ch == ")": + paren -= 1 + elif ch == "[": + bracket += 1 + elif ch == "]": + bracket -= 1 + elif ch == "{": + brace += 1 + elif ch == "}": + brace -= 1 + elif ch == "," and paren == bracket == brace == 0: + args.append(text[start:i].strip()) + start = i + 1 + elif state == "string": + if ch == "\\": + i += 1 + elif ch == '"': + state = "code" + elif state == "char": + if ch == "\\": + i += 1 + elif ch == "'": + state = "code" + i += 1 + args.append(text[start:].strip()) + return args + + +def string_token_bodies(expr: str) -> list[str]: + bodies: list[str] = [] + for token in STRING_TOKEN.finditer(expr): + raw = token.group(0) + quote = raw.find('"') + bodies.append(raw[quote + 1:-1]) + return bodies + + +def literal_string(expr: str) -> str | None: + pos = 0 + chunks: list[str] = [] + while pos < len(expr): + while pos < len(expr) and expr[pos].isspace(): + pos += 1 + match = STRING_TOKEN.match(expr, pos) + if not match: + return None + token = match.group(0) + quote = token.find('"') + chunks.append(token[quote + 1:-1]) + pos = match.end() + return "".join(chunks) + + +def conversions(fmt: str) -> tuple[str, ...]: + result: list[str] = [] + i = 0 + while i < len(fmt): + if fmt[i] != "%": + i += 1 + continue + i += 1 + if i < len(fmt) and fmt[i] == "%": + i += 1 + continue + while i < len(fmt) and fmt[i] in "#0- +'": + i += 1 + if i < len(fmt) and fmt[i] == "*": + i += 1 + else: + while i < len(fmt) and fmt[i].isdigit(): + i += 1 + if i < len(fmt) and fmt[i] == ".": + i += 1 + if i < len(fmt) and fmt[i] == "*": + i += 1 + else: + while i < len(fmt) and fmt[i].isdigit(): + i += 1 + if fmt[i:i + 2] in ("hh", "ll"): + i += 2 + elif i < len(fmt) and fmt[i] in "hljztL": + i += 1 + if i >= len(fmt): + result.append("?") + break + conv = fmt[i] + result.append(conv if conv in VALID_CONVERSIONS else "?") + i += 1 + return tuple(result) + + +def scan_file(path: Path, root: Path) -> list[Site]: + raw = path.read_text(encoding="utf-8", errors="replace") + text = mask_comments(raw) + rel = str(path.relative_to(root)) + sites: list[Site] = [] + for match in IDENT.finditer(text): + api = match.group(0) + fmt_index = FILE_FORMAT_APIS.get((rel, api), FORMAT_APIS.get(api)) + if fmt_index is None: + continue + pos = match.end() + while pos < len(text) and text[pos].isspace(): + pos += 1 + if pos >= len(text) or text[pos] != "(": + continue + closing = matching_paren(text, pos) + if closing is None: + continue + + # Function definitions are not calls. Excluding them removes the most + # misleading kind of dynamic-format noise without trying to parse all C. + after = closing + 1 + while after < len(text) and text[after].isspace(): + after += 1 + if after < len(text) and text[after] == "{": + continue + + args = split_args(text[pos + 1:closing]) + if fmt_index >= len(args): + continue + expr = args[fmt_index] + lit = literal_string(expr) + fragments = "".join(string_token_bodies(expr)) + conv_source = lit if lit is not None else fragments + sites.append(Site( + rel, + raw.count("\n", 0, match.start()) + 1, + api, + " ".join(expr.split()), + lit, + fragments, + conversions(conv_source), + )) + return sites + + +def selftest() -> None: + assert literal_string('"x=%08x"') == "x=%08x" + assert literal_string('"a" "b%llu"') == "ab%llu" + assert literal_string('APP_NAME " %s"') is None + assert string_token_bodies('APP_NAME " %s"') == [" %s"] + assert conversions("x=%08x %% %llu %s") == ("x", "u", "s") + assert conversions("%7.2f %.*g") == ("f", "g") + assert conversions("%zu %p %c") == ("u", "p", "c") + print("printf-format audit selftest: PASS") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", default=".") + parser.add_argument("--output", default="PRINTF_FORMAT_AUDIT.txt") + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + if args.selftest: + selftest() + return 0 + + root = Path(args.root).resolve() + sites = [site for path in source_files(root) for site in scan_file(path, root)] + api_counts = Counter(site.api for site in sites) + conv_counts = Counter(c for site in sites for c in site.conversions) + literal_sites = [site for site in sites if site.literal is not None] + dynamic_sites = [site for site in sites if site.literal is None] + float_sites = [site for site in sites + if any(c in FLOAT_CONVERSIONS for c in site.conversions)] + malformed_sites = [site for site in sites if "?" in site.conversions] + opaque_dynamic = [site for site in dynamic_sites if not site.fragment_text] + + lines = [ + "PS2 HDD Bootstrap Manager - printf-family format contract audit", + "", + "Epistemic status", + " CURRENT IMPLEMENTATION: source-level formatter and local bridge inventory.", + " INFERENCJA: integer-only policy is safe only after opaque dynamic bridges", + " are traced to callers and hardware correctness is tested.", + "", + f"formatter call sites: {len(sites)}", + f"fully literal format sites: {len(literal_sites)}", + f"dynamic/macro format sites: {len(dynamic_sites)}", + f"opaque dynamic format sites: {len(opaque_dynamic)}", + f"floating-conversion sites in visible string tokens: {len(float_sites)}", + f"malformed/unknown visible conversions: {len(malformed_sites)}", + "", + "API counts", + ] + lines += [f" {name:28s} {count}" for name, count in sorted(api_counts.items())] + lines += ["", "Visible conversion counts"] + lines += [f" %{name:3s} {count}" for name, count in sorted(conv_counts.items())] + + lines += ["", "Floating conversion sites"] + if float_sites: + for site in float_sites: + lines.append(f" {site.path}:{site.line}: {site.api}({site.format_expr})") + else: + lines.append(" none") + + lines += ["", "Opaque dynamic format sites requiring caller review"] + if opaque_dynamic: + for site in opaque_dynamic: + lines.append(f" {site.path}:{site.line}: {site.api}({site.format_expr})") + else: + lines.append(" none") + + lines += ["", "Macro/partially literal format sites"] + partial = [site for site in dynamic_sites if site.fragment_text] + if partial: + for site in partial: + lines.append(f" {site.path}:{site.line}: {site.api}({site.format_expr})") + else: + lines.append(" none") + + lines += ["", "Malformed/unknown visible conversion sites"] + if malformed_sites: + for site in malformed_sites: + lines.append(f" {site.path}:{site.line}: {site.api}({site.format_expr})") + else: + lines.append(" none") + + Path(args.output).write_text("\n".join(lines) + "\n", encoding="utf-8") + print("\n".join(lines[:14])) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/resume_hash_ab_preflight.py b/tools/resume_hash_ab_preflight.py new file mode 100644 index 00000000..4c7026b3 --- /dev/null +++ b/tools/resume_hash_ab_preflight.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +"""Bind the optional HDL resume-hash experiment to the frozen A/B baseline. + +The checkpoint experiment is EE-only. A valid CI artifact must therefore keep +both frozen baseline ELFs exact, build one checkpoint ELF for PROFILE OFF and +one for PROFILE ON, and keep each experiment's embedded hdl_stream IRX byte- +identical to the corresponding frozen profiler mode. + +The emitted identity and sample templates are the authoritative bridge between +CI artifacts and real-PS2 measurements. Experiment ELF hashes are intentionally +computed from the current artifact rather than hard-coded forever; any runtime +change therefore creates a new identity instead of silently inheriting an old +benchmark label. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import tempfile +from pathlib import Path +from typing import Any + +FROZEN_SOURCE_GIT_SHA = "7875b14d837d6332f5edc37f1c12a55527d7dd87" +FROZEN = { + "ON": { + "elf_bytes": 638388, + "elf_sha256": "964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7", + "irx_bytes": 9861, + "irx_sha256": "8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db", + }, + "OFF": { + "elf_bytes": 632884, + "elf_sha256": "4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916", + "irx_bytes": 8405, + "irx_sha256": "f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751", + }, +} +DEFAULT_ORDER = ("BASE", "EXP", "EXP", "BASE", "EXP", "BASE", "BASE", "EXP") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def inspect_blob(path: Path) -> dict[str, Any]: + return {"bytes": path.stat().st_size, "sha256": _sha256(path)} + + +def verify_frozen(path: Path, mode: str, kind: str) -> dict[str, Any]: + observed = inspect_blob(path) + expected_bytes = int(FROZEN[mode][f"{kind}_bytes"]) + expected_sha = str(FROZEN[mode][f"{kind}_sha256"]) + if observed["bytes"] != expected_bytes: + raise ValueError( + f"{path}: {mode} {kind} size {observed['bytes']}, expected {expected_bytes}" + ) + if observed["sha256"] != expected_sha: + raise ValueError( + f"{path}: {mode} {kind} sha256 {observed['sha256']}, expected {expected_sha}" + ) + return observed + + +def build_identity( + project_git_sha: str, + baseline: dict[str, dict[str, Any]], + experiment: dict[str, dict[str, Any]], +) -> dict[str, Any]: + if not project_git_sha: + raise ValueError("project_git_sha must be non-empty") + + profiles: dict[str, Any] = {} + for mode in ("OFF", "ON"): + base = baseline[mode] + exp = experiment[mode] + if exp["elf"]["sha256"] == base["elf"]["sha256"]: + raise ValueError(f"PROFILE {mode} experiment ELF is byte-identical to baseline") + if exp["irx"] != base["irx"]: + raise ValueError( + f"PROFILE {mode} experiment IRX differs from frozen baseline; " + "resume-hash must remain EE-only" + ) + profiles[mode] = { + "baseline": base, + "experiment": exp, + } + + return { + "project_git_sha": project_git_sha, + "frozen_baseline_source_git_sha": FROZEN_SOURCE_GIT_SHA, + "experiment": "HDL_RESUME_HASH_CHECKPOINT=1", + "profiles": profiles, + } + + +def sample_template(identity: dict[str, Any], profile: str) -> dict[str, Any]: + profile = profile.upper() + if profile not in ("OFF", "ON"): + raise ValueError("profile must be OFF or ON") + pair = identity["profiles"][profile] + project_git_sha = identity["project_git_sha"] + samples: list[dict[str, Any]] = [] + for run, mode in enumerate(DEFAULT_ORDER, start=1): + variant = pair["baseline" if mode == "BASE" else "experiment"] + samples.append({ + "run": run, + "profile": profile, + "mode": mode, + "project_git_sha": project_git_sha, + "benchmark_elf_sha256": variant["elf"]["sha256"], + "hdl_stream_irx_sha256": variant["irx"]["sha256"], + "workload_kind": "FILL_ME", + "workload_id": "FILL_ME", + "correctness_hash": "FILL_ME", + "source_bytes": 0, + "total_us": 0, + "copy_us": 0, + "verify_us": 0, + "resume_gate_us": 0, + "completed_sectors": 0, + "checkpoint_status": "FILL_ME", + "checkpoint_restored_bytes": 0, + "source_reopen_skipped": False, + }) + return { + "identity": { + "project_git_sha": project_git_sha, + "profile": profile, + "baseline_elf_sha256": pair["baseline"]["elf"]["sha256"], + "experiment_elf_sha256": pair["experiment"]["elf"]["sha256"], + "hdl_stream_irx_sha256": pair["baseline"]["irx"]["sha256"], + }, + "note": ( + "Replace every FILL_ME and zero measurement. For recovery workloads, " + "checkpoint_status is restored or fallback for EXP and not-applicable " + "for BASE. source_reopen_skipped must be true only for a restored " + "PAYLOAD_VERIFIED EXP run whose log confirms the source-free path. " + "Keep one workload/depth per comparator input." + ), + "samples": samples, + } + + +def selftest() -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "fixture.bin" + payload = b"resume-hash-preflight\n" + path.write_bytes(payload) + observed = inspect_blob(path) + assert observed["bytes"] == len(payload) + assert observed["sha256"] == hashlib.sha256(payload).hexdigest() + + baseline: dict[str, dict[str, Any]] = {} + experiment: dict[str, dict[str, Any]] = {} + for mode in ("OFF", "ON"): + base_irx = {"bytes": 10 if mode == "OFF" else 11, "sha256": mode.lower() * 32} + baseline[mode] = { + "elf": {"bytes": 100, "sha256": ("a" if mode == "OFF" else "b") * 64}, + "irx": base_irx, + } + experiment[mode] = { + "elf": {"bytes": 110, "sha256": ("c" if mode == "OFF" else "d") * 64}, + "irx": dict(base_irx), + } + identity = build_identity("head-fixture", baseline, experiment) + template = sample_template(identity, "OFF") + assert len(template["samples"]) == 8 + assert sum(sample["mode"] == "BASE" for sample in template["samples"]) == 4 + assert sum(sample["mode"] == "EXP" for sample in template["samples"]) == 4 + assert all(not sample["source_reopen_skipped"] for sample in template["samples"]) + + broken = {mode: {key: dict(value) if isinstance(value, dict) else value + for key, value in experiment[mode].items()} + for mode in experiment} + broken["OFF"]["irx"] = {"bytes": 99, "sha256": "e" * 64} + try: + build_identity("head-fixture", baseline, broken) + except ValueError as error: + assert "IRX differs" in str(error) + else: + raise AssertionError("IRX drift must fail") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--base-profile-on", type=Path, + default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF")) + parser.add_argument("--base-profile-off", type=Path, + default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF")) + parser.add_argument("--base-irx-on", type=Path, + default=Path("HDL_STREAM_PROFILE_ON.irx")) + parser.add_argument("--base-irx-off", type=Path, + default=Path("HDL_STREAM_PROFILE_OFF.irx")) + parser.add_argument("--exp-profile-on", type=Path, + default=Path("PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_ON.ELF")) + parser.add_argument("--exp-profile-off", type=Path, + default=Path("PS2_HDD_BOOTSTRAP_MANAGER_RESUME_HASH_PROFILE_OFF.ELF")) + parser.add_argument("--exp-irx-on", type=Path, + default=Path("HDL_STREAM_RESUME_HASH_PROFILE_ON.irx")) + parser.add_argument("--exp-irx-off", type=Path, + default=Path("HDL_STREAM_RESUME_HASH_PROFILE_OFF.irx")) + parser.add_argument("--project-git-sha", default=FROZEN_SOURCE_GIT_SHA) + parser.add_argument("--identity-output", type=Path) + parser.add_argument("--profile-off-template", type=Path) + parser.add_argument("--profile-on-template", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("resume_hash_ab_preflight selftest: PASS") + return 0 + + try: + baseline = { + "OFF": { + "elf": verify_frozen(args.base_profile_off, "OFF", "elf"), + "irx": verify_frozen(args.base_irx_off, "OFF", "irx"), + }, + "ON": { + "elf": verify_frozen(args.base_profile_on, "ON", "elf"), + "irx": verify_frozen(args.base_irx_on, "ON", "irx"), + }, + } + experiment = { + "OFF": { + "elf": inspect_blob(args.exp_profile_off), + "irx": inspect_blob(args.exp_irx_off), + }, + "ON": { + "elf": inspect_blob(args.exp_profile_on), + "irx": inspect_blob(args.exp_irx_on), + }, + } + identity = build_identity(args.project_git_sha, baseline, experiment) + except (OSError, ValueError) as error: + print(f"resume_hash_ab_preflight: {error}", file=sys.stderr) + return 2 + + for profile in ("OFF", "ON"): + pair = identity["profiles"][profile] + print( + f"PROFILE {profile} BASE {pair['baseline']['elf']['bytes']} B " + f"{pair['baseline']['elf']['sha256']}" + ) + print( + f"PROFILE {profile} EXP {pair['experiment']['elf']['bytes']} B " + f"{pair['experiment']['elf']['sha256']}" + ) + print( + f"PROFILE {profile} IRX PASS {pair['experiment']['irx']['sha256']}" + ) + + if args.identity_output: + args.identity_output.write_text( + json.dumps(identity, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + if args.profile_off_template: + args.profile_off_template.write_text( + json.dumps(sample_template(identity, "OFF"), indent=2) + "\n", + encoding="utf-8", + ) + if args.profile_on_template: + args.profile_on_template.write_text( + json.dumps(sample_template(identity, "ON"), indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/transaction_workspace_ab_preflight.py b/tools/transaction_workspace_ab_preflight.py new file mode 100644 index 00000000..cc28e480 --- /dev/null +++ b/tools/transaction_workspace_ab_preflight.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Bind the active cold-isolated streaming APAMETA1 v3 experiment. + +V3 changes only code placement relative to streaming v2 (CI #762): the exported +forensic snapshot save boundary is `cold,noinline` so LTO cannot inflate the +repair-plan UI controller. Streaming representation, fileXio sequencing, +workspace ownership and exact read-back semantics remain unchanged. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +FROZEN = { + "OFF": { + "elf_sha256": "4d1458ebf158c21759d1acdd3a44ecca094a5f9948c9e4461ef4a4beb8f23916", + "elf_bytes": 632884, + "irx_sha256": "f0b29957560ce2ef35a53e77fa8250f477d7aa6490037f00cdfe2edc04a39751", + "irx_bytes": 8405, + }, + "ON": { + "elf_sha256": "964d5c30613b16e5a160b51d4473000ce6da5740596a785d100d2c68a09686d7", + "elf_bytes": 638388, + "irx_sha256": "8d3dbeabadbb860888b2c3d2072e8344953bea443faefccefce006b234cdb3db", + "irx_bytes": 9861, + }, +} + +STREAM_V2 = { + "OFF": { + "elf_sha256": "161b68d578e4ac0cbe1a37b34259d630f22822f43973c8462beb169fd494e223", + "elf_bytes": 634420, + "section_text": 288077, + "named_text": 231436, + "instructions": 57911, + "repair_plan_screen_bytes": 6916, + "repair_plan_screen_instructions": 1730, + "execute_transaction_bytes": 6008, + "execute_transaction_instructions": 1502, + }, + "ON": { + "elf_sha256": "08e1694c9fafef6db0f09fc1e696baed3bb5518c675914ca48515e9a8ba10898", + "elf_bytes": 639924, + "section_text": 292037, + "named_text": 234232, + "instructions": 58610, + "repair_plan_screen_bytes": 6916, + "repair_plan_screen_instructions": 1730, + "execute_transaction_bytes": 6008, + "execute_transaction_instructions": 1502, + }, +} + +MAX_PATCHES = 2048 +SNAPSHOT_MAX_BYTES = 64 + MAX_PATCHES * (4 + 32 + 1024) + 32 +STREAM_CHUNK_BYTES = 64 * 1024 +DIGEST_CACHE_BYTES_MAX = MAX_PATCHES * 32 +STREAM_WORKSPACE_BYTES_MAX = STREAM_CHUNK_BYTES * 2 + DIGEST_CACHE_BYTES_MAX +BOUNDED_V2_PEAK_BYTES = SNAPSHOT_MAX_BYTES + STREAM_CHUNK_BYTES +ORIGINAL_PAIR_PEAK_BYTES = SNAPSHOT_MAX_BYTES * 2 +ORDER = ["BASE", "EXP", "EXP", "BASE", "EXP", "BASE", "BASE", "EXP"] + + +def file_info(path: Path) -> dict[str, object]: + data = path.read_bytes() + return {"path": path.name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()} + + +def validate_frozen(label: str, elf: dict[str, object], irx: dict[str, object]) -> None: + expected = FROZEN[label] + if elf["sha256"] != expected["elf_sha256"] or elf["bytes"] != expected["elf_bytes"]: + raise SystemExit(f"{label} baseline ELF is not the frozen Phase-0 binary") + if irx["sha256"] != expected["irx_sha256"] or irx["bytes"] != expected["irx_bytes"]: + raise SystemExit(f"{label} baseline IRX is not frozen Phase-0 identity") + + +def sample_template(profile: str, baseline: dict, experiment: dict, irx: dict) -> dict: + return { + "experiment": "forensic-snapshot-streaming-apameta1-v3-cold", + "profile": profile, + "workload": "forensic-HDDMETA-save-existing-slot-and-new-slot-readback", + "stream_v2_reference": STREAM_V2[profile], + "expected_source_change": { + "dataflow_changed_from_stream_v2": False, + "cold_noinline_boundary_added": True, + "snapshot_format_changed": False, + "exact_byte_compare_preserved": True, + "stream_workspace_bytes_max": STREAM_WORKSPACE_BYTES_MAX, + "peak_reduction_vs_bounded_v2": BOUNDED_V2_PEAK_BYTES - STREAM_WORKSPACE_BYTES_MAX, + "peak_reduction_vs_original_full_pair": ORIGINAL_PAIR_PEAK_BYTES - STREAM_WORKSPACE_BYTES_MAX, + "iop_binary_change": False, + }, + "baseline": baseline, + "experiment_binary": experiment, + "hdl_stream_irx": irx, + "hardware": { + "console_scp": "UNRECORDED", + "hardware_revision": "UNRECORDED", + "romver": "UNRECORDED", + "storage_adapter": "UNRECORDED", + "active_irx": "UNRECORDED", + }, + "runs": [ + { + "index": i + 1, + "variant": variant, + "patch_count": None, + "snapshot_bytes": None, + "elapsed_us": None, + "correctness_hash": None, + "result": None, + } + for i, variant in enumerate(ORDER) + ], + "report": { + "elapsed_us": {"p50": None, "p95": None, "p99": None, "max": None}, + "correctness_failures": None, + }, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--project-git-sha", required=True) + parser.add_argument("--baseline-off", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_OFF.ELF")) + parser.add_argument("--baseline-on", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_PROFILE_ON.ELF")) + parser.add_argument("--baseline-irx-off", type=Path, default=Path("HDL_STREAM_PROFILE_OFF.irx")) + parser.add_argument("--baseline-irx-on", type=Path, default=Path("HDL_STREAM_PROFILE_ON.irx")) + parser.add_argument("--experiment-off", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_OFF.ELF")) + parser.add_argument("--experiment-on", type=Path, default=Path("PS2_HDD_BOOTSTRAP_MANAGER_TX_WORKSPACE_PROFILE_ON.ELF")) + parser.add_argument("--experiment-irx-off", type=Path, default=Path("HDL_STREAM_TX_WORKSPACE_PROFILE_OFF.irx")) + parser.add_argument("--experiment-irx-on", type=Path, default=Path("HDL_STREAM_TX_WORKSPACE_PROFILE_ON.irx")) + parser.add_argument("--identity-output", type=Path, default=Path("TRANSACTION_WORKSPACE_AB_IDENTITY.json")) + parser.add_argument("--profile-off-template", type=Path, default=Path("TRANSACTION_WORKSPACE_AB_PROFILE_OFF_TEMPLATE.json")) + parser.add_argument("--profile-on-template", type=Path, default=Path("TRANSACTION_WORKSPACE_AB_PROFILE_ON_TEMPLATE.json")) + args = parser.parse_args() + + baseline = {"OFF": file_info(args.baseline_off), "ON": file_info(args.baseline_on)} + irx = {"OFF": file_info(args.baseline_irx_off), "ON": file_info(args.baseline_irx_on)} + experiment = {"OFF": file_info(args.experiment_off), "ON": file_info(args.experiment_on)} + experiment_irx = {"OFF": file_info(args.experiment_irx_off), "ON": file_info(args.experiment_irx_on)} + + for label in ("OFF", "ON"): + validate_frozen(label, baseline[label], irx[label]) + if experiment_irx[label]["sha256"] != irx[label]["sha256"]: + raise SystemExit(f"PROFILE {label} streaming v3 changed hdl_stream.irx") + if experiment[label]["sha256"] == STREAM_V2[label]["elf_sha256"]: + raise SystemExit(f"PROFILE {label} cold boundary did not change EE ELF from streaming v2") + + identity = { + "experiment": "forensic-snapshot-streaming-apameta1-v3-cold", + "project_git_sha": args.project_git_sha, + "frozen_phase0_commit": "7875b14d837d6332f5edc37f1c12a55527d7dd87", + "bounded_readback_v2_frozen_ci": 752, + "streaming_v2_frozen_ci": 762, + "stream_v2_reference": STREAM_V2, + "reference_vector": { + "format": "APAMETA1", + "bytes": 2216, + "image_sha256": "601ba74fc619738dac19baa2a6cb53054b67803e00b1fccb6bf89c69ef4bab6f", + }, + "memory_model": { + "max_snapshot_bytes": SNAPSHOT_MAX_BYTES, + "stream_chunk_bytes": STREAM_CHUNK_BYTES, + "digest_cache_bytes_max": DIGEST_CACHE_BYTES_MAX, + "stream_workspace_bytes_max": STREAM_WORKSPACE_BYTES_MAX, + "bounded_v2_peak_bytes": BOUNDED_V2_PEAK_BYTES, + "original_full_pair_peak_bytes": ORIGINAL_PAIR_PEAK_BYTES, + "peak_reduction_vs_bounded_v2": BOUNDED_V2_PEAK_BYTES - STREAM_WORKSPACE_BYTES_MAX, + "peak_reduction_vs_original_full_pair": ORIGINAL_PAIR_PEAK_BYTES - STREAM_WORKSPACE_BYTES_MAX, + }, + "code_placement": { + "forensic_snapshot_save": "cold,noinline", + "reason": "prevent streaming serializer growth from inflating repair_plan_screen through LTO", + }, + "correctness_contract": { + "on_disk_format_changed": False, + "slot_policy_changed": False, + "overwrite_policy_changed": False, + "per_header_sha256_preserved": True, + "trailer_sha256_preserved": True, + "truncation_detection_preserved": True, + "trailing_data_detection_preserved": True, + "exact_byte_compare_preserved": True, + }, + "PROFILE_OFF": {"baseline_elf": baseline["OFF"], "experiment_elf": experiment["OFF"], "hdl_stream_irx": irx["OFF"]}, + "PROFILE_ON": {"baseline_elf": baseline["ON"], "experiment_elf": experiment["ON"], "hdl_stream_irx": irx["ON"]}, + } + args.identity_output.write_text(json.dumps(identity, indent=2, sort_keys=True) + "\n", encoding="utf-8") + args.profile_off_template.write_text(json.dumps(sample_template("OFF", baseline["OFF"], experiment["OFF"], irx["OFF"]), indent=2, sort_keys=True) + "\n", encoding="utf-8") + args.profile_on_template.write_text(json.dumps(sample_template("ON", baseline["ON"], experiment["ON"], irx["ON"]), indent=2, sort_keys=True) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/validate_resume_hash_samples.py b/tools/validate_resume_hash_samples.py new file mode 100644 index 00000000..2c7608c6 --- /dev/null +++ b/tools/validate_resume_hash_samples.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Validate recovery-specific evidence before resume-hash A/B comparison. + +The generic comparator checks timing identity and restored byte counts. This +companion gate proves that PAYLOAD_VERIFIED experiment samples came from the +new source-free stage-4 path rather than an older checkpoint build that still +re-opened the ISO before restoring the same digest. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + + +def validate(raw: Any) -> int: + if isinstance(raw, dict): + raw = raw.get("samples") + if not isinstance(raw, list) or not raw: + raise ValueError("input must contain a non-empty samples array") + + payload_restores = 0 + for index, sample in enumerate(raw): + if not isinstance(sample, dict): + raise ValueError(f"sample {index}: expected object") + for key in ("mode", "workload_kind", "checkpoint_status", + "source_reopen_skipped"): + if key not in sample: + raise ValueError(f"sample {index}: missing {key}") + skipped = sample["source_reopen_skipped"] + if not isinstance(skipped, bool): + raise ValueError( + f"sample {index}: source_reopen_skipped must be boolean" + ) + mode = str(sample["mode"]).upper() + workload = str(sample["workload_kind"]) + status = str(sample["checkpoint_status"]) + + expected = ( + mode == "EXP" + and workload == "payload_verified_resume" + and status == "restored" + ) + if skipped != expected: + if expected: + raise ValueError( + f"sample {index}: restored PAYLOAD_VERIFIED EXP lacks " + "source-reopen-skip evidence" + ) + raise ValueError( + f"sample {index}: source_reopen_skipped is true outside a " + "restored PAYLOAD_VERIFIED EXP run" + ) + if expected: + payload_restores += 1 + return payload_restores + + +def selftest() -> None: + valid = { + "samples": [ + { + "mode": "BASE", + "workload_kind": "payload_verified_resume", + "checkpoint_status": "not-applicable", + "source_reopen_skipped": False, + }, + { + "mode": "EXP", + "workload_kind": "payload_verified_resume", + "checkpoint_status": "restored", + "source_reopen_skipped": True, + }, + { + "mode": "EXP", + "workload_kind": "payload_verified_resume", + "checkpoint_status": "fallback", + "source_reopen_skipped": False, + }, + { + "mode": "EXP", + "workload_kind": "copy_resume", + "checkpoint_status": "restored", + "source_reopen_skipped": False, + }, + ] + } + assert validate(valid) == 1 + + bad = json.loads(json.dumps(valid)) + bad["samples"][1]["source_reopen_skipped"] = False + try: + validate(bad) + except ValueError as error: + assert "lacks" in str(error) + else: + raise AssertionError("missing source-free evidence must fail") + + bad = json.loads(json.dumps(valid)) + bad["samples"][3]["source_reopen_skipped"] = True + try: + validate(bad) + except ValueError as error: + assert "outside" in str(error) + else: + raise AssertionError("false source-free evidence must fail") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("samples", nargs="?", type=Path) + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + if args.selftest: + selftest() + print("validate_resume_hash_samples selftest: PASS") + return 0 + if args.samples is None: + parser.error("samples is required unless --selftest is used") + + try: + count = validate(json.loads(args.samples.read_text(encoding="utf-8"))) + except (OSError, json.JSONDecodeError, ValueError) as error: + print(f"validate_resume_hash_samples: {error}", file=sys.stderr) + return 2 + print(f"resume-hash sample evidence: PASS payload_source_free_restores={count}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())