diff --git a/.clang-format b/.clang-format index 50c829e0..a95b971f 100644 --- a/.clang-format +++ b/.clang-format @@ -3,4 +3,4 @@ IndentWidth: 4 ColumnLimit: 120 PointerAlignment: Left AllowShortFunctionsOnASingleLine: Empty -SortIncludes: CaseSensitive +SortIncludes: Never # include order is load-bearing in spliced TUs (runtime_ui.cpp) diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 00000000..b8a702d1 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,71 @@ +# clang-tidy configuration for first-party C/C++ (runtime-recomp/src, +# runtime-recomp/tests, modding). LLVM major is pinned to 18 everywhere +# (clang-format, clang-tidy, the sanitizer job); bump them together. +# +# CI runs this diff-gated on changed lines (`tidy` job, advisory first, see +# CONTRIBUTING.md). Whole-tree runs are a report, not a gate. +# +# The set is deliberately conservative: bug finders and the static analyzer, +# with the checks that are mostly noise on recompiled N64 code turned off. +# Turning one of the disabled checks on is its own PR with its own baseline. +# +# Disabled, and why: +# bugprone-easily-swappable-parameters style opinion, fires on most (int, int) APIs +# bugprone-implicit-widening-of-multiplication-result +# address math on guest RDRAM is intentional +# bugprone-narrowing-conversions alias of cppcoreguidelines-narrowing-conversions; +# guest u32/s16 conversions are everywhere +# bugprone-reserved-identifier N64 SDK and libultra names (__osXxx, _gXxx) +# bugprone-macro-parentheses decomp-style macros, mostly in extern/ +# bugprone-assignment-in-if-condition style +# bugprone-switch-missing-default-case style; -Wswitch covers enums +# bugprone-branch-clone per-version tables (v77/v80) repeat on purpose +# performance-no-int-to-ptr guest-to-host pointer translation by design +# performance-enum-size enum layout is part of save and wire formats +# clang-analyzer-optin.performance.Padding layout matches guest structs on purpose +# clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling +# flags every memcpy/snprintf (C11 Annex K) +# misc-include-cleaner wrong on spliced .inl translation units +# (runtime_ui.cpp); enable per module once the +# include order is explicit (plan C2/C4) +Checks: > + -*, + bugprone-*, + -bugprone-easily-swappable-parameters, + -bugprone-implicit-widening-of-multiplication-result, + -bugprone-narrowing-conversions, + -bugprone-reserved-identifier, + -bugprone-macro-parentheses, + -bugprone-assignment-in-if-condition, + -bugprone-switch-missing-default-case, + -bugprone-branch-clone, + clang-analyzer-*, + -clang-analyzer-optin.performance.Padding, + -clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling, + performance-*, + -performance-no-int-to-ptr, + -performance-enum-size, + misc-definitions-in-headers, + concurrency-mt-unsafe, + cert-err33-c + +# Promoted to errors even while the job is advisory: these are almost never +# false positives and each one is a crash or memory corruption. +WarningsAsErrors: > + bugprone-use-after-move, + bugprone-dangling-handle, + bugprone-sizeof-expression, + clang-analyzer-core.* + +# First-party headers only. extern/, third_party/ and the build tree are not +# reported even when a first-party TU includes them. +HeaderFilterRegex: '/(runtime-recomp/(src|tests)|modding)/' +FormatStyle: none +SystemHeaders: false + +CheckOptions: + # printf-family results are ignored on purpose for logging. + - key: cert-err33-c.CheckedFunctions + value: '::aligned_alloc;::calloc;::fclose;::fflush;::fgetc;::fgetpos;::fgets;::fopen;::fread;::freopen;::fseek;::fsetpos;::ftell;::fwrite;::malloc;::realloc;::remove;::rename;::setvbuf;::tmpfile' + - key: bugprone-unused-return-value.AllowCastToVoid + value: true diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000..be1cf84c --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,51 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-settings.json", + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CLAUDE_PROJECT_DIR\"/scripts/agent_hooks/run session_start.py", + "timeout": 30 + } + ] + } + ], + "PreToolUse": [ + { + "matcher": "Edit|Write|MultiEdit|NotebookEdit", + "hooks": [ + { + "type": "command", + "command": "\"$CLAUDE_PROJECT_DIR\"/scripts/agent_hooks/run pre_edit_guard.py", + "timeout": 10 + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "Edit|Write|MultiEdit", + "hooks": [ + { + "type": "command", + "command": "\"$CLAUDE_PROJECT_DIR\"/scripts/agent_hooks/run post_edit_check.py", + "timeout": 30 + } + ] + } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CLAUDE_PROJECT_DIR\"/scripts/agent_hooks/run stop_check.py", + "timeout": 120 + } + ] + } + ] + } +} diff --git a/.codex/hooks.json b/.codex/hooks.json new file mode 100644 index 00000000..05cf1f37 --- /dev/null +++ b/.codex/hooks.json @@ -0,0 +1,51 @@ +{ + "description": "DKR-R pattern enforcement. Same scripts as .claude/settings.json; see scripts/agent_hooks/ and AGENTS.md.", + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$(git rev-parse --show-toplevel)\"/scripts/agent_hooks/run session_start.py", + "timeout": 30 + } + ] + } + ], + "PreToolUse": [ + { + "matcher": "apply_patch", + "hooks": [ + { + "type": "command", + "command": "\"$(git rev-parse --show-toplevel)\"/scripts/agent_hooks/run pre_edit_guard.py", + "timeout": 10 + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "apply_patch", + "hooks": [ + { + "type": "command", + "command": "\"$(git rev-parse --show-toplevel)\"/scripts/agent_hooks/run post_edit_check.py", + "timeout": 30 + } + ] + } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$(git rev-parse --show-toplevel)\"/scripts/agent_hooks/run stop_check.py", + "timeout": 120 + } + ] + } + ] + } +} diff --git a/.editorconfig b/.editorconfig index ab2533a3..e5c2dd7d 100644 --- a/.editorconfig +++ b/.editorconfig @@ -16,3 +16,12 @@ indent_size = 2 [*.md] trim_trailing_whitespace = false + +# shfmt reads these when run without flags. Measured against the tracked +# scripts: 2 spaces, no case indent, no -bn/-sr is the style already in use. +[*.sh] +indent_style = space +indent_size = 2 +switch_case_indent = false +binary_next_line = false +space_redirects = false diff --git a/.gersemirc b/.gersemirc new file mode 100644 index 00000000..06008f51 --- /dev/null +++ b/.gersemirc @@ -0,0 +1,9 @@ +# gersemi configuration for first-party CMake. Indent matches .editorconfig +# and the line length matches .clang-format and ruff. favour-inlining is the +# closer of the two expansion modes to the existing hand-written style +# (31 of 37 files drift vs 34 with favour-expansion). +definitions: [runtime-recomp/cmake] +indent: 4 +line_length: 120 +list_expansion: favour-inlining +warn_about_unknown_commands: false diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..70f74c9a --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,4 @@ +# Formatting-only commits. GitHub's blame view reads this file; locally run +# git config blame.ignoreRevsFile .git-blame-ignore-revs +# Apply ruff format, ruff --fix and shfmt to first-party scripts (A2) +d6690f6da8690bdaf79ab898f10e5cee69f6ef07 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..674d383a --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,35 @@ +# Only the files that define the project's rules have owners. Ordinary code +# needs no owner review. Both maintainers own every entry, so either one can +# approve the other's change (GitHub does not let you approve your own PR). +# The ruleset in .github/ruleset-main.json turns these into required reviews. + +# Pattern guard, architecture contracts and the check entry point +/architecture/ @ThatGuyMcd @swhitt +/scripts/check.py @ThatGuyMcd @swhitt +/scripts/pattern_guard.py @ThatGuyMcd @swhitt +/scripts/check_includes.py @ThatGuyMcd @swhitt +/scripts/check_registries.py @ThatGuyMcd @swhitt +/scripts/agent_hooks/ @ThatGuyMcd @swhitt +/scripts/check_pinned_bytes.py @ThatGuyMcd @swhitt +/scripts/bootstrap_hooks.py @ThatGuyMcd @swhitt + +# Lint, format and hook configuration (a new ignore or baseline entry is a rule change) +/.pre-commit-config.yaml @ThatGuyMcd @swhitt +/pyproject.toml @ThatGuyMcd @swhitt +/.clang-format @ThatGuyMcd @swhitt +/.clang-tidy @ThatGuyMcd @swhitt +/.gersemirc @ThatGuyMcd @swhitt +/runtime-recomp/cmake/Warnings.cmake @ThatGuyMcd @swhitt +/.git-blame-ignore-revs @ThatGuyMcd @swhitt + +# CI, ownership and repository policy +/.github/ @ThatGuyMcd @swhitt + +# Agent instructions +/.claude/ @ThatGuyMcd @swhitt +/.codex/ @ThatGuyMcd @swhitt +/AGENTS.md @ThatGuyMcd @swhitt +/CLAUDE.md @ThatGuyMcd @swhitt + +# Online compatibility identity +/runtime-recomp/src/game/netplay/legacy_identity.hpp @ThatGuyMcd @swhitt diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..25723308 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,17 @@ +## What + + + +## Pattern checklist + +- [ ] First-party changes are in-place edits under `runtime-recomp/`, `modding/` or `scripts/` (no new `patches//`) +- [ ] Third-party changes are `*.patch` files registered in `patches/manifest.json` +- [ ] Guest hooks only via `guest-hooks/hooks.toml`; symbols only via `symbols/*.toml` +- [ ] New settings, launcher pages and owned services are registered, and their contract docs are updated +- [ ] `netplay/legacy_identity.hpp` updated (reviewed by both owners) if simulation, wire or owned-service behaviour changed +- [ ] Tests added or updated; `python scripts/check.py --suite all` passes locally +- [ ] Platforms considered: Windows / Linux / macOS / iOS / Android (say which ones you built) + +## Risk + + diff --git a/.github/lint-tools.txt b/.github/lint-tools.txt new file mode 100644 index 00000000..e48b8b0a --- /dev/null +++ b/.github/lint-tools.txt @@ -0,0 +1,170 @@ +# Lint and format tools for the `lint` job, installed with --require-hashes. +# clang-format stays on LLVM 18, the same major as clang-tidy-18 in the `tidy` +# job and clang-18 in the nightly sanitizer job. Bump all three together. +# Regenerate with: +# printf 'ruff==X\nclang-format==X\ngersemi==X\nshellcheck-py==X\n' | +# uv pip compile - --generate-hashes --universal --python-version 3.10 --no-header --no-annotate +clang-format==18.1.8 \ + --hash=sha256:03e0a762b4b504750ce4999174fa50a3c4d6b83d7999481e1b6a1b3bc2915121 \ + --hash=sha256:065ddb7fdd0cb329976115fa03500dd560d9753dd50c25a269776648e6e6bf35 \ + --hash=sha256:2a73a50e17c94325712631c15e5b210e374841bbeefb915f12934b902310df7b \ + --hash=sha256:2de122b8aa78ba49e326f974131caab2c79f4ae877c227cd4e3e5d82a98d21e5 \ + --hash=sha256:310206c21fa8177c019a3871de8c5400e51867376cae630ee3d1610aa6c93816 \ + --hash=sha256:4762e95ea887d522bf664c1411d93d4d41fc9eb059835dd88b9ad54bc3cb08e0 \ + --hash=sha256:4be2b5d983a0cc1ef90a224b599f5928d82ce31154ba69accfdcb670aea62f40 \ + --hash=sha256:6ca6768270e291495174faa6d8c082aa0181722b29ef8e36558b397069fa4a2d \ + --hash=sha256:7c41e2521b7e6ba706cc5d1c3e95eed9a41c1522244e23624e1518991f02a604 \ + --hash=sha256:7d71869103d0f27be3c4930bea59dc6325177a259ba321a04663a231432ec343 \ + --hash=sha256:82c5dd546efa80a838bb9fd4cf6f6e37d85301c0bca13cd79ab5749422ca35a1 \ + --hash=sha256:c11ecdae9cd9068ed010b7cc5fa9d3d2ae08de8bbfa493df865774ad63fd7d76 \ + --hash=sha256:d2db077523bd4517b41fa6adb2e5ee63fc91bc7b641dc6e28b959fa8050cf41b \ + --hash=sha256:d3a41b7c7c3e65fa56763f5712f919111b35d7a8e857f8bef4ad5a4cdc1be131 \ + --hash=sha256:ea103b3ae5b0941152cd29c67eab086aafc98675370ac13581005807680132aa +gersemi==0.29.2 \ + --hash=sha256:0eed0afb642293c3353043a898cd5a1b9b97a3cbf441627c2bc12f4f51b9e007 \ + --hash=sha256:10dd77ff6f2cbd63002ac3375eb236301ff0ab7a67e7b0dc3968893a24e1422d \ + --hash=sha256:144ef0cabad485eedc03786814e1a2b651043427f99867d5aba7086b9d30d9a4 \ + --hash=sha256:1e64857ab198625d62f6ceeaaa33263fd666adebaed3477bf2b9e7ca97f1ab22 \ + --hash=sha256:3acab643bec6c8174fb90ece56d76a8381847f77873a1a6ae3211f5b40472c25 \ + --hash=sha256:3e354644dfc18a9e6fa9e1ff2d8abdc5c92a7db7a7af4210b5d5cb39994770dd \ + --hash=sha256:45fff157fa8bd6920d043aed45f6d8d6ac5bf7b986d042bae6bfed896c888ecd \ + --hash=sha256:4ef4e845985ae5600e715fd54d26898421c2f685b5c8570a32cc3037465b11aa \ + --hash=sha256:50d60f5470cf80baa07bdf993e5d9add610032d14d83e4baa6f9d248cd2811c0 \ + --hash=sha256:5f394ea9b2301e4834cfb072ee312d691aa71e16fe7c722f08368310aac220b9 \ + --hash=sha256:5f5b7c1e66d13caea2810fe39ce11493e769beb236fd435925e9fa5a80d4e3ab \ + --hash=sha256:649bd8b4678980421a292390ad7b0f3a5f350a93a33a178b75c28871c407ae13 \ + --hash=sha256:68a3a67f046240bfe095974d7d37cc7244d51f515590cc091c7892d876c77d25 \ + --hash=sha256:6a13296e33c027b4e438ec20602594b4ce36774712ba99a4ea23fcf47fd5e973 \ + --hash=sha256:6b6eca9d4243a2fe9876fc3418971727dcef6f087108db756e35c1a6558b0ff0 \ + --hash=sha256:7388cb05dde435dc61990a4f5e8ed8b349dd87381bb2ae3d1e54660470e3055a \ + --hash=sha256:73bb7370ca49230c37859e2ed06ed829ca2e53190ce71317e516c413be910e02 \ + --hash=sha256:76b0d29af1412045b0be37d3e7c240146638bcaecab6e7e7866644a54a283b6a \ + --hash=sha256:816e19624cb088aa99dbda70dece853a855b29646b042999bf005f2cf0b587b4 \ + --hash=sha256:84716790f31ffcde14f21999a208517986ca72bd55a8f2e5de778329db776dfe \ + --hash=sha256:8b72f9195d5d00351e4a23475f16a5ac2b8ede4abf82ada589c5a1990ac99289 \ + --hash=sha256:8cdec9adc556d6dbee485c1222b304519117eff609833d28ce2f152b1a447840 \ + --hash=sha256:90d5175c2d73596b83ece8c1fa2374c91f08488b7f02a2a5d6400e55b0d6b53b \ + --hash=sha256:95bddeac76384e3f05c061d8742345ecb36dee9840dfc41e1d5af581d4c4b67a \ + --hash=sha256:9c79d69d3665626d75db925c94045640f0c6c281c0e1dbb7ad368e88bf3ca9e8 \ + --hash=sha256:a5fa40246665bf437f742873ab8a1f75789885555f4e3b80ccfe2b7d4e0d22d8 \ + --hash=sha256:a738e05edd342cc63a3df188eec185cd0fde4e0c8cf52026c409bf5b55c26401 \ + --hash=sha256:a7b88ee4420c6cead3a9768397421390a224422c5d7735955cab474ad8c815d4 \ + --hash=sha256:a8a851f3af50e185beff001e4666b24f4bb24b3f2810671eb4d04b30fff7066c \ + --hash=sha256:aff3003cd402561cbfb7c07d35aae4dfa4ac6b1f17f75b4641914b51d0317fe4 \ + --hash=sha256:b52406f2c226bc05e9e5181844e1d91f0a3dbc64b4f63cb7af2571679399a8ff \ + --hash=sha256:bd862abd026a186e82aecd3fbdbb6b133bb55c7ce98dcab9dec5dfbd98c50316 \ + --hash=sha256:c4caf3aacf3d0b0860893296bd0052d1649aaa42fc665e916094b121c17996e7 \ + --hash=sha256:c9fa61ae5f18f4980a23ba5983a7f0d9d079af8117935db6f9b9a9fc996b5519 \ + --hash=sha256:ce397d9ae2c42acefec206565ba14c08dcc43893d0d3ec682aa8eb613a0d6778 \ + --hash=sha256:d005b5adb93d9826d9b59b88d5ffd70540bb29c04f687b68f1e46ee6af090ba3 \ + --hash=sha256:d44134a79f6213c22bfe2cfcb04e6afbb25e07d81c15a8f323249c336690a95a \ + --hash=sha256:d8a5d126d7f2ea903f8e61e9637f8f8a2b15252e553cf8d7c31d801ad5021792 \ + --hash=sha256:db658e80ae61172628c886245e0d3de570e4b6487adcafa4fcc8ff9bed631f58 \ + --hash=sha256:e39751c3b2f5f1dfb42f4e6a556c74cac3efd1dafceb002e463233e96d9a47c9 \ + --hash=sha256:e422a13b1bdc0ccb58c17bc2c9cde547f91f66b50a0fdd4bf3878e33667dfda7 \ + --hash=sha256:e618edff9d8c772a3599c4bd8b4c155e1349e3f022a221e99c530c1c403bb2d3 \ + --hash=sha256:ed210d1f017421cb5ff6a5318f10adc40caa98dc141bafdffe3cbb55f92c6f0b \ + --hash=sha256:f18690f8ef77e64e99d55c2684a29f2ea54782c9db9abeb8a649f77f4858c71d \ + --hash=sha256:f2050dbadd4c3d96153f453ff12da0d24965640f1c288c3df3440697e4bd2985 \ + --hash=sha256:f26b4cb7c1d9b9ab21109c9ebe226616f7c9e2ae9773056769d5421207e21e3d \ + --hash=sha256:f847982092dc284ac4cb6a9262e3d0a609f074c9300f39892af411e1629bb1ca \ + --hash=sha256:fce3f85213a405f820f413f6b6484c07200fd92a6869063b7e7e05020813ebb8 +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 +ruff==0.17.2 \ + --hash=sha256:122258f6ecde0e0dd1bb60b24263c3edde4338caf723fca711cc729f9bfd414a \ + --hash=sha256:195ad918027f817c119310ca89a42d347887dfc28934aaa869325e9c5dc9d962 \ + --hash=sha256:1b59d447f197bef9ef48a22ba1a6016137753c6099e9bac9c36eb566e422424c \ + --hash=sha256:2d086be452eed686aa51271ecf2cbe5037956517cc6ee39a17b4489ac12f8bbe \ + --hash=sha256:39c5b55c960515cdb32be16e69efb5734c589c06a0eb38a22694524a336b889d \ + --hash=sha256:583bd67bb9e7061dd09bc46a0a884bae2bbd214dbcd48e509813612619683869 \ + --hash=sha256:5875a16b0de70b43cb81eabea5ae5e8f5d1b2fd1b5e6ed07f80cfe5019d0bd5a \ + --hash=sha256:6bce7d1f5165701ae18f5df85c65afe78cddfa6c5fb4b9bdb0014d60388f3edd \ + --hash=sha256:7cfd783590edf2b77317882631900f6a13ea2e8eacefe6ebf2370d2715299160 \ + --hash=sha256:7f8fba429d6ac4128ab29cfb8fa6240d2fa671d04adf7f84d3b22fc7b6b6bea4 \ + --hash=sha256:84d632402d14c23b488f941db23712d6736a013225162062d34558c3f3c780eb \ + --hash=sha256:8c21b4ab24485358f17291a4f8dc5399b834e963b8efedd920fe4fdb02081bb4 \ + --hash=sha256:92a80d2f44584dabb0189b3a6c393e37e525e04942838482010575cb81945be7 \ + --hash=sha256:96a65e755d8dba094707fb23ab602111d9a31ec2835b3a6992a1914a8fcb211e \ + --hash=sha256:b7d0906677d89bbf3cf73cde64b5bbe2ab35df3ae43e49ee2627301e91265bac \ + --hash=sha256:ba405cbd5ec7bda1f555d5e52be1fd68d02608d4d95d24591bdac92cfacb579f \ + --hash=sha256:c555a877f8f41447e318101cfefea9bf12dfa9f82eedf90c376e57e069f917d0 \ + --hash=sha256:c60f93fec7142dba5b8f487f94bf2f6231dae24dc11e4c48648347c8b0720004 +shellcheck-py==0.11.0.1 \ + --hash=sha256:1b274df81de5b000ff78db433e7328b87e52e3c38481c60f8e488c3095beef05 \ + --hash=sha256:5c620c88901e8f1d3be5934b31ea99e3310065e1245253741eafd0a275c8c9cc \ + --hash=sha256:6b88d0a244c82ed07e06a53e444da841f69330ca59ae15d4a66c391655dae7a0 \ + --hash=sha256:784156289ecb17e91c692cd783ab5152333309588cabb10032a047331c63e759 \ + --hash=sha256:b6a3fee28efda2e16e38d6e6d59faf7224300256456639727370d404730849e8 diff --git a/.github/ruleset-main.json b/.github/ruleset-main.json new file mode 100644 index 00000000..47805046 --- /dev/null +++ b/.github/ruleset-main.json @@ -0,0 +1,51 @@ +{ + "name": "main and release branches", + "target": "branch", + "enforcement": "active", + "bypass_actors": [ + { + "actor_id": 5, + "actor_type": "RepositoryRole", + "bypass_mode": "always" + } + ], + "conditions": { + "ref_name": { + "include": ["~DEFAULT_BRANCH", "refs/heads/release/*"], + "exclude": [] + } + }, + "rules": [ + { "type": "deletion" }, + { "type": "non_fast_forward" }, + { + "type": "pull_request", + "parameters": { + "required_approving_review_count": 0, + "require_code_owner_review": true, + "dismiss_stale_reviews_on_push": false, + "require_last_push_approval": false, + "required_review_thread_resolution": false, + "allowed_merge_methods": ["merge", "squash", "rebase"] + } + }, + { + "type": "required_status_checks", + "parameters": { + "strict_required_status_checks_policy": true, + "do_not_enforce_on_create": false, + "required_status_checks": [ + { "context": "lint", "integration_id": 15368 }, + { "context": "linux", "integration_id": 15368 }, + { "context": "macos (macos-15)", "integration_id": 15368 }, + { "context": "macos (macos-15-intel)", "integration_id": 15368 }, + { "context": "windows", "integration_id": 15368 }, + { "context": "asset-scan", "integration_id": 15368 }, + { "context": "dependency-lock", "integration_id": 15368 }, + { "context": "tooling", "integration_id": 15368 }, + { "context": "workflow-lint", "integration_id": 15368 } + ] + } + } + ] +} diff --git a/.github/tidy-skip.txt b/.github/tidy-skip.txt new file mode 100644 index 00000000..c1a641af --- /dev/null +++ b/.github/tidy-skip.txt @@ -0,0 +1,239 @@ +# First-party translation units the `host` preset does not compile, so the +# `tidy` job has no compile command for them. A changed .c/.cpp that is neither +# in build/host/compile_commands.json nor listed here fails the job, which keeps +# gaps in tidy coverage visible. Remove a line once its TU builds in `host` +# (most of these need RT64 or generated code; see host-full-nogen in the plan). +# Paths are repo-relative, one per line, exact match. +modding/examples/ai-boss-difficulty/main.c +modding/examples/core-boot/main.c +modding/examples/custom-trophy-races/main.c +modding/examples/dev-menu/main.c +modding/examples/guide-starting-bananas/main.c +modding/examples/live-ai-difficulty/main.c +modding/runtime.c +runtime-recomp/src/android/android_file_dialog.cpp +runtime-recomp/src/android/android_platform.cpp +runtime-recomp/src/android/android_rt64_window.cpp +runtime-recomp/src/android/android_update_http.cpp +runtime-recomp/src/android/mobile_ui.cpp +runtime-recomp/src/game/camera_obstruction.cpp +runtime-recomp/src/game/custom_music.cpp +runtime-recomp/src/game/custom_tracks_hooks.cpp +runtime-recomp/src/game/custom_tracks.cpp +runtime-recomp/src/game/f3ddkr_rt64.cpp +runtime-recomp/src/game/game_main.cpp +runtime-recomp/src/game/game_payload_v77.cpp +runtime-recomp/src/game/game_payload_v80.cpp +runtime-recomp/src/game/game_payload.cpp +runtime-recomp/src/game/game_registration.cpp +runtime-recomp/src/game/generated/jumpman_font.c +runtime-recomp/src/game/generated/racing_banana_font.c +runtime-recomp/src/game/generated/selawik_font.c +runtime-recomp/src/game/hud_basic_settings.cpp +runtime-recomp/src/game/hud_layout_editor.cpp +runtime-recomp/src/game/legacy_runtime_qualification.cpp +runtime-recomp/src/game/local_model_rt64.cpp +runtime-recomp/src/game/local_scenery_rt64.cpp +runtime-recomp/src/game/modern_camera.cpp +runtime-recomp/src/game/mods/legacy_asset_bank.cpp +runtime-recomp/src/game/mods/legacy_asset_bus.cpp +runtime-recomp/src/game/mods/legacy_asset_directory.cpp +runtime-recomp/src/game/mods/legacy_asset_io.cpp +runtime-recomp/src/game/mods/legacy_audio_bank.cpp +runtime-recomp/src/game/mods/legacy_behaviour.cpp +runtime-recomp/src/game/mods/legacy_cache_namespace.cpp +runtime-recomp/src/game/mods/legacy_character_artifact.cpp +runtime-recomp/src/game/mods/legacy_character_audio.cpp +runtime-recomp/src/game/mods/legacy_character_materialize.cpp +runtime-recomp/src/game/mods/legacy_character_menu_render.cpp +runtime-recomp/src/game/mods/legacy_character_menu.cpp +runtime-recomp/src/game/mods/legacy_character_presentation.cpp +runtime-recomp/src/game/mods/legacy_character_roster.cpp +runtime-recomp/src/game/mods/legacy_character_stage.cpp +runtime-recomp/src/game/mods/legacy_guest_checkpoint.cpp +runtime-recomp/src/game/mods/legacy_import_library.cpp +runtime-recomp/src/game/mods/legacy_mod_dependencies.cpp +runtime-recomp/src/game/mods/legacy_mod_format.cpp +runtime-recomp/src/game/mods/legacy_mod_geometry.cpp +runtime-recomp/src/game/mods/legacy_mod_import.cpp +runtime-recomp/src/game/mods/legacy_mod_launch.cpp +runtime-recomp/src/game/mods/legacy_mod_library.cpp +runtime-recomp/src/game/mods/legacy_mod_lzma_boundary.c +runtime-recomp/src/game/mods/legacy_mod_stage.cpp +runtime-recomp/src/game/mods/legacy_mod_worker.cpp +runtime-recomp/src/game/mods/legacy_resident_assets.cpp +runtime-recomp/src/game/mods/legacy_runtime_assets.cpp +runtime-recomp/src/game/mods/legacy_runtime_character.cpp +runtime-recomp/src/game/mods/legacy_runtime_io.cpp +runtime-recomp/src/game/mods/legacy_runtime_session.cpp +runtime-recomp/src/game/mods/legacy_track_artifact.cpp +runtime-recomp/src/game/mods/legacy_track_catalog.cpp +runtime-recomp/src/game/mods/legacy_track_materialize.cpp +runtime-recomp/src/game/mods/legacy_track_menu_adapter.cpp +runtime-recomp/src/game/mods/legacy_track_menu.cpp +runtime-recomp/src/game/mods/legacy_track_prepare_job.cpp +runtime-recomp/src/game/mods/magic_code_effects.cpp +runtime-recomp/src/game/mods/magic_code_game_services.cpp +runtime-recomp/src/game/mods/magic_code_game_world.cpp +runtime-recomp/src/game/mods/magic_code_library_jobs.cpp +runtime-recomp/src/game/mods/magic_code_library.cpp +runtime-recomp/src/game/mods/magic_code_link.cpp +runtime-recomp/src/game/mods/magic_code_package.cpp +runtime-recomp/src/game/mods/magic_code_trophy_catalog.cpp +runtime-recomp/src/game/mods/magic_code_trophy_definition.cpp +runtime-recomp/src/game/mods/magic_code_vm.cpp +runtime-recomp/src/game/mods/magic_code_world.cpp +runtime-recomp/src/game/mods/online_mod_bundle.cpp +runtime-recomp/src/game/mods/online_mod_cache.cpp +runtime-recomp/src/game/mods/online_mod_manifest.cpp +runtime-recomp/src/game/mods/online_mod_music.cpp +runtime-recomp/src/game/mods/online_mod_native_prepare.cpp +runtime-recomp/src/game/mods/online_mod_package.cpp +runtime-recomp/src/game/mods/online_mod_prepare.cpp +runtime-recomp/src/game/mods/online_mod_profile.cpp +runtime-recomp/src/game/mods/online_mod_runtime.cpp +runtime-recomp/src/game/mods/online_mod_sync.cpp +runtime-recomp/src/game/mods/online_mod_transfer.cpp +runtime-recomp/src/game/netplay/experimental_magic_codes.cpp +runtime-recomp/src/game/netplay/experimental_owned_registry.cpp +runtime-recomp/src/game/netplay/experimental_process.cpp +runtime-recomp/src/game/netplay/experimental_race_frontend.cpp +runtime-recomp/src/game/netplay/experimental_runtime.cpp +runtime-recomp/src/game/null_renderer.cpp +runtime-recomp/src/game/online_nameplates_rt64.cpp +runtime-recomp/src/game/online_view_rt64.cpp +runtime-recomp/src/game/presentation_identity.cpp +runtime-recomp/src/game/replay_probe_capture.cpp +runtime-recomp/src/game/rev_a_asset_mutex.cpp +runtime-recomp/src/game/rt64_renderer.cpp +runtime-recomp/src/game/runtime_audio_controls.cpp +runtime-recomp/src/game/runtime_code_mods.cpp +runtime-recomp/src/game/runtime_crt_overlay.cpp +runtime-recomp/src/game/runtime_enhancements.cpp +runtime-recomp/src/game/runtime_hud_layout.cpp +runtime-recomp/src/game/runtime_legacy_mods.cpp +runtime-recomp/src/game/runtime_netplay.cpp +runtime-recomp/src/game/runtime_platform.cpp +runtime-recomp/src/game/runtime_portable.cpp +runtime-recomp/src/game/runtime_quick_restart.cpp +runtime-recomp/src/game/runtime_rice_texture_import.cpp +runtime-recomp/src/game/runtime_save_routing.cpp +runtime-recomp/src/game/runtime_stubs.cpp +runtime-recomp/src/game/runtime_support.cpp +runtime-recomp/src/game/runtime_telemetry.cpp +runtime-recomp/src/game/runtime_texture_packs.cpp +runtime-recomp/src/game/runtime_ui.cpp +runtime-recomp/src/game/sdl3_input_client.cpp +runtime-recomp/src/game/updates/update_bootstrap_win.cpp +runtime-recomp/src/game/virtual_pak.cpp +runtime-recomp/src/game/visual_model_kernel.c +runtime-recomp/src/game/water_profile.cpp +runtime-recomp/src/input/sdl3_input_host.cpp +runtime-recomp/src/tools/magic_code_inspect.cpp +runtime-recomp/tests/android_imgui_rotation_tests.cpp +runtime-recomp/tests/android_render_pass_gpu_tests.cpp +runtime-recomp/tests/android_texture_copy_tests.cpp +runtime-recomp/tests/android_transfer_gpu_tests.cpp +runtime-recomp/tests/android_upload_gpu_tests.cpp +runtime-recomp/tests/cmake/test_assertions/assertions.c +runtime-recomp/tests/cmake/test_assertions/assertions.cpp +runtime-recomp/tests/cmake/test_assertions/production.cpp +runtime-recomp/tests/code_mod_handoff_tests.cpp +runtime-recomp/tests/code_mod_native_arguments_tests.cpp +runtime-recomp/tests/custom_music_runtime_tests.cpp +runtime-recomp/tests/custom_tracks_guest_tests.cpp +runtime-recomp/tests/custom_tracks_runtime_tests.cpp +runtime-recomp/tests/custom_tracks_tests.cpp +runtime-recomp/tests/enhanced_water_ray_gpu_tests.cpp +runtime-recomp/tests/experimental_gpu_probe.cpp +runtime-recomp/tests/experimental_interactive_probe.cpp +runtime-recomp/tests/experimental_magic_code_tests.cpp +runtime-recomp/tests/experimental_owned_quick_join_tests.cpp +runtime-recomp/tests/experimental_pak_tests.cpp +runtime-recomp/tests/experimental_process_tests.cpp +runtime-recomp/tests/generated_mips_tests.cpp +runtime-recomp/tests/hud_basic_settings_tests.cpp +runtime-recomp/tests/hud_editor_tests.cpp +runtime-recomp/tests/hud_group_layout_tests.cpp +runtime-recomp/tests/ios_event_queue_tests.cpp +runtime-recomp/tests/launcher_recovery_policy_tests.cpp +runtime-recomp/tests/launcher_software_pixels_tests.cpp +runtime-recomp/tests/legacy_asset_capacity_guest_tests.cpp +runtime-recomp/tests/legacy_behaviour_tests.cpp +runtime-recomp/tests/legacy_cache_capacity_tests.cpp +runtime-recomp/tests/legacy_character_ai_guest_tests.cpp +runtime-recomp/tests/legacy_character_ai_tests.cpp +runtime-recomp/tests/legacy_character_audio_tests.cpp +runtime-recomp/tests/legacy_character_catalog_tests.cpp +runtime-recomp/tests/legacy_character_guest_tests.cpp +runtime-recomp/tests/legacy_character_menu_guest_tests.cpp +runtime-recomp/tests/legacy_character_menu_tests.cpp +runtime-recomp/tests/legacy_character_presentation_guest_tests.cpp +runtime-recomp/tests/legacy_character_presentation_tests.cpp +runtime-recomp/tests/legacy_character_tests.cpp +runtime-recomp/tests/legacy_compatibility_tests.cpp +runtime-recomp/tests/legacy_dkrmap_artwork_tests.cpp +runtime-recomp/tests/legacy_guest_pipeline_tests.cpp +runtime-recomp/tests/legacy_heap_guest_tests.cpp +runtime-recomp/tests/legacy_import_library_tests.cpp +runtime-recomp/tests/legacy_memory_guest_tests.cpp +runtime-recomp/tests/legacy_menu_guest_pipeline_tests.cpp +runtime-recomp/tests/legacy_mod_browser_ui_tests.cpp +runtime-recomp/tests/legacy_mod_corpus_tests.cpp +runtime-recomp/tests/legacy_mod_inspect.cpp +runtime-recomp/tests/legacy_mod_library_tests.cpp +runtime-recomp/tests/legacy_mod_management_tests.cpp +runtime-recomp/tests/legacy_mod_tests.cpp +runtime-recomp/tests/legacy_model_cache_guest_tests.cpp +runtime-recomp/tests/legacy_spawn_asset_type_guest_tests.cpp +runtime-recomp/tests/legacy_track_artifact_tests.cpp +runtime-recomp/tests/legacy_track_catalog_tests.cpp +runtime-recomp/tests/legacy_track_menu_adapter_tests.cpp +runtime-recomp/tests/legacy_track_menu_tests.cpp +runtime-recomp/tests/live_lod_tests.cpp +runtime-recomp/tests/memory_probe_test_entries.c +runtime-recomp/tests/mipmap_loading_tests.cpp +runtime-recomp/tests/mobile_ui_tests.cpp +runtime-recomp/tests/native_mod_world_test_boundaries.cpp +runtime-recomp/tests/native_mod_world_tests.cpp +runtime-recomp/tests/offline_mod_test_boundaries.cpp +runtime-recomp/tests/online_game_pak_tests.cpp +runtime-recomp/tests/online_mod_manifest_tests.cpp +runtime-recomp/tests/online_mod_music_tests.cpp +runtime-recomp/tests/online_mod_profile_tests.cpp +runtime-recomp/tests/online_mod_sync_tests.cpp +runtime-recomp/tests/online_mod_transfer_tests.cpp +runtime-recomp/tests/optimisation_microbench.cpp +runtime-recomp/tests/owned_mod_boundary_guest.c +runtime-recomp/tests/owned_mod_boundary_tests.cpp +runtime-recomp/tests/owned_revision_registry_tests.cpp +runtime-recomp/tests/replay_probe/checked_memory_tests.cpp +runtime-recomp/tests/replay_probe/probe_audio_events.c +runtime-recomp/tests/replay_probe/probe_audio_link_canary.cpp +runtime-recomp/tests/replay_probe/probe_audio_rsp.cpp +runtime-recomp/tests/replay_probe/probe_audio.c +runtime-recomp/tests/replay_probe/probe_audio.cpp +runtime-recomp/tests/replay_probe/probe_component.c +runtime-recomp/tests/replay_probe/probe_driver.cpp +runtime-recomp/tests/replay_probe/probe_eeprom.cpp +runtime-recomp/tests/replay_probe/probe_input.c +runtime-recomp/tests/replay_probe/probe_input.cpp +runtime-recomp/tests/replay_probe/probe_link_canary.c +runtime-recomp/tests/replay_probe/probe_magic_tests.cpp +runtime-recomp/tests/replay_probe/probe_magic.c +runtime-recomp/tests/replay_probe/probe_magic.cpp +runtime-recomp/tests/replay_probe/probe_main.cpp +runtime-recomp/tests/replay_probe/probe_native.c +runtime-recomp/tests/replay_probe/probe_pak_retail.c +runtime-recomp/tests/replay_probe/probe_pak_retail.cpp +runtime-recomp/tests/replay_probe/probe_pak_tests.cpp +runtime-recomp/tests/replay_probe/probe_pak.c +runtime-recomp/tests/replay_probe/probe_pak.cpp +runtime-recomp/tests/replay_probe/probe_policies.cpp +runtime-recomp/tests/replay_probe/probe_presentation.c +runtime-recomp/tests/replay_probe/probe_save.c +runtime-recomp/tests/replay_probe/probe_save.cpp +runtime-recomp/tests/sdl3_input_client_tests.cpp +runtime-recomp/tests/split_screen_rt64_tests.cpp +runtime-recomp/tests/track_select_native_draw_tests.cpp +runtime-recomp/tests/water_uv_rt64_tests.cpp diff --git a/.github/workflows/tooling.yml b/.github/workflows/tooling.yml new file mode 100644 index 00000000..6759fd4c --- /dev/null +++ b/.github/workflows/tooling.yml @@ -0,0 +1,169 @@ +name: Lint and static analysis +on: + push: + branches: [main] + pull_request: + # labeled/unlabeled rerun the guard when transition-layer is added or removed. + types: [opened, synchronize, reopened, labeled, unlabeled] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + # Required check. Every step that fails here is green on main today; checks + # that still need the B9 autofix commit run in the advisory step below. + lint: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 # the diff-mode guard needs the merge base + - name: Install verified lint tools + run: | + tool_dir="$RUNNER_TEMP/lint-tools" + python3 -m venv "$tool_dir" + "$tool_dir/bin/python" -m pip install --quiet \ + --require-hashes --only-binary=:all: -r .github/lint-tools.txt + curl --fail --silent --show-error --location \ + https://github.com/mvdan/sh/releases/download/v3.14.1/shfmt_v3.14.1_linux_amd64 \ + --output "$tool_dir/bin/shfmt" + echo "76e77641faa025814b77f153b29796b8e6fa2fca03e0c76a691608b86c7ea7bf $tool_dir/bin/shfmt" | sha256sum --check + chmod +x "$tool_dir/bin/shfmt" + echo "$tool_dir/bin" >>"$GITHUB_PATH" + - name: Ruff (E, F, W, I with the pyproject.toml baseline) + run: ruff check --output-format=github + - name: Ruff format + run: ruff format --check + # Same scope as the pre-commit shfmt hook. No flags, so shfmt reads the + # [*.sh] section of .editorconfig. + - name: shfmt + run: git ls-files -z '*.sh' ':(exclude)patches/' ':(exclude)extern/' ':(exclude)tests/fixtures/' | xargs -0 shfmt -d + - name: Pattern guard and tooling tests + run: python3 scripts/check.py --suite guard,tooling + # The tree check above can't tell new baseline entries from old ones, so + # PRs also run the guard against their base. Baseline growth is only + # allowed with the transition-layer label, until fold-in-base is tagged. + - name: Pattern guard against the PR base + if: github.event_name == 'pull_request' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + TRANSITION_LAYER: ${{ contains(github.event.pull_request.labels.*.name, 'transition-layer') }} + run: | + args=(--diff "$BASE_SHA") + if [[ "$TRANSITION_LAYER" == true ]]; then args+=(--transition-layer); fi + python3 scripts/pattern_guard.py "${args[@]}" + # Report only until their baseline commits land: B9 for C/C++ and CMake, + # and shellcheck once the remaining warnings are fixed by hand. Each line + # moves into a blocking step, or into check.py --suite format, once green. + - name: Format drift report (advisory) + continue-on-error: true + run: | + status=0 + git ls-files -z '*.sh' ':(exclude)patches/' | xargs -0 shellcheck || status=1 + git ls-files -z 'CMakeLists.txt' '*/CMakeLists.txt' '*.cmake' ':(exclude)extern/' ':(exclude)patches/' | + xargs -0 gersemi --check || status=1 + exit "$status" + + # Advisory: not a required check for the first 4 weeks. Findings on changed + # lines show up as PR annotations; WarningsAsErrors hits and changed TUs with + # no compile command fail the job. + tidy: + if: github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + CCACHE_DIR: ${{ github.workspace }}/build/ccache + CCACHE_MAXSIZE: 500M + CCACHE_COMPILERCHECK: content + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - name: Find changed first-party C/C++ + id: changes + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + merge_base="$(git merge-base "$BASE_SHA" HEAD)" + echo "merge-base=$merge_base" >>"$GITHUB_OUTPUT" + git diff --name-only --diff-filter=ACMR "$merge_base" HEAD -- \ + runtime-recomp/src runtime-recomp/tests modding \ + ':(exclude)runtime-recomp/src/game/generated' | + grep -E '\.(c|cc|cpp|h|hpp|inl)$' >"$RUNNER_TEMP/changed.txt" || true + count="$(wc -l <"$RUNNER_TEMP/changed.txt")" + echo "count=$count" >>"$GITHUB_OUTPUT" + echo "$count changed first-party C/C++ files" + - name: Install LLVM 18 and build tools + if: steps.changes.outputs.count != '0' + run: sudo apt-get update && sudo apt-get install -y clang-18 clang-tidy-18 cmake ninja-build ccache libcurl4-openssl-dev + - name: Prepare pinned runtime dependencies + if: steps.changes.outputs.count != '0' + uses: ./.github/actions/prepare-dependencies # zizmor: ignore[self-repository] -- actionlint 1.7.12 compatibility; checkout uses this workflow revision. + - name: Restore compiler cache + if: steps.changes.outputs.count != '0' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: build/ccache + key: ccache-tidy-v1-clang-18-${{ github.sha }} + restore-keys: ccache-tidy-v1-clang-18- + # Generated headers come from the build, so tidy needs a built tree, not + # just a configured one. + - name: Configure and build host preset with clang-18 + if: steps.changes.outputs.count != '0' + working-directory: runtime-recomp + run: | + cmake --preset host -DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + cmake --build --preset host --parallel + - name: Save compiler cache + if: ${{ success() && steps.changes.outputs.count != '0' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: build/ccache + key: ccache-tidy-v1-clang-18-${{ github.sha }} + - name: Changed translation units must have a compile command + if: steps.changes.outputs.count != '0' + run: | + python3 - "$RUNNER_TEMP/changed.txt" <<'EOF' + import json, os, sys + root = os.getcwd() + database = json.load(open("build/host/compile_commands.json")) + compiled = {os.path.relpath(os.path.realpath(os.path.join(e["directory"], e["file"])), root) for e in database} + skipped = {line.strip() for line in open(".github/tidy-skip.txt") if line.strip() and not line.startswith("#")} + changed = [line.strip() for line in open(sys.argv[1]) if line.strip().endswith((".c", ".cc", ".cpp"))] + missing = [path for path in changed if path not in compiled and path not in skipped] + for path in missing: + print(f"::error file={path}::not built by the host preset, so clang-tidy cannot check it; build it in host or add it to .github/tidy-skip.txt") + sys.exit(1 if missing else 0) + EOF + - name: clang-tidy on changed lines + if: steps.changes.outputs.count != '0' + env: + MERGE_BASE: ${{ steps.changes.outputs.merge-base }} + run: | + mapfile -t changed <"$RUNNER_TEMP/changed.txt" + set +e + git diff -U0 "$MERGE_BASE" HEAD -- "${changed[@]}" | + python3 /usr/lib/llvm-18/share/clang/clang-tidy-diff.py -p1 -path build/host \ + -clang-tidy-binary clang-tidy-18 -iregex '.*\.(c|cc|cpp|h|hpp)' -j 4 -quiet >"$RUNNER_TEMP/tidy.log" 2>&1 + status=$? + set -e + sed -nE "s#^($PWD/)?([^:]+):([0-9]+):([0-9]+): (warning|error): (.*)\$#::\5 file=\2,line=\3,col=\4::\6#p" "$RUNNER_TEMP/tidy.log" + { + echo '### clang-tidy (changed lines, advisory)' + echo '```' + grep -E ': (warning|error): ' "$RUNNER_TEMP/tidy.log" | sed "s|^$PWD/||" || echo 'No findings.' + echo '```' + } >>"$GITHUB_STEP_SUMMARY" + exit "$status" diff --git a/.gitignore b/.gitignore index 18aa4284..183b0e42 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,8 @@ runtime-recomp/build/ # Local Cloudflare development state /.wrangler/ + +# AI coding agents: personal permissions and per-session hook state +/.claude/settings.local.json +/.claude/session-base +/.claude/session-base.*.tmp diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..70bbd974 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,112 @@ +# Local hooks for fast feedback. CI (tooling.yml `lint`) runs the same checks +# and is the gate; hooks are a convenience. Install: python3 scripts/bootstrap_hooks.py +# +# Every rev below was checked against that repo's .pre-commit-hooks.yaml at the +# tag, and matches the versions in .github/lint-tools.txt, tooling.yml (shfmt) +# and workflow-lint.yml (actionlint, zizmor). Bump them together. +default_install_hook_types: [pre-commit, pre-push] +default_stages: [pre-commit] +# Pinned-byte files: *.patch sha256s live in patches/manifest.json (141 of 226 +# have trailing whitespace on blank context lines), release-prep source capsules +# are content-addressed, modding/reference-us-v*/* are byte-exact receipts, and +# the 14 CRLF-pinned files all live under patches/. No fixer may touch them; +# the pinned-bytes hook below proves it on every commit. +# The 1.0.7 capsule also pins its first-party "base" files (all of them under +# runtime-recomp/src/ and runtime-recomp/tests/replay_probe/); those two +# entries go when the fold deletes the capsule, before C/C++ formatting starts. +# tests/fixtures/ are pattern-guard inputs and packaging/licenses/ are verbatim +# third-party texts. +exclude: | + (?x)^( + extern/| + runtime-recomp/RecompiledFuncs/| + runtime-recomp/RecompiledPatches/| + runtime-recomp/RecompiledRSP/| + runtime-recomp/third_party/| + runtime-recomp/src/| + runtime-recomp/tests/replay_probe/| + build/| + patches/| + modding/reference-us-v| + tests/fixtures/| + packaging/licenses/| + .*\.patch$ + ) +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: check-added-large-files + args: [--maxkb=512] + - id: check-merge-conflict + - id: check-symlinks + # CRLF .cmd/.bat/.ps1 per .gitattributes. C/C++/CMake and + # runtime-recomp/src wait for the B9 format commit. + - id: end-of-file-fixer + exclude: ^runtime-recomp/src/ + exclude_types: [batch, c, c++, cmake] + - id: trailing-whitespace + exclude: ^runtime-recomp/src/|\.(md|ps1)$ + exclude_types: [batch, c, c++, cmake] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.17.2 + hooks: + - id: ruff-check + args: [--fix] + # pyproject.toml excludes *.md from formatting. + - id: ruff-format + - repo: https://github.com/scop/pre-commit-shfmt + rev: v3.14.1-1 + hooks: + # No flags, so shfmt reads the [*.sh] section of .editorconfig. Limited to + # *.sh to match CI; extensionless scripts keep their own style. + - id: shfmt + files: \.sh$ + - repo: https://github.com/rhysd/actionlint + rev: v1.7.12 + hooks: + - id: actionlint + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.30.1 + hooks: + - id: zizmor + args: [--offline, --no-progress] + # Advisory until their baseline commits land: run with + # `pre-commit run --hook-stage manual --all-files`. + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.11.0.1 + hooks: + # Two SC2155 warnings remain; fixing them changes exit-status behaviour. + - id: shellcheck + args: [-x, --severity=warning] + files: \.sh$ + stages: [manual] + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v18.1.8 + hooks: + # LLVM 18, the same major as clang-tidy in tooling.yml. Blocking after B9. + - id: clang-format + types_or: [c, c++] + stages: [manual] + - repo: https://github.com/BlankSpruce/gersemi-pre-commit + rev: 0.29.2 + hooks: + # Blocking after B9. + - id: gersemi + stages: [manual] + - repo: local + hooks: + - id: pattern-guard + name: pattern guard (no overlay layers against first-party code) + entry: python scripts/pattern_guard.py --staged + language: python + pass_filenames: false + always_run: true + # Runs on every commit, not just ones touching pinned files: the files are + # excluded above, so pre-commit would never pass them to the hook. + - id: pinned-bytes + name: checksum-pinned files untouched by fixers + entry: python scripts/check_pinned_bytes.py + language: python + pass_filenames: false + always_run: true diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..b544799d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,74 @@ +# Working in DKR-R + +Guidance for AI coding agents (Codex, Claude Code, Cursor, Copilot and others). +`CONTRIBUTING.md` says the same things for people; this file is the short, +strict version. + +## The pattern (non-negotiable) + +- Edit first-party code in place: `runtime-recomp/` (including `runtime-recomp/cmake/`), `modding/`, `scripts/`, `docs/`. +- Never create `patches//` for first-party changes. `patches/` holds only third-party patch sets registered in + `patches/manifest.json` against `extern/`. No `*.cmake`, no `*.py`, no `native/` under `patches/`. +- Never stage, copy, string-replace or swap sources at configure or build time. If you find yourself writing + `pipeline.py`, `transforms.py`, `once()`, `region()` or `set_target_properties(... SOURCES ...)`, stop. +- Generated guest code is hooked only through `runtime-recomp/guest-hooks/hooks.toml` (applied by + `scripts/guest_hooks.py`). Never rewrite `RecompiledFuncs`, `funcs_*.c` or `guest_*.c` from a script. +- Symbol tables come from `runtime-recomp/symbols/*.toml` (generated by `scripts/gen_guest_symbols.py`). Don't + hard-code addresses. +- Online or simulation changes that break cross-play need a deliberate edit to + `runtime-recomp/src/game/netplay/legacy_identity.hpp`, approved by both owners. Never splice a digest into an + identity string anywhere else. +- Third-party fixes: a `*.patch` under `patches//` plus a `manifest.json` entry with its sha256. Never vendor a + modified translation unit. + +> Transition note: the fold is in progress. Through v1.0.11 the release was +> built from a chain of first-party layers under `patches/`, and those +> directories and their pipelines still exist. They are listed in +> `architecture/pattern-guard-baseline.txt`; the guard tolerates them and fails +> on anything new. Editing a listed file is fine, adding a new layer is not. +> `guest-hooks/hooks.toml`, `symbols/*.toml`, `scripts/guest_hooks.py`, +> `scripts/gen_guest_symbols.py` and `netplay/legacy_identity.hpp` arrive with +> later fold steps. Until a path exists, don't invent a stand-in for it; ask. + +## How to add a feature + +1. Pick or create a module directory under `runtime-recomp/src/game//` (see `docs/ARCHITECTURE.md`). +2. Register settings, pages and owned services in the registries instead of patching literals. +3. Add tests next to the existing ones in `runtime-recomp/tests/` (flat, named `*_tests.py` for tooling tests). +4. Run the checks below before opening a PR. CI runs the same. + +## Verification you must run + +- `python3 scripts/check.py --suite guard` before claiming any task done. This is mandatory for every agent, + with or without hooks. +- `python3 scripts/check.py --suite guard,tooling` after changing scripts, CMake or tests. +- `pre-commit run --files ` for format and lint, if the git hooks are installed. +- `cmake --preset host -S runtime-recomp && cmake --build --preset host && ctest --preset host` before claiming a + C/C++ change done (`CMakePresets.json` lives in `runtime-recomp/`, so pass `-S` or `cd` there). + +On Windows use `py -3` or `python` in place of `python3`. + +## Agent hooks + +Claude Code (`.claude/settings.json`) and Codex (`.codex/hooks.json`) run the same scripts from +`scripts/agent_hooks/`: the pattern summary at session start, a pre-edit pattern guard that blocks a violating +write, a post-edit lint report that never rewrites your files, and a stop check that refuses to finish while the +session has added a guard problem. Codex loads project hooks only once the project is trusted and the hooks are +reviewed in `/hooks`. The hooks are fast feedback; the git pre-commit hook and the required CI checks are the gates. +Don't disable or route around any of them. + +## Things that are not source + +- Receipts, audits, "contracts" as file hashes, per-feature `package.py`. Release notes go in `CHANGELOG.md`. +- Playtest and investigation notes go in `docs/notes/-.md`. They are welcome there; they are not build + inputs. + +## Pinned bytes + +Some files must keep their exact bytes; `scripts/check_pinned_bytes.py` (a pre-commit hook) fails if they change. +Never reformat or "tidy" them: + +- `*.patch` files listed in `patches/manifest.json`, and `modding/reference-us-v*/`. +- Until the fold deletes the 1.0.7 source capsule: everything under `runtime-recomp/src/` and + `runtime-recomp/tests/replay_probe/` that the capsule pins as a base. Real code changes there are fine; formatting + passes and whitespace fixes are not. diff --git a/Build-Linux.sh b/Build-Linux.sh index afeb6dc9..cdcacf1b 100755 --- a/Build-Linux.sh +++ b/Build-Linux.sh @@ -38,6 +38,9 @@ for argument in "$@"; do esac done +# Prints the hook install command if the hooks are missing; never fails Build. +if [[ -f "${project_root}/scripts/bootstrap_hooks.py" ]]; then python3 "${project_root}/scripts/bootstrap_hooks.py" --check || true; fi + [[ -d "${generated_source}" ]] || { echo "US v1.0/v77 Patch Pipeline output is missing: ${generated_source}" >&2 exit 1 diff --git a/Build-Windows.cmd b/Build-Windows.cmd index e593880e..2e68c995 100644 --- a/Build-Windows.cmd +++ b/Build-Windows.cmd @@ -2,6 +2,11 @@ setlocal cd /d "%~dp0" +rem Prints the hook install command if the hooks are missing; never fails Build. +if exist "%~dp0scripts\bootstrap_hooks.py" ( + py -3 "%~dp0scripts\bootstrap_hooks.py" --check 2>nul || python "%~dp0scripts\bootstrap_hooks.py" --check 2>nul +) + powershell.exe -NoLogo -NoProfile -ExecutionPolicy Bypass -File "%~dp0scripts\Build-DKR-R-Windows.ps1" %* set "RESULT=%ERRORLEVEL%" diff --git a/Build-macOS.sh b/Build-macOS.sh index 39d8500a..9fb45402 100755 --- a/Build-macOS.sh +++ b/Build-macOS.sh @@ -66,6 +66,8 @@ EOF esac shift done +# Prints the hook install command if the hooks are missing; never fails Build. +if [[ -f "${project_root}/scripts/bootstrap_hooks.py" ]]; then python3 "${project_root}/scripts/bootstrap_hooks.py" --check || true; fi jobs="$(dkr_apple_build_jobs "$jobs")" export CMAKE_BUILD_PARALLEL_LEVEL="$jobs" dkr_require_macos diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..bc6fd192 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,3 @@ +Read AGENTS.md first; it is the authoritative project guidance. +Project hooks in `.claude/settings.json` enforce it. +Use `scripts/check.py` (`python3 scripts/check.py --suite guard` before claiming done). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..9fc7413e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,112 @@ +# Contributing to DKR-R + +## The pattern + +First-party code lives in `runtime-recomp/`, `modding/` and `scripts/` and is +edited in place. Third-party code is changed only through `patches/manifest.json` +patch sets against `extern/`. Nothing stages, copies, string-replaces or swaps +first-party sources at configure time. Guest hooks go in +`guest-hooks/hooks.toml`, symbol tables in `symbols/*.toml`, and online +identity lives in `netplay/legacy_identity.hpp` and changes only by deliberate +edit. + +The pattern guard (`scripts/pattern_guard.py`), the local hooks and the +required CI checks all enforce this, so a change that breaks it fails before it +can merge. + +> Transition note: until the patch-layer fold lands, `patches//` +> directories and the configure-time pipelines still exist on `main`. They are +> listed in `architecture/pattern-guard-baseline.txt` and the guard only fails +> on new ones. Don't add a new layer; edit the source in place instead. + +## Local setup + +You need Git and Python 3.10 or newer. `Setup-macOS.sh`, `Setup-Linux.sh` and +`Setup-Windows.cmd` install the git hooks after the prerequisite checks (pass +`--skip-hooks` to opt out), or run `python3 scripts/bootstrap_hooks.py` +directly. With a global `core.hooksPath`, git won't run the repository's hooks, +and the bootstrap prints how to opt this checkout in. Then: + +```sh +python3 scripts/check.py # pattern guard, tooling, metadata and asset checks +python3 scripts/check.py --suite all # everything CI runs that needs no ROM +``` + +On Windows use `python` in place of `python3`. See `docs/DEVELOPMENT.md` for +host builds and tests. + +## Lint and format + +| Area | Tool | Config | +|---|---|---| +| Python | ruff (`E`, `F`, `W`, `I`) | `pyproject.toml` | +| Python types | pyright, strict on new tooling scripts | `pyproject.toml` | +| C/C++ format | clang-format 18 | `.clang-format` | +| C/C++ lint | clang-tidy 18 | `.clang-tidy` | +| C/C++ warnings | `dkr_enable_warnings()` | `runtime-recomp/cmake/Warnings.cmake` | +| CMake | gersemi | `.gersemirc` | +| Shell | shfmt, shellcheck | `.editorconfig` | +| Workflows | actionlint, zizmor | `.github/workflows/` | + +clang-format, clang-tidy and the sanitizer build all use LLVM 18. Bump them +together or not at all, since two majors format the same file differently. + +Existing code is grandfathered, not exempt. `pyproject.toml` carries a +generated per-file baseline of the ruff rules each existing file still trips. +New files get no entry, and the baseline only shrinks. Don't hand-edit it to +make a new finding go away; fix the code. clang-tidy runs on the lines you +changed, so untouched legacy code doesn't block you. + +## Adding a feature + +1. Put the code in a module directory under `runtime-recomp/src/game/`, as a + CMake library. +2. Add the include edges it needs to `architecture/deps.toml`. Anything not + listed is forbidden. +3. Register settings, launcher pages and owned services in their registries + instead of patching literals. +4. Add tests under `runtime-recomp/tests/`. +5. Fill in the PR template, including the platforms you built. + +## Fixing a dependency + +1. Edit the checkout under `extern//`. +2. `git diff` the change into a `*.patch` file under `patches/`. +3. Register it in `patches/manifest.json` with its sha256. +4. Run `python3 scripts/apply_dependency_patches.py --check`. + +## Changing online behaviour + +Anything that changes simulation, the wire format or an owned service changes +online compatibility. Bump the identity in `netplay/legacy_identity.hpp` in the +same PR. Both owners review that file (see `.github/CODEOWNERS`). + +## What happened to `patches/` + +Through v1.0.11, first-party features were shipped as patch layers that the +build composed at configure time. The fold wrote the shipped bytes back into +the source tree and deleted the layers; git history keeps them, and the ADRs in +`docs/adr/` record the decisions that were only written down in layer notes. + +## Review and merging + +`main` and `release/*` are covered by a repository ruleset; the intended +settings are in `.github/ruleset-main.json`: + +- Changes go through a pull request. The repository admin can bypass this for a + hotfix push; the ruleset log records it. +- The required checks are `lint`, `linux`, `macos (macos-15)`, + `macos (macos-15-intel)`, `windows`, `asset-scan`, `dependency-lock`, + `tooling` and `workflow-lint`. Skipping local hooks with `--no-verify` + changes nothing about mergeability. +- No approval is required for ordinary code. Files listed in + `.github/CODEOWNERS` (the rule files) need a review from the other owner. +- Force-pushes and branch deletion are blocked. + +`tidy` is advisory for its first four weeks: it annotates findings on changed +lines and doesn't block merging. + +## Release + +Releases are built by the `release` workflow from a tag; see +`docs/RELEASING.md`. diff --git a/Setup-Linux.sh b/Setup-Linux.sh index 4ce66a50..4d9e03f9 100755 --- a/Setup-Linux.sh +++ b/Setup-Linux.sh @@ -1,5 +1,20 @@ #!/usr/bin/env bash set -euo pipefail +project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +skip_hooks=0 +for argument in "$@"; do + case "$argument" in + --skip-hooks) skip_hooks=1 ;; + -h | --help) + echo 'Usage: ./Setup-Linux.sh [--skip-hooks] (check prerequisites, then install git hooks unless --skip-hooks)' + exit 0 + ;; + *) + echo "Unknown option: $argument" >&2 + exit 1 + ;; + esac +done missing=0 for tool in cmake ninja c++ git pkg-config python3; do @@ -31,4 +46,8 @@ fi cmake --version | sed -n '1p' c++ --version | sed -n '1p' pkg-config --modversion sdl2 gtk+-3.0 vulkan +# Never fails Setup: bootstrap_hooks.py reports problems and exits 0. +if [[ "${skip_hooks}" -eq 0 && -f "${project_root}/scripts/bootstrap_hooks.py" ]]; then + python3 "${project_root}/scripts/bootstrap_hooks.py" || true +fi echo 'Linux runtime prerequisites passed. Prepare generated code, then run ./Build-Linux.sh.' diff --git a/Setup-Windows.cmd b/Setup-Windows.cmd new file mode 100644 index 00000000..f417cc1a --- /dev/null +++ b/Setup-Windows.cmd @@ -0,0 +1,54 @@ +@echo off +setlocal +cd /d "%~dp0" +rem Check the Windows build prerequisites listed in docs\BUILDING.md, then +rem install the git hooks. Setup-Windows.cmd --skip-hooks leaves hooks alone. + +set "SKIP_HOOKS=0" +for %%A in (%*) do ( + if /i "%%~A"=="--skip-hooks" ( + set "SKIP_HOOKS=1" + ) else if /i "%%~A"=="--help" ( + echo Usage: Setup-Windows.cmd [--skip-hooks] ^(check prerequisites, then install git hooks unless --skip-hooks^) + exit /b 0 + ) else ( + echo Unknown option: %%~A + exit /b 1 + ) +) + +set "MISSING=0" +where git >nul 2>nul || (echo Missing required tool: Git & set "MISSING=1") +where powershell.exe >nul 2>nul || (echo Missing required tool: PowerShell & set "MISSING=1") +where wsl.exe >nul 2>nul || (echo Missing required tool: WSL2 ^(used by the preparation workflow^) & set "MISSING=1") + +rem "python" is often the Microsoft Store stub, so prove the interpreter runs. +set "PYTHON=" +py -3 -c "import sys" >nul 2>nul && set "PYTHON=py -3" +rem Keep the probe and the assignment inside the block: on one line, "&& set" +rem would bind to the whole "if" and replace a working "py -3". +if not defined PYTHON ( + python -c "import sys" >nul 2>nul && set "PYTHON=python" +) +if not defined PYTHON (echo Missing required tool: Python 3 & set "MISSING=1") + +set "VSWHERE=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" +set "VS=" +if exist "%VSWHERE%" ( + for /f "usebackq delims=" %%I in (`"%VSWHERE%" -version [17.0^,18.0^) -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do set "VS=%%I" +) +if not defined VS (echo Missing required tool: Visual Studio 2022 with Desktop development with C++ & set "MISSING=1") + +if not "%MISSING%"=="0" ( + echo. + echo Install the missing prerequisites, then rerun Setup-Windows.cmd. + if not "%DKRPORT_NO_PAUSE%"=="1" pause + exit /b 1 +) + +rem Never fails Setup: bootstrap_hooks.py reports problems and exits 0. +if "%SKIP_HOOKS%"=="0" %PYTHON% "%~dp0scripts\bootstrap_hooks.py" + +echo Windows prerequisites passed. docs\BUILDING.md covers preparing and building. +if not "%DKRPORT_NO_PAUSE%"=="1" pause +exit /b 0 diff --git a/Setup-macOS.sh b/Setup-macOS.sh index d873c7e0..3416e0d8 100755 --- a/Setup-macOS.sh +++ b/Setup-macOS.sh @@ -2,22 +2,25 @@ set -euo pipefail project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" skip_deps=0 -case "${1:-}" in -'') ;; ---skip-deps) skip_deps=1 ;; --h | --help) - echo 'Usage: ./Setup-macOS.sh [--skip-deps] (install prerequisites, or only check them)' - exit 0 - ;; -*) - echo "Unknown option: $1" >&2 - exit 1 - ;; -esac -[[ $# -le 1 ]] || { - echo 'Too many arguments.' >&2 - exit 1 -} +skip_hooks=0 +for argument in "$@"; do + case "$argument" in + --skip-deps) skip_deps=1 ;; + --skip-hooks) skip_hooks=1 ;; + -h | --help) + echo 'Usage: ./Setup-macOS.sh [--skip-deps] [--skip-hooks] (install prerequisites, or only check them; --skip-hooks leaves git hooks alone)' + exit 0 + ;; + *) + echo "Unknown option: $argument" >&2 + exit 1 + ;; + esac +done source "$project_root/scripts/macos-host-tools.sh" dkr_macos_host_tools "$skip_deps" +# Never fails Setup: bootstrap_hooks.py reports problems and exits 0. +if [[ "$skip_hooks" -eq 0 && -f "$project_root/scripts/bootstrap_hooks.py" ]]; then + python3 "$project_root/scripts/bootstrap_hooks.py" || true +fi echo 'macOS prerequisites passed (same checks as Build preparation). Run ./Build-macOS.sh --rom-v77 "/path/to/US 1.0.z64" --rom-v80 "/path/to/US Rev A.z64".' diff --git a/architecture/pattern-guard-baseline.txt b/architecture/pattern-guard-baseline.txt new file mode 100644 index 00000000..eb348e43 --- /dev/null +++ b/architecture/pattern-guard-baseline.txt @@ -0,0 +1,1270 @@ +# Pattern guard baseline: first-party patch-layer mechanisms that predate the +# guard. One " " line per finding, sorted, enumerated (never a glob). +# +# This file only ever shrinks. scripts/pattern_guard.py fails on a finding that +# is not listed here, and on a line here whose file no longer produces it. New +# lines are accepted only on PRs labelled transition-layer, until fold-in-base +# is tagged; the fold drives this file to empty. +# +# Regenerate after removing hits: python3 scripts/pattern_guard.py --tree --baseline write +# Explain a rule: python3 scripts/pattern_guard.py --explain PG001 + +patches/achievements/ACHIEVEMENTS.txt PG013 +patches/achievements/PIPELINE.txt PG013 +patches/achievements/ReleaseOverlay.cmake PG001 +patches/achievements/ReleaseOverlay.cmake PG002 +patches/achievements/ReleaseOverlay.cmake PG003 +patches/achievements/audit.py PG013 +patches/achievements/configure.py PG013 +patches/achievements/inspect_rom.py PG013 +patches/achievements/native/achievement_catalogue.cpp PG006 +patches/achievements/native/achievement_ui.inl PG006 +patches/achievements/native/achievements.cpp PG006 +patches/achievements/native/achievements.hpp PG006 +patches/achievements/package.py PG013 +patches/achievements/pipeline.py PG004 +patches/all-player-detail/PIPELINE.txt PG013 +patches/all-player-detail/PLAYTEST.txt PG013 +patches/all-player-detail/ReleaseOverlay.cmake PG001 +patches/all-player-detail/ReleaseOverlay.cmake PG002 +patches/all-player-detail/ReleaseOverlay.cmake PG003 +patches/all-player-detail/ReleaseOverlay.cmake PG010 +patches/all-player-detail/audit.py PG013 +patches/all-player-detail/configure.py PG013 +patches/all-player-detail/native/detail_contract.hpp PG006 +patches/all-player-detail/native/detail_observer.inl PG006 +patches/all-player-detail/package.py PG013 +patches/all-player-detail/pipeline.py PG004 +patches/android/README.md PG013 +patches/android/background-shader-priority.patch PG013 +patches/android/buffer-small-upload.patch PG013 +patches/android/buffer-worker-lifecycle.patch PG013 +patches/android/frame-budget.patch PG013 +patches/android/inspector-prerotation.patch PG013 +patches/android/present-surface-ownership.patch PG013 +patches/android/render-idle-priority.patch PG013 +patches/android/render-pass-retention.patch PG013 +patches/android/render-stage-timing.patch PG013 +patches/android/renderer-transfer-qualification.patch PG013 +patches/android/rt64-rgba-presentation.patch PG013 +patches/android/sdl2-vulkan-surface-lifecycle.patch PG013 +patches/android/texture-memory-pressure.patch PG013 +patches/android/vi-prerotation.patch PG013 +patches/android/video-library-prerotation.patch PG013 +patches/android/vulkan-capability-preflight.patch PG013 +patches/android/vulkan-copy-contract.patch PG013 +patches/android/vulkan-descriptor-safety.patch PG013 +patches/android/vulkan-mapped-memory.patch PG013 +patches/android/vulkan-pass-retention.patch PG013 +patches/android/vulkan-pipeline-cache.patch PG013 +patches/android/vulkan-prerotation.patch PG013 +patches/android/vulkan-read-barriers.patch PG013 +patches/android/vulkan-resource-failures.patch PG013 +patches/android/vulkan-surface-capabilities.patch PG013 +patches/android/vulkan-surface-recovery.patch PG013 +patches/camera-plus/CMakeLists.txt PG001 +patches/camera-plus/IMPLEMENTATION.md PG013 +patches/camera-plus/NativeOverlay.cmake PG001 +patches/camera-plus/NativeOverlay.cmake PG002 +patches/camera-plus/OpenXR.cmake PG001 +patches/camera-plus/README.md PG013 +patches/camera-plus/SceneKernel.cmake PG001 +patches/camera-plus/camera_editor_ui.inl PG006 +patches/camera-plus/inspect_scene_closure.py PG013 +patches/camera-plus/integration_pipeline.py PG013 +patches/camera-plus/mod/dkr.json PG013 +patches/camera-plus/mod/main.c PG006 +patches/camera-plus/mod/settings.json PG013 +patches/camera-plus/native/camera_actions.cpp PG006 +patches/camera-plus/native/camera_actions.hpp PG006 +patches/camera-plus/native/camera_kernel.c PG006 +patches/camera-plus/native/camera_kernel.h PG006 +patches/camera-plus/native/camera_kernel_guest.h PG006 +patches/camera-plus/native/camera_presentation.cpp PG006 +patches/camera-plus/native/camera_presentation.hpp PG006 +patches/camera-plus/native/camera_rt64.cpp PG006 +patches/camera-plus/native/camera_rt64.hpp PG006 +patches/camera-plus/native/camera_runtime.cpp PG006 +patches/camera-plus/native/camera_runtime.hpp PG006 +patches/camera-plus/native/camera_scene.cpp PG006 +patches/camera-plus/native/camera_scene.hpp PG006 +patches/camera-plus/native/camera_session.cpp PG006 +patches/camera-plus/native/camera_session.hpp PG006 +patches/camera-plus/native/camera_settings.cpp PG006 +patches/camera-plus/native/camera_settings.hpp PG006 +patches/camera-plus/native/camera_snapshot.cpp PG006 +patches/camera-plus/native/camera_snapshot.hpp PG006 +patches/camera-plus/native/camera_xr.cpp PG006 +patches/camera-plus/native/camera_xr.hpp PG006 +patches/camera-plus/package_playtest.py PG013 +patches/camera-plus/scene_kernel_pipeline.py PG013 +patches/camera-plus/sdk/dkr/presentation.h PG006 +patches/camera-plus/vr_device_patch.py PG013 +patches/camera-shadow/NativeOverlay.cmake PG001 +patches/camera-shadow/NativeOverlay.cmake PG002 +patches/camera-shadow/NativeOverlay.cmake PG003 +patches/camera-shadow/RELEASE-NOTES.txt PG013 +patches/camera-shadow/camera_shadow_kernel.c PG006 +patches/camera-shadow/camera_shadow_kernel.h PG006 +patches/camera-shadow/package.py PG013 +patches/camera-shadow/pipeline.py PG013 +patches/camera-shadow/shadow_pass.inl PG006 +patches/character-portraits/NativeOverlay.cmake PG001 +patches/character-portraits/NativeOverlay.cmake PG002 +patches/character-portraits/README.md PG013 +patches/character-portraits/inspect_packages.py PG013 +patches/character-portraits/inspect_sources.py PG013 +patches/character-portraits/load_reviewed_portrait.inc PG006 +patches/character-portraits/package-windows.ps1 PG013 +patches/character-portraits/world_portrait.inc PG006 +patches/checkpoint-clock-repair/IMPLEMENTATION.txt PG013 +patches/checkpoint-clock-repair/PLAYTEST.txt PG013 +patches/checkpoint-clock-repair/ReleaseOverlay.cmake PG001 +patches/checkpoint-clock-repair/ReleaseOverlay.cmake PG002 +patches/checkpoint-clock-repair/ReleaseOverlay.cmake PG003 +patches/checkpoint-clock-repair/audit.py PG013 +patches/checkpoint-clock-repair/configure.py PG013 +patches/checkpoint-clock-repair/package.py PG013 +patches/checkpoint-clock-repair/pipeline.py PG004 +patches/checkpoint-clock-repair/transforms.py PG010 +patches/credits-viewport/NativeOverlay.cmake PG001 +patches/credits-viewport/NativeOverlay.cmake PG002 +patches/credits-viewport/NativeOverlay.cmake PG003 +patches/credits-viewport/README.md PG013 +patches/credits-viewport/credit_fade.inl PG006 +patches/credits-viewport/credit_match.inl PG006 +patches/credits-viewport/credits_sky_rt64.hpp PG006 +patches/credits-viewport/package.py PG013 +patches/credits-viewport/pipeline.py PG004 +patches/custom-magic-codes/runtime-code-heap.patch PG013 +patches/custom-vehicles/PLAYTEST.txt PG013 +patches/custom-vehicles/REPAIR-NOTES.txt PG013 +patches/custom-vehicles/ReleaseOverlay.cmake PG001 +patches/custom-vehicles/ReleaseOverlay.cmake PG002 +patches/custom-vehicles/ReleaseOverlay.cmake PG003 +patches/custom-vehicles/ReleaseOverlay.cmake PG010 +patches/custom-vehicles/audit_characters.py PG013 +patches/custom-vehicles/character_ownership_review.py PG013 +patches/custom-vehicles/configure109.py PG013 +patches/custom-vehicles/guest_pipeline.py PG013 +patches/custom-vehicles/inspect_base.py PG013 +patches/custom-vehicles/inspect_character_components.py PG013 +patches/custom-vehicles/inspect_inputs.py PG013 +patches/custom-vehicles/inspect_placement.py PG013 +patches/custom-vehicles/inspect_scale.py PG013 +patches/custom-vehicles/inspect_seating.py PG013 +patches/custom-vehicles/music_pipeline.py PG013 +patches/custom-vehicles/native/asset_bus_mount.inl PG006 +patches/custom-vehicles/native/resident_music.cpp PG006 +patches/custom-vehicles/native/resident_music.hpp PG006 +patches/custom-vehicles/native/resident_music_prepare.inl PG006 +patches/custom-vehicles/native/vehicle_base_hashes.hpp PG006 +patches/custom-vehicles/native/vehicle_binding.cpp PG006 +patches/custom-vehicles/native/vehicle_binding.hpp PG006 +patches/custom-vehicles/native/vehicle_character_masks.hpp PG006 +patches/custom-vehicles/native/vehicle_library.cpp PG006 +patches/custom-vehicles/native/vehicle_library.hpp PG006 +patches/custom-vehicles/native/vehicle_online.hpp PG006 +patches/custom-vehicles/native/vehicle_package.cpp PG006 +patches/custom-vehicles/native/vehicle_package.hpp PG006 +patches/custom-vehicles/native/vehicle_protocol.hpp PG006 +patches/custom-vehicles/native/vehicle_resources.cpp PG006 +patches/custom-vehicles/native/vehicle_resources.hpp PG006 +patches/custom-vehicles/native/vehicle_rt64.cpp PG006 +patches/custom-vehicles/native/vehicle_rt64.hpp PG006 +patches/custom-vehicles/native/vehicle_seat_alignment.hpp PG006 +patches/custom-vehicles/native/vehicle_ui.inl PG006 +patches/custom-vehicles/native/vehicle_wheel_interpolation.hpp PG006 +patches/custom-vehicles/native/vehicle_world.cpp PG006 +patches/custom-vehicles/native/vehicle_world.hpp PG006 +patches/custom-vehicles/online_pipeline.py PG013 +patches/custom-vehicles/package_playtest.py PG013 +patches/custom-vehicles/pipeline.py PG004 +patches/detail-records-repair/PLAYTEST.txt PG013 +patches/detail-records-repair/ReleaseOverlay.cmake PG001 +patches/detail-records-repair/ReleaseOverlay.cmake PG002 +patches/detail-records-repair/ReleaseOverlay.cmake PG003 +patches/detail-records-repair/ReleaseOverlay.cmake PG010 +patches/detail-records-repair/audit.py PG013 +patches/detail-records-repair/configure.py PG013 +patches/detail-records-repair/native/detail_batch.inl PG006 +patches/detail-records-repair/native/detail_metrics.hpp PG006 +patches/detail-records-repair/native/detail_texture_cache.inl PG006 +patches/detail-records-repair/native/records_guest.inl PG006 +patches/detail-records-repair/package.py PG013 +patches/detail-records-repair/pipeline.py PG004 +patches/enhanced-water/LICENSE-WAVE-RACE.txt PG013 +patches/enhanced-water/dkr-adapter.patch PG013 +patches/enhanced-water/dkr-budget.patch PG013 +patches/enhanced-water/dkr-field-portability.patch PG013 +patches/enhanced-water/dkr-lava-glow.patch PG013 +patches/enhanced-water/dkr-lava-lighting.patch PG013 +patches/enhanced-water/dkr-lava.patch PG013 +patches/enhanced-water/dkr-performance.patch PG013 +patches/enhanced-water/dkr-projection.patch PG013 +patches/enhanced-water/dkr-ray-capabilities.patch PG013 +patches/enhanced-water/dkr-rays.patch PG013 +patches/enhanced-water/dkr-recovery.patch PG013 +patches/enhanced-water/dkr-reflection-search.patch PG013 +patches/enhanced-water/dkr-safety.patch PG013 +patches/enhanced-water/dkr-stability.patch PG013 +patches/enhanced-water/dkr-views.patch PG013 +patches/enhanced-water/dkr-waterfall.patch PG013 +patches/enhanced-water/native/LavaFlow.hlsli PG006 +patches/enhanced-water/native/LavaLighting.hlsli PG006 +patches/enhanced-water/native/LavaLightingBounds.h PG006 +patches/enhanced-water/native/LavaLightingCS.hlsl PG006 +patches/enhanced-water/native/LavaLightingPS.hlsl PG006 +patches/enhanced-water/native/LavaLightingVS.hlsl PG006 +patches/enhanced-water/native/WaterBounds.h PG006 +patches/enhanced-water/native/WaterCaptureProbeCS.hlsl PG006 +patches/enhanced-water/native/WaterFieldCompareCS.hlsl PG006 +patches/enhanced-water/native/WaterFieldReferenceCS.hlsl PG006 +patches/enhanced-water/native/WaterPipelineSafety.h PG006 +patches/enhanced-water/native/WaterProjection.h PG006 +patches/enhanced-water/native/WaterProjection.hlsli PG006 +patches/enhanced-water/native/WaterProjectionProbeCS.hlsl PG006 +patches/enhanced-water/native/WaterRayProbeCS.hlsl PG006 +patches/enhanced-water/native/WaterRayScene.cpp PG006 +patches/enhanced-water/native/WaterRayScene.h PG006 +patches/enhanced-water/native/WaterRays.hlsli PG006 +patches/enhanced-water/native/WaterResourcePolicy.h PG006 +patches/enhanced-water/native/WaterWorkPolicy.h PG006 +patches/enhanced-water/native/WaterfallFlow.hlsli PG006 +patches/enhanced-water/native/WaterfallPS.hlsl PG006 +patches/enhanced-water/native/WaterfallVS.hlsl PG006 +patches/enhanced-water/renderer.patch PG013 +patches/graphics-handoff/PIPELINE.txt PG013 +patches/graphics-handoff/PLAYTEST.txt PG013 +patches/graphics-handoff/ReleaseOverlay.cmake PG001 +patches/graphics-handoff/ReleaseOverlay.cmake PG002 +patches/graphics-handoff/ReleaseOverlay.cmake PG003 +patches/graphics-handoff/audit.py PG013 +patches/graphics-handoff/configure.py PG013 +patches/graphics-handoff/finalize.py PG013 +patches/graphics-handoff/native/producer_snapshot.cpp PG006 +patches/graphics-handoff/native/producer_snapshot.hpp PG006 +patches/graphics-handoff/package.py PG013 +patches/graphics-handoff/pipeline.py PG004 +patches/graphics-handoff/run_checks.py PG013 +patches/graphics-handoff/tests/copy_cost.cpp PG006 +patches/graphics-handoff/tests/handoff_tests.cpp PG006 +patches/invite-handoff/IMPLEMENTATION.txt PG013 +patches/invite-handoff/PLAYTEST.txt PG013 +patches/invite-handoff/ReleaseOverlay.cmake PG001 +patches/invite-handoff/ReleaseOverlay.cmake PG002 +patches/invite-handoff/ReleaseOverlay.cmake PG003 +patches/invite-handoff/audit.py PG004 +patches/invite-handoff/configure.py PG013 +patches/invite-handoff/native/invite_handoff.cpp PG006 +patches/invite-handoff/native/invite_handoff.hpp PG006 +patches/invite-handoff/native/invite_ui.inl PG006 +patches/invite-handoff/native/release_inputs.inl PG006 +patches/invite-handoff/package.py PG013 +patches/invite-handoff/pipeline.py PG004 +patches/launcher-polish/NativeOverlay.cmake PG001 +patches/launcher-polish/NativeOverlay.cmake PG002 +patches/launcher-polish/NativeOverlay.cmake PG003 +patches/launcher-polish/pipeline.py PG013 +patches/launcher-polish/support_actions.inl PG006 +patches/library-workflow/DEPLOYMENT.md PG013 +patches/library-workflow/QA.md PG013 +patches/library-workflow/RELEASE-NOTES.txt PG013 +patches/library-workflow/backend.inl PG006 +patches/library-workflow/baseline/assets/hosted-files.js PG013 +patches/library-workflow/baseline/index.html PG013 +patches/library-workflow/batch-client.js PG013 +patches/library-workflow/pipeline.py PG004 +patches/library-workflow/tests/d1-runtime.cjs PG013 +patches/library-workflow/tests/preview.cjs PG013 +patches/library-workflow/tests/run.cjs PG013 +patches/library-workflow/tests/support.cjs PG013 +patches/library-workflow/tests/typecheck.cjs PG013 +patches/library-workflow/transforms.py PG013 +patches/library-workflow/ui.inl PG006 +patches/library-workflow/workflow-schema.sql PG013 +patches/library-workflow/workflow.css PG013 +patches/liquid-streaming/NativeOverlay.cmake PG001 +patches/liquid-streaming/NativeOverlay.cmake PG002 +patches/liquid-streaming/NativeOverlay.cmake PG003 +patches/liquid-streaming/OWNER-NOTES.txt PG013 +patches/liquid-streaming/liquid_replacement_gate.hpp PG006 +patches/liquid-streaming/package.py PG013 +patches/liquid-streaming/pipeline.py PG013 +patches/liquid-streaming/water_residency_rt64.hpp PG006 +patches/lobby-link-repair/IMPLEMENTATION.txt PG013 +patches/lobby-link-repair/PLAYTEST.txt PG013 +patches/lobby-link-repair/ReleaseOverlay.cmake PG001 +patches/lobby-link-repair/ReleaseOverlay.cmake PG002 +patches/lobby-link-repair/ReleaseOverlay.cmake PG003 +patches/lobby-link-repair/audit.py PG004 +patches/lobby-link-repair/configure.py PG013 +patches/lobby-link-repair/package.py PG013 +patches/lobby-link-repair/pipeline.py PG004 +patches/lobby-link-repair/transforms.py PG013 +patches/lobby-restoration/IMPLEMENTATION.txt PG013 +patches/lobby-restoration/PLAYTEST.txt PG013 +patches/lobby-restoration/ReleaseOverlay.cmake PG001 +patches/lobby-restoration/ReleaseOverlay.cmake PG002 +patches/lobby-restoration/ReleaseOverlay.cmake PG003 +patches/lobby-restoration/audit.py PG004 +patches/lobby-restoration/configure.py PG013 +patches/lobby-restoration/native/lobby_adapter.cpp PG006 +patches/lobby-restoration/native/lobby_adapter.hpp PG006 +patches/lobby-restoration/native/lobby_context.inl PG006 +patches/lobby-restoration/native/lobby_controls.inl PG006 +patches/lobby-restoration/native/remove_member.inl PG006 +patches/lobby-restoration/package.py PG013 +patches/lobby-restoration/pipeline.py PG004 +patches/lobby-restoration/transforms.py PG013 +patches/lobby-stability/IMPLEMENTATION.txt PG013 +patches/lobby-stability/PLAYTEST.txt PG013 +patches/lobby-stability/ReleaseOverlay.cmake PG001 +patches/lobby-stability/ReleaseOverlay.cmake PG002 +patches/lobby-stability/ReleaseOverlay.cmake PG003 +patches/lobby-stability/ReleaseOverlay.cmake PG010 +patches/lobby-stability/audit.py PG004 +patches/lobby-stability/configure.py PG013 +patches/lobby-stability/native/cache_repair.hpp PG006 +patches/lobby-stability/native/diagnostics.hpp PG006 +patches/lobby-stability/native/policy.hpp PG006 +patches/lobby-stability/native/task_submission.cpp PG006 +patches/lobby-stability/package.py PG013 +patches/lobby-stability/pipeline.py PG004 +patches/lobby-stability/probe_worker.py PG013 +patches/lobby-stability/tests/CMakeLists.txt PG001 +patches/lobby-stability/tests/policy_tests.cpp PG006 +patches/lobby-stability/tests/sp_fixture.cpp PG006 +patches/lobby-stability/tests/sp_fixture.hpp PG006 +patches/lobby-stability/tests/task_submission_tests.cpp PG006 +patches/lobby-stability/transforms.py PG013 +patches/memory-stability/DIAGNOSIS.txt PG013 +patches/memory-stability/PIPELINE.txt PG013 +patches/memory-stability/PLAYTEST.txt PG013 +patches/memory-stability/ReleaseOverlay.cmake PG001 +patches/memory-stability/ReleaseOverlay.cmake PG002 +patches/memory-stability/ReleaseOverlay.cmake PG003 +patches/memory-stability/ReleaseOverlay.cmake PG010 +patches/memory-stability/audit.py PG013 +patches/memory-stability/configure.py PG013 +patches/memory-stability/finalize.py PG013 +patches/memory-stability/native/primary_heap.h PG006 +patches/memory-stability/package.py PG013 +patches/memory-stability/pipeline.py PG004 +patches/memory-stability/run_checks.py PG013 +patches/memory-stability/tests/allocator_tests.cpp PG006 +patches/memory-stability/tests/owned_entry_tests.cpp PG006 +patches/memory-stability/verify_archive.py PG013 +patches/mod-runtime-repair/PLAYTEST.txt PG013 +patches/mod-runtime-repair/ReleaseOverlay.cmake PG001 +patches/mod-runtime-repair/ReleaseOverlay.cmake PG002 +patches/mod-runtime-repair/ReleaseOverlay.cmake PG003 +patches/mod-runtime-repair/ReleaseOverlay.cmake PG010 +patches/mod-runtime-repair/SDK-WORK-POLICY.txt PG013 +patches/mod-runtime-repair/WORK-NOTES.txt PG013 +patches/mod-runtime-repair/configure.py PG013 +patches/mod-runtime-repair/guest_pipeline.py PG013 +patches/mod-runtime-repair/inspect_sources.py PG004 +patches/mod-runtime-repair/native/magic_code_execution_policy.hpp PG006 +patches/mod-runtime-repair/native/render_heap_support.cpp PG006 +patches/mod-runtime-repair/package.py PG013 +patches/mod-runtime-repair/pipeline.py PG004 +patches/mod-store/INSTALLATION-MANAGER.md PG013 +patches/mod-store/NativeOverlay.cmake PG001 +patches/mod-store/NativeOverlay.cmake PG002 +patches/mod-store/NativeOverlay.cmake PG003 +patches/mod-store/RELEASE-NOTES.txt PG013 +patches/mod-store/backend/OWNER-SETUP.txt PG013 +patches/mod-store/backend/backfill_library.py PG013 +patches/mod-store/backend/launcher-auth.inl PG006 +patches/mod-store/backend/launcher-schema.sql PG013 +patches/mod-store/backend/launcher-service.inl PG006 +patches/mod-store/backend/release_manifest.py PG013 +patches/mod-store/backend/upload-hash.inl PG006 +patches/mod-store/backend/upload-integrity.inl PG006 +patches/mod-store/backend/web.config PG013 +patches/mod-store/frontend_pipeline.py PG013 +patches/mod-store/native/mod_store.cpp PG006 +patches/mod-store/native/mod_store.hpp PG006 +patches/mod-store/native/mod_store_install.cpp PG006 +patches/mod-store/native/mod_store_install.hpp PG006 +patches/mod-store/native/mod_store_management_ui.inl PG006 +patches/mod-store/native/mod_store_names.cpp PG006 +patches/mod-store/native/mod_store_style.inl PG006 +patches/mod-store/native/mod_store_ui.inl PG006 +patches/mod-store/native/store_account_page.inl PG006 +patches/mod-store/native/store_page_shell.inl PG006 +patches/mod-store/native/store_player_activity.inl PG006 +patches/mod-store/native/store_player_details.inl PG006 +patches/mod-store/native/store_player_library.inl PG006 +patches/mod-store/native/store_polish_controls.inl PG006 +patches/mod-store/native/store_polish_modals.inl PG006 +patches/mod-store/native/store_staff_pages.inl PG006 +patches/mod-store/native/store_texture_adapter.inl PG006 +patches/mod-store/native/store_track_adapter.inl PG006 +patches/mod-store/native/store_track_locator.inl PG006 +patches/mod-store/native/store_track_zip.inl PG006 +patches/mod-store/package.py PG013 +patches/mod-store/package_linux.py PG013 +patches/mod-store/pipeline.py PG004 +patches/mod-store/texture_names_pipeline.py PG013 +patches/music-preview-repair/PLAYTEST.txt PG013 +patches/music-preview-repair/REPAIR.txt PG013 +patches/music-preview-repair/ReleaseOverlay.cmake PG001 +patches/music-preview-repair/ReleaseOverlay.cmake PG002 +patches/music-preview-repair/ReleaseOverlay.cmake PG003 +patches/music-preview-repair/ReleaseOverlay.cmake PG010 +patches/music-preview-repair/audit.py PG013 +patches/music-preview-repair/configure.py PG013 +patches/music-preview-repair/package.py PG013 +patches/music-preview-repair/pipeline.py PG004 +patches/netplay/experimental-present-queue.patch PG013 +patches/netplay/experimental-present-state.patch PG013 +patches/netplay/experimental-quiesced-bootstrap.patch PG013 +patches/netplay/experimental-workload-history.patch PG013 +patches/offline-retention-repair/IMPLEMENTATION.txt PG013 +patches/offline-retention-repair/PLAYTEST.txt PG013 +patches/offline-retention-repair/ReleaseOverlay.cmake PG001 +patches/offline-retention-repair/ReleaseOverlay.cmake PG002 +patches/offline-retention-repair/ReleaseOverlay.cmake PG003 +patches/offline-retention-repair/audit.py PG004 +patches/offline-retention-repair/configure.py PG013 +patches/offline-retention-repair/handoff.py PG013 +patches/offline-retention-repair/lobby_layout.py PG013 +patches/offline-retention-repair/native/lobby_sidebar.inl PG006 +patches/offline-retention-repair/native/offline_keep.hpp PG006 +patches/offline-retention-repair/native/offline_keep.inl PG006 +patches/offline-retention-repair/package.py PG013 +patches/offline-retention-repair/pipeline.py PG004 +patches/offline-retention-repair/run_checks.py PG013 +patches/offline-retention-repair/tests/retention_tests.cpp PG006 +patches/online-directory/DEPLOYMENT.md PG013 +patches/online-directory/IMPLEMENTATION.txt PG013 +patches/online-directory/PLAYTEST.txt PG013 +patches/online-directory/QA.md PG013 +patches/online-directory/ReleaseOverlay.cmake PG001 +patches/online-directory/ReleaseOverlay.cmake PG002 +patches/online-directory/ReleaseOverlay.cmake PG003 +patches/online-directory/audit.py PG004 +patches/online-directory/backend/directory.mjs PG013 +patches/online-directory/backend/gateway.inl PG006 +patches/online-directory/backend/wrangler-online.jsonc PG013 +patches/online-directory/backend_pipeline.py PG013 +patches/online-directory/configure.py PG013 +patches/online-directory/delivery.py PG013 +patches/online-directory/native/account_bind.inl PG006 +patches/online-directory/native/admission_contract.hpp PG006 +patches/online-directory/native/directory_tls.hpp PG006 +patches/online-directory/native/directory_ui.inl PG006 +patches/online-directory/native/linux_keyring.hpp PG006 +patches/online-directory/native/online_directory.cpp PG006 +patches/online-directory/native/online_directory.hpp PG006 +patches/online-directory/native/owner_pump.inl PG006 +patches/online-directory/native/owner_state.hpp PG006 +patches/online-directory/package.py PG013 +patches/online-directory/pipeline.py PG004 +patches/online-directory/tests/runtime.cjs PG013 +patches/online-directory/transforms.py PG013 +patches/online-vehicle-menu/PLAYTEST.txt PG013 +patches/online-vehicle-menu/REPAIR.txt PG013 +patches/online-vehicle-menu/ReleaseOverlay.cmake PG001 +patches/online-vehicle-menu/ReleaseOverlay.cmake PG002 +patches/online-vehicle-menu/ReleaseOverlay.cmake PG003 +patches/online-vehicle-menu/ReleaseOverlay.cmake PG010 +patches/online-vehicle-menu/audit.py PG013 +patches/online-vehicle-menu/configure.py PG013 +patches/online-vehicle-menu/native/owned_vehicle_calls.inl PG006 +patches/online-vehicle-menu/native/owned_vehicle_menu.hpp PG006 +patches/online-vehicle-menu/package.py PG013 +patches/online-vehicle-menu/pipeline.py PG004 +patches/party-departure-repair/IMPLEMENTATION.txt PG013 +patches/party-departure-repair/PLAYTEST.txt PG013 +patches/party-departure-repair/ReleaseOverlay.cmake PG001 +patches/party-departure-repair/ReleaseOverlay.cmake PG002 +patches/party-departure-repair/ReleaseOverlay.cmake PG003 +patches/party-departure-repair/audit.py PG013 +patches/party-departure-repair/configure.py PG013 +patches/party-departure-repair/native/close_notice.inl PG006 +patches/party-departure-repair/native/departure.hpp PG006 +patches/party-departure-repair/native/retire_member.inl PG006 +patches/party-departure-repair/package.py PG013 +patches/party-departure-repair/pipeline.py PG004 +patches/party-departure-repair/transforms.py PG010 +patches/party-start-repair/IMPLEMENTATION.txt PG013 +patches/party-start-repair/PLAYTEST.txt PG013 +patches/party-start-repair/ReleaseOverlay.cmake PG001 +patches/party-start-repair/ReleaseOverlay.cmake PG002 +patches/party-start-repair/ReleaseOverlay.cmake PG003 +patches/party-start-repair/audit.py PG004 +patches/party-start-repair/configure.py PG013 +patches/party-start-repair/native/client_start.inl PG006 +patches/party-start-repair/native/host_start.inl PG006 +patches/party-start-repair/native/start_gate.hpp PG006 +patches/party-start-repair/package.py PG013 +patches/party-start-repair/pipeline.py PG004 +patches/party-start-repair/transforms.py PG010 +patches/performance-surgical/NativeOverlay.cmake PG001 +patches/performance-surgical/NativeOverlay.cmake PG002 +patches/performance-surgical/NativeOverlay.cmake PG003 +patches/performance-surgical/OWNER-CHECKS.txt PG013 +patches/performance-surgical/Record-Performance.cmd PG013 +patches/performance-surgical/inspect_sources.py PG013 +patches/performance-surgical/mip_scratch.inl PG006 +patches/performance-surgical/package.py PG013 +patches/performance-surgical/performance_observation.hpp PG006 +patches/performance-surgical/performance_switches.hpp PG006 +patches/performance-surgical/pipeline.py PG004 +patches/performance-surgical/presentation_pacer.hpp PG006 +patches/performance-surgical/reusable_flow_map.hpp PG006 +patches/performance-surgical/texture_binding_index.hpp PG006 +patches/performance/present-queue-wait.patch PG013 +patches/performance/presentation-wait.patch PG013 +patches/performance/renderer-measurements.patch PG013 +patches/performance/renderer-workers.patch PG013 +patches/performance/runtime-deferred-events.patch PG013 +patches/performance/runtime-guest-cancellation.patch PG013 +patches/performance/runtime-guest-lifetime.patch PG013 +patches/performance/runtime-host-task-lifetime.patch PG013 +patches/performance/runtime-idle-diagnostics.patch PG013 +patches/performance/runtime-session-bootstrap.patch PG013 +patches/performance/runtime-snapshot-pool.patch PG013 +patches/performance/runtime-vi-bootstrap.patch PG013 +patches/performance/shader-cache-wait.patch PG013 +patches/performance/vulkan-measurements.patch PG013 +patches/performance/water-renderer-history.patch PG013 +patches/performance/water-scroll-interpolation.patch PG013 +patches/performance/workload-queue-wait.patch PG013 +patches/presentation-polish/NativeOverlay.cmake PG001 +patches/presentation-polish/NativeOverlay.cmake PG002 +patches/presentation-polish/NativeOverlay.cmake PG003 +patches/presentation-polish/README.md PG013 +patches/presentation-polish/intro_caption_rt64.hpp PG006 +patches/presentation-polish/intro_sky_rt64.hpp PG006 +patches/presentation-polish/native_resource_safety.hpp PG006 +patches/presentation-polish/package.py PG013 +patches/presentation-polish/pipeline.py PG004 +patches/presentation-polish/water_residency_rt64.hpp PG006 +patches/race-release-repair/IMPLEMENTATION.txt PG013 +patches/race-release-repair/PLAYTEST.txt PG013 +patches/race-release-repair/ReleaseOverlay.cmake PG001 +patches/race-release-repair/ReleaseOverlay.cmake PG002 +patches/race-release-repair/ReleaseOverlay.cmake PG003 +patches/race-release-repair/ReleaseOverlay.cmake PG010 +patches/race-release-repair/audit.py PG004 +patches/race-release-repair/configure.py PG013 +patches/race-release-repair/handoff.py PG013 +patches/race-release-repair/package.py PG013 +patches/race-release-repair/pipeline.py PG013 +patches/race-release-repair/run_checks.py PG013 +patches/race-release-repair/tests/runtime_tests.cpp PG006 +patches/race-release-repair/transforms.py PG013 +patches/release-prep/v1.0.10/BUILDING.txt PG013 +patches/release-prep/v1.0.10/RELEASE-NOTES.txt PG013 +patches/release-prep/v1.0.10/ReleaseOverlay.cmake PG001 +patches/release-prep/v1.0.10/UPDATER.txt PG013 +patches/release-prep/v1.0.10/audit.py PG013 +patches/release-prep/v1.0.10/check_docs.py PG013 +patches/release-prep/v1.0.10/content.py PG013 +patches/release-prep/v1.0.10/finalize.py PG013 +patches/release-prep/v1.0.10/package.py PG013 +patches/release-prep/v1.0.10/pages/DOCUMENTATION-1.0.10.md PG013 +patches/release-prep/v1.0.10/pages/PLATFORMS.md PG013 +patches/release-prep/v1.0.10/pages/RELEASE-README.md PG013 +patches/release-prep/v1.0.10/pages/RELEASE-VALIDATION.md PG013 +patches/release-prep/v1.0.10/pages/UPDATES.md PG013 +patches/release-prep/v1.0.10/qualify_updates.py PG013 +patches/release-prep/v1.0.10/release.py PG004 +patches/release-prep/v1.0.10/run_checks.py PG013 +patches/release-prep/v1.0.10/seal.py PG013 +patches/release-prep/v1.0.10/update_assets.py PG013 +patches/release-prep/v1.0.11/BUILDING.txt PG013 +patches/release-prep/v1.0.11/RELEASE-NOTES.txt PG013 +patches/release-prep/v1.0.11/ReleaseOverlay.cmake PG001 +patches/release-prep/v1.0.11/ReleaseOverlay.cmake PG002 +patches/release-prep/v1.0.11/UPDATER.txt PG013 +patches/release-prep/v1.0.11/audit.py PG013 +patches/release-prep/v1.0.11/author.py PG013 +patches/release-prep/v1.0.11/build.py PG013 +patches/release-prep/v1.0.11/check_docs.py PG013 +patches/release-prep/v1.0.11/content.py PG013 +patches/release-prep/v1.0.11/contributor_checks.py PG013 +patches/release-prep/v1.0.11/dump_prepared.py PG004 +patches/release-prep/v1.0.11/finalize.py PG013 +patches/release-prep/v1.0.11/package.py PG013 +patches/release-prep/v1.0.11/pages/DOCUMENTATION-1.0.11.md PG013 +patches/release-prep/v1.0.11/pages/PLATFORMS.md PG013 +patches/release-prep/v1.0.11/pages/RELEASE-README.md PG013 +patches/release-prep/v1.0.11/pages/RELEASE-VALIDATION.md PG013 +patches/release-prep/v1.0.11/pages/UPDATES.md PG013 +patches/release-prep/v1.0.11/qualify_updates.py PG013 +patches/release-prep/v1.0.11/release.py PG004 +patches/release-prep/v1.0.11/run_checks.py PG004 +patches/release-prep/v1.0.11/seal.py PG013 +patches/release-prep/v1.0.11/update_assets.py PG013 +patches/release-prep/v1.0.11/worker_probe.py PG013 +patches/release-prep/v1.0.7/.gitattributes PG005 +patches/release-prep/v1.0.7/ACCEPTED-WATER-BASELINE.json PG013 +patches/release-prep/v1.0.7/Archive-ListedResidue.ps1 PG013 +patches/release-prep/v1.0.7/Archive-ReleaseOutput.ps1 PG013 +patches/release-prep/v1.0.7/BUILDING.txt PG013 +patches/release-prep/v1.0.7/DOCUMENTATION-APPLIED.json PG013 +patches/release-prep/v1.0.7/README-1.0.7-DRAFT.txt PG013 +patches/release-prep/v1.0.7/RELEASE-PLAN.txt PG013 +patches/release-prep/v1.0.7/ReleaseOverlay.cmake PG001 +patches/release-prep/v1.0.7/ReleaseOverlay.cmake PG002 +patches/release-prep/v1.0.7/ReleaseOverlay.cmake PG003 +patches/release-prep/v1.0.7/android_store.py PG013 +patches/release-prep/v1.0.7/android_store_compat.hpp PG006 +patches/release-prep/v1.0.7/audit.py PG013 +patches/release-prep/v1.0.7/capsule.py PG004 +patches/release-prep/v1.0.7/capsule.py PG005 +patches/release-prep/v1.0.7/cleanup_plan.py PG013 +patches/release-prep/v1.0.7/documentation_plan.py PG004 +patches/release-prep/v1.0.7/finalize_release.py PG013 +patches/release-prep/v1.0.7/finish_metadata.py PG004 +patches/release-prep/v1.0.7/history/RELEASE-VALIDATION-before-release.txt PG013 +patches/release-prep/v1.0.7/inspect_apk.py PG013 +patches/release-prep/v1.0.7/inspect_elf.py PG013 +patches/release-prep/v1.0.7/inspect_index.py PG013 +patches/release-prep/v1.0.7/linux-host-libraries.txt PG013 +patches/release-prep/v1.0.7/mod_store_android_http.inl PG006 +patches/release-prep/v1.0.7/package_release.py PG004 +patches/release-prep/v1.0.7/prepare_android.py PG004 +patches/release-prep/v1.0.7/record_validation.py PG013 +patches/release-prep/v1.0.7/release.py PG002 +patches/release-prep/v1.0.7/sdk_release.py PG013 +patches/release-prep/v1.0.7/source-capsule-r2/005056dd61f5c65e38216ca6c46f148985d9a081e651b7ea62df7b0c2159bada.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/02750fc3d26c5aa7bd1a250b2218908e04ece846ce046f8e0801e214a0f168c4.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/0291b1fe0790d700142130cad3b8fb270edd674dd9d14520dd39bf069ac0d711.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/038fd287000dacde0b0c5fba7db15036f2b5c222d15c49874ce877dd1016bf5a.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/0424853dd1a0c31276b007d0a2ff8eab58b9800d1dbf1cc8e10f1fd02c8136b8.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/04a9aceaf8c969f3decba2682a7e1be344919ba05dd94524efde67ef6497f51c.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/0638c144dca701b128f729ee4f597394e33ff3811a7a673db309f11cf2f1d852.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/06614652b4bcbd47adcd949d595b337afe1282b21c5f52db141575d148a468df.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/06bc9f5fa0aac55e2fab23ad24da087f17201bb7df8b3a3d8d3a715d7ee14c27.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/071c078a6b3d24af47cb4de4cfdd66a13a711f851686c2641263163968413dea.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/08d360749cb230cb899529586a6b52d774b3f6a1d157a78c12a5f3039b7372c6.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/0bed589aa21e2b30d82690baf2f54b0fe3659e7ecd8b0b3d64878a6d736c96d9.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/0c61f3b1c35bb46a5f78ba5e5e4881d08a4c02811a61cc8bbfebad382517566c.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/0cc3ea198b7496750df75e44cc5c27ea6de507562198284898785d2c68951c00.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/0f00e9ca7731b782bd69da6ae4aa1eec93de6ac6bf685ce8bc545081ddddcc5e.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/1349a9772ad155aa5e395f9f7050d4ae1247f85aab16c61ad8446e03f04b8e8c.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/1452a699671b7969c6554fee5cd179c06cb2303d319efc3103a9374f6246f898.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/14cc1528ecc8acd8228b9cfe410ac9ed970ea580c75f6e9eaad91e1c2f2fa567.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/151e82c73c2265607a3e072086cc60a684e5012782a957d51729b16de65e37c2.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/15c554629a5eda8d5e1ec44db71208887acb64a151a74d7535598563f7ab01e7.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/1831349d7ab11e40a3fe9cefc1b5a389459b9588c1661f5fba602b4e475940ca.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/187bb37cf394eabeddbb6235659c9097367768798281a40cd8d3f37041a1f694.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/1910b74c6e493f04ea2cfcd85824b16ae325216e569f008a14ec493fdce18f3d.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/193c418a41a57dc814b93603898ee2b17dc7e968af1c73f8334224a625479292.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/1a845f5ea53dca134955dc56872777a671ed82cbfc1c0d850b971a80dfbb433c.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/1c6ede4eb1f498b3f7c7dfd8606a644567cf746ec0f623318c48dd2db40de21d.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/1d06998488c2b1ecdef3151d1a21cf7b808ea56c33b5eec034e1e541e55f3c92.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/2187f1dd7c651b41c45826fb2c6550cf057d730d188b98f8d49d78c566e72b80.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/21d812aa27630c74dac16b8ebd41040e20b5eb074077b792ad4990753f530317.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/21fe8296b45b4cff21a2ff26d6bb5800c3926f9e84c249e5d3790cc22c942fb9.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/23e52e4f14278b787047dbb6f8826ec5b8a69a1c780cb1d0d7c6bb2bf57940cc.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/26f4029d8ef88099918111ec0ed6bc7867266399babb370220ea46d0083a2a93.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/279fa9f622335f2862895d210f963d148fe3faa88c438a9a7bb149c5bd9207e1.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/28988f6fd5931c0649f29ac80d4417ac6b2cec9649a52f438d0f591bb502938c.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/2b299fead36f89332f337e444e7ee53abf3bfb920ced4eae8278b22488301c9a.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/2c1cbe78867f68c6676955ee9650100b5129846e62b4b1513ccdeba96b15da6f.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/2c20ef4f18f9e2e62bdc92143d226e7bf5e851949b7a13d67f898b3c6a35fd19.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/2cb7bfcf74d162b325d8421df7351249c8521177d5b72d3a13af49a3119b0448.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/2cf5c30d7ebc65613f9be38eb84fa8b636e6a4a09f0895fcbabe362d5159b83a.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/2d499981716e7517bdb7871ba3b66dd904c1f9a8761e3d9fe5b24e03f779909b.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/2da243480142a9c9771f0b1989a035cf4e2bf7931b6be8b51d7b6abfad569128.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/2e5b4d32342719dbb66988ef9c2e12ef9a0c58287d54f6d02319e32144feb89a.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/2e6f6941e941237b43fe0470b96a22dd86c368b02e2521a77fba796d07a14051.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/2e84f35a5ff12199bb93cb51f40d71f37485bef52c09a48d63188b008e0d52ce.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/2f933d17f6793875e77e3dc5c5fcafc78f47fc5ee7593dd51411d7c6232badac.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/30cf7bfdaaa559e32b1e40c966df6ada5af8033d9c9547c34d365c8d13e4341c.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/34a749109afb7d60a21d9dd02fd30371e1740900641773d445ccd3772b1e032d.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/36d541348d4c3d56bc4db9705da4b94bac7d8f1f6210be5def61e3f2fd4022a9.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/38ea662c3899070d2114094efa8603f0f1e8656ccd689ce683f6c46b2f24bb30.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/3954c49da34fbe726fcd8e7f1e66eb65a76692b769ff22596bc30033d6dbb415.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/398867c578fa9d847c6b598d2d8787e7873d00c9bb2b44329a4a730d7ec69d57.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/3a8f98b03d8908469de7b7d6145cc24ac1c5fca7cbcb93596681edad05eff5f3.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/3e2a7e796a56390e4e644355911b2a7ee1d7696d321113e40e455cf6e47654c8.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/3e6cd2c91e01061b7e8fc92fcf46633d73a5c39a0d00040b4c775fc5d82971e9.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/40e8e6bf7e533bbee43f408a10f9d0c77c5f2be9a27d54ad07c3439622b6c9c9.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/420f1959fa6fe20977509f6e39e4ea4fdd1b25a2a062b740d1e26b11dc9d5a17.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/42ef5e06fae34ef14b7fbc0c54e3e4462e97aa68b9dea6d02d13eacfe07a9049.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/43be50cf84a1ac898e2cc2f769e968e1dc274adf4c092f37f9992e2886ceda75.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/44894f6e85d2bee8d83b983f2f7e38295fca306e2d2b0036aa0f4dd2a6813079.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/459de5e00b23f690155360ef63078310d3a7bef396f475c9d43692160db483b3.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/47df57c52d69010522073a1013f8816f62eb1ab7737a0cbe570ccc93a02a0c6a.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/4879179f2a5df43d2c40e80435902cc935482dc905fc045e47838e96d7880119.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/489591ca629e39aadee540bfce1be104b6668ea689d4c31953d9a434b6e41594.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/49e07c1d395986736652b4736f3d5368d4247784aa09a24e6dc06b543c9b21e8.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/4ab923c66ff6917d5c063fd1c06c779d865fb302878e2f47b81eeebb8a52b362.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/4aca317c29d812adf31c3718e8d04c1b717892dc4c78ecc513bd891195db579b.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/4cd08aaebb9ec349ebff6816a03e6ffe607e85874b23e33bc23c18d7cfb40af2.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/4cd93525825a9866cb18cfe32366d9cbd4c79256bde7793ebf4ce09aa97a041a.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/4d6e62f7801dfd2302d1f3d5354d830762b44e56f6ce5f96c11a1b56b11535bc.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/4e04a323b053b8cf094a719a5915bc0042145580fed46f865eb531133fe8d3f3.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/4e3c6f08c925a6d9ed7633c8fb923042b23f1d8260dc8bd6e706ee1866e0c621.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/4eac568db8fdb0033f69498b00677aa760604e0b58fdd2965ea95a6e49a975d1.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/4f039f59fa4b932bc108c60098dd3318d3427a6a2f7e40bd7cad406e22889342.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/501f4c3678098e8f051d9262515bed6f17f1f9a8566649fa32081c8255f07e9a.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/516011edbe66c7101c15b9b4744f2d85ec33a6fd2b823a6d3e63d5ea7715e6fa.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/526dc2ba1e8ac8f9bec051df50171a645895eda6623f4de7f8bad7dbf11e5fdc.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/56472b54e1df7f708366c3781956d7474b06d550ebb19257af4cbfb7b5b5938c.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/56e959d14d124346c61712e03cf3b95430f246e99e0fc6f3039866d9ea63096d.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/57f24c6012e69eb3acd2b2903dce868539ea8715a5298e107ff58f6ad97375e5.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/57feea89560877bdec23391b10005071e60e0dbc016d4444f9ed405d06e3fc1f.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/586ddb02735426896b929d584d5ffd95c9fd3eb8da41667b8309531913abab05.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/5928438638aa2382bb40e7bc355d7202af807be3186c86a01ca4b426c97d0bca.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/5a2577eaf15f09eb2944d4fdd88dab41c23d3e5eac5aeeb773e9d394ec05c977.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/5a62f310250bb441f05e6c41880d89b912883d33ffdf2877462079b945f5c3f7.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/5b0a44e6a752cab48d32ebfdfc1a6df92227574e369b0e2ce97228422f8d4ca7.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/5b1ebd7864841ac07acf6409b82178f7bc02b5196efbc08f36b7f8f7c6b640a2.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/5b5ede9991065e8e29fec2a462b0972d7a7c752c326c587985cadb81b1b4febc.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/5bf2b33d8e9b3ecf200cbdd297c576a700f30042fc2323c1839b39d2930712e9.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/5c94aae8dc8256e6f0f255e9ca02c5d7b375dddac9957ca2f4a59987ac2caccd.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/5d7fccba6878cacd47417777d5eed867e7c27e5b21fa8c56898f7c2a166e69ed.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/5d8f051a7d8f68889fde8c749f904796dd4db15214818951a9ca8109f18ca82c.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/5dabd7523bf0ab500a01808f904be0a1dcbc08012c4025dcdfaa4b409c2494fd.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/5e04a5f889eeefc14c8e08d670819bbd2d5044784ab1e2eb92b5c773298aee20.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/5ebc5aceb251d28804c91aefe0c4324b3f0a6faf5a4007e4d0d0c77938114fbc.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/604e8fbb0072136980ab979bd3892a14810ac56c4fb26640623d9509837cb231.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/609f640a50f8eb53e907c9fffba8b4ceaaf94d987b4224dc5fbbf8dad8f363ee.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/614d4dfe78e38baf7992a9e6333000d14c8ea086daa1878e03b1d7a78d6de527.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/61be480d3b8c5eab5ad5a9b8876e8340556a2f5b9b0f7d9e63816fa6213bf59e.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/61d88c96a432f270bd13f3980b72746c79268b4dae8307877422d0c032437499.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/6305310b5d2e309003c91a8e47fc99e4d4d4491527a11533b7307e5d29a70894.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/6310019155d7a18785d030a98d66b5bfbcbe6c6780dcf6d7aa0ccf60f33d15b5.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/63a4741035a2871d691a90fc86b95d0858ec98affc5ac5eb6f0dcf565c35c65a.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/642f8baba1efd9d1b438e46c7d29e68b5fb964925edd42b42249e11e15bc9919.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/64bc62e4a41028f710a63ff11e0bbc9e9f3096744b4172a3865b69cc9a5d6e96.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/670a415f4a8bd789d25ce627457cdd599c5efce29430d50c08af104ff64e3f85.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/6749f04846d83a7b30728a9c8fc43ee928e654515d37ca5df92e7cee91644897.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/69353985481430ed275830fc6e0c1ef85d9628415f4d50ee1bb13d28ed64cf73.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/69edb7c9fa1f9c8b373d75eca3fe62091e6888ea76651c5a6ac9052813b6445e.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/6ce848564cba38d7164db097a5adf5aa9675f9fcd6ab55b733c0e0812660908c.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/6d074815631bfb001386b2d09c803656a41e72aa0a272d00740e2ed851771be6.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/6ddf87fcc6667ee0f8e0c70ca06caae03d809a58b3c3b0858bc97061cfd70fbd.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/6e2f3931f43db202a3e723a1da038517dad75c1fdfcbf2eaa8e0fffb3ddd8c67.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/708d7fc6d4b5853d58f2700c31ed45e9c384c366b97ce78c9c6f017193cfd9e5.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/70b22ca90f5a9f83c1b619ab4a136cff8366bc9b7c46adf38fabf49d4863a82b.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/70cc4c87862bc4f9d7b39138bfd51f95ccc1ffc352d5396d89b49e83b48e4167.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/7237742571bff8ce701fd721c121660ea3a550d183fa110970082d6cca945bb1.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/73b30f1cc13a89b116679bcb46045237c6ad856064da2a68be90ba585cdf328c.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/743969ee617fbd455563249287811eef34824b196939e98e86ec1d3ad3c59779.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/7536c661f1fd4debb300881a5cbc50c4933b856546b41143522cd12ac4ea8346.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/7551624d936dc59bbbaa9ca11959d38a40f6f4787145cc7521e94e76dd79495d.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/768e1c61b05c026b65a0fd16334ab099b9d7687b3f652686a038dd47570a9fb9.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/7692137dca97300ab2632918080ec71a2d1f1a385d36a1b19e0004fc14dcbd91.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/78b2f5e62146016d84536a747ef091a3ca2b1409ff75637e1a93e6bee1ff2fed.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/7914ffd7f323b1c82743a53f192ec79c48c81e5a101610c405b417124c37647b.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/792acaf1354342ed1851671fb1ba517c341da1682af92f1d930f4b93f3e23e38.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/7a6f19522d26c5abab09233b8a9cd2d91f152509b0926996aef0bb124651e905.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/7d66b5f6cd9bb392916a990f557a80c7420e413205f434a17447426942d981c5.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/7eaf50feea476e84c30df13f9e81d26215d96410607e0bf32a5498c332dee511.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/8051c44ebda26973ab921d71e90cd15f44d6b93ce00f0fb2fac80253b20dbcb5.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/80815558e0491e30c5745c873442cc3aa36487c85ede964b0ae26eacd40dc484.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/82415ff7d6a2b334b876112295541fba4c2d1dba14627acc688dbd60b6d77e7c.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/82d04a8763f7cd910e8a448287241741b10a56b7c6ff69dc66a82664a6ec6851.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/8359622ae1b0ec581071e4fce9fc30bb17978690f830913f16f63f2901c141a2.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/85020f620b40fdb61506d18f4d541ad4b83d0043bc77f29424545cb3238ab3da.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/852b15b2e755b49ff4c1ae9e251eeda18d2db5efe09e3e24f89e984693327d16.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/87580376144c63d3554f3420597003fedea6c69b1dd3173c9447720a687b1405.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/875f12c4b51d5ab64bd132a3d53390492851acdc4673dfe76ab06d9685121747.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/88155f6417c4e8ce8a3492d2cfbd088f36edf9a3a9c673b1afaae0ef0a58776b.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/8907d38b911e40dfb922cddac3392a1c803db50d2eabede280b9efa961fc26f9.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/893ee74b7b33475b42abbb05b00073d9cb811452438620c13c2236295c1e61cb.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/8ae349e2a7e6e99a33e2d3d094ca8a48241b173856eb59ee2b21e4463e5cb1c6.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/8b508102a08d15048b094b4fd257abee9fce3012705795b29faa54b4c84e9ff0.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/8b93f80952a35e68194b55179ef2e32a7ed12883736d127a20a9b682ad4e67e8.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/8ddc8b373f14b707c148d89eaa9ba4a5694e2e615153d68ea3ab4720163be00e.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/8f8b2bf02f7c3296947900012d963cb92f4242d067fa06d7da17932c443df40e.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/8fda606193ffd01f09e31953753c780a28a66a8477519b651cb6ce6ce5e6fbe5.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/9119f8ab41db037f855ba836434393feb85abf5a1995ee83058177f1d38050dd.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/914aed14d56894f5af22719981e4d0f0c27ff299a2d8687c13b1d28fc986be7b.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/918573b20ceb869c8b5b8e9b565a3a07b0a4af6c1f8b98e6a6a4161330d8d5ba.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/92429fe87c57567f1826f548f67824fc7f9407bb94789572770c4aa4afef02c8.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/9261140546b7ee35cfaffa9a31def5b6b1b833d48e0a97de869b2fc12638ee8a.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/9298054cee0ab7466958c23481ca22dae56ca6b02c5a8a9b362e48ba6ab56d9e.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/92d2323948ef15191689c31e2c0e56efeae427215067f34936955e4380507f11.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/9333b7654cb462b83030736bc7679a48f28d5e0cd9fe2aaf69c29cbae61a6b63.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/970f1666af908eedf456a7557508866b0e7567c7971fdcfbadf6da8a0edd3b98.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/98c4e701941070182bb658f85b7aaae61c923e9ef5808e70dbe919794c4c897a.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/993570d1dd27996b92dbccdcd8a05ff4d6c7d7348235286f12831f1d158b6ac0.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/9a32a6005e83ae2de18afb4f7ae31c0c9d1e1a015b6bb3aa01da1f4b73857694.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/9a81c1a8da313898ce7f5c109bc535b06fc4c843d0182dc07bf91a46f1079ea1.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/9adc7572addedfb662140bb489802799093f6066203d5da151ec00415665036f.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/9de9a251b6522ff4162e1bce04c8d35365b2eeac41ec7a75e96c7164eb6d9f93.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/9e01e2804db2cef7d0e85ff600b0a889d7094ceb0fa5f24c4e74dca26a686f2d.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/9ee6f6ba3620230bb293a703c96e148f75d4ab884232ffc810983d3a3752d63b.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/a1d86d4bdadf4c2e9d88899e272739f2f001af65e8981f685f8afc3c7f7488e4.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/a252d4d871e87da15df30bb22d0b7f0a227ce9ed5370d704acf07bc74a379ea9.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/a42cbd3bf6d7f273ab8e835f8ff8ba8d32dadeae0f853174f8769987f1333501.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/a4942393164219c1196a4f37e9d3c9b23ab72c359b6ef368ec56b12c4c2dfffb.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/a6209f78ac9eac3fca0edac34bfe8a0968fa30d462a6865161dc1994855e33f2.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/a63a604a7abe2e6b23fd12bc695868a1d5b682bd59c21b774cae54e8b8af654b.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/a75c147ddf9389ec5073a5db244c54d792135954dcae7eae590fe6e81064c8d4.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/a76023258d23255e425d70d433db54da07ffc1f9aa4ed9791eb5710988de2a43.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/a7b9c8fd332b4a07e7d137098d9792fa8fd5d380da13d7965196b66f87d3f2d5.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/a830f14880fe8bd0dfec43078560085bc653333381b4654fa8007a152d474518.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/a98f9f6235bc713d1590f4cc2897876868660f19e27a509e0f3fb9020e11411b.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/a99ccc67a4c867680141792a1db0c2236f9e5f228f29dbb88aa57ce906b34ba7.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/ace65e8efd9a98eae192bbbe8693ef17ce878bd6cb366638e9d41686dfef1167.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/ade10d3060b50d50a2a5f2ad428f8399fb959faf0cac2d8d95083ebf9ec1aa1b.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/af5027b58d21d1c3640adc21ad3f6f9df9d8c91ae6565e5ef8aa505af23198ee.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/b0b4bfe713920d9f2dd49d83824da2c6d4a681aba141b80eddc1662a6cf38619.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/b15f640c179ff4882b6b7e4e515c80613b21ecbd43976a1311cc4087b520310e.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/b6bb5c187192af0ba7b99ca839f4abe4ba3e1ad89dffb76ed6e72c3cdd337e25.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/b7c34380d9a3e13a39e31ef4b567e60cc11f3e4910ae9e563b1c84e4dbf886d8.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/b85d28c9dfe7e9c0ef4ce36f6e4c6a7f0079503ddde4b9b9fafa6cb3406c5fbf.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/b943f22b0ad0500e6a5d3931e24fe799483bcfaad22b073aafc55f45fb638800.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/bad557fd563a9e7e485d4fd8626b00ef17318308bd5db4aa6d1bce2e5d7a52f8.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/bdab0d19c96bc0059430122fe039b2133e427ee42c6e7c6756e3b2b81df6302b.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/be5208176efcedbdcc1bb690701e88427f22769bf713e9b826a06c541dd3f8ae.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/bef0c3858f64756974a51fdb80d55fa0694da5f12c706eeb8d1b71b6cdaf9247.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/c039851d4d264d96711472fdc159a208492e92675c565f3e9270b45d2734092b.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/c09e770509cc1479574618201b4b06a812982e26b36a348bcc216b98cfeaae2a.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/c1470bac35edfa0c84ad50b9444cdee33da0a085f9bef0148bae970e5a33e474.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/c16216aceff04f0ce34506ad6292260f350347b84e20c9a6e1b50f8066a1df73.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/c3c69f66bfd890982d4309bdf72490090a2d95ff9c76193d3cfdc2d1d227c676.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/c4777fab27887c55cf770d892d7affd0c5b22964e174f782b607b5aad4092101.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/c60be73c8797751de2e917207b619621db877a7939ea06d451a5682035bf2e4b.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/c9e709d9e17e6b850754d896fefb30d35e2e3e77efa03ac84d9580941330aa9f.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/cb29ff8fb40932821e7bd5cb665dc8ceaced72d35617c816bf066f333e62fad4.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/cb43cb7a4a8ebbaf364b57a329715c5080cd33929f792dcaf165477dc9136591.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/cba572b7501ae1a698f584bad1f51a58a2ae7bfc33a4e50972d03c6cf70611bb.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/cc32731c1f72abf9b0629cb5eb49da19ae33661bed607020b999ecd7a027e348.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/ccd5c44d15ee4fc12a591756d2ecef6ca36dd03e45e88ecc93115386e5141038.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/ceb569ea3e9d8210e2b5d9f0ab6a103ee3c3d2304d1136cee364e76c2a7fc266.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/ced3bdf6a86c69d87bd2166b7eece16fe53df4f7c3fb02ed2fdaaa6271405e0d.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/d088af5f94763fbdad3d83a429abd7e4038832f83e1ef78f7287f16f20f3c479.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/d0cceab6fb4c1aa3fac137cc56f28af1977d12af5b9dc57d0092420ebfc8216f.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/d183f6d2fecbcc23bccb1ff86d4b1712d19577a8826e8bcc83eb91d46a538495.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/d234df70cd4ae7fc0ddd6de8b9b5b65c4c358f4636557e55759ab5d2b4f5a071.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/d36a4975da84d6f7581d96877f896d8f5b8a96654d4645d42742ca11b296578f.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/d5dd288aa554a23b3b76e33d1834ebf9ccb464a1a12f8524a21bdcbc4585fd71.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/d6b1f488ea634917ffcb29b8e297d48a4cb362e7e34a79f476a0a53ed61b3db7.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/d785a6c271ac0490030974bc90557244503135ace717277b36af3a339de46a96.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/d7c2a1e31b1e641da5c58c767f3ac72b9eeb8e25f318ff8913da75bbba41c223.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/d85204652b85528a76c721b2fde7c0df53dbe187b70b405d1d9c5353667738f0.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/d977127fadb8867099607698b8ea7f1e3e0d970513d9baba8380a2c652d90195.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/da3c7aa0b5f7885ec1a612c5774936abe55cd10851f401c50ede9fe851b1a70d.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/dae0a6b6471805c56c4aa9ac4360683aa3a190a1acb1b7874f9aaa75d95b4b55.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/e37ac2281fd5abac8a7d5b1f3608dd063b81e5015aa9668742bbc9dcf8b1f03f.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/e543a6bd0166c685456330c9e03f72b8f15c234bd580c3e1ec7b5e3fcf8f5759.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/e55fd0e22b7051a7c90dc07de71ade8110948a7a7acccc9c003d31c4cce29f8e.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/e57e6126910ed2e3d727b8c5174b988ed911c6d1b83ea4878748ce6371cd6497.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/e6011da6e40d3002979ae8a2fc3ddd06be13900f6c7f649d28cf30e54ebc15fa.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/e612b1c1ffda2d8bb5f5f88c8c1838ba50db6ce0ba3c0883cc527469bc2f57bd.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/e73feb82a5e24502a2c4d416b677f7ed7caf9364c2869da635e459c978ff549c.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/eae16170ee7306238d08e648dbe862dea42857d37b8af6d066071e6128ac2482.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/eb2b6755144655ebba623263a877d4fce019c7acdf6ec98f5c57b0a7f7925212.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/eb4224b88ae60e8b95203f3a127457fc4e95256d6f58b255822f9d980abf0496.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/ebab61adce5778f21338ac8bc4ee2e517c662116bfd5a43dd92035636a7e6481.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/ec2c7cd548a97cde98d4f886f65dbeb36dc671f441b923df732d8a9df71d8972.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/edf1c469026f0da68be1c6855560e8d21ff0633a2b4b8dc2f50c6b608a51e294.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/eeb97503e29340fe841bf2a77b8738c9f18a4048af11832678f0eab3cd68db31.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/ef27691736231c4d323c34425c43d5f64724566970fb37602bd30012925fd456.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/f0a4862b794e1e5cd1b490d99c8655f19d627f52ea10e473441533b8f351b420.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/f1115a2665f2a9ecfef6601172f9bda2e7672cdb5a4e88a7057a3dc4b592c8e7.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/f1d17ac3d4ce4fafafc8e243e08d0109eb790db66a9138385e641e114eaa1aa0.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/f273094b6c4c4baa7e57cc14ea3c064804e6cde188e19742a4724f76971e7527.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/f3745226a5555659cc29213899d5b4cb3c6ed2fa4f15d505ea079d422ae91333.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/f5e7e3d453d68cec5a85ae0afa7922c38ddf8fb809540db11782987f14962df2.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/f624cece137b47d6d7ec923bbd64a5e33f38fe12e42f205ea570d76facfbd08a.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/f7466df109b603997fa417bafc34701ad5bada7c7b2327d5593e44bfb4884458.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/f833a77cba163b7008f4311581e0d0da0c34f8b9bf4ad7f2b19dc6a59bcdc955.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/f8907006dba43b08a40605ac30af17b86fc6bedb321e6bc96ead027054566d1d.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/f8ab891977cd4693fcec92a94a1368259c0888ba484a8411971c7778ea3f4ff7.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/f8cc90ae98da0a1b97eab1fb814caa9174e9e3a43cc48c896c75bec88e43390a.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/f8f51b6dfba8cacac51f2fbdff36b0f07fd278eae9e4716d082e9c6193cd7074.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/fa63238d9e812f7449df57fe7e0a29a512f4857818bde01a497499a0c8684ab3.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/faf91da8f2ced760a7aa331d15e0d2437f04b8dc57a4969c2dee509de64bd2f3.txt PG005 +patches/release-prep/v1.0.7/source-capsule-r2/fca32922d3bb442987c0cfa4bd8a171db22149bede43dfdb2ec5de49c9183e24.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/fcfa7ec86efda3c5fe497f3a9d8945157b2858c3c58d3eb87f48daa7d1d71864.patch PG005 +patches/release-prep/v1.0.7/source-capsule-r2/manifest.json PG005 +patches/release-prep/v1.0.7/static_review.py PG004 +patches/release-prep/v1.0.7/update_docs.py PG004 +patches/release-prep/v1.0.9/BUILDING.txt PG013 +patches/release-prep/v1.0.9/RELEASE-NOTES.txt PG013 +patches/release-prep/v1.0.9/ReleaseOverlay.cmake PG001 +patches/release-prep/v1.0.9/ReleaseOverlay.cmake PG002 +patches/release-prep/v1.0.9/ReleaseOverlay.cmake PG003 +patches/release-prep/v1.0.9/UPDATER.txt PG013 +patches/release-prep/v1.0.9/android_account_input.py PG013 +patches/release-prep/v1.0.9/android_http.py PG013 +patches/release-prep/v1.0.9/android_shared.py PG013 +patches/release-prep/v1.0.9/android_snapshot.py PG013 +patches/release-prep/v1.0.9/android_texture.py PG013 +patches/release-prep/v1.0.9/audit.py PG004 +patches/release-prep/v1.0.9/documentation/check.py PG004 +patches/release-prep/v1.0.9/documentation/content.py PG013 +patches/release-prep/v1.0.9/documentation/github_export.py PG013 +patches/release-prep/v1.0.9/documentation/pages/ACHIEVEMENTS.md PG013 +patches/release-prep/v1.0.9/documentation/pages/DKR-R-v1.0.9-Changelog.txt PG013 +patches/release-prep/v1.0.9/documentation/pages/MOD_LIBRARY_ADMIN.md PG013 +patches/release-prep/v1.0.9/documentation/pages/ONLINE_MULTIPLAYER.md PG013 +patches/release-prep/v1.0.9/documentation/pages/PUBLIC_LOBBIES.md PG013 +patches/release-prep/v1.0.9/documentation/pages/README.md PG013 +patches/release-prep/v1.0.9/documentation/pages/RELEASE-VALIDATION.md PG013 +patches/release-prep/v1.0.9/documentation/pages/SPECTATORS.md PG013 +patches/release-prep/v1.0.9/documentation/pages/VEHICLES.md PG013 +patches/release-prep/v1.0.9/documentation/pipeline.py PG004 +patches/release-prep/v1.0.9/finalize.py PG013 +patches/release-prep/v1.0.9/package.py PG013 +patches/release-prep/v1.0.9/release.py PG002 +patches/release-prep/v1.0.9/release.py PG004 +patches/release-prep/v1.0.9/update_assets.py PG013 +patches/render-stability/PIPELINE.txt PG013 +patches/render-stability/PLAYTEST.txt PG013 +patches/render-stability/ReleaseOverlay.cmake PG001 +patches/render-stability/ReleaseOverlay.cmake PG002 +patches/render-stability/ReleaseOverlay.cmake PG003 +patches/render-stability/audit.py PG013 +patches/render-stability/configure.py PG013 +patches/render-stability/finalize.py PG013 +patches/render-stability/native/liquid_replacement_gate.hpp PG006 +patches/render-stability/native/render_provenance.hpp PG006 +patches/render-stability/native/water_residency_rt64.hpp PG006 +patches/render-stability/package.py PG013 +patches/render-stability/pipeline.py PG004 +patches/render-stability/run_checks.py PG013 +patches/render-stability/tests/residency_tests.cpp PG006 +patches/render-stability/tests/selection_tests.cpp PG006 +patches/render-stability/tests/stubs/enhanced_water_rt64.hpp PG006 +patches/render-stability/tests/stubs/performance_observation.hpp PG006 +patches/render-stability/tests/stubs/polish_symbols.hpp PG006 +patches/render-stability/tests/stubs/polish_symbols.hpp PG011 +patches/render-stability/tests/stubs/render/rt64_texture_cache.h PG006 +patches/render-stability/tests/stubs/rice_texture_pack_policy.hpp PG006 +patches/review-editor/BUILDING.txt PG013 +patches/review-editor/ReleaseOverlay.cmake PG001 +patches/review-editor/ReleaseOverlay.cmake PG002 +patches/review-editor/ReleaseOverlay.cmake PG003 +patches/review-editor/configure109.py PG013 +patches/review-editor/native/review_form.inl PG006 +patches/review-editor/native/wrapped_input.inl PG006 +patches/review-editor/pipeline.py PG004 +patches/rollback-connection/IMPLEMENTATION.txt PG013 +patches/rollback-connection/PLAYTEST.txt PG013 +patches/rollback-connection/ReleaseOverlay.cmake PG001 +patches/rollback-connection/ReleaseOverlay.cmake PG002 +patches/rollback-connection/ReleaseOverlay.cmake PG003 +patches/rollback-connection/audit.py PG004 +patches/rollback-connection/configure.py PG013 +patches/rollback-connection/native/connection_ui.inl PG006 +patches/rollback-connection/native/owner_context.inl PG006 +patches/rollback-connection/native/owner_send.inl PG006 +patches/rollback-connection/native/rollback_check.cpp PG006 +patches/rollback-connection/native/rollback_check.hpp PG006 +patches/rollback-connection/package.py PG013 +patches/rollback-connection/pipeline.py PG004 +patches/rollback-connection/transforms.py PG013 +patches/rollback-only/IMPLEMENTATION.txt PG013 +patches/rollback-only/PLAYTEST.txt PG013 +patches/rollback-only/ReleaseOverlay.cmake PG001 +patches/rollback-only/ReleaseOverlay.cmake PG002 +patches/rollback-only/ReleaseOverlay.cmake PG003 +patches/rollback-only/audit.py PG004 +patches/rollback-only/configure.py PG013 +patches/rollback-only/native/host_settings.inl PG006 +patches/rollback-only/native/rollback_policy.hpp PG006 +patches/rollback-only/package.py PG013 +patches/rollback-only/pipeline.py PG004 +patches/rollback-scoreboard/IMPLEMENTATION.txt PG013 +patches/rollback-scoreboard/PLAYTEST.txt PG013 +patches/rollback-scoreboard/ReleaseOverlay.cmake PG001 +patches/rollback-scoreboard/ReleaseOverlay.cmake PG002 +patches/rollback-scoreboard/ReleaseOverlay.cmake PG003 +patches/rollback-scoreboard/audit.py PG004 +patches/rollback-scoreboard/configure.py PG013 +patches/rollback-scoreboard/native/assessment.hpp PG006 +patches/rollback-scoreboard/native/report_ui.inl PG006 +patches/rollback-scoreboard/native/scoreboard_wire.inl PG006 +patches/rollback-scoreboard/package.py PG013 +patches/rollback-scoreboard/pipeline.py PG004 +patches/rollback-scoreboard/transforms.py PG013 +patches/rt64/texture-layer-mip-cache.patch PG007 +patches/rt64/texture-layer-priority.patch PG007 +patches/save-layout/README.md PG013 +patches/save-layout/guest_save_layout.h PG006 +patches/save-layout/inspect_packages.py PG013 +patches/save-layout/inspect_sources.py PG013 +patches/save-layout/package-linux.sh PG013 +patches/save-layout/package-windows.ps1 PG013 +patches/scene-preparation/PLAYTEST.txt PG013 +patches/scene-preparation/REPAIR.txt PG013 +patches/scene-preparation/ReleaseOverlay.cmake PG001 +patches/scene-preparation/ReleaseOverlay.cmake PG002 +patches/scene-preparation/ReleaseOverlay.cmake PG003 +patches/scene-preparation/ReleaseOverlay.cmake PG010 +patches/scene-preparation/audit.py PG004 +patches/scene-preparation/configure.py PG013 +patches/scene-preparation/native/scene_load_work.hpp PG006 +patches/scene-preparation/package.py PG013 +patches/scene-preparation/pipeline.py PG013 +patches/sdk2/BIBLE.txt PG013 +patches/sdk2/DEV-MENU-IMPORT-FIX.txt PG013 +patches/sdk2/IMPLEMENTATION-PLAN.txt PG013 +patches/sdk2/NativeOverlay.cmake PG001 +patches/sdk2/NativeOverlay.cmake PG002 +patches/sdk2/NativeOverlay.cmake PG003 +patches/sdk2/NativeOverlay.cmake PG010 +patches/sdk2/PLAYTEST-README.txt PG013 +patches/sdk2/RELEASE-NOTES.txt PG013 +patches/sdk2/SDK-README.txt PG013 +patches/sdk2/STARTUP-FIX-README.txt PG013 +patches/sdk2/WORK-STATUS.txt PG013 +patches/sdk2/android_startup_init.gradle PG013 +patches/sdk2/catalogue_pipeline.py PG013 +patches/sdk2/compatibility.py PG013 +patches/sdk2/control_pipeline.py PG013 +patches/sdk2/distribution.py PG013 +patches/sdk2/documentation.py PG013 +patches/sdk2/documentation_qa.py PG013 +patches/sdk2/finalize_delivery.py PG013 +patches/sdk2/inspect_delivery.py PG013 +patches/sdk2/lobby_pipeline.py PG013 +patches/sdk2/mod/dkr.json PG013 +patches/sdk2/mod/documents.inl PG006 +patches/sdk2/mod/main.c PG006 +patches/sdk2/mod/mod.toml PG006 +patches/sdk2/mod/tools.py PG013 +patches/sdk2/mod/track.inl PG006 +patches/sdk2/native/sdk2_asset_inventory.inl PG006 +patches/sdk2/native/sdk2_bridge.cpp PG006 +patches/sdk2/native/sdk2_bridge.hpp PG006 +patches/sdk2/native/sdk2_broker.inl PG006 +patches/sdk2/native/sdk2_camera.inl PG006 +patches/sdk2/native/sdk2_collision.inl PG006 +patches/sdk2/native/sdk2_command.hpp PG006 +patches/sdk2/native/sdk2_debug.inl PG006 +patches/sdk2/native/sdk2_documents.inl PG006 +patches/sdk2/native/sdk2_operations.inl PG006 +patches/sdk2/native/sdk2_overrides.inl PG006 +patches/sdk2/native/sdk2_properties.inl PG006 +patches/sdk2/native/sdk2_reload.inl PG006 +patches/sdk2/native/sdk2_schema.hpp PG006 +patches/sdk2/native/sdk2_settings.inl PG006 +patches/sdk2/native/sdk2_simulation.inl PG006 +patches/sdk2/native/sdk2_tools_camera.inl PG006 +patches/sdk2/native/sdk2_tools_console.inl PG006 +patches/sdk2/native/sdk2_tools_documents.inl PG006 +patches/sdk2/native/sdk2_tools_reload.inl PG006 +patches/sdk2/native/sdk2_tools_ui.inl PG006 +patches/sdk2/native/sdk2_tools_workspace.inl PG006 +patches/sdk2/native/sdk2_track_catalogue.inl PG006 +patches/sdk2/native/sdk2_tuning.inl PG006 +patches/sdk2/native/sdk2_world.inl PG006 +patches/sdk2/network_pipeline.py PG013 +patches/sdk2/package_import_fix.py PG013 +patches/sdk2/package_startup_fix.py PG004 +patches/sdk2/pipeline.py PG004 +patches/sdk2/presentation_pipeline.py PG013 +patches/sdk2/reload_pipeline.py PG013 +patches/sdk2/reuse_stage.py PG013 +patches/sdk2/sdk/development.h PG006 +patches/sdk2/sdk/development.hpp PG006 +patches/sdk2/sdk/json.h PG006 +patches/sdk2/sdk/json_read.h PG006 +patches/sdk2/sdk/relocations.py PG013 +patches/sdk2/sdk/tools.py PG013 +patches/sdk2/settings_pipeline.py PG013 +patches/shared-saves/NativeOverlay.cmake PG001 +patches/shared-saves/NativeOverlay.cmake PG002 +patches/shared-saves/NativeOverlay.cmake PG003 +patches/shared-saves/NativeOverlay.cmake PG010 +patches/shared-saves/README.md PG013 +patches/shared-saves/RELEASE-NOTES.txt PG013 +patches/shared-saves/bundle_dialog.inl PG006 +patches/shared-saves/cache_ui.inl PG006 +patches/shared-saves/custom_records.hpp PG006 +patches/shared-saves/import_dialog.inl PG006 +patches/shared-saves/inspect_build.py PG013 +patches/shared-saves/online_seed.inl PG006 +patches/shared-saves/overview_ui.inl PG006 +patches/shared-saves/package.py PG013 +patches/shared-saves/page_ui.inl PG006 +patches/shared-saves/pipeline.py PG004 +patches/shared-saves/record_guest.inl PG006 +patches/shared-saves/record_store.inl PG006 +patches/shared-saves/shared_save_backend.inl PG006 +patches/shared-saves/shared_save_jobs.inl PG006 +patches/shared-saves/shared_saves.hpp PG006 +patches/spectator-admission-repair/IMPLEMENTATION.txt PG013 +patches/spectator-admission-repair/PLAYTEST.txt PG013 +patches/spectator-admission-repair/ReleaseOverlay.cmake PG001 +patches/spectator-admission-repair/ReleaseOverlay.cmake PG002 +patches/spectator-admission-repair/ReleaseOverlay.cmake PG003 +patches/spectator-admission-repair/audit.py PG004 +patches/spectator-admission-repair/configure.py PG013 +patches/spectator-admission-repair/handoff.py PG013 +patches/spectator-admission-repair/native/admission_retry.hpp PG006 +patches/spectator-admission-repair/native/observer_music.hpp PG006 +patches/spectator-admission-repair/package.py PG013 +patches/spectator-admission-repair/pipeline.py PG004 +patches/spectator-admission-repair/run_checks.py PG013 +patches/spectator-admission-repair/tests/handler_tests.cpp PG006 +patches/spectator-admission-repair/tests/music_tests.cpp PG006 +patches/spectator-admission-repair/transforms.py PG013 +patches/spectators/IMPLEMENTATION.txt PG013 +patches/spectators/PLAYTEST.txt PG013 +patches/spectators/ReleaseOverlay.cmake PG001 +patches/spectators/ReleaseOverlay.cmake PG002 +patches/spectators/ReleaseOverlay.cmake PG003 +patches/spectators/audit.py PG010 +patches/spectators/configure.py PG013 +patches/spectators/core/CMakeLists.txt PG001 +patches/spectators/link_pipeline.py PG013 +patches/spectators/native/admit_component.inl PG006 +patches/spectators/native/admit_eeprom.inl PG006 +patches/spectators/native/admit_magic.inl PG006 +patches/spectators/native/admit_paks.inl PG006 +patches/spectators/native/checkpoint_transfer.cpp PG006 +patches/spectators/native/checkpoint_transfer.hpp PG006 +patches/spectators/native/confirmed_feed.cpp PG006 +patches/spectators/native/confirmed_feed.hpp PG006 +patches/spectators/native/confirmed_observer.cpp PG006 +patches/spectators/native/confirmed_observer.hpp PG006 +patches/spectators/native/director_pose.inl PG006 +patches/spectators/native/link_approve.inl PG006 +patches/spectators/native/link_receive.inl PG006 +patches/spectators/native/link_service.inl PG006 +patches/spectators/native/observer_playback.cpp PG006 +patches/spectators/native/observer_playback.hpp PG006 +patches/spectators/native/owned_observer.cpp PG006 +patches/spectators/native/owned_observer.hpp PG006 +patches/spectators/native/owner_export.cpp PG006 +patches/spectators/native/owner_export.hpp PG006 +patches/spectators/native/party.cpp PG006 +patches/spectators/native/party.hpp PG006 +patches/spectators/native/party_camera.hpp PG006 +patches/spectators/native/party_codec.cpp PG006 +patches/spectators/native/party_codec.hpp PG006 +patches/spectators/native/party_link.cpp PG006 +patches/spectators/native/party_link.hpp PG006 +patches/spectators/native/party_runtime.cpp PG006 +patches/spectators/native/party_runtime.hpp PG006 +patches/spectators/native/party_ui.inl PG006 +patches/spectators/native/runtime_runner.inl PG006 +patches/spectators/owned_pipeline.py PG013 +patches/spectators/package.py PG013 +patches/spectators/pipeline.py PG004 +patches/spectators/runtime_pipeline.py PG010 +patches/spectators/stage_core.py PG013 +patches/spectators/ui_pipeline.py PG004 +patches/startup-memory/BUILDING.txt PG013 +patches/startup-memory/DIAGNOSIS.txt PG013 +patches/startup-memory/RuntimeMemory.cmake PG001 +patches/startup-memory/RuntimeMemory.cmake PG007 +patches/startup-memory/history/RELEASE-VALIDATION-1.0.7.txt PG013 +patches/startup-memory/inspect108.py PG004 +patches/startup-memory/pipeline.py PG004 +patches/startup-memory/release108.py PG004 +patches/startup-memory/runtime-memory.patch PG013 +patches/store-vehicles/IMPLEMENTATION.txt PG013 +patches/store-vehicles/PLAYTEST.txt PG013 +patches/store-vehicles/ReleaseOverlay.cmake PG001 +patches/store-vehicles/ReleaseOverlay.cmake PG002 +patches/store-vehicles/ReleaseOverlay.cmake PG003 +patches/store-vehicles/audit.py PG004 +patches/store-vehicles/configure.py PG013 +patches/store-vehicles/native/store_vehicle_adapter.cpp PG006 +patches/store-vehicles/native/store_vehicle_adapter.hpp PG006 +patches/store-vehicles/native/vehicle_store_ui.inl PG006 +patches/store-vehicles/package.py PG013 +patches/store-vehicles/pipeline.py PG004 +patches/store-vehicles/transforms.py PG013 +patches/texture-dumping/ReleaseOverlay.cmake PG001 +patches/texture-dumping/ReleaseOverlay.cmake PG002 +patches/texture-dumping/ReleaseOverlay.cmake PG003 +patches/texture-dumping/TEXTURE-DUMPING.txt PG013 +patches/texture-dumping/configure.py PG013 +patches/texture-dumping/inspect_dump.py PG013 +patches/texture-dumping/native/texture_capture_bridge.hpp PG006 +patches/texture-dumping/native/texture_decode.hpp PG006 +patches/texture-dumping/native/texture_dump.cpp PG006 +patches/texture-dumping/native/texture_dump.hpp PG006 +patches/texture-dumping/native/texture_dump_ui.inl PG006 +patches/texture-dumping/native/texture_gallery.inl PG006 +patches/texture-dumping/package.py PG013 +patches/texture-dumping/pipeline.py PG004 +patches/title-text/NativeOverlay.cmake PG001 +patches/title-text/NativeOverlay.cmake PG002 +patches/title-text/NativeOverlay.cmake PG003 +patches/title-text/README.md PG013 +patches/title-text/package.py PG013 +patches/title-text/pipeline.py PG004 +patches/title-text/title_text_rt64.hpp PG006 +patches/upstream-sync/IMPLEMENTATION.txt PG013 +patches/upstream-sync/PLAYTEST.txt PG013 +patches/upstream-sync/ReleaseOverlay.cmake PG001 +patches/upstream-sync/ReleaseOverlay.cmake PG002 +patches/upstream-sync/ReleaseOverlay.cmake PG003 +patches/upstream-sync/audit.py PG013 +patches/upstream-sync/configure.py PG013 +patches/upstream-sync/package.py PG013 +patches/upstream-sync/pins.json PG013 +patches/upstream-sync/pipeline.py PG004 +patches/upstream-sync/restore_capsule.py PG005 +patches/upstream-sync/transforms.py PG013 +patches/vanilla-water/NativeOverlay.cmake PG001 +patches/vanilla-water/NativeOverlay.cmake PG002 +patches/vanilla-water/NativeOverlay.cmake PG003 +patches/vanilla-water/README.md PG013 +patches/vanilla-water/package.py PG013 +patches/vanilla-water/pipeline.py PG004 +patches/vehicle-original-textures/PIPELINE.txt PG013 +patches/vehicle-original-textures/PLAYTEST.txt PG013 +patches/vehicle-original-textures/ReleaseOverlay.cmake PG001 +patches/vehicle-original-textures/ReleaseOverlay.cmake PG002 +patches/vehicle-original-textures/ReleaseOverlay.cmake PG003 +patches/vehicle-original-textures/audit.py PG013 +patches/vehicle-original-textures/collect_checks.py PG013 +patches/vehicle-original-textures/configure.py PG013 +patches/vehicle-original-textures/finalize.py PG013 +patches/vehicle-original-textures/native/driver_anchor_units.hpp PG006 +patches/vehicle-original-textures/native/package_compatibility.hpp PG006 +patches/vehicle-original-textures/package.py PG013 +patches/vehicle-original-textures/pipeline.py PG013 +patches/vehicle-original-textures/run_checks.py PG013 +patches/vehicle-original-textures/tests/package_checks.cpp PG006 +patches/vehicle-seating/FORMAT.txt PG013 +patches/vehicle-seating/PLAYTEST.txt PG013 +patches/vehicle-seating/ReleaseOverlay.cmake PG001 +patches/vehicle-seating/ReleaseOverlay.cmake PG002 +patches/vehicle-seating/ReleaseOverlay.cmake PG003 +patches/vehicle-seating/ReleaseOverlay.cmake PG010 +patches/vehicle-seating/configure.py PG013 +patches/vehicle-seating/driver_detail_pipeline.py PG013 +patches/vehicle-seating/inspect_sources.py PG013 +patches/vehicle-seating/package.py PG013 +patches/viewport-flow/NativeOverlay.cmake PG001 +patches/viewport-flow/NativeOverlay.cmake PG002 +patches/viewport-flow/NativeOverlay.cmake PG003 +patches/viewport-flow/README.md PG013 +patches/viewport-flow/flow_cadence_rt64.hpp PG006 +patches/viewport-flow/flow_history_rt64.hpp PG006 +patches/viewport-flow/flow_phase_rt64.hpp PG006 +patches/viewport-flow/flow_vertex_scope_rt64.hpp PG006 +patches/viewport-flow/intro_viewport_rt64.hpp PG006 +patches/viewport-flow/package.py PG013 +patches/viewport-flow/pipeline.py PG004 +runtime-recomp/CMakeLists.txt PG007 +runtime-recomp/cmake/PerformancePatches.cmake PG007 +runtime-recomp/cmake/android/PreRotation.cmake PG007 +scripts/adapt-water-reference.py PG004 +scripts/camera_plus_pipeline.py PG004 +scripts/character_portrait_pipeline.py PG004 +scripts/compose-lava-glow.py PG004 +scripts/compose-lava-lighting.py PG004 +scripts/compose-lava.py PG004 +scripts/compose-water-performance.py PG004 +scripts/compose-water-projection.py PG004 +scripts/compose-water-rays.py PG004 +scripts/compose-water-recovery.py PG004 +scripts/compose-water-safety.py PG004 +scripts/compose-water-stability.py PG004 +scripts/compose-water-views.py PG004 +scripts/compose-waterfall.py PG004 +scripts/normalize-water-adapter.py PG004 +scripts/save_layout_pipeline.py PG004 diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 5bf88200..106f295b 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -23,8 +23,9 @@ python3 scripts/check.py --list python3 scripts/check.py --suite tooling ``` -The default command runs synthetic tooling tests, release metadata checks and -the source asset scan. It stops at the first failure and never downloads +The default command runs the pattern guard, synthetic tooling tests, release +metadata checks and the source asset scan. `--suite` repeats and takes comma +lists (`--suite guard,tooling`). It stops at the first failure and never downloads anything or changes dependency checkouts. `--list` shows the selected commands. On Windows, use `python` in place of `python3`; POSIX-only tests report skips. To run the Track Lab Python suites, use `--suite track-lab`. Missing private diff --git a/modding/audit_dispatch.py b/modding/audit_dispatch.py index fc9c7e25..110734e6 100644 --- a/modding/audit_dispatch.py +++ b/modding/audit_dispatch.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Read-only CPU/SDK source audit. Does not run a game or a mod callback.""" + import argparse import hashlib import json @@ -7,47 +8,91 @@ import re import sys -sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts')) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) import stage_magic_code_payload as pipeline -def audit(source,normal,owned,revision): - originals={} - for file in sorted(source.glob('*.c')): - text=file.read_text(encoding='utf-8');matches=list(pipeline.DEFINITION.finditer(text)) - for i,m in enumerate(matches):originals[m[1]]=text[m.start():matches[i+1].start() if i+1r\d+|0X[0-9A-F]+), (ctx->r\d+|0X[0-9A-F]+)\)') - for root,receipt in zip((normal,owned),receipts): - functions=receipt.get('functions',receipt.get('code_dispatch_functions',[]));by_name={f['name']:f for f in functions} - assert functions and receipt['revision']==revision - for name in fixes&by_name.keys():assert by_name[name]['protected'],f'Unprotected port fix {name}' - for name in pipeline.CURATED_AUDIO:assert by_name[name]['curated_call'],f'No reviewed audio handler {name}' - for name in ('get_misc_asset','get_settings','set_level_default_vehicle','set_time_trial_enabled','draw_text','set_text_font','set_text_colour','set_text_background_colour'): - assert name in by_name and not by_name[name]['protected_call'],f'Trophy SDK call unavailable: {name}' - text='\n'.join(p.read_text(encoding='utf-8') for p in sorted(root.glob('*.c'))) - for name,event in pipeline.TRACK_OPTIONS.items(): - assert name in by_name,f'Missing six-column menu owner {name}' - for number in (event,event+1): - assert f'dkr_legacy_track_menu(rdram,ctx,{number}U,menu_fields,0U)' in text,f'Missing cursor scope {name}/{number}' - laps={} - for name,fields in pipeline.LAP_FIELDS.items(): - if name not in by_name:continue - expected=sum(any(x.startswith('0X') and int(x,16) in fields for x in m.groups()) for m in pattern.finditer(originals[name])) - match=re.search(r'static void dkr_code_body_'+re.escape(name)+r'\(.*?(?=\nRECOMP_FUNC void '+re.escape(name)+r'\()',text,re.S) - assert match,f'Missing body wrapper {name}' - body=match[0] - actual=len(re.findall(r'\b(?:dkr_code_lap_address|dkr_probe_code_lap_address)\(',body)) - assert actual==expected and expected>0,f'Lap coverage {name}: {actual}/{expected}' - assert not any(any(x.startswith('0X') and int(x,16) in fields for x in m.groups()) for m in pattern.finditer(body)),f'Unmapped lap memory {name}' - laps[name]=actual - result['namespaces'].append({'private':root==owned,'functions':len(functions),'lap_accesses':laps, - 'dispatch_sha256':hashlib.sha256((root/'magic_code_dispatch.c').read_bytes()).hexdigest()}) + +def audit(source, normal, owned, revision): + originals = {} + for file in sorted(source.glob("*.c")): + text = file.read_text(encoding="utf-8") + matches = list(pipeline.DEFINITION.finditer(text)) + for i, m in enumerate(matches): + originals[m[1]] = text[m.start() : matches[i + 1].start() if i + 1 < len(matches) else len(text)] + receipts = [ + json.loads((normal / "magic-code-dispatch.json").read_text()), + json.loads((owned / "manifest.json").read_text()), + ] + fixes = pipeline.policy_names(source, revision) | set(pipeline.TRACK_OPTIONS) + result = {"revision": revision, "executed": False, "protected_functions": len(fixes), "namespaces": []} + pattern = re.compile(r"MEM_W\((ctx->r\d+|0X[0-9A-F]+), (ctx->r\d+|0X[0-9A-F]+)\)") + for root, receipt in zip((normal, owned), receipts): + functions = receipt.get("functions", receipt.get("code_dispatch_functions", [])) + by_name = {f["name"]: f for f in functions} + assert functions and receipt["revision"] == revision + for name in fixes & by_name.keys(): + assert by_name[name]["protected"], f"Unprotected port fix {name}" + for name in pipeline.CURATED_AUDIO: + assert by_name[name]["curated_call"], f"No reviewed audio handler {name}" + for name in ( + "get_misc_asset", + "get_settings", + "set_level_default_vehicle", + "set_time_trial_enabled", + "draw_text", + "set_text_font", + "set_text_colour", + "set_text_background_colour", + ): + assert name in by_name and not by_name[name]["protected_call"], f"Trophy SDK call unavailable: {name}" + text = "\n".join(p.read_text(encoding="utf-8") for p in sorted(root.glob("*.c"))) + for name, event in pipeline.TRACK_OPTIONS.items(): + assert name in by_name, f"Missing six-column menu owner {name}" + for number in (event, event + 1): + assert f"dkr_legacy_track_menu(rdram,ctx,{number}U,menu_fields,0U)" in text, ( + f"Missing cursor scope {name}/{number}" + ) + laps = {} + for name, fields in pipeline.LAP_FIELDS.items(): + if name not in by_name: + continue + expected = sum( + any(x.startswith("0X") and int(x, 16) in fields for x in m.groups()) + for m in pattern.finditer(originals[name]) + ) + match = re.search( + r"static void dkr_code_body_" + + re.escape(name) + + r"\(.*?(?=\nRECOMP_FUNC void " + + re.escape(name) + + r"\()", + text, + re.S, + ) + assert match, f"Missing body wrapper {name}" + body = match[0] + actual = len(re.findall(r"\b(?:dkr_code_lap_address|dkr_probe_code_lap_address)\(", body)) + assert actual == expected and expected > 0, f"Lap coverage {name}: {actual}/{expected}" + assert not any( + any(x.startswith("0X") and int(x, 16) in fields for x in m.groups()) for m in pattern.finditer(body) + ), f"Unmapped lap memory {name}" + laps[name] = actual + result["namespaces"].append( + { + "private": root == owned, + "functions": len(functions), + "lap_accesses": laps, + "dispatch_sha256": hashlib.sha256((root / "magic_code_dispatch.c").read_bytes()).hexdigest(), + } + ) return result -if __name__=='__main__': - p=argparse.ArgumentParser(description=__doc__) - p.add_argument('--source',type=Path,required=True);p.add_argument('--normal',type=Path,required=True) - p.add_argument('--owned',type=Path,required=True);p.add_argument('--revision',type=int,choices=(77,80),required=True) - a=p.parse_args();print(json.dumps(audit(a.source,a.normal,a.owned,a.revision),indent=2)) + +if __name__ == "__main__": + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--source", type=Path, required=True) + p.add_argument("--normal", type=Path, required=True) + p.add_argument("--owned", type=Path, required=True) + p.add_argument("--revision", type=int, choices=(77, 80), required=True) + a = p.parse_args() + print(json.dumps(audit(a.source, a.normal, a.owned, a.revision), indent=2)) diff --git a/modding/build.py b/modding/build.py index 9706a43a..354c177d 100644 --- a/modding/build.py +++ b/modding/build.py @@ -5,20 +5,23 @@ All output goes into a new directory; game code/dependencies remain read-only. Raw-address mods require a separately reviewed revision-specific authoring kit. """ + from __future__ import annotations + import argparse -import json import hashlib +import json import math from pathlib import Path, PurePosixPath import re import shutil import subprocess -import tomllib import zipfile +import tomllib + SDK = Path(__file__).resolve().parent -ID = re.compile(r'[a-z0-9_-]{1,96}\Z') +ID = re.compile(r"[a-z0-9_-]{1,96}\Z") def literal(value): @@ -27,23 +30,27 @@ def literal(value): if isinstance(value, float) and math.isfinite(value): return repr(value) if isinstance(value, list): - return '[' + ', '.join(literal(v) for v in value) + ']' - raise ValueError('Unsupported TOML value in mod manifest') + return "[" + ", ".join(literal(v) for v in value) + "]" + raise ValueError("Unsupported TOML value in mod manifest") def local(root: Path, value: str) -> Path: p = PurePosixPath(value) - if (not value or p.is_absolute() or any(x in ('', '.', '..') for x in value.split('/')) or - any(ord(c) < 32 or ord(c) > 126 or c in '\\:*?"<>|' for c in value)): - raise ValueError('Unsafe project resource path') + if ( + not value + or p.is_absolute() + or any(x in ("", ".", "..") for x in value.split("/")) + or any(ord(c) < 32 or ord(c) > 126 or c in '\\:*?"<>|' for c in value) + ): + raise ValueError("Unsafe project resource path") path = root.joinpath(*p.parts) for part in (path, *path.parents): if part == root: break if part.is_symlink(): - raise ValueError('Project sources/resources must not be symbolic links') + raise ValueError("Project sources/resources must not be symbolic links") if not path.resolve().is_relative_to(root) or not path.is_file(): - raise ValueError(f'Missing or external project file: {value}') + raise ValueError(f"Missing or external project file: {value}") return path @@ -51,136 +58,210 @@ def unique_object(pairs): result = {} for k, v in pairs: if k in result: - raise ValueError(f'Duplicate metadata field: {k}') + raise ValueError(f"Duplicate metadata field: {k}") result[k] = v return result def main() -> int: p = argparse.ArgumentParser(description=__doc__) - p.add_argument('project', type=Path) - p.add_argument('--output', type=Path, required=True) - p.add_argument('--clang', default='clang') - p.add_argument('--lld', default='ld.lld') - p.add_argument('--source-map', action='store_true', help='Include callback source locations in this development package') - p.add_argument('--mod-tool', type=Path, required=True) - p.add_argument('--reference-kit',type=Path,help='Address-only pinned kit for a single-revision raw mod') + p.add_argument("project", type=Path) + p.add_argument("--output", type=Path, required=True) + p.add_argument("--clang", default="clang") + p.add_argument("--lld", default="ld.lld") + p.add_argument( + "--source-map", action="store_true", help="Include callback source locations in this development package" + ) + p.add_argument("--mod-tool", type=Path, required=True) + p.add_argument("--reference-kit", type=Path, help="Address-only pinned kit for a single-revision raw mod") a = p.parse_args() project = a.project.resolve(strict=True) - source_config = local(project, 'mod.toml') - config = tomllib.loads(source_config.read_text(encoding='utf-8')) - manifest = config['manifest'] - if not ID.fullmatch(manifest['id']) or manifest.get('game_id') != 'dkr': - raise ValueError('Expected a DKR mod manifest with a portable ID') - if manifest.get('native_libraries') or manifest.get('custom_gamemode'): - raise ValueError('Native libraries and unmanaged custom game modes are not supported') - meta_path = local(project, 'dkr.json') + source_config = local(project, "mod.toml") + config = tomllib.loads(source_config.read_text(encoding="utf-8")) + manifest = config["manifest"] + if not ID.fullmatch(manifest["id"]) or manifest.get("game_id") != "dkr": + raise ValueError("Expected a DKR mod manifest with a portable ID") + if manifest.get("native_libraries") or manifest.get("custom_gamemode"): + raise ValueError("Native libraries and unmanaged custom game modes are not supported") + meta_path = local(project, "dkr.json") if meta_path.stat().st_size > 1024 * 1024: - raise ValueError('DKR metadata exceeds its budget') - meta = json.loads(meta_path.read_text(encoding='utf-8'), object_pairs_hook=unique_object) - if meta.get('schema') != 1 or meta.get('api') not in (1, 2): - raise ValueError('Unsupported DKR SDK metadata version') - reference=SDK/'reference-sdk-only.toml';data_references=[];extra_link=[] + raise ValueError("DKR metadata exceeds its budget") + meta = json.loads(meta_path.read_text(encoding="utf-8"), object_pairs_hook=unique_object) + if meta.get("schema") != 1 or meta.get("api") not in (1, 2): + raise ValueError("Unsupported DKR SDK metadata version") + reference = SDK / "reference-sdk-only.toml" + data_references = [] + extra_link = [] if a.reference_kit: - kit=a.reference_kit.resolve(strict=True) - receipt=json.loads(local(kit,'receipt.json').read_text(encoding='utf-8'),object_pairs_hook=unique_object) - if receipt.get('schema')!=1 or meta.get('revisions')!=[receipt.get('revision')]: - raise ValueError('Raw mods must declare exactly the reference-kit revision') - for name,digest in receipt['files'].items(): - if hashlib.sha256(local(kit,name).read_bytes()).hexdigest()!=digest:raise ValueError('Reference kit was modified') - reference=local(kit,'functions.toml');data_references=[local(kit,'data.toml').as_posix()] - extra_link=['-T',str(local(kit,'symbols.ld'))] + kit = a.reference_kit.resolve(strict=True) + receipt = json.loads(local(kit, "receipt.json").read_text(encoding="utf-8"), object_pairs_hook=unique_object) + if receipt.get("schema") != 1 or meta.get("revisions") != [receipt.get("revision")]: + raise ValueError("Raw mods must declare exactly the reference-kit revision") + for name, digest in receipt["files"].items(): + if hashlib.sha256(local(kit, name).read_bytes()).hexdigest() != digest: + raise ValueError("Reference kit was modified") + reference = local(kit, "functions.toml") + data_references = [local(kit, "data.toml").as_posix()] + extra_link = ["-T", str(local(kit, "symbols.ld"))] resources = {} - for name, value in meta.get('resources', {}).items(): + for name, value in meta.get("resources", {}).items(): if not ID.fullmatch(name) or value in resources: - raise ValueError('Invalid or aliased resource') + raise ValueError("Invalid or aliased resource") resources[value] = local(project, value) if len(resources) > 256 or sum(f.stat().st_size for f in resources.values()) > 32 * 1024 * 1024: - raise ValueError('Resources exceed the portable package budget') + raise ValueError("Resources exceed the portable package budget") sources = set() - for pattern in config['build']['sources']: - if not isinstance(pattern, str) or '..' in PurePosixPath(pattern).parts or PurePosixPath(pattern).is_absolute(): - raise ValueError('Unsafe source pattern') + for pattern in config["build"]["sources"]: + if not isinstance(pattern, str) or ".." in PurePosixPath(pattern).parts or PurePosixPath(pattern).is_absolute(): + raise ValueError("Unsafe source pattern") matches = list(project.glob(pattern)) if not matches: - raise ValueError(f'No source files match {pattern}') + raise ValueError(f"No source files match {pattern}") for match in matches: relative = match.relative_to(project).as_posix() source = local(project, relative) - if source.suffix not in ('.c', '.cpp', '.cc') or source.stat().st_size > 1024 * 1024: - raise ValueError('Only bounded freestanding C or C++ sources are supported') + if source.suffix not in (".c", ".cpp", ".cc") or source.stat().st_size > 1024 * 1024: + raise ValueError("Only bounded freestanding C or C++ sources are supported") sources.add(source) if not sources or len(sources) > 256: - raise ValueError('Source file count exceeds its budget') + raise ValueError("Source file count exceeds its budget") clang, lld = shutil.which(a.clang), shutil.which(a.lld) tool = a.mod_tool.resolve(strict=True) if not clang or not lld or not tool.is_file(): - raise ValueError('Clang, LLD and the pinned RecompModTool are required') + raise ValueError("Clang, LLD and the pinned RecompModTool are required") output = a.output.resolve() if output.exists() or output == project or output in project.parents: - raise ValueError('Output must be a NEW directory, not an input/ancestor') + raise ValueError("Output must be a NEW directory, not an input/ancestor") # Upstream Windows tooling uses a PowerShell ZIP command internally. Keep # its temporary paths strictly free of shell metacharacters; the compiler # invocations themselves always use argument arrays, never shell=True. if any(c in str(output) for c in "'`$\r\n"): - raise ValueError('Choose a tool output path without shell metacharacters') + raise ValueError("Choose a tool output path without shell metacharacters") output.mkdir(parents=True) objects = [] - flags = ['--target=mips-unknown-none', '-march=mips2', '-mabi=32', '-mno-abicalls', - '-fno-pic', '-G0', '-mfp32', '-mno-odd-spreg', '-ffreestanding', '-fno-builtin', - '-fno-stack-protector', '-ffp-contract=off', '-O2', '-Wall', '-Wextra', '-Werror', - '-I' + str(SDK / 'include'), '-I' + str(project)] + flags = [ + "--target=mips-unknown-none", + "-march=mips2", + "-mabi=32", + "-mno-abicalls", + "-fno-pic", + "-G0", + "-mfp32", + "-mno-odd-spreg", + "-ffreestanding", + "-fno-builtin", + "-fno-stack-protector", + "-ffp-contract=off", + "-O2", + "-Wall", + "-Wextra", + "-Werror", + "-I" + str(SDK / "include"), + "-I" + str(project), + ] # Keep complete HI16/LO16 address expressions for relocation at arbitrary # mod bases; machine CSE can otherwise share a LUI across different carries. - flags += ['-fdata-sections', '-mllvm', '-disable-machine-cse'] + flags += ["-fdata-sections", "-mllvm", "-disable-machine-cse"] if a.source_map: - flags += ['-gline-tables-only', '-fdebug-compilation-dir=/mod', '-ffile-prefix-map='+str(project)+'=/mod', '-ffile-prefix-map='+str(SDK)+'=/sdk'] - for index, source in enumerate([SDK/'runtime.c',*sorted(sources)]): - obj = output / f'unit-{index}.o' - extra = ['-fno-exceptions', '-fno-rtti', '-fno-threadsafe-statics', '-std=c++17'] if source.suffix in ('.cpp','.cc') else [] - subprocess.run([clang, *flags, *extra, '-c', str(source), '-o', str(obj)], check=True) + flags += [ + "-gline-tables-only", + "-fdebug-compilation-dir=/mod", + "-ffile-prefix-map=" + str(project) + "=/mod", + "-ffile-prefix-map=" + str(SDK) + "=/sdk", + ] + for index, source in enumerate([SDK / "runtime.c", *sorted(sources)]): + obj = output / f"unit-{index}.o" + extra = ( + ["-fno-exceptions", "-fno-rtti", "-fno-threadsafe-statics", "-std=c++17"] + if source.suffix in (".cpp", ".cc") + else [] + ) + subprocess.run([clang, *flags, *extra, "-c", str(source), "-o", str(obj)], check=True) objects.append(obj) - elf = output / 'mod.elf' - subprocess.run([lld, '-m', 'elf32btsmip', '-static', '--emit-relocs', '-T', str(SDK / 'mod.ld'), - *extra_link,'-e', '0', '-o', str(elf), *(str(o) for o in objects)], check=True) - lines = ['[manifest]'] + elf = output / "mod.elf" + subprocess.run( + [ + lld, + "-m", + "elf32btsmip", + "-static", + "--emit-relocs", + "-T", + str(SDK / "mod.ld"), + *extra_link, + "-e", + "0", + "-o", + str(elf), + *(str(o) for o in objects), + ], + check=True, + ) + lines = ["[manifest]"] for key, value in manifest.items(): - if key != 'config_options': - if not re.fullmatch(r'[a-z_]+', key): - raise ValueError('Invalid manifest key') - lines.append(f'{key} = {literal(value)}') - for option in manifest.get('config_options', []): - lines.append('\n[[manifest.config_options]]') + if key != "config_options": + if not re.fullmatch(r"[a-z_]+", key): + raise ValueError("Invalid manifest key") + lines.append(f"{key} = {literal(value)}") + for option in manifest.get("config_options", []): + lines.append("\n[[manifest.config_options]]") for key, value in option.items(): - if not re.fullmatch(r'[a-z_]+', key): - raise ValueError('Invalid option key') - lines.append(f'{key} = {literal(value)}') - lines += ['\n[inputs]', 'elf_path = "mod.elf"', 'mod_filename = "tool-output"', - 'func_reference_syms_file = ' + literal(reference.as_posix()), - 'data_reference_syms_files = '+literal(data_references), 'additional_files = []'] - tool_config = output / 'tool.toml' - tool_config.write_text('\n'.join(lines) + '\n', encoding='utf-8') + if not re.fullmatch(r"[a-z_]+", key): + raise ValueError("Invalid option key") + lines.append(f"{key} = {literal(value)}") + lines += [ + "\n[inputs]", + 'elf_path = "mod.elf"', + 'mod_filename = "tool-output"', + "func_reference_syms_file = " + literal(reference.as_posix()), + "data_reference_syms_files = " + literal(data_references), + "additional_files = []", + ] + tool_config = output / "tool.toml" + tool_config.write_text("\n".join(lines) + "\n", encoding="utf-8") subprocess.run([str(tool), str(tool_config), str(output)], check=True) import sys - subprocess.run([sys.executable, '-B', str(SDK/'relocations.py'), '--elf', str(elf), - '--symbols', str(output/'mod_syms.bin'), '--binary', str(output/'mod_binary.bin'), - '--report', str(output/'relocation-report.json')], check=True) - package = output / (manifest['id'] + '.nrm') + + subprocess.run( + [ + sys.executable, + "-B", + str(SDK / "relocations.py"), + "--elf", + str(elf), + "--symbols", + str(output / "mod_syms.bin"), + "--binary", + str(output / "mod_binary.bin"), + "--report", + str(output / "relocation-report.json"), + ], + check=True, + ) + package = output / (manifest["id"] + ".nrm") if a.source_map: import sys - if 'source-map' in meta.get('resources', {}) or 'source-map.json' in resources: - raise ValueError('The project already declares a source map; omit --source-map to retain it') - source_map = output/'source-map.json' - subprocess.run([sys.executable, '-B', str(SDK/'tools.py'), 'source-map', str(elf), '--output', str(source_map)], check=True) - meta.setdefault('resources', {})['source-map']='source-map.json' - resources['source-map.json']=source_map - meta_path=output/'dkr-staged.json' - meta_path.write_text(json.dumps(meta, indent=2)+'\n', encoding='utf-8') - fixed = {'mod.json': output / 'mod.json', 'mod_syms.bin': output / 'mod_syms.bin', - 'mod_binary.bin': output / 'mod_binary.bin', 'dkr.json': meta_path} + + if "source-map" in meta.get("resources", {}) or "source-map.json" in resources: + raise ValueError("The project already declares a source map; omit --source-map to retain it") + source_map = output / "source-map.json" + subprocess.run( + [sys.executable, "-B", str(SDK / "tools.py"), "source-map", str(elf), "--output", str(source_map)], + check=True, + ) + meta.setdefault("resources", {})["source-map"] = "source-map.json" + resources["source-map.json"] = source_map + meta_path = output / "dkr-staged.json" + meta_path.write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8") + fixed = { + "mod.json": output / "mod.json", + "mod_syms.bin": output / "mod_syms.bin", + "mod_binary.bin": output / "mod_binary.bin", + "dkr.json": meta_path, + } if any(name in fixed for name in resources): - raise ValueError('A resource cannot replace reserved package metadata') - with zipfile.ZipFile(package, 'x', zipfile.ZIP_DEFLATED, compresslevel=6) as archive: + raise ValueError("A resource cannot replace reserved package metadata") + with zipfile.ZipFile(package, "x", zipfile.ZIP_DEFLATED, compresslevel=6) as archive: for name, file in sorted({**fixed, **resources}.items()): # Portable/reproducible archive metadata, not the developer's # platform attributes, usernames or source modification times. @@ -190,11 +271,11 @@ def main() -> int: info.compress_type = zipfile.ZIP_DEFLATED archive.writestr(info, file.read_bytes(), compresslevel=6) if package.stat().st_size > 32 * 1024 * 1024: - raise ValueError('Compiled package exceeds its import size limit') - print(f'Compiled portable code package: {package}') - print('No game/mod execution or runtime qualification was performed.') + raise ValueError("Compiled package exceeds its import size limit") + print(f"Compiled portable code package: {package}") + print("No game/mod execution or runtime qualification was performed.") return 0 -if __name__ == '__main__': +if __name__ == "__main__": raise SystemExit(main()) diff --git a/modding/examples/dev-menu/tools.py b/modding/examples/dev-menu/tools.py index 25a71262..da3f95d8 100644 --- a/modding/examples/dev-menu/tools.py +++ b/modding/examples/dev-menu/tools.py @@ -1,89 +1,309 @@ """Immutable Dev Menu UI resource, generated into the SDK staging copy.""" + + def schema(): - def text(id,label):return dict(id=id,type='text',label=label) - def button(id,label,action,op=None,confirm=False): - value=dict(id=id,type='button',label=label,action=action) - if op is not None:value['visible']='op_'+str(op) - if confirm:value['confirm']=True + def text(id, label): + return dict(id=id, type="text", label=label) + + def button(id, label, action, op=None, confirm=False): + value = dict(id=id, type="button", label=label, action=action) + if op is not None: + value["visible"] = "op_" + str(op) + if confirm: + value["confirm"] = True return value - def integer(id,label,argument,default=0,minimum=0,maximum=100): - return dict(id=id,type='integer',label=label,argument=argument,default=default,min=minimum,max=maximum) - def vector(id,label,argument):return dict(id=id,type='vector',label=label,argument=argument) - def number(id,label,argument,default=0.0,minimum=-1000.0,maximum=1000.0):return dict(id=id,type='number',label=label,argument=argument,default=default,min=minimum,max=maximum) - def table(id,label,select=False):return dict(id=id,type='table',label=label,select=select) - def page(id,label,*fields):return dict(id=id,label=label,fields=list(fields)) - target=[dict(id='live-inspector',type='inspector',label='Live entity inspector',edit_action=101)] - return dict(schema=2,title='DKR-R Dev Menu',pages=[ - page('session','Session',text('message','Last operation'),text('authority','Session authority'), - text('scene','Scene generation'),text('tick','Simulation tick'),text('save_status','Save protection'), - button('pause','Pause shared gameplay',20),button('resume','Resume / reset normal speed',21),button('step','Advance one update while paused',22), - dict(id='speed',type='enum',label='Gameplay speed (retail logic units, not audio pitch)',argument=2,default=1,choices=['0.5x','1x','1.5x','2x']),button('set-speed','Apply gameplay speed',23), - button('restart','Restart this scene',150,30,True), - integer('restart-entrance','Existing entrance for this scene',2,0,0,15),button('restart-at','Restart at this entrance (all racer anchors required)',153,30,True), - integer('race-laps','Ordinary race laps: 1..3 (0 resets)',2,3,0,3),button('apply-laps','Set laps before anyone completes the first lap',154,40,True), - integer('course','Course catalogue index',2,0,0,1023),integer('vehicle','Vehicle: 0 Car / 1 Hovercraft / 2 Plane',3,0,0,2), - integer('entrance','Entrance (other courses use their default spawn)',4,0,0,15),button('warp','Load this admitted course',151,31,True), - dict(id='return-target',type='enum',label='Return using the normal game transition',argument=2,choices=['Track Select','Character Select','Start menu']),button('return-menu','Return to selected menu',152,32,True), - button('reset','Reset Dev Menu racer overrides',2,None,True)), - page('racers','Racers / AI',*target,integer('bananas','Bananas',2,10,0,99),button('set-bananas','Set selected racer bananas',105), - integer('skill','AI skill',2,50,0,100),button('set-skill','Set selected racer AI skill',106), - integer('ammo','Item ammunition',2,3,0,10),button('set-ammo','Set selected racer ammunition',107), - button('stop','Stop selected racer',104),button('freeze','Toggle CPU freeze',160), - button('boost','Toggle selected racer continuous boost',161),button('shield','Toggle selected racer shield timer',162), - button('keep-bananas','Toggle selected racer 10 bananas',163),text('overrides','Active overrides'), - number('weight','Selected racer weight / gravity factor',2,1,0.1,4),number('handling','Handling factor',3,1,0.1,4),number('acceleration','Acceleration curve factor (standard vehicles)',4,1,0.1,4),number('speed-cap','Forward speed cap (0 = off)',5,0,0,100), - button('apply-tune','Apply selected racer tuning',182),button('reset-tune','Reset this tool\'s racer tuning',181,None,True)), - page('objects','Objects',*target, - integer('property','Property ID (see SDK reference)',2,102,1,301),button('inspect','Inspect selected property',100), - integer('value','Integer property value',3,10,-32768,65535),button('edit','Apply selected integer property',101), - dict(id='float-value',type='number',label='Float property value',argument=3,default=1.0,min=-1000.0,max=1000.0),button('edit-float','Apply selected float property',101), - button('undo','Undo last property edit',102),button('redo','Redo property edit',103), - vector('position','Destination / clone position: X Y Z',2),button('teleport','Teleport selected entity',110,1), - button('clone','Clone selected scenery template',111,2),button('delete','Delete selected tool-owned spawn',112,3,True)), - page('track','Track / Routes',integer('catalogue-page','First course catalogue index (8 entries per page)',2,0,0,1016), - button('read-catalogue','Read this course page',12),table('courses','Admitted stock, legacy and Track Lab courses'), - dict(id='debug-display',type='debug',label='Show local route, selected entity and probe overlays'), - dict(id='scene-catalogue',type='enum',label='Loaded-scene inspector',argument=2,choices=['Scene / rules','Segment bounds','AI node links','Spawn points / entrances','Trigger volumes','Triangle geometry']), - integer('scene-entry','First scene entry (4 per page)',3,0,0,8191),integer('scene-segment','Segment for triangle inspection',4,0,0,1023),button('inspect-scene','Read current-scene metadata',17),table('geometry','Loaded scene, geometry and navigation'), - integer('draw-segment','Segment to highlight',2,0,0,1023),integer('draw-triangle','Triangle to highlight',3,0,0,8191), - dict(id='geometry-mode',type='enum',label='Overlay highlight',argument=4,default=0,choices=['Racing route','Segment bounds','Triangle wireframe / normal']),button('draw-geometry','Update geometry highlight',18), - integer('route-page','Checkpoint index / first route on this page',2,0,0,126),button('read-route','Inspect native checkpoint / AI racing line',14),table('routes','Native main and alternate racing routes',True), - integer('checkpoint-index','Checkpoint to move',2,0,0,126),vector('checkpoint-position','Checkpoint position: X Y Z',3), - integer('checkpoint-yaw','Yaw (native angle16)',6,0,-32768,32767),integer('checkpoint-pitch','Pitch (native angle16)',7,0,-32768,32767),integer('checkpoint-roll','Roll (native angle16)',8,0,-32768,32767), - number('checkpoint-scale','Checkpoint scale',9,1.0,0.01,16.0),button('move-checkpoint','Move checkpoint and rebuild native route planes',171,4,True), - button('prepare-checkpoint','Prepare the inspected checkpoint as a patch',183),dict(id='checkpoint-document',type='document',label='Export or review a checkpoint adjustment',document='checkpoint-patch',import_action=184), - vector('probe-from','Probe from: X Y Z',2),vector('probe-to','Probe to: X Y Z',5),number('probe-radius','Probe radius (0 = ray)',8,0,0,512), - integer('probe-flags','0 Static / 1 Static + interaction spheres / 3 Spheres only',9,0,0,3),button('cast','Run bounded collision probe',170),text('probe','Probe result'), - button('prepare-document','Prepare property-edit / bookmark document',179),dict(id='scene-document',type='document',label='Export or review a scene adjustment document',document='scene-patch',import_action=180), - text('entity_count','Live entity count'),text('racer_count','Current racer count'),text('message','Last operation')), - page('camera','Camera / Bookmarks',*target,button('bookmark','Mark selected entity position',120), - button('return','Return selected entity to marked position',121,1), - dict(id='local-camera',type='camera',label='Local free, follow and spectator camera')), - page('environment','Environment / Audio',integer('fog-player','Fog viewport (0..3)',2,0,0,3), - integer('near','Fog near',3,900,0,1018),integer('far','Fog far',4,1000,5,1023), - integer('red','Fog red',5,128,0,255),integer('green','Fog green',6,128,0,255),integer('blue','Fog blue',7,128,0,255), - button('fog','Apply original game fog',130,10),*target, - number('ambient','Selected object ambient light',2,1.0,0,2),button('ambient-apply','Apply selected object ambient light',190), - number('diffuse','Selected object diffuse light',2,1.0,0,2),button('diffuse-apply','Apply selected object diffuse light',191), - integer('weather-x','Existing weather X velocity',2,0,-1048576,1048576),integer('weather-y','Existing weather Y velocity',3,-256,-1048576,1048576),integer('weather-z','Existing weather Z velocity',4,0,-1048576,1048576), - integer('weather-intensity','Weather intensity',5,16384,0,65536),integer('weather-opacity','Weather opacity',6,32768,0,65536),integer('weather-transition','Weather transition ticks',7,120,1,3600),button('weather','Adjust initialized scene weather',131,11), - integer('sound','Sound / sequence ID',2,0,0,65535), - button('sound-play','Play sound',140,20),button('music-play','Play music sequence',141,21), - button('music-stop','Stop music',142,22),integer('fade-value','Music fade (-4096..4096)',2,-60,-4096,4096),button('fade','Fade music',143,23)), - page('mods','Mods / Values',integer('package-page','First code package index (8 per page)',2,0,0,120),button('read-packages','Read this package page',13), - table('packages','Active portable code packages and capabilities'),dict(id='reload-code',type='reload',label='Offline compatible code reload'),text('instructions','Remaining callback instruction budget'), - dict(id='resource-kind',type='enum',label='Resource catalogue',argument=2,choices=['Characters','Character models / headers','Object models','3D textures','2D textures','Sprites','Object templates','Level models','Custom cups','Sounds','Music sequences','All mod settings','Hooks','Callbacks','This mod resources']), - integer('resource-page','First resource index (8 per page)',3,0,0,16376),button('read-resources','Browse this resource page',16),table('resources','Verified resource identities and live-setting policy'), - integer('setting-package','Setting owner package index',2,0,0,127),integer('setting-option','Option index in that package',3,0,0,127), - number('setting-number','Numeric setting value',4,1,-1e6,1e6),button('apply-setting-number','Apply opted-in numeric setting',200), - integer('setting-choice','Enum choice index',4,0,0,255),button('apply-setting-enum','Apply opted-in enum setting',201),button('reset-setting','Reset selected live setting',202), - text('settings-policy','Live setting policy'),text('watch','Selected property watch'), - text('native_calls','Remaining native-call budget'),text('last_action','Last deterministic command ID'),dict(id='costs',type='diagnostics',label='Per-package work and memory'), - integer('symbol-page','First decomp symbol index (12 entries per page)',2,0,0,8192),button('read-symbols','Read reviewed symbol / hook capabilities',15),table('symbols','Decomp symbols: callable, hookable or protected'),table('trace','Recent deterministic lifecycle/racer events')), - page('commands','Commands / History',text('message','Last command result'),text('last_action','Last deterministic command ID'), - dict(id='command-console',type='console',label='Registered SDK commands and recorded macros'), - button('refresh','Refresh tools snapshot',1),button('undo','Undo last typed property edit',102),button('redo','Redo typed property edit',103)), - page('online','Online Development',text('authority','Command authority'),text('save_status','Development save scope'), - dict(id='online-authority',type='authority',label='Host permissions, pending requests and shared action audit'), - text('scene','Scene generation'),text('tick','Current simulated tick'),text('last_action','Last ordered action'),text('message','Last operation')) - ]) + + def integer(id, label, argument, default=0, minimum=0, maximum=100): + return dict(id=id, type="integer", label=label, argument=argument, default=default, min=minimum, max=maximum) + + def vector(id, label, argument): + return dict(id=id, type="vector", label=label, argument=argument) + + def number(id, label, argument, default=0.0, minimum=-1000.0, maximum=1000.0): + return dict(id=id, type="number", label=label, argument=argument, default=default, min=minimum, max=maximum) + + def table(id, label, select=False): + return dict(id=id, type="table", label=label, select=select) + + def page(id, label, *fields): + return dict(id=id, label=label, fields=list(fields)) + + target = [dict(id="live-inspector", type="inspector", label="Live entity inspector", edit_action=101)] + return dict( + schema=2, + title="DKR-R Dev Menu", + pages=[ + page( + "session", + "Session", + text("message", "Last operation"), + text("authority", "Session authority"), + text("scene", "Scene generation"), + text("tick", "Simulation tick"), + text("save_status", "Save protection"), + button("pause", "Pause shared gameplay", 20), + button("resume", "Resume / reset normal speed", 21), + button("step", "Advance one update while paused", 22), + dict( + id="speed", + type="enum", + label="Gameplay speed (retail logic units, not audio pitch)", + argument=2, + default=1, + choices=["0.5x", "1x", "1.5x", "2x"], + ), + button("set-speed", "Apply gameplay speed", 23), + button("restart", "Restart this scene", 150, 30, True), + integer("restart-entrance", "Existing entrance for this scene", 2, 0, 0, 15), + button("restart-at", "Restart at this entrance (all racer anchors required)", 153, 30, True), + integer("race-laps", "Ordinary race laps: 1..3 (0 resets)", 2, 3, 0, 3), + button("apply-laps", "Set laps before anyone completes the first lap", 154, 40, True), + integer("course", "Course catalogue index", 2, 0, 0, 1023), + integer("vehicle", "Vehicle: 0 Car / 1 Hovercraft / 2 Plane", 3, 0, 0, 2), + integer("entrance", "Entrance (other courses use their default spawn)", 4, 0, 0, 15), + button("warp", "Load this admitted course", 151, 31, True), + dict( + id="return-target", + type="enum", + label="Return using the normal game transition", + argument=2, + choices=["Track Select", "Character Select", "Start menu"], + ), + button("return-menu", "Return to selected menu", 152, 32, True), + button("reset", "Reset Dev Menu racer overrides", 2, None, True), + ), + page( + "racers", + "Racers / AI", + *target, + integer("bananas", "Bananas", 2, 10, 0, 99), + button("set-bananas", "Set selected racer bananas", 105), + integer("skill", "AI skill", 2, 50, 0, 100), + button("set-skill", "Set selected racer AI skill", 106), + integer("ammo", "Item ammunition", 2, 3, 0, 10), + button("set-ammo", "Set selected racer ammunition", 107), + button("stop", "Stop selected racer", 104), + button("freeze", "Toggle CPU freeze", 160), + button("boost", "Toggle selected racer continuous boost", 161), + button("shield", "Toggle selected racer shield timer", 162), + button("keep-bananas", "Toggle selected racer 10 bananas", 163), + text("overrides", "Active overrides"), + number("weight", "Selected racer weight / gravity factor", 2, 1, 0.1, 4), + number("handling", "Handling factor", 3, 1, 0.1, 4), + number("acceleration", "Acceleration curve factor (standard vehicles)", 4, 1, 0.1, 4), + number("speed-cap", "Forward speed cap (0 = off)", 5, 0, 0, 100), + button("apply-tune", "Apply selected racer tuning", 182), + button("reset-tune", "Reset this tool's racer tuning", 181, None, True), + ), + page( + "objects", + "Objects", + *target, + integer("property", "Property ID (see SDK reference)", 2, 102, 1, 301), + button("inspect", "Inspect selected property", 100), + integer("value", "Integer property value", 3, 10, -32768, 65535), + button("edit", "Apply selected integer property", 101), + dict( + id="float-value", + type="number", + label="Float property value", + argument=3, + default=1.0, + min=-1000.0, + max=1000.0, + ), + button("edit-float", "Apply selected float property", 101), + button("undo", "Undo last property edit", 102), + button("redo", "Redo property edit", 103), + vector("position", "Destination / clone position: X Y Z", 2), + button("teleport", "Teleport selected entity", 110, 1), + button("clone", "Clone selected scenery template", 111, 2), + button("delete", "Delete selected tool-owned spawn", 112, 3, True), + ), + page( + "track", + "Track / Routes", + integer("catalogue-page", "First course catalogue index (8 entries per page)", 2, 0, 0, 1016), + button("read-catalogue", "Read this course page", 12), + table("courses", "Admitted stock, legacy and Track Lab courses"), + dict(id="debug-display", type="debug", label="Show local route, selected entity and probe overlays"), + dict( + id="scene-catalogue", + type="enum", + label="Loaded-scene inspector", + argument=2, + choices=[ + "Scene / rules", + "Segment bounds", + "AI node links", + "Spawn points / entrances", + "Trigger volumes", + "Triangle geometry", + ], + ), + integer("scene-entry", "First scene entry (4 per page)", 3, 0, 0, 8191), + integer("scene-segment", "Segment for triangle inspection", 4, 0, 0, 1023), + button("inspect-scene", "Read current-scene metadata", 17), + table("geometry", "Loaded scene, geometry and navigation"), + integer("draw-segment", "Segment to highlight", 2, 0, 0, 1023), + integer("draw-triangle", "Triangle to highlight", 3, 0, 0, 8191), + dict( + id="geometry-mode", + type="enum", + label="Overlay highlight", + argument=4, + default=0, + choices=["Racing route", "Segment bounds", "Triangle wireframe / normal"], + ), + button("draw-geometry", "Update geometry highlight", 18), + integer("route-page", "Checkpoint index / first route on this page", 2, 0, 0, 126), + button("read-route", "Inspect native checkpoint / AI racing line", 14), + table("routes", "Native main and alternate racing routes", True), + integer("checkpoint-index", "Checkpoint to move", 2, 0, 0, 126), + vector("checkpoint-position", "Checkpoint position: X Y Z", 3), + integer("checkpoint-yaw", "Yaw (native angle16)", 6, 0, -32768, 32767), + integer("checkpoint-pitch", "Pitch (native angle16)", 7, 0, -32768, 32767), + integer("checkpoint-roll", "Roll (native angle16)", 8, 0, -32768, 32767), + number("checkpoint-scale", "Checkpoint scale", 9, 1.0, 0.01, 16.0), + button("move-checkpoint", "Move checkpoint and rebuild native route planes", 171, 4, True), + button("prepare-checkpoint", "Prepare the inspected checkpoint as a patch", 183), + dict( + id="checkpoint-document", + type="document", + label="Export or review a checkpoint adjustment", + document="checkpoint-patch", + import_action=184, + ), + vector("probe-from", "Probe from: X Y Z", 2), + vector("probe-to", "Probe to: X Y Z", 5), + number("probe-radius", "Probe radius (0 = ray)", 8, 0, 0, 512), + integer("probe-flags", "0 Static / 1 Static + interaction spheres / 3 Spheres only", 9, 0, 0, 3), + button("cast", "Run bounded collision probe", 170), + text("probe", "Probe result"), + button("prepare-document", "Prepare property-edit / bookmark document", 179), + dict( + id="scene-document", + type="document", + label="Export or review a scene adjustment document", + document="scene-patch", + import_action=180, + ), + text("entity_count", "Live entity count"), + text("racer_count", "Current racer count"), + text("message", "Last operation"), + ), + page( + "camera", + "Camera / Bookmarks", + *target, + button("bookmark", "Mark selected entity position", 120), + button("return", "Return selected entity to marked position", 121, 1), + dict(id="local-camera", type="camera", label="Local free, follow and spectator camera"), + ), + page( + "environment", + "Environment / Audio", + integer("fog-player", "Fog viewport (0..3)", 2, 0, 0, 3), + integer("near", "Fog near", 3, 900, 0, 1018), + integer("far", "Fog far", 4, 1000, 5, 1023), + integer("red", "Fog red", 5, 128, 0, 255), + integer("green", "Fog green", 6, 128, 0, 255), + integer("blue", "Fog blue", 7, 128, 0, 255), + button("fog", "Apply original game fog", 130, 10), + *target, + number("ambient", "Selected object ambient light", 2, 1.0, 0, 2), + button("ambient-apply", "Apply selected object ambient light", 190), + number("diffuse", "Selected object diffuse light", 2, 1.0, 0, 2), + button("diffuse-apply", "Apply selected object diffuse light", 191), + integer("weather-x", "Existing weather X velocity", 2, 0, -1048576, 1048576), + integer("weather-y", "Existing weather Y velocity", 3, -256, -1048576, 1048576), + integer("weather-z", "Existing weather Z velocity", 4, 0, -1048576, 1048576), + integer("weather-intensity", "Weather intensity", 5, 16384, 0, 65536), + integer("weather-opacity", "Weather opacity", 6, 32768, 0, 65536), + integer("weather-transition", "Weather transition ticks", 7, 120, 1, 3600), + button("weather", "Adjust initialized scene weather", 131, 11), + integer("sound", "Sound / sequence ID", 2, 0, 0, 65535), + button("sound-play", "Play sound", 140, 20), + button("music-play", "Play music sequence", 141, 21), + button("music-stop", "Stop music", 142, 22), + integer("fade-value", "Music fade (-4096..4096)", 2, -60, -4096, 4096), + button("fade", "Fade music", 143, 23), + ), + page( + "mods", + "Mods / Values", + integer("package-page", "First code package index (8 per page)", 2, 0, 0, 120), + button("read-packages", "Read this package page", 13), + table("packages", "Active portable code packages and capabilities"), + dict(id="reload-code", type="reload", label="Offline compatible code reload"), + text("instructions", "Remaining callback instruction budget"), + dict( + id="resource-kind", + type="enum", + label="Resource catalogue", + argument=2, + choices=[ + "Characters", + "Character models / headers", + "Object models", + "3D textures", + "2D textures", + "Sprites", + "Object templates", + "Level models", + "Custom cups", + "Sounds", + "Music sequences", + "All mod settings", + "Hooks", + "Callbacks", + "This mod resources", + ], + ), + integer("resource-page", "First resource index (8 per page)", 3, 0, 0, 16376), + button("read-resources", "Browse this resource page", 16), + table("resources", "Verified resource identities and live-setting policy"), + integer("setting-package", "Setting owner package index", 2, 0, 0, 127), + integer("setting-option", "Option index in that package", 3, 0, 0, 127), + number("setting-number", "Numeric setting value", 4, 1, -1e6, 1e6), + button("apply-setting-number", "Apply opted-in numeric setting", 200), + integer("setting-choice", "Enum choice index", 4, 0, 0, 255), + button("apply-setting-enum", "Apply opted-in enum setting", 201), + button("reset-setting", "Reset selected live setting", 202), + text("settings-policy", "Live setting policy"), + text("watch", "Selected property watch"), + text("native_calls", "Remaining native-call budget"), + text("last_action", "Last deterministic command ID"), + dict(id="costs", type="diagnostics", label="Per-package work and memory"), + integer("symbol-page", "First decomp symbol index (12 entries per page)", 2, 0, 0, 8192), + button("read-symbols", "Read reviewed symbol / hook capabilities", 15), + table("symbols", "Decomp symbols: callable, hookable or protected"), + table("trace", "Recent deterministic lifecycle/racer events"), + ), + page( + "commands", + "Commands / History", + text("message", "Last command result"), + text("last_action", "Last deterministic command ID"), + dict(id="command-console", type="console", label="Registered SDK commands and recorded macros"), + button("refresh", "Refresh tools snapshot", 1), + button("undo", "Undo last typed property edit", 102), + button("redo", "Redo typed property edit", 103), + ), + page( + "online", + "Online Development", + text("authority", "Command authority"), + text("save_status", "Development save scope"), + dict( + id="online-authority", + type="authority", + label="Host permissions, pending requests and shared action audit", + ), + text("scene", "Scene generation"), + text("tick", "Current simulated tick"), + text("last_action", "Last ordered action"), + text("message", "Last operation"), + ), + ], + ) diff --git a/modding/relocations.py b/modding/relocations.py index 518e012c..0843fcae 100644 --- a/modding/relocations.py +++ b/modding/relocations.py @@ -7,6 +7,7 @@ Resolve LOCAL records from the fully linked ELF, preserving every other byte. No dependency, game loader, permission or memory budget is changed here. """ + import argparse import hashlib import json @@ -16,156 +17,228 @@ MAX_CODE = 4 * 1024 * 1024 MAX_FUNCTIONS = 16384 LOCAL = 0x80000000 -IMPORT, EVENT = 0xfffffffe, 0xfffffffd +IMPORT, EVENT = 0xFFFFFFFE, 0xFFFFFFFD SUPPORTED = (0, 2, 4, 5, 6) def require(condition, message): if not condition: - raise ValueError('Static relocation inspection: ' + message) + raise ValueError("Static relocation inspection: " + message) def region(data, offset, size): - require(0 <= offset <= len(data) and 0 <= size <= len(data)-offset, 'truncated binary table') - return data[offset:offset+size] + require(0 <= offset <= len(data) and 0 <= size <= len(data) - offset, "truncated binary table") + return data[offset : offset + size] def symbols(data, binary, *, targets=True): """Mirror the existing loader's complete symbol-wire preflight, read-only.""" - require(len(data) >= 52 and data[:8] == b'N64RSYMS', 'invalid symbol header') - word = lambda at: struct.unpack('= 52 and data[:8] == b"N64RSYMS", "invalid symbol header") + word = lambda at: struct.unpack("= 4 and kind in SUPPORTED, 'unsupported or out-of-bounds relocation') - if owner == IMPORT: require(kind == 4 and target < counts[2], 'invalid import relocation') - elif owner == EVENT: require(kind == 4 and target < counts[7], 'invalid event relocation') - elif owner & LOCAL: require(owner & ~LOCAL < counts[0], 'invalid local relocation owner') - else: require(owner == 0x1000 and target < 0x800000, 'invalid base-game relocation') + slot = r + 16 * i + off, kind, target, owner = struct.unpack_from("<4I", data, slot) + require( + off % 4 == 0 and off <= size and size - off >= 4 and kind in SUPPORTED, + "unsupported or out-of-bounds relocation", + ) + if owner == IMPORT: + require(kind == 4 and target < counts[2], "invalid import relocation") + elif owner == EVENT: + require(kind == 4 and target < counts[7], "invalid event relocation") + elif owner & LOCAL: + require(owner & ~LOCAL < counts[0], "invalid local relocation owner") + else: + require(owner == 0x1000 and target < 0x800000, "invalid base-game relocation") relocs.append(dict(section=index, off=off, kind=kind, target=target, owner=owner, slot=slot)) if targets: for r in relocs: - if r['owner'] & LOCAL and r['owner'] not in (IMPORT, EVENT): - s = sections[r['owner'] & ~LOCAL] - require(r['target'] <= s['size']+s['bss'], 'local relocation is outside its section at '+hex(sections[r['section']]['ram']+r['off'])) - t = table(12, counts[1]); deps = set() + if r["owner"] & LOCAL and r["owner"] not in (IMPORT, EVENT): + s = sections[r["owner"] & ~LOCAL] + require( + r["target"] <= s["size"] + s["bss"], + "local relocation is outside its section at " + hex(sections[r["section"]]["ram"] + r["off"]), + ) + t = table(12, counts[1]) + deps = set() for i in range(counts[1]): - value = name(word(t+12*i+4), word(t+12*i+8)) - require(value not in deps, 'duplicate dependency'); deps.add(value) + value = name(word(t + 12 * i + 4), word(t + 12 * i + 8)) + require(value not in deps, "duplicate dependency") + deps.add(value) for group in (2, 3): - t = table(12, counts[group]); seen = set() + t = table(12, counts[group]) + seen = set() for i in range(counts[group]): - start, length, dep = struct.unpack_from('<3I', data, t+12*i) + start, length, dep = struct.unpack_from("<3I", data, t + 12 * i) value = (dep, name(start, length)) - require(dep < counts[1] and value not in seen, 'invalid dependency symbol'); seen.add(value) + require(dep < counts[1] and value not in seen, "invalid dependency symbol") + seen.add(value) t = table(16, counts[4]) for i in range(counts[4]): - func, rom, off, flags = struct.unpack_from('<4I', data, t+16*i) - require(func < functions and rom == 0x1000 and off % 4 == 0 and flags <= 1, 'invalid replacement') - t = table(12, counts[5]); exported = set(); names = set() + func, rom, off, flags = struct.unpack_from("<4I", data, t + 16 * i) + require(func < functions and rom == 0x1000 and off % 4 == 0 and flags <= 1, "invalid replacement") + t = table(12, counts[5]) + exported = set() + names = set() for i in range(counts[5]): - func, start, length = struct.unpack_from('<3I', data, t+12*i); value = name(start, length) - require(func < functions and func not in exported and value not in names, 'invalid export') - exported.add(func); names.add(value) + func, start, length = struct.unpack_from("<3I", data, t + 12 * i) + value = name(start, length) + require(func < functions and func not in exported and value not in names, "invalid export") + exported.add(func) + names.add(value) t = table(8, counts[6]) for i in range(counts[6]): - event, func = struct.unpack_from('<2I', data, t+8*i) - require(event < counts[3] and func < functions, 'invalid callback') - t = table(8, counts[7]); seen = set() + event, func = struct.unpack_from("<2I", data, t + 8 * i) + require(event < counts[3] and func < functions, "invalid callback") + t = table(8, counts[7]) + seen = set() for i in range(counts[7]): - value = name(word(t+8*i), word(t+8*i+4)); require(value not in seen, 'duplicate event'); seen.add(value) + value = name(word(t + 8 * i), word(t + 8 * i + 4)) + require(value not in seen, "duplicate event") + seen.add(value) t = table(16, counts[8]) for i in range(counts[8]): - func, rom, off, flags = struct.unpack_from('<4I', data, t+16*i) - require(func < functions and rom == 0x1000 and off % 4 == 0 and flags <= 1, 'invalid hook') - require(at == len(data), 'unexpected trailing symbol bytes') + func, rom, off, flags = struct.unpack_from("<4I", data, t + 16 * i) + require(func < functions and rom == 0x1000 and off % 4 == 0 and flags <= 1, "invalid hook") + require(at == len(data), "unexpected trailing symbol bytes") return sections, relocs def linked_elf(data): - require(data[:7] == b'\x7fELF\x01\x02\x01', 'expected big-endian ELF32') - header = struct.unpack('>HHIIIIIHHHHHH', region(data, 16, 36)) - require(header[0:2] == (2, 8) and header[10] == 40 and 0 < header[11] < 65536, 'expected linked MIPS ELF') + require(data[:7] == b"\x7fELF\x01\x02\x01", "expected big-endian ELF32") + header = struct.unpack(">HHIIIIIHHHHHH", region(data, 16, 36)) + require(header[0:2] == (2, 8) and header[10] == 40 and 0 < header[11] < 65536, "expected linked MIPS ELF") sections = [] for i in range(header[11]): - values = struct.unpack('>10I', region(data, header[5]+40*i, 40)) - section = dict(zip(('name','type','flags','addr','offset','size','link','info','align','entsize'), values)) - if section['type'] != 8: region(data, section['offset'], section['size']) - require(section['addr']+section['size'] <= 0x100000000, 'ELF address overflow') + values = struct.unpack(">10I", region(data, header[5] + 40 * i, 40)) + section = dict( + zip(("name", "type", "flags", "addr", "offset", "size", "link", "info", "align", "entsize"), values) + ) + if section["type"] != 8: + region(data, section["offset"], section["size"]) + require(section["addr"] + section["size"] <= 0x100000000, "ELF address overflow") sections.append(section) - result = {}; pairs = []; orphan_lows = [] + result = {} + pairs = [] + orphan_lows = [] for rel in sections: - if rel['type'] != 9: continue - require(rel['info'] < len(sections) and rel['link'] < len(sections) and rel['entsize'] == 8 and rel['size'] % 8 == 0, 'invalid ELF relocation table') - source = sections[rel['info']] - if not source['flags'] & 2: continue # Debug-only relocations are not loaded. - syms = sections[rel['link']] - require(source['type'] != 8 and syms['type'] == 2 and syms['entsize'] == 16 and syms['size'] % 16 == 0, 'invalid ELF symbol/source table') + if rel["type"] != 9: + continue + require( + rel["info"] < len(sections) + and rel["link"] < len(sections) + and rel["entsize"] == 8 + and rel["size"] % 8 == 0, + "invalid ELF relocation table", + ) + source = sections[rel["info"]] + if not source["flags"] & 2: + continue # Debug-only relocations are not loaded. + syms = sections[rel["link"]] + require( + source["type"] != 8 and syms["type"] == 2 and syms["entsize"] == 16 and syms["size"] % 16 == 0, + "invalid ELF symbol/source table", + ) pending = [] - for at in range(rel['offset'], rel['offset']+rel['size'], 8): - address, info = struct.unpack_from('>2I', data, at); kind = info & 255; symbol_index = info >> 8 - require(symbol_index < syms['size']//16 and kind in SUPPORTED, 'unsupported ELF relocation') - require(source['addr'] <= address and address+4 <= source['addr']+source['size'] and address % 4 == 0, 'ELF relocation outside initialized section') - value = struct.unpack_from('>IIIBBH', data, syms['offset']+16*symbol_index) + for at in range(rel["offset"], rel["offset"] + rel["size"], 8): + address, info = struct.unpack_from(">2I", data, at) + kind = info & 255 + symbol_index = info >> 8 + require(symbol_index < syms["size"] // 16 and kind in SUPPORTED, "unsupported ELF relocation") + require( + source["addr"] <= address and address + 4 <= source["addr"] + source["size"] and address % 4 == 0, + "ELF relocation outside initialized section", + ) + value = struct.unpack_from(">IIIBBH", data, syms["offset"] + 16 * symbol_index) symbol = dict(value=value[1], size=value[2], type=value[3] & 15, section=value[5]) - word = struct.unpack_from('>I', data, source['offset']+address-source['addr'])[0] + word = struct.unpack_from(">I", data, source["offset"] + address - source["addr"])[0] key = (address, kind) - require(key not in result, 'duplicate ELF relocation address/type') + require(key not in result, "duplicate ELF relocation address/type") row = dict(address=address, kind=kind, symbol=symbol, word=word) result[key] = row if kind == 5: - require(not pending or pending[0]['symbol'] == symbol, 'mixed-symbol pending HI16 group') + require(not pending or pending[0]["symbol"] == symbol, "mixed-symbol pending HI16 group") pending.append(row) elif kind == 6: if pending: - require(pending[0]['symbol'] == symbol, 'HI16/LO16 symbol disagreement') - low = (word & 0xffff) - (0x10000 if word & 0x8000 else 0) + require(pending[0]["symbol"] == symbol, "HI16/LO16 symbol disagreement") + low = (word & 0xFFFF) - (0x10000 if word & 0x8000 else 0) for high in pending: - target = (((high['word'] & 0xffff) << 16)+low) & 0xffffffff - high['resolved'] = target; pairs.append((high['address'], address)) + target = (((high["word"] & 0xFFFF) << 16) + low) & 0xFFFFFFFF + high["resolved"] = target + pairs.append((high["address"], address)) pending = [] - else: orphan_lows.append(address) + else: + orphan_lows.append(address) else: - require(not pending, 'unpaired HI16 before another relocation') - require(not pending, 'unterminated HI16 group') + require(not pending, "unpaired HI16 before another relocation") + require(not pending, "unterminated HI16 group") return sections, result, pairs, orphan_lows @@ -173,45 +246,68 @@ def normalize(elf, syms, binary): """Return proven complete local targets; never infer an ambiguous address.""" sections, relocs = symbols(syms, binary, targets=False) elf_sections, entries, pairs, orphan_lows = linked_elf(elf) - out = bytearray(syms); changes = []; local = 0; addresses = set() + out = bytearray(syms) + changes = [] + local = 0 + addresses = set() highs = {} for entry in entries.values(): - if entry['kind'] == 5 and 'resolved' in entry: - highs.setdefault(tuple(entry['symbol'].items()), []).append(entry['resolved']) + if entry["kind"] == 5 and "resolved" in entry: + highs.setdefault(tuple(entry["symbol"].items()), []).append(entry["resolved"]) checked_orphans = 0 for row in relocs: - source = sections[row['section']]; address = source['ram']+row['off']; key = (address, row['kind']) - require(key not in addresses, 'duplicate output relocation'); addresses.add(key) - require(key in entries, 'output relocation has no ELF provenance at '+hex(address)) - entry = entries[key]; symbol = entry['symbol']; kind = row['kind'] - if row['owner'] in (IMPORT, EVENT) or not row['owner'] & LOCAL: + source = sections[row["section"]] + address = source["ram"] + row["off"] + key = (address, row["kind"]) + require(key not in addresses, "duplicate output relocation") + addresses.add(key) + require(key in entries, "output relocation has no ELF provenance at " + hex(address)) + entry = entries[key] + symbol = entry["symbol"] + kind = row["kind"] + if row["owner"] in (IMPORT, EVENT) or not row["owner"] & LOCAL: continue # The pinned tool owns imports/events/per-ROM references. local += 1 - require(0 < symbol['section'] < len(elf_sections), 'invalid local ELF target section') - target_section = elf_sections[symbol['section']] - require(symbol['section'] != 0 and target_section['flags'] & 2 and target_section['size'], 'local record does not reference an allocated ELF section') - lo = target_section['addr']; end = lo+target_section['size']; word = entry['word'] - if kind == 0: continue + require(0 < symbol["section"] < len(elf_sections), "invalid local ELF target section") + target_section = elf_sections[symbol["section"]] + require( + symbol["section"] != 0 and target_section["flags"] & 2 and target_section["size"], + "local record does not reference an allocated ELF section", + ) + lo = target_section["addr"] + end = lo + target_section["size"] + word = entry["word"] + if kind == 0: + continue if kind == 2: target = word # Fully linked pointer, INCLUDING its addend. elif kind == 4: - target = ((address+4) & 0xf0000000) | ((word & 0x3ffffff) << 2) + target = ((address + 4) & 0xF0000000) | ((word & 0x3FFFFFF) << 2) elif kind == 5: - require('resolved' in entry, 'HI16 has no checked LO16 pair') - target = entry['resolved'] + require("resolved" in entry, "HI16 has no checked LO16 pair") + target = entry["resolved"] else: # A LO16's encoded low half is exact even if an unrelated LUI was # encountered since its matching high relocation. A small original # ELF data section makes the full address uniquely recoverable. - first = lo+(((word & 0xffff)-lo) & 0xffff) - candidates = list(range(first, end+1, 0x10000)) - require(len(candidates) == 1, 'ambiguous LO16 target at '+hex(address)+'; split oversized data sections (use -fdata-sections)') + first = lo + (((word & 0xFFFF) - lo) & 0xFFFF) + candidates = list(range(first, end + 1, 0x10000)) + require( + len(candidates) == 1, + "ambiguous LO16 target at " + hex(address) + "; split oversized data sections (use -fdata-sections)", + ) target = candidates[0] - require(lo <= target <= end, 'linked target outside its original ELF section at '+hex(address)) - dest = sections[row['owner'] & ~LOCAL] - require(dest['ram'] <= lo and end <= dest['ram']+dest['size']+dest['bss'], 'ELF target section not contained in declared NRM owner') - require(region(binary, source['rom']+row['off'], 4) == struct.pack('>I', word), 'tool modified a local relocation instruction/pointer') - corrected = target-dest['ram'] + require(lo <= target <= end, "linked target outside its original ELF section at " + hex(address)) + dest = sections[row["owner"] & ~LOCAL] + require( + dest["ram"] <= lo and end <= dest["ram"] + dest["size"] + dest["bss"], + "ELF target section not contained in declared NRM owner", + ) + require( + region(binary, source["rom"] + row["off"], 4) == struct.pack(">I", word), + "tool modified a local relocation instruction/pointer", + ) + corrected = target - dest["ram"] if kind == 6 and address in orphan_lows: # LLVM may reuse the same symbol's LUI inside a basic block even # with MachineCSE disabled. This is safe at EVERY runtime base @@ -220,33 +316,52 @@ def normalize(elf, syms, binary): # simple condition: all offsets lie in the same 16-byte cell. # Otherwise refuse to publish, rather than depend on mod order. candidates = highs.get(tuple(symbol.items()), []) - require(candidates and all((h-dest['ram'])//16 == corrected//16 for h in candidates), - 'shared LUI has an ambiguous relocation carry at '+hex(address)) + require( + candidates and all((h - dest["ram"]) // 16 == corrected // 16 for h in candidates), + "shared LUI has an ambiguous relocation carry at " + hex(address), + ) checked_orphans += 1 - if corrected != row['target']: - changes.append(dict(address=hex(address), kind=kind, before=hex(row['target']), after=hex(corrected))) - struct.pack_into('32*1024*1024:raise ValueError('Package exceeds the 32 MiB portable budget') + path = args.package.resolve(strict=True) + if path.stat().st_size > 32 * 1024 * 1024: + raise ValueError("Package exceeds the 32 MiB portable budget") with zipfile.ZipFile(path) as archive: - entries=archive.infolist();names=[entry.filename for entry in entries] - if len(entries)>512 or len(set(name.casefold() for name in names))!=len(names):raise ValueError('Duplicate or excessive archive members') - if sum(entry.file_size for entry in entries)>64*1024*1024:raise ValueError('Expanded package exceeds 64 MiB') + entries = archive.infolist() + names = [entry.filename for entry in entries] + if len(entries) > 512 or len(set(name.casefold() for name in names)) != len(names): + raise ValueError("Duplicate or excessive archive members") + if sum(entry.file_size for entry in entries) > 64 * 1024 * 1024: + raise ValueError("Expanded package exceeds 64 MiB") for entry in entries: - parts=PurePosixPath(entry.filename) - if parts.is_absolute() or '..' in parts.parts or '\\' in entry.filename or ':' in entry.filename:raise ValueError('Unsafe archive path') - meta=json.loads(archive.read('dkr.json'));manifest=json.loads(archive.read('mod.json')) - sections,relocations=inspect_symbols(archive.read('mod_syms.bin'),archive.read('mod_binary.bin')) - print(json.dumps({'id':manifest.get('id'),'name':manifest.get('display_name'),'version':manifest.get('version'), - 'sha256':hashlib.sha256(path.read_bytes()).hexdigest(),'api':meta.get('api'),'development':meta.get('development',False), - 'online':meta.get('online'),'revisions':meta.get('revisions'),'requires':meta.get('requires',{}), - 'resources':meta.get('resources',{}),'files':len(entries),'sections':len(sections),'relocations':len(relocations)},indent=2)) - print('Static metadata and symbol/relocation inspection only. This is not runtime qualification or a trust guarantee.') + parts = PurePosixPath(entry.filename) + if parts.is_absolute() or ".." in parts.parts or "\\" in entry.filename or ":" in entry.filename: + raise ValueError("Unsafe archive path") + meta = json.loads(archive.read("dkr.json")) + manifest = json.loads(archive.read("mod.json")) + sections, relocations = inspect_symbols(archive.read("mod_syms.bin"), archive.read("mod_binary.bin")) + print( + json.dumps( + { + "id": manifest.get("id"), + "name": manifest.get("display_name"), + "version": manifest.get("version"), + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "api": meta.get("api"), + "development": meta.get("development", False), + "online": meta.get("online"), + "revisions": meta.get("revisions"), + "requires": meta.get("requires", {}), + "resources": meta.get("resources", {}), + "files": len(entries), + "sections": len(sections), + "relocations": len(relocations), + }, + indent=2, + ) + ) + print( + "Static metadata and symbol/relocation inspection only. This is not runtime qualification or a trust guarantee." + ) def source_map(args): - elf=args.elf.resolve(strict=True);output=args.output.resolve() - if output.exists():raise ValueError('Source-map output must be a NEW file') - nm=shutil.which(args.nm);symbolizer=shutil.which(args.symbolizer) - if not nm or not symbolizer:raise ValueError('llvm-nm and llvm-symbolizer are required for source maps') - result=subprocess.run([nm,'--defined-only','--numeric-sort','--print-size',str(elf)],capture_output=True,text=True,check=True) - functions=[] + elf = args.elf.resolve(strict=True) + output = args.output.resolve() + if output.exists(): + raise ValueError("Source-map output must be a NEW file") + nm = shutil.which(args.nm) + symbolizer = shutil.which(args.symbolizer) + if not nm or not symbolizer: + raise ValueError("llvm-nm and llvm-symbolizer are required for source maps") + result = subprocess.run( + [nm, "--defined-only", "--numeric-sort", "--print-size", str(elf)], capture_output=True, text=True, check=True + ) + functions = [] for line in result.stdout.splitlines(): - found=re.fullmatch(r'([0-9a-fA-F]+)\s+([0-9a-fA-F]+)\s+([TtWw])\s+(.+)',line) - if found:functions.append((int(found[1],16),int(found[2],16),found[4])) - if len(functions)>16384:raise ValueError('Source map exceeds the function budget') - resolved=subprocess.run([symbolizer,'--obj='+str(elf),'--output-style=JSON','--no-inlines'], - input=''.join(hex(address)+'\n' for address,_,_ in functions),capture_output=True,text=True,check=True) - rows=[json.loads(line) for line in resolved.stdout.splitlines() if line.strip()] - if len(rows)!=len(functions):raise ValueError('Symbolizer did not return one result per function') - mapped=[] - for (address,size,name),entry in zip(functions,rows): - symbols=entry.get('Symbol',[]);symbol=symbols[0] if symbols else {} - file=symbol.get('FileName','') + found = re.fullmatch(r"([0-9a-fA-F]+)\s+([0-9a-fA-F]+)\s+([TtWw])\s+(.+)", line) + if found: + functions.append((int(found[1], 16), int(found[2], 16), found[4])) + if len(functions) > 16384: + raise ValueError("Source map exceeds the function budget") + resolved = subprocess.run( + [symbolizer, "--obj=" + str(elf), "--output-style=JSON", "--no-inlines"], + input="".join(hex(address) + "\n" for address, _, _ in functions), + capture_output=True, + text=True, + check=True, + ) + rows = [json.loads(line) for line in resolved.stdout.splitlines() if line.strip()] + if len(rows) != len(functions): + raise ValueError("Symbolizer did not return one result per function") + mapped = [] + for (address, size, name), entry in zip(functions, rows): + symbols = entry.get("Symbol", []) + symbol = symbols[0] if symbols else {} + file = symbol.get("FileName", "") # Kits build with /mod and /sdk prefix mappings. Refuse to publish a # developer's local absolute path accidentally. - if file and not file.startswith(('/mod/','/sdk/')):file=PurePosixPath(file.replace('\\','/')).name - mapped.append({'name':name,'elf_address':address,'size':size,'file':file,'line':symbol.get('Line',0),'column':symbol.get('Column',0)}) - document={'schema':2,'elf_sha256':hashlib.sha256(elf.read_bytes()).hexdigest(),'functions':mapped} - if not any(item['line'] for item in mapped):raise ValueError('The ELF has no source lines. Build with --source-map or -gline-tables-only.') - encoded=json.dumps(document,indent=2)+'\n' - if len(encoded.encode())>1024*1024:raise ValueError('Source map exceeds 1 MiB') - output.parent.mkdir(parents=True,exist_ok=True);output.write_text(encoded,encoding='utf-8') - print('Created source map: '+str(output)) + if file and not file.startswith(("/mod/", "/sdk/")): + file = PurePosixPath(file.replace("\\", "/")).name + mapped.append( + { + "name": name, + "elf_address": address, + "size": size, + "file": file, + "line": symbol.get("Line", 0), + "column": symbol.get("Column", 0), + } + ) + document = {"schema": 2, "elf_sha256": hashlib.sha256(elf.read_bytes()).hexdigest(), "functions": mapped} + if not any(item["line"] for item in mapped): + raise ValueError("The ELF has no source lines. Build with --source-map or -gline-tables-only.") + encoded = json.dumps(document, indent=2) + "\n" + if len(encoded.encode()) > 1024 * 1024: + raise ValueError("Source map exceeds 1 MiB") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(encoded, encoding="utf-8") + print("Created source map: " + str(output)) def main(): - parser=argparse.ArgumentParser(description=__doc__);commands=parser.add_subparsers(dest='command',required=True) - new=commands.add_parser('new');new.add_argument('--id',required=True);new.add_argument('--name');new.add_argument('--language',choices=('c','cpp'),default='c');new.add_argument('--output',type=Path,required=True) - read=commands.add_parser('inspect');read.add_argument('package',type=Path) - source=commands.add_parser('source-map');source.add_argument('elf',type=Path);source.add_argument('--output',type=Path,required=True);source.add_argument('--nm',default='llvm-nm');source.add_argument('--symbolizer',default='llvm-symbolizer') - args=parser.parse_args();{'new':new_project,'inspect':inspect,'source-map':source_map}[args.command](args) + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + new = commands.add_parser("new") + new.add_argument("--id", required=True) + new.add_argument("--name") + new.add_argument("--language", choices=("c", "cpp"), default="c") + new.add_argument("--output", type=Path, required=True) + read = commands.add_parser("inspect") + read.add_argument("package", type=Path) + source = commands.add_parser("source-map") + source.add_argument("elf", type=Path) + source.add_argument("--output", type=Path, required=True) + source.add_argument("--nm", default="llvm-nm") + source.add_argument("--symbolizer", default="llvm-symbolizer") + args = parser.parse_args() + {"new": new_project, "inspect": inspect, "source-map": source_map}[args.command](args) -if __name__=='__main__':main() +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..70408a12 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,118 @@ +# Tool configuration only. DKR-R is not a Python package and nothing here is +# installed, so there is no [project] table. + +[tool.ruff] +# The floor documented in scripts/README.md. +target-version = "py310" +# Matches ColumnLimit in .clang-format so both languages wrap at the same width. +line-length = 120 +# First-party Python is scripts/, runtime-recomp/ (tests and cmake helpers) and +# modding/. patches/ holds byte-pinned layer copies that the fold deletes, +# extern/ is third-party, tools/blender/ is the Track Lab addon with its own +# release cycle, build/ is generated. +extend-exclude = [ + "build", + "extern", + "patches", + "tools/blender", + # The 1.0.7 source capsule pins this file's bytes as a base (see + # scripts/check_pinned_bytes.py). Lint and format it after the fold. + "runtime-recomp/tests/replay_probe/test_generator.py", +] +src = [".", "scripts", "runtime-recomp/tests", "modding"] + +[tool.ruff.format] +# ruff 0.17 also formats fenced code in Markdown, and the docs keep aligned +# columns there on purpose. The pattern-guard fixtures are inputs to its tests. +exclude = ["*.md", "tests/fixtures/**"] + +[tool.ruff.lint] +# Ratchet further groups in one PR each (B, UP, SIM, PTH, ...). +select = ["E", "F", "W", "I"] + +[tool.ruff.lint.isort] +# Closest to the existing import style ("from x import y" sorted among "import z"). +force-sort-within-sections = true + +[tool.ruff.lint.extend-per-file-ignores] +# Files listed here are grandfathered for exactly these rules; new files get no +# entry. Regenerate the block after each autofix pass instead of editing it. +# BEGIN GENERATED BASELINE +"modding/relocations.py" = ["E731"] +"modding/tools.py" = ["E402"] +"runtime-recomp/tests/ci_summary_tests.py" = ["E402", "E501"] +"runtime-recomp/tests/host_task_policy_tests.py" = ["E402"] +"runtime-recomp/tests/legacy_behaviour_pipeline_tests.py" = ["E402"] +"runtime-recomp/tests/legacy_mod_pipeline_tests.py" = ["E501"] +"runtime-recomp/tests/regression_harness_tests.py" = ["E402", "E501"] +"runtime-recomp/tests/stress_runner_tests.py" = ["E402"] +"runtime-recomp/tests/test_track_select_draw_pipeline.py" = ["E402"] +"runtime-recomp/tests/water_profile_policy_tests.py" = ["E402"] +"scripts/adapt-water-reference.py" = ["E501"] +"scripts/apply_dependency_patches.py" = ["E501"] +"scripts/audio_queue_policy.py" = ["E501"] +"scripts/camera_plus_pipeline.py" = ["E501"] +"scripts/character_portrait_pipeline.py" = ["E501", "E731"] +"scripts/ci_summary.py" = ["E501"] +"scripts/compose-lava-glow.py" = ["E501"] +"scripts/compose-lava-lighting.py" = ["E501"] +"scripts/compose-water-performance.py" = ["E501", "F841"] +"scripts/compose-water-projection.py" = ["E501"] +"scripts/compose-water-rays.py" = ["E501"] +"scripts/compose-water-recovery.py" = ["E501"] +"scripts/compose-water-safety.py" = ["E501"] +"scripts/compose-water-stability.py" = ["E501"] +"scripts/compose-water-views.py" = ["E501"] +"scripts/compose-waterfall.py" = ["E501"] +"scripts/compose_legacy_mod_policy.py" = ["E501"] +"scripts/generate_replay_probe.py" = ["E501"] +"scripts/generate_visual_model_kernel.py" = ["E501", "F841"] +"scripts/harness_report.py" = ["E501"] +"scripts/legacy_asset_capacity_policy.py" = ["E501"] +"scripts/legacy_behaviour_policy.py" = ["E501"] +"scripts/legacy_character_ai_policy.py" = ["E501"] +"scripts/legacy_character_presentation_policy.py" = ["E501"] +"scripts/legacy_memory_safety_policy.py" = ["E501"] +"scripts/legacy_model_cache_policy.py" = ["E501"] +"scripts/legacy_model_reference_policy.py" = ["E501"] +"scripts/legacy_spawn_asset_type_policy.py" = ["E501"] +"scripts/macos_build_lock.py" = ["E501"] +"scripts/prepare_legacy_menu_fragment.py" = ["F841"] +"scripts/regression.py" = ["E501"] +"scripts/save_layout_pipeline.py" = ["E501", "E731"] +"scripts/stage_magic_code_payload.py" = ["E501"] +"scripts/test_update_appimage.py" = ["E501", "F841"] +"scripts/test_update_package.py" = ["E501"] +"scripts/track_select_draw_policy.py" = ["E501"] +"scripts/update_release.py" = ["E501"] +"scripts/water_patch_baseline.py" = ["E501"] +"scripts/water_profile_policy.py" = ["E501"] +# END GENERATED BASELINE + +[tool.pyright] +# Strict only for new tooling scripts; everything else is ratcheted in per path. +include = [ + "scripts/agent_hooks", + "scripts/bootstrap_hooks.py", + "scripts/check_pinned_bytes.py", + "scripts/pattern_guard.py", + "scripts/guest_hooks.py", + "scripts/gen_guest_symbols.py", + "scripts/check_includes.py", + "scripts/check_registries.py", + "scripts/release", +] +strict = [ + "scripts/agent_hooks", + "scripts/bootstrap_hooks.py", + "scripts/check_pinned_bytes.py", + "scripts/pattern_guard.py", + "scripts/guest_hooks.py", + "scripts/gen_guest_symbols.py", + "scripts/check_includes.py", + "scripts/check_registries.py", + "scripts/release", +] +exclude = ["build", "extern", "patches", "tools/blender"] +extraPaths = ["scripts"] +pythonVersion = "3.10" diff --git a/runtime-recomp/CMakeLists.txt b/runtime-recomp/CMakeLists.txt index 691cf7d9..be6679ff 100644 --- a/runtime-recomp/CMakeLists.txt +++ b/runtime-recomp/CMakeLists.txt @@ -1423,6 +1423,8 @@ if(BUILD_TESTING) "DKRCiSummary|ci_summary_tests.py" "DKRStressRunner|stress_runner_tests.py" "DKROwnedContentIdentity|owned_content_identity_tests.py" + "DKRPatternGuard|pattern_guard_tests.py" + "DKRAgentHooks|agent_hooks_tests.py" "DKRLauncherCopySource|launcher_copy_source_tests.py") string(REPLACE "|" ";" tooling_test "${tooling_test}") list(GET tooling_test 0 test_name) diff --git a/runtime-recomp/cmake/Warnings.cmake b/runtime-recomp/cmake/Warnings.cmake new file mode 100644 index 00000000..21d5e413 --- /dev/null +++ b/runtime-recomp/cmake/Warnings.cmake @@ -0,0 +1,72 @@ +# Compiler warnings for first-party targets. Nothing here applies itself: +# call dkr_enable_warnings() per first-party target and +# dkr_disable_warnings() for recompiled or generated code. Third-party +# targets under extern/ keep their own flags. +# +# DKR_WERROR promotes only a curated subset to errors (each one is a real bug +# on at least one platform we ship). Blanket -Werror is per target through +# DKR_STRICT_MODULES, which starts empty and grows as a module is cleaned up. +include_guard(GLOBAL) +option(DKR_WERROR "Promote the curated warning subset to errors on first-party targets" OFF) +set(DKR_STRICT_MODULES "" CACHE STRING "First-party targets built with blanket -Werror or /WX") + +function(dkr_enable_warnings target) + if(NOT TARGET ${target}) + message(FATAL_ERROR "dkr_enable_warnings: ${target} is not a target") + endif() + get_target_property(type ${target} TYPE) + if(type STREQUAL "INTERFACE_LIBRARY") + message(FATAL_ERROR "dkr_enable_warnings: ${target} is an INTERFACE library; use the targets that compile it") + endif() + + if(MSVC) + # Also covers clang-cl, which maps /W4 to -Wall -Wextra. + target_compile_options(${target} PRIVATE /W4 /permissive- /utf-8) + if(DKR_WERROR) + target_compile_options( + ${target} + PRIVATE + /we4013 # undefined function assumed extern returning int + /we4700 # uninitialized local variable used + /we4716 # function must return a value + /we4477 # format string does not match argument type + $<$:/we4047> # differs in levels of indirection (int/pointer) + ) + endif() + set(strict /WX) + elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options( + ${target} + PRIVATE -Wall -Wextra -Wshadow -Wformat=2 -Wundef -Wvla -Wimplicit-fallthrough -Wmissing-declarations + ) + if(DKR_WERROR) + target_compile_options( + ${target} + PRIVATE + -Werror=return-type + -Werror=uninitialized + -Werror=format-security + $<$:-Werror=implicit-function-declaration> + $<$:-Werror=int-conversion> + ) + endif() + set(strict -Werror) + else() + message(WARNING "dkr_enable_warnings: no warning set for ${CMAKE_CXX_COMPILER_ID}; ${target} uses defaults") + return() + endif() + + if(target IN_LIST DKR_STRICT_MODULES) + target_compile_options(${target} PRIVATE ${strict}) + endif() +endfunction() + +# Recompiled N64 code and generated tables are not edited by hand, so warnings +# there are noise that hides first-party ones. +function(dkr_disable_warnings target) + if(MSVC) + target_compile_options(${target} PRIVATE /w) + else() + target_compile_options(${target} PRIVATE -w) + endif() +endfunction() diff --git a/runtime-recomp/cmake/generate_revision_symbol_map.py b/runtime-recomp/cmake/generate_revision_symbol_map.py index 0acc24f8..52965fd9 100644 --- a/runtime-recomp/cmake/generate_revision_symbol_map.py +++ b/runtime-recomp/cmake/generate_revision_symbol_map.py @@ -18,9 +18,7 @@ def main() -> int: if len(sys.argv) != 4: - raise SystemExit( - "usage: generate_revision_symbol_map.py " - ) + raise SystemExit("usage: generate_revision_symbol_map.py ") source_dir = pathlib.Path(sys.argv[1]) prefix = sys.argv[2] diff --git a/runtime-recomp/tests/agent_hooks_tests.py b/runtime-recomp/tests/agent_hooks_tests.py new file mode 100644 index 00000000..05e26560 --- /dev/null +++ b/runtime-recomp/tests/agent_hooks_tests.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python3 +"""The AI agent hooks in scripts/agent_hooks/ block what they should, allow +what they should, and are wired identically for Claude Code and Codex. + +Every hook gets a synthetic stdin payload shaped like the agents' own.""" + +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import time +import unittest + +ROOT = Path(__file__).resolve().parents[2] +HOOKS = ROOT / "scripts/agent_hooks" +HOOK_SCRIPTS = {"session_start.py", "pre_edit_guard.py", "post_edit_check.py", "stop_check.py"} +SPEC = importlib.util.spec_from_file_location("hooklib", HOOKS / "hooklib.py") +hooklib = importlib.util.module_from_spec(SPEC) +sys.modules["hooklib"] = hooklib # dataclasses resolve their module by name +SPEC.loader.exec_module(hooklib) + + +def run_hook(script, payload, env=None): + # The developer's own git config (a global core.hooksPath, say) must not leak in. + environment = {key: value for key, value in os.environ.items() if key != "CLAUDE_PROJECT_DIR"} + environment["GIT_CONFIG_GLOBAL"] = os.devnull + environment.update(env or {}) + return subprocess.run( + [sys.executable, str(HOOKS / script)], + input=json.dumps(payload), + capture_output=True, + text=True, + env=environment, + timeout=120, + ) + + +def edit_payload(tool, cwd, **tool_input): + return { + "session_id": "agent-hooks-test", + "cwd": str(cwd), + "hook_event_name": "PreToolUse", + "tool_name": tool, + "tool_input": tool_input, + } + + +def git(root, *args): + subprocess.run( + ["git", "-C", str(root), *args], + check=True, + capture_output=True, + env={**os.environ, "GIT_CONFIG_GLOBAL": os.devnull}, + ) + + +class PreEditGuardTests(unittest.TestCase): + def test_blocks_a_new_overlay_cmake_layer(self): + result = run_hook( + "pre_edit_guard.py", + edit_payload( + "Write", + ROOT, + file_path=str(ROOT / "patches/x/ReleaseOverlay.cmake"), + content="set(DKR_OVERLAY ON)\n", + ), + ) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("patches/x/ReleaseOverlay.cmake:1: PG001", result.stderr) + self.assertIn("instead:", result.stderr) + self.assertFalse((ROOT / "patches/x").exists()) + + def test_blocks_the_same_layer_from_a_codex_apply_patch(self): + patch = "*** Begin Patch\n*** Add File: patches/x/ReleaseOverlay.cmake\n+set(DKR_OVERLAY ON)\n*** End Patch\n" + result = run_hook("pre_edit_guard.py", edit_payload("apply_patch", ROOT, command=patch)) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("PG001", result.stderr) + + def test_blocks_new_text_that_breaks_a_rule(self): + result = run_hook( + "pre_edit_guard.py", + edit_payload( + "Write", + ROOT, + file_path=str(ROOT / "runtime-recomp/src/game/agent_hooks_probe.cpp"), + content='#include "build/generated/funcs.h"\n', + ), + ) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("PG009", result.stderr) + + def test_allows_an_extern_edit(self): + target = ROOT / "extern/RT64/src/agent_hooks_probe.cpp" + result = run_hook( + "pre_edit_guard.py", + edit_payload("Edit", ROOT, file_path=str(target), old_string="a", new_string='#include "/abs/x.h"'), + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_allows_an_ordinary_edit(self): + result = run_hook( + "pre_edit_guard.py", + edit_payload( + "Edit", ROOT, file_path=str(ROOT / "AGENTS.md"), old_string="# Working in DKR-R", new_string="# DKR-R" + ), + ) + self.assertEqual((result.returncode, result.stderr), (0, "")) + + def test_allows_editing_a_baselined_file(self): + baseline = (ROOT / "architecture/pattern-guard-baseline.txt").read_text(encoding="utf-8") + path = next(line.split()[0] for line in baseline.splitlines() if line and not line.startswith("#")) + current = (ROOT / path).read_text(encoding="utf-8", errors="replace") + result = run_hook("pre_edit_guard.py", edit_payload("Write", ROOT, file_path=str(ROOT / path), content=current)) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_ignores_files_outside_the_checkout(self): + with tempfile.TemporaryDirectory() as temp: + result = run_hook( + "pre_edit_guard.py", + edit_payload("Write", temp, file_path=f"{temp}/patches/x/ReleaseOverlay.cmake", content="x\n"), + ) + self.assertEqual(result.returncode, 0, result.stderr) + + +class PlannedEditTests(unittest.TestCase): + def test_edit_multiedit_and_apply_patch_reconstruct_the_new_text(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "a.txt" + path.write_text("one\ntwo\nthree\n", encoding="utf-8") + edit = hooklib.planned_edits( + edit_payload("Edit", temp, file_path=str(path), old_string="two", new_string="2") + ) + self.assertEqual(edit[0].text, "one\n2\nthree\n") + multi = hooklib.planned_edits( + edit_payload( + "MultiEdit", + temp, + file_path=str(path), + edits=[{"old_string": "one", "new_string": "1"}, {"old_string": "three", "new_string": "3"}], + ) + ) + self.assertEqual(multi[0].text, "1\ntwo\n3\n") + patch = ( + "*** Begin Patch\n*** Update File: a.txt\n*** Move to: b.txt\n@@\n one\n-two\n+zwei\n" + "*** Delete File: gone.txt\n*** End Patch\n" + ) + applied = hooklib.planned_edits(edit_payload("apply_patch", temp, command=patch)) + self.assertEqual([(edit.path.name, edit.text) for edit in applied], [("b.txt", "one\nzwei\nthree\n")]) + + +class StopCheckTests(unittest.TestCase): + """A scratch DKR-R checkout with one baselined layer and its baseline.""" + + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name).resolve() + (self.root / "scripts").mkdir() + shutil.copy2(ROOT / "scripts/pattern_guard.py", self.root / "scripts/pattern_guard.py") + self.write("patches/old/ReleaseOverlay.cmake", "set(OLD ON)\n") + self.write("architecture/pattern-guard-baseline.txt", "patches/old/ReleaseOverlay.cmake PG001\n") + self.write(".gitignore", "/.claude/session-base\n/.claude/session-base.*.tmp\n") + git(self.root, "init", "-q") + git(self.root, "add", ".") + git(self.root, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "commit", "-qm", "base") + + def write(self, path, text): + target = self.root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding="utf-8") + + def event(self, name, **fields): + return {"session_id": "s1", "cwd": str(self.root), "hook_event_name": name, **fields} + + def start(self): + result = run_hook("session_start.py", self.event("SessionStart", source="startup")) + self.assertEqual(result.returncode, 0, result.stderr) + return result + + def stop(self, active=False): + return run_hook("stop_check.py", self.event("Stop", stop_hook_active=active)) + + def test_session_start_records_the_base_and_prints_the_pattern(self): + self.write("AGENTS.md", "# x\n\n## The pattern (non-negotiable)\n- Edit in place.\n\n## Next\nno\n") + result = self.start() + self.assertIn("- Edit in place.", result.stdout) + self.assertNotIn("## Next", result.stdout) + sessions = json.loads((self.root / ".claude/session-base").read_text(encoding="utf-8"))["sessions"] + head = subprocess.run(["git", "-C", str(self.root), "rev-parse", "HEAD"], capture_output=True, text=True) + self.assertEqual(sessions["s1"]["head"], head.stdout.strip()) + + def test_pre_existing_baseline_hits_do_not_block_stop(self): + self.start() + self.write("patches/old/ReleaseOverlay.cmake", "set(OLD ON)\nset(STILL_OLD ON)\n") + self.write("docs/notes.md", "unrelated\n") + result = self.stop() + self.assertEqual(result.returncode, 0, result.stderr) + + def test_a_new_violation_blocks_stop(self): + self.start() + self.write("patches/new/ReleaseOverlay.cmake", "set(NEW ON)\n") + result = self.stop() + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("patches/new/ReleaseOverlay.cmake:1: PG001", result.stderr) + self.assertNotIn("patches/old", result.stderr) + + def test_baselining_a_new_violation_still_blocks_stop(self): + self.start() + self.write("patches/new/ReleaseOverlay.cmake", "set(NEW ON)\n") + self.write( + "architecture/pattern-guard-baseline.txt", + "patches/new/ReleaseOverlay.cmake PG001\npatches/old/ReleaseOverlay.cmake PG001\n", + ) + result = self.stop() + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("new baseline entry for patches/new/ReleaseOverlay.cmake", result.stderr) + + def test_violations_present_at_session_start_do_not_block_stop(self): + self.write("patches/dirty/ReleaseOverlay.cmake", "set(DIRTY ON)\n") + self.start() + result = self.stop() + self.assertEqual(result.returncode, 0, result.stderr) + + def test_stop_hook_active_exits_zero(self): + self.start() + self.write("patches/new/ReleaseOverlay.cmake", "set(NEW ON)\n") + self.assertEqual(self.stop(active=False).returncode, 2) + result = self.stop(active=True) + self.assertEqual((result.returncode, result.stderr), (0, "")) + + def test_stop_without_a_recorded_session_uses_head(self): + self.write("patches/new/ReleaseOverlay.cmake", "set(NEW ON)\n") + self.assertEqual(self.stop().returncode, 2) + + +class PostEditCheckTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name).resolve() + (self.root / "scripts").mkdir() + (self.root / "scripts/pattern_guard.py").write_text("", encoding="utf-8") + self.file = self.root / "a.py" + self.file.write_text("x=1\n", encoding="utf-8") + git(self.root, "init", "-q") + + def payload(self): + return { + "session_id": "s1", + "cwd": str(self.root), + "hook_event_name": "PostToolUse", + "tool_name": "Edit", + "tool_input": {"file_path": str(self.file), "old_string": "x", "new_string": "x"}, + } + + def test_without_a_pre_commit_config_it_does_nothing(self): + result = run_hook("post_edit_check.py", self.payload()) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("hookSpecificOutput", result.stdout) + + @unittest.skipIf(os.name == "nt", "fake pre-commit is a shell script") + def test_reports_a_fixer_diff_and_never_rewrites_the_file(self): + (self.root / ".pre-commit-config.yaml").write_text("repos: []\n", encoding="utf-8") + hook = self.root / ".git/hooks/pre-commit" + hook.write_text("#!/bin/sh\n# File generated by pre-commit\n", encoding="utf-8") + bin_dir = self.root / "bin" + bin_dir.mkdir() + fake = bin_dir / "pre-commit" + fake.write_text( + '#!/bin/sh\necho "ruff format.............Failed"\necho "- files were modified by this hook"\n' + 'printf "x = 1\\n" > a.py\nexit 1\n', + encoding="utf-8", + ) + fake.chmod(0o755) + before = self.file.stat().st_mtime_ns + result = run_hook("post_edit_check.py", self.payload(), {"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}"}) + self.assertEqual(result.returncode, 0, result.stderr) + context = json.loads(result.stdout)["hookSpecificOutput"] + self.assertEqual(context["hookEventName"], "PostToolUse") + self.assertIn("+x = 1", context["additionalContext"]) + self.assertIn("ruff format", context["additionalContext"]) + self.assertEqual(self.file.read_text(encoding="utf-8"), "x=1\n") + self.assertEqual(self.file.stat().st_mtime_ns, before) + + @unittest.skipIf(os.name == "nt", "fake pre-commit is a shell script") + def test_a_save_made_while_hooks_run_is_kept(self): + (self.root / ".pre-commit-config.yaml").write_text("repos: []\n", encoding="utf-8") + (self.root / ".git/hooks/pre-commit").write_text( + "#!/bin/sh\n# File generated by pre-commit\n", encoding="utf-8" + ) + bin_dir = self.root / "bin" + bin_dir.mkdir() + fake = bin_dir / "pre-commit" + # A fixer rewrites its copy while another editor saves the real file. + fake.write_text( + f'#!/bin/sh\nprintf "x = 1\\n" > a.py\nprintf "x=2\\n" > "{self.file}"\nexit 1\n', encoding="utf-8" + ) + fake.chmod(0o755) + result = run_hook("post_edit_check.py", self.payload(), {"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}"}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("+x = 1", json.loads(result.stdout)["hookSpecificOutput"]["additionalContext"]) + self.assertEqual(self.file.read_text(encoding="utf-8"), "x=2\n") + + def test_without_the_git_hook_it_says_how_to_bootstrap(self): + (self.root / ".pre-commit-config.yaml").write_text("repos: []\n", encoding="utf-8") + result = run_hook("post_edit_check.py", self.payload()) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("Bootstrap with", result.stdout) + self.assertEqual(len(result.stdout.strip().splitlines()), 1) + + +class ProcessTreeTests(unittest.TestCase): + def test_timeout_kills_formatters_with_pre_commit(self): + spec = importlib.util.spec_from_file_location("post_edit_check", HOOKS / "post_edit_check.py") + module = importlib.util.module_from_spec(spec) + sys.modules["post_edit_check"] = module # dataclasses resolve their module by name + spec.loader.exec_module(module) + with tempfile.TemporaryDirectory() as temp: + target = Path(temp) / "edited.py" + target.write_text("original\n", encoding="utf-8") + # A "pre-commit" that starts a slow fixer and then hangs past the timeout. + fixer = f"import time; time.sleep(1.5); open({str(target)!r}, 'w').write('rewritten')" + parent = ( + f"import subprocess, sys, time; subprocess.Popen([sys.executable, '-c', {fixer!r}]); time.sleep(30)" + ) + output, failed = module.run_process_tree([sys.executable, "-c", parent], Path(temp), 0.5) + self.assertTrue(failed) + self.assertIn("did not finish", output) + time.sleep(2.5) # long enough for an orphaned fixer to have written + self.assertEqual(target.read_text(encoding="utf-8"), "original\n") + + +class ConfigTests(unittest.TestCase): + @staticmethod + def scripts(config): + commands = [ + handler["command"] + for groups in config["hooks"].values() + for group in groups + for handler in group["hooks"] + if handler.get("type") == "command" + ] + launched = [re.search(r"/scripts/agent_hooks/run (\w+\.py)$", command) for command in commands] + return commands, launched + + def load(self, path): + return json.loads((ROOT / path).read_text(encoding="utf-8")) + + def test_claude_and_codex_reference_the_same_existing_scripts(self): + claude_commands, claude = self.scripts(self.load(".claude/settings.json")) + codex_commands, codex = self.scripts(self.load(".codex/hooks.json")) + self.assertTrue(all(claude), claude_commands) + self.assertTrue(all(codex), codex_commands) + claude_set = {match.group(1) for match in claude} + codex_set = {match.group(1) for match in codex} + self.assertEqual(claude_set, HOOK_SCRIPTS) + self.assertEqual(codex_set, claude_set) + for script in claude_set | {"run", "hooklib.py"}: + self.assertTrue((HOOKS / script).is_file(), script) + + def test_events_line_up(self): + claude = self.load(".claude/settings.json")["hooks"] + codex = self.load(".codex/hooks.json")["hooks"] + self.assertEqual(set(claude), {"SessionStart", "PreToolUse", "PostToolUse", "Stop"}) + self.assertEqual(set(codex), set(claude)) + for event in claude: + self.assertEqual( + self.scripts({"hooks": {event: claude[event]}})[0][0].rsplit(" ", 1)[1], + self.scripts({"hooks": {event: codex[event]}})[0][0].rsplit(" ", 1)[1], + event, + ) + + def test_claude_commands_are_rooted_on_the_project_dir(self): + commands, _ = self.scripts(self.load(".claude/settings.json")) + for command in commands: + self.assertTrue(command.startswith('"$CLAUDE_PROJECT_DIR"/scripts/agent_hooks/run '), command) + + def test_session_state_is_gitignored(self): + for path in (".claude/session-base", ".claude/settings.local.json"): + result = subprocess.run(["git", "-C", str(ROOT), "check-ignore", "-q", path]) + self.assertEqual(result.returncode, 0, f"{path} is not ignored") + + @unittest.skipIf(os.name == "nt" or shutil.which("sh") is None, "the launcher is a POSIX sh script") + def test_launcher_finds_python_and_passes_stdin(self): + payload = json.dumps(self.stop_payload()) + result = subprocess.run( + [str(HOOKS / "run"), "stop_check.py"], input=payload, capture_output=True, text=True, timeout=60 + ) + self.assertEqual((result.returncode, result.stderr), (0, "")) + rejected = subprocess.run([str(HOOKS / "run"), "../x.py"], capture_output=True, text=True) + self.assertEqual(rejected.returncode, 1) + + @staticmethod + def stop_payload(): + return {"session_id": "s1", "cwd": str(ROOT), "hook_event_name": "Stop", "stop_hook_active": True} + + +if __name__ == "__main__": + unittest.main() diff --git a/runtime-recomp/tests/android_startup_contract_tests.py b/runtime-recomp/tests/android_startup_contract_tests.py index 1fc3b32d..4f8132f2 100644 --- a/runtime-recomp/tests/android_startup_contract_tests.py +++ b/runtime-recomp/tests/android_startup_contract_tests.py @@ -1,4 +1,5 @@ """Patch Pipeline contracts for the 1050022 and 1050023 startup regressions.""" + from pathlib import Path import shutil import subprocess @@ -17,15 +18,15 @@ def test_session_gate_reset_precedes_workers(self): shutil.copyfile(ROOT / "extern/n64-modern-runtime" / relative, stage / relative) subprocess.run(["git", "init", "--quiet", str(stage)], check=True) patch = str(ROOT / "patches/performance/runtime-session-bootstrap.patch") - subprocess.run(["git", "-C", str(stage), "apply", "--recount", - "--whitespace=error", patch], check=True) + subprocess.run(["git", "-C", str(stage), "apply", "--recount", "--whitespace=error", patch], check=True) body = (stage / relative).read_text().split("void recomp::start(", 1)[1] - self.assertLess(body.index("game_status.store(GameStatus::None)"), - body.index("exited.store(false)")) + self.assertLess(body.index("game_status.store(GameStatus::None)"), body.index("exited.store(false)")) self.assertLess(body.index("current_game.reset()"), body.index("std::thread game_thread")) cmake = (ROOT / "runtime-recomp/cmake/PerformancePatches.cmake").read_text() self.assertIn('dkr_stage_patch(librecomp "${DKR_MODERN_RUNTIME_SOURCE}" session-bootstrap', cmake) - self.assertIn('dkr_stage_patch(ultramodern "${CMAKE_BINARY_DIR}/generated/host-task-lifetime" vi-bootstrap', cmake) + self.assertIn( + 'dkr_stage_patch(ultramodern "${CMAKE_BINARY_DIR}/generated/host-task-lifetime" vi-bootstrap', cmake + ) def test_no_colour_readback_probe_in_startup(self): helper = (ROOT / "runtime-recomp/src/game/android_transfer_qualification.hpp").read_text() @@ -33,8 +34,7 @@ def test_no_colour_readback_probe_in_startup(self): self.assertTrue("qualify_color_target" not in helper + patch, "Unsafe colour probe must not ship") self.assertNotRegex(helper, r"copyTextureRegion\s*\(") self.assertIn("qualify_transfer(*textureCopyWorker)", patch) - self.assertLess(helper.index("worker.execute(); worker.wait();"), - helper.index("readback->map")) + self.assertLess(helper.index("worker.execute(); worker.wait();"), helper.index("readback->map")) def test_guard_patch_precedes_destination_texture_access(self): # Apply the real checked patch to a private copy, never to the submodule. @@ -46,8 +46,9 @@ def test_guard_patch_precedes_destination_texture_access(self): subprocess.run(["git", "init", "--quiet", str(stage)], check=True) patch = str(ROOT / "patches/android/vulkan-copy-contract.patch") for extra in (["--check"], []): - subprocess.run(["git", "-C", str(stage), "apply", "--recount", - "--whitespace=error", *extra, patch], check=True) + subprocess.run( + ["git", "-C", str(stage), "apply", "--recount", "--whitespace=error", *extra, patch], check=True + ) source = (stage / relative).read_text() body = source.split("void VulkanCommandList::copyTextureRegion(", 1)[1] body = body.split("void VulkanCommandList::copyBuffer(", 1)[0] @@ -57,8 +58,11 @@ def test_guard_patch_precedes_destination_texture_access(self): def test_android_pipeline_selects_guarded_source(self): cmake = (ROOT / "runtime-recomp/CMakeLists.txt").read_text() - self.assertTrue('dkr_stage_patch(plume "${CMAKE_BINARY_DIR}/generated/android-surface-recovery" android-copy-contract' in cmake, - "Android must compile the guard through the checked patch pipeline") + self.assertTrue( + 'dkr_stage_patch(plume "${CMAKE_BINARY_DIR}/generated/android-surface-recovery" android-copy-contract' + in cmake, + "Android must compile the guard through the checked patch pipeline", + ) self.assertIn('"${DKRPORT_ROOT}/patches/android/vulkan-copy-contract.patch"', cmake) diff --git a/runtime-recomp/tests/asset_scan_tests.py b/runtime-recomp/tests/asset_scan_tests.py index 8eddec8b..dae6c4b5 100644 --- a/runtime-recomp/tests/asset_scan_tests.py +++ b/runtime-recomp/tests/asset_scan_tests.py @@ -1,4 +1,5 @@ """The fast source scan must prune build outputs, while still checking sources.""" + import contextlib import io import os diff --git a/runtime-recomp/tests/build_iteration_tests.py b/runtime-recomp/tests/build_iteration_tests.py index 278c6585..ccba3ae0 100644 --- a/runtime-recomp/tests/build_iteration_tests.py +++ b/runtime-recomp/tests/build_iteration_tests.py @@ -1,6 +1,7 @@ """Exercise CMake reconfiguration using isolated synthetic sources and patches.""" -from pathlib import Path + import os +from pathlib import Path import shutil import subprocess import tempfile @@ -23,12 +24,21 @@ def setUp(self): (self.source / f"{name}.patch").write_text( "diff --git a/fixture.c b/fixture.c\n--- a/fixture.c\n+++ b/fixture.c\n" "@@ -1 +1 @@\n" - f"-int answer(void) {{ return {old}; }}\n+int answer(void) {{ return {new}; }}\n") + f"-int answer(void) {{ return {old}; }}\n+int answer(void) {{ return {new}; }}\n" + ) (self.source / "payload").mkdir() (self.source / "payload/functions.c").write_text("void recomp_entrypoint(void) {}\n") - self.configure = ["cmake", "-S", str(self.source), "-B", str(self.build), - "-G", "Ninja", f"-DDKR_CMAKE_DIR={CMAKE}", - f"-DDKR_DEPENDENCIES_DIR={self.source / 'shared-extern'}"] + self.configure = [ + "cmake", + "-S", + str(self.source), + "-B", + str(self.build), + "-G", + "Ninja", + f"-DDKR_CMAKE_DIR={CMAKE}", + f"-DDKR_DEPENDENCIES_DIR={self.source / 'shared-extern'}", + ] def run_ok(self, *args): result = subprocess.run(args, capture_output=True, text=True, timeout=60) @@ -80,24 +90,33 @@ def test_changed_patch_rebuilds_and_invalid_patch_preserves_published_source(sel def test_preflight_checks_only_consumed_headers_and_requires_test_python(self): dependencies = self.source / "minimal-extern" - for relative in ("n64-modern-runtime/CMakeLists.txt", - "n64-modern-runtime/N64Recomp/include/recomp.h", - "n64-modern-runtime/N64Recomp/lib/fmt/CMakeLists.txt", - "gekkonet/GekkoLib/CMakeLists.txt", "monocypher/src/monocypher.c"): + for relative in ( + "n64-modern-runtime/CMakeLists.txt", + "n64-modern-runtime/N64Recomp/include/recomp.h", + "n64-modern-runtime/N64Recomp/lib/fmt/CMakeLists.txt", + "gekkonet/GekkoLib/CMakeLists.txt", + "monocypher/src/monocypher.c", + ): path = dependencies / relative path.parent.mkdir(parents=True, exist_ok=True) path.touch() - command = ["cmake", f"-DDKR_DEPENDENCIES_DIR={dependencies}", - "-DDKR_NETPLAY_WEBRTC=OFF", "-DDKR_RUNTIME_BUILD_RT64=OFF"] + command = [ + "cmake", + f"-DDKR_DEPENDENCIES_DIR={dependencies}", + "-DDKR_NETPLAY_WEBRTC=OFF", + "-DDKR_RUNTIME_BUILD_RT64=OFF", + ] script = ["-P", str(CMAKE / "DependencyPreflight.cmake")] self.run_ok(*command, "-DBUILD_TESTING=OFF", *script) - host = subprocess.run([*command, "-DBUILD_TESTING=ON", *script], - capture_output=True, text=True, timeout=60) + host = subprocess.run([*command, "-DBUILD_TESTING=ON", *script], capture_output=True, text=True, timeout=60) self.assertNotEqual(host.returncode, 0) self.assertIn("hlsl++.h", host.stderr) - android = subprocess.run([*command, "-DBUILD_TESTING=ON", "-DANDROID=ON", - "-DCMAKE_DISABLE_FIND_PACKAGE_Python3=TRUE", *script], - capture_output=True, text=True, timeout=60) + android = subprocess.run( + [*command, "-DBUILD_TESTING=ON", "-DANDROID=ON", "-DCMAKE_DISABLE_FIND_PACKAGE_Python3=TRUE", *script], + capture_output=True, + text=True, + timeout=60, + ) self.assertNotEqual(android.returncode, 0) self.assertIn("Python3", android.stderr) self.assertIn("REQUIRED", android.stderr) @@ -109,8 +128,9 @@ def test_source_root_isolation_and_legacy_override_diagnostic(self): # CMake canonicalizes Windows short-name aliases and macOS symlinks. self.assertEqual(Path(source_root).resolve(), ROOT.resolve()) self.assertEqual(Path(dependencies).resolve(), (self.source / "shared-extern").resolve()) - result = subprocess.run([*self.configure, f"-DDKRPORT_ROOT={self.source}"], - capture_output=True, text=True, timeout=60) + result = subprocess.run( + [*self.configure, f"-DDKRPORT_ROOT={self.source}"], capture_output=True, text=True, timeout=60 + ) self.assertNotEqual(result.returncode, 0) self.assertIn("DKRPORT_ROOT can no longer redirect", result.stderr) self.assertIn("DKR_DEPENDENCIES_DIR", result.stderr) diff --git a/runtime-recomp/tests/build_tooling_tests.py b/runtime-recomp/tests/build_tooling_tests.py index 6106e699..76528a33 100644 --- a/runtime-recomp/tests/build_tooling_tests.py +++ b/runtime-recomp/tests/build_tooling_tests.py @@ -6,6 +6,7 @@ import hashlib import json import os +from pathlib import Path import re import shutil import struct @@ -13,7 +14,6 @@ import sys import tempfile import unittest -from pathlib import Path from unittest.mock import patch ROOT = Path(__file__).resolve().parents[2] @@ -22,20 +22,26 @@ import apply_dependency_patches as patches # noqa: E402 import bootstrap_dependencies as bootstrap # noqa: E402 import legacy_character_presentation_policy as presentation # noqa: E402 -import validate_release_metadata as release_metadata # noqa: E402 import package_source # noqa: E402 - +import validate_release_metadata as release_metadata # noqa: E402 # Hermetic: the developer's global config (signing, hooks, autocrlf) must not # reach the fixture repositories. -GIT_ENV = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1", - "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@example.invalid", - "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@example.invalid"} +GIT_ENV = { + **os.environ, + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@example.invalid", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@example.invalid", +} def git(repo, *args): - return subprocess.run(["git", "-C", str(repo), *args], check=True, env=GIT_ENV, - capture_output=True, text=True).stdout.strip() + return subprocess.run( + ["git", "-C", str(repo), *args], check=True, env=GIT_ENV, capture_output=True, text=True + ).stdout.strip() def write(path, text): @@ -45,19 +51,26 @@ def write(path, text): class SourcePackageTests(unittest.TestCase): def test_new_custom_backgrounds_and_authored_shaders_are_included(self): - selected = [Path("assets/ui/CustomTrackBackgrounds/custom-row-1.png"), - Path("assets/ui/CustomTrackBackgrounds/rt64.json"), - Path("patches/enhanced-water/native/LavaLightingCS.hlsl"), - Path("patches/enhanced-water/native/LavaFlow.hlsli")] - excluded = [Path("downloads/example.png"), Path("assets/ui/CustomTrackBackgrounds/game.z64"), - Path("patches/private.dkr-probe")] + selected = [ + Path("assets/ui/CustomTrackBackgrounds/custom-row-1.png"), + Path("assets/ui/CustomTrackBackgrounds/rt64.json"), + Path("patches/enhanced-water/native/LavaLightingCS.hlsl"), + Path("patches/enhanced-water/native/LavaFlow.hlsli"), + ] + excluded = [ + Path("downloads/example.png"), + Path("assets/ui/CustomTrackBackgrounds/game.z64"), + Path("patches/private.dkr-probe"), + ] with tempfile.TemporaryDirectory() as directory: root = Path(directory) for path in selected + excluded: (root / path).parent.mkdir(parents=True, exist_ok=True) (root / path).write_bytes(b"fixture") - with patch.object(package_source, "ROOT", root), \ - patch.object(package_source, "git_paths", side_effect=[[], selected + excluded]): + with ( + patch.object(package_source, "ROOT", root), + patch.object(package_source, "git_paths", side_effect=[[], selected + excluded]), + ): self.assertEqual(package_source.source_files(), sorted(selected)) @@ -71,15 +84,20 @@ def test_patched_dependencies_require_matching_lock_entries(self): (root / "packaging/linux").mkdir(parents=True) (root / "patches").mkdir() (root / "VERSION").write_text("1.0.5") - (root / "packaging/linux/dkr-port.appdata.xml").write_text( - '') - for name in ("README.md", "THIRD_PARTY.md", "docs/README.md", "docs/BUILDING.md", - "docs/PLATFORMS.md", "packaging/RELEASE-README.md", "packaging/MACOS-BUILD-README.md"): + (root / "packaging/linux/dkr-port.appdata.xml").write_text('') + for name in ( + "README.md", + "THIRD_PARTY.md", + "docs/README.md", + "docs/BUILDING.md", + "docs/PLATFORMS.md", + "packaging/RELEASE-README.md", + "packaging/MACOS-BUILD-README.md", + ): target = root / name target.parent.mkdir(parents=True, exist_ok=True) target.write_text("") - manifest = {"dependencies": [{"name": "NewDependency", - "expectedCommit": "abc123", "patches": []}]} + manifest = {"dependencies": [{"name": "NewDependency", "expectedCommit": "abc123", "patches": []}]} (root / "patches/manifest.json").write_text(json.dumps(manifest)) lock_path = root / "dependencies.lock.json" with patch.object(release_metadata, "ROOT", root): @@ -97,8 +115,12 @@ def test_patched_dependencies_require_matching_lock_entries(self): def test_invalid_version_is_rejected(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) - for version in ("1.0.5-beta.0", "1.0.5-beta.15-playtest.0", - "1.0.5-beta.15-playtest.15-fix.0", "1.0.5-garbage"): + for version in ( + "1.0.5-beta.0", + "1.0.5-beta.15-playtest.0", + "1.0.5-beta.15-playtest.15-fix.0", + "1.0.5-garbage", + ): with self.subTest(version=version): (root / "VERSION").write_text(version) with patch.object(release_metadata, "ROOT", root): @@ -108,8 +130,15 @@ def test_invalid_version_is_rejected(self): def test_unknown_dependency_names_and_historical_notes(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) - for name in ("README.md", "THIRD_PARTY.md", "docs/README.md", "docs/BUILDING.md", - "docs/PLATFORMS.md", "packaging/RELEASE-README.md", "packaging/MACOS-BUILD-README.md"): + for name in ( + "README.md", + "THIRD_PARTY.md", + "docs/README.md", + "docs/BUILDING.md", + "docs/PLATFORMS.md", + "packaging/RELEASE-README.md", + "packaging/MACOS-BUILD-README.md", + ): target = root / name target.parent.mkdir(parents=True, exist_ok=True) target.write_text("Current documentation\n") @@ -117,10 +146,12 @@ def test_unknown_dependency_names_and_historical_notes(self): (root / "packaging/linux").mkdir() (root / "packaging/linux/dkr-port.appdata.xml").write_text('') (root / "patches").mkdir() - (root / "patches/manifest.json").write_text(json.dumps({"dependencies": [ - {"name": "new-library", "expectedCommit": "abc", "patches": []}]})) - (root / "dependencies.lock.json").write_text(json.dumps({"dependencies": [ - {"name": "new-library", "commit": "abc"}]})) + (root / "patches/manifest.json").write_text( + json.dumps({"dependencies": [{"name": "new-library", "expectedCommit": "abc", "patches": []}]}) + ) + (root / "dependencies.lock.json").write_text( + json.dumps({"dependencies": [{"name": "new-library", "commit": "abc"}]}) + ) (root / "docs/historical-notes.md").write_text("Earlier milestone and release candidate\n") with patch.object(release_metadata, "ROOT", root): self.assertEqual(release_metadata.main(), 0) @@ -159,11 +190,16 @@ def make_patch(name, edit): first = make_patch("0001", lambda r: write(r / "a.txt", "one\nTWO\nthree\n")) # The second patch also creates a binary and an empty file and deletes # one: shapes a patch-text parser gets wrong. - second = make_patch("0002", lambda r: (write(r / "a.txt", "one\nTWO\nTHREE\n"), - write(r / "new.txt", "created\n"), - (r / "new.bin").write_bytes(b"\0\xff" * 8), - (r / "empty.txt").write_bytes(b""), - (r / "blob.bin").unlink())) + second = make_patch( + "0002", + lambda r: ( + write(r / "a.txt", "one\nTWO\nTHREE\n"), + write(r / "new.txt", "created\n"), + (r / "new.bin").write_bytes(b"\0\xff" * 8), + (r / "empty.txt").write_bytes(b""), + (r / "blob.bin").unlink(), + ), + ) self.final_a = "one\nTWO\nTHREE\n" self.patches = [first, second] # The dependency is checked out through `git worktree`, giving it a @@ -171,11 +207,27 @@ def make_patch(name, edit): self.repo = self.root / "dep" git(upstream, "worktree", "add", "-q", "--detach", str(self.repo), self.commit) self.manifest = self.root / "manifest.json" - self.manifest.write_text(json.dumps({"schemaVersion": 1, "dependencies": [{ - "name": "dep", "repositoryPath": "dep", "expectedCommit": self.commit, - "patches": [{"path": str(p.relative_to(self.root)), - "sha256": hashlib.sha256(p.read_bytes()).hexdigest()} - for p in self.patches]}]})) + self.manifest.write_text( + json.dumps( + { + "schemaVersion": 1, + "dependencies": [ + { + "name": "dep", + "repositoryPath": "dep", + "expectedCommit": self.commit, + "patches": [ + { + "path": str(p.relative_to(self.root)), + "sha256": hashlib.sha256(p.read_bytes()).hexdigest(), + } + for p in self.patches + ], + } + ], + } + ) + ) def tearDown(self): self.temp.cleanup() @@ -236,14 +288,14 @@ def test_exact_intermediate_patch_state_resumes(self): self.assert_patched() def test_check_validates_stacked_patches_without_checkout_mutation(self): - before = {p.name: (p.read_bytes(), p.stat().st_mtime_ns) - for p in self.repo.iterdir() if p.is_file()} + before = {p.name: (p.read_bytes(), p.stat().st_mtime_ns) for p in self.repo.iterdir() if p.is_file()} index_path = Path(git(self.repo, "rev-parse", "--git-path", "index")) index = index_path.read_bytes() self.assertEqual(patches.apply_manifest(self.root, self.manifest, check=True), 0) self.assertEqual(index_path.read_bytes(), index) - self.assertEqual(before, {p.name: (p.read_bytes(), p.stat().st_mtime_ns) - for p in self.repo.iterdir() if p.is_file()}) + self.assertEqual( + before, {p.name: (p.read_bytes(), p.stat().st_mtime_ns) for p in self.repo.iterdir() if p.is_file()} + ) def test_required_missing_checkout_and_empty_check_are_rejected(self): data = json.loads(self.manifest.read_text()) @@ -258,8 +310,15 @@ def test_required_missing_checkout_and_empty_check_are_rejected(self): patches.apply_manifest(self.root, self.manifest, check=True) def test_cli_check_and_failure_exit_codes(self): - command = [sys.executable, str(ROOT / "scripts/apply_dependency_patches.py"), - "--root", str(self.root), "--manifest", str(self.manifest), "--check"] + command = [ + sys.executable, + str(ROOT / "scripts/apply_dependency_patches.py"), + "--root", + str(self.root), + "--manifest", + str(self.manifest), + "--check", + ] result = subprocess.run(command, capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("validated", result.stdout) @@ -327,8 +386,12 @@ def setUp(self): self.addCleanup(self.env.stop) self.manifest.rename(self.root / "patches" / "manifest.json") self.manifest = self.root / "patches" / "manifest.json" - self.entry = {"name": "dep", "destination": "dep", "commit": self.commit, - "repository": str(self.root / "upstream")} + self.entry = { + "name": "dep", + "destination": "dep", + "commit": self.commit, + "repository": str(self.root / "upstream"), + } git(self.repo, "remote", "add", "origin", self.entry["repository"]) def bootstrap(self): @@ -342,8 +405,9 @@ def test_bootstrap_patch_bootstrap_preserves_files_and_index(self): index = git(self.repo, "write-tree") with patch.object(bootstrap, "run", wraps=bootstrap.run) as commands: self.assertEqual(self.bootstrap()["commit"], self.commit) - self.assertFalse(any(call.args[0][1] in ("fetch", "checkout", "reset", "clean") - for call in commands.call_args_list)) + self.assertFalse( + any(call.args[0][1] in ("fetch", "checkout", "reset", "clean") for call in commands.call_args_list) + ) self.assert_patched() self.assertEqual(git(self.repo, "write-tree"), index) self.assertEqual(before, {p.name: p.stat().st_mtime_ns for p in self.repo.iterdir()}) @@ -352,8 +416,15 @@ def test_bootstrap_rejects_unexpected_work_without_mutation(self): for kind in ("tracked", "managed", "untracked", "staged"): with self.subTest(kind=kind): self.apply() - target = self.repo / {"tracked": "unrelated.txt", "managed": "a.txt", - "untracked": "leftover.txt", "staged": "unrelated.txt"}[kind] + target = ( + self.repo + / { + "tracked": "unrelated.txt", + "managed": "a.txt", + "untracked": "leftover.txt", + "staged": "unrelated.txt", + }[kind] + ) original = target.read_bytes() if target.exists() else None target.write_bytes(b"developer work\n") if kind == "staged": @@ -390,10 +461,14 @@ def test_shallow_bootstrap_pins_old_revision_and_preserves_reviewed_edits(self): manifest["dependencies"][0]["repositoryPath"] = "fresh" self.manifest.write_text(json.dumps(manifest)) self.apply() - with patch.object(bootstrap, "ROOT", self.root), patch.object(bootstrap, "run", wraps=bootstrap.run) as commands: + with ( + patch.object(bootstrap, "ROOT", self.root), + patch.object(bootstrap, "run", wraps=bootstrap.run) as commands, + ): bootstrap.prepare_dependency(self.entry, False, shallow=True) - self.assertFalse(any(call.args[0][1] in ("fetch", "checkout", "reset", "clean") - for call in commands.call_args_list)) + self.assertFalse( + any(call.args[0][1] in ("fetch", "checkout", "reset", "clean") for call in commands.call_args_list) + ) self.assertEqual((checkout / "a.txt").read_text(), self.final_a) write(checkout / "a.txt", "developer work\n") with patch.object(bootstrap, "ROOT", self.root), self.assertRaisesRegex(RuntimeError, "contains local changes"): @@ -430,8 +505,7 @@ def test_bootstrap_preserves_reviewed_submodule_and_rejects_local_edits(self): git(child_origin, "add", ".") git(child_origin, "commit", "-qm", "child base") child_commit = git(child_origin, "rev-parse", "HEAD") - git(self.repo, "-c", "protocol.file.allow=always", "submodule", "add", - str(child_origin), "nested child") + git(self.repo, "-c", "protocol.file.allow=always", "submodule", "add", str(child_origin), "nested child") git(self.repo, "commit", "-qm", "add child") self.commit = git(self.repo, "rev-parse", "HEAD") self.entry.update(commit=self.commit, recursive=True) @@ -441,11 +515,16 @@ def test_bootstrap_preserves_reviewed_submodule_and_rejects_local_edits(self): write(patch_path, git(child, "diff") + "\n") manifest = json.loads(self.manifest.read_text()) manifest["dependencies"][0]["expectedCommit"] = self.commit - manifest["dependencies"].append({ - "name": "child", "repositoryPath": "dep/nested child", - "expectedCommit": child_commit, "patches": [{ - "path": "patches/child.patch", - "sha256": hashlib.sha256(patch_path.read_bytes()).hexdigest()}]}) + manifest["dependencies"].append( + { + "name": "child", + "repositoryPath": "dep/nested child", + "expectedCommit": child_commit, + "patches": [ + {"path": "patches/child.patch", "sha256": hashlib.sha256(patch_path.read_bytes()).hexdigest()} + ], + } + ) self.manifest.write_text(json.dumps(manifest)) self.apply() before = (child / "child.txt").stat().st_mtime_ns @@ -482,23 +561,28 @@ def test_nested_selection_initializes_parent_and_preserves_resolution_records(se git(child_origin, "add", ".") git(child_origin, "commit", "-qm", "child base") upstream = self.root / "upstream" - git(upstream, "-c", "protocol.file.allow=always", "submodule", "add", - str(child_origin), "nested child") + git(upstream, "-c", "protocol.file.allow=always", "submodule", "add", str(child_origin), "nested child") git(upstream, "commit", "-qm", "add child") parent = {**self.entry, "destination": "fresh", "commit": git(upstream, "rev-parse", "HEAD")} - child = {"name": "child", "destination": "fresh/nested child", - "repository": str(child_origin), "commit": git(child_origin, "rev-parse", "HEAD"), - "managedBy": "dep git submodule"} - self.assertEqual([e["name"] for e in bootstrap.select_entries([child, parent], {"child"})], - ["dep", "child"]) + child = { + "name": "child", + "destination": "fresh/nested child", + "repository": str(child_origin), + "commit": git(child_origin, "rev-parse", "HEAD"), + "managedBy": "dep git submodule", + } + self.assertEqual([e["name"] for e in bootstrap.select_entries([child, parent], {"child"})], ["dep", "child"]) lock = self.root / "lock.json" lock.write_text(json.dumps({"dependencies": [child, parent]})) resolved = self.root / "resolved.json" resolved.write_text(json.dumps({"dependencies": [{"name": "unrelated", "commit": "keep"}]})) - with patch.object(bootstrap, "ROOT", self.root), patch.object(bootstrap, "LOCK_FILE", lock), \ - patch.object(bootstrap, "RESOLVED_FILE", resolved), \ - patch.object(sys, "argv", ["bootstrap", "--only", "child"]), \ - patch.dict(os.environ, {"GIT_ALLOW_PROTOCOL": "file"}): + with ( + patch.object(bootstrap, "ROOT", self.root), + patch.object(bootstrap, "LOCK_FILE", lock), + patch.object(bootstrap, "RESOLVED_FILE", resolved), + patch.object(sys, "argv", ["bootstrap", "--only", "child"]), + patch.dict(os.environ, {"GIT_ALLOW_PROTOCOL": "file"}), + ): self.assertEqual(bootstrap.main(), 0) checkout = self.root / child["destination"] self.assertTrue((checkout / ".git").is_file(), "nested dependency must be a submodule, never a clone") @@ -530,8 +614,11 @@ def elf32be(sections): name_at += len(name) + 1 headers += struct.pack(">10I", name_at, 3, 0, 0, shstr_offset, len(names), 0, 0, 1, 0) count = len(sections) + 2 - header = b"\x7fELF\x01\x02\x01" + b"\0" * 9 + struct.pack( - ">HHIIIIIHHHHHH", 2, 8, 1, 0, 0, cursor, 0, 52, 0, 0, 40, count, count - 1) + header = ( + b"\x7fELF\x01\x02\x01" + + b"\0" * 9 + + struct.pack(">HHIIIIIHHHHHH", 2, 8, 1, 0, 0, cursor, 0, 52, 0, 0, 40, count, count - 1) + ) return header + body + headers @@ -545,9 +632,11 @@ def digest(self, sections): os.unlink(handle.name) def base(self, symtab=b"\0" * 16, code=b"\x27\xbd\xff\xe0", bss=8): - return [(".main", 1, 6, 0x80000400, code), - (".bss", 8, 3, 0x80100000, b"\0" * bss), - (".symtab", 2, 0, 0, symtab)] + return [ + (".main", 1, 6, 0x80000400, code), + (".bss", 8, 3, 0x80100000, b"\0" * bss), + (".symtab", 2, 0, 0, symtab), + ] def test_symbol_table_differences_are_ignored(self): self.assertEqual(self.digest(self.base()), self.digest(self.base(symtab=b"\1" * 32))) @@ -598,14 +687,14 @@ def test_build_prepares_without_cli_arguments_in_system_bash(self): prepare.write_text('#!/bin/sh\necho "prepare called"\nexit 42\n') prepare.chmod(0o755) uname = root / "bin" / "uname" - uname.write_text('#!/bin/sh\necho Darwin\n') + uname.write_text("#!/bin/sh\necho Darwin\n") uname.chmod(0o755) - env = dict(os.environ, PATH=str(root / "bin") + os.pathsep + os.environ["PATH"], - CMAKE_BUILD_PARALLEL_LEVEL="1") + env = dict( + os.environ, PATH=str(root / "bin") + os.pathsep + os.environ["PATH"], CMAKE_BUILD_PARALLEL_LEVEL="1" + ) env.pop("DKR_MAC_GENERATED_SOURCE", None) env.pop("DKR_MAC_V80_GENERATED_SOURCE", None) - result = subprocess.run(["/bin/bash", str(build)], env=env, - capture_output=True, text=True) + result = subprocess.run(["/bin/bash", str(build)], env=env, capture_output=True, text=True) self.assertEqual(result.returncode, 42, result.stdout + result.stderr) self.assertIn("prepare called", result.stdout) @@ -622,8 +711,7 @@ def test_macos_option_errors_are_actionable(self): (root / script).write_text((ROOT / script).read_text()) for option in ("--rom-v77", "--rom-v80", "--jobs"): with self.subTest(script=script, option=option): - result = subprocess.run(["bash", str(root / script), option], - capture_output=True, text=True) + result = subprocess.run(["bash", str(root / script), option], capture_output=True, text=True) self.assertNotEqual(result.returncode, 0) self.assertIn("requires", result.stdout + result.stderr) self.assertNotIn("unbound variable", result.stdout + result.stderr) @@ -636,8 +724,7 @@ def test_macos_decomp_ready_requires_all_stage_tools(self): with tempfile.TemporaryDirectory() as temporary: tree = Path(temporary) / "isolated revision with spaces" python = tree / ".venv/bin/python3" - tools = [tree / "tools/dkr_assets_tool", tree / "tools/n64crc", - tree / "tools/ido-recomp/macos/cc"] + tools = [tree / "tools/dkr_assets_tool", tree / "tools/n64crc", tree / "tools/ido-recomp/macos/cc"] def executable(path, status=0): path.parent.mkdir(parents=True, exist_ok=True) @@ -645,10 +732,15 @@ def executable(path, status=0): path.chmod(0o755) def ready(): - return subprocess.run( - ["bash", "-c", helper.group(0) + '\ndecomp_ready "$1"', - "stage-test", str(tree)], capture_output=True, text=True, - timeout=5).returncode == 0 + return ( + subprocess.run( + ["bash", "-c", helper.group(0) + '\ndecomp_ready "$1"', "stage-test", str(tree)], + capture_output=True, + text=True, + timeout=5, + ).returncode + == 0 + ) self.assertFalse(ready()) executable(python) @@ -664,8 +756,9 @@ def ready(): self.assertFalse(ready()) def tracked(self): - output = subprocess.run(["git", "-C", str(ROOT), "ls-files", "-s"], check=True, - capture_output=True, text=True).stdout + output = subprocess.run( + ["git", "-C", str(ROOT), "ls-files", "-s"], check=True, capture_output=True, text=True + ).stdout for line in output.splitlines(): mode, _, _, path = line.split(maxsplit=3) yield mode, path @@ -674,7 +767,9 @@ def test_macos_shell_entry_points_are_executable(self): if not (ROOT / ".git").exists(): self.skipTest("not a git checkout") entry_points = { - "Build-macOS.sh", "Run-macOS.sh", "Setup-macOS.sh", + "Build-macOS.sh", + "Run-macOS.sh", + "Setup-macOS.sh", "scripts/Prepare-DKR-Runtime-macOS.sh", "scripts/Prepare-Game-Source-macOS.sh", "scripts/apply-dependency-patches.sh", @@ -693,38 +788,47 @@ def test_shell_script_references_match_on_disk_case(self): wrong.append(f"{script.name}: {ref} -> {lower[ref.lower()]}") self.assertEqual(wrong, []) + @unittest.skipIf(os.name == "nt", "macOS packaging and POSIX build lock") class MacOSDevexTests(unittest.TestCase): def test_package_versions_are_distinct_and_overridable(self): import package_macos as package + with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) with patch.object(package.time, "time", return_value=1800000000): first = package.bundle_version(root, 1) second = package.bundle_version(root, 1) - self.assertLess(tuple(map(int, first.split('.'))), tuple(map(int, second.split('.')))) - self.assertEqual(package.bundle_version(root, 1, '42.3.2'), '42.3.2') - for invalid in ('0', '10000', '1.100.1', '1.2.100', 'beta', '-1'): + self.assertLess(tuple(map(int, first.split("."))), tuple(map(int, second.split(".")))) + self.assertEqual(package.bundle_version(root, 1, "42.3.2"), "42.3.2") + for invalid in ("0", "10000", "1.100.1", "1.2.100", "beta", "-1"): with self.assertRaises(ValueError): package.bundle_version(root, 1, invalid) - (root / 'bin/Release').mkdir(parents=True) - (root / 'bin/Release/dkr-release.json').write_text('{"version":"1.0.5-old","build_number":1}') - with self.assertRaisesRegex(RuntimeError, 'differs from configured'): - package.package(root, '1.0.5-new') + (root / "bin/Release").mkdir(parents=True) + (root / "bin/Release/dkr-release.json").write_text('{"version":"1.0.5-old","build_number":1}') + with self.assertRaisesRegex(RuntimeError, "differs from configured"): + package.package(root, "1.0.5-new") def test_notarize_requires_identity_and_profile(self): env = {k: v for k, v in os.environ.items() if not k.startswith("DKR_MAC_")} - for extra in ({}, {"DKR_MAC_SIGN_IDENTITY": "-", "DKR_MAC_NOTARY_PROFILE": "p"}, - {"DKR_MAC_SIGN_IDENTITY": "Developer ID Application: X (T)"}): - result = subprocess.run([sys.executable, str(ROOT / "scripts/package_macos.py"), - "build", "1.0.8", "--notarize"], - capture_output=True, text=True, env={**env, **extra}) + for extra in ( + {}, + {"DKR_MAC_SIGN_IDENTITY": "-", "DKR_MAC_NOTARY_PROFILE": "p"}, + {"DKR_MAC_SIGN_IDENTITY": "Developer ID Application: X (T)"}, + ): + result = subprocess.run( + [sys.executable, str(ROOT / "scripts/package_macos.py"), "build", "1.0.8", "--notarize"], + capture_output=True, + text=True, + env={**env, **extra}, + ) self.assertEqual(result.returncode, 2, result.stderr) self.assertIn("DKR_MAC_SIGN_IDENTITY and DKR_MAC_NOTARY_PROFILE", result.stderr) @unittest.skipUnless(shutil.which("ditto"), "ditto is macOS-only") def test_archive_rejects_appledouble_entries(self): import package_macos as package + with tempfile.TemporaryDirectory() as tmp: stage = Path(tmp) / "stage" (stage / "DKR-R.app").mkdir(parents=True) @@ -737,29 +841,30 @@ def test_archive_rejects_appledouble_entries(self): def test_lock_rejects_concurrent_builds_and_allows_nested_prepare(self): import fcntl - helper = ROOT / 'scripts/macos_build_lock.py' + + helper = ROOT / "scripts/macos_build_lock.py" with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) - child = root / 'child.sh' - child.write_text('echo nested-ok\n') - parent = root / 'parent.sh' + child = root / "child.sh" + child.write_text("echo nested-ok\n") + parent = root / "parent.sh" parent.write_text(f'"{sys.executable}" "{helper}" "{root}" "{child}"\n') cmd = [sys.executable, str(helper), str(root), str(parent)] result = subprocess.run(cmd, capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn('nested-ok', result.stdout) - with (root / 'build/.macos-build.lock').open('r+') as held: + self.assertIn("nested-ok", result.stdout) + with (root / "build/.macos-build.lock").open("r+") as held: fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) result = subprocess.run(cmd, capture_output=True, text=True) self.assertNotEqual(result.returncode, 0) - self.assertIn('Another macOS build/prepare', result.stderr) - parent.write_text('exit 42\n') + self.assertIn("Another macOS build/prepare", result.stderr) + parent.write_text("exit 42\n") self.assertEqual(subprocess.run(cmd).returncode, 42) - parent.write_text('exit 0\n') + parent.write_text("exit 0\n") self.assertEqual(subprocess.run(cmd).returncode, 0) def test_apple_helpers_validate_jobs_without_source_side_effects(self): - helper = ROOT / 'scripts/lib/apple-build.sh' + helper = ROOT / "scripts/lib/apple-build.sh" script = r""" set -euo pipefail before_pwd="$PWD" @@ -794,38 +899,51 @@ def test_apple_helpers_validate_jobs_without_source_side_effects(self): xcodebuild() { return 0; } dkr_require_xcode """ - result = subprocess.run(['/bin/bash', '-c', script, 'apple-helper-test', str(helper)], - capture_output=True, text=True) + result = subprocess.run( + ["/bin/bash", "-c", script, "apple-helper-test", str(helper)], capture_output=True, text=True + ) self.assertEqual(result.returncode, 0, result.stderr) def test_host_check_rejects_missing_sha1sum_and_old_gmake(self): - helper = ROOT / 'scripts/macos-host-tools.sh' + helper = ROOT / "scripts/macos-host-tools.sh" with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) - def executable(name, body='exit 0'): + + def executable(name, body="exit 0"): p = root / name - p.write_text('#!/bin/sh\n' + body + '\n') + p.write_text("#!/bin/sh\n" + body + "\n") p.chmod(0o755) - for name in ('xcodebuild', 'git', 'python3', 'cmake', 'ninja', 'c++', 'shasum', 'pcre2-config', 'sdl2-config'): + + for name in ( + "xcodebuild", + "git", + "python3", + "cmake", + "ninja", + "c++", + "shasum", + "pcre2-config", + "sdl2-config", + ): executable(name) - executable('uname', 'echo Darwin') - executable('gmake', 'echo "GNU Make 3.81"') - cmd = ['/bin/bash', '-c', f'source "{helper}"; dkr_macos_host_tools 1'] + executable("uname", "echo Darwin") + executable("gmake", 'echo "GNU Make 3.81"') + cmd = ["/bin/bash", "-c", f'source "{helper}"; dkr_macos_host_tools 1'] env = dict(os.environ, PATH=str(root)) missing = subprocess.run(cmd, env=env, capture_output=True, text=True) self.assertNotEqual(missing.returncode, 0) - self.assertIn('missing from PATH: sha1sum', missing.stderr) - executable('sha1sum') + self.assertIn("missing from PATH: sha1sum", missing.stderr) + executable("sha1sum") old_make = subprocess.run(cmd, env=env, capture_output=True, text=True) self.assertNotEqual(old_make.returncode, 0) - self.assertIn('GNU Make 4', old_make.stderr) - executable('gmake', 'echo "GNU Make 4.4"') - executable('sha1sum', 'exit 1') - for name in ('mktemp', 'rm'): - (root / name).symlink_to('/usr/bin/' + name if Path('/usr/bin/' + name).exists() else '/bin/' + name) + self.assertIn("GNU Make 4", old_make.stderr) + executable("gmake", 'echo "GNU Make 4.4"') + executable("sha1sum", "exit 1") + for name in ("mktemp", "rm"): + (root / name).symlink_to("/usr/bin/" + name if Path("/usr/bin/" + name).exists() else "/bin/" + name) incompatible = subprocess.run(cmd, env=env, capture_output=True, text=True) self.assertNotEqual(incompatible.returncode, 0) - self.assertIn('sha1sum must support -c --quiet', incompatible.stderr) + self.assertIn("sha1sum must support -c --quiet", incompatible.stderr) if __name__ == "__main__": diff --git a/runtime-recomp/tests/ci_summary_tests.py b/runtime-recomp/tests/ci_summary_tests.py index 2ad89d87..6685969a 100644 --- a/runtime-recomp/tests/ci_summary_tests.py +++ b/runtime-recomp/tests/ci_summary_tests.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Exercise CI summaries using CTest XML and the real harness report writer.""" + import contextlib import io import json @@ -9,7 +10,7 @@ import unittest ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / 'scripts')) +sys.path.insert(0, str(ROOT / "scripts")) import ci_summary from harness_report import write_reports @@ -22,106 +23,132 @@ def setUp(self): def artifact(self, name, text): path = self.root / name - path.write_text(text, encoding='utf-8') + path.write_text(text, encoding="utf-8") return path def report(self): - return {'schema_version': 1, 'suite': 'Synthetic checkpoint regression', - 'summary': {'passed': 1, 'failed': 0, 'skipped': 0}, - 'scenarios': [{'id': 'checkpoint-benchmark', 'status': 'passed', 'errors': [], - 'attempts': [], 'comparisons': [], - 'metrics': {'process.median_ms': 42, 'checkpoint.hash_median_ms': .25}, - 'fingerprints': {'Checkpoint fingerprint: [0-9a-f]+': ['Checkpoint fingerprint: deadbeef']}}]} + return { + "schema_version": 1, + "suite": "Synthetic checkpoint regression", + "summary": {"passed": 1, "failed": 0, "skipped": 0}, + "scenarios": [ + { + "id": "checkpoint-benchmark", + "status": "passed", + "errors": [], + "attempts": [], + "comparisons": [], + "metrics": {"process.median_ms": 42, "checkpoint.hash_median_ms": 0.25}, + "fingerprints": {"Checkpoint fingerprint: [0-9a-f]+": ["Checkpoint fingerprint: deadbeef"]}, + } + ], + } def test_nested_suites_count_cases_once_and_keep_failures_skips_and_slowest(self): - junit = self.artifact('junit.xml', ''' + junit = self.artifact( + "junit.xml", + """ stackdetails crashed - ''') + """, + ) text, invalid = ci_summary.render(junit=junit) self.assertFalse(invalid) - self.assertIn('1 passed, 2 failed, 2 skipped', text) - self.assertIn('broken\nstack\ndetails', text.replace('\n\n', '\n')) - self.assertIn('| slow failure | 9.000 | failed |', text) - self.assertLess(text.index('| slow failure'), text.index('| error')) - self.assertNotIn('999', text) + self.assertIn("1 passed, 2 failed, 2 skipped", text) + self.assertIn("broken\nstack\ndetails", text.replace("\n\n", "\n")) + self.assertIn("| slow failure | 9.000 | failed |", text) + self.assertLess(text.index("| slow failure"), text.index("| error")) + self.assertNotIn("999", text) def test_real_harness_writer_schema_and_checkpoint_evidence(self): write_reports(self.root, self.report()) - text, invalid = ci_summary.render(junit=self.root / 'junit.xml', regression=self.root / 'report.json') + text, invalid = ci_summary.render(junit=self.root / "junit.xml", regression=self.root / "report.json") self.assertFalse(invalid) - self.assertIn('1 passed, 0 failed, 0 skipped', text) - self.assertIn('0.250 ms', text) - self.assertIn('Checkpoint fingerprint: deadbeef', text) - self.assertIn('42.000', text) + self.assertIn("1 passed, 0 failed, 0 skipped", text) + self.assertIn("0.250 ms", text) + self.assertIn("Checkpoint fingerprint: deadbeef", text) + self.assertIn("42.000", text) def test_failed_and_skipped_scenarios_do_not_imply_passing_coverage(self): report = self.report() - report['scenarios'][0].update(status='failed', errors=['baseline fingerprint changed']) - report['scenarios'].append({'id': 'missing-rom', 'status': 'skipped', 'errors': ['Missing ROM'], 'metrics': {}}) - report['summary'] = {'passed': 0, 'failed': 1, 'skipped': 1} - text, invalid = ci_summary.render(regression=self.artifact('report.json', json.dumps(report))) + report["scenarios"][0].update(status="failed", errors=["baseline fingerprint changed"]) + report["scenarios"].append({"id": "missing-rom", "status": "skipped", "errors": ["Missing ROM"], "metrics": {}}) + report["summary"] = {"passed": 0, "failed": 1, "skipped": 1} + text, invalid = ci_summary.render(regression=self.artifact("report.json", json.dumps(report))) self.assertFalse(invalid) - self.assertIn('0 passed, 1 failed, 1 skipped', text) - self.assertIn('baseline fingerprint changed', text) - self.assertIn('Missing ROM', text) + self.assertIn("0 passed, 1 failed, 1 skipped", text) + self.assertIn("baseline fingerprint changed", text) + self.assertIn("Missing ROM", text) def test_missing_and_empty_reports_are_explicit(self): - text, invalid = ci_summary.render(junit=self.root / 'missing.xml') + text, invalid = ci_summary.render(junit=self.root / "missing.xml") self.assertFalse(invalid) - self.assertIn('earlier configure, build, or test step may have failed', text) - self.assertNotIn('0 passed', text) - text, invalid = ci_summary.render(junit=self.artifact('empty.xml', '')) + self.assertIn("earlier configure, build, or test step may have failed", text) + self.assertNotIn("0 passed", text) + text, invalid = ci_summary.render(junit=self.artifact("empty.xml", "")) self.assertFalse(invalid) - self.assertIn('not evidence of a passing test run', text) + self.assertIn("not evidence of a passing test run", text) def test_malformed_artifacts_are_reported_and_exit_nonzero(self): bad_report = self.report() - bad_report['summary']['passed'] = 99 - for text in ('{', '[]', json.dumps(bad_report)): + bad_report["summary"]["passed"] = 99 + for text in ("{", "[]", json.dumps(bad_report)): with self.subTest(text=text): - path = self.artifact('bad.json', text) - output = self.root / 'summary.md' - self.assertEqual(ci_summary.main(['--regression', str(path), '--output', str(output)]), 1) - self.assertIn('Invalid artifact:', output.read_text()) - for xml in ('', '', ''): - text, invalid = ci_summary.render(junit=self.artifact('bad.xml', xml)) + path = self.artifact("bad.json", text) + output = self.root / "summary.md" + self.assertEqual(ci_summary.main(["--regression", str(path), "--output", str(output)]), 1) + self.assertIn("Invalid artifact:", output.read_text()) + for xml in ( + "", + "", + '', + ): + text, invalid = ci_summary.render(junit=self.artifact("bad.xml", xml)) self.assertTrue(invalid) - self.assertIn('No successful result is inferred', text) + self.assertIn("No successful result is inferred", text) def test_markdown_html_and_log_sizes_are_bounded(self): - xml = '</pre><script>' + 'x' * 4000 + '' - text, _ = ci_summary.render(junit=self.artifact('escape.xml', xml), label='|bad') - self.assertNotIn('', text) - self.assertNotIn('')) - self.assertIn('Unrecognized cache statistics', text) - self.assertIn('<script>', text) + self.assertIn("60", text) + self.assertNotIn("60%", text) # Never invent a hit ratio. + text, _ = ci_summary.render(cache_stats=self.artifact("cache.txt", "")) + self.assertIn("Unrecognized cache statistics", text) + self.assertIn("<script>", text) def test_output_appends_and_stdout_is_default(self): - path = self.artifact('junit.xml', '') - output = self.artifact('summary.md', 'Earlier summary\n') - self.assertEqual(ci_summary.main(['--junit', str(path), '--label', 'Windows host tests', '--output', str(output)]), 0) - self.assertTrue(output.read_text().startswith('Earlier summary\n')) - self.assertIn('## Windows host tests', output.read_text()) + path = self.artifact("junit.xml", '') + output = self.artifact("summary.md", "Earlier summary\n") + self.assertEqual( + ci_summary.main(["--junit", str(path), "--label", "Windows host tests", "--output", str(output)]), 0 + ) + self.assertTrue(output.read_text().startswith("Earlier summary\n")) + self.assertIn("## Windows host tests", output.read_text()) with contextlib.redirect_stdout(io.StringIO()) as stdout: - self.assertEqual(ci_summary.main(['--junit', str(path)]), 0) - self.assertIn('1 passed', stdout.getvalue()) + self.assertEqual(ci_summary.main(["--junit", str(path)]), 0) + self.assertIn("1 passed", stdout.getvalue()) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/runtime-recomp/tests/developer_entrypoint_tests.py b/runtime-recomp/tests/developer_entrypoint_tests.py index 7d49c69e..67a091d2 100644 --- a/runtime-recomp/tests/developer_entrypoint_tests.py +++ b/runtime-recomp/tests/developer_entrypoint_tests.py @@ -1,4 +1,5 @@ """Exercise shell command failures with fake compilers and no private inputs.""" + import os from pathlib import Path import shutil @@ -18,8 +19,11 @@ def setUp(self): self.bin = self.root / "tools" self.bin.mkdir() self.trace = self.root / "trace" - self.env = {**os.environ, "PATH": f"{self.bin}{os.pathsep}{os.environ['PATH']}", - "DKR_TEST_TRACE": str(self.trace)} + self.env = { + **os.environ, + "PATH": f"{self.bin}{os.pathsep}{os.environ['PATH']}", + "DKR_TEST_TRACE": str(self.trace), + } def executable(self, path, body): path.parent.mkdir(parents=True, exist_ok=True) @@ -27,17 +31,18 @@ def executable(self, path, body): path.chmod(0o755) def invoke(self, script, *args): - return subprocess.run([str(self.root / script), *args], env=self.env, - capture_output=True, text=True) + return subprocess.run([str(self.root / script), *args], env=self.env, capture_output=True, text=True) def linux_fixture(self): shutil.copy2(ROOT / "Build-Linux.sh", self.root / "Build-Linux.sh") (self.root / "VERSION").write_text("1.0.5\n") for name in ("v77", "v80", "extern/rt64", "extern/n64-modern-runtime"): (self.root / name).mkdir(parents=True) - self.env.update(DKR_LINUX_GENERATED_SOURCE=str(self.root / "v77"), - DKR_LINUX_V80_GENERATED_SOURCE=str(self.root / "v80"), - DKR_LINUX_BUILD_DIR=str(self.root / "build")) + self.env.update( + DKR_LINUX_GENERATED_SOURCE=str(self.root / "v77"), + DKR_LINUX_V80_GENERATED_SOURCE=str(self.root / "v80"), + DKR_LINUX_BUILD_DIR=str(self.root / "build"), + ) self.executable(self.bin / "cmake", 'echo cmake >> "$DKR_TEST_TRACE"') self.executable(self.bin / "ctest", 'echo ctest >> "$DKR_TEST_TRACE"; exit "${DKR_TEST_EXIT:-0}"') self.executable(self.root / "build/bin/Release/DKR-R", 'echo runtime >> "$DKR_TEST_TRACE"') @@ -79,9 +84,17 @@ def test_missing_linux_libraries_print_install_command(self): def test_failed_build_kit_does_not_poison_retry(self): (self.root / "scripts").mkdir() shutil.copy2(ROOT / "scripts/Package-macOS-Build-Kit.sh", self.root / "scripts/Package-macOS-Build-Kit.sh") - for name in ("VERSION", "dependencies.lock.json", "docs/BUILDING.md", "LICENSE.md", - "THIRD_PARTY.md", "packaging/MACOS-BUILD-README.md", "Build-macOS.sh", - "Setup-macOS.sh", "Run-macOS.sh"): + for name in ( + "VERSION", + "dependencies.lock.json", + "docs/BUILDING.md", + "LICENSE.md", + "THIRD_PARTY.md", + "packaging/MACOS-BUILD-README.md", + "Build-macOS.sh", + "Setup-macOS.sh", + "Run-macOS.sh", + ): path = self.root / name path.parent.mkdir(parents=True, exist_ok=True) path.write_text("1.0.5\n") diff --git a/runtime-recomp/tests/finish_presentation_pipeline_tests.py b/runtime-recomp/tests/finish_presentation_pipeline_tests.py index b5fdbaf5..c0f18a30 100644 --- a/runtime-recomp/tests/finish_presentation_pipeline_tests.py +++ b/runtime-recomp/tests/finish_presentation_pipeline_tests.py @@ -1,4 +1,5 @@ """Verify both new hooks against generated placement AND original ELF opcodes.""" + import json import pathlib import struct @@ -6,6 +7,7 @@ ROOT = pathlib.Path(__file__).resolve().parents[1] + def instruction(elf, address): data = pathlib.Path(elf).read_bytes() assert data[:5] == b"\x7fELF\x01" @@ -19,6 +21,7 @@ def instruction(elf, address): return struct.unpack_from(endian + "I", data, pos + address - start)[0] raise AssertionError(f"instruction not found: {address:x}") + expected = {"v77": (0x80058DC4, 0x800955F0), "v80": (0x80058E04, 0x80095AF4)} assert len(sys.argv) == 5, "generated v77, generated v80, ELF v77, ELF v80" for (rev, anchors), generated_dir, elf in zip(expected.items(), sys.argv[1:3], sys.argv[3:5], strict=True): @@ -27,7 +30,9 @@ def instruction(elf, address): generated = "\n".join(p.read_text() for p in pathlib.Path(generated_dir).glob("*.c")) for symbol, function, address in zip( ("dkr_presentation_finish_camera_node", "dkr_postrace_wooden_frame_draw"), - ("update_camera_finish_race", "postrace_viewport"), anchors, strict=True + ("update_camera_finish_race", "postrace_viewport"), + anchors, + strict=True, ): matches = [h for h in hooks if symbol in h["text"]] assert len(matches) == 1 @@ -42,5 +47,7 @@ def instruction(elf, address): assert all(obsolete not in h["text"] for h in hooks) postrace = [h for h in hooks if "dkr_postrace_presentation_start(" in h["text"]] assert len(postrace) == 1 and postrace[0]["function"] == "postrace_start" - assert postrace[0]["text"].startswith("extern void dkr_netplay_postrace_barrier(uint8_t*, recomp_context*); dkr_netplay_postrace_barrier(rdram, ctx);") + assert postrace[0]["text"].startswith( + "extern void dkr_netplay_postrace_barrier(uint8_t*, recomp_context*); dkr_netplay_postrace_barrier(rdram, ctx);" + ) print(f"[test][finish-pipeline] {rev}: ELF instructions, unique hooks, obsolete disables removed PASS") diff --git a/runtime-recomp/tests/harness_report_tests.py b/runtime-recomp/tests/harness_report_tests.py index a859ac8e..9befcb20 100644 --- a/runtime-recomp/tests/harness_report_tests.py +++ b/runtime-recomp/tests/harness_report_tests.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Exercise report portability, failure semantics and untrusted diagnostics.""" + import json from pathlib import Path import sys @@ -20,11 +21,28 @@ def report(self): "identity": {"revision": "a&b"}, "summary": {"passed": 1, "failed": 1, "skipped": 1}, "scenarios": [ - {"id": "pass", "status": "passed", "metrics": {"wall_ms": 12.5}, "attempts": [ - {"index": 1, "phase": "measured", "status": "passed", "elapsed_ms": 12.5, - "exit_code": 0, "stdout": "logs/log one.txt", "stderr": "logs/stderr.txt"}]}, - {"id": "failure", "status": "failed", "errors": ["bad \x1b"], "comparisons": [ - {"metric": "time", "baseline": 1, "current": 2, "limit": 1.1, "status": "failed"}]}, + { + "id": "pass", + "status": "passed", + "metrics": {"wall_ms": 12.5}, + "attempts": [ + { + "index": 1, + "phase": "measured", + "status": "passed", + "elapsed_ms": 12.5, + "exit_code": 0, + "stdout": "logs/log one.txt", + "stderr": "logs/stderr.txt", + } + ], + }, + { + "id": "failure", + "status": "failed", + "errors": ["bad \x1b"], + "comparisons": [{"metric": "time", "baseline": 1, "current": 2, "limit": 1.1, "status": "failed"}], + }, {"id": "skip", "status": "skipped", "errors": ["ROM absent"]}, ], } @@ -49,7 +67,17 @@ def test_reports_preserve_results_and_escape_markup(self): self.assertEqual(cases["skip"].find("skipped").attrib["message"], "ROM absent") def test_log_links_reject_active_and_escaping_paths(self): - for path in ["javascript:alert(1)", "https://example.org", "//example.org", "../secret", "/secret", "logs/../../secret", "C:\\secret", "//[bad", "\nmalicious"]: + for path in [ + "javascript:alert(1)", + "https://example.org", + "//example.org", + "../secret", + "/secret", + "logs/../../secret", + "C:\\secret", + "//[bad", + "\nmalicious", + ]: with self.subTest(path=path), tempfile.TemporaryDirectory() as directory: report = self.report() report["scenarios"][0]["attempts"][0]["stdout"] = path diff --git a/runtime-recomp/tests/host_task_policy_tests.py b/runtime-recomp/tests/host_task_policy_tests.py index 509cdba3..7c485a3f 100644 --- a/runtime-recomp/tests/host_task_policy_tests.py +++ b/runtime-recomp/tests/host_task_policy_tests.py @@ -16,14 +16,20 @@ def fixture(self, revision): policy = json.loads((ROOT / f"runtime-recomp/dkr.us.v{revision}.recomp-policy.json").read_text()) hook = next(h for h in policy["functionHooks"] if "dkr_scheduler_keep_host_tasks" in h["text"]) pc = int(hook["beforeVram"], 0) - target, sp, dp = ((0x80079818, 0xE754, 0xE758) if revision == 77 else - (0x80079C68, 0xECD4, 0xECD8)) - words = {pc: 0x0C000000 | ((target >> 2) & 0x03FFFFFF), pc+4: 0x02402025, - target+0x30: 0x3C03800E, target+0x34: 0x24630000 | sp, - target+0x58: 0x3C08800E, target+0x5C: 0x8D080000 | dp, - target+0x74: 0x2941000B, target+0xBC: 0x2981000B} + target, sp, dp = (0x80079818, 0xE754, 0xE758) if revision == 77 else (0x80079C68, 0xECD4, 0xECD8) + words = { + pc: 0x0C000000 | ((target >> 2) & 0x03FFFFFF), + pc + 4: 0x02402025, + target + 0x30: 0x3C03800E, + target + 0x34: 0x24630000 | sp, + target + 0x58: 0x3C08800E, + target + 0x5C: 0x8D080000 | dp, + target + 0x74: 0x2941000B, + target + 0xBC: 0x2981000B, + } data = bytearray(0x400) - for address, word in words.items(): struct.pack_into(">I", data, address-pc, word) + for address, word in words.items(): + struct.pack_into(">I", data, address - pc, word) return policy, hook, pc, data, words def test_reviewed_revisions(self): @@ -37,24 +43,33 @@ def test_changed_instructions_fail_closed(self): policy, _, pc, data, words = self.fixture(revision) for address in words: corrupted = bytearray(data) - corrupted[address-pc+3] ^= 1 - with self.subTest(revision=revision, address=address), patch( - "host_task_policy.elf_sections", return_value=[(pc, corrupted)]): - with self.assertRaises(ValueError): verify_host_task_policy(policy, None) + corrupted[address - pc + 3] ^= 1 + with ( + self.subTest(revision=revision, address=address), + patch("host_task_policy.elf_sections", return_value=[(pc, corrupted)]), + ): + with self.assertRaises(ValueError): + verify_host_task_policy(policy, None) def test_wrong_owner_register_counter_and_duplicate(self): for revision in (77, 80): for modification in ("owner", "register", "counter", "duplicate"): policy, hook, _, _, _ = self.fixture(revision) - if modification == "owner": hook["function"] = "__scHandleRSP" - if modification == "register": hook["text"] = hook["text"].replace("ctx->r18", "ctx->r16") - if modification == "counter": hook["text"] = hook["text"].replace("0x800D", "0x800E") - if modification == "duplicate": policy["functionHooks"].append(copy.deepcopy(hook)) + if modification == "owner": + hook["function"] = "__scHandleRSP" + if modification == "register": + hook["text"] = hook["text"].replace("ctx->r18", "ctx->r16") + if modification == "counter": + hook["text"] = hook["text"].replace("0x800D", "0x800E") + if modification == "duplicate": + policy["functionHooks"].append(copy.deepcopy(hook)) with self.subTest(revision=revision, modification=modification): - with self.assertRaises(ValueError): verify_host_task_policy(policy, None) + with self.assertRaises(ValueError): + verify_host_task_policy(policy, None) def test_unrelated_policy_unchanged(self): verify_host_task_policy({"functionHooks": []}, None) -if __name__ == "__main__": unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/runtime-recomp/tests/hud_pipeline_tests.py b/runtime-recomp/tests/hud_pipeline_tests.py index bdecf78c..994c93ff 100644 --- a/runtime-recomp/tests/hud_pipeline_tests.py +++ b/runtime-recomp/tests/hud_pipeline_tests.py @@ -1,4 +1,5 @@ """HUD hook instruction/mapping and protected-baseline checks, without a game.""" + import json import pathlib import re @@ -8,93 +9,112 @@ ROOT = pathlib.Path(__file__).resolve().parents[2] + def symbols(path): """Read the retail ELF symbols; do not duplicate addresses in the test.""" data = pathlib.Path(path).read_bytes() - assert data[:5] == b'\x7fELF\x01' - endian = '>' if data[5] == 2 else '<' - offset = struct.unpack_from(endian+'I', data, 32)[0] - size, count = struct.unpack_from(endian+'HH', data, 46) - sections = [struct.unpack_from(endian+'10I', data, offset+i*size) for i in range(count)] + assert data[:5] == b"\x7fELF\x01" + endian = ">" if data[5] == 2 else "<" + offset = struct.unpack_from(endian + "I", data, 32)[0] + size, count = struct.unpack_from(endian + "HH", data, 46) + sections = [struct.unpack_from(endian + "10I", data, offset + i * size) for i in range(count)] result = {} for section in sections: - if section[1] != 2: continue # SHT_SYMTAB + if section[1] != 2: + continue # SHT_SYMTAB strings = sections[section[6]] - for pos in range(section[4], section[4]+section[5], section[9]): - name, value, *_ = struct.unpack_from(endian+'IIIBBH', data, pos) - start = strings[4]+name - result[data[start:data.index(b'\0', start)].decode()] = value + for pos in range(section[4], section[4] + section[5], section[9]): + name, value, *_ = struct.unpack_from(endian + "IIIBBH", data, pos) + start = strings[4] + name + result[data[start : data.index(b"\0", start)].decode()] = value return result + def hud_display_list(header, revision): - fields = re.findall(r'std::uint32_t\s+(\w+)\s*;', - header.split('struct AddressTable {',1)[1].split('};',1)[0]) - table = header.split('AddressTable kUs'+revision.upper()+'{',1)[1].split('};',1)[0] - values = [int(value,16) for value in re.findall(r'0x([0-9A-Fa-f]+)U',table)] + fields = re.findall(r"std::uint32_t\s+(\w+)\s*;", header.split("struct AddressTable {", 1)[1].split("};", 1)[0]) + table = header.split("AddressTable kUs" + revision.upper() + "{", 1)[1].split("};", 1)[0] + values = [int(value, 16) for value in re.findall(r"0x([0-9A-Fa-f]+)U", table)] assert len(fields) == len(values) - return dict(zip(fields,values))['HudDisplayList'] + return dict(zip(fields, values))["HudDisplayList"] + def instruction(path, address): data = pathlib.Path(path).read_bytes() - assert data[:5] == b'\x7fELF\x01' - endian = '>' if data[5] == 2 else '<' - offset = struct.unpack_from(endian+'I', data, 32)[0] - size, count = struct.unpack_from(endian+'HH', data, 46) + assert data[:5] == b"\x7fELF\x01" + endian = ">" if data[5] == 2 else "<" + offset = struct.unpack_from(endian + "I", data, 32)[0] + size, count = struct.unpack_from(endian + "HH", data, 46) for i in range(count): - section = struct.unpack_from(endian+'10I', data, offset+i*size) + section = struct.unpack_from(endian + "10I", data, offset + i * size) _, kind, _, start, pos, length, *_ = section - if kind == 1 and start <= address < start+length: - return struct.unpack_from(endian+'I', data, pos+address-start)[0] + if kind == 1 and start <= address < start + length: + return struct.unpack_from(endian + "I", data, pos + address - start)[0] raise AssertionError(hex(address)) -assert len(sys.argv) == 6, 'generated v77, generated v80, ELF v77, ELF v80, baseline source ZIP' -runtime = (ROOT/'runtime-recomp/src/game/runtime_hud_layout.cpp').read_text() -minimap_begin = runtime.split('void dkr::runtime::hud::begin_minimap(',1)[1].split('\n}',1)[0] -minimap_end = runtime.split('void dkr::runtime::hud::end_minimap(',1)[1].split('\n}',1)[0] -assert 'rdram && context && g_group_frame' in minimap_begin -assert 'g_map_holder = revision_addresses::HudDisplayList;' in minimap_begin -assert 'BeginWidget(rdram,g_map_holder,Widget::Minimap,0)' in minimap_begin -assert 'EndWidget(rdram,g_map_holder,g_map_widget_active)' in minimap_end + +assert len(sys.argv) == 6, "generated v77, generated v80, ELF v77, ELF v80, baseline source ZIP" +runtime = (ROOT / "runtime-recomp/src/game/runtime_hud_layout.cpp").read_text() +minimap_begin = runtime.split("void dkr::runtime::hud::begin_minimap(", 1)[1].split("\n}", 1)[0] +minimap_end = runtime.split("void dkr::runtime::hud::end_minimap(", 1)[1].split("\n}", 1)[0] +assert "rdram && context && g_group_frame" in minimap_begin +assert "g_map_holder = revision_addresses::HudDisplayList;" in minimap_begin +assert "BeginWidget(rdram,g_map_holder,Widget::Minimap,0)" in minimap_begin +assert "EndWidget(rdram,g_map_holder,g_map_widget_active)" in minimap_end with zipfile.ZipFile(sys.argv[5]) as baseline: + def previous(path): matches = [n for n in baseline.namelist() if n.endswith(path)] assert len(matches) == 1, (path, matches) - return baseline.read(matches[0]).decode('utf-8-sig').replace('\r\n','\n') - for rev, generated, elf in zip(('v77','v80'),sys.argv[1:3],sys.argv[3:5]): - header = 'runtime-recomp/src/game/revision_addresses.hpp' + return baseline.read(matches[0]).decode("utf-8-sig").replace("\r\n", "\n") + + for rev, generated, elf in zip(("v77", "v80"), sys.argv[1:3], sys.argv[3:5]): + header = "runtime-recomp/src/game/revision_addresses.hpp" retail = symbols(elf) - current_holder = hud_display_list((ROOT/header).read_text(), rev) - assert current_holder == retail['gHudDL'], (rev, hex(current_holder), hex(retail['gHudDL'])) - assert current_holder != retail['gHudMtx'] - print(rev, 'minimap command holder matches retail gHudDL:', hex(current_holder)) - path = f'runtime-recomp/dkr.us.{rev}.recomp-policy.json' - policy = json.loads((ROOT/path).read_text()) + current_holder = hud_display_list((ROOT / header).read_text(), rev) + assert current_holder == retail["gHudDL"], (rev, hex(current_holder), hex(retail["gHudDL"])) + assert current_holder != retail["gHudMtx"] + print(rev, "minimap command holder matches retail gHudDL:", hex(current_holder)) + path = f"runtime-recomp/dkr.us.{rev}.recomp-policy.json" + policy = json.loads((ROOT / path).read_text()) old = json.loads(previous(path)) # No existing hook, replacement or revision mapping was changed. for key, value in old.items(): - if key != 'functionHooks': assert policy[key] == value, key - for hook in old['functionHooks']: assert hook in policy['functionHooks'], hook - added = [h for h in policy['functionHooks'] if h not in old['functionHooks']] - assert len(added) == 13, (rev,len(added)) - code = '\n'.join(p.read_text() for p in pathlib.Path(generated).glob('*.c')) + if key != "functionHooks": + assert policy[key] == value, key + for hook in old["functionHooks"]: + assert hook in policy["functionHooks"], hook + added = [h for h in policy["functionHooks"] if h not in old["functionHooks"]] + assert len(added) == 13, (rev, len(added)) + code = "\n".join(p.read_text() for p in pathlib.Path(generated).glob("*.c")) for h in added: - address = int(h['beforeVram'],16) - assert code.count(h['text']+'\n // '+h['beforeVram']+':') == 1, h - op = instruction(elf,address) - if 'dkr_hud_rect_begin' in h['text']: - assert op == (0x27BDFFD0 if h['function']=='texrect_draw' else 0x27BDFF50) - elif 'dkr_hud_rect_extent' in h['text']: assert op == 0x8FA900C8 - elif 'dkr_hud_text_begin' in h['text']: assert op == 0x27BDFFD8 - elif 'dkr_hud_rect_end' in h['text'] or 'dkr_hud_text_end' in h['text']: assert op == 0x03E00008 - print(rev, len(added), 'new hooks verified; every pre-existing patch retained') - protected = ['split_screen_rt64.hpp','split_screen_policy.hpp','postrace_viewport_rt64.hpp', - 'runtime_netplay.cpp','runtime_input.cpp','rt64_renderer.cpp','runtime_enhancements.cpp'] + address = int(h["beforeVram"], 16) + assert code.count(h["text"] + "\n // " + h["beforeVram"] + ":") == 1, h + op = instruction(elf, address) + if "dkr_hud_rect_begin" in h["text"]: + assert op == (0x27BDFFD0 if h["function"] == "texrect_draw" else 0x27BDFF50) + elif "dkr_hud_rect_extent" in h["text"]: + assert op == 0x8FA900C8 + elif "dkr_hud_text_begin" in h["text"]: + assert op == 0x27BDFFD8 + elif "dkr_hud_rect_end" in h["text"] or "dkr_hud_text_end" in h["text"]: + assert op == 0x03E00008 + print(rev, len(added), "new hooks verified; every pre-existing patch retained") + protected = [ + "split_screen_rt64.hpp", + "split_screen_policy.hpp", + "postrace_viewport_rt64.hpp", + "runtime_netplay.cpp", + "runtime_input.cpp", + "rt64_renderer.cpp", + "runtime_enhancements.cpp", + ] for name in protected: - path='runtime-recomp/src/game/'+name - if not (ROOT/path).exists(): continue - assert (ROOT/path).read_text() == previous(path), 'Unrelated change: '+path - path='runtime-recomp/src/game/hud_layout_policy.hpp' - current=(ROOT/path).read_text().replace(', FitToViewport, Custom',', FitToViewport') - current=current.replace('// Legacy numeric values are persistent: SafeArea must never become Custom.\n','') - assert current.strip() == previous(path).strip(), 'Protected quadrant HUD policy changed' -print('Protected world/online/3-4-player policies remain byte-for-byte unchanged') + path = "runtime-recomp/src/game/" + name + if not (ROOT / path).exists(): + continue + assert (ROOT / path).read_text() == previous(path), "Unrelated change: " + path + path = "runtime-recomp/src/game/hud_layout_policy.hpp" + current = (ROOT / path).read_text().replace(", FitToViewport, Custom", ", FitToViewport") + current = current.replace("// Legacy numeric values are persistent: SafeArea must never become Custom.\n", "") + assert current.strip() == previous(path).strip(), "Protected quadrant HUD policy changed" +print("Protected world/online/3-4-player policies remain byte-for-byte unchanged") diff --git a/runtime-recomp/tests/hud_reference_source_tests.py b/runtime-recomp/tests/hud_reference_source_tests.py index 2b919bb4..c4652173 100644 --- a/runtime-recomp/tests/hud_reference_source_tests.py +++ b/runtime-recomp/tests/hud_reference_source_tests.py @@ -3,6 +3,7 @@ Usage: python hud_reference_source_tests.py The reference is a test input, never a build/runtime dependency. """ + import pathlib import re import subprocess @@ -10,47 +11,51 @@ root = pathlib.Path(__file__).resolve().parents[2] reference = pathlib.Path(sys.argv[1]) -commit = subprocess.check_output(['git', '-C', str(reference), 'rev-parse', 'HEAD'], text=True).strip() -assert commit == '83a847ccbd3e6334c9cc9c697122d5b3cddb91c1', commit +commit = subprocess.check_output(["git", "-C", str(reference), "rev-parse", "HEAD"], text=True).strip() +assert commit == "83a847ccbd3e6334c9cc9c697122d5b3cddb91c1", commit -header = (root/'extern/dkr-decomp/src/game_ui.h').read_text() -enum = re.search(r'enum HudTypes\s*\{(.*?)\}', header, re.S).group(1) -names = re.findall(r'\b(HUD_[A-Z0-9_]+)\b', re.sub(r'//[^\n]*', '', enum)) -names = [name for name in names if name != 'HUD_ELEMENT_COUNT'] +header = (root / "extern/dkr-decomp/src/game_ui.h").read_text() +enum = re.search(r"enum HudTypes\s*\{(.*?)\}", header, re.S).group(1) +names = re.findall(r"\b(HUD_[A-Z0-9_]+)\b", re.sub(r"//[^\n]*", "", enum)) +names = [name for name in names if name != "HUD_ELEMENT_COUNT"] assert len(names) == 59, names -gb = (reference/'game/src/game_ui.c').read_text(encoding='utf-8') -table = gb.split('sHudWidescreenAnchor[HUD_ELEMENT_COUNT][HUD_WIDESCREEN_MODE_COUNT] = {', 1)[1].split('\n};', 1)[0] +gb = (reference / "game/src/game_ui.c").read_text(encoding="utf-8") +table = gb.split("sHudWidescreenAnchor[HUD_ELEMENT_COUNT][HUD_WIDESCREEN_MODE_COUNT] = {", 1)[1].split("\n};", 1)[0] upstream = {} -for name, kind, value in re.findall(r'\[(HUD_[A-Z0-9_]+)\]\s*=\s*HUD_ANCHOR_(ALL_MODES|MODES)\((.*?)\)', table, re.S): - row = [{'LEFT': -1, 'CENTER': 0, 'RIGHT': 1}[anchor] - for anchor in re.findall(r'MDKR_HUD_ANCHOR_(LEFT|CENTER|RIGHT)', value)] - upstream[name] = row*5 if kind == 'ALL_MODES' else row +for name, kind, value in re.findall(r"\[(HUD_[A-Z0-9_]+)\]\s*=\s*HUD_ANCHOR_(ALL_MODES|MODES)\((.*?)\)", table, re.S): + row = [ + {"LEFT": -1, "CENTER": 0, "RIGHT": 1}[anchor] + for anchor in re.findall(r"MDKR_HUD_ANCHOR_(LEFT|CENTER|RIGHT)", value) + ] + upstream[name] = row * 5 if kind == "ALL_MODES" else row assert set(names) == set(upstream) -code = (root/'runtime-recomp/src/game/hud_reference_layout.hpp').read_text() -aliases = {name: [int(v) for v in value.split(',')] - for name, value in re.findall(r'\b(left|centre|right|lap|banana|challenge)\{([^{}]+)\}', code)} -local = code.split('anchors{{', 1)[1].split('}};', 1)[0] -tokens = re.findall(r'Row\{[^{}]+\}|\b[a-z]+\b', local) +code = (root / "runtime-recomp/src/game/hud_reference_layout.hpp").read_text() +aliases = { + name: [int(v) for v in value.split(",")] + for name, value in re.findall(r"\b(left|centre|right|lap|banana|challenge)\{([^{}]+)\}", code) +} +local = code.split("anchors{{", 1)[1].split("}};", 1)[0] +tokens = re.findall(r"Row\{[^{}]+\}|\b[a-z]+\b", local) assert len(tokens) == 59 for name, token in zip(names, tokens): - row = [int(v) for v in token[4:-1].split(',')] if token.startswith('Row{') else aliases[token] + row = [int(v) for v in token[4:-1].split(",")] if token.startswith("Row{") else aliases[token] assert row == upstream[name], (name, row, upstream[name]) -runtime = (root/'runtime-recomp/src/game/runtime_hud_layout.cpp').read_text() -begin = runtime.split('bool BeginWidget(', 1)[1].split('void EndWidget', 1)[0] -assert 'hg::bounds' not in begin and 'hg::transform' not in begin and 'hg::editable' not in begin -assert 'reference::anchor' in begin and 'reference::transform' in begin -assert 'MEM_H(6,address) != 40' in runtime -assert names[33] == 'HUD_COURSE_ARROWS' -assert 'g_frame_layout == 1, g_general_pass_active, authored_x)' in begin -element_begin = runtime.split('void dkr::runtime::hud::begin_element(',1)[1].split('// Accepted 3/4-player path',1)[0] -assert 'if (g_group_frame)' in element_begin -assert '*widget == Widget::CourseArrows ? ReadFloat(rdram,address,0x0C) : 0.0F' in element_begin -bridge = (root/'runtime-recomp/src/game/f3ddkr_rt64.cpp').read_text() -assert 'presentation_variant == hud::reference::kPassVariant' in bridge -assert 'hud::push_hud_projection' in bridge and bridge.count('hud::pop_hud_projection') == 2 -assert 'hud_color_image_count' in bridge -assert 'hud_layout_editor.cpp' not in (root/'runtime-recomp/CMakeLists.txt').read_text() -print('Golden Balloon reference parity: all 59 slots x 5 modes (295 anchors) match; production routing guards passed') +runtime = (root / "runtime-recomp/src/game/runtime_hud_layout.cpp").read_text() +begin = runtime.split("bool BeginWidget(", 1)[1].split("void EndWidget", 1)[0] +assert "hg::bounds" not in begin and "hg::transform" not in begin and "hg::editable" not in begin +assert "reference::anchor" in begin and "reference::transform" in begin +assert "MEM_H(6,address) != 40" in runtime +assert names[33] == "HUD_COURSE_ARROWS" +assert "g_frame_layout == 1, g_general_pass_active, authored_x)" in begin +element_begin = runtime.split("void dkr::runtime::hud::begin_element(", 1)[1].split("// Accepted 3/4-player path", 1)[0] +assert "if (g_group_frame)" in element_begin +assert "*widget == Widget::CourseArrows ? ReadFloat(rdram,address,0x0C) : 0.0F" in element_begin +bridge = (root / "runtime-recomp/src/game/f3ddkr_rt64.cpp").read_text() +assert "presentation_variant == hud::reference::kPassVariant" in bridge +assert "hud::push_hud_projection" in bridge and bridge.count("hud::pop_hud_projection") == 2 +assert "hud_color_image_count" in bridge +assert "hud_layout_editor.cpp" not in (root / "runtime-recomp/CMakeLists.txt").read_text() +print("Golden Balloon reference parity: all 59 slots x 5 modes (295 anchors) match; production routing guards passed") diff --git a/runtime-recomp/tests/launcher_copy_source_tests.py b/runtime-recomp/tests/launcher_copy_source_tests.py index 18769aad..5292527c 100644 --- a/runtime-recomp/tests/launcher_copy_source_tests.py +++ b/runtime-recomp/tests/launcher_copy_source_tests.py @@ -1,9 +1,10 @@ """Read-only regression contracts for player-facing launcher copy.""" + import ast import json +from pathlib import Path import re import unittest -from pathlib import Path ROOT = Path(__file__).resolve().parents[2] GAME = ROOT / "runtime-recomp/src/game" @@ -18,8 +19,7 @@ def literals(text): result = [] for chunk in chunks: try: - result.append("".join(ast.literal_eval(token) for token in - re.findall(r'"(?:\\.|[^"\\])*"', chunk))) + result.append("".join(ast.literal_eval(token) for token in re.findall(r'"(?:\\.|[^"\\])*"', chunk))) except (SyntaxError, ValueError): pass return result @@ -65,8 +65,14 @@ def test_background_policy_is_preserved(self): def test_release_language(self): files = [p for p in GAME.glob("runtime_*ui.inl")] + [GAME / "runtime_ui.cpp"] displayed = "\n".join(value for p in files for value in literals(p.read_text(encoding="utf-8"))) - for forbidden in ("playtest qualification", "Online is in beta", "in development", - "not finished cooking", "WHATâ\x80\x99S NEW SINCE 1.0.0", "developer build"): + for forbidden in ( + "playtest qualification", + "Online is in beta", + "in development", + "not finished cooking", + "WHATâ\x80\x99S NEW SINCE 1.0.0", + "developer build", + ): self.assertNotIn(forbidden, displayed) def test_raw_network_classification_stays_before_display(self): @@ -118,7 +124,7 @@ def test_streamer_privacy_is_shared_and_persistent(self): self.assertIn('g_online_hide_lobby_code ? "Show" : "Hide"', online) self.assertIn('OlNotify("Lobby code copied.")', online) self.assertIn('g_online_hide_lobby_code ? "*****"', online) - self.assertIn('OnlineTextForDisplay(text)', online) + self.assertIn("OnlineTextForDisplay(text)", online) def test_save_and_mod_safety_stays_visible(self): saves = source("runtime_saves_ui.inl") diff --git a/runtime-recomp/tests/legacy_behaviour_pipeline_tests.py b/runtime-recomp/tests/legacy_behaviour_pipeline_tests.py index 2486e91d..e861d4cd 100644 --- a/runtime-recomp/tests/legacy_behaviour_pipeline_tests.py +++ b/runtime-recomp/tests/legacy_behaviour_pipeline_tests.py @@ -1,40 +1,44 @@ """Read-only qualification of the closed behaviour hook ABI.""" + import copy +from pathlib import Path import sys import unittest -from pathlib import Path + sys.dont_write_bytecode = True -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'scripts')) +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "scripts")) from legacy_behaviour_policy import compose_behaviour INPUTS = [] + class PolicyTests(unittest.TestCase): def test_unreviewed_source(self): with self.assertRaises(ValueError): - compose_behaviour({}, Path(__file__), 'us.v77') + compose_behaviour({}, Path(__file__), "us.v77") with self.assertRaises(ValueError): - compose_behaviour({}, Path(__file__), 'unknown') + compose_behaviour({}, Path(__file__), "unknown") def test_revisions_conflicts_and_idempotence(self): if not INPUTS: - self.skipTest('Pass both retail ELFs for private revision qualification') + self.skipTest("Pass both retail ELFs for private revision qualification") for revision, path in INPUTS: - baseline = {'functionHooks': [], 'instructionPatches': []} + baseline = {"functionHooks": [], "instructionPatches": []} output = compose_behaviour(baseline, path, revision) - self.assertEqual(baseline, {'functionHooks': [], 'instructionPatches': []}) - self.assertEqual(len(output['functionHooks']), 10) + self.assertEqual(baseline, {"functionHooks": [], "instructionPatches": []}) + self.assertEqual(len(output["functionHooks"]), 10) self.assertEqual(compose_behaviour(output, path, revision), output) - for hook in output['functionHooks']: + for hook in output["functionHooks"]: damaged = copy.deepcopy(baseline) - damaged['functionHooks'] = [dict(hook, text='return;')] + damaged["functionHooks"] = [dict(hook, text="return;")] with self.assertRaises(ValueError): compose_behaviour(damaged, path, revision) damaged = copy.deepcopy(baseline) - damaged['instructionPatches'] = [{'vram': hook['beforeVram']}] + damaged["instructionPatches"] = [{"vram": hook["beforeVram"]}] with self.assertRaises(ValueError): compose_behaviour(damaged, path, revision) -if __name__ == '__main__': - INPUTS = [('us.v77', Path(sys.argv[1])), ('us.v80', Path(sys.argv[2]))] if len(sys.argv) == 3 else [] + +if __name__ == "__main__": + INPUTS = [("us.v77", Path(sys.argv[1])), ("us.v80", Path(sys.argv[2]))] if len(sys.argv) == 3 else [] unittest.main(argv=[sys.argv[0]]) diff --git a/runtime-recomp/tests/legacy_memory_pipeline_tests.py b/runtime-recomp/tests/legacy_memory_pipeline_tests.py index 545c18f0..500ca707 100644 --- a/runtime-recomp/tests/legacy_memory_pipeline_tests.py +++ b/runtime-recomp/tests/legacy_memory_pipeline_tests.py @@ -1,34 +1,41 @@ """Headless, fail-closed policy review. Inputs/ELFs are strictly read-only.""" + import copy +from pathlib import Path import sys import unittest -from pathlib import Path -sys.path.insert(0,str(Path(__file__).resolve().parents[2]/'scripts')) + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "scripts")) from legacy_memory_safety_policy import compose_memory_safety ELF_INPUTS = [] + class PolicyTests(unittest.TestCase): def test_requires_reviewed_elf(self): with self.assertRaises(ValueError): - compose_memory_safety({},Path(__file__),'us.v77') + compose_memory_safety({}, Path(__file__), "us.v77") def test_revisions_and_conflicts(self): # Explicit inputs let CI qualify hash-pinned ELFs without shipping ROMs. if not ELF_INPUTS: - self.skipTest('Pass both reviewed retail ELFs to qualify revision hooks') - for rev,elf in ELF_INPUTS: - baseline={'schemaVersion':1,'functionHooks':[],'instructionPatches':[]} - result=compose_memory_safety(baseline,elf,rev) - self.assertEqual(baseline,{'schemaVersion':1,'functionHooks':[],'instructionPatches':[]}) - self.assertEqual(result,compose_memory_safety(result,elf,rev)) - self.assertEqual(len(result['functionHooks']),20) - self.assertEqual(len(result['instructionPatches']),8) - for hook in result['functionHooks']: - damaged=copy.deepcopy(baseline);bad=copy.deepcopy(hook);bad['text']='return;' - damaged['functionHooks'].append(bad) - with self.assertRaises(ValueError):compose_memory_safety(damaged,elf,rev) - -if __name__=='__main__': - ELF_INPUTS=[('us.v77',Path(sys.argv[1])),('us.v80',Path(sys.argv[2]))] if len(sys.argv)==3 else [] + self.skipTest("Pass both reviewed retail ELFs to qualify revision hooks") + for rev, elf in ELF_INPUTS: + baseline = {"schemaVersion": 1, "functionHooks": [], "instructionPatches": []} + result = compose_memory_safety(baseline, elf, rev) + self.assertEqual(baseline, {"schemaVersion": 1, "functionHooks": [], "instructionPatches": []}) + self.assertEqual(result, compose_memory_safety(result, elf, rev)) + self.assertEqual(len(result["functionHooks"]), 20) + self.assertEqual(len(result["instructionPatches"]), 8) + for hook in result["functionHooks"]: + damaged = copy.deepcopy(baseline) + bad = copy.deepcopy(hook) + bad["text"] = "return;" + damaged["functionHooks"].append(bad) + with self.assertRaises(ValueError): + compose_memory_safety(damaged, elf, rev) + + +if __name__ == "__main__": + ELF_INPUTS = [("us.v77", Path(sys.argv[1])), ("us.v80", Path(sys.argv[2]))] if len(sys.argv) == 3 else [] unittest.main(argv=[sys.argv[0]]) diff --git a/runtime-recomp/tests/legacy_mod_pipeline_tests.py b/runtime-recomp/tests/legacy_mod_pipeline_tests.py index 9524465b..f0cd4f09 100644 --- a/runtime-recomp/tests/legacy_mod_pipeline_tests.py +++ b/runtime-recomp/tests/legacy_mod_pipeline_tests.py @@ -1,4 +1,5 @@ """Synthetic Patch Pipeline guards; no private ROMs, generated C or submodules edited.""" + import copy import hashlib import importlib.util @@ -13,7 +14,9 @@ spec = importlib.util.spec_from_file_location("legacy_compose", ROOT / "scripts/compose_legacy_mod_policy.py") compose = importlib.util.module_from_spec(spec) spec.loader.exec_module(compose) -spec = importlib.util.spec_from_file_location("legacy_presentation", ROOT / "scripts/legacy_character_presentation_policy.py") +spec = importlib.util.spec_from_file_location( + "legacy_presentation", ROOT / "scripts/legacy_character_presentation_policy.py" +) presentation = importlib.util.module_from_spec(spec) spec.loader.exec_module(presentation) @@ -21,332 +24,458 @@ class PipelineTests(unittest.TestCase): def test_spawn_asset_type_is_hash_pinned_and_branch_scoped(self): import sys - with patch.object(sys, 'path', [str(ROOT/'scripts'), *sys.path]): + + with patch.object(sys, "path", [str(ROOT / "scripts"), *sys.path]): import legacy_spawn_asset_type_policy as spawn + class Elf: - def read_bytes(self): return b'isolated trophy signpost fixture' - elf=Elf() - pc=0x8000ed8c - for rev in ('us.v77', 'us.v80'): - base={'functionHooks': [], 'instructionPatches': []} - symbols={'spawn_object': {(0x8000ea54, 0xbdc)}} - sections=[(pc, struct.pack('>I', 0xa246003a))] - with self.assertRaises(ValueError): spawn.compose_spawn_asset_type(base, elf, rev) - with patch.object(spawn, 'reviewed_revision', return_value=rev), \ - patch.object(spawn, 'elf_sections', return_value=sections), \ - patch.object(spawn, 'elf_functions', return_value=symbols): - result=spawn.compose_spawn_asset_type(base, elf, rev) - self.assertEqual(base, {'functionHooks': [], 'instructionPatches': []}) - self.assertEqual(len(result['functionHooks']), 1) - self.assertEqual(result['instructionPatches'], []) - hook=result['functionHooks'][0] - self.assertEqual(int(hook['beforeVram'], 0), pc) - self.assertIn('MEM_W(0x64, ctx->r29)', hook['text']) - self.assertIn('MEM_B(0x53, MEM_W(0x40, ctx->r18))', hook['text']) + def read_bytes(self): + return b"isolated trophy signpost fixture" + + elf = Elf() + pc = 0x8000ED8C + for rev in ("us.v77", "us.v80"): + base = {"functionHooks": [], "instructionPatches": []} + symbols = {"spawn_object": {(0x8000EA54, 0xBDC)}} + sections = [(pc, struct.pack(">I", 0xA246003A))] + with self.assertRaises(ValueError): + spawn.compose_spawn_asset_type(base, elf, rev) + with ( + patch.object(spawn, "reviewed_revision", return_value=rev), + patch.object(spawn, "elf_sections", return_value=sections), + patch.object(spawn, "elf_functions", return_value=symbols), + ): + result = spawn.compose_spawn_asset_type(base, elf, rev) + self.assertEqual(base, {"functionHooks": [], "instructionPatches": []}) + self.assertEqual(len(result["functionHooks"]), 1) + self.assertEqual(result["instructionPatches"], []) + hook = result["functionHooks"][0] + self.assertEqual(int(hook["beforeVram"], 0), pc) + self.assertIn("MEM_W(0x64, ctx->r29)", hook["text"]) + self.assertIn("MEM_B(0x53, MEM_W(0x40, ctx->r18))", hook["text"]) self.assertEqual(result, spawn.compose_spawn_asset_type(result, elf, rev)) - conflicting_result=copy.deepcopy(result) - conflicting_result['instructionPatches'].append({'vram': hex(pc), 'value': '0x0'}) - with self.assertRaises(ValueError): spawn.compose_spawn_asset_type(conflicting_result, elf, rev) - with patch.object(spawn, 'elf_sections', return_value=[(pc, struct.pack('>I', 0xa246003b))]): - with self.assertRaises(ValueError): spawn.compose_spawn_asset_type(base, elf, rev) - for bad_symbols in ({'spawn_object': set()}, {'spawn_object': {(pc,4),(pc+4,4)}}, - {'spawn_object': {(pc+4,4)}}): - with patch.object(spawn, 'elf_functions', return_value=bad_symbols): - with self.assertRaises(ValueError): spawn.compose_spawn_asset_type(base, elf, rev) - for kind, value in [('functionHooks', {**hook, 'text': 'unknown owner'}), - ('instructionPatches', {'vram': hex(pc), 'value': '0x0'})]: - conflict=copy.deepcopy(base); conflict[kind].append(value) - with self.assertRaises(ValueError): spawn.compose_spawn_asset_type(conflict, elf, rev) + conflicting_result = copy.deepcopy(result) + conflicting_result["instructionPatches"].append({"vram": hex(pc), "value": "0x0"}) + with self.assertRaises(ValueError): + spawn.compose_spawn_asset_type(conflicting_result, elf, rev) + with patch.object(spawn, "elf_sections", return_value=[(pc, struct.pack(">I", 0xA246003B))]): + with self.assertRaises(ValueError): + spawn.compose_spawn_asset_type(base, elf, rev) + for bad_symbols in ( + {"spawn_object": set()}, + {"spawn_object": {(pc, 4), (pc + 4, 4)}}, + {"spawn_object": {(pc + 4, 4)}}, + ): + with patch.object(spawn, "elf_functions", return_value=bad_symbols): + with self.assertRaises(ValueError): + spawn.compose_spawn_asset_type(base, elf, rev) + for kind, value in [ + ("functionHooks", {**hook, "text": "unknown owner"}), + ("instructionPatches", {"vram": hex(pc), "value": "0x0"}), + ]: + conflict = copy.deepcopy(base) + conflict[kind].append(value) + with self.assertRaises(ValueError): + spawn.compose_spawn_asset_type(conflict, elf, rev) def test_asset_capacity_pairs_allocation_limit_and_prewrite_guards(self): import sys - with patch.object(sys,'path',[str(ROOT/'scripts'),*sys.path]): + + with patch.object(sys, "path", [str(ROOT / "scripts"), *sys.path]): import legacy_asset_capacity_policy as capacity + class Elf: - def read_bytes(self):return b'isolated capacity fixture' - elf=Elf() - for rev in ('us.v77','us.v80'): - words={};bounds={} - def put(name,pc,word): - words[pc]=word;lo,hi=bounds.get(name,(pc,pc));bounds[name]=(min(lo,pc),max(hi,pc)) - for name,v77,v80,word,kind,stride in capacity.ALLOCATIONS:put(name,v77 if rev=='us.v77' else v80,word) - for name,pc,word,branch,*_ in capacity.COMPARISONS[rev]:put(name,pc,word);put(name,pc+4,branch) - for name,pc,word,*_ in capacity.STORES[rev]:put(name,pc,word) - put('track_setup_racers',0x8000d7a4,0xa44f0000) - symbols={n:{(lo,hi-lo+4)} for n,(lo,hi) in bounds.items()} - sections=[(pc,struct.pack('>I',word)) for pc,word in words.items()] - base={'functionHooks':[],'instructionPatches':[]} - with self.assertRaises(ValueError):capacity.compose_asset_capacity(base,elf,rev,sections,symbols) - with patch.object(capacity,'reviewed_revision',return_value=rev): - result=capacity.compose_asset_capacity(base,elf,rev,sections,symbols) - self.assertEqual(base,{'functionHooks':[],'instructionPatches':[]}) - self.assertEqual(len(result['functionHooks']),11) - self.assertEqual(len(result['instructionPatches']),4) - self.assertEqual(result,capacity.compose_asset_capacity(result,elf,rev,sections,symbols)) - for i,(pc,data) in enumerate(sections): - bad=sections.copy();bad[i]=(pc,struct.pack('>I',words[pc]^1)) - with self.assertRaises(ValueError):capacity.compose_asset_capacity(base,elf,rev,bad,symbols) - for h in result['functionHooks']: - conflict=copy.deepcopy(base);conflict['functionHooks'].append({**h,'text':'unknown owner'}) - with self.assertRaises(ValueError):capacity.compose_asset_capacity(conflict,elf,rev,sections,symbols) - for p in result['instructionPatches']: - conflict=copy.deepcopy(base);conflict['instructionPatches'].append({**p,'value':'0x0'}) - with self.assertRaises(ValueError):capacity.compose_asset_capacity(conflict,elf,rev,sections,symbols) + def read_bytes(self): + return b"isolated capacity fixture" + + elf = Elf() + for rev in ("us.v77", "us.v80"): + words = {} + bounds = {} + + def put(name, pc, word): + words[pc] = word + lo, hi = bounds.get(name, (pc, pc)) + bounds[name] = (min(lo, pc), max(hi, pc)) + + for name, v77, v80, word, kind, stride in capacity.ALLOCATIONS: + put(name, v77 if rev == "us.v77" else v80, word) + for name, pc, word, branch, *_ in capacity.COMPARISONS[rev]: + put(name, pc, word) + put(name, pc + 4, branch) + for name, pc, word, *_ in capacity.STORES[rev]: + put(name, pc, word) + put("track_setup_racers", 0x8000D7A4, 0xA44F0000) + symbols = {n: {(lo, hi - lo + 4)} for n, (lo, hi) in bounds.items()} + sections = [(pc, struct.pack(">I", word)) for pc, word in words.items()] + base = {"functionHooks": [], "instructionPatches": []} + with self.assertRaises(ValueError): + capacity.compose_asset_capacity(base, elf, rev, sections, symbols) + with patch.object(capacity, "reviewed_revision", return_value=rev): + result = capacity.compose_asset_capacity(base, elf, rev, sections, symbols) + self.assertEqual(base, {"functionHooks": [], "instructionPatches": []}) + self.assertEqual(len(result["functionHooks"]), 11) + self.assertEqual(len(result["instructionPatches"]), 4) + self.assertEqual(result, capacity.compose_asset_capacity(result, elf, rev, sections, symbols)) + for i, (pc, data) in enumerate(sections): + bad = sections.copy() + bad[i] = (pc, struct.pack(">I", words[pc] ^ 1)) + with self.assertRaises(ValueError): + capacity.compose_asset_capacity(base, elf, rev, bad, symbols) + for h in result["functionHooks"]: + conflict = copy.deepcopy(base) + conflict["functionHooks"].append({**h, "text": "unknown owner"}) + with self.assertRaises(ValueError): + capacity.compose_asset_capacity(conflict, elf, rev, sections, symbols) + for p in result["instructionPatches"]: + conflict = copy.deepcopy(base) + conflict["instructionPatches"].append({**p, "value": "0x0"}) + with self.assertRaises(ValueError): + capacity.compose_asset_capacity(conflict, elf, rev, sections, symbols) def test_cache_failure_policy_is_hash_pinned_and_conflict_checked(self): import sys - with patch.object(sys,'path',[str(ROOT/'scripts'),*sys.path]): + + with patch.object(sys, "path", [str(ROOT / "scripts"), *sys.path]): import legacy_model_cache_policy as cache from legacy_model_cache_policy import compose_model_cache + class Elf: - def read_bytes(self): return b'isolated cache failure fixture' - elf=Elf() - for rev in ('us.v77','us.v80'): - words={0x80001000:0x27bdffe0} - symbols={'model_instance_init':{(0x80001000,4)}} - heap=0x80070b50 if rev=='us.v77' else 0x80070d90 - hud=0x800aa7ac if rev=='us.v77' else 0x800aad08 - words.update({heap:0x01e42823,hud:0}) - symbols.update({'mempool_init_main':{(heap,4)},'hud_element_render':{(hud,4)}}) - if rev=='us.v77': - words.update({0x8005f99c:0x27bdffa8,0x8005fcb4:0x8fbf0024}) - symbols.update({'object_model_init':{(0x8005f99c,0x400)}, - 'gModelCacheCount':{(0x8011d62c,4)},'D_8011D634':{(0x8011d634,4)}}) - sections=[(pc,struct.pack('>I',word)) for pc,word in words.items()] - base={'functionHooks':[],'instructionPatches':[]} - with self.assertRaises(ValueError):compose_model_cache(base,elf,rev,sections,symbols) - with patch.object(cache,'reviewed_revision',return_value=rev): - result=compose_model_cache(base,elf,rev,sections,symbols) - self.assertEqual(base,{'functionHooks':[],'instructionPatches':[]}) - self.assertEqual(len(result['functionHooks']),5 if rev=='us.v77' else 3) - for site in result['functionHooks']: - for key,field in [('functionHooks','beforeVram'),('instructionPatches','vram')]: - conflict=copy.deepcopy(base);conflict[key].append({field:site['beforeVram']}) - with self.assertRaises(ValueError):compose_model_cache(conflict,elf,rev,sections,symbols) - for i,(pc,data) in enumerate(sections): - bad=sections.copy();bad[i]=(pc,bytes([data[0]^1])+data[1:]) - with self.assertRaises(ValueError):compose_model_cache(base,elf,rev,bad,symbols) + def read_bytes(self): + return b"isolated cache failure fixture" + + elf = Elf() + for rev in ("us.v77", "us.v80"): + words = {0x80001000: 0x27BDFFE0} + symbols = {"model_instance_init": {(0x80001000, 4)}} + heap = 0x80070B50 if rev == "us.v77" else 0x80070D90 + hud = 0x800AA7AC if rev == "us.v77" else 0x800AAD08 + words.update({heap: 0x01E42823, hud: 0}) + symbols.update({"mempool_init_main": {(heap, 4)}, "hud_element_render": {(hud, 4)}}) + if rev == "us.v77": + words.update({0x8005F99C: 0x27BDFFA8, 0x8005FCB4: 0x8FBF0024}) + symbols.update( + { + "object_model_init": {(0x8005F99C, 0x400)}, + "gModelCacheCount": {(0x8011D62C, 4)}, + "D_8011D634": {(0x8011D634, 4)}, + } + ) + sections = [(pc, struct.pack(">I", word)) for pc, word in words.items()] + base = {"functionHooks": [], "instructionPatches": []} + with self.assertRaises(ValueError): + compose_model_cache(base, elf, rev, sections, symbols) + with patch.object(cache, "reviewed_revision", return_value=rev): + result = compose_model_cache(base, elf, rev, sections, symbols) + self.assertEqual(base, {"functionHooks": [], "instructionPatches": []}) + self.assertEqual(len(result["functionHooks"]), 5 if rev == "us.v77" else 3) + for site in result["functionHooks"]: + for key, field in [("functionHooks", "beforeVram"), ("instructionPatches", "vram")]: + conflict = copy.deepcopy(base) + conflict[key].append({field: site["beforeVram"]}) + with self.assertRaises(ValueError): + compose_model_cache(conflict, elf, rev, sections, symbols) + for i, (pc, data) in enumerate(sections): + bad = sections.copy() + bad[i] = (pc, bytes([data[0] ^ 1]) + data[1:]) + with self.assertRaises(ValueError): + compose_model_cache(base, elf, rev, bad, symbols) def test_presentation_rejects_conflicts_and_changed_signatures(self): class FixtureElf: - def read_bytes(self): return b'isolated synthetic presentation fixture' + def read_bytes(self): + return b"isolated synthetic presentation fixture" + elf = FixtureElf() - for rev in ('us.v77', 'us.v80'): - words = {}; bounds = {} + for rev in ("us.v77", "us.v80"): + words = {} + bounds = {} + def put(name, pc, word): words[pc] = word lo, hi = bounds.get(name, (pc, pc)) bounds[name] = min(lo, pc), max(hi, pc) + for name, pc, subject, lookup, expected in presentation.PORTRAITS[rev]: put(name, pc, expected) - voice, horn = (0x800571f0, 0x8005708c) if rev == 'us.v77' else (0x80057230, 0x800570cc) - entries = ('audspat_play_sound_at_position', 'sound_play_direct', 'sound_play', 'sound_play_spatial') - for i, name in enumerate(entries): put(name, 0x80001000 + i * 0x100, 0x27bdffe0) - put('play_random_character_voice', voice, 0x0c000400) - put('play_random_character_voice', voice+4, 0x3104ffff) - put('racer_play_sound', horn, 0x0c0004c0) - put('racer_play_sound', horn+4, 0xafa00010) - bonus,banana=(0x8003b30c,0x8003db10) if rev=='us.v77' else (0x8003b34c,0x8003db50) - for name,pc,delay in [('obj_loop_bonus',bonus,0x01202025),('obj_loop_banana',banana,0xafa00010)]: - put(name,pc,0x0c0004c0);put(name,pc+4,delay) - renderer=0x800aa600 if rev=='us.v77' else 0x800aab5c - put('hud_element_render',renderer,0x27bdff48) - for name,pc,owner,offset,delay in presentation.HUD_CALLS[rev]: - put(name,pc,0x0c000000|((renderer>>2)&0x3ffffff));put(name,pc+4,delay);put(name,pc+8,0) - for pc,lookup,expected in presentation.HUD_LOADS[rev]:put('hud_element_render',pc,expected) - capture,draw=presentation.CINEMATIC[rev] - put('menu_trophy_race_rankings_loop',capture-4,0x80820059) - put('menu_trophy_race_rankings_loop',capture,0x14600005) - put('menu_cinematic_loop',draw,0x8f050000) - sections = [(pc, struct.pack('>I', word)) for pc, word in words.items()] - symbols = {name: {(lo, hi-lo+4)} for name, (lo, hi) in bounds.items()} - base = {'instructionPatches': [], 'functionHooks': []} - with self.assertRaises(ValueError): presentation.compose_presentation(base, elf, rev, sections, symbols) - with patch.object(presentation, 'reviewed_revision', return_value=rev): + voice, horn = (0x800571F0, 0x8005708C) if rev == "us.v77" else (0x80057230, 0x800570CC) + entries = ("audspat_play_sound_at_position", "sound_play_direct", "sound_play", "sound_play_spatial") + for i, name in enumerate(entries): + put(name, 0x80001000 + i * 0x100, 0x27BDFFE0) + put("play_random_character_voice", voice, 0x0C000400) + put("play_random_character_voice", voice + 4, 0x3104FFFF) + put("racer_play_sound", horn, 0x0C0004C0) + put("racer_play_sound", horn + 4, 0xAFA00010) + bonus, banana = (0x8003B30C, 0x8003DB10) if rev == "us.v77" else (0x8003B34C, 0x8003DB50) + for name, pc, delay in [("obj_loop_bonus", bonus, 0x01202025), ("obj_loop_banana", banana, 0xAFA00010)]: + put(name, pc, 0x0C0004C0) + put(name, pc + 4, delay) + renderer = 0x800AA600 if rev == "us.v77" else 0x800AAB5C + put("hud_element_render", renderer, 0x27BDFF48) + for name, pc, owner, offset, delay in presentation.HUD_CALLS[rev]: + put(name, pc, 0x0C000000 | ((renderer >> 2) & 0x3FFFFFF)) + put(name, pc + 4, delay) + put(name, pc + 8, 0) + for pc, lookup, expected in presentation.HUD_LOADS[rev]: + put("hud_element_render", pc, expected) + capture, draw = presentation.CINEMATIC[rev] + put("menu_trophy_race_rankings_loop", capture - 4, 0x80820059) + put("menu_trophy_race_rankings_loop", capture, 0x14600005) + put("menu_cinematic_loop", draw, 0x8F050000) + sections = [(pc, struct.pack(">I", word)) for pc, word in words.items()] + symbols = {name: {(lo, hi - lo + 4)} for name, (lo, hi) in bounds.items()} + base = {"instructionPatches": [], "functionHooks": []} + with self.assertRaises(ValueError): + presentation.compose_presentation(base, elf, rev, sections, symbols) + with patch.object(presentation, "reviewed_revision", return_value=rev): result = presentation.compose_presentation(base, elf, rev, sections, symbols) - self.assertEqual(base, {'instructionPatches': [], 'functionHooks': []}) - self.assertEqual(len(result['functionHooks']), 25) - self.assertEqual(result,presentation.refresh_presentation(result,elf,rev,sections,symbols)) - legacy=presentation.compose_presentation(base,elf,rev,sections,symbols,extended=False) - self.assertEqual(result,presentation.refresh_presentation(legacy,elf,rev,sections,symbols)) - for site in result['functionHooks']: - for key, address in (('functionHooks', 'beforeVram'), ('instructionPatches', 'vram')): - bad = copy.deepcopy(base); bad[key].append({address: site['beforeVram'], 'function': site['function'], 'text': 'unreviewed'}) - with self.assertRaises(ValueError): presentation.compose_presentation(bad, elf, rev, sections, symbols) - bad = copy.deepcopy(sections); pc, data = bad[0]; bad[0] = pc, bytes([data[0]^1])+data[1:] - with self.assertRaises(ValueError): presentation.compose_presentation(base, elf, rev, bad, symbols) + self.assertEqual(base, {"instructionPatches": [], "functionHooks": []}) + self.assertEqual(len(result["functionHooks"]), 25) + self.assertEqual(result, presentation.refresh_presentation(result, elf, rev, sections, symbols)) + legacy = presentation.compose_presentation(base, elf, rev, sections, symbols, extended=False) + self.assertEqual(result, presentation.refresh_presentation(legacy, elf, rev, sections, symbols)) + for site in result["functionHooks"]: + for key, address in (("functionHooks", "beforeVram"), ("instructionPatches", "vram")): + bad = copy.deepcopy(base) + bad[key].append( + {address: site["beforeVram"], "function": site["function"], "text": "unreviewed"} + ) + with self.assertRaises(ValueError): + presentation.compose_presentation(bad, elf, rev, sections, symbols) + bad = copy.deepcopy(sections) + pc, data = bad[0] + bad[0] = pc, bytes([data[0] ^ 1]) + data[1:] + with self.assertRaises(ValueError): + presentation.compose_presentation(base, elf, rev, bad, symbols) owned = copy.deepcopy(base) - text = '{ extern int dkr_legacy_character_menu(uint8_t*, recomp_context*, unsigned, const uint32_t*); if (dkr_legacy_character_menu(rdram, ctx, 8U, dkr_character_menu_fields)) return; }' - owned['functionHooks'].append({'function':'sound_play','beforeVram':'0x80001200','text':text}) + text = "{ extern int dkr_legacy_character_menu(uint8_t*, recomp_context*, unsigned, const uint32_t*); if (dkr_legacy_character_menu(rdram, ctx, 8U, dkr_character_menu_fields)) return; }" + owned["functionHooks"].append({"function": "sound_play", "beforeVram": "0x80001200", "text": text}) merged = presentation.compose_presentation(owned, elf, rev, sections, symbols) - self.assertTrue(merged['functionHooks'][0]['text'].startswith(text+' ')) + self.assertTrue(merged["functionHooks"][0]["text"].startswith(text + " ")) def test_sidebar_visual_and_navigation_order_match(self): - source = (ROOT / 'runtime-recomp/src/game/runtime_ui.cpp').read_text() - pages = {name: int(value) for name, value in re.findall(r'constexpr int (kPage\w+) = (\d+);', source)} - order = re.search(r'kSidebarOrder\{([\d,\s]+)\}', source).group(1) - order = [int(page) for page in order.split(',')][:len(pages)] + source = (ROOT / "runtime-recomp/src/game/runtime_ui.cpp").read_text() + pages = {name: int(value) for name, value in re.findall(r"constexpr int (kPage\w+) = (\d+);", source)} + order = re.search(r"kSidebarOrder\{([\d,\s]+)\}", source).group(1) + order = [int(page) for page in order.split(",")][: len(pages)] # Launcher and in-game overlay draw their buttons in the order the D-pad walks. - for button in ('LauncherSidebarButton', 'SidebarButton'): - drawn = [int(page) if page.isdigit() else pages[page] - for page in re.findall(r'\b' + button + r'\("[^"]+",\s*(\w+)', source)] + for button in ("LauncherSidebarButton", "SidebarButton"): + drawn = [ + int(page) if page.isdigit() else pages[page] + for page in re.findall(r"\b" + button + r'\("[^"]+",\s*(\w+)', source) + ] self.assertEqual(drawn, order, button) def test_track_lab_is_in_mods_and_keeps_its_texture_modal(self): - source=(ROOT/'runtime-recomp/src/game/runtime_ui.cpp').read_text() - details=(ROOT/'runtime-recomp/src/game/runtime_details_ui.inl').read_text() - mods=source.split('void DrawModsHacks(',1)[1].split('std::string FormatRecordTime',1)[0] - textures=details.split('void DrawTexturesPage(float width) {',1)[1].split('\n}\n',1)[0] + source = (ROOT / "runtime-recomp/src/game/runtime_ui.cpp").read_text() + details = (ROOT / "runtime-recomp/src/game/runtime_details_ui.inl").read_text() + mods = source.split("void DrawModsHacks(", 1)[1].split("std::string FormatRecordTime", 1)[0] + textures = details.split("void DrawTexturesPage(float width) {", 1)[1].split("\n}\n", 1)[0] # Track Lab is a section of the mods page and never of the textures page. - self.assertIn('kModsSectionTrackLab',mods) - self.assertIn('DrawTrackLabSection(',mods) - self.assertNotIn('DrawTrackLabSection',textures) + self.assertIn("kModsSectionTrackLab", mods) + self.assertIn("DrawTrackLabSection(", mods) + self.assertNotIn("DrawTrackLabSection", textures) # One shared single-pack modal, rendered once by each page that reaches it. - for page in (mods,textures):self.assertEqual(page.count('DrawSharedTexturePackModal();'),1) - section=source.split('void DrawTrackLabSection(float width, bool locked,',1)[1] - section=section.split('void DrawSharedTexturePackModal',1)[0] + for page in (mods, textures): + self.assertEqual(page.count("DrawSharedTexturePackModal();"), 1) + section = source.split("void DrawTrackLabSection(float width, bool locked,", 1)[1] + section = section.split("void DrawSharedTexturePackModal", 1)[0] # The section still owns arming and the auto-boot switch. - self.assertIn('armed_track_id()',section) - self.assertIn('set_auto_boot(',section) + self.assertIn("armed_track_id()", section) + self.assertIn("set_auto_boot(", section) def test_custom_stage_reuses_stock_clock_and_revision_camera(self): # The custom early-return hook must not bypass the stock beat override. - source = ROOT / 'runtime-recomp/src/game' - owner = (source / 'runtime_legacy_mods.cpp').read_text() - self.assertIn('dkr_character_select_animation_fraction,p->stage_camera', owner) - stage = (source / 'mods/legacy_character_stage.cpp').read_text() - self.assertLess(stage.index('calls.animation_fraction(memory.data(),&music)'), - stage.index('calls.animation_override(memory.data(),&music)')) - self.assertNotIn('advance_character_select_phase', stage) # never tick per actor + source = ROOT / "runtime-recomp/src/game" + owner = (source / "runtime_legacy_mods.cpp").read_text() + self.assertIn("dkr_character_select_animation_fraction,p->stage_camera", owner) + stage = (source / "mods/legacy_character_stage.cpp").read_text() + self.assertLess( + stage.index("calls.animation_fraction(memory.data(),&music)"), + stage.index("calls.animation_override(memory.data(),&music)"), + ) + self.assertNotIn("advance_character_select_phase", stage) # never tick per actor for rev in (77, 80): - payload = (source / f'game_payload_v{rev}.cpp').read_text() - self.assertIn('.stage_camera = cam_get_active_camera,', payload) - policy = json.loads((ROOT / f'runtime-recomp/dkr.us.v{rev}.recomp-policy.json').read_text()) - clocks = [h for h in policy['functionHooks'] - if h['function'] == 'obj_loop_char_select' - and 'dkr_character_select_animation_fraction' in h['text']] + payload = (source / f"game_payload_v{rev}.cpp").read_text() + self.assertIn(".stage_camera = cam_get_active_camera,", payload) + policy = json.loads((ROOT / f"runtime-recomp/dkr.us.v{rev}.recomp-policy.json").read_text()) + clocks = [ + h + for h in policy["functionHooks"] + if h["function"] == "obj_loop_char_select" and "dkr_character_select_animation_fraction" in h["text"] + ] self.assertEqual(len(clocks), 1) def test_character_menu_composes_without_replacing_native_owners(self): - for rev in (77,80): - fragment=json.loads((ROOT/f'runtime-recomp/legacy-character-menu.v{rev}.recomp-fragment.json').read_text()) - symbols={f['name']:{(int(f['address'],0),f['size'])} for f in fragment['fields']} - sections=[] - for name,body in fragment['functions'].items(): - start=int(body['vram'],0);data=bytearray(body['size']) - for site in fragment['hooks']: - if site['function']==name: - for i,word in enumerate(site['expected']):struct.pack_into('>I',data,int(site['expectedAt'],0)-start+4*i,int(word,0)) - body['sha256']=hashlib.sha256(data).hexdigest();sections.append((start,bytes(data)));symbols[name]={(start,len(data))} - policy=json.loads((ROOT/f'runtime-recomp/dkr.us.v{rev}.recomp-policy.json').read_text()) + for rev in (77, 80): + fragment = json.loads( + (ROOT / f"runtime-recomp/legacy-character-menu.v{rev}.recomp-fragment.json").read_text() + ) + symbols = {f["name"]: {(int(f["address"], 0), f["size"])} for f in fragment["fields"]} + sections = [] + for name, body in fragment["functions"].items(): + start = int(body["vram"], 0) + data = bytearray(body["size"]) + for site in fragment["hooks"]: + if site["function"] == name: + for i, word in enumerate(site["expected"]): + struct.pack_into(">I", data, int(site["expectedAt"], 0) - start + 4 * i, int(word, 0)) + body["sha256"] = hashlib.sha256(data).hexdigest() + sections.append((start, bytes(data))) + symbols[name] = {(start, len(data))} + policy = json.loads((ROOT / f"runtime-recomp/dkr.us.v{rev}.recomp-policy.json").read_text()) # Exact resource commit already authored/validated by its own tests. - owner=next(h for h in policy['functionHooks'] if h['function']=='charselect_assign_ai') - owner['text']+=' extern void dkr_legacy_character_event(uint8_t*, recomp_context*, unsigned, uint32_t); '+f'dkr_legacy_character_event(rdram, ctx, 1U, {int(fragment["fields"][3]["address"],0):#x}U);' - before=copy.deepcopy(policy) - result=compose.compose_character_menu(policy,fragment,sections,symbols) - self.assertEqual(policy,before);self.assertEqual(result['instructionPatches'],policy['instructionPatches']) - for old,new in zip(policy['functionHooks'],result['functionHooks']): - if old['function']=='charselect_assign_ai':self.assertTrue(new['text'].endswith(old['text'])) - elif old['function']=='menu_character_select_loop':self.assertTrue(new['text'].startswith(old['text'])) - else:self.assertEqual(old,new) - self.assertEqual(len(result['functionHooks']),len(policy['functionHooks'])+7) - with self.assertRaises(ValueError):compose.compose_character_menu(result,fragment,sections,symbols) + owner = next(h for h in policy["functionHooks"] if h["function"] == "charselect_assign_ai") + owner["text"] += ( + " extern void dkr_legacy_character_event(uint8_t*, recomp_context*, unsigned, uint32_t); " + + f"dkr_legacy_character_event(rdram, ctx, 1U, {int(fragment['fields'][3]['address'], 0):#x}U);" + ) + before = copy.deepcopy(policy) + result = compose.compose_character_menu(policy, fragment, sections, symbols) + self.assertEqual(policy, before) + self.assertEqual(result["instructionPatches"], policy["instructionPatches"]) + for old, new in zip(policy["functionHooks"], result["functionHooks"]): + if old["function"] == "charselect_assign_ai": + self.assertTrue(new["text"].endswith(old["text"])) + elif old["function"] == "menu_character_select_loop": + self.assertTrue(new["text"].startswith(old["text"])) + else: + self.assertEqual(old, new) + self.assertEqual(len(result["functionHooks"]), len(policy["functionHooks"]) + 7) + with self.assertRaises(ValueError): + compose.compose_character_menu(result, fragment, sections, symbols) for i in range(len(sections)): - bad=copy.deepcopy(sections);data=bytearray(bad[i][1]);data[-1]^=1;bad[i]=(bad[i][0],bytes(data)) - with self.assertRaises(ValueError):compose.compose_character_menu(policy,fragment,bad,symbols) - for site in fragment['hooks']: - bad=copy.deepcopy(policy);bad['instructionPatches'].append({'vram':site['vram']}) - with self.assertRaises(ValueError):compose.compose_character_menu(bad,fragment,sections,symbols) - for field in fragment['fields']: - bad=copy.deepcopy(symbols);bad[field['name']]={(int(field['address'],0)+4,field['size'])} - with self.assertRaises(ValueError):compose.compose_character_menu(policy,fragment,sections,bad) + bad = copy.deepcopy(sections) + data = bytearray(bad[i][1]) + data[-1] ^= 1 + bad[i] = (bad[i][0], bytes(data)) + with self.assertRaises(ValueError): + compose.compose_character_menu(policy, fragment, bad, symbols) + for site in fragment["hooks"]: + bad = copy.deepcopy(policy) + bad["instructionPatches"].append({"vram": site["vram"]}) + with self.assertRaises(ValueError): + compose.compose_character_menu(bad, fragment, sections, symbols) + for field in fragment["fields"]: + bad = copy.deepcopy(symbols) + bad[field["name"]] = {(int(field["address"], 0) + 4, field["size"])} + with self.assertRaises(ValueError): + compose.compose_character_menu(policy, fragment, sections, bad) def test_character_boundaries_and_original_owners(self): - for rev in (77,80): - fragment=json.loads((ROOT/f'runtime-recomp/legacy-characters.v{rev}.recomp-fragment.json').read_text()) - symbols={f['name']:{(int(f['address'],0),f['size'])} for f in fragment['fields']} - sections=[] - for name,body in fragment['functions'].items(): - start=int(body['vram'],0);data=bytearray(body['size']) - for site in fragment['hooks']: - if site['function']==name: - for i,word in enumerate(site['expected']):struct.pack_into('>I',data,int(site['vram'],0)-start+i*4,int(word,0)) - body['sha256']=hashlib.sha256(data).hexdigest() - sections.append((start,bytes(data)));symbols[name]={(start,len(data))} - policy=json.loads((ROOT/f'runtime-recomp/dkr.us.v{rev}.recomp-policy.json').read_text()) - before=copy.deepcopy(policy) - result=compose.compose_characters(policy,fragment,sections,symbols) - self.assertEqual(policy,before) - self.assertEqual(result['instructionPatches'],policy['instructionPatches']) - self.assertEqual(len(result['functionHooks']),len(policy['functionHooks'])+2) - for old,new in zip(policy['functionHooks'],result['functionHooks']): - if old['function']=='charselect_assign_ai':self.assertTrue(new['text'].startswith(old['text']+' ')) - else:self.assertEqual(old,new) - with self.assertRaises(ValueError):compose.compose_characters(result,fragment,sections,symbols) + for rev in (77, 80): + fragment = json.loads((ROOT / f"runtime-recomp/legacy-characters.v{rev}.recomp-fragment.json").read_text()) + symbols = {f["name"]: {(int(f["address"], 0), f["size"])} for f in fragment["fields"]} + sections = [] + for name, body in fragment["functions"].items(): + start = int(body["vram"], 0) + data = bytearray(body["size"]) + for site in fragment["hooks"]: + if site["function"] == name: + for i, word in enumerate(site["expected"]): + struct.pack_into(">I", data, int(site["vram"], 0) - start + i * 4, int(word, 0)) + body["sha256"] = hashlib.sha256(data).hexdigest() + sections.append((start, bytes(data))) + symbols[name] = {(start, len(data))} + policy = json.loads((ROOT / f"runtime-recomp/dkr.us.v{rev}.recomp-policy.json").read_text()) + before = copy.deepcopy(policy) + result = compose.compose_characters(policy, fragment, sections, symbols) + self.assertEqual(policy, before) + self.assertEqual(result["instructionPatches"], policy["instructionPatches"]) + self.assertEqual(len(result["functionHooks"]), len(policy["functionHooks"]) + 2) + for old, new in zip(policy["functionHooks"], result["functionHooks"]): + if old["function"] == "charselect_assign_ai": + self.assertTrue(new["text"].startswith(old["text"] + " ")) + else: + self.assertEqual(old, new) + with self.assertRaises(ValueError): + compose.compose_characters(result, fragment, sections, symbols) for index in range(len(sections)): - bad=copy.deepcopy(sections);data=bytearray(bad[index][1]);data[-1]^=1;bad[index]=(bad[index][0],bytes(data)) - with self.assertRaises(ValueError):compose.compose_characters(policy,fragment,bad,symbols) - for site in fragment['hooks']: - bad=copy.deepcopy(policy);bad['instructionPatches'].append({'vram':site['vram']}) - with self.assertRaises(ValueError):compose.compose_characters(bad,fragment,sections,symbols) - for field in fragment['fields']: - bad=copy.deepcopy(symbols);bad[field['name']]={(int(field['address'],0)+4,field['size'])} - with self.assertRaises(ValueError):compose.compose_characters(policy,fragment,sections,bad) + bad = copy.deepcopy(sections) + data = bytearray(bad[index][1]) + data[-1] ^= 1 + bad[index] = (bad[index][0], bytes(data)) + with self.assertRaises(ValueError): + compose.compose_characters(policy, fragment, bad, symbols) + for site in fragment["hooks"]: + bad = copy.deepcopy(policy) + bad["instructionPatches"].append({"vram": site["vram"]}) + with self.assertRaises(ValueError): + compose.compose_characters(bad, fragment, sections, symbols) + for field in fragment["fields"]: + bad = copy.deepcopy(symbols) + bad[field["name"]] = {(int(field["address"], 0) + 4, field["size"])} + with self.assertRaises(ValueError): + compose.compose_characters(policy, fragment, sections, bad) def menu_fixture(self, revision): fragment = json.loads((ROOT / f"runtime-recomp/legacy-track-menu.v{revision}.recomp-fragment.json").read_text()) # No ROM dependency in CI: exercise guards with bounded synthetic # bodies, and keep the committed real-input hashes unchanged. sections = [] - symbols = {f["name"]: {(int(f["address"],0), f["size"])} for f in fragment["fields"]} + symbols = {f["name"]: {(int(f["address"], 0), f["size"])} for f in fragment["fields"]} for name, body in fragment["functions"].items(): - start = int(body["vram"],0); data = bytearray(body["size"]) + start = int(body["vram"], 0) + data = bytearray(body["size"]) for site in fragment["hooks"]: if site["function"] == name: for i, word in enumerate(site["expected"]): - struct.pack_into(">I", data, int(site["expectedAt"],0)-start+i*4, int(word,0)) + struct.pack_into(">I", data, int(site["expectedAt"], 0) - start + i * 4, int(word, 0)) body["sha256"] = hashlib.sha256(data).hexdigest() - sections.append((start,bytes(data))); symbols[name] = {(start,len(data))} + sections.append((start, bytes(data))) + symbols[name] = {(start, len(data))} policy = json.loads((ROOT / f"runtime-recomp/dkr.us.v{revision}.recomp-policy.json").read_text()) return policy, fragment, sections, symbols def test_menu_preserves_native_owners_and_data_abi(self): - for rev in (77,80): + for rev in (77, 80): policy, fragment, sections, symbols = self.menu_fixture(rev) before = copy.deepcopy(policy) - result = compose.compose_track_menu(policy,fragment,sections,symbols) - self.assertEqual(policy,before) - self.assertEqual(policy["instructionPatches"],result["instructionPatches"]) - for old, new in zip(policy["functionHooks"],result["functionHooks"]): + result = compose.compose_track_menu(policy, fragment, sections, symbols) + self.assertEqual(policy, before) + self.assertEqual(policy["instructionPatches"], result["instructionPatches"]) + for old, new in zip(policy["functionHooks"], result["functionHooks"]): if old["function"] == "load_level_game" and "gameplay_level_begin" in old["text"]: self.assertTrue(new["text"].startswith(old["text"])) - self.assertIn("14U, dkr_legacy_fields",new["text"]) + self.assertIn("14U, dkr_legacy_fields", new["text"]) else: - self.assertEqual(old,new) + self.assertEqual(old, new) with self.assertRaises(ValueError): - compose.compose_track_menu(result,fragment,sections,symbols) - for old_abi in (1,2): - bad=copy.deepcopy(fragment);bad["abi"]=old_abi - with self.assertRaises(ValueError):compose.compose_track_menu(policy,bad,sections,symbols) - for index in range(len(fragment["fields"])-1): - bad=copy.deepcopy(fragment) - bad["fields"][index],bad["fields"][index+1]=bad["fields"][index+1],bad["fields"][index] - with self.assertRaises(ValueError):compose.compose_track_menu(policy,bad,sections,symbols) + compose.compose_track_menu(result, fragment, sections, symbols) + for old_abi in (1, 2): + bad = copy.deepcopy(fragment) + bad["abi"] = old_abi + with self.assertRaises(ValueError): + compose.compose_track_menu(policy, bad, sections, symbols) + for index in range(len(fragment["fields"]) - 1): + bad = copy.deepcopy(fragment) + bad["fields"][index], bad["fields"][index + 1] = bad["fields"][index + 1], bad["fields"][index] + with self.assertRaises(ValueError): + compose.compose_track_menu(policy, bad, sections, symbols) for index in range(len(sections)): - bad=copy.deepcopy(sections); data=bytearray(bad[index][1]);data[-1]^=1 - bad[index]=(bad[index][0],bytes(data)) - with self.assertRaises(ValueError):compose.compose_track_menu(policy,fragment,bad,symbols) + bad = copy.deepcopy(sections) + data = bytearray(bad[index][1]) + data[-1] ^= 1 + bad[index] = (bad[index][0], bytes(data)) + with self.assertRaises(ValueError): + compose.compose_track_menu(policy, fragment, bad, symbols) for site in fragment["hooks"]: - bad=copy.deepcopy(policy);bad["instructionPatches"].append({"vram":site["vram"]}) - with self.assertRaises(ValueError):compose.compose_track_menu(bad,fragment,sections,symbols) + bad = copy.deepcopy(policy) + bad["instructionPatches"].append({"vram": site["vram"]}) + with self.assertRaises(ValueError): + compose.compose_track_menu(bad, fragment, sections, symbols) for field in fragment["fields"]: - bad=copy.deepcopy(symbols);bad[field["name"]]={(int(field["address"],0),field["size"]+4)} - with self.assertRaises(ValueError):compose.compose_track_menu(policy,fragment,sections,bad) + bad = copy.deepcopy(symbols) + bad[field["name"]] = {(int(field["address"], 0), field["size"] + 4)} + with self.assertRaises(ValueError): + compose.compose_track_menu(policy, fragment, sections, bad) def fixture(self, revision): fragment = json.loads((ROOT / f"runtime-recomp/legacy-mods.v{revision}.recomp-fragment.json").read_text()) - sections = [(int(site["vram"], 0), b"".join(struct.pack(">I", int(v, 0)) for v in site["expected"])) - for site in fragment["sites"] + fragment["assetApis"]] + sections = [ + (int(site["vram"], 0), b"".join(struct.pack(">I", int(v, 0)) for v in site["expected"])) + for site in fragment["sites"] + fragment["assetApis"] + ] return fragment, sections def test_verified_entries_compose_without_changing_existing_policy(self): @@ -357,17 +486,22 @@ def test_verified_entries_compose_without_changing_existing_policy(self): after = compose.compose(policy, fragment, sections) self.assertEqual(before, policy) self.assertEqual(after["instructionPatches"][:-2], before["instructionPatches"]) - for old in before['functionHooks']: - matches=[h for h in after['functionHooks'] if h['function']==old['function'] and h['beforeVram']==old['beforeVram']] - self.assertEqual(len(matches),1) - new=matches[0] - if old['function'] in ('asset_table_load','asset_load') and '_begin(rdram, ctx)' in old['text']: - self.assertTrue(new['text'].endswith(' '+old['text'])) - self.assertEqual(new['reason'],old['reason']) - else:self.assertEqual(new,old) + for old in before["functionHooks"]: + matches = [ + h + for h in after["functionHooks"] + if h["function"] == old["function"] and h["beforeVram"] == old["beforeVram"] + ] + self.assertEqual(len(matches), 1) + new = matches[0] + if old["function"] in ("asset_table_load", "asset_load") and "_begin(rdram, ctx)" in old["text"]: + self.assertTrue(new["text"].endswith(" " + old["text"])) + self.assertEqual(new["reason"], old["reason"]) + else: + self.assertEqual(new, old) self.assertTrue(all(p["value"] == "0x00000000" for p in after["instructionPatches"][-2:])) - bridges=[h for h in after['functionHooks'] if 'dkr_legacy_pi_start_dma' in h['text']] - for site, hook in zip(fragment["sites"], bridges,strict=True): + bridges = [h for h in after["functionHooks"] if "dkr_legacy_pi_start_dma" in h["text"]] + for site, hook in zip(fragment["sites"], bridges, strict=True): self.assertEqual(int(hook["beforeVram"], 0), int(site["vram"], 0) + 8) def test_every_instruction_signature_is_required(self): @@ -376,7 +510,8 @@ def test_every_instruction_signature_is_required(self): for index in range(len(sections)): for word in range(3): bad = copy.deepcopy(sections) - data = bytearray(bad[index][1]); data[word * 4 + 3] ^= 1 + data = bytearray(bad[index][1]) + data[word * 4 + 3] ^= 1 bad[index] = (bad[index][0], bytes(data)) with self.subTest(revision=revision, entry=index, word=word): with self.assertRaises(ValueError): @@ -395,44 +530,54 @@ def test_conflicting_sites_and_incomplete_coverage_fail(self): with self.assertRaises(ValueError): compose.compose({"schemaVersion": 1}, fragment, extra) for key in ("sites", "assetApis"): - bad = copy.deepcopy(fragment);bad[key].pop() + bad = copy.deepcopy(fragment) + bad[key].pop() with self.assertRaises(ValueError): compose.compose({"schemaVersion": 1}, bad, sections) for site in fragment["assetApis"]: with self.assertRaises(ValueError): - compose.compose({"schemaVersion": 1, "functionHooks": [{"beforeVram": site["vram"]}]}, fragment, sections) + compose.compose( + {"schemaVersion": 1, "functionHooks": [{"beforeVram": site["vram"]}]}, fragment, sections + ) def test_unmodelled_branch_into_bridge_fails(self): for revision in (77, 80): fragment, sections = self.fixture(revision) for offset in (4, 8): destination = int(fragment["sites"][0]["vram"], 0) + offset - jump = 0x08000000 | ((destination >> 2) & 0x03ffffff) + jump = 0x08000000 | ((destination >> 2) & 0x03FFFFFF) with self.assertRaises(ValueError): compose.compose({"schemaVersion": 1}, fragment, sections + [(0x80001000, struct.pack(">I", jump))]) def test_function_bounds_and_manual_function_identity(self): for revision in (77, 80): fragment, _ = self.fixture(revision) - symbols = {site["function"]: {(int(site["vram"], 0), 12)} - for site in fragment["sites"] + fragment["assetApis"]} + symbols = { + site["function"]: {(int(site["vram"], 0), 12)} for site in fragment["sites"] + fragment["assetApis"] + } symbols[fragment["callTarget"]] = {(int(fragment["callTargetVram"], 0), 12)} compose.verify_function_bounds({}, fragment, symbols) for name in symbols: - bad = copy.deepcopy(symbols); address, size = next(iter(bad[name])) + bad = copy.deepcopy(symbols) + address, size = next(iter(bad[name])) bad[name] = {(address + 4, size)} with self.assertRaises(ValueError): compose.verify_function_bounds({}, fragment, bad) - site = fragment["sites"][1]; del symbols[site["function"]] + site = fragment["sites"][1] + del symbols[site["function"]] manual = {"name": site["function"], "vram": site["vram"], "size": "0xC"} compose.verify_function_bounds({"manualFunctions": [manual]}, fragment, symbols) def test_scene_composition_preserves_existing_owner_and_checks_signatures(self): - for revision, level, menu, word in ((77, 0x8006B250, 0x8006E2E8, 0x81CE3514), - (80, 0x8006B490, 0x8006E528, 0x81CE3A94)): + for revision, level, menu, word in ( + (77, 0x8006B250, 0x8006E2E8, 0x81CE3514), + (80, 0x8006B490, 0x8006E528, 0x81CE3A94), + ): fragment, sections = self.fixture(revision) - sections += [(level, struct.pack(">III", 0x27BDFFA0, 0xAFBF002C, 0xAFB10028)), - (menu, struct.pack(">III", 0x27BDFFE0, 0x3C0E8012, word))] + sections += [ + (level, struct.pack(">III", 0x27BDFFA0, 0xAFBF002C, 0xAFB10028)), + (menu, struct.pack(">III", 0x27BDFFE0, 0x3C0E8012, word)), + ] symbols = {"level_load": {(level, 64)}, "load_level_for_menu": {(menu, 64)}} base = json.loads((ROOT / f"runtime-recomp/dkr.us.v{revision}.recomp-policy.json").read_text()) before = copy.deepcopy(base) @@ -454,12 +599,14 @@ def test_scene_composition_preserves_existing_owner_and_checks_signatures(self): for index in (-1, -2): for offset in range(12): broken = copy.deepcopy(sections) - data = bytearray(broken[index][1]); data[offset] ^= 1 + data = bytearray(broken[index][1]) + data[offset] ^= 1 broken[index] = (broken[index][0], bytes(data)) with self.assertRaises(ValueError): compose.compose_scene_runtime(base, fragment, broken, symbols, True) for name in symbols: - broken = copy.deepcopy(symbols); broken[name].add((0x80001000, 64)) + broken = copy.deepcopy(symbols) + broken[name].add((0x80001000, 64)) with self.assertRaises(ValueError): compose.compose_scene_runtime(base, fragment, sections, broken, True) conflicting = copy.deepcopy(base) diff --git a/runtime-recomp/tests/live_lod_pipeline_tests.py b/runtime-recomp/tests/live_lod_pipeline_tests.py index 9b0a8faf..93490dbf 100644 --- a/runtime-recomp/tests/live_lod_pipeline_tests.py +++ b/runtime-recomp/tests/live_lod_pipeline_tests.py @@ -1,4 +1,5 @@ """Guard the live LOD wiring; pixel correctness is tested by live_lod_probe.hlsl.""" + import hashlib from pathlib import Path import unittest @@ -13,7 +14,8 @@ def read(name): class LiveLodPipelineTests(unittest.TestCase): def test_ui_publishes_before_hidden_overlay_early_return(self): draw = read("runtime-recomp/src/game/runtime_ui.cpp").split( - "void dkr::runtime::ui::draw(RT64::Application& application) {", 1)[1] + "void dkr::runtime::ui::draw(RT64::Application& application) {", 1 + )[1] before_return = draw.split("return;", 1)[0] self.assertIn("RT64::setDefaultSamplerMipLODBias(", before_return) self.assertIn("dkr::runtime::enhancements::effective_texture_lod_bias()", before_return) @@ -40,8 +42,11 @@ def test_existing_patch_is_the_single_bias_owner(self): path = "patches/rt64/0015-configurable-default-mip-lod-bias.patch" digest = hashlib.sha256((ROOT / path).read_bytes()).hexdigest() self.assertIn(digest.lower(), read("patches/manifest.json").lower()) - owners = [p.name for p in (ROOT / "patches/rt64").glob("*.patch") - if "gDefaultSamplerMipLODBias" in p.read_text(encoding="utf-8")] + owners = [ + p.name + for p in (ROOT / "patches/rt64").glob("*.patch") + if "gDefaultSamplerMipLODBias" in p.read_text(encoding="utf-8") + ] self.assertEqual(owners, [Path(path).name]) @@ -53,7 +58,9 @@ def test_manifest_and_separate_patch_ownership(self): def test_generation_only_at_session_boundary(self): renderer = read("runtime-recomp/src/game/rt64_renderer.cpp") - self.assertLess(renderer.index("RT64::beginGeneratedMipSession"), renderer.index("std::make_unique")) + self.assertLess( + renderer.index("RT64::beginGeneratedMipSession"), renderer.index("std::make_unique") + ) ui = read("runtime-recomp/src/game/runtime_ui.cpp") self.assertNotIn("beginGeneratedMipSession", ui) draw = ui.split("void dkr::runtime::ui::draw(RT64::Application& application) {", 1)[1].split("return;", 1)[0] @@ -76,7 +83,15 @@ def test_upload_order_lifetime_and_dds_preservation(self): def test_draw_scope_and_sampling_exclusions(self): sampler = read("extern/rt64/src/shaders/TextureSampler.hlsli") - for guard in ("!renderFlagRect", "!otherMode.forceBlend", "!otherMode.cvgXAlpha", "otherMode.alphaCompare() == G_AC_NONE", "!gpuTileFlagRawTMEM", "!gpuTileFlagFromCopy", "wholeS && wholeT"): + for guard in ( + "!renderFlagRect", + "!otherMode.forceBlend", + "!otherMode.cvgXAlpha", + "otherMode.alphaCompare() == G_AC_NONE", + "!gpuTileFlagRawTMEM", + "!gpuTileFlagFromCopy", + "wholeS && wholeT", + ): self.assertIn(guard, sampler) renderer = read("extern/rt64/src/render/rt64_framebuffer_renderer.cpp") self.assertIn("generatedMipPerspectiveScope(proj.type == Projection::Type::Perspective", renderer) @@ -85,7 +100,9 @@ def test_draw_scope_and_sampling_exclusions(self): def test_loading_feedback_releases_upload_lock_and_does_not_replay_game(self): cache = read("extern/rt64/src/render/rt64_texture_cache.cpp") - wait = cache.split("void TextureCache::waitForGPUUploads()", 1)[1].split("void TextureCache::addResolvedPaths", 1)[0] + wait = cache.split("void TextureCache::waitForGPUUploads()", 1)[1].split( + "void TextureCache::addResolvedPaths", 1 + )[0] self.assertIn("!generateMipmaps || !MipConfiguration::waitFeedback", wait) self.assertLess(wait.index("queueLock.unlock()"), wait.index("MipConfiguration::waitFeedback();")) self.assertLess(wait.index("MipConfiguration::waitFeedback();"), wait.index("queueLock.lock()")) diff --git a/runtime-recomp/tests/magic_code_pipeline_tests.py b/runtime-recomp/tests/magic_code_pipeline_tests.py index 16fb0ca1..fcb2c5b7 100644 --- a/runtime-recomp/tests/magic_code_pipeline_tests.py +++ b/runtime-recomp/tests/magic_code_pipeline_tests.py @@ -1,4 +1,5 @@ """Audit both retail-specific Magic Code hook anchors and generated placement.""" + import json import pathlib import sys @@ -17,8 +18,9 @@ generated = "\n".join(p.read_text() for p in pathlib.Path(generated_path).glob("*.c")) for function, symbol, address in zip( ("menu_magic_codes_loop", "menu_file_select_loop", "main_game_loop"), - ("dkr_magic_code_credits_started", "dkr_magic_code_balloon_awarded", - "dkr_magic_codes_frame_complete"), EXPECTED[revision], strict=True + ("dkr_magic_code_credits_started", "dkr_magic_code_balloon_awarded", "dkr_magic_codes_frame_complete"), + EXPECTED[revision], + strict=True, ): matches = [h for h in hooks if symbol in h["text"]] assert len(matches) == 1, (revision, symbol) @@ -29,11 +31,11 @@ # Preserve the mode-filter implementation. Do not 'fix' intentional retail # Adventure/time-trial/challenge exclusions by bypassing get_filtered_cheats. assert all(h["function"] != "get_filtered_cheats" for h in hooks) - assert all(p["function"] != "get_filtered_cheats" - for p in policy["instructionPatches"]) + assert all(p["function"] != "get_filtered_cheats" for p in policy["instructionPatches"]) balloon_anchor = generated.index( "extern void dkr_magic_code_balloon_awarded(uint8_t*, recomp_context*); " - "dkr_magic_code_balloon_awarded(rdram, ctx);") - assert "sh $t4, 0x0($v0)" in generated[balloon_anchor:balloon_anchor + 280] - assert "MEM_H(0X0, ctx->r2) = ctx->r12;" in generated[balloon_anchor:balloon_anchor + 280] + "dkr_magic_code_balloon_awarded(rdram, ctx);" + ) + assert "sh $t4, 0x0($v0)" in generated[balloon_anchor : balloon_anchor + 280] + assert "MEM_H(0X0, ctx->r2) = ctx->r12;" in generated[balloon_anchor : balloon_anchor + 280] print(f"[test][magic-code-pipeline] {revision}: unique native success and commit anchors PASS") diff --git a/runtime-recomp/tests/owned_content_identity_tests.py b/runtime-recomp/tests/owned_content_identity_tests.py index 943b4bbd..83013265 100644 --- a/runtime-recomp/tests/owned_content_identity_tests.py +++ b/runtime-recomp/tests/owned_content_identity_tests.py @@ -1,4 +1,5 @@ """Contract IDs ignore text newline conventions, not semantic or binary changes.""" + import pathlib import shutil import subprocess @@ -8,6 +9,7 @@ HELPER = pathlib.Path(__file__).resolve().parents[1] / "cmake/OwnedContentIdentity.cmake" +@unittest.skipUnless(shutil.which("cmake"), "needs CMake (python3 scripts/check.py --suite host-tools)") class OwnedContentIdentityTests(unittest.TestCase): def identity(self, root, files, *, outside=False): root = pathlib.Path(root) @@ -26,9 +28,10 @@ def identity(self, root, files, *, outside=False): f'include("{HELPER.as_posix()}")\n' f'dkr_owned_content_identity(value "{authored.as_posix()}" ' f'"{generated.as_posix()}" {" ".join(inputs)})\n' - f'file(WRITE "{output.as_posix()}" "${{value}}")\n', encoding="utf-8") - result = subprocess.run([shutil.which("cmake"), "-P", str(script)], - capture_output=True, text=True) + f'file(WRITE "{output.as_posix()}" "${{value}}")\n', + encoding="utf-8", + ) + result = subprocess.run([shutil.which("cmake"), "-P", str(script)], capture_output=True, text=True) if outside: self.assertNotEqual(result.returncode, 0) self.assertIn("outside its checked roots", result.stderr) @@ -41,17 +44,20 @@ def test_text_newlines_and_relocated_roots_are_portable(self): roots = [pathlib.Path(temp) / name for name in ("windows layout", "linux-layout")] for root in roots: root.mkdir() - files = [(f"generated/sample{ext}", b'int x = 1;\n// checked input\n') - for ext in (".c", ".cpp", ".h", ".hpp")] - files += [("generated/manifest.json", b'{"revision":77,\n"full_scenes":true}\n'), - ("authored/src/native.cpp", b'int native_value = 2;\n')] + files = [ + (f"generated/sample{ext}", b"int x = 1;\n// checked input\n") for ext in (".c", ".cpp", ".h", ".hpp") + ] + files += [ + ("generated/manifest.json", b'{"revision":77,\n"full_scenes":true}\n'), + ("authored/src/native.cpp", b"int native_value = 2;\n"), + ] other = [(name, data.replace(b"\n", b"\r\n")) for name, data in reversed(files)] self.assertEqual(self.identity(roots[0], files), self.identity(roots[1], other)) def test_semantic_change_changes_identity(self): with tempfile.TemporaryDirectory() as temp: - before = self.identity(temp, [("generated/guest.c", b'int x = 1;\n')]) - after = self.identity(temp, [("generated/guest.c", b'int x = 2;\n')]) + before = self.identity(temp, [("generated/guest.c", b"int x = 1;\n")]) + after = self.identity(temp, [("generated/guest.c", b"int x = 2;\n")]) self.assertNotEqual(before, after) def test_manifest_change_changes_identity(self): diff --git a/runtime-recomp/tests/pattern_guard_tests.py b/runtime-recomp/tests/pattern_guard_tests.py new file mode 100644 index 00000000..37af62d2 --- /dev/null +++ b/runtime-recomp/tests/pattern_guard_tests.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Every pattern guard rule fires on its positive fixture and never on its negative one.""" + +import contextlib +import importlib.util +import io +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +FIXTURES = ROOT / "tests/fixtures/pattern-guard" +SPEC = importlib.util.spec_from_file_location("pattern_guard", ROOT / "scripts/pattern_guard.py") +pattern_guard = importlib.util.module_from_spec(SPEC) +sys.modules["pattern_guard"] = pattern_guard # dataclasses resolve their module by name +SPEC.loader.exec_module(pattern_guard) + + +def findings(case): + tree = pattern_guard.Tree(FIXTURES / case) + return pattern_guard.scan(tree, tree.files) + + +def run_guard(root, *args): + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + status = pattern_guard.main(["--root", str(root), *args]) + return status, out.getvalue(), err.getvalue() + + +def git(root, *args): + subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True) + + +class FixtureTests(unittest.TestCase): + def test_every_rule_has_a_positive_and_a_negative_fixture(self): + cases = {path.relative_to(FIXTURES).as_posix() for path in FIXTURES.glob("*/*") if path.is_dir()} + expected = {f"{rule}/{kind}" for rule in pattern_guard.RULES for kind in ("positive", "negative")} + self.assertEqual(cases, expected) + + def test_positive_fixtures_fire_only_their_rule(self): + for rule in pattern_guard.RULES: + with self.subTest(rule=rule): + hits = findings(f"{rule}/positive") + self.assertTrue(hits, f"{rule} did not fire") + self.assertEqual({hit.rule for hit in hits}, {rule}, [hit.render() for hit in hits]) + + def test_positive_fixtures_flag_every_file(self): + for rule in pattern_guard.RULES: + with self.subTest(rule=rule): + tree = pattern_guard.Tree(FIXTURES / rule / "positive") + flagged = {hit.path for hit in pattern_guard.scan(tree, tree.files)} + unflagged = set(tree.files) - flagged - {"patches/manifest.json"} + self.assertEqual(unflagged, set(), "each positive file should be one violating form") + + def test_negative_fixtures_are_clean(self): + for rule in pattern_guard.RULES: + with self.subTest(rule=rule): + self.assertEqual([hit.render() for hit in findings(f"{rule}/negative")], []) + + def test_every_rule_explains_itself(self): + for rule in pattern_guard.RULES: + status, out, _ = run_guard(ROOT, "--explain", rule) + self.assertEqual(status, 0) + self.assertIn("instead:", out) + self.assertEqual(run_guard(ROOT, "--explain", "PG999")[0], 2) + + +class CMakeTokenizerTests(unittest.TestCase): + def test_comments_quotes_brackets_and_lines(self): + commands = pattern_guard.parse_cmake( + '# set(ignored)\n#[[ set(also ignored)\n]]\nSET(a "b c" [=[d)e]=] f) # tail\nif(x AND (y OR z))\nendif()\n' + ) + self.assertEqual( + [(c.name, c.args, c.line) for c in commands], + [ + ("set", ("a", "b c", "d)e", "f"), 4), + ("if", ("x", "AND", "y", "OR", "z"), 5), + ("endif", (), 6), + ], + ) + + +class BaselineTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) / "repo" + shutil.copytree(FIXTURES / "PG001/positive", self.root) + self.baseline = self.root / pattern_guard.BASELINE + + def write_baseline(self, *entries): + self.baseline.parent.mkdir(parents=True, exist_ok=True) + self.baseline.write_text(pattern_guard.BASELINE_HEADER + "".join(f"{e}\n" for e in entries)) + + def test_unlisted_findings_fail_and_listed_ones_pass(self): + status, out, _ = run_guard(self.root, "--tree") + self.assertEqual(status, 1) + self.assertIn("patches/feature/ReleaseOverlay.cmake:1: PG001 ", out) + self.assertIn(" instead: ", out) + self.assertEqual(run_guard(self.root, "--tree", "--baseline", "write")[0], 0) + self.assertEqual(run_guard(self.root, "--tree")[0], 0) + self.assertIn("patches/feature/CMakeLists.txt PG001\n", self.baseline.read_text()) + + def test_fixed_findings_must_leave_the_baseline(self): + run_guard(self.root, "--tree", "--baseline", "write") + (self.root / "patches/feature/CMakeLists.txt").unlink() + status, out, _ = run_guard(self.root, "--tree") + self.assertEqual(status, 1) + self.assertIn("PG001 stale entry, patches/feature/CMakeLists.txt no longer matches", out) + + def test_malformed_baselines_are_usage_errors(self): + for entries in ( + ["patches/feature/*.cmake PG001"], + ["patches/feature/ReleaseOverlay.cmake PG001", "patches/feature/CMakeLists.txt PG001"], + ["patches/feature/CMakeLists.txt PG999"], + ): + with self.subTest(entries=entries): + self.write_baseline(*entries) + status, _, err = run_guard(self.root, "--tree") + self.assertEqual(status, 2, err) + + def test_ignore_reports_every_hit(self): + run_guard(self.root, "--tree", "--baseline", "write") + status, out, _ = run_guard(self.root, "--tree", "--baseline", "ignore") + self.assertEqual((status, out.count(": PG001 ")), (1, 2)) + + +@unittest.skipIf(shutil.which("git") is None, "diff scopes need git") +class DiffScopeTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + git(self.root, "init", "--quiet") + git(self.root, "config", "user.email", "guard@example.invalid") + git(self.root, "config", "user.name", "Guard") + shutil.copytree(FIXTURES / "PG001/positive", self.root, dirs_exist_ok=True) + run_guard(self.root, "--tree", "--baseline", "write") + git(self.root, "add", ".") + git(self.root, "commit", "--quiet", "-m", "base") + + def test_staged_scope_checks_only_staged_paths(self): + layer = self.root / "patches/new-layer/ReleaseOverlay.cmake" + layer.parent.mkdir(parents=True) + layer.write_text('message(STATUS "layer 44")\n') + self.assertEqual(run_guard(self.root, "--staged")[0], 0) + git(self.root, "add", str(layer)) + status, out, _ = run_guard(self.root, "--staged") + self.assertEqual(status, 1) + self.assertIn("patches/new-layer/ReleaseOverlay.cmake:1: PG001", out) + + def test_diff_scope_sees_untracked_files(self): + (self.root / "patches/feature/NOTES.txt").write_text("notes\n") + status, out, _ = run_guard(self.root, "--diff", "HEAD") + self.assertEqual(status, 1) + self.assertIn("patches/feature/NOTES.txt:1: PG013", out) + + def test_baseline_growth_needs_the_transition_label(self): + layer = self.root / "patches/new-layer/ReleaseOverlay.cmake" + layer.parent.mkdir(parents=True) + layer.write_text('message(STATUS "layer 44")\n') + run_guard(self.root, "--tree", "--baseline", "write") + status, out, _ = run_guard(self.root, "--diff", "HEAD") + self.assertEqual(status, 1) + self.assertIn("PG001 new baseline entry for patches/new-layer/ReleaseOverlay.cmake", out) + self.assertEqual(run_guard(self.root, "--diff", "HEAD", "--transition-layer")[0], 0) + + def test_shrinking_the_baseline_passes(self): + (self.root / "patches/feature/CMakeLists.txt").unlink() + self.assertEqual(run_guard(self.root, "--diff", "HEAD")[0], 1) + run_guard(self.root, "--tree", "--baseline", "write") + self.assertEqual(run_guard(self.root, "--diff", "HEAD")[0], 0) + + def test_unknown_base_is_a_usage_error(self): + self.assertEqual(run_guard(self.root, "--diff", "no-such-ref")[0], 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/runtime-recomp/tests/pinned_bytes_tests.py b/runtime-recomp/tests/pinned_bytes_tests.py new file mode 100644 index 00000000..e13387a5 --- /dev/null +++ b/runtime-recomp/tests/pinned_bytes_tests.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Checksum-pinned files stay byte-exact: the checker catches changes, and the +pre-commit exclude list keeps every fixer away from them.""" + +import contextlib +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location("check_pinned_bytes", ROOT / "scripts/check_pinned_bytes.py") +check_pinned_bytes = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(check_pinned_bytes) +CONFIG = (ROOT / ".pre-commit-config.yaml").read_text(encoding="utf-8") +CAPSULE_TEXT = b"pinned capsule line \r\n" +CAPSULE_NAME = hashlib.sha256(CAPSULE_TEXT).hexdigest() + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def run_checker(root, *paths): + err = io.StringIO() + with contextlib.redirect_stderr(err): + status = check_pinned_bytes.main(["--root", str(root), *paths]) + return status, err.getvalue() + + +def top_level_exclude(): + """The top-level `exclude: |` block scalar, without needing PyYAML.""" + match = re.search(r"^exclude: \|\n((?: .*\n)+)", CONFIG, re.M) + return re.compile("".join(line[2:] for line in match[1].splitlines(keepends=True))) + + +def write(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + +class RepositoryTests(unittest.TestCase): + def test_every_pin_in_the_tree_matches(self): + self.assertEqual(check_pinned_bytes.check(ROOT), []) + + def test_each_kind_of_pin_is_found(self): + sources = {pin.source for pin in check_pinned_bytes.pins(ROOT).values()} + self.assertIn("patches/manifest.json", sources) + self.assertIn("patches/release-prep/v1.0.7/source-capsule-r2/manifest.json", sources) + self.assertTrue(any(source.startswith("modding/reference-us-v") for source in sources), sources) + + def test_pre_commit_never_hands_a_pinned_file_to_a_fixer(self): + exclude = top_level_exclude() + reachable = [path for path in check_pinned_bytes.pins(ROOT) if not exclude.search(path)] + self.assertEqual(reachable, []) + + def test_pinned_bytes_hook_runs_without_file_arguments(self): + # The pinned files are excluded, so pre-commit would never pass them in. + hook = CONFIG[CONFIG.index("id: pinned-bytes") :] + self.assertIn("always_run: true", hook) + self.assertIn("pass_filenames: false", hook) + + def test_hook_revisions_match_the_ci_tool_pins(self): + lint_tools = (ROOT / ".github/lint-tools.txt").read_text(encoding="utf-8") + workflows = "".join( + (ROOT / ".github/workflows" / name).read_text(encoding="utf-8") + for name in ("tooling.yml", "workflow-lint.yml") + ) + + def pinned(name): + return re.search(rf"^{re.escape(name)}==(\S+)", lint_tools, re.M)[1] + + def released(tool): + return re.search(rf"/{tool}/releases/download/v([0-9.]+)/", workflows)[1] + + expected = { + "astral-sh/ruff-pre-commit": f"v{pinned('ruff')}", + "pre-commit/mirrors-clang-format": f"v{pinned('clang-format')}", + "BlankSpruce/gersemi-pre-commit": pinned("gersemi"), + "shellcheck-py/shellcheck-py": f"v{pinned('shellcheck-py')}", + "scop/pre-commit-shfmt": f"v{released('sh')}-1", + "rhysd/actionlint": f"v{released('actionlint')}", + "zizmorcore/zizmor-pre-commit": f"v{released('zizmor')}", + } + revs = dict(re.findall(r"- repo: https://github\.com/(\S+)\n\s+rev: (\S+)", CONFIG)) + self.assertEqual({repo: revs.get(repo) for repo in expected}, expected) + + +class CheckerTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.patch = b"--- a/x\n+++ b/x\n@@ -1,3 +1,3 @@\n \n-old\n+new\n \n" + self.symbols = b"symbol = 0x80000000\r\n" + manifest = {"dependencies": [{"patches": [{"path": "patches/dep/0001.patch", "sha256": sha(self.patch)}]}]} + write(self.root / "patches/manifest.json", json.dumps(manifest).encode()) + write(self.root / "patches/dep/0001.patch", self.patch) + receipt = {"files": {"symbols.ld": sha(self.symbols)}} + write(self.root / "modding/reference-us-v77/receipt.json", json.dumps(receipt).encode()) + write(self.root / "modding/reference-us-v77/symbols.ld", self.symbols) + write(self.root / f"patches/release-prep/v1/source-capsule-r1/{CAPSULE_NAME}.txt", CAPSULE_TEXT) + + def test_untouched_pins_pass(self): + self.assertEqual(run_checker(self.root), (0, "")) + + def test_stripped_whitespace_on_a_patch_context_line_fails(self): + # What trailing-whitespace does to a blank context line. + write(self.root / "patches/dep/0001.patch", b"--- a/x\n+++ b/x\n@@ -1,3 +1,3 @@\n\n-old\n+new\n\n") + status, err = run_checker(self.root) + self.assertEqual(status, 1) + self.assertIn("patches/dep/0001.patch: sha256", err) + self.assertIn("pinned by patches/manifest.json", err) + + def test_crlf_normalisation_of_a_receipt_file_fails(self): + write(self.root / "modding/reference-us-v77/symbols.ld", b"symbol = 0x80000000\n") + status, err = run_checker(self.root) + self.assertEqual(status, 1) + self.assertIn("pinned by modding/reference-us-v77/receipt.json", err) + + def test_changed_capsule_file_fails(self): + write(self.root / f"patches/release-prep/v1/source-capsule-r1/{CAPSULE_NAME}.txt", CAPSULE_TEXT.strip()) + status, err = run_checker(self.root) + self.assertEqual(status, 1) + self.assertIn("pinned by its file name", err) + + def write_capsule_manifest(self, base_text): + capsule = { + "roles": { + "ROLE": { + "object.txt": {"sha256": CAPSULE_NAME, "object": f"{CAPSULE_NAME}.txt"}, + "base.py": {"sha256": "0" * 64, "base": "src/base.py", "base_sha256": sha(base_text)}, + } + } + } + write(self.root / "patches/release-prep/v1/source-capsule-r1/manifest.json", json.dumps(capsule).encode()) + + def test_deleted_object_the_capsule_manifest_references_fails(self): + self.write_capsule_manifest(b"base\n") + write(self.root / "src/base.py", b"base\n") + self.assertEqual(run_checker(self.root), (0, "")) + (self.root / f"patches/release-prep/v1/source-capsule-r1/{CAPSULE_NAME}.txt").unlink() + status, err = run_checker(self.root, "patches/release-prep/v1/source-capsule-r1/manifest.json") + self.assertEqual(status, 1) + self.assertIn(f"{CAPSULE_NAME}.txt: missing", err) + + def test_capsule_base_is_compared_after_crlf_normalisation(self): + self.write_capsule_manifest(b"base\n") + write(self.root / "src/base.py", b"base\r\n") + self.assertEqual(run_checker(self.root), (0, "")) + write(self.root / "src/base.py", b"base \n") # what a formatter might leave + status, err = run_checker(self.root, "src/base.py") + self.assertEqual(status, 1) + self.assertIn("pinned by patches/release-prep/v1/source-capsule-r1/manifest.json", err) + + def test_missing_pinned_file_fails(self): + (self.root / "patches/dep/0001.patch").unlink() + status, err = run_checker(self.root) + self.assertEqual(status, 1) + self.assertIn("missing", err) + + def test_paths_limit_the_check_and_manifests_expand_to_their_files(self): + write(self.root / "patches/dep/0001.patch", b"changed\n") + self.assertEqual(run_checker(self.root, "modding/reference-us-v77/symbols.ld")[0], 0) + self.assertEqual(run_checker(self.root, "unrelated.txt")[0], 0) + self.assertEqual(run_checker(self.root, "patches/dep/0001.patch")[0], 1) + self.assertEqual(run_checker(self.root, "./patches/manifest.json")[0], 1) + + +@unittest.skipUnless( + os.environ.get("DKR_PRE_COMMIT_INTEGRATION") == "1", + "real pre-commit run; use python3 scripts/check.py --suite host-tools", +) +@unittest.skipUnless(shutil.which("pre-commit"), "pre-commit is not installed (python3 scripts/bootstrap_hooks.py)") +class HookRunTests(unittest.TestCase): + """Run the real hook set over copies of pinned files and a control file.""" + + def test_full_hook_run_leaves_pinned_bytes_alone(self): + manifest = json.loads((ROOT / "patches/manifest.json").read_text(encoding="utf-8")) + patches = [patch for dependency in manifest["dependencies"] for patch in dependency.get("patches", [])] + # A patch with whitespace on a blank context line is what a fixer would break. + patch = next(p for p in patches if re.search(rb"\n[ \t]+\r?\n", (ROOT / p["path"]).read_bytes())) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for name in (".pre-commit-config.yaml", "pyproject.toml", ".editorconfig", "scripts/check_pinned_bytes.py"): + write(root / name, (ROOT / name).read_bytes()) + write(root / patch["path"], (ROOT / patch["path"]).read_bytes()) + reduced = {"dependencies": [{"patches": [patch]}]} + write(root / "patches/manifest.json", json.dumps(reduced).encode()) + shutil.copytree(ROOT / "modding/reference-us-v77", root / "modding/reference-us-v77") + write(root / "notes.txt", b"trailing space \nno final newline") + pinned = {path: sha((root / path).read_bytes()) for path in check_pinned_bytes.pins(root)} + self.assertGreater(len(pinned), 1) + for args in (("init", "-q"), ("add", "-A")): + subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True) + env = {**os.environ, "SKIP": "pattern-guard"} + result = subprocess.run( + ["pre-commit", "run", "--all-files", "--color", "never"], + cwd=root, + env=env, + capture_output=True, + text=True, + ) + output = result.stdout + result.stderr + # The control file proves the fixers ran; they fail the run by fixing it. + self.assertEqual((root / "notes.txt").read_bytes(), b"trailing space\nno final newline\n", output) + self.assertRegex(output, r"checksum-pinned files untouched by fixers\.+Passed", output) + self.assertEqual({path: sha((root / path).read_bytes()) for path in pinned}, pinned, output) + + +if __name__ == "__main__": + unittest.main() diff --git a/runtime-recomp/tests/regression_harness_tests.py b/runtime-recomp/tests/regression_harness_tests.py index aa405594..b526668a 100644 --- a/runtime-recomp/tests/regression_harness_tests.py +++ b/runtime-recomp/tests/regression_harness_tests.py @@ -1,119 +1,139 @@ #!/usr/bin/env python3 """Exercise the real harness CLI with tiny synthetic subprocess workloads.""" + from __future__ import annotations import concurrent.futures import json import os +from pathlib import Path import shutil import socket -import threading -import time -from pathlib import Path import subprocess import sys import tempfile +import threading +import time import unittest from unittest import mock import xml.etree.ElementTree as ET ROOT = Path(__file__).resolve().parents[2] -RUNNER = ROOT / 'scripts' / 'regression.py' -sys.path.insert(0, str(ROOT / 'scripts')) +RUNNER = ROOT / "scripts" / "regression.py" +sys.path.insert(0, str(ROOT / "scripts")) import regression class RegressionHarnessTests(unittest.TestCase): def setUp(self): - self.temporary = tempfile.TemporaryDirectory(prefix='dkr harness tests ') + self.temporary = tempfile.TemporaryDirectory(prefix="dkr harness tests ") self.addCleanup(self.temporary.cleanup) self.root = Path(self.temporary.name) - self.suite_path = self.root / 'suite with spaces.json' + self.suite_path = self.root / "suite with spaces.json" self.next_output = 0 - def scenario(self, name='synthetic', code='print("OK")', **options): - return {'id': name, 'command': [sys.executable, '-c', code], **options} + def scenario(self, name="synthetic", code='print("OK")', **options): + return {"id": name, "command": [sys.executable, "-c", code], **options} def suite(self, *scenarios): - value = {'schema_version': 1, 'name': 'Synthetic CLI acceptance', 'scenarios': list(scenarios)} - self.suite_path.write_text(json.dumps(value), encoding='utf-8') + value = {"schema_version": 1, "name": "Synthetic CLI acceptance", "scenarios": list(scenarios)} + self.suite_path.write_text(json.dumps(value), encoding="utf-8") return value def run_cli(self, *options, output=None): self.next_output += 1 - output = output or self.root / f'run with spaces {self.next_output}' - command = [sys.executable, str(RUNNER), '--suite', str(self.suite_path), - '--build-dir', str(self.root / 'fake build'), '--repeat', '1', '--warmup', '0', - '--output', str(output), *map(str, options)] + output = output or self.root / f"run with spaces {self.next_output}" + command = [ + sys.executable, + str(RUNNER), + "--suite", + str(self.suite_path), + "--build-dir", + str(self.root / "fake build"), + "--repeat", + "1", + "--warmup", + "0", + "--output", + str(output), + *map(str, options), + ] child = subprocess.run(command, cwd=self.root, capture_output=True, text=True, timeout=20) - report_path = output / 'report.json' + report_path = output / "report.json" report = json.loads(report_path.read_text()) if report_path.is_file() else None return child, report, output def assert_exit(self, child, code): - self.assertEqual(child.returncode, code, child.stdout + '\n' + child.stderr) + self.assertEqual(child.returncode, code, child.stdout + "\n" + child.stderr) def test_warmup_is_checked_but_excluded_from_measurements(self): - code = '''from pathlib import Path + code = """from pathlib import Path import json p = Path('count') n = int(p.read_text()) + 1 if p.exists() else 1 p.write_text(str(n)) print('DKR_HARNESS ' + json.dumps({'metrics': {'stage_ms': 999 if n == 1 else n}})) print('STATE=stable') -''' - self.suite(self.scenario(code=code, required_metrics=['stage_ms'], - fingerprints=['STATE=\\w+'], budgets={'stage_ms': 3})) - child, report, output = self.run_cli('--warmup', '1', '--repeat', '2') +""" + self.suite( + self.scenario( + code=code, required_metrics=["stage_ms"], fingerprints=["STATE=\\w+"], budgets={"stage_ms": 3} + ) + ) + child, report, output = self.run_cli("--warmup", "1", "--repeat", "2") self.assert_exit(child, 0) - result = report['scenarios'][0] - self.assertEqual(result['metrics']['stage_ms'], 2.5) - self.assertEqual([x['phase'] for x in result['attempts']], ['warmup', 'measured', 'measured']) - self.assertEqual(result['attempts'][0]['metrics']['stage_ms'], 999) - for attempt in result['attempts']: - self.assertTrue((output / attempt['stdout']).is_file()) - self.assertTrue((output / 'report.html').is_file()) - self.assertEqual(ET.parse(output / 'junit.xml').getroot().attrib['failures'], '0') + result = report["scenarios"][0] + self.assertEqual(result["metrics"]["stage_ms"], 2.5) + self.assertEqual([x["phase"] for x in result["attempts"]], ["warmup", "measured", "measured"]) + self.assertEqual(result["attempts"][0]["metrics"]["stage_ms"], 999) + for attempt in result["attempts"]: + self.assertTrue((output / attempt["stdout"]).is_file()) + self.assertTrue((output / "report.html").is_file()) + self.assertEqual(ET.parse(output / "junit.xml").getroot().attrib["failures"], "0") def test_child_crash_timeout_and_missing_input_are_not_passes(self): self.suite( - self.scenario('crash', 'import os; print("before crash", flush=True); os._exit(9)'), - self.scenario('timeout', 'import time; print("started", flush=True); time.sleep(.5)', timeout_seconds=.05), - self.scenario('missing', inputs=[str(self.root / 'missing.bin')]), - self.scenario('healthy'), + self.scenario("crash", 'import os; print("before crash", flush=True); os._exit(9)'), + self.scenario("timeout", 'import time; print("started", flush=True); time.sleep(.5)', timeout_seconds=0.05), + self.scenario("missing", inputs=[str(self.root / "missing.bin")]), + self.scenario("healthy"), ) child, report, output = self.run_cli() self.assert_exit(child, 1) - self.assertEqual(report['summary'], {'passed': 1, 'failed': 2, 'skipped': 1}) - results = {item['id']: item for item in report['scenarios']} - self.assertEqual(results['crash']['attempts'][0]['exit_code'], 9) - self.assertIn('before crash', (output / results['crash']['attempts'][0]['stdout']).read_text()) - self.assertTrue(any('Timed out' in x for x in results['timeout']['errors'])) - self.assertEqual(results['missing']['attempts'], []) - xml = ET.parse(output / 'junit.xml').getroot() - self.assertEqual(xml.attrib['failures'], '2') - self.assertEqual(xml.attrib['skipped'], '1') + self.assertEqual(report["summary"], {"passed": 1, "failed": 2, "skipped": 1}) + results = {item["id"]: item for item in report["scenarios"]} + self.assertEqual(results["crash"]["attempts"][0]["exit_code"], 9) + self.assertIn("before crash", (output / results["crash"]["attempts"][0]["stdout"]).read_text()) + self.assertTrue(any("Timed out" in x for x in results["timeout"]["errors"])) + self.assertEqual(results["missing"]["attempts"], []) + xml = ET.parse(output / "junit.xml").getroot() + self.assertEqual(xml.attrib["failures"], "2") + self.assertEqual(xml.attrib["skipped"], "1") def test_missing_policy_is_explicit_and_all_skipped_still_fails(self): - missing = self.scenario('missing', inputs=['${private_fixture}']) - self.suite(missing, self.scenario('healthy')) + missing = self.scenario("missing", inputs=["${private_fixture}"]) + self.suite(missing, self.scenario("healthy")) child, report, _ = self.run_cli() self.assert_exit(child, 2) - self.assertEqual(report['summary']['skipped'], 1) - child, _, _ = self.run_cli('--allow-missing') + self.assertEqual(report["summary"]["skipped"], 1) + child, _, _ = self.run_cli("--allow-missing") self.assert_exit(child, 0) self.suite(missing) - child, report, _ = self.run_cli('--allow-missing') + child, report, _ = self.run_cli("--allow-missing") self.assert_exit(child, 2) - self.assertEqual(report['summary']['passed'], 0) + self.assertEqual(report["summary"]["passed"], 0) def measurement_suite(self): # This external control file intentionally is not a fixture input: it # emulates varying runtime performance while keeping the workload fixed. - control = self.root / 'measurement control.json' - code = "import json; from pathlib import Path; x=json.loads(Path(" + repr(str(control)) + ").read_text()); print('DKR_HARNESS ' + json.dumps({'metrics': {'stage_ms': x['time']}})); print('STATE=' + x['state'])" - self.suite(self.scenario(code=code, required_metrics=['stage_ms'], fingerprints=['STATE=\\w+'])) - control.write_text(json.dumps({'time': 10, 'state': 'stable'})) + control = self.root / "measurement control.json" + code = ( + "import json; from pathlib import Path; x=json.loads(Path(" + + repr(str(control)) + + ").read_text()); print('DKR_HARNESS ' + json.dumps({'metrics': {'stage_ms': x['time']}})); print('STATE=' + x['state'])" + ) + self.suite(self.scenario(code=code, required_metrics=["stage_ms"], fingerprints=["STATE=\\w+"])) + control.write_text(json.dumps({"time": 10, "state": "stable"})) return control def test_same_machine_baseline_detects_regression_and_fingerprint_change(self): @@ -122,210 +142,259 @@ def test_same_machine_baseline_detects_regression_and_fingerprint_change(self): self.assert_exit(child, 0) # Process timing is noisy; a generous ms floor removes that source of # flakiness while the deterministic stage regression is much larger. - options = ['--baseline', baseline / 'report.json', '--absolute-tolerance-ms', '100'] + options = ["--baseline", baseline / "report.json", "--absolute-tolerance-ms", "100"] child, report, _ = self.run_cli(*options) self.assert_exit(child, 0) - self.assertTrue(report['scenarios'][0]['comparisons']) - control.write_text(json.dumps({'time': 1000, 'state': 'stable'})) + self.assertTrue(report["scenarios"][0]["comparisons"]) + control.write_text(json.dumps({"time": 1000, "state": "stable"})) child, report, _ = self.run_cli(*options) self.assert_exit(child, 1) - comparison = next(x for x in report['scenarios'][0]['comparisons'] if x['metric'] == 'stage_ms') - self.assertEqual(comparison['status'], 'failed') - control.write_text(json.dumps({'time': 10, 'state': 'changed'})) + comparison = next(x for x in report["scenarios"][0]["comparisons"] if x["metric"] == "stage_ms") + self.assertEqual(comparison["status"], "failed") + control.write_text(json.dumps({"time": 10, "state": "changed"})) child, report, _ = self.run_cli(*options) self.assert_exit(child, 1) - self.assertTrue(any('fingerprint' in error.lower() for error in report['scenarios'][0]['errors'])) + self.assertTrue(any("fingerprint" in error.lower() for error in report["scenarios"][0]["errors"])) def test_incompatible_baseline_is_rejected_before_execution(self): self.measurement_suite() child, _, baseline = self.run_cli() self.assert_exit(child, 0) - child, report, output = self.run_cli('--baseline', baseline / 'report.json', '--config', 'Debug') + child, report, output = self.run_cli("--baseline", baseline / "report.json", "--config", "Debug") self.assert_exit(child, 2) self.assertIsNone(report) self.assertFalse(output.exists()) - self.assertIn('Baseline', child.stderr) + self.assertIn("Baseline", child.stderr) def test_list_does_not_execute_and_existing_output_is_preserved(self): - sentinel = self.root / 'executed' + sentinel = self.root / "executed" self.suite(self.scenario(code=f'from pathlib import Path; Path({str(sentinel)!r}).write_text("executed")')) - child, report, output = self.run_cli('--list') + child, report, output = self.run_cli("--list") self.assert_exit(child, 0) - self.assertIn('synthetic', child.stdout) + self.assertIn("synthetic", child.stdout) self.assertFalse(sentinel.exists()) self.assertFalse(output.exists()) self.assertIsNone(report) output.mkdir() - preserved = output / 'important.txt' - preserved.write_text('keep') + preserved = output / "important.txt" + preserved.write_text("keep") child, _, _ = self.run_cli(output=output) self.assert_exit(child, 2) self.assertFalse(sentinel.exists()) - self.assertEqual(preserved.read_text(), 'keep') + self.assertEqual(preserved.read_text(), "keep") def test_command_arguments_are_literal_and_fixture_changes_invalidate_baseline(self): - fixture = self.root / 'private fixture with spaces.bin' - fixture.write_bytes(b'first') - marker = self.root / 'should never execute' + fixture = self.root / "private fixture with spaces.bin" + fixture.write_bytes(b"first") + marker = self.root / "should never execute" payload = f'$(touch "{marker}"); echo injected' - item = self.scenario(code='import sys; print(sys.argv[1]); print("OK")', - inputs=['${fixture}'], expect_stdout=[payload, 'OK']) - item['command'].append(payload) + item = self.scenario( + code='import sys; print(sys.argv[1]); print("OK")', inputs=["${fixture}"], expect_stdout=[payload, "OK"] + ) + item["command"].append(payload) self.suite(item) - options = ['--var', f'fixture={fixture}'] + options = ["--var", f"fixture={fixture}"] child, report, baseline = self.run_cli(*options) self.assert_exit(child, 0) self.assertFalse(marker.exists()) - self.assertEqual(len(report['scenarios'][0]['input_sha256']), 1) - fixture.write_bytes(b'changed') - child, report, _ = self.run_cli(*options, '--baseline', baseline / 'report.json') + self.assertEqual(len(report["scenarios"][0]["input_sha256"]), 1) + fixture.write_bytes(b"changed") + child, report, _ = self.run_cli(*options, "--baseline", baseline / "report.json") self.assert_exit(child, 1) - self.assertTrue(any('baseline' in error.lower() for error in report['scenarios'][0]['errors'])) + self.assertTrue(any("baseline" in error.lower() for error in report["scenarios"][0]["errors"])) def test_expanded_variables_are_part_of_the_baseline_contract(self): - item = self.scenario(code='import sys; print(sys.argv[1])') - item['command'].append('${workload}') + item = self.scenario(code="import sys; print(sys.argv[1])") + item["command"].append("${workload}") self.suite(item) - child, _, baseline = self.run_cli('--var', 'workload=small') + child, _, baseline = self.run_cli("--var", "workload=small") self.assert_exit(child, 0) - child, report, _ = self.run_cli('--var', 'workload=large', '--baseline', baseline / 'report.json') + child, report, _ = self.run_cli("--var", "workload=large", "--baseline", baseline / "report.json") self.assert_exit(child, 1) - self.assertTrue(any('baseline' in error.lower() for error in report['scenarios'][0]['errors'])) + self.assertTrue(any("baseline" in error.lower() for error in report["scenarios"][0]["errors"])) def test_vars_file_resolves_paths_beside_file_and_cli_overrides_are_literal(self): - directory = self.root / 'private configuration' + directory = self.root / "private configuration" directory.mkdir() - fixture = (directory / 'fixture with spaces.bin').resolve() - fixture.write_bytes(b'private fixture') - variables = directory / 'paths.json' - variables.write_text(json.dumps({'fixture': fixture.name, 'absolute': str(fixture), - 'workload': 'unused', 'unused_for_selected_scenario': 'missing.bin'})) - item = self.scenario(code='import sys; print(sys.argv[1]); print(sys.argv[2])', - inputs=['${fixture}', '${absolute}'], expect_stdout=[str(fixture), 'small']) - item['command'].extend(['${fixture}', '${workload}']) + fixture = (directory / "fixture with spaces.bin").resolve() + fixture.write_bytes(b"private fixture") + variables = directory / "paths.json" + variables.write_text( + json.dumps( + { + "fixture": fixture.name, + "absolute": str(fixture), + "workload": "unused", + "unused_for_selected_scenario": "missing.bin", + } + ) + ) + item = self.scenario( + code="import sys; print(sys.argv[1]); print(sys.argv[2])", + inputs=["${fixture}", "${absolute}"], + expect_stdout=[str(fixture), "small"], + ) + item["command"].extend(["${fixture}", "${workload}"]) self.suite(item) # Overrides win regardless of argument order, and remain scalar strings. - child, report, baseline = self.run_cli('--var', 'workload=small', '--vars-file', variables) + child, report, baseline = self.run_cli("--var", "workload=small", "--vars-file", variables) self.assert_exit(child, 0) - self.assertEqual(report['scenarios'][0]['input_paths'], [str(fixture)] * 2) + self.assertEqual(report["scenarios"][0]["input_paths"], [str(fixture)] * 2) # Equivalent CLI inputs produce the same baseline contract. - child, _, _ = self.run_cli('--var', f'fixture={fixture}', '--var', f'absolute={fixture}', - '--var', 'workload=small', '--baseline', baseline / 'report.json') + child, _, _ = self.run_cli( + "--var", + f"fixture={fixture}", + "--var", + f"absolute={fixture}", + "--var", + "workload=small", + "--baseline", + baseline / "report.json", + ) self.assert_exit(child, 0) def test_invalid_vars_files_fail_before_execution_or_output(self): self.suite(self.scenario()) - variables = self.root / 'invalid paths.json' - cases = ['[]', 'null', '{', '{"fixture":"one", "fixture":"two"}', - '{"bad-name":"value"}', '{"root":"value"}', '{"python":"value"}', - '{"build":"value"}', '{"fixture":1}', '{"fixture":null}', - '{"fixture":{}}', '{"fixture":""}', '{"fixture":"bad\\u0000path"}'] + variables = self.root / "invalid paths.json" + cases = [ + "[]", + "null", + "{", + '{"fixture":"one", "fixture":"two"}', + '{"bad-name":"value"}', + '{"root":"value"}', + '{"python":"value"}', + '{"build":"value"}', + '{"fixture":1}', + '{"fixture":null}', + '{"fixture":{}}', + '{"fixture":""}', + '{"fixture":"bad\\u0000path"}', + ] for content in cases: with self.subTest(content=content): variables.write_text(content) - child, report, output = self.run_cli('--vars-file', variables) + child, report, output = self.run_cli("--vars-file", variables) self.assert_exit(child, 2) self.assertIsNone(report) self.assertFalse(output.exists()) - self.assertNotIn('Traceback', child.stderr) + self.assertNotIn("Traceback", child.stderr) variables.unlink() - child, _, output = self.run_cli('--vars-file', variables) + child, _, output = self.run_cli("--vars-file", variables) self.assert_exit(child, 2) self.assertFalse(output.exists()) - self.assertNotIn('Traceback', child.stderr) + self.assertNotIn("Traceback", child.stderr) def test_vars_file_does_not_permit_duplicate_or_reserved_cli_assignments(self): self.suite(self.scenario()) - variables = self.root / 'paths.json' + variables = self.root / "paths.json" variables.write_text('{"fixture":"unused.bin"}') - for assignments in [('fixture=one', 'fixture=two'), ('root=bad',), ('python=bad',), - ('build=bad',), ('no_equals',), ('bad-name=value',), ('fixture=',)]: + for assignments in [ + ("fixture=one", "fixture=two"), + ("root=bad",), + ("python=bad",), + ("build=bad",), + ("no_equals",), + ("bad-name=value",), + ("fixture=",), + ]: with self.subTest(assignments=assignments): - options = [argument for assignment in assignments for argument in ('--var', assignment)] - child, report, output = self.run_cli('--vars-file', variables, *options) + options = [argument for assignment in assignments for argument in ("--var", assignment)] + child, report, output = self.run_cli("--vars-file", variables, *options) self.assert_exit(child, 2) self.assertIsNone(report) self.assertFalse(output.exists()) - self.assertNotIn('Traceback', child.stderr) + self.assertNotIn("Traceback", child.stderr) def test_check_inputs_accepts_paths_with_spaces_without_running_commands(self): - sentinel = self.root / 'must not execute' - fixture = self.root / 'private input.bin' - fixture.write_bytes(b'fixture') - variables = self.root / 'paths.json' - variables.write_text(json.dumps({'fixture': fixture.name})) - self.suite(self.scenario(code=f'from pathlib import Path; Path({str(sentinel)!r}).touch()', - inputs=['${fixture}'])) - child, report, output = self.run_cli('--vars-file', variables, '--check-inputs') + sentinel = self.root / "must not execute" + fixture = self.root / "private input.bin" + fixture.write_bytes(b"fixture") + variables = self.root / "paths.json" + variables.write_text(json.dumps({"fixture": fixture.name})) + self.suite( + self.scenario(code=f"from pathlib import Path; Path({str(sentinel)!r}).touch()", inputs=["${fixture}"]) + ) + child, report, output = self.run_cli("--vars-file", variables, "--check-inputs") self.assert_exit(child, 0) - self.assertIn('Inputs ready for 1 selected scenarios', child.stdout) + self.assertIn("Inputs ready for 1 selected scenarios", child.stdout) self.assertIsNone(report) self.assertFalse(output.exists()) self.assertFalse(sentinel.exists()) def test_check_inputs_aggregates_independent_failures_and_honors_selection(self): - missing_file = self.root / 'missing fixture.bin' - not_executable = self.root / 'nonexecutable file' - not_executable.write_text('not a program') - sentinel = self.root / 'must not execute' - self.suite(self.scenario('missing-vars', inputs=['${missing_one}', '${missing_two}', str(missing_file)], - env={'EXAMPLE': '${missing_env}'}), - {'id': 'bad-command', 'command': [str(not_executable)]}, - self.scenario('ready', code=f'from pathlib import Path; Path({str(sentinel)!r}).touch()')) - child, report, output = self.run_cli('--check-inputs', '--allow-missing') + missing_file = self.root / "missing fixture.bin" + not_executable = self.root / "nonexecutable file" + not_executable.write_text("not a program") + sentinel = self.root / "must not execute" + self.suite( + self.scenario( + "missing-vars", + inputs=["${missing_one}", "${missing_two}", str(missing_file)], + env={"EXAMPLE": "${missing_env}"}, + ), + {"id": "bad-command", "command": [str(not_executable)]}, + self.scenario("ready", code=f"from pathlib import Path; Path({str(sentinel)!r}).touch()"), + ) + child, report, output = self.run_cli("--check-inputs", "--allow-missing") self.assert_exit(child, 2) # Inputs are canonicalized; a failed executable lookup preserves its argument. - for expected in ('missing_one', 'missing_two', 'missing_env', str(missing_file.resolve()), str(not_executable)): + for expected in ("missing_one", "missing_two", "missing_env", str(missing_file.resolve()), str(not_executable)): self.assertIn(expected, child.stderr) self.assertIsNone(report) self.assertFalse(output.exists()) self.assertFalse(sentinel.exists()) - child, _, output = self.run_cli('--check-inputs', '--scenario', 'ready') + child, _, output = self.run_cli("--check-inputs", "--scenario", "ready") self.assert_exit(child, 0) self.assertFalse(output.exists()) self.assertFalse(sentinel.exists()) def test_check_inputs_and_execution_resolve_relative_executable_from_invocation_directory(self): - item = self.scenario(expect_stdout=['OK']) - item['command'][0] = os.path.relpath(sys.executable, self.root.resolve()) + item = self.scenario(expect_stdout=["OK"]) + item["command"][0] = os.path.relpath(sys.executable, self.root.resolve()) self.suite(item) - child, _, output = self.run_cli('--check-inputs') + child, _, output = self.run_cli("--check-inputs") self.assert_exit(child, 0) self.assertFalse(output.exists()) child, report, _ = self.run_cli() self.assert_exit(child, 0) - self.assertEqual(report['scenarios'][0]['command'][0], str(Path(sys.executable).resolve())) + self.assertEqual(report["scenarios"][0]["command"][0], str(Path(sys.executable).resolve())) def test_check_inputs_validates_variables_and_needs_no_output_option(self): self.suite(self.scenario()) - child = subprocess.run([sys.executable, str(RUNNER), '--suite', str(self.suite_path), - '--check-inputs'], capture_output=True, text=True, timeout=20) + child = subprocess.run( + [sys.executable, str(RUNNER), "--suite", str(self.suite_path), "--check-inputs"], + capture_output=True, + text=True, + timeout=20, + ) self.assert_exit(child, 0) - child, _, output = self.run_cli('--check-inputs', '--var', 'root=forbidden') + child, _, output = self.run_cli("--check-inputs", "--var", "root=forbidden") self.assert_exit(child, 2) self.assertFalse(output.exists()) - @unittest.skipUnless(shutil.which('ctest'), 'CTest is not installed') + @unittest.skipUnless(shutil.which("ctest"), "CTest is not installed") def test_check_inputs_inspects_ctest_registration_without_executing_tests(self): - build = self.root / 'fake build' + build = self.root / "fake build" build.mkdir() - sentinel = self.root / 'must not execute' - child_script = self.root / 'registered child.py' - child_script.write_text(f'from pathlib import Path; Path({str(sentinel)!r}).touch()\n') - (build / 'CTestTestfile.cmake').write_text( - f'add_test(Ready [==[{sys.executable}]==] [==[{child_script}]==])\n' - f'add_test(Disabled [==[{sys.executable}]==] [==[{child_script}]==])\n' - 'set_tests_properties(Disabled PROPERTIES DISABLED TRUE)\n' - 'add_test(Unbuilt /missing/dkr/executable)\n') - self.suite(*({'id': name.lower(), 'ctest': name} for name in ('Ready', 'Disabled', 'Unbuilt', 'Missing'))) - child, _, output = self.run_cli('--check-inputs') + sentinel = self.root / "must not execute" + child_script = self.root / "registered child.py" + child_script.write_text(f"from pathlib import Path; Path({str(sentinel)!r}).touch()\n") + (build / "CTestTestfile.cmake").write_text( + f"add_test(Ready [==[{sys.executable}]==] [==[{child_script}]==])\n" + f"add_test(Disabled [==[{sys.executable}]==] [==[{child_script}]==])\n" + "set_tests_properties(Disabled PROPERTIES DISABLED TRUE)\n" + "add_test(Unbuilt /missing/dkr/executable)\n" + ) + self.suite(*({"id": name.lower(), "ctest": name} for name in ("Ready", "Disabled", "Unbuilt", "Missing"))) + child, _, output = self.run_cli("--check-inputs") self.assert_exit(child, 2) - self.assertIn('CTest test is disabled: Disabled', child.stderr) - self.assertIn('build Unbuilt first', child.stderr) - self.assertIn('CTest test not configured: Missing', child.stderr) + self.assertIn("CTest test is disabled: Disabled", child.stderr) + self.assertIn("build Unbuilt first", child.stderr) + self.assertIn("CTest test not configured: Missing", child.stderr) self.assertFalse(output.exists()) self.assertFalse(sentinel.exists()) - child, _, output = self.run_cli('--check-inputs', '--scenario', 'ready') + child, _, output = self.run_cli("--check-inputs", "--scenario", "ready") self.assert_exit(child, 0) self.assertFalse(output.exists()) self.assertFalse(sentinel.exists()) @@ -336,145 +405,171 @@ def test_process_time_is_informational_unless_explicitly_compared(self): self.assert_exit(child, 0) # Force a deterministic process regression without introducing sleeps or # depending on scheduler variance between two real executions. - for key in baseline_report['scenarios'][0]['metrics']: - if key.startswith('process.'): - baseline_report['scenarios'][0]['metrics'][key] = 0 - (baseline / 'report.json').write_text(json.dumps(baseline_report)) - options = ['--baseline', baseline / 'report.json', '--relative-tolerance', '0', '--absolute-tolerance-ms', '0'] + for key in baseline_report["scenarios"][0]["metrics"]: + if key.startswith("process."): + baseline_report["scenarios"][0]["metrics"][key] = 0 + (baseline / "report.json").write_text(json.dumps(baseline_report)) + options = ["--baseline", baseline / "report.json", "--relative-tolerance", "0", "--absolute-tolerance-ms", "0"] child, report, _ = self.run_cli(*options) self.assert_exit(child, 0) - self.assertFalse(any(x['metric'].startswith('process.') for x in report['scenarios'][0]['comparisons'])) - child, report, _ = self.run_cli(*options, '--compare-process-time') + self.assertFalse(any(x["metric"].startswith("process.") for x in report["scenarios"][0]["comparisons"])) + child, report, _ = self.run_cli(*options, "--compare-process-time") self.assert_exit(child, 1) - self.assertTrue(any(x['metric'].startswith('process.') and x['status'] == 'failed' - for x in report['scenarios'][0]['comparisons'])) + self.assertTrue( + any( + x["metric"].startswith("process.") and x["status"] == "failed" + for x in report["scenarios"][0]["comparisons"] + ) + ) def test_instrumented_metrics_strip_ctest_prefix_and_exclude_unbounded_histogram(self): - text = '\n'.join([ - '42: [rollback][timing] stage=tick count=8 mean-us=1250 p95-upper-us=2000 p99-upper-us=18446744073709551615 max-us=3000', - '42: DKR_HARNESS {"metrics": {"custom_ms": 3}}', - '42: [perf][present-interval] ms(p50/p95/p99/max)=16.0/18.0/20.0/25.0', - '42: [perf][present-interval] ms(p50/p95/p99/max)=17.0/17.0/19.0/27.0', - ]) + text = "\n".join( + [ + "42: [rollback][timing] stage=tick count=8 mean-us=1250 p95-upper-us=2000 p99-upper-us=18446744073709551615 max-us=3000", + '42: DKR_HARNESS {"metrics": {"custom_ms": 3}}', + "42: [perf][present-interval] ms(p50/p95/p99/max)=16.0/18.0/20.0/25.0", + "42: [perf][present-interval] ms(p50/p95/p99/max)=17.0/17.0/19.0/27.0", + ] + ) metrics = regression.extract_metrics(text) - self.assertEqual(metrics['rollback.tick.count'], 8) - self.assertEqual(metrics['rollback.tick.mean_ms'], 1.25) - self.assertEqual(metrics['rollback.tick.p95_upper_ms'], 2) - self.assertNotIn('rollback.tick.p99_upper_ms', metrics) - self.assertEqual(metrics['rollback.tick.max_ms'], 3) - self.assertEqual(metrics['custom_ms'], 3) - self.assertEqual(metrics['present.window_p50_max_ms'], 17) - self.assertEqual(metrics['present.window_p95_max_ms'], 18) - self.assertEqual(metrics['present.window_max_max_ms'], 27) - - @unittest.skipUnless(shutil.which('ctest'), 'CTest is not installed') + self.assertEqual(metrics["rollback.tick.count"], 8) + self.assertEqual(metrics["rollback.tick.mean_ms"], 1.25) + self.assertEqual(metrics["rollback.tick.p95_upper_ms"], 2) + self.assertNotIn("rollback.tick.p99_upper_ms", metrics) + self.assertEqual(metrics["rollback.tick.max_ms"], 3) + self.assertEqual(metrics["custom_ms"], 3) + self.assertEqual(metrics["present.window_p50_max_ms"], 17) + self.assertEqual(metrics["present.window_p95_max_ms"], 18) + self.assertEqual(metrics["present.window_max_max_ms"], 27) + + @unittest.skipUnless(shutil.which("ctest"), "CTest is not installed") def test_ctest_skip_xml_is_not_a_passing_scenario(self): - build = self.root / 'fake build' + build = self.root / "fake build" build.mkdir() - child_script = self.root / 'skip child.py' - child_script.write_text('raise SystemExit(77)\n') + child_script = self.root / "skip child.py" + child_script.write_text("raise SystemExit(77)\n") # A minimal CTest registration needs no compiler or configure step. - (build / 'CTestTestfile.cmake').write_text( - f'add_test(SyntheticSkip [==[{sys.executable}]==] [==[{child_script}]==])\n' - 'set_tests_properties(SyntheticSkip PROPERTIES SKIP_RETURN_CODE 77)\n') - self.suite({'id': 'ctest-skip', 'ctest': 'SyntheticSkip'}) - child, report, output = self.run_cli('--allow-missing') + (build / "CTestTestfile.cmake").write_text( + f"add_test(SyntheticSkip [==[{sys.executable}]==] [==[{child_script}]==])\n" + "set_tests_properties(SyntheticSkip PROPERTIES SKIP_RETURN_CODE 77)\n" + ) + self.suite({"id": "ctest-skip", "ctest": "SyntheticSkip"}) + child, report, output = self.run_cli("--allow-missing") self.assert_exit(child, 2) - self.assertEqual(report['summary'], {'passed': 0, 'failed': 0, 'skipped': 1}) - self.assertEqual(report['scenarios'][0]['attempts'][0]['status'], 'skipped') - xml = ET.parse(output / 'junit.xml').getroot() - self.assertEqual(xml.attrib['skipped'], '1') + self.assertEqual(report["summary"], {"passed": 0, "failed": 0, "skipped": 1}) + self.assertEqual(report["scenarios"][0]["attempts"][0]["status"], "skipped") + xml = ET.parse(output / "junit.xml").getroot() + self.assertEqual(xml.attrib["skipped"], "1") def test_timeout_is_classified_when_watchdog_has_not_been_scheduled(self): # Simulate a starved watchdog deterministically; the child exits normally # after the observed deadline, so its exit code alone cannot enforce it. - for duration, expected in ((.2, 'Timed out after 0.15 seconds'), (.1, None)): - with self.subTest(duration=duration), \ - mock.patch.object(regression.threading, 'Thread'), \ - mock.patch.object(regression.time, 'perf_counter', side_effect=[100.0, 100.0 + duration]): + for duration, expected in ((0.2, "Timed out after 0.15 seconds"), (0.1, None)): + with ( + self.subTest(duration=duration), + mock.patch.object(regression.threading, "Thread"), + mock.patch.object(regression.time, "perf_counter", side_effect=[100.0, 100.0 + duration]), + ): code, elapsed, error, interrupted = regression.execute( - [sys.executable, '-c', 'pass'], self.root, os.environ.copy(), .15, - self.root / 'stdout.log', self.root / 'stderr.log') + [sys.executable, "-c", "pass"], + self.root, + os.environ.copy(), + 0.15, + self.root / "stdout.log", + self.root / "stderr.log", + ) self.assertEqual(code, 0) self.assertAlmostEqual(elapsed, duration * 1000) self.assertEqual(error, expected) self.assertFalse(interrupted) - @unittest.skipUnless(os.name == 'posix', 'POSIX process-group teardown') + @unittest.skipUnless(os.name == "posix", "POSIX process-group teardown") def test_timeout_retires_started_descendant(self): # A socket handshake proves the descendant is alive before advancing the # deadline. EOF after execute returns proves teardown, without racing a # delayed sentinel against a heavily loaded CI machine's scheduler. expired = threading.Event() real_counter = time.perf_counter + def controlled_counter(): return real_counter() + (60 if expired.is_set() else 0) with socket.socket() as listener: - listener.bind(('127.0.0.1', 0)) + listener.bind(("127.0.0.1", 0)) listener.listen(1) listener.settimeout(10) - descendant = ('import socket; ' - f's=socket.create_connection({listener.getsockname()!r}); ' - 's.sendall(b"ready"); s.recv(1)') - code = ('import subprocess, sys; ' - f'p=subprocess.Popen([sys.executable, "-c", {descendant!r}]); p.wait()') - with mock.patch.object(regression.time, 'perf_counter', side_effect=controlled_counter), \ - mock.patch.object(regression, 'stop_process', wraps=regression.stop_process) as cleanup, \ - concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: - future = pool.submit(regression.execute, [sys.executable, '-c', code], - self.root, os.environ.copy(), 30, - self.root / 'stdout.log', self.root / 'stderr.log') + descendant = ( + f'import socket; s=socket.create_connection({listener.getsockname()!r}); s.sendall(b"ready"); s.recv(1)' + ) + code = f'import subprocess, sys; p=subprocess.Popen([sys.executable, "-c", {descendant!r}]); p.wait()' + with ( + mock.patch.object(regression.time, "perf_counter", side_effect=controlled_counter), + mock.patch.object(regression, "stop_process", wraps=regression.stop_process) as cleanup, + concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool, + ): + future = pool.submit( + regression.execute, + [sys.executable, "-c", code], + self.root, + os.environ.copy(), + 30, + self.root / "stdout.log", + self.root / "stderr.log", + ) try: connection, _ = listener.accept() with connection: connection.settimeout(10) - self.assertEqual(connection.recv(5, socket.MSG_WAITALL), b'ready') + self.assertEqual(connection.recv(5, socket.MSG_WAITALL), b"ready") expired.set() result = future.result(timeout=10) self.assertEqual(cleanup.call_count, 1) - self.assertIn('Timed out', result[2] or '', repr(result)) + self.assertIn("Timed out", result[2] or "", repr(result)) self.assertNotEqual(result[0], 0, repr(result)) - self.assertEqual(connection.recv(1), b'', 'Descendant still holds its socket open') + self.assertEqual(connection.recv(1), b"", "Descendant still holds its socket open") finally: expired.set() def test_malformed_baseline_array_has_concise_error(self): self.suite(self.scenario()) - baseline = self.root / 'bad baseline.json' - baseline.write_text('[]') - child, report, output = self.run_cli('--baseline', baseline) + baseline = self.root / "bad baseline.json" + baseline.write_text("[]") + child, report, output = self.run_cli("--baseline", baseline) self.assert_exit(child, 2) - self.assertNotIn('Traceback', child.stderr) + self.assertNotIn("Traceback", child.stderr) self.assertIsNone(report) self.assertFalse(output.exists()) def test_crlf_anchored_fingerprints_are_stable_across_attempts_and_baseline(self): # Write bytes so the fixture produces CRLF on every host without Python # text-mode newline translation changing the intended Windows input. - self.suite(self.scenario( - code='import sys; sys.stdout.buffer.write(b"STATE=stable\\r\\nDONE\\r\\n")', - fingerprints=['(?m)^STATE=stable$'], expect_stdout=['STATE=stable'])) - options = ['--warmup', '1', '--repeat', '2'] + self.suite( + self.scenario( + code='import sys; sys.stdout.buffer.write(b"STATE=stable\\r\\nDONE\\r\\n")', + fingerprints=["(?m)^STATE=stable$"], + expect_stdout=["STATE=stable"], + ) + ) + options = ["--warmup", "1", "--repeat", "2"] child, report, baseline = self.run_cli(*options) self.assert_exit(child, 0) - result = report['scenarios'][0] - self.assertEqual(len(result['attempts']), 3) - self.assertEqual(result['fingerprints'], {'(?m)^STATE=stable$': ['STATE=stable']}) - self.assertTrue(all(attempt['fingerprints'] == result['fingerprints'] for attempt in result['attempts'])) - self.assertIn(b'\r\n', (baseline / result['attempts'][0]['stdout']).read_bytes()) - child, report, _ = self.run_cli(*options, '--baseline', baseline / 'report.json') + result = report["scenarios"][0] + self.assertEqual(len(result["attempts"]), 3) + self.assertEqual(result["fingerprints"], {"(?m)^STATE=stable$": ["STATE=stable"]}) + self.assertTrue(all(attempt["fingerprints"] == result["fingerprints"] for attempt in result["attempts"])) + self.assertIn(b"\r\n", (baseline / result["attempts"][0]["stdout"]).read_bytes()) + child, report, _ = self.run_cli(*options, "--baseline", baseline / "report.json") self.assert_exit(child, 0) - self.assertEqual(report['scenarios'][0]['fingerprints'], result['fingerprints']) + self.assertEqual(report["scenarios"][0]["fingerprints"], result["fingerprints"]) def test_bad_schema_is_rejected_without_output(self): cases = [ - {'schema_version': 1, 'name': 'bad', 'scenarios': [self.scenario('../escape')]}, - {'schema_version': 1, 'name': 'bad', 'scenarios': [self.scenario('same'), self.scenario('same')]}, - {'schema_version': 1, 'name': 'bad', 'scenarios': [{'id': 'shell', 'command': 'echo unsafe'}]}, - {'schema_version': 1, 'name': 'bad', 'scenarios': [self.scenario(timeout_seconds=float('nan'))]}, - {'schema_version': 1, 'name': 'bad', 'scenarios': [self.scenario(unrecognized=True)]}, - {'schema_version': 1, 'name': 'bad', 'scenarios': [self.scenario(fingerprints=['[invalid'])]}, + {"schema_version": 1, "name": "bad", "scenarios": [self.scenario("../escape")]}, + {"schema_version": 1, "name": "bad", "scenarios": [self.scenario("same"), self.scenario("same")]}, + {"schema_version": 1, "name": "bad", "scenarios": [{"id": "shell", "command": "echo unsafe"}]}, + {"schema_version": 1, "name": "bad", "scenarios": [self.scenario(timeout_seconds=float("nan"))]}, + {"schema_version": 1, "name": "bad", "scenarios": [self.scenario(unrecognized=True)]}, + {"schema_version": 1, "name": "bad", "scenarios": [self.scenario(fingerprints=["[invalid"])]}, ] for suite in cases: with self.subTest(suite=suite): @@ -483,20 +578,24 @@ def test_bad_schema_is_rejected_without_output(self): self.assert_exit(child, 2) self.assertIsNone(report) self.assertFalse(output.exists()) - self.assertNotIn('Traceback', child.stderr) + self.assertNotIn("Traceback", child.stderr) def test_bad_metrics_and_unstable_fingerprints_fail_as_reports(self): self.suite( - self.scenario('reserved', "print('DKR_HARNESS {\"metrics\": {\"process.median_ms\": 0}}')"), - self.scenario('nonfinite', "print('DKR_HARNESS {\"metrics\": {\"stage_ms\": NaN}}')"), - self.scenario('missingmetric', required_metrics=['expected_ms']), - self.scenario('unstable', "from pathlib import Path; p=Path('counter'); n=int(p.read_text())+1 if p.exists() else 1; p.write_text(str(n)); print('STATE='+str(n))", fingerprints=['STATE=\\d+']), + self.scenario("reserved", 'print(\'DKR_HARNESS {"metrics": {"process.median_ms": 0}}\')'), + self.scenario("nonfinite", 'print(\'DKR_HARNESS {"metrics": {"stage_ms": NaN}}\')'), + self.scenario("missingmetric", required_metrics=["expected_ms"]), + self.scenario( + "unstable", + "from pathlib import Path; p=Path('counter'); n=int(p.read_text())+1 if p.exists() else 1; p.write_text(str(n)); print('STATE='+str(n))", + fingerprints=["STATE=\\d+"], + ), ) - child, report, _ = self.run_cli('--repeat', '2') + child, report, _ = self.run_cli("--repeat", "2") self.assert_exit(child, 1) - self.assertEqual(report['summary'], {'passed': 0, 'failed': 4, 'skipped': 0}) - self.assertTrue(any('fingerprint' in error.lower() for error in report['scenarios'][-1]['errors'])) + self.assertEqual(report["summary"], {"passed": 0, "failed": 4, "skipped": 0}) + self.assertTrue(any("fingerprint" in error.lower() for error in report["scenarios"][-1]["errors"])) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/runtime-recomp/tests/replay_capture_tests.py b/runtime-recomp/tests/replay_capture_tests.py index 68275656..3be3408e 100644 --- a/runtime-recomp/tests/replay_capture_tests.py +++ b/runtime-recomp/tests/replay_capture_tests.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Private capture orchestration tests; no ROM or game-derived fixture data.""" + import json import os from pathlib import Path @@ -8,107 +9,114 @@ import unittest from unittest.mock import patch -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'scripts')) +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "scripts")) import capture_replay_fixture as capture class ReplayCaptureTests(unittest.TestCase): def setUp(self): - temporary = tempfile.TemporaryDirectory(prefix='dkr capture tests ') + temporary = tempfile.TemporaryDirectory(prefix="dkr capture tests ") self.addCleanup(temporary.cleanup) self.root = Path(temporary.name).resolve() - self.runtime = self.root / 'runtime' - self.runtime.write_text('synthetic executable') + self.runtime = self.root / "runtime" + self.runtime.write_text("synthetic executable") self.runtime.chmod(0o700) - self.rom = self.root / 'synthetic.z64' - self.rom.write_text('not a ROM') - self.output = self.root / 'capture' + self.rom = self.root / "synthetic.z64" + self.rom.write_text("not a ROM") + self.output = self.root / "capture" self.original_hash = capture.file_hash - self.hash_patch = patch.object(capture, 'file_hash', side_effect=self.hash_input) + self.hash_patch = patch.object(capture, "file_hash", side_effect=self.hash_input) self.hash_patch.start() self.addCleanup(self.hash_patch.stop) - def hash_input(self, path, algorithm='sha256'): - if path == self.rom and algorithm == 'sha1': + def hash_input(self, path, algorithm="sha256"): + if path == self.rom and algorithm == "sha1": return next(iter(capture.ROM_REVISIONS)) return self.original_hash(path, algorithm) def child(self, command, cwd, env, timeout, stdout, stderr): self.assertEqual(timeout, 90) - self.assertEqual(command, [str(self.runtime), '--rom', str(self.rom), '--config', - str(self.output / 'profile'), '--timeout', '60']) - self.assertEqual({key for key in env if key.startswith('DKR_')}, { - 'DKR_REPLAY_CAPTURE_FILE', 'DKR_REPLAY_CAPTURE_MAP', 'DKR_REPLAY_CAPTURE_PLAYERS'}) - self.assertEqual(env['DKR_REPLAY_CAPTURE_PLAYERS'], '2') - self.assertEqual(env['DKR_REPLAY_CAPTURE_MAP'], '3') - Path(env['DKR_REPLAY_CAPTURE_FILE']).write_bytes(b'synthetic fixture') - self.log = cwd / 'profile' / 'logs' / 'runtime.log' + self.assertEqual( + command, + [str(self.runtime), "--rom", str(self.rom), "--config", str(self.output / "profile"), "--timeout", "60"], + ) + self.assertEqual( + {key for key in env if key.startswith("DKR_")}, + {"DKR_REPLAY_CAPTURE_FILE", "DKR_REPLAY_CAPTURE_MAP", "DKR_REPLAY_CAPTURE_PLAYERS"}, + ) + self.assertEqual(env["DKR_REPLAY_CAPTURE_PLAYERS"], "2") + self.assertEqual(env["DKR_REPLAY_CAPTURE_MAP"], "3") + Path(env["DKR_REPLAY_CAPTURE_FILE"]).write_bytes(b"synthetic fixture") + self.log = cwd / "profile" / "logs" / "runtime.log" self.log.parent.mkdir(parents=True) - self.log.write_text('[replay-probe][quiesced] revision=77 bytes=17 success=1\n') - stdout.write_text('') - stderr.write_text('') - return 0, 100.0, '', False + self.log.write_text("[replay-probe][quiesced] revision=77 bytes=17 success=1\n") + stdout.write_text("") + stderr.write_text("") + return 0, 100.0, "", False def run_capture(self, child=None): - with patch.object(capture, 'execute', side_effect=child or self.child): + with patch.object(capture, "execute", side_effect=child or self.child): return capture.capture(self.runtime, self.rom, self.output) def test_success_and_environment_isolation(self): - with patch.dict(os.environ, {'DKR_WATER_TEST_MAP': '5', 'DKR_PROBE_REFERENCE_OUTPUT': 'unsafe'}): + with patch.dict(os.environ, {"DKR_WATER_TEST_MAP": "5", "DKR_PROBE_REFERENCE_OUTPUT": "unsafe"}): manifest = self.run_capture() metadata = json.loads(manifest.read_text()) - self.assertEqual(metadata['revision'], 77) - self.assertEqual(metadata['fixture_bytes'], 17) - self.assertEqual(metadata['fixture_sha256'], self.original_hash(self.output / 'race.dkr-component')) - self.assertNotIn('synthetic fixture', manifest.read_text()) + self.assertEqual(metadata["revision"], 77) + self.assertEqual(metadata["fixture_bytes"], 17) + self.assertEqual(metadata["fixture_sha256"], self.original_hash(self.output / "race.dkr-component")) + self.assertNotIn("synthetic fixture", manifest.read_text()) def test_wrong_rom_rejected_before_writes(self): - with patch.object(capture, 'file_hash', return_value='wrong'), patch.object(capture, 'execute') as child: - with self.assertRaisesRegex(ValueError, 'SHA-1'): + with patch.object(capture, "file_hash", return_value="wrong"), patch.object(capture, "execute") as child: + with self.assertRaisesRegex(ValueError, "SHA-1"): capture.capture(self.runtime, self.rom, self.output) child.assert_not_called() self.assertFalse(self.output.exists()) def test_missing_runtime_rejected_before_writes(self): self.runtime.unlink() - with self.assertRaisesRegex(ValueError, 'executable'): + with self.assertRaisesRegex(ValueError, "executable"): self.run_capture() self.assertFalse(self.output.exists()) def test_existing_output_preserved(self): self.output.mkdir() - marker = self.output / 'keep' - marker.write_text('unchanged') + marker = self.output / "keep" + marker.write_text("unchanged") with self.assertRaises(FileExistsError): self.run_capture() - self.assertEqual(marker.read_text(), 'unchanged') + self.assertEqual(marker.read_text(), "unchanged") self.assertEqual(list(self.output.iterdir()), [marker]) def assert_failed_child(self, mutate, message): def child(*args): result = self.child(*args) return mutate(result) or result + with self.assertRaisesRegex(RuntimeError, message): self.run_capture(child) - self.assertFalse((self.output / 'fixture.json').exists()) + self.assertFalse((self.output / "fixture.json").exists()) self.assertTrue(self.log.exists()) def test_missing_success_marker(self): - self.assert_failed_child(lambda _: self.log.write_text('not a capture\n') and None, 'quiescence') + self.assert_failed_child(lambda _: self.log.write_text("not a capture\n") and None, "quiescence") def test_wrong_revision(self): - self.assert_failed_child(lambda _: self.log.write_text( - '[replay-probe][quiesced] revision=80 bytes=17 success=1\n') and None, 'revision 77') + self.assert_failed_child( + lambda _: self.log.write_text("[replay-probe][quiesced] revision=80 bytes=17 success=1\n") and None, + "revision 77", + ) def test_truncated_fixture(self): - self.assert_failed_child(lambda _: (self.output / 'race.dkr-component').write_bytes(b'x') and None, 'size') + self.assert_failed_child(lambda _: (self.output / "race.dkr-component").write_bytes(b"x") and None, "size") def test_nonzero_exit_despite_success_marker(self): - self.assert_failed_child(lambda _: (5, 100, '', False), 'exited 5') + self.assert_failed_child(lambda _: (5, 100, "", False), "exited 5") def test_watchdog_failure_despite_success_marker(self): - self.assert_failed_child(lambda _: (-15, 90000, 'Timed out after 90 seconds', False), 'Timed out') + self.assert_failed_child(lambda _: (-15, 90000, "Timed out after 90 seconds", False), "Timed out") -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/runtime-recomp/tests/source_packaging_tests.py b/runtime-recomp/tests/source_packaging_tests.py index 028557cb..2ae10016 100644 --- a/runtime-recomp/tests/source_packaging_tests.py +++ b/runtime-recomp/tests/source_packaging_tests.py @@ -1,4 +1,5 @@ """Exercise archive publication with synthetic source trees only.""" + import contextlib import importlib.util import io @@ -20,76 +21,107 @@ def load(relative): class PackagingTests(unittest.TestCase): def test_new_source_modules_and_excluded_source_diagnostic(self): - module = load('scripts/package_source.py') + module = load("scripts/package_source.py") with tempfile.TemporaryDirectory() as directory: root = Path(directory) - paths = [Path(p) for p in ('runtime-recomp/src/new/module.cpp', 'runtime-recomp/include/new.hpp', - 'tools/blender/track_lab/new.py', 'downloads/new.py', 'tools/blender/local.zip')] + paths = [ + Path(p) + for p in ( + "runtime-recomp/src/new/module.cpp", + "runtime-recomp/include/new.hpp", + "tools/blender/track_lab/new.py", + "downloads/new.py", + "tools/blender/local.zip", + ) + ] for path in paths: - (root/path).parent.mkdir(parents=True, exist_ok=True) - (root/path).write_text('synthetic') - with mock.patch.object(module, 'ROOT', root), mock.patch.object(module, 'git_paths', side_effect=[[], paths]), \ - contextlib.redirect_stderr(io.StringIO()) as err: + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_text("synthetic") + with ( + mock.patch.object(module, "ROOT", root), + mock.patch.object(module, "git_paths", side_effect=[[], paths]), + contextlib.redirect_stderr(io.StringIO()) as err, + ): self.assertEqual(set(module.source_files()), set(paths[:3])) - self.assertIn(str(Path('downloads/new.py')), err.getvalue()) + self.assertIn(str(Path("downloads/new.py")), err.getvalue()) def test_source_success_is_deterministic_and_no_clobber(self): - module = load('scripts/package_source.py') + module = load("scripts/package_source.py") with tempfile.TemporaryDirectory() as directory: root = Path(directory) - (root/'module.py').write_text('value = 1\n') - with mock.patch.object(module, 'ROOT', root), mock.patch.object(module, 'source_files', return_value=[Path('module.py')]), \ - contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): - self.assertEqual(module.main([str(root/'one.zip')]), 0) - self.assertEqual(module.main([str(root/'two.zip')]), 0) - self.assertEqual((root/'one.zip').read_bytes(), (root/'two.zip').read_bytes()) - self.assertEqual(module.main([str(root/'one.zip')]), 2) + (root / "module.py").write_text("value = 1\n") + with ( + mock.patch.object(module, "ROOT", root), + mock.patch.object(module, "source_files", return_value=[Path("module.py")]), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual(module.main([str(root / "one.zip")]), 0) + self.assertEqual(module.main([str(root / "two.zip")]), 0) + self.assertEqual((root / "one.zip").read_bytes(), (root / "two.zip").read_bytes()) + self.assertEqual(module.main([str(root / "one.zip")]), 2) def test_source_validation_failure_leaves_no_artifact(self): - module = load('scripts/package_source.py') + module = load("scripts/package_source.py") with tempfile.TemporaryDirectory() as directory: root = Path(directory) - (root/'bad.py').write_bytes(b'\x80\x37\x12\x40') - with mock.patch.object(module, 'ROOT', root), mock.patch.object(module, 'source_files', return_value=[Path('bad.py')]), \ - contextlib.redirect_stderr(io.StringIO()): - self.assertEqual(module.main([str(root/'out.zip')]), 2) - self.assertEqual(sorted(p.name for p in root.iterdir()), ['bad.py']) + (root / "bad.py").write_bytes(b"\x80\x37\x12\x40") + with ( + mock.patch.object(module, "ROOT", root), + mock.patch.object(module, "source_files", return_value=[Path("bad.py")]), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual(module.main([str(root / "out.zip")]), 2) + self.assertEqual(sorted(p.name for p in root.iterdir()), ["bad.py"]) def test_source_publish_race_preserves_existing_artifact(self): - module = load('scripts/package_source.py') + module = load("scripts/package_source.py") with tempfile.TemporaryDirectory() as directory: root = Path(directory) - output = root/'out.zip' + output = root / "out.zip" + def validate(path): - output.write_bytes(b'concurrent archive') - with mock.patch.object(module, 'create_archive'), mock.patch.object(module, 'validate_archive', side_effect=validate), \ - contextlib.redirect_stderr(io.StringIO()): + output.write_bytes(b"concurrent archive") + + with ( + mock.patch.object(module, "create_archive"), + mock.patch.object(module, "validate_archive", side_effect=validate), + contextlib.redirect_stderr(io.StringIO()), + ): self.assertEqual(module.main([str(output)]), 2) - self.assertEqual(output.read_bytes(), b'concurrent archive') + self.assertEqual(output.read_bytes(), b"concurrent archive") self.assertEqual(list(root.iterdir()), [output]) def test_addon_reproducible_and_failure_preserves_previous(self): - module = load('tools/blender/package_addon.py') + module = load("tools/blender/package_addon.py") with tempfile.TemporaryDirectory() as directory: root = Path(directory) - package = root/'package' - (package/'data').mkdir(parents=True) - (package/'data/catalog.json').write_text('{}') - (package/'module.py').write_text('x = 1\n') - output = root/'out' - with mock.patch.object(module, 'PACKAGE_DIR', str(package)), contextlib.redirect_stdout(io.StringIO()): - self.assertEqual(module.main(['--output', str(output)]), 0) - archive = output/'track_lab.zip' + package = root / "package" + (package / "data").mkdir(parents=True) + (package / "data/catalog.json").write_text("{}") + (package / "module.py").write_text("x = 1\n") + output = root / "out" + with mock.patch.object(module, "PACKAGE_DIR", str(package)), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(module.main(["--output", str(output)]), 0) + archive = output / "track_lab.zip" first = archive.read_bytes() - self.assertEqual(module.main(['--output', str(output)]), 0) + self.assertEqual(module.main(["--output", str(output)]), 0) self.assertEqual(first, archive.read_bytes()) with zipfile.ZipFile(archive) as zipped: - self.assertTrue(all(e.date_time == (1980,1,1,0,0,0) and e.external_attr >> 16 == 0o100644 for e in zipped.infolist())) - with mock.patch.object(module, 'iter_files', side_effect=OSError('read failed')), contextlib.redirect_stderr(io.StringIO()): - self.assertEqual(module.main(['--output', str(output)]), 1) + self.assertTrue( + all( + e.date_time == (1980, 1, 1, 0, 0, 0) and e.external_attr >> 16 == 0o100644 + for e in zipped.infolist() + ) + ) + with ( + mock.patch.object(module, "iter_files", side_effect=OSError("read failed")), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual(module.main(["--output", str(output)]), 1) self.assertEqual(first, archive.read_bytes()) self.assertEqual(list(output.iterdir()), [archive]) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/runtime-recomp/tests/stress_runner_tests.py b/runtime-recomp/tests/stress_runner_tests.py index 6c0f3099..d729d57b 100644 --- a/runtime-recomp/tests/stress_runner_tests.py +++ b/runtime-recomp/tests/stress_runner_tests.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 """Exercise the stress runner with real, ROM-free CTest fixtures.""" + import argparse import contextlib import io import json +from pathlib import Path import re import shutil -from pathlib import Path import subprocess import sys import tempfile @@ -15,14 +16,14 @@ import xml.etree.ElementTree as ET ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / 'scripts')) +sys.path.insert(0, str(ROOT / "scripts")) import stress_tests as runner def ctest_supported(): try: - text = subprocess.check_output(['ctest', '--version'], text=True) - match = re.search(r'(\d+)\.(\d+)\.(\d+)', text) + text = subprocess.check_output(["ctest", "--version"], text=True) + match = re.search(r"(\d+)\.(\d+)\.(\d+)", text) return bool(match and tuple(map(int, match.groups())) >= (4, 1, 0)) except (OSError, subprocess.SubprocessError): return False @@ -33,137 +34,167 @@ def ctest_supported(): class StressTests(unittest.TestCase): def setUp(self): - self.temp = tempfile.TemporaryDirectory(prefix='dkr stress ') + self.temp = tempfile.TemporaryDirectory(prefix="dkr stress ") self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) - self.build = self.root / 'build' + self.build = self.root / "build" self.build.mkdir() - self.output = self.root / 'results' + self.output = self.root / "results" def fixture(self, fail=False, disabled=False): # Use the actual installed CMake executable, portable across platforms. - executable = Path(shutil.which('cmake')).as_posix() - command = 'false' if fail else 'true' - (self.build / 'CTestTestfile.cmake').write_text( + executable = Path(shutil.which("cmake")).as_posix() + command = "false" if fail else "true" + (self.build / "CTestTestfile.cmake").write_text( f'add_test(example "{executable}" -E {command})\n' 'set_tests_properties(example PROPERTIES LABELS "flake")\n' - + ('set_tests_properties(example PROPERTIES DISABLED TRUE)\n' if disabled else '')) + + ("set_tests_properties(example PROPERTIES DISABLED TRUE)\n" if disabled else "") + ) def invoke(self, *options): - return subprocess.run([sys.executable, str(ROOT / 'scripts/stress_tests.py'), - '--build-dir', str(self.build), '--output', str(self.output), - '--seed', '100', '--rounds', '2', '--test-regex', 'example', *options], - capture_output=True, text=True) - - @unittest.skipUnless(CTEST_SUPPORTED, 'Real seeded CTest fixtures require CTest >= 4.1') + return subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/stress_tests.py"), + "--build-dir", + str(self.build), + "--output", + str(self.output), + "--seed", + "100", + "--rounds", + "2", + "--test-regex", + "example", + *options, + ], + capture_output=True, + text=True, + ) + + @unittest.skipUnless(CTEST_SUPPORTED, "Real seeded CTest fixtures require CTest >= 4.1") def test_real_pass_rounds_preserve_and_merge(self): self.fixture() result = self.invoke() self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - data = json.loads((self.output / 'manifest.json').read_text()) - self.assertEqual([r['seed'] for r in data['rounds']], [100, 101]) - self.assertEqual(data['selected_tests'], ['example']) - self.assertEqual(data['config'], 'Release') - self.assertEqual(Path(data['build_dir']), self.build.resolve()) - discovery = json.loads((self.output / 'ctest-tests.json').read_text()) - self.assertEqual(discovery['tests'][0]['name'], 'example') - self.assertTrue(discovery['tests'][0]['command']) - self.assertTrue(discovery['tests'][0]['properties']) - cases = list(ET.parse(self.output / 'junit.xml').iter('testcase')) - self.assertEqual([c.get('name') for c in cases], ['round-001: example', 'round-002: example']) + data = json.loads((self.output / "manifest.json").read_text()) + self.assertEqual([r["seed"] for r in data["rounds"]], [100, 101]) + self.assertEqual(data["selected_tests"], ["example"]) + self.assertEqual(data["config"], "Release") + self.assertEqual(Path(data["build_dir"]), self.build.resolve()) + discovery = json.loads((self.output / "ctest-tests.json").read_text()) + self.assertEqual(discovery["tests"][0]["name"], "example") + self.assertTrue(discovery["tests"][0]["command"]) + self.assertTrue(discovery["tests"][0]["properties"]) + cases = list(ET.parse(self.output / "junit.xml").iter("testcase")) + self.assertEqual([c.get("name") for c in cases], ["round-001: example", "round-002: example"]) for number in (1, 2): - folder = self.output / f'round-{number:03d}' - self.assertTrue((folder / 'ctest.log').is_file()) - self.assertTrue((folder / 'CTestTemporary' / 'LastTest.log').is_file()) - self.assertIn('2 passed', (self.output / 'summary.md').read_text()) - self.assertNotEqual(self.invoke().returncode, 0, 'must not overwrite evidence') + folder = self.output / f"round-{number:03d}" + self.assertTrue((folder / "ctest.log").is_file()) + self.assertTrue((folder / "CTestTemporary" / "LastTest.log").is_file()) + self.assertIn("2 passed", (self.output / "summary.md").read_text()) + self.assertNotEqual(self.invoke().returncode, 0, "must not overwrite evidence") - @unittest.skipUnless(CTEST_SUPPORTED, 'Real seeded CTest fixtures require CTest >= 4.1') + @unittest.skipUnless(CTEST_SUPPORTED, "Real seeded CTest fixtures require CTest >= 4.1") def test_failure_stops_without_retry(self): self.fixture(fail=True) result = self.invoke() self.assertEqual(result.returncode, 1, result.stdout + result.stderr) - data = json.loads((self.output / 'manifest.json').read_text()) - self.assertFalse(data['passed']) - self.assertEqual(len(data['rounds']), 1) - self.assertFalse((self.output / 'round-002').exists()) - self.assertIn('1 failed', (self.output / 'summary.md').read_text()) + data = json.loads((self.output / "manifest.json").read_text()) + self.assertFalse(data["passed"]) + self.assertEqual(len(data["rounds"]), 1) + self.assertFalse((self.output / "round-002").exists()) + self.assertIn("1 failed", (self.output / "summary.md").read_text()) - @unittest.skipUnless(CTEST_SUPPORTED, 'Real seeded CTest fixtures require CTest >= 4.1') + @unittest.skipUnless(CTEST_SUPPORTED, "Real seeded CTest fixtures require CTest >= 4.1") def test_disabled_not_false_pass(self): self.fixture(disabled=True) self.assertEqual(self.invoke().returncode, 1) - @unittest.skipUnless(CTEST_SUPPORTED, 'Real seeded CTest fixtures require CTest >= 4.1') + @unittest.skipUnless(CTEST_SUPPORTED, "Real seeded CTest fixtures require CTest >= 4.1") def test_empty_selection_and_list(self): self.fixture() - result = self.invoke('--test-regex', '^absent$') + result = self.invoke("--test-regex", "^absent$") self.assertEqual(result.returncode, 2) self.assertFalse(self.output.exists()) - result = self.invoke('--list') + result = self.invoke("--list") self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn('example', result.stdout) + self.assertIn("example", result.stdout) self.assertFalse(self.output.exists()) - @unittest.skipUnless(CTEST_SUPPORTED, 'Real seeded CTest fixtures require CTest >= 4.1') + @unittest.skipUnless(CTEST_SUPPORTED, "Real seeded CTest fixtures require CTest >= 4.1") def test_missing_default_core_suite(self): self.fixture() - result = self.invoke('--test-regex', '') + result = self.invoke("--test-regex", "") self.assertEqual(result.returncode, 2) - self.assertIn('Default flake suite is incomplete', result.stderr) + self.assertIn("Default flake suite is incomplete", result.stderr) self.assertFalse(self.output.exists()) def test_seed_validation(self): - for seed in ('-1', '4294967296', '4294967295', 'word'): - self.assertEqual(self.invoke('--seed', seed).returncode, 2, seed) + for seed in ("-1", "4294967296", "4294967295", "word"): + self.assertEqual(self.invoke("--seed", seed).returncode, 2, seed) self.assertFalse(self.output.exists()) - @unittest.skipUnless(CTEST_SUPPORTED, 'Real seeded CTest fixtures require CTest >= 4.1') + @unittest.skipUnless(CTEST_SUPPORTED, "Real seeded CTest fixtures require CTest >= 4.1") def test_missing_executable_preflight(self): - (self.build / 'CTestTestfile.cmake').write_text( + (self.build / "CTestTestfile.cmake").write_text( 'add_test(example "definitely_missing_dkr_fixture_executable")\n' - 'set_tests_properties(example PROPERTIES LABELS "flake")\n') + 'set_tests_properties(example PROPERTIES LABELS "flake")\n' + ) result = self.invoke() self.assertEqual(result.returncode, 2) - self.assertIn('missing executables', result.stderr) + self.assertIn("missing executables", result.stderr) self.assertFalse(self.output.exists()) - @unittest.skipUnless(CTEST_SUPPORTED, 'Real seeded CTest fixtures require CTest >= 4.1') + @unittest.skipUnless(CTEST_SUPPORTED, "Real seeded CTest fixtures require CTest >= 4.1") def test_missing_and_malformed_real_reports_fail_with_evidence(self): self.fixture() collect = runner.collect_junit for malformed in (False, True): output = self.root / str(malformed) + def damage(path, selected, number): if malformed: - path.write_text('', '', - '', - ''): + runner.collect_junit(path, ["example"], 1) + for value in ( + "", + "", + '', + '', + ): path.write_text(value) with self.assertRaises((ValueError, ET.ParseError), msg=value): - runner.collect_junit(path, ['example'], 1) + runner.collect_junit(path, ["example"], 1) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/runtime-recomp/tests/test_startup_flow_pipeline.py b/runtime-recomp/tests/test_startup_flow_pipeline.py index d77a7ae1..a459f905 100644 --- a/runtime-recomp/tests/test_startup_flow_pipeline.py +++ b/runtime-recomp/tests/test_startup_flow_pipeline.py @@ -1,11 +1,12 @@ """No-ROM signature/bounds/conflict regressions for the intro-skip pipeline.""" + +from pathlib import Path +import struct import sys import unittest -from pathlib import Path from unittest.mock import patch -import struct -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'scripts')) +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "scripts")) import startup_flow_policy as pipeline @@ -14,40 +15,49 @@ def compose(self, revision, *, opcode_changed=False, outside=False, bad_store=Fa sites = pipeline.SITES[revision] symbols = {} for name, address, _, _ in sites: - if name not in symbols: symbols[name] = {(address+4 if outside else address-4,16)} + if name not in symbols: + symbols[name] = {(address + 4 if outside else address - 4, 16)} elif not outside: - begin=next(iter(symbols[name]))[0] - symbols[name]={(begin,address+16-begin)} + begin = next(iter(symbols[name]))[0] + symbols[name] = {(begin, address + 16 - begin)} sections = [] for name, address, word, _ in sites: - sections.append((address - 4, struct.pack('>II', 0 if bad_store else 0xAC820000, word ^ int(opcode_changed)))) - with patch.object(pipeline, 'reviewed_revision', return_value=revision), \ - patch.object(pipeline, 'elf_functions', return_value=symbols), \ - patch.object(pipeline, 'elf_sections', return_value=sections): - return pipeline.compose_startup_flow(policy or {}, Path('not-a-ROM'), revision) + sections.append( + (address - 4, struct.pack(">II", 0 if bad_store else 0xAC820000, word ^ int(opcode_changed))) + ) + with ( + patch.object(pipeline, "reviewed_revision", return_value=revision), + patch.object(pipeline, "elf_functions", return_value=symbols), + patch.object(pipeline, "elf_sections", return_value=sections), + ): + return pipeline.compose_startup_flow(policy or {}, Path("not-a-ROM"), revision) def test_both_revisions_and_idempotence(self): for revision in pipeline.SITES: result = self.compose(revision) - self.assertEqual(len(result['functionHooks']), 4) + self.assertEqual(len(result["functionHooks"]), 4) self.assertEqual(result, self.compose(revision, policy=result)) def test_opcode_and_function_bounds_fail_closed(self): for revision in pipeline.SITES: - with self.assertRaises(ValueError): self.compose(revision, opcode_changed=True) - with self.assertRaises(ValueError): self.compose(revision, outside=True) - with self.assertRaises(ValueError): self.compose(revision, bad_store=True) + with self.assertRaises(ValueError): + self.compose(revision, opcode_changed=True) + with self.assertRaises(ValueError): + self.compose(revision, outside=True) + with self.assertRaises(ValueError): + self.compose(revision, bad_store=True) def test_conflicting_hook_is_rejected(self): - result = self.compose('us.v77') - result['functionHooks'][0]['text'] = 'wrong callback' - with self.assertRaises(ValueError): self.compose('us.v77', policy=result) + result = self.compose("us.v77") + result["functionHooks"][0]["text"] = "wrong callback" + with self.assertRaises(ValueError): + self.compose("us.v77", policy=result) def test_unreviewed_image_is_rejected(self): - with patch.object(pipeline, 'reviewed_revision', return_value=None): + with patch.object(pipeline, "reviewed_revision", return_value=None): with self.assertRaises(ValueError): - pipeline.compose_startup_flow({}, Path('not-a-ROM'), 'us.v77') + pipeline.compose_startup_flow({}, Path("not-a-ROM"), "us.v77") -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/runtime-recomp/tests/test_track_select_draw_pipeline.py b/runtime-recomp/tests/test_track_select_draw_pipeline.py index c68c4e67..a6f9d086 100644 --- a/runtime-recomp/tests/test_track_select_draw_pipeline.py +++ b/runtime-recomp/tests/test_track_select_draw_pipeline.py @@ -1,67 +1,99 @@ """Read-only checks of the scoped draw Patch Pipeline on both reviewed revisions.""" + import copy import json import os from pathlib import Path import sys import unittest -ROOT=Path(__file__).resolve().parents[2] -sys.path.insert(0,str(ROOT/'scripts')) -from compose_legacy_mod_policy import elf_functions,elf_sections,compose_track_menu + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts")) +from compose_legacy_mod_policy import compose_track_menu, elf_functions, elf_sections from track_select_draw_policy import compose_track_draw + class TrackDrawPipeline(unittest.TestCase): - def inputs(self,revision): - default=ROOT/'extern/dkr-decomp/build/dkr.us.v77.elf' if revision==77 else Path( - 'G:/DiddykongWorkFolder/hud-fit-viewport-20260824/dkr-decomp-v80/build/dkr.us.v80.elf') - elf=Path(os.environ.get(f'DKR_REVIEWED_ELF_V{revision}',default)) - if not elf.exists():self.skipTest('Reviewed ELF unavailable') - return elf,elf_sections(elf),elf_functions(elf,(1,2)) + def inputs(self, revision): + default = ( + ROOT / "extern/dkr-decomp/build/dkr.us.v77.elf" + if revision == 77 + else Path("G:/DiddykongWorkFolder/hud-fit-viewport-20260824/dkr-decomp-v80/build/dkr.us.v80.elf") + ) + elf = Path(os.environ.get(f"DKR_REVIEWED_ELF_V{revision}", default)) + if not elf.exists(): + self.skipTest("Reviewed ELF unavailable") + return elf, elf_sections(elf), elf_functions(elf, (1, 2)) + def test_signed_draw_sites_and_stack_table_are_bounded(self): - for revision in (77,80): - elf,sections,symbols=self.inputs(revision) - result=compose_track_draw({'instructionPatches':[],'functionHooks':[]},elf,f'us.v{revision}',sections,symbols) - self.assertEqual(len(result['instructionPatches']),13) - self.assertEqual(len(result['functionHooks']),13) - self.assertEqual({p['function'] for p in result['instructionPatches']}, - {'trackmenu_render_names','trackmenu_render_2D','texrect_draw_scaled','texrect_draw'}) - table=[p for p in result['instructionPatches'] if p['function']=='trackmenu_render_names'] - self.assertEqual({int(p['value'],0) for p in table}, - {0x27BDFE80,0x27BD0180,0x27B00090,0x27B30180,0x2416FFFE,0x24010003}) - self.assertEqual(sum(int(p['value'],0)==0x27B00090 for p in table),2) - self.assertEqual(sum('9U' in h['text'] for h in result['functionHooks']),1) - original=copy.deepcopy(result) - with self.assertRaises(ValueError):compose_track_draw(result,elf,f'us.v{revision}',sections,symbols) - self.assertEqual(result,original) + for revision in (77, 80): + elf, sections, symbols = self.inputs(revision) + result = compose_track_draw( + {"instructionPatches": [], "functionHooks": []}, elf, f"us.v{revision}", sections, symbols + ) + self.assertEqual(len(result["instructionPatches"]), 13) + self.assertEqual(len(result["functionHooks"]), 13) + self.assertEqual( + {p["function"] for p in result["instructionPatches"]}, + {"trackmenu_render_names", "trackmenu_render_2D", "texrect_draw_scaled", "texrect_draw"}, + ) + table = [p for p in result["instructionPatches"] if p["function"] == "trackmenu_render_names"] + self.assertEqual( + {int(p["value"], 0) for p in table}, + {0x27BDFE80, 0x27BD0180, 0x27B00090, 0x27B30180, 0x2416FFFE, 0x24010003}, + ) + self.assertEqual(sum(int(p["value"], 0) == 0x27B00090 for p in table), 2) + self.assertEqual(sum("9U" in h["text"] for h in result["functionHooks"]), 1) + original = copy.deepcopy(result) + with self.assertRaises(ValueError): + compose_track_draw(result, elf, f"us.v{revision}", sections, symbols) + self.assertEqual(result, original) + def test_custom_rows_receive_separate_fifteen_cell_storage(self): - for revision in (77,80): - elf,sections,symbols=self.inputs(revision) - fragment=json.loads((ROOT/f'runtime-recomp/legacy-track-menu.v{revision}.recomp-fragment.json').read_text()) - result=compose_track_menu({'instructionPatches':[],'functionHooks':[]},fragment,sections,symbols) - hooks=[h for h in result['functionHooks'] if h['function']=='trackmenu_render_names'] - self.assertEqual(len(hooks),1) - text=hooks[0]['text'] - self.assertIn('(uint32_t)ctx->r29 + 0x90U',text) - self.assertIn('4U, dkr_legacy_fields, 5U',text) - self.assertNotIn('static const uint32_t',text) + for revision in (77, 80): + elf, sections, symbols = self.inputs(revision) + fragment = json.loads( + (ROOT / f"runtime-recomp/legacy-track-menu.v{revision}.recomp-fragment.json").read_text() + ) + result = compose_track_menu({"instructionPatches": [], "functionHooks": []}, fragment, sections, symbols) + hooks = [h for h in result["functionHooks"] if h["function"] == "trackmenu_render_names"] + self.assertEqual(len(hooks), 1) + text = hooks[0]["text"] + self.assertIn("(uint32_t)ctx->r29 + 0x90U", text) + self.assertIn("4U, dkr_legacy_fields, 5U", text) + self.assertNotIn("static const uint32_t", text) + def test_instruction_drift_and_wrong_revision_are_rejected(self): - for revision in (77,80): - elf,sections,symbols=self.inputs(revision) - with self.assertRaises(ValueError):compose_track_draw({'instructionPatches':[],'functionHooks':[]}, - elf,f'us.v{80 if revision==77 else 77}',sections,symbols) - start=next(iter(symbols['trackmenu_render_names']))[0] - corrupted=[] - for address,data in sections: - changed=bytearray(data) - if address<=startpending=0',helper) + helper = (ROOT / "runtime-recomp/src/game/track_select_draw_guest.h").read_text() + for forbidden in ("SDL", "get_graphics_config", "filesystem", "malloc", "new "): + self.assertNotIn(forbidden, helper) + self.assertIn("end-24U", helper) + self.assertIn("for(unsigned i=0;i<240;i+=4)", helper) + self.assertIn("state->pending=0", helper) + -if __name__=='__main__':unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/runtime-recomp/tests/tooling_utilities_tests.py b/runtime-recomp/tests/tooling_utilities_tests.py index 22046798..a6f14ed9 100644 --- a/runtime-recomp/tests/tooling_utilities_tests.py +++ b/runtime-recomp/tests/tooling_utilities_tests.py @@ -1,4 +1,5 @@ """Fixture-free regression checks for developer command-line utilities.""" + import contextlib import importlib.util import io @@ -6,6 +7,7 @@ import subprocess import sys import tempfile + try: import tomllib except ImportError: # Python 3.10 remains usable for the other utility checks. @@ -14,7 +16,7 @@ from unittest import mock ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / 'scripts')) +sys.path.insert(0, str(ROOT / "scripts")) def load(relative): @@ -27,81 +29,106 @@ def load(relative): class ToolingTests(unittest.TestCase): @unittest.skipIf(tomllib is None, "TOML parser roundtrip requires Python 3.11+") def test_toml_control_roundtrip(self): - module = load('scripts/generate_recomp_config.py') - value = '"\\\n\r\t' + ''.join(chr(c) for c in range(32)) + chr(127) + ' café' - self.assertEqual(tomllib.loads('value = ' + module.toml_string(value))['value'], value) + module = load("scripts/generate_recomp_config.py") + value = '"\\\n\r\t' + "".join(chr(c) for c in range(32)) + chr(127) + " café" + self.assertEqual(tomllib.loads("value = " + module.toml_string(value))["value"], value) def test_shader_wrapper_preserves_exit_and_arguments(self): - module = load('scripts/run_host_shader_tool.py') - with mock.patch.object(module.subprocess, 'call', return_value=23) as call: - self.assertEqual(module.main([sys.executable, '/target/file_to_c', '--flag', 'a b']), 23) - self.assertEqual(call.call_args.args[0][1:], ['--flag', 'a b']) + module = load("scripts/run_host_shader_tool.py") + with mock.patch.object(module.subprocess, "call", return_value=23) as call: + self.assertEqual(module.main([sys.executable, "/target/file_to_c", "--flag", "a b"]), 23) + self.assertEqual(call.call_args.args[0][1:], ["--flag", "a b"]) with contextlib.redirect_stderr(io.StringIO()) as err: - self.assertEqual(module.main(['/missing/file_to_c', 'file_to_c']), 127) - self.assertIn('Build the host file_to_c', err.getvalue()) + self.assertEqual(module.main(["/missing/file_to_c", "file_to_c"]), 127) + self.assertIn("Build the host file_to_c", err.getvalue()) def test_fragment_generation_failure_does_not_create_output(self): - module = load('scripts/prepare_legacy_character_menu_fragment.py') + module = load("scripts/prepare_legacy_character_menu_fragment.py") with tempfile.TemporaryDirectory() as directory: - output = Path(directory) / 'fragment.json' - with mock.patch.object(module, 'prepare', side_effect=ValueError('bad ELF')): + output = Path(directory) / "fragment.json" + with mock.patch.object(module, "prepare", side_effect=ValueError("bad ELF")): with self.assertRaises(ValueError): - module.main(['--elf', 'missing', '--revision', next(iter(module.ELFS)), '--output', str(output)]) + module.main(["--elf", "missing", "--revision", next(iter(module.ELFS)), "--output", str(output)]) self.assertFalse(output.exists()) def test_qualification_preflight_before_generation(self): - module = load('scripts/generate_legacy_menu_qualification.py') + module = load("scripts/generate_legacy_menu_qualification.py") with tempfile.TemporaryDirectory() as directory: - output = Path(directory) / 'out' - args = ['--output', str(output), '--v77-build', directory, '--v80-build', directory, - '--recompiler', sys.executable] - for extra in (['--character-menu'], ['--refresh-presentation'], []): - with mock.patch.object(module.subprocess, 'run') as run, contextlib.redirect_stderr(io.StringIO()): + output = Path(directory) / "out" + args = [ + "--output", + str(output), + "--v77-build", + directory, + "--v80-build", + directory, + "--recompiler", + sys.executable, + ] + for extra in (["--character-menu"], ["--refresh-presentation"], []): + with mock.patch.object(module.subprocess, "run") as run, contextlib.redirect_stderr(io.StringIO()): with self.assertRaises(SystemExit): module.main(args + extra) run.assert_not_called() self.assertFalse(output.exists()) def test_qualification_checks_second_revision_before_starting(self): - module = load('scripts/generate_legacy_menu_qualification.py') + module = load("scripts/generate_legacy_menu_qualification.py") with tempfile.TemporaryDirectory() as directory: root = Path(directory) - (root/'dkr.us.v77.elf').write_bytes(b'fixture') - (root/'dkr.us.v77.z64').write_bytes(b'fixture') - output = root/'out' - with mock.patch.object(module.subprocess, 'run') as run, contextlib.redirect_stderr(io.StringIO()) as err: + (root / "dkr.us.v77.elf").write_bytes(b"fixture") + (root / "dkr.us.v77.z64").write_bytes(b"fixture") + output = root / "out" + with mock.patch.object(module.subprocess, "run") as run, contextlib.redirect_stderr(io.StringIO()) as err: with self.assertRaises(SystemExit): - module.main(['--output', str(output), '--v77-build', directory, - '--v80-build', directory, '--recompiler', sys.executable]) - self.assertIn('Missing v80 input', err.getvalue()) + module.main( + [ + "--output", + str(output), + "--v77-build", + directory, + "--v80-build", + directory, + "--recompiler", + sys.executable, + ] + ) + self.assertIn("Missing v80 input", err.getvalue()) run.assert_not_called() self.assertFalse(output.exists()) def test_qualification_stage_failure_is_concise(self): - module = load('scripts/generate_legacy_menu_qualification.py') - with mock.patch.object(module.subprocess, 'run', side_effect=subprocess.CalledProcessError(9, ['tool'])): - with self.assertRaisesRegex(RuntimeError, 'test stage failed.*exit 9'): - module.run_stage(['tool'], 'test stage') + module = load("scripts/generate_legacy_menu_qualification.py") + with mock.patch.object(module.subprocess, "run", side_effect=subprocess.CalledProcessError(9, ["tool"])): + with self.assertRaisesRegex(RuntimeError, "test stage failed.*exit 9"): + module.run_stage(["tool"], "test stage") def test_blender_invocation_and_skip_status(self): - module = load('tools/blender/run_tests.py') + module = load("tools/blender/run_tests.py") self.assertEqual(Path(module.REPO_ROOT), ROOT) - with mock.patch.object(module, 'PLAIN_SUITES', ['plain.py']), mock.patch.object(module, 'BLENDER_SUITES', ['blend.py']), \ - mock.patch.object(module, 'find_blender', return_value='/blender'), \ - mock.patch.object(module, 'run', return_value=77) as run, contextlib.redirect_stdout(io.StringIO()) as out: + with ( + mock.patch.object(module, "PLAIN_SUITES", ["plain.py"]), + mock.patch.object(module, "BLENDER_SUITES", ["blend.py"]), + mock.patch.object(module, "find_blender", return_value="/blender"), + mock.patch.object(module, "run", return_value=77) as run, + contextlib.redirect_stdout(io.StringIO()) as out, + ): self.assertEqual(module.main([]), 0) - self.assertIn('SKIP plain.py', out.getvalue()) - self.assertIn('SKIP blend.py', out.getvalue()) - self.assertIn('--python-exit-code', run.call_args.args[1]) - with mock.patch.object(module, 'run') as run, contextlib.redirect_stderr(io.StringIO()): + self.assertIn("SKIP plain.py", out.getvalue()) + self.assertIn("SKIP blend.py", out.getvalue()) + self.assertIn("--python-exit-code", run.call_args.args[1]) + with mock.patch.object(module, "run") as run, contextlib.redirect_stderr(io.StringIO()): with self.assertRaises(SystemExit): - module.main(['--blender', '/missing/blender']) + module.main(["--blender", "/missing/blender"]) run.assert_not_called() def test_discovery_defaults(self): - for name, variable in [('legacy_behaviour_pipeline_tests.py', 'INPUTS'), ('legacy_memory_pipeline_tests.py', 'ELF_INPUTS')]: - self.assertEqual(getattr(load('runtime-recomp/tests/' + name), variable), []) + for name, variable in [ + ("legacy_behaviour_pipeline_tests.py", "INPUTS"), + ("legacy_memory_pipeline_tests.py", "ELF_INPUTS"), + ]: + self.assertEqual(getattr(load("runtime-recomp/tests/" + name), variable), []) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/runtime-recomp/tests/water_profile_policy_tests.py b/runtime-recomp/tests/water_profile_policy_tests.py index 1cbfde6e..def01a7f 100644 --- a/runtime-recomp/tests/water_profile_policy_tests.py +++ b/runtime-recomp/tests/water_profile_policy_tests.py @@ -1,46 +1,58 @@ import copy import json -import re from pathlib import Path +import re import sys -root=Path(__file__).resolve().parents[2] -sys.path.insert(0,str(root/'scripts')) -from water_profile_policy import compose_water_profile + +root = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(root / "scripts")) from compose_legacy_mod_policy import elf_functions +from water_profile_policy import compose_water_profile + def check_scroll_symbols(elf77, elf80): - header=(root/'runtime-recomp/src/game/revision_addresses.hpp').read_text() - fields=re.findall(r'std::uint32_t (\w+);',header.split('struct AddressTable {',1)[1].split('};',1)[0]) - for revision,elf in [('kUsV77',elf77),('kUsV80',elf80)]: - table=header.split('AddressTable '+revision+'{',1)[1].split('};',1)[0] - values=[int(v,16) for v in re.findall(r'0x([0-9A-Fa-f]+)U',table)] - assert len(fields)==len(values) - addresses=dict(zip(fields,values)) - symbols=elf_functions(Path(elf),(1,)) - for field in ['WaveTexUVMaskX','WaveTexUVMaskY']: - assert symbols['g'+field]=={(addresses[field],4)} - print('Verified both water scroll mask address tables against retail ELF symbols') + header = (root / "runtime-recomp/src/game/revision_addresses.hpp").read_text() + fields = re.findall(r"std::uint32_t (\w+);", header.split("struct AddressTable {", 1)[1].split("};", 1)[0]) + for revision, elf in [("kUsV77", elf77), ("kUsV80", elf80)]: + table = header.split("AddressTable " + revision + "{", 1)[1].split("};", 1)[0] + values = [int(v, 16) for v in re.findall(r"0x([0-9A-Fa-f]+)U", table)] + assert len(fields) == len(values) + addresses = dict(zip(fields, values)) + symbols = elf_functions(Path(elf), (1,)) + for field in ["WaveTexUVMaskX", "WaveTexUVMaskY"]: + assert symbols["g" + field] == {(addresses[field], 4)} + print("Verified both water scroll mask address tables against retail ELF symbols") + + +def check(elf, policy_path): + policy = json.loads(Path(policy_path).read_text()) + original = copy.deepcopy(policy) + result = compose_water_profile(policy, Path(elf)) + assert policy == original + hooks = [h for h in result["functionHooks"] if "dkr_water_" in h["text"]] + assert len(hooks) == 21 + assert result.get("instructionPatches") == policy.get("instructionPatches") + assert len({int(h["beforeVram"], 0) for h in result["functionHooks"]}) == len(result["functionHooks"]) + try: + compose_water_profile(result, Path(elf)) + except ValueError: + pass + else: + raise AssertionError("Duplicate composition must fail") + conflict = copy.deepcopy(policy) + conflict.setdefault("instructionPatches", []).append(dict(vram=hooks[-2]["beforeVram"])) + try: + compose_water_profile(conflict, Path(elf)) + except ValueError: + pass + else: + raise AssertionError("Instruction collision must fail") + print("Verified water timing hook coverage and existing-policy preservation:", elf) -def check(elf,policy_path): - policy=json.loads(Path(policy_path).read_text()) - original=copy.deepcopy(policy) - result=compose_water_profile(policy,Path(elf)) - assert policy==original - hooks=[h for h in result['functionHooks'] if 'dkr_water_' in h['text']] - assert len(hooks)==21 - assert result.get('instructionPatches')==policy.get('instructionPatches') - assert len({int(h['beforeVram'],0) for h in result['functionHooks']})==len(result['functionHooks']) - try: compose_water_profile(result,Path(elf)) - except ValueError: pass - else: raise AssertionError('Duplicate composition must fail') - conflict=copy.deepcopy(policy) - conflict.setdefault('instructionPatches',[]).append(dict(vram=hooks[-2]['beforeVram'])) - try: compose_water_profile(conflict,Path(elf)) - except ValueError: pass - else: raise AssertionError('Instruction collision must fail') - print('Verified water timing hook coverage and existing-policy preservation:',elf) -if __name__=='__main__': - if len(sys.argv)!=5: raise SystemExit('v77 ELF policy v80 ELF policy required') - check(sys.argv[1],sys.argv[2]);check(sys.argv[3],sys.argv[4]) - check_scroll_symbols(sys.argv[1],sys.argv[3]) +if __name__ == "__main__": + if len(sys.argv) != 5: + raise SystemExit("v77 ELF policy v80 ELF policy required") + check(sys.argv[1], sys.argv[2]) + check(sys.argv[3], sys.argv[4]) + check_scroll_symbols(sys.argv[1], sys.argv[3]) diff --git a/scripts/Package-Linux-AppImage.sh b/scripts/Package-Linux-AppImage.sh index 1a5a3c78..39917e09 100755 --- a/scripts/Package-Linux-AppImage.sh +++ b/scripts/Package-Linux-AppImage.sh @@ -4,32 +4,32 @@ set -euo pipefail PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" BUILD_DIRECTORY="${DKR_LINUX_BUILD_DIR:-${PROJECT_ROOT}/build/dkr-runtime-linux}" if [[ -f "${BUILD_DIRECTORY}/CMakeCache.txt" ]] && - grep -Eq '^DKR_EXPERIMENTAL_RENDER_QUALIFICATION:BOOL=(ON|1|TRUE|YES)[[:space:]]*$' "${BUILD_DIRECTORY}/CMakeCache.txt"; then + grep -Eq '^DKR_EXPERIMENTAL_RENDER_QUALIFICATION:BOOL=(ON|1|TRUE|YES)[[:space:]]*$' "${BUILD_DIRECTORY}/CMakeCache.txt"; then echo 'Refusing to package private experimental GPU retirement qualification. Reconfigure and rebuild with DKR_EXPERIMENTAL_RENDER_QUALIFICATION=OFF.' >&2 exit 1 fi if [[ -f "${BUILD_DIRECTORY}/CMakeCache.txt" ]] && - grep -Eq '^DKR_REPLAY_QUALIFICATION:BOOL=(ON|1|TRUE|YES)[[:space:]]*$' "${BUILD_DIRECTORY}/CMakeCache.txt"; then + grep -Eq '^DKR_REPLAY_QUALIFICATION:BOOL=(ON|1|TRUE|YES)[[:space:]]*$' "${BUILD_DIRECTORY}/CMakeCache.txt"; then echo 'Refusing to package private replay fixture capture.' >&2 exit 1 fi if [[ -f "${BUILD_DIRECTORY}/CMakeCache.txt" ]] && - grep -Eq '^DKR_ANDROID_RENDER_QUALIFICATION:BOOL=(ON|1|TRUE|YES)[[:space:]]*$' "${BUILD_DIRECTORY}/CMakeCache.txt"; then + grep -Eq '^DKR_ANDROID_RENDER_QUALIFICATION:BOOL=(ON|1|TRUE|YES)[[:space:]]*$' "${BUILD_DIRECTORY}/CMakeCache.txt"; then echo 'Refusing to package private Android renderer qualification.' >&2 exit 1 fi if [[ -f "${BUILD_DIRECTORY}/CMakeCache.txt" ]] && - grep -Eq '^DKR_WATER_QUALIFICATION:BOOL=(ON|1|TRUE|YES)[[:space:]]*$' "${BUILD_DIRECTORY}/CMakeCache.txt"; then + grep -Eq '^DKR_WATER_QUALIFICATION:BOOL=(ON|1|TRUE|YES)[[:space:]]*$' "${BUILD_DIRECTORY}/CMakeCache.txt"; then echo 'Refusing to package private water-scene qualification.' >&2 exit 1 fi if [[ -f "${BUILD_DIRECTORY}/CMakeCache.txt" ]] && - grep -Eq '^DKR_TASK_QUALIFICATION:BOOL=(ON|1|TRUE|YES)[[:space:]]*$' "${BUILD_DIRECTORY}/CMakeCache.txt"; then + grep -Eq '^DKR_TASK_QUALIFICATION:BOOL=(ON|1|TRUE|YES)[[:space:]]*$' "${BUILD_DIRECTORY}/CMakeCache.txt"; then echo 'Refusing to package private host-task fault injection. Rebuild with DKR_TASK_QUALIFICATION=OFF.' >&2 exit 1 fi if [[ -f "${BUILD_DIRECTORY}/CMakeCache.txt" ]] && - grep -Eq '^DKR_LEGACY_QUALIFICATION:BOOL=(ON|1|TRUE|YES)[[:space:]]*$' "${BUILD_DIRECTORY}/CMakeCache.txt"; then + grep -Eq '^DKR_LEGACY_QUALIFICATION:BOOL=(ON|1|TRUE|YES)[[:space:]]*$' "${BUILD_DIRECTORY}/CMakeCache.txt"; then echo 'Refusing to package a private legacy-content qualification build. Reconfigure and rebuild with DKR_LEGACY_QUALIFICATION=OFF first.' >&2 exit 1 fi @@ -43,36 +43,45 @@ INPUT_HOST="${INPUT_HOST_DIRECTORY}/DKR-R-InputHost" MOD_WORKER="${INPUT_HOST_DIRECTORY}/DKR-R-ModWorker" UPDATE_HELPER="${INPUT_HOST_DIRECTORY}/DKR-R-Updater" SDL3_LIBRARY="${INPUT_HOST_DIRECTORY}/libSDL3.so.0" -VERSION_FILE_VALUE="$(tr -d '\r\n' < "${PROJECT_ROOT}/VERSION")" +VERSION_FILE_VALUE="$(tr -d '\r\n' <"${PROJECT_ROOT}/VERSION")" VERSION="${DKR_RELEASE_VERSION:-${VERSION_FILE_VALUE}}" EXPERIMENTAL_RACE_TEST=0 if grep -Eq '^DKR_EXPERIMENTAL_RACE_TEST:BOOL=(ON|1|TRUE|YES)[[:space:]]*$' "${BUILD_DIRECTORY}/CMakeCache.txt"; then EXPERIMENTAL_RACE_TEST=1 if [[ "${VERSION}" != 1.0.5 && "${VERSION}" != 1.0.6 && "${VERSION}" != 1.0.7 && "${VERSION}" != 1.0.7+beta.2 && "${VERSION}" != 1.0.7+beta.3 && "${VERSION}" != 1.0.7+beta.4 && "${VERSION}" != 1.0.7+beta.5 ]]; then - [[ "${VERSION}" == *experimental* || "${VERSION}" == *rollback-test* || "${VERSION}" == 1.0.5-beta.15 || "${VERSION}" == 1.0.5-beta.15-playtest.[23456789] || "${VERSION}" == 1.0.5-beta.15-playtest.10 || "${VERSION}" == 1.0.5-beta.15-playtest.11 || "${VERSION}" == 1.0.5-beta.15-playtest.12 || "${VERSION}" == 1.0.5-beta.15-playtest.14 || "${VERSION}" == 1.0.5-beta.15-playtest.15 || "${VERSION}" == 1.0.5-beta.15-playtest.15-fix.[12] || "${VERSION}" == 1.0.5-beta.15-playtest.16 || "${VERSION}" == 1.0.5-beta.15-playtest.16-fix.[12] ]] || { echo 'Experimental builds require a distinct experimental version or an approved Beta 15 playtest candidate.' >&2; exit 1; } -fi + [[ "${VERSION}" == *experimental* || "${VERSION}" == *rollback-test* || "${VERSION}" == 1.0.5-beta.15 || "${VERSION}" == 1.0.5-beta.15-playtest.[23456789] || "${VERSION}" == 1.0.5-beta.15-playtest.10 || "${VERSION}" == 1.0.5-beta.15-playtest.11 || "${VERSION}" == 1.0.5-beta.15-playtest.12 || "${VERSION}" == 1.0.5-beta.15-playtest.14 || "${VERSION}" == 1.0.5-beta.15-playtest.15 || "${VERSION}" == 1.0.5-beta.15-playtest.15-fix.[12] || "${VERSION}" == 1.0.5-beta.15-playtest.16 || "${VERSION}" == 1.0.5-beta.15-playtest.16-fix.[12] ]] || { + echo 'Experimental builds require a distinct experimental version or an approved Beta 15 playtest candidate.' >&2 + exit 1 + } + fi fi # Runtime checks are on by default. Release maintainers can explicitly reserve # execution/acceptance for their own machines without running the game here. SKIP_RUNTIME_TESTS="${DKR_SKIP_RUNTIME_TESTS:-0}" DEFER_RUNTIME_TESTS_TO_USER="${DKR_DEFER_RUNTIME_TESTS_TO_USER:-0}" [[ "${SKIP_RUNTIME_TESTS}" == 0 || "${SKIP_RUNTIME_TESTS}" == 1 ]] || { - echo 'DKR_SKIP_RUNTIME_TESTS must be 0 or 1' >&2; exit 1; + echo 'DKR_SKIP_RUNTIME_TESTS must be 0 or 1' >&2 + exit 1 } [[ "${DEFER_RUNTIME_TESTS_TO_USER}" == 0 || "${DEFER_RUNTIME_TESTS_TO_USER}" == 1 ]] || { - echo 'DKR_DEFER_RUNTIME_TESTS_TO_USER must be 0 or 1' >&2; exit 1; + echo 'DKR_DEFER_RUNTIME_TESTS_TO_USER must be 0 or 1' >&2 + exit 1 } if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ && "${SKIP_RUNTIME_TESTS}" != 0 && "${DEFER_RUNTIME_TESTS_TO_USER}" != 1 ]]; then - echo 'Public release runtime checks need explicit maintainer deferral: DKR_DEFER_RUNTIME_TESTS_TO_USER=1.' >&2; exit 1 + echo 'Public release runtime checks need explicit maintainer deferral: DKR_DEFER_RUNTIME_TESTS_TO_USER=1.' >&2 + exit 1 fi if [[ "${SKIP_RUNTIME_TESTS}" == 1 ]]; then echo 'Packaging only: runtime execution is deferred to the maintainer; this is not an acceptance pass.' fi ARCH="${DKR_LINUX_ARCH:-x86_64}" case "${ARCH}" in - x86_64) ELF_MACHINE='Advanced Micro Devices X86-64' ;; - aarch64) ELF_MACHINE='AArch64' ;; - *) echo "Unsupported AppImage architecture: ${ARCH}" >&2; exit 1 ;; +x86_64) ELF_MACHINE='Advanced Micro Devices X86-64' ;; +aarch64) ELF_MACHINE='AArch64' ;; +*) + echo "Unsupported AppImage architecture: ${ARCH}" >&2 + exit 1 + ;; esac APPDIR="${DKR_APPDIR:-${PROJECT_ROOT}/dist/DKR-R-${VERSION}-Linux-${ARCH}.AppDir}" OUTPUT="${DKR_APPIMAGE_OUTPUT:-${PROJECT_ROOT}/dist/DKR-R-${VERSION}-Linux-${ARCH}.AppImage}" @@ -93,17 +102,17 @@ validate_release_tree() { extension="${file##*.}" extension="${extension,,}" case "${extension}" in - z64|v64|n64|eep|mpk|sra|fla|o2r|otr|dkr-probe|dkr-bootstrap|dkr-component) - echo "Release staging contains prohibited game data: ${file}" >&2 - return 1 - ;; + z64 | v64 | n64 | eep | mpk | sra | fla | o2r | otr | dkr-probe | dkr-bootstrap | dkr-component) + echo "Release staging contains prohibited game data: ${file}" >&2 + return 1 + ;; esac magic="$(od -An -tx1 -N4 "${file}" 2>/dev/null | tr -d '[:space:]')" case "${magic}" in - 80371240|37804012|40123780) - echo "Release staging contains an N64 ROM header: ${file}" >&2 - return 1 - ;; + 80371240 | 37804012 | 40123780) + echo "Release staging contains an N64 ROM header: ${file}" >&2 + return 1 + ;; esac done < <(find "${root}" -type f -print0) [[ "${inspected}" -gt 0 ]] || { @@ -121,14 +130,14 @@ collect_linux_dependency_notices() { local library_count copyright_count mkdir -p "${notice_root}/common-licenses" - : > "${manifest}" + : >"${manifest}" printf '%s\n' \ 'Debian package notice directories deployed with the Linux dependencies:' \ - >> "${manifest}" + >>"${manifest}" while IFS= read -r -d '' copyright_file; do package_directory="$(basename "$(dirname "${copyright_file}")")" - printf '%s\n' "${package_directory}" >> "${manifest}" + printf '%s\n' "${package_directory}" >>"${manifest}" done < <(find "${root}/usr/share/doc" -mindepth 2 -maxdepth 2 \ -type f -name copyright -print0 | sort -z) @@ -147,7 +156,10 @@ collect_linux_dependency_notices() { echo "Collected ${copyright_count} dependency copyright records for ${library_count} bundled libraries" } -[[ -x "${BINARY}" ]] || { echo "Missing Linux release binary: ${BINARY}" >&2; exit 1; } +[[ -x "${BINARY}" ]] || { + echo "Missing Linux release binary: ${BINARY}" >&2 + exit 1 +} if ! strings "${BINARY}" | grep -Fx "${VERSION}" >/dev/null; then echo "Release executable does not contain requested version ${VERSION}; reconfigure and rebuild before packaging." >&2 exit 1 @@ -157,21 +169,52 @@ if grep -aEq 'DKR_WATER_TEST_MAP|\[perf\]\[private-water-preview\]' "${BINARY}"; exit 1 fi READELF="${READELF:-readelf}" -command -v "${READELF}" >/dev/null || { echo "Missing ELF inspector: ${READELF}" >&2; exit 1; } +command -v "${READELF}" >/dev/null || { + echo "Missing ELF inspector: ${READELF}" >&2 + exit 1 +} for architecture_file in "${BINARY}" "${MOD_WORKER}" "${INPUT_HOST}" "${SDL3_LIBRARY}" "${UPDATE_HELPER}"; do "${READELF}" -h "${architecture_file}" | grep -F "${ELF_MACHINE}" >/dev/null || { - echo "Wrong architecture (expected ${ARCH}): ${architecture_file}" >&2; exit 1; + echo "Wrong architecture (expected ${ARCH}): ${architecture_file}" >&2 + exit 1 } done -[[ -x "${MOD_WORKER}" ]] || { echo "Missing legacy importer: ${MOD_WORKER}" >&2; exit 1; } -[[ -x "${INPUT_HOST}" ]] || { echo "Missing private SDL3 input host: ${INPUT_HOST}" >&2; exit 1; } -[[ -f "${SDL3_LIBRARY}" ]] || { echo "Missing private SDL3 runtime: ${SDL3_LIBRARY}" >&2; exit 1; } -[[ -x "${LINUXDEPLOY}" ]] || { echo "Missing linuxdeploy: ${LINUXDEPLOY}" >&2; exit 1; } -[[ -x "${APPIMAGE_PLUGIN}" ]] || { echo "Missing linuxdeploy AppImage plugin: ${APPIMAGE_PLUGIN}" >&2; exit 1; } -[[ -f "${ICON_FILE}" ]] || { echo "Missing Linux application icon: ${ICON_FILE}" >&2; exit 1; } -command -v patchelf >/dev/null || { echo "Missing required command: patchelf" >&2; exit 1; } -[[ ! -e "${APPDIR}" ]] || { echo "AppDir already exists; choose a fresh DKR_APPDIR: ${APPDIR}" >&2; exit 1; } -[[ ! -e "${OUTPUT}" ]] || { echo "Output already exists; choose a fresh DKR_APPIMAGE_OUTPUT: ${OUTPUT}" >&2; exit 1; } +[[ -x "${MOD_WORKER}" ]] || { + echo "Missing legacy importer: ${MOD_WORKER}" >&2 + exit 1 +} +[[ -x "${INPUT_HOST}" ]] || { + echo "Missing private SDL3 input host: ${INPUT_HOST}" >&2 + exit 1 +} +[[ -f "${SDL3_LIBRARY}" ]] || { + echo "Missing private SDL3 runtime: ${SDL3_LIBRARY}" >&2 + exit 1 +} +[[ -x "${LINUXDEPLOY}" ]] || { + echo "Missing linuxdeploy: ${LINUXDEPLOY}" >&2 + exit 1 +} +[[ -x "${APPIMAGE_PLUGIN}" ]] || { + echo "Missing linuxdeploy AppImage plugin: ${APPIMAGE_PLUGIN}" >&2 + exit 1 +} +[[ -f "${ICON_FILE}" ]] || { + echo "Missing Linux application icon: ${ICON_FILE}" >&2 + exit 1 +} +command -v patchelf >/dev/null || { + echo "Missing required command: patchelf" >&2 + exit 1 +} +[[ ! -e "${APPDIR}" ]] || { + echo "AppDir already exists; choose a fresh DKR_APPDIR: ${APPDIR}" >&2 + exit 1 +} +[[ ! -e "${OUTPUT}" ]] || { + echo "Output already exists; choose a fresh DKR_APPIMAGE_OUTPUT: ${OUTPUT}" >&2 + exit 1 +} mkdir -p "${APPDIR}/usr/share/doc/dkr-port/licenses" "$(dirname "${OUTPUT}")" mkdir -p "${APPDIR}/usr/share/metainfo" @@ -375,22 +418,31 @@ install -m 0755 "${SDL3_LIBRARY}" \ patchelf --set-rpath '$ORIGIN' \ "${APPDIR}/usr/libexec/dkr-r/DKR-R-InputHost" if [[ "${SKIP_RUNTIME_TESTS}" == 0 ]]; then -[[ "$(timeout 15 "${APPDIR}/usr/libexec/dkr-r/DKR-R-Updater" --self-test)" == 'DKR-R updater protocol 1' ]] || { echo 'Packaged updater helper is not compatible.' >&2; exit 1; } -timeout 15 "${APPDIR}/usr/libexec/dkr-r/DKR-R-ModWorker" --self-test -[[ "$(timeout 15 "${APPDIR}/usr/libexec/dkr-r/DKR-R-ModWorker" --online-capability)" == 'DKR-R online-prepare protocol 1' ]] || { echo 'Packaged ModWorker lacks online preparation support.' >&2; exit 1; } -LD_LIBRARY_PATH="${APPDIR}/usr/libexec/dkr-r${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" \ - "${APPDIR}/usr/libexec/dkr-r/DKR-R-InputHost" --self-test --mappings \ - "${APPDIR}/usr/bin/assets/controllers/gamecontrollerdb.txt" + [[ "$(timeout 15 "${APPDIR}/usr/libexec/dkr-r/DKR-R-Updater" --self-test)" == 'DKR-R updater protocol 1' ]] || { + echo 'Packaged updater helper is not compatible.' >&2 + exit 1 + } + timeout 15 "${APPDIR}/usr/libexec/dkr-r/DKR-R-ModWorker" --self-test + [[ "$(timeout 15 "${APPDIR}/usr/libexec/dkr-r/DKR-R-ModWorker" --online-capability)" == 'DKR-R online-prepare protocol 1' ]] || { + echo 'Packaged ModWorker lacks online preparation support.' >&2 + exit 1 + } + LD_LIBRARY_PATH="${APPDIR}/usr/libexec/dkr-r${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" \ + "${APPDIR}/usr/libexec/dkr-r/DKR-R-InputHost" --self-test --mappings \ + "${APPDIR}/usr/bin/assets/controllers/gamecontrollerdb.txt" fi collect_linux_dependency_notices "${APPDIR}" validate_release_tree "${APPDIR}" run_appimage "${APPIMAGE_PLUGIN}" --appdir "${APPDIR}" -[[ -s "${OUTPUT}" ]] || { echo "AppImage output is missing or empty: ${OUTPUT}" >&2; exit 1; } +[[ -s "${OUTPUT}" ]] || { + echo "AppImage output is missing or empty: ${OUTPUT}" >&2 + exit 1 +} if [[ "${SKIP_RUNTIME_TESTS}" == 0 ]]; then -APPIMAGE_EXTRACT_AND_RUN=1 run_appimage "${OUTPUT}" --self-test-pak "${PAK_TEST}" -SDL_VIDEODRIVER=dummy SDL_AUDIODRIVER=dummy APPIMAGE_EXTRACT_AND_RUN=1 \ - run_appimage "${OUTPUT}" --self-test-input-switch "${INPUT_SWITCH_TEST}" + APPIMAGE_EXTRACT_AND_RUN=1 run_appimage "${OUTPUT}" --self-test-pak "${PAK_TEST}" + SDL_VIDEODRIVER=dummy SDL_AUDIODRIVER=dummy APPIMAGE_EXTRACT_AND_RUN=1 \ + run_appimage "${OUTPUT}" --self-test-input-switch "${INPUT_SWITCH_TEST}" else echo 'WARNING: Runtime tests explicitly deferred; package runtime qualification remains pending.' >&2 fi diff --git a/scripts/README.md b/scripts/README.md index 4558d0a0..958ec07b 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -10,7 +10,9 @@ Windows). Commands with options describe them with `--help`. | Task | Command | Inputs | |---|---|---| | Fast contributor checks | `python3 scripts/check.py` | Git and Python; no ROM or downloads | -| List checks or choose a suite | `python3 scripts/check.py --list` / `--suite tooling` | No C++ build | +| Build-tool checks | `python3 scripts/check.py --suite host-tools` | Also CMake, Ninja and pre-commit; may download hook environments | +| List checks or choose suites | `python3 scripts/check.py --list` / `--suite guard,tooling` | No C++ build | +| Refuse new patch layers | `python3 scripts/pattern_guard.py --tree` (`--explain all` lists rules) | Git; checks `architecture/pattern-guard-baseline.txt` | | Prepare pinned dependencies | `python3 scripts/bootstrap_dependencies.py` | Git; network for missing revisions | | Apply reviewed patches | `python3 scripts/apply_dependency_patches.py` | Prepared dependencies | | Inspect patch readiness | `python3 scripts/apply_dependency_patches.py --check` | Read-only checkout validation | diff --git a/scripts/adapt-water-reference.py b/scripts/adapt-water-reference.py index 0917c8a4..bcdae363 100644 --- a/scripts/adapt-water-reference.py +++ b/scripts/adapt-water-reference.py @@ -3,6 +3,7 @@ This is a mechanical patch transformation, not a dependency checkout edit. Usage: python scripts/adapt-water-reference.py path/to/rt64-water.patch """ + import hashlib from pathlib import Path import re @@ -15,21 +16,33 @@ sections = re.split(r"(?=^diff --git )", source.decode().replace("\r\n", "\n"), flags=re.M) selected = [] existing = { - "src/hle/rt64_draw_call.h", "src/hle/rt64_state.cpp", "src/hle/rt64_state.h", - "src/hle/rt64_workload.cpp", "src/hle/rt64_workload.h", - "src/render/rt64_descriptor_sets.h", "src/render/rt64_framebuffer_renderer.cpp", - "src/render/rt64_framebuffer_renderer.h", "src/render/rt64_framebuffer_renderer_call.h", - "src/render/rt64_rsp_processor.cpp", "src/shaders/RSPProcessCS.hlsl", + "src/hle/rt64_draw_call.h", + "src/hle/rt64_state.cpp", + "src/hle/rt64_state.h", + "src/hle/rt64_workload.cpp", + "src/hle/rt64_workload.h", + "src/render/rt64_descriptor_sets.h", + "src/render/rt64_framebuffer_renderer.cpp", + "src/render/rt64_framebuffer_renderer.h", + "src/render/rt64_framebuffer_renderer_call.h", + "src/render/rt64_rsp_processor.cpp", + "src/shaders/RSPProcessCS.hlsl", } for section in sections: match = re.match(r"diff --git a/(\S+) b/\S+", section) if not match: continue path = match[1] - if path in existing or path.startswith("src/shaders/Water") or path in { - "src/shared/rt64_water_params.h", "src/render/rt64_water_renderer.cpp", - "src/render/rt64_water_renderer.h", - }: + if ( + path in existing + or path.startswith("src/shaders/Water") + or path + in { + "src/shared/rt64_water_params.h", + "src/render/rt64_water_renderer.cpp", + "src/render/rt64_water_renderer.h", + } + ): # The reference's diagnostics are not part of the product renderer. if path == "src/render/rt64_framebuffer_renderer.cpp": begin = section.index("+ static const bool waterDiagnostics=") @@ -40,8 +53,14 @@ # on both sides of the insertion, preserving those statements. marker = "+ uint32_t waterProfileToken = UINT32_MAX, waterProfileScene = UINT32_MAX, activeWaterDraw = UINT32_MAX;\n" trailing = " InstanceDrawCall::Type previousCallType = InstanceDrawCall::Type::Unknown;\n bool previousVertexTestZ = false;\n const RenderPipeline *previousPipeline = nullptr;\n" - original = "@@ -466,6 +471,7 @@\n }\n \n void FramebufferRenderer::submitRasterScene(RenderWorker *worker, const Framebuffer &framebuffer, RenderFramebufferStorage *fbStorage, const RasterScene &rasterScene, bool &depthState) {\n" + marker + trailing - replacement = "@@ -469,3 +474,4 @@\n" + trailing.splitlines(True)[0] + marker + "".join(trailing.splitlines(True)[1:]) + original = ( + "@@ -466,6 +471,7 @@\n }\n \n void FramebufferRenderer::submitRasterScene(RenderWorker *worker, const Framebuffer &framebuffer, RenderFramebufferStorage *fbStorage, const RasterScene &rasterScene, bool &depthState) {\n" + + marker + + trailing + ) + replacement = ( + "@@ -469,3 +474,4 @@\n" + trailing.splitlines(True)[0] + marker + "".join(trailing.splitlines(True)[1:]) + ) if section.count(original) != 1: raise SystemExit("Reference function-entry context changed") section = section.replace(original, replacement, 1) diff --git a/scripts/agent_hooks/hooklib.py b/scripts/agent_hooks/hooklib.py new file mode 100644 index 00000000..26dc2e67 --- /dev/null +++ b/scripts/agent_hooks/hooklib.py @@ -0,0 +1,348 @@ +"""Shared plumbing for the AI coding agent hooks in this directory. + +Claude Code (.claude/settings.json) and Codex (.codex/hooks.json) both run +these scripts through ``run`` with the hook payload as JSON on stdin. The two +agents use the same event names and the same exit status contract: 0 lets the +action proceed, 2 blocks it and feeds stderr back to the agent. Anything else +is a hook error that the agent reports but does not act on, so a broken hook +never wedges a session. + +Edits arrive in two shapes. Claude sends ``tool_input.file_path`` with +``content`` (Write), ``old_string``/``new_string`` (Edit), ``edits`` (MultiEdit) +or ``notebook_path`` (NotebookEdit). Codex sends ``apply_patch`` with the patch +text in ``tool_input.command``. ``planned_edits`` turns either into the +would-be content of each file. + +Standard library only. +""" + +# pyright: strict + +from __future__ import annotations + +from collections.abc import Iterator, Mapping +import dataclasses +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +from typing import Any, cast + +SCRIPTS = Path(__file__).resolve().parents[1] +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +import pattern_guard # noqa: E402 (needs the sys.path entry above) + +# A checkout counts as DKR-R when it carries the guard; the hooks stay out of +# any other repository an agent happens to touch. +MARKER = "scripts/pattern_guard.py" +SESSION_BASE = ".claude/session-base" +SESSION_LIMIT = 32 +# Every hook script `pre-commit install` writes carries this line. +PRE_COMMIT_MARK = "File generated by pre-commit" + +Payload = Mapping[str, Any] + + +def read_payload() -> Payload: + """The hook payload from stdin; an empty or malformed payload reads as {}.""" + try: + payload = json.loads(sys.stdin.read() or "{}") + except json.JSONDecodeError: + return {} + return cast(Payload, payload) if isinstance(payload, dict) else {} + + +def git(root: Path, *args: str) -> str | None: + """Stripped stdout of a git command in root, or None if it failed.""" + try: + result = subprocess.run( + ["git", "-C", str(root), *args], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True + ) + except OSError: + return None + return result.stdout.strip() if result.returncode == 0 else None + + +def checkout_of(path: Path) -> Path | None: + """The DKR-R work tree containing path (which may not exist yet).""" + directory = path if path.is_dir() else path.parent + while not directory.is_dir() and directory != directory.parent: + directory = directory.parent + top = git(directory, "rev-parse", "--show-toplevel") + if top is None: + return None + root = Path(top).resolve() + return root if (root / MARKER).is_file() else None + + +def project_root(payload: Payload) -> Path | None: + """The checkout the session works in: the payload's cwd (which follows the + agent into a worktree), then $CLAUDE_PROJECT_DIR, then this script's own.""" + candidates = [payload.get("cwd"), os.environ.get("CLAUDE_PROJECT_DIR"), str(SCRIPTS.parent)] + for candidate in candidates: + if isinstance(candidate, str) and candidate: + root = checkout_of(Path(candidate)) + if root is not None: + return root + return None + + +def relative(root: Path, path: Path) -> str | None: + """path as a root-relative POSIX string, or None if it lies outside root.""" + resolved = path.parent.resolve() / path.name + try: + return resolved.relative_to(root).as_posix() + except ValueError: + return None + + +# --- Planned edits ------------------------------------------------------------- + + +@dataclasses.dataclass(frozen=True) +class PlannedEdit: + """One file an edit tool is about to write. ``text`` is the content after + the edit, or None when the hook can't reconstruct it (the rules then see + the current file).""" + + path: Path + text: str | None + + +def _current(path: Path) -> str: + try: + return path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "" + + +def _replace(text: str, old: str, new: str, every: bool) -> str: + """Apply one Edit. If old_string is missing the tool call will fail, but + the new text is still judged, appended, so a violation can't slip past.""" + if old and old in text: + return text.replace(old, new) if every else text.replace(old, new, 1) + return f"{text}\n{new}" + + +def planned_edits(payload: Payload) -> list[PlannedEdit]: + """Files the payload's edit tool call would write, with their new text.""" + tool = payload.get("tool_name") + raw = payload.get("tool_input") + tool_input: Mapping[str, Any] = cast(Mapping[str, Any], raw) if isinstance(raw, dict) else {} + cwd = Path(str(payload.get("cwd") or os.getcwd())) + + def resolve(name: object) -> Path | None: + return cwd / name if isinstance(name, str) and name else None + + if tool == "apply_patch" or (tool in ("Edit", "Write") and "command" in tool_input): + command = tool_input.get("command") + return list(_apply_patch(str(command), cwd)) if isinstance(command, str) else [] + if tool == "NotebookEdit": + path = resolve(tool_input.get("notebook_path")) + return [PlannedEdit(path, None)] if path else [] + path = resolve(tool_input.get("file_path")) + if path is None: + return [] + if tool == "Write": + content = tool_input.get("content") + return [PlannedEdit(path, content if isinstance(content, str) else None)] + if tool == "Edit": + text = _replace( + _current(path), + str(tool_input.get("old_string") or ""), + str(tool_input.get("new_string") or ""), + bool(tool_input.get("replace_all")), + ) + return [PlannedEdit(path, text)] + if tool == "MultiEdit": + text = _current(path) + edits = tool_input.get("edits") + for edit in cast(list[Any], edits) if isinstance(edits, list) else []: + if isinstance(edit, dict): + fields = cast(Mapping[str, Any], edit) + text = _replace( + text, + str(fields.get("old_string") or ""), + str(fields.get("new_string") or ""), + bool(fields.get("replace_all")), + ) + return [PlannedEdit(path, text)] + return [PlannedEdit(path, None)] + + +def _apply_patch(patch: str, cwd: Path) -> Iterator[PlannedEdit]: + """Files touched by a Codex apply_patch envelope (``*** Begin Patch``). + + Add File yields the added lines. Update File applies each hunk by finding + its context and removed lines in the current text; a hunk that doesn't + match contributes its added lines at the end instead, which errs towards + judging the new text. Deleted files yield nothing.""" + path: Path | None = None + source: Path | None = None + added: list[str] | None = None + hunks: list[list[str]] = [] + + def flush() -> Iterator[PlannedEdit]: + if path is None: + return + if added is not None: + yield PlannedEdit(path, "".join(f"{line}\n" for line in added)) + else: + yield PlannedEdit(path, _apply_hunks(_current(source or path), hunks)) + + for line in patch.splitlines(): + header = line.strip() + for prefix in ("*** Add File: ", "*** Update File: ", "*** Delete File: "): + if header.startswith(prefix): + yield from flush() + target = cwd / header[len(prefix) :].strip() + path = source = None if prefix == "*** Delete File: " else target + added = [] if prefix == "*** Add File: " else None + hunks = [] + break + else: + if header.startswith("*** Move to: ") and path is not None: + path = cwd / header[len("*** Move to: ") :].strip() + elif header.startswith("***"): + continue + elif added is not None: + added.append(line[1:] if line.startswith("+") else line) + elif path is not None: + if line.startswith("@@"): + hunks.append([]) + elif line[:1] in (" ", "+", "-") or line == "": + if not hunks: + hunks.append([]) + hunks[-1].append(line or " ") + yield from flush() + + +def _apply_hunks(text: str, hunks: list[list[str]]) -> str: + lines = text.splitlines() + cursor = 0 + tail: list[str] = [] + for hunk in hunks: + before = [line[1:] for line in hunk if line[0] in " -"] + after = [line[1:] for line in hunk if line[0] in " +"] + for start in range(cursor, len(lines) - len(before) + 1): + if lines[start : start + len(before)] == before: + lines[start : start + len(before)] = after + cursor = start + len(after) + break + else: + tail.extend(line[1:] for line in hunk if line[0] == "+") + return "".join(f"{line}\n" for line in lines + tail) + + +# --- Session base -------------------------------------------------------------- + + +def load_sessions(root: Path) -> dict[str, dict[str, Any]]: + """session id -> what session_start.py recorded for it.""" + try: + data = json.loads((root / SESSION_BASE).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + sessions = cast(dict[str, Any], data).get("sessions") if isinstance(data, dict) else None + if not isinstance(sessions, dict): + return {} + return { + str(key): cast(dict[str, Any], value) + for key, value in cast(dict[Any, Any], sessions).items() + if isinstance(value, dict) + } + + +def save_sessions(root: Path, sessions: Mapping[str, Mapping[str, Any]]) -> None: + """Keep the newest SESSION_LIMIT entries (dicts keep insertion order).""" + kept = dict(list(sessions.items())[-SESSION_LIMIT:]) + target = root / SESSION_BASE + target.parent.mkdir(parents=True, exist_ok=True) + scratch = target.with_name(f"{target.name}.{os.getpid()}.tmp") + scratch.write_text(json.dumps({"sessions": kept}, indent=1, sort_keys=True) + "\n", encoding="utf-8") + os.replace(scratch, target) + + +# --- Guard problems ------------------------------------------------------------ + + +def changed_paths(root: Path, base: str) -> set[str]: + """Tracked paths that differ from base (commits and local edits), plus + untracked, non-ignored files.""" + changed: set[str] = set() + for args in ( + ("diff", "--name-only", "--no-renames", "-z", base), + ("ls-files", "--others", "--exclude-standard", "-z"), + ): + changed |= {path for path in (git(root, *args) or "").split("\0") if path} + return changed + + +def guard_problems(root: Path, paths: set[str], base: str | None) -> dict[str, str]: + """Pattern guard problems for the given paths, keyed so that two runs can + be compared: a finding the baseline doesn't list, a baseline entry for one + of the paths that no longer fires, and a baseline entry added since base. + Values are the rendered messages.""" + tree = pattern_guard.Tree(root) + scanned = sorted(path for path in paths if tree.is_scanned(path)) + findings = pattern_guard.scan(tree, scanned) + baseline_path = root / pattern_guard.BASELINE + try: + baseline = pattern_guard.parse_baseline(baseline_path.read_text(encoding="utf-8"), str(pattern_guard.BASELINE)) + except FileNotFoundError: + baseline = {} + problems: dict[str, str] = {} + for finding in findings: + if (finding.path, finding.rule) not in baseline: + problems.setdefault(f"new {finding.path} {finding.rule}", finding.render()) + found = {(finding.path, finding.rule) for finding in findings} + for (path, rule), number in baseline.items(): + if path in paths and (path, rule) not in found: + problems[f"stale {path} {rule}"] = ( + f"{pattern_guard.BASELINE}:{number}: {rule} stale entry, {path} no longer matches\n" + " instead: delete the line (the baseline only shrinks)" + ) + before = pattern_guard.committed_baseline(tree, base) if base else None + grown = [entry for entry in baseline.items() if before is not None and entry[0] not in before] + for (path, rule), number in grown: + problems[f"grown {path} {rule}"] = ( + f"{pattern_guard.BASELINE}:{number}: {rule} new baseline entry for {path}\n" + " instead: fix the finding; the baseline only shrinks" + ) + return problems + + +def hooks_status(root: Path) -> str | None: + """None when the pre-commit git hook is installed, otherwise how to fix it.""" + if not (root / ".pre-commit-config.yaml").is_file(): + return None + bootstrap = ( + "python3 scripts/bootstrap_hooks.py" + if (root / "scripts/bootstrap_hooks.py").is_file() + else "pre-commit install" + ) + if shutil.which("pre-commit") is None: + return f"pre-commit is not installed, so local git hooks don't run. Bootstrap with `{bootstrap}`." + hook = git(root, "rev-parse", "--path-format=absolute", "--git-path", "hooks/pre-commit") + try: + installed = hook is not None and PRE_COMMIT_MARK in Path(hook).read_text(encoding="utf-8", errors="replace") + except OSError: + installed = False + return None if installed else f"The pre-commit git hook is not installed. Bootstrap with `{bootstrap}`." + + +def registries_failure(root: Path) -> str | None: + """scripts/check_registries.py output if it exists and fails, else None.""" + script = root / "scripts/check_registries.py" + if not script.is_file(): + return None + result = subprocess.run( + [sys.executable, str(script)], cwd=root, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True + ) + if result.returncode == 0: + return None + return result.stdout.strip() or "scripts/check_registries.py failed" diff --git a/scripts/agent_hooks/post_edit_check.py b/scripts/agent_hooks/post_edit_check.py new file mode 100755 index 00000000..67d9bacd --- /dev/null +++ b/scripts/agent_hooks/post_edit_check.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""PostToolUse hook: report lint and format findings for the edited files. + +Runs ``pre-commit run --files `` from .pre-commit-config.yaml and hands +the agent the failing hooks' output plus, for any file a fixer would have +changed, the diff it wanted. The live files are never written: pre-commit runs +in a scratch git repo holding copies of the edited files and the lint config, +so a fixer can't make the agent's next Edit fail with "file modified since +read", and a save made by someone else meanwhile can't be rolled back. The +agent applies the fix itself. + +The two local hooks need the real tree, so they run differently: +pattern-guard is skipped (pre_edit_guard and stop_check cover it), and +check_pinned_bytes.py runs read-only against the real files. + +Feedback goes out as ``hookSpecificOutput.additionalContext`` JSON on stdout, +which Claude Code and Codex both add next to the tool result. Always exits 0. + +Without pre-commit, its config or the installed git hook, it prints one line +(the agent's debug log) and exits instead of spending the hook timeout +installing hook environments. session_start.py already told the agent how to +bootstrap. +""" + +# pyright: strict + +from __future__ import annotations + +import difflib +import json +import os +from pathlib import Path +import re +import shutil +import signal +import subprocess +import sys +import tempfile + +import hooklib + +# Below the 30s hook timeout in .claude/settings.json and .codex/hooks.json. +PRE_COMMIT_TIMEOUT = 25 +# Files the configured hooks read for their settings, copied into the scratch repo. +CONFIG_FILES = ( + ".pre-commit-config.yaml", + "pyproject.toml", + ".editorconfig", + ".clang-format", + ".clang-tidy", + ".gersemirc", +) +# Local hooks that read the rest of the tree; see the module docstring. +SCRATCH_SKIP = "pattern-guard,pinned-bytes" +CONTEXT_LIMIT = 6000 +HOOK_LINE = re.compile(r"^(.+?)\.{3,}(\(no files to check\))?(Passed|Failed|Skipped)$") + + +def wanted_change(name: str, before: bytes, after: bytes) -> str | None: + """The unified diff a fixer made to a scratch copy, or None if it left it alone.""" + if after == before: + return None + old = before.decode("utf-8", errors="replace").splitlines(keepends=True) + new = after.decode("utf-8", errors="replace").splitlines(keepends=True) + return "".join(difflib.unified_diff(old, new, f"a/{name}", f"b/{name}")) + + +def run_in_scratch(root: Path, relative: list[str]) -> tuple[str, bool, list[str]]: + """Run pre-commit over copies of the files; returns (output, failed, diffs).""" + with tempfile.TemporaryDirectory(prefix="dkr-post-edit-") as temporary: + scratch = Path(temporary) + originals: dict[str, bytes] = {} + for name in (*CONFIG_FILES, *relative): + source = root / name + if source.is_file(): + target = scratch / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + if name in relative: + originals[name] = source.read_bytes() + environment = {**os.environ, "SKIP": SCRATCH_SKIP, "GIT_CONFIG_GLOBAL": os.devnull} + for command in (["git", "init", "-q"], ["git", "add", "--", *originals]): + subprocess.run(command, cwd=scratch, env=environment, capture_output=True, check=True) + output, failed = run_process_tree( + ["pre-commit", "run", "--color", "never", "--files", *originals], scratch, PRE_COMMIT_TIMEOUT, environment + ) + diffs = [ + diff + for name, before in originals.items() + if (diff := wanted_change(name, before, (scratch / name).read_bytes())) is not None + ] + return output, failed, diffs + + +def pinned_bytes_findings(root: Path, relative: list[str]) -> str | None: + """check_pinned_bytes.py's report for these files, read-only, or None if clean.""" + script = root / "scripts/check_pinned_bytes.py" + if not script.is_file(): + return None + result = subprocess.run( + [sys.executable, str(script), *relative], cwd=root, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True + ) + if result.returncode == 0: + return None + return result.stdout.strip() or "check_pinned_bytes.py failed" + + +def run_process_tree( + command: list[str], cwd: Path, timeout: float, env: dict[str, str] | None = None +) -> tuple[str, bool]: + """Run command and return (output, failed); on timeout kill it and every child. + + pre-commit launches the formatters as child processes. Killing only + pre-commit on timeout would leave a fixer running after this returns (and + holding the scratch directory open), so the command gets its own process + group (session on POSIX) and the whole group is killed and reaped first. + """ + if sys.platform == "win32": + process = subprocess.Popen( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env=env, + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, + ) + else: + process = subprocess.Popen( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env=env, + start_new_session=True, + ) + try: + output, _ = process.communicate(timeout=timeout) + return output, process.returncode != 0 + except subprocess.TimeoutExpired: + if sys.platform == "win32": + subprocess.run(["taskkill", "/F", "/T", "/PID", str(process.pid)], capture_output=True, check=False) + else: + os.killpg(process.pid, signal.SIGKILL) + process.communicate() + return f"pre-commit did not finish in {timeout:g}s; run it yourself.", True + + +def failing_sections(output: str) -> str: + """Only the output of hooks that failed (pre-commit prints every hook).""" + kept: list[str] = [] + keep = False + for line in output.splitlines(): + status = HOOK_LINE.match(line.strip()) + if status: + keep = status.group(3) == "Failed" + if keep: + kept.append(line) + return "\n".join(kept) + + +def main() -> int: + payload = hooklib.read_payload() + root = hooklib.project_root(payload) + if root is None: + return 0 + paths = [edit.path for edit in hooklib.planned_edits(payload) if edit.path.is_file()] + relative = [rel for rel in (hooklib.relative(root, path) for path in paths) if rel is not None] + if not relative: + return 0 + status = hooklib.hooks_status(root) + if not (root / ".pre-commit-config.yaml").is_file() or status: + print(status or "post_edit_check: no .pre-commit-config.yaml, nothing to check") + return 0 + output, failed, diffs = run_in_scratch(root, relative) + pinned = pinned_bytes_findings(root, relative) + if not failed and not diffs and not pinned: + return 0 + parts = [f"pre-commit findings for {', '.join(relative)} (nothing was rewritten; fix them yourself):"] + if diffs: + parts.append("A fixer wanted these changes:\n" + "\n".join(diffs)) + sections = failing_sections(output) if failed else "" + if sections or (failed and not diffs): + parts.append(sections or output.strip()) + if pinned: + parts.append(pinned) + context = "\n\n".join(parts) + if len(context) > CONTEXT_LIMIT: + context = context[:CONTEXT_LIMIT] + "\n[truncated; run pre-commit run --files on the file for the rest]" + print(json.dumps({"hookSpecificOutput": {"hookEventName": "PostToolUse", "additionalContext": context}})) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except OSError as error: + print(f"post_edit_check: skipped ({error})") + raise SystemExit(0) from None diff --git a/scripts/agent_hooks/pre_edit_guard.py b/scripts/agent_hooks/pre_edit_guard.py new file mode 100755 index 00000000..cb34e424 --- /dev/null +++ b/scripts/agent_hooks/pre_edit_guard.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""PreToolUse hook: refuse an edit that would add a pattern-guard violation. + +Reconstructs the content each edited file would have after the tool call and +runs the pattern guard rules on it (scripts/pattern_guard.py check_content). +A finding that architecture/pattern-guard-baseline.txt already lists for that +file is tolerated, so editing a grandfathered pipeline still works. + +Two rules are left to the Stop hook, pre-commit and CI, because a correct +change trips them halfway through: PG007 (a dependency patch whose manifest +sha256 is updated after the patch) and PG013 on a ``*.patch`` file (a new +dependency dir is registered in the manifest after its first patch). PG008 +still stops a patch against first-party code. + +This is fast feedback, not the gate. A file written through a shell command +skips it; the Stop hook, the git pre-commit hook and CI see the tree. + +Exit status: 0 allow, 2 block (the reason is on stderr). +""" + +# pyright: strict + +from __future__ import annotations + +import sys + +import hooklib + +import pattern_guard + +DEFERRED_RULES = {"PG007"} + + +def blocking(edit: hooklib.PlannedEdit) -> list[pattern_guard.Finding]: + """Findings that should stop this edit.""" + root = hooklib.checkout_of(edit.path) + if root is None: + return [] + path = hooklib.relative(root, edit.path) + if path is None: + return [] + text = edit.text + if text is None: + current = root / path + text = current.read_text(encoding="utf-8", errors="replace") if current.is_file() else "" + try: + baseline = pattern_guard.parse_baseline( + (root / pattern_guard.BASELINE).read_text(encoding="utf-8"), str(pattern_guard.BASELINE) + ) + except FileNotFoundError: + baseline = {} + return [ + finding + for finding in pattern_guard.check_content(root, path, text) + if (finding.path, finding.rule) not in baseline + and finding.rule not in DEFERRED_RULES + and not (finding.rule == "PG013" and finding.path.endswith(".patch")) + ] + + +def main() -> int: + payload = hooklib.read_payload() + findings = [finding for edit in hooklib.planned_edits(payload) for finding in blocking(edit)] + if not findings: + return 0 + print("pattern-guard: this edit brings back a patch-layer mechanism (see AGENTS.md).", file=sys.stderr) + for finding in findings: + print(finding.render(), file=sys.stderr) + print( + "Edit the first-party source in place instead. Rule details: scripts/pattern_guard.py --explain PGnnn", + file=sys.stderr, + ) + return 2 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (pattern_guard.GuardError, OSError, UnicodeError) as error: + # A broken checkout must not wedge the agent; the Stop hook and CI still run. + print(f"pre_edit_guard: skipped ({error})", file=sys.stderr) + raise SystemExit(0) from None diff --git a/scripts/agent_hooks/run b/scripts/agent_hooks/run new file mode 100755 index 00000000..893f461b --- /dev/null +++ b/scripts/agent_hooks/run @@ -0,0 +1,26 @@ +#!/bin/sh +# Run one agent hook script (scripts/agent_hooks/.py) with the first +# Python 3.10+ found, passing the hook payload through on stdin. +# +# Order: py -3 first, because on Windows a bare `python` can be the Microsoft +# Store stub; then python, then python3. Each candidate is probed with stdin +# closed so the probe can't consume the payload. +# +# Usage: run [args...] + +dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) || exit 1 +script=$1 +shift +case $script in +*/* | '') echo "agent hooks: expected a script name in $dir, got '$script'" >&2 && exit 1 ;; +esac + +for candidate in "py -3" python python3; do + # shellcheck disable=SC2086 # "py -3" is a command plus argument + if $candidate -c 'import sys; sys.exit(sys.version_info < (3, 10))' /dev/null 2>&1; then + exec $candidate "$dir/$script" "$@" + fi +done +# Not exit 2: a missing interpreter must not block the agent's tool calls. +echo "agent hooks: no Python 3.10+ found (tried py -3, python, python3)" >&2 +exit 1 diff --git a/scripts/agent_hooks/session_start.py b/scripts/agent_hooks/session_start.py new file mode 100755 index 00000000..d38ca665 --- /dev/null +++ b/scripts/agent_hooks/session_start.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""SessionStart hook: put the pattern in context and record the session base. + +Prints the "The pattern" section of AGENTS.md (both agents add SessionStart +stdout to the model's context), plus a line if the pre-commit git hook is not +installed. + +Records, per session id, the HEAD the session started from and the pattern +guard problems already present in locally changed files, in +.claude/session-base (gitignored). stop_check.py reports only problems that +are not in that record, so a dirty tree at startup never blocks a stop. A +resumed or compacted session keeps its original record. + +Always exits 0. +""" + +# pyright: strict + +from __future__ import annotations + +from pathlib import Path +import re +import sys +import time + +import hooklib + + +def pattern_summary(root: Path) -> str: + """The "## The pattern" section of AGENTS.md, without its heading.""" + try: + text = (root / "AGENTS.md").read_text(encoding="utf-8") + except OSError: + return "Read AGENTS.md before editing; it is the project guidance." + match = re.search(r"^## The pattern[^\n]*\n(.*?)(?=^## |\Z)", text, re.MULTILINE | re.DOTALL) + return match.group(1).strip() if match else "Read AGENTS.md before editing; it is the project guidance." + + +def record_base(root: Path, session: str) -> None: + sessions = hooklib.load_sessions(root) + if session in sessions: + return + head = hooklib.git(root, "rev-parse", "--verify", "HEAD") + if head is None: + return + dirty = hooklib.changed_paths(root, head) + sessions[session] = { + "head": head, + "tree": hooklib.git(root, "rev-parse", f"{head}^{{tree}}"), + "started": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "problems": sorted(hooklib.guard_problems(root, dirty, head)), + "registries_failed": hooklib.registries_failure(root) is not None, + } + hooklib.save_sessions(root, sessions) + + +def main() -> int: + payload = hooklib.read_payload() + root = hooklib.project_root(payload) + if root is None: + return 0 + print("DKR-R project rules (from AGENTS.md, which is authoritative):") + print(pattern_summary(root)) + print("Before claiming done: python3 scripts/check.py --suite guard,tooling") + status = hooklib.hooks_status(root) + if status: + print(status) + session = payload.get("session_id") + if isinstance(session, str) and session: + try: + record_base(root, session) + except (hooklib.pattern_guard.GuardError, OSError) as error: + print(f"session_start: could not record the session base ({error})", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/agent_hooks/stop_check.py b/scripts/agent_hooks/stop_check.py new file mode 100755 index 00000000..ecd33424 --- /dev/null +++ b/scripts/agent_hooks/stop_check.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Stop hook: don't let the agent finish with a new pattern-guard problem. + +Compares the work tree with the HEAD recorded by session_start.py and runs the +pattern guard on every path changed since then (commits, local edits and +untracked files). It reports only problems the session introduced: a finding +the baseline doesn't list, a stale baseline entry for a changed file, or a +baseline entry added since the session base. Problems already present when the +session started are subtracted, and untouched baselined files are never +scanned. scripts/check_registries.py runs too once it exists, and blocks only +if it passed when the session started and fails now. + +When ``stop_hook_active`` is set the agent is already continuing because of +this hook, so it exits 0: an unfixable finding never loops the session. The +git pre-commit hook and CI remain the gates. + +Without a recorded session (hooks enabled mid-session, or a Codex session that +started before the hooks were trusted), the base is HEAD. + +Exit status: 0 allow, 2 keep working (the problems are on stderr). +""" + +# pyright: strict + +from __future__ import annotations + +import sys +from typing import cast + +import hooklib + + +def main() -> int: + payload = hooklib.read_payload() + if payload.get("stop_hook_active"): + return 0 + root = hooklib.project_root(payload) + if root is None: + return 0 + session = hooklib.load_sessions(root).get(str(payload.get("session_id")), {}) + base = session.get("head") or hooklib.git(root, "rev-parse", "--verify", "HEAD") + if not isinstance(base, str) or hooklib.git(root, "cat-file", "-e", f"{base}^{{commit}}") is None: + return 0 + changed = hooklib.changed_paths(root, base) + if not changed: + return 0 + recorded = session.get("problems") + known = {str(key) for key in cast(list[object], recorded)} if isinstance(recorded, list) else set[str]() + problems = {key: text for key, text in hooklib.guard_problems(root, changed, base).items() if key not in known} + messages = [problems[key] for key in sorted(problems)] + registries = hooklib.registries_failure(root) + if registries is not None and not session.get("registries_failed"): + messages.append(registries) + if not messages: + return 0 + print("This session added pattern-guard problems; fix them before finishing (see AGENTS.md):", file=sys.stderr) + for message in messages: + print(message, file=sys.stderr) + print("Check with: python3 scripts/check.py --suite guard", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (hooklib.pattern_guard.GuardError, OSError, UnicodeError) as error: + print(f"stop_check: skipped ({error})", file=sys.stderr) + raise SystemExit(0) from None diff --git a/scripts/apply_dependency_patches.py b/scripts/apply_dependency_patches.py index 419fee58..6bd3c150 100755 --- a/scripts/apply_dependency_patches.py +++ b/scripts/apply_dependency_patches.py @@ -12,11 +12,11 @@ import hashlib import json import os +from pathlib import Path import shutil import subprocess import sys import tempfile -from pathlib import Path def git(repo: Path, *args: str, index: Path | None = None) -> str: @@ -25,8 +25,9 @@ def git(repo: Path, *args: str, index: Path | None = None) -> str: env = dict(os.environ) if index is not None: env["GIT_INDEX_FILE"] = str(index) - return subprocess.run(["git", "-C", str(repo), *args], env=env, check=True, - stdout=subprocess.PIPE, text=True).stdout.strip() + return subprocess.run( + ["git", "-C", str(repo), *args], env=env, check=True, stdout=subprocess.PIPE, text=True + ).stdout.strip() def index_tree(repo: Path, scratch_index: Path) -> str: @@ -41,8 +42,9 @@ def index_tree(repo: Path, scratch_index: Path) -> str: return git(repo, "write-tree", index=scratch_index) -def patch_trees(repo: Path, patches: list[Path], expected_index: Path, - current_index: Path, states: list[str] | None = None) -> tuple[str, str]: +def patch_trees( + repo: Path, patches: list[Path], expected_index: Path, current_index: Path, states: list[str] | None = None +) -> tuple[str, str]: """Compute reviewed and working trees without touching the real index.""" git(repo, "read-tree", "HEAD", index=expected_index) trees = [git(repo, "write-tree", index=expected_index)] @@ -58,8 +60,9 @@ def patch_trees(repo: Path, patches: list[Path], expected_index: Path, # Files the patches create are untracked in the checkout; stage the ones # present so they take part in the comparison (lexists: symlinks too). - added = "\0".join(git(repo, "diff", "--name-only", "-z", "--no-renames", - "--diff-filter=A", "HEAD", tree) for tree in trees) + added = "\0".join( + git(repo, "diff", "--name-only", "-z", "--no-renames", "--diff-filter=A", "HEAD", tree) for tree in trees + ) git(repo, "read-tree", "HEAD", index=current_index) git(repo, "add", "-u", index=current_index) present = [p for p in added.split("\0") if p and os.path.lexists(repo / p)] @@ -92,21 +95,20 @@ def is_reviewed_checkout(root: Path, repo: Path, commit: str) -> bool: data = json.loads(manifest.read_text(encoding="utf-8")) if data.get("schemaVersion") != 1: raise SystemExit("Unsupported patch manifest schema") - matches = [d for d in data["dependencies"] - if (root / d["repositoryPath"]).resolve() == repo.resolve()] + matches = [d for d in data["dependencies"] if (root / d["repositoryPath"]).resolve() == repo.resolve()] if len(matches) != 1 or matches[0]["expectedCommit"].lower() != commit.lower(): return False reviewed = checked_patches(root, matches[0]) with tempfile.TemporaryDirectory() as scratch: - expected, current = patch_trees(repo, reviewed, Path(scratch) / "expected", - Path(scratch) / "current") + expected, current = patch_trees(repo, reviewed, Path(scratch) / "expected", Path(scratch) / "current") if current != expected: return False base = git(repo, "rev-parse", "HEAD^{tree}") if index_tree(repo, Path(scratch) / "staged") not in (base, expected): return False - added = set(git(repo, "diff", "--name-only", "-z", "--no-renames", - "--diff-filter=A", "HEAD", expected).split("\0")) + added = set( + git(repo, "diff", "--name-only", "-z", "--no-renames", "--diff-filter=A", "HEAD", expected).split("\0") + ) untracked = set(git(repo, "ls-files", "--others", "--exclude-standard", "-z").split("\0")) if untracked - added - {""}: return False @@ -121,34 +123,35 @@ def is_reviewed_checkout(root: Path, repo: Path, commit: str) -> bool: child = repo / path if mode != "160000" or not (child / ".git").exists(): continue - if git(child, "status", "--porcelain", "--untracked-files=all", - "--ignore-submodules=none") and not is_reviewed_checkout(root, child, child_commit): + if git( + child, "status", "--porcelain", "--untracked-files=all", "--ignore-submodules=none" + ) and not is_reviewed_checkout(root, child, child_commit): return False return True -def plan_dependency(repo: Path, patches: list[Path], expected_index: Path, - current_index: Path, repair: bool = False) -> tuple[list[str], list[str]]: +def plan_dependency( + repo: Path, patches: list[Path], expected_index: Path, current_index: Path, repair: bool = False +) -> tuple[list[str], list[str]]: states: list[str] = [] expected, current = patch_trees(repo, patches, expected_index, current_index, states) managed = set() for tree in states: - managed.update(git(repo, "diff", "--name-only", "--no-renames", "-z", - "HEAD", tree).split("\0")) + managed.update(git(repo, "diff", "--name-only", "--no-renames", "-z", "HEAD", tree).split("\0")) for tree in (current, index_tree(repo, current_index.with_name(current_index.name + "-staged"))): - changed = set(git(repo, "diff", "--name-only", "--no-renames", "-z", - "HEAD", tree).split("\0")) + changed = set(git(repo, "diff", "--name-only", "--no-renames", "-z", "HEAD", tree).split("\0")) unrelated = sorted(changed - managed - {""}) if unrelated: - raise SystemExit(f"{repo}: local edits outside the patch set; commit or stash: " - + ", ".join(unrelated)) + raise SystemExit(f"{repo}: local edits outside the patch set; commit or stash: " + ", ".join(unrelated)) if tree not in states and not repair: - raise SystemExit(f"{repo}: unrecognized local edits in managed patch paths; " - "commit/stash them, or use --repair to discard them") - paths = [p for p in git(repo, "diff", "--name-only", "--no-renames", "-z", - current, expected).split("\0") if p] - present = set(git(repo, "ls-tree", "-r", "-z", "--name-only", expected, - "--", *paths).split("\0")) if paths else set() + raise SystemExit( + f"{repo}: unrecognized local edits in managed patch paths; " + "commit/stash them, or use --repair to discard them" + ) + paths = [p for p in git(repo, "diff", "--name-only", "--no-renames", "-z", current, expected).split("\0") if p] + present = ( + set(git(repo, "ls-tree", "-r", "-z", "--name-only", expected, "--", *paths).split("\0")) if paths else set() + ) return [p for p in paths if p not in present], [p for p in paths if p in present] @@ -168,8 +171,9 @@ def sync_dependency(repo: Path, patches: list[Path], repair: bool = False) -> st return execute_plan(repo, expected_index, plan) -def apply_manifest(root: Path, manifest_path: Path, check: bool = False, - repair: bool = False, require_all: bool = False) -> int: +def apply_manifest( + root: Path, manifest_path: Path, check: bool = False, repair: bool = False, require_all: bool = False +) -> int: data = json.loads(manifest_path.read_text(encoding="utf-8")) if data.get("schemaVersion") != 1: raise SystemExit("Unsupported patch manifest schema") @@ -205,13 +209,27 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--root", type=Path, default=root) parser.add_argument("--manifest", type=Path) - parser.add_argument("--require-all", action="store_true", help="Fail if any manifest checkout is absent (used by Windows preparation).") + parser.add_argument( + "--require-all", + action="store_true", + help="Fail if any manifest checkout is absent (used by Windows preparation).", + ) mode = parser.add_mutually_exclusive_group() - mode.add_argument("--check", action="store_true", help="Validate the full ordered patch stack without changing checkout files or the real index.") - mode.add_argument("--repair", action="store_true", help="Discard unknown working-tree edits on managed patch paths; unrelated tracked edits remain protected. The real index is preserved.") + mode.add_argument( + "--check", + action="store_true", + help="Validate the full ordered patch stack without changing checkout files or the real index.", + ) + mode.add_argument( + "--repair", + action="store_true", + help="Discard unknown working-tree edits on managed patch paths; unrelated tracked edits remain protected. The real index is preserved.", + ) args = parser.parse_args() root = args.root.resolve() # patch paths are passed to `git -C ` - return apply_manifest(root, args.manifest or root / "patches" / "manifest.json", args.check, args.repair, args.require_all) + return apply_manifest( + root, args.manifest or root / "patches" / "manifest.json", args.check, args.repair, args.require_all + ) if __name__ == "__main__": diff --git a/scripts/audio_queue_policy.py b/scripts/audio_queue_policy.py index ebe5c5f9..4175390c 100644 --- a/scripts/audio_queue_policy.py +++ b/scripts/audio_queue_policy.py @@ -1,31 +1,39 @@ """Register retail audio pools through checked Patch Pipeline output only.""" + import copy import struct + +from compose_legacy_mod_policy import elf_functions, elf_sections from legacy_character_presentation_policy import reviewed_revision -from compose_legacy_mod_policy import elf_sections, elf_functions def compose_audio_queue(policy, elf, revision): if reviewed_revision(elf) != revision: - raise ValueError('Audio queue ownership requires a reviewed retail ELF') + raise ValueError("Audio queue ownership requires a reviewed retail ELF") symbols = elf_functions(elf, (1, 2)) - matches = symbols.get('alEvtqNew', set()) + matches = symbols.get("alEvtqNew", set()) if len(matches) != 1: - raise ValueError('Ambiguous audio queue initialization') + raise ValueError("Ambiguous audio queue initialization") entry, size = next(iter(matches)) - words = {base+i: struct.unpack_from('>I', data, i)[0] - for base, data in elf_sections(elf) for i in range(0, len(data), 4)} + words = { + base + i: struct.unpack_from(">I", data, i)[0] + for base, data in elf_sections(elf) + for i in range(0, len(data), 4) + } if size < 4 or words.get(entry) != 0x27BDFFD8: - raise ValueError('Audio queue initialization signature changed') - hook = dict(function='alEvtqNew', beforeVram=hex(entry), - text='extern void dkr_audio_event_queue_initialize(uint8_t*, recomp_context*); ' - 'dkr_audio_event_queue_initialize(rdram, ctx);', - reason='Register the exact retail event pool before initialization; bounded recovery never invents allocation ownership.') + raise ValueError("Audio queue initialization signature changed") + hook = dict( + function="alEvtqNew", + beforeVram=hex(entry), + text="extern void dkr_audio_event_queue_initialize(uint8_t*, recomp_context*); " + "dkr_audio_event_queue_initialize(rdram, ctx);", + reason="Register the exact retail event pool before initialization; bounded recovery never invents allocation ownership.", + ) result = copy.deepcopy(policy) - hooks = result.setdefault('functionHooks', []) - existing = [x for x in hooks if int(x['beforeVram'], 0) == entry] + hooks = result.setdefault("functionHooks", []) + existing = [x for x in hooks if int(x["beforeVram"], 0) == entry] if existing and existing != [hook]: - raise ValueError('Conflicting audio queue initialization hook') + raise ValueError("Conflicting audio queue initialization hook") if not existing: hooks.append(hook) return result diff --git a/scripts/audit_replay_boundary.py b/scripts/audit_replay_boundary.py index e48a99d4..1b2bd890 100644 --- a/scripts/audit_replay_boundary.py +++ b/scripts/audit_replay_boundary.py @@ -11,7 +11,6 @@ from pathlib import Path import re - FUNCTION = re.compile(r"RECOMP_FUNC\s+void\s+(\w+)\s*\(") CALL = re.compile(r"\b(\w+)\s*\(\s*rdram\s*[,)]") @@ -23,7 +22,7 @@ def audit(directory, root): source = file.read_text(encoding="utf-8") matches = list(FUNCTION.finditer(source)) for index, match in enumerate(matches): - body = source[match.end(): matches[index + 1].start() if index + 1 < len(matches) else len(source)] + body = source[match.end() : matches[index + 1].start() if index + 1 < len(matches) else len(source)] body = re.sub(r"/\*.*?\*/|//[^\n]*", "", body, flags=re.S) # Function declarations inside hooks are not invocations because # their first parameter is a type, not the rdram variable. @@ -52,10 +51,12 @@ def audit(directory, root): "directly_reachable_guest_functions": sum(name in functions for name in seen), "native_boundaries": dict(sorted(native.items())), "indirect_dispatch": dict(sorted(indirect.items())), - "limitations": ["Only direct calls whose first argument is rdram are followed.", - "Indirect calls are reported, not assumed safe or fully enumerated.", - "Native helper calls without rdram and shared-memory ownership need a separate audit.", - "Reachability includes branches, not a claim that every call runs on every frame."] + "limitations": [ + "Only direct calls whose first argument is rdram are followed.", + "Indirect calls are reported, not assumed safe or fully enumerated.", + "Native helper calls without rdram and shared-memory ownership need a separate audit.", + "Reachability includes branches, not a claim that every call runs on every frame.", + ], } @@ -70,9 +71,11 @@ def main(): if args.output: # This is a derived audit artifact, never a generated-code rewrite. args.output.write_text(encoded + "\n", encoding="utf-8") - print(f"{args.root}: {result['directly_reachable_guest_functions']} directly reachable guest functions; " - f"{len(result['native_boundaries'])} native boundaries; " - f"{len(result['indirect_dispatch'])} indirect-dispatch functions") + print( + f"{args.root}: {result['directly_reachable_guest_functions']} directly reachable guest functions; " + f"{len(result['native_boundaries'])} native boundaries; " + f"{len(result['indirect_dispatch'])} indirect-dispatch functions" + ) else: print(encoded) diff --git a/scripts/bootstrap_dependencies.py b/scripts/bootstrap_dependencies.py index 7138f00e..392d4ace 100644 --- a/scripts/bootstrap_dependencies.py +++ b/scripts/bootstrap_dependencies.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Clone and verify the exact external revisions used by DKR-R.""" + from __future__ import annotations import argparse @@ -40,7 +41,13 @@ def ensure_tool(name: str) -> None: def is_dirty(directory: pathlib.Path) -> bool: - return bool(run(["git", "status", "--porcelain", "--untracked-files=all", "--ignore-submodules=none"], cwd=directory, capture=True)) + return bool( + run( + ["git", "status", "--porcelain", "--untracked-files=all", "--ignore-submodules=none"], + cwd=directory, + capture=True, + ) + ) def dirty_children(directory: pathlib.Path) -> list[str]: @@ -68,21 +75,25 @@ def select_entries(entries: list[dict[str, Any]], selected: set[str]) -> list[di while entry.get("managedBy"): entry = parent_entry(entry, entries) wanted.add(entry["name"]) - return sorted((entry for entry in entries if entry["name"] in wanted), - key=lambda entry: len(pathlib.PurePosixPath(entry["destination"]).parts)) + return sorted( + (entry for entry in entries if entry["name"] in wanted), + key=lambda entry: len(pathlib.PurePosixPath(entry["destination"]).parts), + ) def parent_entry(entry: dict[str, Any], entries: list[dict[str, Any]]) -> dict[str, Any]: destination = pathlib.PurePosixPath(entry["destination"]) - parents = [candidate for candidate in entries - if pathlib.PurePosixPath(candidate["destination"]) in destination.parents] + parents = [ + candidate for candidate in entries if pathlib.PurePosixPath(candidate["destination"]) in destination.parents + ] if not parents: raise RuntimeError(f"No locked parent for managed dependency {entry['name']}") return max(parents, key=lambda candidate: len(pathlib.PurePosixPath(candidate["destination"]).parts)) -def prepare_dependency(entry: dict[str, Any], force: bool, - parent: dict[str, Any] | None = None, shallow: bool = False) -> dict[str, str]: +def prepare_dependency( + entry: dict[str, Any], force: bool, parent: dict[str, Any] | None = None, shallow: bool = False +) -> dict[str, str]: destination = ROOT / entry["destination"] repository = entry["repository"] commit = entry["commit"] @@ -98,7 +109,19 @@ def prepare_dependency(entry: dict[str, Any], force: bool, if len(fields) < 3 or fields[0] != "160000" or fields[2].lower() != commit.lower(): raise RuntimeError(f"Parent submodule pin does not match lock for {entry['name']}") if not (destination / ".git").exists(): - run(["git", "submodule", "update", "--init", "--recursive", *(["--depth", "1"] if shallow else []), "--", relative], cwd=parent_path) + run( + [ + "git", + "submodule", + "update", + "--init", + "--recursive", + *(["--depth", "1"] if shallow else []), + "--", + relative, + ], + cwd=parent_path, + ) if run(["git", "rev-parse", "HEAD"], cwd=destination, capture=True).lower() != commit.lower(): raise RuntimeError(f"{entry['name']} submodule is at another revision; restore its locked parent pin first") @@ -118,7 +141,10 @@ def prepare_dependency(entry: dict[str, Any], force: bool, run(["git", "clone", "--filter=blob:none", "--no-checkout", repository, str(destination)]) else: current_remote = run(["git", "remote", "get-url", "origin"], cwd=destination, capture=True) - if current_remote.rstrip("/").removesuffix(".git").lower() != repository.rstrip("/").removesuffix(".git").lower(): + if ( + current_remote.rstrip("/").removesuffix(".git").lower() + != repository.rstrip("/").removesuffix(".git").lower() + ): raise RuntimeError(f"Unexpected origin for {entry['name']}: {current_remote}") dirty = is_dirty(destination) reviewed = dirty and is_reviewed_checkout(ROOT, destination, commit) @@ -129,14 +155,18 @@ def prepare_dependency(entry: dict[str, Any], force: bool, if force and dirty and not reviewed: children = dirty_children(destination) if children: - raise RuntimeError(f"{entry['name']} contains dirty submodule(s): {', '.join(children)}. " - "Commit/stash or explicitly restore changes inside each child before --force; " - "parent reset/clean cannot safely discard nested work.") + raise RuntimeError( + f"{entry['name']} contains dirty submodule(s): {', '.join(children)}. " + "Commit/stash or explicitly restore changes inside each child before --force; " + "parent reset/clean cannot safely discard nested work." + ) run(["git", "reset", "--hard"], cwd=destination) run(["git", "clean", "-fd"], cwd=destination) if is_dirty(destination): - raise RuntimeError(f"{entry['name']} still contains local changes after --force. " - "Inspect git status and handle nested repositories explicitly; none were deleted.") + raise RuntimeError( + f"{entry['name']} still contains local changes after --force. " + "Inspect git status and handle nested repositories explicitly; none were deleted." + ) # A pinned checkout is already resolved, including one with exactly the # reviewed patches. Avoid checkout/fetch churn and support offline reruns. @@ -145,7 +175,10 @@ def prepare_dependency(entry: dict[str, Any], force: bool, run(["git", "fetch", "--no-tags", *(["--depth", "1"] if shallow else []), "origin", commit], cwd=destination) run(["git", "checkout", "--detach", commit], cwd=destination) if recursive: - run(["git", "submodule", "update", "--init", "--recursive", *(["--depth", "1"] if shallow else [])], cwd=destination) + run( + ["git", "submodule", "update", "--init", "--recursive", *(["--depth", "1"] if shallow else [])], + cwd=destination, + ) resolved = run(["git", "rev-parse", "HEAD"], cwd=destination, capture=True) if resolved.lower() != commit.lower(): @@ -155,23 +188,41 @@ def prepare_dependency(entry: dict[str, Any], force: bool, def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--only", action="append", help="Prepare this dependency and its required parents (repeatable).") - parser.add_argument("--force", action="store_true", help="Discard local changes in selected checkouts; dirty nested submodules must be handled explicitly first.") - parser.add_argument("--shallow", action="store_true", help="Fetch only pinned revisions and shallow submodules for compact CI source caches.") + parser.add_argument( + "--only", action="append", help="Prepare this dependency and its required parents (repeatable)." + ) + parser.add_argument( + "--force", + action="store_true", + help="Discard local changes in selected checkouts; dirty nested submodules must be handled explicitly first.", + ) + parser.add_argument( + "--shallow", + action="store_true", + help="Fetch only pinned revisions and shallow submodules for compact CI source caches.", + ) args = parser.parse_args() ensure_tool("git") lock = json.loads(LOCK_FILE.read_text(encoding="utf-8")) selected = set(args.only or []) entries = select_entries(lock["dependencies"], selected) - resolved = [prepare_dependency(entry, args.force, - parent_entry(entry, lock["dependencies"]) if entry.get("managedBy") else None, shallow=args.shallow) - for entry in entries] + resolved = [ + prepare_dependency( + entry, + args.force, + parent_entry(entry, lock["dependencies"]) if entry.get("managedBy") else None, + shallow=args.shallow, + ) + for entry in entries + ] previous = json.loads(RESOLVED_FILE.read_text(encoding="utf-8")) if RESOLVED_FILE.exists() else {} records = {entry["name"]: entry for entry in previous.get("dependencies", [])} records.update({entry["name"]: entry for entry in resolved}) RESOLVED_FILE.parent.mkdir(parents=True, exist_ok=True) - RESOLVED_FILE.write_text(json.dumps({"schemaVersion": 1, "dependencies": list(records.values())}, indent=2) + "\n", encoding="utf-8") + RESOLVED_FILE.write_text( + json.dumps({"schemaVersion": 1, "dependencies": list(records.values())}, indent=2) + "\n", encoding="utf-8" + ) print(f"\nPrepared {len(resolved)} dependency checkout(s).") print(f"Resolution record: {RESOLVED_FILE.relative_to(ROOT)}") return 0 diff --git a/scripts/bootstrap_hooks.py b/scripts/bootstrap_hooks.py new file mode 100755 index 00000000..8d4967ad --- /dev/null +++ b/scripts/bootstrap_hooks.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Install the repository's pre-commit hooks, or say why they were not. + +Setup-macOS.sh, Setup-Linux.sh and Setup-Windows.cmd run this after their +prerequisite checks; the Build entry points run it with --check, which only +prints the install command when the hooks are missing or out of date. It never +fails the caller: every problem is a notice and exit status 0. CI is the gate, +hooks are a convenience. + +pre-commit itself comes from `uv tool install`, then `pipx install`, then +`pip install --user`, whichever works first. + +pre-commit refuses to install while core.hooksPath is set, because git then +runs hooks from that directory and not the repository's own. This installs +into the repository's own hooks directory anyway (blanking core.hooksPath for +the install only). If core.hooksPath points somewhere else, a global hooks +directory say, git won't run them, and this prints the two ways to fix that. +""" + +from __future__ import annotations + +import argparse +import hashlib +import os +from pathlib import Path +import shutil +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +CONFIG = ".pre-commit-config.yaml" +# Written next to the installed hooks; holds the sha256 of the config they were +# installed from so --check can tell when they need reinstalling. +STAMP = ".dkr-bootstrap-version" +INSTALLERS = ( + ("uv", "tool", "install", "pre-commit"), + ("pipx", "install", "pre-commit"), + (sys.executable, "-m", "pip", "install", "--user", "--quiet", "pre-commit"), +) + + +def git(*args: str) -> str | None: + try: + result = subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True) + except OSError: + return None + return result.stdout.strip() if result.returncode == 0 else None + + +def resolve(path: str) -> Path: + return (ROOT / path).resolve() + + +def config_digest() -> str: + return hashlib.sha256((ROOT / CONFIG).read_bytes()).hexdigest() + + +def find_pre_commit() -> list[str] | None: + if found := shutil.which("pre-commit"): + return [found] + # `pip install --user` puts it next to this interpreter's user scripts, + # which are often not on PATH. + probe = subprocess.run([sys.executable, "-m", "pre_commit", "--version"], capture_output=True) + return [sys.executable, "-m", "pre_commit"] if probe.returncode == 0 else None + + +def install_pre_commit() -> list[str] | None: + for command in INSTALLERS: + if not shutil.which(command[0]): + continue + print(f"bootstrap_hooks: installing pre-commit with {command[0]}", flush=True) + if subprocess.run(command).returncode == 0 and (found := find_pre_commit()): + return found + return None + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=(__doc__ or "").splitlines()[0]) + parser.add_argument("--check", action="store_true", help="only report whether the hooks are installed and current") + args = parser.parse_args(argv) + bootstrap = f"{'python' if os.name == 'nt' else 'python3'} scripts/bootstrap_hooks.py" + + if git("rev-parse", "--is-inside-work-tree") != "true": + if not args.check: + print("bootstrap_hooks: not a git checkout, skipping git hooks") + return 0 + common = git("rev-parse", "--git-common-dir") + active = git("rev-parse", "--git-path", "hooks") + if common is None or active is None: + print("bootstrap_hooks: could not locate the git hooks directory, skipping git hooks") + return 0 + # pre-commit always installs into /hooks. --git-path hooks + # is where git runs them from: the same directory (in every worktree) + # unless core.hooksPath points somewhere else. + own, runs_from = resolve(common) / "hooks", resolve(active) + stamp = own / STAMP + + if args.check: + installed = (own / "pre-commit").is_file() and stamp.is_file() + if not installed or stamp.read_text().strip() != config_digest(): + print(f"Git hooks are not installed or are out of date. Install them with: {bootstrap}") + elif runs_from != own: + print(f"Git hooks are installed but core.hooksPath bypasses them here. {bootstrap} prints the fix.") + return 0 + + pre_commit = find_pre_commit() or install_pre_commit() + if pre_commit is None: + print(f"bootstrap_hooks: could not install pre-commit. Run `uv tool install pre-commit`, then {bootstrap}") + return 0 + env = dict(os.environ) + if git("config", "--get", "core.hooksPath") is not None: + # Blank core.hooksPath for pre-commit's "Cowardly refusing" check only; + # it installs into /hooks either way. + index = int(env.get("GIT_CONFIG_COUNT", "0")) + env.update( + { + "GIT_CONFIG_COUNT": str(index + 1), + f"GIT_CONFIG_KEY_{index}": "core.hooksPath", + f"GIT_CONFIG_VALUE_{index}": "", + } + ) + result = subprocess.run([*pre_commit, "install", "--install-hooks"], cwd=ROOT, env=env) + if result.returncode != 0: + print(f"bootstrap_hooks: pre-commit install failed (exit {result.returncode}); rerun {bootstrap} to retry") + return 0 + stamp.write_text(config_digest() + "\n") + if runs_from == own: + print(f"bootstrap_hooks: hooks installed in {own}") + else: + print(not_active(own, runs_from)) + return 0 + + +def not_active(own: Path, runs_from: Path) -> str: + origin = (git("config", "--show-origin", "--get", "core.hooksPath") or str(runs_from)).replace("\t", " ") + return ( + f"bootstrap_hooks: hooks installed in {own}, but core.hooksPath ({origin})\n" + " makes git run hooks from there instead. To use them in this checkout, either\n" + f' git config --local core.hooksPath "{own}"\n' + " (your global hooks then stop running here), or call them from your global\n" + f' pre-commit and pre-push hooks: [ -x "{own}/pre-commit" ] && "{own}/pre-commit" "$@"' + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/camera_obstruction_policy.py b/scripts/camera_obstruction_policy.py index 6b338419..9bd77f40 100644 --- a/scripts/camera_obstruction_policy.py +++ b/scripts/camera_obstruction_policy.py @@ -1,63 +1,87 @@ """Qualify the follow-camera Patch Pipeline ABI against both pinned retail ELFs.""" + import hashlib import struct -from compose_legacy_mod_policy import elf_sections, elf_functions + +from compose_legacy_mod_policy import elf_functions, elf_sections from legacy_character_presentation_policy import reviewed_revision BODY = { - 'us.v77': (0x80057A40, '4ea4efe47c86284398a2498af7eac0ed66112f99dde87153c7efd1b88d59203c'), - 'us.v80': (0x80057A80, '49cf0f61f401286458ff813975d24a0da7b8aaaa36d8c8c88ff582952c2f51d0'), + "us.v77": (0x80057A40, "4ea4efe47c86284398a2498af7eac0ed66112f99dde87153c7efd1b88d59203c"), + "us.v80": (0x80057A80, "49cf0f61f401286458ff813975d24a0da7b8aaaa36d8c8c88ff582952c2f51d0"), } FIELDS = { - 'us.v77': (0x801234EC,0x80121168,0x80120CE0,0x80120D14,0x800DC918,0x8011D508,0x8011D586,0x8011D55C), - 'us.v80': (0x80123A6C,0x801216E8,0x80121260,0x80121294,0x800DCE88,0x8011DA88,0x8011DB06,0x8011DADC), + "us.v77": (0x801234EC, 0x80121168, 0x80120CE0, 0x80120D14, 0x800DC918, 0x8011D508, 0x8011D586, 0x8011D55C), + "us.v80": (0x80123A6C, 0x801216E8, 0x80121260, 0x80121294, 0x800DCE88, 0x8011DA88, 0x8011DB06, 0x8011DADC), } -NAMES = ('gGameMode','gCurrentLevelHeader','gViewportLayout','gCutsceneCameraActive', - 'gCurrentLevelModel','gCameraObject','gDialogueCameraAngle','gCurrentPlayerIndex') +NAMES = ( + "gGameMode", + "gCurrentLevelHeader", + "gViewportLayout", + "gCutsceneCameraActive", + "gCurrentLevelModel", + "gCameraObject", + "gDialogueCameraAngle", + "gCurrentPlayerIndex", +) def verify_camera_obstruction_policy(policy, elf): - hooks = [h for h in policy.get('functionHooks', []) - if 'dkr_resolve_follow_camera' in h.get('text', '')] + hooks = [h for h in policy.get("functionHooks", []) if "dkr_resolve_follow_camera" in h.get("text", "")] if not hooks: return # Older fixtures are allowed; release CMake requires the new hook. revision = reviewed_revision(elf) if revision is None: - raise ValueError('Camera collision requires the reviewed retail ELF') + raise ValueError("Camera collision requires the reviewed retail ELF") start, expected_body = BODY[revision] symbols, sections = elf_functions(elf, (1, 2)), elf_sections(elf) size = 0x674 - if symbols.get('update_player_camera') != {(start, size)}: - raise ValueError('Camera-update function bounds changed') - raw = [data[start-base:start-base+size] for base, data in sections - if base <= start and start+size <= base+len(data)] + if symbols.get("update_player_camera") != {(start, size)}: + raise ValueError("Camera-update function bounds changed") + raw = [ + data[start - base : start - base + size] + for base, data in sections + if base <= start and start + size <= base + len(data) + ] if len(raw) != 1 or hashlib.sha256(raw[0]).hexdigest() != expected_body: - raise ValueError('Camera-update body changed') - signatures = {0:0x27BDFFC8,0x18:0xAFA40038,size-12:0x27BD0038,size-8:0x03E00008,size-4:0} + raise ValueError("Camera-update body changed") + signatures = {0: 0x27BDFFC8, 0x18: 0xAFA40038, size - 12: 0x27BD0038, size - 8: 0x03E00008, size - 4: 0} for offset, word in signatures.items(): - if struct.unpack_from('>I', raw[0], offset)[0] != word: - raise ValueError('Camera stack/original object/return ABI changed') - expected_text = ('extern void dkr_resolve_follow_camera(uint8_t*, recomp_context*, uint32_t); ' - 'dkr_resolve_follow_camera(rdram, ctx, (uint32_t)MEM_W(0x38, ctx->r29));') - if len(hooks) != 1 or hooks[0]['function'] != 'update_player_camera' or \ - int(hooks[0]['beforeVram'], 0) != start+size-12 or hooks[0]['text'] != expected_text: - raise ValueError('Camera hook moved away from the shared stack-release boundary') - for name, address, size in zip(NAMES, FIELDS[revision], (4,4,4,1,4,4,2,4)): - if symbols.get(name) != {(address,size)}: - raise ValueError('Camera field ABI changed: '+name) - site = int(hooks[0]['beforeVram'], 0) - if sum(int(h['beforeVram'],0)==site for h in policy['functionHooks']) != 1 or any( - int(p['vram'],0)==site for p in policy.get('instructionPatches', [])): - raise ValueError('Conflicting camera return owner') + if struct.unpack_from(">I", raw[0], offset)[0] != word: + raise ValueError("Camera stack/original object/return ABI changed") + expected_text = ( + "extern void dkr_resolve_follow_camera(uint8_t*, recomp_context*, uint32_t); " + "dkr_resolve_follow_camera(rdram, ctx, (uint32_t)MEM_W(0x38, ctx->r29));" + ) + if ( + len(hooks) != 1 + or hooks[0]["function"] != "update_player_camera" + or int(hooks[0]["beforeVram"], 0) != start + size - 12 + or hooks[0]["text"] != expected_text + ): + raise ValueError("Camera hook moved away from the shared stack-release boundary") + for name, address, size in zip(NAMES, FIELDS[revision], (4, 4, 4, 1, 4, 4, 2, 4)): + if symbols.get(name) != {(address, size)}: + raise ValueError("Camera field ABI changed: " + name) + site = int(hooks[0]["beforeVram"], 0) + if sum(int(h["beforeVram"], 0) == site for h in policy["functionHooks"]) != 1 or any( + int(p["vram"], 0) == site for p in policy.get("instructionPatches", []) + ): + raise ValueError("Conflicting camera return owner") # The local Modern camera replaces the pose after the hook above and puts # the retail pose back here, at the first instruction, before the next update. - restore = [h for h in policy['functionHooks'] - if 'dkr_modern_camera_restore' in h.get('text', '')] - restore_text = ('extern void dkr_modern_camera_restore(uint8_t*, recomp_context*); ' - 'dkr_modern_camera_restore(rdram, ctx);') - if len(restore) != 1 or restore[0]['function'] != 'update_player_camera' or \ - int(restore[0]['beforeVram'], 0) != start or restore[0]['text'] != restore_text: - raise ValueError('Modern-camera restore must own the camera-update entry') - if sum(int(h['beforeVram'],0)==start for h in policy['functionHooks']) != 1 or any( - int(p['vram'],0)==start for p in policy.get('instructionPatches', [])): - raise ValueError('Conflicting camera entry owner') + restore = [h for h in policy["functionHooks"] if "dkr_modern_camera_restore" in h.get("text", "")] + restore_text = ( + "extern void dkr_modern_camera_restore(uint8_t*, recomp_context*); dkr_modern_camera_restore(rdram, ctx);" + ) + if ( + len(restore) != 1 + or restore[0]["function"] != "update_player_camera" + or int(restore[0]["beforeVram"], 0) != start + or restore[0]["text"] != restore_text + ): + raise ValueError("Modern-camera restore must own the camera-update entry") + if sum(int(h["beforeVram"], 0) == start for h in policy["functionHooks"]) != 1 or any( + int(p["vram"], 0) == start for p in policy.get("instructionPatches", []) + ): + raise ValueError("Conflicting camera entry owner") diff --git a/scripts/camera_plus_pipeline.py b/scripts/camera_plus_pipeline.py index 37a804c7..9f3ed174 100644 --- a/scripts/camera_plus_pipeline.py +++ b/scripts/camera_plus_pipeline.py @@ -4,16 +4,18 @@ This initial native stage builds the isolated presentation domain. Renderer/XR capabilities must not be advertised before their real consumers are connected. """ -from pathlib import Path + import argparse import hashlib import json +from pathlib import Path import shutil import sys + sys.dont_write_bytecode = True ROOT = Path(__file__).resolve().parents[1] -AUTH = ROOT / 'patches/camera-plus' +AUTH = ROOT / "patches/camera-plus" def digest(path): @@ -21,193 +23,253 @@ def digest(path): def authored_inputs(): - return {p.relative_to(ROOT).as_posix(): digest(p) for p in sorted(AUTH.rglob('*')) - if p.is_file() and '__pycache__' not in p.parts} + return { + p.relative_to(ROOT).as_posix(): digest(p) + for p in sorted(AUTH.rglob("*")) + if p.is_file() and "__pycache__" not in p.parts + } def new_output(path, *inputs): path = path.resolve() - if path.exists() or any(path == p.resolve() or p.resolve() in path.parents or path in p.resolve().parents for p in inputs): - raise ValueError('Choose a NEW Camera++ output, outside and not above all inputs') - if any(part.lower() in ('extern', 'recompiledfuncs', 'recompiledpatches') for part in path.parts): - raise ValueError('Camera++ output cannot enter a protected tree') - if not (ROOT / 'build').resolve() in path.parents: - raise ValueError('Camera++ generated output must stay inside the workspace build directory') + if path.exists() or any( + path == p.resolve() or p.resolve() in path.parents or path in p.resolve().parents for p in inputs + ): + raise ValueError("Choose a NEW Camera++ output, outside and not above all inputs") + if any(part.lower() in ("extern", "recompiledfuncs", "recompiledpatches") for part in path.parts): + raise ValueError("Camera++ output cannot enter a protected tree") + if (ROOT / "build").resolve() not in path.parents: + raise ValueError("Camera++ generated output must stay inside the workspace build directory") return path def replace(text, old, new, count=1): if text.count(old) != count: - raise ValueError('Unreviewed Camera++ transformation site: ' + old[:100]) + raise ValueError("Unreviewed Camera++ transformation site: " + old[:100]) return text.replace(old, new) def transform(name, text): - if name == 'magic_code_library.hpp': - text = replace(text, 'approved=false,deterministic=false;', 'approved=false,deterministic=false,presentation=false;') - return replace(text, 'const std::map& modules,std::stop_token stop={})const;', - 'const std::map& modules,std::stop_token stop={},bool presentation=false)const;') - if name == 'magic_code_library.cpp': - text = replace(text, '!item.is_object()||item.size()!=7||!item.contains("id")', - '''!item.is_object()||(item.size()!=7&&item.size()!=8)|| + if name == "magic_code_library.hpp": + text = replace( + text, "approved=false,deterministic=false;", "approved=false,deterministic=false,presentation=false;" + ) + return replace( + text, + "const std::map& modules,std::stop_token stop={})const;", + "const std::map& modules,std::stop_token stop={},bool presentation=false)const;", + ) + if name == "magic_code_library.cpp": + text = replace( + text, + '!item.is_object()||item.size()!=7||!item.contains("id")', + """!item.is_object()||(item.size()!=7&&item.size()!=8)|| (item.size()==8&&(!item.contains("presentation")||!item["presentation"].is_boolean()))|| - (item.size()==7&&item.contains("presentation"))||!item.contains("id")''') - text = replace(text, 'entry.compatibility=compatibility_issue(hash);', - 'entry.presentation=item.value("presentation",false);\n entry.compatibility=compatibility_issue(hash);') - text = replace(text, '{"deterministic",package->deterministic},{"kept_online",kept_online||was_kept}', - '{"deterministic",package->deterministic},{"presentation",package->modules.contains("presentation")},{"kept_online",kept_online||was_kept}') - text = replace(text, 'const std::map& modules,std::stop_token stop)const {', - 'const std::map& modules,std::stop_token stop,bool presentation)const {') - text = replace(text, ' if(p->id!=id)bad("package ID differs from its offline profile");', - ''' if(p->id!=id)bad("package ID differs from its offline profile"); + (item.size()==7&&item.contains("presentation"))||!item.contains("id")""", + ) + text = replace( + text, + "entry.compatibility=compatibility_issue(hash);", + 'entry.presentation=item.value("presentation",false);\n entry.compatibility=compatibility_issue(hash);', + ) + text = replace( + text, + '{"deterministic",package->deterministic},{"kept_online",kept_online||was_kept}', + '{"deterministic",package->deterministic},{"presentation",package->modules.contains("presentation")},{"kept_online",kept_online||was_kept}', + ) + text = replace( + text, + "const std::map& modules,std::stop_token stop)const {", + "const std::map& modules,std::stop_token stop,bool presentation)const {", + ) + text = replace( + text, + ' if(p->id!=id)bad("package ID differs from its offline profile");', + """ if(p->id!=id)bad("package ID differs from its offline profile"); // Derive the domain from authenticated package metadata, never the // mutable catalogue hint. Local cameras cannot enter race/save hashes. - if(p->modules.contains("presentation")!=presentation)continue;''') - return replace(text, 'resolve(packages,enabled,revision,online,modules)', - 'resolve(packages,enabled,revision,presentation?false:online,modules)') - raise ValueError('Unknown Camera++ transformation: ' + name) + if(p->modules.contains("presentation")!=presentation)continue;""", + ) + return replace( + text, + "resolve(packages,enabled,revision,online,modules)", + "resolve(packages,enabled,revision,presentation?false:online,modules)", + ) + raise ValueError("Unknown Camera++ transformation: " + name) def native(baseline, output): # The previous stage checks every copied byte against its authored patches # and current first-party inputs, not merely a self-issued receipt. from character_portrait_pipeline import verify_native + baseline = baseline.resolve(strict=True) verify_native(baseline) - output = new_output(output, baseline, ROOT / 'runtime-recomp/src') - for path in baseline.rglob('*'): + output = new_output(output, baseline, ROOT / "runtime-recomp/src") + for path in baseline.rglob("*"): if path.is_symlink(): - raise ValueError('Native baseline must not contain links') - shutil.copytree(baseline / 'src', output / 'src') - shutil.copytree(ROOT / 'runtime-recomp/tests/replay_probe', output / 'tests/replay_probe') - shutil.copytree(baseline.parent / 'modding', output.parent / 'modding') - for name in ('magic_code_library.hpp', 'magic_code_library.cpp'): - source = baseline / 'src/game/mods' / name - (output / 'src/game/mods' / name).write_text(transform(name, source.read_text()), newline='\n') - sys.path.insert(0,str(AUTH)) - from integration_pipeline import TRANSFORMS,transform as integrate + raise ValueError("Native baseline must not contain links") + shutil.copytree(baseline / "src", output / "src") + shutil.copytree(ROOT / "runtime-recomp/tests/replay_probe", output / "tests/replay_probe") + shutil.copytree(baseline.parent / "modding", output.parent / "modding") + for name in ("magic_code_library.hpp", "magic_code_library.cpp"): + source = baseline / "src/game/mods" / name + (output / "src/game/mods" / name).write_text(transform(name, source.read_text()), newline="\n") + sys.path.insert(0, str(AUTH)) + from integration_pipeline import TRANSFORMS + from integration_pipeline import transform as integrate + for name in sorted(TRANSFORMS): - source=(ROOT/'runtime-recomp'/name) if name.startswith('tests/') else baseline/name - (output/name).write_text(integrate(name,source.read_text(),replace,AUTH),newline='\n') - for path in (AUTH / 'native').iterdir(): + source = (ROOT / "runtime-recomp" / name) if name.startswith("tests/") else baseline / name + (output / name).write_text(integrate(name, source.read_text(), replace, AUTH), newline="\n") + for path in (AUTH / "native").iterdir(): if path.is_file(): - shutil.copyfile(path, output / 'src/game' / path.name) - shutil.copytree(AUTH / 'sdk', output / 'include') - files = {p.relative_to(output).as_posix(): digest(p) for p in sorted(output.rglob('*')) if p.is_file()} - receipt = {'schema': 2, 'baseline': baseline.as_posix(), - 'baseline_receipt': digest(baseline / 'native-receipt.json'), 'files': files, - 'symbols': {p.relative_to(output.parent / 'modding').as_posix(): digest(p) - for p in sorted((output.parent / 'modding').rglob('*')) if p.is_file()}, - 'authored': authored_inputs()} - (output / 'camera-receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') - print('Fresh Camera++ native stage:', output) + shutil.copyfile(path, output / "src/game" / path.name) + shutil.copytree(AUTH / "sdk", output / "include") + files = {p.relative_to(output).as_posix(): digest(p) for p in sorted(output.rglob("*")) if p.is_file()} + receipt = { + "schema": 2, + "baseline": baseline.as_posix(), + "baseline_receipt": digest(baseline / "native-receipt.json"), + "files": files, + "symbols": { + p.relative_to(output.parent / "modding").as_posix(): digest(p) + for p in sorted((output.parent / "modding").rglob("*")) + if p.is_file() + }, + "authored": authored_inputs(), + } + (output / "camera-receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + print("Fresh Camera++ native stage:", output) def verify(output): - receipt = json.loads((output / 'camera-receipt.json').read_text()) - baseline_name=receipt['baseline'] + receipt = json.loads((output / "camera-receipt.json").read_text()) + baseline_name = receipt["baseline"] # Receipts remain portable between this Windows workspace and its WSL # mount; the input itself is still checked byte-for-byte below. - if baseline_name.lower().startswith('c:/dkrport/'): - baseline=ROOT/Path(baseline_name[len('c:/dkrport/'):]) - else:baseline=Path(baseline_name) + if baseline_name.lower().startswith("c:/dkrport/"): + baseline = ROOT / Path(baseline_name[len("c:/dkrport/") :]) + else: + baseline = Path(baseline_name) from character_portrait_pipeline import verify_native + verify_native(baseline) - if receipt['schema'] != 2 or digest(baseline / 'native-receipt.json') != receipt['baseline_receipt']: - raise ValueError('Camera++ baseline receipt changed') + if receipt["schema"] != 2 or digest(baseline / "native-receipt.json") != receipt["baseline_receipt"]: + raise ValueError("Camera++ baseline receipt changed") authored = authored_inputs() - if authored != receipt['authored']: - raise ValueError('Regenerate Camera++ after authored inputs change') - actual = {p.relative_to(output).as_posix(): digest(p) for p in output.rglob('*') if p.is_file() and p.name != 'camera-receipt.json'} - if actual != receipt['files']: - raise ValueError('Camera++ output contains unreviewed edits or unexpected files') - symbols = {p.relative_to(output.parent / 'modding').as_posix(): digest(p) - for p in sorted((output.parent / 'modding').rglob('*')) if p.is_file()} - baseline_symbols = {p.relative_to(baseline.parent / 'modding').as_posix(): digest(p) - for p in sorted((baseline.parent / 'modding').rglob('*')) if p.is_file()} - if symbols != receipt['symbols'] or symbols != baseline_symbols: - raise ValueError('Camera++ reference symbol inputs changed') - for name in ('magic_code_library.hpp', 'magic_code_library.cpp'): - rel = Path('src/game/mods') / name + if authored != receipt["authored"]: + raise ValueError("Regenerate Camera++ after authored inputs change") + actual = { + p.relative_to(output).as_posix(): digest(p) + for p in output.rglob("*") + if p.is_file() and p.name != "camera-receipt.json" + } + if actual != receipt["files"]: + raise ValueError("Camera++ output contains unreviewed edits or unexpected files") + symbols = { + p.relative_to(output.parent / "modding").as_posix(): digest(p) + for p in sorted((output.parent / "modding").rglob("*")) + if p.is_file() + } + baseline_symbols = { + p.relative_to(baseline.parent / "modding").as_posix(): digest(p) + for p in sorted((baseline.parent / "modding").rglob("*")) + if p.is_file() + } + if symbols != receipt["symbols"] or symbols != baseline_symbols: + raise ValueError("Camera++ reference symbol inputs changed") + for name in ("magic_code_library.hpp", "magic_code_library.cpp"): + rel = Path("src/game/mods") / name if (output / rel).read_text() != transform(name, (baseline / rel).read_text()): - raise ValueError('Camera++ library composition changed') - sys.path.insert(0,str(AUTH)) - from integration_pipeline import TRANSFORMS,transform as integrate + raise ValueError("Camera++ library composition changed") + sys.path.insert(0, str(AUTH)) + from integration_pipeline import TRANSFORMS + from integration_pipeline import transform as integrate + for name in sorted(TRANSFORMS): - source=(ROOT/'runtime-recomp'/name) if name.startswith('tests/') else baseline/name - if (output/name).read_text()!=integrate(name,source.read_text(),replace,AUTH): - raise ValueError('Camera++ integration changed: '+name) + source = (ROOT / "runtime-recomp" / name) if name.startswith("tests/") else baseline / name + if (output / name).read_text() != integrate(name, source.read_text(), replace, AUTH): + raise ValueError("Camera++ integration changed: " + name) # Verify every other baseline file remains identical, not only its hash. - for path in (baseline / 'src').rglob('*'): + for path in (baseline / "src").rglob("*"): if path.is_file(): rel = path.relative_to(baseline) - if rel.as_posix() in TRANSFORMS:continue - if rel.as_posix() in ('src/game/mods/magic_code_library.hpp', 'src/game/mods/magic_code_library.cpp'): + if rel.as_posix() in TRANSFORMS: + continue + if rel.as_posix() in ("src/game/mods/magic_code_library.hpp", "src/game/mods/magic_code_library.cpp"): continue if path.read_bytes() != (output / rel).read_bytes(): - raise ValueError('Unrelated native source changed: ' + str(rel)) - for source in (ROOT/'runtime-recomp/tests/replay_probe').rglob('*'): + raise ValueError("Unrelated native source changed: " + str(rel)) + for source in (ROOT / "runtime-recomp/tests/replay_probe").rglob("*"): if source.is_file(): - name=source.relative_to(ROOT/'runtime-recomp').as_posix() - if name not in TRANSFORMS and source.read_bytes()!=(output/name).read_bytes(): - raise ValueError('Unrelated owned bridge source changed: '+name) - print('Camera++ native composition and protected inputs inspected') + name = source.relative_to(ROOT / "runtime-recomp").as_posix() + if name not in TRANSFORMS and source.read_bytes() != (output / name).read_bytes(): + raise ValueError("Unrelated owned bridge source changed: " + name) + print("Camera++ native composition and protected inputs inspected") def sdk(output): - output = new_output(output, ROOT / 'modding', AUTH) - base = ROOT / 'modding' - for name in ('build.py', 'runtime.c', 'mod.ld', 'reference-sdk-only.toml'): + output = new_output(output, ROOT / "modding", AUTH) + base = ROOT / "modding" + for name in ("build.py", "runtime.c", "mod.ld", "reference-sdk-only.toml"): dest = output / name dest.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(base / name, dest) - shutil.copytree(base / 'include', output / 'include') - shutil.copyfile(AUTH / 'sdk/dkr/presentation.h', output / 'include/dkr/presentation.h') - project = output / 'examples/camera-plus' + shutil.copytree(base / "include", output / "include") + shutil.copyfile(AUTH / "sdk/dkr/presentation.h", output / "include/dkr/presentation.h") + project = output / "examples/camera-plus" project.mkdir(parents=True) - for name in ('main.c', 'dkr.json', 'settings.json'): - shutil.copyfile(AUTH / 'mod' / name, project / name) - schema = json.loads((project / 'settings.json').read_text()) - config = ['[manifest]', 'id = "dkr_camera_plus"', 'version = "0.1.0"', - 'display_name = "ModernCamera++"', 'description = "Choose and tune your own driving camera."', - 'authors = ["DKR-R contributors"]', 'game_id = "dkr"', - 'minimum_recomp_version = "1.0.7"', 'dependencies = []'] + for name in ("main.c", "dkr.json", "settings.json"): + shutil.copyfile(AUTH / "mod" / name, project / name) + schema = json.loads((project / "settings.json").read_text()) + config = [ + "[manifest]", + 'id = "dkr_camera_plus"', + 'version = "0.1.0"', + 'display_name = "ModernCamera++"', + 'description = "Choose and tune your own driving camera."', + 'authors = ["DKR-R contributors"]', + 'game_id = "dkr"', + 'minimum_recomp_version = "1.0.7"', + "dependencies = []", + ] known = set() - for group in schema['groups']: - for option in group['options']: + for group in schema["groups"]: + for option in group["options"]: option = dict(option) - if option['id'] in known: - raise ValueError('Duplicate Camera++ setting') - known.add(option['id']) - option.setdefault('type', 'Number') - config += ['\n[[manifest.config_options]]'] - config += [key + ' = ' + json.dumps(value) for key, value in option.items()] + if option["id"] in known: + raise ValueError("Duplicate Camera++ setting") + known.add(option["id"]) + option.setdefault("type", "Number") + config += ["\n[[manifest.config_options]]"] + config += [key + " = " + json.dumps(value) for key, value in option.items()] if len(known) > 128: - raise ValueError('Camera++ settings exceed API 1 schema capacity') - config += ['\n[build]', 'sources = ["main.c"]'] - (project / 'mod.toml').write_text('\n'.join(config) + '\n') - print('Camera++ SDK/example staged without executing code:', output) + raise ValueError("Camera++ settings exceed API 1 schema capacity") + config += ["\n[build]", 'sources = ["main.c"]'] + (project / "mod.toml").write_text("\n".join(config) + "\n") + print("Camera++ SDK/example staged without executing code:", output) def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('operation', choices=['native', 'verify', 'sdk']) - parser.add_argument('--output', type=Path, required=True) - parser.add_argument('--baseline', type=Path) + parser.add_argument("operation", choices=["native", "verify", "sdk"]) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--baseline", type=Path) args = parser.parse_args() - if args.operation == 'native': + if args.operation == "native": if not args.baseline: - parser.error('native requires --baseline') + parser.error("native requires --baseline") native(args.baseline, args.output) - elif args.operation == 'sdk': + elif args.operation == "sdk": sdk(args.output) else: verify(args.output.resolve(strict=True)) -if __name__ == '__main__': +if __name__ == "__main__": try: main() except (OSError, ValueError, KeyError) as error: - print('Camera++ pipeline:', error, file=sys.stderr) + print("Camera++ pipeline:", error, file=sys.stderr) sys.exit(1) diff --git a/scripts/capture_replay_fixture.py b/scripts/capture_replay_fixture.py index ef2568c0..1a680d91 100644 --- a/scripts/capture_replay_fixture.py +++ b/scripts/capture_replay_fixture.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Capture a private, quiesced retail replay fixture with a qualification runtime.""" + from __future__ import annotations import argparse @@ -13,24 +14,22 @@ from regression import execute ROM_REVISIONS = { - '0cb115d8716dbbc2922fda38e533b9fe63bb9670': 77, - '6d96743d46f8c0cd0edb0ec5600b003c89b93755': 80, + "0cb115d8716dbbc2922fda38e533b9fe63bb9670": 77, + "6d96743d46f8c0cd0edb0ec5600b003c89b93755": 80, } MAPS = (3, 4, 5, 7, 8, 10, 14, 26, 30, 40, 53) -CAPTURE_RECORD = re.compile( - r'^\[replay-probe\]\[quiesced\] revision=(77|80) bytes=([0-9]+) success=1$', re.MULTILINE) +CAPTURE_RECORD = re.compile(r"^\[replay-probe\]\[quiesced\] revision=(77|80) bytes=([0-9]+) success=1$", re.MULTILINE) -def file_hash(path: Path, algorithm: str = 'sha256') -> str: +def file_hash(path: Path, algorithm: str = "sha256") -> str: digest = hashlib.new(algorithm) - with path.open('rb') as source: - for chunk in iter(lambda: source.read(1024 * 1024), b''): + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() -def capture(runtime: Path, rom: Path, output: Path, *, players: int = 2, - map_id: int = 3, timeout: int = 60) -> Path: +def capture(runtime: Path, rom: Path, output: Path, *, players: int = 2, map_id: int = 3, timeout: int = 60) -> Path: """Return fixture.json only after exit, quiescence, revision and size agree. Failures retain their private output directory and logs for diagnosis. @@ -39,85 +38,97 @@ def capture(runtime: Path, rom: Path, output: Path, *, players: int = 2, runtime, rom = Path(runtime).resolve(), Path(rom).resolve() output = Path(output).absolute() if players not in (1, 2, 3, 4) or map_id not in MAPS or timeout <= 0: - raise ValueError('Capture requires 1–4 players, a supported map and a positive timeout.') + raise ValueError("Capture requires 1–4 players, a supported map and a positive timeout.") if not runtime.is_file() or not os.access(runtime, os.X_OK): - raise ValueError(f'Runtime must be an existing executable: {runtime}') - if not rom.is_file() or rom.suffix.lower() != '.z64': - raise ValueError('ROM must be an existing canonical .z64 file.') - rom_sha1 = file_hash(rom, 'sha1') + raise ValueError(f"Runtime must be an existing executable: {runtime}") + if not rom.is_file() or rom.suffix.lower() != ".z64": + raise ValueError("ROM must be an existing canonical .z64 file.") + rom_sha1 = file_hash(rom, "sha1") revision = ROM_REVISIONS.get(rom_sha1) if revision is None: - raise ValueError('ROM SHA-1 does not match either supported canonical US retail revision.') + raise ValueError("ROM SHA-1 does not match either supported canonical US retail revision.") runtime_sha256 = file_hash(runtime) # mkdir is the exclusive claim: neither an existing directory nor a symlink # may cause a profile or snapshot from another run to be overwritten. output.mkdir(parents=True, exist_ok=False) - fixture = output / 'race.dkr-component' - profile = output / 'profile' - stdout, stderr = output / 'stdout.log', output / 'stderr.log' - env = {key: value for key, value in os.environ.items() if not key.startswith('DKR_')} - env.update(DKR_REPLAY_CAPTURE_FILE=str(fixture), DKR_REPLAY_CAPTURE_MAP=str(map_id), - DKR_REPLAY_CAPTURE_PLAYERS=str(players)) - command = [str(runtime), '--rom', str(rom), '--config', str(profile), '--timeout', str(timeout)] - code, elapsed_ms, error, interrupted = execute( - command, output, env, timeout + 30, stdout, stderr) - logs = f'Private logs: {output} (including profile/logs/runtime.log).' + fixture = output / "race.dkr-component" + profile = output / "profile" + stdout, stderr = output / "stdout.log", output / "stderr.log" + env = {key: value for key, value in os.environ.items() if not key.startswith("DKR_")} + env.update( + DKR_REPLAY_CAPTURE_FILE=str(fixture), + DKR_REPLAY_CAPTURE_MAP=str(map_id), + DKR_REPLAY_CAPTURE_PLAYERS=str(players), + ) + command = [str(runtime), "--rom", str(rom), "--config", str(profile), "--timeout", str(timeout)] + code, elapsed_ms, error, interrupted = execute(command, output, env, timeout + 30, stdout, stderr) + logs = f"Private logs: {output} (including profile/logs/runtime.log)." if interrupted: raise KeyboardInterrupt if error or code != 0: - raise RuntimeError(f'Capture failed: {error or f"runtime exited {code}"}. {logs}') - runtime_log = profile / 'logs' / 'runtime.log' + raise RuntimeError(f"Capture failed: {error or f'runtime exited {code}'}. {logs}") + runtime_log = profile / "logs" / "runtime.log" if not runtime_log.is_file(): - raise RuntimeError(f'Capture has no runtime quiescence log. {logs}') - records = CAPTURE_RECORD.findall(runtime_log.read_text(encoding='utf-8', errors='replace')) + raise RuntimeError(f"Capture has no runtime quiescence log. {logs}") + records = CAPTURE_RECORD.findall(runtime_log.read_text(encoding="utf-8", errors="replace")) if len(records) != 1 or int(records[0][0]) != revision: - raise RuntimeError(f'Capture lacks one successful quiescence record for revision {revision}. {logs}') + raise RuntimeError(f"Capture lacks one successful quiescence record for revision {revision}. {logs}") expected_bytes = int(records[0][1]) if expected_bytes <= 0 or not fixture.is_file() or fixture.stat().st_size != expected_bytes: - raise RuntimeError(f'Captured fixture size does not match its quiescence record. {logs}') - manifest = output / 'fixture.json' + raise RuntimeError(f"Captured fixture size does not match its quiescence record. {logs}") + manifest = output / "fixture.json" metadata = { - 'schema_version': 1, 'revision': revision, 'map': map_id, 'players': players, - 'rom_sha1': rom_sha1, 'runtime_sha256': runtime_sha256, - 'fixture': fixture.name, 'fixture_sha256': file_hash(fixture), - 'fixture_bytes': expected_bytes, 'capture_elapsed_ms': elapsed_ms, - 'runtime_log': str(runtime_log.relative_to(output)), + "schema_version": 1, + "revision": revision, + "map": map_id, + "players": players, + "rom_sha1": rom_sha1, + "runtime_sha256": runtime_sha256, + "fixture": fixture.name, + "fixture_sha256": file_hash(fixture), + "fixture_bytes": expected_bytes, + "capture_elapsed_ms": elapsed_ms, + "runtime_log": str(runtime_log.relative_to(output)), } - with manifest.open('x', encoding='utf-8') as target: + with manifest.open("x", encoding="utf-8") as target: json.dump(metadata, target, indent=2) - target.write('\n') + target.write("\n") return manifest def positive_int(value: str) -> int: number = int(value) if number <= 0: - raise argparse.ArgumentTypeError('must be positive') + raise argparse.ArgumentTypeError("must be positive") return number def main(argv=None) -> int: - parser = argparse.ArgumentParser(description=__doc__, epilog=( - 'Requires a DKR_REPLAY_QUALIFICATION runtime. Keep the entire output private; ' - 'fixtures contain ROM-derived data. RT64 is optional for CPU-only capture.')) - parser.add_argument('--runtime', type=Path, required=True) - parser.add_argument('--rom', type=Path, required=True) - parser.add_argument('--output', type=Path, required=True, help='New private output directory; never overwritten') - parser.add_argument('--players', type=int, choices=(1, 2, 3, 4), default=2) - parser.add_argument('--map', dest='map_id', type=int, choices=MAPS, default=3) - parser.add_argument('--timeout', type=positive_int, default=60) + parser = argparse.ArgumentParser( + description=__doc__, + epilog=( + "Requires a DKR_REPLAY_QUALIFICATION runtime. Keep the entire output private; " + "fixtures contain ROM-derived data. RT64 is optional for CPU-only capture." + ), + ) + parser.add_argument("--runtime", type=Path, required=True) + parser.add_argument("--rom", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True, help="New private output directory; never overwritten") + parser.add_argument("--players", type=int, choices=(1, 2, 3, 4), default=2) + parser.add_argument("--map", dest="map_id", type=int, choices=MAPS, default=3) + parser.add_argument("--timeout", type=positive_int, default=60) args = parser.parse_args(argv) try: manifest = capture(**vars(args)) except (OSError, ValueError, RuntimeError) as failure: - print(f'Capture failed: {failure}', file=sys.stderr) + print(f"Capture failed: {failure}", file=sys.stderr) return 1 except KeyboardInterrupt: - print('Capture interrupted; private logs were retained.', file=sys.stderr) + print("Capture interrupted; private logs were retained.", file=sys.stderr) return 130 - print(f'Captured verified private fixture: {manifest}') + print(f"Captured verified private fixture: {manifest}") return 0 -if __name__ == '__main__': +if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/character_portrait_pipeline.py b/scripts/character_portrait_pipeline.py index 6514af29..f3e678a0 100644 --- a/scripts/character_portrait_pipeline.py +++ b/scripts/character_portrait_pipeline.py @@ -4,78 +4,127 @@ missing portrait from its retained source review. New imports use the correct table. An explicit legacy preparation mode is solely for frozen online v2 IDs. """ + +import argparse +import hashlib +import json from pathlib import Path -import argparse, hashlib, json, re, shutil - -ROOT=Path(__file__).resolve().parents[1] -AUTH=ROOT/'patches/character-portraits' -MODS=ROOT/'runtime-recomp/src/game/mods' -PINS={ - 'legacy_mod_format.cpp':'74adeea776dc34cb2516fd14b4578c4f81f048f61f4fd452faa1e1e5d66642b3', - 'legacy_character_materialize.hpp':'bd4b2f1104257de526bbe05cbfa1aac0ec0f3b528f908de55d7ac16e5e992397', - 'legacy_character_materialize.cpp':'6d9657e9dabb748a13f9733e201411b02cb8f94cb1b33f14bca64fa90f4db203', - 'legacy_track_catalog.cpp':'37cf82e6987a5cc06743fc86144eabb5e299f797c37a8ab7e959771c68bd3875', - 'legacy_track_prepare_job.hpp':'2672949857d0bf106ba702a47d153725878f10058934c11ba3d2cf83429e542a', - 'legacy_track_prepare_job.cpp':'716b637ca843b7f1fd87d95bd3dbaaf37df80828fa3adb43e0b7f742cc2a771d', - 'online_mod_prepare.cpp':'73921b793921da7bb0bab3763eb1c42dc40fafa067a01c57155a38533d22cf06'} -GUEST_PINS={77:'006cc572caf0097392aeccac974dc4574813eeea81c09b6eabc5b1b7a9199842', - 80:'9fc26cc8bb1762db7a78f5979cea65835ccd7c84e6ed325d411ad149f98df070'} -sha=lambda b:hashlib.sha256(b).hexdigest() - -def new_output(path,*inputs): - path=path.resolve() - if path.exists() or any(path==p.resolve() or p.resolve() in path.parents for p in inputs): - raise ValueError('Portrait pipeline output must be NEW and outside every read-only input') - if any(part.lower() in ('extern','recompiledfuncs','recompiledpatches') for part in path.parts): - raise ValueError('Portrait output must not enter a protected dependency/generated tree') +import shutil + +ROOT = Path(__file__).resolve().parents[1] +AUTH = ROOT / "patches/character-portraits" +MODS = ROOT / "runtime-recomp/src/game/mods" +PINS = { + "legacy_mod_format.cpp": "74adeea776dc34cb2516fd14b4578c4f81f048f61f4fd452faa1e1e5d66642b3", + "legacy_character_materialize.hpp": "bd4b2f1104257de526bbe05cbfa1aac0ec0f3b528f908de55d7ac16e5e992397", + "legacy_character_materialize.cpp": "6d9657e9dabb748a13f9733e201411b02cb8f94cb1b33f14bca64fa90f4db203", + "legacy_track_catalog.cpp": "37cf82e6987a5cc06743fc86144eabb5e299f797c37a8ab7e959771c68bd3875", + "legacy_track_prepare_job.hpp": "2672949857d0bf106ba702a47d153725878f10058934c11ba3d2cf83429e542a", + "legacy_track_prepare_job.cpp": "716b637ca843b7f1fd87d95bd3dbaaf37df80828fa3adb43e0b7f742cc2a771d", + "online_mod_prepare.cpp": "73921b793921da7bb0bab3763eb1c42dc40fafa067a01c57155a38533d22cf06", +} +GUEST_PINS = { + 77: "006cc572caf0097392aeccac974dc4574813eeea81c09b6eabc5b1b7a9199842", + 80: "9fc26cc8bb1762db7a78f5979cea65835ccd7c84e6ed325d411ad149f98df070", +} +sha = lambda b: hashlib.sha256(b).hexdigest() + + +def new_output(path, *inputs): + path = path.resolve() + if path.exists() or any(path == p.resolve() or p.resolve() in path.parents for p in inputs): + raise ValueError("Portrait pipeline output must be NEW and outside every read-only input") + if any(part.lower() in ("extern", "recompiledfuncs", "recompiledpatches") for part in path.parts): + raise ValueError("Portrait output must not enter a protected dependency/generated tree") return path -def replace(text,old,new,count=1): - if text.count(old)!=count:raise ValueError('Unreviewed portrait edit site: '+old[:100]) - return text.replace(old,new) -def transform(name,text): - if name=='legacy_mod_format.cpp': - return replace(text,'portraits[]{122,123,125,126,124,127,129,128,130,131}', - 'portraits[]{122,123,124,126,125,127,128,129,130,131}') - if name=='legacy_character_materialize.hpp': +def replace(text, old, new, count=1): + if text.count(old) != count: + raise ValueError("Unreviewed portrait edit site: " + old[:100]) + return text.replace(old, new) + + +def transform(name, text): + if name == "legacy_mod_format.cpp": + return replace( + text, + "portraits[]{122,123,125,126,124,127,129,128,130,131}", + "portraits[]{122,123,124,126,125,127,128,129,130,131}", + ) + if name == "legacy_character_materialize.hpp": # Only PreparedCharacter grows; AllocatedCharacter's ABI stays intact. - at=text.index('struct AllocatedCharacter') - first=replace(text[:at],' CharacterRaceAudio race_audio;',''' CharacterRaceAudio race_audio; + at = text.index("struct AllocatedCharacter") + first = replace( + text[:at], + " CharacterRaceAudio race_audio;", + """ CharacterRaceAudio race_audio; // Recovered only by the authenticated source-review loader. Bytes source_portrait; - unsigned source_portrait_id=0;''') - text=first+text[at:] - return replace(text,'bool behaviour_enabled=true);','bool behaviour_enabled=true,bool legacy_portrait_layout=false);') - if name=='legacy_character_materialize.cpp': - text=replace(text,'unsigned base_character,bool behaviour_enabled) {','unsigned base_character,bool behaviour_enabled,bool legacy_portrait_layout) {') - text=replace(text,' result.base_fingerprint=sha256(original);',''' result.base_fingerprint=sha256(original); + unsigned source_portrait_id=0;""", + ) + text = first + text[at:] + return replace( + text, "bool behaviour_enabled=true);", "bool behaviour_enabled=true,bool legacy_portrait_layout=false);" + ) + if name == "legacy_character_materialize.cpp": + text = replace( + text, + "unsigned base_character,bool behaviour_enabled) {", + "unsigned base_character,bool behaviour_enabled,bool legacy_portrait_layout) {", + ) + text = replace( + text, + " result.base_fingerprint=sha256(original);", + """ result.base_fingerprint=sha256(original); if(legacy_portrait_layout) { constexpr unsigned previous[]{122,123,125,126,124,127,129,128,130,131}; result.root.portrait=previous[base_character]; - }''') - text=replace(text,' const auto& portrait=character.records.at({4,character.root.portrait});',''' constexpr unsigned portrait_ids[]{122,123,124,126,125,127,128,129,130,131}; + }""", + ) + text = replace( + text, + " const auto& portrait=character.records.at({4,character.root.portrait});", + """ constexpr unsigned portrait_ids[]{122,123,124,126,125,127,128,129,130,131}; const bool recovered=!character.source_portrait.empty(); if(recovered && (character.source_portrait_id!=portrait_ids[character.root.base_character] || character.source_portrait.size()>4*MiB)) throw Error("Invalid recovered source portrait."); const auto portrait_id=recovered?character.source_portrait_id:character.root.portrait; const auto& portrait=recovered?character.source_portrait:character.records.at({4,character.root.portrait}); - validate_texture_record(portrait);''') - text=replace(text,' for(unsigned i=0;i<3;++i)slot.headers[i]=remap.at({34,character.root.headers[i]});',''' if(recovered) { + validate_texture_record(portrait);""", + ) + text = replace( + text, + " for(unsigned i=0;i<3;++i)slot.headers[i]=remap.at({34,character.root.headers[i]});", + """ if(recovered) { if(next[4]>=32767)throw Error("Recovered portrait namespace is full."); slot.portrait=next[4]++; result.additions.emplace(AssetKey{4,slot.portrait},portrait); } - for(unsigned i=0;i<3;++i)slot.headers[i]=remap.at({34,character.root.headers[i]});''') - text=replace(text,'slot.custom_portrait=character_portrait_changed(portrait,stock->record(4,character.root.portrait));', - 'slot.custom_portrait=character_portrait_changed(portrait,stock->record(4,portrait_id));') - return replace(text,'for(const auto& c:result.characters)roster_ids+=c.id;', - 'for(const auto& c:result.characters)roster_ids+=c.id+":portrait-source-v1:"+c.portrait_identity;') - if name=='legacy_track_catalog.cpp': - text=replace(text,' std::size_t cached_audio_bytes=0;',' std::size_t cached_audio_bytes=0,source_portrait_bytes=0;') - anchor=' std::array audio;' - text=replace(text,anchor,(AUTH/'load_reviewed_portrait.inc').read_text()+'\n'+anchor) - return replace(text,' result.push_back(std::move(character));',''' constexpr unsigned correct_portraits[]{122,123,124,126,125,127,128,129,130,131}; + for(unsigned i=0;i<3;++i)slot.headers[i]=remap.at({34,character.root.headers[i]});""", + ) + text = replace( + text, + "slot.custom_portrait=character_portrait_changed(portrait,stock->record(4,character.root.portrait));", + "slot.custom_portrait=character_portrait_changed(portrait,stock->record(4,portrait_id));", + ) + return replace( + text, + "for(const auto& c:result.characters)roster_ids+=c.id;", + 'for(const auto& c:result.characters)roster_ids+=c.id+":portrait-source-v1:"+c.portrait_identity;', + ) + if name == "legacy_track_catalog.cpp": + text = replace( + text, + " std::size_t cached_audio_bytes=0;", + " std::size_t cached_audio_bytes=0,source_portrait_bytes=0;", + ) + anchor = " std::array audio;" + text = replace(text, anchor, (AUTH / "load_reviewed_portrait.inc").read_text() + "\n" + anchor) + return replace( + text, + " result.push_back(std::move(character));", + """ constexpr unsigned correct_portraits[]{122,123,124,126,125,127,128,129,130,131}; const auto correct_portrait=correct_portraits[character.root.base_character]; if(character.root.portrait!=correct_portrait && character.source_portrait.empty()) { const auto bytes=stock->record(4,correct_portrait); @@ -85,21 +134,39 @@ def transform(name,text): if(character.source_portrait.size()>32*MiB-source_portrait_bytes) throw Error("Recovered character portraits exceed the presentation memory budget."); source_portrait_bytes+=character.source_portrait.size(); - result.push_back(std::move(character));''') - if name=='legacy_track_prepare_job.hpp': - at=text.index('void prepare_imported_characters') - return text[:at]+replace(text[at:],'bool behaviour_enabled=true);','bool behaviour_enabled=true,bool legacy_portrait_layout=false);') - if name=='legacy_track_prepare_job.cpp': - text=replace(text,'bool characters,const ImportPreparation* preparation,bool behaviour_enabled) {', - 'bool characters,const ImportPreparation* preparation,bool behaviour_enabled,bool legacy_portrait_layout=false) {') - text=replace(text,'prepare_character(base,target,patch_id,root.base_character,behaviour_enabled);', - 'prepare_character(base,target,patch_id,root.base_character,behaviour_enabled,legacy_portrait_layout);') - at=text.index('void prepare_imported_characters(') - tail=replace(text[at:],'bool behaviour_enabled) {','bool behaviour_enabled,bool legacy_portrait_layout) {') - tail=replace(tail,'progress,true,preparation,behaviour_enabled);','progress,true,preparation,behaviour_enabled,legacy_portrait_layout);') - return text[:at]+tail - if name=='online_mod_prepare.cpp': - text=replace(text,' const auto prepared=job/(std::string(kind==Kind::Track?', ''' // Old host selections retain v2 IDs/save associations. Rebuild + result.push_back(std::move(character));""", + ) + if name == "legacy_track_prepare_job.hpp": + at = text.index("void prepare_imported_characters") + return text[:at] + replace( + text[at:], + "bool behaviour_enabled=true);", + "bool behaviour_enabled=true,bool legacy_portrait_layout=false);", + ) + if name == "legacy_track_prepare_job.cpp": + text = replace( + text, + "bool characters,const ImportPreparation* preparation,bool behaviour_enabled) {", + "bool characters,const ImportPreparation* preparation,bool behaviour_enabled,bool legacy_portrait_layout=false) {", + ) + text = replace( + text, + "prepare_character(base,target,patch_id,root.base_character,behaviour_enabled);", + "prepare_character(base,target,patch_id,root.base_character,behaviour_enabled,legacy_portrait_layout);", + ) + at = text.index("void prepare_imported_characters(") + tail = replace(text[at:], "bool behaviour_enabled) {", "bool behaviour_enabled,bool legacy_portrait_layout) {") + tail = replace( + tail, + "progress,true,preparation,behaviour_enabled);", + "progress,true,preparation,behaviour_enabled,legacy_portrait_layout);", + ) + return text[:at] + tail + if name == "online_mod_prepare.cpp": + text = replace( + text, + " const auto prepared=job/(std::string(kind==Kind::Track?", + """ // Old host selections retain v2 IDs/save associations. Rebuild // that exact edition only when requested, never accept a mismatch. for(const bool previous_portraits:{false,true}) { if(previous_portraits && kind!=Kind::Character)continue; @@ -108,133 +175,202 @@ def transform(name,text): if(content.payload==payload.digest && content.kind==kind && (content.behaviour_abi!=0)==enabled && !found.contains({kind,content.id}))pending=true; if(!pending)continue; - const auto prepared=job/(std::string(previous_portraits?"previous-portraits-":"")+std::string(kind==Kind::Track?''') - text=replace(text,'prepare_imported_characters(review,owned_roms,prepared,progress,&preparation,enabled);', - 'prepare_imported_characters(review,owned_roms,prepared,progress,&preparation,enabled,previous_portraits);') - text=replace(text,' if(entry==expected.end() || entry->second.payload!=payload.digest)continue;', - ' if(entry==expected.end() || entry->second.payload!=payload.digest || found.contains(key))continue;') - text=replace(text,' receipt[entries]=std::move(selected);',''' // A corrected pass may have no roots matching an old host ID. + const auto prepared=job/(std::string(previous_portraits?"previous-portraits-":"")+std::string(kind==Kind::Track?""", + ) + text = replace( + text, + "prepare_imported_characters(review,owned_roms,prepared,progress,&preparation,enabled);", + "prepare_imported_characters(review,owned_roms,prepared,progress,&preparation,enabled,previous_portraits);", + ) + text = replace( + text, + " if(entry==expected.end() || entry->second.payload!=payload.digest)continue;", + " if(entry==expected.end() || entry->second.payload!=payload.digest || found.contains(key))continue;", + ) + text = replace( + text, + " receipt[entries]=std::move(selected);", + """ // A corrected pass may have no roots matching an old host ID. // Leave that scratch unpublished; empty groups are not catalogues. if(selected.empty())continue; - receipt[entries]=std::move(selected);''') - return replace(text,' }\n }\n storage_budget(destination);',' }\n }\n }\n storage_budget(destination);') + receipt[entries]=std::move(selected);""", + ) + return replace( + text, + " }\n }\n storage_budget(destination);", + " }\n }\n }\n storage_budget(destination);", + ) raise ValueError(name) + def native(output): - sources=ROOT/'runtime-recomp/src' - output=new_output(output,sources) - for name,pin in PINS.items(): - if sha((MODS/name).read_bytes())!=pin:raise ValueError('Portrait source pin changed: '+name) - for p in sources.rglob('*'): - if p.is_symlink():raise ValueError('Native source input must not contain links') - shutil.copytree(sources,output/'src') + sources = ROOT / "runtime-recomp/src" + output = new_output(output, sources) + for name, pin in PINS.items(): + if sha((MODS / name).read_bytes()) != pin: + raise ValueError("Portrait source pin changed: " + name) + for p in sources.rglob("*"): + if p.is_symlink(): + raise ValueError("Native source input must not contain links") + shutil.copytree(sources, output / "src") for name in PINS: - dest=output/'src/game/mods'/name - dest.write_text(transform(name,(MODS/name).read_text()),newline='\n') + dest = output / "src/game/mods" / name + dest.write_text(transform(name, (MODS / name).read_text()), newline="\n") # Every staged consumer must use the same struct definition. The complete # first-party src tree is copied unchanged except the seven pinned files. - files={p.relative_to(output).as_posix():sha(p.read_bytes()) for p in sorted((output/'src').rglob('*')) if p.is_file()} - receipt=dict(schema=1,original_pins=PINS,files=files) - (output/'native-receipt.json').write_text(json.dumps(receipt,indent=2)+'\n') - print('Native portrait stage:',output,len(files),'first-party inputs, seven pinned edits') - symbols(output.parent/'modding') + files = { + p.relative_to(output).as_posix(): sha(p.read_bytes()) + for p in sorted((output / "src").rglob("*")) + if p.is_file() + } + receipt = dict(schema=1, original_pins=PINS, files=files) + (output / "native-receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + print("Native portrait stage:", output, len(files), "first-party inputs, seven pinned edits") + symbols(output.parent / "modding") + def symbols(output): - output=new_output(output,ROOT/'modding') - for revision in (77,80): - rel=Path('reference-us-v'+str(revision))/'game_symbols.inc' - source=ROOT/'modding'/rel - if source.is_symlink() or not source.is_file():raise ValueError('Invalid symbol dependency') - dest=output/rel;dest.parent.mkdir(parents=True,exist_ok=True) - shutil.copyfile(source,dest) - print('Read-only source-relative symbol dependencies copied to',output) + output = new_output(output, ROOT / "modding") + for revision in (77, 80): + rel = Path("reference-us-v" + str(revision)) / "game_symbols.inc" + source = ROOT / "modding" / rel + if source.is_symlink() or not source.is_file(): + raise ValueError("Invalid symbol dependency") + dest = output / rel + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, dest) + print("Read-only source-relative symbol dependencies copied to", output) + def verify_native(output): - receipt=json.loads((output/'native-receipt.json').read_text()) - if receipt['schema']!=1 or receipt['original_pins']!=PINS:raise ValueError('Unknown portrait stage') - originals=ROOT/'runtime-recomp/src' - expected={p.relative_to(output).as_posix() for p in (output/'src').rglob('*') if p.is_file()} - if expected!=set(receipt['files']):raise ValueError('Portrait stage file set changed') - for rel,digest in receipt['files'].items(): - file=output/rel - if file.is_symlink() or sha(file.read_bytes())!=digest:raise ValueError('Portrait staged file changed: '+rel) - source=originals/Path(rel).relative_to('src') - name=source.name - if source.parent==MODS and name in PINS: - if sha(source.read_bytes())!=PINS[name] or file.read_text()!=transform(name,source.read_text()): - raise ValueError('Unreviewed portrait source difference: '+rel) - elif file.read_bytes()!=source.read_bytes():raise ValueError('Unrelated native source changed: '+rel) - for revision in (77,80): - rel=Path('reference-us-v'+str(revision))/'game_symbols.inc' - copy=output.parent/'modding'/rel - if copy.is_symlink() or copy.read_bytes()!=(ROOT/'modding'/rel).read_bytes(): - raise ValueError('Source-relative game symbols differ from the checked online contract') - print('Portrait native stage matches authored pipeline and current first-party inputs') - -def guest(source,output,revision,config,output_config,native_stage): + receipt = json.loads((output / "native-receipt.json").read_text()) + if receipt["schema"] != 1 or receipt["original_pins"] != PINS: + raise ValueError("Unknown portrait stage") + originals = ROOT / "runtime-recomp/src" + expected = {p.relative_to(output).as_posix() for p in (output / "src").rglob("*") if p.is_file()} + if expected != set(receipt["files"]): + raise ValueError("Portrait stage file set changed") + for rel, digest in receipt["files"].items(): + file = output / rel + if file.is_symlink() or sha(file.read_bytes()) != digest: + raise ValueError("Portrait staged file changed: " + rel) + source = originals / Path(rel).relative_to("src") + name = source.name + if source.parent == MODS and name in PINS: + if sha(source.read_bytes()) != PINS[name] or file.read_text() != transform(name, source.read_text()): + raise ValueError("Unreviewed portrait source difference: " + rel) + elif file.read_bytes() != source.read_bytes(): + raise ValueError("Unrelated native source changed: " + rel) + for revision in (77, 80): + rel = Path("reference-us-v" + str(revision)) / "game_symbols.inc" + copy = output.parent / "modding" / rel + if copy.is_symlink() or copy.read_bytes() != (ROOT / "modding" / rel).read_bytes(): + raise ValueError("Source-relative game symbols differ from the checked online contract") + print("Portrait native stage matches authored pipeline and current first-party inputs") + + +def guest(source, output, revision, config, output_config, native_stage): from save_layout_pipeline import DEF - source=source.resolve() - output=new_output(output,source) - output_config=new_output(output_config,source,config) - for path in source.rglob('*'): - if path.is_symlink():raise ValueError('Guest portrait input cannot contain links') - edits={};found=None - for file in source.glob('*.c'): - text=file.read_text();matches=list(DEF.finditer(text)) - for i,m in enumerate(matches): - if m[1]!='obj_loop_characterflag':continue - if found:raise ValueError('Duplicate world portrait function') - end=matches[i+1].start() if i+1r3) = ctx->r11;', - (AUTH/'world_portrait.inc').read_text()+'\n MEM_W(0X24, ctx->r3) = ctx->r11;') - edits[file.name]=text[:m.start()]+replacement+text[end:] - found=dict(original_sha256=sha(body.encode()),patched_sha256=sha(replacement.encode())) - if not found:raise ValueError('Missing in-world portrait guest function') - shutil.copytree(source,output) - for name,text in edits.items():(output/name).write_text(text,newline='\n') - receipt=dict(schema=1,revision=revision,function=found, - native_sha256=sha((native_stage/'native-receipt.json').read_bytes()), - world_source_sha256=sha((AUTH/'world_portrait.inc').read_bytes())) - (output/'character-portrait-receipt.json').write_text(json.dumps(receipt,indent=2)+'\n') - content=config.read_text().replace(source.resolve().as_posix(),output.resolve().as_posix()) - address={77:'80035FE4',80:'80036024'}[revision] + + source = source.resolve() + output = new_output(output, source) + output_config = new_output(output_config, source, config) + for path in source.rglob("*"): + if path.is_symlink(): + raise ValueError("Guest portrait input cannot contain links") + edits = {} + found = None + for file in source.glob("*.c"): + text = file.read_text() + matches = list(DEF.finditer(text)) + for i, m in enumerate(matches): + if m[1] != "obj_loop_characterflag": + continue + if found: + raise ValueError("Duplicate world portrait function") + end = matches[i + 1].start() if i + 1 < len(matches) else len(text) + body = text[m.start() : end] + if sha(body.encode()) != GUEST_PINS[revision]: + raise ValueError("Unreviewed CharacterFlag guest instructions") + replacement = replace( + body, + " MEM_W(0X24, ctx->r3) = ctx->r11;", + (AUTH / "world_portrait.inc").read_text() + "\n MEM_W(0X24, ctx->r3) = ctx->r11;", + ) + edits[file.name] = text[: m.start()] + replacement + text[end:] + found = dict(original_sha256=sha(body.encode()), patched_sha256=sha(replacement.encode())) + if not found: + raise ValueError("Missing in-world portrait guest function") + shutil.copytree(source, output) + for name, text in edits.items(): + (output / name).write_text(text, newline="\n") + receipt = dict( + schema=1, + revision=revision, + function=found, + native_sha256=sha((native_stage / "native-receipt.json").read_bytes()), + world_source_sha256=sha((AUTH / "world_portrait.inc").read_bytes()), + ) + (output / "character-portrait-receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + content = config.read_text().replace(source.resolve().as_posix(), output.resolve().as_posix()) + address = {77: "80035FE4", 80: "80036024"}[revision] # Protect the entire function from code-mod replacement, like other port hooks. - content+='\n# In-world custom participant portraits\n[[patches.hook]]\nfunc = "obj_loop_characterflag"\nbefore_vram = 0x'+address+'\ntext = "/* dkr_character_portrait_pipeline: protected source hook */"\n' - output_config.write_text(content,newline='\n') - print('World portrait guest stage v'+str(revision)+':',output) - -def owned_receipt(source,revision): - path=source/'character-portrait-receipt.json' - if not path.exists():return {} - receipt=json.loads(path.read_text()) - if receipt['schema']!=1 or receipt['revision']!=revision or receipt['function']['original_sha256']!=GUEST_PINS[revision]: - raise ValueError('Unreviewed portrait guest provenance') - if receipt['world_source_sha256']!=sha((AUTH/'world_portrait.inc').read_bytes()): - raise ValueError('World portrait source has changed') - native_stage=source.parent/'native' + content += ( + '\n# In-world custom participant portraits\n[[patches.hook]]\nfunc = "obj_loop_characterflag"\nbefore_vram = 0x' + + address + + '\ntext = "/* dkr_character_portrait_pipeline: protected source hook */"\n' + ) + output_config.write_text(content, newline="\n") + print("World portrait guest stage v" + str(revision) + ":", output) + + +def owned_receipt(source, revision): + path = source / "character-portrait-receipt.json" + if not path.exists(): + return {} + receipt = json.loads(path.read_text()) + if ( + receipt["schema"] != 1 + or receipt["revision"] != revision + or receipt["function"]["original_sha256"] != GUEST_PINS[revision] + ): + raise ValueError("Unreviewed portrait guest provenance") + if receipt["world_source_sha256"] != sha((AUTH / "world_portrait.inc").read_bytes()): + raise ValueError("World portrait source has changed") + native_stage = source.parent / "native" verify_native(native_stage) - if receipt['native_sha256']!=sha((native_stage/'native-receipt.json').read_bytes()): - raise ValueError('Portrait guest/native stages disagree') + if receipt["native_sha256"] != sha((native_stage / "native-receipt.json").read_bytes()): + raise ValueError("Portrait guest/native stages disagree") from save_layout_pipeline import DEF - found=[] - for file in source.glob('*.c'): - text=file.read_text();matches=list(DEF.finditer(text)) - for i,m in enumerate(matches): - if m[1]=='obj_loop_characterflag': - end=matches[i+1].start() if i+1 str | None: + """The first extern/ checkout a tooling test needs that is absent or empty.""" + for needed in NEEDS_EXTERN.get(test, ()): + directory = ROOT / needed + if not directory.is_dir() or not any(directory.iterdir()): + return needed + return None + + +SUITES = ("guard", "tooling", "metadata", "assets", "track-lab", "host-tools") +DEFAULT_SUITES = ("guard", "tooling", "metadata", "assets") + + +def suite_list(value: str) -> list[str]: + """One --suite value: a suite name, "all", or a comma-separated list of them.""" + names = [name.strip() for name in value.split(",") if name.strip()] + names = [suite for name in names for suite in (SUITES if name == "all" else (name,))] + unknown = [name for name in names if name not in SUITES] + if not names or unknown: + raise argparse.ArgumentTypeError( + f"unknown suite {', '.join(unknown) or repr(value)} (choose from {', '.join(SUITES)} or all)" + ) + return names def main(argv=None) -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--suite", action="append", choices=("tooling", "metadata", "assets", "track-lab"), - help="select checks (repeatable); defaults to tooling, metadata and assets") + parser.add_argument( + "--suite", + action="append", + type=suite_list, + metavar="NAME[,NAME...]", + help=f"select checks (repeatable, comma lists allowed: {', '.join(SUITES)} or all); " + f"defaults to {', '.join(DEFAULT_SUITES)}", + ) parser.add_argument("--list", action="store_true", help="print selected checks without running them") args = parser.parse_args(argv) - suites = args.suite or ["tooling", "metadata", "assets"] + suites = [name for names in args.suite for name in names] if args.suite else list(DEFAULT_SUITES) commands = { + "guard": [[sys.executable, "scripts/pattern_guard.py", "--tree", "--baseline", "check"]], "tooling": [[sys.executable, f"runtime-recomp/tests/{test}"] for test in TOOLING_TESTS], + "host-tools": [ + [sys.executable, f"runtime-recomp/tests/{test}"] + for test in (*sorted(HOST_TOOL_TESTS), "pinned_bytes_tests.py") + ], "metadata": [[sys.executable, "scripts/validate_release_metadata.py"]], "assets": [[sys.executable, "scripts/scan_for_game_assets.py"]], "track-lab": [[sys.executable, "tools/blender/run_tests.py", "--skip-blender"]], } for suite in dict.fromkeys(suites): for command in commands[suite]: + absent = missing_checkout(Path(command[-1]).name) if suite == "tooling" else None + if absent: + print(f"[{suite}] skip {command[-1]}: {absent} is not checked out", flush=True) + continue print(f"[{suite}] {' '.join(command[1:])}", flush=True) if args.list: continue try: - result = subprocess.run(command, cwd=ROOT) + environment = {**os.environ, PRE_COMMIT_INTEGRATION: "1"} if suite == "host-tools" else None + result = subprocess.run(command, cwd=ROOT, env=environment) except OSError as error: print(f"Could not run {command[1]}: {error}", file=sys.stderr) return 1 diff --git a/scripts/check_pinned_bytes.py b/scripts/check_pinned_bytes.py new file mode 100755 index 00000000..726afaf9 --- /dev/null +++ b/scripts/check_pinned_bytes.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Fail when a checksum-pinned file no longer matches its pin. + +Three kinds of file are pinned by sha256 and must keep their exact bytes, even +where they carry trailing whitespace or CRLF line endings that a formatter +would "fix": + +* every patch listed in patches/manifest.json (``dependencies[].patches[]``); +* the address-only SDK kits, modding/reference-us-v*/, against the ``files`` + map in each kit's receipt.json; +* the content-addressed source capsules, patches/release-prep/*/source-capsule-*/, + whose file names are their own sha256; +* the capsules' "base" files, first-party sources whose CRLF-normalised bytes + the capsule manifest pins (``base_sha256``) because capsule.py rebuilds its + roles from them. + +With no arguments every pin is checked. With paths (what pre-commit passes), +only the pins those paths belong to are checked; a manifest or receipt in the +list checks every file it pins. Bytes are read from the work tree, which is +what pre-commit stashes to before running hooks. Standard library only. + +Exit status: 0 all checked pins match, 1 a pinned file is missing or changed. +""" + +from __future__ import annotations + +import argparse +from collections.abc import Iterable +import hashlib +import json +from pathlib import Path +import re +import sys +from typing import NamedTuple + +ROOT = Path(__file__).resolve().parents[1] +PATCH_MANIFEST = "patches/manifest.json" +RECEIPTS = "modding/reference-us-v*/receipt.json" +CAPSULE_MANIFESTS = "patches/release-prep/*/source-capsule-*/manifest.json" +# Bases that main had already changed when the capsule was frozen. +# patches/upstream-sync/restore_capsule.py reads these from the pinned commit +# instead, so they are free to change; every other base must keep its bytes. +DRIFTED_BASES = frozenset( + { + "runtime-recomp/src/game/custom_tracks.cpp", + "runtime-recomp/src/game/mobile_graphics_preset.hpp", + "runtime-recomp/src/game/mods/magic_code_dispatch.h", + "runtime-recomp/src/game/runtime_code_mods.cpp", + "runtime-recomp/src/game/runtime_graphics_ui.inl", + } +) +CAPSULE = re.compile(r"^patches/release-prep/[^/]+/source-capsule-[^/]+/(?P[0-9a-f]{64})\.[A-Za-z0-9]+$") + + +class Pin(NamedTuple): + """One pinned file: its expected sha256 and the file that pins it.""" + + sha256: str + source: str + # capsule.py hashes its bases after CRLF -> LF (capsule.py canonical()). + crlf_normalised: bool = False + + +def pins(root: Path) -> dict[str, Pin]: + """Map each pinned path to (expected sha256, file that pins it).""" + pinned: dict[str, Pin] = {} + manifest = json.loads((root / PATCH_MANIFEST).read_text(encoding="utf-8")) + for dependency in manifest["dependencies"]: + for patch in dependency.get("patches", []): + pinned[patch["path"]] = Pin(patch["sha256"], PATCH_MANIFEST) + for receipt in sorted(root.glob(RECEIPTS)): + source = receipt.relative_to(root).as_posix() + kit = receipt.parent.relative_to(root).as_posix() + for name, sha in json.loads(receipt.read_text(encoding="utf-8"))["files"].items(): + pinned[f"{kit}/{name}"] = Pin(sha, source) + for path in sorted(root.glob("patches/release-prep/*/source-capsule-*/*")): + relative = path.relative_to(root).as_posix() + match = CAPSULE.match(relative) + if match: + pinned[relative] = Pin(match["sha"], "its file name") + for manifest_path in sorted(root.glob(CAPSULE_MANIFESTS)): + source = manifest_path.relative_to(root).as_posix() + capsule_dir = manifest_path.parent.relative_to(root).as_posix() + capsule = json.loads(manifest_path.read_text(encoding="utf-8")) + for entries in capsule["roles"].values(): + for entry in entries.values(): + # Objects and deltas the manifest needs must exist, not just + # match when present: capsule.py can't rebuild a role without them. + for stored in (entry.get("object"), entry.get("patch")): + if stored: + relative = f"{capsule_dir}/{stored}" + match = CAPSULE.match(relative) + if match: + pinned[relative] = Pin(match["sha"], source) + base = entry.get("base") + if base and base not in DRIFTED_BASES: + pinned[base] = Pin(entry["base_sha256"], source, crlf_normalised=True) + return pinned + + +def selected(pinned: dict[str, Pin], paths: Iterable[str]) -> list[str]: + """The pinned paths that the given paths (files or pinning manifests) cover.""" + chosen: set[str] = set() + for path in (Path(p).as_posix().removeprefix("./") for p in paths): + if path in pinned: + chosen.add(path) + chosen.update(name for name, pin in pinned.items() if pin.source == path) + return sorted(chosen) + + +def check(root: Path, paths: Iterable[str] | None = None) -> list[str]: + """One message per pinned file that is missing or whose sha256 changed.""" + pinned = pins(root) + problems: list[str] = [] + for path in sorted(pinned) if paths is None else selected(pinned, paths): + expected, source, crlf_normalised = pinned[path] + try: + raw = (root / path).read_bytes() + except FileNotFoundError: + problems.append(f"{path}: missing, but pinned by {source}") + continue + if crlf_normalised: + raw = raw.replace(b"\r\n", b"\n") + actual = hashlib.sha256(raw).hexdigest() + if actual != expected: + problems.append(f"{path}: sha256 {actual} does not match {expected} pinned by {source}") + return problems + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=(__doc__ or "").splitlines()[0]) + parser.add_argument("paths", nargs="*", help="limit the check to these files (default: every pin)") + parser.add_argument("--root", type=Path, default=ROOT, help=argparse.SUPPRESS) + args = parser.parse_args(argv) + problems = check(args.root, args.paths or None) + for problem in problems: + print(problem, file=sys.stderr) + if problems: + print( + "Pinned bytes changed. If a formatter or editor did this, restore the file " + "(git checkout -- ); if the change is intended, update the pin in the same commit.", + file=sys.stderr, + ) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci_summary.py b/scripts/ci_summary.py index ad40f337..86a2d465 100644 --- a/scripts/ci_summary.py +++ b/scripts/ci_summary.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Render bounded GitHub Markdown from existing CI artifacts (never run tests).""" + from __future__ import annotations import argparse @@ -13,175 +14,222 @@ import xml.etree.ElementTree as ET READ_LIMIT = 16 * 1024 * 1024 -STATUSES = ('passed', 'failed', 'skipped') +STATUSES = ("passed", "failed", "skipped") def clipped(value, limit=180): - text = ''.join(c for c in str(value) if c in '\n\t' or (ord(c) >= 32 and ord(c) != 127)) - return text if len(text) <= limit else text[:limit] + '… [truncated]' + text = "".join(c for c in str(value) if c in "\n\t" or (ord(c) >= 32 and ord(c) != 127)) + return text if len(text) <= limit else text[:limit] + "… [truncated]" def escape(value, limit=180): - text = html.escape(clipped(value, limit).replace('\n', ' ').replace('\t', ' ')) + text = html.escape(clipped(value, limit).replace("\n", " ").replace("\t", " ")) # Numeric entities keep artifact names literal in tables and inline Markdown. - return re.sub(r'[\\`*_{}\[\]()#+.!|~-]', lambda match: f'&#{ord(match[0])};', text) + return re.sub(r"[\\`*_{}\[\]()#+.!|~-]", lambda match: f"&#{ord(match[0])};", text) def block(value): - return '
' + html.escape(clipped(value, 1200)) + '
' + return "
" + html.escape(clipped(value, 1200)) + "
" def duration(value): if isinstance(value, bool): - raise ValueError('Boolean duration/metric') + raise ValueError("Boolean duration/metric") number = float(value) if not math.isfinite(number) or number < 0: - raise ValueError('Nonfinite or negative duration/metric') + raise ValueError("Nonfinite or negative duration/metric") return number def totals(counts): - return ', '.join(f'{counts.get(status, 0)} {status}' for status in STATUSES) + return ", ".join(f"{counts.get(status, 0)} {status}" for status in STATUSES) def junit_summary(text): - if ' 12: - lines.append(f'{len(failures) - 12} additional failures omitted; inspect the JUnit artifact.') - lines += ['', 'Slowest tests:', '', '| Test | Seconds | Result |', '| --- | ---: | --- |'] + lines.append(f"{len(failures) - 12} additional failures omitted; inspect the JUnit artifact.") + lines += ["", "Slowest tests:", "", "| Test | Seconds | Result |", "| --- | ---: | --- |"] for case, status, seconds in sorted(cases, key=lambda row: row[2], reverse=True)[:5]: - lines.append(f'| {escape(case.get("name", "(unnamed)"))} | {seconds:.3f} | {status} |') + lines.append(f"| {escape(case.get('name', '(unnamed)'))} | {seconds:.3f} | {status} |") return lines def regression_summary(text): report = json.loads(text) - if not isinstance(report, dict) or report.get('schema_version') != 1 or not isinstance(report.get('scenarios'), list): - raise ValueError('Expected harness schema_version 1 with scenarios') - scenarios = report['scenarios'] + if ( + not isinstance(report, dict) + or report.get("schema_version") != 1 + or not isinstance(report.get("scenarios"), list) + ): + raise ValueError("Expected harness schema_version 1 with scenarios") + scenarios = report["scenarios"] counts = Counter() seen = set() for item in scenarios: - if not isinstance(item, dict) or not isinstance(item.get('id'), str) or not item['id'] or item['id'] in seen or item.get('status') not in STATUSES: - raise ValueError('Invalid or duplicate harness scenario') - seen.add(item['id']) - counts[item['status']] += 1 - if not isinstance(item.get('errors', []), list) or not all(isinstance(value, str) for value in item.get('errors', [])): - raise ValueError('Invalid scenario errors') - if not isinstance(item.get('metrics', {}), dict): - raise ValueError('Invalid scenario metrics') - for value in item.get('metrics', {}).values(): + if ( + not isinstance(item, dict) + or not isinstance(item.get("id"), str) + or not item["id"] + or item["id"] in seen + or item.get("status") not in STATUSES + ): + raise ValueError("Invalid or duplicate harness scenario") + seen.add(item["id"]) + counts[item["status"]] += 1 + if not isinstance(item.get("errors", []), list) or not all( + isinstance(value, str) for value in item.get("errors", []) + ): + raise ValueError("Invalid scenario errors") + if not isinstance(item.get("metrics", {}), dict): + raise ValueError("Invalid scenario metrics") + for value in item.get("metrics", {}).values(): duration(value) - fingerprints = item.get('fingerprints', {}) - if not isinstance(fingerprints, dict) or not all(isinstance(values, list) and all(isinstance(value, str) for value in values) for values in fingerprints.values()): - raise ValueError('Invalid scenario fingerprints') - declared = report.get('summary') - if not isinstance(declared, dict) or any(type(declared.get(status)) is not int or declared[status] != counts[status] for status in STATUSES): - raise ValueError('Harness summary does not match scenario outcomes') - lines = [escape(report.get('suite', 'Regression/replay')) + ': ' + totals(counts) + '.', ''] + fingerprints = item.get("fingerprints", {}) + if not isinstance(fingerprints, dict) or not all( + isinstance(values, list) and all(isinstance(value, str) for value in values) + for values in fingerprints.values() + ): + raise ValueError("Invalid scenario fingerprints") + declared = report.get("summary") + if not isinstance(declared, dict) or any( + type(declared.get(status)) is not int or declared[status] != counts[status] for status in STATUSES + ): + raise ValueError("Harness summary does not match scenario outcomes") + lines = [escape(report.get("suite", "Regression/replay")) + ": " + totals(counts) + ".", ""] if not scenarios: - return lines + ['No scenarios recorded; no passing regression evidence.'] - lines += ['| Scenario | Result | Process median (ms) |', '| --- | --- | ---: |'] + return lines + ["No scenarios recorded; no passing regression evidence."] + lines += ["| Scenario | Result | Process median (ms) |", "| --- | --- | ---: |"] for item in scenarios[:50]: - value = item.get('metrics', {}).get('process.median_ms') - measured = f'{duration(value):.3f}' if value is not None else '—' - lines.append(f'| {escape(item["id"])} | {item["status"]} | {measured} |') + value = item.get("metrics", {}).get("process.median_ms") + measured = f"{duration(value):.3f}" if value is not None else "—" + lines.append(f"| {escape(item['id'])} | {item['status']} | {measured} |") if len(scenarios) > 50: - lines.append(f'{len(scenarios) - 50} additional scenarios omitted; inspect report.json.') + lines.append(f"{len(scenarios) - 50} additional scenarios omitted; inspect report.json.") for item in scenarios[:50]: - errors = item.get('errors', []) - metrics = {key: value for key, value in item.get('metrics', {}).items() - if (key.startswith('checkpoint.') and key.endswith('_ms')) or key == 'rollback.tick.mean_ms'} - prints = [value for values in item.get('fingerprints', {}).values() for value in values] + errors = item.get("errors", []) + metrics = { + key: value + for key, value in item.get("metrics", {}).items() + if (key.startswith("checkpoint.") and key.endswith("_ms")) or key == "rollback.tick.mean_ms" + } + prints = [value for values in item.get("fingerprints", {}).values() for value in values] if not errors and not metrics and not prints: continue - lines += ['', '**' + escape(item['id']) + '**'] + lines += ["", "**" + escape(item["id"]) + "**"] if errors: - lines.append(block('\n'.join(clipped(error, 300) for error in errors[:4]))) + lines.append(block("\n".join(clipped(error, 300) for error in errors[:4]))) if metrics: - lines.append('; '.join(f'{escape(key)}: {duration(value):.3f} ms' for key, value in sorted(metrics.items())[:8])) + lines.append( + "; ".join(f"{escape(key)}: {duration(value):.3f} ms" for key, value in sorted(metrics.items())[:8]) + ) if prints: - lines.append(block('\n'.join(clipped(value, 240) for value in prints[:4]))) - lines += ['', 'Process timing includes launch overhead. Checkpoint benchmarks use synthetic data; replay results reflect only the scenarios shown.'] + lines.append(block("\n".join(clipped(value, 240) for value in prints[:4]))) + lines += [ + "", + "Process timing includes launch overhead. Checkpoint benchmarks use synthetic data; replay results reflect only the scenarios shown.", + ] return lines def cache_summary(text): - lines = [line for line in text.splitlines() if re.search(r'compile requests|cacheable|hits|misses|hit rate|uncacheable|cache size', line, re.IGNORECASE)] - return [block('\n'.join(lines[:14]))] if lines else ['Unrecognized cache statistics format; supplied details:', block(text)] - - -def render(junit=None, regression=None, cache_stats=None, label='CI results'): - lines, invalid = ['## ' + escape(label), ''], False - for title, path, renderer in (('Host tests', junit, junit_summary), ('Regression and replay', regression, regression_summary), ('Compiler cache', cache_stats, cache_summary)): + lines = [ + line + for line in text.splitlines() + if re.search(r"compile requests|cacheable|hits|misses|hit rate|uncacheable|cache size", line, re.IGNORECASE) + ] + return ( + [block("\n".join(lines[:14]))] + if lines + else ["Unrecognized cache statistics format; supplied details:", block(text)] + ) + + +def render(junit=None, regression=None, cache_stats=None, label="CI results"): + lines, invalid = ["## " + escape(label), ""], False + for title, path, renderer in ( + ("Host tests", junit, junit_summary), + ("Regression and replay", regression, regression_summary), + ("Compiler cache", cache_stats, cache_summary), + ): if path is None: continue - lines += ['### ' + title, ''] + lines += ["### " + title, ""] try: - with Path(path).open('rb') as source: + with Path(path).open("rb") as source: payload = source.read(READ_LIMIT + 1) if len(payload) > READ_LIMIT: - raise ValueError('Artifact exceeds 16 MiB summary input limit') - lines += renderer(payload.decode('utf-8')) + raise ValueError("Artifact exceeds 16 MiB summary input limit") + lines += renderer(payload.decode("utf-8")) except FileNotFoundError: - lines += ['Missing artifact: ' + escape(path) + '. An earlier configure, build, or test step may have failed; no result is inferred.'] + lines += [ + "Missing artifact: " + + escape(path) + + ". An earlier configure, build, or test step may have failed; no result is inferred." + ] except (OSError, ValueError, TypeError, KeyError, ET.ParseError) as error: invalid = True - lines += ['Invalid artifact: ' + escape(path) + '. ' + escape(error, 300) + '. No successful result is inferred.'] - lines.append('') + lines += [ + "Invalid artifact: " + escape(path) + ". " + escape(error, 300) + ". No successful result is inferred." + ] + lines.append("") bounded, size = [], 0 for line in lines: if size + len(line) > 60000: - bounded.append('\nSummary truncated; inspect full artifacts.') + bounded.append("\nSummary truncated; inspect full artifacts.") break bounded.append(line) size += len(line) + 1 - return '\n'.join(bounded), invalid + return "\n".join(bounded), invalid def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--junit', type=Path) - parser.add_argument('--regression', type=Path) - parser.add_argument('--cache-stats', type=Path) - parser.add_argument('--label', default='CI results', help='summary heading') - parser.add_argument('--output', type=Path, help='append Markdown to this file instead of stdout') + parser.add_argument("--junit", type=Path) + parser.add_argument("--regression", type=Path) + parser.add_argument("--cache-stats", type=Path) + parser.add_argument("--label", default="CI results", help="summary heading") + parser.add_argument("--output", type=Path, help="append Markdown to this file instead of stdout") args = parser.parse_args(argv) if not any((args.junit, args.regression, args.cache_stats)): - parser.error('provide at least one report path') + parser.error("provide at least one report path") text, invalid = render(args.junit, args.regression, args.cache_stats, args.label) try: if args.output: - with args.output.open('a', encoding='utf-8') as output: - output.write('\n' + text + '\n') + with args.output.open("a", encoding="utf-8") as output: + output.write("\n" + text + "\n") else: print(text) except OSError as error: - print(f'ci-summary: {error}', file=sys.stderr) + print(f"ci-summary: {error}", file=sys.stderr) return 1 return int(invalid) -if __name__ == '__main__': +if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/compose-lava-glow.py b/scripts/compose-lava-glow.py index b1972adf..f68a0b9b 100644 --- a/scripts/compose-lava-glow.py +++ b/scripts/compose-lava-glow.py @@ -1,7 +1,9 @@ """Compose cached local glow into the existing RSP colour pass, in authored patches only.""" -from pathlib import Path + import difflib +from pathlib import Path import sys + from water_patch_baseline import read_before root = Path(__file__).resolve().parents[1] @@ -23,143 +25,233 @@ def change(path, fn): if before == after: raise SystemExit(f"No glow change: {path}") changes.append(f"diff --git a/{path} b/{path}\n") - changes.extend(difflib.unified_diff(before.splitlines(True), after.splitlines(True), - fromfile=f"a/{path}", tofile=f"b/{path}", n=2)) + changes.extend( + difflib.unified_diff( + before.splitlines(True), after.splitlines(True), fromfile=f"a/{path}", tofile=f"b/{path}", n=2 + ) + ) def workload_header(s): - s = replace(s, " std::vector posFloats;", """ // Immutable local irradiance; the existing RSP pass interpolates its pulse. + s = replace( + s, + " std::vector posFloats;", + """ // Immutable local irradiance; the existing RSP pass interpolates its pulse. std::vector lavaGlow; bool lavaGlowActive=false; float lavaGlowTime=0,lavaGlowStep=0,lavaGlowStrength=0,lavaGlowSpeed=1,lavaGlowIntensity=0; - std::vector posFloats;""") - return replace(s, " BufferPair normalColorBuffer;", " BufferPair normalColorBuffer;\n BufferPair lavaGlowBuffer;") + std::vector posFloats;""", + ) + return replace( + s, + " BufferPair normalColorBuffer;", + " BufferPair normalColorBuffer;\n BufferPair lavaGlowBuffer;", + ) change("src/hle/rt64_workload.h", workload_header) def workload(s): - s = replace(s, " drawData.posFloats.clear();", """ drawData.lavaGlow.clear();drawData.lavaGlowActive=false; + s = replace( + s, + " drawData.posFloats.clear();", + """ drawData.lavaGlow.clear();drawData.lavaGlowActive=false; drawData.lavaGlowTime=drawData.lavaGlowStep=drawData.lavaGlowStrength=drawData.lavaGlowIntensity=0; drawData.lavaGlowSpeed=1; - drawData.posFloats.clear();""") - s = replace(s, " void Workload::uploadDrawData(RenderWorker *worker, BufferUploader *bufferUploader) {", + drawData.posFloats.clear();""", + ) + s = replace( + s, + " void Workload::uploadDrawData(RenderWorker *worker, BufferUploader *bufferUploader) {", """ void Workload::uploadDrawData(RenderWorker *worker, BufferUploader *bufferUploader) { // Only lava scenes upload a per-vertex stream; otherwise bind a 4-byte dummy. - drawData.lavaGlow.resize(drawData.lavaGlowActive ? drawData.vertexCount() : 1,0);""") - return replace(s, " bufferUploader->submit(worker, {", """ bufferUploader->submit(worker, { + drawData.lavaGlow.resize(drawData.lavaGlowActive ? drawData.vertexCount() : 1,0);""", + ) + return replace( + s, + " bufferUploader->submit(worker, {", + """ bufferUploader->submit(worker, { {drawData.lavaGlow.data(),{0,drawData.lavaGlow.size()},sizeof(float),RenderBufferFlag::FORMATTED, - {RenderFormat::R32_FLOAT},&drawBuffers.lavaGlowBuffer},""") + {RenderFormat::R32_FLOAT},&drawBuffers.lavaGlowBuffer},""", + ) change("src/hle/rt64_workload.cpp", workload) -change("src/hle/rt64_rsp.cpp", lambda s: replace(s, - " const uint32_t newIndex = workload.drawData.vertexCount();", """ const uint32_t newIndex = workload.drawData.vertexCount(); +change( + "src/hle/rt64_rsp.cpp", + lambda s: replace( + s, + " const uint32_t newIndex = workload.drawData.vertexCount();", + """ const uint32_t newIndex = workload.drawData.vertexCount(); if(workload.drawData.lavaGlowActive) { auto& glow=workload.drawData.lavaGlow; const float value=globalIndexwaterWorldActive ? 1U : 0U;", """ processCB.waterWorldActive = p.outputBuffers->waterWorldActive ? 1U : 0U; + s = replace( + s, '#include "rt64_buffer_uploader.h"', '#include "rt64_buffer_uploader.h"\n#include "lava_lighting_policy.hpp"' + ) + s = replace( + s, + " processCB.waterWorldActive = p.outputBuffers->waterWorldActive ? 1U : 0U;", + """ processCB.waterWorldActive = p.outputBuffers->waterWorldActive ? 1U : 0U; const float glowTime=drawData.lavaGlowTime-drawData.lavaGlowStep*(1-std::clamp(p.curFrameWeight,0.0f,1.0f)); processCB.lavaGlowCooling=dkr::runtime::water::enhanced::lava_cooling(glowTime,drawData.lavaGlowStrength,drawData.lavaGlowSpeed); processCB.lavaGlowScale=drawData.lavaGlowActive ? std::clamp(drawData.lavaGlowIntensity,0.0f,3.0f) * - dkr::runtime::water::enhanced::lava_pulse(glowTime,drawData.lavaGlowStrength,drawData.lavaGlowSpeed) : 0;""") - return replace(s, " const uint32_t modifyCount = drawData.modifyCount();", """ processSet->setBuffer(processSet->srcLavaGlow,p.drawBuffers->lavaGlowBuffer.get(),p.drawBuffers->lavaGlowBuffer.getView(0)); - const uint32_t modifyCount = drawData.modifyCount();""") + dkr::runtime::water::enhanced::lava_pulse(glowTime,drawData.lavaGlowStrength,drawData.lavaGlowSpeed) : 0;""", + ) + return replace( + s, + " const uint32_t modifyCount = drawData.modifyCount();", + """ processSet->setBuffer(processSet->srcLavaGlow,p.drawBuffers->lavaGlowBuffer.get(),p.drawBuffers->lavaGlowBuffer.getView(0)); + const uint32_t modifyCount = drawData.modifyCount();""", + ) change("src/render/rt64_rsp_processor.cpp", processor) def shader(s): - s = replace(s, " uint waterWorldActive;", " uint waterWorldActive;\n float lavaGlowScale;\n float lavaGlowCooling;\n uint reserved;") - s = replace(s, "RWStructuredBuffer dstWorld : register(u22);", - "RWStructuredBuffer dstWorld : register(u22);\nBuffer srcLavaGlow : register(t23);") - return replace(s, " dstCol[vertexOffsetIndex] = vertexColor;", """ if(gConstants.lavaGlowScale>0) { + s = replace( + s, + " uint waterWorldActive;", + " uint waterWorldActive;\n float lavaGlowScale;\n float lavaGlowCooling;\n uint reserved;", + ) + s = replace( + s, + "RWStructuredBuffer dstWorld : register(u22);", + "RWStructuredBuffer dstWorld : register(u22);\nBuffer srcLavaGlow : register(t23);", + ) + return replace( + s, + " dstCol[vertexOffsetIndex] = vertexColor;", + """ if(gConstants.lavaGlowScale>0) { const float energy=min(0.18,min(0.14,max(0,srcLavaGlow[vertexOffsetIndex]))*gConstants.lavaGlowScale); const float3 warmth=lerp(float3(1,0.19,0.012),float3(1,0.07,0.006),saturate(gConstants.lavaGlowCooling)); vertexColor.rgb=min(vertexColor.rgb+max(vertexColor.rgb,0.12)*warmth*energy,1); } - dstCol[vertexOffsetIndex] = vertexColor;""") + dstCol[vertexOffsetIndex] = vertexColor;""", + ) change("src/shaders/RSPProcessCS.hlsl", shader) def surface(s): - s = replace(s, " float4 molten=lavaSurface(worldPosition.xz,p.camera.w,footprint);", + s = replace( + s, + " float4 molten=lavaSurface(worldPosition.xz,p.camera.w,footprint);", """ float4 molten=lavaSurface(worldPosition.xz,p.camera.w,footprint); // Surface heat and nearby glow share the interpolated clock. Pulse // between hot and cooling palettes, never between lit and black. const float cooling=saturate(m.sunColor.w)*0.5*(1-sin(p.camera.w*m.optics.y*1.7)); - molten.rgb=lavaHeatPalette(molten.rgb,cooling)*m.shoreCounts[0].z;""") - return replace(s, " float4 traced=hardwareWaterReflection(worldPosition,N,V,p,fallingHit,capturedHit);", - """ float4 traced=m.shoreCounts[0].y>0.5 ? hardwareWaterReflection(worldPosition,N,V,p,fallingHit,capturedHit) : float4(0,0,0,0);""") + molten.rgb=lavaHeatPalette(molten.rgb,cooling)*m.shoreCounts[0].z;""", + ) + return replace( + s, + " float4 traced=hardwareWaterReflection(worldPosition,N,V,p,fallingHit,capturedHit);", + """ float4 traced=m.shoreCounts[0].y>0.5 ? hardwareWaterReflection(worldPosition,N,V,p,fallingHit,capturedHit) : float4(0,0,0,0);""", + ) change("src/shaders/WaterPS.hlsl", surface) -change("src/render/rt64_water_renderer.h", lambda s: replace(s, - "std::array profileTotals{};", "std::array profileTotals{};")) +change( + "src/render/rt64_water_renderer.h", + lambda s: replace(s, "std::array profileTotals{};", "std::array profileTotals{};"), +) def profiler(s): s = replace(s, "double milliseconds[4] = {};", "double milliseconds[5] = {};") - s = replace(s, "for(unsigned i=0;i<4;i++)profileTotals[i]+=milliseconds[i];", "for(unsigned i=0;i<5;i++)profileTotals[i]+=milliseconds[i];") + s = replace( + s, + "for(unsigned i=0;i<4;i++)profileTotals[i]+=milliseconds[i];", + "for(unsigned i=0;i<5;i++)profileTotals[i]+=milliseconds[i];", + ) s = replace(s, "profileTotals[4]+=frameMilliseconds;", "profileTotals[5]+=frameMilliseconds;") s = replace(s, "field-ms=%.3f frame-gpu-ms=%.3f", "field-ms=%.3f lava-ms=%.3f frame-gpu-ms=%.3f") - s = replace(s, "profileTotals[3]/profileFrames,profileTotals[4]/profileFrames,static_cast(allocationBytes())", - "profileTotals[3]/profileFrames,profileTotals[4]/profileFrames,profileTotals[5]/profileFrames,static_cast(allocationBytes())") + s = replace( + s, + "profileTotals[3]/profileFrames,profileTotals[4]/profileFrames,static_cast(allocationBytes())", + "profileTotals[3]/profileFrames,profileTotals[4]/profileFrames,profileTotals[5]/profileFrames,static_cast(allocationBytes())", + ) s = replace(s, "field_ms,frame_gpu_ms", "field_ms,lava_ms,frame_gpu_ms") s = replace(s, "%.5f,%.5f,%.5f,%.5f,%.5f\\n", "%.5f,%.5f,%.5f,%.5f,%.5f,%.5f\\n") - return replace(s, "milliseconds[0],milliseconds[1],milliseconds[2],milliseconds[3],frameMilliseconds", - "milliseconds[0],milliseconds[1],milliseconds[2],milliseconds[3],milliseconds[4],frameMilliseconds") + return replace( + s, + "milliseconds[0],milliseconds[1],milliseconds[2],milliseconds[3],frameMilliseconds", + "milliseconds[0],milliseconds[1],milliseconds[2],milliseconds[3],milliseconds[4],frameMilliseconds", + ) change("src/render/rt64_water_renderer.cpp", profiler) def framebuffer(s): - s = replace(s, " if(waterRenderer->drawLavaLighting(worker,pendingLava,depthState,framebuffer.viewport)) {", + s = replace( + s, + " if(waterRenderer->drawLavaLighting(worker,pendingLava,depthState,framebuffer.viewport)) {", """ const auto lavaProfileToken=waterRenderer->beginProfile(worker,4); - if(waterRenderer->drawLavaLighting(worker,pendingLava,depthState,framebuffer.viewport)) {""") - return replace(s, " pendingLava=UINT32_MAX;pendingLavaProjection=UINT32_MAX;", - " waterRenderer->endProfile(worker,lavaProfileToken);\n pendingLava=UINT32_MAX;pendingLavaProjection=UINT32_MAX;") + if(waterRenderer->drawLavaLighting(worker,pendingLava,depthState,framebuffer.viewport)) {""", + ) + return replace( + s, + " pendingLava=UINT32_MAX;pendingLavaProjection=UINT32_MAX;", + " waterRenderer->endProfile(worker,lavaProfileToken);\n pendingLava=UINT32_MAX;pendingLavaProjection=UINT32_MAX;", + ) change("src/render/rt64_framebuffer_renderer.cpp", framebuffer) diff --git a/scripts/compose-lava-lighting.py b/scripts/compose-lava-lighting.py index 8579f6f2..314d703d 100644 --- a/scripts/compose-lava-lighting.py +++ b/scripts/compose-lava-lighting.py @@ -1,55 +1,86 @@ """Compose local ray-tested lava lighting; never edit dependency/generated sources.""" -from pathlib import Path + import difflib +from pathlib import Path import sys + from water_patch_baseline import read_before -root=Path(__file__).resolve().parents[1] -stage=Path(sys.argv[1]).resolve() -output=root/'patches/enhanced-water/dkr-lava-lighting.patch' -layers=[output] if 'finishLava' in (stage/'src/render/rt64_framebuffer_renderer.cpp').read_text() else [] -if 'lavaGlowActive' in (stage/'src/hle/rt64_workload.h').read_text(): - layers.insert(0,root/'patches/enhanced-water/dkr-lava-glow.patch') -changes=[] -def replace(text,before,after): - if text.count(before)!=1:raise SystemExit(f'Non-unique lighting context: {before[:120]}') - return text.replace(before,after,1) -def change(path,fn): - overlay=None - if layers and path=='src/render/rt64_framebuffer_renderer.cpp': - overlay=(stage/path).read_text() - if ' instanceDrawCall.worldProjection=false;' not in overlay: - overlay=replace(overlay,' instanceDrawCall.waterIndex = UINT32_MAX;', - ' instanceDrawCall.waterIndex = UINT32_MAX;\n instanceDrawCall.worldProjection=false;\n instanceDrawCall.projectionIndex=UINT32_MAX;') - before=read_before(stage,path,layers,overlay);after=fn(before) - if before==after:raise SystemExit(f'No lighting change: {path}') - changes.append(f'diff --git a/{path} b/{path}\n') +root = Path(__file__).resolve().parents[1] +stage = Path(sys.argv[1]).resolve() +output = root / "patches/enhanced-water/dkr-lava-lighting.patch" +layers = [output] if "finishLava" in (stage / "src/render/rt64_framebuffer_renderer.cpp").read_text() else [] +if "lavaGlowActive" in (stage / "src/hle/rt64_workload.h").read_text(): + layers.insert(0, root / "patches/enhanced-water/dkr-lava-glow.patch") +changes = [] + + +def replace(text, before, after): + if text.count(before) != 1: + raise SystemExit(f"Non-unique lighting context: {before[:120]}") + return text.replace(before, after, 1) + + +def change(path, fn): + overlay = None + if layers and path == "src/render/rt64_framebuffer_renderer.cpp": + overlay = (stage / path).read_text() + if " instanceDrawCall.worldProjection=false;" not in overlay: + overlay = replace( + overlay, + " instanceDrawCall.waterIndex = UINT32_MAX;", + " instanceDrawCall.waterIndex = UINT32_MAX;\n instanceDrawCall.worldProjection=false;\n instanceDrawCall.projectionIndex=UINT32_MAX;", + ) + before = read_before(stage, path, layers, overlay) + after = fn(before) + if before == after: + raise SystemExit(f"No lighting change: {path}") + changes.append(f"diff --git a/{path} b/{path}\n") # Narrow context preserves Android's adjacent draw-metrics instrumentation. # Every edited semantic anchor above is still required to be unique. - changes.extend(difflib.unified_diff(before.splitlines(True),after.splitlines(True),fromfile=f'a/{path}',tofile=f'b/{path}',n=1)) + changes.extend( + difflib.unified_diff( + before.splitlines(True), after.splitlines(True), fromfile=f"a/{path}", tofile=f"b/{path}", n=1 + ) + ) + + +change( + "src/render/rt64_framebuffer_renderer_call.h", + lambda s: replace( + s, + " uint32_t waterIndex = UINT32_MAX;", + " uint32_t waterIndex = UINT32_MAX;\n bool worldProjection = false;\n uint32_t projectionIndex = UINT32_MAX;", + ), +) + -change('src/render/rt64_framebuffer_renderer_call.h',lambda s:replace(s, - ' uint32_t waterIndex = UINT32_MAX;', - ' uint32_t waterIndex = UINT32_MAX;\n bool worldProjection = false;\n uint32_t projectionIndex = UINT32_MAX;')) def header(s): - s=replace(s,'struct WaterDescriptorSet : RenderDescriptorSetBase {', -'''struct LavaLightingSet : RenderDescriptorSetBase { + s = replace( + s, + "struct WaterDescriptorSet : RenderDescriptorSetBase {", + """struct LavaLightingSet : RenderDescriptorSetBase { uint32_t output,input; explicit LavaLightingSet(RenderDevice *device=nullptr) { builder.begin();output=builder.addReadWriteTexture(1);input=builder.addTexture(2);builder.end(); if(device)create(device); } }; -struct WaterDescriptorSet : RenderDescriptorSetBase {''') - s=replace(s,' std::unique_ptr originalColor;', -''' std::unique_ptr originalColor, lavaLight; +struct WaterDescriptorSet : RenderDescriptorSetBase {""", + ) + s = replace( + s, + " std::unique_ptr originalColor;", + """ std::unique_ptr originalColor, lavaLight; std::unique_ptr lavaLightView; std::unique_ptr lavaLightSet; uint32_t lavaWidth=0,lavaHeight=0; - bool lavaPrepared=false;''') - return replace(s, - ' WaterRayScene rays;', - ''' WaterRayScene rays; + bool lavaPrepared=false;""", + ) + return replace( + s, + " WaterRayScene rays;", + """ WaterRayScene rays; std::unique_ptr lavaLightingVS,lavaLightingPS,lavaLightingCS; std::unique_ptr lavaLightingLayout; std::unique_ptr lavaLightingCompute; @@ -58,10 +89,13 @@ def header(s): RenderPipeline *lavaLightingPipeline(RenderDevice *device,RenderShaderFormat format,const RenderMultisampling& samples); bool prepareLavaLighting(RenderWorker *worker,uint32_t sceneIndex,const interop::WaterDrawParams& draw); bool drawLavaLighting(RenderWorker *worker,uint32_t drawIndex,bool depthReadOnly,const RenderViewport& viewport); -''') -change('src/render/rt64_water_renderer.h',header) +""", + ) + + +change("src/render/rt64_water_renderer.h", header) -implementation=r''' +implementation = r""" RenderPipeline *WaterRenderer::lavaLightingPipeline(RenderDevice *device,RenderShaderFormat format,const RenderMultisampling& samples) { if(lavaLightingUnavailable || !device->getCapabilities().rayQuery || !rayLayout)return nullptr; const uint32_t key=uint32_t(samples.sampleCount); @@ -181,23 +215,41 @@ def header(s): worker->commandList->setGraphicsPushConstants(0,pc.data()); return true; } -''' +""" + + # drawLavaLighting prepares bindings; caller binds the target and issues the draw. def renderer(s): - s=replace(s,'#include "rt64_water_renderer.h"','#include "rt64_water_renderer.h"\n#include "LavaLightingBounds.h"') - s=replace(s,'scene.captured=false; scene.opticalCaptured=false; scene.surfaceCleared=false;', - 'scene.captured=false; scene.opticalCaptured=false; scene.surfaceCleared=false; scene.lavaPrepared=false;') - for fmt in ('spirv','dxil'): - s=replace(s,f'#include "shaders/WaterHardwarePS.hlsl.{fmt}.h"', - f'#include "shaders/WaterHardwarePS.hlsl.{fmt}.h"\n#include "shaders/LavaLightingVS.hlsl.{fmt}.h"\n#include "shaders/LavaLightingPS.hlsl.{fmt}.h"\n#include "shaders/LavaLightingCS.hlsl.{fmt}.h"') - s=replace(s,'bool(s.waterDepth),s.waterDepthWidth,s.waterDepthHeight,s.waterDepthSamples,s.originalWidth,s.originalHeight) : uint64_t(0);', - 'bool(s.waterDepth),s.waterDepthWidth,s.waterDepthHeight,s.waterDepthSamples,s.originalWidth,s.originalHeight)+uint64_t(s.lavaWidth)*s.lavaHeight*16 : uint64_t(0);') - s=replace(s,'classic,owner!=index,selfDepth,depth->width,depth->height,depth->multisampling.sampleCount,ow,oh);', - 'classic,owner!=index,selfDepth,depth->width,depth->height,depth->multisampling.sampleCount,ow,oh)+(existing ? uint64_t(scenes[index].lavaWidth)*scenes[index].lavaHeight*16 : 0);') - return replace(s,'bool WaterRenderer::prepareRays(',implementation+'\nbool WaterRenderer::prepareRays(') -change('src/render/rt64_water_renderer.cpp',renderer) - -finish=r''' uint32_t pendingLava=UINT32_MAX,pendingLavaProjection=UINT32_MAX; + s = replace( + s, '#include "rt64_water_renderer.h"', '#include "rt64_water_renderer.h"\n#include "LavaLightingBounds.h"' + ) + s = replace( + s, + "scene.captured=false; scene.opticalCaptured=false; scene.surfaceCleared=false;", + "scene.captured=false; scene.opticalCaptured=false; scene.surfaceCleared=false; scene.lavaPrepared=false;", + ) + for fmt in ("spirv", "dxil"): + s = replace( + s, + f'#include "shaders/WaterHardwarePS.hlsl.{fmt}.h"', + f'#include "shaders/WaterHardwarePS.hlsl.{fmt}.h"\n#include "shaders/LavaLightingVS.hlsl.{fmt}.h"\n#include "shaders/LavaLightingPS.hlsl.{fmt}.h"\n#include "shaders/LavaLightingCS.hlsl.{fmt}.h"', + ) + s = replace( + s, + "bool(s.waterDepth),s.waterDepthWidth,s.waterDepthHeight,s.waterDepthSamples,s.originalWidth,s.originalHeight) : uint64_t(0);", + "bool(s.waterDepth),s.waterDepthWidth,s.waterDepthHeight,s.waterDepthSamples,s.originalWidth,s.originalHeight)+uint64_t(s.lavaWidth)*s.lavaHeight*16 : uint64_t(0);", + ) + s = replace( + s, + "classic,owner!=index,selfDepth,depth->width,depth->height,depth->multisampling.sampleCount,ow,oh);", + "classic,owner!=index,selfDepth,depth->width,depth->height,depth->multisampling.sampleCount,ow,oh)+(existing ? uint64_t(scenes[index].lavaWidth)*scenes[index].lavaHeight*16 : 0);", + ) + return replace(s, "bool WaterRenderer::prepareRays(", implementation + "\nbool WaterRenderer::prepareRays(") + + +change("src/render/rt64_water_renderer.cpp", renderer) + +finish = r""" uint32_t pendingLava=UINT32_MAX,pendingLavaProjection=UINT32_MAX; auto finishLava = [&]() { if(pendingLava==UINT32_MAX)return; finishWater(); @@ -209,45 +261,81 @@ def renderer(s): switchToGraphicsPipeline(); }; -''' +""" + + def framebuffer(s): - s=replace(s,' instanceDrawCall.waterIndex = UINT32_MAX;', -''' instanceDrawCall.waterIndex = UINT32_MAX; + s = replace( + s, + " instanceDrawCall.waterIndex = UINT32_MAX;", + """ instanceDrawCall.waterIndex = UINT32_MAX; instanceDrawCall.worldProjection=false; - instanceDrawCall.projectionIndex=UINT32_MAX;''') - s=replace(s,' for (uint32_t i : rasterScene.instanceIndices) {',finish+' for (uint32_t i : rasterScene.instanceIndices) {') - s=replace(s,' const InstanceDrawCall &drawCall = instanceDrawCallVector[i];', -''' const InstanceDrawCall &drawCall = instanceDrawCallVector[i]; + instanceDrawCall.projectionIndex=UINT32_MAX;""", + ) + s = replace( + s, + " for (uint32_t i : rasterScene.instanceIndices) {", + finish + " for (uint32_t i : rasterScene.instanceIndices) {", + ) + s = replace( + s, + " const InstanceDrawCall &drawCall = instanceDrawCallVector[i];", + """ const InstanceDrawCall &drawCall = instanceDrawCallVector[i]; if(pendingLava!=UINT32_MAX && (!drawCall.worldProjection || drawCall.projectionIndex!=pendingLavaProjection))finishLava(); if(pendingLava==UINT32_MAX && drawCall.waterIndex!=UINT32_MAX) { const auto& m=waterRenderer->draws[drawCall.waterIndex].material; if(m.effects.z==5 && m.effects.w>0.5f && m.sunDirection.w>0) { pendingLava=drawCall.waterIndex;pendingLavaProjection=drawCall.projectionIndex; } - }''') - s=replace(s,' // Mark targets for resolve.\n finishWater();', - ' // Mark targets for resolve.\n finishLava();\n finishWater();') - s=replace(s,' switch (proj.type) {', -''' instanceDrawCall.worldProjection=proj.type==Projection::Type::Perspective; + }""", + ) + s = replace( + s, + " // Mark targets for resolve.\n finishWater();", + " // Mark targets for resolve.\n finishLava();\n finishWater();", + ) + s = replace( + s, + " switch (proj.type) {", + """ instanceDrawCall.worldProjection=proj.type==Projection::Type::Perspective; instanceDrawCall.projectionIndex=proj.transformsIndex; - switch (proj.type) {''') - s=replace(s,'bool hardware=!classic && wp.material.appearance.x>0 && wp.material.effects.w>0.5f &&', - 'bool hardware=!classic && (wp.material.appearance.x>0 || (wp.material.effects.z==5 && wp.material.sunDirection.w>0)) && wp.material.effects.w>0.5f &&') - s=replace(s,' wp.material.effects.w=hardware ? 1.0f : 0.0f;', -''' wp.material.effects.w=hardware ? 1.0f : 0.0f; + switch (proj.type) {""", + ) + s = replace( + s, + "bool hardware=!classic && wp.material.appearance.x>0 && wp.material.effects.w>0.5f &&", + "bool hardware=!classic && (wp.material.appearance.x>0 || (wp.material.effects.z==5 && wp.material.sunDirection.w>0)) && wp.material.effects.w>0.5f &&", + ) + s = replace( + s, + " wp.material.effects.w=hardware ? 1.0f : 0.0f;", + """ wp.material.effects.w=hardware ? 1.0f : 0.0f; if(hardware && wp.material.effects.z==5 && wp.material.sunDirection.w>0 && !waterRenderer->lavaLightingPipeline(p.worker->device,p.rasterShaderCache->shaderFormat,p.fbStorage->colorTarget->multisampling)) - wp.material.sunDirection.w=0;''') - s=replace(s,' if(waterScene!=water_policy::invalidScene) {', -''' if(waterScene!=water_policy::invalidScene) { + wp.material.sunDirection.w=0;""", + ) + s = replace( + s, + " if(waterScene!=water_policy::invalidScene) {", + """ if(waterScene!=water_policy::invalidScene) { if(hardware && wp.material.effects.z==5 && wp.material.sunDirection.w>0 && - !waterRenderer->prepareLavaLighting(p.worker,waterScene,wp))wp.material.sunDirection.w=0;''') + !waterRenderer->prepareLavaLighting(p.worker,waterScene,wp))wp.material.sunDirection.w=0;""", + ) return s -change('src/render/rt64_framebuffer_renderer.cpp',framebuffer) + + +change("src/render/rt64_framebuffer_renderer.cpp", framebuffer) + + def lava_shader(s): - s=replace(s,' float4 reflection=0;\n#ifdef HARDWARE_WATER', - ' float4 reflection=0;\n if(m.appearance.x>0) {\n#ifdef HARDWARE_WATER') - return replace(s,'#endif\n float fresnel=0.04', '#endif\n }\n float fresnel=0.04') -change('src/shaders/WaterPS.hlsl',lava_shader) -output.write_text(''.join(changes),newline='\n') -print(f'Composed {output.name}: {len(changes)} diff records') + s = replace( + s, + " float4 reflection=0;\n#ifdef HARDWARE_WATER", + " float4 reflection=0;\n if(m.appearance.x>0) {\n#ifdef HARDWARE_WATER", + ) + return replace(s, "#endif\n float fresnel=0.04", "#endif\n }\n float fresnel=0.04") + + +change("src/shaders/WaterPS.hlsl", lava_shader) +output.write_text("".join(changes), newline="\n") +print(f"Composed {output.name}: {len(changes)} diff records") diff --git a/scripts/compose-lava.py b/scripts/compose-lava.py index 29797736..a335b410 100644 --- a/scripts/compose-lava.py +++ b/scripts/compose-lava.py @@ -1,29 +1,38 @@ """Compose the molten-lava patch; dependency and generated sources stay read-only.""" -from pathlib import Path + import difflib +from pathlib import Path import sys + from water_patch_baseline import read_before -root=Path(__file__).resolve().parents[1] -stage=Path(sys.argv[1]).resolve() -output=root/'patches/enhanced-water/dkr-lava.patch' -layers=[output] if 'lavaSurface' in (stage/'src/shaders/WaterPS.hlsl').read_text() else [] -changes=[] +root = Path(__file__).resolve().parents[1] +stage = Path(sys.argv[1]).resolve() +output = root / "patches/enhanced-water/dkr-lava.patch" +layers = [output] if "lavaSurface" in (stage / "src/shaders/WaterPS.hlsl").read_text() else [] +changes = [] + -def replace(text,before,after): - if text.count(before)!=1: - raise SystemExit(f'Non-unique lava context: {before[:120]}') - return text.replace(before,after,1) +def replace(text, before, after): + if text.count(before) != 1: + raise SystemExit(f"Non-unique lava context: {before[:120]}") + return text.replace(before, after, 1) -def change(path,fn): - before=read_before(stage,path,layers) - after=fn(before) - if before==after:raise SystemExit(f'No lava change: {path}') - changes.append(f'diff --git a/{path} b/{path}\n') - changes.extend(difflib.unified_diff(before.splitlines(True),after.splitlines(True), - fromfile=f'a/{path}',tofile=f'b/{path}',n=3)) -lava=r''' if (m.effects.z==5) { +def change(path, fn): + before = read_before(stage, path, layers) + after = fn(before) + if before == after: + raise SystemExit(f"No lava change: {path}") + changes.append(f"diff --git a/{path} b/{path}\n") + changes.extend( + difflib.unified_diff( + before.splitlines(True), after.splitlines(True), fromfile=f"a/{path}", tofile=f"b/{path}", n=3 + ) + ) + + +lava = r""" if (m.effects.z==5) { // Keep the authored static/tessellated lava mesh and depth contract. // Molten material is opaque: no water transmission, caustics or wakes. float3 geometric=surfaceNormal; @@ -54,16 +63,25 @@ def change(path,fn): float3 color=molten.rgb*(m.shallowColor.rgb/float3(1,0.18,0.015))+reflected*reflectionWeight; return float4(lavaDisplayColor(color,p.fogColor.rgb,p.fogColor.a>0.5 ? sourceColor.a : 0),1); } -''' +""" + def shader(text): - text=replace(text,'#include "WaterProjection.hlsli"', - '#include "WaterProjection.hlsli"\n#include "LavaFlow.hlsli"') - return replace(text,' if (m.effects.z>=3) {',lava+' if (m.effects.z>=3 && m.effects.z<=4) {') -change('src/shaders/WaterPS.hlsl',shader) -change('src/render/rt64_water_renderer.cpp',lambda text:replace(text, - ' if(draws.back().material.effects.z>=3)return;', - ' if(draws.back().material.effects.z>=3 && draws.back().material.effects.z<=4)return;')) + text = replace( + text, '#include "WaterProjection.hlsli"', '#include "WaterProjection.hlsli"\n#include "LavaFlow.hlsli"' + ) + return replace(text, " if (m.effects.z>=3) {", lava + " if (m.effects.z>=3 && m.effects.z<=4) {") + + +change("src/shaders/WaterPS.hlsl", shader) +change( + "src/render/rt64_water_renderer.cpp", + lambda text: replace( + text, + " if(draws.back().material.effects.z>=3)return;", + " if(draws.back().material.effects.z>=3 && draws.back().material.effects.z<=4)return;", + ), +) -output.write_text(''.join(changes),newline='\n') -print(f'Composed {output.name}: {len(changes)} diff records') +output.write_text("".join(changes), newline="\n") +print(f"Composed {output.name}: {len(changes)} diff records") diff --git a/scripts/compose-water-performance.py b/scripts/compose-water-performance.py index 14869b7c..207d4f5d 100644 --- a/scripts/compose-water-performance.py +++ b/scripts/compose-water-performance.py @@ -1,89 +1,160 @@ """Author Playtest 16 renderer changes; never mutate protected dependencies.""" -from pathlib import Path + import difflib +from pathlib import Path import sys + from water_patch_baseline import read_before -root=Path(__file__).resolve().parents[1] -stage=Path(sys.argv[1]).resolve() -output=root/'patches/enhanced-water/dkr-performance.patch' -applied=Path(sys.argv[2]).resolve() if len(sys.argv)>2 else output -layers=[applied] if 'WaterWorkPolicy.h' in (stage/'src/render/rt64_water_renderer.h').read_text() else [] -changes=[] +root = Path(__file__).resolve().parents[1] +stage = Path(sys.argv[1]).resolve() +output = root / "patches/enhanced-water/dkr-performance.patch" +applied = Path(sys.argv[2]).resolve() if len(sys.argv) > 2 else output +layers = [applied] if "WaterWorkPolicy.h" in (stage / "src/render/rt64_water_renderer.h").read_text() else [] +changes = [] -def replace(t,a,b): - if t.count(a)!=1:raise SystemExit(f'Non-unique performance context: {a[:140]}') - return t.replace(a,b,1) -def change(path,fn): - before=read_before(stage,path,layers);after=fn(before) - if before==after:raise SystemExit(f'No performance change: {path}') - changes.append(f'diff --git a/{path} b/{path}\n') - changes.extend(difflib.unified_diff(before.splitlines(True),after.splitlines(True),fromfile=f'a/{path}',tofile=f'b/{path}',n=3)) +def replace(t, a, b): + if t.count(a) != 1: + raise SystemExit(f"Non-unique performance context: {a[:140]}") + return t.replace(a, b, 1) + + +def change(path, fn): + before = read_before(stage, path, layers) + after = fn(before) + if before == after: + raise SystemExit(f"No performance change: {path}") + changes.append(f"diff --git a/{path} b/{path}\n") + changes.extend( + difflib.unified_diff( + before.splitlines(True), after.splitlines(True), fromfile=f"a/{path}", tofile=f"b/{path}", n=3 + ) + ) + def header(t): - t=replace(t,'#include "WaterResourcePolicy.h"','#include "WaterResourcePolicy.h"\n#include "WaterWorkPolicy.h"') - t=replace(t,' float fieldTime = 0, fieldDesired = 0;',' bool fieldWasActive=false;\n float fieldTime = 0, fieldDesired = 0;') - t=replace(t,' std::set vertices;',' std::vector vertices;') - return replace(t,' std::vector profileKinds;', - ' std::vector profileKinds;\n uint32_t profileFrames=0;\n std::array profileTotals{};') -change('src/render/rt64_water_renderer.h',header) + t = replace(t, '#include "WaterResourcePolicy.h"', '#include "WaterResourcePolicy.h"\n#include "WaterWorkPolicy.h"') + t = replace( + t, + " float fieldTime = 0, fieldDesired = 0;", + " bool fieldWasActive=false;\n float fieldTime = 0, fieldDesired = 0;", + ) + t = replace(t, " std::set vertices;", " std::vector vertices;") + return replace( + t, + " std::vector profileKinds;", + " std::vector profileKinds;\n uint32_t profileFrames=0;\n std::array profileTotals{};", + ) + + +change("src/render/rt64_water_renderer.h", header) + def renderer(t): - t=replace(t,'#include "rt64_water_renderer.h"','#include "rt64_water_renderer.h"\n#include ') + t = replace(t, '#include "rt64_water_renderer.h"', '#include "rt64_water_renderer.h"\n#include ') # Capture/reduction push constants include the dispatch region (32 bytes). - t=t.replace('builder.addPushConstant(0,0,16,RenderShaderStageFlag::COMPUTE);', - 'builder.addPushConstant(0,0,32,RenderShaderStageFlag::COMPUTE);') - t=t.replace('builder.addPushConstant(0, 0, 16, RenderShaderStageFlag::COMPUTE);', - 'builder.addPushConstant(0, 0, 32, RenderShaderStageFlag::COMPUTE);') - t=replace(t,' const uint32_t params[] = {scene.width,scene.height,samples,0};', - ''' auto region=water_work::captureRegion(scene.width,scene.height,view.x,view.y,view.z,view.w); - const uint32_t params[] = {scene.width,scene.height,samples,0,region.x,region.y,region.width,region.height};''') - t=replace(t,' worker->commandList->dispatch((scene.width+7)/8,(scene.height+7)/8,1);\n if (draws[drawIndex].material.identity.x >= 2)', - ' worker->commandList->dispatch((region.width+7)/8,(region.height+7)/8,1);\n if (draws[drawIndex].material.identity.x >= 2)') - t=replace(t,' const uint32_t reduceParams[] = {w,h,sourceWidth,sourceHeight};', - ''' region=water_work::reduceRegion(region,w,h); - const uint32_t reduceParams[] = {w,h,sourceWidth,sourceHeight,region.x,region.y,region.width,region.height};''') - t=replace(t,' worker->commandList->dispatch((w+7)/8,(h+7)/8,1);', - ' worker->commandList->dispatch((region.width+7)/8,(region.height+7)/8,1);') - t=replace(t,' const uint32_t params[]={scene.width,scene.height,samples,0};', - ''' const auto& view=draws[drawIndex].viewBounds; + t = t.replace( + "builder.addPushConstant(0,0,16,RenderShaderStageFlag::COMPUTE);", + "builder.addPushConstant(0,0,32,RenderShaderStageFlag::COMPUTE);", + ) + t = t.replace( + "builder.addPushConstant(0, 0, 16, RenderShaderStageFlag::COMPUTE);", + "builder.addPushConstant(0, 0, 32, RenderShaderStageFlag::COMPUTE);", + ) + t = replace( + t, + " const uint32_t params[] = {scene.width,scene.height,samples,0};", + """ auto region=water_work::captureRegion(scene.width,scene.height,view.x,view.y,view.z,view.w); + const uint32_t params[] = {scene.width,scene.height,samples,0,region.x,region.y,region.width,region.height};""", + ) + t = replace( + t, + " worker->commandList->dispatch((scene.width+7)/8,(scene.height+7)/8,1);\n if (draws[drawIndex].material.identity.x >= 2)", + " worker->commandList->dispatch((region.width+7)/8,(region.height+7)/8,1);\n if (draws[drawIndex].material.identity.x >= 2)", + ) + t = replace( + t, + " const uint32_t reduceParams[] = {w,h,sourceWidth,sourceHeight};", + """ region=water_work::reduceRegion(region,w,h); + const uint32_t reduceParams[] = {w,h,sourceWidth,sourceHeight,region.x,region.y,region.width,region.height};""", + ) + t = replace( + t, + " worker->commandList->dispatch((w+7)/8,(h+7)/8,1);", + " worker->commandList->dispatch((region.width+7)/8,(region.height+7)/8,1);", + ) + t = replace( + t, + " const uint32_t params[]={scene.width,scene.height,samples,0};", + """ const auto& view=draws[drawIndex].viewBounds; const auto region=water_work::captureRegion(scene.width,scene.height,view.x,view.y,view.z,view.w); - const uint32_t params[]={scene.width,scene.height,samples,0,region.x,region.y,region.width,region.height};''') - t=replace(t,' worker->commandList->dispatch((scene.width+7)/8,(scene.height+7)/8,1);', - ' worker->commandList->dispatch((region.width+7)/8,(region.height+7)/8,1);') - t=replace(t,' const uint32_t resolution=active ? (m.identity.x>=3 ? 512 : 256) : 1;', - ' const uint32_t resolution=water_work::fieldResolution(active,bool(fieldTextures[0]),fieldResolution,uint32_t(m.identity.x));') - t=replace(t,' for (uint32_t i=0;i<2;i++) {\n fieldTextures[i]=', - ''' // Descriptors/views must retire before replacing their textures. + const uint32_t params[]={scene.width,scene.height,samples,0,region.x,region.y,region.width,region.height};""", + ) + t = replace( + t, + " worker->commandList->dispatch((scene.width+7)/8,(scene.height+7)/8,1);", + " worker->commandList->dispatch((region.width+7)/8,(region.height+7)/8,1);", + ) + t = replace( + t, + " const uint32_t resolution=active ? (m.identity.x>=3 ? 512 : 256) : 1;", + " const uint32_t resolution=water_work::fieldResolution(active,bool(fieldTextures[0]),fieldResolution,uint32_t(m.identity.x));", + ) + t = replace( + t, + " for (uint32_t i=0;i<2;i++) {\n fieldTextures[i]=", + """ // Descriptors/views must retire before replacing their textures. for(uint32_t i=0;i<2;i++) {fieldSets[i].reset();fieldViews[i].reset();} for (uint32_t i=0;i<2;i++) { - fieldTextures[i]=''') - t=replace(t,' const bool reset=fieldGeneration!=uint32_t(m.identity.w) || desiredfieldDesired+0.25f;', - ''' const bool reset=fieldGeneration!=uint32_t(m.identity.w) || active!=fieldWasActive || + fieldTextures[i]=""", + ) + t = replace( + t, + " const bool reset=fieldGeneration!=uint32_t(m.identity.w) || desiredfieldDesired+0.25f;", + """ const bool reset=fieldGeneration!=uint32_t(m.identity.w) || active!=fieldWasActive || desiredfieldDesired+0.25f; - fieldWasActive=active;''') + fieldWasActive=active;""", + ) # Normal grouping decodes raw (not interpolated) source positions. Keep # that exact camera inverse, caching it across same-camera water batches. - t=replace(t,' const auto cameraInverse=hlslpp::inverse(hlslpp::float4x4(draws.back().material.cameraViewProjection));', - ''' const auto camera=hlslpp::float4x4(draws.back().material.cameraViewProjection); + t = replace( + t, + " const auto cameraInverse=hlslpp::inverse(hlslpp::float4x4(draws.back().material.cameraViewProjection));", + """ const auto camera=hlslpp::float4x4(draws.back().material.cameraViewProjection); struct CameraCache {hlslpp::float4x4 matrix,inverse;bool valid=false;}; thread_local CameraCache cache; if(!cache.valid || std::memcmp(&cache.matrix,&camera,sizeof(camera))) { cache.matrix=camera;cache.inverse=hlslpp::inverse(camera);cache.valid=true; } - const auto cameraInverse=cache.inverse;''') - t=replace(t,' g.vertices.insert(v);', - ' if(std::find(g.vertices.begin(),g.vertices.end(),v)==g.vertices.end())g.vertices.push_back(v);') - t=replace(t,' for (const auto &[key,g] : normalGroups) {', - ''' for (auto &[key,g] : normalGroups) { + const auto cameraInverse=cache.inverse;""", + ) + t = replace( + t, + " g.vertices.insert(v);", + " if(std::find(g.vertices.begin(),g.vertices.end(),v)==g.vertices.end())g.vertices.push_back(v);", + ) + t = replace( + t, + " for (const auto &[key,g] : normalGroups) {", + """ for (auto &[key,g] : normalGroups) { // Preserve the original set's ascending vertex order without a heap // allocation/tree node for every vertex in every frame. - std::sort(g.vertices.begin(),g.vertices.end());''') - t=replace(t,' if (!profilePool || profileKinds.empty() || draws.empty()) return;', - ' if (!profilePool || profileKinds.empty() || draws.empty()) return;') if False else t - t=replace(t,' static FILE *csv = []() {', - ''' for(unsigned i=0;i<4;i++)profileTotals[i]+=milliseconds[i]; + std::sort(g.vertices.begin(),g.vertices.end());""", + ) + t = ( + replace( + t, + " if (!profilePool || profileKinds.empty() || draws.empty()) return;", + " if (!profilePool || profileKinds.empty() || draws.empty()) return;", + ) + if False + else t + ) + t = replace( + t, + " static FILE *csv = []() {", + """ for(unsigned i=0;i<4;i++)profileTotals[i]+=milliseconds[i]; profileTotals[4]+=frameMilliseconds; if(++profileFrames==120) { std::fprintf(stderr,"[water][timing] frames=%u normal-ms=%.3f capture-ms=%.3f surface-ms=%.3f field-ms=%.3f frame-gpu-ms=%.3f bytes=%llu\\n", @@ -91,37 +162,65 @@ def renderer(t): profileTotals[3]/profileFrames,profileTotals[4]/profileFrames,static_cast(allocationBytes())); profileFrames=0;profileTotals.fill(0); } - static FILE *csv = []() {''') + static FILE *csv = []() {""", + ) return t -change('src/render/rt64_water_renderer.cpp',renderer) + + +change("src/render/rt64_water_renderer.cpp", renderer) + def capture(t): - t=replace(t,'struct CaptureParams { uint width; uint height; uint samples; uint unused; };', - 'struct CaptureParams { uint width; uint height; uint samples; uint unused; uint2 origin; uint2 extent; };') - return replace(t,' if (pixel.x >= params.width || pixel.y >= params.height) return;', - ' if (any(pixel>=params.extent)) return;\n pixel+=params.origin;\n if (pixel.x >= params.width || pixel.y >= params.height) return;') -change('src/shaders/WaterCaptureCS.hlsl',capture) -change('src/shaders/WaterOriginalCS.hlsl',capture) + t = replace( + t, + "struct CaptureParams { uint width; uint height; uint samples; uint unused; };", + "struct CaptureParams { uint width; uint height; uint samples; uint unused; uint2 origin; uint2 extent; };", + ) + return replace( + t, + " if (pixel.x >= params.width || pixel.y >= params.height) return;", + " if (any(pixel>=params.extent)) return;\n pixel+=params.origin;\n if (pixel.x >= params.width || pixel.y >= params.height) return;", + ) + + +change("src/shaders/WaterCaptureCS.hlsl", capture) +change("src/shaders/WaterOriginalCS.hlsl", capture) + + def reduce(t): - t=replace(t,'struct ReduceParams { uint width; uint height; uint sourceWidth; uint sourceHeight; };', - 'struct ReduceParams { uint width; uint height; uint sourceWidth; uint sourceHeight; uint2 origin; uint2 extent; };') - return replace(t,' if (pixel.x >= params.width || pixel.y >= params.height) return;', - ' if (any(pixel>=params.extent)) return;\n pixel+=params.origin;\n if (pixel.x >= params.width || pixel.y >= params.height) return;') -change('src/shaders/WaterDepthReduceCS.hlsl',reduce) + t = replace( + t, + "struct ReduceParams { uint width; uint height; uint sourceWidth; uint sourceHeight; };", + "struct ReduceParams { uint width; uint height; uint sourceWidth; uint sourceHeight; uint2 origin; uint2 extent; };", + ) + return replace( + t, + " if (pixel.x >= params.width || pixel.y >= params.height) return;", + " if (any(pixel>=params.extent)) return;\n pixel+=params.origin;\n if (pixel.x >= params.width || pixel.y >= params.height) return;", + ) + + +change("src/shaders/WaterDepthReduceCS.hlsl", reduce) + def field(t): # Immutable pre-optimisation shader for the headless equivalence probe. - (root/'patches/enhanced-water/native/WaterFieldReferenceCS.hlsl').write_text(t,newline='\n') - t=replace(t,'[numthreads(8,8,1)]\nvoid CSMain(uint2 pixel : SV_DispatchThreadID)', - '''// Racer motion is uniform for all 64 cells in this group. Computing it + (root / "patches/enhanced-water/native/WaterFieldReferenceCS.hlsl").write_text(t, newline="\n") + t = replace( + t, + "[numthreads(8,8,1)]\nvoid CSMain(uint2 pixel : SV_DispatchThreadID)", + """// Racer motion is uniform for all 64 cells in this group. Computing it // once per racer avoids repeating square roots and segment setup per cell. groupshared float4 craftPath[WATER_CRAFT_COUNT]; groupshared float4 craftMotion[WATER_CRAFT_COUNT]; groupshared float4 craftResponse[WATER_CRAFT_COUNT]; [numthreads(8,8,1)] -void CSMain(uint2 pixel : SV_DispatchThreadID, uint lane : SV_GroupIndex)''') - t=replace(t,' float cell=params.simulation.w/params.resolution;', - ''' // Reset still injects this step's wake/shore response, like the reference. +void CSMain(uint2 pixel : SV_DispatchThreadID, uint lane : SV_GroupIndex)""", + ) + t = replace( + t, + " float cell=params.simulation.w/params.resolution;", + """ // Reset still injects this step's wake/shore response, like the reference. float cell=params.simulation.w/params.resolution; float interval=max(m.animation.y-m.animation.x,0.00001); float a=saturate((params.simulation.y-m.animation.x)/interval); @@ -141,22 +240,29 @@ def field(t): if (pixel.x>=params.resolution*WATER_CRAFT_COUNT || pixel.y>=params.resolution) return; uint tile=pixel.x/params.resolution; uint2 local=uint2(pixel.x%params.resolution,pixel.y); - if(m.craftCurrent[tile].w<0) {nextField[pixel]=0;return;}''') - t=replace(t,''' if (pixel.x>=params.resolution*WATER_CRAFT_COUNT || pixel.y>=params.resolution) return; + if(m.craftCurrent[tile].w<0) {nextField[pixel]=0;return;}""", + ) + t = replace( + t, + """ if (pixel.x>=params.resolution*WATER_CRAFT_COUNT || pixel.y>=params.resolution) return; uint tile=pixel.x/params.resolution; uint2 local=uint2(pixel.x%params.resolution,pixel.y); - WaterDrawParams draw=waterDraws[params.drawIndex];''',''' // Zero-duration reset has no history, swept motion, or shoreline growth. + WaterDrawParams draw=waterDraws[params.drawIndex];""", + """ // Zero-duration reset has no history, swept motion, or shoreline growth. // This dispatch-uniform fast path preserves the exact reference result. if(params.reset && params.simulation.x==0 && params.simulation.y==params.simulation.z) { if(pixel.x0 && params.simulation.y=m.animation.y; + float speed=craftMotion[i].z,density=craftMotion[i].w;""", + ) + t = replace( + t, + """ bool landed=old.w==0 && current.w>0 && params.simulation.y=m.animation.y; if (landed && !params.reset) { float impact=saturate((old.y-current.y)/interval/180); - float radius=length(world-current.xz);''', - ''' if (craftResponse[i].y>0) { + float radius=length(world-current.xz);""", + """ if (craftResponse[i].y>0) { float impact=craftResponse[i].x; - float radius=length(world-m.craftCurrent[i].xz);''') + float radius=length(world-m.craftCurrent[i].xz);""", + ) return t -change('src/shaders/WaterFieldCS.hlsl',field) + + +change("src/shaders/WaterFieldCS.hlsl", field) + def pixels(t): # Exact dimensions are already known; avoid a texture dimensions query # inside every SSR search iteration. Reflection sample counts are unchanged. - return replace(t,' uint mipWidth,mipHeight,levels; sceneDepth.GetDimensions(mip,mipWidth,mipHeight,levels);', - ' uint mipWidth=max(depthWidth>>mip,1u),mipHeight=max(depthHeight>>mip,1u);') -change('src/shaders/WaterPS.hlsl',pixels) + return replace( + t, + " uint mipWidth,mipHeight,levels; sceneDepth.GetDimensions(mip,mipWidth,mipHeight,levels);", + " uint mipWidth=max(depthWidth>>mip,1u),mipHeight=max(depthHeight>>mip,1u);", + ) + + +change("src/shaders/WaterPS.hlsl", pixels) + def framebuffer(t): - t=replace(t,' wp.inverseViewProjection = hlslpp::inverse(wp.viewProjection);', - ' if(wp.material.cameraPosition.w!=1.0f)\n wp.inverseViewProjection = hlslpp::inverse(wp.viewProjection);') - start=t.index(' worker->commandList->setGraphicsPipelineLayout(waterRenderer->layout.get());', - t.index('if (drawCall.waterIndex != UINT32_MAX)')) - end=t.index(' if (selfDepth) waterRenderer->beginColor',start) - old=t[start:end] + t = replace( + t, + " wp.inverseViewProjection = hlslpp::inverse(wp.viewProjection);", + " if(wp.material.cameraPosition.w!=1.0f)\n wp.inverseViewProjection = hlslpp::inverse(wp.viewProjection);", + ) + start = t.index( + " worker->commandList->setGraphicsPipelineLayout(waterRenderer->layout.get());", + t.index("if (drawCall.waterIndex != UINT32_MAX)"), + ) + end = t.index(" if (selfDepth) waterRenderer->beginColor", start) + old = t[start:end] # The depth prepass remains mandatory for translucent modern water. The # mask-only pass is unused when there are no particles to draw. - oldprefix=''' worker->commandList->setGraphicsPipelineLayout(waterRenderer->layout.get());''' - oldend=''' drawCallTriangles(drawCall); -''' - body=old[old.index(oldprefix):] - body=body.replace(' const bool classic=waterRenderer->draws[drawCall.waterIndex].material.optics.w>0.5f;\n','') - body=body.replace(' const bool selfDepth=waterRenderer->scenes[waterRenderer->drawScenes[drawCall.waterIndex]].selfDepth;\n','') - t=t[:start]+''' const bool classic=waterRenderer->draws[drawCall.waterIndex].material.optics.w>0.5f; + oldprefix = """ worker->commandList->setGraphicsPipelineLayout(waterRenderer->layout.get());""" + oldend = """ drawCallTriangles(drawCall); +""" + body = old[old.index(oldprefix) :] + body = body.replace( + " const bool classic=waterRenderer->draws[drawCall.waterIndex].material.optics.w>0.5f;\n", "" + ) + body = body.replace( + " const bool selfDepth=waterRenderer->scenes[waterRenderer->drawScenes[drawCall.waterIndex]].selfDepth;\n", + "", + ) + t = ( + t[:start] + + """ const bool classic=waterRenderer->draws[drawCall.waterIndex].material.optics.w>0.5f; const bool selfDepth=waterRenderer->scenes[waterRenderer->drawScenes[drawCall.waterIndex]].selfDepth; const bool needsSurface=selfDepth || (!waterRenderer->particles.empty() && waterRenderer->draws[drawCall.waterIndex].material.effects.x>0.5f); if(needsSurface) { -'''+body+''' } -'''+t[end:] +""" + + body + + """ } +""" + + t[end:] + ) return t -change('src/render/rt64_framebuffer_renderer.cpp',framebuffer) + + +change("src/render/rt64_framebuffer_renderer.cpp", framebuffer) + def workload(t): - return replace(t,' rendererGPUProfiler.log(double(frameTimestamps[1] - frameTimestamps[0]) / 1000000.0);', - ''' const auto gpuMilliseconds=double(frameTimestamps[1] - frameTimestamps[0]) / 1000000.0; + return replace( + t, + " rendererGPUProfiler.log(double(frameTimestamps[1] - frameTimestamps[0]) / 1000000.0);", + """ const auto gpuMilliseconds=double(frameTimestamps[1] - frameTimestamps[0]) / 1000000.0; rendererGPUProfiler.log(gpuMilliseconds); // Read only after the existing completed-workload fence. This // adds no wait; opt-in instrumentation is disabled by default. if(framebufferRenderer->waterRenderer) - framebufferRenderer->waterRenderer->finishProfile(gpuMilliseconds);''') -change('src/hle/rt64_workload_queue.cpp',workload) + framebufferRenderer->waterRenderer->finishProfile(gpuMilliseconds);""", + ) + + +change("src/hle/rt64_workload_queue.cpp", workload) -output.write_text(''.join(changes),newline='\n') -print('Generated quality-preserving water workload/performance patch.') +output.write_text("".join(changes), newline="\n") +print("Generated quality-preserving water workload/performance patch.") diff --git a/scripts/compose-water-projection.py b/scripts/compose-water-projection.py index d4db62e0..58e6b09a 100644 --- a/scripts/compose-water-projection.py +++ b/scripts/compose-water-projection.py @@ -3,100 +3,159 @@ Read the fully composed stage; emit one final pipeline patch. Re-runs reverse only this layer in memory. Shared native shader functions are authored inputs. """ -from pathlib import Path + import difflib +from pathlib import Path import sys + from water_patch_baseline import read_before -root=Path(__file__).resolve().parents[1] -stage=Path(sys.argv[1]).resolve() -output=root/'patches/enhanced-water/dkr-projection.patch' -layers=[output] if 'inverseWorldViewProjection' in (stage/'src/shared/rt64_water_params.h').read_text() else [] -changes=[] - -def replace(text,old,new): - if text.count(old)!=1: - raise SystemExit(f'Non-unique projection context: {old[:120]}') - return text.replace(old,new,1) - -def change(path,fn): - before=read_before(stage,path,layers) - after=fn(before) - if before==after:raise SystemExit(f'No projection changes: {path}') - changes.append(f'diff --git a/{path} b/{path}\n') - changes.extend(difflib.unified_diff(before.splitlines(True),after.splitlines(True), - fromfile=f'a/{path}',tofile=f'b/{path}',n=3)) +root = Path(__file__).resolve().parents[1] +stage = Path(sys.argv[1]).resolve() +output = root / "patches/enhanced-water/dkr-projection.patch" +layers = [output] if "inverseWorldViewProjection" in (stage / "src/shared/rt64_water_params.h").read_text() else [] +changes = [] + + +def replace(text, old, new): + if text.count(old) != 1: + raise SystemExit(f"Non-unique projection context: {old[:120]}") + return text.replace(old, new, 1) + + +def change(path, fn): + before = read_before(stage, path, layers) + after = fn(before) + if before == after: + raise SystemExit(f"No projection changes: {path}") + changes.append(f"diff --git a/{path} b/{path}\n") + changes.extend( + difflib.unified_diff( + before.splitlines(True), after.splitlines(True), fromfile=f"a/{path}", tofile=f"b/{path}", n=3 + ) + ) + def params(t): - t=replace(t,' float4x4 inverseViewProjection;', - ' float4x4 inverseViewProjection; // inverse of final raster camera\n' - ' float4x4 inverseWorldViewProjection; // decode DKR model MVP output only') - return replace(t,'sizeof(WaterDrawParams) == 1936','sizeof(WaterDrawParams) == 2000') -change('src/shared/rt64_water_params.h',params) + t = replace( + t, + " float4x4 inverseViewProjection;", + " float4x4 inverseViewProjection; // inverse of final raster camera\n" + " float4x4 inverseWorldViewProjection; // decode DKR model MVP output only", + ) + return replace(t, "sizeof(WaterDrawParams) == 1936", "sizeof(WaterDrawParams) == 2000") + + +change("src/shared/rt64_water_params.h", params) + def framebuffer(t): - t=replace(t,'#include "rt64_framebuffer_renderer.h"','#include "rt64_framebuffer_renderer.h"\n#include "WaterProjection.h"') - t=replace(t,' wp.viewProjection = wp.material.cameraViewProjection;', - ' auto cameraViewProjection = hlslpp::float4x4(wp.material.cameraViewProjection);') - t=replace(t,'if (i0.000001f ? world.w : 1.0f; } - worldPosition = world.xyz;''',' worldPosition = waterDecodeWorld(world,p);') -change('src/shaders/WaterVS.hlsl',vertices) + worldPosition = world.xyz;""", + " worldPosition = waterDecodeWorld(world,p);", + ) + + +change("src/shaders/WaterVS.hlsl", vertices) + def normals(t): - t=replace(t,'#include "shared/rt64_water_params.h"','#include "WaterProjection.hlsli"') - return replace(t,''' float4 v = positions[index]; + t = replace(t, '#include "shared/rt64_water_params.h"', '#include "WaterProjection.hlsli"') + return replace( + t, + """ float4 v = positions[index]; if (p.material.cameraPosition.w == 1.0f) { v = mul(p.inverseViewProjection, v); v.xyz /= abs(v.w)>0.000001f ? v.w : 1.0f; } - return v.xyz;''',' return waterDecodeWorld(positions[index],p);') -change('src/shaders/WaterNormalsCS.hlsl',normals) + return v.xyz;""", + " return waterDecodeWorld(positions[index],p);", + ) + + +change("src/shaders/WaterNormalsCS.hlsl", normals) + def spray(t): - t=replace(t,'#include "shared/rt64_water_params.h"','#include "WaterProjection.hlsli"') - return replace(t,''' float2 ndc=position.xy/float2(p.viewportScale.w,p.viewportTranslate.w)*float2(2,-2)+float2(-1,1); + t = replace(t, '#include "shared/rt64_water_params.h"', '#include "WaterProjection.hlsli"') + return replace( + t, + """ float2 ndc=position.xy/float2(p.viewportScale.w,p.viewportTranslate.w)*float2(2,-2)+float2(-1,1); ndc=(ndc-p.screenTransform.zw)/p.screenTransform.xy; float2 screen=(ndc*float2(0.5,-0.5)+0.5)*p.screen.xy; float3 projected=float3((screen-p.viewportTranslate.xy)/p.viewportScale.xy*float2(1,-1),(depth-p.viewportTranslate.z)/p.viewportScale.z); float4 background=mul(p.inverseViewProjection,float4(projected,1)); - alpha*=saturate((length(background.xyz/background.w-p.camera.xyz)-fade.y)/12);''', - ''' float3 background=waterWorldFromPixel(position.xy,depth,p); - alpha*=saturate((length(background-p.camera.xyz)-fade.y)/12);''') -change('src/shaders/WaterSprayPS.hlsl',spray) + alpha*=saturate((length(background.xyz/background.w-p.camera.xyz)-fade.y)/12);""", + """ float3 background=waterWorldFromPixel(position.xy,depth,p); + alpha*=saturate((length(background-p.camera.xyz)-fade.y)/12);""", + ) + + +change("src/shaders/WaterSprayPS.hlsl", spray) -output.write_text(''.join(changes),newline='\n') -print('Generated the final camera/viewport projection correction patch.') +output.write_text("".join(changes), newline="\n") +print("Generated the final camera/viewport projection correction patch.") diff --git a/scripts/compose-water-rays.py b/scripts/compose-water-rays.py index df37f5a3..5edb84f9 100644 --- a/scripts/compose-water-rays.py +++ b/scripts/compose-water-rays.py @@ -1,76 +1,172 @@ """Generate authored water-ray patches; dependency/staged inputs stay read-only.""" -from pathlib import Path + import difflib +from pathlib import Path import sys -root=Path(__file__).resolve().parents[1] -stage=Path(sys.argv[1]).resolve() -plume=root/'extern/rt64/src/contrib/plume' -def replace(text,old,new): - if text.count(old)!=1:raise SystemExit(f'Non-unique ray context: {old[:100]}') - return text.replace(old,new,1) -def patch(name,changes): - out=[] - for path,before,after in changes: - if before==after:raise SystemExit(f'No changes in {path}') - out.append(f'diff --git a/{path} b/{path}\n') - out.extend(difflib.unified_diff(before.splitlines(True),after.splitlines(True),fromfile=f'a/{path}',tofile=f'b/{path}',n=3)) - (root/'patches/enhanced-water'/name).write_text(''.join(out),newline='\n') - -changes=[] +root = Path(__file__).resolve().parents[1] +stage = Path(sys.argv[1]).resolve() +plume = root / "extern/rt64/src/contrib/plume" + + +def replace(text, old, new): + if text.count(old) != 1: + raise SystemExit(f"Non-unique ray context: {old[:100]}") + return text.replace(old, new, 1) + + +def patch(name, changes): + out = [] + for path, before, after in changes: + if before == after: + raise SystemExit(f"No changes in {path}") + out.append(f"diff --git a/{path} b/{path}\n") + out.extend( + difflib.unified_diff( + before.splitlines(True), after.splitlines(True), fromfile=f"a/{path}", tofile=f"b/{path}", n=3 + ) + ) + (root / "patches/enhanced-water" / name).write_text("".join(out), newline="\n") + + +changes = [] + + def baseline(path): - before=(stage/path).read_text() - if 'waterRayGeometry' not in (stage/'src/hle/rt64_state.h').read_text():return before + before = (stage / path).read_text() + if "waterRayGeometry" not in (stage / "src/hle/rt64_state.h").read_text(): + return before # The stage can predate the last generator invocation. Normalize the one # reviewed transactional-layout correction before removing that layer. - if path=='src/render/rt64_water_renderer.cpp': - before=before.replace(' if(!rayShader) {',' if(!rayShader || !rayLayout) {\n // Do not keep a shader if its companion layout creation failed.\n rayShader.reset();rayLayout.reset();') - patch_path=root/'patches/enhanced-water/dkr-rays.patch' - hunks=[];active=False;old=[];new=[] + if path == "src/render/rt64_water_renderer.cpp": + before = before.replace( + " if(!rayShader) {", + " if(!rayShader || !rayLayout) {\n // Do not keep a shader if its companion layout creation failed.\n rayShader.reset();rayLayout.reset();", + ) + patch_path = root / "patches/enhanced-water/dkr-rays.patch" + hunks = [] + active = False + old = [] + new = [] + def finish(): - if active and (old or new):hunks.append((''.join(new),''.join(old))) + if active and (old or new): + hunks.append(("".join(new), "".join(old))) + for line in patch_path.read_text().splitlines(True): - if line.startswith('diff --git '): - finish();old=[];new=[];active=line.split()[2][2:]==path - elif line.startswith('@@ '):finish();old=[];new=[] - elif line.startswith(('--- ','+++ ')):continue - elif line.startswith(' '):old.append(line[1:]);new.append(line[1:]) - elif line.startswith('-'):old.append(line[1:]) - elif line.startswith('+'):new.append(line[1:]) + if line.startswith("diff --git "): + finish() + old = [] + new = [] + active = line.split()[2][2:] == path + elif line.startswith("@@ "): + finish() + old = [] + new = [] + elif line.startswith(("--- ", "+++ ")): + continue + elif line.startswith(" "): + old.append(line[1:]) + new.append(line[1:]) + elif line.startswith("-"): + old.append(line[1:]) + elif line.startswith("+"): + new.append(line[1:]) finish() - for new,old in reversed(hunks): - if before.count(new)!=1:raise SystemExit(f'Cannot reverse previous ray patch in {path}') - before=before.replace(new,old,1) + for new, old in reversed(hunks): + if before.count(new) != 1: + raise SystemExit(f"Cannot reverse previous ray patch in {path}") + before = before.replace(new, old, 1) return before -def native(path,fn): - before=baseline(path);changes.append((path,before,fn(before))) -native('src/hle/rt64_state.h',lambda t:replace(replace(t,'#pragma once','#pragma once\n#include "enhanced_water_ray_geometry.hpp"'), - ' interop::WaterMaterial waterMaterial{};', ' interop::WaterMaterial waterMaterial{};\n std::shared_ptr waterRayGeometry;')) -native('src/hle/rt64_workload.h',lambda t:replace(replace(t,'#pragma once','#pragma once\n#include "enhanced_water_ray_geometry.hpp"'), - ' std::vector waterMaterials;', ' std::vector waterMaterials;\n std::shared_ptr waterRayGeometry;')) -native('src/hle/rt64_workload.cpp',lambda t:replace(t,' waterMaterials.clear();',' waterMaterials.clear();\n waterRayGeometry.reset();')) -native('src/hle/rt64_state.cpp',lambda t:replace(t,' drawCall.waterMaterialIndex = uint32_t(workload.waterMaterials.size() - 1);', - ' drawCall.waterMaterialIndex = uint32_t(workload.waterMaterials.size() - 1);\n if(waterMaterial.effects.w>0.5f)workload.waterRayGeometry=waterRayGeometry;')) + +def native(path, fn): + before = baseline(path) + changes.append((path, before, fn(before))) + + +native( + "src/hle/rt64_state.h", + lambda t: replace( + replace(t, "#pragma once", '#pragma once\n#include "enhanced_water_ray_geometry.hpp"'), + " interop::WaterMaterial waterMaterial{};", + " interop::WaterMaterial waterMaterial{};\n std::shared_ptr waterRayGeometry;", + ), +) +native( + "src/hle/rt64_workload.h", + lambda t: replace( + replace(t, "#pragma once", '#pragma once\n#include "enhanced_water_ray_geometry.hpp"'), + " std::vector waterMaterials;", + " std::vector waterMaterials;\n std::shared_ptr waterRayGeometry;", + ), +) +native( + "src/hle/rt64_workload.cpp", + lambda t: replace( + t, " waterMaterials.clear();", " waterMaterials.clear();\n waterRayGeometry.reset();" + ), +) +native( + "src/hle/rt64_state.cpp", + lambda t: replace( + t, + " drawCall.waterMaterialIndex = uint32_t(workload.waterMaterials.size() - 1);", + " drawCall.waterMaterialIndex = uint32_t(workload.waterMaterials.size() - 1);\n if(waterMaterial.effects.w>0.5f)workload.waterRayGeometry=waterRayGeometry;", + ), +) + + def water_header(t): - t=replace(t,'#pragma once','#pragma once\n#include "WaterRayScene.h"') - t=replace(t,' std::unique_ptr pixelShader, classicShader, classicMSShader;', - ' std::unique_ptr pixelShader, classicShader, classicMSShader;\n std::unique_ptr rayShader;\n std::unique_ptr rayLayout;\n WaterRayScene rays;\n bool rayUnavailable=false;\n bool prepareRays(RenderWorker *worker,RenderShaderFormat format,std::shared_ptr geometry);') - return replace(t,'bool selfDepth=false);','bool selfDepth=false, bool hardware=false);') -native('src/render/rt64_water_renderer.h',water_header) + t = replace(t, "#pragma once", '#pragma once\n#include "WaterRayScene.h"') + t = replace( + t, + " std::unique_ptr pixelShader, classicShader, classicMSShader;", + " std::unique_ptr pixelShader, classicShader, classicMSShader;\n std::unique_ptr rayShader;\n std::unique_ptr rayLayout;\n WaterRayScene rays;\n bool rayUnavailable=false;\n bool prepareRays(RenderWorker *worker,RenderShaderFormat format,std::shared_ptr geometry);", + ) + return replace(t, "bool selfDepth=false);", "bool selfDepth=false, bool hardware=false);") + + +native("src/render/rt64_water_renderer.h", water_header) + + def water_cpp(t): - t=replace(t,'#include "shaders/WaterPS.hlsl.spirv.h"','#include "shaders/WaterPS.hlsl.spirv.h"\n#include "shaders/WaterHardwarePS.hlsl.spirv.h"') - t=replace(t,'#include "shaders/WaterPS.hlsl.dxil.h"','#include "shaders/WaterPS.hlsl.dxil.h"\n#include "shaders/WaterHardwarePS.hlsl.dxil.h"') - t=replace(t,' draws.clear(); normalGroups.clear(); sceneKeys.clear(); drawScenes.clear();', - ' draws.clear(); normalGroups.clear(); sceneKeys.clear(); drawScenes.clear();\n rays.resetFrame();') - t=replace(t,'bool wireframe, bool surface, bool classic, bool selfDepth) {','bool wireframe, bool surface, bool classic, bool selfDepth, bool hardware) {') - t=replace(t,'(uint32_t(selfDepth) << 21);','(uint32_t(selfDepth) << 21) | (uint32_t(hardware) << 22);') - t=replace(t,' desc.pipelineLayout = layout.get();',' desc.pipelineLayout = hardware ? rayLayout.get() : layout.get();') - t=replace(t,'(samples.sampleCount>1 ? classicMSShader.get() : classicShader.get()) : pixelShader.get();', - '(samples.sampleCount>1 ? classicMSShader.get() : classicShader.get()) : hardware ? rayShader.get() : pixelShader.get();') - point='void WaterRenderer::collectNormals(' - at=t.index(point) - t=t[:at]+'''bool WaterRenderer::prepareRays(RenderWorker *worker,RenderShaderFormat format,std::shared_ptr geometry) { + t = replace( + t, + '#include "shaders/WaterPS.hlsl.spirv.h"', + '#include "shaders/WaterPS.hlsl.spirv.h"\n#include "shaders/WaterHardwarePS.hlsl.spirv.h"', + ) + t = replace( + t, + '#include "shaders/WaterPS.hlsl.dxil.h"', + '#include "shaders/WaterPS.hlsl.dxil.h"\n#include "shaders/WaterHardwarePS.hlsl.dxil.h"', + ) + t = replace( + t, + " draws.clear(); normalGroups.clear(); sceneKeys.clear(); drawScenes.clear();", + " draws.clear(); normalGroups.clear(); sceneKeys.clear(); drawScenes.clear();\n rays.resetFrame();", + ) + t = replace( + t, + "bool wireframe, bool surface, bool classic, bool selfDepth) {", + "bool wireframe, bool surface, bool classic, bool selfDepth, bool hardware) {", + ) + t = replace(t, "(uint32_t(selfDepth) << 21);", "(uint32_t(selfDepth) << 21) | (uint32_t(hardware) << 22);") + t = replace( + t, + " desc.pipelineLayout = layout.get();", + " desc.pipelineLayout = hardware ? rayLayout.get() : layout.get();", + ) + t = replace( + t, + "(samples.sampleCount>1 ? classicMSShader.get() : classicShader.get()) : pixelShader.get();", + "(samples.sampleCount>1 ? classicMSShader.get() : classicShader.get()) : hardware ? rayShader.get() : pixelShader.get();", + ) + point = "void WaterRenderer::collectNormals(" + at = t.index(point) + t = ( + t[:at] + + """bool WaterRenderer::prepareRays(RenderWorker *worker,RenderShaderFormat format,std::shared_ptr geometry) { if(rayUnavailable || !worker->device->getCapabilities().rayQuery)return false; try { if(!rayShader || !rayLayout) { @@ -91,44 +187,88 @@ def water_cpp(t): } } -'''+t[at:] +""" + + t[at:] + ) return t -native('src/render/rt64_water_renderer.cpp',water_cpp) + + +native("src/render/rt64_water_renderer.cpp", water_cpp) + + def framebuffer(t): - t=replace(t,' const bool classic=wp.material.optics.w>0.5f;', ''' const bool classic=wp.material.optics.w>0.5f; + t = replace( + t, + " const bool classic=wp.material.optics.w>0.5f;", + """ const bool classic=wp.material.optics.w>0.5f; bool hardware=!classic && wp.material.effects.w>0.5f && waterRenderer->prepareRays(p.worker,p.rasterShaderCache->shaderFormat,p.curWorkload->waterRayGeometry); - wp.material.effects.w=hardware ? 1.0f : 0.0f;''') - t=replace(t,'wp.material.identity.y==12,false,classic,selfDepth);','wp.material.identity.y==12,false,classic,selfDepth,hardware);') - t=replace(t,' waterMatricesValid = waterPipeline && waterRenderer->pipeline(',''' if(hardware && !waterPipeline) { + wp.material.effects.w=hardware ? 1.0f : 0.0f;""", + ) + t = replace( + t, + "wp.material.identity.y==12,false,classic,selfDepth);", + "wp.material.identity.y==12,false,classic,selfDepth,hardware);", + ) + t = replace( + t, + " waterMatricesValid = waterPipeline && waterRenderer->pipeline(", + """ if(hardware && !waterPipeline) { // Failure of the optional hardware pipeline must // retain the software water, not remove the effect. hardware=false;wp.material.effects.w=0;waterRenderer->rayUnavailable=true; waterPipeline=waterRenderer->pipeline(p.worker->device,p.fbStorage->colorTarget->multisampling, call.callDesc.otherMode.zCmp() || selfDepth,call.callDesc.otherMode.zUpd() && !selfDepth,wp.material.identity.y==12,false,classic,selfDepth,false); } - waterMatricesValid = waterPipeline && waterRenderer->pipeline(''') - t=replace(t,' worker->commandList->setGraphicsPipelineLayout(waterRenderer->layout.get());', ''' const bool hardware=waterRenderer->draws[drawCall.waterIndex].material.effects.w>0.5f; + waterMatricesValid = waterPipeline && waterRenderer->pipeline(""", + ) + t = replace( + t, + " worker->commandList->setGraphicsPipelineLayout(waterRenderer->layout.get());", + """ const bool hardware=waterRenderer->draws[drawCall.waterIndex].material.effects.w>0.5f; worker->commandList->setGraphicsPipelineLayout(hardware ? waterRenderer->rayLayout.get() : waterRenderer->layout.get()); - if(hardware)worker->commandList->setGraphicsDescriptorSet(waterRenderer->rays.set->get(),1);''') + if(hardware)worker->commandList->setGraphicsDescriptorSet(waterRenderer->rays.set->get(),1);""", + ) # The surface mask uses the same vertex inputs, but its layout is always # non-ray. Rebind the colour layout after that auxiliary pass. - t=replace(t,' worker->commandList->setPipeline(triangles.pipeline);\n activeWaterDraw=drawCall.waterIndex;', ''' worker->commandList->setGraphicsPipelineLayout(hardware ? waterRenderer->rayLayout.get() : waterRenderer->layout.get()); + t = replace( + t, + " worker->commandList->setPipeline(triangles.pipeline);\n activeWaterDraw=drawCall.waterIndex;", + """ worker->commandList->setGraphicsPipelineLayout(hardware ? waterRenderer->rayLayout.get() : waterRenderer->layout.get()); worker->commandList->setGraphicsDescriptorSet(waterRenderer->materialSet(drawCall.waterIndex),0); if(hardware)worker->commandList->setGraphicsDescriptorSet(waterRenderer->rays.set->get(),1); worker->commandList->setGraphicsPushConstants(0,&pc); worker->commandList->setPipeline(triangles.pipeline); - activeWaterDraw=drawCall.waterIndex;''') - t=replace(t,' waterRenderer->beginSurface(worker,drawCall.waterIndex);', ''' worker->commandList->setGraphicsPipelineLayout(waterRenderer->layout.get()); + activeWaterDraw=drawCall.waterIndex;""", + ) + t = replace( + t, + " waterRenderer->beginSurface(worker,drawCall.waterIndex);", + """ worker->commandList->setGraphicsPipelineLayout(waterRenderer->layout.get()); worker->commandList->setGraphicsDescriptorSet(waterRenderer->materialSet(drawCall.waterIndex),0); worker->commandList->setGraphicsPushConstants(0,&pc); - waterRenderer->beginSurface(worker,drawCall.waterIndex);''') - return replace(t,' if (waterRenderer && !waterRenderer->draws.empty()) waterRenderer->recordNormals(worker, *outputBuffers);', - ' if (waterRenderer && !waterRenderer->draws.empty()) waterRenderer->rays.record(worker);\n if (waterRenderer && !waterRenderer->draws.empty()) waterRenderer->recordNormals(worker, *outputBuffers);') -native('src/render/rt64_framebuffer_renderer.cpp',framebuffer) + waterRenderer->beginSurface(worker,drawCall.waterIndex);""", + ) + return replace( + t, + " if (waterRenderer && !waterRenderer->draws.empty()) waterRenderer->recordNormals(worker, *outputBuffers);", + " if (waterRenderer && !waterRenderer->draws.empty()) waterRenderer->rays.record(worker);\n if (waterRenderer && !waterRenderer->draws.empty()) waterRenderer->recordNormals(worker, *outputBuffers);", + ) + + +native("src/render/rt64_framebuffer_renderer.cpp", framebuffer) + + def shader(t): - t=replace(t,'float2 rippleCornerGradient(', '#ifdef HARDWARE_WATER\n#include "WaterRays.hlsli"\n#endif\n\nfloat2 rippleCornerGradient(') - return replace(t,' float4 reflection=screenReflection(worldPosition,N,V,roughness,p);', ''' float4 reflection=screenReflection(worldPosition,N,V,roughness,p); + t = replace( + t, + "float2 rippleCornerGradient(", + '#ifdef HARDWARE_WATER\n#include "WaterRays.hlsli"\n#endif\n\nfloat2 rippleCornerGradient(', + ) + return replace( + t, + " float4 reflection=screenReflection(worldPosition,N,V,roughness,p);", + """ float4 reflection=screenReflection(worldPosition,N,V,roughness,p); #ifdef HARDWARE_WATER // SSR covers moving actors using their exact rendered colours; hardware // rays add static off-screen world intersections, not a second game pass. @@ -136,25 +276,58 @@ def shader(t): float4 hardwareReflection=hardwareWaterReflection(worldPosition,N,V,p); if(hardwareReflection.a>reflection.a)reflection=hardwareReflection; } -#endif''') -native('src/shaders/WaterPS.hlsl',shader) -patch('dkr-rays.patch',changes) - -changes=[] -def backend(name,fn): - before=(plume/name).read_text();changes.append(('src/contrib/plume/'+name,before,fn(before))) -backend('plume_render_interface_types.h',lambda t:replace(t,' bool raytracingStateUpdate = false;', - ' bool raytracingStateUpdate = false;\n bool rayQuery = false; // Feature actually enabled, not inferred from GPU model.')) -backend('plume_vulkan.h',lambda t:replace(t,' VkPhysicalDeviceRayTracingPipelinePropertiesKHR rtPipelineProperties = {};', - ' VkPhysicalDeviceRayTracingPipelinePropertiesKHR rtPipelineProperties = {};\n VkPhysicalDeviceAccelerationStructurePropertiesKHR waterASProperties = {};')) +#endif""", + ) + + +native("src/shaders/WaterPS.hlsl", shader) +patch("dkr-rays.patch", changes) + +changes = [] + + +def backend(name, fn): + before = (plume / name).read_text() + changes.append(("src/contrib/plume/" + name, before, fn(before))) + + +backend( + "plume_render_interface_types.h", + lambda t: replace( + t, + " bool raytracingStateUpdate = false;", + " bool raytracingStateUpdate = false;\n bool rayQuery = false; // Feature actually enabled, not inferred from GPU model.", + ), +) +backend( + "plume_vulkan.h", + lambda t: replace( + t, + " VkPhysicalDeviceRayTracingPipelinePropertiesKHR rtPipelineProperties = {};", + " VkPhysicalDeviceRayTracingPipelinePropertiesKHR rtPipelineProperties = {};\n VkPhysicalDeviceAccelerationStructurePropertiesKHR waterASProperties = {};", + ), +) + + def vulkan(t): - t=replace(t,' VK_KHR_RAY_TRACING_PIPELINE_EXTENSION_NAME,',' VK_KHR_RAY_TRACING_PIPELINE_EXTENSION_NAME,\n VK_KHR_RAY_QUERY_EXTENSION_NAME,') - t=replace(t,' VkPhysicalDeviceRayTracingPipelineFeaturesKHR rayTracingPipelineFeatures = {};', ''' const bool rayQueryFound=bufferDeviceAddressFound && + t = replace( + t, + " VK_KHR_RAY_TRACING_PIPELINE_EXTENSION_NAME,", + " VK_KHR_RAY_TRACING_PIPELINE_EXTENSION_NAME,\n VK_KHR_RAY_QUERY_EXTENSION_NAME,", + ) + t = replace( + t, + " VkPhysicalDeviceRayTracingPipelineFeaturesKHR rayTracingPipelineFeatures = {};", + """ const bool rayQueryFound=bufferDeviceAddressFound && supportedOptionalExtensions.count(VK_KHR_ACCELERATION_STRUCTURE_EXTENSION_NAME) && supportedOptionalExtensions.count(VK_KHR_DEFERRED_HOST_OPERATIONS_EXTENSION_NAME) && supportedOptionalExtensions.count(VK_KHR_RAY_QUERY_EXTENSION_NAME); - VkPhysicalDeviceRayTracingPipelineFeaturesKHR rayTracingPipelineFeatures = {};''') - t=replace(t,' VkPhysicalDeviceFeatures2 deviceFeatures = {};', ''' VkPhysicalDeviceRayQueryFeaturesKHR rayQueryFeatures = {}; + VkPhysicalDeviceRayTracingPipelineFeaturesKHR rayTracingPipelineFeatures = {};""", + ) + t = replace( + t, + " VkPhysicalDeviceFeatures2 deviceFeatures = {};", + """ VkPhysicalDeviceRayQueryFeaturesKHR rayQueryFeatures = {}; // Query-only GPUs still need acceleration structures, not a ray // tracing pipeline. Do not query unsupported pipeline features. if(rayQueryFound && !rayTracingFound) { @@ -165,9 +338,14 @@ def vulkan(t): rayQueryFeatures.sType=VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_RAY_QUERY_FEATURES_KHR; rayQueryFeatures.pNext=featuresChain;featuresChain=&rayQueryFeatures; } - VkPhysicalDeviceFeatures2 deviceFeatures = {};''') - point=' const bool rayTracingSupported = rayTracingPipelineFeatures.rayTracingPipeline && accelerationStructureFeatures.accelerationStructure;' - t=replace(t,point,point+''' + VkPhysicalDeviceFeatures2 deviceFeatures = {};""", + ) + point = " const bool rayTracingSupported = rayTracingPipelineFeatures.rayTracingPipeline && accelerationStructureFeatures.accelerationStructure;" + t = replace( + t, + point, + point + + """ const bool rayQuerySupported=rayQueryFound && accelerationStructureFeatures.accelerationStructure && bufferDeviceAddressFeatures.bufferDeviceAddress && rayQueryFeatures.rayQuery; if(rayQuerySupported) { @@ -175,42 +353,86 @@ def vulkan(t): if(!rayTracingSupported) { accelerationStructureFeatures.pNext=createDeviceChain;createDeviceChain=&accelerationStructureFeatures; } - }''') - t=replace(t,' capabilities.raytracing = rayTracingSupported;', ' capabilities.raytracing = rayTracingSupported;\n capabilities.rayQuery = rayQuerySupported;') - t=replace(t,'static VkPipelineStageFlags toStageFlags(RenderBarrierStages stages, bool geometrySupported, bool rtSupported)', - 'static VkPipelineStageFlags toStageFlags(RenderBarrierStages stages, bool geometrySupported, bool rtSupported, bool querySupported=false)') - t=replace(t,''' if (rtSupported) { + }""", + ) + t = replace( + t, + " capabilities.raytracing = rayTracingSupported;", + " capabilities.raytracing = rayTracingSupported;\n capabilities.rayQuery = rayQuerySupported;", + ) + t = replace( + t, + "static VkPipelineStageFlags toStageFlags(RenderBarrierStages stages, bool geometrySupported, bool rtSupported)", + "static VkPipelineStageFlags toStageFlags(RenderBarrierStages stages, bool geometrySupported, bool rtSupported, bool querySupported=false)", + ) + t = replace( + t, + """ if (rtSupported) { flags |= VK_PIPELINE_STAGE_ACCELERATION_STRUCTURE_BUILD_BIT_KHR; flags |= VK_PIPELINE_STAGE_RAY_TRACING_SHADER_BIT_KHR; - }''', ''' if (rtSupported || querySupported) flags |= VK_PIPELINE_STAGE_ACCELERATION_STRUCTURE_BUILD_BIT_KHR; - if (rtSupported) flags |= VK_PIPELINE_STAGE_RAY_TRACING_SHADER_BIT_KHR;''') - t=replace(t,' const bool rtEnabled = queue->device->capabilities.raytracing;', - ' const bool rtEnabled = queue->device->capabilities.raytracing;\n const bool queryEnabled = queue->device->capabilities.rayQuery;') - for stages in ['stages','interfaceBuffer->barrierStages','interfaceTexture->barrierStages']: - t=replace(t,f'toStageFlags({stages}, geometryEnabled, rtEnabled)',f'toStageFlags({stages}, geometryEnabled, rtEnabled, queryEnabled)') - t=replace(t,' // Check for properties.', ''' if(rayQueryFound || rayTracingFound) { + }""", + """ if (rtSupported || querySupported) flags |= VK_PIPELINE_STAGE_ACCELERATION_STRUCTURE_BUILD_BIT_KHR; + if (rtSupported) flags |= VK_PIPELINE_STAGE_RAY_TRACING_SHADER_BIT_KHR;""", + ) + t = replace( + t, + " const bool rtEnabled = queue->device->capabilities.raytracing;", + " const bool rtEnabled = queue->device->capabilities.raytracing;\n const bool queryEnabled = queue->device->capabilities.rayQuery;", + ) + for stages in ["stages", "interfaceBuffer->barrierStages", "interfaceTexture->barrierStages"]: + t = replace( + t, + f"toStageFlags({stages}, geometryEnabled, rtEnabled)", + f"toStageFlags({stages}, geometryEnabled, rtEnabled, queryEnabled)", + ) + t = replace( + t, + " // Check for properties.", + """ if(rayQueryFound || rayTracingFound) { waterASProperties.sType=VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ACCELERATION_STRUCTURE_PROPERTIES_KHR; VkPhysicalDeviceProperties2 properties={};properties.sType=VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; properties.pNext=&waterASProperties;vkGetPhysicalDeviceProperties2(physicalDevice,&properties); } - // Check for properties.''') - t=replace(t,' VkResult res;\n if (minAlignment > 0)', ''' if(desc.flags & RenderBufferFlag::ACCELERATION_STRUCTURE_SCRATCH) + // Check for properties.""", + ) + t = replace( + t, + " VkResult res;\n if (minAlignment > 0)", + """ if(desc.flags & RenderBufferFlag::ACCELERATION_STRUCTURE_SCRATCH) minAlignment=std::max(minAlignment,device->waterASProperties.minAccelerationStructureScratchOffsetAlignment); VkResult res; - if (minAlignment > 0)''') + if (minAlignment > 0)""", + ) return t -backend('plume_vulkan.cpp',vulkan) + + +backend("plume_vulkan.cpp", vulkan) + + def d3d(t): - t=replace(t,' // Use copy-optimized states.', ''' if(stages==RenderBarrierStage::COMPUTE && accessBits==RenderBufferAccess::READ && + t = replace( + t, + " // Use copy-optimized states.", + """ if(stages==RenderBarrierStage::COMPUTE && accessBits==RenderBufferAccess::READ && (bufferFlags & RenderBufferFlag::ACCELERATION_STRUCTURE_INPUT))return D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE; - // Use copy-optimized states.''') - t=replace(t,' bool triangleFanSupportOption = false;', ''' D3D12_FEATURE_DATA_SHADER_MODEL rayShaderModel = {D3D_SHADER_MODEL_6_5}; + // Use copy-optimized states.""", + ) + t = replace( + t, + " bool triangleFanSupportOption = false;", + """ D3D12_FEATURE_DATA_SHADER_MODEL rayShaderModel = {D3D_SHADER_MODEL_6_5}; const bool rayQueryOption=rtStateUpdateSupportOption && SUCCEEDED(deviceOption->CheckFeatureSupport(D3D12_FEATURE_SHADER_MODEL,&rayShaderModel,sizeof(rayShaderModel))) && rayShaderModel.HighestShaderModel>=D3D_SHADER_MODEL_6_5; - bool triangleFanSupportOption = false;''') - return replace(t,' capabilities.raytracingStateUpdate = rtStateUpdateSupportOption;', - ' capabilities.raytracingStateUpdate = rtStateUpdateSupportOption;\n capabilities.rayQuery = rayQueryOption;') -backend('plume_d3d12.cpp',d3d) -patch('dkr-ray-capabilities.patch',changes) -print('Generated renderer and capability patches without changing dependency inputs.') + bool triangleFanSupportOption = false;""", + ) + return replace( + t, + " capabilities.raytracingStateUpdate = rtStateUpdateSupportOption;", + " capabilities.raytracingStateUpdate = rtStateUpdateSupportOption;\n capabilities.rayQuery = rayQueryOption;", + ) + + +backend("plume_d3d12.cpp", d3d) +patch("dkr-ray-capabilities.patch", changes) +print("Generated renderer and capability patches without changing dependency inputs.") diff --git a/scripts/compose-water-recovery.py b/scripts/compose-water-recovery.py index ac0743d3..4847ab99 100644 --- a/scripts/compose-water-recovery.py +++ b/scripts/compose-water-recovery.py @@ -1,39 +1,67 @@ """Author bounded water scratch pooling/recovery; dependencies stay read-only.""" -from pathlib import Path + import difflib +from pathlib import Path import sys + from water_patch_baseline import read_before -root=Path(__file__).resolve().parents[1] -stage=Path(sys.argv[1]).resolve() -output=root/'patches/enhanced-water/dkr-recovery.patch' -layers=[output] if 'waterRecovery' in (stage/'src/render/rt64_framebuffer_renderer.h').read_text() else [] -changes=[] -def replace(t,a,b): - if t.count(a)!=1:raise SystemExit(f'Non-unique recovery context: {a[:120]}') - return t.replace(a,b,1) -def change(path,fn): - before=read_before(stage,path,layers);after=fn(before) - if before==after:raise SystemExit(f'No recovery change: {path}') - changes.append(f'diff --git a/{path} b/{path}\n') - changes.extend(difflib.unified_diff(before.splitlines(True),after.splitlines(True),fromfile=f'a/{path}',tofile=f'b/{path}',n=3)) +root = Path(__file__).resolve().parents[1] +stage = Path(sys.argv[1]).resolve() +output = root / "patches/enhanced-water/dkr-recovery.patch" +layers = [output] if "waterRecovery" in (stage / "src/render/rt64_framebuffer_renderer.h").read_text() else [] +changes = [] + + +def replace(t, a, b): + if t.count(a) != 1: + raise SystemExit(f"Non-unique recovery context: {a[:120]}") + return t.replace(a, b, 1) + + +def change(path, fn): + before = read_before(stage, path, layers) + after = fn(before) + if before == after: + raise SystemExit(f"No recovery change: {path}") + changes.append(f"diff --git a/{path} b/{path}\n") + changes.extend( + difflib.unified_diff( + before.splitlines(True), after.splitlines(True), fromfile=f"a/{path}", tofile=f"b/{path}", n=3 + ) + ) + + def header(t): - t=replace(t,'#include "WaterRayScene.h"','#include "WaterRayScene.h"\n#include "WaterResourcePolicy.h"') - t=replace(t,' std::vector drawScenes;', - ' std::set> rejectedScenes;\n' - ' uint32_t budgetLogCooldown=0;\n' - ' std::vector drawScenes;') + t = replace(t, '#include "WaterRayScene.h"', '#include "WaterRayScene.h"\n#include "WaterResourcePolicy.h"') + t = replace( + t, + " std::vector drawScenes;", + " std::set> rejectedScenes;\n" + " uint32_t budgetLogCooldown=0;\n" + " std::vector drawScenes;", + ) return t + + def resources(t): - t=replace(t,' draws.clear(); normalGroups.clear(); sceneKeys.clear(); drawScenes.clear();', - ''' // The preceding workload fence has completed before this reset. + t = replace( + t, + " draws.clear(); normalGroups.clear(); sceneKeys.clear(); drawScenes.clear();", + """ // The preceding workload fence has completed before this reset. // Keep only scratch targets actually used, not historical menu targets. scenes.resize(water_policy::retainedScenes(uint32_t(scenes.size()),uint32_t(sceneKeys.size()))); draws.clear(); normalGroups.clear(); sceneKeys.clear(); drawScenes.clear(); rejectedScenes.clear(); - if(budgetLogCooldown) --budgetLogCooldown;''') - a=t.index(' const uint64_t pixels=uint64_t(color->width)*color->height;',t.index('uint32_t WaterRenderer::prepareScene')) - b=t.index(' auto &scene = scenes[index];',a) - t=t[:a]+''' if(rejectedScenes.count(sceneKey))return water_policy::invalidScene; + if(budgetLogCooldown) --budgetLogCooldown;""", + ) + a = t.index( + " const uint64_t pixels=uint64_t(color->width)*color->height;", + t.index("uint32_t WaterRenderer::prepareScene"), + ) + b = t.index(" auto &scene = scenes[index];", a) + t = ( + t[:a] + + """ if(rejectedScenes.count(sceneKey))return water_policy::invalidScene; const uint32_t index=uint32_t(sceneKeys.size()); const auto bytes=[](const SceneInputs &s) { return s.color ? water_policy::sceneBytes(s.width,s.height,s.surfaceSamples, @@ -72,9 +100,13 @@ def resources(t): // Publish the key only after every texture, framebuffer and binding exists. // A failed candidate must never be looked up as a prepared scene. try { -'''+t[b:] - t=replace(t,' return index;\n}\n\nbool WaterRenderer::captureScene', - ''' sceneKeys.emplace(sceneKey,index); +""" + + t[b:] + ) + t = replace( + t, + " return index;\n}\n\nbool WaterRenderer::captureScene", + """ sceneKeys.emplace(sceneKey,index); return index; } catch(...) { {auto retired=std::move(scenes[index]);scenes[index]=SceneInputs{};} @@ -83,47 +115,102 @@ def resources(t): } } -bool WaterRenderer::captureScene''') - t=replace(t,' scene.surfaceFramebuffer=checkedWaterResource(worker->device->createFramebuffer(RenderFramebufferDesc(&surfaceAttachment,1,', - ''' const bool attachmentsChanged=scene.sourceColor!=color || scene.sourceDepth!=depth || +bool WaterRenderer::captureScene""", + ) + t = replace( + t, + " scene.surfaceFramebuffer=checkedWaterResource(worker->device->createFramebuffer(RenderFramebufferDesc(&surfaceAttachment,1,", + """ const bool attachmentsChanged=scene.sourceColor!=color || scene.sourceDepth!=depth || scene.sourceColorTexture!=color->texture.get() || scene.sourceDepthTexture!=depth->texture.get() || scene.sourceColorRevision!=color->textureRevision || scene.sourceDepthRevision!=depth->textureRevision; if(!scene.surfaceFramebuffer || attachmentsChanged) - scene.surfaceFramebuffer=checkedWaterResource(worker->device->createFramebuffer(RenderFramebufferDesc(&surfaceAttachment,1,''') - t=replace(t,' if (scene.selfDepth) {\n const RenderTexture *colorAttachment=color->texture.get();', - ' if (scene.selfDepth && (!scene.waterColorFramebuffer || attachmentsChanged)) {\n const RenderTexture *colorAttachment=color->texture.get();') - t=replace(t,' else scene.waterColorFramebuffer.reset();',' else if(!scene.selfDepth) scene.waterColorFramebuffer.reset();') - t=replace(t,' scene.sourceColor = color; scene.sourceDepth = depth;', - ' scene.sourceColor = color; scene.sourceDepth = depth;\n' - ' scene.sourceColorTexture=color->texture.get();scene.sourceDepthTexture=depth->texture.get();\n' - ' scene.sourceColorRevision=color->textureRevision;scene.sourceDepthRevision=depth->textureRevision;') + scene.surfaceFramebuffer=checkedWaterResource(worker->device->createFramebuffer(RenderFramebufferDesc(&surfaceAttachment,1,""", + ) + t = replace( + t, + " if (scene.selfDepth) {\n const RenderTexture *colorAttachment=color->texture.get();", + " if (scene.selfDepth && (!scene.waterColorFramebuffer || attachmentsChanged)) {\n const RenderTexture *colorAttachment=color->texture.get();", + ) + t = replace( + t, + " else scene.waterColorFramebuffer.reset();", + " else if(!scene.selfDepth) scene.waterColorFramebuffer.reset();", + ) + t = replace( + t, + " scene.sourceColor = color; scene.sourceDepth = depth;", + " scene.sourceColor = color; scene.sourceDepth = depth;\n" + " scene.sourceColorTexture=color->texture.get();scene.sourceDepthTexture=depth->texture.get();\n" + " scene.sourceColorRevision=color->textureRevision;scene.sourceDepthRevision=depth->textureRevision;", + ) return t + + # Track the attachment, not just the wrapper which may be reused after resize. def attachment(t): - return replace(t,' RenderTarget *sourceColor = nullptr, *sourceDepth = nullptr;', - ' RenderTarget *sourceColor = nullptr, *sourceDepth = nullptr;\n' - ' RenderTexture *sourceColorTexture=nullptr,*sourceDepthTexture=nullptr;\n' - ' uint64_t sourceColorRevision=0,sourceDepthRevision=0;') -change('src/render/rt64_water_renderer.h',lambda t:attachment(header(t))) -change('src/render/rt64_water_renderer.cpp',resources) + return replace( + t, + " RenderTarget *sourceColor = nullptr, *sourceDepth = nullptr;", + " RenderTarget *sourceColor = nullptr, *sourceDepth = nullptr;\n" + " RenderTexture *sourceColorTexture=nullptr,*sourceDepthTexture=nullptr;\n" + " uint64_t sourceColorRevision=0,sourceDepthRevision=0;", + ) + + +change("src/render/rt64_water_renderer.h", lambda t: attachment(header(t))) +change("src/render/rt64_water_renderer.cpp", resources) + + def framebuffer_header(t): - t=replace(t,'#pragma once','#pragma once\n#include "WaterResourcePolicy.h"') - return replace(t,' bool waterUnavailable = false;',' bool waterUnavailable = false; // shader/device support only\n water_policy::Recovery waterRecovery;') -change('src/render/rt64_framebuffer_renderer.h',framebuffer_header) + t = replace(t, "#pragma once", '#pragma once\n#include "WaterResourcePolicy.h"') + return replace( + t, + " bool waterUnavailable = false;", + " bool waterUnavailable = false; // shader/device support only\n water_policy::Recovery waterRecovery;", + ) + + +change("src/render/rt64_framebuffer_renderer.h", framebuffer_header) + + def framebuffer(t): - t=replace(t,' if (waterRenderer) waterRenderer->reset();',' waterRecovery.beginFrame();\n if (waterRenderer) waterRenderer->reset();') - t=replace(t,'if (wantsWater && waterRenderer && !waterUnavailable && waterSamplesSupported)', - 'if (wantsWater && waterRenderer && !waterUnavailable && waterRecovery.ready() && waterSamplesSupported)') - t=replace(t,' instanceDrawCall.originalWaterPipeline=triangles.pipeline;', - ' if(waterScene!=water_policy::invalidScene) {\n instanceDrawCall.originalWaterPipeline=triangles.pipeline;') - t=replace(t,' waterRenderer->worldView = RenderVertexBufferView(RenderBufferReference(outputBuffers.rasterWorldPosBuffer.buffer.get()), PosStride * vertexCount);', - ' waterRenderer->worldView = RenderVertexBufferView(RenderBufferReference(outputBuffers.rasterWorldPosBuffer.buffer.get()), PosStride * vertexCount);\n }') - t=replace(t,' waterUnavailable=true;\n std::fprintf(stderr,"[water] resource fallback', - ' waterRecovery.fail();\n std::fprintf(stderr,"[water] resource fallback') - t=replace(t,' if (waterUnavailable) throw std::runtime_error("water preparation was incomplete");', - ' if (waterRecovery.failed) throw std::runtime_error("water preparation was incomplete; bounded retry scheduled");') - return replace(t,' waterUnavailable=true;\n std::fprintf(stderr,"[water] frame fallback', - ' waterRecovery.fail();\n std::fprintf(stderr,"[water] frame fallback') -change('src/render/rt64_framebuffer_renderer.cpp',framebuffer) -output.write_text(''.join(changes)) -print(f'Authored {output}; no dependency/stage files changed.') + t = replace( + t, + " if (waterRenderer) waterRenderer->reset();", + " waterRecovery.beginFrame();\n if (waterRenderer) waterRenderer->reset();", + ) + t = replace( + t, + "if (wantsWater && waterRenderer && !waterUnavailable && waterSamplesSupported)", + "if (wantsWater && waterRenderer && !waterUnavailable && waterRecovery.ready() && waterSamplesSupported)", + ) + t = replace( + t, + " instanceDrawCall.originalWaterPipeline=triangles.pipeline;", + " if(waterScene!=water_policy::invalidScene) {\n instanceDrawCall.originalWaterPipeline=triangles.pipeline;", + ) + t = replace( + t, + " waterRenderer->worldView = RenderVertexBufferView(RenderBufferReference(outputBuffers.rasterWorldPosBuffer.buffer.get()), PosStride * vertexCount);", + " waterRenderer->worldView = RenderVertexBufferView(RenderBufferReference(outputBuffers.rasterWorldPosBuffer.buffer.get()), PosStride * vertexCount);\n }", + ) + t = replace( + t, + ' waterUnavailable=true;\n std::fprintf(stderr,"[water] resource fallback', + ' waterRecovery.fail();\n std::fprintf(stderr,"[water] resource fallback', + ) + t = replace( + t, + ' if (waterUnavailable) throw std::runtime_error("water preparation was incomplete");', + ' if (waterRecovery.failed) throw std::runtime_error("water preparation was incomplete; bounded retry scheduled");', + ) + return replace( + t, + ' waterUnavailable=true;\n std::fprintf(stderr,"[water] frame fallback', + ' waterRecovery.fail();\n std::fprintf(stderr,"[water] frame fallback', + ) + + +change("src/render/rt64_framebuffer_renderer.cpp", framebuffer) +output.write_text("".join(changes)) +print(f"Authored {output}; no dependency/stage files changed.") diff --git a/scripts/compose-water-safety.py b/scripts/compose-water-safety.py index 8507f8dd..39ca173f 100644 --- a/scripts/compose-water-safety.py +++ b/scripts/compose-water-safety.py @@ -3,68 +3,106 @@ Reads generated input; writes only the authored patch, never RT64 or a stage. Run after renderer + adapter + budget patches have been applied. """ -from pathlib import Path + import difflib +from pathlib import Path import re import sys -stage=Path(sys.argv[1]).resolve() -root=Path(__file__).resolve().parents[1] -output=[] +stage = Path(sys.argv[1]).resolve() +root = Path(__file__).resolve().parents[1] +output = [] # Rebuilding the authored safety patch from an already-composed build stage # must first remove the PREVIOUS patch in memory. No generated file is edited. -previous_patch=root/"patches/enhanced-water/dkr-safety.patch" -reverse_hunks={} -if previous_patch.exists() and "fieldDesired" in (stage/"src/render/rt64_water_renderer.h").read_text(): - path=None;old=[];new=[] +previous_patch = root / "patches/enhanced-water/dkr-safety.patch" +reverse_hunks = {} +if previous_patch.exists() and "fieldDesired" in (stage / "src/render/rt64_water_renderer.h").read_text(): + path = None + old = [] + new = [] + def finish_hunk(): - if path and (old or new):reverse_hunks.setdefault(path,[]).append(("".join(new),"".join(old))) + if path and (old or new): + reverse_hunks.setdefault(path, []).append(("".join(new), "".join(old))) + for line in previous_patch.read_text().splitlines(True): if line.startswith("diff --git "): - finish_hunk();old=[];new=[];path=line.split()[2][2:] + finish_hunk() + old = [] + new = [] + path = line.split()[2][2:] elif line.startswith("@@ "): - finish_hunk();old=[];new=[] - elif line.startswith(("--- ","+++ ")):continue - elif line.startswith(" "):old.append(line[1:]);new.append(line[1:]) - elif line.startswith("-"):old.append(line[1:]) - elif line.startswith("+"):new.append(line[1:]) + finish_hunk() + old = [] + new = [] + elif line.startswith(("--- ", "+++ ")): + continue + elif line.startswith(" "): + old.append(line[1:]) + new.append(line[1:]) + elif line.startswith("-"): + old.append(line[1:]) + elif line.startswith("+"): + new.append(line[1:]) finish_hunk() + def change(path, transform): - before=(stage/path).read_text() - for new,old in reversed(reverse_hunks.get(path,[])): - if before.count(new)!=1:raise SystemExit(f"Cannot reverse the previous authored safety hunk in {path}") - before=before.replace(new,old,1) - after=transform(before) - if before==after: raise SystemExit(f"No reviewed changes in {path}") + before = (stage / path).read_text() + for new, old in reversed(reverse_hunks.get(path, [])): + if before.count(new) != 1: + raise SystemExit(f"Cannot reverse the previous authored safety hunk in {path}") + before = before.replace(new, old, 1) + after = transform(before) + if before == after: + raise SystemExit(f"No reviewed changes in {path}") output.append(f"diff --git a/{path} b/{path}\n") - output.extend(difflib.unified_diff(before.splitlines(True),after.splitlines(True), - fromfile=f"a/{path}",tofile=f"b/{path}",n=3)) + output.extend( + difflib.unified_diff( + before.splitlines(True), after.splitlines(True), fromfile=f"a/{path}", tofile=f"b/{path}", n=3 + ) + ) + def replace(text, old, new): - if text.count(old)!=1: raise SystemExit(f"Non-unique reviewed water context: {old[:100]}") - return text.replace(old,new,1) + if text.count(old) != 1: + raise SystemExit(f"Non-unique reviewed water context: {old[:100]}") + return text.replace(old, new, 1) + def resources(text): # Balanced calls can span lines; requireWater checks before dereferencing # every optional texture/view/buffer/framebuffer allocation. - pattern=r"(?:worker->device->create(?:Texture|Framebuffer|Buffer)|(?:scene\.\w+|fieldTextures\[i\])->createTextureView)\(" - matches=list(re.finditer(pattern,text)) - if len(matches)<12: raise SystemExit("Allocation coverage changed") + pattern = r"(?:worker->device->create(?:Texture|Framebuffer|Buffer)|(?:scene\.\w+|fieldTextures\[i\])->createTextureView)\(" + matches = list(re.finditer(pattern, text)) + if len(matches) < 12: + raise SystemExit("Allocation coverage changed") for m in reversed(matches): - end=m.end(); depth=1 - while depth and end=2 ? 512 : 256;", - "const uint32_t resolution=m.identity.x>=3 ? 512 : 256;") - text=replace(text,"if (reset || m.identity.x<2) {","if (reset || m.identity.x<2 || m.effects.x<0.5f) {") - text=replace(text," const auto sceneKey=std::make_pair(key,selfDepth);", """ const auto sceneKey=std::make_pair(key,selfDepth); + end = m.end() + depth = 1 + while depth and end < len(text): + if text[end] == "(": + depth += 1 + elif text[end] == ")": + depth -= 1 + end += 1 + if depth: + raise SystemExit("Unbalanced allocation") + text = text[: m.start()] + "checkedWaterResource(" + text[m.start() : end] + ")" + text[end:] + text = replace( + text, '"water shader or pipeline creation failed"', '"water resource creation failed; retaining Original"' + ) + text = replace( + text, + "const uint32_t resolution=m.identity.x>=2 ? 512 : 256;", + "const uint32_t resolution=m.identity.x>=3 ? 512 : 256;", + ) + text = replace(text, "if (reset || m.identity.x<2) {", "if (reset || m.identity.x<2 || m.effects.x<0.5f) {") + text = replace( + text, + " const auto sceneKey=std::make_pair(key,selfDepth);", + """ const auto sceneKey=std::make_pair(key,selfDepth); auto found = sceneKeys.find(sceneKey); if (found != sceneKeys.end()) return found->second; const uint64_t pixels=uint64_t(color->width)*color->height; @@ -84,11 +122,19 @@ def resources(text): if (!color->width || !color->height || color->width>8192 || color->height>8192 || resident>256ULL*1024*1024 || sceneKeys.size()>=16) throw std::runtime_error("water intermediate target exceeds its bounded budget"); -""") - text=replace(text,""" auto found = sceneKeys.find(sceneKey); +""", + ) + text = replace( + text, + """ auto found = sceneKeys.find(sceneKey); if (found != sceneKeys.end()) return found->second; - const uint32_t index = uint32_t(sceneKeys.size());"""," const uint32_t index = uint32_t(sceneKeys.size());") - text=replace(text," auto key = std::make_tuple(group,int(data.posFloats[v*3]),int(data.posFloats[v*3+1]),int(data.posFloats[v*3+2]));", """ const auto &material=draws.back().material; + const uint32_t index = uint32_t(sceneKeys.size());""", + " const uint32_t index = uint32_t(sceneKeys.size());", + ) + text = replace( + text, + " auto key = std::make_tuple(group,int(data.posFloats[v*3]),int(data.posFloats[v*3+1]),int(data.posFloats[v*3+2]));", + """ const auto &material=draws.back().material; hlslpp::float4 position(data.posFloats[v*3],data.posFloats[v*3+1],data.posFloats[v*3+2],1); if (material.cameraPosition.w==1.0f) { if (v>=data.worldIndices.size() || data.worldIndices[v]>=data.worldTransforms.size()) continue; @@ -99,21 +145,34 @@ def resources(text): } if (!std::isfinite(float(position.x)) || !std::isfinite(float(position.y)) || !std::isfinite(float(position.z)) || std::abs(float(position.x))>10000000 || std::abs(float(position.y))>10000000 || std::abs(float(position.z))>10000000) continue; - auto key = std::make_tuple(group,int(std::round(float(position.x)*4)),int(std::round(float(position.y)*4)),int(std::round(float(position.z)*4)));""") - text=replace(text," wireStarts.push_back(uint32_t(wireIndices.size()));", """ wireStarts.push_back(uint32_t(wireIndices.size())); - const auto cameraInverse=hlslpp::inverse(hlslpp::float4x4(draws.back().material.cameraViewProjection));""") - text=replace(text,"const bool reset=fieldGeneration!=uint32_t(m.identity.w) || desiredfieldTime+0.5f;", - "const bool reset=fieldGeneration!=uint32_t(m.identity.w) || desiredfieldDesired+0.25f;\n fieldDesired=desired;") - text=text.replace("WR64_WATER_","DKRR_WATER_") + auto key = std::make_tuple(group,int(std::round(float(position.x)*4)),int(std::round(float(position.y)*4)),int(std::round(float(position.z)*4)));""", + ) + text = replace( + text, + " wireStarts.push_back(uint32_t(wireIndices.size()));", + """ wireStarts.push_back(uint32_t(wireIndices.size())); + const auto cameraInverse=hlslpp::inverse(hlslpp::float4x4(draws.back().material.cameraViewProjection));""", + ) + text = replace( + text, + "const bool reset=fieldGeneration!=uint32_t(m.identity.w) || desiredfieldTime+0.5f;", + "const bool reset=fieldGeneration!=uint32_t(m.identity.w) || desiredfieldDesired+0.25f;\n fieldDesired=desired;", + ) + text = text.replace("WR64_WATER_", "DKRR_WATER_") return text -change("src/render/rt64_water_renderer.cpp",resources) + +change("src/render/rt64_water_renderer.cpp", resources) + def fallback(text): - text=replace(text,""" wp.viewProjection = wp.material.cameraViewProjection; + text = replace( + text, + """ wp.viewProjection = wp.material.cameraViewProjection; wp.camera.x = wp.material.cameraPosition.x; wp.camera.y = wp.material.cameraPosition.y; - wp.camera.z = wp.material.cameraPosition.z;""", """ wp.viewProjection = wp.material.cameraViewProjection; + wp.camera.z = wp.material.cameraPosition.z;""", + """ wp.viewProjection = wp.material.cameraViewProjection; // Match the matrix used by interpolated water vertices, // not the current simulation camera (DKR stores MVPs). const auto address=uint32_t(wp.material.effects.y); @@ -125,35 +184,65 @@ def fallback(text): const auto eye=hlslpp::mul(hlslpp::float4(0,0,1,0),hlslpp::inverse(wp.viewProjection)); wp.camera.x=float(eye.x/eye.w); wp.camera.y=float(eye.y/eye.w); - wp.camera.z=float(eye.z/eye.w);""") - text=replace(text," if (wantsWater && waterRenderer && waterSamplesSupported) {", - " if (wantsWater && waterRenderer && !waterUnavailable && waterSamplesSupported) {\n try {") - text=replace(text," instanceDrawCall.waterIndex = uint32_t(waterRenderer->draws.size());", """ const auto waterScene=waterRenderer->prepareScene(p.worker,p.fbStorage->colorTarget,p.fbStorage->depthTarget, + wp.camera.z=float(eye.z/eye.w);""", + ) + text = replace( + text, + " if (wantsWater && waterRenderer && waterSamplesSupported) {", + " if (wantsWater && waterRenderer && !waterUnavailable && waterSamplesSupported) {\n try {", + ) + text = replace( + text, + " instanceDrawCall.waterIndex = uint32_t(waterRenderer->draws.size());", + """ const auto waterScene=waterRenderer->prepareScene(p.worker,p.fbStorage->colorTarget,p.fbStorage->depthTarget, (uint64_t(framebufferCount) << 32) | proj.transformsIndex,classic,selfDepth); instanceDrawCall.originalWaterPipeline=triangles.pipeline; - instanceDrawCall.waterIndex = uint32_t(waterRenderer->draws.size());""") - text=replace(text," if (wp.material.optics.w>0.5f) instanceDrawCall.originalWaterPipeline=triangles.pipeline;","") - text=replace(text," if (drawCall.originalWaterPipeline) {"," if (waterRenderer->draws[drawCall.waterIndex].material.optics.w>0.5f) {") - text=replace(text," const bool classic=drawCall.originalWaterPipeline!=nullptr;"," const bool classic=waterRenderer->draws[drawCall.waterIndex].material.optics.w>0.5f;") - text=replace(text,""" waterRenderer->drawScenes.push_back(waterRenderer->prepareScene(p.worker,p.fbStorage->colorTarget,p.fbStorage->depthTarget, + instanceDrawCall.waterIndex = uint32_t(waterRenderer->draws.size());""", + ) + text = replace( + text, + " if (wp.material.optics.w>0.5f) instanceDrawCall.originalWaterPipeline=triangles.pipeline;", + "", + ) + text = replace( + text, + " if (drawCall.originalWaterPipeline) {", + " if (waterRenderer->draws[drawCall.waterIndex].material.optics.w>0.5f) {", + ) + text = replace( + text, + " const bool classic=drawCall.originalWaterPipeline!=nullptr;", + " const bool classic=waterRenderer->draws[drawCall.waterIndex].material.optics.w>0.5f;", + ) + text = replace( + text, + """ waterRenderer->drawScenes.push_back(waterRenderer->prepareScene(p.worker,p.fbStorage->colorTarget,p.fbStorage->depthTarget, (uint64_t(framebufferCount) << 32) | proj.transformsIndex,classic,selfDepth));""", - " waterRenderer->drawScenes.push_back(waterScene);") - text=replace(text,""" } + " waterRenderer->drawScenes.push_back(waterScene);", + ) + text = replace( + text, + """ } } - // Determine to use the draw call either in the RT scene or the raster scene.""", """ } + // Determine to use the draw call either in the RT scene or the raster scene.""", + """ } } catch (const std::exception &error) { waterUnavailable=true; std::fprintf(stderr,"[water] resource fallback to Original: %s\\n",error.what()); } } - // Determine to use the draw call either in the RT scene or the raster scene.""") - text=replace(text,""" if (waterRenderer && !waterRenderer->draws.empty()) { + // Determine to use the draw call either in the RT scene or the raster scene.""", + ) + text = replace( + text, + """ if (waterRenderer && !waterRenderer->draws.empty()) { waterRenderer->prepareField(worker); waterRenderer->prepareNormals(worker, *outputBuffers, shaderUploads); shaderUploads.push_back({waterRenderer->draws.data(), {0,waterRenderer->draws.size()}, sizeof(interop::WaterDrawParams), RenderBufferFlag::STORAGE, {}, &waterRenderer->drawBuffer}); - }""", """ if (waterRenderer && !waterRenderer->draws.empty()) { + }""", + """ if (waterRenderer && !waterRenderer->draws.empty()) { const auto uploadStart=shaderUploads.size(); try { if (waterUnavailable) throw std::runtime_error("water preparation was incomplete"); @@ -171,15 +260,28 @@ def fallback(text): waterUnavailable=true; std::fprintf(stderr,"[water] frame fallback to Original: %s\\n",error.what()); } - }""") + }""", + ) return text -change("src/render/rt64_framebuffer_renderer.cpp",fallback) -change("src/render/rt64_water_renderer.h",lambda text:replace(text," float fieldTime = 0;"," float fieldTime = 0, fieldDesired = 0;")) + + +change("src/render/rt64_framebuffer_renderer.cpp", fallback) +change( + "src/render/rt64_water_renderer.h", + lambda text: replace(text, " float fieldTime = 0;", " float fieldTime = 0, fieldDesired = 0;"), +) + def quality(text): - text=replace(text,"[loop] for (uint step=0;step<36;step++) {", - "[loop] for (uint step=0;step<(p.material.identity.x>=3 ? 36u : 24u);step++) {") - return replace(text," if (position.z < 0 || position.z > 1022.0/1024.0) discard;", """ if (position.z < 0 || position.z > 1022.0/1024.0) discard; + text = replace( + text, + "[loop] for (uint step=0;step<36;step++) {", + "[loop] for (uint step=0;step<(p.material.identity.x>=3 ? 36u : 24u);step++) {", + ) + return replace( + text, + " if (position.z < 0 || position.z > 1022.0/1024.0) discard;", + """ if (position.z < 0 || position.z > 1022.0/1024.0) discard; if (m.effects.z>=3) { // Preserve the original waterfall/fountain's transparency, palette // and course fog. This is NOT a horizontal ocean plane. Continuous @@ -196,7 +298,10 @@ def quality(text): float visibility=p.fogColor.a>0.5 ? 1-saturate(sourceColor.a) : 1; float3 result=lerp(base,base+float3(0.025,0.033,0.035),saturate(streak*visibility)); return float4(pow(saturate(result),1.0/2.2),1); - }""") -change("src/shaders/WaterPS.hlsl",quality) -(root/"patches/enhanced-water/dkr-safety.patch").write_text("".join(output),newline="\n") + }""", + ) + + +change("src/shaders/WaterPS.hlsl", quality) +(root / "patches/enhanced-water/dkr-safety.patch").write_text("".join(output), newline="\n") print("Generated reviewed allocation, normal-space, quality and fallback patch.") diff --git a/scripts/compose-water-stability.py b/scripts/compose-water-stability.py index a7307f2d..5ca55e92 100644 --- a/scripts/compose-water-stability.py +++ b/scripts/compose-water-stability.py @@ -1,46 +1,80 @@ """Author the final water stability/tuning layer; never edit staged/dependency sources.""" -from pathlib import Path + import difflib +from pathlib import Path import sys + from water_patch_baseline import read_before -root=Path(__file__).resolve().parents[1] -stage=Path(sys.argv[1]).resolve() -output=root/'patches/enhanced-water/dkr-stability.patch' -layers=[] -if 'lavaSurface' in (stage/'src/shaders/WaterPS.hlsl').read_text(): - layers.append(root/'patches/enhanced-water/dkr-lava.patch') -if 'waterfallPipeline' in (stage/'src/render/rt64_water_renderer.h').read_text(): - layers.append(root/'patches/enhanced-water/dkr-waterfall.patch') -if 'captureOwner' in (stage/'src/render/rt64_water_renderer.h').read_text(): +root = Path(__file__).resolve().parents[1] +stage = Path(sys.argv[1]).resolve() +output = root / "patches/enhanced-water/dkr-stability.patch" +layers = [] +if "lavaSurface" in (stage / "src/shaders/WaterPS.hlsl").read_text(): + layers.append(root / "patches/enhanced-water/dkr-lava.patch") +if "waterfallPipeline" in (stage / "src/render/rt64_water_renderer.h").read_text(): + layers.append(root / "patches/enhanced-water/dkr-waterfall.patch") +if "captureOwner" in (stage / "src/render/rt64_water_renderer.h").read_text(): layers.append(output) -changes=[] -def replace(t,a,b): - if t.count(a)!=1:raise SystemExit(f'Non-unique stability context: {a[:150]}') - return t.replace(a,b,1) -def change(path,fn): - before=read_before(stage,path,layers);after=fn(before) - if before==after:raise SystemExit(f'No stability change: {path}') - changes.append(f'diff --git a/{path} b/{path}\n') - changes.extend(difflib.unified_diff(before.splitlines(True),after.splitlines(True),fromfile=f'a/{path}',tofile=f'b/{path}',n=3)) +changes = [] + + +def replace(t, a, b): + if t.count(a) != 1: + raise SystemExit(f"Non-unique stability context: {a[:150]}") + return t.replace(a, b, 1) + + +def change(path, fn): + before = read_before(stage, path, layers) + after = fn(before) + if before == after: + raise SystemExit(f"No stability change: {path}") + changes.append(f"diff --git a/{path} b/{path}\n") + changes.extend( + difflib.unified_diff( + before.splitlines(True), after.splitlines(True), fromfile=f"a/{path}", tofile=f"b/{path}", n=3 + ) + ) + def params(t): - t=replace(t,' float4 effects;', ' float4 performance; // SSR steps, wake resolution, hardware distance, spray tier\n float4 appearance; // reflection, refraction, wake and waterfall strengths\n float4 effects;') - t=replace(t,' float4 viewBounds;', ' float4 originalRegion; // compact original capture origin XY, atlas cell size ZW\n float4 viewBounds;') - t=t.replace('sizeof(WaterMaterial) == 1552','sizeof(WaterMaterial) == 1584').replace('sizeof(WaterDrawParams) == 2000','sizeof(WaterDrawParams) == 2048') + t = replace( + t, + " float4 effects;", + " float4 performance; // SSR steps, wake resolution, hardware distance, spray tier\n float4 appearance; // reflection, refraction, wake and waterfall strengths\n float4 effects;", + ) + t = replace( + t, + " float4 viewBounds;", + " float4 originalRegion; // compact original capture origin XY, atlas cell size ZW\n float4 viewBounds;", + ) + t = t.replace("sizeof(WaterMaterial) == 1552", "sizeof(WaterMaterial) == 1584").replace( + "sizeof(WaterDrawParams) == 2000", "sizeof(WaterDrawParams) == 2048" + ) return t -change('src/shared/rt64_water_params.h',params) + + +change("src/shared/rt64_water_params.h", params) + def header(t): - t=replace(t,' std::unique_ptr color, depth, originalColor;', - ' std::shared_ptr color,depth;\n std::unique_ptr originalColor;\n uint32_t originalWidth=0,originalHeight=0,captureOwner=UINT32_MAX;\n bool classic=false,opticalCaptured=false;\n uint32_t opticalCamera=UINT32_MAX;\n interop::float4 opticalView{};') - t=replace(t,' std::unique_ptr surface;', ' std::shared_ptr surface;') - t=replace(t,'bool classic, bool selfDepth);','bool classic, bool selfDepth,water_work::Region originalRegion={});') - t=replace(t,' bool rayUnavailable=false;',' bool rayUnavailable=false;\n uint32_t rayRetry=0;') + t = replace( + t, + " std::unique_ptr color, depth, originalColor;", + " std::shared_ptr color,depth;\n std::unique_ptr originalColor;\n uint32_t originalWidth=0,originalHeight=0,captureOwner=UINT32_MAX;\n bool classic=false,opticalCaptured=false;\n uint32_t opticalCamera=UINT32_MAX;\n interop::float4 opticalView{};", + ) + t = replace(t, " std::unique_ptr surface;", " std::shared_ptr surface;") + t = replace( + t, "bool classic, bool selfDepth);", "bool classic, bool selfDepth,water_work::Region originalRegion={});" + ) + t = replace(t, " bool rayUnavailable=false;", " bool rayUnavailable=false;\n uint32_t rayRetry=0;") return t -change('src/render/rt64_water_renderer.h',header) -prepare=r'''uint32_t WaterRenderer::prepareScene(RenderWorker *worker, RenderTarget *color, RenderTarget *depth, uint64_t key, bool classic, bool selfDepth,water_work::Region originalRegion) { + +change("src/render/rt64_water_renderer.h", header) + +prepare = r"""uint32_t WaterRenderer::prepareScene(RenderWorker *worker, RenderTarget *color, RenderTarget *depth, uint64_t key, bool classic, bool selfDepth,water_work::Region originalRegion) { const auto sceneKey=std::make_tuple(key,classic,selfDepth); if(rejectedScenes.count(sceneKey))return water_policy::invalidScene; const auto found=sceneKeys.find(sceneKey); @@ -179,132 +213,272 @@ def header(t): throw; } } -''' +""" + def renderer(t): - t=replace(t,'#include "rt64_water_renderer.h"','#include "rt64_water_renderer.h"\n#include "enhanced_water_settings.hpp"') - t=replace(t,' if (std::getenv("WR64_TEST_WATER_SHADER_FAILURE"))', - ' dkr::runtime::water::enhanced::active_reflection=-1;\n dkr::runtime::water::enhanced::scratch_fallbacks=0;\n std::fprintf(stderr,"[water] shared-optics/compact-vertical scratch policy active; budget=256 MiB; quality defaults preserved\\n");\n if (std::getenv("WR64_TEST_WATER_SHADER_FAILURE"))') - t=replace(t,'if (std::getenv("DKRR_WATER_PROFILE") && device->getCapabilities().queryPools)', - 'if ((std::getenv("DKRR_WATER_PROFILE") || dkr::runtime::water::enhanced::timing_enabled.load()) && device->getCapabilities().queryPools)') - start=t.index('uint32_t WaterRenderer::prepareScene(');end=t.index('\nbool WaterRenderer::captureScene(',start) - t=t[:start]+prepare+t[end:] - t=replace(t,' spray->setBuffer(spray->draws,drawBuffer.get(),RenderBufferStructuredView(sizeof(interop::WaterDrawParams)));', - ' if(spray)spray->setBuffer(spray->draws,drawBuffer.get(),RenderBufferStructuredView(sizeof(interop::WaterDrawParams)));') - t=replace(t,' if (particleBuffer.get()) spray->setBuffer(', ' if (spray && particleBuffer.get()) spray->setBuffer(') - t=replace(t,' rays.resetFrame();',' rays.resetFrame();\n if(rayRetry && !--rayRetry)rayUnavailable=false;') - t=replace(t,'scene.captured=false; scene.surfaceCleared=false;', 'scene.captured=false; scene.opticalCaptured=false; scene.surfaceCleared=false;') - t=replace(t,' rayUnavailable=true;', ' rayUnavailable=true;rayRetry=120;') - t=replace(t,' pipelines.emplace(key, std::move(result));', - ' if(!result && hardware)return nullptr; // Transient failure must not poison the cache forever.\n pipelines.emplace(key, std::move(result));') - t=replace(t,' if(scene.captured && scene.capturedCamera==camera &&', - ' if(scene.classic) {\n if(scene.captured)return false;scene.captured=true;\n worker->commandList->setFramebuffer(nullptr);\n worker->commandList->barriers(RenderBarrierStage::GRAPHICS,{\n RenderTextureBarrier(scene.color.get(),RenderTextureLayout::SHADER_READ),\n RenderTextureBarrier(scene.depth.get(),RenderTextureLayout::SHADER_READ)});\n return true;\n }\n auto& optical=scenes[scene.captureOwner];\n const bool same=optical.opticalCaptured && optical.opticalCamera==camera &&\n optical.opticalView.x==view.x && optical.opticalView.y==view.y &&\n optical.opticalView.z==view.z && optical.opticalView.w==view.w;\n if(same && scene.captured && scene.capturedCamera==camera &&') + t = replace( + t, + '#include "rt64_water_renderer.h"', + '#include "rt64_water_renderer.h"\n#include "enhanced_water_settings.hpp"', + ) + t = replace( + t, + ' if (std::getenv("WR64_TEST_WATER_SHADER_FAILURE"))', + ' dkr::runtime::water::enhanced::active_reflection=-1;\n dkr::runtime::water::enhanced::scratch_fallbacks=0;\n std::fprintf(stderr,"[water] shared-optics/compact-vertical scratch policy active; budget=256 MiB; quality defaults preserved\\n");\n if (std::getenv("WR64_TEST_WATER_SHADER_FAILURE"))', + ) + t = replace( + t, + 'if (std::getenv("DKRR_WATER_PROFILE") && device->getCapabilities().queryPools)', + 'if ((std::getenv("DKRR_WATER_PROFILE") || dkr::runtime::water::enhanced::timing_enabled.load()) && device->getCapabilities().queryPools)', + ) + start = t.index("uint32_t WaterRenderer::prepareScene(") + end = t.index("\nbool WaterRenderer::captureScene(", start) + t = t[:start] + prepare + t[end:] + t = replace( + t, + " spray->setBuffer(spray->draws,drawBuffer.get(),RenderBufferStructuredView(sizeof(interop::WaterDrawParams)));", + " if(spray)spray->setBuffer(spray->draws,drawBuffer.get(),RenderBufferStructuredView(sizeof(interop::WaterDrawParams)));", + ) + t = replace( + t, + " if (particleBuffer.get()) spray->setBuffer(", + " if (spray && particleBuffer.get()) spray->setBuffer(", + ) + t = replace( + t, " rays.resetFrame();", " rays.resetFrame();\n if(rayRetry && !--rayRetry)rayUnavailable=false;" + ) + t = replace( + t, + "scene.captured=false; scene.surfaceCleared=false;", + "scene.captured=false; scene.opticalCaptured=false; scene.surfaceCleared=false;", + ) + t = replace(t, " rayUnavailable=true;", " rayUnavailable=true;rayRetry=120;") + t = replace( + t, + " pipelines.emplace(key, std::move(result));", + " if(!result && hardware)return nullptr; // Transient failure must not poison the cache forever.\n pipelines.emplace(key, std::move(result));", + ) + t = replace( + t, + " if(scene.captured && scene.capturedCamera==camera &&", + " if(scene.classic) {\n if(scene.captured)return false;scene.captured=true;\n worker->commandList->setFramebuffer(nullptr);\n worker->commandList->barriers(RenderBarrierStage::GRAPHICS,{\n RenderTextureBarrier(scene.color.get(),RenderTextureLayout::SHADER_READ),\n RenderTextureBarrier(scene.depth.get(),RenderTextureLayout::SHADER_READ)});\n return true;\n }\n auto& optical=scenes[scene.captureOwner];\n const bool same=optical.opticalCaptured && optical.opticalCamera==camera &&\n optical.opticalView.x==view.x && optical.opticalView.y==view.y &&\n optical.opticalView.z==view.z && optical.opticalView.w==view.w;\n if(same && scene.captured && scene.capturedCamera==camera &&", + ) # A second legacy depth mode shares the immutable pre-water optical image, # but still initializes its own isolated depth attachment when needed. - t=replace(t,' worker->commandList->barriers(RenderBarrierStage::COMPUTE, {\n RenderTextureBarrier(scene.sourceColor->texture.get()', - ' if(!same) {\n optical.opticalCaptured=true;optical.opticalCamera=camera;optical.opticalView=view;\n worker->commandList->barriers(RenderBarrierStage::COMPUTE, {\n RenderTextureBarrier(scene.sourceColor->texture.get()') - t=replace(t,' endProfile(worker,profileToken);\n return true;', - ''' } else if(scene.selfDepth) { + t = replace( + t, + " worker->commandList->barriers(RenderBarrierStage::COMPUTE, {\n RenderTextureBarrier(scene.sourceColor->texture.get()", + " if(!same) {\n optical.opticalCaptured=true;optical.opticalCamera=camera;optical.opticalView=view;\n worker->commandList->barriers(RenderBarrierStage::COMPUTE, {\n RenderTextureBarrier(scene.sourceColor->texture.get()", + ) + t = replace( + t, + " endProfile(worker,profileToken);\n return true;", + """ } else if(scene.selfDepth) { // A reused optical image can still require this mode's private depth // copy. Restore the source attachment even without another capture. worker->commandList->barriers(RenderBarrierStage::GRAPHICS, RenderTextureBarrier(scene.sourceDepth->texture.get(),depthReadOnly ? RenderTextureLayout::DEPTH_READ : RenderTextureLayout::DEPTH_WRITE)); } endProfile(worker,profileToken); - return true;''') - t=replace(t,' const auto& view=draws[drawIndex].viewBounds;\n const auto region=water_work::captureRegion(scene.width,scene.height,view.x,view.y,view.z,view.w);\n const uint32_t params[]={scene.width,scene.height,samples,0,region.x,region.y,region.width,region.height};', - ''' const auto& region=draws[drawIndex].originalRegion; + return true;""", + ) + t = replace( + t, + " const auto& view=draws[drawIndex].viewBounds;\n const auto region=water_work::captureRegion(scene.width,scene.height,view.x,view.y,view.z,view.w);\n const uint32_t params[]={scene.width,scene.height,samples,0,region.x,region.y,region.width,region.height};", + """ const auto& region=draws[drawIndex].originalRegion; const uint32_t x=uint32_t(region.x),y=uint32_t(region.y); const uint32_t width=std::min(scene.originalWidth,scene.sourceColor->width-x); const uint32_t height=std::min(scene.originalHeight,scene.sourceColor->height-y); - const uint32_t params[]={scene.originalWidth,scene.originalHeight,samples,0,x,y,width,height};''') - t=replace(t,' worker->commandList->dispatch((region.width+7)/8,(region.height+7)/8,1);\n worker->commandList->barriers(RenderBarrierStage::GRAPHICS,{', - ' worker->commandList->dispatch((width+7)/8,(height+7)/8,1);\n worker->commandList->barriers(RenderBarrierStage::GRAPHICS,{') - t=replace(t,' if (!scene.surfaceCleared) {\n worker->commandList->clearColor(0,RenderColor(0,0,0,0));\n scene.surfaceCleared=true;', - ' auto& maskOwner=scenes[scene.captureOwner];\n if (!maskOwner.surfaceCleared) {\n worker->commandList->clearColor(0,RenderColor(0,0,0,0));\n maskOwner.surfaceCleared=true;') - t=replace(t,' const uint32_t budget=m.identity.x>=3 ? 4096 : 2048;', - ' const uint32_t budget=m.performance.w>=2 ? 4096 : m.performance.w>=1 ? 2048 : 512;') - t=replace(t,'distance*(b-a)*std::clamp(speed/250.0f', 'distance*(b-a)*(m.performance.w<1 ? 0.5f : 1.0f)*std::clamp(speed/250.0f') - t=replace(t,' const uint32_t resolution=water_work::fieldResolution(active,bool(fieldTextures[0]),fieldResolution,uint32_t(m.identity.x));', - ' const uint32_t resolution=active && m.performance.y>=128 ? uint32_t(m.performance.y) : water_work::fieldResolution(active,bool(fieldTextures[0]),fieldResolution,uint32_t(m.identity.x));') - t=replace(t,' wireStarts.push_back(uint32_t(wireIndices.size()));', - ' wireStarts.push_back(uint32_t(wireIndices.size()));\n if(draws.back().material.effects.z>=3)return;') - t=replace(t,' normalVertices.clear(); normalFaces.clear();', - ''' for(uint32_t i=0;icommandList->dispatch((region.width+7)/8,(region.height+7)/8,1);\n worker->commandList->barriers(RenderBarrierStage::GRAPHICS,{", + " worker->commandList->dispatch((width+7)/8,(height+7)/8,1);\n worker->commandList->barriers(RenderBarrierStage::GRAPHICS,{", + ) + t = replace( + t, + " if (!scene.surfaceCleared) {\n worker->commandList->clearColor(0,RenderColor(0,0,0,0));\n scene.surfaceCleared=true;", + " auto& maskOwner=scenes[scene.captureOwner];\n if (!maskOwner.surfaceCleared) {\n worker->commandList->clearColor(0,RenderColor(0,0,0,0));\n maskOwner.surfaceCleared=true;", + ) + t = replace( + t, + " const uint32_t budget=m.identity.x>=3 ? 4096 : 2048;", + " const uint32_t budget=m.performance.w>=2 ? 4096 : m.performance.w>=1 ? 2048 : 512;", + ) + t = replace( + t, + "distance*(b-a)*std::clamp(speed/250.0f", + "distance*(b-a)*(m.performance.w<1 ? 0.5f : 1.0f)*std::clamp(speed/250.0f", + ) + t = replace( + t, + " const uint32_t resolution=water_work::fieldResolution(active,bool(fieldTextures[0]),fieldResolution,uint32_t(m.identity.x));", + " const uint32_t resolution=active && m.performance.y>=128 ? uint32_t(m.performance.y) : water_work::fieldResolution(active,bool(fieldTextures[0]),fieldResolution,uint32_t(m.identity.x));", + ) + t = replace( + t, + " wireStarts.push_back(uint32_t(wireIndices.size()));", + " wireStarts.push_back(uint32_t(wireIndices.size()));\n if(draws.back().material.effects.z>=3)return;", + ) + t = replace( + t, + " normalVertices.clear(); normalFaces.clear();", + """ for(uint32_t i=0;i0.5f &&', - ' bool hardware=!classic && wp.material.appearance.x>0 && wp.material.effects.w>0.5f &&') - t=t.replace('waterRenderer->rayUnavailable=true;','waterRenderer->rayUnavailable=true;waterRenderer->rayRetry=120;') - t=replace(t,' worker->commandList->setFramebuffer(nullptr);\n worker->commandList->barriers(RenderBarrierStage::GRAPHICS,RenderTextureBarrier(scene.surface.get()', - ' if(scene.classic) {activeWaterDraw=UINT32_MAX;switchToGraphicsPipeline();return;}\n worker->commandList->setFramebuffer(nullptr);\n worker->commandList->barriers(RenderBarrierStage::GRAPHICS,RenderTextureBarrier(scene.surface.get()') - t=replace(t,' const bool needsSurface=selfDepth || (!waterRenderer->particles.empty() &&\n waterRenderer->draws[drawCall.waterIndex].material.effects.x>0.5f);', - ' const bool needsSurface=!classic && (selfDepth || (!waterRenderer->particles.empty() &&\n waterRenderer->draws[drawCall.waterIndex].material.effects.x>0.5f));') - t=replace(t,' const auto waterScene=waterRenderer->prepareScene(p.worker,p.fbStorage->colorTarget,p.fbStorage->depthTarget,\n uint64_t(framebufferCount),classic,selfDepth);', - ''' const auto originalRegion=classic ? waterOriginalRegion(drawData,call.meshDesc.faceIndicesStart, + t = replace( + t, + '#include "WaterProjection.h"', + '#include "WaterProjection.h"\n#include "WaterBounds.h"\n#include "enhanced_water_settings.hpp"', + ) + t = replace( + t, + " bool hardware=!classic && wp.material.effects.w>0.5f &&", + " bool hardware=!classic && wp.material.appearance.x>0 && wp.material.effects.w>0.5f &&", + ) + t = t.replace( + "waterRenderer->rayUnavailable=true;", "waterRenderer->rayUnavailable=true;waterRenderer->rayRetry=120;" + ) + t = replace( + t, + " worker->commandList->setFramebuffer(nullptr);\n worker->commandList->barriers(RenderBarrierStage::GRAPHICS,RenderTextureBarrier(scene.surface.get()", + " if(scene.classic) {activeWaterDraw=UINT32_MAX;switchToGraphicsPipeline();return;}\n worker->commandList->setFramebuffer(nullptr);\n worker->commandList->barriers(RenderBarrierStage::GRAPHICS,RenderTextureBarrier(scene.surface.get()", + ) + t = replace( + t, + " const bool needsSurface=selfDepth || (!waterRenderer->particles.empty() &&\n waterRenderer->draws[drawCall.waterIndex].material.effects.x>0.5f);", + " const bool needsSurface=!classic && (selfDepth || (!waterRenderer->particles.empty() &&\n waterRenderer->draws[drawCall.waterIndex].material.effects.x>0.5f));", + ) + t = replace( + t, + " const auto waterScene=waterRenderer->prepareScene(p.worker,p.fbStorage->colorTarget,p.fbStorage->depthTarget,\n uint64_t(framebufferCount),classic,selfDepth);", + """ const auto originalRegion=classic ? waterOriginalRegion(drawData,call.meshDesc.faceIndicesStart, call.callDesc.triangleCount*3,wp,p.interpolationWeight) : water_work::Region{}; wp.originalRegion={float(originalRegion.x),float(originalRegion.y),0,0}; const auto waterScene=waterRenderer->prepareScene(p.worker,p.fbStorage->colorTarget,p.fbStorage->depthTarget, - uint64_t(framebufferCount),classic,selfDepth,originalRegion);''') - t=replace(t,' if(waterScene!=water_policy::invalidScene) {', - ''' if(waterScene!=water_policy::invalidScene) { - if(!classic)dkr::runtime::water::enhanced::active_reflection.store(hardware ? 2 : wp.material.identity.x>=2 ? 1 : 0);''') + uint64_t(framebufferCount),classic,selfDepth,originalRegion);""", + ) + t = replace( + t, + " if(waterScene!=water_policy::invalidScene) {", + """ if(waterScene!=water_policy::invalidScene) { + if(!classic)dkr::runtime::water::enhanced::active_reflection.store(hardware ? 2 : wp.material.identity.x>=2 ? 1 : 0);""", + ) return t -change('src/render/rt64_framebuffer_renderer.cpp',framebuffer) + + +change("src/render/rt64_framebuffer_renderer.cpp", framebuffer) + def original(t): - t=replace(t,'RWTexture2D originalColor', - '[[vk::image_format("rgba16f")]] RWTexture2D originalColor') - t=replace(t,' pixel+=params.origin;\n if (pixel.x >= params.width || pixel.y >= params.height) return;', - ' const uint2 sourcePixel=pixel+params.origin;') - return t.replace('sourceColor.Load(pixel,i)','sourceColor.Load(sourcePixel,i)').replace('sourceColor.Load(uint3(pixel,0))','sourceColor.Load(uint3(sourcePixel,0))') -change('src/shaders/WaterOriginalCS.hlsl',original) + t = replace( + t, "RWTexture2D originalColor", '[[vk::image_format("rgba16f")]] RWTexture2D originalColor' + ) + t = replace( + t, + " pixel+=params.origin;\n if (pixel.x >= params.width || pixel.y >= params.height) return;", + " const uint2 sourcePixel=pixel+params.origin;", + ) + return t.replace("sourceColor.Load(pixel,i)", "sourceColor.Load(sourcePixel,i)").replace( + "sourceColor.Load(uint3(pixel,0))", "sourceColor.Load(uint3(sourcePixel,0))" + ) + + +change("src/shaders/WaterOriginalCS.hlsl", original) + def capture_format(t): - return replace(t,'RWTexture2D capturedColor', - '[[vk::image_format("rgba16f")]] RWTexture2D capturedColor') -change('src/shaders/WaterCaptureCS.hlsl',capture_format) + return replace( + t, "RWTexture2D capturedColor", '[[vk::image_format("rgba16f")]] RWTexture2D capturedColor' + ) + + +change("src/shaders/WaterCaptureCS.hlsl", capture_format) + + def field_format(t): - return replace(t,'RWTexture2D nextField', - '[[vk::image_format("rgba16f")]] RWTexture2D nextField') -change('src/shaders/WaterFieldCS.hlsl',field_format) + return replace(t, "RWTexture2D nextField", '[[vk::image_format("rgba16f")]] RWTexture2D nextField') + + +change("src/shaders/WaterFieldCS.hlsl", field_format) + def pixel(t): - t=replace(t,' uint depthWidth,depthHeight,depthLevels; sceneDepth.GetDimensions(0,depthWidth,depthHeight,depthLevels);', - ' if(p.material.appearance.x<=0)return 0;\n uint depthWidth,depthHeight,depthLevels; sceneDepth.GetDimensions(0,depthWidth,depthHeight,depthLevels);') - t=replace(t,'step<(p.material.identity.x>=3 ? 36u : 24u)', 'step0 ? p.material.performance.x : (p.material.identity.x>=3 ? 36 : 24))') - t=replace(t,' uint2 at=uint2(position.xy);',' uint2 at=uint2(position.xy-p.originalRegion.xy);') - t=replace(t,' uint w,h; sceneColor.GetDimensions(w,h);\n at+=uint2((sampleIndex&1)*w,(sampleIndex>>1)*h);', - ' at+=uint2((sampleIndex&1)*uint(p.originalRegion.z),(sampleIndex>>1)*uint(p.originalRegion.w));') - t=replace(t,'float streak=smoothstep(0.67,0.91,foamNoise(flow))*filter*m.animation.w;', - 'float streak=m.appearance.w<=0 ? 0 : smoothstep(0.67,0.91,foamNoise(flow))*filter*m.animation.w*m.appearance.w;') - t=replace(t,' float3 wake=interaction(worldPosition.xz,p);',' float3 wake=interaction(worldPosition.xz,p)*m.appearance.z;') - t=replace(t,'position.xy + ripples * 13 * saturate(thickness/80)', 'position.xy + ripples * 13 * m.appearance.y * saturate(thickness/80)') - t=replace(t,' color += reflected*fresnel;', ' color += reflected*fresnel*m.appearance.x;') - t=replace(t,' if(reflection.a<0.75) {',' if(reflection.a<0.75 && m.appearance.x>0) {') - t=replace(t,' uint2 originalPixel=uint2(position.xy);',' uint2 originalPixel=uint2(position.xy-p.originalRegion.xy);') - t=replace(t,' uint originalWidth,originalHeight;\n sceneColor.GetDimensions(originalWidth,originalHeight);\n originalPixel+=uint2((sampleIndex&1)*originalWidth,(sampleIndex>>1)*originalHeight);', - ' originalPixel+=uint2((sampleIndex&1)*uint(p.originalRegion.z),(sampleIndex>>1)*uint(p.originalRegion.w));') + t = replace( + t, + " uint depthWidth,depthHeight,depthLevels; sceneDepth.GetDimensions(0,depthWidth,depthHeight,depthLevels);", + " if(p.material.appearance.x<=0)return 0;\n uint depthWidth,depthHeight,depthLevels; sceneDepth.GetDimensions(0,depthWidth,depthHeight,depthLevels);", + ) + t = replace( + t, + "step<(p.material.identity.x>=3 ? 36u : 24u)", + "step0 ? p.material.performance.x : (p.material.identity.x>=3 ? 36 : 24))", + ) + t = replace(t, " uint2 at=uint2(position.xy);", " uint2 at=uint2(position.xy-p.originalRegion.xy);") + t = replace( + t, + " uint w,h; sceneColor.GetDimensions(w,h);\n at+=uint2((sampleIndex&1)*w,(sampleIndex>>1)*h);", + " at+=uint2((sampleIndex&1)*uint(p.originalRegion.z),(sampleIndex>>1)*uint(p.originalRegion.w));", + ) + t = replace( + t, + "float streak=smoothstep(0.67,0.91,foamNoise(flow))*filter*m.animation.w;", + "float streak=m.appearance.w<=0 ? 0 : smoothstep(0.67,0.91,foamNoise(flow))*filter*m.animation.w*m.appearance.w;", + ) + t = replace( + t, + " float3 wake=interaction(worldPosition.xz,p);", + " float3 wake=interaction(worldPosition.xz,p)*m.appearance.z;", + ) + t = replace( + t, + "position.xy + ripples * 13 * saturate(thickness/80)", + "position.xy + ripples * 13 * m.appearance.y * saturate(thickness/80)", + ) + t = replace(t, " color += reflected*fresnel;", " color += reflected*fresnel*m.appearance.x;") + t = replace(t, " if(reflection.a<0.75) {", " if(reflection.a<0.75 && m.appearance.x>0) {") + t = replace( + t, + " uint2 originalPixel=uint2(position.xy);", + " uint2 originalPixel=uint2(position.xy-p.originalRegion.xy);", + ) + t = replace( + t, + " uint originalWidth,originalHeight;\n sceneColor.GetDimensions(originalWidth,originalHeight);\n originalPixel+=uint2((sampleIndex&1)*originalWidth,(sampleIndex>>1)*originalHeight);", + " originalPixel+=uint2((sampleIndex&1)*uint(p.originalRegion.z),(sampleIndex>>1)*uint(p.originalRegion.w));", + ) return t -change('src/shaders/WaterPS.hlsl',pixel) -output.write_text(''.join(changes),newline='\n') -print(f'Authored {output}; no dependency/stage files changed.') + + +change("src/shaders/WaterPS.hlsl", pixel) +output.write_text("".join(changes), newline="\n") +print(f"Authored {output}; no dependency/stage files changed.") diff --git a/scripts/compose-water-views.py b/scripts/compose-water-views.py index 97adc09a..e19b2ae7 100644 --- a/scripts/compose-water-views.py +++ b/scripts/compose-water-views.py @@ -2,45 +2,100 @@ No dependency or generated stage file is changed. Output is a pipeline patch. """ -from pathlib import Path + import difflib +from pathlib import Path import sys + from water_patch_baseline import read_before -root=Path(__file__).resolve().parents[1] -stage=Path(sys.argv[1]).resolve() -directory=root/'patches/enhanced-water' -layers=[] -if 'waterRayGeometry' in (stage/'src/hle/rt64_state.h').read_text():layers.append(directory/'dkr-rays.patch') -if 'WATER_CRAFT_COUNT' in (stage/'src/shared/rt64_water_params.h').read_text():layers.append(directory/'dkr-views.patch') -changes=[] -def replace(text,old,new): - if text.count(old)!=1:raise SystemExit(f'Non-unique view context: {old[:100]}') - return text.replace(old,new,1) -def change(path,fn): - before=read_before(stage,path,layers);after=fn(before) - if before==after:raise SystemExit(f'No view changes: {path}') - changes.append(f'diff --git a/{path} b/{path}\n') - changes.extend(difflib.unified_diff(before.splitlines(True),after.splitlines(True),fromfile=f'a/{path}',tofile=f'b/{path}',n=3)) +root = Path(__file__).resolve().parents[1] +stage = Path(sys.argv[1]).resolve() +directory = root / "patches/enhanced-water" +layers = [] +if "waterRayGeometry" in (stage / "src/hle/rt64_state.h").read_text(): + layers.append(directory / "dkr-rays.patch") +if "WATER_CRAFT_COUNT" in (stage / "src/shared/rt64_water_params.h").read_text(): + layers.append(directory / "dkr-views.patch") +changes = [] + + +def replace(text, old, new): + if text.count(old) != 1: + raise SystemExit(f"Non-unique view context: {old[:100]}") + return text.replace(old, new, 1) + + +def change(path, fn): + before = read_before(stage, path, layers) + after = fn(before) + if before == after: + raise SystemExit(f"No view changes: {path}") + changes.append(f"diff --git a/{path} b/{path}\n") + changes.extend( + difflib.unified_diff( + before.splitlines(True), after.splitlines(True), fromfile=f"a/{path}", tofile=f"b/{path}", n=3 + ) + ) + def params(t): - t=replace(t,'#define WATER_SHORE_SEGMENTS_PER_TILE 20','#define WATER_CRAFT_COUNT 8\n#define WATER_SHORE_SEGMENTS_PER_TILE 8') - t=t.replace('craftPrevious[4]','craftPrevious[WATER_CRAFT_COUNT]').replace('craftCurrent[4]','craftCurrent[WATER_CRAFT_COUNT]') - t=t.replace('float4 shoreCounts;','float4 shoreCounts[2];').replace('[4 * WATER_SHORE_SEGMENTS_PER_TILE]','[WATER_CRAFT_COUNT * WATER_SHORE_SEGMENTS_PER_TILE]') - t=t.replace('fieldOrigins[4]','fieldOrigins[WATER_CRAFT_COUNT]').replace('origins[4]','origins[WATER_CRAFT_COUNT]') - t=t.replace('== 1664','== 1552').replace('== 1984','== 1936').replace('== 96','== 160') + t = replace( + t, + "#define WATER_SHORE_SEGMENTS_PER_TILE 20", + "#define WATER_CRAFT_COUNT 8\n#define WATER_SHORE_SEGMENTS_PER_TILE 8", + ) + t = t.replace("craftPrevious[4]", "craftPrevious[WATER_CRAFT_COUNT]").replace( + "craftCurrent[4]", "craftCurrent[WATER_CRAFT_COUNT]" + ) + t = t.replace("float4 shoreCounts;", "float4 shoreCounts[2];").replace( + "[4 * WATER_SHORE_SEGMENTS_PER_TILE]", "[WATER_CRAFT_COUNT * WATER_SHORE_SEGMENTS_PER_TILE]" + ) + t = t.replace("fieldOrigins[4]", "fieldOrigins[WATER_CRAFT_COUNT]").replace( + "origins[4]", "origins[WATER_CRAFT_COUNT]" + ) + t = t.replace("== 1664", "== 1552").replace("== 1984", "== 1936").replace("== 96", "== 160") return t -change('src/shared/rt64_water_params.h',params) + + +change("src/shared/rt64_water_params.h", params) + def header(t): - t=t.replace('std::array sprayDistance','std::array sprayDistance').replace('std::array sprayHeading','std::array sprayHeading').replace('std::array fieldOrigins','std::array fieldOrigins') - t=replace(t,' bool captured = false;',' bool captured = false;\n interop::float4 capturedView{};\n uint32_t capturedCamera=UINT32_MAX;') - return replace(t,'std::map,uint32_t> sceneKeys;','std::map,uint32_t> sceneKeys;') -change('src/render/rt64_water_renderer.h',header) + t = ( + t.replace("std::array sprayDistance", "std::array sprayDistance") + .replace( + "std::array sprayHeading", "std::array sprayHeading" + ) + .replace( + "std::array fieldOrigins", "std::array fieldOrigins" + ) + ) + t = replace( + t, + " bool captured = false;", + " bool captured = false;\n interop::float4 capturedView{};\n uint32_t capturedCamera=UINT32_MAX;", + ) + return replace( + t, + "std::map,uint32_t> sceneKeys;", + "std::map,uint32_t> sceneKeys;", + ) + + +change("src/render/rt64_water_renderer.h", header) + def renderer(t): - t=replace(t,' const auto sceneKey=std::make_pair(key,selfDepth);',' const auto sceneKey=std::make_tuple(key,classic,selfDepth);') - t=replace(t,' if (scene.captured) return false;',''' const auto &draw=draws[drawIndex]; + t = replace( + t, + " const auto sceneKey=std::make_pair(key,selfDepth);", + " const auto sceneKey=std::make_tuple(key,classic,selfDepth);", + ) + t = replace( + t, + " if (scene.captured) return false;", + """ const auto &draw=draws[drawIndex]; const uint32_t camera=uint32_t(draw.material.effects.y); const auto &view=draw.viewBounds; if(scene.captured && scene.capturedCamera==camera && @@ -50,20 +105,49 @@ def renderer(t): // Earlier view commands have already consumed it; all sampling stays // inside the corresponding view bounds. Four views do not need four // full-window colour/depth copies resident simultaneously. - scene.capturedCamera=camera;scene.capturedView=view;''') - t=replace(t,' const auto sourceDraw=draws.front();',''' const auto horizontal=std::find_if(draws.begin(),draws.end(),[](const auto& d){return d.material.effects.z<3;}); - const auto sourceDraw=horizontal==draws.end() ? draws.front() : *horizontal;''') - t=replace(t,' const uint32_t resolution=m.identity.x>=3 ? 512 : 256;',''' const bool active=horizontal!=draws.end() && std::any_of(std::begin(m.craftCurrent),std::end(m.craftCurrent),[](const auto& c){return c.w>0;}); - const uint32_t resolution=active ? (m.identity.x>=3 ? 512 : 256) : 1;''') - t=replace(t,' const float desired=draws.front().camera.w;',' const float desired=sourceDraw.camera.w;') - t=replace(t,' if (reset) fieldInputs.clear();',' if (reset || !active) fieldInputs.clear();\n if(!active)stepSpray(m,fieldTime,desired,true);') - t=replace(t,' while (fieldTime+0.00001f=3 ? 512 : 256;", + """ const bool active=horizontal!=draws.end() && std::any_of(std::begin(m.craftCurrent),std::end(m.craftCurrent),[](const auto& c){return c.w>0;}); + const uint32_t resolution=active ? (m.identity.x>=3 ? 512 : 256) : 1;""", + ) + t = replace(t, " const float desired=draws.front().camera.w;", " const float desired=sourceDraw.camera.w;") + t = replace( + t, + " if (reset) fieldInputs.clear();", + " if (reset || !active) fieldInputs.clear();\n if(!active)stepSpray(m,fieldTime,desired,true);", + ) + t = replace(t, " while (fieldTime+0.00001fdrawScenes[activeWaterDraw]) finishWater();', ''' if(activeWaterDraw!=UINT32_MAX) { + t = replace( + t, + "(uint64_t(framebufferCount) << 32) | proj.transformsIndex,classic,selfDepth);", + "uint64_t(framebufferCount),classic,selfDepth);", + ) + return replace( + t, + " if (activeWaterDraw!=UINT32_MAX && waterScene!=waterRenderer->drawScenes[activeWaterDraw]) finishWater();", + """ if(activeWaterDraw!=UINT32_MAX) { bool changed=waterScene!=waterRenderer->drawScenes[activeWaterDraw]; if(!changed && drawCall.waterIndex!=UINT32_MAX) { const auto &a=waterRenderer->draws[activeWaterDraw],&b=waterRenderer->draws[drawCall.waterIndex]; @@ -71,17 +155,31 @@ def framebuffer(t): a.viewBounds.y!=b.viewBounds.y || a.viewBounds.z!=b.viewBounds.z || a.viewBounds.w!=b.viewBounds.w; } if(changed)finishWater(); - }''') -change('src/render/rt64_framebuffer_renderer.cpp',framebuffer) + }""", + ) + + +change("src/render/rt64_framebuffer_renderer.cpp", framebuffer) + def field(t): - t=t.replace('resolution*4','resolution*WATER_CRAFT_COUNT').replace('tile>=4','tile>=WATER_CRAFT_COUNT') - t=t.replace('i<4;i++','i=4", "tile>=WATER_CRAFT_COUNT") + t = t.replace("i<4;i++", "i0.5; @@ -44,21 +63,36 @@ def raster(t): combinerColor.rgb=waterfallComposite(combinerColor.rgb, flow,anchored ? waterfall.camera.w : 0,direction,vertexColor.rgb,footprint, waterfall.material.appearance.w); -#endif''') - t=replace(t,' , in float2 vertexUV : TEXCOORD',''' , in float2 vertexUV : TEXCOORD +#endif""", + ) + t = replace( + t, + " , in float2 vertexUV : TEXCOORD", + """ , in float2 vertexUV : TEXCOORD #ifdef DKR_WATERFALL , in float4 waterfallFlow : TEXCOORD1 -#endif''') - t=replace(t,'isFrontFace, resultColor, resultAlpha)) {','''isFrontFace, resultColor, resultAlpha +#endif""", + ) + t = replace( + t, + "isFrontFace, resultColor, resultAlpha)) {", + """isFrontFace, resultColor, resultAlpha #ifdef DKR_WATERFALL , waterfallFlow #endif - )) {''') - return t.replace(' ) \n',' )\n') -change('src/shaders/RasterPS.hlsl',raster,True) + )) {""", + ) + return t.replace(" ) \n", " )\n") + + +change("src/shaders/RasterPS.hlsl", raster, True) + def header(t): - t=replace(t,' std::unique_ptr layout;', ''' std::unique_ptr layout; + t = replace( + t, + " std::unique_ptr layout;", + """ std::unique_ptr layout; struct WaterfallSet : RenderDescriptorSetBase { uint32_t draws; explicit WaterfallSet(RenderDevice *device=nullptr) { @@ -74,12 +108,18 @@ def header(t): BufferPair waterfallFlowBuffer; RenderVertexBufferView waterfallFlowView; void collectWaterfall(const DrawData& data,uint32_t start,uint32_t count); - RenderPipeline *waterfallPipeline(RenderDevice *device,const RenderMultisampling &samples,bool zCompare,bool zWrite,bool coverageAdd);''') - return replace(t,'WaterRenderer(RenderDevice *device, RenderShaderFormat format, bool usesHDR);', - 'WaterRenderer(RenderDevice *device, RenderShaderFormat format, bool usesHDR,const ShaderLibrary *library);') -change('src/render/rt64_water_renderer.h',header) + RenderPipeline *waterfallPipeline(RenderDevice *device,const RenderMultisampling &samples,bool zCompare,bool zWrite,bool coverageAdd);""", + ) + return replace( + t, + "WaterRenderer(RenderDevice *device, RenderShaderFormat format, bool usesHDR);", + "WaterRenderer(RenderDevice *device, RenderShaderFormat format, bool usesHDR,const ShaderLibrary *library);", + ) + -pipeline=r''' +change("src/render/rt64_water_renderer.h", header) + +pipeline = r""" RenderPipeline *WaterRenderer::waterfallPipeline(RenderDevice *device,const RenderMultisampling &samples,bool zCompare,bool zWrite,bool coverageAdd) { const uint32_t key=samples.sampleCount|(zCompare ? 16 : 0)|(zWrite ? 32 : 0)|(coverageAdd ? 64 : 0); if(auto found=waterfallPipelines.find(key);found!=waterfallPipelines.end())return found->second.get(); @@ -115,15 +155,33 @@ def header(t): // remains visible and unrelated horizontal-water effects stay enabled. auto *pointer=result.get();waterfallPipelines.emplace(key,std::move(result));return pointer; } -''' +""" + + def renderer(t): - for suffix in ('spirv','dxil'): - t=replace(t,f'#include "shaders/WaterVS.hlsl.{suffix}.h"', - '\n'.join(f'#include "shaders/{name}.hlsl.{suffix}.h"' for name in ('WaterVS','WaterfallVS','WaterfallPS','WaterfallMSPS'))) - t=replace(t,'WaterRenderer::WaterRenderer(RenderDevice *device, RenderShaderFormat format, bool hdr) : usesHDR(hdr)', - 'WaterRenderer::WaterRenderer(RenderDevice *device, RenderShaderFormat format, bool hdr,const ShaderLibrary *library) : usesHDR(hdr)') - t=replace(t,'shared-optics/compact-vertical scratch policy active','shared-optics/direct-waterfall scratch policy active') - t=replace(t,' descriptorSet = std::make_unique(device);', ''' waterfallSet=std::make_unique(device); + for suffix in ("spirv", "dxil"): + t = replace( + t, + f'#include "shaders/WaterVS.hlsl.{suffix}.h"', + "\n".join( + f'#include "shaders/{name}.hlsl.{suffix}.h"' + for name in ("WaterVS", "WaterfallVS", "WaterfallPS", "WaterfallMSPS") + ), + ) + t = replace( + t, + "WaterRenderer::WaterRenderer(RenderDevice *device, RenderShaderFormat format, bool hdr) : usesHDR(hdr)", + "WaterRenderer::WaterRenderer(RenderDevice *device, RenderShaderFormat format, bool hdr,const ShaderLibrary *library) : usesHDR(hdr)", + ) + t = replace( + t, + "shared-optics/compact-vertical scratch policy active", + "shared-optics/direct-waterfall scratch policy active", + ) + t = replace( + t, + " descriptorSet = std::make_unique(device);", + """ waterfallSet=std::make_unique(device); FramebufferRendererDescriptorCommonSet common(library->samplerLibrary,device->getCapabilities().raytracing); FramebufferRendererDescriptorTextureSet textures; FramebufferRendererDescriptorFramebufferSet framebuffer; @@ -137,19 +195,40 @@ def renderer(t): waterfallVS=checkedWaterResource(device->createShader(CREATE_SHADER_INPUTS(WaterfallVSBlobDXIL,WaterfallVSBlobSPIRV,WaterfallVSBlobMSL,"VSMain",format))); waterfallPS=checkedWaterResource(device->createShader(CREATE_SHADER_INPUTS(WaterfallPSBlobDXIL,WaterfallPSBlobSPIRV,WaterfallPSBlobMSL,"PSMain",format))); waterfallMSPS=checkedWaterResource(device->createShader(CREATE_SHADER_INPUTS(WaterfallMSPSBlobDXIL,WaterfallMSPSBlobSPIRV,WaterfallMSPSBlobMSL,"PSMain",format))); - descriptorSet = std::make_unique(device);''') - t=replace(t,'#include "rt64_render_target.h"','#include "rt64_render_target.h"\n#include "rt64_descriptor_sets.h"\n#include "shared/rt64_raster_params.h"\n#include "WaterPipelineSafety.h"') - t=replace(t,' auto result = device->createGraphicsPipeline(desc);', ''' auto result = device->createGraphicsPipeline(desc); - if(!waterGraphicsPipelineValid(result.get()))result.reset();''') - t=replace(t,'void WaterRenderer::bindBuffer() {','''void WaterRenderer::bindBuffer() { + descriptorSet = std::make_unique(device);""", + ) + t = replace( + t, + '#include "rt64_render_target.h"', + '#include "rt64_render_target.h"\n#include "rt64_descriptor_sets.h"\n#include "shared/rt64_raster_params.h"\n#include "WaterPipelineSafety.h"', + ) + t = replace( + t, + " auto result = device->createGraphicsPipeline(desc);", + """ auto result = device->createGraphicsPipeline(desc); + if(!waterGraphicsPipelineValid(result.get()))result.reset();""", + ) + t = replace( + t, + "void WaterRenderer::bindBuffer() {", + """void WaterRenderer::bindBuffer() { waterfallSet->setBuffer(waterfallSet->draws,drawBuffer.get(),RenderBufferStructuredView(sizeof(interop::WaterDrawParams))); - waterfallFlowView=RenderVertexBufferView(waterfallFlowBuffer.get()->at(0),waterfallFlow.size()*sizeof(interop::float4));''') - t=replace(t,' draws.clear(); normalGroups.clear(); sceneKeys.clear(); drawScenes.clear(); rejectedScenes.clear();', - ' draws.clear(); normalGroups.clear(); sceneKeys.clear(); drawScenes.clear(); rejectedScenes.clear();\n waterfallFlow.clear();') - t=replace(t,'drawBuffer.allocatedSize+normalVertexBuffer.allocatedSize+normalFaceBuffer.allocatedSize+wireBuffer.allocatedSize+particleBuffer.allocatedSize', - 'drawBuffer.allocatedSize+normalVertexBuffer.allocatedSize+normalFaceBuffer.allocatedSize+wireBuffer.allocatedSize+particleBuffer.allocatedSize+waterfallFlowBuffer.allocatedSize') - t=replace(t,'void WaterRenderer::prepareNormals(RenderWorker *worker, const OutputBuffers &outputs, std::vector &uploads) {', - '''void WaterRenderer::collectWaterfall(const DrawData& data,uint32_t start,uint32_t count) { + waterfallFlowView=RenderVertexBufferView(waterfallFlowBuffer.get()->at(0),waterfallFlow.size()*sizeof(interop::float4));""", + ) + t = replace( + t, + " draws.clear(); normalGroups.clear(); sceneKeys.clear(); drawScenes.clear(); rejectedScenes.clear();", + " draws.clear(); normalGroups.clear(); sceneKeys.clear(); drawScenes.clear(); rejectedScenes.clear();\n waterfallFlow.clear();", + ) + t = replace( + t, + "drawBuffer.allocatedSize+normalVertexBuffer.allocatedSize+normalFaceBuffer.allocatedSize+wireBuffer.allocatedSize+particleBuffer.allocatedSize", + "drawBuffer.allocatedSize+normalVertexBuffer.allocatedSize+normalFaceBuffer.allocatedSize+wireBuffer.allocatedSize+particleBuffer.allocatedSize+waterfallFlowBuffer.allocatedSize", + ) + t = replace( + t, + "void WaterRenderer::prepareNormals(RenderWorker *worker, const OutputBuffers &outputs, std::vector &uploads) {", + """void WaterRenderer::collectWaterfall(const DrawData& data,uint32_t start,uint32_t count) { // Same indexed vertex IDs as the raster streams. Missing/object metadata // uses the retail UV stream, never a guessed world-axis projection. waterfallFlow.resize(data.vertexCount(),interop::float4(0,0,0,0)); @@ -163,30 +242,46 @@ def renderer(t): void WaterRenderer::prepareNormals(RenderWorker *worker, const OutputBuffers &outputs, std::vector &uploads) { if(waterfallFlow.empty())waterfallFlow.emplace_back(0,0,0,0); - uploads.push_back({waterfallFlow.data(),{0,waterfallFlow.size()},sizeof(interop::float4),RenderBufferFlag::VERTEX,{},&waterfallFlowBuffer});''') - a=t.index(' uint32_t ow=classic ? water_policy::bucketExtent') - b=t.index(' const auto requested=',a) - t=t[:a]+' const uint32_t ow=0,oh=0; // Direct waterfall shader: no framebuffer atlas.\n'+t[b:] - a=t.index(' if(classic && (!scene.originalColor') - b=t.index(' if(selfDepth &&',a) - t=t[:a]+t[b:] - a=t.index(' if(classic) {\n scene.originalSet->setTexture') - b=t.index(' ms->setTexture(ms->sceneColor',a) - t=t[:a]+''' auto& ms=scene.materialSet; + uploads.push_back({waterfallFlow.data(),{0,waterfallFlow.size()},sizeof(interop::float4),RenderBufferFlag::VERTEX,{},&waterfallFlowBuffer});""", + ) + a = t.index(" uint32_t ow=classic ? water_policy::bucketExtent") + b = t.index(" const auto requested=", a) + t = t[:a] + " const uint32_t ow=0,oh=0; // Direct waterfall shader: no framebuffer atlas.\n" + t[b:] + a = t.index(" if(classic && (!scene.originalColor") + b = t.index(" if(selfDepth &&", a) + t = t[:a] + t[b:] + a = t.index(" if(classic) {\n scene.originalSet->setTexture") + b = t.index(" ms->setTexture(ms->sceneColor", a) + t = ( + t[:a] + + """ auto& ms=scene.materialSet; ms->setTexture(ms->originalColor,scene.color.get(),RenderTextureLayout::SHADER_READ,scene.colorView.get()); -'''+t[b:] - closing=t.rfind('\n}') - return t[:closing]+pipeline+t[closing:] -change('src/render/rt64_water_renderer.cpp',renderer) +""" + + t[b:] + ) + closing = t.rfind("\n}") + return t[:closing] + pipeline + t[closing:] + + +change("src/render/rt64_water_renderer.cpp", renderer) + def framebuffer(t): - t=replace(t,'p.rasterShaderCache->shaderFormat, shaderLibrary->usesHDR);', - 'p.rasterShaderCache->shaderFormat, shaderLibrary->usesHDR,shaderLibrary);') - t=replace(t,'const bool wantsWater = waterMaterial && waterMaterial->identity.x > 0.0f &&', - 'const bool wantsWater = waterMaterial && waterMaterial->identity.x > 0.0f &&\n (waterMaterial->optics.w<0.5f || waterMaterial->appearance.w>0.0f) &&') - a=t.index(' // Use the original texture combiner') - b=t.index(' const bool hardware=',a) - t=t[:a]+''' // Single retail-compatible draw. No composite capture, full-screen + t = replace( + t, + "p.rasterShaderCache->shaderFormat, shaderLibrary->usesHDR);", + "p.rasterShaderCache->shaderFormat, shaderLibrary->usesHDR,shaderLibrary);", + ) + t = replace( + t, + "const bool wantsWater = waterMaterial && waterMaterial->identity.x > 0.0f &&", + "const bool wantsWater = waterMaterial && waterMaterial->identity.x > 0.0f &&\n (waterMaterial->optics.w<0.5f || waterMaterial->appearance.w>0.0f) &&", + ) + a = t.index(" // Use the original texture combiner") + b = t.index(" const bool hardware=", a) + t = ( + t[:a] + + """ // Single retail-compatible draw. No composite capture, full-screen // allocation or second shading pass for vertical water. worker->commandList->setGraphicsPipelineLayout(waterRenderer->waterfallLayout.get()); worker->commandList->setGraphicsDescriptorSet(descCommonSet->get(),0); @@ -206,54 +301,99 @@ def framebuffer(t): rasterParams.padding=0;activeWaterDraw=drawCall.waterIndex; switchToGraphicsPipeline();break; } -'''+t[b:] - t=replace(t,' waterPipeline = waterRenderer->pipeline(p.worker->device,p.fbStorage->colorTarget->multisampling,', - ''' waterPipeline = classic ? waterRenderer->waterfallPipeline(p.worker->device,p.fbStorage->colorTarget->multisampling, +""" + + t[b:] + ) + t = replace( + t, + " waterPipeline = waterRenderer->pipeline(p.worker->device,p.fbStorage->colorTarget->multisampling,", + """ waterPipeline = classic ? waterRenderer->waterfallPipeline(p.worker->device,p.fbStorage->colorTarget->multisampling, call.callDesc.otherMode.zCmp(),call.callDesc.otherMode.zUpd(),call.callDesc.otherMode.cvgDst()==CVG_DST_WRAP) : - waterRenderer->pipeline(p.worker->device,p.fbStorage->colorTarget->multisampling,''') - t=replace(t,'waterMatricesValid = waterPipeline && waterRenderer->pipeline(', - 'waterMatricesValid = waterPipeline && (classic || waterRenderer->pipeline(') - t=replace(t,'selfDepth,false,true,classic,selfDepth);','selfDepth,false,true,classic,selfDepth));') - a=t.index(' const auto originalRegion=classic ? waterOriginalRegion(') - b=t.index(' const auto waterScene=',a) - t=t[:a]+' const auto originalRegion=water_work::Region{};\n'+t[b:] - t=replace(t,' waterRenderer->collectNormals(drawData,',''' if(classic)waterRenderer->collectWaterfall(drawData,call.meshDesc.faceIndicesStart,call.callDesc.triangleCount*3); - else waterRenderer->collectNormals(drawData,''') + waterRenderer->pipeline(p.worker->device,p.fbStorage->colorTarget->multisampling,""", + ) + t = replace( + t, + "waterMatricesValid = waterPipeline && waterRenderer->pipeline(", + "waterMatricesValid = waterPipeline && (classic || waterRenderer->pipeline(", + ) + t = replace(t, "selfDepth,false,true,classic,selfDepth);", "selfDepth,false,true,classic,selfDepth));") + a = t.index(" const auto originalRegion=classic ? waterOriginalRegion(") + b = t.index(" const auto waterScene=", a) + t = t[:a] + " const auto originalRegion=water_work::Region{};\n" + t[b:] + t = replace( + t, + " waterRenderer->collectNormals(drawData,", + """ if(classic)waterRenderer->collectWaterfall(drawData,call.meshDesc.faceIndicesStart,call.callDesc.triangleCount*3); + else waterRenderer->collectNormals(drawData,""", + ) return t -change('src/render/rt64_framebuffer_renderer.cpp',framebuffer) + + +change("src/render/rt64_framebuffer_renderer.cpp", framebuffer) + def workload_header(t): - t=replace(t,'#include "rt64_transform_group.h"','#include "rt64_transform_group.h"\n#include ') - return replace(t,' std::vector tcVelFloats;', ''' std::vector tcVelFloats; + t = replace(t, '#include "rt64_transform_group.h"', '#include "rt64_transform_group.h"\n#include ') + return replace( + t, + " std::vector tcVelFloats;", + """ std::vector tcVelFloats; // Renderer-only metadata, immutable for this submitted task. Does not // change generic group matching or guest/netplay state. std::map waterfallPeriods; - std::map waterfallFlow;''') -change('src/hle/rt64_workload.h',workload_header) -change('src/hle/rt64_workload.cpp',lambda t:replace(t,' drawData.tcVelFloats.clear();', - ' drawData.tcVelFloats.clear();\n drawData.waterfallPeriods.clear();\n drawData.waterfallFlow.clear();')) + std::map waterfallFlow;""", + ) + + +change("src/hle/rt64_workload.h", workload_header) +change( + "src/hle/rt64_workload.cpp", + lambda t: replace( + t, + " drawData.tcVelFloats.clear();", + " drawData.tcVelFloats.clear();\n drawData.waterfallPeriods.clear();\n drawData.waterfallFlow.clear();", + ), +) + def rsp(t): - return replace(t,' // Indicates the vertex has been used in a tri.', ''' if(state->waterMaterial.identity.x>0 && state->waterMaterial.optics.w>0.5f) { + return replace( + t, + " // Indicates the vertex has been used in a tri.", + """ if(state->waterMaterial.identity.x>0 && state->waterMaterial.optics.w>0.5f) { const auto& optics=state->waterMaterial.optics; if(std::isfinite(optics.y) && std::isfinite(optics.z) && optics.y>=1 && optics.z>=1 && optics.y<=2048 && optics.z<=2048) for(unsigned i=0;i<3;i++)workload.drawData.waterfallPeriods[globalIndices[i]]={optics.y,optics.z}; } - // Indicates the vertex has been used in a tri.''') -change('src/hle/rt64_rsp.cpp',rsp) + // Indicates the vertex has been used in a tri.""", + ) + + +change("src/hle/rt64_rsp.cpp", rsp) + def frame(t): - t='#include "waterfall_scroll_policy.hpp"\n'+t - t=replace(t,' if ((curGroup.texcoordInterpolation != G_EX_COMPONENT_SKIP) && (curVertexCount == prevVertexCount)) {', ''' const auto& currentPeriods=curWorkload.drawData.waterfallPeriods; + t = '#include "waterfall_scroll_policy.hpp"\n' + t + t = replace( + t, + " if ((curGroup.texcoordInterpolation != G_EX_COMPONENT_SKIP) && (curVertexCount == prevVertexCount)) {", + """ const auto& currentPeriods=curWorkload.drawData.waterfallPeriods; const auto& previousPeriods=prevWorkload.drawData.waterfallPeriods; const auto firstWaterfall=currentPeriods.lower_bound(curVertexIndex); const bool verticalUV=firstWaterfall!=currentPeriods.end() && firstWaterfall->first WrappingModuloHalf[0]) {', ''' if(waterfall) { + t = t.replace("/ WrappingModulo[0]) * WrappingModulo[0]", "/ period[0]) * period[0]") + t = t.replace("/ WrappingModulo[1]) * WrappingModulo[1]", "/ period[1]) * period[1]") + t = replace( + t, + " if (fabsf(curVelFloatsRef[i * 2]) > WrappingModuloHalf[0]) {", + """ if(waterfall) { curVelFloatsRef[i*2]=dkr::runtime::water::waterfall_uv_velocity(prevTcFloatsRef[i*2],curTcFloatsRef[i*2],period[0]); curVelFloatsRef[i*2+1]=dkr::runtime::water::waterfall_uv_velocity(prevTcFloatsRef[i*2+1],curTcFloatsRef[i*2+1],period[1]); } - if (fabsf(curVelFloatsRef[i * 2]) > WrappingModuloHalf[0]) {''') + if (fabsf(curVelFloatsRef[i * 2]) > WrappingModuloHalf[0]) {""", + ) return t -change('src/hle/rt64_game_frame.cpp',frame) + + +change("src/hle/rt64_game_frame.cpp", frame) + def pixel(t): - a=t.index(' float4 reflection=screenReflection(') - b=t.index(' float3 reflected=',a) - return t[:a]+''' float4 reflection=0; + a = t.index(" float4 reflection=screenReflection(") + b = t.index(" float3 reflected=", a) + return ( + t[:a] + + """ float4 reflection=0; #ifdef HARDWARE_WATER bool reflectedWaterfall=false,capturedHit=false; float4 hardwareReflection=hardwareWaterReflection(worldPosition,N,V,p,reflectedWaterfall,capturedHit); @@ -291,8 +441,12 @@ def pixel(t): #else reflection=screenReflection(worldPosition,N,V,roughness,p); #endif -'''+t[b:] -change('src/shaders/WaterPS.hlsl',pixel) +""" + + t[b:] + ) + + +change("src/shaders/WaterPS.hlsl", pixel) -output.write_text(''.join(changes),newline='\n') -print(f'Authored {output} ({len(changes)} diff records)') +output.write_text("".join(changes), newline="\n") +print(f"Authored {output} ({len(changes)} diff records)") diff --git a/scripts/compose_legacy_mod_policy.py b/scripts/compose_legacy_mod_policy.py index d5fc5199..ae2a64e1 100644 --- a/scripts/compose_legacy_mod_policy.py +++ b/scripts/compose_legacy_mod_policy.py @@ -5,6 +5,7 @@ retained; the bridge runs immediately after it, before the caller reuses a0. Ordinary PI calls must still dispatch through osPiStartDma_recomp unchanged. """ + from __future__ import annotations import argparse @@ -15,26 +16,56 @@ import struct MENU_FIELDS = [ - ("gTrackSelectCursorX", 4), ("gTrackSelectCursorY", 4), - ("gTrackSelectX", 4), ("gTrackSelectY", 4), - ("gTrackSelectTargetX", 4), ("gTrackSelectTargetY", 4), - ("gSelectedTrackX", 4), ("gSelectedTrackY", 4), - ("gTrackSelectViewportY", 4), ("gTrackSelectViewPortHalfY", 4), - ("gTrackIdForPreview", 4), ("gTrackmenuLoadedLevel", 4), - ("gOpacityDecayTimer", 4), ("gFFLUnlocked", 2), - ("gTrackSelectRenderDetails", 144), ("gMenuDelay", 4), - ("gMenuButtons", 20), ("gMenuStickX", 10), ("gMenuStickY", 10), - ("gTrackSelectRow", 4), ("gTrackSelectIDs", 48), - ("gTrackNameVoiceDelay", 4), ("gTrackmenuType", 4), - ("gIsInTracksMenu", 4), ("gTrackIdToLoad", 4), - ("gThread30NeedToLoadLevel", 4), ("gTitleScreenLoaded", 4), + ("gTrackSelectCursorX", 4), + ("gTrackSelectCursorY", 4), + ("gTrackSelectX", 4), + ("gTrackSelectY", 4), + ("gTrackSelectTargetX", 4), + ("gTrackSelectTargetY", 4), + ("gSelectedTrackX", 4), + ("gSelectedTrackY", 4), + ("gTrackSelectViewportY", 4), + ("gTrackSelectViewPortHalfY", 4), + ("gTrackIdForPreview", 4), + ("gTrackmenuLoadedLevel", 4), + ("gOpacityDecayTimer", 4), + ("gFFLUnlocked", 2), + ("gTrackSelectRenderDetails", 144), + ("gMenuDelay", 4), + ("gMenuButtons", 20), + ("gMenuStickX", 10), + ("gMenuStickY", 10), + ("gTrackSelectRow", 4), + ("gTrackSelectIDs", 48), + ("gTrackNameVoiceDelay", 4), + ("gTrackmenuType", 4), + ("gIsInTracksMenu", 4), + ("gTrackIdToLoad", 4), + ("gThread30NeedToLoadLevel", 4), + ("gTitleScreenLoaded", 4), ("sMenuCurrDisplayList", 4), ("gTrackSelectVertsFlip", 4), ] -MENU_EVENTS = ["menu_track_select_init", "menu_track_select_init", "trackmenu_input", "trackmenu_input", - "trackmenu_render_names", "trackmenu_track_view", "bgload_start", "bgload_start", - "load_level_for_menu", "level_name", "leveltable_vehicle_default", "leveltable_vehicle_usable", - "trackmenu_assets", "func_80092188", "load_level_game", "menu_init", "func_8008F618", "func_8008F618"] +MENU_EVENTS = [ + "menu_track_select_init", + "menu_track_select_init", + "trackmenu_input", + "trackmenu_input", + "trackmenu_render_names", + "trackmenu_track_view", + "bgload_start", + "bgload_start", + "load_level_for_menu", + "level_name", + "leveltable_vehicle_default", + "leveltable_vehicle_usable", + "trackmenu_assets", + "func_80092188", + "load_level_game", + "menu_init", + "func_8008F618", + "func_8008F618", +] def elf_sections(path: Path): @@ -51,7 +82,7 @@ def elf_sections(path: Path): if kind == 1 and flags & 4: if offset + size > len(data) or size % 4 or address % 4: raise ValueError("Invalid executable ELF section") - sections.append((address, data[offset:offset + size])) + sections.append((address, data[offset : offset + size])) return sections @@ -74,7 +105,7 @@ def elf_functions(path: Path, symbol_types=(2,)) -> dict: strings_at, strings_size = strings_header[4:6] if strings_header[1] != 3 or strings_at + strings_size > len(data): raise ValueError("Invalid ELF symbol string table") - strings = data[strings_at:strings_at + strings_size] + strings = data[strings_at : strings_at + strings_size] for at in range(offset, offset + size, stride): name, address, length, info, _, _ = struct.unpack_from(">IIIBBH", data, at) if info & 15 not in symbol_types or not address or not length: @@ -91,11 +122,13 @@ def verify_function_bounds(policy: dict, fragment: dict, functions: dict) -> Non functions = copy.deepcopy(functions) for manual in policy.get("manualFunctions", []): functions[manual["name"]] = {(int(manual["vram"], 0), int(manual["size"], 0))} + def bounds(name): entries = functions.get(name, set()) if len(entries) != 1: raise ValueError(f"Missing or ambiguous function bounds: {name}") return next(iter(entries)) + if bounds(fragment["callTarget"])[0] != int(fragment["callTargetVram"], 0): raise ValueError("PI target does not match its declared ELF symbol") for site in fragment["sites"] + fragment["assetApis"]: @@ -115,8 +148,9 @@ def compose(policy: dict, fragment: dict, sections: list) -> dict: result = copy.deepcopy(policy) patches = result.setdefault("instructionPatches", []) hooks = result.setdefault("functionHooks", []) - words = {start + i: struct.unpack_from(">I", data, i)[0] - for start, data in sections for i in range(0, len(data), 4)} + words = { + start + i: struct.unpack_from(">I", data, i)[0] for start, data in sections for i in range(0, len(data), 4) + } target = int(fragment["callTargetVram"], 0) jal = 0x0C000000 | ((target >> 2) & 0x03FFFFFF) sites = fragment["sites"] @@ -150,15 +184,30 @@ def compose(policy: dict, fragment: dict, sections: list) -> dict: for entry in hooks: if int(entry["beforeVram"], 0) in (address, address + 4, address + 8): raise ValueError("Legacy PI bridge conflicts with an existing hook") - patches.append({ - "function": site["function"], "vram": f"0x{address:08X}", "value": "0x00000000", - "reason": "Route the checked PI JAL through the project asset bus; retain the original delay-slot instruction."}) - hooks.append({ - "function": site["function"], "beforeVram": f"0x{address + 8:08X}", - "text": "extern void dkr_legacy_pi_start_dma(uint8_t*, recomp_context*); dkr_legacy_pi_start_dma(rdram, ctx);", - "reason": "Dispatch after the original delay slot has populated the DMA arguments and before a0 is reused. Stock uses the original PI handler."}) - api_names = ["asset_table_load", "asset_load", "asset_rom_offset", "asset_table_size", - "asset_table_load_addr", "asset_table_load_zipped"] + patches.append( + { + "function": site["function"], + "vram": f"0x{address:08X}", + "value": "0x00000000", + "reason": "Route the checked PI JAL through the project asset bus; retain the original delay-slot instruction.", + } + ) + hooks.append( + { + "function": site["function"], + "beforeVram": f"0x{address + 8:08X}", + "text": "extern void dkr_legacy_pi_start_dma(uint8_t*, recomp_context*); dkr_legacy_pi_start_dma(rdram, ctx);", + "reason": "Dispatch after the original delay slot has populated the DMA arguments and before a0 is reused. Stock uses the original PI handler.", + } + ) + api_names = [ + "asset_table_load", + "asset_load", + "asset_rom_offset", + "asset_table_size", + "asset_table_load_addr", + "asset_table_load_zipped", + ] api_sites = fragment.get("assetApis", []) if len(api_sites) != len(api_names): raise ValueError("Incomplete asset API coverage") @@ -170,26 +219,40 @@ def compose(policy: dict, fragment: dict, sections: list) -> dict: expected = [int(word, 0) for word in site["expected"]] if len(expected) != 3 or [words.get(address + i * 4) for i in range(3)] != expected: raise ValueError(f"Asset API entry signature changed: {name}") - owners=[entry for entry in hooks if int(entry["beforeVram"],0)==address] - text="extern int dkr_legacy_asset_api(uint8_t*, recomp_context*, unsigned); " + f"if (dkr_legacy_asset_api(rdram, ctx, {operation}U)) return;" - shared={"asset_table_load":"dkr_custom_tracks_table_load_begin", "asset_load":"dkr_custom_tracks_asset_load_begin"} - callback=shared.get(name) - allowed=f"extern void {callback}(uint8_t*, recomp_context*); {callback}(rdram, ctx);" if callback else None - if any(int(entry["vram"], 0) == address for entry in patches) or (owners and - (len(owners)!=1 or owners[0].get('function')!=name or owners[0].get('text')!=allowed)): + owners = [entry for entry in hooks if int(entry["beforeVram"], 0) == address] + text = ( + "extern int dkr_legacy_asset_api(uint8_t*, recomp_context*, unsigned); " + + f"if (dkr_legacy_asset_api(rdram, ctx, {operation}U)) return;" + ) + shared = { + "asset_table_load": "dkr_custom_tracks_table_load_begin", + "asset_load": "dkr_custom_tracks_asset_load_begin", + } + callback = shared.get(name) + allowed = f"extern void {callback}(uint8_t*, recomp_context*); {callback}(rdram, ctx);" if callback else None + if any(int(entry["vram"], 0) == address for entry in patches) or ( + owners and (len(owners) != 1 or owners[0].get("function") != name or owners[0].get("text") != allowed) + ): raise ValueError(f"Asset API entry conflicts with an existing policy: {name}") if owners: # Mounted legacy reads explicitly run the Blender extension via # the runtime bridge. Unmounted reads retain its retail hooks. - owners[0]['text']=text+' '+owners[0]['text'] - else:hooks.append({"function": name, "beforeVram": f"0x{address:08X}", - "text": text, - "reason": "Resolve a mounted immutable bank at the original function entry; leave the complete retail path intact without a mount."}) + owners[0]["text"] = text + " " + owners[0]["text"] + else: + hooks.append( + { + "function": name, + "beforeVram": f"0x{address:08X}", + "text": text, + "reason": "Resolve a mounted immutable bank at the original function entry; leave the complete retail path intact without a mount.", + } + ) return result -def compose_scene_runtime(policy: dict, fragment: dict, sections: list, functions: dict, - qualification: bool = False) -> dict: +def compose_scene_runtime( + policy: dict, fragment: dict, sections: list, functions: dict, qualification: bool = False +) -> dict: """Compose with, never replace, the existing presentation scene owner.""" result = copy.deepcopy(policy) revision = fragment["revision"] @@ -200,8 +263,9 @@ def compose_scene_runtime(policy: dict, fragment: dict, sections: list, function if revision not in entries: raise ValueError("Unsupported scene lifecycle revision") level, menu, load_word = entries[revision] - words = {start + i: struct.unpack_from(">I", data, i)[0] - for start, data in sections for i in range(0, len(data), 4)} + words = { + start + i: struct.unpack_from(">I", data, i)[0] for start, data in sections for i in range(0, len(data), 4) + } def verify(name, address, expected): bounds = functions.get(name, set()) @@ -212,22 +276,31 @@ def verify(name, address, expected): verify("level_load", level, [0x27BDFFA0, 0xAFBF002C, 0xAFB10028]) owner = [hook for hook in result["functionHooks"] if int(hook["beforeVram"], 0) == level] - expected = ("extern void dkr_runtime_scene_reset(uint8_t*, recomp_context*); " - "extern void dkr_presentation_scene_begin(uint8_t*, recomp_context*); " - "dkr_runtime_scene_reset(rdram, ctx); dkr_presentation_scene_begin(rdram, ctx);") + expected = ( + "extern void dkr_runtime_scene_reset(uint8_t*, recomp_context*); " + "extern void dkr_presentation_scene_begin(uint8_t*, recomp_context*); " + "dkr_runtime_scene_reset(rdram, ctx); dkr_presentation_scene_begin(rdram, ctx);" + ) if len(owner) != 1 or owner[0]["function"] != "level_load" or owner[0]["text"] != expected: raise ValueError("The existing scene owner changed; review lifecycle composition before enabling mods") - owner[0]["text"] += (" extern void dkr_legacy_scene_begin(uint8_t*, recomp_context*); " - "dkr_legacy_scene_begin(rdram, ctx);") + owner[0]["text"] += ( + " extern void dkr_legacy_scene_begin(uint8_t*, recomp_context*); dkr_legacy_scene_begin(rdram, ctx);" + ) owner[0]["reason"] += " Then publish only an explicitly prepared custom asset scene at this same boundary." if qualification: verify("load_level_for_menu", menu, [0x27BDFFE0, 0x3C0E8012, load_word]) if any(int(hook["beforeVram"], 0) == menu for hook in result["functionHooks"]) or any( - int(patch["vram"], 0) == menu for patch in result["instructionPatches"]): + int(patch["vram"], 0) == menu for patch in result["instructionPatches"] + ): raise ValueError("Private menu-load qualification conflicts with an existing hook") - result["functionHooks"].append({"function": "load_level_for_menu", "beforeVram": f"0x{menu:08X}", - "text": "extern void dkr_legacy_qualification_menu_load(uint8_t*, recomp_context*); dkr_legacy_qualification_menu_load(rdram, ctx);", - "reason": "PRIVATE QUALIFICATION ONLY: exercise custom/stock scenes through the real menu/background loader."}) + result["functionHooks"].append( + { + "function": "load_level_for_menu", + "beforeVram": f"0x{menu:08X}", + "text": "extern void dkr_legacy_qualification_menu_load(uint8_t*, recomp_context*); dkr_legacy_qualification_menu_load(rdram, ctx);", + "reason": "PRIVATE QUALIFICATION ONLY: exercise custom/stock scenes through the real menu/background loader.", + } + ) return result @@ -235,8 +308,9 @@ def compose_track_menu(policy: dict, fragment: dict, sections: list, symbols: di """Keep native drawing/zoom functions; add bounded catalogue data hooks.""" if fragment.get("schema") != 1 or fragment.get("abi") != 4 or fragment.get("revision") not in ("us.v77", "us.v80"): raise ValueError("Unsupported track-menu adapter") - words = {start+i: struct.unpack_from(">I", data, i)[0] - for start, data in sections for i in range(0, len(data), 4)} + words = { + start + i: struct.unpack_from(">I", data, i)[0] for start, data in sections for i in range(0, len(data), 4) + } fields = fragment.get("fields", []) if [(f["name"], f["size"]) for f in fields] != MENU_FIELDS: raise ValueError("Incomplete track-menu field ABI") @@ -249,7 +323,9 @@ def compose_track_menu(policy: dict, fragment: dict, sections: list, symbols: di if size <= 0 or size % 4 or symbols.get(name) != {(start, size)}: raise ValueError(f"Track-menu function bounds changed: {name}") try: - digest = hashlib.sha256(b"".join(struct.pack(">I", words[pc]) for pc in range(start, start+size, 4))).hexdigest() + digest = hashlib.sha256( + b"".join(struct.pack(">I", words[pc]) for pc in range(start, start + size, 4)) + ).hexdigest() except KeyError as error: raise ValueError(f"Track-menu body missing: {name}") from error if digest != body["sha256"]: @@ -258,161 +334,282 @@ def compose_track_menu(policy: dict, fragment: dict, sections: list, symbols: di hooks = result.setdefault("functionHooks", []) patches = result.setdefault("instructionPatches", []) seen = set() - field_data = ", ".join(f"{int(f['address'],0):#010x}U" for f in fields) + field_data = ", ".join(f"{int(f['address'], 0):#010x}U" for f in fields) for site in fragment["hooks"]: - name, pc, event = site["function"], int(site["vram"],0), site["event"] - body = bodies[name]; begin = int(body["vram"],0); end = begin+body["size"] - at = int(site["expectedAt"],0) - if pc in seen or pc % 4 or not begin <= pc < end or at < begin or at+12 > end or not 0 <= event <= 17: + name, pc, event = site["function"], int(site["vram"], 0), site["event"] + body = bodies[name] + begin = int(body["vram"], 0) + end = begin + body["size"] + at = int(site["expectedAt"], 0) + if pc in seen or pc % 4 or not begin <= pc < end or at < begin or at + 12 > end or not 0 <= event <= 17: raise ValueError("Invalid/duplicate track-menu hook") if name != MENU_EVENTS[event] or site["returns"] != (event in (9, 10, 11)): raise ValueError("Track-menu event/return ABI changed") observed = 0 if event == 7: - delay = words.get(pc+4) - if words.get(pc) != 0x03e00008: + delay = words.get(pc + 4) + if words.get(pc) != 0x03E00008: raise ValueError("Background-loader outcome hook is not at a return") if delay == 0x24020001: observed = 1 - elif delay != 0 or words.get(pc-4) != 0x00001025: + elif delay != 0 or words.get(pc - 4) != 0x00001025: raise ValueError("Background-loader outcome delay slot changed") if site.get("observedReturn") != observed: raise ValueError("Background-loader observation differs from its actual return sequence") seen.add(pc) - if len(site["expected"]) != 3 or [words.get(at+i*4) for i in range(3)] != [int(w,0) for w in site["expected"]]: + if len(site["expected"]) != 3 or [words.get(at + i * 4) for i in range(3)] != [ + int(w, 0) for w in site["expected"] + ]: raise ValueError(f"Track-menu hook signature changed: {name}") - if any(int(p["vram"],0) == pc for p in patches): + if any(int(p["vram"], 0) == pc for p in patches): raise ValueError(f"Track-menu hook collides with an instruction patch: {name}") - owners = [h for h in hooks if int(h["beforeVram"],0) == pc] - event_fields=field_data - if event==4: - event_fields=", ".join("(uint32_t)ctx->r29 + 0x90U" if f['name']=='gTrackSelectRenderDetails' - else f"{int(f['address'],0):#010x}U" for f in fields) - observed=5 # Five draw-only columns, three rows; separate stack storage. - text = ("extern int dkr_legacy_track_menu(uint8_t*, recomp_context*, unsigned, const uint32_t*, unsigned); " - "{ "+("" if event==4 else "static ")+"const uint32_t dkr_legacy_fields[] = {"+event_fields+"}; " - + ("if (" if site["returns"] else "") - + f"dkr_legacy_track_menu(rdram, ctx, {event}U, dkr_legacy_fields, {observed}U)" - + (") return; }" if site["returns"] else "; }")) + owners = [h for h in hooks if int(h["beforeVram"], 0) == pc] + event_fields = field_data + if event == 4: + event_fields = ", ".join( + "(uint32_t)ctx->r29 + 0x90U" + if f["name"] == "gTrackSelectRenderDetails" + else f"{int(f['address'], 0):#010x}U" + for f in fields + ) + observed = 5 # Five draw-only columns, three rows; separate stack storage. + text = ( + "extern int dkr_legacy_track_menu(uint8_t*, recomp_context*, unsigned, const uint32_t*, unsigned); " + "{ " + + ("" if event == 4 else "static ") + + "const uint32_t dkr_legacy_fields[] = {" + + event_fields + + "}; " + + ("if (" if site["returns"] else "") + + f"dkr_legacy_track_menu(rdram, ctx, {event}U, dkr_legacy_fields, {observed}U)" + + (") return; }" if site["returns"] else "; }") + ) if owners: # One deliberate composition point, not a blanket hook override. - expected = ("extern void dkr_netplay_gameplay_level_begin(uint8_t*, recomp_context*); " - "dkr_netplay_gameplay_level_begin(rdram, ctx);") + expected = ( + "extern void dkr_netplay_gameplay_level_begin(uint8_t*, recomp_context*); " + "dkr_netplay_gameplay_level_begin(rdram, ctx);" + ) if len(owners) != 1 or event != 14 or name != "load_level_game" or owners[0]["text"] != expected: raise ValueError(f"Unreviewed existing owner at track-menu hook: {name}") owners[0]["text"] += " " + text - owners[0]["reason"] += " Bind an explicitly selected custom course after the unchanged online entry callback." + owners[0]["reason"] += ( + " Bind an explicitly selected custom course after the unchanged online entry callback." + ) else: - hooks.append({"function":name,"beforeVram":hex(pc),"text":text, - "reason":"Additive custom-track catalogue adapter; preserve the complete original stock menu/drawing/zoom path."}) + hooks.append( + { + "function": name, + "beforeVram": hex(pc), + "text": text, + "reason": "Additive custom-track catalogue adapter; preserve the complete original stock menu/drawing/zoom path.", + } + ) if {h["event"] for h in fragment["hooks"]} != set(range(18)): raise ValueError("Incomplete track-menu event coverage") return result def compose_characters(policy: dict, fragment: dict, sections: list, symbols: dict) -> dict: - if fragment.get('schema')!=1 or fragment.get('abi')!=1 or fragment.get('revision') not in ('us.v77','us.v80'): - raise ValueError('Unsupported character adapter') - fields=fragment.get('fields',[]) - if [(f['name'],f['size']) for f in fields]!=[('gCharacterIdSlots',8)]: - raise ValueError('Incomplete character field ABI') + if fragment.get("schema") != 1 or fragment.get("abi") != 1 or fragment.get("revision") not in ("us.v77", "us.v80"): + raise ValueError("Unsupported character adapter") + fields = fragment.get("fields", []) + if [(f["name"], f["size"]) for f in fields] != [("gCharacterIdSlots", 8)]: + raise ValueError("Incomplete character field ABI") for f in fields: - if symbols.get(f['name'])!={(int(f['address'],0),f['size'])}:raise ValueError('Character symbol changed') - words={base+i:struct.unpack_from('>I',data,i)[0] for base,data in sections for i in range(0,len(data),4)} - names=['spawn_object','charselect_assign_ai','menu_character_select_init'] - if set(fragment.get('functions',{}))!=set(names) or len(fragment.get('hooks',[]))!=3: - raise ValueError('Incomplete character lifecycle coverage') - for name,body in fragment['functions'].items(): - start,size=int(body['vram'],0),body['size'] - if size<=0 or size%4 or symbols.get(name)!={(start,size)}:raise ValueError('Character function bounds changed') - try:digest=hashlib.sha256(b''.join(struct.pack('>I',words[pc]) for pc in range(start,start+size,4))).hexdigest() - except KeyError as e:raise ValueError('Character function backing is missing') from e - if digest!=body['sha256']:raise ValueError('Character function body changed: '+name) - result=copy.deepcopy(policy) - hooks=result.setdefault('functionHooks',[]);patches=result.setdefault('instructionPatches',[]) - for event,site in enumerate(fragment['hooks']): - name,pc=site['function'],int(site['vram'],0) - body=fragment['functions'].get(name,{}) - start=int(body.get('vram','0'),0);end=start+body.get('size',0) - if name!=names[event] or event!=site['event'] or pc%4 or not start<=pc<=end-12: - raise ValueError('Character event ABI changed') - if len(site['expected'])!=3 or [words.get(pc+i*4) for i in range(3)]!=[int(w,0) for w in site['expected']]: - raise ValueError('Character site signature changed') - if event==0 and words[pc]!=0xa7a4004e:raise ValueError('Character hook must precede the header lifetime store') - if event!=0 and pc!=start:raise ValueError('Character lifecycle hook moved away from entry') - if any(int(p['vram'],0)==pc for p in patches): - raise ValueError('Character hook conflicts with an existing owner') - text='extern void dkr_legacy_character_event(uint8_t*, recomp_context*, unsigned, uint32_t); '+f'dkr_legacy_character_event(rdram, ctx, {event}U, {int(fields[0]["address"],0):#x}U);' - owners=[h for h in hooks if int(h['beforeVram'],0)==pc] + if symbols.get(f["name"]) != {(int(f["address"], 0), f["size"])}: + raise ValueError("Character symbol changed") + words = {base + i: struct.unpack_from(">I", data, i)[0] for base, data in sections for i in range(0, len(data), 4)} + names = ["spawn_object", "charselect_assign_ai", "menu_character_select_init"] + if set(fragment.get("functions", {})) != set(names) or len(fragment.get("hooks", [])) != 3: + raise ValueError("Incomplete character lifecycle coverage") + for name, body in fragment["functions"].items(): + start, size = int(body["vram"], 0), body["size"] + if size <= 0 or size % 4 or symbols.get(name) != {(start, size)}: + raise ValueError("Character function bounds changed") + try: + digest = hashlib.sha256( + b"".join(struct.pack(">I", words[pc]) for pc in range(start, start + size, 4)) + ).hexdigest() + except KeyError as e: + raise ValueError("Character function backing is missing") from e + if digest != body["sha256"]: + raise ValueError("Character function body changed: " + name) + result = copy.deepcopy(policy) + hooks = result.setdefault("functionHooks", []) + patches = result.setdefault("instructionPatches", []) + for event, site in enumerate(fragment["hooks"]): + name, pc = site["function"], int(site["vram"], 0) + body = fragment["functions"].get(name, {}) + start = int(body.get("vram", "0"), 0) + end = start + body.get("size", 0) + if name != names[event] or event != site["event"] or pc % 4 or not start <= pc <= end - 12: + raise ValueError("Character event ABI changed") + if len(site["expected"]) != 3 or [words.get(pc + i * 4) for i in range(3)] != [ + int(w, 0) for w in site["expected"] + ]: + raise ValueError("Character site signature changed") + if event == 0 and words[pc] != 0xA7A4004E: + raise ValueError("Character hook must precede the header lifetime store") + if event != 0 and pc != start: + raise ValueError("Character lifecycle hook moved away from entry") + if any(int(p["vram"], 0) == pc for p in patches): + raise ValueError("Character hook conflicts with an existing owner") + text = ( + "extern void dkr_legacy_character_event(uint8_t*, recomp_context*, unsigned, uint32_t); " + + f"dkr_legacy_character_event(rdram, ctx, {event}U, {int(fields[0]['address'], 0):#x}U);" + ) + owners = [h for h in hooks if int(h["beforeVram"], 0) == pc] if owners: - expected='extern void dkr_netplay_character_select_ai_seed(uint8_t*, recomp_context*); dkr_netplay_character_select_ai_seed(rdram, ctx);' - if len(owners)!=1 or event!=1 or owners[0]['function']!='charselect_assign_ai' or owners[0]['text']!=expected: - raise ValueError('Unreviewed existing character lifecycle owner') - owners[0]['text']+=' '+text - owners[0]['reason']+=' Then record the explicitly chosen logical human identities without changing native AI assignment.' + expected = "extern void dkr_netplay_character_select_ai_seed(uint8_t*, recomp_context*); dkr_netplay_character_select_ai_seed(rdram, ctx);" + if ( + len(owners) != 1 + or event != 1 + or owners[0]["function"] != "charselect_assign_ai" + or owners[0]["text"] != expected + ): + raise ValueError("Unreviewed existing character lifecycle owner") + owners[0]["text"] += " " + text + owners[0]["reason"] += ( + " Then record the explicitly chosen logical human identities without changing native AI assignment." + ) else: - hooks.append({'function':name,'beforeVram':hex(pc),'text':text, - 'reason':'Additive per-racer assets before native header ownership; no original character IDs or cache IDs are replaced.'}) + hooks.append( + { + "function": name, + "beforeVram": hex(pc), + "text": text, + "reason": "Additive per-racer assets before native header ownership; no original character IDs or cache IDs are replaced.", + } + ) return result -def compose_character_menu(policy,fragment,sections,symbols): +def compose_character_menu(policy, fragment, sections, symbols): # Independent menu adapter: the existing per-racer resource ABI stays fixed. - fields_spec=[('gActivePlayersArray',4),('gCharselectStatus',4),('gPlayersCharacterArray',8), - ('gCharacterIdSlots',8),('gMenuButtons',20),('gMenuStickX',10),('gMenuStickY',10), - ('gNumberOfReadyPlayers',4),('gNumberOfActivePlayers',4),('gMenuDelay',4), - ('sMenuCurrDisplayList',4),('gCurrCharacterSelectData',4),('dDialogueBoxBegin',56),('dDialogueBoxDrawModes',32),('gMenuSoundMasks',16), - ('gMenuCurrentCharacter',4),('gObjectCount',4),('gFreeListCount',4)] - names=['menu_character_select_init','menu_character_select_loop','charselect_render_text', - 'charselect_free','charselect_assign_ai','charselect_move','obj_loop_char_select','obj_loop_char_select','sound_play'] - if fragment.get('schema')!=1 or fragment.get('abi')!=3 or fragment.get('revision') not in ('us.v77','us.v80'): - raise ValueError('Unsupported character-menu adapter') - fields=fragment.get('fields',[]) - if [(f['name'],f['size']) for f in fields]!=fields_spec:raise ValueError('Incomplete character-menu field ABI') + fields_spec = [ + ("gActivePlayersArray", 4), + ("gCharselectStatus", 4), + ("gPlayersCharacterArray", 8), + ("gCharacterIdSlots", 8), + ("gMenuButtons", 20), + ("gMenuStickX", 10), + ("gMenuStickY", 10), + ("gNumberOfReadyPlayers", 4), + ("gNumberOfActivePlayers", 4), + ("gMenuDelay", 4), + ("sMenuCurrDisplayList", 4), + ("gCurrCharacterSelectData", 4), + ("dDialogueBoxBegin", 56), + ("dDialogueBoxDrawModes", 32), + ("gMenuSoundMasks", 16), + ("gMenuCurrentCharacter", 4), + ("gObjectCount", 4), + ("gFreeListCount", 4), + ] + names = [ + "menu_character_select_init", + "menu_character_select_loop", + "charselect_render_text", + "charselect_free", + "charselect_assign_ai", + "charselect_move", + "obj_loop_char_select", + "obj_loop_char_select", + "sound_play", + ] + if fragment.get("schema") != 1 or fragment.get("abi") != 3 or fragment.get("revision") not in ("us.v77", "us.v80"): + raise ValueError("Unsupported character-menu adapter") + fields = fragment.get("fields", []) + if [(f["name"], f["size"]) for f in fields] != fields_spec: + raise ValueError("Incomplete character-menu field ABI") for f in fields: - if symbols.get(f['name'])!={(int(f['address'],0),f['size'])}:raise ValueError('Character-menu symbol changed') - words={base+i:struct.unpack_from('>I',data,i)[0] for base,data in sections for i in range(0,len(data),4)} - if set(fragment.get('functions',{}))!=set(names) or len(fragment.get('hooks',[]))!=9: - raise ValueError('Incomplete character-menu lifecycle') - for name,body in fragment['functions'].items(): - start,size=int(body['vram'],0),body['size'] - if size<=0 or size%4 or symbols.get(name)!={(start,size)}:raise ValueError('Character-menu function changed') - try:digest=hashlib.sha256(b''.join(struct.pack('>I',words[p]) for p in range(start,start+size,4))).hexdigest() - except KeyError as e:raise ValueError('Missing menu function backing') from e - if digest!=body['sha256']:raise ValueError('Character-menu body changed: '+name) - result=copy.deepcopy(policy);hooks=result.setdefault('functionHooks',[]) - values=', '.join(f'{int(f["address"],0):#x}U' for f in fields) - for event,site in enumerate(fragment['hooks']): - name=site['function'];pc=int(site['vram'],0);at=int(site['expectedAt'],0) - body=fragment['functions'].get(name,{});start=int(body.get('vram','0'),0);end=start+body.get('size',0) - if name!=names[event] or site['event']!=event or pc%4 or not start<=pcI", data, i)[0] for base, data in sections for i in range(0, len(data), 4)} + if set(fragment.get("functions", {})) != set(names) or len(fragment.get("hooks", [])) != 9: + raise ValueError("Incomplete character-menu lifecycle") + for name, body in fragment["functions"].items(): + start, size = int(body["vram"], 0), body["size"] + if size <= 0 or size % 4 or symbols.get(name) != {(start, size)}: + raise ValueError("Character-menu function changed") + try: + digest = hashlib.sha256( + b"".join(struct.pack(">I", words[p]) for p in range(start, start + size, 4)) + ).hexdigest() + except KeyError as e: + raise ValueError("Missing menu function backing") from e + if digest != body["sha256"]: + raise ValueError("Character-menu body changed: " + name) + result = copy.deepcopy(policy) + hooks = result.setdefault("functionHooks", []) + values = ", ".join(f"{int(f['address'], 0):#x}U" for f in fields) + for event, site in enumerate(fragment["hooks"]): + name = site["function"] + pc = int(site["vram"], 0) + at = int(site["expectedAt"], 0) + body = fragment["functions"].get(name, {}) + start = int(body.get("vram", "0"), 0) + end = start + body.get("size", 0) + if ( + name != names[event] + or site["event"] != event + or pc % 4 + or not start <= pc < end + or not start <= at <= end - 12 + ): + raise ValueError("Character-menu event moved") + if len(site["expected"]) != 3 or [words.get(at + 4 * i) for i in range(3)] != [ + int(w, 0) for w in site["expected"] + ]: + raise ValueError("Character-menu site signature changed") + if event in (0, 2) and words[pc] != 0x03E00008: + raise ValueError("Menu initialization/render hook must precede the native return") + if event in (3, 4, 5, 6, 8) and pc != start: + raise ValueError("Character-menu lifecycle entry moved") + if any(int(p["vram"], 0) == pc for p in result.get("instructionPatches", [])): + raise ValueError("Menu hook overlaps an instruction patch") + call = f"dkr_legacy_character_menu(rdram, ctx, {event}U, dkr_character_menu_fields)" + text = ( + "{ extern int dkr_legacy_character_menu(uint8_t*, recomp_context*, unsigned, const uint32_t*); " + + f"static const uint32_t dkr_character_menu_fields[] = {{ {values} }}; " + ) + text += (f"if ({call}) return;" if event in (5, 6, 8) else call + ";") + " }" + owners = [h for h in hooks if int(h["beforeVram"], 0) == pc] if owners: - if len(owners)!=1 or owners[0]['function']!=name:raise ValueError('Ambiguous character-menu hook owner') - old=owners[0]['text'] - if event==1: - expected='extern void dkr_netplay_character_select_lock(uint8_t*, recomp_context*); dkr_netplay_character_select_lock(rdram, ctx);' - if old!=expected:raise ValueError('Unknown collected-input owner') - owners[0]['text']=old+' '+text - elif event==4: - expected='extern void dkr_netplay_character_select_ai_seed(uint8_t*, recomp_context*); dkr_netplay_character_select_ai_seed(rdram, ctx); ' - expected+='extern void dkr_legacy_character_event(uint8_t*, recomp_context*, unsigned, uint32_t); '+f'dkr_legacy_character_event(rdram, ctx, 1U, {int(fields[3]["address"],0):#x}U);' - if old!=expected:raise ValueError('Logical commit requires the exact existing AI/resource owner') - owners[0]['text']=text+' '+old - else:raise ValueError('Unreviewed character-menu owner') - owners[0]['reason']+=' Compose the separately validated logical custom-character selection without bypassing native launch.' + if len(owners) != 1 or owners[0]["function"] != name: + raise ValueError("Ambiguous character-menu hook owner") + old = owners[0]["text"] + if event == 1: + expected = "extern void dkr_netplay_character_select_lock(uint8_t*, recomp_context*); dkr_netplay_character_select_lock(rdram, ctx);" + if old != expected: + raise ValueError("Unknown collected-input owner") + owners[0]["text"] = old + " " + text + elif event == 4: + expected = "extern void dkr_netplay_character_select_ai_seed(uint8_t*, recomp_context*); dkr_netplay_character_select_ai_seed(rdram, ctx); " + expected += ( + "extern void dkr_legacy_character_event(uint8_t*, recomp_context*, unsigned, uint32_t); " + + f"dkr_legacy_character_event(rdram, ctx, 1U, {int(fields[3]['address'], 0):#x}U);" + ) + if old != expected: + raise ValueError("Logical commit requires the exact existing AI/resource owner") + owners[0]["text"] = text + " " + old + else: + raise ValueError("Unreviewed character-menu owner") + owners[0]["reason"] += ( + " Compose the separately validated logical custom-character selection without bypassing native launch." + ) else: - if event==4:raise ValueError('Character-menu adapter requires the resource-commit hook') - hooks.append({'function':name,'beforeVram':hex(pc),'text':text,'reason':'Additive native-stage selection; original ready/launch and controller mapping remain authoritative.'}) + if event == 4: + raise ValueError("Character-menu adapter requires the resource-commit hook") + hooks.append( + { + "function": name, + "beforeVram": hex(pc), + "text": text, + "reason": "Additive native-stage selection; original ready/launch and controller mapping remain authoritative.", + } + ) return result @@ -423,11 +620,14 @@ def main() -> None: parser.add_argument("--elf", required=True, type=Path) parser.add_argument("--output", required=True, type=Path) parser.add_argument("--scene-runtime", action="store_true") - parser.add_argument("--qualification", action="store_true", - help="Add PRIVATE real-menu-load probe; requires DKR_LEGACY_QUALIFICATION build") + parser.add_argument( + "--qualification", + action="store_true", + help="Add PRIVATE real-menu-load probe; requires DKR_LEGACY_QUALIFICATION build", + ) parser.add_argument("--track-menu", type=Path, help="Verified additive native Track Select fragment") - parser.add_argument("--characters",type=Path,help="Verified additive character resource fragment") - parser.add_argument("--character-menu",type=Path,help="Verified additive character selection fragment") + parser.add_argument("--characters", type=Path, help="Verified additive character resource fragment") + parser.add_argument("--character-menu", type=Path, help="Verified additive character selection fragment") args = parser.parse_args() if args.output.resolve() in (args.policy.resolve(), args.fragment.resolve(), args.elf.resolve()): raise ValueError("Output must not overwrite an input") @@ -437,8 +637,9 @@ def main() -> None: if args.qualification and not args.scene_runtime: raise ValueError("Qualification requires the scene runtime bridge") if args.scene_runtime: - result = compose_scene_runtime(result, fragment, elf_sections(args.elf), - elf_functions(args.elf), args.qualification) + result = compose_scene_runtime( + result, fragment, elf_sections(args.elf), elf_functions(args.elf), args.qualification + ) if args.track_menu: if not args.scene_runtime or args.qualification: raise ValueError("Track Select requires scene ownership and cannot use the cyclic menu-load probe") @@ -447,20 +648,32 @@ def main() -> None: raise ValueError("Track-menu and asset revisions differ") result = compose_track_menu(result, menu, elf_sections(args.elf), elf_functions(args.elf, (1, 2))) from track_select_draw_policy import compose_track_draw - result = compose_track_draw(result,args.elf,fragment['revision'],elf_sections(args.elf),elf_functions(args.elf,(1,2))) + + result = compose_track_draw( + result, args.elf, fragment["revision"], elf_sections(args.elf), elf_functions(args.elf, (1, 2)) + ) if args.characters: - if not args.scene_runtime:raise ValueError('Character adapter requires boot/scene asset ownership') - character=json.loads(args.characters.read_text()) - if character.get('revision')!=fragment.get('revision'):raise ValueError('Character and asset revisions differ') - result=compose_characters(result,character,elf_sections(args.elf),elf_functions(args.elf,(1,2))) + if not args.scene_runtime: + raise ValueError("Character adapter requires boot/scene asset ownership") + character = json.loads(args.characters.read_text()) + if character.get("revision") != fragment.get("revision"): + raise ValueError("Character and asset revisions differ") + result = compose_characters(result, character, elf_sections(args.elf), elf_functions(args.elf, (1, 2))) if args.character_menu: - menu=json.loads(args.character_menu.read_text()) - if not args.characters or menu.get('revision')!=fragment.get('revision'):raise ValueError('Character menu requires matching resource ownership') - result=compose_character_menu(result,menu,elf_sections(args.elf),elf_functions(args.elf,(1,2))) + menu = json.loads(args.character_menu.read_text()) + if not args.characters or menu.get("revision") != fragment.get("revision"): + raise ValueError("Character menu requires matching resource ownership") + result = compose_character_menu(result, menu, elf_sections(args.elf), elf_functions(args.elf, (1, 2))) from legacy_character_presentation_policy import compose_presentation - result=compose_presentation(result,args.elf,fragment['revision'],elf_sections(args.elf),elf_functions(args.elf,(1,2))) + + result = compose_presentation( + result, args.elf, fragment["revision"], elf_sections(args.elf), elf_functions(args.elf, (1, 2)) + ) from legacy_model_cache_policy import compose_model_cache - result=compose_model_cache(result,args.elf,fragment['revision'],elf_sections(args.elf),elf_functions(args.elf,(1,2))) + + result = compose_model_cache( + result, args.elf, fragment["revision"], elf_sections(args.elf), elf_functions(args.elf, (1, 2)) + ) args.output.parent.mkdir(parents=True, exist_ok=True) # A build artifact, never a mutation of a versioned policy or protected C. args.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") diff --git a/scripts/generate_legacy_menu_qualification.py b/scripts/generate_legacy_menu_qualification.py index 4265636c..e05ff443 100644 --- a/scripts/generate_legacy_menu_qualification.py +++ b/scripts/generate_legacy_menu_qualification.py @@ -4,29 +4,34 @@ Uses only the existing checked Patch Pipeline and N64Recomp executable. No generated C edits, no changes to the input ELF, ROM or versioned base policy. """ + import argparse +import json import os from pathlib import Path import subprocess import sys -import json def main(argv=None): - parser=argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output",type=Path,required=True) - parser.add_argument("--v77-build",type=Path,required=True) - parser.add_argument("--v80-build",type=Path,required=True) - parser.add_argument("--recompiler",type=Path,required=True) - parser.add_argument("--characters",action="store_true",help="Include the checked character resource adapter") - parser.add_argument("--character-menu",action="store_true",help="Include the checked additive character selector") - parser.add_argument("--refresh-presentation",action="store_true",help="Refresh the reviewed presentation layer in an already-composed committed policy") - parser.add_argument("--v77-policy",type=Path,help="Explicit reviewed baseline policy for v1.0") - parser.add_argument("--v80-policy",type=Path,help="Explicit reviewed baseline policy for v1.1") - args=parser.parse_args(argv) - root=Path(__file__).resolve().parents[1] - output=args.output.resolve() - if output==root or output==Path(output.anchor) or output.is_relative_to(root/"extern"): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--v77-build", type=Path, required=True) + parser.add_argument("--v80-build", type=Path, required=True) + parser.add_argument("--recompiler", type=Path, required=True) + parser.add_argument("--characters", action="store_true", help="Include the checked character resource adapter") + parser.add_argument("--character-menu", action="store_true", help="Include the checked additive character selector") + parser.add_argument( + "--refresh-presentation", + action="store_true", + help="Refresh the reviewed presentation layer in an already-composed committed policy", + ) + parser.add_argument("--v77-policy", type=Path, help="Explicit reviewed baseline policy for v1.0") + parser.add_argument("--v80-policy", type=Path, help="Explicit reviewed baseline policy for v1.1") + args = parser.parse_args(argv) + root = Path(__file__).resolve().parents[1] + output = args.output.resolve() + if output == root or output == Path(output.anchor) or output.is_relative_to(root / "extern"): parser.error("Private generation must target a dedicated build directory") if args.character_menu and not args.characters: parser.error("Character selector requires --characters") @@ -37,15 +42,17 @@ def main(argv=None): if not args.recompiler.is_file() or not os.access(args.recompiler, os.X_OK): parser.error(f"Recompiler must be an existing executable: {args.recompiler}") # Validate both revisions before any subprocess or output mutation. - for revision, source, explicit in ((77,args.v77_build,args.v77_policy), (80,args.v80_build,args.v80_policy)): - required = [source/f"dkr.us.v{revision}.elf", source/f"dkr.us.v{revision}.z64"] - policies = [explicit or root/f"runtime-recomp/dkr.us.v{revision}.recomp-policy.json", - root/f"runtime-recomp/legacy-mods.v{revision}.recomp-fragment.json", - root/f"runtime-recomp/legacy-track-menu.v{revision}.recomp-fragment.json"] + for revision, source, explicit in ((77, args.v77_build, args.v77_policy), (80, args.v80_build, args.v80_policy)): + required = [source / f"dkr.us.v{revision}.elf", source / f"dkr.us.v{revision}.z64"] + policies = [ + explicit or root / f"runtime-recomp/dkr.us.v{revision}.recomp-policy.json", + root / f"runtime-recomp/legacy-mods.v{revision}.recomp-fragment.json", + root / f"runtime-recomp/legacy-track-menu.v{revision}.recomp-fragment.json", + ] if args.characters: - policies.append(root/f"runtime-recomp/legacy-characters.v{revision}.recomp-fragment.json") + policies.append(root / f"runtime-recomp/legacy-characters.v{revision}.recomp-fragment.json") if args.character_menu: - policies.append(root/f"runtime-recomp/legacy-character-menu.v{revision}.recomp-fragment.json") + policies.append(root / f"runtime-recomp/legacy-character-menu.v{revision}.recomp-fragment.json") for path in required + policies: if not path.is_file(): parser.error(f"Missing v{revision} input: {path}") @@ -70,41 +77,83 @@ def run_stage(command, stage): def generate(args, root, output): - for revision,source,entry in ((77,args.v77_build,"0x80065D40"),(80,args.v80_build,"0x80065F80")): - elf=source/f"dkr.us.v{revision}.elf" - rom=source/f"dkr.us.v{revision}.z64" - policy=output/f"menu-v{revision}.policy.json" - source_policy=(args.v77_policy if revision==77 else args.v80_policy) or root/f'runtime-recomp/dkr.us.v{revision}.recomp-policy.json' - compose_command=[sys.executable,str(root/"scripts/compose_legacy_mod_policy.py"), - "--policy",str(source_policy), - "--fragment",str(root/f"runtime-recomp/legacy-mods.v{revision}.recomp-fragment.json"), - "--elf",str(elf),"--scene-runtime","--track-menu",str(root/f"runtime-recomp/legacy-track-menu.v{revision}.recomp-fragment.json"), - "--output",str(policy)] - if args.characters:compose_command += ['--characters',str(root/f'runtime-recomp/legacy-characters.v{revision}.recomp-fragment.json')] + for revision, source, entry in ((77, args.v77_build, "0x80065D40"), (80, args.v80_build, "0x80065F80")): + elf = source / f"dkr.us.v{revision}.elf" + rom = source / f"dkr.us.v{revision}.z64" + policy = output / f"menu-v{revision}.policy.json" + source_policy = ( + args.v77_policy if revision == 77 else args.v80_policy + ) or root / f"runtime-recomp/dkr.us.v{revision}.recomp-policy.json" + compose_command = [ + sys.executable, + str(root / "scripts/compose_legacy_mod_policy.py"), + "--policy", + str(source_policy), + "--fragment", + str(root / f"runtime-recomp/legacy-mods.v{revision}.recomp-fragment.json"), + "--elf", + str(elf), + "--scene-runtime", + "--track-menu", + str(root / f"runtime-recomp/legacy-track-menu.v{revision}.recomp-fragment.json"), + "--output", + str(policy), + ] + if args.characters: + compose_command += [ + "--characters", + str(root / f"runtime-recomp/legacy-characters.v{revision}.recomp-fragment.json"), + ] if args.character_menu: - compose_command += ['--character-menu',str(root/f'runtime-recomp/legacy-character-menu.v{revision}.recomp-fragment.json')] + compose_command += [ + "--character-menu", + str(root / f"runtime-recomp/legacy-character-menu.v{revision}.recomp-fragment.json"), + ] if args.refresh_presentation: - from compose_legacy_mod_policy import elf_sections,elf_functions + from compose_legacy_mod_policy import elf_functions, elf_sections from legacy_character_presentation_policy import refresh_presentation - base=json.loads(source_policy.read_text()) + + base = json.loads(source_policy.read_text()) try: - updated=refresh_presentation(base,elf,f'us.v{revision}',elf_sections(elf),elf_functions(elf,(1,2))) + updated = refresh_presentation( + base, elf, f"us.v{revision}", elf_sections(elf), elf_functions(elf, (1, 2)) + ) except (ValueError, OSError) as error: - raise RuntimeError(f'v{revision} presentation refresh failed: {error}') from None - policy.parent.mkdir(parents=True,exist_ok=True) - policy.write_text(json.dumps(updated,indent=2)+'\n') - else:run_stage(compose_command, f"v{revision} policy composition") - for isolated in (True,False): - functions=output/"menu-pipeline"/f"RecompiledFuncs-v{revision}" if isolated else output/f"generated-v{revision}" - config=output/f"menu-{'pipeline-' if isolated else ''}v{revision}.toml" - command=[sys.executable,str(root/"scripts/generate_recomp_config.py"),"--policy",str(policy), - "--elf",str(elf),"--rom",str(rom),"--output-functions",str(functions),"--entrypoint",entry,"--output",str(config)] - if isolated:command += ["--functions-per-output-file","1"] + raise RuntimeError(f"v{revision} presentation refresh failed: {error}") from None + policy.parent.mkdir(parents=True, exist_ok=True) + policy.write_text(json.dumps(updated, indent=2) + "\n") + else: + run_stage(compose_command, f"v{revision} policy composition") + for isolated in (True, False): + functions = ( + output / "menu-pipeline" / f"RecompiledFuncs-v{revision}" + if isolated + else output / f"generated-v{revision}" + ) + config = output / f"menu-{'pipeline-' if isolated else ''}v{revision}.toml" + command = [ + sys.executable, + str(root / "scripts/generate_recomp_config.py"), + "--policy", + str(policy), + "--elf", + str(elf), + "--rom", + str(rom), + "--output-functions", + str(functions), + "--entrypoint", + entry, + "--output", + str(config), + ] + if isolated: + command += ["--functions-per-output-file", "1"] run_stage(command, f"v{revision} config generation") - run_stage([str(args.recompiler.resolve()),str(config)], f"v{revision} recompilation ({config.name})") + run_stage([str(args.recompiler.resolve()), str(config)], f"v{revision} recompilation ({config.name})") -if __name__=="__main__": +if __name__ == "__main__": try: raise SystemExit(main()) except (RuntimeError, ValueError, OSError) as error: diff --git a/scripts/generate_recomp_config.py b/scripts/generate_recomp_config.py index 1e8a7214..82974039 100644 --- a/scripts/generate_recomp_config.py +++ b/scripts/generate_recomp_config.py @@ -4,8 +4,8 @@ from __future__ import annotations import argparse -import json import hashlib +import json from pathlib import Path @@ -27,55 +27,81 @@ def main() -> int: parser.add_argument("--output-functions", required=True, type=Path) parser.add_argument("--entrypoint", required=True) parser.add_argument("--output", required=True, type=Path) - parser.add_argument("--composed-policy", type=Path, - help="Write the complete checked policy for code-mod hook protection") + parser.add_argument( + "--composed-policy", type=Path, help="Write the complete checked policy for code-mod hook protection" + ) parser.add_argument("--use-mdebug", action="store_true") parser.add_argument("--water-profile", action="store_true", help="Compose checked water timing hooks") - parser.add_argument("--functions-per-output-file", type=int, - help="Optional source granularity for isolated Patch Pipeline qualification") + parser.add_argument( + "--functions-per-output-file", + type=int, + help="Optional source granularity for isolated Patch Pipeline qualification", + ) arguments = parser.parse_args() if arguments.functions_per_output_file is not None and arguments.functions_per_output_file < 1: parser.error("--functions-per-output-file must be positive") - source_granularity = (f"functions_per_output_file = {arguments.functions_per_output_file}\n" - if arguments.functions_per_output_file is not None else "") + source_granularity = ( + f"functions_per_output_file = {arguments.functions_per_output_file}\n" + if arguments.functions_per_output_file is not None + else "" + ) policy = json.loads(arguments.policy.read_text(encoding="utf-8")) if policy.get("schemaVersion") != 1: raise ValueError("unsupported Patch Pipeline policy schema") from host_task_policy import verify_host_task_policy + verify_host_task_policy(policy, arguments.elf) from camera_obstruction_policy import verify_camera_obstruction_policy + verify_camera_obstruction_policy(policy, arguments.elf) from legacy_character_ai_policy import verify_character_ai_policy + verify_character_ai_policy(policy, arguments.elf) from legacy_asset_capacity_policy import compose_asset_capacity - revision = 'us.v77' if arguments.entrypoint.lower() == '0x80065d40' else 'us.v80' + + revision = "us.v77" if arguments.entrypoint.lower() == "0x80065d40" else "us.v80" policy = compose_asset_capacity(policy, arguments.elf, revision) from legacy_model_reference_policy import compose_model_references + policy = compose_model_references(policy, arguments.elf, revision) from legacy_spawn_asset_type_policy import compose_spawn_asset_type + policy = compose_spawn_asset_type(policy, arguments.elf, revision) from legacy_memory_safety_policy import compose_memory_safety + policy = compose_memory_safety(policy, arguments.elf, revision) from legacy_behaviour_policy import compose_behaviour + policy = compose_behaviour(policy, arguments.elf, revision) from audio_queue_policy import compose_audio_queue + policy = compose_audio_queue(policy, arguments.elf, revision) from startup_flow_policy import compose_startup_flow + policy = compose_startup_flow(policy, arguments.elf, revision) if arguments.water_profile: from water_profile_policy import compose_water_profile + policy = compose_water_profile(policy, arguments.elf) if arguments.composed_policy: # Baseline-only inventories miss project hooks added by composition. # This receipt is emitted only after every selected policy has run. - receipt = dict(schema=1, revision=revision, - elf_sha256=hashlib.sha256(arguments.elf.read_bytes()).hexdigest(), - complete=True, policy=policy) - if arguments.composed_policy.resolve() in (arguments.policy.resolve(), arguments.elf.resolve(), arguments.output.resolve()): - raise ValueError('Composed policy must not overwrite an input or the generated TOML') + receipt = dict( + schema=1, + revision=revision, + elf_sha256=hashlib.sha256(arguments.elf.read_bytes()).hexdigest(), + complete=True, + policy=policy, + ) + if arguments.composed_policy.resolve() in ( + arguments.policy.resolve(), + arguments.elf.resolve(), + arguments.output.resolve(), + ): + raise ValueError("Composed policy must not overwrite an input or the generated TOML") arguments.composed_policy.parent.mkdir(parents=True, exist_ok=True) - arguments.composed_policy.write_text(json.dumps(receipt, indent=2) + '\n', encoding='utf-8') + arguments.composed_policy.write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8") manual = ", ".join( "{ name = %s, section = %s, vram = %s, size = %s }" @@ -88,8 +114,7 @@ def main() -> int: for entry in policy.get("manualFunctions", []) ) sizes = ", ".join( - "{ name = %s, size = %s }" - % (toml_string(str(entry["name"])), entry["size"]) + "{ name = %s, size = %s }" % (toml_string(str(entry["name"])), entry["size"]) for entry in policy.get("functionSizes", []) ) patches = "\n\n".join( @@ -120,9 +145,9 @@ def main() -> int: function_sizes = [{sizes}] [patches] -stubs = [{string_list(policy.get('stubs', []))}] -renamed = [{string_list(policy.get('renamed', []))}] -ignored = [{string_list(policy.get('ignored', []))}] +stubs = [{string_list(policy.get("stubs", []))}] +renamed = [{string_list(policy.get("renamed", []))}] +ignored = [{string_list(policy.get("ignored", []))}] # BEGIN DKR_INSTRUCTION_PATCHES {patches} diff --git a/scripts/generate_replay_probe.py b/scripts/generate_replay_probe.py index 577353e6..232e81b9 100644 --- a/scripts/generate_replay_probe.py +++ b/scripts/generate_replay_probe.py @@ -6,17 +6,22 @@ adapter: every native import traps, rather than silently pretending success. The standalone target links no launcher, audio, renderer or online runtime. """ + from __future__ import annotations + import argparse import hashlib import json from pathlib import Path import re + import stage_magic_code_payload as magic_code_pipeline HEADER_HASH = "58977ed0a489f7df8b17c377a87cc6670857a5fba15af78fc7d38132fc59b33f" DEF = re.compile(r"RECOMP_FUNC\s+void\s+(\w+)\s*\([^{};]*\)\s*\{") -PROTO = re.compile(r"(?:extern\s+)?\b(void|int|unsigned|float|double|gpr|u?int(?:8|16|32|64)_t|recomp_func_t\s*\*)\s+(\w+)\s*\(([^();{}]*)\)\s*;") +PROTO = re.compile( + r"(?:extern\s+)?\b(void|int|unsigned|float|double|gpr|u?int(?:8|16|32|64)_t|recomp_func_t\s*\*)\s+(\w+)\s*\(([^();{}]*)\)\s*;" +) CALL = re.compile(r"\b([A-Za-z_]\w*)\s*\(") # Project-owned import absent from N64Recomp's emitted funcs.h. ABI checked # against runtime-recomp/src/game/virtual_pak.cpp, not inferred from its name. @@ -44,7 +49,10 @@ "dkr_legacy_character_cinematic_id": ("unsigned", ["uint8_t*", "recomp_context*", "unsigned", "unsigned"]), "dkr_legacy_character_race_sound": ("unsigned", ["uint8_t*", "recomp_context*", "uint32_t", "unsigned"]), "dkr_legacy_character_ai_event": ("void", ["uint8_t*", "recomp_context*", "unsigned", "recomp_func_t*"]), - "dkr_legacy_memory_service": ("uint32_t", ["uint8_t*", "recomp_context*", "unsigned", "unsigned", "uint32_t", "uint32_t"]), + "dkr_legacy_memory_service": ( + "uint32_t", + ["uint8_t*", "recomp_context*", "unsigned", "unsigned", "uint32_t", "uint32_t"], + ), "dkr_legacy_course_component": ("uint32_t", ["uint8_t*", "recomp_context*", "unsigned", "uint32_t"]), "dkr_legacy_racer_stat": ("uint32_t", ["uint8_t*", "recomp_context*", "unsigned", "uint32_t", "uint32_t"]), "dkr_legacy_texture_animation": ("int", ["uint8_t*", "recomp_context*", "recomp_func_t*"]), @@ -54,47 +62,68 @@ # entries. Keep the complete closure explicit: function-pointer arguments do # not appear as calls in the normal call scan. Never use active_payload(). OWNED_MOD_CALLS = { - "get_settings": "get_settings", "asset_allocate": "mempool_alloc_safe", - "asset_release": "mempool_free", "asset_copy": "dmacopy", + "get_settings": "get_settings", + "asset_allocate": "mempool_alloc_safe", + "asset_release": "mempool_free", + "asset_copy": "dmacopy", "menu_material_set": "material_set", - "menu_sound_play": "sound_play", "menu_texture_load": "load_texture", - "menu_texture_free": "tex_free", "menu_texture_draw": "texrect_draw_scaled", - "menu_font": "set_text_font", "menu_colour": "set_text_colour", - "menu_background": "set_text_background_colour", "menu_text": "draw_text", - "menu_render_reset": "rendermode_reset", "filtered_cheats": "get_filtered_cheats", - "stage_spawn": "spawn_object", "stage_free": "free_object", - "stage_music_fraction": "music_animation_fraction", "stage_particles": "obj_spawn_particle", - "stage_camera": "cam_get_active_camera", "unlock_drumstick": "is_drumstick_unlocked", - "unlock_tt": "is_tt_unlocked", "sound_bank_relocate": "alBnkfNew", - "sound_bank_play": "sndp_play_with_priority", "sound_parameter": "sndp_set_param", + "menu_sound_play": "sound_play", + "menu_texture_load": "load_texture", + "menu_texture_free": "tex_free", + "menu_texture_draw": "texrect_draw_scaled", + "menu_font": "set_text_font", + "menu_colour": "set_text_colour", + "menu_background": "set_text_background_colour", + "menu_text": "draw_text", + "menu_render_reset": "rendermode_reset", + "filtered_cheats": "get_filtered_cheats", + "stage_spawn": "spawn_object", + "stage_free": "free_object", + "stage_music_fraction": "music_animation_fraction", + "stage_particles": "obj_spawn_particle", + "stage_camera": "cam_get_active_camera", + "unlock_drumstick": "is_drumstick_unlocked", + "unlock_tt": "is_tt_unlocked", + "sound_bank_relocate": "alBnkfNew", + "sound_bank_play": "sndp_play_with_priority", + "sound_parameter": "sndp_set_param", "sound_spatial_point": "audspat_point_create", } OWNED_MOD_NATIVE_CALLS = {"menu_background_cover": "dkr_track_select_background_cover"} + def owned_mod_calls_header(): - lines=['// New Patch Pipeline output: private C fault trampolines only.', '#pragma once', - 'extern "C" {', '#include "imports.h"', '}', - 'namespace dkr_owned_mod_calls {'] - calls={**OWNED_MOD_CALLS,**OWNED_MOD_NATIVE_CALLS} - names=set(calls.values()) | {"dkr_character_select_animation_fraction", "rand_range", "tex_animate_texture"} + lines = [ + "// New Patch Pipeline output: private C fault trampolines only.", + "#pragma once", + 'extern "C" {', + '#include "imports.h"', + "}", + "namespace dkr_owned_mod_calls {", + ] + calls = {**OWNED_MOD_CALLS, **OWNED_MOD_NATIVE_CALLS} + names = set(calls.values()) | {"dkr_character_select_animation_fraction", "rand_range", "tex_animate_texture"} for name in sorted(names): - lines.append(f'inline void call_{name}(uint8_t* ram,recomp_context* ctx) {{') - lines.append(' ctx->f_odd=ctx->mips3_float_mode?&ctx->f1.u32l:&ctx->f0.u32h;') - lines.append(f' const auto result=dkr_probe_run_native({name},ram,ctx);') - lines.append(' if(!result.completed)throw dkr::mods::Error(result.blocked?result.blocked:"Owned guest callback failed.");') - lines.append('}') + lines.append(f"inline void call_{name}(uint8_t* ram,recomp_context* ctx) {{") + lines.append(" ctx->f_odd=ctx->mips3_float_mode?&ctx->f1.u32l:&ctx->f0.u32h;") + lines.append(f" const auto result=dkr_probe_run_native({name},ram,ctx);") + lines.append( + ' if(!result.completed)throw dkr::mods::Error(result.blocked?result.blocked:"Owned guest callback failed.");' + ) + lines.append("}") # Isolated-symbol macros also rename a few names used as C++ field names. # Suspend those macros only AFTER the private wrapper bodies are emitted. for field in sorted(calls): - lines.extend([f'#pragma push_macro("{field}")', f'#undef {field}']) - lines.extend(['inline dkr::runtime::GamePayload payload() {', ' dkr::runtime::GamePayload out{};']) - for field,name in calls.items(): - lines.append(f' out.{field}=call_{name};') - lines.extend([' return out;', '}']) + lines.extend([f'#pragma push_macro("{field}")', f"#undef {field}"]) + lines.extend(["inline dkr::runtime::GamePayload payload() {", " dkr::runtime::GamePayload out{};"]) + for field, name in calls.items(): + lines.append(f" out.{field}=call_{name};") + lines.extend([" return out;", "}"]) for field in reversed(sorted(calls)): lines.append(f'#pragma pop_macro("{field}")') - lines.append('}') - return '\n'.join(lines)+'\n' + lines.append("}") + return "\n".join(lines) + "\n" + # EXACT reviewed emitted mode_game bodies. Two unload call sites have no # live host local across the split: hi/lo/result/c1cs are declaration-only; @@ -115,10 +144,14 @@ def owned_mod_calls_header(): 80: "29b360351b098f56d112f51b629bb88b929440153465a251339402fdc0fa15f5", } VIDEO_CPU_HASH = { - 77: {"fb_update": "0985bdc3788dcd9dcb3f9a087762510426f573e4e69c9feec1872bca99197d99", - "fb_swap": "263cb9dae5645bd95a93148533e04dc65b40a16309c2b0b6e58098064e9a5850"}, - 80: {"fb_update": "dab5246c92faea42ecc1e9d5225a39a11ce5aac3d0f0a42b71f52546a7b5b69f", - "fb_swap": "68891ec728466b61473c722cff4263c48c9ff773653659ad204999fb878f889d"}, + 77: { + "fb_update": "0985bdc3788dcd9dcb3f9a087762510426f573e4e69c9feec1872bca99197d99", + "fb_swap": "263cb9dae5645bd95a93148533e04dc65b40a16309c2b0b6e58098064e9a5850", + }, + 80: { + "fb_update": "dab5246c92faea42ecc1e9d5225a39a11ce5aac3d0f0a42b71f52546a7b5b69f", + "fb_swap": "68891ec728466b61473c722cff4263c48c9ff773653659ad204999fb878f889d", + }, } UNLOAD_CPU_HASH = { 77: "1e32bb1436b0a5b59c56e5936b02001d246b24bac4b3c48434fddd62ef3b3abe", @@ -127,10 +160,14 @@ def owned_mod_calls_header(): LOAD_CPU_HASH = { # ABI 4 appends DisplayList and VertexFlip to event 14's field table; # the exact retail constructor instructions and ownership cuts are unchanged. - 77: {"load_level_game": "c435b2bf270048c3a3e0d2825ec788e7642c84453c5139693da945822e64bd9b", - "level_load": "57dc04a00e938e23f1e48115d7e111f4763a53f92fbf4fd1f9a789d49f71b93c"}, - 80: {"load_level_game": "e84e7259d83a3299058073e0fda5b82c2aa6ec777a50430f22500d4e71555633", - "level_load": "93236d6aa9579ad4b8c856aca32b70b5be7ec23c12c4c8462495eeeb236890de"}, + 77: { + "load_level_game": "c435b2bf270048c3a3e0d2825ec788e7642c84453c5139693da945822e64bd9b", + "level_load": "57dc04a00e938e23f1e48115d7e111f4763a53f92fbf4fd1f9a789d49f71b93c", + }, + 80: { + "load_level_game": "e84e7259d83a3299058073e0fda5b82c2aa6ec777a50430f22500d4e71555633", + "level_load": "93236d6aa9579ad4b8c856aca32b70b5be7ec23c12c4c8462495eeeb236890de", + }, } # Full menu ownership is a separate opt-in payload. Pins cover the complete @@ -176,82 +213,117 @@ def owned_mod_calls_header(): def menu_cpu_body_reviewed(name, body, revision): digest = hashlib.sha256(body.encode()).hexdigest() expected = MENU_CPU_HASH.get(revision, {}).get(name) - return digest == expected or (name == "load_level_for_menu" and - digest == MENU_STOCK_LEVEL_HASH.get(revision)) + return digest == expected or (name == "load_level_for_menu" and digest == MENU_STOCK_LEVEL_HASH.get(revision)) def menu_scene_cuts(body, revision): if hashlib.sha256(body.encode()).hexdigest() != MENU_CPU_HASH.get(revision, {}).get("mode_menu"): raise ValueError("Unreviewed mode_menu continuation") for name in ("hi", "lo", "result", "c1cs"): - if len(re.findall(r"\b"+name+r"\b", strip(body))) != 1: + if len(re.findall(r"\b" + name + r"\b", strip(body))) != 1: raise ValueError("Live menu host temporary cannot cross a scene cut") - calls=list(re.finditer(r" unload_level_(?:menu|game)\(rdram, ctx\);",body)) - if len(calls)!=4 or [x[0] for x in calls] != [ - " unload_level_menu(rdram, ctx);", " unload_level_game(rdram, ctx);", - " unload_level_menu(rdram, ctx);", " unload_level_menu(rdram, ctx);"]: + calls = list(re.finditer(r" unload_level_(?:menu|game)\(rdram, ctx\);", body)) + if len(calls) != 4 or [x[0] for x in calls] != [ + " unload_level_menu(rdram, ctx);", + " unload_level_game(rdram, ctx);", + " unload_level_menu(rdram, ctx);", + " unload_level_menu(rdram, ctx);", + ]: raise ValueError("Menu unload sites changed") # Before the prologue, resume at the exact call site: never poll input or # execute menu_loop twice for the same logical frame. - entry="\n switch (dkr_probe_scene_entry()) {\n case 0: break;\n" - entry+="".join(f" case {site}: goto dkr_probe_menu_unload_{site};\n" for site in range(3,7)) - entry+=' default: dkr_probe_block("invalid-menu-scene-continuation");\n }\n' - for site,match in reversed(list(enumerate(calls,3))): - body=body[:match.start()]+f"dkr_probe_menu_unload_{site}:\n if (dkr_probe_scene_cut({site})) return;\n"+body[match.start():] - return body.replace(" int c1cs = 0;", " int c1cs = 0;"+entry,1) + entry = "\n switch (dkr_probe_scene_entry()) {\n case 0: break;\n" + entry += "".join(f" case {site}: goto dkr_probe_menu_unload_{site};\n" for site in range(3, 7)) + entry += ' default: dkr_probe_block("invalid-menu-scene-continuation");\n }\n' + for site, match in reversed(list(enumerate(calls, 3))): + body = ( + body[: match.start()] + + f"dkr_probe_menu_unload_{site}:\n if (dkr_probe_scene_cut({site})) return;\n" + + body[match.start() :] + ) + return body.replace(" int c1cs = 0;", " int c1cs = 0;" + entry, 1) def owned_menu_cpu(functions, revision): """Confirmed menu constructors, not live bgload/scheduler permission.""" - for name in MENU_CPU_HASH.get(revision,{}): + for name in MENU_CPU_HASH.get(revision, {}): if name not in functions or not menu_cpu_body_reviewed(name, functions[name], revision): raise ValueError(f"Unreviewed owned menu dependency: {name}") if revision not in MENU_CPU_HASH: raise ValueError("Unsupported owned menu revision") - replacements={} + replacements = {} # INTRO is also a confirmed resource-changing tick. Its first constructor # must consume the existing cold-world capability through the private # menu loader, not call the live scene-reset/interpolation hooks. Preserve # every retail input, timer, argument, delay slot and guest stack write. - intro=functions["mode_intro"] - if intro.count(" load_menu_with_level_background(rdram, ctx);")!=1: + intro = functions["mode_intro"] + if intro.count(" load_menu_with_level_background(rdram, ctx);") != 1: raise ValueError("INTRO menu constructor site changed") - replacements["mode_intro"]=intro.replace( - " load_menu_with_level_background(rdram, ctx);", - " dkr_probe_menu_background_cpu(rdram, ctx);",1) - unload=functions["unload_level_menu"].replace("void unload_level_menu(","void dkr_probe_menu_unload_cpu(",1) - unload=unload.replace(" int c1cs = 0;", " int c1cs = 0;\n dkr_probe_scene_unload_begin();\n dkr_probe_scene_unload_render_drained();",1) - if unload.count(" return;")!=1: raise ValueError("Menu teardown epilogue changed") - replacements["dkr_probe_menu_unload_cpu"]=unload.replace(" return;"," dkr_probe_scene_unload_end();\n return;",1) - load=functions["load_level_menu"].replace("void load_level_menu(","void dkr_probe_menu_load_cpu(",1) - if load.count(" level_load(rdram, ctx);")!=1 or load.count(" return;")!=1: + replacements["mode_intro"] = intro.replace( + " load_menu_with_level_background(rdram, ctx);", " dkr_probe_menu_background_cpu(rdram, ctx);", 1 + ) + unload = functions["unload_level_menu"].replace("void unload_level_menu(", "void dkr_probe_menu_unload_cpu(", 1) + unload = unload.replace( + " int c1cs = 0;", + " int c1cs = 0;\n dkr_probe_scene_unload_begin();\n dkr_probe_scene_unload_render_drained();", + 1, + ) + if unload.count(" return;") != 1: + raise ValueError("Menu teardown epilogue changed") + replacements["dkr_probe_menu_unload_cpu"] = unload.replace( + " return;", " dkr_probe_scene_unload_end();\n return;", 1 + ) + load = functions["load_level_menu"].replace("void load_level_menu(", "void dkr_probe_menu_load_cpu(", 1) + if load.count(" level_load(rdram, ctx);") != 1 or load.count(" return;") != 1: raise ValueError("Menu constructor sites changed") - load=load.replace(" int c1cs = 0;"," int c1cs = 0;\n dkr_probe_menu_load_begin(rdram, ctx);",1) - load=load.replace(" level_load(rdram, ctx);"," dkr_probe_level_load_cpu(rdram, ctx);",1) - replacements["dkr_probe_menu_load_cpu"]=load.replace(" return;"," dkr_probe_scene_load_ready(rdram, ctx);\n return;",1) - wrapper=functions["load_menu_with_level_background"].replace("void load_menu_with_level_background(","void dkr_probe_menu_background_cpu(",1) - if wrapper.count(" load_level_menu(rdram, ctx);")!=1 or wrapper.count(" return;")!=1: + load = load.replace(" int c1cs = 0;", " int c1cs = 0;\n dkr_probe_menu_load_begin(rdram, ctx);", 1) + load = load.replace(" level_load(rdram, ctx);", " dkr_probe_level_load_cpu(rdram, ctx);", 1) + replacements["dkr_probe_menu_load_cpu"] = load.replace( + " return;", " dkr_probe_scene_load_ready(rdram, ctx);\n return;", 1 + ) + wrapper = functions["load_menu_with_level_background"].replace( + "void load_menu_with_level_background(", "void dkr_probe_menu_background_cpu(", 1 + ) + if wrapper.count(" load_level_menu(rdram, ctx);") != 1 or wrapper.count(" return;") != 1: raise ValueError("Menu background constructor sites changed") - wrapper=wrapper.replace(" int c1cs = 0;"," int c1cs = 0;\n dkr_probe_menu_construct_begin();",1) - wrapper=wrapper.replace(" load_level_menu(rdram, ctx);"," dkr_probe_menu_load_cpu(rdram, ctx);",1) - replacements["dkr_probe_menu_background_cpu"]=wrapper.replace(" return;"," dkr_probe_menu_construct_ready(rdram, ctx);\n return;",1) - next_level=functions["load_next_ingame_level"].replace("void load_next_ingame_level(","void dkr_probe_next_ingame_cpu(",1) - if next_level.count(" load_level_game(rdram, ctx);")!=1: + wrapper = wrapper.replace(" int c1cs = 0;", " int c1cs = 0;\n dkr_probe_menu_construct_begin();", 1) + wrapper = wrapper.replace(" load_level_menu(rdram, ctx);", " dkr_probe_menu_load_cpu(rdram, ctx);", 1) + replacements["dkr_probe_menu_background_cpu"] = wrapper.replace( + " return;", " dkr_probe_menu_construct_ready(rdram, ctx);\n return;", 1 + ) + next_level = functions["load_next_ingame_level"].replace( + "void load_next_ingame_level(", "void dkr_probe_next_ingame_cpu(", 1 + ) + if next_level.count(" load_level_game(rdram, ctx);") != 1: raise ValueError("Next-level constructor changed") - replacements["dkr_probe_next_ingame_cpu"]=next_level.replace(" load_level_game(rdram, ctx);"," dkr_probe_scene_load_cpu(rdram, ctx);",1) - menu_level=functions["load_level_for_menu"] - if menu_level.count(" unload_level_menu(rdram, ctx);")!=1 or menu_level.count(" load_level_menu(rdram, ctx);")!=1 or menu_level.count(" return;")!=1: + replacements["dkr_probe_next_ingame_cpu"] = next_level.replace( + " load_level_game(rdram, ctx);", " dkr_probe_scene_load_cpu(rdram, ctx);", 1 + ) + menu_level = functions["load_level_for_menu"] + if ( + menu_level.count(" unload_level_menu(rdram, ctx);") != 1 + or menu_level.count(" load_level_menu(rdram, ctx);") != 1 + or menu_level.count(" return;") != 1 + ): raise ValueError("Menu level-change ownership sites changed") - menu_level=menu_level.replace(" int c1cs = 0;"," int c1cs = 0;\n dkr_probe_menu_level_change_begin(rdram, ctx);",1) - menu_level=menu_level.replace(" unload_level_menu(rdram, ctx);"," dkr_probe_menu_unload_cpu(rdram, ctx);",1) - menu_level=menu_level.replace(" load_level_menu(rdram, ctx);"," dkr_probe_menu_load_cpu(rdram, ctx);",1) - replacements["load_level_for_menu"]=menu_level.replace(" return;"," dkr_probe_menu_level_change_ready(rdram, ctx);\n return;",1) - preview=functions["bgload_tick"] - if preview.count(" osSendMesg_recomp(rdram, ctx);")!=1: + menu_level = menu_level.replace( + " int c1cs = 0;", " int c1cs = 0;\n dkr_probe_menu_level_change_begin(rdram, ctx);", 1 + ) + menu_level = menu_level.replace( + " unload_level_menu(rdram, ctx);", " dkr_probe_menu_unload_cpu(rdram, ctx);", 1 + ) + menu_level = menu_level.replace(" load_level_menu(rdram, ctx);", " dkr_probe_menu_load_cpu(rdram, ctx);", 1) + replacements["load_level_for_menu"] = menu_level.replace( + " return;", " dkr_probe_menu_level_change_ready(rdram, ctx);\n return;", 1 + ) + preview = functions["bgload_tick"] + if preview.count(" osSendMesg_recomp(rdram, ctx);") != 1: raise ValueError("Background preview request site changed") # Preserve the retail four-tick delay. This is an owned request, NOT a # fabricated OS queue completion. The loader runs at the agreed boundary. - replacements["bgload_tick"]=preview.replace(" osSendMesg_recomp(rdram, ctx);"," dkr_probe_menu_background_request(rdram, ctx);",1) + replacements["bgload_tick"] = preview.replace( + " osSendMesg_recomp(rdram, ctx);", " dkr_probe_menu_background_request(rdram, ctx);", 1 + ) return replacements @@ -262,22 +334,32 @@ def owned_load_cpu(body, level, revision): load hooks nor a live interpolation registry is entered. The replacement owner must implement the reviewed no-mod/Accurate state explicitly. """ - pins=LOAD_CPU_HASH.get(revision,{}) - if any(hashlib.sha256(source.encode()).hexdigest()!=pins.get(name) - for name,source in (("load_level_game",body),("level_load",level))): + pins = LOAD_CPU_HASH.get(revision, {}) + if any( + hashlib.sha256(source.encode()).hexdigest() != pins.get(name) + for name, source in (("load_level_game", body), ("level_load", level)) + ): raise ValueError("Unreviewed confirmed scene constructor") - begin='extern void dkr_netplay_gameplay_level_begin(uint8_t*, recomp_context*); dkr_netplay_gameplay_level_begin(rdram, ctx);' - ready='extern void dkr_netplay_gameplay_level_ready(uint8_t*, recomp_context*); dkr_netplay_gameplay_level_ready(rdram, ctx);' - reset='extern void dkr_runtime_scene_reset(uint8_t*, recomp_context*); extern void dkr_presentation_scene_begin(uint8_t*, recomp_context*); dkr_runtime_scene_reset(rdram, ctx); dkr_presentation_scene_begin(rdram, ctx); extern void dkr_legacy_scene_begin(uint8_t*, recomp_context*); dkr_legacy_scene_begin(rdram, ctx);' - if body.count(begin)!=1 or body.count(ready)!=1 or body.count(' level_load(rdram, ctx);')!=1 or level.count(reset)!=1: + begin = "extern void dkr_netplay_gameplay_level_begin(uint8_t*, recomp_context*); dkr_netplay_gameplay_level_begin(rdram, ctx);" + ready = "extern void dkr_netplay_gameplay_level_ready(uint8_t*, recomp_context*); dkr_netplay_gameplay_level_ready(rdram, ctx);" + reset = "extern void dkr_runtime_scene_reset(uint8_t*, recomp_context*); extern void dkr_presentation_scene_begin(uint8_t*, recomp_context*); dkr_runtime_scene_reset(rdram, ctx); dkr_presentation_scene_begin(rdram, ctx); extern void dkr_legacy_scene_begin(uint8_t*, recomp_context*); dkr_legacy_scene_begin(rdram, ctx);" + if ( + body.count(begin) != 1 + or body.count(ready) != 1 + or body.count(" level_load(rdram, ctx);") != 1 + or level.count(reset) != 1 + ): raise ValueError("Confirmed scene constructor ownership sites changed") - game=body.replace('void load_level_game(','void dkr_probe_scene_load_cpu(',1).replace( - begin,'dkr_probe_scene_load_begin(rdram, ctx);',1).replace( - ready,'dkr_probe_scene_load_ready(rdram, ctx);',1).replace( - ' level_load(rdram, ctx);',' dkr_probe_level_load_cpu(rdram, ctx);',1) - child=level.replace('void level_load(','void dkr_probe_level_load_cpu(',1).replace( - reset,'dkr_probe_scene_load_reset(rdram, ctx);',1) - return game,child + game = ( + body.replace("void load_level_game(", "void dkr_probe_scene_load_cpu(", 1) + .replace(begin, "dkr_probe_scene_load_begin(rdram, ctx);", 1) + .replace(ready, "dkr_probe_scene_load_ready(rdram, ctx);", 1) + .replace(" level_load(rdram, ctx);", " dkr_probe_level_load_cpu(rdram, ctx);", 1) + ) + child = level.replace("void level_load(", "void dkr_probe_level_load_cpu(", 1).replace( + reset, "dkr_probe_scene_load_reset(rdram, ctx);", 1 + ) + return game, child def owned_unload_cpu(body, revision): @@ -287,69 +369,87 @@ def owned_unload_cpu(body, revision): ownership gate is checked BEFORE the first guest stack or allocator write. This does not synthesize a graphics message or skip a live worker wait. """ - if hashlib.sha256(body.encode()).hexdigest()!=UNLOAD_CPU_HASH.get(revision): + if hashlib.sha256(body.encode()).hexdigest() != UNLOAD_CPU_HASH.get(revision): raise ValueError("Unreviewed confirmed scene teardown") - entry='extern void dkr_netplay_gameplay_level_end(uint8_t*, recomp_context*); dkr_netplay_gameplay_level_end(rdram, ctx);' - wait=' gfxtask_wait(rdram, ctx);' - if body.count(entry)!=1 or body.count(wait)!=1 or body.count(' return;')!=1: + entry = "extern void dkr_netplay_gameplay_level_end(uint8_t*, recomp_context*); dkr_netplay_gameplay_level_end(rdram, ctx);" + wait = " gfxtask_wait(rdram, ctx);" + if body.count(entry) != 1 or body.count(wait) != 1 or body.count(" return;") != 1: raise ValueError("Confirmed teardown ownership sites changed") - return body.replace('void unload_level_game(','void dkr_probe_scene_unload_cpu(',1).replace( - entry,'dkr_probe_scene_unload_begin();',1).replace( - wait,' dkr_probe_scene_unload_render_drained();',1).replace( - ' return;',' dkr_probe_scene_unload_end();\n return;',1) + return ( + body.replace("void unload_level_game(", "void dkr_probe_scene_unload_cpu(", 1) + .replace(entry, "dkr_probe_scene_unload_begin();", 1) + .replace(wait, " dkr_probe_scene_unload_render_drained();", 1) + .replace(" return;", " dkr_probe_scene_unload_end();\n return;", 1) + ) def authored_video_cpu(body, swap, revision): """Keep retail blackout/role writes, not VI queue waits or fake events.""" - pins=VIDEO_CPU_HASH.get(revision,{}) - if any(hashlib.sha256(source.encode()).hexdigest()!=pins.get(name) - for name,source in (("fb_update",body),("fb_swap",swap))): + pins = VIDEO_CPU_HASH.get(revision, {}) + if any( + hashlib.sha256(source.encode()).hexdigest() != pins.get(name) + for name, source in (("fb_update", body), ("fb_swap", swap)) + ): raise ValueError("Unreviewed framebuffer CPU state phase") - offset=0 if revision==77 else 0x450 - split=f"L_{0x8007A9E0+offset:08X}:" - epilogue=f" // 0x{0x8007AB08+offset:08X}: lw" - if body.count(split)!=1 or body.count(epilogue)!=1: + offset = 0 if revision == 77 else 0x450 + split = f"L_{0x8007A9E0 + offset:08X}:" + epilogue = f" // 0x{0x8007AB08 + offset:08X}: lw" + if body.count(split) != 1 or body.count(epilogue) != 1: raise ValueError("Framebuffer CPU split/epilogue changed") - prefix=body[:body.index(split)].replace("void fb_update(","void dkr_probe_authored_video_cpu(",1) + prefix = body[: body.index(split)].replace("void fb_update(", "void dkr_probe_authored_video_cpu(", 1) # Own the fixed-cadence return slot; s0/s1/s2, RA and SP retain the exact # retail epilogue. No VI receive is invoked or claimed to have completed. - return prefix+split+"\n MEM_W(0X28, ctx->r29) = 2;\n"+body[body.index(epilogue):] + return prefix + split + "\n MEM_W(0X28, ctx->r29) = 2;\n" + body[body.index(epilogue) :] def authored_declarations(full_scenes=False): # funcs.h closes its original extern-C block. New guest entries need their # own linkage block because both C and C++ private owners include it. - menu = ''.join(f'void {name}(uint8_t*, recomp_context*);\n' for name in ( - "dkr_probe_scene_resume_menu_cpu", "dkr_probe_menu_unload_cpu", "dkr_probe_menu_load_cpu", - "dkr_probe_menu_background_cpu", "dkr_probe_next_ingame_cpu")) if full_scenes else '' - return '\n#ifdef __cplusplus\nextern "C" {\n#endif\n' + \ - 'void dkr_probe_authored_main_cpu(uint8_t*, recomp_context*);\n' + \ - 'void dkr_probe_authored_video_cpu(uint8_t*, recomp_context*);\n' + \ - 'void dkr_probe_scene_unload_cpu(uint8_t*, recomp_context*);\n' + \ - 'void dkr_probe_scene_load_cpu(uint8_t*, recomp_context*);\n' + \ - 'void dkr_probe_level_load_cpu(uint8_t*, recomp_context*);\n' + \ - 'void dkr_probe_scene_resume_main_cpu(uint8_t*, recomp_context*);\n' + \ - 'void dkr_probe_scene_resume_mode_cpu(uint8_t*, recomp_context*);\n' + \ - menu + '#ifdef __cplusplus\n}\n#endif\n' + menu = ( + "".join( + f"void {name}(uint8_t*, recomp_context*);\n" + for name in ( + "dkr_probe_scene_resume_menu_cpu", + "dkr_probe_menu_unload_cpu", + "dkr_probe_menu_load_cpu", + "dkr_probe_menu_background_cpu", + "dkr_probe_next_ingame_cpu", + ) + ) + if full_scenes + else "" + ) + return ( + '\n#ifdef __cplusplus\nextern "C" {\n#endif\n' + + "void dkr_probe_authored_main_cpu(uint8_t*, recomp_context*);\n" + + "void dkr_probe_authored_video_cpu(uint8_t*, recomp_context*);\n" + + "void dkr_probe_scene_unload_cpu(uint8_t*, recomp_context*);\n" + + "void dkr_probe_scene_load_cpu(uint8_t*, recomp_context*);\n" + + "void dkr_probe_level_load_cpu(uint8_t*, recomp_context*);\n" + + "void dkr_probe_scene_resume_main_cpu(uint8_t*, recomp_context*);\n" + + "void dkr_probe_scene_resume_mode_cpu(uint8_t*, recomp_context*);\n" + + menu + + "#ifdef __cplusplus\n}\n#endif\n" + ) def authored_main_cpu(body, revision, full_scenes=False): if hashlib.sha256(body.encode()).hexdigest() != MAIN_CPU_HASH.get(revision): raise ValueError("Unreviewed authored main-loop phase") for name in ("hi", "lo", "result", "c1cs"): - if len(re.findall(r"\b"+name+r"\b", strip(body))) != 1: + if len(re.findall(r"\b" + name + r"\b", strip(body))) != 1: raise ValueError("Live main-loop host temporary crosses the CPU boundary") offset = 0 if revision == 77 else 0x240 - start = f" // 0x{0x8006C724+offset:08X}: lw" - cleanup = f"L_{0x8006CA3C+offset:08X}:" - copy_tail = f"L_{0x8006CA64+offset:08X}:" - timer_reset = f"L_{0x8006CA34+offset:08X}:" - video_wait = f"L_{0x8006CAA8+offset:08X}:" - epilogue = f"L_{0x8006CAD4+offset:08X}:" + start = f" // 0x{0x8006C724 + offset:08X}: lw" + cleanup = f"L_{0x8006CA3C + offset:08X}:" + copy_tail = f"L_{0x8006CA64 + offset:08X}:" + timer_reset = f"L_{0x8006CA34 + offset:08X}:" + video_wait = f"L_{0x8006CAA8 + offset:08X}:" + epilogue = f"L_{0x8006CAD4 + offset:08X}:" for marker in (start, " after_30:", cleanup, copy_tail, timer_reset, video_wait, epilogue): if body.count(marker) != 1: raise ValueError("Authored main-loop instruction boundary changed") - cpu = body[body.index(start):body.index(" after_30:")] + " after_30:\n" + cpu = body[body.index(start) : body.index(" after_30:")] + " after_30:\n" # The separate driver confirms after replay. Never invoke the stable # authority commitment from inside a speculative CPU tick. stable = "extern void dkr_netplay_authoritative_frame_commit(uint8_t*, recomp_context*); " @@ -357,32 +457,41 @@ def authored_main_cpu(body, revision, full_scenes=False): if cpu.count(stable) != 1 or cpu.count(call) != 1 or cpu.count(" mode_game(rdram, ctx);") != 1: raise ValueError("Authored main-loop owner/continuation sites changed") cpu = cpu.replace(stable, "", 1).replace(call, "", 1) - cpu = cpu.replace(" mode_game(rdram, ctx);", - " mode_game(rdram, ctx);\n if (dkr_probe_scene_pending()) return;", 1) + cpu = cpu.replace( + " mode_game(rdram, ctx);", " mode_game(rdram, ctx);\n if (dkr_probe_scene_pending()) return;", 1 + ) if full_scenes: - if cpu.count(" mode_menu(rdram, ctx);")!=1 or cpu.count(" after_18:")!=1: + if cpu.count(" mode_menu(rdram, ctx);") != 1 or cpu.count(" after_18:") != 1: raise ValueError("Authored main menu continuation site changed") - cpu=cpu.replace(" mode_menu(rdram, ctx);", - " mode_menu(rdram, ctx);\n if (dkr_probe_scene_pending()) return;",1) + cpu = cpu.replace( + " mode_menu(rdram, ctx);", " mode_menu(rdram, ctx);\n if (dkr_probe_scene_pending()) return;", 1 + ) # Retain the exact timer reset, allocator/camera tail and framebuffer CPU # copy. Only the scheduler wait and VI-derived cadence remain outside this # fixed-30-Hz phase. A confirmed loader sets timer=2; dropping its copy or # leaving timer=1 forever would corrupt the very next authored frame. - timer_address=0x800DD3F0 if revision==77 else 0x800DD960 + timer_address = 0x800DD3F0 if revision == 77 else 0x800DD960 tail = f" if (MEM_B(0, (gpr)(int32_t)0x{timer_address:08X}U) != 1) goto {cleanup[:-1]};\n" - tail += body[body.index(timer_reset):body.index(video_wait)] + video_wait + "\n" - end = body[body.index(epilogue):] - pending_menu=''' if (dkr_probe_scene_pending() >= 3) { + tail += body[body.index(timer_reset) : body.index(video_wait)] + video_wait + "\n" + end = body[body.index(epilogue) :] + pending_menu = ( + """ if (dkr_probe_scene_pending() >= 3) { mode_menu(rdram, ctx); if (dkr_probe_scene_pending()) return; goto after_18; } -''' if full_scenes else '' - return '''RECOMP_FUNC void dkr_probe_authored_main_cpu(uint8_t* rdram, recomp_context* ctx) { +""" + if full_scenes + else "" + ) + return ( + """RECOMP_FUNC void dkr_probe_authored_main_cpu(uint8_t* rdram, recomp_context* ctx) { uint64_t hi = 0, lo = 0, result = 0; int c1cs = 0; if (dkr_probe_scene_pending()) { -''' + pending_menu + ''' mode_game(rdram, ctx); +""" + + pending_menu + + """ mode_game(rdram, ctx); if (dkr_probe_scene_pending()) return; goto after_19; } @@ -390,17 +499,37 @@ def authored_main_cpu(body, revision, full_scenes=False): // Preserve the exact guest frame and return-address layout for scene cuts. ctx->r29 = ADD32(ctx->r29, -0X18); MEM_W(0X14, ctx->r29) = ctx->r31; -''' + f" ctx->r7 = (gpr)(int32_t)0x{0x801234E8+(0 if revision==77 else 0x580):08X}U;\n" + cpu + tail + end +""" + + f" ctx->r7 = (gpr)(int32_t)0x{0x801234E8 + (0 if revision == 77 else 0x580):08X}U;\n" + + cpu + + tail + + end + ) + # The retail audio filter/player constructors assign exactly these callbacks. # This is an immutable private dispatch table, never a live get_function or a # permissive native fallback. Unknown/custom code pointers remain fenced. AUDIO_CALLBACKS = ( - "sndp_voice_handler", "__CSPVoiceHandler", "__amDMA", - "alSavePull", "alSaveParam", "alMainBusPull", "alMainBusParam", - "alAuxBusPull", "alAuxBusParam", "alFxPull", "alFxParam", "alFxParamHdl", - "alEnvmixerPull", "alEnvmixerParam", "alResamplePull", "alResampleParam", - "alAdpcmPull", "alRaw16Pull", "alLoadParam", + "sndp_voice_handler", + "__CSPVoiceHandler", + "__amDMA", + "alSavePull", + "alSaveParam", + "alMainBusPull", + "alMainBusParam", + "alAuxBusPull", + "alAuxBusParam", + "alFxPull", + "alFxParam", + "alFxParamHdl", + "alEnvmixerPull", + "alEnvmixerParam", + "alResamplePull", + "alResampleParam", + "alAdpcmPull", + "alRaw16Pull", + "alLoadParam", ) AUDIO_SYMBOL_HASH = { 77: "fb8b4f0e790c35b1c93e8e3221670bd253432ac6a60ff33f575a259366341082", @@ -568,7 +697,11 @@ def audio_rsp_payload(rsp_source=None, dependencies_dir=None): """Read only; return checked private copies, never modify the RSP/runtime.""" sources = {} for name, expected in RSP_INPUT_HASH.items(): - source_path = (rsp_source / Path(name).name) if rsp_source is not None and name.startswith("runtime-recomp/RecompiledRSP/") else audited_input_path(name, dependencies_dir) + source_path = ( + (rsp_source / Path(name).name) + if rsp_source is not None and name.startswith("runtime-recomp/RecompiledRSP/") + else audited_input_path(name, dependencies_dir) + ) raw = source_path.read_bytes() if hashlib.sha256(raw).hexdigest() != expected: # The reviewed Windows-generated input uses CRLF, whereas a Git @@ -581,28 +714,39 @@ def audio_rsp_payload(rsp_source=None, dependencies_dir=None): raw = pinned_newlines sources[Path(name).name] = raw.decode() header = sources["rsp.hpp"] - header = header.replace('#include "ultramodern/ultra64.h"', - '#include \nstruct OSTask; // Opaque: private probe never calls the native scheduler.', 1) + header = header.replace( + '#include "ultramodern/ultra64.h"', + "#include \nstruct OSTask; // Opaque: private probe never calls the native scheduler.", + 1, + ) # Replace ONLY the hash-reviewed DMA implementations in our private copy. # The retail 24-bit/8-byte hardware address normalization is unchanged. # Validate full RAM/DMEM spans once, before writing, rather than calling # the scalar guest-memory trap/budget checker for every transferred byte. - for direction, length, write in (("rdram_to_dmem", "rd_len", 0), - ("dmem_to_rdram", "wr_len", 1)): + for direction, length, write in (("rdram_to_dmem", "rd_len", 0), ("dmem_to_rdram", "wr_len", 1)): signature = f"static inline void dma_{direction}(uint8_t* rdram, uint32_t dmem_addr, uint32_t dram_addr, uint32_t {length})" pattern = re.escape(signature) + r" \{\n.*?^\}" - replacement = signature + " {\n" + ( - f" dkr_probe_audio_dma(rdram, dmem, dmem_addr, dram_addr & 0xFFFFF8U, {length}, {write}, __FILE__, __LINE__);\n}}") + replacement = ( + signature + + " {\n" + + ( + f" dkr_probe_audio_dma(rdram, dmem, dmem_addr, dram_addr & 0xFFFFF8U, {length}, {write}, __FILE__, __LINE__);\n}}" + ) + ) header, count = re.subn(pattern, lambda _: replacement, header, flags=re.S | re.M) if count != 1: raise ValueError("Unreviewed private RSP DMA implementation shape") sources["rsp.hpp"] = header source = sources.pop("aspMain.cpp") - source = '#include \n'+source - source = source.replace(" RSP rsp{};", " static_assert(std::is_trivially_destructible_v);\n RSP rsp{};", 1) + source = "#include \n" + source + source = source.replace( + " RSP rsp{};", " static_assert(std::is_trivially_destructible_v);\n RSP rsp{};", 1 + ) # The RSP function has only primitive/trivially destructible locals; the # private C trap may cross it. Every loop/dispatcher label is budgeted. - source, count = re.subn(r"^(L_[A-Fa-f0-9]+|do_indirect_jump):\s*$", r"\1: dkr_probe_checkpoint();", source, flags=re.M) + source, count = re.subn( + r"^(L_[A-Fa-f0-9]+|do_indirect_jump):\s*$", r"\1: dkr_probe_checkpoint();", source, flags=re.M + ) if count < 30: raise ValueError("Unreviewed RSP control-flow budget shape") return sources, source @@ -628,94 +772,122 @@ def audio_dispatch(functions, revision, dependencies_dir=None): def menu_dispatch(functions, revision, dependencies_dir=None): pins = MENU_INDIRECT_HASH.get(revision) - if not pins or any(name not in functions or hashlib.sha256(functions[name].encode()).hexdigest()!=expected - for name,expected in pins.items()): + if not pins or any( + name not in functions or hashlib.sha256(functions[name].encode()).hexdigest() != expected + for name, expected in pins.items() + ): raise ValueError("Unreviewed retail menu indirect-call boundary") path = audited_input_path(f"extern/dkr-decomp/ver/symbols/symbol_addrs.us.v{revision}.txt", dependencies_dir) data = path.read_bytes() - if hashlib.sha256(data).hexdigest()!=AUDIO_SYMBOL_HASH[revision]: + if hashlib.sha256(data).hexdigest() != AUDIO_SYMBOL_HASH[revision]: raise ValueError("Unreviewed retail menu address map") - symbols=dict(re.findall(r"^(\w+)\s*=\s*(0x[0-9A-Fa-f]+);",data.decode(),re.M)) - result={} + symbols = dict(re.findall(r"^(\w+)\s*=\s*(0x[0-9A-Fa-f]+);", data.decode(), re.M)) + result = {} for name in MENU_CALLBACKS: - entry=re.search(r"// (0x[0-9A-Fa-f]+):",functions[name]) - address=int(symbols[name],16) - if not entry or int(entry[1],16)!=address or address in result: + entry = re.search(r"// (0x[0-9A-Fa-f]+):", functions[name]) + address = int(symbols[name], 16) + if not entry or int(entry[1], 16) != address or address in result: raise ValueError("Menu callback address mismatch") - result[address]=name + result[address] = name return result def indirect_stub(audio, menu): - if set(audio)&set(menu):raise ValueError("Overlapping indirect owner permissions") - cases=[] - for dispatch,permission,label in ((audio,"dkr_probe_audio_enabled","audio"), - (menu,"dkr_probe_menu_callback_enabled","menu")): - for address,target in sorted(dispatch.items()): - cases.append(f'case 0x{address:08X}U: if (!{permission}()) dkr_probe_block("unowned-{label}-indirect-dispatch"); return {target};') - return 'recomp_func_t* get_function(int32_t vram) {\n' + \ - 'dkr_probe_checkpoint();\nswitch ((uint32_t)vram) {\n' + '\n'.join(cases) + \ - '\ndefault: dkr_probe_block("unreviewed-indirect-callback");\n}\n}' + if set(audio) & set(menu): + raise ValueError("Overlapping indirect owner permissions") + cases = [] + for dispatch, permission, label in ( + (audio, "dkr_probe_audio_enabled", "audio"), + (menu, "dkr_probe_menu_callback_enabled", "menu"), + ): + for address, target in sorted(dispatch.items()): + cases.append( + f'case 0x{address:08X}U: if (!{permission}()) dkr_probe_block("unowned-{label}-indirect-dispatch"); return {target};' + ) + return ( + "recomp_func_t* get_function(int32_t vram) {\n" + + "dkr_probe_checkpoint();\nswitch ((uint32_t)vram) {\n" + + "\n".join(cases) + + '\ndefault: dkr_probe_block("unreviewed-indirect-callback");\n}\n}' + ) def mode_scene_cuts(body, revision): if revision not in MODE_SCENE_HASH or hashlib.sha256(body.encode()).hexdigest() != MODE_SCENE_HASH[revision]: raise ValueError("Unreviewed mode_game continuation: audit guest/host liveness before splitting") for name in ("hi", "lo", "result", "c1cs"): - if len(re.findall(r"\b"+name+r"\b", strip(body))) != 1: + if len(re.findall(r"\b" + name + r"\b", strip(body))) != 1: raise ValueError("Live host temporary cannot be checkpointed across a scene cut") - marker=" int c1cs = 0;" - if body.count(marker)!=1 or body.count(" unload_level_game(rdram, ctx);")!=2: + marker = " int c1cs = 0;" + if body.count(marker) != 1 or body.count(" unload_level_game(rdram, ctx);") != 2: raise ValueError("Scene-cut prologue/call sites changed") - entry=''' + entry = """ switch (dkr_probe_scene_entry()) { case 0: break; case 1: goto dkr_probe_unload_1; case 2: goto dkr_probe_unload_2; default: dkr_probe_block("invalid-mode-scene-continuation"); } -''' - body=body.replace(marker,marker+entry,1) - needle=" unload_level_game(rdram, ctx);" - parts=body.split(needle) - return parts[0]+'''dkr_probe_unload_1: +""" + body = body.replace(marker, marker + entry, 1) + needle = " unload_level_game(rdram, ctx);" + parts = body.split(needle) + return ( + parts[0] + + """dkr_probe_unload_1: if (dkr_probe_scene_cut(1)) return; -'''+needle+parts[1]+'''dkr_probe_unload_2: +""" + + needle + + parts[1] + + """dkr_probe_unload_2: if (dkr_probe_scene_cut(2)) return; -'''+needle+parts[2] +""" + + needle + + parts[2] + ) -def owned_scene_resume(main,mode,revision,menu=None): +def owned_scene_resume(main, mode, revision, menu=None): # Reuse the EXACT previously audited guest/host continuation boundaries. # No guessed stack unwind or second old-scene object/render update. - cpu=authored_main_cpu(main,revision,menu is not None).replace('void dkr_probe_authored_main_cpu(', - 'void dkr_probe_scene_resume_main_cpu(',1).replace('mode_game(rdram, ctx);', - 'dkr_probe_scene_resume_mode_cpu(rdram, ctx);') - cpu=cpu.replace(' int c1cs = 0;',' int c1cs = 0;\n if (!dkr_probe_scene_pending()) dkr_probe_block("no-confirmed-scene-continuation");',1) - child=mode_scene_cuts(mode,revision).replace('void mode_game(','void dkr_probe_scene_resume_mode_cpu(',1) - child=child.replace('dkr_probe_scene_cut(', 'dkr_probe_scene_transaction_cut(') - child=child.replace('unload_level_game(rdram, ctx);','dkr_probe_scene_unload_cpu(rdram, ctx);') - child=child.replace('load_level_game(rdram, ctx);','dkr_probe_scene_load_cpu(rdram, ctx);') + cpu = ( + authored_main_cpu(main, revision, menu is not None) + .replace("void dkr_probe_authored_main_cpu(", "void dkr_probe_scene_resume_main_cpu(", 1) + .replace("mode_game(rdram, ctx);", "dkr_probe_scene_resume_mode_cpu(rdram, ctx);") + ) + cpu = cpu.replace( + " int c1cs = 0;", + ' int c1cs = 0;\n if (!dkr_probe_scene_pending()) dkr_probe_block("no-confirmed-scene-continuation");', + 1, + ) + child = mode_scene_cuts(mode, revision).replace("void mode_game(", "void dkr_probe_scene_resume_mode_cpu(", 1) + child = child.replace("dkr_probe_scene_cut(", "dkr_probe_scene_transaction_cut(") + child = child.replace("unload_level_game(rdram, ctx);", "dkr_probe_scene_unload_cpu(rdram, ctx);") + child = child.replace("load_level_game(rdram, ctx);", "dkr_probe_scene_load_cpu(rdram, ctx);") if menu is not None: - cpu=cpu.replace('mode_menu(rdram, ctx);','dkr_probe_scene_resume_menu_cpu(rdram, ctx);') - child=child.replace('load_menu_with_level_background(rdram, ctx);','dkr_probe_menu_background_cpu(rdram, ctx);') - if child.count(' return;')!=1: + cpu = cpu.replace("mode_menu(rdram, ctx);", "dkr_probe_scene_resume_menu_cpu(rdram, ctx);") + child = child.replace( + "load_menu_with_level_background(rdram, ctx);", "dkr_probe_menu_background_cpu(rdram, ctx);" + ) + if child.count(" return;") != 1: raise ValueError("Confirmed mode continuation epilogue changed") - child=child.replace(' return;',' dkr_probe_scene_transaction_end();\n return;',1) - if menu is None: return cpu,child - menu_child=menu_scene_cuts(menu,revision).replace('void mode_menu(', 'void dkr_probe_scene_resume_menu_cpu(',1) - for original,owned in ( - ('dkr_probe_scene_cut','dkr_probe_scene_transaction_cut'), - ('unload_level_game','dkr_probe_scene_unload_cpu'), - ('unload_level_menu','dkr_probe_menu_unload_cpu'), - ('load_level_game','dkr_probe_scene_load_cpu'), - ('load_menu_with_level_background','dkr_probe_menu_background_cpu'), - ('load_next_ingame_level','dkr_probe_next_ingame_cpu')): - menu_child=menu_child.replace(original+'(',owned+'(') - if menu_child.count(' return;')!=1: + child = child.replace(" return;", " dkr_probe_scene_transaction_end();\n return;", 1) + if menu is None: + return cpu, child + menu_child = menu_scene_cuts(menu, revision).replace("void mode_menu(", "void dkr_probe_scene_resume_menu_cpu(", 1) + for original, owned in ( + ("dkr_probe_scene_cut", "dkr_probe_scene_transaction_cut"), + ("unload_level_game", "dkr_probe_scene_unload_cpu"), + ("unload_level_menu", "dkr_probe_menu_unload_cpu"), + ("load_level_game", "dkr_probe_scene_load_cpu"), + ("load_menu_with_level_background", "dkr_probe_menu_background_cpu"), + ("load_next_ingame_level", "dkr_probe_next_ingame_cpu"), + ): + menu_child = menu_child.replace(original + "(", owned + "(") + if menu_child.count(" return;") != 1: raise ValueError("Confirmed menu continuation epilogue changed") - menu_child=menu_child.replace(' return;',' dkr_probe_scene_transaction_end();\n return;',1) - return cpu,child,menu_child + menu_child = menu_child.replace(" return;", " dkr_probe_scene_transaction_end();\n return;", 1) + return cpu, child, menu_child def strip(source): @@ -734,12 +906,27 @@ def declarations(source): # signature we cannot preserve instead of guessing an ABI. if not re.fullmatch(r"(?:const\s+)?(?:unsigned\s+)?[A-Za-z_]\w*(?:\s*\*)*(?:\s+\w+)?", param): raise ValueError(f"Unsupported native signature {name}: {param}") - known = {"void", "int", "float", "double", "gpr", "uint8_t", "int8_t", - "uint16_t", "int16_t", "uint32_t", "int32_t", "uint64_t", "int64_t", - "uintptr_t", "recomp_context", "char"} + known = { + "void", + "int", + "float", + "double", + "gpr", + "uint8_t", + "int8_t", + "uint16_t", + "int16_t", + "uint32_t", + "int32_t", + "uint64_t", + "int64_t", + "uintptr_t", + "recomp_context", + "char", + } words = param.split() if len(words) > 1 and words[-1] not in known and "*" not in words[-1]: - param = param[:param.rfind(words[-1])].strip() + param = param[: param.rfind(words[-1])].strip() # Handle uint8_t* rdram and uint8_t *rdram as well. param = re.sub(r"(\*)\s*[A-Za-z_]\w*$", r"\1", param) param = re.sub(r"\s*\*\s*", "*", param) @@ -772,8 +959,7 @@ def local_world_observation_body(body, revision): """ if hashlib.sha256(body.encode()).hexdigest() != LOCAL_WORLD_SOURCE_HASH[revision]: raise ValueError("Unreviewed world-render scenery observation source") - calls = (" sort_objects_by_dist(rdram, ctx);\n", - " func_80012C3C(rdram, ctx);\n") + calls = (" sort_objects_by_dist(rdram, ctx);\n", " func_80012C3C(rdram, ctx);\n") observations = tuple(f" dkr_probe_local_world_draw(rdram, ctx, {phase});\n" for phase in range(2)) patched = body for call, observation in zip(calls, observations): @@ -803,21 +989,30 @@ def boss_finish_diagnostic_body(body, revision): raise ValueError("Unexpected boss-finish entry/return shape") body = body.replace(opening, opening + ' dkr_probe_boss_trace("begin", rdram, ctx);\n', 1) expected_calls = { - "get_settings": 1, "get_misc_asset": 2, "music_play": 2, - "level_properties_push": 22, "set_eeprom_settings_value": 1, - "level_transition_begin": 6, "instShowBearBar": 2, - "is_in_two_player_adventure": 1, "swap_lead_player": 1, - "get_save_file_index": 1, "safe_mark_write_save_file": 1, + "get_settings": 1, + "get_misc_asset": 2, + "music_play": 2, + "level_properties_push": 22, + "set_eeprom_settings_value": 1, + "level_transition_begin": 6, + "instShowBearBar": 2, + "is_in_two_player_adventure": 1, + "swap_lead_player": 1, + "get_save_file_index": 1, + "safe_mark_write_save_file": 1, } for callee, expected in expected_calls.items(): call = f" {callee}(rdram, ctx);\n" if body.count(call) != expected: raise ValueError(f"Unexpected boss-finish call count: {callee}") - body = body.replace(call, - f' dkr_probe_boss_trace("before-{callee}", rdram, ctx);\n' + call + - f' dkr_probe_boss_trace("after-{callee}", rdram, ctx);\n') + body = body.replace( + call, + f' dkr_probe_boss_trace("before-{callee}", rdram, ctx);\n' + + call + + f' dkr_probe_boss_trace("after-{callee}", rdram, ctx);\n', + ) body = body.replace(" return;\n", ' dkr_probe_boss_trace("end", rdram, ctx);\n return;\n', 1) - stripped = re.sub(r'^ dkr_probe_boss_trace\("[\w-]+", rdram, ctx\);\n', '', body, flags=re.M) + stripped = re.sub(r'^ dkr_probe_boss_trace\("[\w-]+", rdram, ctx\);\n', "", body, flags=re.M) if stripped != original: raise ValueError("Boss diagnostics changed original guest instructions") return body @@ -826,23 +1021,33 @@ def boss_finish_diagnostic_body(body, revision): def instrument_function(name, body): # Read-only renderer observations. Preserve every original instruction, # return and native function call; only a fresh private output is written. - mesh=name=='render_mesh' + mesh = name == "render_mesh" if mesh: - body=body.replace('{','{\n dkr_owned_draw_event dkr_mesh_event;\n dkr_probe_mesh_begin(rdram,ctx,&dkr_mesh_event);',1) - observed=name in ('viewport_reset','divider_draw','divider_clear_coverage','hud_render_general','hud_timer_render') - observe=f' dkr_probe_online_function("{name}", 1, rdram, ctx);' if observed else '' - if mesh:observe+=' dkr_probe_mesh_end(rdram,ctx,&dkr_mesh_event);' + body = body.replace( + "{", "{\n dkr_owned_draw_event dkr_mesh_event;\n dkr_probe_mesh_begin(rdram,ctx,&dkr_mesh_event);", 1 + ) + observed = name in ( + "viewport_reset", + "divider_draw", + "divider_clear_coverage", + "hud_render_general", + "hud_timer_render", + ) + observe = f' dkr_probe_online_function("{name}", 1, rdram, ctx);' if observed else "" + if mesh: + observe += " dkr_probe_mesh_end(rdram,ctx,&dkr_mesh_event);" if observed: - body=body.replace('{', '{\n dkr_probe_online_function("'+name+'", 0, rdram, ctx);',1) + body = body.replace("{", '{\n dkr_probe_online_function("' + name + '", 0, rdram, ctx);', 1) body = body.replace("{", '{\n dkr_probe_enter("' + name + '");', 1) # Keep an unbraced `if (hook()) return;` a single statement. Injecting two # unbraced statements would accidentally make the return unconditional. - body = re.sub(r"\breturn\s*;", "{"+observe+" dkr_probe_leave(); return; }", body) + body = re.sub(r"\breturn\s*;", "{" + observe + " dkr_probe_leave(); return; }", body) # Some emitted functions have an implicit void return. Cover fallthrough # as well; after an explicit return this statement is simply unreachable. closing = body.rfind("}") - if closing < 0: raise ValueError(f"Guest function has no closing brace: {name}") - body = body[:closing] + "\n "+observe+"\n dkr_probe_leave();\n" + body[closing:] + if closing < 0: + raise ValueError(f"Guest function has no closing brace: {name}") + body = body[:closing] + "\n " + observe + "\n dkr_probe_leave();\n" + body[closing:] body = re.sub(r"^(L_[A-Fa-f0-9]+:)\s*$", r"\1 dkr_probe_checkpoint();", body, flags=re.M) return body @@ -867,71 +1072,147 @@ def native_stub(name, signature, full_scenes=False): audited_assets = { "dkr_track_select_draw": ("unsigned", ["uint8_t*", "recomp_context*", "unsigned", "unsigned"]), "dkr_legacy_asset_api": ("int", ["uint8_t*", "recomp_context*", "unsigned"]), - **{n: ("void", ["uint8_t*", "recomp_context*"]) for n in ( - "dkr_legacy_pi_start_dma", "osPiStartDma_recomp", "osRecvMesg_recomp", - "osInvalDCache_recomp", "dkr_v11_asset_mutex_acquire", "dkr_v11_asset_mutex_release", - "dkr_custom_tracks_asset_load_begin", "dkr_custom_tracks_asset_load_end", - "dkr_custom_tracks_table_load_begin", "dkr_custom_tracks_table_load_end")}, - **{n: ("void", ["uint8_t*", "recomp_context*"]) for n in ( - "osEepromProbe_recomp", "osEepromRead_recomp", "osEepromWrite_recomp", - "osEepromLongRead_recomp", "osEepromLongWrite_recomp")}, - **{n: ("void", ["uint8_t*", "recomp_context*"]) for n in ( - "__f_to_ll_recomp", "__ll_to_f_recomp", "__ull_rshift_recomp", - "__ll_lshift_recomp", "__ull_div_recomp", "__ull_rem_recomp")}, - **{n: ("void", ["uint8_t*", "recomp_context*"]) for n in ( - "osGetCount_recomp", "osGetTime_recomp", "osSetTime_recomp")}, + **{ + n: ("void", ["uint8_t*", "recomp_context*"]) + for n in ( + "dkr_legacy_pi_start_dma", + "osPiStartDma_recomp", + "osRecvMesg_recomp", + "osInvalDCache_recomp", + "dkr_v11_asset_mutex_acquire", + "dkr_v11_asset_mutex_release", + "dkr_custom_tracks_asset_load_begin", + "dkr_custom_tracks_asset_load_end", + "dkr_custom_tracks_table_load_begin", + "dkr_custom_tracks_table_load_end", + ) + }, + **{ + n: ("void", ["uint8_t*", "recomp_context*"]) + for n in ( + "osEepromProbe_recomp", + "osEepromRead_recomp", + "osEepromWrite_recomp", + "osEepromLongRead_recomp", + "osEepromLongWrite_recomp", + ) + }, + **{ + n: ("void", ["uint8_t*", "recomp_context*"]) + for n in ( + "__f_to_ll_recomp", + "__ll_to_f_recomp", + "__ull_rshift_recomp", + "__ll_lshift_recomp", + "__ull_div_recomp", + "__ull_rem_recomp", + ) + }, + **{ + n: ("void", ["uint8_t*", "recomp_context*"]) + for n in ("osGetCount_recomp", "osGetTime_recomp", "osSetTime_recomp") + }, "dkr_audio_event_queue_next": ("uint32_t", ["uint8_t*", "recomp_context*"]), "dkr_audio_event_queue_initialize": ("void", ["uint8_t*", "recomp_context*"]), - **{n: ("int", ["uint8_t*", "recomp_context*"]) for n in ( - "dkr_audio_voice_guard", "dkr_audio_bus_guard")}, - **{n: ("void", ["uint8_t*", "recomp_context*"]) for n in ( - "osContGetReadData_recomp", "osContStartReadData_recomp", - "osPfsIsPlug_recomp", "osMotorInit_recomp", "osMotorStart_recomp", - "osMotorStop_recomp", "__osMotorAccess_recomp", - "dkr_netplay_resolve_authored_input_frame")}, - **{n: ("void", ["uint8_t*", "recomp_context*"]) for n in ( - "osPfsInitPak_recomp", "osPfsInit_recomp", "osPfsFreeBlocks_recomp", "osPfsNumFiles_recomp", - "osPfsFindFile_recomp", "osPfsAllocateFile_recomp", "osPfsDeleteFile_recomp", "osPfsFileState_recomp", - "osPfsReadWriteFile_recomp", "osPfsChecker_recomp", "osPfsRepairId_recomp", - "__osPfsSelectBank_recomp", "__osContRamWrite_recomp", "__osPfsGetStatus_recomp", - "__osGetId_recomp", "__osContRamRead_recomp")}, - **{n: ("int", ["uint8_t*", "recomp_context*"]) for n in ( - "dkr_virtual_pak_preferred_status", "dkr_virtual_pak_reformat")}, - **{n: ("void", ["uint8_t*", "recomp_context*"]) for n in ( - "dkr_apply_launch_magic_codes", "dkr_magic_code_credits_started", - "dkr_magic_code_balloon_awarded", "dkr_magic_codes_frame_complete", - "dkr_fix_fullscreen_clear_scissor", "osViBlack_recomp")}, - **{n: ("void", ["uint8_t*", "recomp_context*"]) for n in ( - "dkr_title_intro_audio_tail", "dkr_netplay_character_select_enter", - "dkr_netplay_character_select_lock", "dkr_netplay_character_select_ai_seed", - "dkr_character_select_music_unblock", "dkr_character_select_music_mask", - "dkr_character_menu_music_mask", "dkr_character_select_animation_tick", - "dkr_character_select_animation_fraction", "dkr_track_select_fullscreen_preview", - "dkr_track_select_background_cover", "dkr_track_select_lens_flare_tint_begin", - "dkr_track_select_lens_flare_tint_end", "dkr_custom_tracks_track_id_override")}, + **{n: ("int", ["uint8_t*", "recomp_context*"]) for n in ("dkr_audio_voice_guard", "dkr_audio_bus_guard")}, + **{ + n: ("void", ["uint8_t*", "recomp_context*"]) + for n in ( + "osContGetReadData_recomp", + "osContStartReadData_recomp", + "osPfsIsPlug_recomp", + "osMotorInit_recomp", + "osMotorStart_recomp", + "osMotorStop_recomp", + "__osMotorAccess_recomp", + "dkr_netplay_resolve_authored_input_frame", + ) + }, + **{ + n: ("void", ["uint8_t*", "recomp_context*"]) + for n in ( + "osPfsInitPak_recomp", + "osPfsInit_recomp", + "osPfsFreeBlocks_recomp", + "osPfsNumFiles_recomp", + "osPfsFindFile_recomp", + "osPfsAllocateFile_recomp", + "osPfsDeleteFile_recomp", + "osPfsFileState_recomp", + "osPfsReadWriteFile_recomp", + "osPfsChecker_recomp", + "osPfsRepairId_recomp", + "__osPfsSelectBank_recomp", + "__osContRamWrite_recomp", + "__osPfsGetStatus_recomp", + "__osGetId_recomp", + "__osContRamRead_recomp", + ) + }, + **{ + n: ("int", ["uint8_t*", "recomp_context*"]) + for n in ("dkr_virtual_pak_preferred_status", "dkr_virtual_pak_reformat") + }, + **{ + n: ("void", ["uint8_t*", "recomp_context*"]) + for n in ( + "dkr_apply_launch_magic_codes", + "dkr_magic_code_credits_started", + "dkr_magic_code_balloon_awarded", + "dkr_magic_codes_frame_complete", + "dkr_fix_fullscreen_clear_scissor", + "osViBlack_recomp", + ) + }, + **{ + n: ("void", ["uint8_t*", "recomp_context*"]) + for n in ( + "dkr_title_intro_audio_tail", + "dkr_netplay_character_select_enter", + "dkr_netplay_character_select_lock", + "dkr_netplay_character_select_ai_seed", + "dkr_character_select_music_unblock", + "dkr_character_select_music_mask", + "dkr_character_menu_music_mask", + "dkr_character_select_animation_tick", + "dkr_character_select_animation_fraction", + "dkr_track_select_fullscreen_preview", + "dkr_track_select_background_cover", + "dkr_track_select_lens_flare_tint_begin", + "dkr_track_select_lens_flare_tint_end", + "dkr_custom_tracks_track_id_override", + ) + }, } if name in audited_assets and signature != audited_assets[name]: raise ValueError(f"Unreviewed owned-asset ABI: {name}") # The Play checkbox is offline-only. Private checkpoint replay must not # consult launcher state, even if a stale request exists during handoff. - if name in ("dkr_skip_intro_audio", "dkr_skip_intro_logos", "dkr_skip_intro_title", "dkr_skip_intro_character_select"): + if name in ( + "dkr_skip_intro_audio", + "dkr_skip_intro_logos", + "dkr_skip_intro_title", + "dkr_skip_intro_character_select", + ): if signature != ("void", ["uint8_t*", "recomp_context*"]): raise ValueError("Unreviewed startup-skip ABI") - return f'void {name}({params}) {{ /* Offline preference: disabled in owned replay. */ }}' + return f"void {name}({params}) {{ /* Offline preference: disabled in owned replay. */ }}" # water_profile.cpp: these diagnostic-only hooks return before reading RAM # or changing any state when DKR_WATER_PROFILE is disabled. The private # services explicitly keep it disabled; no generic scalar-import bypass. if name in ("dkr_water_profile_begin", "dkr_water_profile_end"): expected = ("void", ["uint32_t"]) if name.endswith("begin") else ("void", ["uint8_t*", "uint32_t"]) - if signature != expected: raise ValueError("Unreviewed water-profile ABI") + if signature != expected: + raise ValueError("Unreviewed water-profile ABI") return f'void {name}({params}) {{ dkr_probe_profile_disabled("{name}"); }}' # librecomp/src/recomp.cpp: these operate solely on checkpointed guest # registers. Preserve its FR handling and refusal of other changed bits. if name in ("cop0_status_read", "cop0_status_write"): expected = ("gpr", ["recomp_context*"]) if name.endswith("read") else ("void", ["recomp_context*", "gpr"]) - if signature != expected: raise ValueError("Unreviewed COP0 status ABI") + if signature != expected: + raise ValueError("Unreviewed COP0 status ABI") invocation = "dkr_probe_cop0_read(p0)" if name.endswith("read") else "dkr_probe_cop0_write(p0,p1)" - return f'{ret} {name}({params}) {{ {"return " if ret != "void" else ""}{invocation}; }}' + return f"{ret} {name}({params}) {{ {'return ' if ret != 'void' else ''}{invocation}; }}" # This pointer is a native constant field table, not a guest address. The # private no-mods service never dereferences it. Only this audited ABI may # pass it through; unknown pointer signatures remain fenced. @@ -944,14 +1225,18 @@ def native_stub(name, signature, full_scenes=False): raise ValueError("Unreviewed track-menu hook ABI") # The private owner permits ONLY notification 14 with no observed # return, never the interactive track-menu or a native field pointer. - observed_guard='' if full_scenes else 'if (p4) dkr_probe_block("unowned-track-menu-observation"); ' + observed_guard = "" if full_scenes else 'if (p4) dkr_probe_block("unowned-track-menu-observation"); ' return f'int {name}({params}) {{ {observed_guard}uint64_t args[]={{p4}}, result=0; if(dkr_probe_mod_dispatch("{name}",p0,p1,args,1,p3,p2,&result))return (int)result; return dkr_probe_native_fields("{name}", p0, p1, p2, p3); }}' - if types[:2] == ["uint8_t*", "recomp_context*"] and ret in ("void", "int", "unsigned", "uint32_t") and all(t in ("int", "unsigned", "uint32_t", "int32_t") for t in types[2:]): + if ( + types[:2] == ["uint8_t*", "recomp_context*"] + and ret in ("void", "int", "unsigned", "uint32_t") + and all(t in ("int", "unsigned", "uint32_t", "int32_t") for t in types[2:]) + ): args = ", ".join(f"(uint64_t)p{i}" for i in range(2, len(types))) if args: - return f'{ret} {name}({params}) {{ uint64_t args[] = {{{args}}}; {"return " if ret != "void" else ""}dkr_probe_native_args("{name}", p0, p1, args, {len(types)-2}); }}' + return f'{ret} {name}({params}) {{ uint64_t args[] = {{{args}}}; {"return " if ret != "void" else ""}dkr_probe_native_args("{name}", p0, p1, args, {len(types) - 2}); }}' invocation = f'dkr_probe_native("{name}", p0, p1)' - return f'{ret} {name}({params}) {{ {"return " if ret != "void" else ""}{invocation}; }}' + return f"{ret} {name}({params}) {{ {'return ' if ret != 'void' else ''}{invocation}; }}" return f'{ret} {name}({params}) {{ dkr_probe_block("{name}"); }}' @@ -969,16 +1254,30 @@ def isolation_header(guest_names, import_names, revision): if any(not re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]*", name) for name in names): raise ValueError("Invalid isolated guest/import identifier") prefix = f"dkr_experimental_v{revision}_" - return "#pragma once\n// Patch Pipeline-owned C link boundary; NOT live admission.\n" + \ - "\n".join(f"#define {name} {prefix}{name}" for name in names) + "\n" + return ( + "#pragma once\n// Patch Pipeline-owned C link boundary; NOT live admission.\n" + + "\n".join(f"#define {name} {prefix}{name}" for name in names) + + "\n" + ) def audio_isolation_header(revision): # Types matter too: inline RSP member functions otherwise retain the # stable runtime's C++ link names even when DMEM itself is renamed. - return isolation_header(("dkrAspMain", "dmem", "rspReciprocals", - "rspInverseSquareRoots", "RSP", "RspContext", "RspExitReason", - "RspUcodeFunc"), (), revision) + return isolation_header( + ( + "dkrAspMain", + "dmem", + "rspReciprocals", + "rspInverseSquareRoots", + "RSP", + "RspContext", + "RspExitReason", + "RspUcodeFunc", + ), + (), + revision, + ) def runtime_isolation_header(guest_names, import_names, revision): @@ -997,7 +1296,25 @@ def runtime_isolation_header(guest_names, import_names, revision): return isolation_header(names, (), revision) -def generate(source_dir: Path, header_path: Path, output: Path, roots, revision, scene_cuts=False, audio_services=False, input_services=False, authored_cpu=False, isolate_symbols=False, full_scenes=False, boss_finish_diagnostics=False, rsp_source=None, mod_services=False, inline_memory=False, dependencies_dir=None, code_policy_config=None): +def generate( + source_dir: Path, + header_path: Path, + output: Path, + roots, + revision, + scene_cuts=False, + audio_services=False, + input_services=False, + authored_cpu=False, + isolate_symbols=False, + full_scenes=False, + boss_finish_diagnostics=False, + rsp_source=None, + mod_services=False, + inline_memory=False, + dependencies_dir=None, + code_policy_config=None, +): source_dir, header_path, output = source_dir.resolve(), header_path.resolve(), output.resolve() if output.exists() or output == source_dir or source_dir in output.parents or header_path.parent in output.parents: raise ValueError("Probe output must be a new, separate directory; protected inputs are never rewritten") @@ -1018,60 +1335,93 @@ def generate(source_dir: Path, header_path: Path, output: Path, roots, revision, # An authored leaf staged into NEW CPU output, never a native import or # mutable global service. The same checked MEM macros instrument it here. import save_layout_pipeline + save_layout_header = save_layout_pipeline.owned_support(source_dir) import character_portrait_pipeline + character_portrait_receipt = character_portrait_pipeline.owned_receipt(source_dir, revision) functions, prototypes, origins = {}, declarations(header), {} + def merge(items): for name, signature in items.items(): if name in prototypes and prototypes[name] != signature: raise ValueError(f"Conflicting native declaration for {name}") prototypes[name] = signature + merge(PROJECT_IMPORTS) merge(declarations((source_dir / "funcs.h").read_text(encoding="utf-8"))) for file in sorted(source_dir.glob("*.c")): source = file.read_text(encoding="utf-8") matches = list(DEF.finditer(source)) for i, match in enumerate(matches): - body = source[match.start():matches[i + 1].start() if i + 1 < len(matches) else len(source)] + body = source[match.start() : matches[i + 1].start() if i + 1 < len(matches) else len(source)] name = match[1] - if name in functions: raise ValueError(f"Duplicate guest definition {name}") + if name in functions: + raise ValueError(f"Duplicate guest definition {name}") functions[name] = body merge(declarations(body)) origins[name] = {"file": str(file), "sha256": hashlib.sha256(body.encode()).hexdigest()} # Keep protection decisions tied to the complete ordinary Patch Pipeline # input, BEFORE adding checked private instrumentation/continuations. - code_originals=dict(functions) + code_originals = dict(functions) if boss_finish_diagnostics: functions["racer_boss_finish"] = boss_finish_diagnostic_body(functions["racer_boss_finish"], revision) if full_scenes: - functions["render_level_geometry_and_objects"] = local_world_observation_body(functions["render_level_geometry_and_objects"], revision) + functions["render_level_geometry_and_objects"] = local_world_observation_body( + functions["render_level_geometry_and_objects"], revision + ) if authored_cpu: if not (scene_cuts and audio_services and input_services): raise ValueError("Authored CPU requires explicit scene/audio/input ownership") functions["dkr_probe_authored_main_cpu"] = authored_main_cpu(functions["main_game_loop"], revision, full_scenes) - origins["dkr_probe_authored_main_cpu"] = {**origins["main_game_loop"], "phase": "private-fixed-cadence-cpu-only"} - functions["dkr_probe_authored_video_cpu"]=authored_video_cpu(functions["fb_update"],functions["fb_swap"],revision) - origins["dkr_probe_authored_video_cpu"]={**origins["fb_update"],"phase":"private-blackout-and-buffer-role-cpu-only"} - functions["dkr_probe_scene_unload_cpu"]=owned_unload_cpu(functions["unload_level_game"],revision) - origins["dkr_probe_scene_unload_cpu"]={**origins["unload_level_game"],"phase":"private-confirmed-drained-teardown"} - functions["dkr_probe_scene_load_cpu"],functions["dkr_probe_level_load_cpu"]=owned_load_cpu( - functions["load_level_game"],functions["level_load"],revision) - for new,original in (("dkr_probe_scene_load_cpu","load_level_game"),("dkr_probe_level_load_cpu","level_load")): - origins[new]={**origins[original],"phase":"private-confirmed-constructor"} - resumed=owned_scene_resume(functions["main_game_loop"],functions["mode_game"],revision, - functions["mode_menu"] if full_scenes else None) - functions["dkr_probe_scene_resume_main_cpu"],functions["dkr_probe_scene_resume_mode_cpu"]=resumed[:2] + origins["dkr_probe_authored_main_cpu"] = { + **origins["main_game_loop"], + "phase": "private-fixed-cadence-cpu-only", + } + functions["dkr_probe_authored_video_cpu"] = authored_video_cpu( + functions["fb_update"], functions["fb_swap"], revision + ) + origins["dkr_probe_authored_video_cpu"] = { + **origins["fb_update"], + "phase": "private-blackout-and-buffer-role-cpu-only", + } + functions["dkr_probe_scene_unload_cpu"] = owned_unload_cpu(functions["unload_level_game"], revision) + origins["dkr_probe_scene_unload_cpu"] = { + **origins["unload_level_game"], + "phase": "private-confirmed-drained-teardown", + } + functions["dkr_probe_scene_load_cpu"], functions["dkr_probe_level_load_cpu"] = owned_load_cpu( + functions["load_level_game"], functions["level_load"], revision + ) + for new, original in ( + ("dkr_probe_scene_load_cpu", "load_level_game"), + ("dkr_probe_level_load_cpu", "level_load"), + ): + origins[new] = {**origins[original], "phase": "private-confirmed-constructor"} + resumed = owned_scene_resume( + functions["main_game_loop"], + functions["mode_game"], + revision, + functions["mode_menu"] if full_scenes else None, + ) + functions["dkr_probe_scene_resume_main_cpu"], functions["dkr_probe_scene_resume_mode_cpu"] = resumed[:2] if full_scenes: - functions["dkr_probe_scene_resume_menu_cpu"]=resumed[2] - origins["dkr_probe_scene_resume_menu_cpu"]={**origins["mode_menu"],"phase":"confirmed-menu-continuation"} - menu_source_pins = {name: hashlib.sha256(functions[name].encode()).hexdigest() - for name in MENU_CPU_HASH[revision]} - for name,body in owned_menu_cpu(functions,revision).items(): - functions[name]=body - origins[name]={"phase":"confirmed-menu-constructor","source_pins":menu_source_pins} - for new,original in (("dkr_probe_scene_resume_main_cpu","main_game_loop"),("dkr_probe_scene_resume_mode_cpu","mode_game")): - origins[new]={**origins[original],"phase":"private-confirmed-continuation"} + functions["dkr_probe_scene_resume_menu_cpu"] = resumed[2] + origins["dkr_probe_scene_resume_menu_cpu"] = { + **origins["mode_menu"], + "phase": "confirmed-menu-continuation", + } + menu_source_pins = { + name: hashlib.sha256(functions[name].encode()).hexdigest() for name in MENU_CPU_HASH[revision] + } + for name, body in owned_menu_cpu(functions, revision).items(): + functions[name] = body + origins[name] = {"phase": "confirmed-menu-constructor", "source_pins": menu_source_pins} + for new, original in ( + ("dkr_probe_scene_resume_main_cpu", "main_game_loop"), + ("dkr_probe_scene_resume_mode_cpu", "mode_game"), + ): + origins[new] = {**origins[original], "phase": "private-confirmed-continuation"} helpers = set(re.findall(r"^#define\s+(\w+)\s*\(", header, re.M)) if save_layout_header: helpers.update(re.findall(r"static\s+inline\s+\w+\s+(\w+)\s*\(", save_layout_header)) @@ -1080,15 +1430,43 @@ def merge(items): helpers.add("dkr_probe_scene_pending") helpers.add("dkr_probe_boss_trace") helpers.add("dkr_probe_local_world_draw") - helpers.update(("dkr_probe_mesh_begin","dkr_probe_mesh_end")) - helpers.update(("dkr_probe_scene_entry","dkr_probe_block")) - helpers.update(("dkr_probe_scene_unload_begin","dkr_probe_scene_unload_render_drained","dkr_probe_scene_unload_end")) - helpers.update(("dkr_probe_scene_load_begin","dkr_probe_scene_load_reset","dkr_probe_scene_load_ready")) - helpers.update(("dkr_probe_scene_transaction_cut","dkr_probe_scene_transaction_end")) - helpers.update(("dkr_probe_menu_load_begin", "dkr_probe_menu_construct_begin", "dkr_probe_menu_construct_ready", - "dkr_probe_menu_level_change_begin", "dkr_probe_menu_level_change_ready", "dkr_probe_menu_background_request")) - helpers.update(("sqrtf", "sqrt", "truncf", "trunc", "roundf", "round", "fabsf", "fabs", - "floorf", "floor", "ceilf", "ceil", "fmodf", "fmod", "abs", "assert")) + helpers.update(("dkr_probe_mesh_begin", "dkr_probe_mesh_end")) + helpers.update(("dkr_probe_scene_entry", "dkr_probe_block")) + helpers.update( + ("dkr_probe_scene_unload_begin", "dkr_probe_scene_unload_render_drained", "dkr_probe_scene_unload_end") + ) + helpers.update(("dkr_probe_scene_load_begin", "dkr_probe_scene_load_reset", "dkr_probe_scene_load_ready")) + helpers.update(("dkr_probe_scene_transaction_cut", "dkr_probe_scene_transaction_end")) + helpers.update( + ( + "dkr_probe_menu_load_begin", + "dkr_probe_menu_construct_begin", + "dkr_probe_menu_construct_ready", + "dkr_probe_menu_level_change_begin", + "dkr_probe_menu_level_change_ready", + "dkr_probe_menu_background_request", + ) + ) + helpers.update( + ( + "sqrtf", + "sqrt", + "truncf", + "trunc", + "roundf", + "round", + "fabsf", + "fabs", + "floorf", + "floor", + "ceilf", + "ceil", + "fmodf", + "fmod", + "abs", + "assert", + ) + ) dispatch = audio_dispatch(functions, revision, dependencies_dir) if audio_services else {} menu_callbacks = menu_dispatch(functions, revision, dependencies_dir) if full_scenes else {} rsp_headers, rsp_source = audio_rsp_payload(rsp_source, dependencies_dir) if audio_services else ({}, None) @@ -1108,20 +1486,36 @@ def merge(items): pending, used, blocked, unknown = list(roots), set(), set(), set() while pending: name = pending.pop() - if name in used: continue - if name not in functions: raise ValueError(f"Missing guest root/callee {name}") + if name in used: + continue + if name not in functions: + raise ValueError(f"Missing guest root/callee {name}") used.add(name) for callee in CALL.findall(strip(functions[name])): - if callee in functions: pending.append(callee) - elif callee in prototypes: blocked.add(callee) - elif callee not in helpers: unknown.add((name, callee)) - if unknown: raise ValueError(f"Unclassified calls: {sorted(unknown)}") + if callee in functions: + pending.append(callee) + elif callee in prototypes: + blocked.add(callee) + elif callee not in helpers: + unknown.add((name, callee)) + if unknown: + raise ValueError(f"Unclassified calls: {sorted(unknown)}") # These can be invoked by header macros/inline helpers, outside the body # call scan. They must not resolve to a live runtime via indirect dispatch. - blocked.update(("get_function", "switch_error", "do_break", "cop0_status_write", - "cop0_status_read", "recomp_syscall_handler", "pause_self")) + blocked.update( + ( + "get_function", + "switch_error", + "do_break", + "cop0_status_write", + "cop0_status_read", + "recomp_syscall_handler", + "pause_self", + ) + ) for name in blocked: - if name not in prototypes: raise ValueError(f"No checked prototype for native import {name}") + if name not in prototypes: + raise ValueError(f"No checked prototype for native import {name}") # Substitute ALL direct header guest memory accesses, including inline # doubleword/unaligned helpers. Macro shape is protected by the header hash. @@ -1130,30 +1524,44 @@ def merge(items): # level_id, etc.). C++ adopts the private names explicitly at funcs.h, # after its ordinary native headers; C translation units retain the # closed private import names from their first header. - insert = ('#ifndef __cplusplus\n#include "isolated_symbols.h"\n#endif\n' if isolate_symbols else '') + '#include "probe_bridge.h"\n' + insert = ( + '#ifndef __cplusplus\n#include "isolated_symbols.h"\n#endif\n' if isolate_symbols else "" + ) + '#include "probe_bridge.h"\n' if inline_memory: insert += '#include "probe_memory_inline.h"\n' header = header.replace("// Compiler definition", insert + "// Compiler definition", 1) - for name, typ, width in (("W", "int32_t", 4), ("H", "int16_t", 2), - ("B", "int8_t", 1), ("HU", "uint16_t", 2), ("BU", "uint8_t", 1)): + for name, typ, width in ( + ("W", "int32_t", 4), + ("H", "int16_t", 2), + ("B", "int8_t", 1), + ("HU", "uint16_t", 2), + ("BU", "uint8_t", 1), + ): pattern = r"#define MEM_" + name + r"\(offset, reg\) \\\n[^\n]*" replacement = f"#define MEM_{name}(offset, reg) (*({typ}*)dkr_probe_memory_{width}_at(rdram, (uint64_t)(reg) + (uint64_t)(offset), __FILE__, __LINE__))" if inline_memory: lane = 3 if width == 1 else 2 if width == 2 else 0 replacement = f"#define MEM_{name}(offset, reg) (*({typ}*)dkr_probe_memory_inline_at(rdram, (uint64_t)(reg) + (uint64_t)(offset), {width}, {lane}, __FILE__, __LINE__))" header, count = re.subn(pattern, replacement, header) - if count != 1: raise ValueError(f"Memory macro mismatch MEM_{name}") - header, count = re.subn(r"#define SD\(val, offset, reg\) \{ \\\n.*?\n\}", - "#define SD(val, offset, reg) { MEM_W((offset) + 4, reg) = (uint32_t)(val); MEM_W(offset, reg) = (uint32_t)((gpr)(val) >> 32); }", header, flags=re.S) - if count != 1 or re.search(r"rdram\s*\+", header): raise ValueError("Uninstrumented guest memory access remains") + if count != 1: + raise ValueError(f"Memory macro mismatch MEM_{name}") + header, count = re.subn( + r"#define SD\(val, offset, reg\) \{ \\\n.*?\n\}", + "#define SD(val, offset, reg) { MEM_W((offset) + 4, reg) = (uint32_t)(val); MEM_W(offset, reg) = (uint32_t)((gpr)(val) >> 32); }", + header, + flags=re.S, + ) + if count != 1 or re.search(r"rdram\s*\+", header): + raise ValueError("Uninstrumented guest memory access remains") if any(re.search(r"\brdram\s*\[|\brdram\s*\+", strip(functions[n])) for n in used): raise ValueError("Guest body has direct pointer arithmetic outside the checked memory macros") if scene_cuts: - if "mode_game" not in used: raise ValueError("Scene cuts require the reviewed mode_game closure") - functions["mode_game"]=mode_scene_cuts(functions["mode_game"],revision) + if "mode_game" not in used: + raise ValueError("Scene cuts require the reviewed mode_game closure") + functions["mode_game"] = mode_scene_cuts(functions["mode_game"], revision) if full_scenes: - functions["mode_menu"]=menu_scene_cuts(functions["mode_menu"],revision) + functions["mode_menu"] = menu_scene_cuts(functions["mode_menu"], revision) isolated_header = isolation_header(used, blocked, revision) if isolate_symbols else "" audio_isolated_header = audio_isolation_header(revision) if isolate_symbols and audio_services else "" @@ -1166,7 +1574,9 @@ def merge(items): (output / "save_layout_guest.h").write_text(save_layout_header, encoding="utf-8") if isolate_symbols: (output / "isolated_symbols.h").write_text(isolated_header, encoding="utf-8") - (output / "isolated_runtime_symbols.h").write_text(runtime_isolation_header(used, blocked, revision), encoding="utf-8") + (output / "isolated_runtime_symbols.h").write_text( + runtime_isolation_header(used, blocked, revision), encoding="utf-8" + ) if mod_services: (output / "owned_mod_calls.hpp").write_text(owned_mod_calls_header(), encoding="utf-8") if audio_services: @@ -1178,33 +1588,63 @@ def merge(items): (output / "private_audio_rsp.cpp").write_text(rsp_source, encoding="utf-8") (output / "recomp.h").write_text(header, encoding="utf-8") extra = authored_declarations(full_scenes) if authored_cpu else "" - private_names = '#include "isolated_symbols.h"\n' if isolate_symbols else '' - (output / "funcs.h").write_text(private_names + (source_dir / "funcs.h").read_text(encoding="utf-8") + extra + - ('\n#include "save_layout_guest.h"\n' if save_layout_header else ''), encoding="utf-8") + private_names = '#include "isolated_symbols.h"\n' if isolate_symbols else "" + (output / "funcs.h").write_text( + private_names + + (source_dir / "funcs.h").read_text(encoding="utf-8") + + extra + + ('\n#include "save_layout_guest.h"\n' if save_layout_header else ""), + encoding="utf-8", + ) ordered = sorted(used) - code_policy=(magic_code_pipeline.policy_names(source_dir,revision,code_policy_config)|set(magic_code_pipeline.TRACK_OPTIONS)) if mod_services else set() - code_menu_fields=magic_code_pipeline.track_menu_fields(code_originals) if mod_services else '' - code_entries=magic_code_pipeline.entries({n:functions[n] for n in ordered},code_originals,code_policy) if mod_services else [] - code_by_address={e['address']:e for e in code_entries} - code_grants={n:code_by_address[int(m[1],16)] for n in ordered if n in code_originals and - (m:=magic_code_pipeline.ADDRESS.search(code_originals[n])) and int(m[1],16) in code_by_address} + code_policy = ( + ( + magic_code_pipeline.policy_names(source_dir, revision, code_policy_config) + | set(magic_code_pipeline.TRACK_OPTIONS) + ) + if mod_services + else set() + ) + code_menu_fields = magic_code_pipeline.track_menu_fields(code_originals) if mod_services else "" + code_entries = ( + magic_code_pipeline.entries({n: functions[n] for n in ordered}, code_originals, code_policy) + if mod_services + else [] + ) + code_by_address = {e["address"]: e for e in code_entries} + code_grants = { + n: code_by_address[int(m[1], 16)] + for n in ordered + if n in code_originals + and (m := magic_code_pipeline.ADDRESS.search(code_originals[n])) + and int(m[1], 16) in code_by_address + } checked_declarations = ['#include "recomp.h"'] for name in sorted(blocked): ret, types = prototypes[name] - checked_declarations.append(f'{ret} {name}({", ".join(types) or "void"});') + checked_declarations.append(f"{ret} {name}({', '.join(types) or 'void'});") (output / "imports.h").write_text("\n".join(checked_declarations) + "\n", encoding="utf-8") for index in range(0, len(ordered), 50): - chunks=[] - for n in ordered[index:index+50]: - body=instrument_function(n,functions[n]) - if n in code_grants:body=magic_code_pipeline.wrap(n,body,code_grants[n]['address'],owned=True,menu_fields=code_menu_fields) - elif mod_services:body=magic_code_pipeline.indirect_calls(body,True) + chunks = [] + for n in ordered[index : index + 50]: + body = instrument_function(n, functions[n]) + if n in code_grants: + body = magic_code_pipeline.wrap( + n, body, code_grants[n]["address"], owned=True, menu_fields=code_menu_fields + ) + elif mod_services: + body = magic_code_pipeline.indirect_calls(body, True) chunks.append(body) - (output / f"guest_{index // 50}.c").write_text('#include "funcs.h"\n#include "imports.h"\n' + - (magic_code_pipeline.owned_header() if mod_services else '')+ - "\n".join(chunks), encoding="utf-8") + (output / f"guest_{index // 50}.c").write_text( + '#include "funcs.h"\n#include "imports.h"\n' + + (magic_code_pipeline.owned_header() if mod_services else "") + + "\n".join(chunks), + encoding="utf-8", + ) if mod_services: - (output/'magic_code_dispatch.c').write_text(magic_code_pipeline.table_source(code_entries,revision,owned=True),encoding='utf-8') + (output / "magic_code_dispatch.c").write_text( + magic_code_pipeline.table_source(code_entries, revision, owned=True), encoding="utf-8" + ) stubs = ['#include "recomp.h"', '#include "probe_bridge.h"'] if audio_services: stubs.append('#include "funcs.h"') @@ -1216,38 +1656,61 @@ def merge(items): else: stubs.append(native_stub(name, prototypes[name], full_scenes)) (output / "blocked_imports.c").write_text("\n".join(stubs) + "\n", encoding="utf-8") - (output / "manifest.json").write_text(json.dumps({"revision": revision, "roots": roots, "scene_cuts": scene_cuts, "full_scenes": full_scenes, - "character_portrait_pipeline": character_portrait_receipt, - "save_layout_schema": 1 if save_layout_header else 0, - "save_layout_helper_sha256": hashlib.sha256(save_layout_header.encode()).hexdigest() if save_layout_header else "", - "boss_finish_diagnostics": boss_finish_diagnostics, - "owned_mod_services": mod_services, - "code_dispatch_schema": 2 if mod_services else 0, - "code_protected_policy": sorted(code_policy), - "code_dispatch_functions": code_entries, - "checked_inline_memory": inline_memory, - "owned_mod_calls": OWNED_MOD_CALLS if mod_services else {}, - "local_world_observations": full_scenes, - "online_view_observations": 2 if full_scenes else 0, - "local_model_observations": 1 if full_scenes else 0, - "local_world_observation_source": origins["render_level_geometry_and_objects"] if full_scenes else {}, - "boss_finish_diagnostic_source": origins["racer_boss_finish"] if boss_finish_diagnostics else {}, - "symbol_prefix": f"dkr_experimental_v{revision}_" if isolate_symbols else "", - "runtime_symbol_isolation": bool(isolate_symbols), - "audio_symbol_isolation": bool(audio_isolated_header), - "audio_services": audio_services, "input_services": input_services, "authored_cpu": authored_cpu, - "authored_cpu_source": origins["main_game_loop"] if authored_cpu else {}, - "authored_video_sources": {name: origins[name] for name in ("fb_update","fb_swap")} if authored_cpu else {}, - "confirmed_scene_sources": {name: origins[name] for name in ("unload_level_game","load_level_game","level_load")} if authored_cpu else {}, - "input_native_inputs": INPUT_NATIVE_HASH if input_services else {}, - "full_scene_native_inputs": FULL_SCENE_NATIVE_HASH if full_scenes else {}, - "audio_dispatch": {f"0x{address:08X}": name for address, name in sorted(dispatch.items())}, - "menu_dispatch": {f"0x{address:08X}": name for address,name in sorted(menu_callbacks.items())}, - "menu_indirect_sources": MENU_INDIRECT_HASH[revision] if full_scenes else {}, - "audio_rsp_inputs": RSP_INPUT_HASH if audio_services else {}, - "guest_functions": {n: origins[n] for n in ordered}, "blocked_native_imports": sorted(blocked), - "recomp_header_sha256": HEADER_HASH, - "safety": "Isolated guest CPU closure. Only audited owned imports are admitted; normal boot and live admission are separately enforced by the owned adapter."}, indent=2) + "\n", encoding="utf-8") + (output / "manifest.json").write_text( + json.dumps( + { + "revision": revision, + "roots": roots, + "scene_cuts": scene_cuts, + "full_scenes": full_scenes, + "character_portrait_pipeline": character_portrait_receipt, + "save_layout_schema": 1 if save_layout_header else 0, + "save_layout_helper_sha256": hashlib.sha256(save_layout_header.encode()).hexdigest() + if save_layout_header + else "", + "boss_finish_diagnostics": boss_finish_diagnostics, + "owned_mod_services": mod_services, + "code_dispatch_schema": 2 if mod_services else 0, + "code_protected_policy": sorted(code_policy), + "code_dispatch_functions": code_entries, + "checked_inline_memory": inline_memory, + "owned_mod_calls": OWNED_MOD_CALLS if mod_services else {}, + "local_world_observations": full_scenes, + "online_view_observations": 2 if full_scenes else 0, + "local_model_observations": 1 if full_scenes else 0, + "local_world_observation_source": origins["render_level_geometry_and_objects"] if full_scenes else {}, + "boss_finish_diagnostic_source": origins["racer_boss_finish"] if boss_finish_diagnostics else {}, + "symbol_prefix": f"dkr_experimental_v{revision}_" if isolate_symbols else "", + "runtime_symbol_isolation": bool(isolate_symbols), + "audio_symbol_isolation": bool(audio_isolated_header), + "audio_services": audio_services, + "input_services": input_services, + "authored_cpu": authored_cpu, + "authored_cpu_source": origins["main_game_loop"] if authored_cpu else {}, + "authored_video_sources": {name: origins[name] for name in ("fb_update", "fb_swap")} + if authored_cpu + else {}, + "confirmed_scene_sources": { + name: origins[name] for name in ("unload_level_game", "load_level_game", "level_load") + } + if authored_cpu + else {}, + "input_native_inputs": INPUT_NATIVE_HASH if input_services else {}, + "full_scene_native_inputs": FULL_SCENE_NATIVE_HASH if full_scenes else {}, + "audio_dispatch": {f"0x{address:08X}": name for address, name in sorted(dispatch.items())}, + "menu_dispatch": {f"0x{address:08X}": name for address, name in sorted(menu_callbacks.items())}, + "menu_indirect_sources": MENU_INDIRECT_HASH[revision] if full_scenes else {}, + "audio_rsp_inputs": RSP_INPUT_HASH if audio_services else {}, + "guest_functions": {n: origins[n] for n in ordered}, + "blocked_native_imports": sorted(blocked), + "recomp_header_sha256": HEADER_HASH, + "safety": "Isolated guest CPU closure. Only audited owned imports are admitted; normal boot and live admission are separately enforced by the owned adapter.", + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) print(f"Probe v{revision}: {len(used)} real guest functions; {len(blocked)} fenced native imports") @@ -1257,18 +1720,80 @@ def merge(items): p.add_argument("header", type=Path) p.add_argument("output", type=Path) p.add_argument("--revision", type=int, choices=(77, 80), required=True) - p.add_argument("--inline-memory", action="store_true", help="Inline the fully checked guest-memory fast path; retain all fault/budget/watch gates") + p.add_argument( + "--inline-memory", + action="store_true", + help="Inline the fully checked guest-memory fast path; retain all fault/budget/watch gates", + ) p.add_argument("--roots", nargs="+", default=["main_game_loop", "obj_update", "input_swap_id"]) - p.add_argument("--scene-cuts", action="store_true", help="Private audited resumable mode_game unload intents; not live scene admission") - p.add_argument("--audio-services", action="store_true", help="Private retail audio CPU closure and exact callback map; not live audio admission") - p.add_argument("--input-services", action="store_true", help="Private audited logical controller/save/rumble owner; not live SI/Pak admission") - p.add_argument("--authored-cpu", action="store_true", help="Exact private retail main-loop CPU phase; excludes scheduler/VI/renderer ownership") - p.add_argument("--isolate-symbols", action="store_true", help="Revision-specific C link isolation; preserves all native fences, NOT live admission") - p.add_argument("--full-scenes", action="store_true", help="Confirmed menu continuations and constructors; release qualification remains separate") - p.add_argument("--boss-finish-diagnostics", action="store_true", help="Bounded observation-only boss-finish tracing; no gameplay changes") - p.add_argument("--mod-services", action="store_true", help="Reviewed private mod callback closure; session/runtime admission remains separately gated") - p.add_argument("--rsp-source", type=Path, help="Read-only RSP input directory for an isolated worktree; original SHA-256 pins remain mandatory") - p.add_argument("--dependencies-dir", type=Path, help="Read-only prepared extern directory; first-party sources and all SHA-256 pins remain unchanged") - p.add_argument("--code-policy-config",type=Path,help="Complete CPU Patch Pipeline config for code-mod protection") + p.add_argument( + "--scene-cuts", + action="store_true", + help="Private audited resumable mode_game unload intents; not live scene admission", + ) + p.add_argument( + "--audio-services", + action="store_true", + help="Private retail audio CPU closure and exact callback map; not live audio admission", + ) + p.add_argument( + "--input-services", + action="store_true", + help="Private audited logical controller/save/rumble owner; not live SI/Pak admission", + ) + p.add_argument( + "--authored-cpu", + action="store_true", + help="Exact private retail main-loop CPU phase; excludes scheduler/VI/renderer ownership", + ) + p.add_argument( + "--isolate-symbols", + action="store_true", + help="Revision-specific C link isolation; preserves all native fences, NOT live admission", + ) + p.add_argument( + "--full-scenes", + action="store_true", + help="Confirmed menu continuations and constructors; release qualification remains separate", + ) + p.add_argument( + "--boss-finish-diagnostics", + action="store_true", + help="Bounded observation-only boss-finish tracing; no gameplay changes", + ) + p.add_argument( + "--mod-services", + action="store_true", + help="Reviewed private mod callback closure; session/runtime admission remains separately gated", + ) + p.add_argument( + "--rsp-source", + type=Path, + help="Read-only RSP input directory for an isolated worktree; original SHA-256 pins remain mandatory", + ) + p.add_argument( + "--dependencies-dir", + type=Path, + help="Read-only prepared extern directory; first-party sources and all SHA-256 pins remain unchanged", + ) + p.add_argument("--code-policy-config", type=Path, help="Complete CPU Patch Pipeline config for code-mod protection") a = p.parse_args() - generate(a.generated, a.header, a.output, a.roots, a.revision, a.scene_cuts, a.audio_services, a.input_services, a.authored_cpu, a.isolate_symbols, a.full_scenes, a.boss_finish_diagnostics, a.rsp_source, a.mod_services, a.inline_memory, a.dependencies_dir,a.code_policy_config) + generate( + a.generated, + a.header, + a.output, + a.roots, + a.revision, + a.scene_cuts, + a.audio_services, + a.input_services, + a.authored_cpu, + a.isolate_symbols, + a.full_scenes, + a.boss_finish_diagnostics, + a.rsp_source, + a.mod_services, + a.inline_memory, + a.dependencies_dir, + a.code_policy_config, + ) diff --git a/scripts/generate_revision_policy.py b/scripts/generate_revision_policy.py index cef6cdb3..1e420b89 100644 --- a/scripts/generate_revision_policy.py +++ b/scripts/generate_revision_policy.py @@ -11,9 +11,8 @@ import argparse import json -import re from pathlib import Path - +import re SYMBOL = re.compile(r"^([A-Za-z_.$][\w.$]*)\s*=\s*0x([0-9A-Fa-f]+);") @@ -58,10 +57,7 @@ def map_site( raise ValueError(f"destination symbol is missing: {name}") offset = source_site - source_symbols[name] if offset < 0 or offset % 4 != 0: - raise ValueError( - f"invalid site for {name}: {format_address(source_site)} " - f"(offset {offset})" - ) + raise ValueError(f"invalid site for {name}: {format_address(source_site)} (offset {offset})") destination_site = destination_symbols[name] + offset next_address = next_symbol_address(destination_ordered, destination_symbols[name]) if next_address is not None and destination_site >= next_address: @@ -88,9 +84,7 @@ def main() -> int: arguments = parser.parse_args() source_by_name, _ = read_symbols(arguments.source_symbols) - destination_by_name, destination_ordered = read_symbols( - arguments.destination_symbols - ) + destination_by_name, destination_ordered = read_symbols(arguments.destination_symbols) for override in arguments.manual_override: name, separator, value = override.partition("=") if not separator: @@ -145,9 +139,7 @@ def main() -> int: ) arguments.output.parent.mkdir(parents=True, exist_ok=True) - arguments.output.write_text( - json.dumps(policy, indent=2) + "\n", encoding="utf-8", newline="\n" - ) + arguments.output.write_text(json.dumps(policy, indent=2) + "\n", encoding="utf-8", newline="\n") return 0 diff --git a/scripts/generate_visual_model_kernel.py b/scripts/generate_visual_model_kernel.py index 04bacc8f..4b4e50ea 100644 --- a/scripts/generate_visual_model_kernel.py +++ b/scripts/generate_visual_model_kernel.py @@ -6,117 +6,167 @@ The five reviewed leaf routines run against a decoder image with copy-on-write scalar globals and a separate vertex/command arena; they cannot write guest RAM. """ -from pathlib import Path + import argparse import hashlib import json +from pathlib import Path import re -from generate_replay_probe import DEF, CALL, HEADER_HASH, strip from compose_legacy_mod_policy import elf_functions +from generate_replay_probe import CALL, DEF, HEADER_HASH, strip PINS = { 77: { - 'obj_shade_fast': 'ae2f5e243b3928109f1c2c76117b6d7e9bb49e257bb71f56a96631a76b747b34', - 'material_set': 'e892d9ba93dc3ed57b96385c61b75892b8c87e3147ae4ba4025637c697130850', - 'obj_animate': '129f0cac28948697172e4765307a6f306a5734b635060b98f46c466a1ab287a3', - 'set_animated_texture_header': '03c4a910a5ac8706c063f68f3c0a2e0034ac61d372827fda32f9b3cf2aeac1bc', - 'render_mesh': 'cb84a1ea9a41e0ea2f61147aa0d46a9b08564aa2fbf76e1d7d697163efbfa38c', + "obj_shade_fast": "ae2f5e243b3928109f1c2c76117b6d7e9bb49e257bb71f56a96631a76b747b34", + "material_set": "e892d9ba93dc3ed57b96385c61b75892b8c87e3147ae4ba4025637c697130850", + "obj_animate": "129f0cac28948697172e4765307a6f306a5734b635060b98f46c466a1ab287a3", + "set_animated_texture_header": "03c4a910a5ac8706c063f68f3c0a2e0034ac61d372827fda32f9b3cf2aeac1bc", + "render_mesh": "cb84a1ea9a41e0ea2f61147aa0d46a9b08564aa2fbf76e1d7d697163efbfa38c", }, 80: { - 'obj_shade_fast': 'af718882437565cf1fbe859572da676b1495b7a7ac4f2faafb4daa3a3bc03349', - 'material_set': 'b3e4b17b7e207a4d1164d3b25a73f1532d12354f18ec387c90c3cbdc29f16c20', - 'obj_animate': '69afe6c4255996372f4b55caf1212ab37f92ffe39a0cd365be856a34a79b9028', - 'set_animated_texture_header': '50fa347945c1031846d214c88368f806316cf8880d1bf8e134ee5cc6f03417e3', - 'render_mesh': '37038c5b33b9932e8624417ea52aaed7c6b1221321a99146cd1cce49988c0c10', + "obj_shade_fast": "af718882437565cf1fbe859572da676b1495b7a7ac4f2faafb4daa3a3bc03349", + "material_set": "b3e4b17b7e207a4d1164d3b25a73f1532d12354f18ec387c90c3cbdc29f16c20", + "obj_animate": "69afe6c4255996372f4b55caf1212ab37f92ffe39a0cd365be856a34a79b9028", + "set_animated_texture_header": "50fa347945c1031846d214c88368f806316cf8880d1bf8e134ee5cc6f03417e3", + "render_mesh": "37038c5b33b9932e8624417ea52aaed7c6b1221321a99146cd1cce49988c0c10", }, } -GLOBALS = ('gObjectCurrDisplayList', 'D_8011D644', 'gObjectTexAnim', - 'gForceFlags', 'gCurrentTextureHeader', 'gCurrentRenderFlags', - 'gUsingTexture', 'gUseDirectionalLighting', 'gBlockedRenderFlags') +GLOBALS = ( + "gObjectCurrDisplayList", + "D_8011D644", + "gObjectTexAnim", + "gForceFlags", + "gCurrentTextureHeader", + "gCurrentRenderFlags", + "gUsingTexture", + "gUseDirectionalLighting", + "gBlockedRenderFlags", +) def generate(source, header_path, elf, revision, output): source, output = source.resolve(), output.resolve() - if output.exists() or output == source or source in output.parents or header_path.resolve().parent in output.parents: - raise ValueError('Visual kernel output must be a NEW separate directory') + if ( + output.exists() + or output == source + or source in output.parents + or header_path.resolve().parent in output.parents + ): + raise ValueError("Visual kernel output must be a NEW separate directory") header = header_path.read_bytes() if hashlib.sha256(header).hexdigest() != HEADER_HASH: - raise ValueError('Unreviewed visual kernel register/memory ABI') + raise ValueError("Unreviewed visual kernel register/memory ABI") functions = {} - for path in sorted(source.glob('*.c')): - text = path.read_text(encoding='utf-8') + for path in sorted(source.glob("*.c")): + text = path.read_text(encoding="utf-8") matches = list(DEF.finditer(text)) for i, match in enumerate(matches): if match[1] in PINS[revision]: if match[1] in functions: - raise ValueError('Duplicate visual kernel function') - body = text[match.start():matches[i+1].start() if i+1 < len(matches) else len(text)] + raise ValueError("Duplicate visual kernel function") + body = text[match.start() : matches[i + 1].start() if i + 1 < len(matches) else len(text)] if hashlib.sha256(body.encode()).hexdigest() != PINS[revision][match[1]]: - raise ValueError('Unreviewed visual kernel: '+match[1]) + raise ValueError("Unreviewed visual kernel: " + match[1]) functions[match[1]] = body if set(functions) != set(PINS[revision]): - raise ValueError('Incomplete model-only closure') + raise ValueError("Incomplete model-only closure") symbols = elf_functions(elf, (0, 1, 2)) addresses = {} for name in GLOBALS: entries = symbols.get(name, set()) - if len(entries) != 1 or next(iter(entries))[1] not in (2,4): - raise ValueError('Unreviewed model scalar global: '+name) + if len(entries) != 1 or next(iter(entries))[1] not in (2, 4): + raise ValueError("Unreviewed model scalar global: " + name) addresses[name] = next(iter(entries))[0] - prefix = f'dkr_visual_v{revision}_' - names = set(functions) | {'dkr_fix_car_steering_wheel_material'} + prefix = f"dkr_visual_v{revision}_" + names = set(functions) | {"dkr_fix_car_steering_wheel_material"} h = header.decode() - h = h.replace('// Compiler definition', '#include "visual_model_kernel_guest.h"\n// Compiler definition', 1) - for name, typ, width in (('W','int32_t',4),('H','int16_t',2),('B','int8_t',1),('HU','uint16_t',2),('BU','uint8_t',1)): - pattern = r'#define MEM_'+name+r'\(offset, reg\) \\\n[^\n]*' - h, count = re.subn(pattern, f'#define MEM_{name}(offset, reg) (*({typ}*)dkr_visual_read((uint64_t)(reg)+(uint64_t)(offset),{width}))', h) + h = h.replace("// Compiler definition", '#include "visual_model_kernel_guest.h"\n// Compiler definition', 1) + for name, typ, width in ( + ("W", "int32_t", 4), + ("H", "int16_t", 2), + ("B", "int8_t", 1), + ("HU", "uint16_t", 2), + ("BU", "uint8_t", 1), + ): + pattern = r"#define MEM_" + name + r"\(offset, reg\) \\\n[^\n]*" + h, count = re.subn( + pattern, + f"#define MEM_{name}(offset, reg) (*({typ}*)dkr_visual_read((uint64_t)(reg)+(uint64_t)(offset),{width}))", + h, + ) if count != 1: - raise ValueError('Visual memory macro shape changed') + raise ValueError("Visual memory macro shape changed") # The selected leaf routines do not use doubleword/unaligned helpers. # Keep their definitions unreachable rather than granting unchecked access. - allowed = set(re.findall(r'^#define\s+(\w+)\s*\(', header.decode(), re.M)) - allowed.update(re.findall(r'static\s+inline\s+\w+\s+(\w+)\s*\(',header.decode())) - allowed |= {'if','while','switch','for','return','sizeof','void'} | names + allowed = set(re.findall(r"^#define\s+(\w+)\s*\(", header.decode(), re.M)) + allowed.update(re.findall(r"static\s+inline\s+\w+\s+(\w+)\s*\(", header.decode())) + allowed |= {"if", "while", "switch", "for", "return", "sizeof", "void"} | names emitted = [] for name, body in sorted(functions.items()): extra = set(CALL.findall(strip(body))) - allowed - if extra or re.search(r'\brdram\s*(?:\[|\+)', strip(body)): - raise ValueError('Visual kernel escaped its leaf closure: '+str(extra)) + if extra or re.search(r"\brdram\s*(?:\[|\+)", strip(body)): + raise ValueError("Visual kernel escaped its leaf closure: " + str(extra)) # Every store is independently checked. Reads of retail scalar globals # see the kernel's shadow words, never mutations of the immutable image. - stores = len(re.findall(r'MEM_[WHB]\([^\n]+?\)\s*=', body)) - body, changed = re.subn(r'MEM_([WHB])\(([^,\n]+), ([^)\n]+)\) = ([^;\n]+);', - lambda m: f'dkr_visual_write((uint64_t)({m[3]})+(uint64_t)({m[2]}), {dict(W=4,H=2,B=1)[m[1]]}, (uint32_t)({m[4]}));', body) + stores = len(re.findall(r"MEM_[WHB]\([^\n]+?\)\s*=", body)) + body, changed = re.subn( + r"MEM_([WHB])\(([^,\n]+), ([^)\n]+)\) = ([^;\n]+);", + lambda m: ( + f"dkr_visual_write((uint64_t)({m[3]})+(uint64_t)({m[2]}), {dict(W=4, H=2, B=1)[m[1]]}, (uint32_t)({m[4]}));" + ), + body, + ) if changed != stores: - raise ValueError('Uninstrumented visual store') - body = body.replace('{', '{\n dkr_visual_step();', 1) - body = re.sub(r'^(L_[A-Fa-f0-9]+:)\s*$', r'\1 dkr_visual_step();', body, flags=re.M) + raise ValueError("Uninstrumented visual store") + body = body.replace("{", "{\n dkr_visual_step();", 1) + body = re.sub(r"^(L_[A-Fa-f0-9]+:)\s*$", r"\1 dkr_visual_step();", body, flags=re.M) for symbol in sorted(names, key=len, reverse=True): - body = re.sub(r'\b'+symbol+r'\b', prefix+symbol, body) + body = re.sub(r"\b" + symbol + r"\b", prefix + symbol, body) emitted.append(body) root = Path(__file__).resolve().parents[1] output.mkdir(parents=True) - (output/'recomp.h').write_text(h, encoding='utf-8') - (output/'funcs.h').write_text('#pragma once\n#include "recomp.h"\n'+''.join( - f'void {prefix}{n}(uint8_t*,recomp_context*);\n' for n in sorted(names)), encoding='utf-8') - (output/'visual_model_functions.c').write_text('#include "funcs.h"\n'+''.join(emitted), encoding='utf-8') - constants = '\n'.join(f'#define DKR_VISUAL_{n} 0x{v:08X}U' for n,v in addresses.items()) - (output/'visual_model_addresses.h').write_text('#pragma once\n'+constants+'\n'+ - '\n'.join(f'#define {n} {prefix}{n}' for n in names)+'\n'+ - f'#define DKR_VISUAL_ENTRY {prefix}model\n', encoding='utf-8') + (output / "recomp.h").write_text(h, encoding="utf-8") + (output / "funcs.h").write_text( + '#pragma once\n#include "recomp.h"\n' + + "".join(f"void {prefix}{n}(uint8_t*,recomp_context*);\n" for n in sorted(names)), + encoding="utf-8", + ) + (output / "visual_model_functions.c").write_text('#include "funcs.h"\n' + "".join(emitted), encoding="utf-8") + constants = "\n".join(f"#define DKR_VISUAL_{n} 0x{v:08X}U" for n, v in addresses.items()) + (output / "visual_model_addresses.h").write_text( + "#pragma once\n" + + constants + + "\n" + + "\n".join(f"#define {n} {prefix}{n}" for n in names) + + "\n" + + f"#define DKR_VISUAL_ENTRY {prefix}model\n", + encoding="utf-8", + ) # The project-owned bridge is compiled normally, not copied or rewritten. - (output/'manifest.json').write_text(json.dumps({'revision':revision,'schema':1, - 'functions':PINS[revision],'scalar_globals':addresses,'guest_writes':False, - 'world_redraw':False}, indent=2)+'\n', encoding='utf-8') + (output / "manifest.json").write_text( + json.dumps( + { + "revision": revision, + "schema": 1, + "functions": PINS[revision], + "scalar_globals": addresses, + "guest_writes": False, + "world_redraw": False, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--generated',type=Path,required=True) - parser.add_argument('--header',type=Path,required=True) - parser.add_argument('--elf',type=Path,required=True) - parser.add_argument('--revision',type=int,choices=(77,80),required=True) - parser.add_argument('--output',type=Path,required=True) - args=parser.parse_args() - generate(args.generated,args.header,args.elf,args.revision,args.output) + parser.add_argument("--generated", type=Path, required=True) + parser.add_argument("--header", type=Path, required=True) + parser.add_argument("--elf", type=Path, required=True) + parser.add_argument("--revision", type=int, choices=(77, 80), required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + generate(args.generated, args.header, args.elf, args.revision, args.output) diff --git a/scripts/harness_report.py b/scripts/harness_report.py index 78b3d09e..92111879 100644 --- a/scripts/harness_report.py +++ b/scripts/harness_report.py @@ -102,12 +102,18 @@ def _junit(report: dict) -> bytes: indent=2, allow_nan=False, ) + # Child diagnostics can contain terminal controls, which XML 1.0 rejects. def xml_text(value: str) -> str: return "".join( - char if (char in "\t\n\r" or 0x20 <= ord(char) <= 0xD7FF - or 0xE000 <= ord(char) <= 0xFFFD - or 0x10000 <= ord(char) <= 0x10FFFF) else "\ufffd" + char + if ( + char in "\t\n\r" + or 0x20 <= ord(char) <= 0xD7FF + or 0xE000 <= ord(char) <= 0xFFFD + or 0x10000 <= ord(char) <= 0x10FFFF + ) + else "\ufffd" for char in value ) @@ -122,13 +128,23 @@ def xml_text(value: str) -> str: def _html(report: dict) -> str: scenarios = report.get("scenarios", []) - counts = {status: sum(item.get("status") == status for item in scenarios) for status in ("passed", "failed", "skipped")} - cards = "".join(f'
{count}{status}
' for status, count in counts.items()) - identity = "".join(f"
{_escape(key)}
{_escape(value)}
" for key, value in sorted(report.get("identity", {}).items())) + counts = { + status: sum(item.get("status") == status for item in scenarios) for status in ("passed", "failed", "skipped") + } + cards = "".join( + f'
{count}{status}
' + for status, count in counts.items() + ) + identity = "".join( + f"
{_escape(key)}
{_escape(value)}
" for key, value in sorted(report.get("identity", {}).items()) + ) rows = [] for scenario in scenarios: attempts = scenario.get("attempts", []) - metrics = "".join(f"{_escape(key)}{_number(value)}" for key, value in sorted(scenario.get("metrics", {}).items())) + metrics = "".join( + f"{_escape(key)}{_number(value)}" + for key, value in sorted(scenario.get("metrics", {}).items()) + ) comparisons = "".join( f"{_escape(item.get('metric', ''))}{_number(item.get('baseline'))}" f"{_number(item.get('current'))}{_number(item.get('limit'))}{_status(item.get('status'))}" @@ -142,19 +158,25 @@ def _html(report: dict) -> str: ) details = [] if metrics: - details.append(f'

Metrics

{metrics}
MetricValue
') + details.append( + f'

Metrics

{metrics}
MetricValue
' + ) if comparisons: - details.append(f'

Baseline comparisons

{comparisons}
MetricBaselineCurrentLimitStatus
') + details.append( + f'

Baseline comparisons

{comparisons}
MetricBaselineCurrentLimitStatus
' + ) if attempt_rows: - details.append(f'

Attempts

{attempt_rows}
AttemptPhaseStatusWall time (ms)Exit codeLogs
') + details.append( + f'

Attempts

{attempt_rows}
AttemptPhaseStatusWall time (ms)Exit codeLogs
' + ) opened = " open" if scenario.get("status") == "failed" else "" rows.append( f'
{_status(scenario.get("status"))} ' f'{_escape(scenario["id"])}{_number(_elapsed(scenario))} s total' - f'

{_escape(scenario.get("description", ""))}

{_errors(scenario.get("errors", []))}{"".join(details)}
' + f"

{_escape(scenario.get('description', ''))}

{_errors(scenario.get('errors', []))}{''.join(details)}" ) title = _escape(report.get("suite", "Regression harness")) - return f''' + return f""" {title} — regression report
-

DKR-R · Regression & performance

{title}

Started {_escape(report.get('started_at', ''))} · {len(scenarios)} scenarios

+

DKR-R · Regression & performance

{title}

Started {_escape(report.get("started_at", ""))} · {len(scenarios)} scenarios

{cards}

Process wall time includes startup and shutdown; it is not frame time. Skipped scenarios are not passes. This report covers the configured checks and does not certify real gameplay or hardware acceptance.

Machine-readable JSON · JUnit XML

-
Run identity
{identity or '
Identity
Not recorded
'}
-

Scenarios

{''.join(rows) or '

No scenarios were recorded.

'} +
Run identity
{identity or "
Identity
Not recorded
"}
+

Scenarios

{"".join(rows) or "

No scenarios were recorded.

"}
Warmup and measured attempts are shown separately. Scenario totals include all attempts. Metric names define their units.
-
''' +""" def write_reports(output: Path, report: dict) -> None: diff --git a/scripts/host_task_policy.py b/scripts/host_task_policy.py index b428bf85..d110add9 100644 --- a/scripts/host_task_policy.py +++ b/scripts/host_task_policy.py @@ -1,11 +1,12 @@ """Qualify the project-owned retail watchdog hook against the input MIPS ELF.""" + import struct + from compose_legacy_mod_policy import elf_sections def verify_host_task_policy(policy, elf): - hooks = [h for h in policy.get("functionHooks", []) - if "dkr_scheduler_keep_host_tasks" in h.get("text", "")] + hooks = [h for h in policy.get("functionHooks", []) if "dkr_scheduler_keep_host_tasks" in h.get("text", "")] if not hooks: return # Older policies and unrelated pipeline fixtures remain valid. if len(hooks) != 1 or hooks[0]["function"] != "__scMain": @@ -19,9 +20,11 @@ def verify_host_task_policy(policy, elf): if pc not in revisions: raise ValueError("Unreviewed scheduler revision") target, sp_low, dp_low = revisions[pc] - expected = ("extern void dkr_scheduler_keep_host_tasks(uint8_t*, uint32_t, uint32_t, uint32_t); " - f"dkr_scheduler_keep_host_tasks(rdram, (uint32_t)ctx->r18, 0x{0x800D0000 + sp_low:08X}U, " - f"0x{0x800D0000 + dp_low:08X}U);") + expected = ( + "extern void dkr_scheduler_keep_host_tasks(uint8_t*, uint32_t, uint32_t, uint32_t); " + f"dkr_scheduler_keep_host_tasks(rdram, (uint32_t)ctx->r18, 0x{0x800D0000 + sp_low:08X}U, " + f"0x{0x800D0000 + dp_low:08X}U);" + ) if hook["text"] != expected: raise ValueError("Scheduler register/counter ownership changed") sections = elf_sections(elf) @@ -36,7 +39,10 @@ def verify_host_task_policy(policy, elf): target + 0xBC: 0x2981000B, # slti at,t4,11 } for address, opcode in signatures.items(): - matches = [struct.unpack_from(">I", data, address - base)[0] - for base, data in sections if base <= address <= base + len(data) - 4] + matches = [ + struct.unpack_from(">I", data, address - base)[0] + for base, data in sections + if base <= address <= base + len(data) - 4 + ] if matches != [opcode]: raise ValueError(f"Scheduler hook signature mismatch at {address:08X}") diff --git a/scripts/inspect_legacy_character_assets.py b/scripts/inspect_legacy_character_assets.py index 524f387c..b4910507 100644 --- a/scripts/inspect_legacy_character_assets.py +++ b/scripts/inspect_legacy_character_assets.py @@ -4,6 +4,7 @@ An explicit alternate LUT is analysis input, not importer authorization. This tool never patches a ROM or enables executable changes. Output is metadata. """ + import argparse import hashlib import json @@ -29,21 +30,21 @@ def sections(data, at): start = at + 208 if offsets[-1] > len(data) - start: raise ValueError("Directory exceeds decoded image") - return [data[start+a:start+b] for a, b in zip(offsets, offsets[1:])] + return [data[start + a : start + b] for a, b in zip(offsets, offsets[1:])] def records(sec, section): - table = section+1 if section in (0,2,4,6,8,10,12,15) else section-1 + table = section + 1 if section in (0, 2, 4, 6, 8, 10, 12, 15) else section - 1 data = sec[table] offsets = [0] if section == 39 else [] step = 8 if section == 49 else 4 - start = 4 if section in (8,49) else 0 - for at in range(start, len(data)-3, step): + start = 4 if section in (8, 49) else 0 + for at in range(start, len(data) - 3, step): value = u32(data, at) - if value == 0xffffffff: + if value == 0xFFFFFFFF: break if section == 6: - value &= 0x7fffffff + value &= 0x7FFFFFFF if section == 15: value *= 4 if value > len(sec[section]) or (offsets and value < offsets[-1]): @@ -51,17 +52,17 @@ def records(sec, section): offsets.append(value) else: raise ValueError(f"No section {section} table terminator") - return [sec[section][a:b] for a,b in zip(offsets,offsets[1:])] + return [sec[section][a:b] for a, b in zip(offsets, offsets[1:])] def unpack(data): if len(data) < 6: raise ValueError("Compressed record is truncated") - size = int.from_bytes(data[:4], 'little') - if not 0 < size <= 4*1024*1024 or data[4] > 9: + size = int.from_bytes(data[:4], "little") + if not 0 < size <= 4 * 1024 * 1024 or data[4] > 9: raise ValueError("Invalid compressed record size/level") decoder = zlib.decompressobj(-15) - result = decoder.decompress(data[5:], size+1) + result = decoder.decompress(data[5:], size + 1) if len(result) != size or not decoder.eof: raise ValueError("Compressed record did not match declared size") return result @@ -69,8 +70,9 @@ def unpack(data): def elf_object_ids(path): from compose_legacy_mod_policy import elf_functions + data = path.read_bytes() - entries = elf_functions(path, (1,))['gRacerObjectTable'] + entries = elf_functions(path, (1,))["gRacerObjectTable"] if len(entries) != 1: raise ValueError("Ambiguous racer object table") address, size = next(iter(entries)) @@ -79,70 +81,112 @@ def elf_object_ids(path): shoff = u32(data, 32) count = u16(data, 48) for i in range(count): - _,kind,_,vram,offset,length,*_ = struct.unpack_from('>10I',data,shoff+i*40) - if kind == 1 and vram <= address and address+size <= vram+length: - return list(struct.unpack_from('>59H', data, offset+address-vram)) + _, kind, _, vram, offset, length, *_ = struct.unpack_from(">10I", data, shoff + i * 40) + if kind == 1 and vram <= address and address + size <= vram + length: + return list(struct.unpack_from(">59H", data, offset + address - vram)) raise ValueError("Racer table has no ELF backing") def main(): - p=argparse.ArgumentParser(description=__doc__) - p.add_argument('--base',type=Path,required=True) - p.add_argument('--target',type=Path,required=True) - p.add_argument('--elf',type=Path,required=True) - p.add_argument('--target-lut',type=lambda s:int(s,0),default=0xecb60) - p.add_argument('--output',type=Path) - args=p.parse_args() - base=args.base.read_bytes();target=args.target.read_bytes() - old=sections(base,0xecb60);new=sections(target,args.target_lut) - ids=elf_object_ids(args.elf) - old_records={s:records(old,s) for s in (2,4,29,32,34)} - new_records={s:records(new,s) for s in (2,4,29,32,34)} - names=('Krunch','Bumper','Tiptup','Conker','Timber','Banjo','Drumstick','Pipsy','T.T.','Diddy') - audit={'target_sha256':hashlib.sha256(target).hexdigest(),'target_lut':hex(args.target_lut), - 'asset_sha256':hashlib.sha256(target[args.target_lut:]).hexdigest(), - 'record_counts':{s:[len(old_records[s]),len(new_records[s])] for s in old_records},'characters':[]} - for character,name in enumerate(names): - item={'base':name,'id':character,'vehicles':[]} + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--base", type=Path, required=True) + p.add_argument("--target", type=Path, required=True) + p.add_argument("--elf", type=Path, required=True) + p.add_argument("--target-lut", type=lambda s: int(s, 0), default=0xECB60) + p.add_argument("--output", type=Path) + args = p.parse_args() + base = args.base.read_bytes() + target = args.target.read_bytes() + old = sections(base, 0xECB60) + new = sections(target, args.target_lut) + ids = elf_object_ids(args.elf) + old_records = {s: records(old, s) for s in (2, 4, 29, 32, 34)} + new_records = {s: records(new, s) for s in (2, 4, 29, 32, 34)} + names = ("Krunch", "Bumper", "Tiptup", "Conker", "Timber", "Banjo", "Drumstick", "Pipsy", "T.T.", "Diddy") + audit = { + "target_sha256": hashlib.sha256(target).hexdigest(), + "target_lut": hex(args.target_lut), + "asset_sha256": hashlib.sha256(target[args.target_lut :]).hexdigest(), + "record_counts": {s: [len(old_records[s]), len(new_records[s])] for s in old_records}, + "characters": [], + } + for character, name in enumerate(names): + item = {"base": name, "id": character, "vehicles": []} for vehicle in range(3): - oid=ids[vehicle*10+character] - hid=u16(new[35],2*oid) - header=new_records[34][hid] - n=header[0x55];at=u32(header,0x10) - models=[u32(header,at+4*i) for i in range(n)] - model_data=[] + oid = ids[vehicle * 10 + character] + hid = u16(new[35], 2 * oid) + header = new_records[34][hid] + n = header[0x55] + at = u32(header, 0x10) + models = [u32(header, at + 4 * i) for i in range(n)] + model_data = [] for mid in models: - if mid==0xffffffff: + if mid == 0xFFFFFFFF: continue - b=new_records[29][mid];d=unpack(b) - changed=mid>=len(old_records[29]) or d!=unpack(old_records[29][mid]) - first=u16(new[30],mid*2);end=u16(new[30],mid*2+2) - textures=[u32(d,u32(d,0)+8*i) for i in range(u16(d,0x22))] - model_data.append({'id':mid,'changed':changed,'bytes':len(b),'decoded':len(d), - 'vertices':u16(d,0x24),'animated_vertices':u16(d,0x4a),'animations':[first,end], - 'animation_sizes': [{'id':a,'frames':u32(unpack(new_records[32][a]),0), - 'bytes':len(unpack(new_records[32][a]))} for a in range(first,end)], - 'textures':textures,'changed_textures':[t for t in textures if t>=len(old_records[2]) or new_records[2][t]!=old_records[2][t]]}) - item['vehicles'].append({'vehicle':vehicle,'object':oid,'header':hid, - 'name':header[0x60:0x70].split(b'\0')[0].decode('ascii','replace'), - 'header_changed':hid>=len(old_records[34]) or header!=old_records[34][hid], - 'models':model_data}) - audit['characters'].append(item) - audit['attachment_headers']=[] - for hid in (188,189,190,191,192): - header=new_records[34][hid] - audit['attachment_headers'].append({'id':hid,'type':header[0x53], - 'models':[u32(header,u32(header,0x10)+i*4) for i in range(header[0x55])], - 'changed':header!=old_records[34][hid], - 'differences':[i for i,(a,b) in enumerate(zip(header,old_records[34][hid])) if a!=b]}) - audit['changed_portraits']=[i for i,b in enumerate(new_records[4]) if 120<=i<=139 and (i>=len(old_records[4]) or b!=old_records[4][i])] - text=json.dumps(audit,indent=2)+'\n' + b = new_records[29][mid] + d = unpack(b) + changed = mid >= len(old_records[29]) or d != unpack(old_records[29][mid]) + first = u16(new[30], mid * 2) + end = u16(new[30], mid * 2 + 2) + textures = [u32(d, u32(d, 0) + 8 * i) for i in range(u16(d, 0x22))] + model_data.append( + { + "id": mid, + "changed": changed, + "bytes": len(b), + "decoded": len(d), + "vertices": u16(d, 0x24), + "animated_vertices": u16(d, 0x4A), + "animations": [first, end], + "animation_sizes": [ + { + "id": a, + "frames": u32(unpack(new_records[32][a]), 0), + "bytes": len(unpack(new_records[32][a])), + } + for a in range(first, end) + ], + "textures": textures, + "changed_textures": [ + t for t in textures if t >= len(old_records[2]) or new_records[2][t] != old_records[2][t] + ], + } + ) + item["vehicles"].append( + { + "vehicle": vehicle, + "object": oid, + "header": hid, + "name": header[0x60:0x70].split(b"\0")[0].decode("ascii", "replace"), + "header_changed": hid >= len(old_records[34]) or header != old_records[34][hid], + "models": model_data, + } + ) + audit["characters"].append(item) + audit["attachment_headers"] = [] + for hid in (188, 189, 190, 191, 192): + header = new_records[34][hid] + audit["attachment_headers"].append( + { + "id": hid, + "type": header[0x53], + "models": [u32(header, u32(header, 0x10) + i * 4) for i in range(header[0x55])], + "changed": header != old_records[34][hid], + "differences": [i for i, (a, b) in enumerate(zip(header, old_records[34][hid])) if a != b], + } + ) + audit["changed_portraits"] = [ + i + for i, b in enumerate(new_records[4]) + if 120 <= i <= 139 and (i >= len(old_records[4]) or b != old_records[4][i]) + ] + text = json.dumps(audit, indent=2) + "\n" if args.output: - with args.output.open('x',encoding='utf-8') as f: + with args.output.open("x", encoding="utf-8") as f: f.write(text) else: print(text) -if __name__=='__main__': +if __name__ == "__main__": main() diff --git a/scripts/legacy_asset_capacity_policy.py b/scripts/legacy_asset_capacity_policy.py index fc00e52d..38c21c3d 100644 --- a/scripts/legacy_asset_capacity_policy.py +++ b/scripts/legacy_asset_capacity_policy.py @@ -4,81 +4,113 @@ must expand both allocations and comparisons; pre-store guards prevent an unexpected exhaustion from corrupting adjacent guest allocations. """ + import copy import struct + +from compose_legacy_mod_policy import elf_functions, elf_sections from legacy_character_presentation_policy import reviewed_revision -from compose_legacy_mod_policy import elf_sections, elf_functions # function, pc v77/v80, expected operand, cache kind (textures/sprites/models/free) ALLOCATIONS = [ - ('tex_init_textures', 0x8007ac84, 0x8007b0d4, 0x240415e0, 0, 8), - ('tex_init_textures', 0x8007ad48, 0x8007b198, 0x24040320, 1, 8), - ('allocate_object_model_pools', 0x8005f864, 0x8005fa04, 0x24040230, 2, 8), - ('allocate_object_model_pools', 0x8005f87c, 0x8005fa1c, 0x24040190, 3, 4), + ("tex_init_textures", 0x8007AC84, 0x8007B0D4, 0x240415E0, 0, 8), + ("tex_init_textures", 0x8007AD48, 0x8007B198, 0x24040320, 1, 8), + ("allocate_object_model_pools", 0x8005F864, 0x8005FA04, 0x24040230, 2, 8), + ("allocate_object_model_pools", 0x8005F87C, 0x8005FA1C, 0x24040190, 3, 4), ] COMPARISONS = { - 'us.v77': [('load_texture',0x8007b288,0x2b0102bd,0x14200003,24,0,'<='), - ('tex_load_sprite',0x8007c4c4,0x29c10064,0x14200003,14,1,'<'), - ('object_model_init',0x8005fc94,0x29210046,0x10200003,9,2,'<')], - 'us.v80': [('load_texture',0x8007b6d8,0x2b0102bd,0x14200003,24,0,'<='), - ('tex_load_sprite',0x8007c914,0x29c10064,0x14200003,14,1,'<'), - ('object_model_init',0x8005fe98,0x2b210046,0x10200004,25,2,'<')], + "us.v77": [ + ("load_texture", 0x8007B288, 0x2B0102BD, 0x14200003, 24, 0, "<="), + ("tex_load_sprite", 0x8007C4C4, 0x29C10064, 0x14200003, 14, 1, "<"), + ("object_model_init", 0x8005FC94, 0x29210046, 0x10200003, 9, 2, "<"), + ], + "us.v80": [ + ("load_texture", 0x8007B6D8, 0x2B0102BD, 0x14200003, 24, 0, "<="), + ("tex_load_sprite", 0x8007C914, 0x29C10064, 0x14200003, 14, 1, "<"), + ("object_model_init", 0x8005FE98, 0x2B210046, 0x10200004, 25, 2, "<"), + ], } STORES = { - 'us.v77': [('load_texture',0x8007b104,0xadb00000,5,0), - ('tex_load_sprite',0x8007c4ec,0xad8b0000,15,1), - ('object_model_init',0x8005fc78,0xaf130000,13,2)], - 'us.v80': [('load_texture',0x8007b554,0xadb00000,5,0), - ('tex_load_sprite',0x8007c93c,0xad8b0000,15,1), - ('object_model_init',0x8005fe7c,0xadb20000,11,2)], + "us.v77": [ + ("load_texture", 0x8007B104, 0xADB00000, 5, 0), + ("tex_load_sprite", 0x8007C4EC, 0xAD8B0000, 15, 1), + ("object_model_init", 0x8005FC78, 0xAF130000, 13, 2), + ], + "us.v80": [ + ("load_texture", 0x8007B554, 0xADB00000, 5, 0), + ("tex_load_sprite", 0x8007C93C, 0xAD8B0000, 15, 1), + ("object_model_init", 0x8005FE7C, 0xADB20000, 11, 2), + ], } -REASON = 'Immutable additive asset cache capacities and pre-write bounds; preserve stock limits and prevent adjacent guest-memory corruption.' -DECL = 'extern uint32_t dkr_legacy_asset_cache_capacity(uint8_t*, recomp_context*, unsigned); ' +REASON = "Immutable additive asset cache capacities and pre-write bounds; preserve stock limits and prevent adjacent guest-memory corruption." +DECL = "extern uint32_t dkr_legacy_asset_cache_capacity(uint8_t*, recomp_context*, unsigned); " def compose_asset_capacity(policy, elf, revision, sections=None, symbols=None): if reviewed_revision(elf) != revision: - raise ValueError('Asset cache capacity requires a reviewed retail ELF') - if sections is None: sections=elf_sections(elf) - if symbols is None: symbols=elf_functions(elf) - words={base+i:struct.unpack_from('>I',data,i)[0] - for base,data in sections for i in range(0,len(data),4)} - result=copy.deepcopy(policy) - result.setdefault('functionHooks',[]);result.setdefault('instructionPatches',[]) + raise ValueError("Asset cache capacity requires a reviewed retail ELF") + if sections is None: + sections = elf_sections(elf) + if symbols is None: + symbols = elf_functions(elf) + words = {base + i: struct.unpack_from(">I", data, i)[0] for base, data in sections for i in range(0, len(data), 4)} + result = copy.deepcopy(policy) + result.setdefault("functionHooks", []) + result.setdefault("instructionPatches", []) - def checked(name,pc,expected): - matches=symbols.get(name,set()) - if len(matches)!=1:raise ValueError('Ambiguous asset cache symbol: '+name) - start,size=next(iter(matches)) - if not start<=pcr4 = dkr_legacy_asset_cache_capacity(rdram, ctx, {kind}U) * {stride}U;',True) - insert('instructionPatches','vram',dict(function=name,vram=hex(pc),value='0x00802025',reason=REASON)) - for name,pc,word,branch,register,kind,op in COMPARISONS[revision]: - checked(name,pc,word) - hook(name,pc+4,branch,DECL+f'ctx->r1 = (int32_t)ctx->r{register} {op} (int32_t)dkr_legacy_asset_cache_capacity(rdram, ctx, {kind}U);') - for name,pc,word,register,kind in STORES[revision]: - hook(name,pc,word,'extern void dkr_legacy_asset_cache_guard(uint8_t*, recomp_context*, unsigned, uint32_t); '+ - f'dkr_legacy_asset_cache_guard(rdram, ctx, {kind}U, (uint32_t)ctx->r{register});') - hook('track_setup_racers',0x8000d7a4,0xa44f0000, - 'if (ctx->r2 == 0) { extern void dkr_legacy_racer_spawn_failed(uint8_t*, recomp_context*); dkr_legacy_racer_spawn_failed(rdram, ctx); }') + hook( + name, pc, word, DECL + f"ctx->r4 = dkr_legacy_asset_cache_capacity(rdram, ctx, {kind}U) * {stride}U;", True + ) + insert("instructionPatches", "vram", dict(function=name, vram=hex(pc), value="0x00802025", reason=REASON)) + for name, pc, word, branch, register, kind, op in COMPARISONS[revision]: + checked(name, pc, word) + hook( + name, + pc + 4, + branch, + DECL + + f"ctx->r1 = (int32_t)ctx->r{register} {op} (int32_t)dkr_legacy_asset_cache_capacity(rdram, ctx, {kind}U);", + ) + for name, pc, word, register, kind in STORES[revision]: + hook( + name, + pc, + word, + "extern void dkr_legacy_asset_cache_guard(uint8_t*, recomp_context*, unsigned, uint32_t); " + + f"dkr_legacy_asset_cache_guard(rdram, ctx, {kind}U, (uint32_t)ctx->r{register});", + ) + hook( + "track_setup_racers", + 0x8000D7A4, + 0xA44F0000, + "if (ctx->r2 == 0) { extern void dkr_legacy_racer_spawn_failed(uint8_t*, recomp_context*); dkr_legacy_racer_spawn_failed(rdram, ctx); }", + ) return result diff --git a/scripts/legacy_behaviour_policy.py b/scripts/legacy_behaviour_policy.py index c3007eb0..1b82ea6a 100644 --- a/scripts/legacy_behaviour_policy.py +++ b/scripts/legacy_behaviour_policy.py @@ -4,82 +4,114 @@ and only a currently admitted custom course can request the replacement value. No retail unlock globals, files, assets or protected generated source edited. """ + import copy import struct + from compose_legacy_mod_policy import elf_functions, elf_sections from legacy_character_presentation_policy import reviewed_revision -SITES = ((0x8000c928, 0x8d030000, 3), - (0x8000c92c, 0x9450000c, 16), - (0x8000c974, 0x8d090000, 9)) +SITES = ((0x8000C928, 0x8D030000, 3), (0x8000C92C, 0x9450000C, 16), (0x8000C974, 0x8D090000, 9)) STATS = { - 'us.v77': (('update_player_racer',0x8004e244,0xc5480000,8,0), - ('update_player_racer',0x8004e2a8,0xc5e40000,4,1), - ('update_player_racer',0x8004e2d4,0xc5260000,6,2), - ('update_AI_racer',0x8005a7bc,0xc5c40000,4,0), - ('update_AI_racer',0x8005a820,0xc5c60000,6,1), - ('update_AI_racer',0x8005a84c,0xc5a80000,8,2)), - 'us.v80': (('update_player_racer',0x8004e284,0xc5480000,8,0), - ('update_player_racer',0x8004e2e8,0xc5e40000,4,1), - ('update_player_racer',0x8004e314,0xc5260000,6,2), - ('update_AI_racer',0x8005a828,0xc5c40000,4,0), - ('update_AI_racer',0x8005a88c,0xc5c60000,6,1), - ('update_AI_racer',0x8005a8b8,0xc5a80000,8,2))} + "us.v77": ( + ("update_player_racer", 0x8004E244, 0xC5480000, 8, 0), + ("update_player_racer", 0x8004E2A8, 0xC5E40000, 4, 1), + ("update_player_racer", 0x8004E2D4, 0xC5260000, 6, 2), + ("update_AI_racer", 0x8005A7BC, 0xC5C40000, 4, 0), + ("update_AI_racer", 0x8005A820, 0xC5C60000, 6, 1), + ("update_AI_racer", 0x8005A84C, 0xC5A80000, 8, 2), + ), + "us.v80": ( + ("update_player_racer", 0x8004E284, 0xC5480000, 8, 0), + ("update_player_racer", 0x8004E2E8, 0xC5E40000, 4, 1), + ("update_player_racer", 0x8004E314, 0xC5260000, 6, 2), + ("update_AI_racer", 0x8005A828, 0xC5C40000, 4, 0), + ("update_AI_racer", 0x8005A88C, 0xC5C60000, 6, 1), + ("update_AI_racer", 0x8005A8B8, 0xC5A80000, 8, 2), + ), +} + def compose_behaviour(policy, elf, revision): if reviewed_revision(elf) != revision: - raise ValueError('Behaviour bridge requires the pinned retail ELF') - symbols=elf_functions(elf).get('track_spawn_objects',set()) - if len(symbols)!=1: - raise ValueError('Behaviour spawn function has ambiguous bounds') - begin,size=next(iter(symbols)) - words={base+i:struct.unpack_from('>I',data,i)[0] - for base,data in elf_sections(elf) for i in range(0,len(data),4)} - result=copy.deepcopy(policy) - for event,(pc,word,reg) in enumerate(SITES): - if not begin<=pc or pc+8>begin+size or words.get(pc)!=word or pc+4 not in words: - raise ValueError('Behaviour spawn register/instruction boundary changed') - hook=dict(function='track_spawn_objects',beforeVram=hex(pc+4), - text=('extern uint32_t dkr_legacy_course_component(uint8_t*, recomp_context*, unsigned, uint32_t); ' - f'ctx->r{reg} = (gpr)(int32_t)dkr_legacy_course_component(rdram, ctx, {event}U, (uint32_t)ctx->r{reg});'), - reason='Execute only the admitted course\'s reviewed constant-load component; native instructions and original unlock/save state retained.') - owners=[h for h in result.setdefault('functionHooks',[]) if int(h['beforeVram'],0)==pc+4] - if any(int(p['vram'],0) in (pc,pc+4) for p in result.get('instructionPatches',[])): - raise ValueError('Behaviour bridge conflicts with an instruction owner') + raise ValueError("Behaviour bridge requires the pinned retail ELF") + symbols = elf_functions(elf).get("track_spawn_objects", set()) + if len(symbols) != 1: + raise ValueError("Behaviour spawn function has ambiguous bounds") + begin, size = next(iter(symbols)) + words = { + base + i: struct.unpack_from(">I", data, i)[0] + for base, data in elf_sections(elf) + for i in range(0, len(data), 4) + } + result = copy.deepcopy(policy) + for event, (pc, word, reg) in enumerate(SITES): + if not begin <= pc or pc + 8 > begin + size or words.get(pc) != word or pc + 4 not in words: + raise ValueError("Behaviour spawn register/instruction boundary changed") + hook = dict( + function="track_spawn_objects", + beforeVram=hex(pc + 4), + text=( + "extern uint32_t dkr_legacy_course_component(uint8_t*, recomp_context*, unsigned, uint32_t); " + f"ctx->r{reg} = (gpr)(int32_t)dkr_legacy_course_component(rdram, ctx, {event}U, (uint32_t)ctx->r{reg});" + ), + reason="Execute only the admitted course's reviewed constant-load component; native instructions and original unlock/save state retained.", + ) + owners = [h for h in result.setdefault("functionHooks", []) if int(h["beforeVram"], 0) == pc + 4] + if any(int(p["vram"], 0) in (pc, pc + 4) for p in result.get("instructionPatches", [])): + raise ValueError("Behaviour bridge conflicts with an instruction owner") if owners: - if owners!=[hook]:raise ValueError('Behaviour bridge conflicts with a hook owner') - else:result['functionHooks'].append(hook) - functions=elf_functions(elf) - name='obj_tex_animate' - bounds=functions.get(name,set()) - if len(bounds)!=1:raise ValueError('Texture animation entry has ambiguous bounds') - pc,size=next(iter(bounds)) - if size!=304 or words.get(pc)!=0x27bdff98 or len(functions.get('tex_animate_texture',set()))!=1: - raise ValueError('Texture animation entry/callback ABI changed') - hook=dict(function=name,beforeVram=hex(pc), - text=('extern int dkr_legacy_texture_animation(uint8_t*, recomp_context*, recomp_func_t*); ' - 'if (dkr_legacy_texture_animation(rdram, ctx, tex_animate_texture)) return;'), - reason='Reviewed random-texture phase sharing for admitted custom character headers only; original animation/RNG routine and guest checkpoint state retained.') - owners=[h for h in result['functionHooks'] if int(h['beforeVram'],0)==pc] - if any(int(p['vram'],0)==pc for p in result.get('instructionPatches',[])): - raise ValueError('Texture animation entry conflicts with an instruction owner') + if owners != [hook]: + raise ValueError("Behaviour bridge conflicts with a hook owner") + else: + result["functionHooks"].append(hook) + functions = elf_functions(elf) + name = "obj_tex_animate" + bounds = functions.get(name, set()) + if len(bounds) != 1: + raise ValueError("Texture animation entry has ambiguous bounds") + pc, size = next(iter(bounds)) + if size != 304 or words.get(pc) != 0x27BDFF98 or len(functions.get("tex_animate_texture", set())) != 1: + raise ValueError("Texture animation entry/callback ABI changed") + hook = dict( + function=name, + beforeVram=hex(pc), + text=( + "extern int dkr_legacy_texture_animation(uint8_t*, recomp_context*, recomp_func_t*); " + "if (dkr_legacy_texture_animation(rdram, ctx, tex_animate_texture)) return;" + ), + reason="Reviewed random-texture phase sharing for admitted custom character headers only; original animation/RNG routine and guest checkpoint state retained.", + ) + owners = [h for h in result["functionHooks"] if int(h["beforeVram"], 0) == pc] + if any(int(p["vram"], 0) == pc for p in result.get("instructionPatches", [])): + raise ValueError("Texture animation entry conflicts with an instruction owner") if owners: - if owners!=[hook]:raise ValueError('Texture animation entry conflicts with a hook owner') - else:result['functionHooks'].append(hook) - for name,pc,word,reg,stat in STATS[revision]: - bounds=functions.get(name,set()) - if len(bounds)!=1:raise ValueError('Racer parameter function has ambiguous bounds') - begin,size=next(iter(bounds)) - if not begin<=pc or pc+8>begin+size or words.get(pc)!=word or pc+4 not in words: - raise ValueError('Racer parameter load/register ABI changed') - hook=dict(function=name,beforeVram=hex(pc+4), - text=('extern uint32_t dkr_legacy_racer_stat(uint8_t*, recomp_context*, unsigned, uint32_t, uint32_t); ' - f'ctx->f{reg}.u32l = dkr_legacy_racer_stat(rdram, ctx, {stat}U, (uint32_t)ctx->r16, ctx->f{reg}.u32l);'), - reason='Apply finite per-instance custom racer stats immediately after the original table lookup; stock, other donors, ghosts and boss objects unchanged.') - owners=[h for h in result['functionHooks'] if int(h['beforeVram'],0)==pc+4] - if any(int(p['vram'],0) in (pc,pc+4) for p in result.get('instructionPatches',[])): - raise ValueError('Racer parameter bridge conflicts with an instruction owner') + if owners != [hook]: + raise ValueError("Texture animation entry conflicts with a hook owner") + else: + result["functionHooks"].append(hook) + for name, pc, word, reg, stat in STATS[revision]: + bounds = functions.get(name, set()) + if len(bounds) != 1: + raise ValueError("Racer parameter function has ambiguous bounds") + begin, size = next(iter(bounds)) + if not begin <= pc or pc + 8 > begin + size or words.get(pc) != word or pc + 4 not in words: + raise ValueError("Racer parameter load/register ABI changed") + hook = dict( + function=name, + beforeVram=hex(pc + 4), + text=( + "extern uint32_t dkr_legacy_racer_stat(uint8_t*, recomp_context*, unsigned, uint32_t, uint32_t); " + f"ctx->f{reg}.u32l = dkr_legacy_racer_stat(rdram, ctx, {stat}U, (uint32_t)ctx->r16, ctx->f{reg}.u32l);" + ), + reason="Apply finite per-instance custom racer stats immediately after the original table lookup; stock, other donors, ghosts and boss objects unchanged.", + ) + owners = [h for h in result["functionHooks"] if int(h["beforeVram"], 0) == pc + 4] + if any(int(p["vram"], 0) in (pc, pc + 4) for p in result.get("instructionPatches", [])): + raise ValueError("Racer parameter bridge conflicts with an instruction owner") if owners: - if owners!=[hook]:raise ValueError('Racer parameter bridge conflicts with a hook owner') - else:result['functionHooks'].append(hook) + if owners != [hook]: + raise ValueError("Racer parameter bridge conflicts with a hook owner") + else: + result["functionHooks"].append(hook) return result diff --git a/scripts/legacy_character_ai_policy.py b/scripts/legacy_character_ai_policy.py index 0e6920e2..fce64a25 100644 --- a/scripts/legacy_character_ai_policy.py +++ b/scripts/legacy_character_ai_policy.py @@ -1,81 +1,105 @@ """Checked, offline-only AI identity hooks; never edit generated functions.""" + import argparse import copy import hashlib import json -import struct from pathlib import Path +import struct + from compose_legacy_mod_policy import elf_functions, elf_sections from legacy_character_presentation_policy import reviewed_revision BODY = { - 'us.v77': '293ecb84ee5f83794329821a968781bb283ee98ccbef329c3a8a219fea43228e', - 'us.v80': '61d525b58f578bc7816ea402b577fdc9721a0fae8edc2526a1f9df0d4e320013', + "us.v77": "293ecb84ee5f83794329821a968781bb283ee98ccbef329c3a8a219fea43228e", + "us.v80": "61d525b58f578bc7816ea402b577fdc9721a0fae8edc2526a1f9df0d4e320013", } + def checked_sites(elf): revision = reviewed_revision(elf) if revision is None: - raise ValueError('AI adapter requires the pinned retail ELF') + raise ValueError("AI adapter requires the pinned retail ELF") symbols = elf_functions(elf) - start, size = 0x8000cc7c, 0x1434 - if symbols.get('track_setup_racers') != {(start, size)}: - raise ValueError('Race-construction bounds changed') - raw = next(data[start-base:start-base+size] for base, data in elf_sections(elf) - if base <= start and start+size <= base+len(data)) + start, size = 0x8000CC7C, 0x1434 + if symbols.get("track_setup_racers") != {(start, size)}: + raise ValueError("Race-construction bounds changed") + raw = next( + data[start - base : start - base + size] + for base, data in elf_sections(elf) + if base <= start and start + size <= base + len(data) + ) if hashlib.sha256(raw).hexdigest() != BODY[revision]: - raise ValueError('Race-construction body changed') + raise ValueError("Race-construction body changed") # Includes topology stack slots, Settings register, racer-index register, # entry register, primary (not ghost) spawn call and its continuation. signatures = { - 0: 0x27bdfeb0, 0x10c: 0xafa20138, 0x110: 0x0040b825, - 0x318: 0xafa20144, 0x474: 0x8cc30000, - 0x7c8: 0x0200a025, 0xb00: 0x02c02025, - 0xb0c: 0x0c003a95, 0xb10: 0xafa7005c, 0xb14: 0x8fb80044, + 0: 0x27BDFEB0, + 0x10C: 0xAFA20138, + 0x110: 0x0040B825, + 0x318: 0xAFA20144, + 0x474: 0x8CC30000, + 0x7C8: 0x0200A025, + 0xB00: 0x02C02025, + 0xB0C: 0x0C003A95, + 0xB10: 0xAFA7005C, + 0xB14: 0x8FB80044, } for offset, word in signatures.items(): - if struct.unpack_from('>I', raw, offset)[0] != word: - raise ValueError(f'AI register/stack ABI changed at {start+offset:#x}') - return [(start, 0), (start+0x474, 1), (start+0xb0c, 2), (start+0xb14, 3)] + if struct.unpack_from(">I", raw, offset)[0] != word: + raise ValueError(f"AI register/stack ABI changed at {start + offset:#x}") + return [(start, 0), (start + 0x474, 1), (start + 0xB0C, 2), (start + 0xB14, 3)] + def hook_text(event): - return ('extern void dkr_legacy_character_ai_event(uint8_t*, recomp_context*, unsigned, ' - 'void (*)(uint8_t*, recomp_context*)); ' - f'dkr_legacy_character_ai_event(rdram, ctx, {event}U, rand_range);') + return ( + "extern void dkr_legacy_character_ai_event(uint8_t*, recomp_context*, unsigned, " + "void (*)(uint8_t*, recomp_context*)); " + f"dkr_legacy_character_ai_event(rdram, ctx, {event}U, rand_range);" + ) + def verify_character_ai_policy(policy, elf): - hooks = [h for h in policy.get('functionHooks', []) if 'dkr_legacy_character_ai_event' in h.get('text', '')] + hooks = [h for h in policy.get("functionHooks", []) if "dkr_legacy_character_ai_event" in h.get("text", "")] if not hooks: return sites = checked_sites(elf) expected = {(pc, hook_text(event)) for pc, event in sites} - actual = {(int(h['beforeVram'], 0), h['text']) for h in hooks if h['function'] == 'track_setup_racers'} + actual = {(int(h["beforeVram"], 0), h["text"]) for h in hooks if h["function"] == "track_setup_racers"} if len(hooks) != 4 or actual != expected: - raise ValueError('AI hook ownership changed') + raise ValueError("AI hook ownership changed") for pc, _ in sites: - if sum(int(h['beforeVram'], 0) == pc for h in policy['functionHooks']) != 1 or any( - int(p['vram'], 0) == pc for p in policy.get('instructionPatches', [])): - raise ValueError('Conflicting AI hook owner') + if sum(int(h["beforeVram"], 0) == pc for h in policy["functionHooks"]) != 1 or any( + int(p["vram"], 0) == pc for p in policy.get("instructionPatches", []) + ): + raise ValueError("Conflicting AI hook owner") + def compose_character_ai(policy, elf): result = copy.deepcopy(policy) - if any('dkr_legacy_character_ai_event' in h.get('text', '') for h in result.get('functionHooks', [])): + if any("dkr_legacy_character_ai_event" in h.get("text", "") for h in result.get("functionHooks", [])): verify_character_ai_policy(result, elf) return result for pc, event in checked_sites(elf): - result.setdefault('functionHooks', []).append({ - 'function': 'track_setup_racers', 'beforeVram': hex(pc), 'text': hook_text(event), - 'reason': 'Offline custom AI identities are selected after native topology and scoped to the primary racer spawn; boss, hub, demo and ghost paths remain stock.'}) + result.setdefault("functionHooks", []).append( + { + "function": "track_setup_racers", + "beforeVram": hex(pc), + "text": hook_text(event), + "reason": "Offline custom AI identities are selected after native topology and scoped to the primary racer spawn; boss, hub, demo and ghost paths remain stock.", + } + ) verify_character_ai_policy(result, elf) return result -if __name__ == '__main__': + +if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--policy', type=Path, required=True) - parser.add_argument('--elf', type=Path, required=True) - parser.add_argument('--output', type=Path, required=True) + parser.add_argument("--policy", type=Path, required=True) + parser.add_argument("--elf", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() updated = compose_character_ai(json.loads(args.policy.read_text()), args.elf) args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(updated, indent=2)+'\n') - print('Verified all four AI identity boundaries; original instructions preserved.') + args.output.write_text(json.dumps(updated, indent=2) + "\n") + print("Verified all four AI identity boundaries; original instructions preserved.") diff --git a/scripts/legacy_character_presentation_policy.py b/scripts/legacy_character_presentation_policy.py index 97567ebe..00522d91 100644 --- a/scripts/legacy_character_presentation_policy.py +++ b/scripts/legacy_character_presentation_policy.py @@ -3,13 +3,14 @@ Only the two retail ELF images are admitted (see elf_image_digest). The native lookup/call instructions remain unchanged: callbacks redirect their local arguments. """ + import copy import hashlib import struct ELFS = { - 'us.v77': '1903f3d12581a3beb7288f32094dfe60ca766ada637de9d7bb27044146a25316', - 'us.v80': '1d8cf535e78e64ce5848338b13fd0a4674c3f44d68c9bdebec228d40e346a47c', + "us.v77": "1903f3d12581a3beb7288f32094dfe60ca766ada637de9d7bb27044146a25316", + "us.v80": "1d8cf535e78e64ce5848338b13fd0a4674c3f44d68c9bdebec228d40e346a47c", } @@ -20,21 +21,21 @@ def elf_image_digest(path): data, and therefore the ROM, stay byte-identical.""" data = path.read_bytes() if len(data) < 52 or data[:6] != b"\x7fELF\x01\x02": - raise ValueError('Expected a big-endian ELF32 DKR image') - start = struct.unpack_from('>I', data, 32)[0] - stride, count = struct.unpack_from('>HH', data, 46) + raise ValueError("Expected a big-endian ELF32 DKR image") + start = struct.unpack_from(">I", data, 32)[0] + stride, count = struct.unpack_from(">HH", data, 46) if stride != 40 or start + stride * count > len(data): - raise ValueError('Invalid ELF section directory') + raise ValueError("Invalid ELF section directory") digest = hashlib.sha256() for index in range(count): - _, kind, flags, address, offset, size, *_ = struct.unpack_from('>10I', data, start + index * stride) + _, kind, flags, address, offset, size, *_ = struct.unpack_from(">10I", data, start + index * stride) if not flags & 2: # SHF_ALLOC continue - digest.update(struct.pack('>III', kind, address, size)) + digest.update(struct.pack(">III", kind, address, size)) if kind != 8: # SHT_NOBITS if offset + size > len(data): - raise ValueError('Invalid allocated ELF section') - digest.update(data[offset:offset + size]) + raise ValueError("Invalid allocated ELF section") + digest.update(data[offset : offset + size]) return digest.hexdigest() @@ -42,109 +43,179 @@ def reviewed_revision(path): """The retail revision whose reviewed image this ELF matches, or None.""" digest = elf_image_digest(path) return next((revision for revision, expected in ELFS.items() if expected == digest), None) + + # function, address, Settings+racer*24 register, lookup-address register, lw word PORTRAITS = { - 'us.v77': [('postrace_load',0x80094ae8,10,13,0x8dae0000), - ('postrace_load',0x80094b44,2,10,0x8d4b0000), - ('results_render',0x80096b94,16,25,0x8f250000), - ('func_80098774',0x80098984,3,25,0x8f2e0000), - ('func_80098774',0x800989c0,15,14,0x8dcf0000)], - 'us.v80': [('postrace_load',0x80094fec,10,13,0x8dae0000), - ('postrace_load',0x80095048,2,10,0x8d4b0000), - ('results_render',0x800970bc,16,11,0x8d650000), - ('func_80098774',0x80098ec0,3,25,0x8f2e0000), - ('func_80098774',0x80098efc,15,14,0x8dcf0000)], + "us.v77": [ + ("postrace_load", 0x80094AE8, 10, 13, 0x8DAE0000), + ("postrace_load", 0x80094B44, 2, 10, 0x8D4B0000), + ("results_render", 0x80096B94, 16, 25, 0x8F250000), + ("func_80098774", 0x80098984, 3, 25, 0x8F2E0000), + ("func_80098774", 0x800989C0, 15, 14, 0x8DCF0000), + ], + "us.v80": [ + ("postrace_load", 0x80094FEC, 10, 13, 0x8DAE0000), + ("postrace_load", 0x80095048, 2, 10, 0x8D4B0000), + ("results_render", 0x800970BC, 16, 11, 0x8D650000), + ("func_80098774", 0x80098EC0, 3, 25, 0x8F2E0000), + ("func_80098774", 0x80098EFC, 15, 14, 0x8DCF0000), + ], } # Call site, owner expression, HUD argument adjustment, original delay slot. HUD_CALLS = { - 'us.v77': [('hud_eggs_portrait',0x800a1a6c,'ctx->r19',0x640,0x24e70640), - ('hud_lives_render',0x800a2380,'MEM_W(ctx->r29, 0x20)',0x640,0x24e70640), - ('hud_treasure',0x800a46bc,'ctx->r22',0x640,0x24e70640), - ('hud_main_hub',0x800a275c,'1U',0,0xa4f90006)], - 'us.v80': [('hud_eggs_portrait',0x800a1fac,'ctx->r19',0x640,0x24e70640), - ('hud_lives_render',0x800a28c0,'MEM_W(ctx->r29, 0x20)',0x640,0x24e70640), - ('hud_treasure',0x800a4c04,'ctx->r22',0x640,0x24e70640), - ('hud_main_hub',0x800a2c9c,'1U',0,0xa4f90006)], + "us.v77": [ + ("hud_eggs_portrait", 0x800A1A6C, "ctx->r19", 0x640, 0x24E70640), + ("hud_lives_render", 0x800A2380, "MEM_W(ctx->r29, 0x20)", 0x640, 0x24E70640), + ("hud_treasure", 0x800A46BC, "ctx->r22", 0x640, 0x24E70640), + ("hud_main_hub", 0x800A275C, "1U", 0, 0xA4F90006), + ], + "us.v80": [ + ("hud_eggs_portrait", 0x800A1FAC, "ctx->r19", 0x640, 0x24E70640), + ("hud_lives_render", 0x800A28C0, "MEM_W(ctx->r29, 0x20)", 0x640, 0x24E70640), + ("hud_treasure", 0x800A4C04, "ctx->r22", 0x640, 0x24E70640), + ("hud_main_hub", 0x800A2C9C, "1U", 0, 0xA4F90006), + ], } # PAL, unit-scale (batched/immediate), and arbitrary-scale texture lookups. # Finish-position digits and all layout instructions remain unmodified. -HUD_LOADS = {'us.v77':[(0x800aab2c,12,0x8d820000),(0x800aabd8,14,0x8dc30000),(0x800aad30,24,0x8f020000)], - 'us.v80':[(0x800ab088,12,0x8d820000),(0x800ab134,14,0x8dc30000),(0x800ab28c,24,0x8f020000)]} -CINEMATIC = {'us.v77':(0x800993d8,0x8009ae94),'us.v80':(0x80099914,0x8009b3d0)} +HUD_LOADS = { + "us.v77": [(0x800AAB2C, 12, 0x8D820000), (0x800AABD8, 14, 0x8DC30000), (0x800AAD30, 24, 0x8F020000)], + "us.v80": [(0x800AB088, 12, 0x8D820000), (0x800AB134, 14, 0x8DC30000), (0x800AB28C, 24, 0x8F020000)], +} +CINEMATIC = {"us.v77": (0x800993D8, 0x8009AE94), "us.v80": (0x80099914, 0x8009B3D0)} + def compose_presentation(policy, elf, revision, sections, symbols, *, extended=True): if reviewed_revision(elf) != revision: - raise ValueError('Custom presentation requires a reviewed retail ELF') - result=copy.deepcopy(policy) - words={base+i:struct.unpack_from('>I',data,i)[0] for base,data in sections for i in range(0,len(data),4)} + raise ValueError("Custom presentation requires a reviewed retail ELF") + result = copy.deepcopy(policy) + words = {base + i: struct.unpack_from(">I", data, i)[0] for base, data in sections for i in range(0, len(data), 4)} + def one(name): - values=symbols.get(name,set()) - if len(values)!=1:raise ValueError('Ambiguous presentation function '+name) + values = symbols.get(name, set()) + if len(values) != 1: + raise ValueError("Ambiguous presentation function " + name) return next(iter(values)) - def hook(name,pc,text,expected): - start,size=one(name) - if pc%4 or not start<=pcr{subject}); ' - f'if (cell) ctx->r{lookup} = (int32_t)cell; }}',expected) - voice,horn=(0x800571f0,0x8005708c) if revision=='us.v77' else (0x80057230,0x800570cc) - bonus,banana=(0x8003b30c,0x8003db10) if revision=='us.v77' else (0x8003b34c,0x8003db50) - voices=[ - ('play_random_character_voice',voice,'audspat_play_sound_at_position',18,8,0x3104ffff), - ('racer_play_sound',horn,'sound_play_spatial',2,4,0xafa00010)] - if extended:voices += [ - ('obj_loop_bonus',bonus,'sound_play_spatial',17,9,0x01202025), - ('obj_loop_banana',banana,'sound_play_spatial',8,4,0xafa00010)] - for name,pc,target,racer,value,delay in voices: - if words.get(pc+4)!=delay:raise ValueError('Race voice delay-slot ownership changed') - hook(name,pc,'{ extern unsigned dkr_legacy_character_race_sound(uint8_t*, recomp_context*, uint32_t, unsigned); ' - f'ctx->r{value} = dkr_legacy_character_race_sound(rdram, ctx, (uint32_t)ctx->r{racer}, (unsigned)ctx->r{value}); }}', - 0x0c000000|((one(target)[0]>>2)&0x3ffffff)) - if extended and words.get(one('hud_element_render')[0])!=0x27bdff48: - raise ValueError('HUD renderer stack ownership changed') - for name,pc,owner,offset,delay in (HUD_CALLS[revision] if extended else []): - if words.get(pc+4)!=delay:raise ValueError('HUD portrait delay-slot ownership changed') - hook(name,pc,'{ extern void dkr_legacy_character_hud_bind(uint8_t*, recomp_context*, uint32_t, uint32_t); ' - f'dkr_legacy_character_hud_bind(rdram, ctx, (uint32_t)ctx->r7 + {offset}U, (uint32_t)({owner})); }}', - 0x0c000000|((one('hud_element_render')[0]>>2)&0x3ffffff)) - hook(name,pc+8,'{ extern void dkr_legacy_character_hud_unbind(uint8_t*, recomp_context*); ' - 'dkr_legacy_character_hud_unbind(rdram, ctx); }',words[pc+8]) - for pc,lookup,expected in (HUD_LOADS[revision] if extended else []): - hook('hud_element_render',pc,'{ extern uint32_t dkr_legacy_character_hud_lookup(uint8_t*, recomp_context*, uint32_t, uint32_t); ' - 'uint32_t cell = dkr_legacy_character_hud_lookup(rdram, ctx, (uint32_t)ctx->r29 + 0xB8U, (uint32_t)ctx->r16); ' - f'if (cell) ctx->r{lookup} = (int32_t)cell; }}',expected) + result["functionHooks"].append( + { + "function": name, + "beforeVram": hex(pc), + "text": text, + "reason": "Session-owned custom portraits/voices for committed human racers only; preserve native layout, RNG, gating and audio lifetimes.", + } + ) + + for name, pc, subject, lookup, expected in PORTRAITS[revision]: + hook( + name, + pc, + "{ extern uint32_t dkr_legacy_character_portrait_lookup(uint8_t*, recomp_context*, uint32_t); " + f"uint32_t cell = dkr_legacy_character_portrait_lookup(rdram, ctx, (uint32_t)ctx->r{subject}); " + f"if (cell) ctx->r{lookup} = (int32_t)cell; }}", + expected, + ) + voice, horn = (0x800571F0, 0x8005708C) if revision == "us.v77" else (0x80057230, 0x800570CC) + bonus, banana = (0x8003B30C, 0x8003DB10) if revision == "us.v77" else (0x8003B34C, 0x8003DB50) + voices = [ + ("play_random_character_voice", voice, "audspat_play_sound_at_position", 18, 8, 0x3104FFFF), + ("racer_play_sound", horn, "sound_play_spatial", 2, 4, 0xAFA00010), + ] + if extended: + voices += [ + ("obj_loop_bonus", bonus, "sound_play_spatial", 17, 9, 0x01202025), + ("obj_loop_banana", banana, "sound_play_spatial", 8, 4, 0xAFA00010), + ] + for name, pc, target, racer, value, delay in voices: + if words.get(pc + 4) != delay: + raise ValueError("Race voice delay-slot ownership changed") + hook( + name, + pc, + "{ extern unsigned dkr_legacy_character_race_sound(uint8_t*, recomp_context*, uint32_t, unsigned); " + f"ctx->r{value} = dkr_legacy_character_race_sound(rdram, ctx, (uint32_t)ctx->r{racer}, (unsigned)ctx->r{value}); }}", + 0x0C000000 | ((one(target)[0] >> 2) & 0x3FFFFFF), + ) + if extended and words.get(one("hud_element_render")[0]) != 0x27BDFF48: + raise ValueError("HUD renderer stack ownership changed") + for name, pc, owner, offset, delay in HUD_CALLS[revision] if extended else []: + if words.get(pc + 4) != delay: + raise ValueError("HUD portrait delay-slot ownership changed") + hook( + name, + pc, + "{ extern void dkr_legacy_character_hud_bind(uint8_t*, recomp_context*, uint32_t, uint32_t); " + f"dkr_legacy_character_hud_bind(rdram, ctx, (uint32_t)ctx->r7 + {offset}U, (uint32_t)({owner})); }}", + 0x0C000000 | ((one("hud_element_render")[0] >> 2) & 0x3FFFFFF), + ) + hook( + name, + pc + 8, + "{ extern void dkr_legacy_character_hud_unbind(uint8_t*, recomp_context*); " + "dkr_legacy_character_hud_unbind(rdram, ctx); }", + words[pc + 8], + ) + for pc, lookup, expected in HUD_LOADS[revision] if extended else []: + hook( + "hud_element_render", + pc, + "{ extern uint32_t dkr_legacy_character_hud_lookup(uint8_t*, recomp_context*, uint32_t, uint32_t); " + "uint32_t cell = dkr_legacy_character_hud_lookup(rdram, ctx, (uint32_t)ctx->r29 + 0xB8U, (uint32_t)ctx->r16); " + f"if (cell) ctx->r{lookup} = (int32_t)cell; }}", + expected, + ) if extended: - capture,draw=CINEMATIC[revision] - if words.get(capture-4)!=0x80820059:raise ValueError('Cinematic owner load changed') - hook('menu_trophy_race_rankings_loop',capture, - '{ extern unsigned dkr_legacy_character_cinematic_id(uint8_t*, recomp_context*, unsigned, unsigned); ' - 'ctx->r2 = dkr_legacy_character_cinematic_id(rdram, ctx, (unsigned)ctx->r14, (unsigned)ctx->r2); }',0x14600005) - hook('menu_cinematic_loop',draw, - '{ extern uint32_t dkr_legacy_character_cinematic_portrait(uint8_t*, recomp_context*, unsigned); ' - 'uint32_t cell = dkr_legacy_character_cinematic_portrait(rdram, ctx, (unsigned)ctx->r17); ' - 'if (cell) ctx->r24 = (int32_t)cell; }',0x8f050000) - for kind,name in enumerate(('audspat_play_sound_at_position','sound_play_direct','sound_play')): - pc=one(name)[0] - hook(name,pc,'{ extern int dkr_legacy_character_play_sound(uint8_t*, recomp_context*, unsigned); ' - f'if (dkr_legacy_character_play_sound(rdram, ctx, {kind}U)) return; }}',words[pc]) + capture, draw = CINEMATIC[revision] + if words.get(capture - 4) != 0x80820059: + raise ValueError("Cinematic owner load changed") + hook( + "menu_trophy_race_rankings_loop", + capture, + "{ extern unsigned dkr_legacy_character_cinematic_id(uint8_t*, recomp_context*, unsigned, unsigned); " + "ctx->r2 = dkr_legacy_character_cinematic_id(rdram, ctx, (unsigned)ctx->r14, (unsigned)ctx->r2); }", + 0x14600005, + ) + hook( + "menu_cinematic_loop", + draw, + "{ extern uint32_t dkr_legacy_character_cinematic_portrait(uint8_t*, recomp_context*, unsigned); " + "uint32_t cell = dkr_legacy_character_cinematic_portrait(rdram, ctx, (unsigned)ctx->r17); " + "if (cell) ctx->r24 = (int32_t)cell; }", + 0x8F050000, + ) + for kind, name in enumerate(("audspat_play_sound_at_position", "sound_play_direct", "sound_play")): + pc = one(name)[0] + hook( + name, + pc, + "{ extern int dkr_legacy_character_play_sound(uint8_t*, recomp_context*, unsigned); " + f"if (dkr_legacy_character_play_sound(rdram, ctx, {kind}U)) return; }}", + words[pc], + ) return result + def refresh_presentation(policy, elf, revision, sections, symbols): """Replace only an exact, known presentation layer in a composed policy. @@ -152,20 +223,35 @@ def refresh_presentation(policy, elf, revision, sections, symbols): adapter would duplicate its hooks. Strip only this layer's verified text, preserving shared owners and every unrelated patch byte-for-byte. """ - callbacks=('dkr_legacy_character_portrait_lookup','dkr_legacy_character_race_sound', - 'dkr_legacy_character_play_sound','dkr_legacy_character_hud_','dkr_legacy_character_cinematic_') - if not any(any(c in h['text'] for c in callbacks) for h in policy['functionHooks']): - return compose_presentation(policy,elf,revision,sections,symbols) - for extended in (True,False): - expected=compose_presentation({'functionHooks':[],'instructionPatches':[]},elf,revision,sections,symbols,extended=extended) - result=copy.deepcopy(policy) - for old in expected['functionHooks']: - matches=[h for h in result['functionHooks'] if h['function']==old['function'] and int(h['beforeVram'],0)==int(old['beforeVram'],0)] - if len(matches)!=1:break - h=matches[0] - if h['text']==old['text']:result['functionHooks'].remove(h) - elif old['function']=='sound_play' and h['text'].endswith(' '+old['text']): - h['text']=h['text'][:-len(old['text'])-1] - else:break - else:return compose_presentation(result,elf,revision,sections,symbols) - raise ValueError('Existing custom presentation is not an exact reviewed layer') + callbacks = ( + "dkr_legacy_character_portrait_lookup", + "dkr_legacy_character_race_sound", + "dkr_legacy_character_play_sound", + "dkr_legacy_character_hud_", + "dkr_legacy_character_cinematic_", + ) + if not any(any(c in h["text"] for c in callbacks) for h in policy["functionHooks"]): + return compose_presentation(policy, elf, revision, sections, symbols) + for extended in (True, False): + expected = compose_presentation( + {"functionHooks": [], "instructionPatches": []}, elf, revision, sections, symbols, extended=extended + ) + result = copy.deepcopy(policy) + for old in expected["functionHooks"]: + matches = [ + h + for h in result["functionHooks"] + if h["function"] == old["function"] and int(h["beforeVram"], 0) == int(old["beforeVram"], 0) + ] + if len(matches) != 1: + break + h = matches[0] + if h["text"] == old["text"]: + result["functionHooks"].remove(h) + elif old["function"] == "sound_play" and h["text"].endswith(" " + old["text"]): + h["text"] = h["text"][: -len(old["text"]) - 1] + else: + break + else: + return compose_presentation(result, elf, revision, sections, symbols) + raise ValueError("Existing custom presentation is not an exact reviewed layer") diff --git a/scripts/legacy_memory_safety_policy.py b/scripts/legacy_memory_safety_policy.py index 15335c40..b72fd351 100644 --- a/scripts/legacy_memory_safety_policy.py +++ b/scripts/legacy_memory_safety_policy.py @@ -3,125 +3,201 @@ No generated source or dependency is patched in place. Native/owned adapters share guest-resident ownership; all instruction/delay slots are verified. """ + import copy import struct + +from compose_legacy_mod_policy import elf_functions, elf_sections from legacy_character_presentation_policy import reviewed_revision -from compose_legacy_mod_policy import elf_sections, elf_functions def compose_memory_safety(policy, elf, revision): if reviewed_revision(elf) != revision: - raise ValueError('Memory safety requires a reviewed retail ELF') + raise ValueError("Memory safety requires a reviewed retail ELF") symbols = elf_functions(elf, (1, 2)) - words = {base+i: struct.unpack_from('>I', data, i)[0] - for base, data in elf_sections(elf) for i in range(0, len(data), 4)} + words = { + base + i: struct.unpack_from(">I", data, i)[0] + for base, data in elf_sections(elf) + for i in range(0, len(data), 4) + } result = copy.deepcopy(policy) - rev = 77 if revision == 'us.v77' else 80 + rev = 77 if revision == "us.v77" else 80 memory_delta = 0 if rev == 77 else 0x240 transition_delta = 0 if rev == 77 else 0x520 - declaration = 'extern uint32_t dkr_legacy_memory_service(uint8_t*, recomp_context*, unsigned, unsigned, uint32_t, uint32_t); ' - reason = 'Pre-write guest heap/queue bounds, deferred reserve ownership, and allocation-free transition fallback.' + declaration = ( + "extern uint32_t dkr_legacy_memory_service(uint8_t*, recomp_context*, unsigned, unsigned, uint32_t, uint32_t); " + ) + reason = "Pre-write guest heap/queue bounds, deferred reserve ownership, and allocation-free transition fallback." def symbol(name): matches = symbols.get(name, set()) if len(matches) != 1: - raise ValueError('Ambiguous memory safety symbol: '+name) + raise ValueError("Ambiguous memory safety symbol: " + name) return next(iter(matches))[0] - def service(op, a='0U', b='0U'): - return f'dkr_legacy_memory_service(rdram, ctx, {op}U, {rev}U, (uint32_t)({a}), (uint32_t)({b}))' + def service(op, a="0U", b="0U"): + return f"dkr_legacy_memory_service(rdram, ctx, {op}U, {rev}U, (uint32_t)({a}), (uint32_t)({b}))" def insert(key, address, entry): matches = [x for x in result.setdefault(key, []) if int(x[address], 0) == int(entry[address], 0)] if matches: if matches != [entry]: - raise ValueError('Conflicting memory safety patch '+entry[address]) + raise ValueError("Conflicting memory safety patch " + entry[address]) else: result[key].append(entry) def hook(name, pc, expected, text, replacement=None): start, size = next(iter(symbols[name])) - if not start <= pc < start+size or words.get(pc) != expected: - raise ValueError('Memory safety instruction changed '+name+' '+hex(pc)) - existing = [x for x in result.get('instructionPatches', []) if int(x['vram'], 0) == pc] - paired = dict(function=name,vram=hex(pc),value=hex(replacement),reason=reason) if replacement is not None else None + if not start <= pc < start + size or words.get(pc) != expected: + raise ValueError("Memory safety instruction changed " + name + " " + hex(pc)) + existing = [x for x in result.get("instructionPatches", []) if int(x["vram"], 0) == pc] + paired = ( + dict(function=name, vram=hex(pc), value=hex(replacement), reason=reason) + if replacement is not None + else None + ) if existing and existing != [paired]: - raise ValueError('Conflicting memory safety instruction '+hex(pc)) - insert('functionHooks', 'beforeVram', dict(function=name, beforeVram=hex(pc), - text=declaration+text, reason=reason)) + raise ValueError("Conflicting memory safety instruction " + hex(pc)) + insert( + "functionHooks", + "beforeVram", + dict(function=name, beforeVram=hex(pc), text=declaration + text, reason=reason), + ) if replacement is not None: - insert('instructionPatches', 'vram', dict(function=name, vram=hex(pc), value=hex(replacement), reason=reason)) + insert( + "instructionPatches", "vram", dict(function=name, vram=hex(pc), value=hex(replacement), reason=reason) + ) def memhook(name, pc, expected, text, replacement=None): - hook(name, pc+memory_delta, expected, text, replacement) + hook(name, pc + memory_delta, expected, text, replacement) # Clear the descriptor before the pool is made; stock remains 1600 slots. - memhook('mempool_init_main', 0x80070b58, 0x24060640, - 'ctx->r6 = '+service(0)+';', 0) - memhook('mempool_init_main', 0x80070b5c, 0x0c01c42c if rev == 77 else 0x0c01c4bc, - '{ recomp_context dkr_alloc = *ctx; dkr_alloc.f_odd = dkr_alloc.mips3_float_mode ? &dkr_alloc.f1.u32l : &dkr_alloc.f0.u32h; ' - 'dkr_alloc.r4 = '+service(1)+'; dkr_alloc.r5 = (int32_t)0xffff00ffU; ' - 'mempool_alloc(rdram, &dkr_alloc); '+service(2, 'dkr_alloc.r2')+'; }') + memhook("mempool_init_main", 0x80070B58, 0x24060640, "ctx->r6 = " + service(0) + ";", 0) + memhook( + "mempool_init_main", + 0x80070B5C, + 0x0C01C42C if rev == 77 else 0x0C01C4BC, + "{ recomp_context dkr_alloc = *ctx; dkr_alloc.f_odd = dkr_alloc.mips3_float_mode ? &dkr_alloc.f1.u32l : &dkr_alloc.f0.u32h; " + "dkr_alloc.r4 = " + service(1) + "; dkr_alloc.r5 = (int32_t)0xffff00ffU; " + "mempool_alloc(rdram, &dkr_alloc); " + service(2, "dkr_alloc.r2") + "; }", + ) # Replace enqueue at its leaf entry, before ANY original store. Draining # still uses the retail timers/order and real mempool_free_addr owner. - memhook('mempool_free_queue', 0x80071440, 0x3c058012, - service(4, 'ctx->r4')+'; return;') + memhook("mempool_free_queue", 0x80071440, 0x3C058012, service(4, "ctx->r4") + "; return;") for name, pc, word, register in [ - ('mempool_free_queue_clear', 0x800711cc, 0x265235c8 if rev == 77 else 0x26523b48, 18), - ('mempool_free_queue_clear', 0x800711d0, 0x261035c8 if rev == 77 else 0x26103b48, 16), - ('mempool_free_queue_clear', 0x8007121c, 0x258c35c8 if rev == 77 else 0x258c3b48, 12), - ('mempool_free_queue_clear', 0x80071238, 0x25ce35c8 if rev == 77 else 0x25ce3b48, 14), - ('mempool_free_timer', 0x800710f0, 0x265235c8 if rev == 77 else 0x26523b48, 18), + ("mempool_free_queue_clear", 0x800711CC, 0x265235C8 if rev == 77 else 0x26523B48, 18), + ("mempool_free_queue_clear", 0x800711D0, 0x261035C8 if rev == 77 else 0x26103B48, 16), + ("mempool_free_queue_clear", 0x8007121C, 0x258C35C8 if rev == 77 else 0x258C3B48, 12), + ("mempool_free_queue_clear", 0x80071238, 0x25CE35C8 if rev == 77 else 0x25CE3B48, 14), + ("mempool_free_timer", 0x800710F0, 0x265235C8 if rev == 77 else 0x26523B48, 18), ]: # Use symbol-relative verification for the differing static queue base. # v80's BSS base differs by 0x580, unlike its code offset (0x240). - expected = (word & 0xffff0000) | (symbol('gFreeQueue') & 0xffff) - memhook(name, pc, expected, f'ctx->r{register} = (int32_t)'+service(3)+';', 0) - memhook('mempool_free_queue_clear', 0x80071198, 0x27bdffd0, service(3)+';') - memhook('mempool_free_timer', 0x800710b0, 0x27bdffd0, service(3)+';') - memhook('mempool_free_addr', 0x80071278, 0x27bdffe0, - 'if ('+service(6, 'ctx->r4')+') return; unsigned dkr_free_visits = 0; unsigned dkr_free_max = 0;') - memhook('mempool_free_addr', 0x80071288, 0x00027100, - 'dkr_free_max = '+service(7, 'ctx->r2')+';') - memhook('mempool_free_addr', 0x800712ac, 0x00a60019, - 'if ((uint32_t)ctx->r5 >= dkr_free_max || ++dkr_free_visits > dkr_free_max) '+service(11, 'ctx->r4', 'ctx->r31')+';') + expected = (word & 0xFFFF0000) | (symbol("gFreeQueue") & 0xFFFF) + memhook(name, pc, expected, f"ctx->r{register} = (int32_t)" + service(3) + ";", 0) + memhook("mempool_free_queue_clear", 0x80071198, 0x27BDFFD0, service(3) + ";") + memhook("mempool_free_timer", 0x800710B0, 0x27BDFFD0, service(3) + ";") + memhook( + "mempool_free_addr", + 0x80071278, + 0x27BDFFE0, + "if (" + service(6, "ctx->r4") + ") return; unsigned dkr_free_visits = 0; unsigned dkr_free_max = 0;", + ) + memhook("mempool_free_addr", 0x80071288, 0x00027100, "dkr_free_max = " + service(7, "ctx->r2") + ";") + memhook( + "mempool_free_addr", + 0x800712AC, + 0x00A60019, + "if ((uint32_t)ctx->r5 >= dkr_free_max || ++dkr_free_visits > dkr_free_max) " + + service(11, "ctx->r4", "ctx->r31") + + ";", + ) - memhook('mempool_slot_find', 0x80070d3c, 0x27bdffc0, - 'unsigned dkr_find_visits = 0; unsigned dkr_find_max = '+service(7, 'ctx->r4')+'; ' - 'if ((int32_t)ctx->r5 <= 0 || (uint32_t)ctx->r5 > 0x7ffffff0U) '+service(11, 'ctx->r5', 'ctx->r31')+';') - memhook('mempool_slot_find', 0x80070dc8, 0x00880019, - 'if ((uint32_t)ctx->r4 >= dkr_find_max || ++dkr_find_visits > dkr_find_max) '+service(11, 'ctx->r4', 'ctx->r31')+';') - for name, pc, word in [('mempool_slot_assign', 0x8007178c, 0x240b0014), ('mempool_slot_clear', 0x8007164c, 0x240a0014)]: - memhook(name, pc, word, service(8, 'ctx->r4', 'ctx->r5')+';') - memhook('mempool_alloc_safe', 0x80070cdc, 0x14400008, - 'if (!ctx->r2) '+service(10, 'MEM_W(0x20, ctx->r29)', 'MEM_W(0x14, ctx->r29)')+';') + memhook( + "mempool_slot_find", + 0x80070D3C, + 0x27BDFFC0, + "unsigned dkr_find_visits = 0; unsigned dkr_find_max = " + service(7, "ctx->r4") + "; " + "if ((int32_t)ctx->r5 <= 0 || (uint32_t)ctx->r5 > 0x7ffffff0U) " + service(11, "ctx->r5", "ctx->r31") + ";", + ) + memhook( + "mempool_slot_find", + 0x80070DC8, + 0x00880019, + "if ((uint32_t)ctx->r4 >= dkr_find_max || ++dkr_find_visits > dkr_find_max) " + + service(11, "ctx->r4", "ctx->r31") + + ";", + ) + for name, pc, word in [ + ("mempool_slot_assign", 0x8007178C, 0x240B0014), + ("mempool_slot_clear", 0x8007164C, 0x240A0014), + ]: + memhook(name, pc, word, service(8, "ctx->r4", "ctx->r5") + ";") + memhook( + "mempool_alloc_safe", + 0x80070CDC, + 0x14400008, + "if (!ctx->r2) " + service(10, "MEM_W(0x20, ctx->r29)", "MEM_W(0x14, ctx->r29)") + ";", + ) # Optional transition allocations get a permanent three-buffer reserve. # If reserve/optional allocation fail, retain transition_begin's colour, # duration, inversion and end timer, but use the nonallocating full fade. for name, alloc_pc, next_pc, next_word, frame, arg, saves in [ - ('transition_init_shape', 0x800c0b64, 0x800c0b6c, 0x8fa30030, 0x50, 0x50, [(16,0x18),(17,0x1c),(18,0x20),(31,0x24)]), - ('transition_init_circle', 0x800c1608, 0x800c1610, 0x3c04800e, 0x38, 0x38, [(i,0x1c+4*(i-16)) for i in range(16,22)]+[(31,0x34)]), + ( + "transition_init_shape", + 0x800C0B64, + 0x800C0B6C, + 0x8FA30030, + 0x50, + 0x50, + [(16, 0x18), (17, 0x1C), (18, 0x20), (31, 0x24)], + ), + ( + "transition_init_circle", + 0x800C1608, + 0x800C1610, + 0x3C04800E, + 0x38, + 0x38, + [(i, 0x1C + 4 * (i - 16)) for i in range(16, 22)] + [(31, 0x34)], + ), ]: - alloc_pc += transition_delta;next_pc += transition_delta - expected = 0x0c000000 | ((symbol('mempool_alloc_safe') >> 2) & 0x3ffffff) + alloc_pc += transition_delta + next_pc += transition_delta + expected = 0x0C000000 | ((symbol("mempool_alloc_safe") >> 2) & 0x3FFFFFF) # Leave the original delay instruction intact. start, size = next(iter(symbols[name])) - if words.get(alloc_pc) != expected or not start <= alloc_pc < start+size: - raise ValueError('Transition allocation changed') - insert('instructionPatches','vram',dict(function=name,vram=hex(alloc_pc),value='0x0',reason=reason)) - clear = ''.join(f'MEM_W(0, (int32_t)0x{symbol(s)+offset:08x}U) = 0; ' for s,offset in - [('sTransitionVtx',0),('sTransitionVtx',4),('sTransitionTris',0),('sTransitionTris',4), - ('gTransitionNextVtx',0),('gTransitionVtxStep',0),('gTransitionVertexTarget',0),('gCurFadeTransition',0)]) - restore = ''.join(f'ctx->r{reg} = MEM_W(0x{off:x}, ctx->r29); ' for reg,off in saves) - if name == 'transition_init_circle': - restore += 'ctx->f_odd[(21 - 1) * 2] = MEM_W(0x10, ctx->r29); ctx->f20.u32l = MEM_W(0x14, ctx->r29); ' - text = '{ unsigned dkr_request = (uint32_t)ctx->r4; uint32_t dkr_reserved = '+service(5, 'dkr_request')+'; ' - text += 'recomp_context dkr_alloc = *ctx; dkr_alloc.f_odd = dkr_alloc.mips3_float_mode ? &dkr_alloc.f1.u32l : &dkr_alloc.f0.u32h; ' - text += 'if (!dkr_reserved) { mempool_alloc(rdram, &dkr_alloc); dkr_reserved = (uint32_t)dkr_alloc.r2; } ctx->r2 = (int32_t)dkr_reserved; ' - text += 'if (!dkr_reserved) { '+service(9, 'dkr_request', 'MEM_W(0x'+hex(saves[-1][1])[2:]+', ctx->r29)')+'; '+clear - text += f'dkr_alloc.r4 = MEM_W(0x{arg:x}, ctx->r29); transition_fullscreen_start(rdram, &dkr_alloc); ' - text += restore+f'ctx->r29 = ADD32(ctx->r29, 0x{frame:x}); return; '+'} }' - hook(name,next_pc,next_word,text) + if words.get(alloc_pc) != expected or not start <= alloc_pc < start + size: + raise ValueError("Transition allocation changed") + insert("instructionPatches", "vram", dict(function=name, vram=hex(alloc_pc), value="0x0", reason=reason)) + clear = "".join( + f"MEM_W(0, (int32_t)0x{symbol(s) + offset:08x}U) = 0; " + for s, offset in [ + ("sTransitionVtx", 0), + ("sTransitionVtx", 4), + ("sTransitionTris", 0), + ("sTransitionTris", 4), + ("gTransitionNextVtx", 0), + ("gTransitionVtxStep", 0), + ("gTransitionVertexTarget", 0), + ("gCurFadeTransition", 0), + ] + ) + restore = "".join(f"ctx->r{reg} = MEM_W(0x{off:x}, ctx->r29); " for reg, off in saves) + if name == "transition_init_circle": + restore += "ctx->f_odd[(21 - 1) * 2] = MEM_W(0x10, ctx->r29); ctx->f20.u32l = MEM_W(0x14, ctx->r29); " + text = "{ unsigned dkr_request = (uint32_t)ctx->r4; uint32_t dkr_reserved = " + service(5, "dkr_request") + "; " + text += "recomp_context dkr_alloc = *ctx; dkr_alloc.f_odd = dkr_alloc.mips3_float_mode ? &dkr_alloc.f1.u32l : &dkr_alloc.f0.u32h; " + text += "if (!dkr_reserved) { mempool_alloc(rdram, &dkr_alloc); dkr_reserved = (uint32_t)dkr_alloc.r2; } ctx->r2 = (int32_t)dkr_reserved; " + text += ( + "if (!dkr_reserved) { " + + service(9, "dkr_request", "MEM_W(0x" + hex(saves[-1][1])[2:] + ", ctx->r29)") + + "; " + + clear + ) + text += f"dkr_alloc.r4 = MEM_W(0x{arg:x}, ctx->r29); transition_fullscreen_start(rdram, &dkr_alloc); " + text += restore + f"ctx->r29 = ADD32(ctx->r29, 0x{frame:x}); return; " + "} }" + hook(name, next_pc, next_word, text) return result diff --git a/scripts/legacy_model_cache_policy.py b/scripts/legacy_model_cache_policy.py index 53416c19..fd2f3769 100644 --- a/scripts/legacy_model_cache_policy.py +++ b/scripts/legacy_model_cache_policy.py @@ -3,58 +3,81 @@ Project Patch Pipeline only: native model loading and successful cache entries remain untouched. Verified against the same reviewed retail ELF images as portraits. """ + import copy import struct + from legacy_character_presentation_policy import reviewed_revision def compose_model_cache(policy, elf, revision, sections, symbols): if reviewed_revision(elf) != revision: - raise ValueError('Model cache requires a reviewed retail ELF') + raise ValueError("Model cache requires a reviewed retail ELF") result = copy.deepcopy(policy) - words = {base+i: struct.unpack_from('>I', data, i)[0] - for base, data in sections for i in range(0, len(data), 4)} + words = {base + i: struct.unpack_from(">I", data, i)[0] for base, data in sections for i in range(0, len(data), 4)} def one(name): matches = symbols.get(name, set()) if len(matches) != 1: - raise ValueError('Ambiguous model-cache symbol: '+name) + raise ValueError("Ambiguous model-cache symbol: " + name) return next(iter(matches)) def hook(name, pc, expected, text): start, size = one(name) - if not start <= pc < start+size or words.get(pc) != expected: - raise ValueError('Model-cache instruction changed: '+name) - if any(int(h['beforeVram'], 0) == pc for h in result['functionHooks']) or any( - int(p['vram'], 0) == pc for p in result.get('instructionPatches', [])): - raise ValueError('Conflicting model-cache patch') - result['functionHooks'].append(dict(function=name, beforeVram=hex(pc), text=text, - reason='Failed model loads must not leave a phantom cache entry; preserve successful native allocations and ownership.')) + if not start <= pc < start + size or words.get(pc) != expected: + raise ValueError("Model-cache instruction changed: " + name) + if any(int(h["beforeVram"], 0) == pc for h in result["functionHooks"]) or any( + int(p["vram"], 0) == pc for p in result.get("instructionPatches", []) + ): + raise ValueError("Conflicting model-cache patch") + result["functionHooks"].append( + dict( + function=name, + beforeVram=hex(pc), + text=text, + reason="Failed model loads must not leave a phantom cache entry; preserve successful native allocations and ownership.", + ) + ) # Retail logs a warning then dereferences NULL. Callers already handle a # failed instance; propagate NULL without a host access violation instead. - hook('model_instance_init', one('model_instance_init')[0], 0x27bdffe0, - 'if (ctx->r4 == 0) { ctx->r2 = 0; return; }') - heap = 0x80070b50 if revision=='us.v77' else 0x80070d90 - hook('mempool_init_main', heap, 0x01e42823, - 'extern void dkr_legacy_heap_capacity(uint8_t*, recomp_context*); dkr_legacy_heap_capacity(rdram, ctx);') + hook("model_instance_init", one("model_instance_init")[0], 0x27BDFFE0, "if (ctx->r4 == 0) { ctx->r2 = 0; return; }") + heap = 0x80070B50 if revision == "us.v77" else 0x80070D90 + hook( + "mempool_init_main", + heap, + 0x01E42823, + "extern void dkr_legacy_heap_capacity(uint8_t*, recomp_context*); dkr_legacy_heap_capacity(rdram, ctx);", + ) # The retail HUD silently returns when a texture/sprite/model cannot load. # Observe only that failed-load branch; do not change its draw state. - hud_failure = 0x800aa7ac if revision == 'us.v77' else 0x800aad08 - hook('hud_element_render', hud_failure, 0, - 'if (ctx->r15 == 0) { extern void dkr_hud_asset_load_failed(uint8_t*, recomp_context*); dkr_hud_asset_load_failed(rdram, ctx); }') - if revision == 'us.v77': - count, count_size = one('gModelCacheCount') - free, free_size = one('D_8011D634') + hud_failure = 0x800AA7AC if revision == "us.v77" else 0x800AAD08 + hook( + "hud_element_render", + hud_failure, + 0, + "if (ctx->r15 == 0) { extern void dkr_hud_asset_load_failed(uint8_t*, recomp_context*); dkr_hud_asset_load_failed(rdram, ctx); }", + ) + if revision == "us.v77": + count, count_size = one("gModelCacheCount") + free, free_size = one("D_8011D634") if count_size != 4 or free_size != 4: - raise ValueError('Model-cache counter sizes changed') - hook('object_model_init', 0x8005f99c, 0x27bdffa8, - f'int32_t dkr_model_count_before = MEM_W(0, (int32_t)0x{count:08x}U); ' - f'int32_t dkr_model_free_before = MEM_W(0, (int32_t)0x{free:08x}U);') + raise ValueError("Model-cache counter sizes changed") + hook( + "object_model_init", + 0x8005F99C, + 0x27BDFFA8, + f"int32_t dkr_model_count_before = MEM_W(0, (int32_t)0x{count:08x}U); " + f"int32_t dkr_model_free_before = MEM_W(0, (int32_t)0x{free:08x}U);", + ) # Includes model allocation, texture, normals, animation and instance # failures. Cache hits do not change these counters. Snapshot locals # are invocation-owned, not shared state across guest threads/calls. - hook('object_model_init', 0x8005fcb4, 0x8fbf0024, - f'if (ctx->r2 == 0) {{ MEM_W(0, (int32_t)0x{count:08x}U) = dkr_model_count_before; ' - f'MEM_W(0, (int32_t)0x{free:08x}U) = dkr_model_free_before; }}') + hook( + "object_model_init", + 0x8005FCB4, + 0x8FBF0024, + f"if (ctx->r2 == 0) {{ MEM_W(0, (int32_t)0x{count:08x}U) = dkr_model_count_before; " + f"MEM_W(0, (int32_t)0x{free:08x}U) = dkr_model_free_before; }}", + ) return result diff --git a/scripts/legacy_model_reference_policy.py b/scripts/legacy_model_reference_policy.py index 07955b01..b9e36dba 100644 --- a/scripts/legacy_model_reference_policy.py +++ b/scripts/legacy_model_reference_policy.py @@ -3,47 +3,65 @@ This layer preserves valid retail loaders and shading. Corrupt allocations are never guessed/rebound; their native owner reports the context and ends safely. """ + import copy import struct + +from compose_legacy_mod_policy import elf_functions, elf_sections from legacy_character_presentation_policy import reviewed_revision -from compose_legacy_mod_policy import elf_sections, elf_functions def compose_model_references(policy, elf, revision): - if reviewed_revision(elf)!=revision: - raise ValueError('Model reference fences require a reviewed retail ELF') - result=copy.deepcopy(policy) - words={base+i:struct.unpack_from('>I',data,i)[0] for base,data in elf_sections(elf) - for i in range(0,len(data),4)} - symbols=elf_functions(elf) - declaration='extern int dkr_legacy_model_safety(uint8_t*, recomp_context*, unsigned, uint32_t); ' + if reviewed_revision(elf) != revision: + raise ValueError("Model reference fences require a reviewed retail ELF") + result = copy.deepcopy(policy) + words = { + base + i: struct.unpack_from(">I", data, i)[0] + for base, data in elf_sections(elf) + for i in range(0, len(data), 4) + } + symbols = elf_functions(elf) + declaration = "extern int dkr_legacy_model_safety(uint8_t*, recomp_context*, unsigned, uint32_t); " - def hook(name,pc,word,text): - matches=symbols.get(name,set()) - if len(matches)!=1:raise ValueError('Ambiguous model reference function '+name) - begin,size=next(iter(matches)) - if not begin<=pcr4)) { MEM_W(0x54, ctx->r4) = 0; ctx->r2 = 4; return; }') + hook( + "init_object_shading", + 0x8000F7EC, + 0x27BDFFD8, + "if (dkr_legacy_model_safety(rdram, ctx, 0U, (uint32_t)ctx->r4)) { MEM_W(0x54, ctx->r4) = 0; ctx->r2 = 4; return; }", + ) # Observe every loaded slot BEFORE behaviour initialization can obscure # where a reference was damaged. Slots intentionally skipped stay NULL. - hook('spawn_object',0x8000f0ec,0x8e480040, - 'dkr_legacy_model_safety(rdram, ctx, 1U, (uint32_t)ctx->r18);') - instance=0x8005fcd0 if revision=='us.v77' else 0x8005ff10 - hook('model_instance_init',instance+8,0x848e0048, - 'dkr_legacy_model_safety(rdram, ctx, 2U, (uint32_t)ctx->r4);') - hook('model_instance_init',instance+0x260,0x8fbf0014, - 'if (ctx->r2) dkr_legacy_model_safety(rdram, ctx, 3U, (uint32_t)ctx->r2);') + hook("spawn_object", 0x8000F0EC, 0x8E480040, "dkr_legacy_model_safety(rdram, ctx, 1U, (uint32_t)ctx->r18);") + instance = 0x8005FCD0 if revision == "us.v77" else 0x8005FF10 + hook("model_instance_init", instance + 8, 0x848E0048, "dkr_legacy_model_safety(rdram, ctx, 2U, (uint32_t)ctx->r4);") + hook( + "model_instance_init", + instance + 0x260, + 0x8FBF0014, + "if (ctx->r2) dkr_legacy_model_safety(rdram, ctx, 3U, (uint32_t)ctx->r2);", + ) return result diff --git a/scripts/legacy_spawn_asset_type_policy.py b/scripts/legacy_spawn_asset_type_policy.py index d409b13f..fc835382 100644 --- a/scripts/legacy_spawn_asset_type_policy.py +++ b/scripts/legacy_spawn_asset_type_policy.py @@ -3,34 +3,42 @@ Only the reviewed rocket-signpost-2 branch is changed. Trophy progress, model index and sparse model slots remain retail; no saves or resource IDs change. """ + import copy import struct + +from compose_legacy_mod_policy import elf_functions, elf_sections from legacy_character_presentation_policy import reviewed_revision -from compose_legacy_mod_policy import elf_sections, elf_functions def compose_spawn_asset_type(policy, elf, revision): if reviewed_revision(elf) != revision: - raise ValueError('Spawn asset-type correction requires a reviewed retail ELF') - pc = 0x8000ed8c - symbols = elf_functions(elf).get('spawn_object', set()) + raise ValueError("Spawn asset-type correction requires a reviewed retail ELF") + pc = 0x8000ED8C + symbols = elf_functions(elf).get("spawn_object", set()) if len(symbols) != 1: - raise ValueError('Ambiguous spawn asset-type function') + raise ValueError("Ambiguous spawn asset-type function") begin, size = next(iter(symbols)) - words = {base+i: struct.unpack_from('>I', data, i)[0] - for base, data in elf_sections(elf) for i in range(0, len(data), 4)} - if not begin <= pc < begin+size or words.get(pc) != 0xa246003a: - raise ValueError('Spawn trophy-count boundary changed') + words = { + base + i: struct.unpack_from(">I", data, i)[0] + for base, data in elf_sections(elf) + for i in range(0, len(data), 4) + } + if not begin <= pc < begin + size or words.get(pc) != 0xA246003A: + raise ValueError("Spawn trophy-count boundary changed") result = copy.deepcopy(policy) - entry = dict(function='spawn_object', beforeVram=hex(pc), - text='MEM_W(0x64, ctx->r29) = MEM_B(0x53, MEM_W(0x40, ctx->r18));', - reason='After counting the first four world trophies, restore header modelType; fifth-world trophy bits must not choose a Sprite loader for a 3D model.') - owners = [h for h in result.setdefault('functionHooks', []) if int(h['beforeVram'], 0) == pc] - if any(int(p['vram'], 0) == pc for p in result.get('instructionPatches', [])): - raise ValueError('Conflicting spawn asset-type instruction') + entry = dict( + function="spawn_object", + beforeVram=hex(pc), + text="MEM_W(0x64, ctx->r29) = MEM_B(0x53, MEM_W(0x40, ctx->r18));", + reason="After counting the first four world trophies, restore header modelType; fifth-world trophy bits must not choose a Sprite loader for a 3D model.", + ) + owners = [h for h in result.setdefault("functionHooks", []) if int(h["beforeVram"], 0) == pc] + if any(int(p["vram"], 0) == pc for p in result.get("instructionPatches", [])): + raise ValueError("Conflicting spawn asset-type instruction") if owners: if owners != [entry]: - raise ValueError('Conflicting spawn asset-type hook') + raise ValueError("Conflicting spawn asset-type hook") else: - result['functionHooks'].append(entry) + result["functionHooks"].append(entry) return result diff --git a/scripts/macos_build_lock.py b/scripts/macos_build_lock.py index 3dc0b035..27787ea1 100644 --- a/scripts/macos_build_lock.py +++ b/scripts/macos_build_lock.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Hold a checkout lock across the whole build and its nested prepare scripts.""" + import fcntl import os from pathlib import Path @@ -8,11 +9,11 @@ def main(): root, script, *args = sys.argv[1:] - path = Path(root).resolve() / 'build' / '.macos-build.lock' + path = Path(root).resolve() / "build" / ".macos-build.lock" path.parent.mkdir(parents=True, exist_ok=True) fd = None try: - inherited = int(os.environ.get('DKR_MACOS_LOCK_FD', '-1')) + inherited = int(os.environ.get("DKR_MACOS_LOCK_FD", "-1")) if os.path.samestat(os.fstat(inherited), path.stat()): fd = inherited except (ValueError, OSError): @@ -22,12 +23,15 @@ def main(): try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: - print(f'Another macOS build/prepare is using {root}. Wait for it to finish; shared extern/, runtime-tools and generated files are locked.', file=sys.stderr) + print( + f"Another macOS build/prepare is using {root}. Wait for it to finish; shared extern/, runtime-tools and generated files are locked.", + file=sys.stderr, + ) return 1 os.set_inheritable(fd, True) env = dict(os.environ, DKR_MACOS_LOCK_FD=str(fd), DKR_MACOS_LOCK_ENTRY=script) - os.execvpe('/bin/bash', ['/bin/bash', script, *args], env) + os.execvpe("/bin/bash", ["/bin/bash", script, *args], env) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) diff --git a/scripts/magic_code_authoring_kit.py b/scripts/magic_code_authoring_kit.py index 4f3727c0..c6f83bae 100644 --- a/scripts/magic_code_authoring_kit.py +++ b/scripts/magic_code_authoring_kit.py @@ -4,7 +4,9 @@ No ROM bytes or original implementations are distributed. Inputs are read-only; output is always a new directory. The receipt pins the full ELF and symbol list. """ + from __future__ import annotations + import argparse import hashlib import json @@ -14,62 +16,94 @@ def symbols(path: Path): - data=path.read_bytes() - if data[:6]!=b'\x7fELF\x01\x02':raise ValueError('Expected big-endian ELF32') - start=struct.unpack_from('>I',data,32)[0] - stride,count=struct.unpack_from('>HH',data,46) - if stride!=40 or count>4096 or start+stride*count>len(data):raise ValueError('Invalid ELF directory') - headers=[struct.unpack_from('>10I',data,start+i*stride) for i in range(count)] - found={} + data = path.read_bytes() + if data[:6] != b"\x7fELF\x01\x02": + raise ValueError("Expected big-endian ELF32") + start = struct.unpack_from(">I", data, 32)[0] + stride, count = struct.unpack_from(">HH", data, 46) + if stride != 40 or count > 4096 or start + stride * count > len(data): + raise ValueError("Invalid ELF directory") + headers = [struct.unpack_from(">10I", data, start + i * stride) for i in range(count)] + found = {} for h in headers: - if h[1]!=2:continue - if h[9]!=16 or h[5]%16 or h[4]+h[5]>len(data) or h[6]>=count:raise ValueError('Invalid symbols') - strings=headers[h[6]];names=data[strings[4]:strings[4]+strings[5]] - for at in range(h[4],h[4]+h[5],16): - name,address,size,info,_,section=struct.unpack_from('>IIIBBH',data,at) - kind=info&15 - if kind not in (1,2) or not size or not 0x80000400<=address<0x80800000 or size>0x80800000-address:continue - if section>=count or not(headers[section][2]&2):continue - end=names.find(b'\0',name) - if end len(data) or h[6] >= count: + raise ValueError("Invalid symbols") + strings = headers[h[6]] + names = data[strings[4] : strings[4] + strings[5]] + for at in range(h[4], h[4] + h[5], 16): + name, address, size, info, _, section = struct.unpack_from(">IIIBBH", data, at) + kind = info & 15 + if kind not in (1, 2) or not size or not 0x80000400 <= address < 0x80800000 or size > 0x80800000 - address: + continue + if section >= count or not (headers[section][2] & 2): + continue + end = names.find(b"\0", name) + if end < name: + raise ValueError("Unterminated symbol") + label = names[name:end].decode("ascii") + if not re.fullmatch(r"[A-Za-z_]\w*", label): + continue + entry = (address, size, kind) + if label in found and found[label] != entry: + raise ValueError("Ambiguous symbol " + label) + found[label] = entry + if len(found) < 1000: + raise ValueError("Unexpectedly incomplete DKR symbol table") return found def main(): - p=argparse.ArgumentParser(description=__doc__) - p.add_argument('--elf',type=Path,required=True) - p.add_argument('--revision',type=int,choices=(77,80),required=True) - p.add_argument('--output',type=Path,required=True) - a=p.parse_args();out=a.output.resolve() - if out.exists() or out in a.elf.resolve().parents:raise ValueError('A NEW output directory is required') - values=symbols(a.elf);out.mkdir(parents=True) - rows=['// Generated address-only SDK bindings; see receipt.json.'] - rows += [f'{{"{name}",0x{address:08X}U,{size}U,{kind==2 and "true" or "false"}}},' - for name,(address,size,kind) in sorted(values.items())] - (out/'game_symbols.inc').write_text('\n'.join(rows)+'\n',encoding='utf-8') - funcs=['[[section]]','name = ".main"','rom = 0x1000','vram = 0x80000400','size = 0x7FFC00','functions = ['] - funcs += [f'{{ name = "{name}", vram = 0x{address:08X}, size = 0x{size:X} }},' - for name,(address,size,kind) in sorted(values.items()) if kind==2] - funcs += [']'] - (out/'functions.toml').write_text('\n'.join(funcs)+'\n',encoding='utf-8') - objects=['[[section]]','name = ".main"','rom = 0x1000','vram = 0x80000400','size = 0x7FFC00','symbols = ['] - objects += [f'{{ name = "{name}", vram = 0x{address:08X} }},' - for name,(address,_,kind) in sorted(values.items()) if kind==1] - (out/'data.toml').write_text('\n'.join(objects+["]"])+'\n',encoding='utf-8') + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--elf", type=Path, required=True) + p.add_argument("--revision", type=int, choices=(77, 80), required=True) + p.add_argument("--output", type=Path, required=True) + a = p.parse_args() + out = a.output.resolve() + if out.exists() or out in a.elf.resolve().parents: + raise ValueError("A NEW output directory is required") + values = symbols(a.elf) + out.mkdir(parents=True) + rows = ["// Generated address-only SDK bindings; see receipt.json."] + rows += [ + f'{{"{name}",0x{address:08X}U,{size}U,{kind == 2 and "true" or "false"}}},' + for name, (address, size, kind) in sorted(values.items()) + ] + (out / "game_symbols.inc").write_text("\n".join(rows) + "\n", encoding="utf-8") + funcs = ["[[section]]", 'name = ".main"', "rom = 0x1000", "vram = 0x80000400", "size = 0x7FFC00", "functions = ["] + funcs += [ + f'{{ name = "{name}", vram = 0x{address:08X}, size = 0x{size:X} }},' + for name, (address, size, kind) in sorted(values.items()) + if kind == 2 + ] + funcs += ["]"] + (out / "functions.toml").write_text("\n".join(funcs) + "\n", encoding="utf-8") + objects = ["[[section]]", 'name = ".main"', "rom = 0x1000", "vram = 0x80000400", "size = 0x7FFC00", "symbols = ["] + objects += [ + f'{{ name = "{name}", vram = 0x{address:08X} }},' + for name, (address, _, kind) in sorted(values.items()) + if kind == 1 + ] + (out / "data.toml").write_text("\n".join(objects + ["]"]) + "\n", encoding="utf-8") # Data may be absolute addresses in this non-relocatable retail namespace. # The explicit revision declaration is mandatory for raw-address packages. - (out/'symbols.ld').write_text('\n'.join(f'{name} = 0x{address:08X};' - for name,(address,_,_) in sorted(values.items()))+'\n',encoding='utf-8') - receipt=dict(schema=1,revision=f'us.v{a.revision}',elf_sha256=hashlib.sha256(a.elf.read_bytes()).hexdigest(), - files={name:hashlib.sha256((out/name).read_bytes()).hexdigest() for name in ('game_symbols.inc','functions.toml','data.toml','symbols.ld')}) - (out/'receipt.json').write_text(json.dumps(receipt,indent=2)+'\n',encoding='utf-8') - print(f'Address-only SDK kit: {len(values)} symbols -> {out}') + (out / "symbols.ld").write_text( + "\n".join(f"{name} = 0x{address:08X};" for name, (address, _, _) in sorted(values.items())) + "\n", + encoding="utf-8", + ) + receipt = dict( + schema=1, + revision=f"us.v{a.revision}", + elf_sha256=hashlib.sha256(a.elf.read_bytes()).hexdigest(), + files={ + name: hashlib.sha256((out / name).read_bytes()).hexdigest() + for name in ("game_symbols.inc", "functions.toml", "data.toml", "symbols.ld") + }, + ) + (out / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8") + print(f"Address-only SDK kit: {len(values)} symbols -> {out}") -if __name__=='__main__':main() +if __name__ == "__main__": + main() diff --git a/scripts/magic_code_hook_inventory.py b/scripts/magic_code_hook_inventory.py index b30b591d..99eef469 100644 --- a/scripts/magic_code_hook_inventory.py +++ b/scripts/magic_code_hook_inventory.py @@ -4,68 +4,101 @@ Inputs are read-only. This is an inventory, not executable code or a claim that the private owned closure can dispatch every retail function. """ + from __future__ import annotations + import argparse import hashlib import json from pathlib import Path + from compose_legacy_mod_policy import elf_functions def inventory(elf: Path, receipt: dict) -> dict: - if (receipt.get('schema') != 1 or receipt.get('complete') is not True or - receipt.get('revision') not in ('us.v77', 'us.v80') or - receipt.get('elf_sha256') != hashlib.sha256(elf.read_bytes()).hexdigest()): - raise ValueError('Hook inventory requires the complete matching composed policy') - policy = receipt['policy'] + if ( + receipt.get("schema") != 1 + or receipt.get("complete") is not True + or receipt.get("revision") not in ("us.v77", "us.v80") + or receipt.get("elf_sha256") != hashlib.sha256(elf.read_bytes()).hexdigest() + ): + raise ValueError("Hook inventory requires the complete matching composed policy") + policy = receipt["policy"] symbols = elf_functions(elf) - for item in policy.get('manualFunctions', []): - symbols.setdefault(item['name'], set()).add((int(item['vram'], 0), int(item['size'], 0))) + for item in policy.get("manualFunctions", []): + symbols.setdefault(item["name"], set()).add((int(item["vram"], 0), int(item["size"], 0))) protected = {} - for key, name_key in (('functionHooks', 'function'), ('instructionPatches', 'function'), - ('stubs', 'name'), ('ignored', 'name'), ('renamed', 'name')): + for key, name_key in ( + ("functionHooks", "function"), + ("instructionPatches", "function"), + ("stubs", "name"), + ("ignored", "name"), + ("renamed", "name"), + ): for item in policy.get(key, []): protected.setdefault(item[name_key], set()).add(key) # Scheduler, saves, native IO and render ownership are not optional fixes. # Broad raw code cannot remove the runtime's lifetime/security invariants. - prefixes = ('os', '__os', '__sc', 'al', '_al', 'sndp_', 'sound_', 'music_', - 'mempool_', 'dmacopy', 'mainproc', 'thread', 'bgload_', 'rsp_', - 'fb_', 'input_', 'rumble_', 'contpak_', 'eeprom_', 'save_') + prefixes = ( + "os", + "__os", + "__sc", + "al", + "_al", + "sndp_", + "sound_", + "music_", + "mempool_", + "dmacopy", + "mainproc", + "thread", + "bgload_", + "rsp_", + "fb_", + "input_", + "rumble_", + "contpak_", + "eeprom_", + "save_", + ) functions = [] for name, definitions in sorted(symbols.items()): if len(definitions) != 1: - raise ValueError(f'Ambiguous retail function: {name}') + raise ValueError(f"Ambiguous retail function: {name}") address, size = next(iter(definitions)) reasons = set(protected.get(name, ())) if name.startswith(prefixes): - reasons.add('native-system-ownership') - if any(part in name.lower() for part in ('eeprom', 'mempak', 'mempool', 'controller_pak', 'flashrom')): - reasons.add('persistent-storage-ownership') - functions.append(dict(name=name, address=address, bytes=size, - protected=bool(reasons), reasons=sorted(reasons))) + reasons.add("native-system-ownership") + if any(part in name.lower() for part in ("eeprom", "mempak", "mempool", "controller_pak", "flashrom")): + reasons.add("persistent-storage-ownership") + functions.append(dict(name=name, address=address, bytes=size, protected=bool(reasons), reasons=sorted(reasons))) # Alias symbols at an identical entry must share the strictest grant. - protected_addresses = {f['address'] for f in functions if f['protected']} + protected_addresses = {f["address"] for f in functions if f["protected"]} for f in functions: - if f['address'] in protected_addresses and not f['protected']: - f.update(protected=True, reasons=['protected-symbol-alias']) - return dict(schema=1, revision=receipt['revision'], elf_sha256=receipt['elf_sha256'], - policy_sha256=hashlib.sha256(json.dumps(policy, sort_keys=True, separators=(',', ':')).encode()).hexdigest(), - functions=functions) + if f["address"] in protected_addresses and not f["protected"]: + f.update(protected=True, reasons=["protected-symbol-alias"]) + return dict( + schema=1, + revision=receipt["revision"], + elf_sha256=receipt["elf_sha256"], + policy_sha256=hashlib.sha256(json.dumps(policy, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), + functions=functions, + ) def main() -> int: p = argparse.ArgumentParser(description=__doc__) - p.add_argument('--elf', type=Path, required=True) - p.add_argument('--composed-policy', type=Path, required=True) - p.add_argument('--output', type=Path, required=True) + p.add_argument("--elf", type=Path, required=True) + p.add_argument("--composed-policy", type=Path, required=True) + p.add_argument("--output", type=Path, required=True) a = p.parse_args() if a.output.exists() or a.output.resolve() in (a.elf.resolve(), a.composed_policy.resolve()): - raise ValueError('Hook inventory output must be a NEW file') - result = inventory(a.elf, json.loads(a.composed_policy.read_text(encoding='utf-8'))) + raise ValueError("Hook inventory output must be a NEW file") + result = inventory(a.elf, json.loads(a.composed_policy.read_text(encoding="utf-8"))) a.output.parent.mkdir(parents=True, exist_ok=True) - a.output.write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8') + a.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") return 0 -if __name__ == '__main__': +if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/normalize-water-adapter.py b/scripts/normalize-water-adapter.py index 5d7f0bce..94c57d39 100644 --- a/scripts/normalize-water-adapter.py +++ b/scripts/normalize-water-adapter.py @@ -1,32 +1,43 @@ """Expand reviewed adapter hunks to full checked contexts (mechanical only).""" + import difflib from pathlib import Path import re import sys -root=Path(__file__).resolve().parents[1] -patch=root/"patches/enhanced-water"/(sys.argv[2] if len(sys.argv)>2 else "dkr-adapter.patch") -stage=Path(sys.argv[1]).resolve() -parts=[] -for section in re.split(r"(?=^diff --git )",patch.read_text(),flags=re.M): - match=re.match(r"diff --git a/(\S+) b/\S+",section) - if not match: continue - relative=match[1] - if relative.startswith("/") or ".." in Path(relative).parts: raise SystemExit("Invalid stage path") - before=(stage/relative).read_text() - after=before - for hunk in re.split(r"(?=^@@ )",section,flags=re.M)[1:]: - old=[];new=[] +root = Path(__file__).resolve().parents[1] +patch = root / "patches/enhanced-water" / (sys.argv[2] if len(sys.argv) > 2 else "dkr-adapter.patch") +stage = Path(sys.argv[1]).resolve() +parts = [] +for section in re.split(r"(?=^diff --git )", patch.read_text(), flags=re.M): + match = re.match(r"diff --git a/(\S+) b/\S+", section) + if not match: + continue + relative = match[1] + if relative.startswith("/") or ".." in Path(relative).parts: + raise SystemExit("Invalid stage path") + before = (stage / relative).read_text() + after = before + for hunk in re.split(r"(?=^@@ )", section, flags=re.M)[1:]: + old = [] + new = [] for line in hunk.splitlines(keepends=True)[1:]: - if line.startswith(" "): old.append(line[1:]);new.append(line[1:]) - elif line.startswith("-"): old.append(line[1:]) - elif line.startswith("+"): new.append(line[1:]) - old="".join(old);new="".join(new) - if not old or after.count(old)!=1: raise SystemExit(f"Ambiguous adapter hunk in {relative}: {old!r}") - after=after.replace(old,new,1) + if line.startswith(" "): + old.append(line[1:]) + new.append(line[1:]) + elif line.startswith("-"): + old.append(line[1:]) + elif line.startswith("+"): + new.append(line[1:]) + old = "".join(old) + new = "".join(new) + if not old or after.count(old) != 1: + raise SystemExit(f"Ambiguous adapter hunk in {relative}: {old!r}") + after = after.replace(old, new, 1) parts.append(f"diff --git a/{relative} b/{relative}\n") - for line in difflib.unified_diff(before.splitlines(True),after.splitlines(True), - fromfile=f"a/{relative}",tofile=f"b/{relative}",n=3): - parts.append(line if line.endswith("\n") else line+"\n\\ No newline at end of file\n") -patch.write_text("".join(parts),newline="\n") + for line in difflib.unified_diff( + before.splitlines(True), after.splitlines(True), fromfile=f"a/{relative}", tofile=f"b/{relative}", n=3 + ): + parts.append(line if line.endswith("\n") else line + "\n\\ No newline at end of file\n") +patch.write_text("".join(parts), newline="\n") print("Adapter hunks expanded with exact staged-source context.") diff --git a/scripts/package_ios.py b/scripts/package_ios.py index 0d2b467a..7beb9d92 100755 --- a/scripts/package_ios.py +++ b/scripts/package_ios.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 """Wrap the iOS device build as an ad-hoc-signed, sideloadable IPA.""" + from __future__ import annotations import json +from pathlib import Path import plistlib import re import shutil @@ -12,7 +14,6 @@ import tempfile import zipfile import zlib -from pathlib import Path import scan_for_game_assets as asset_scan @@ -33,9 +34,7 @@ def run(args: list[str], cwd: Path | None = None) -> str: result = subprocess.run(args, cwd=cwd, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if result.returncode != 0: - raise RuntimeError( - f"{' '.join(args)} failed ({result.returncode})\n{result.stdout}\n{result.stderr}" - ) + raise RuntimeError(f"{' '.join(args)} failed ({result.returncode})\n{result.stdout}\n{result.stderr}") return result.stdout @@ -146,16 +145,27 @@ def install_icon(app: Path, scratch: Path) -> None: ) partial = scratch / "ActoolPartial.plist" try: - run([ - "xcrun", "actool", str(scratch / "Assets.xcassets"), - "--compile", str(app), - "--platform", "iphoneos", - "--minimum-deployment-target", "18.0", - "--target-device", "iphone", - "--target-device", "ipad", - "--app-icon", "AppIcon", - "--output-partial-info-plist", str(partial), - ]) + run( + [ + "xcrun", + "actool", + str(scratch / "Assets.xcassets"), + "--compile", + str(app), + "--platform", + "iphoneos", + "--minimum-deployment-target", + "18.0", + "--target-device", + "iphone", + "--target-device", + "ipad", + "--app-icon", + "AppIcon", + "--output-partial-info-plist", + str(partial), + ] + ) except RuntimeError as exc: print(f"actool failed, using classic PNG icons ({exc})", file=sys.stderr) install_classic_icons(app, scratch, logo) @@ -203,10 +213,11 @@ def stage_assets(app: Path, build: Path) -> None: (ROOT / "assets/ui/Icons/DKR-R-Logo.bmp", app / "assets/ui/Icons/DKR-R-Logo.bmp"), (ROOT / "assets/ui/Icons/DKR-R-Spinning-Icon.png", app / "assets/ui/Icons/DKR-R-Spinning-Icon.png"), (ROOT / "assets/ui/Icons/DKR-R-Short-Logo.png", app / "assets/ui/Icons/DKR-R-Short-Logo.png"), - (ROOT / "assets/ui/Backgrounds/DKR-R-Launcher-Background.png", - app / "assets/ui/Backgrounds/DKR-R-Launcher-Background.png"), - (ROOT / "assets/controllers/gamecontrollerdb.txt", - app / "assets/controllers/gamecontrollerdb.txt"), + ( + ROOT / "assets/ui/Backgrounds/DKR-R-Launcher-Background.png", + app / "assets/ui/Backgrounds/DKR-R-Launcher-Background.png", + ), + (ROOT / "assets/controllers/gamecontrollerdb.txt", app / "assets/controllers/gamecontrollerdb.txt"), ] for source, dest in copies: dest.parent.mkdir(parents=True, exist_ok=True) @@ -240,15 +251,24 @@ def compile_launch_screen(app: Path) -> None: compiled = app / "LaunchScreen.storyboardc" if compiled.exists(): shutil.rmtree(compiled) - run([ - "xcrun", "ibtool", - "--errors", "--warnings", "--notices", - "--minimum-deployment-target", "18.0", - "--target-device", "iphone", - "--target-device", "ipad", - "--compile", str(compiled), - str(ROOT / "packaging/ios/LaunchScreen.storyboard"), - ]) + run( + [ + "xcrun", + "ibtool", + "--errors", + "--warnings", + "--notices", + "--minimum-deployment-target", + "18.0", + "--target-device", + "iphone", + "--target-device", + "ipad", + "--compile", + str(compiled), + str(ROOT / "packaging/ios/LaunchScreen.storyboard"), + ] + ) def require_plist(app: Path, version: str, bundle_version: str) -> None: @@ -279,8 +299,12 @@ def require_plist(app: Path, version: str, bundle_version: str) -> None: def sign_app(app: Path) -> None: entitlements = ROOT / "packaging/ios/entitlements.plist" command = [ - "codesign", "--force", "--sign", "-", - "--entitlements", str(entitlements), + "codesign", + "--force", + "--sign", + "-", + "--entitlements", + str(entitlements), "--timestamp=none", str(app), ] @@ -370,7 +394,9 @@ def write_ipa(app: Path, destination: Path) -> None: def release_number(version: str) -> str: - match = re.fullmatch(r"([1-9][0-9]{0,3})\.(0|[1-9][0-9]?)\.(0|[1-9][0-9]?)(?:-rc[1-9][0-9]*)?(?:\+[0-9A-Za-z.-]+)?", version) + match = re.fullmatch( + r"([1-9][0-9]{0,3})\.(0|[1-9][0-9]?)\.(0|[1-9][0-9]?)(?:-rc[1-9][0-9]*)?(?:\+[0-9A-Za-z.-]+)?", version + ) if match is None: raise RuntimeError(f"Invalid iOS release version: {version!r}; expected major.minor.patch[-rcN] (4/2/2 digits)") return ".".join(match.groups()) diff --git a/scripts/package_macos.py b/scripts/package_macos.py index b8491160..7b8abd2e 100755 --- a/scripts/package_macos.py +++ b/scripts/package_macos.py @@ -5,10 +5,10 @@ (DKR_MAC_SIGN_IDENTITY), notarizes with a notarytool keychain profile (DKR_MAC_NOTARY_PROFILE), staples, and re-zips. """ + import argparse import hashlib import json -import time import os from pathlib import Path import plistlib @@ -16,9 +16,10 @@ import shutil import subprocess import tempfile +import time import zipfile -import scan_for_game_assets as asset_scan +import scan_for_game_assets as asset_scan ROOT = Path(__file__).resolve().parents[1] @@ -40,8 +41,7 @@ def rpaths(path): def dependencies(path): - names = [line.strip().split(" (compatibility", 1)[0] - for line in run("otool", "-L", path).splitlines()[1:]] + names = [line.strip().split(" (compatibility", 1)[0] for line in run("otool", "-L", path).splitlines()[1:]] if path.suffix == ".dylib": names = names[1:] # LC_ID_DYLIB, not a dependency return names @@ -49,11 +49,15 @@ def dependencies(path): def resolve(name, source, executable): def expand(value): - return Path(value.replace("@loader_path", str(source.parent)) - .replace("@executable_path", str(executable.parent))) - candidates = ([expand(p) / name[len("@rpath/"):] - for p in rpaths(source) + rpaths(executable)] - if name.startswith("@rpath/") else [expand(name)]) + return Path( + value.replace("@loader_path", str(source.parent)).replace("@executable_path", str(executable.parent)) + ) + + candidates = ( + [expand(p) / name[len("@rpath/") :] for p in rpaths(source) + rpaths(executable)] + if name.startswith("@rpath/") + else [expand(name)] + ) for candidate in candidates: if candidate.is_absolute() and candidate.is_file(): return candidate.resolve() @@ -87,17 +91,30 @@ def archive(stage, destination): destination.unlink(missing_ok=True) run("ditto", "-c", "-k", "--norsrc", "--noextattr", "--noacl", "--keepParent", stage, destination) with zipfile.ZipFile(destination) as handle: - junk = [n for n in handle.namelist() - if re.search(r"(^|/)(\._|\.DS_Store$|__MACOSX/)", n)] + junk = [n for n in handle.namelist() if re.search(r"(^|/)(\._|\.DS_Store$|__MACOSX/)", n)] if junk: raise RuntimeError(f"Archive contains metadata files: {junk[:5]}") def notarize(submission, app, profile): # notarytool can exit non-zero for a rejected upload; read its JSON either way. - result = subprocess.run(["xcrun", "notarytool", "submit", str(submission), "--keychain-profile", - profile, "--wait", "--timeout", "1h", "--output-format", "json"], - capture_output=True, text=True) + result = subprocess.run( + [ + "xcrun", + "notarytool", + "submit", + str(submission), + "--keychain-profile", + profile, + "--wait", + "--timeout", + "1h", + "--output-format", + "json", + ], + capture_output=True, + text=True, + ) try: status = json.loads(result.stdout) except json.JSONDecodeError: @@ -105,13 +122,17 @@ def notarize(submission, app, profile): print(f"Notarization {status.get('id')}: {status.get('status')}") if status.get("status") != "Accepted": if status.get("id"): - print(subprocess.run(["xcrun", "notarytool", "log", status["id"], "--keychain-profile", profile], - capture_output=True, text=True).stdout) + print( + subprocess.run( + ["xcrun", "notarytool", "log", status["id"], "--keychain-profile", profile], + capture_output=True, + text=True, + ).stdout + ) raise RuntimeError("Notarization was not accepted") run("xcrun", "stapler", "staple", app) run("xcrun", "stapler", "validate", app) - assessment = subprocess.run(["spctl", "-a", "-vv", "-t", "exec", str(app)], - capture_output=True, text=True) + assessment = subprocess.run(["spctl", "-a", "-vv", "-t", "exec", str(app)], capture_output=True, text=True) print(assessment.stderr.strip()) if assessment.returncode != 0 or "source=Notarized Developer ID" not in assessment.stderr: raise RuntimeError("Gatekeeper did not accept the notarized app") @@ -135,6 +156,7 @@ def bundle_version(build, configured_build, override=None): raise ValueError("bundle version must be positive N[.N[.N]] (4/2/2 digits)") return override import fcntl # macOS-only; keep pure metadata helpers importable on Windows. + with (build / ".macos-package-sequence").open("a+") as counter: fcntl.flock(counter, fcntl.LOCK_EX) counter.seek(0) @@ -213,15 +235,16 @@ def library(source, owner): raise RuntimeError(f"{source} is not a thin {architecture} image") commands = load_commands(source) versions = re.findall(r"\bminos (\d+\.\d+(?:\.\d+)?)", commands) - versions += re.findall(r"cmd LC_VERSION_MIN_MACOSX\s+cmdsize \d+\s+version (\d+\.\d+(?:\.\d+)?)", commands) + versions += re.findall( + r"cmd LC_VERSION_MIN_MACOSX\s+cmdsize \d+\s+version (\d+\.\d+(?:\.\d+)?)", commands + ) for value in versions: - minimum = max(minimum, tuple(int(n) for n in value.split('.'))) + minimum = max(minimum, tuple(int(n) for n in value.split("."))) for dependency in dependencies(source): if system_library(dependency): continue target = library(resolve(dependency, source, owner), owner) - run("install_name_tool", "-change", dependency, - f"@rpath/{target.name}", destination) + run("install_name_tool", "-change", dependency, f"@rpath/{target.name}", destination) # SDL2-compat loads SDL3 with dlopen, so otool cannot discover it. if b"@loader_path/libSDL3.dylib" in source.read_bytes(): sdl3 = Path(run("brew", "--prefix", "sdl3")) / "lib/libSDL3.dylib" @@ -242,28 +265,41 @@ def library(source, owner): iconset.mkdir() for size in (16, 32, 128, 256, 512): for scale, suffix in ((1, ""), (2, "@2x")): - run("sips", "-z", size * scale, size * scale, ROOT / "assets/ui/Icons/DKR-R-Icon.png", - "--out", iconset / f"icon_{size}x{size}{suffix}.png") + run( + "sips", + "-z", + size * scale, + size * scale, + ROOT / "assets/ui/Icons/DKR-R-Icon.png", + "--out", + iconset / f"icon_{size}x{size}{suffix}.png", + ) run("iconutil", "-c", "icns", iconset, "-o", resources / "DKR-R.icns") - plist = dict(CFBundleExecutable="DKR-R", CFBundleName="DKR-R", - CFBundleIdentifier="io.github.thatguymcd.dkr-r", - CFBundlePackageType="APPL", CFBundleVersion=bundle_build, - CFBundleShortVersionString=release_number(version), - CFBundleIconFile="DKR-R", - LSMinimumSystemVersion=".".join(map(str, minimum)), - NSHighResolutionCapable=True) + plist = dict( + CFBundleExecutable="DKR-R", + CFBundleName="DKR-R", + CFBundleIdentifier="io.github.thatguymcd.dkr-r", + CFBundlePackageType="APPL", + CFBundleVersion=bundle_build, + CFBundleShortVersionString=release_number(version), + CFBundleIconFile="DKR-R", + LSMinimumSystemVersion=".".join(map(str, minimum)), + NSHighResolutionCapable=True, + ) with (contents / "Info.plist").open("wb") as handle: plistlib.dump(plist, handle) notices = resources / "Licenses" notices.mkdir() - for source, label in [(ROOT / "LICENSE.md", "DKR-R-LICENSE.md"), - (ROOT / "THIRD_PARTY.md", "DKR-R-THIRD_PARTY.md"), - (ROOT / "runtime-recomp/COPYING-NOTICE.md", "DKR-R-COPYING-NOTICE.md"), - (ROOT / "extern/rt64/LICENSE", "RT64-LICENSE.txt"), - (ROOT / "extern/rt64/src/contrib/imgui/LICENSE.txt", "Dear-ImGui-LICENSE.txt"), - (ROOT / "extern/n64-modern-runtime/COPYING", "N64ModernRuntime-COPYING.txt"), - (ROOT / "extern/n64-modern-runtime/N64Recomp/LICENSE", "N64Recomp-LICENSE.txt"), - (ROOT / "extern/sdl3/LICENSE.txt", "SDL3-LICENSE.txt")]: + for source, label in [ + (ROOT / "LICENSE.md", "DKR-R-LICENSE.md"), + (ROOT / "THIRD_PARTY.md", "DKR-R-THIRD_PARTY.md"), + (ROOT / "runtime-recomp/COPYING-NOTICE.md", "DKR-R-COPYING-NOTICE.md"), + (ROOT / "extern/rt64/LICENSE", "RT64-LICENSE.txt"), + (ROOT / "extern/rt64/src/contrib/imgui/LICENSE.txt", "Dear-ImGui-LICENSE.txt"), + (ROOT / "extern/n64-modern-runtime/COPYING", "N64ModernRuntime-COPYING.txt"), + (ROOT / "extern/n64-modern-runtime/N64Recomp/LICENSE", "N64Recomp-LICENSE.txt"), + (ROOT / "extern/sdl3/LICENSE.txt", "SDL3-LICENSE.txt"), + ]: shutil.copy2(source, notices / label) shutil.copytree(ROOT / "packaging/licenses", notices / "additional") # Include licences for the Homebrew SDL libraries, where used. @@ -279,7 +315,7 @@ def library(source, owner): for dependency in dependencies(destination): if system_library(dependency): continue - if not (dependency.startswith("@rpath/") and (frameworks / dependency[len("@rpath/"):]).is_file()): + if not (dependency.startswith("@rpath/") and (frameworks / dependency[len("@rpath/") :]).is_file()): raise RuntimeError(f"Unbundled dependency: {dependency}") run("xattr", "-crs", stage) sign(app, images.values(), identity, Path(temporary)) @@ -309,11 +345,17 @@ def library(source, owner): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("build", type=Path) parser.add_argument("version") - parser.add_argument("--build-number", default=os.environ.get("DKR_MAC_BUNDLE_VERSION"), - help="Explicit CFBundleVersion; otherwise reserve a distinct increasing build") - parser.add_argument("--notarize", action="store_true", - help="Developer ID sign (DKR_MAC_SIGN_IDENTITY), notarize with a notarytool " - "keychain profile (DKR_MAC_NOTARY_PROFILE) and staple") + parser.add_argument( + "--build-number", + default=os.environ.get("DKR_MAC_BUNDLE_VERSION"), + help="Explicit CFBundleVersion; otherwise reserve a distinct increasing build", + ) + parser.add_argument( + "--notarize", + action="store_true", + help="Developer ID sign (DKR_MAC_SIGN_IDENTITY), notarize with a notarytool " + "keychain profile (DKR_MAC_NOTARY_PROFILE) and staple", + ) args = parser.parse_args() if release_number(args.version) is None or not re.fullmatch(r"[A-Za-z0-9._+-]+", args.version): parser.error("version must start with a numeric major.minor.patch") diff --git a/scripts/package_source.py b/scripts/package_source.py index 8acc3603..55452c7d 100644 --- a/scripts/package_source.py +++ b/scripts/package_source.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 """Create and validate a deterministic source ZIP from the current source tree.""" + from __future__ import annotations import argparse -import pathlib import os -import tempfile +import pathlib import subprocess import sys +import tempfile import zipfile ROOT = pathlib.Path(__file__).resolve().parents[1] @@ -30,11 +31,12 @@ def git_paths(*arguments: str) -> list[pathlib.Path]: result = subprocess.run( - ["git", *arguments, "-z"], cwd=ROOT, check=True, + ["git", *arguments, "-z"], + cwd=ROOT, + check=True, capture_output=True, ).stdout - return [pathlib.Path(item.decode("utf-8")) - for item in result.split(b"\0") if item] + return [pathlib.Path(item.decode("utf-8")) for item in result.split(b"\0") if item] def source_files() -> list[pathlib.Path]: @@ -66,9 +68,33 @@ def source_files() -> list[pathlib.Path]: pathlib.PurePosixPath("scripts"), ) allowed_untracked_suffixes = { - ".c", ".cc", ".cpp", ".h", ".hpp", ".json", ".md", ".png", - ".toml", ".hlsl", ".hlsli", ".in", ".inc", ".inl", ".ld", ".ps1", ".py", ".sh", ".txt", ".patch", ".yaml", ".yml", ".cmake", - ".java", ".kt", ".xml", ".gradle", + ".c", + ".cc", + ".cpp", + ".h", + ".hpp", + ".json", + ".md", + ".png", + ".toml", + ".hlsl", + ".hlsli", + ".in", + ".inc", + ".inl", + ".ld", + ".ps1", + ".py", + ".sh", + ".txt", + ".patch", + ".yaml", + ".yml", + ".cmake", + ".java", + ".kt", + ".xml", + ".gradle", } selected = set(tracked) allowed_exact_untracked = { @@ -87,13 +113,9 @@ def source_files() -> list[pathlib.Path]: } for path in untracked: pure = pathlib.PurePosixPath(path.as_posix()) - if ( - pure in allowed_exact_untracked - or ( - path.suffix.lower() in allowed_untracked_suffixes - and any(pure.is_relative_to(root) - for root in allowed_untracked_roots) - ) + if pure in allowed_exact_untracked or ( + path.suffix.lower() in allowed_untracked_suffixes + and any(pure.is_relative_to(root) for root in allowed_untracked_roots) ): selected.add(path) elif path.suffix.lower() in allowed_untracked_suffixes: @@ -115,7 +137,7 @@ def create_archive(output: pathlib.Path) -> None: info.create_system = 3 info.compress_type = zipfile.ZIP_DEFLATED executable = relative.suffix.lower() == ".sh" - info.external_attr = ((0o100755 if executable else 0o100644) << 16) + info.external_attr = (0o100755 if executable else 0o100644) << 16 archive.writestr(info, source.read_bytes()) diff --git a/scripts/pattern_guard.py b/scripts/pattern_guard.py new file mode 100644 index 00000000..1c44fa02 --- /dev/null +++ b/scripts/pattern_guard.py @@ -0,0 +1,1130 @@ +#!/usr/bin/env python3 +"""Refuse new configure-time patch layers on first-party code. + +DKR-R used to ship from runtime-recomp/src plus a chain of first-party layers +(patches//ReleaseOverlay.cmake, Python string-replace pipelines, a frozen +source capsule). The fold replaces that chain with real source. This guard +keeps any of those mechanisms from coming back, while third-party dependency +patches registered in patches/manifest.json stay allowed. + +Every finding names a rule (PG001...), prints as ``path:line: PGnnn message`` +and is followed by a one-line pointer to the supported alternative. Existing +hits are inventoried in architecture/pattern-guard-baseline.txt, one +`` `` line per pair. The baseline only ever shrinks: + +* a finding not in the baseline fails; +* a baseline entry whose file no longer produces that finding fails (delete it); +* with --diff, a baseline that gained entries since the base fails unless + --transition-layer is passed (CI passes it for PRs labelled + ``transition-layer`` until ``fold-in-base`` is tagged). + +Scopes: --tree (default) scans tracked and untracked, non-ignored files; +--diff BASE scans files changed since merge-base(BASE, HEAD) including local +edits; --staged scans the index's changed paths (contents are read from the +work tree, which is what pre-commit stashes to). extern/, the untracked build/ +directory and the guard's own fixtures are never scanned. + +The guard is deliberately small: globs, regexes, a CMake command tokenizer and +an AST walk. Standard library only. + +Exit status: + 0 no new findings, and the baseline matches the scanned files + 1 new findings, stale baseline entries, or unlabelled baseline growth + 2 usage or environment error (bad arguments, unknown base ref, not a git + work tree where one is needed, malformed baseline) +""" + +from __future__ import annotations + +import argparse +import ast +from collections.abc import Callable, Iterable, Iterator, Mapping +import dataclasses +import fnmatch +import functools +import hashlib +import json +from pathlib import Path, PurePosixPath +import re +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +BASELINE = PurePosixPath("architecture/pattern-guard-baseline.txt") +# Never scanned: submodules, and the guard, its tests and fixtures, which spell +# out every violation on purpose. The local build/ tree is skipped too, unless +# git tracks a file there (PG011 reports it). +EXCLUDED_PREFIXES = ("extern/", "tests/fixtures/pattern-guard/") +EXCLUDED_FILES = {"scripts/pattern_guard.py", "runtime-recomp/tests/pattern_guard_tests.py"} +LOCAL_BUILD = "build/" + +BASELINE_HEADER = """\ +# Pattern guard baseline: first-party patch-layer mechanisms that predate the +# guard. One " " line per finding, sorted, enumerated (never a glob). +# +# This file only ever shrinks. scripts/pattern_guard.py fails on a finding that +# is not listed here, and on a line here whose file no longer produces it. New +# lines are accepted only on PRs labelled transition-layer, until fold-in-base +# is tagged; the fold drives this file to empty. +# +# Regenerate after removing hits: python3 scripts/pattern_guard.py --tree --baseline write +# Explain a rule: python3 scripts/pattern_guard.py --explain PG001 +""" + + +@dataclasses.dataclass(frozen=True) +class Rule: + """One guarded mechanism, with the supported alternative.""" + + id: str + name: str + detects: str + instead: str + + +RULES = { + rule.id: rule + for rule in ( + Rule( + "PG001", + "overlay-cmake", + "a CMake file under patches/ (a configure-time overlay)", + "change runtime-recomp/CMakeLists.txt or runtime-recomp/cmake/*.cmake directly", + ), + Rule( + "PG002", + "project-include", + "CMAKE_PROJECT_INCLUDE or cmake_language(DEFER outside GuestHooks.cmake and ProjectTests.cmake", + "declare targets and sources in runtime-recomp/CMakeLists.txt; no injected configure hooks", + ), + Rule( + "PG003", + "source-swap", + "rewriting a first-party target's SOURCES or pointing its sources at build/generated", + "edit the source file in runtime-recomp/src and list it with target_sources()", + ), + Rule( + "PG004", + "string-replace-pipeline", + "Python that reads first-party text, rewrites it with replace/re.sub/once/region and writes it out", + "edit the source file itself; generators must emit new files, not rewritten copies", + ), + Rule( + "PG005", + "capsule", + "a frozen source capsule: *capsule* paths, manifest.json with roles, patches/**/*-receipt.json, " + "source-capsule .gitattributes lines", + "commit the real source; git history is the archive", + ), + Rule( + "PG006", + "first-party-under-patches", + "a C/C++/shader/TOML source file under patches/", + "move it into runtime-recomp/src and add it to its target", + ), + Rule( + "PG007", + "unregistered-dependency-patch", + "a dependency *.patch, or a dkr_stage_patch() patch, missing from patches/manifest.json " + "or with a stale sha256", + "register the patch with its sha256 in patches/manifest.json", + ), + Rule( + "PG008", + "patch-against-first-party", + "a *.patch whose paths land in runtime-recomp/, modding/ or scripts/", + "commit the change to the first-party file instead of a patch", + ), + Rule( + "PG009", + "build-dir-include", + "#include of an absolute path or of a build/ directory", + "include a path relative to the source tree or a target include directory", + ), + Rule( + "PG010", + "identity-literal", + "a digest literal or ledger spliced into an online identity string", + "change identity only in netplay/legacy_identity.hpp", + ), + Rule( + "PG011", + "generated-committed", + "a committed file under a build/ directory, or a committed *_symbols.hpp generator output", + "generate it at build time and keep it out of git", + ), + Rule( + "PG012", + "guest-rewrite", + "Python or CMake rewriting recompiled guest code (RecompiledFuncs, funcs_*.c, guest_*.c) in place", + "declare the hook in runtime-recomp/guest-hooks/hooks.toml (scripts/guest_hooks.py applies it)", + ), + Rule( + "PG013", + "patches-layout", + "anything else under patches/: only manifest.json, README.md and registered " + "dependency *.patch files belong there", + "put first-party code, tooling and notes in runtime-recomp/, scripts/ or docs/", + ), + ) +} + +# Explicit allow-lists. Each entry is a reviewed, permanent exception. +PROJECT_INCLUDE_ALLOWED = { + "runtime-recomp/cmake/GuestHooks.cmake", + "runtime-recomp/cmake/ProjectTests.cmake", # deferred test assertions and metadata +} +SOURCE_SWAP_ALLOWED = { + "runtime-recomp/cmake/StagedPatch.cmake", # dependency-patch staging + "runtime-recomp/cmake/GuestHooks.cmake", +} +PIPELINE_ALLOWED = { + "scripts/gen_guest_symbols.py": "writes new symbol headers", + "scripts/guest_hooks.py": "applies declared hooks to generated guest code", + "scripts/generate_replay_probe.py": "reads first-party headers only to collect probe symbol names", +} +GUEST_REWRITE_ALLOWED = {"scripts/guest_hooks.py", "runtime-recomp/cmake/GuestHooks.cmake"} +IDENTITY_ALLOWED_SUFFIX = "netplay/legacy_identity.hpp" + +CMAKE_SUFFIXES = (".cmake",) +SCRIPT_SUFFIXES = (".cmake", ".py", ".sh", ".ps1", ".cmd", ".bat", ".yml", ".yaml") +C_FAMILY_SUFFIXES = (".c", ".cc", ".cpp", ".cxx", ".h", ".hh", ".hpp", ".hxx", ".inl", ".inc", ".m", ".mm") +NATIVE_SUFFIXES = C_FAMILY_SUFFIXES + (".hlsl", ".hlsli") +CODE_SUFFIXES = NATIVE_SUFFIXES + SCRIPT_SUFFIXES +FIRST_PARTY_SOURCE_SUFFIXES = (".c", ".cc", ".cpp", ".h", ".hpp", ".inl", ".mm") +# Where a patch target is our own code; src/game/ is runtime-recomp-relative. +FIRST_PARTY_PATCH_TARGETS = ("runtime-recomp/", "modding/", "scripts/", "src/game/") + +PROJECT_INCLUDE = re.compile(r"CMAKE_PROJECT_INCLUDE|cmake_language\s*\(\s*DEFER\b", re.IGNORECASE) +INCLUDE = re.compile(r'^\s*#\s*include\s*[<"]([^">]+)[">]') +ABSOLUTE_OR_BUILD = re.compile(r"^(?:/|\\|[A-Za-z]:[/\\])|(?:^|[/\\])build[/\\]") +# Identity splices, not every 64-hex string: mods carry content hashes. +IDENTITY_SPLICE = re.compile( + r"""\bbase\s*\+=\s*"(?:[0-9a-fA-F]{64}|'+\s*\+)""" # base+="" or code emitting one + r"""|\|(?:sdk2|shared-saves):""" # salted identity joins + r"""|string\s*\(\s*SHA256\s+\S+\s+"[^"]*\$\{ledger\}""" +) +GUEST_CODE = re.compile(r"RecompiledFuncs|(?:^|[/\\*])(?:funcs|guest)_(?:\d+|\*|\$?\{[^}]*\})\.c\b") +# A string naming first-party source or a patch layer, as a pipeline would. +FIRST_PARTY_TEXT = re.compile(r"(?:^|[/\\])(?:runtime-recomp[/\\]src|src[/\\]game|modding|patches)[/\\]") +STABLE_PATH = re.compile(r"dkr_\w+_stable_path") +SWAP_COMMANDS = {"dkr_repair_replace_source", "dkr_sdk2_rebind"} +LIST_MUTATIONS = {"TRANSFORM", "FILTER", "REMOVE_ITEM", "REMOVE_AT", "INSERT"} + + +@dataclasses.dataclass(frozen=True, order=True) +class Finding: + """A rule hit at a path and 1-based line.""" + + path: str + line: int + rule: str + message: str + + def render(self) -> str: + return f"{self.path}:{self.line}: {self.rule} {self.message}\n instead: {RULES[self.rule].instead}" + + +class GuardError(Exception): + """A usage or environment problem (exit status 2).""" + + +# --- CMake tokenizer -------------------------------------------------------- + + +@dataclasses.dataclass(frozen=True) +class Command: + """One CMake command invocation; quoted arguments are unquoted.""" + + name: str + args: tuple[str, ...] + line: int + + +_BRACKET_OPEN = re.compile(r"\[(=*)\[") + + +def parse_cmake(text: str) -> list[Command]: + """Tokenize CMake into commands. Good enough for guard checks, not a + full CMake parser: it understands comments, quoting, bracket arguments + and nested parentheses, and nothing about evaluation.""" + commands: list[Command] = [] + i, line, size = 0, 1, len(text) + + def skip_bracket(start: int, equals: str) -> int: + nonlocal line + end = text.find(f"]{equals}]", start) + end = size if end < 0 else end + len(equals) + 2 + line += text.count("\n", start, end) + return end + + while i < size: + char = text[i] + if char == "\n": + line, i = line + 1, i + 1 + elif char == "#": + bracket = _BRACKET_OPEN.match(text, i + 1) + if bracket: + i = skip_bracket(bracket.end(), bracket.group(1)) + else: + newline = text.find("\n", i) + i = size if newline < 0 else newline + elif char.isalpha() or char == "_": + start = i + while i < size and (text[i].isalnum() or text[i] == "_"): + i += 1 + name, opener = text[start:i], i + while opener < size and text[opener] in " \t": + opener += 1 + if opener < size and text[opener] == "(": + command_line = line + args, i = _parse_args(text, opener + 1) + commands.append(Command(name.lower(), tuple(args), command_line)) + line += text.count("\n", opener, i) + else: + i += 1 + return commands + + +def _parse_args(text: str, i: int) -> tuple[list[str], int]: + """Arguments from just after ``(`` to the matching ``)``; returns the + index after it.""" + args: list[str] = [] + token: list[str] = [] + depth, size = 1, len(text) + + def flush() -> None: + if token: + args.append("".join(token)) + token.clear() + + while i < size: + char = text[i] + if char == '"': + flush() + i += 1 + quoted: list[str] = [] + while i < size and text[i] != '"': + if text[i] == "\\" and i + 1 < size: + quoted.append(text[i : i + 2]) + i += 2 + continue + quoted.append(text[i]) + i += 1 + args.append("".join(quoted)) + i += 1 + elif char == "#": + flush() + newline = text.find("\n", i) + i = size if newline < 0 else newline + elif char == "[" and _BRACKET_OPEN.match(text, i): + flush() + bracket = _BRACKET_OPEN.match(text, i) + assert bracket + close = f"]{bracket.group(1)}]" + end = text.find(close, bracket.end()) + end = size if end < 0 else end + args.append(text[bracket.end() : end]) + i = end + len(close) + elif char.isspace(): + flush() + i += 1 + elif char == "(": + flush() + depth += 1 + i += 1 + elif char == ")": + flush() + depth -= 1 + i += 1 + if depth == 0: + break + else: + token.append(char) + i += 1 + return args, i + + +# --- The tree under scan ---------------------------------------------------- + + +class Tree: + """Files of one checkout plus the context rules need (manifest, targets). + + A git work tree is enumerated with git; any other directory (the test + fixtures) is walked, and every file in it counts as committed. + + ``overlay`` maps paths to text that replaces (or creates) the file on + disk, so rules can judge an edit before it is written. Overlay paths count + as committed outside build/, as they will be once the edit lands.""" + + def __init__(self, root: Path, overlay: Mapping[str, str] | None = None) -> None: + self.root = root.resolve() + self.overlay = dict(overlay or {}) + self.is_git = _git_toplevel(self.root) == self.root + if self.is_git: + self.tracked = set(_git_lines(self.root, "ls-files", "--cached")) + listed = self.tracked | set(_git_lines(self.root, "ls-files", "--others", "--exclude-standard")) + else: + relative = (path.relative_to(self.root) for path in self.root.rglob("*") if path.is_file()) + self.tracked = listed = { + path.as_posix() for path in relative if ".git" not in path.parts and path.parts[0] != LOCAL_BUILD[:-1] + } + self.tracked |= {path for path in self.overlay if not path.startswith(LOCAL_BUILD)} + self.files = sorted( + path for path in listed | set(self.overlay) if self.is_scanned(path) and self.has_file(path) + ) + + def is_scanned(self, path: str) -> bool: + """Outside the excluded paths, and not untracked local build output.""" + return ( + not path.startswith(EXCLUDED_PREFIXES) + and path not in EXCLUDED_FILES + and (path in self.tracked or not path.startswith(LOCAL_BUILD)) + ) + + def has_file(self, path: str) -> bool: + return path in self.overlay or (self.root / path).is_file() + + @functools.cache + def text(self, path: str) -> str | None: + """Decoded file text, or None for binary or unreadable files.""" + if path in self.overlay: + return self.overlay[path] + try: + data = (self.root / path).read_bytes() + except OSError: + return None + if b"\0" in data[:8192]: + return None + return data.decode("utf-8", errors="replace") + + @functools.cache + def cmake(self, path: str) -> list[Command]: + return parse_cmake(self.text(path) or "") + + @functools.cache + def python(self, path: str) -> ast.Module | None: + try: + return ast.parse(self.text(path) or "", filename=path) + except (SyntaxError, ValueError): + return None + + @functools.cached_property + def registered_patches(self) -> dict[str, str]: + """patches/manifest.json path -> sha256 for every dependency patch.""" + try: + manifest = json.loads(self.text("patches/manifest.json") or "{}") + except json.JSONDecodeError: + return {} + return { + patch["path"]: patch.get("sha256", "") + for dependency in manifest.get("dependencies", []) + for patch in dependency.get("patches", []) + if "path" in patch + } + + @functools.cached_property + def dependency_dirs(self) -> set[str]: + """patches/ directories that hold at least one registered patch.""" + return {"/".join(path.split("/")[:2]) for path in self.registered_patches} + + @functools.cached_property + def first_party_targets(self) -> set[str]: + """Targets declared by add_library/add_executable under runtime-recomp/.""" + return { + command.args[0] + for path in self.files + if path.startswith("runtime-recomp/") and is_cmake(path) + for command in self.cmake(path) + if command.name in ("add_library", "add_executable") and command.args and "${" not in command.args[0] + } + + @functools.cached_property + def first_party_sources(self) -> set[str]: + """Basenames of first-party C/C++ sources, for spotting rewritten copies.""" + return { + PurePosixPath(path).name + for path in self.files + if path.startswith("runtime-recomp/src/") and path.endswith(FIRST_PARTY_SOURCE_SUFFIXES) + } + + def is_first_party_target(self, name: str) -> bool: + """Literal targets are looked up; a variable target cannot be proven + third-party, so it counts as first-party.""" + return "${" in name or name in self.first_party_targets + + +def _git_toplevel(root: Path) -> Path | None: + result = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--show-toplevel"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + return Path(result.stdout.strip()).resolve() if result.returncode == 0 else None + + +def _git_lines(root: Path, *args: str) -> list[str]: + result = subprocess.run( + ["git", "-C", str(root), *args, "-z"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + if result.returncode: + raise GuardError(f"git {' '.join(args)} failed: {result.stderr.strip()}") + return [line for line in result.stdout.split("\0") if line] + + +def is_cmake(path: str) -> bool: + return path.endswith(CMAKE_SUFFIXES) or PurePosixPath(path).name == "CMakeLists.txt" + + +def is_workflow(path: str) -> bool: + return path.startswith(".github/workflows/") + + +# --- Rules ----------------------------------------------------------------- + +FileCheck = Callable[[Tree, str], Iterator[Finding]] +FILE_CHECKS: list[FileCheck] = [] + + +def file_check(function: FileCheck) -> FileCheck: + FILE_CHECKS.append(function) + return function + + +def _lines(tree: Tree, path: str) -> Iterator[tuple[int, str]]: + yield from enumerate((tree.text(path) or "").splitlines(), 1) + + +@file_check +def overlay_cmake(tree: Tree, path: str) -> Iterator[Finding]: + """PG001: CMake under patches/ is a configure-time overlay.""" + if path.startswith("patches/") and is_cmake(path): + yield Finding(path, 1, "PG001", "CMake overlay under patches/") + + +@file_check +def project_include(tree: Tree, path: str) -> Iterator[Finding]: + """PG002: injected configure hooks in build scripts and workflows.""" + if path in PROJECT_INCLUDE_ALLOWED: + return + if not (is_cmake(path) or path.endswith(SCRIPT_SUFFIXES) or is_workflow(path)): + return + for number, line in _lines(tree, path): + if not line.lstrip().startswith("#") and PROJECT_INCLUDE.search(line): + yield Finding(path, number, "PG002", "injects a configure hook") + + +@file_check +def source_swap(tree: Tree, path: str) -> Iterator[Finding]: + """PG003: swapping first-party target sources at configure time. + + Third-party SOURCES swaps (rt64, plume, SDL2 ...) are the staged + dependency-patch mechanism and stay allowed.""" + if not is_cmake(path) or path in SOURCE_SWAP_ALLOWED: + return + commands = tree.cmake(path) + first_party = tree.is_first_party_target + for index, command in enumerate(commands): + args = command.args + if command.name in SWAP_COMMANDS or STABLE_PATH.fullmatch(command.name): + yield Finding(path, command.line, "PG003", f"source swap helper {command.name}()") + elif ( + command.name in ("function", "macro") + and args + and (args[0].lower() in SWAP_COMMANDS or STABLE_PATH.fullmatch(args[0].lower())) + ): + yield Finding(path, command.line, "PG003", f"defines source swap helper {args[0]}()") + elif command.name == "set_target_properties" and "PROPERTIES" in args: + split = args.index("PROPERTIES") + if "SOURCES" in args[split + 1 :: 2] and any(map(first_party, args[:split])): + yield Finding(path, command.line, "PG003", "replaces a first-party target's SOURCES") + elif command.name == "set_property" and args[:1] == ("TARGET",) and "PROPERTY" in args: + split = args.index("PROPERTY") + targets = [arg for arg in args[1:split] if arg not in ("APPEND", "APPEND_STRING")] + if args[split + 1 : split + 2] == ("SOURCES",) and any(map(first_party, targets)): + yield Finding(path, command.line, "PG003", "replaces a first-party target's SOURCES") + elif command.name == "get_target_property" and len(args) == 3 and args[2] == "SOURCES": + variable, target = args[0], args[1] + mutated = any( + later.name == "list" + and len(later.args) > 1 + and later.args[0] in LIST_MUTATIONS + and later.args[1] == variable + for later in commands[index + 1 :] + ) + if mutated and first_party(target): + yield Finding(path, command.line, "PG003", f"edits the SOURCES list of first-party {target}") + elif _generated_include_dirs(command, first_party): + yield Finding(path, command.line, "PG003", "points first-party sources at build/generated includes") + + +def _generated_include_dirs(command: Command, first_party: Callable[[str], bool]) -> bool: + """set_source_files_properties / set_property(SOURCE) giving sources an + INCLUDE_DIRECTORIES entry in the generated build tree.""" + args = command.args + if command.name == "set_source_files_properties": + keyword = "PROPERTIES" + elif command.name == "set_property" and args[:1] == ("SOURCE",): + keyword = "PROPERTY" + else: + return False + if keyword not in args: + return False + split = args.index(keyword) + values = args[split + 1 :] + if "INCLUDE_DIRECTORIES" not in values[:1] + values[::2] or not any( + "${CMAKE_BINARY_DIR}/generated" in value for value in values + ): + return False + scope = args[:split] + if "TARGET_DIRECTORY" not in scope: + return True # the calling directory is first-party + targets = scope[scope.index("TARGET_DIRECTORY") + 1 :] + return any(map(first_party, targets)) + + +@dataclasses.dataclass +class _PythonFacts: + """What a Python module does with text, gathered in one AST walk.""" + + reads: list[ast.Call] = dataclasses.field(default_factory=list[ast.Call]) + rewrites: list[ast.Call] = dataclasses.field(default_factory=list[ast.Call]) + writes: list[ast.Call] = dataclasses.field(default_factory=list[ast.Call]) + strings: list[str] = dataclasses.field(default_factory=list[str]) + read_targets: set[str] = dataclasses.field(default_factory=set[str]) + write_targets: list[tuple[str, int, ast.AST]] = dataclasses.field(default_factory=list[tuple[str, int, ast.AST]]) + + +def _open_mode(call: ast.Call) -> str | None: + """Mode of an open()/Path.open() call, or None when it is not one.""" + function = call.func + if isinstance(function, ast.Name) and function.id == "open": + position = 1 + elif isinstance(function, ast.Attribute) and function.attr == "open": + builtin_like = isinstance(function.value, ast.Name) and function.value.id in ("io", "codecs") + position = 1 if builtin_like else 0 + else: + return None + mode = ( + call.args[position] + if len(call.args) > position + else next((keyword.value for keyword in call.keywords if keyword.arg == "mode"), None) + ) + if mode is None: + return "r" + return mode.value if isinstance(mode, ast.Constant) and isinstance(mode.value, str) else "?" + + +def _open_target(call: ast.Call) -> ast.AST | None: + function = call.func + if isinstance(function, ast.Name): + return call.args[0] if call.args else None + assert isinstance(function, ast.Attribute) + return function.value + + +def _is_rewrite(call: ast.Call) -> bool: + function = call.func + if isinstance(function, ast.Name): + return function.id in ("once", "region") + if not isinstance(function, ast.Attribute) or len(call.args) < 2: + return False + if function.attr == "replace": + return not (isinstance(function.value, ast.Name) and function.value.id == "os") + return function.attr in ("sub", "subn") + + +def _walk_scope(node: ast.AST) -> Iterator[ast.AST]: + """Nodes of one scope, without descending into nested functions or classes.""" + for child in ast.iter_child_nodes(node): + yield child + if not isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)): + yield from _walk_scope(child) + + +def _python_facts(nodes: Iterable[ast.AST]) -> _PythonFacts: + facts = _PythonFacts() + for node in nodes: + if isinstance(node, ast.Constant) and isinstance(node.value, str): + facts.strings.append(node.value) + if not isinstance(node, ast.Call): + continue + function, mode = node.func, _open_mode(node) + attribute, receiver = (function.attr, function.value) if isinstance(function, ast.Attribute) else (None, None) + if mode is not None: + target = _open_target(node) + if any(flag in mode for flag in "wax+"): + facts.writes.append(node) + if target is not None: + facts.write_targets.append((ast.dump(target), node.lineno, target)) + else: + facts.reads.append(node) + if target is not None: + facts.read_targets.add(ast.dump(target)) + elif receiver is not None and attribute in ("read_text", "read_bytes", "readlines", "read"): + facts.reads.append(node) + facts.read_targets.add(ast.dump(receiver)) + elif receiver is not None and ( + attribute in ("write_text", "write_bytes", "writelines") or (attribute == "write" and node.args) + ): + facts.writes.append(node) + facts.write_targets.append((ast.dump(receiver), node.lineno, receiver)) + elif _is_rewrite(node): + facts.rewrites.append(node) + return facts + + +def _docstrings(module: ast.Module) -> set[int]: + """ids of docstring constants, which describe code rather than drive it.""" + owners = [module] + [ + node for node in ast.walk(module) if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + ] + return { + id(owner.body[0].value) + for owner in owners + if owner.body and isinstance(owner.body[0], ast.Expr) and isinstance(owner.body[0].value, ast.Constant) + } + + +@file_check +def string_replace_pipeline(tree: Tree, path: str) -> Iterator[Finding]: + """PG004: read text, rewrite it, write it, with first-party source or a + patch layer named as data. File-level on purpose: pipelines spread the + read, the edits and the write over helpers.""" + if not path.endswith(".py") or path in PIPELINE_ALLOWED: + return + module = tree.python(path) + if module is None: + return + docstrings = _docstrings(module) + facts = _python_facts(node for node in ast.walk(module) if id(node) not in docstrings) + if not (facts.reads and facts.rewrites and facts.writes): + return + named = next((text for text in facts.strings if _names_first_party(tree, text)), None) + if named is not None: + yield Finding( + path, facts.rewrites[0].lineno, "PG004", f"string-replace pipeline over first-party text ({named[:60]!r})" + ) + + +def _names_first_party(tree: Tree, text: str) -> bool: + """A first-party source tree or patch layer path, or a first-party source + file name (which is what a staged rewritten copy is called).""" + return ( + bool(FIRST_PARTY_TEXT.search(text)) or PurePosixPath(text.replace("\\", "/")).name in tree.first_party_sources + ) + + +@file_check +def capsule(tree: Tree, path: str) -> Iterator[Finding]: + """PG005: a frozen source capsule in any of its forms.""" + posix = PurePosixPath(path) + if posix.name == ".gitattributes": + for number, line in _lines(tree, path): + if "source-capsule" in line: + yield Finding(path, number, "PG005", "capsule attributes") + return + if posix.suffix != ".md" and any("capsule" in part.lower() for part in posix.parts): + yield Finding(path, 1, "PG005", "capsule file") + elif path.startswith("patches/") and posix.name.endswith("-receipt.json"): + yield Finding(path, 1, "PG005", "capsule receipt") + elif posix.name == "manifest.json": + try: + manifest = json.loads(tree.text(path) or "") + except json.JSONDecodeError: + return + if isinstance(manifest, dict) and "roles" in manifest: + yield Finding(path, 1, "PG005", "capsule manifest (has roles)") + + +@file_check +def first_party_under_patches(tree: Tree, path: str) -> Iterator[Finding]: + """PG006: source files living in a patch layer.""" + if path.startswith("patches/") and path.endswith(NATIVE_SUFFIXES + (".toml",)): + yield Finding(path, 1, "PG006", "first-party source under patches/") + + +@file_check +def unregistered_dependency_patch(tree: Tree, path: str) -> Iterator[Finding]: + """PG007: dependency patches and staged patches go through one registry.""" + registered = tree.registered_patches + if path.endswith(".patch") and "/".join(path.split("/")[:2]) in tree.dependency_dirs: + if path not in registered: + yield Finding(path, 1, "PG007", "dependency patch is not in patches/manifest.json") + elif registered[path] != hashlib.sha256((tree.root / path).read_bytes()).hexdigest(): + yield Finding(path, 1, "PG007", "sha256 differs from patches/manifest.json") + if is_cmake(path): + for command in tree.cmake(path): + if command.name != "dkr_stage_patch" or len(command.args) < 4: + continue + # Patches outside patches/ (build-iteration test data) are not dependency patches. + match = re.search(r"(?:^|[/}])(patches/.+\.patch)$", command.args[3]) + if match and match.group(1) not in registered: + yield Finding( + path, command.line, "PG007", f"staged patch {match.group(1)} is not in patches/manifest.json" + ) + + +@file_check +def patch_against_first_party(tree: Tree, path: str) -> Iterator[Finding]: + """PG008: a diff whose target is our own code.""" + if not path.endswith(".patch"): + return + for number, line in _lines(tree, path): + if line.startswith(("--- ", "+++ ")): + target = line[4:].split("\t")[0].strip() + target = target[2:] if target.startswith(("a/", "b/")) else target + if target.startswith(FIRST_PARTY_PATCH_TARGETS): + yield Finding(path, number, "PG008", f"patches first-party file {target}") + return + + +@file_check +def build_dir_include(tree: Tree, path: str) -> Iterator[Finding]: + """PG009: sources that only compile inside one configured build tree.""" + if not path.endswith(NATIVE_SUFFIXES): + return + for number, line in _lines(tree, path): + include = INCLUDE.match(line) + if include and ABSOLUTE_OR_BUILD.search(include.group(1)): + yield Finding(path, number, "PG009", f"includes {include.group(1)}") + + +@file_check +def identity_literal(tree: Tree, path: str) -> Iterator[Finding]: + """PG010: identity built by splicing digests instead of named constants.""" + if not (path.endswith(CODE_SUFFIXES) or is_cmake(path)) or path.endswith(IDENTITY_ALLOWED_SUFFIX): + return + for number, line in _lines(tree, path): + if IDENTITY_SPLICE.search(line): + yield Finding(path, number, "PG010", "digest spliced into an identity string") + + +@file_check +def generated_committed(tree: Tree, path: str) -> Iterator[Finding]: + """PG011: generator output in git.""" + posix = PurePosixPath(path) + if path not in tree.tracked: + return + if "build" in posix.parts[:-1]: + yield Finding(path, 1, "PG011", "committed file under a build/ directory") + elif fnmatch.fnmatch(posix.name, "*_symbols.hpp"): + yield Finding(path, 1, "PG011", "committed generated symbol header") + + +@file_check +def guest_rewrite(tree: Tree, path: str) -> Iterator[Finding]: + """PG012: read-modify-write of recompiled guest code. Naming those paths + is fine (build scripts and generators do); rewriting a file in place whose + path derives from one is not.""" + if path in GUEST_REWRITE_ALLOWED or not GUEST_CODE.search(tree.text(path) or ""): + return + if path.endswith(".py"): + module = tree.python(path) + if module is None: + return + functions = (node for node in ast.walk(module) if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))) + for scope in (module, *functions): + nodes = list(_walk_scope(scope)) + facts = _python_facts(nodes) + if not facts.rewrites: + continue + guest_names = _guest_bound_names(nodes) + for target, number, expression in facts.write_targets: + if target in facts.read_targets and _names_guest_code(expression, guest_names): + yield Finding(path, number, "PG012", "rewrites guest code in place") + return + elif is_cmake(path): + commands = tree.cmake(path) + guest_vars = _guest_bound_variables(commands) + read: dict[str, int] = {} + for index, command in enumerate(commands): + if command.name != "file" or len(command.args) < 2: + continue + mode, target = command.args[0], command.args[1] + if mode == "READ": + read[target] = index + elif ( + mode in ("WRITE", "APPEND") + and target in read + and (GUEST_CODE.search(target) or _references(target, guest_vars)) + and any( + later.name == "string" and later.args[:1] in (("REPLACE",), ("REGEX",)) + for later in commands[read[target] : index] + ) + ): + yield Finding(path, command.line, "PG012", "rewrites guest code in place") + return + + +def _guest_bound_names(nodes: list[ast.AST]) -> set[str]: + """Names in one scope whose value derives from a guest code path, to a + fixpoint (``funcs = build / "RecompiledFuncs"; for f in funcs.glob(...)``).""" + bindings: list[tuple[list[str], ast.AST]] = [] + for node in nodes: + if isinstance(node, ast.Assign): + bindings.append(([name for target in node.targets for name in _bound(target)], node.value)) + elif isinstance(node, (ast.AnnAssign, ast.AugAssign, ast.NamedExpr)) and node.value is not None: + bindings.append((_bound(node.target), node.value)) + elif isinstance(node, (ast.For, ast.AsyncFor, ast.comprehension)): + bindings.append((_bound(node.target), node.iter)) + elif isinstance(node, ast.withitem) and node.optional_vars is not None: + bindings.append((_bound(node.optional_vars), node.context_expr)) + names: set[str] = set() + while True: + grown = {name for targets, value in bindings if _names_guest_code(value, names) for name in targets} + if grown <= names: + return names + names |= grown + + +def _bound(target: ast.AST) -> list[str]: + return [node.id for node in ast.walk(target) if isinstance(node, ast.Name)] + + +def _names_guest_code(expression: ast.AST, guest_names: set[str]) -> bool: + return any( + (isinstance(node, ast.Constant) and isinstance(node.value, str) and GUEST_CODE.search(node.value)) + or (isinstance(node, ast.Name) and node.id in guest_names) + for node in ast.walk(expression) + ) + + +def _guest_bound_variables(commands: list[Command]) -> set[str]: + """CMake variables set (set, list, file(GLOB), foreach) from guest code paths.""" + bindings: list[tuple[str, tuple[str, ...]]] = [] + for command in commands: + args = command.args + if command.name in ("set", "foreach") and args: + bindings.append((args[0], args[1:])) + elif command.name in ("file", "list") and len(args) > 1 and args[0] in ("GLOB", "GLOB_RECURSE", "APPEND"): + bindings.append((args[1], args[2:])) + variables: set[str] = set() + while True: + grown = { + name + for name, values in bindings + if any(GUEST_CODE.search(value) or _references(value, variables) for value in values) + } + if grown <= variables: + return variables + variables |= grown + + +def _references(value: str, variables: set[str]) -> bool: + """``${name}`` anywhere in the value, or a bare name (``IN LISTS name``).""" + return value in variables or any(f"${{{name}}}" in value for name in variables) + + +def patches_layout(tree: Tree, path: str, flagged: set[str]) -> Iterator[Finding]: + """PG013: catch-all for patches/ contents no specific rule claimed.""" + if not path.startswith("patches/") or path in flagged: + return + parts = PurePosixPath(path).parts + top = "/".join(parts[:2]) + allowed = ( + path == "patches/manifest.json" + or path == "patches/README.md" + or (top in tree.dependency_dirs and (path.endswith(".patch") or parts[-1] == "README.md")) + ) + if not allowed: + yield Finding(path, 1, "PG013", "patches/ holds only manifest.json, READMEs and registered dependency patches") + + +def scan(tree: Tree, paths: Iterable[str]) -> list[Finding]: + """Every finding for the given tree-relative paths, sorted.""" + findings: list[Finding] = [] + scoped = [path for path in paths if tree.has_file(path)] + for path in scoped: + for check in FILE_CHECKS: + findings.extend(check(tree, path)) + flagged = {finding.path for finding in findings} + for path in scoped: + findings.extend(patches_layout(tree, path, flagged)) + return sorted(set(findings)) + + +def check_content(root: Path, path: str, text: str) -> list[Finding]: + """Findings for one root-relative path if its content were ``text``. + + Public entry point for scripts/agent_hooks/pre_edit_guard.py. It applies + every rule but does not consult the baseline; callers filter. Excluded + paths (extern/, the guard's own files) yield nothing.""" + tree = Tree(root, {path: text}) + return scan(tree, [path]) if tree.is_scanned(path) else [] + + +# --- Baseline ---------------------------------------------------------------- + +Entry = tuple[str, str] # (path, rule) + + +def parse_baseline(text: str, source: str) -> dict[Entry, int]: + """Baseline entries mapped to their line numbers; malformed input raises.""" + entries: dict[Entry, int] = {} + previous: Entry | None = None + for number, raw in enumerate(text.splitlines(), 1): + line = raw.strip() + if not line or line.startswith("#"): + continue + path, _, rule = line.rpartition(" ") + if rule not in RULES or not path or any(char in path for char in "*?["): + raise GuardError(f"{source}:{number}: expected ' ' with an enumerated path") + entry = (path, rule) + if previous is not None and entry <= previous: + raise GuardError(f"{source}:{number}: entries must be sorted and unique") + entries[entry] = number + previous = entry + return entries + + +def render_baseline(findings: Iterable[Finding]) -> str: + entries = sorted({(finding.path, finding.rule) for finding in findings}) + return BASELINE_HEADER + "\n" + "".join(f"{path} {rule}\n" for path, rule in entries) + + +# --- Scopes ------------------------------------------------------------------ + + +@dataclasses.dataclass(frozen=True) +class Scope: + """What one invocation checks: paths to scan, paths whose baseline entries + must still hold (None: all of them), and the revision the baseline may not + grow beyond (None: no growth check).""" + + paths: list[str] + touched: set[str] | None = None + base: str | None = None + + +def select_scope(tree: Tree, args: argparse.Namespace) -> Scope: + if not (args.diff or args.staged): + return Scope(tree.files) + if not tree.is_git: + raise GuardError(f"{tree.root} is not a git work tree; use --tree") + if args.staged: + touched = set(_git_lines(tree.root, "diff", "--cached", "--name-only", "--no-renames")) + base = None + else: + result = subprocess.run( + ["git", "-C", str(tree.root), "merge-base", args.diff, "HEAD"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode: + raise GuardError(f"unknown base {args.diff!r}: {result.stderr.strip()}") + base = result.stdout.strip() + touched = set(_git_lines(tree.root, "diff", "--name-only", "--no-renames", base)) + touched |= set(_git_lines(tree.root, "ls-files", "--others", "--exclude-standard")) + return Scope(sorted(path for path in touched if tree.is_scanned(path)), touched, base) + + +def committed_baseline(tree: Tree, revision: str) -> dict[Entry, int] | None: + """The baseline as committed at a revision; None before it existed.""" + result = subprocess.run( + ["git", "-C", str(tree.root), "show", f"{revision}:{BASELINE}"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + return parse_baseline(result.stdout, f"{revision}:{BASELINE}") if result.returncode == 0 else None + + +# --- Command line ------------------------------------------------------------ + + +def explain(rule_id: str) -> str: + if rule_id == "all": + return "\n".join(f"{rule.id} {rule.name}: {rule.detects}" for rule in RULES.values()) + rule = RULES.get(rule_id.upper()) + if rule is None: + raise GuardError(f"unknown rule {rule_id}; try --explain all") + return f"{rule.id} {rule.name}\n detects: {rule.detects}\n instead: {rule.instead}" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + scope = parser.add_mutually_exclusive_group() + scope.add_argument("--tree", action="store_true", help="scan the whole checkout (default)") + scope.add_argument("--diff", metavar="BASE", help="scan files changed since merge-base(BASE, HEAD)") + scope.add_argument("--staged", action="store_true", help="scan staged paths (pre-commit)") + parser.add_argument( + "--baseline", + choices=("check", "write", "ignore"), + default="check", + help="check findings against the baseline (default), rewrite it, or report every hit", + ) + parser.add_argument( + "--transition-layer", action="store_true", help="accept new baseline entries (PRs labelled transition-layer)" + ) + parser.add_argument("--explain", metavar="PGnnn", help="describe a rule ('all' lists them) and exit") + parser.add_argument("--root", type=Path, default=ROOT, help=argparse.SUPPRESS) + args = parser.parse_args(argv) + try: + if args.explain: + print(explain(args.explain)) + return 0 + return run(args) + except GuardError as error: + print(f"pattern-guard: {error}", file=sys.stderr) + return 2 + + +def run(args: argparse.Namespace) -> int: + tree = Tree(args.root) + baseline_path = tree.root / BASELINE + if args.baseline == "write" and (args.diff or args.staged): + raise GuardError("--baseline write needs the whole tree (--tree)") + scope = select_scope(tree, args) + findings = scan(tree, scope.paths) + + if args.baseline == "write": + baseline_path.parent.mkdir(parents=True, exist_ok=True) + baseline_path.write_text(render_baseline(findings), encoding="utf-8") + print(f"pattern-guard: wrote {len({(f.path, f.rule) for f in findings})} entries to {BASELINE}") + return 0 + if args.baseline == "ignore": + for finding in findings: + print(finding.render()) + print(f"pattern-guard: {len(findings)} finding(s)", file=sys.stderr) + return 1 if findings else 0 + + try: + baseline = parse_baseline(baseline_path.read_text(encoding="utf-8"), str(BASELINE)) + except FileNotFoundError: + baseline = {} + except OSError as error: + raise GuardError(f"cannot read {BASELINE}: {error}") from error + + new = [finding for finding in findings if (finding.path, finding.rule) not in baseline] + problems = [finding.render() for finding in new] + found = {(finding.path, finding.rule) for finding in findings} + for (path, rule), number in baseline.items(): + if (scope.touched is None or path in scope.touched) and (path, rule) not in found: + problems.append( + f"{BASELINE}:{number}: {rule} stale entry, {path} no longer matches\n" + " instead: delete the line (the baseline only shrinks)" + ) + before = committed_baseline(tree, scope.base) if scope.base and not args.transition_layer else None + if before is not None: + problems.extend( + f"{BASELINE}:{number}: {rule} new baseline entry for {path}\n" + " instead: fix the finding; only transition-layer PRs may add entries" + for (path, rule), number in baseline.items() + if (path, rule) not in before + ) + + for problem in problems: + print(problem) + print( + f"pattern-guard: {len(problems)} problem(s), {len(findings) - len(new)} baselined finding(s) tolerated", + file=sys.stderr, + ) + return 1 if problems else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prepare_legacy_character_fragment.py b/scripts/prepare_legacy_character_fragment.py index fee72d7c..1de264c0 100644 --- a/scripts/prepare_legacy_character_fragment.py +++ b/scripts/prepare_legacy_character_fragment.py @@ -5,55 +5,76 @@ remain. Only the selected, prepared actor's header ID may change before both the header loader and object lifetime store observe it. """ + import argparse import hashlib import json from pathlib import Path import struct + from compose_legacy_mod_policy import elf_functions, elf_sections from prepare_legacy_menu_fragment import ELFS, reviewed_revision def prepare(elf, revision): - if reviewed_revision(elf)!=revision: - raise ValueError('The ELF is not the reviewed character-adapter input') - symbols=elf_functions(elf,(1,2)) - words={base+i:struct.unpack_from('>I',data,i)[0] for base,data in elf_sections(elf) for i in range(0,len(data),4)} + if reviewed_revision(elf) != revision: + raise ValueError("The ELF is not the reviewed character-adapter input") + symbols = elf_functions(elf, (1, 2)) + words = { + base + i: struct.unpack_from(">I", data, i)[0] + for base, data in elf_sections(elf) + for i in range(0, len(data), 4) + } + def one(name): - entries=symbols.get(name,set()) - if len(entries)!=1:raise ValueError('Missing or ambiguous symbol '+name) + entries = symbols.get(name, set()) + if len(entries) != 1: + raise ValueError("Missing or ambiguous symbol " + name) return next(iter(entries)) - fields=[] - for name,size in [('gCharacterIdSlots',8)]: - address,actual=one(name) - if actual!=size:raise ValueError('Character field allocation changed') - fields.append({'name':name,'address':hex(address),'size':size}) - begin,size=one('spawn_object');loader=one('load_object_header')[0] - sites=[pc for pc in range(begin,begin+size-8,4) - if words[pc]==0xa7a4004e and words[pc+4]==0x0c000000|((loader>>2)&0x3ffffff)] - if len(sites)!=1:raise ValueError('Object header store/load boundary is ambiguous') + + fields = [] + for name, size in [("gCharacterIdSlots", 8)]: + address, actual = one(name) + if actual != size: + raise ValueError("Character field allocation changed") + fields.append({"name": name, "address": hex(address), "size": size}) + begin, size = one("spawn_object") + loader = one("load_object_header")[0] + sites = [ + pc + for pc in range(begin, begin + size - 8, 4) + if words[pc] == 0xA7A4004E and words[pc + 4] == 0x0C000000 | ((loader >> 2) & 0x3FFFFFF) + ] + if len(sites) != 1: + raise ValueError("Object header store/load boundary is ambiguous") # Both reviewed functions preserve the original entry pointer at caller-SP # and allocate 0x68 bytes. A hook cannot reinterpret a changed stack layout. - if words[begin]!=0x27bdff98 or 0xafa40068 not in [words[pc] for pc in range(begin,begin+32,4)]: - raise ValueError('Object entry stack lifetime changed') - definitions=[('spawn_object',sites[0],0),('charselect_assign_ai',one('charselect_assign_ai')[0],1), - ('menu_character_select_init',one('menu_character_select_init')[0],2)] - hooks=[];bodies={} - for name,pc,event in definitions: - start,length=one(name) - body=b''.join(struct.pack('>I',words[p]) for p in range(start,start+length,4)) - bodies[name]={'vram':hex(start),'size':length,'sha256':hashlib.sha256(body).hexdigest()} - hooks.append({'function':name,'vram':hex(pc),'event':event, - 'expected':[hex(words[pc+i*4]) for i in range(3)]}) - return {'schema':1,'revision':revision,'abi':1,'fields':fields,'functions':bodies,'hooks':hooks} - - -if __name__=='__main__': - p=argparse.ArgumentParser(description=__doc__) - p.add_argument('--elf',type=Path,required=True) - p.add_argument('--revision',choices=ELFS,required=True) - p.add_argument('--output',type=Path,required=True) - args=p.parse_args() - result=prepare(args.elf,args.revision) - with args.output.open('x',encoding='utf-8') as f:f.write(json.dumps(result,indent=2)+'\n') - print('Verified original header lifetime and three character hook boundaries.') + if words[begin] != 0x27BDFF98 or 0xAFA40068 not in [words[pc] for pc in range(begin, begin + 32, 4)]: + raise ValueError("Object entry stack lifetime changed") + definitions = [ + ("spawn_object", sites[0], 0), + ("charselect_assign_ai", one("charselect_assign_ai")[0], 1), + ("menu_character_select_init", one("menu_character_select_init")[0], 2), + ] + hooks = [] + bodies = {} + for name, pc, event in definitions: + start, length = one(name) + body = b"".join(struct.pack(">I", words[p]) for p in range(start, start + length, 4)) + bodies[name] = {"vram": hex(start), "size": length, "sha256": hashlib.sha256(body).hexdigest()} + hooks.append( + {"function": name, "vram": hex(pc), "event": event, "expected": [hex(words[pc + i * 4]) for i in range(3)]} + ) + return {"schema": 1, "revision": revision, "abi": 1, "fields": fields, "functions": bodies, "hooks": hooks} + + +if __name__ == "__main__": + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--elf", type=Path, required=True) + p.add_argument("--revision", choices=ELFS, required=True) + p.add_argument("--output", type=Path, required=True) + args = p.parse_args() + result = prepare(args.elf, args.revision) + with args.output.open("x", encoding="utf-8") as f: + f.write(json.dumps(result, indent=2) + "\n") + print("Verified original header lifetime and three character hook boundaries.") diff --git a/scripts/prepare_legacy_character_menu_fragment.py b/scripts/prepare_legacy_character_menu_fragment.py index 4d03a618..c2cbf09e 100644 --- a/scripts/prepare_legacy_character_menu_fragment.py +++ b/scripts/prepare_legacy_character_menu_fragment.py @@ -1,64 +1,122 @@ #!/usr/bin/env python3 """Author additive character-menu hooks from the pinned retail ELF only.""" + import argparse import hashlib import json -import struct from pathlib import Path +import struct + from compose_legacy_mod_policy import elf_functions, elf_sections from prepare_legacy_menu_fragment import ELFS, reviewed_revision -FIELDS=[('gActivePlayersArray',4),('gCharselectStatus',4),('gPlayersCharacterArray',8), - ('gCharacterIdSlots',8),('gMenuButtons',20),('gMenuStickX',10),('gMenuStickY',10), - ('gNumberOfReadyPlayers',4),('gNumberOfActivePlayers',4),('gMenuDelay',4), - ('sMenuCurrDisplayList',4),('gCurrCharacterSelectData',4),('dDialogueBoxBegin',56),('dDialogueBoxDrawModes',32),('gMenuSoundMasks',16), - ('gMenuCurrentCharacter',4),('gObjectCount',4),('gFreeListCount',4)] +FIELDS = [ + ("gActivePlayersArray", 4), + ("gCharselectStatus", 4), + ("gPlayersCharacterArray", 8), + ("gCharacterIdSlots", 8), + ("gMenuButtons", 20), + ("gMenuStickX", 10), + ("gMenuStickY", 10), + ("gNumberOfReadyPlayers", 4), + ("gNumberOfActivePlayers", 4), + ("gMenuDelay", 4), + ("sMenuCurrDisplayList", 4), + ("gCurrCharacterSelectData", 4), + ("dDialogueBoxBegin", 56), + ("dDialogueBoxDrawModes", 32), + ("gMenuSoundMasks", 16), + ("gMenuCurrentCharacter", 4), + ("gObjectCount", 4), + ("gFreeListCount", 4), +] + + +def prepare(elf, revision): + if reviewed_revision(elf) != revision: + raise ValueError("Unreviewed character-menu ELF") + symbols = elf_functions(elf, (1, 2)) + words = { + base + i: struct.unpack_from(">I", data, i)[0] + for base, data in elf_sections(elf) + for i in range(0, len(data), 4) + } -def prepare(elf,revision): - if reviewed_revision(elf)!=revision:raise ValueError('Unreviewed character-menu ELF') - symbols=elf_functions(elf,(1,2)) - words={base+i:struct.unpack_from('>I',data,i)[0] for base,data in elf_sections(elf) for i in range(0,len(data),4)} def one(name): - values=symbols.get(name,set()) - if len(values)!=1:raise ValueError('Missing/ambiguous character-menu symbol '+name) + values = symbols.get(name, set()) + if len(values) != 1: + raise ValueError("Missing/ambiguous character-menu symbol " + name) return next(iter(values)) - fields=[] - for name,size in FIELDS: - address,actual=one(name) - if actual!=size:raise ValueError(f'{name} has changed size: {actual}, expected {size}') - fields.append({'name':name,'address':hex(address),'size':size}) + + fields = [] + for name, size in FIELDS: + address, actual = one(name) + if actual != size: + raise ValueError(f"{name} has changed size: {actual}, expected {size}") + fields.append({"name": name, "address": hex(address), "size": size}) + def exit_pc(name): - begin,size=one(name);sites=[pc for pc in range(begin,begin+size,4) if words[pc]==0x03e00008] - if len(sites)!=1:raise ValueError('Ambiguous menu exit: '+name) + begin, size = one(name) + sites = [pc for pc in range(begin, begin + size, 4) if words[pc] == 0x03E00008] + if len(sites) != 1: + raise ValueError("Ambiguous menu exit: " + name) return sites[0] - begin,size=one('menu_character_select_loop');target=one('menu_input')[0] - sites=[pc+8 for pc in range(begin,begin+size-8,4) if words[pc]==(0x0c000000|((target>>2)&0x3ffffff))] - if len(sites)!=1:raise ValueError('Ambiguous collected-input boundary') - definitions=[('menu_character_select_init',exit_pc('menu_character_select_init')), - ('menu_character_select_loop',sites[0]),('charselect_render_text',exit_pc('charselect_render_text')), - ('charselect_free',one('charselect_free')[0]),('charselect_assign_ai',one('charselect_assign_ai')[0]), - ('charselect_move',one('charselect_move')[0]),('obj_loop_char_select',one('obj_loop_char_select')[0])] - begin,size=one('obj_loop_char_select');target=one('get_player_character')[0] - cursor_sites=[pc+8 for pc in range(begin,begin+size-8,4) if words[pc]==(0x0c000000|((target>>2)&0x3ffffff))] - if len(cursor_sites)!=1:raise ValueError('Ambiguous native stage cursor boundary') - definitions.append(('obj_loop_char_select',cursor_sites[0])) - definitions.append(('sound_play',one('sound_play')[0])) - hooks=[];bodies={} - for event,(name,pc) in enumerate(definitions): - start,length=one(name);at=min(pc,start+length-12) - body=b''.join(struct.pack('>I',words[p]) for p in range(start,start+length,4)) - bodies[name]={'vram':hex(start),'size':length,'sha256':hashlib.sha256(body).hexdigest()} - hooks.append({'function':name,'vram':hex(pc),'event':event,'expectedAt':hex(at), - 'expected':[hex(words[at+i*4]) for i in range(3)]}) - return {'schema':1,'revision':revision,'abi':3,'fields':fields,'functions':bodies,'hooks':hooks} + + begin, size = one("menu_character_select_loop") + target = one("menu_input")[0] + sites = [ + pc + 8 for pc in range(begin, begin + size - 8, 4) if words[pc] == (0x0C000000 | ((target >> 2) & 0x3FFFFFF)) + ] + if len(sites) != 1: + raise ValueError("Ambiguous collected-input boundary") + definitions = [ + ("menu_character_select_init", exit_pc("menu_character_select_init")), + ("menu_character_select_loop", sites[0]), + ("charselect_render_text", exit_pc("charselect_render_text")), + ("charselect_free", one("charselect_free")[0]), + ("charselect_assign_ai", one("charselect_assign_ai")[0]), + ("charselect_move", one("charselect_move")[0]), + ("obj_loop_char_select", one("obj_loop_char_select")[0]), + ] + begin, size = one("obj_loop_char_select") + target = one("get_player_character")[0] + cursor_sites = [ + pc + 8 for pc in range(begin, begin + size - 8, 4) if words[pc] == (0x0C000000 | ((target >> 2) & 0x3FFFFFF)) + ] + if len(cursor_sites) != 1: + raise ValueError("Ambiguous native stage cursor boundary") + definitions.append(("obj_loop_char_select", cursor_sites[0])) + definitions.append(("sound_play", one("sound_play")[0])) + hooks = [] + bodies = {} + for event, (name, pc) in enumerate(definitions): + start, length = one(name) + at = min(pc, start + length - 12) + body = b"".join(struct.pack(">I", words[p]) for p in range(start, start + length, 4)) + bodies[name] = {"vram": hex(start), "size": length, "sha256": hashlib.sha256(body).hexdigest()} + hooks.append( + { + "function": name, + "vram": hex(pc), + "event": event, + "expectedAt": hex(at), + "expected": [hex(words[at + i * 4]) for i in range(3)], + } + ) + return {"schema": 1, "revision": revision, "abi": 3, "fields": fields, "functions": bodies, "hooks": hooks} + def main(argv=None): - p=argparse.ArgumentParser(description=__doc__) - p.add_argument('--elf',type=Path,required=True);p.add_argument('--revision',choices=ELFS,required=True) - p.add_argument('--output',type=Path,required=True);a=p.parse_args(argv) - payload = json.dumps(prepare(a.elf,a.revision),indent=2)+'\n' - with a.output.open('x',encoding='utf-8') as f:f.write(payload) + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--elf", type=Path, required=True) + p.add_argument("--revision", choices=ELFS, required=True) + p.add_argument("--output", type=Path, required=True) + a = p.parse_args(argv) + payload = json.dumps(prepare(a.elf, a.revision), indent=2) + "\n" + with a.output.open("x", encoding="utf-8") as f: + f.write(payload) return 0 -if __name__=='__main__': + +if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/prepare_legacy_menu_fragment.py b/scripts/prepare_legacy_menu_fragment.py index 387aaf82..5785d466 100644 --- a/scripts/prepare_legacy_menu_fragment.py +++ b/scripts/prepare_legacy_menu_fragment.py @@ -4,14 +4,16 @@ This is an explicit adapter-authoring step, not automatic permission to patch an unknown binary. Only the reviewed ELF images are admitted; no generated C is edited. """ + import argparse import hashlib import json from pathlib import Path import struct -from compose_legacy_mod_policy import elf_functions, elf_sections, MENU_FIELDS +from compose_legacy_mod_policy import MENU_FIELDS, elf_functions, elf_sections from legacy_character_presentation_policy import ELFS, reviewed_revision + # ABI order is mirrored by legacy_track_menu_adapter.hpp. Every field comes # from an actual symbol, including its expected allocation size. FIELDS = MENU_FIELDS @@ -22,8 +24,11 @@ def prepare(elf: Path, revision: str): raise ValueError("The ELF differs from the reviewed menu adapter input") symbols = elf_functions(elf, (1, 2)) functions = elf_functions(elf) - words = {start + i: struct.unpack_from(">I", data, i)[0] - for start, data in elf_sections(elf) for i in range(0, len(data), 4)} + words = { + start + i: struct.unpack_from(">I", data, i)[0] + for start, data in elf_sections(elf) + for i in range(0, len(data), 4) + } def one(name): entries = symbols.get(name, set()) @@ -44,16 +49,23 @@ def hook(name, address, event, returns=False): at = min(address, begin + size - 12) if at < begin or at % 4 or not begin <= address < begin + size: raise ValueError(f"Hook outside function {name}") - hooks.append({"function": name, "vram": hex(address), "event": event, - "returns": returns, "expectedAt": hex(at), - "expected": [hex(words[at + i * 4]) for i in range(3)]}) + hooks.append( + { + "function": name, + "vram": hex(address), + "event": event, + "returns": returns, + "expectedAt": hex(at), + "expected": [hex(words[at + i * 4]) for i in range(3)], + } + ) if event == 7: # Both revisions set success in the JR delay slot. A before-JR # callback must not inspect the caller's previous v0 value. - delay = words[address+4] + delay = words[address + 4] if delay == 0x24020001: hooks[-1]["observedReturn"] = 1 - elif delay == 0 and words[address-4] == 0x00001025: + elif delay == 0 and words[address - 4] == 0x00001025: hooks[-1]["observedReturn"] = 0 else: raise ValueError("Unreviewed background-loader return sequence") @@ -63,23 +75,25 @@ def entry(name, event, returns=False): def exits(name, event): begin, size = one(name) - sites = [pc for pc in range(begin, begin + size, 4) if words[pc] == 0x03e00008] + sites = [pc for pc in range(begin, begin + size, 4) if words[pc] == 0x03E00008] if not sites: raise ValueError(f"Missing bounded return in {name}") for pc in sites: hook(name, pc, event) - entry("menu_track_select_init", 0); exits("menu_track_select_init", 1) - entry("trackmenu_input", 2); exits("trackmenu_input", 3) + entry("menu_track_select_init", 0) + exits("menu_track_select_init", 1) + entry("trackmenu_input", 2) + exits("trackmenu_input", 3) begin, size = one("trackmenu_render_names") target = one("camDisableUserView")[0] - sites = [pc for pc in range(begin, begin + size, 4) - if words[pc] == 0x0c000000 | ((target >> 2) & 0x03ffffff)] + sites = [pc for pc in range(begin, begin + size, 4) if words[pc] == 0x0C000000 | ((target >> 2) & 0x03FFFFFF)] if len(sites) != 1: raise ValueError("The render-details/draw boundary is ambiguous") hook("trackmenu_render_names", sites[0], 4) entry("trackmenu_track_view", 5) - entry("bgload_start", 6); exits("bgload_start", 7) + entry("bgload_start", 6) + exits("bgload_start", 7) entry("load_level_for_menu", 8) entry("level_name", 9, True) entry("leveltable_vehicle_default", 10, True) @@ -88,7 +102,8 @@ def exits(name, event): entry("func_80092188", 13) entry("load_level_game", 14) entry("menu_init", 15) - entry("func_8008F618", 16); exits("func_8008F618", 17) + entry("func_8008F618", 16) + exits("func_8008F618", 17) bodies = {} for name in {h["function"] for h in hooks}: begin, size = one(name) @@ -107,5 +122,5 @@ def exits(name, event): if args.output.exists() and not args.replace: raise ValueError("Refusing to overwrite an existing adapter fragment") result = prepare(args.elf, args.revision) - args.output.write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") + args.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") print(f"Verified {len(result['fields'])} data symbols and {len(result['hooks'])} bounded menu hook sites.") diff --git a/scripts/qualify-updater-arm.sh b/scripts/qualify-updater-arm.sh index eff39843..70da422e 100755 --- a/scripts/qualify-updater-arm.sh +++ b/scripts/qualify-updater-arm.sh @@ -5,7 +5,7 @@ task="${DKR_UPDATER_TASK_ROOT:?Set the qualification log directory}" build="${DKR_LINUX_BUILD_DIR:?Set the configured ARM64 build directory}" pipeline="${DKR_UPDATER_PIPELINE:?Set the reviewed patch-pipeline directory}" sysroot="${DKR_ARM64_SYSROOT:?Set the ARM64 sysroot}" -version="${DKR_RELEASE_VERSION:-$(tr -d '\r\n' < "$project_root/VERSION")}" +version="${DKR_RELEASE_VERSION:-$(tr -d '\r\n' <"$project_root/VERSION")}" mkdir -p "$task" export QEMU_LD_PREFIX="$sysroot" PKG_CONFIG_SYSROOT_DIR="$sysroot" export PKG_CONFIG_LIBDIR="$sysroot/usr/lib/aarch64-linux-gnu/pkgconfig:$sysroot/usr/share/pkgconfig" @@ -22,7 +22,7 @@ cmake -S "$project_root/runtime-recomp" -B "$build" -G Ninja \ -DDKR_EXPERIMENTAL_OWNED_GENERATED="$pipeline/owned-v77" -DDKR_EXPERIMENTAL_OWNED_GENERATED_V80="$pipeline/owned-v80" \ -DDKR_EXPERIMENTAL_RACE_TEST=ON -DDKR_LEGACY_QUALIFICATION=OFF -DDKR_REPLAY_QUALIFICATION=OFF \ -DDKR_ANDROID_RENDER_QUALIFICATION=OFF -DDKR_EXPERIMENTAL_RENDER_QUALIFICATION=OFF \ - -DDKR_WATER_QUALIFICATION=OFF -DDKR_TASK_QUALIFICATION=OFF > "$task/ARM-configure.log" 2>&1 + -DDKR_WATER_QUALIFICATION=OFF -DDKR_TASK_QUALIFICATION=OFF >"$task/ARM-configure.log" 2>&1 cmake --build "$build" --target DKRPortGame DKRUpdateHelper DKRUpdateCoreTests DKRUpdateServiceTests DKRUpdateProcessTests \ - --parallel 2 > "$task/ARM-build.log" 2>&1 -ctest --test-dir "$build" -R DKRUpdate --output-on-failure > "$task/ARM-update-tests.log" 2>&1 + --parallel 2 >"$task/ARM-build.log" 2>&1 +ctest --test-dir "$build" -R DKRUpdate --output-on-failure >"$task/ARM-update-tests.log" 2>&1 diff --git a/scripts/qualify-updater-linux.sh b/scripts/qualify-updater-linux.sh index 14d7df1e..b3e555b3 100755 --- a/scripts/qualify-updater-linux.sh +++ b/scripts/qualify-updater-linux.sh @@ -4,7 +4,7 @@ project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" task="${DKR_UPDATER_TASK_ROOT:?Set the qualification log directory}" pipeline="${DKR_UPDATER_PIPELINE:?Set the reviewed patch-pipeline directory}" build="${DKR_LINUX_BUILD_DIR:?Set the configured native Linux build directory}" -version="${DKR_RELEASE_VERSION:-$(tr -d '\r\n' < "$project_root/VERSION")}" +version="${DKR_RELEASE_VERSION:-$(tr -d '\r\n' <"$project_root/VERSION")}" mkdir -p "$task" export DKR_REVIEWED_GENERATED_V77="$pipeline/generated-v77" export DKR_REVIEWED_GENERATED_V80="$pipeline/generated-v80" @@ -13,7 +13,7 @@ cmake -S "$project_root/runtime-recomp" -B "$build" -DDKR_RELEASE_VERSION="$vers -DDKR_EXPERIMENTAL_OWNED_GENERATED="$pipeline/owned-v77" -DDKR_EXPERIMENTAL_OWNED_GENERATED_V80="$pipeline/owned-v80" \ -DDKR_EXPERIMENTAL_RACE_TEST=ON -DDKR_REPLAY_QUALIFICATION=OFF -DDKR_LEGACY_QUALIFICATION=OFF \ -DDKR_ANDROID_RENDER_QUALIFICATION=OFF -DDKR_EXPERIMENTAL_RENDER_QUALIFICATION=OFF \ - -DDKR_WATER_QUALIFICATION=OFF -DDKR_TASK_QUALIFICATION=OFF > "$task/Linux-configure.log" 2>&1 + -DDKR_WATER_QUALIFICATION=OFF -DDKR_TASK_QUALIFICATION=OFF >"$task/Linux-configure.log" 2>&1 cmake --build "$build" --target DKRPortGame DKRLegacyModWorker DKRUpdateHelper DKRUpdateCoreTests DKRUpdateServiceTests DKRUpdateProcessTests \ - --parallel 2 > "$task/Linux-build.log" 2>&1 -ctest --test-dir "$build" -R DKRUpdate --output-on-failure > "$task/Linux-update-tests.log" 2>&1 + --parallel 2 >"$task/Linux-build.log" 2>&1 +ctest --test-dir "$build" -R DKRUpdate --output-on-failure >"$task/Linux-update-tests.log" 2>&1 diff --git a/scripts/regression.py b/scripts/regression.py index b91633f6..2d3e3dae 100644 --- a/scripts/regression.py +++ b/scripts/regression.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Repeatable DKR regression checks and same-machine performance comparisons.""" + from __future__ import annotations import argparse @@ -12,8 +13,8 @@ from pathlib import Path import platform import re -import signal import shutil +import signal import statistics import subprocess import sys @@ -29,9 +30,9 @@ def digest(path): - with Path(path).open('rb') as source: + with Path(path).open("rb") as source: value = hashlib.sha256() - for block in iter(lambda: source.read(1024 * 1024), b''): + for block in iter(lambda: source.read(1024 * 1024), b""): value.update(block) return value.hexdigest() @@ -42,54 +43,69 @@ def canonical(value): def number(value, label, minimum=0): if isinstance(value, bool) or not isinstance(value, (float, int)) or not math.isfinite(value) or value < minimum: - raise ValueError(f'{label} must be a finite number >= {minimum}') + raise ValueError(f"{label} must be a finite number >= {minimum}") return value def load_suite(path): suite = json.loads(path.read_text()) - if not isinstance(suite, dict) or suite.get('schema_version') != 1: - raise ValueError('Suite requires schema_version 1') - if set(suite) - {'schema_version', 'name', 'scenarios'}: - raise ValueError('Unknown suite field') - if not isinstance(suite.get('name'), str) or not suite['name']: - raise ValueError('Suite requires a name') - scenarios = suite.get('scenarios') + if not isinstance(suite, dict) or suite.get("schema_version") != 1: + raise ValueError("Suite requires schema_version 1") + if set(suite) - {"schema_version", "name", "scenarios"}: + raise ValueError("Unknown suite field") + if not isinstance(suite.get("name"), str) or not suite["name"]: + raise ValueError("Suite requires a name") + scenarios = suite.get("scenarios") if not isinstance(scenarios, list) or not scenarios: - raise ValueError('Suite requires nonempty scenarios') + raise ValueError("Suite requires nonempty scenarios") seen = set() - allowed = {'id', 'description', 'command', 'ctest', 'env', 'inputs', 'expect_stdout', - 'timeout_seconds', 'budgets', 'required_metrics', 'fingerprints'} + allowed = { + "id", + "description", + "command", + "ctest", + "env", + "inputs", + "expect_stdout", + "timeout_seconds", + "budgets", + "required_metrics", + "fingerprints", + } for item in scenarios: if not isinstance(item, dict) or set(item) - allowed: - raise ValueError('Unknown scenario field or invalid scenario') - name = item.get('id', '') - if not isinstance(name, str) or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', name) or name in seen: - raise ValueError(f'Invalid or duplicate scenario id: {name}') + raise ValueError("Unknown scenario field or invalid scenario") + name = item.get("id", "") + if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", name) or name in seen: + raise ValueError(f"Invalid or duplicate scenario id: {name}") seen.add(name) - if not isinstance(item.get('description', ''), str): - raise ValueError(f'{name}: description must be a string') - if ('command' in item) == ('ctest' in item): - raise ValueError(f'{name}: provide exactly one of command or ctest') - if 'ctest' in item and (not isinstance(item['ctest'], str) or not item['ctest']): - raise ValueError(f'{name}: ctest must name exactly one test') - for key in ('command', 'inputs', 'expect_stdout', 'required_metrics', 'fingerprints'): - if key in item and (not isinstance(item[key], list) or - not all(isinstance(x, str) and x and '\0' not in x for x in item[key])): - raise ValueError(f'{name}: {key} must be a string array') - if 'command' in item and not item['command']: - raise ValueError(f'{name}: empty command') - env = item.get('env', {}) - if not isinstance(env, dict) or not all(isinstance(k, str) and re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', k) - and isinstance(v, str) and '\0' not in v for k, v in env.items()): - raise ValueError(f'{name}: env must map strings to strings') - for pattern in item.get('fingerprints', []): + if not isinstance(item.get("description", ""), str): + raise ValueError(f"{name}: description must be a string") + if ("command" in item) == ("ctest" in item): + raise ValueError(f"{name}: provide exactly one of command or ctest") + if "ctest" in item and (not isinstance(item["ctest"], str) or not item["ctest"]): + raise ValueError(f"{name}: ctest must name exactly one test") + for key in ("command", "inputs", "expect_stdout", "required_metrics", "fingerprints"): + if key in item and ( + not isinstance(item[key], list) + or not all(isinstance(x, str) and x and "\0" not in x for x in item[key]) + ): + raise ValueError(f"{name}: {key} must be a string array") + if "command" in item and not item["command"]: + raise ValueError(f"{name}: empty command") + env = item.get("env", {}) + if not isinstance(env, dict) or not all( + isinstance(k, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", k) and isinstance(v, str) and "\0" not in v + for k, v in env.items() + ): + raise ValueError(f"{name}: env must map strings to strings") + for pattern in item.get("fingerprints", []): re.compile(pattern) - number(item.get('timeout_seconds', 180), f'{name}: timeout_seconds', 0.01) - if not isinstance(item.get('budgets', {}), dict): - raise ValueError(f'{name}: budgets must map metric names to maxima') - for key, value in item.get('budgets', {}).items(): - number(value, f'{name}: budget {key}') + number(item.get("timeout_seconds", 180), f"{name}: timeout_seconds", 0.01) + if not isinstance(item.get("budgets", {}), dict): + raise ValueError(f"{name}: budgets must map metric names to maxima") + for key, value in item.get("budgets", {}).items(): + number(value, f"{name}: budget {key}") return suite @@ -98,52 +114,57 @@ def expand(value, variables): def load_variables(build_dir, vars_file, assignments): - variables = {'root': str(ROOT), 'python': sys.executable, 'build': str(build_dir)} + variables = {"root": str(ROOT), "python": sys.executable, "build": str(build_dir)} reserved = set(variables) def validate(key, value, source): - if not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]*', key) or key in reserved: - raise ValueError(f'{source}: invalid or reserved variable name: {key}') - if not isinstance(value, str) or not value or '\0' in value: - raise ValueError(f'{source}: variable {key} requires a nonempty string without NUL characters') + if not re.fullmatch(r"[A-Za-z][A-Za-z0-9_]*", key) or key in reserved: + raise ValueError(f"{source}: invalid or reserved variable name: {key}") + if not isinstance(value, str) or not value or "\0" in value: + raise ValueError(f"{source}: variable {key} requires a nonempty string without NUL characters") if vars_file: + def unique_object(pairs): values = {} for key, value in pairs: if key in values: - raise ValueError(f'{vars_file}: duplicate variable: {key}') + raise ValueError(f"{vars_file}: duplicate variable: {key}") values[key] = value return values - values = json.loads(vars_file.read_text(encoding='utf-8'), object_pairs_hook=unique_object) + values = json.loads(vars_file.read_text(encoding="utf-8"), object_pairs_hook=unique_object) if not isinstance(values, dict): - raise ValueError(f'{vars_file}: variables file must be a JSON object of path strings') + raise ValueError(f"{vars_file}: variables file must be a JSON object of path strings") for key, value in values.items(): validate(key, value, str(vars_file)) variables[key] = str((vars_file.resolve().parent / value).resolve()) seen = set() for assignment in assignments: - key, separator, value = assignment.partition('=') + key, separator, value = assignment.partition("=") if not separator: - raise ValueError(f'--var requires NAME=VALUE: {assignment}') - validate(key, value, '--var') + raise ValueError(f"--var requires NAME=VALUE: {assignment}") + validate(key, value, "--var") if key in seen: - raise ValueError(f'--var: duplicate variable: {key}') + raise ValueError(f"--var: duplicate variable: {key}") seen.add(key) variables[key] = value return variables def stop_process(process): - if os.name == 'posix': + if os.name == "posix": try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass elif process.poll() is None: - subprocess.run(['taskkill', '/PID', str(process.pid), '/T', '/F'], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10) + subprocess.run( + ["taskkill", "/PID", str(process.pid), "/T", "/F"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + ) process.kill() process.wait() @@ -153,26 +174,35 @@ def execute(command, cwd, env, timeout, stdout, stderr): error = None interrupted = False stopped = False - with stdout.open('wb') as out, stderr.open('wb') as err: + with stdout.open("wb") as out, stderr.open("wb") as err: try: - process = subprocess.Popen(command, cwd=cwd, env=env, stdout=out, stderr=err, - stdin=subprocess.DEVNULL, start_new_session=os.name == 'posix') + process = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdout=out, + stderr=err, + stdin=subprocess.DEVNULL, + start_new_session=os.name == "posix", + ) except OSError as failure: - return None, 0.0, f'Could not launch: {failure}', False + return None, 0.0, f"Could not launch: {failure}", False finished = threading.Event() + def watchdog(): nonlocal error, stopped while not finished.wait(0.02): if process.poll() is not None: return if time.perf_counter() - started > timeout: - error = f'Timed out after {timeout:g} seconds' + error = f"Timed out after {timeout:g} seconds" elif stdout.stat().st_size + stderr.stat().st_size > LOG_LIMIT: - error = 'Output exceeded the 32 MiB per-attempt limit' + error = "Output exceeded the 32 MiB per-attempt limit" if error: stop_process(process) stopped = True return + watcher = threading.Thread(target=watchdog, daemon=True) watcher.start() try: @@ -180,7 +210,7 @@ def watchdog(): # to a polling interval. A separate watchdog bounds time/log size. process.wait() except KeyboardInterrupt: - error, interrupted = 'Interrupted by user', True + error, interrupted = "Interrupted by user", True finally: elapsed = (time.perf_counter() - started) * 1000 finished.set() @@ -194,102 +224,115 @@ def watchdog(): # observed deadline independently of its polling cadence, without adding # polling latency to successful process measurements. if error is None and elapsed > timeout * 1000: - error = f'Timed out after {timeout:g} seconds' + error = f"Timed out after {timeout:g} seconds" if stdout.stat().st_size + stderr.stat().st_size > LOG_LIMIT: - error = 'Output exceeded the 32 MiB per-attempt limit' + error = "Output exceeded the 32 MiB per-attempt limit" return process.returncode, elapsed, error, interrupted def read_log(path): - with path.open('rb') as source: - return source.read(LOG_LIMIT).decode('utf-8', errors='replace').replace('\r\n', '\n') + with path.open("rb") as source: + return source.read(LOG_LIMIT).decode("utf-8", errors="replace").replace("\r\n", "\n") def extract_metrics(text): """Keep histogram bounds and window maxima distinct from exact percentiles.""" metrics = {} for line in text.splitlines(): - line = re.sub(r'^\d+: ', '', line) # CTest -V prefixes child output. - if line.startswith('[rollback][timing] '): - fields = dict(re.findall(r'([\w-]+)=([^\s]+)', line)) - stage = fields.get('stage') + line = re.sub(r"^\d+: ", "", line) # CTest -V prefixes child output. + if line.startswith("[rollback][timing] "): + fields = dict(re.findall(r"([\w-]+)=([^\s]+)", line)) + stage = fields.get("stage") if not stage: - raise ValueError('Timing record has no stage') - for field in ('count', 'mean-us', 'p95-upper-us', 'p99-upper-us', 'max-us'): - value = number(float(fields[field]), f'{stage}.{field}') + raise ValueError("Timing record has no stage") + for field in ("count", "mean-us", "p95-upper-us", "p99-upper-us", "max-us"): + value = number(float(fields[field]), f"{stage}.{field}") # UINT64_MAX is the unbounded histogram bucket, not a duration. - if field.endswith('upper-us') and value >= 2**64 - 1: + if field.endswith("upper-us") and value >= 2**64 - 1: continue - suffix = field.replace('-', '_') - if field.endswith('-us'): - suffix, value = suffix[:-2] + 'ms', value / 1000 - metrics[f'rollback.{stage}.{suffix}'] = value - elif line.startswith('DKR_HARNESS '): - event = json.loads(line[len('DKR_HARNESS '):]) - if not isinstance(event, dict) or set(event) != {'metrics'} or not isinstance(event['metrics'], dict): - raise ValueError('Invalid DKR_HARNESS metrics record') - for key, value in event['metrics'].items(): - if not re.fullmatch(r'[A-Za-z][A-Za-z0-9_.-]*', key) or key in metrics or key.startswith('process.'): - raise ValueError(f'Duplicate, reserved or invalid metric: {key}') + suffix = field.replace("-", "_") + if field.endswith("-us"): + suffix, value = suffix[:-2] + "ms", value / 1000 + metrics[f"rollback.{stage}.{suffix}"] = value + elif line.startswith("DKR_HARNESS "): + event = json.loads(line[len("DKR_HARNESS ") :]) + if not isinstance(event, dict) or set(event) != {"metrics"} or not isinstance(event["metrics"], dict): + raise ValueError("Invalid DKR_HARNESS metrics record") + for key, value in event["metrics"].items(): + if not re.fullmatch(r"[A-Za-z][A-Za-z0-9_.-]*", key) or key in metrics or key.startswith("process."): + raise ValueError(f"Duplicate, reserved or invalid metric: {key}") metrics[key] = number(value, key) - elif '[perf][present-interval]' in line: - match = re.search(r'ms\(p50/p95/p99/max\)=([\d.]+)/([\d.]+)/([\d.]+)/([\d.]+)', line) + elif "[perf][present-interval]" in line: + match = re.search(r"ms\(p50/p95/p99/max\)=([\d.]+)/([\d.]+)/([\d.]+)/([\d.]+)", line) if not match: - raise ValueError('Malformed present-interval record') - for label, raw in zip(('p50', 'p95', 'p99', 'max'), match.groups()): - key = f'present.window_{label}_max_ms' + raise ValueError("Malformed present-interval record") + for label, raw in zip(("p50", "p95", "p99", "max"), match.groups()): + key = f"present.window_{label}_max_ms" metrics[key] = max(metrics.get(key, 0), number(float(raw), key)) return metrics def ctest_status(path): if not path.is_file(): - raise ValueError('CTest did not produce its result XML') - cases = list(ET.parse(path).getroot().iter('testcase')) + raise ValueError("CTest did not produce its result XML") + cases = list(ET.parse(path).getroot().iter("testcase")) if len(cases) != 1: - raise ValueError(f'Expected one CTest result, got {len(cases)}') + raise ValueError(f"Expected one CTest result, got {len(cases)}") case = cases[0] - if case.find('skipped') is not None or case.get('status') in ('notrun', 'disabled'): - return 'skipped' - if case.find('failure') is not None or case.find('error') is not None: - return 'failed' - return 'passed' + if case.find("skipped") is not None or case.get("status") in ("notrun", "disabled"): + return "skipped" + if case.find("failure") is not None or case.find("error") is not None: + return "failed" + return "passed" def run_scenario(item, variables, output, args): - result = {'id': item['id'], 'description': item.get('description', ''), 'status': 'passed', - 'errors': [], 'attempts': [], 'metrics': {}, 'comparisons': [], 'fingerprints': {}} + result = { + "id": item["id"], + "description": item.get("description", ""), + "status": "passed", + "errors": [], + "attempts": [], + "metrics": {}, + "comparisons": [], + "fingerprints": {}, + } try: - command = [expand(x, variables) for x in item.get('command', [])] - overrides = {k: expand(v, variables) for k, v in item.get('env', {}).items()} - inputs = [Path(expand(x, variables)).resolve() for x in item.get('inputs', [])] + command = [expand(x, variables) for x in item.get("command", [])] + overrides = {k: expand(v, variables) for k, v in item.get("env", {}).items()} + inputs = [Path(expand(x, variables)).resolve() for x in item.get("inputs", [])] except KeyError as missing: - result.update(status='skipped', errors=[f'Missing variable {missing.args[0]} (set --var {missing.args[0]}=VALUE or --vars-file)']) + result.update( + status="skipped", + errors=[f"Missing variable {missing.args[0]} (set --var {missing.args[0]}=VALUE or --vars-file)"], + ) return result, False absent = [str(path) for path in inputs if not path.is_file()] if absent: - result.update(status='skipped', errors=['Missing input: ' + ', '.join(absent)]) + result.update(status="skipped", errors=["Missing input: " + ", ".join(absent)]) return result, False - ctest_entry = args.ctest_tests.get(item.get('ctest')) if 'ctest' in item else None - if 'ctest' in item and ctest_entry is None: - result.update(status='skipped', errors=[f"CTest test not configured: {item['ctest']}"]) + ctest_entry = args.ctest_tests.get(item.get("ctest")) if "ctest" in item else None + if "ctest" in item and ctest_entry is None: + result.update(status="skipped", errors=[f"CTest test not configured: {item['ctest']}"]) return result, False if command: executable = shutil.which(command[0]) if not executable: - result.update(status='skipped', errors=[f'Missing executable: {command[0]}']) + result.update(status="skipped", errors=[f"Missing executable: {command[0]}"]) return result, False command[0] = str(Path(executable).resolve()) - result['input_sha256'] = [digest(path) for path in inputs] + result["input_sha256"] = [digest(path) for path in inputs] # Bind substituted workload parameters, while allowing a rebuilt executable # and identical private inputs to move between build directories. - aliases = {str(path): 'sha256:' + sha for path, sha in zip(inputs, result['input_sha256'])} + aliases = {str(path): "sha256:" + sha for path, sha in zip(inputs, result["input_sha256"])} + def semantic(value): for path, alias in sorted(aliases.items(), key=lambda x: -len(x[0])): - value = value.replace(path, alias).replace(path.replace('\\', '/'), alias) - for path, alias in ((args.build_dir, '${build}'), (ROOT, '${root}')): + value = value.replace(path, alias).replace(path.replace("\\", "/"), alias) + for path, alias in ((args.build_dir, "${build}"), (ROOT, "${root}")): value = value.replace(str(path), alias).replace(path.as_posix(), alias) return value + def normalize(value): if isinstance(value, str): return semantic(value) @@ -298,229 +341,297 @@ def normalize(value): if isinstance(value, dict): return {k: normalize(v) for k, v in value.items()} return value - contract = {'scenario': item, 'inputs': result['input_sha256'], - 'arguments': [semantic(x) for x in command[1:]], - 'environment': {k: semantic(v) for k, v in overrides.items()}} + + contract = { + "scenario": item, + "inputs": result["input_sha256"], + "arguments": [semantic(x) for x in command[1:]], + "environment": {k: semantic(v) for k, v in overrides.items()}, + } if ctest_entry: - contract['ctest'] = {k: v for k, v in ctest_entry.items() if k in ('command', 'properties')} - contract['ctest'] = normalize(contract['ctest']) - result['ctest_definition'] = ctest_entry - result['contract'] = canonical(contract) - result['command'] = command or ['ctest', item['ctest']] - result['environment'] = overrides - result['input_paths'] = [str(path) for path in inputs] - executable = command[0] if command else (ctest_entry.get('command') or [''])[0] + contract["ctest"] = {k: v for k, v in ctest_entry.items() if k in ("command", "properties")} + contract["ctest"] = normalize(contract["ctest"]) + result["ctest_definition"] = ctest_entry + result["contract"] = canonical(contract) + result["command"] = command or ["ctest", item["ctest"]] + result["environment"] = overrides + result["input_paths"] = [str(path) for path in inputs] + executable = command[0] if command else (ctest_entry.get("command") or [""])[0] if Path(executable).is_file(): - result['executable_sha256'] = digest(executable) + result["executable_sha256"] = digest(executable) # An unrelated local debug switch must not silently change the workload or # make a probe dump private checkpoint bytes. Suites declare their switches. - env = {k: v for k, v in os.environ.items() if not k.startswith('DKR_')} + env = {k: v for k, v in os.environ.items() if not k.startswith("DKR_")} env.update(overrides) - directory = output / item['id'] + directory = output / item["id"] directory.mkdir() measured = [] interrupted = False for index in range(args.warmup + args.repeat): - phase = 'warmup' if index < args.warmup else 'measured' - prefix = f'{index + 1:02d}-{phase}' - stdout, stderr = directory / f'{prefix}.stdout.log', directory / f'{prefix}.stderr.log' - xml = directory / f'{prefix}.ctest.xml' - actual = command or ['ctest', '--test-dir', str(args.build_dir), '-C', args.config, - '-R', '^' + re.escape(item['ctest']) + '$', '--no-tests=error', - '-V', '--output-junit', str(xml)] - code, elapsed, error, interrupted = execute(actual, directory, env, - item.get('timeout_seconds', 180), stdout, stderr) - attempt = {'index': index + 1, 'phase': phase, 'elapsed_ms': elapsed, 'exit_code': code, - 'stdout': stdout.relative_to(output).as_posix(), 'stderr': stderr.relative_to(output).as_posix(), - 'status': 'passed', 'errors': [], 'command': actual} - result['attempts'].append(attempt) + phase = "warmup" if index < args.warmup else "measured" + prefix = f"{index + 1:02d}-{phase}" + stdout, stderr = directory / f"{prefix}.stdout.log", directory / f"{prefix}.stderr.log" + xml = directory / f"{prefix}.ctest.xml" + actual = command or [ + "ctest", + "--test-dir", + str(args.build_dir), + "-C", + args.config, + "-R", + "^" + re.escape(item["ctest"]) + "$", + "--no-tests=error", + "-V", + "--output-junit", + str(xml), + ] + code, elapsed, error, interrupted = execute( + actual, directory, env, item.get("timeout_seconds", 180), stdout, stderr + ) + attempt = { + "index": index + 1, + "phase": phase, + "elapsed_ms": elapsed, + "exit_code": code, + "stdout": stdout.relative_to(output).as_posix(), + "stderr": stderr.relative_to(output).as_posix(), + "status": "passed", + "errors": [], + "command": actual, + } + result["attempts"].append(attempt) if error: - attempt['errors'].append(error) + attempt["errors"].append(error) if code != 0: - attempt['errors'].append(f'Process exit: {code}') - text = read_log(stdout) + '\n' + read_log(stderr) + attempt["errors"].append(f"Process exit: {code}") + text = read_log(stdout) + "\n" + read_log(stderr) try: - if not command and not attempt['errors']: - attempt['status'] = ctest_status(xml) - if attempt['status'] == 'failed': - attempt['errors'].append('CTest reported a failure') - elif attempt['status'] == 'skipped': - result.update(status='skipped', errors=['CTest skipped the scenario']) + if not command and not attempt["errors"]: + attempt["status"] = ctest_status(xml) + if attempt["status"] == "failed": + attempt["errors"].append("CTest reported a failure") + elif attempt["status"] == "skipped": + result.update(status="skipped", errors=["CTest skipped the scenario"]) break - for expected in item.get('expect_stdout', []): + for expected in item.get("expect_stdout", []): if expected not in read_log(stdout): - attempt['errors'].append(f'Missing success marker: {expected}') - attempt['metrics'] = extract_metrics(text) - for metric in item.get('required_metrics', []): - if metric not in attempt['metrics']: - attempt['errors'].append(f'Missing required metric: {metric}') + attempt["errors"].append(f"Missing success marker: {expected}") + attempt["metrics"] = extract_metrics(text) + for metric in item.get("required_metrics", []): + if metric not in attempt["metrics"]: + attempt["errors"].append(f"Missing required metric: {metric}") fingerprints = {} - for pattern in item.get('fingerprints', []): + for pattern in item.get("fingerprints", []): matches = [match.group(0) for match in re.finditer(pattern, text, re.MULTILINE)] if not matches: - attempt['errors'].append(f'Missing fingerprint: {pattern}') + attempt["errors"].append(f"Missing fingerprint: {pattern}") fingerprints[pattern] = matches - attempt['fingerprints'] = fingerprints - if index and fingerprints != result['fingerprints']: - attempt['errors'].append('Deterministic fingerprints changed between attempts') - result['fingerprints'] = fingerprints + attempt["fingerprints"] = fingerprints + if index and fingerprints != result["fingerprints"]: + attempt["errors"].append("Deterministic fingerprints changed between attempts") + result["fingerprints"] = fingerprints except (ValueError, KeyError, TypeError, ET.ParseError) as failure: - attempt['errors'].append(f'Invalid measurement: {failure}') - if attempt['errors']: - attempt['status'] = 'failed' - if attempt['status'] != 'passed': - result['status'] = attempt['status'] - result['errors'].extend(attempt['errors'] or ['CTest skipped the scenario']) + attempt["errors"].append(f"Invalid measurement: {failure}") + if attempt["errors"]: + attempt["status"] = "failed" + if attempt["status"] != "passed": + result["status"] = attempt["status"] + result["errors"].extend(attempt["errors"] or ["CTest skipped the scenario"]) break - if phase == 'measured': + if phase == "measured": measured.append(attempt) - if measured and result['status'] == 'passed': - elapsed = sorted(x['elapsed_ms'] for x in measured) - result['metrics'] = {'process.median_ms': statistics.median(elapsed), - 'process.p95_ms': elapsed[math.ceil(len(elapsed) * .95) - 1], - 'process.max_ms': max(elapsed)} - keys = set().union(*(x['metrics'] for x in measured)) + if measured and result["status"] == "passed": + elapsed = sorted(x["elapsed_ms"] for x in measured) + result["metrics"] = { + "process.median_ms": statistics.median(elapsed), + "process.p95_ms": elapsed[math.ceil(len(elapsed) * 0.95) - 1], + "process.max_ms": max(elapsed), + } + keys = set().union(*(x["metrics"] for x in measured)) for key in keys: - if not all(key in x['metrics'] for x in measured): - result['errors'].append(f'Metric missing in some measured attempts: {key}') + if not all(key in x["metrics"] for x in measured): + result["errors"].append(f"Metric missing in some measured attempts: {key}") else: - result['metrics'][key] = statistics.median(x['metrics'][key] for x in measured) - for key, limit in item.get('budgets', {}).items(): - value = result['metrics'].get(key) + result["metrics"][key] = statistics.median(x["metrics"][key] for x in measured) + for key, limit in item.get("budgets", {}).items(): + value = result["metrics"].get(key) if value is None or value > limit: - result['errors'].append(f'Budget exceeded or unavailable: {key}={value}, maximum {limit}') - if result['errors']: - result['status'] = 'failed' + result["errors"].append(f"Budget exceeded or unavailable: {key}={value}, maximum {limit}") + if result["errors"]: + result["status"] = "failed" return result, interrupted def compare(result, baseline, percent, absolute, compare_process=False): - if result['status'] != 'passed': + if result["status"] != "passed": return - if not baseline or baseline.get('status') != 'passed' or baseline.get('contract') != result['contract']: - result['errors'].append('No compatible passing baseline for this scenario') - elif baseline.get('fingerprints', {}) != result['fingerprints']: - result['fingerprint_difference'] = {'expected': baseline.get('fingerprints', {}), - 'actual': result['fingerprints']} - result['errors'].append('Deterministic fingerprint differs from baseline: ' + - json.dumps(result['fingerprint_difference'], sort_keys=True)) + if not baseline or baseline.get("status") != "passed" or baseline.get("contract") != result["contract"]: + result["errors"].append("No compatible passing baseline for this scenario") + elif baseline.get("fingerprints", {}) != result["fingerprints"]: + result["fingerprint_difference"] = { + "expected": baseline.get("fingerprints", {}), + "actual": result["fingerprints"], + } + result["errors"].append( + "Deterministic fingerprint differs from baseline: " + + json.dumps(result["fingerprint_difference"], sort_keys=True) + ) else: - for metric, previous in baseline['metrics'].items(): - if metric.startswith('process.') and not compare_process: + for metric, previous in baseline["metrics"].items(): + if metric.startswith("process.") and not compare_process: continue # Corrections/worker counts can vary with scheduling. Retain them as # provenance; they are not a lower-is-faster performance metric. - if metric.endswith('.count'): + if metric.endswith(".count"): continue - current = result['metrics'].get(metric) - number(previous, f'baseline {metric}') - limit = previous + max(previous * percent / 100, absolute if metric.endswith('_ms') else 0) + current = result["metrics"].get(metric) + number(previous, f"baseline {metric}") + limit = previous + max(previous * percent / 100, absolute if metric.endswith("_ms") else 0) passed = current is not None and current <= limit - result['comparisons'].append({'metric': metric, 'baseline': previous, 'current': current, - 'limit': limit, 'status': 'passed' if passed else 'failed'}) + result["comparisons"].append( + { + "metric": metric, + "baseline": previous, + "current": current, + "limit": limit, + "status": "passed" if passed else "failed", + } + ) if not passed: - result['errors'].append(f'Performance regression: {metric}={current}; limit {limit:.3f}') - if result['errors']: - result['status'] = 'failed' + result["errors"].append(f"Performance regression: {metric}={current}; limit {limit:.3f}") + if result["errors"]: + result["status"] = "failed" def build_identity(args, suite): - cache = args.build_dir / 'CMakeCache.txt' + cache = args.build_dir / "CMakeCache.txt" settings = {} if cache.is_file(): - for line in cache.read_text(errors='replace').splitlines(): - match = re.match(r'((?:CMAKE_(?:BUILD_TYPE|CXX_COMPILER|CXX_FLAGS[^:]*|C_FLAGS[^:]*|SYSTEM_PROCESSOR))|DKR_[A-Z0-9_]+):[^=]+=(.*)', line) - if match and not match[1].endswith(('_SOURCE', '_DIR', '_ROOT', '_GENERATED')): + for line in cache.read_text(errors="replace").splitlines(): + match = re.match( + r"((?:CMAKE_(?:BUILD_TYPE|CXX_COMPILER|CXX_FLAGS[^:]*|C_FLAGS[^:]*|SYSTEM_PROCESSOR))|DKR_[A-Z0-9_]+):[^=]+=(.*)", + line, + ) + if match and not match[1].endswith(("_SOURCE", "_DIR", "_ROOT", "_GENERATED")): settings[match[1]] = match[2] - compatibility = {'suite': canonical(suite), 'system': platform.system(), 'release': platform.release(), - 'machine': platform.machine(), 'machine_tag': args.machine_tag or canonical(platform.node())[:16], - 'config': args.config, 'build_settings': settings, 'warmup': args.warmup, 'repeat': args.repeat} - git = subprocess.run(['git', '-C', str(ROOT), 'rev-parse', 'HEAD'], capture_output=True, text=True) - dirty = subprocess.run(['git', '-C', str(ROOT), 'status', '--porcelain'], capture_output=True, text=True) - return {'compatibility': compatibility, 'commit': git.stdout.strip(), 'dirty': bool(dirty.stdout.strip()), - 'python': platform.python_version(), 'measurement': 'Serial process wall time; stage values are medians across measured runs. No physical scanout or input-latency measurement.'} + compatibility = { + "suite": canonical(suite), + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "machine_tag": args.machine_tag or canonical(platform.node())[:16], + "config": args.config, + "build_settings": settings, + "warmup": args.warmup, + "repeat": args.repeat, + } + git = subprocess.run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture_output=True, text=True) + dirty = subprocess.run(["git", "-C", str(ROOT), "status", "--porcelain"], capture_output=True, text=True) + return { + "compatibility": compatibility, + "commit": git.stdout.strip(), + "dirty": bool(dirty.stdout.strip()), + "python": platform.python_version(), + "measurement": "Serial process wall time; stage values are medians across measured runs. No physical scanout or input-latency measurement.", + } def discover_ctest(args, items): - if not any('ctest' in item for item in items): + if not any("ctest" in item for item in items): return {} - result = subprocess.run(['ctest', '--test-dir', str(args.build_dir), '-C', args.config, - '--show-only=json-v1'], capture_output=True, text=True, timeout=30) + result = subprocess.run( + ["ctest", "--test-dir", str(args.build_dir), "-C", args.config, "--show-only=json-v1"], + capture_output=True, + text=True, + timeout=30, + ) if result.returncode: - raise ValueError('Could not discover CTest tests: ' + result.stderr.strip()) - return {test['name']: test for test in json.loads(result.stdout)['tests']} + raise ValueError("Could not discover CTest tests: " + result.stderr.strip()) + return {test["name"]: test for test in json.loads(result.stdout)["tests"]} def check_inputs(items, variables, ctest_tests): errors = [] for item in items: - name = item['id'] - values = item.get('command', []) + item.get('inputs', []) + list(item.get('env', {}).values()) + name = item["id"] + values = item.get("command", []) + item.get("inputs", []) + list(item.get("env", {}).values()) missing = sorted({key for value in values for key in TOKEN.findall(value) if key not in variables}) - errors.extend(f'{name}: missing variable {key} (set --var {key}=VALUE or --vars-file)' for key in missing) + errors.extend(f"{name}: missing variable {key} (set --var {key}=VALUE or --vars-file)" for key in missing) # Still inspect independent paths even when another substitution is missing. - for value in item.get('inputs', []): + for value in item.get("inputs", []): if any(key not in variables for key in TOKEN.findall(value)): continue path = Path(expand(value, variables)).resolve() if not path.is_file(): - errors.append(f'{name}: missing input: {path}') - if 'command' in item: - executable = item['command'][0] + errors.append(f"{name}: missing input: {path}") + if "command" in item: + executable = item["command"][0] if not any(key not in variables for key in TOKEN.findall(executable)): executable = expand(executable, variables) if not shutil.which(executable): - errors.append(f'{name}: missing or nonexecutable command: {executable}') - elif item['ctest'] not in ctest_tests: - errors.append(f'{name}: CTest test not configured: {item["ctest"]}') + errors.append(f"{name}: missing or nonexecutable command: {executable}") + elif item["ctest"] not in ctest_tests: + errors.append(f"{name}: CTest test not configured: {item['ctest']}") else: - entry = ctest_tests[item['ctest']] - command = entry.get('command', []) + entry = ctest_tests[item["ctest"]] + command = entry.get("command", []) if not command or not shutil.which(command[0]): - errors.append(f'{name}: CTest executable unavailable; build {item["ctest"]} first') - if any(prop.get('name') == 'DISABLED' and prop.get('value') for prop in entry.get('properties', [])): - errors.append(f'{name}: CTest test is disabled: {item["ctest"]}') + errors.append(f"{name}: CTest executable unavailable; build {item['ctest']} first") + if any(prop.get("name") == "DISABLED" and prop.get("value") for prop in entry.get("properties", [])): + errors.append(f"{name}: CTest test is disabled: {item['ctest']}") return errors def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--suite', type=Path, default=ROOT / 'regression/host.json') - parser.add_argument('--build-dir', type=Path, default=ROOT / 'build/host') - parser.add_argument('--config', default='Release') - parser.add_argument('--output', type=Path, help='new directory for reports and logs') - parser.add_argument('--baseline', type=Path, help='passing report.json from the same machine/configuration') - parser.add_argument('--scenario', action='append', default=[], help='scenario glob (repeatable)') - parser.add_argument('--vars-file', type=Path, metavar='PATH', - help='JSON path-variable map; relative paths resolve beside this file') - parser.add_argument('--var', action='append', default=[], metavar='NAME=VALUE', - help='literal variable value, overriding --vars-file (repeatable)') - parser.add_argument('--repeat', type=int, default=3) - parser.add_argument('--warmup', type=int, default=1) - parser.add_argument('--relative-tolerance', type=float, default=20, metavar='PERCENT') - parser.add_argument('--absolute-tolerance-ms', type=float, default=2) - parser.add_argument('--compare-process-time', action='store_true', - help='also gate whole-process timings (includes launch/test overhead)') - parser.add_argument('--machine-tag', help='explicit machine identity for controlled benchmark hosts') - parser.add_argument('--allow-missing', action='store_true', help='allow partial runs; all-skipped still fails') + parser.add_argument("--suite", type=Path, default=ROOT / "regression/host.json") + parser.add_argument("--build-dir", type=Path, default=ROOT / "build/host") + parser.add_argument("--config", default="Release") + parser.add_argument("--output", type=Path, help="new directory for reports and logs") + parser.add_argument("--baseline", type=Path, help="passing report.json from the same machine/configuration") + parser.add_argument("--scenario", action="append", default=[], help="scenario glob (repeatable)") + parser.add_argument( + "--vars-file", type=Path, metavar="PATH", help="JSON path-variable map; relative paths resolve beside this file" + ) + parser.add_argument( + "--var", + action="append", + default=[], + metavar="NAME=VALUE", + help="literal variable value, overriding --vars-file (repeatable)", + ) + parser.add_argument("--repeat", type=int, default=3) + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--relative-tolerance", type=float, default=20, metavar="PERCENT") + parser.add_argument("--absolute-tolerance-ms", type=float, default=2) + parser.add_argument( + "--compare-process-time", + action="store_true", + help="also gate whole-process timings (includes launch/test overhead)", + ) + parser.add_argument("--machine-tag", help="explicit machine identity for controlled benchmark hosts") + parser.add_argument("--allow-missing", action="store_true", help="allow partial runs; all-skipped still fails") inspection = parser.add_mutually_exclusive_group() - inspection.add_argument('--list', action='store_true') - inspection.add_argument('--check-inputs', action='store_true', - help='check selected variables, files, and executables without running scenarios') + inspection.add_argument("--list", action="store_true") + inspection.add_argument( + "--check-inputs", + action="store_true", + help="check selected variables, files, and executables without running scenarios", + ) args = parser.parse_args(argv) try: if not 1 <= args.repeat <= 100 or not 0 <= args.warmup <= 20: - raise ValueError('repeat must be 1..100 and warmup 0..20') - number(args.relative_tolerance, 'relative tolerance') - number(args.absolute_tolerance_ms, 'absolute tolerance') + raise ValueError("repeat must be 1..100 and warmup 0..20") + number(args.relative_tolerance, "relative tolerance") + number(args.absolute_tolerance_ms, "absolute tolerance") args.build_dir = args.build_dir.resolve() suite = load_suite(args.suite) - items = suite['scenarios'] + items = suite["scenarios"] if args.scenario: for pattern in args.scenario: - if not any(fnmatch.fnmatchcase(x['id'], pattern) for x in items): - raise ValueError(f'No scenario matches {pattern}') - items = [x for x in items if any(fnmatch.fnmatchcase(x['id'], p) for p in args.scenario)] + if not any(fnmatch.fnmatchcase(x["id"], pattern) for x in items): + raise ValueError(f"No scenario matches {pattern}") + items = [x for x in items if any(fnmatch.fnmatchcase(x["id"], p) for p in args.scenario)] if args.list: for item in items: print(f"{item['id']}: {item.get('description', '')}") @@ -536,33 +647,51 @@ def main(argv=None): errors.extend(check_inputs(items, variables, ctest_tests)) if errors: for error in errors: - print(f'regression: {error}', file=sys.stderr) + print(f"regression: {error}", file=sys.stderr) return 2 - print(f'Inputs ready for {len(items)} selected scenarios; no scenarios executed.') + print(f"Inputs ready for {len(items)} selected scenarios; no scenarios executed.") return 0 identity = build_identity(args, suite) args.ctest_tests = discover_ctest(args, items) baseline = None if args.baseline: baseline = json.loads(args.baseline.read_text()) - if not isinstance(baseline, dict) or baseline.get('schema_version') != 1: - raise ValueError('Baseline requires a schema_version 1 report object') - if not isinstance(baseline.get('identity'), dict) or baseline['identity'].get('compatibility') != identity['compatibility']: - raise ValueError('Baseline machine, suite, build configuration, or sampling settings do not match') - if not isinstance(baseline.get('summary'), dict) or baseline['summary'].get('failed', 1) or not baseline['summary'].get('passed', 0): - raise ValueError('Baseline must contain passing scenarios and no failures') - if not isinstance(baseline.get('scenarios'), list) or not all(isinstance(x, dict) and - isinstance(x.get('id'), str) and isinstance(x.get('metrics'), dict) for x in baseline['scenarios']): - raise ValueError('Invalid baseline scenarios') + if not isinstance(baseline, dict) or baseline.get("schema_version") != 1: + raise ValueError("Baseline requires a schema_version 1 report object") + if ( + not isinstance(baseline.get("identity"), dict) + or baseline["identity"].get("compatibility") != identity["compatibility"] + ): + raise ValueError("Baseline machine, suite, build configuration, or sampling settings do not match") + if ( + not isinstance(baseline.get("summary"), dict) + or baseline["summary"].get("failed", 1) + or not baseline["summary"].get("passed", 0) + ): + raise ValueError("Baseline must contain passing scenarios and no failures") + if not isinstance(baseline.get("scenarios"), list) or not all( + isinstance(x, dict) and isinstance(x.get("id"), str) and isinstance(x.get("metrics"), dict) + for x in baseline["scenarios"] + ): + raise ValueError("Invalid baseline scenarios") if not args.output: - raise ValueError('--output NEW_DIRECTORY is required') + raise ValueError("--output NEW_DIRECTORY is required") output = args.output.resolve() output.mkdir(parents=True, exist_ok=False) - report = {'schema_version': 1, 'suite': suite['name'], 'started_at': datetime.now(timezone.utc).isoformat(), - 'identity': identity, 'summary': {}, 'scenarios': [], - 'comparison_policy': {'relative_percent': args.relative_tolerance, 'absolute_ms': args.absolute_tolerance_ms, - 'compare_process_time': args.compare_process_time}} - (output / 'suite.json').write_text(json.dumps(suite, indent=2) + '\n') + report = { + "schema_version": 1, + "suite": suite["name"], + "started_at": datetime.now(timezone.utc).isoformat(), + "identity": identity, + "summary": {}, + "scenarios": [], + "comparison_policy": { + "relative_percent": args.relative_tolerance, + "absolute_ms": args.absolute_tolerance_ms, + "compare_process_time": args.compare_process_time, + }, + } + (output / "suite.json").write_text(json.dumps(suite, indent=2) + "\n") interrupted = False try: for item in items: @@ -570,31 +699,49 @@ def main(argv=None): try: result, interrupted = run_scenario(item, variables, output, args) if baseline: - previous = next((x for x in baseline['scenarios'] if x['id'] == item['id']), None) - compare(result, previous, args.relative_tolerance, args.absolute_tolerance_ms, args.compare_process_time) + previous = next((x for x in baseline["scenarios"] if x["id"] == item["id"]), None) + compare( + result, + previous, + args.relative_tolerance, + args.absolute_tolerance_ms, + args.compare_process_time, + ) except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError) as error: - result = {'id': item['id'], 'status': 'failed', 'errors': [str(error)], - 'attempts': [], 'metrics': {}, 'comparisons': []} - report['scenarios'].append(result) - print(f" {result['status'].upper()} " + '; '.join(result['errors']), flush=True) - report['summary'] = {s: sum(x['status'] == s for x in report['scenarios']) for s in ('passed', 'failed', 'skipped')} + result = { + "id": item["id"], + "status": "failed", + "errors": [str(error)], + "attempts": [], + "metrics": {}, + "comparisons": [], + } + report["scenarios"].append(result) + print(f" {result['status'].upper()} " + "; ".join(result["errors"]), flush=True) + report["summary"] = { + s: sum(x["status"] == s for x in report["scenarios"]) for s in ("passed", "failed", "skipped") + } write_reports(output, report) if interrupted: break finally: write_reports(output, report) print(f"Reports: {output / 'report.html'}") - summary = report['summary'] + summary = report["summary"] if interrupted: return 130 - return 1 if summary.get('failed') else (2 if not summary.get('passed') or (summary.get('skipped') and not args.allow_missing) else 0) + return ( + 1 + if summary.get("failed") + else (2 if not summary.get("passed") or (summary.get("skipped") and not args.allow_missing) else 0) + ) except KeyboardInterrupt: - print('regression: interrupted', file=sys.stderr) + print("regression: interrupted", file=sys.stderr) return 130 except (OSError, ValueError, KeyError, TypeError, re.error, subprocess.SubprocessError) as error: - print(f'regression: {error}', file=sys.stderr) + print(f"regression: {error}", file=sys.stderr) return 2 -if __name__ == '__main__': +if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/render_magic_code_guide.py b/scripts/render_magic_code_guide.py index 5ba69b12..404c8187 100644 --- a/scripts/render_magic_code_guide.py +++ b/scripts/render_magic_code_guide.py @@ -4,7 +4,9 @@ Document rendering/coverage only: never loads a mod or launches the game. The source remains the SDK's plain-text book; no online assets are required. """ + from __future__ import annotations + import argparse import html import json @@ -14,36 +16,36 @@ def inline(value: str) -> str: escaped = html.escape(value) - return re.sub(r'`([^`]+)`', r'\1', escaped) + return re.sub(r"`([^`]+)`", r"\1", escaped) def render(sdk: Path, output: Path) -> dict: if output.exists(): - raise ValueError('Choose a new guide output file') - source = (sdk / 'MODDERS-GUIDE.txt').read_text(encoding='utf-8') - headers = [(name, (sdk / 'include/dkr' / name).read_text(encoding='utf-8')) - for name in ('magic_codes.h', 'game.h')] - public = sorted(set(re.findall(r'(?:DKR_IMPORT|static inline)[^;{]*?\b(dkr_\w+)\s*\(', - '\n'.join(text for _, text in headers)))) + raise ValueError("Choose a new guide output file") + source = (sdk / "MODDERS-GUIDE.txt").read_text(encoding="utf-8") + headers = [(name, (sdk / "include/dkr" / name).read_text(encoding="utf-8")) for name in ("magic_codes.h", "game.h")] + public = sorted( + set(re.findall(r"(?:DKR_IMPORT|static inline)[^;{]*?\b(dkr_\w+)\s*\(", "\n".join(text for _, text in headers))) + ) missing = [name for name in public if name not in source] if missing: - raise ValueError('Undocumented public functions: ' + ', '.join(missing)) + raise ValueError("Undocumented public functions: " + ", ".join(missing)) sections, body, paragraph, code = [], [], [], None list_open = False def flush(): nonlocal paragraph, list_open if paragraph: - body.append('

' + inline(' '.join(paragraph)) + '

') + body.append("

" + inline(" ".join(paragraph)) + "

") paragraph = [] if list_open: - body.append('') + body.append("") list_open = False for line in source.splitlines(): - if line.startswith('```'): + if line.startswith("```"): if code is not None: - body.append('
' + html.escape('\n'.join(code)) + '
') + body.append("
" + html.escape("\n".join(code)) + "
") code = None else: flush() @@ -52,40 +54,46 @@ def flush(): if code is not None: code.append(line) continue - heading = re.match(r'^(#{1,3}) (.+)$', line) + heading = re.match(r"^(#{1,3}) (.+)$", line) if heading: flush() level, title = len(heading[1]), heading[2] - anchor = 'section-' + str(len(sections) + 1) + anchor = "section-" + str(len(sections) + 1) sections.append((level, title, anchor)) body.append(f'{inline(title)}') elif not line.strip(): flush() - elif line.startswith('- '): + elif line.startswith("- "): if paragraph: flush() if not list_open: - body.append('
    ') + body.append("
      ") list_open = True - body.append('
    • ' + inline(line[2:]) + '
      • ') - elif list_open and line.startswith(' '): + body.append("
    • " + inline(line[2:]) + "
      • ") + elif list_open and line.startswith(" "): # Continuations belong to the preceding list item. - body[-1] = body[-1][:-5] + ' ' + inline(line.strip()) + '' + body[-1] = body[-1][:-5] + " " + inline(line.strip()) + "" else: if list_open: flush() paragraph.append(line.strip()) flush() if code is not None: - raise ValueError('Unclosed code fence in guide') - body.append('

      Appendix Full SDK headers

      ' - '

      These are the exact shipped declarations, comments, fields and constants. ' - 'Weak import bodies are metadata, not runtime fallbacks.

      ') + raise ValueError("Unclosed code fence in guide") + body.append( + '

      Appendix Full SDK headers

      ' + "

      These are the exact shipped declarations, comments, fields and constants. " + "Weak import bodies are metadata, not runtime fallbacks.

      " + ) for name, text in headers: - body.append('

      ' + html.escape(name) + '

      ' + html.escape(text) + '
      ') - toc = ''.join('
    • ' + html.escape(title) + '
      • ' - for level, title, anchor in sections if level == 2) - page = ''' + body.append("

      " + html.escape(name) + "

      " + html.escape(text) + "
      ") + toc = "".join( + '
    • ' + html.escape(title) + "
      • " + for level, title, anchor in sections + if level == 2 + ) + page = ( + """ Modders Guide to Custom Magic Code Creation
      ''' + '\n'.join(body) + ''' +DKR-R v1.0.7 beta 2 · API 1
        """ + + toc + + """
      1. Full SDK headers
        2. +
      Back to top
      """ + + "\n".join(body) + + """
      DKR-R Custom Magic Codes API 1. Keep this guide with the matching headers and examples. -This document is self-contained and makes no network requests.
      ''' +This document is self-contained and makes no network requests.
      """ + ) output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(page, encoding='utf-8') - return {'chapters': sum(level == 2 for level, _, _ in sections), - 'public_functions': len(public), 'documented_functions': public, - 'source_words': len(source.split()), 'output': str(output), - 'executed_mod_code': False} + output.write_text(page, encoding="utf-8") + return { + "chapters": sum(level == 2 for level, _, _ in sections), + "public_functions": len(public), + "documented_functions": public, + "source_words": len(source.split()), + "output": str(output), + "executed_mod_code": False, + } def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--sdk', type=Path, default=Path(__file__).resolve().parents[1] / 'modding') - parser.add_argument('--output', type=Path, required=True) + parser.add_argument("--sdk", type=Path, default=Path(__file__).resolve().parents[1] / "modding") + parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() print(json.dumps(render(args.sdk, args.output), indent=2)) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/scripts/run_host_shader_tool.py b/scripts/run_host_shader_tool.py index 873fd958..df47fe31 100644 --- a/scripts/run_host_shader_tool.py +++ b/scripts/run_host_shader_tool.py @@ -1,4 +1,5 @@ """Run the renderer's separately built host file_to_c tool when cross-compiling.""" + import argparse import pathlib import subprocess @@ -17,8 +18,11 @@ def main(argv=None): host = args.host.resolve(strict=True) return subprocess.call([str(host), *args.arguments]) except OSError as error: - print(f"Cannot run host shader tool {args.host}: {error}. " - "Build the host file_to_c target before cross-compiling.", file=sys.stderr) + print( + f"Cannot run host shader tool {args.host}: {error}. " + "Build the host file_to_c target before cross-compiling.", + file=sys.stderr, + ) return 127 diff --git a/scripts/save_layout_pipeline.py b/scripts/save_layout_pipeline.py index dce812fd..b865fd80 100644 --- a/scripts/save_layout_pipeline.py +++ b/scripts/save_layout_pipeline.py @@ -3,90 +3,129 @@ Reuses the complete composed CPU pipeline (characters, tracks, code SDK, graphics fixes) rather than regenerating from an incomplete base policy. """ + +import argparse +import hashlib +import json from pathlib import Path -import argparse, hashlib, json, re, shutil +import re +import shutil + +DEF = re.compile(r"RECOMP_FUNC\s+void\s+(\w+)\s*\([^{};]*\)\s*\{") +OPERATIONS = { + "populate_settings_from_save_data": 0, + "func_800732E8": 1, + "func_80073588": 2, + "func_800738A4": 3, + "clear_lap_records": 4, +} +PINS = { + 77: { + "func_800738A4": "ace7fa4ceff245c2152085f37289f31c950f541fc2fe4e8e86169fe2d094a98c", + "func_80073588": "80cf0a25375965fbfaadaa92ade2097c62832a65fb01c569be6ae3db8689f164", + "populate_settings_from_save_data": "4a8e586f1791fc699731631b0c84a14d19614237a0103e847af1b80ef9a60d56", + "func_800732E8": "c6262419220c3fa67ba3ab795b4fba94bce2829c25cdabea2f62d9c7332e6928", + "clear_lap_records": "82fc9f4782fd420e965566f16ad954b4e83b55c61a417f9e11c979855caaa38f", + }, + 80: { + "func_800738A4": "9c573baeedc19859e0781ffb862720dce5e8497ade8c7c099c3fbe0a618ec451", + "func_80073588": "566e81cc90879325ddde9667559a6541c1de50594afdf95eeaab2ce0da1b31a8", + "populate_settings_from_save_data": "1a2e858683bb6162d1d806a1ea4ab24c9ded1fa0e8d2775c295740c3cadb0b22", + "func_800732E8": "d169fe3f22787dfa281d69f0aa69b708dfdad03caaa5c6391e2593dcc157bc76", + "clear_lap_records": "2bd81ee2b02842af88bca9fdd7517623fe1e15922c0484ac35c50965873e04cf", + }, +} +ROOT = Path(__file__).resolve().parents[1] +HEADER = ROOT / "patches/save-layout/guest_save_layout.h" +sha = lambda text: hashlib.sha256(text.encode()).hexdigest() -DEF=re.compile(r'RECOMP_FUNC\s+void\s+(\w+)\s*\([^{};]*\)\s*\{') -OPERATIONS={'populate_settings_from_save_data':0,'func_800732E8':1, - 'func_80073588':2,'func_800738A4':3,'clear_lap_records':4} -PINS={77:{ - 'func_800738A4':'ace7fa4ceff245c2152085f37289f31c950f541fc2fe4e8e86169fe2d094a98c', - 'func_80073588':'80cf0a25375965fbfaadaa92ade2097c62832a65fb01c569be6ae3db8689f164', - 'populate_settings_from_save_data':'4a8e586f1791fc699731631b0c84a14d19614237a0103e847af1b80ef9a60d56', - 'func_800732E8':'c6262419220c3fa67ba3ab795b4fba94bce2829c25cdabea2f62d9c7332e6928', - 'clear_lap_records':'82fc9f4782fd420e965566f16ad954b4e83b55c61a417f9e11c979855caaa38f'},80:{ - 'func_800738A4':'9c573baeedc19859e0781ffb862720dce5e8497ade8c7c099c3fbe0a618ec451', - 'func_80073588':'566e81cc90879325ddde9667559a6541c1de50594afdf95eeaab2ce0da1b31a8', - 'populate_settings_from_save_data':'1a2e858683bb6162d1d806a1ea4ab24c9ded1fa0e8d2775c295740c3cadb0b22', - 'func_800732E8':'d169fe3f22787dfa281d69f0aa69b708dfdad03caaa5c6391e2593dcc157bc76', - 'clear_lap_records':'2bd81ee2b02842af88bca9fdd7517623fe1e15922c0484ac35c50965873e04cf'}} -ROOT=Path(__file__).resolve().parents[1] -HEADER=ROOT/'patches/save-layout/guest_save_layout.h' -sha=lambda text:hashlib.sha256(text.encode()).hexdigest() def owned_support(source): - receipt=source/'save-layout-receipt.json' - if not receipt.exists():return '' - data=json.loads(receipt.read_text()) - header=(source/'save_layout_guest.h').read_text() - if data['schema']!=1 or data['helper_sha256']!=sha(header) or header!=HEADER.read_text(): - raise ValueError('Save-layout helper differs from the reviewed authored pipeline input') + receipt = source / "save-layout-receipt.json" + if not receipt.exists(): + return "" + data = json.loads(receipt.read_text()) + header = (source / "save_layout_guest.h").read_text() + if data["schema"] != 1 or data["helper_sha256"] != sha(header) or header != HEADER.read_text(): + raise ValueError("Save-layout helper differs from the reviewed authored pipeline input") return header -def stage(source,output,revision,config,output_config): - source,output=source.resolve(),output.resolve() - config,output_config=config.resolve(),output_config.resolve() - if revision not in PINS or output.exists() or source==output or source in output.parents: - raise ValueError('Save layout requires a new separate output directory') - if output_config.exists() or output_config==config or source in output_config.parents: - raise ValueError('Save-layout configuration must also be a NEW separate output') - reserved=('save_layout_guest.h','save_layout_functions.c','save_layout_fault.cpp','save-layout-receipt.json') - if any((source/name).exists() for name in reserved): - raise ValueError('Input already contains save-layout output; do not compose it twice') + +def stage(source, output, revision, config, output_config): + source, output = source.resolve(), output.resolve() + config, output_config = config.resolve(), output_config.resolve() + if revision not in PINS or output.exists() or source == output or source in output.parents: + raise ValueError("Save layout requires a new separate output directory") + if output_config.exists() or output_config == config or source in output_config.parents: + raise ValueError("Save-layout configuration must also be a NEW separate output") + reserved = ("save_layout_guest.h", "save_layout_functions.c", "save_layout_fault.cpp", "save-layout-receipt.json") + if any((source / name).exists() for name in reserved): + raise ValueError("Input already contains save-layout output; do not compose it twice") # Minimal ABI declaration avoids importing scheduler implementation headers # (and their private concurrentqueue include paths) into a guest CPU target. # The exception definition is token-identical across translation units. - scheduler=(ROOT/'extern/n64-modern-runtime/ultramodern/include/ultramodern/ultramodern.hpp').read_text() - if 'struct thread_terminated : std::exception {};' not in scheduler or 'void quit();' not in scheduler: - raise ValueError('Unreviewed scheduler termination ABI; inspect before regenerating') - edits={};found={};hooks=[] - for file in sorted(source.glob('*.c')): - text=file.read_text();matches=list(DEF.finditer(text));changed=text - for i,m in reversed(list(enumerate(matches))): - name=m[1] - if name not in OPERATIONS:continue - end=matches[i+1].start() if i+1 @@ -100,16 +139,23 @@ def stage(source,output,revision,config,output_config): ultramodern::quit(); throw ultramodern::thread_terminated{}; } -''',newline='\n') - (output/'save-layout-receipt.json').write_text(json.dumps(dict(schema=1,revision=revision, - helper_sha256=sha(header),functions=found),indent=2)+'\n') - content=config.read_text().replace(source.as_posix(),output.as_posix()) - output_config.write_text(content+'\n# Authored fixed retail save projection\n'+''.join(hooks),newline='\n') - print(f'Save-layout v{revision}: five pinned functions, new output {output}') +""", + newline="\n", + ) + (output / "save-layout-receipt.json").write_text( + json.dumps(dict(schema=1, revision=revision, helper_sha256=sha(header), functions=found), indent=2) + "\n" + ) + content = config.read_text().replace(source.as_posix(), output.as_posix()) + output_config.write_text(content + "\n# Authored fixed retail save projection\n" + "".join(hooks), newline="\n") + print(f"Save-layout v{revision}: five pinned functions, new output {output}") + -if __name__=='__main__': - p=argparse.ArgumentParser(description=__doc__) - p.add_argument('--source',type=Path,required=True);p.add_argument('--output',type=Path,required=True) - p.add_argument('--revision',type=int,required=True);p.add_argument('--config',type=Path,required=True) - p.add_argument('--output-config',type=Path,required=True);a=p.parse_args() - stage(a.source,a.output,a.revision,a.config,a.output_config) +if __name__ == "__main__": + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--source", type=Path, required=True) + p.add_argument("--output", type=Path, required=True) + p.add_argument("--revision", type=int, required=True) + p.add_argument("--config", type=Path, required=True) + p.add_argument("--output-config", type=Path, required=True) + a = p.parse_args() + stage(a.source, a.output, a.revision, a.config, a.output_config) diff --git a/scripts/scan_for_game_assets.py b/scripts/scan_for_game_assets.py index 56146ff6..3566b791 100644 --- a/scripts/scan_for_game_assets.py +++ b/scripts/scan_for_game_assets.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Fail when repository files look like ROMs or generated game-resource archives.""" + from __future__ import annotations import argparse @@ -10,8 +11,17 @@ ROOT = pathlib.Path(__file__).resolve().parents[1] SKIP_PARTS = { - ".git", ".deps", "build", "build-logs", "dist", "runtime", "extern", - "run-data", "--portable", "squashfs-root", "__pycache__", + ".git", + ".deps", + "build", + "build-logs", + "dist", + "runtime", + "extern", + "run-data", + "--portable", + "squashfs-root", + "__pycache__", } DENIED_SUFFIXES = {".z64", ".v64", ".n64", ".o2r", ".otr", ".eep", ".mpk"} N64_HEADERS = { diff --git a/scripts/stage_magic_code_payload.py b/scripts/stage_magic_code_payload.py index 54a41b25..a368019c 100644 --- a/scripts/stage_magic_code_payload.py +++ b/scripts/stage_magic_code_payload.py @@ -5,157 +5,260 @@ body gets a private TU-local name. A dispatch table contains only functions actually emitted in this namespace, with project/system replacements fenced. """ + from __future__ import annotations + import argparse import hashlib import json +from pathlib import Path import re + import tomllib -from pathlib import Path -DEFINITION=re.compile(r'RECOMP_FUNC void (\w+)\(uint8_t\* rdram, recomp_context\* ctx\) \{') -ADDRESS=re.compile(r'// (0x[0-9a-fA-F]{8}):') -SYSTEM=('os','__os','__sc','al','_al','sndp_','sound_','music_','mempool_', - 'dmacopy','mainproc','thread','bgload_','rsp_','fb_','input_','rumble_', - 'contpak_','eeprom_','save_') +DEFINITION = re.compile(r"RECOMP_FUNC void (\w+)\(uint8_t\* rdram, recomp_context\* ctx\) \{") +ADDRESS = re.compile(r"// (0x[0-9a-fA-F]{8}):") +SYSTEM = ( + "os", + "__os", + "__sc", + "al", + "_al", + "sndp_", + "sound_", + "music_", + "mempool_", + "dmacopy", + "mainproc", + "thread", + "bgload_", + "rsp_", + "fb_", + "input_", + "rumble_", + "contpak_", + "eeprom_", + "save_", +) + def protected(name: str, body: str) -> bool: - return (name.startswith(SYSTEM) or - any(s in name.lower() for s in ('eeprom','mempak','mempool','controller_pak','flashrom')) or - bool(re.search(r'\bdkr_\w+\s*\(',body))) + return ( + name.startswith(SYSTEM) + or any(s in name.lower() for s in ("eeprom", "mempak", "mempool", "controller_pak", "flashrom")) + or bool(re.search(r"\bdkr_\w+\s*\(", body)) + ) + def protected_hook(name: str) -> bool: - return name.startswith(SYSTEM) or any(s in name.lower() for s in - ('eeprom','mempak','mempool','controller_pak','flashrom')) - -EVENTS={'main_game_loop':(1,2),'unload_level_game':(3,0),'level_load':(0,4), - 'obj_init_racer':(0,5),'postrace_start':(6,0),'menu_init':(0,7)} -CALL=re.compile(r'\b([A-Za-z_]\w*)\s*\(') -BLOCKING=('osRecvMesg','osSendMesg','osJamMesg','osCreateThread','osStartThread','osDestroyThread', - 'osStopThread','osSetThreadPri','osPiStartDma','osEPiStartDma','pause_self','yield_self', - 'recomp_syscall_handler') -CURATED_AUDIO={'sound_play','music_play','music_stop','music_fade'} - -def policy_names(source: Path,revision: int,config: Path|None=None) -> set[str]: + return name.startswith(SYSTEM) or any( + s in name.lower() for s in ("eeprom", "mempak", "mempool", "controller_pak", "flashrom") + ) + + +EVENTS = { + "main_game_loop": (1, 2), + "unload_level_game": (3, 0), + "level_load": (0, 4), + "obj_init_racer": (0, 5), + "postrace_start": (6, 0), + "menu_init": (0, 7), +} +CALL = re.compile(r"\b([A-Za-z_]\w*)\s*\(") +BLOCKING = ( + "osRecvMesg", + "osSendMesg", + "osJamMesg", + "osCreateThread", + "osStartThread", + "osDestroyThread", + "osStopThread", + "osSetThreadPri", + "osPiStartDma", + "osEPiStartDma", + "pause_self", + "yield_self", + "recomp_syscall_handler", +) +CURATED_AUDIO = {"sound_play", "music_play", "music_stop", "music_fade"} + + +def policy_names(source: Path, revision: int, config: Path | None = None) -> set[str]: """Use the COMPLETE config that produced this CPU input, not name guesses. Instruction-only patches have no dkr_* call in their body. They must be protected from replacement just like explicit native ownership hooks. """ - config=config or source.parent/f'v{revision}.toml' - doc=tomllib.loads(config.read_text(encoding='utf-8')) - if doc.get('input',{}).get('entrypoint')!={77:0x80065d40,80:0x80065f80}[revision]: - raise ValueError('Code protection requires the matching complete CPU Patch Pipeline config') - patches=doc['patches'] - if 'instruction' not in patches or 'hook' not in patches: - raise ValueError('Incomplete code-mod port-fix protection policy') - names=set() - for key in ('stubs','ignored','renamed'):names.update(patches.get(key,[])) - for key in ('instruction','hook'): - for entry in patches[key]:names.add(entry['func']) + config = config or source.parent / f"v{revision}.toml" + doc = tomllib.loads(config.read_text(encoding="utf-8")) + if doc.get("input", {}).get("entrypoint") != {77: 0x80065D40, 80: 0x80065F80}[revision]: + raise ValueError("Code protection requires the matching complete CPU Patch Pipeline config") + patches = doc["patches"] + if "instruction" not in patches or "hook" not in patches: + raise ValueError("Incomplete code-mod port-fix protection policy") + names = set() + for key in ("stubs", "ignored", "renamed"): + names.update(patches.get(key, [])) + for key in ("instruction", "hook"): + for entry in patches[key]: + names.add(entry["func"]) return names + + # Reviewed against Object_Racer.lap_times[5] and every source use in the # pinned decomp (racer.c, game_ui.c, objects.c). Offsets are byte displacements, # not arbitrary memory ranges. No unrelated racer fields are redirected. -LAP_FIELDS={ - 'update_player_racer':{0x128},'hud_main_time_trial':{0x124,0x128}, - 'hud_bananas':{0x128},'hud_lap_count':{0x128}, - 'hud_race_finish_multiplayer':{0x128},'hud_race_time':{0x124,0x128}, - 'race_finish_time_trial':{0x128}, - 'mode_init_taj_race':{0x128,0x12c,0x130,0x134,0x138}, +LAP_FIELDS = { + "update_player_racer": {0x128}, + "hud_main_time_trial": {0x124, 0x128}, + "hud_bananas": {0x128}, + "hud_lap_count": {0x128}, + "hud_race_finish_multiplayer": {0x128}, + "hud_race_time": {0x124, 0x128}, + "race_finish_time_trial": {0x128}, + "mode_init_taj_race": {0x128, 0x12C, 0x130, 0x134, 0x138}, } -def lap_memory(name: str,body: str,owned: bool) -> str: - if name not in LAP_FIELDS:return body - bridge='dkr_probe_code_lap_address' if owned else 'dkr_code_lap_address' - pattern=re.compile(r'MEM_W\((ctx->r\d+|0X[0-9A-F]+), (ctx->r\d+|0X[0-9A-F]+)\)') + +def lap_memory(name: str, body: str, owned: bool) -> str: + if name not in LAP_FIELDS: + return body + bridge = "dkr_probe_code_lap_address" if owned else "dkr_code_lap_address" + pattern = re.compile(r"MEM_W\((ctx->r\d+|0X[0-9A-F]+), (ctx->r\d+|0X[0-9A-F]+)\)") + def replace(m): - displacement=next((int(x,16) for x in m.groups() if x.startswith('0X')),None) - if displacement not in LAP_FIELDS[name]:return m[0] - address=f'(uint32_t)ADD32({m[1]}, {m[2]})' - return f'MEM_W(0, (gpr)(int32_t)(!dkr_code_callbacks_enabled ? {address} : {bridge}({address},rdram,ctx)))' - return pattern.sub(replace,body) - -def blocking_closure(functions: dict[str,str]) -> set[str]: - calls={name:set(CALL.findall(re.sub(r'//[^\n]*|/\*.*?\*/','',body,flags=re.S))) - for name,body in functions.items()} - blocked={name for name,targets in calls.items() if any(t.startswith(BLOCKING) for t in targets) - or 'LOOKUP_FUNC' in targets} + displacement = next((int(x, 16) for x in m.groups() if x.startswith("0X")), None) + if displacement not in LAP_FIELDS[name]: + return m[0] + address = f"(uint32_t)ADD32({m[1]}, {m[2]})" + return f"MEM_W(0, (gpr)(int32_t)(!dkr_code_callbacks_enabled ? {address} : {bridge}({address},rdram,ctx)))" + + return pattern.sub(replace, body) + + +def blocking_closure(functions: dict[str, str]) -> set[str]: + calls = { + name: set(CALL.findall(re.sub(r"//[^\n]*|/\*.*?\*/", "", body, flags=re.S))) for name, body in functions.items() + } + blocked = { + name + for name, targets in calls.items() + if any(t.startswith(BLOCKING) for t in targets) or "LOOKUP_FUNC" in targets + } # A raw native call must not suspend while a code VM/native lease is held. # Transitive calls are fenced too, not just functions with an OS-like name. while True: - more={name for name,targets in calls.items() if targets&blocked}-blocked - if not more:return blocked + more = {name for name, targets in calls.items() if targets & blocked} - blocked + if not more: + return blocked blocked.update(more) -def indirect_calls(body: str,owned: bool) -> str: - pattern=re.compile(r'LOOKUP_FUNC\((ctx->r[0-9]+)\)\(rdram, ctx\);') - if body.count('LOOKUP_FUNC(')!=len(pattern.findall(body)): - raise ValueError('Unreviewed indirect guest call shape') - bridge='dkr_probe_code_indirect' if owned else 'dkr_code_call_indirect' - return pattern.sub(lambda m:'{ const uint32_t code_target=(uint32_t)('+m[1]+'); ' - 'if(!dkr_code_callbacks_enabled || !'+bridge+'(code_target,rdram,ctx)) ' - 'LOOKUP_FUNC(code_target)(rdram,ctx); }',body) - -def entries(functions: dict[str,str], originals: dict[str,str]|None=None,port_fixes: set[str]|None=None) -> list[dict]: - result=[] - blocked=blocking_closure(originals or functions) - for name,body in sorted(functions.items()): - original=(originals or functions).get(name) - if not original:continue # Authored private continuations have no retail grant. - match=ADDRESS.search(original) - if not match:continue - before,after=EVENTS.get(name,(0,0)) - result.append(dict(name=name,address=int(match[1],16),protected=protected(name,original) or name in blocked or name in (port_fixes or set()), - protected_call=protected_hook(name) or name in blocked, - curated_call=name in CURATED_AUDIO and name not in blocked, - protected_hook=protected_hook(name),before_event=before,after_event=after)) + +def indirect_calls(body: str, owned: bool) -> str: + pattern = re.compile(r"LOOKUP_FUNC\((ctx->r[0-9]+)\)\(rdram, ctx\);") + if body.count("LOOKUP_FUNC(") != len(pattern.findall(body)): + raise ValueError("Unreviewed indirect guest call shape") + bridge = "dkr_probe_code_indirect" if owned else "dkr_code_call_indirect" + return pattern.sub( + lambda m: ( + "{ const uint32_t code_target=(uint32_t)(" + m[1] + "); " + "if(!dkr_code_callbacks_enabled || !" + bridge + "(code_target,rdram,ctx)) " + "LOOKUP_FUNC(code_target)(rdram,ctx); }" + ), + body, + ) + + +def entries( + functions: dict[str, str], originals: dict[str, str] | None = None, port_fixes: set[str] | None = None +) -> list[dict]: + result = [] + blocked = blocking_closure(originals or functions) + for name, body in sorted(functions.items()): + original = (originals or functions).get(name) + if not original: + continue # Authored private continuations have no retail grant. + match = ADDRESS.search(original) + if not match: + continue + before, after = EVENTS.get(name, (0, 0)) + result.append( + dict( + name=name, + address=int(match[1], 16), + protected=protected(name, original) or name in blocked or name in (port_fixes or set()), + protected_call=protected_hook(name) or name in blocked, + curated_call=name in CURATED_AUDIO and name not in blocked, + protected_hook=protected_hook(name), + before_event=before, + after_event=after, + ) + ) # Alias ambiguity must never resolve to an arbitrary native function. - by_address={} + by_address = {} for e in result: - if e['address'] in by_address: - by_address[e['address']]['protected']=True;e['protected']=True - by_address[e['address']]['protected_hook']=True;e['protected_hook']=True - by_address[e['address']]['protected_call']=True;e['protected_call']=True - by_address[e['address']]['curated_call']=False;e['curated_call']=False - else:by_address[e['address']]=e - return sorted(by_address.values(),key=lambda e:e['address']) + if e["address"] in by_address: + by_address[e["address"]]["protected"] = True + e["protected"] = True + by_address[e["address"]]["protected_hook"] = True + e["protected_hook"] = True + by_address[e["address"]]["protected_call"] = True + e["protected_call"] = True + by_address[e["address"]]["curated_call"] = False + e["curated_call"] = False + else: + by_address[e["address"]] = e + return sorted(by_address.values(), key=lambda e: e["address"]) -TRACK_OPTIONS={'trackmenu_setup_render':18,'func_80092188':20,'menu_track_select_loop':22,'trackmenu_input':24} -def track_menu_fields(functions: dict[str,str]) -> str: +TRACK_OPTIONS = {"trackmenu_setup_render": 18, "func_80092188": 20, "menu_track_select_loop": 22, "trackmenu_input": 24} + + +def track_menu_fields(functions: dict[str, str]) -> str: # Reuse ONLY the field table already verified by the legacy Patch Pipeline # against this revision's ELF. Never guess globals from decomp addresses. - source=functions.get('func_80092188','') - fields=re.findall(r'const uint32_t dkr_legacy_fields\[\] = \{([^}]+)\}',source) - if len(fields)!=1 or len(fields[0].split(','))!=29: - raise ValueError('Six-column menus require the verified native field table') - if not re.fullmatch(r'[0-9a-fA-FxXUu,\s]+',fields[0]):raise ValueError('Nonconstant native menu fields') + source = functions.get("func_80092188", "") + fields = re.findall(r"const uint32_t dkr_legacy_fields\[\] = \{([^}]+)\}", source) + if len(fields) != 1 or len(fields[0].split(",")) != 29: + raise ValueError("Six-column menus require the verified native field table") + if not re.fullmatch(r"[0-9a-fA-FxXUu,\s]+", fields[0]): + raise ValueError("Nonconstant native menu fields") return fields[0] -def wrap(name: str, body: str, address: int, owned: bool=False,menu_fields: str='') -> str: - body=lap_memory(name,body,owned) - body=indirect_calls(body,owned) - match=DEFINITION.match(body) - if not match or match[1]!=name:raise ValueError(f'Unreviewed guest ABI: {name}') - internal='dkr_code_body_'+name - renamed=body[:match.start()]+body[match.start():match.end()].replace( - f'RECOMP_FUNC void {name}(',f'static void {internal}(',1)+body[match.end():] + +def wrap(name: str, body: str, address: int, owned: bool = False, menu_fields: str = "") -> str: + body = lap_memory(name, body, owned) + body = indirect_calls(body, owned) + match = DEFINITION.match(body) + if not match or match[1] != name: + raise ValueError(f"Unreviewed guest ABI: {name}") + internal = "dkr_code_body_" + name + renamed = ( + body[: match.start()] + + body[match.start() : match.end()].replace(f"RECOMP_FUNC void {name}(", f"static void {internal}(", 1) + + body[match.end() :] + ) if owned: - enter=f'dkr_probe_code_hook(0,0x{address:08X}U,rdram,ctx)' - leave=f'dkr_probe_code_hook(1,0x{address:08X}U,rdram,ctx)' + enter = f"dkr_probe_code_hook(0,0x{address:08X}U,rdram,ctx)" + leave = f"dkr_probe_code_hook(1,0x{address:08X}U,rdram,ctx)" else: - enter=f'dkr_code_hook_enter(0x{address:08X}U,rdram,ctx)' - leave=f'dkr_code_hook_leave(0x{address:08X}U,rdram,ctx)' - invocation=f'{internal}(rdram,ctx);' + enter = f"dkr_code_hook_enter(0x{address:08X}U,rdram,ctx)" + leave = f"dkr_code_hook_leave(0x{address:08X}U,rdram,ctx)" + invocation = f"{internal}(rdram,ctx);" if name in TRACK_OPTIONS: - if not menu_fields:raise ValueError('Missing verified six-column menu fields') - event=TRACK_OPTIONS[name] - invocation=('{ extern int dkr_legacy_track_menu(uint8_t*,recomp_context*,unsigned,const uint32_t*,unsigned); ' - 'static const uint32_t menu_fields[]={'+menu_fields+'}; ' - f'dkr_legacy_track_menu(rdram,ctx,{event}U,menu_fields,0U); ' - +invocation+f' dkr_legacy_track_menu(rdram,ctx,{event+1}U,menu_fields,0U); }}') - return renamed+f''' + if not menu_fields: + raise ValueError("Missing verified six-column menu fields") + event = TRACK_OPTIONS[name] + invocation = ( + "{ extern int dkr_legacy_track_menu(uint8_t*,recomp_context*,unsigned,const uint32_t*,unsigned); " + "static const uint32_t menu_fields[]={" + menu_fields + "}; " + f"dkr_legacy_track_menu(rdram,ctx,{event}U,menu_fields,0U); " + + invocation + + f" dkr_legacy_track_menu(rdram,ctx,{event + 1}U,menu_fields,0U); }}" + ) + return ( + renamed + + f""" RECOMP_FUNC void {name}(uint8_t* rdram, recomp_context* ctx) {{ if(!dkr_code_hook_active(0x{address:08X}U)) {{ {invocation} return; }} gpr code_in[4]={{ctx->r4,ctx->r5,ctx->r6,ctx->r7}}; @@ -169,12 +272,17 @@ def wrap(name: str, body: str, address: int, owned: bool=False,menu_fields: str= if(!code_repeat)break; }} }} -''' +""" + ) -def table_source(items: list[dict],revision: int,owned: bool=False) -> str: - prefix=f'dkr_experimental_v{revision}' if owned else f'dkr_v{revision}' - rows='\n'.join(f' {{0x{e["address"]:08X}U,{e["name"]},{int(e["protected"])},{int(e["protected_hook"])},{e["before_event"]},{e["after_event"]},"{e["name"]}",{int(e["protected_call"])},{int(e["curated_call"])}}},' for e in items) - return f'''#include "funcs.h" + +def table_source(items: list[dict], revision: int, owned: bool = False) -> str: + prefix = f"dkr_experimental_v{revision}" if owned else f"dkr_v{revision}" + rows = "\n".join( + f' {{0x{e["address"]:08X}U,{e["name"]},{int(e["protected"])},{int(e["protected_hook"])},{e["before_event"]},{e["after_event"]},"{e["name"]}",{int(e["protected_call"])},{int(e["curated_call"])}}},' + for e in items + ) + return f"""#include "funcs.h" #include "mods/magic_code_dispatch.h" static const dkr_code_dispatch_entry entries[]={{ {rows} @@ -182,10 +290,11 @@ def table_source(items: list[dict],revision: int,owned: bool=False) -> str: const dkr_code_dispatch_entry* {prefix}_code_dispatch_table(size_t* count) {{ *count=sizeof(entries)/sizeof(entries[0]);return entries; }} -''' +""" + def owned_header() -> str: - return '''#include "mods/magic_code_dispatch.h" + return """#include "mods/magic_code_dispatch.h" static inline int dkr_probe_code_hook(unsigned leaving,uint32_t address,uint8_t* ram,recomp_context* context) { const uint64_t args[]={address};uint64_t result=0; const int handled=dkr_probe_mod_dispatch(leaving?"dkr_code_leave":"dkr_code_enter", @@ -207,43 +316,72 @@ def owned_header() -> str: dkr_probe_block("code-world-lap-address-unavailable"); return (uint32_t)result; } -''' - -def stage(source: Path,output: Path,revision: int,config: Path|None=None) -> None: - source=source.resolve();output=output.resolve() - if output.exists() or source==output or source in output.parents: - raise ValueError('Code dispatch staging requires a NEW directory outside its protected input') - functions={};files={} - for path in sorted(source.glob('*.c')): - text=path.read_text(encoding='utf-8');matches=list(DEFINITION.finditer(text));parts=[] - for i,m in enumerate(matches): - name=m[1] - if name in functions:raise ValueError(f'Duplicate guest definition {name}') - body=text[m.start():matches[i+1].start() if i+1 None: + source = source.resolve() + output = output.resolve() + if output.exists() or source == output or source in output.parents: + raise ValueError("Code dispatch staging requires a NEW directory outside its protected input") + functions = {} + files = {} + for path in sorted(source.glob("*.c")): + text = path.read_text(encoding="utf-8") + matches = list(DEFINITION.finditer(text)) + parts = [] + for i, m in enumerate(matches): + name = m[1] + if name in functions: + raise ValueError(f"Duplicate guest definition {name}") + body = text[m.start() : matches[i + 1].start() if i + 1 < len(matches) else len(text)] + functions[name] = body + parts.append(name) + files[path.name] = (text[: matches[0].start()] if matches else text, parts) + fixes = policy_names(source, revision, config) | set(TRACK_OPTIONS) + menu_fields = track_menu_fields(functions) + items = entries(functions, port_fixes=fixes) + by_address = {e["address"]: e for e in items} + grants = { + name: by_address[int(m[1], 16)] + for name, body in functions.items() + if (m := ADDRESS.search(body)) and int(m[1], 16) in by_address + } output.mkdir(parents=True) for path in sorted(source.iterdir()): - if path.is_symlink() or not path.is_file():raise ValueError('Unexpected generated payload entry') - if path.suffix!='.c':(output/path.name).write_bytes(path.read_bytes()) - for filename,(header,names) in files.items(): - pieces=[header,'#include "mods/magic_code_dispatch.h"\n'] + if path.is_symlink() or not path.is_file(): + raise ValueError("Unexpected generated payload entry") + if path.suffix != ".c": + (output / path.name).write_bytes(path.read_bytes()) + for filename, (header, names) in files.items(): + pieces = [header, '#include "mods/magic_code_dispatch.h"\n'] for name in names: - pieces.append(wrap(name,functions[name],grants[name]['address'],menu_fields=menu_fields) if name in grants else indirect_calls(functions[name],False)) - (output/filename).write_text(''.join(pieces),encoding='utf-8',newline='\n') - (output/'magic_code_dispatch.c').write_text(table_source(items,revision),encoding='utf-8',newline='\n') - manifest=dict(schema=2,revision=revision,protected_policy=sorted(fixes),functions=items,inputs={ - p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(source.iterdir()) if p.is_file()}) - (output/'magic-code-dispatch.json').write_text(json.dumps(manifest,sort_keys=True,indent=2)+'\n',encoding='utf-8',newline='\n') - -if __name__=='__main__': - p=argparse.ArgumentParser(description=__doc__) - p.add_argument('source',type=Path);p.add_argument('output',type=Path) - p.add_argument('--revision',type=int,choices=(77,80),required=True) - p.add_argument('--policy-config',type=Path,help='Complete config that generated this input (default: sibling v77/v80.toml)') - a=p.parse_args();stage(a.source,a.output,a.revision,a.policy_config) + pieces.append( + wrap(name, functions[name], grants[name]["address"], menu_fields=menu_fields) + if name in grants + else indirect_calls(functions[name], False) + ) + (output / filename).write_text("".join(pieces), encoding="utf-8", newline="\n") + (output / "magic_code_dispatch.c").write_text(table_source(items, revision), encoding="utf-8", newline="\n") + manifest = dict( + schema=2, + revision=revision, + protected_policy=sorted(fixes), + functions=items, + inputs={p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(source.iterdir()) if p.is_file()}, + ) + (output / "magic-code-dispatch.json").write_text( + json.dumps(manifest, sort_keys=True, indent=2) + "\n", encoding="utf-8", newline="\n" + ) + + +if __name__ == "__main__": + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("source", type=Path) + p.add_argument("output", type=Path) + p.add_argument("--revision", type=int, choices=(77, 80), required=True) + p.add_argument( + "--policy-config", type=Path, help="Complete config that generated this input (default: sibling v77/v80.toml)" + ) + a = p.parse_args() + stage(a.source, a.output, a.revision, a.policy_config) diff --git a/scripts/stage_magic_code_sdk.py b/scripts/stage_magic_code_sdk.py index 05ce82aa..fd9c8a6f 100644 --- a/scripts/stage_magic_code_sdk.py +++ b/scripts/stage_magic_code_sdk.py @@ -1,37 +1,75 @@ #!/usr/bin/env python3 """Stage authored SDK files and an optional explicit sample; never activate code.""" + import argparse import json from pathlib import Path import shutil import zipfile + from render_magic_code_guide import render as render_guide + def main(): - p=argparse.ArgumentParser(description=__doc__) - p.add_argument('--output',type=Path,required=True) - p.add_argument('--example',type=Path) - p.add_argument('--difficulty-example',type=Path) - a=p.parse_args();root=Path(__file__).resolve().parents[1];sdk=root/'modding' - if a.output.exists():raise ValueError('Choose a new SDK staging directory') - sources=[sdk/name for name in ('SDK.txt','USER-GUIDE.txt','MODDERS-GUIDE.txt','build.py','tools.py','relocations.py','runtime.c','mod.ld','reference-sdk-only.toml')] - for name in ('include','examples','reference-us-v77','reference-us-v80'): - sources+=sorted(f for f in (sdk/name).rglob('*') if f.is_file()) + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--output", type=Path, required=True) + p.add_argument("--example", type=Path) + p.add_argument("--difficulty-example", type=Path) + a = p.parse_args() + root = Path(__file__).resolve().parents[1] + sdk = root / "modding" + if a.output.exists(): + raise ValueError("Choose a new SDK staging directory") + sources = [ + sdk / name + for name in ( + "SDK.txt", + "USER-GUIDE.txt", + "MODDERS-GUIDE.txt", + "build.py", + "tools.py", + "relocations.py", + "runtime.c", + "mod.ld", + "reference-sdk-only.toml", + ) + ] + for name in ("include", "examples", "reference-us-v77", "reference-us-v80"): + sources += sorted(f for f in (sdk / name).rglob("*") if f.is_file()) for source in sources: - if source.is_symlink() or source.suffix not in ('.txt','.py','.c','.h','.hpp','.inl','.inc','.ld','.toml','.json'): - raise ValueError('SDK contains an unexpected source type') - target=a.output/source.relative_to(sdk);target.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(source,target) - shutil.copyfile(root/'LICENSE.md',a.output/'LICENSE.txt') - render_guide(sdk,a.output/'MODDERS-GUIDE.html') - for example,identity in ((a.example,'dkr_custom_trophies'),(a.difficulty_example,'dkr_ai_boss_difficulty')): - if not example:continue - if not example.is_file() or example.is_symlink() or example.stat().st_size>32*1024*1024:raise ValueError('Invalid example package') + if source.is_symlink() or source.suffix not in ( + ".txt", + ".py", + ".c", + ".h", + ".hpp", + ".inl", + ".inc", + ".ld", + ".toml", + ".json", + ): + raise ValueError("SDK contains an unexpected source type") + target = a.output / source.relative_to(sdk) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + shutil.copyfile(root / "LICENSE.md", a.output / "LICENSE.txt") + render_guide(sdk, a.output / "MODDERS-GUIDE.html") + for example, identity in ((a.example, "dkr_custom_trophies"), (a.difficulty_example, "dkr_ai_boss_difficulty")): + if not example: + continue + if not example.is_file() or example.is_symlink() or example.stat().st_size > 32 * 1024 * 1024: + raise ValueError("Invalid example package") with zipfile.ZipFile(example) as package: - expected={'dkr.json','mod.json','mod_syms.bin','mod_binary.bin'} - if len(package.infolist())!=4 or set(package.namelist())!=expected:raise ValueError('Only source-only examples are allowed') - meta=json.loads(package.read('mod.json')) - if meta.get('id')!=identity or meta.get('game_id')!='dkr':raise ValueError('Unexpected example identity') - shutil.copyfile(example,a.output/(identity+'.nrm')) - print('Staged SDK sources and optional manually importable mods. No activation or execution.') + expected = {"dkr.json", "mod.json", "mod_syms.bin", "mod_binary.bin"} + if len(package.infolist()) != 4 or set(package.namelist()) != expected: + raise ValueError("Only source-only examples are allowed") + meta = json.loads(package.read("mod.json")) + if meta.get("id") != identity or meta.get("game_id") != "dkr": + raise ValueError("Unexpected example identity") + shutil.copyfile(example, a.output / (identity + ".nrm")) + print("Staged SDK sources and optional manually importable mods. No activation or execution.") + -if __name__=='__main__':main() +if __name__ == "__main__": + main() diff --git a/scripts/startup_flow_policy.py b/scripts/startup_flow_policy.py index 35e4f805..f3a5bfd9 100644 --- a/scripts/startup_flow_policy.py +++ b/scripts/startup_flow_policy.py @@ -1,4 +1,5 @@ """Checked offline intro skip hooks; retail menu constructors stay intact.""" + import copy import struct @@ -8,40 +9,50 @@ # Exact retail sites, not a guessed delta between revisions. Title injection # is after storing the unmodified demo-table base and before reading its entry. SITES = { - 'us.v77': [('menu_title_screen_init', 0x8008353C, 0x240E0001, 'dkr_skip_intro_audio'), - ('menu_logo_screen_loop', 0x80082B84, 0x3C0E8000, 'dkr_skip_intro_logos'), - ('menu_title_screen_init', 0x80083704, 0x80450001, 'dkr_skip_intro_title'), - ('menu_title_screen_loop', 0x800839E4, 0x27BDFFD0, 'dkr_skip_intro_character_select')], - 'us.v80': [('menu_title_screen_init', 0x80083974, 0x240E0001, 'dkr_skip_intro_audio'), - ('menu_logo_screen_loop', 0x80082FD4, 0x3C0E8000, 'dkr_skip_intro_logos'), - ('menu_title_screen_init', 0x80083B3C, 0x80450001, 'dkr_skip_intro_title'), - ('menu_title_screen_loop', 0x80083E1C, 0x27BDFFD0, 'dkr_skip_intro_character_select')], + "us.v77": [ + ("menu_title_screen_init", 0x8008353C, 0x240E0001, "dkr_skip_intro_audio"), + ("menu_logo_screen_loop", 0x80082B84, 0x3C0E8000, "dkr_skip_intro_logos"), + ("menu_title_screen_init", 0x80083704, 0x80450001, "dkr_skip_intro_title"), + ("menu_title_screen_loop", 0x800839E4, 0x27BDFFD0, "dkr_skip_intro_character_select"), + ], + "us.v80": [ + ("menu_title_screen_init", 0x80083974, 0x240E0001, "dkr_skip_intro_audio"), + ("menu_logo_screen_loop", 0x80082FD4, 0x3C0E8000, "dkr_skip_intro_logos"), + ("menu_title_screen_init", 0x80083B3C, 0x80450001, "dkr_skip_intro_title"), + ("menu_title_screen_loop", 0x80083E1C, 0x27BDFFD0, "dkr_skip_intro_character_select"), + ], } def compose_startup_flow(policy, elf, revision): if reviewed_revision(elf) != revision or revision not in SITES: - raise ValueError('Intro skip requires a reviewed retail ELF') + raise ValueError("Intro skip requires a reviewed retail ELF") symbols = elf_functions(elf, (1, 2)) - words = {base + i: struct.unpack_from('>I', data, i)[0] - for base, data in elf_sections(elf) for i in range(0, len(data), 4)} + words = { + base + i: struct.unpack_from(">I", data, i)[0] + for base, data in elf_sections(elf) + for i in range(0, len(data), 4) + } result = copy.deepcopy(policy) - hooks = result.setdefault('functionHooks', []) + hooks = result.setdefault("functionHooks", []) for function, address, expected, callback in SITES[revision]: matches = symbols.get(function, set()) if len(matches) != 1: - raise ValueError(f'Ambiguous startup function: {function}') + raise ValueError(f"Ambiguous startup function: {function}") entry, size = next(iter(matches)) if not entry <= address < entry + size or words.get(address) != expected: - raise ValueError(f'Startup signature changed: {function}') - if callback == 'dkr_skip_intro_title' and words.get(address - 4) != 0xAC820000: - raise ValueError('Title demo-table storage boundary changed') - hook = dict(function=function, beforeVram=hex(address), - text=f'extern void {callback}(uint8_t*, recomp_context*); {callback}(rdram, ctx);', - reason='One-shot offline Play preference; preserve retail saves, title input, music and online startup.') - existing = [item for item in hooks if int(item['beforeVram'], 0) == address] + raise ValueError(f"Startup signature changed: {function}") + if callback == "dkr_skip_intro_title" and words.get(address - 4) != 0xAC820000: + raise ValueError("Title demo-table storage boundary changed") + hook = dict( + function=function, + beforeVram=hex(address), + text=f"extern void {callback}(uint8_t*, recomp_context*); {callback}(rdram, ctx);", + reason="One-shot offline Play preference; preserve retail saves, title input, music and online startup.", + ) + existing = [item for item in hooks if int(item["beforeVram"], 0) == address] if existing and existing != [hook]: - raise ValueError(f'Conflicting startup hook: {function}') + raise ValueError(f"Conflicting startup hook: {function}") if not existing: hooks.append(hook) return result diff --git a/scripts/stress_tests.py b/scripts/stress_tests.py index c1662115..66d158fd 100644 --- a/scripts/stress_tests.py +++ b/scripts/stress_tests.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Run seeded CTest flake rounds, stopping on failure and retaining evidence.""" + import argparse import copy import json @@ -15,174 +16,212 @@ MAX_SEED = 2**32 - 1 REQUIRED_TESTS = { - 'DKRDirectSession', 'DKRFriendService', 'DKRQuickJoinTransport', - 'DKRExperimentalSession', 'DKRExperimentalNetwork', - 'DKRExperimentalLaunch', 'DKRNetplayAdverseNetwork', + "DKRDirectSession", + "DKRFriendService", + "DKRQuickJoinTransport", + "DKRExperimentalSession", + "DKRExperimentalNetwork", + "DKRExperimentalLaunch", + "DKRNetplayAdverseNetwork", } OPTIONAL_TESTS = { - 'DKRFriendDelivery', 'DKRFriendLocalIntegration', - 'DKRQuickJoinLifecycle', 'DKRExperimentalQuickJoin', + "DKRFriendDelivery", + "DKRFriendLocalIntegration", + "DKRQuickJoinLifecycle", + "DKRExperimentalQuickJoin", } def positive(value): number = int(value) if number < 1: - raise argparse.ArgumentTypeError('must be positive') + raise argparse.ArgumentTypeError("must be positive") return number def seed_value(value): number = int(value) if not 0 <= number <= MAX_SEED: - raise argparse.ArgumentTypeError('seed must be an unsigned 32-bit integer') + raise argparse.ArgumentTypeError("seed must be an unsigned 32-bit integer") return number def collect_junit(path, selected, number): - text = path.read_text(encoding='utf-8') - if '= 4.1 is required for reproducible random seeds') - command = ['ctest', '--test-dir', str(build), '-C', args.config, '-L', '^flake$'] + raise ValueError("CTest >= 4.1 is required for reproducible random seeds") + command = ["ctest", "--test-dir", str(build), "-C", args.config, "-L", "^flake$"] if args.test_regex: - command += ['-R', args.test_regex] - discovery = subprocess.run([*command, '--show-only=json-v1'], capture_output=True, text=True, check=True) - tests = json.loads(discovery.stdout)['tests'] - selected = [test['name'] for test in tests] + command += ["-R", args.test_regex] + discovery = subprocess.run([*command, "--show-only=json-v1"], capture_output=True, text=True, check=True) + tests = json.loads(discovery.stdout)["tests"] + selected = [test["name"] for test in tests] if not selected or len(set(selected)) != len(selected): - raise ValueError('Selection must contain a nonempty set of distinct flake tests') + raise ValueError("Selection must contain a nonempty set of distinct flake tests") if not args.test_regex: - inventory = subprocess.run(['ctest', '--test-dir', str(build), '-C', args.config, - '--show-only=json-v1'], capture_output=True, text=True, check=True) - available = {test['name'] for test in json.loads(inventory.stdout)['tests']} + inventory = subprocess.run( + ["ctest", "--test-dir", str(build), "-C", args.config, "--show-only=json-v1"], + capture_output=True, + text=True, + check=True, + ) + available = {test["name"] for test in json.loads(inventory.stdout)["tests"]} expected = REQUIRED_TESTS | (OPTIONAL_TESTS & available) if expected - set(selected): - raise ValueError('Default flake suite is incomplete; missing labels/tests: ' - + ', '.join(sorted(expected - set(selected)))) - missing = [test['name'] for test in tests if not test.get('command')] + raise ValueError( + "Default flake suite is incomplete; missing labels/tests: " + + ", ".join(sorted(expected - set(selected))) + ) + missing = [test["name"] for test in tests if not test.get("command")] if missing: - raise ValueError('Selected tests have missing executables: ' + ', '.join(missing)) + raise ValueError("Selected tests have missing executables: " + ", ".join(missing)) seed = args.seed if args.seed is not None else secrets.randbelow(MAX_SEED - args.rounds + 2) if seed + args.rounds - 1 > MAX_SEED: - raise ValueError('Seed plus rounds exceeds the 32-bit seed range') - print(f'{version}\nBase seed: {seed}\nSelected {len(selected)} tests:\n' + '\n'.join(selected), flush=True) + raise ValueError("Seed plus rounds exceeds the 32-bit seed range") + print(f"{version}\nBase seed: {seed}\nSelected {len(selected)} tests:\n" + "\n".join(selected), flush=True) if args.list: return 0 output = args.output.resolve() output.mkdir(parents=True, exist_ok=False) - (output / 'ctest-tests.json').write_text(discovery.stdout, encoding='utf-8') - manifest = dict(schema=1, ctest_version=version, config=args.config, build_dir=str(build), - seed=seed, requested_rounds=args.rounds, - parallel=args.parallel, selected_tests=selected, rounds=[]) - aggregate = ET.Element('testsuites') + (output / "ctest-tests.json").write_text(discovery.stdout, encoding="utf-8") + manifest = dict( + schema=1, + ctest_version=version, + config=args.config, + build_dir=str(build), + seed=seed, + requested_rounds=args.rounds, + parallel=args.parallel, + selected_tests=selected, + rounds=[], + ) + aggregate = ET.Element("testsuites") status = 0 for number in range(1, args.rounds + 1): - folder = output / f'round-{number:03d}' + folder = output / f"round-{number:03d}" folder.mkdir() - junit = folder / 'junit.xml' - invocation = [*command, '--parallel', str(args.parallel), '--schedule-random', - '--schedule-random-seed', str(seed + number - 1), '--no-tests=error', - '--output-on-failure', '--output-junit', str(junit)] + junit = folder / "junit.xml" + invocation = [ + *command, + "--parallel", + str(args.parallel), + "--schedule-random", + "--schedule-random-seed", + str(seed + number - 1), + "--no-tests=error", + "--output-on-failure", + "--output-junit", + str(junit), + ] record = dict(round=number, seed=seed + number - 1, command=invocation) - manifest['rounds'].append(record) - (output / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n') - print(f'Round {number}/{args.rounds}, seed {record["seed"]}', flush=True) - with (folder / 'ctest.log').open('w', encoding='utf-8') as log: + manifest["rounds"].append(record) + (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + print(f"Round {number}/{args.rounds}, seed {record['seed']}", flush=True) + with (folder / "ctest.log").open("w", encoding="utf-8") as log: result = subprocess.run(invocation, stdout=log, stderr=subprocess.STDOUT, text=True) - record['returncode'] = result.returncode - temporary = build / 'Testing' / 'Temporary' + record["returncode"] = result.returncode + temporary = build / "Testing" / "Temporary" if temporary.is_dir(): - shutil.copytree(temporary, folder / 'CTestTemporary') + shutil.copytree(temporary, folder / "CTestTemporary") failed = False try: suite, failed = collect_junit(junit, selected, number) aggregate.append(suite) if failed: - record['error'] = 'Selected tests failed, skipped, or did not run' + record["error"] = "Selected tests failed, skipped, or did not run" except (OSError, ValueError, ET.ParseError) as error: - record['error'] = f'Incomplete or invalid JUnit: {error}' - record['passed'] = result.returncode == 0 and 'error' not in record - if not record['passed']: + record["error"] = f"Incomplete or invalid JUnit: {error}" + record["passed"] = result.returncode == 0 and "error" not in record + if not record["passed"]: status = 1 if not failed: # Make infrastructure failures visible to JUnit-only consumers. - suite = ET.SubElement(aggregate, 'testsuite', name=f'Round {number} infrastructure') - case = ET.SubElement(suite, 'testcase', name=f'round-{number:03d}: CTest evidence', time='0') - ET.SubElement(case, 'error').text = record.get('error', f'CTest exited {result.returncode}') - print((folder / 'ctest.log').read_text(encoding='utf-8', errors='replace'), end='') + suite = ET.SubElement(aggregate, "testsuite", name=f"Round {number} infrastructure") + case = ET.SubElement(suite, "testcase", name=f"round-{number:03d}: CTest evidence", time="0") + ET.SubElement(case, "error").text = record.get("error", f"CTest exited {result.returncode}") + print((folder / "ctest.log").read_text(encoding="utf-8", errors="replace"), end="") break - manifest['passed'] = status == 0 - (output / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n') + manifest["passed"] = status == 0 + (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") ET.indent(aggregate) - ET.ElementTree(aggregate).write(output / 'junit.xml', encoding='utf-8', xml_declaration=True) - lines = ['## Flake stress tests', '', f'Base seed: {seed}; attempted {len(manifest["rounds"])}/{args.rounds} rounds.', - f'Result: {"passed" if not status else "FAILED (stopped at first failed round)"}.', '', - 'Seeds reproduce CTest scheduling order, not operating-system thread scheduling.', ''] - for record in manifest['rounds']: - lines.append(f'- Round {record["round"]}, seed {record["seed"]}: ' + ('passed' if record['passed'] else 'FAILED')) - if 'error' in record: - lines.append(' ' + escape(record['error'], 1000)) - lines += ['', *junit_summary(ET.tostring(aggregate, encoding='unicode'))] - (output / 'summary.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') - print(f'Evidence: {output}', flush=True) + ET.ElementTree(aggregate).write(output / "junit.xml", encoding="utf-8", xml_declaration=True) + lines = [ + "## Flake stress tests", + "", + f"Base seed: {seed}; attempted {len(manifest['rounds'])}/{args.rounds} rounds.", + f"Result: {'passed' if not status else 'FAILED (stopped at first failed round)'}.", + "", + "Seeds reproduce CTest scheduling order, not operating-system thread scheduling.", + "", + ] + for record in manifest["rounds"]: + lines.append( + f"- Round {record['round']}, seed {record['seed']}: " + ("passed" if record["passed"] else "FAILED") + ) + if "error" in record: + lines.append(" " + escape(record["error"], 1000)) + lines += ["", *junit_summary(ET.tostring(aggregate, encoding="unicode"))] + (output / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + print(f"Evidence: {output}", flush=True) return status def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--build-dir', type=Path, required=True) - parser.add_argument('--config', default='Release') - parser.add_argument('--rounds', type=positive, default=5) - parser.add_argument('--parallel', type=positive, default=2) - parser.add_argument('--seed', type=seed_value) - parser.add_argument('--output', type=Path, default=Path('build/stress')) - parser.add_argument('--test-regex') - parser.add_argument('--list', action='store_true') + parser.add_argument("--build-dir", type=Path, required=True) + parser.add_argument("--config", default="Release") + parser.add_argument("--rounds", type=positive, default=5) + parser.add_argument("--parallel", type=positive, default=2) + parser.add_argument("--seed", type=seed_value) + parser.add_argument("--output", type=Path, default=Path("build/stress")) + parser.add_argument("--test-regex") + parser.add_argument("--list", action="store_true") args = parser.parse_args() if args.rounds > MAX_SEED + 1: - parser.error('Too many rounds for the seed range') + parser.error("Too many rounds for the seed range") try: return run(args) except (OSError, ValueError, KeyError, subprocess.SubprocessError) as error: - print(f'stress_tests: {error}', file=sys.stderr) + print(f"stress_tests: {error}", file=sys.stderr) return 2 -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) diff --git a/scripts/test_update_appimage.py b/scripts/test_update_appimage.py index 81162648..1e21f4d6 100644 --- a/scripts/test_update_appimage.py +++ b/scripts/test_update_appimage.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Run the copied updater against an isolated AppImage and retain its data.""" + import argparse import hashlib import json @@ -12,45 +13,86 @@ import time import uuid + def main(): - parser=argparse.ArgumentParser(description=__doc__) - for name in ("package","manifest","signature","helper","parent"): - parser.add_argument("--"+name) - args=parser.parse_args() + parser = argparse.ArgumentParser(description=__doc__) + for name in ("package", "manifest", "signature", "helper", "parent"): + parser.add_argument("--" + name) + args = parser.parse_args() if args.parent: - request=Path(args.parent);value=json.loads(request.read_text());job=request.parent - value["parent"]=os.getpid();request.write_text(json.dumps(value)) - subprocess.Popen([str(job/"DKR-R-Updater"),"--apply",str(request)],cwd=job) - end=time.monotonic()+10 - while time.monotonic()I',d,i)[0] for s,d in sections for i in range(0,len(d),4)} - sizes={'trackmenu_render_names':1484,'trackmenu_render_2D':1224, - 'texrect_draw_scaled':1604,'texrect_draw':584,'render_text_string':2044,'viewport_main':3424} - starts={} - for name,size in sizes.items(): - if len(symbols.get(name,()))!=1:raise ValueError('Ambiguous draw symbol: '+name) - start,actual=next(iter(symbols[name])) - if actual!=size:raise ValueError('Changed draw function bounds: '+name) - starts[name]=start - def checked(name,offset,expected): - pc=starts[name]+offset - if words.get(pc)!=expected:raise ValueError(f'Changed draw instruction: {name}+{offset:#x}') + if reviewed_revision(elf) != revision: + raise ValueError("Unreviewed Track Select draw ELF") + result = copy.deepcopy(policy) + words = {s + i: struct.unpack_from(">I", d, i)[0] for s, d in sections for i in range(0, len(d), 4)} + sizes = { + "trackmenu_render_names": 1484, + "trackmenu_render_2D": 1224, + "texrect_draw_scaled": 1604, + "texrect_draw": 584, + "render_text_string": 2044, + "viewport_main": 3424, + } + starts = {} + for name, size in sizes.items(): + if len(symbols.get(name, ())) != 1: + raise ValueError("Ambiguous draw symbol: " + name) + start, actual = next(iter(symbols[name])) + if actual != size: + raise ValueError("Changed draw function bounds: " + name) + starts[name] = start + + def checked(name, offset, expected): + pc = starts[name] + offset + if words.get(pc) != expected: + raise ValueError(f"Changed draw instruction: {name}+{offset:#x}") return pc - def hook(name,offset,expected,event,param='0U',extra=''): - pc=checked(name,offset,expected) - if any(int(h['beforeVram'],0)==pc for h in result['functionHooks']): - raise ValueError(f'Draw hook owner conflict: {name}+{offset:#x}') - call=f'dkr_track_select_draw(rdram, ctx, {event}U, {param})' - result['functionHooks'].append({'function':name,'beforeVram':hex(pc), - 'text':'extern unsigned dkr_track_select_draw(uint8_t*, recomp_context*, unsigned, unsigned); '+ - (extra.replace('$CALL',call) if extra else call+';'), - 'reason':'Bounded Track Select-only signed drawing; local aspect never enters CPU replay.'}) - def patch(name,offset,expected,value): - pc=checked(name,offset,expected) - if any(int(p['vram'],0)==pc for p in result['instructionPatches']): - raise ValueError(f'Draw patch owner conflict: {name}+{offset:#x}') - result['instructionPatches'].append({'function':name,'vram':hex(pc),'value':hex(value), - 'reason':'Use the dedicated 15-cell stack table / scoped rectangle guard; preserve native delay slots.'}) - name='trackmenu_render_names' - patch(name,0,0x27BDFF80,0x27BDFE80) # 0x180-byte frame, table [sp+0x90,sp+0x180) - patch(name,0x5C8,0x27BD0080,0x27BD0180) - details=next(iter(symbols['gTrackSelectRenderDetails']))[0] - patch(name,0xAC,0x26100000|(details&0xFFFF),0x27B00090) - patch(name,0x508,0x26100000|(details&0xFFFF),0x27B00090) - patch(name,0x500,0x26730000|((details+144)&0xFFFF),0x27B30180) - patch(name,0x11C,0x2416FFFF,0x2416FFFE) # j=-2 - patch(name,0x45C,0x24010002,0x24010003) # j<3 - hook(name,4,0xAFBF0054,10) - dl=next(iter(symbols['sMenuCurrDisplayList']))[0] - hook(name,0x4B4,0x8E230000,0,hex(dl)+'U') - hook(name,0x584,0x3C028012,1,hex(dl)+'U') + + def hook(name, offset, expected, event, param="0U", extra=""): + pc = checked(name, offset, expected) + if any(int(h["beforeVram"], 0) == pc for h in result["functionHooks"]): + raise ValueError(f"Draw hook owner conflict: {name}+{offset:#x}") + call = f"dkr_track_select_draw(rdram, ctx, {event}U, {param})" + result["functionHooks"].append( + { + "function": name, + "beforeVram": hex(pc), + "text": "extern unsigned dkr_track_select_draw(uint8_t*, recomp_context*, unsigned, unsigned); " + + (extra.replace("$CALL", call) if extra else call + ";"), + "reason": "Bounded Track Select-only signed drawing; local aspect never enters CPU replay.", + } + ) + + def patch(name, offset, expected, value): + pc = checked(name, offset, expected) + if any(int(p["vram"], 0) == pc for p in result["instructionPatches"]): + raise ValueError(f"Draw patch owner conflict: {name}+{offset:#x}") + result["instructionPatches"].append( + { + "function": name, + "vram": hex(pc), + "value": hex(value), + "reason": "Use the dedicated 15-cell stack table / scoped rectangle guard; preserve native delay slots.", + } + ) + + name = "trackmenu_render_names" + patch(name, 0, 0x27BDFF80, 0x27BDFE80) # 0x180-byte frame, table [sp+0x90,sp+0x180) + patch(name, 0x5C8, 0x27BD0080, 0x27BD0180) + details = next(iter(symbols["gTrackSelectRenderDetails"]))[0] + patch(name, 0xAC, 0x26100000 | (details & 0xFFFF), 0x27B00090) + patch(name, 0x508, 0x26100000 | (details & 0xFFFF), 0x27B00090) + patch(name, 0x500, 0x26730000 | ((details + 144) & 0xFFFF), 0x27B30180) + patch(name, 0x11C, 0x2416FFFF, 0x2416FFFE) # j=-2 + patch(name, 0x45C, 0x24010002, 0x24010003) # j<3 + hook(name, 4, 0xAFBF0054, 10) + dl = next(iter(symbols["sMenuCurrDisplayList"]))[0] + hook(name, 0x4B4, 0x8E230000, 0, hex(dl) + "U") + hook(name, 0x584, 0x3C028012, 1, hex(dl) + "U") # Native horizontal image-half tests only; vertical/opacity gates unchanged. - name='trackmenu_render_2D' - for offset,word in [(0x28C,0x10200018),(0x294,0x1A800016),(0x2F4,0x10200013),(0x304,0x1940000F)]: - patch(name,offset,word,0) - name='texrect_draw_scaled' - hook(name,0x330,0x102000A5,2,extra='if ($CALL) ctx->r1=1;') - hook(name,0x3F8,0x1A400073,2,extra='ctx->r1=$CALL || (int32_t)ctx->r18>0;') - patch(name,0x3F8,0x1A400073,0x10200073) - hook(name,0x4E0,0x05610006,3) - hook(name,0x5C8,0x8E3F0008,4) - name='texrect_draw' - hook(name,0xEC,0x19600039,2,extra='ctx->r1=$CALL || (int32_t)ctx->r11>0;') - patch(name,0xEC,0x19600039,0x10200039) - hook(name,0xFC,0x04C10004,5) - hook(name,0x1D4,0x8D480008,6) - name='render_text_string' - hook(name,0x670,0x04A10006,7) - hook(name,0x720,0x3C0E8013,8) + name = "trackmenu_render_2D" + for offset, word in [(0x28C, 0x10200018), (0x294, 0x1A800016), (0x2F4, 0x10200013), (0x304, 0x1940000F)]: + patch(name, offset, word, 0) + name = "texrect_draw_scaled" + hook(name, 0x330, 0x102000A5, 2, extra="if ($CALL) ctx->r1=1;") + hook(name, 0x3F8, 0x1A400073, 2, extra="ctx->r1=$CALL || (int32_t)ctx->r18>0;") + patch(name, 0x3F8, 0x1A400073, 0x10200073) + hook(name, 0x4E0, 0x05610006, 3) + hook(name, 0x5C8, 0x8E3F0008, 4) + name = "texrect_draw" + hook(name, 0xEC, 0x19600039, 2, extra="ctx->r1=$CALL || (int32_t)ctx->r11>0;") + patch(name, 0xEC, 0x19600039, 0x10200039) + hook(name, 0xFC, 0x04C10004, 5) + hook(name, 0x1D4, 0x8D480008, 6) + name = "render_text_string" + hook(name, 0x670, 0x04A10006, 7) + hook(name, 0x720, 0x3C0E8013, 8) # Observe the ORIGINAL viewport immediately after its native scissor packet. # The renderer repairs clipping; the CPU viewport and fullscreen zoom stay untouched. - name='viewport_main' - target=next(iter(symbols['viewport_rsp_set']))[0] - menu=next(iter(symbols['gCurrentMenuId']))[0] - hook(name,0x1F4,0x0C000000|((target>>2)&0x3FFFFFF),9,hex(menu)+'U') + name = "viewport_main" + target = next(iter(symbols["viewport_rsp_set"]))[0] + menu = next(iter(symbols["gCurrentMenuId"]))[0] + hook(name, 0x1F4, 0x0C000000 | ((target >> 2) & 0x3FFFFFF), 9, hex(menu) + "U") return result diff --git a/scripts/update_release.py b/scripts/update_release.py index 152d3b5c..00c1c19b 100644 --- a/scripts/update_release.py +++ b/scripts/update_release.py @@ -4,12 +4,13 @@ Uses OpenSSL's standard Ed25519, compatible with the unmodified Monocypher optional verifier. Package creation and signing are distinct from publishing. """ + import argparse import hashlib import json import os -import plistlib from pathlib import Path, PurePosixPath +import plistlib import re import shutil import stat @@ -19,12 +20,19 @@ MAX_PACKAGE = 512 * 1024 * 1024 MAX_EXTRACTED = 1024 * 1024 * 1024 -PROTECTED = {"portable.txt", "dkr-runtime-data", "mods", "online-mods", "texture-packs", - "saves", "logs", "crash-dumps"} +PROTECTED = {"portable.txt", "dkr-runtime-data", "mods", "online-mods", "texture-packs", "saves", "logs", "crash-dumps"} GAME_EXTENSIONS = {".z64", ".v64", ".n64", ".eep", ".mpk", ".sra", ".fla"} -PLATFORMS = {"windows-x64": "zip", "windows-arm64": "zip", "linux-x86_64": "appimage", - "linux-aarch64": "appimage", "macos-arm64": "zip", "macos-x86_64": "zip", - "android-arm64": "apk", "ios-arm64": "ipa"} +PLATFORMS = { + "windows-x64": "zip", + "windows-arm64": "zip", + "linux-x86_64": "appimage", + "linux-aarch64": "appimage", + "macos-arm64": "zip", + "macos-x86_64": "zip", + "android-arm64": "apk", + "ios-arm64": "ipa", +} + def safe_path(name): if not name or len(name) > 1024 or name.startswith("/") or name.endswith("/"): @@ -39,29 +47,35 @@ def safe_path(name): return False return True + def protected(name): top = name.split("/", 1)[0].lower() return top in PROTECTED or top.startswith(".dkr-update") or PurePosixPath(name).suffix.lower() in GAME_EXTENSIONS + def canonical(value): return (json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + "\n").encode() + def digest(path): with open(path, "rb") as source: return hashlib.file_digest(source, "sha256").hexdigest() + def run_ssl(*args): openssl = shutil.which("openssl") if not openssl: raise ValueError("OpenSSL with Ed25519 support is required for release signing.") return subprocess.check_output([openssl, *map(str, args)], stderr=subprocess.PIPE) + def public_key(private): der = run_ssl("pkey", "-in", private, "-pubout", "-outform", "DER") prefix = bytes.fromhex("302a300506032b6570032100") if len(der) != len(prefix) + 32 or not der.startswith(prefix): raise ValueError("Release key is not Ed25519.") - return der[len(prefix):].hex() + return der[len(prefix) :].hex() + def generate_key(path): path = Path(path).absolute() @@ -71,12 +85,14 @@ def generate_key(path): run_ssl("genpkey", "-algorithm", "ED25519", "-out", path) if os.name == "nt": owner = subprocess.check_output(["whoami"], text=True).strip() - subprocess.run(["icacls", str(path), "/inheritance:r", "/grant:r", f"{owner}:(F)"], - check=True, stdout=subprocess.DEVNULL) + subprocess.run( + ["icacls", str(path), "/inheritance:r", "/grant:r", f"{owner}:(F)"], check=True, stdout=subprocess.DEVNULL + ) else: path.chmod(0o600) return public_key(path) + def prepare_tree(directory, version, platform): """Call after staging all program files, before building the archive.""" # Match native Version::parse's build-metadata support. Candidate bundles @@ -99,8 +115,8 @@ def prepare_tree(directory, version, platform): if "assets/updates/install.json" not in owned: owned.append("assets/updates/install.json") receipt.parent.mkdir(parents=True, exist_ok=True) - receipt.write_bytes(canonical({"schema": 1, "version": version, "platform": platform, - "files": sorted(owned)})) + receipt.write_bytes(canonical({"schema": 1, "version": version, "platform": platform, "files": sorted(owned)})) + def zip_table(path, platform): table = [] @@ -133,8 +149,7 @@ def zip_table(path, platform): raise ValueError("Windows update archives may not contain links.") with archive.open(entry) as source: sha = hashlib.file_digest(source, "sha256").hexdigest() - item = {"path": parts[1], "bytes": entry.file_size, "sha256": sha, - "mode": (mode & 0o777) or 0o644} + item = {"path": parts[1], "bytes": entry.file_size, "sha256": sha, "mode": (mode & 0o777) or 0o644} if linked: link = archive.read(entry).decode("utf-8") if len(link) > 1024 or link.startswith("/") or "\\" in link: @@ -153,9 +168,13 @@ def zip_table(path, platform): if not table: raise ValueError("Empty update archive.") if platform.startswith("windows"): - required = {"DKR-R.exe", "libexec/dkr-r/DKR-R-Updater.exe", - "libexec/dkr-r/DKR-R-ModWorker.exe", "libexec/dkr-r/DKR-R-InputHost.exe", - "assets/updates/install.json"} + required = { + "DKR-R.exe", + "libexec/dkr-r/DKR-R-Updater.exe", + "libexec/dkr-r/DKR-R-ModWorker.exe", + "libexec/dkr-r/DKR-R-InputHost.exe", + "assets/updates/install.json", + } paths = {f["path"] for f in table} prefix = "libexec/dkr-r/" if "libexec/dkr-r/DKR-R-Runtime.exe" in paths else "" required |= {prefix + name for name in ("DKR-R-Runtime.exe", "SDL2.dll", "dxcompiler.dll", "dxil.dll")} @@ -163,10 +182,13 @@ def zip_table(path, platform): raise ValueError("Windows package lacks updater bootstrap/runtime/helper or ownership receipt.") return root, table + def create_manifest(args): expected = json.loads(Path(args.verification_key).read_text(encoding="utf-8"))["public_key"] if public_key(args.private_key) != expected: - raise ValueError("Signing key does not match the selected trusted public key. Use an explicit previous verification key only for a deliberate, old-key-signed bridge release.") + raise ValueError( + "Signing key does not match the selected trusted public key. Use an explicit previous verification key only for a deliberate, old-key-signed bridge release." + ) if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?", args.version): raise ValueError("Invalid release version.") if args.channel == "stable" and "-" in args.version: @@ -187,8 +209,14 @@ def create_manifest(args): kind = PLATFORMS[platform] if path.suffix.lower() != {"zip": ".zip", "appimage": ".appimage", "apk": ".apk", "ipa": ".ipa"}[kind]: raise ValueError("Package kind and platform disagree.") - item = {"platform": platform, "name": path.name, "kind": kind, "bytes": size, - "sha256": digest(path), "files": []} + item = { + "platform": platform, + "name": path.name, + "kind": kind, + "bytes": size, + "sha256": digest(path), + "files": [], + } minimums = {"windows-x64": "10.0", "windows-arm64": "10.0", "android-arm64": "28", "ios-arm64": "18.0"} if platform in minimums: item["minimum_os"] = minimums[platform] @@ -218,10 +246,16 @@ def create_manifest(args): signature = output / "DKR-R-update.sig" if manifest.exists() or signature.exists(): raise ValueError("Refusing to overwrite already signed update assets.") - value = {"schema": 1, "minimum_updater": 1, "version": args.version, "tag": args.tag, - "channel": args.channel, "commit": args.commit, - "notes": f"https://github.com/ThatGuyMcd/DKR-R/releases/tag/{args.tag}", - "artifacts": artifacts} + value = { + "schema": 1, + "minimum_updater": 1, + "version": args.version, + "tag": args.tag, + "channel": args.channel, + "commit": args.commit, + "notes": f"https://github.com/ThatGuyMcd/DKR-R/releases/tag/{args.tag}", + "artifacts": artifacts, + } encoded = canonical(value) if len(encoded) > 2 * 1024 * 1024: raise ValueError("Update manifest is too large.") @@ -229,8 +263,7 @@ def create_manifest(args): with tempfile.TemporaryDirectory(prefix="dkrr-release-sign-") as scratch: input_file, sig_file = Path(scratch) / "manifest.json", Path(scratch) / "signature.bin" input_file.write_bytes(encoded) - run_ssl("pkeyutl", "-sign", "-rawin", "-inkey", args.private_key, - "-in", input_file, "-out", sig_file) + run_ssl("pkeyutl", "-sign", "-rawin", "-inkey", args.private_key, "-in", input_file, "-out", sig_file) sig = sig_file.read_bytes() if len(sig) != 64: raise ValueError("Invalid Ed25519 signature length.") @@ -238,29 +271,40 @@ def create_manifest(args): signature.write_text(sig.hex(), encoding="ascii") return public_key(args.private_key) + def main(): parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="action", required=True) - key = sub.add_parser("keygen"); key.add_argument("--private-key", required=True) - pub = sub.add_parser("public-key"); pub.add_argument("--private-key", required=True) + key = sub.add_parser("keygen") + key.add_argument("--private-key", required=True) + pub = sub.add_parser("public-key") + pub.add_argument("--private-key", required=True) stage = sub.add_parser("prepare-tree") - stage.add_argument("--directory", required=True); stage.add_argument("--version", required=True) + stage.add_argument("--directory", required=True) + stage.add_argument("--version", required=True) stage.add_argument("--platform", choices=PLATFORMS, required=True) sign = sub.add_parser("sign") for name in ("version", "tag", "commit", "private-key", "output"): sign.add_argument("--" + name, required=True) sign.add_argument("--channel", choices=("stable", "beta"), default="stable") - sign.add_argument("--verification-key", default=str(Path(__file__).resolve().parent.parent / "packaging/update-public-key.json")) + sign.add_argument( + "--verification-key", default=str(Path(__file__).resolve().parent.parent / "packaging/update-public-key.json") + ) sign.add_argument("--artifact", action="append", required=True, metavar="PLATFORM=FILE") sign.add_argument("--minimum-os", action="append", default=[], metavar="PLATFORM=VERSION") args = parser.parse_args() try: - if args.action == "keygen": print("Public Ed25519 key:", generate_key(args.private_key)) - elif args.action == "public-key": print("Public Ed25519 key:", public_key(args.private_key)) - elif args.action == "prepare-tree": prepare_tree(args.directory, args.version, args.platform) - else: print("Signed update assets; public Ed25519 key:", create_manifest(args)) + if args.action == "keygen": + print("Public Ed25519 key:", generate_key(args.private_key)) + elif args.action == "public-key": + print("Public Ed25519 key:", public_key(args.private_key)) + elif args.action == "prepare-tree": + prepare_tree(args.directory, args.version, args.platform) + else: + print("Signed update assets; public Ed25519 key:", create_manifest(args)) except (ValueError, OSError, subprocess.CalledProcessError, zipfile.BadZipFile) as error: parser.exit(1, f"Update release preparation failed: {error}\n") + if __name__ == "__main__": main() diff --git a/scripts/validate_release_metadata.py b/scripts/validate_release_metadata.py index 0cf45e08..72393598 100644 --- a/scripts/validate_release_metadata.py +++ b/scripts/validate_release_metadata.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Validate DKR-R release versions, dependency patches, and final documentation.""" + from __future__ import annotations import hashlib @@ -23,16 +24,13 @@ def main() -> int: ): fail(f"VERSION is not a supported release/beta semantic version: {version!r}") - appdata = (ROOT / "packaging/linux/dkr-port.appdata.xml").read_text( - encoding="utf-8" - ) + appdata = (ROOT / "packaging/linux/dkr-port.appdata.xml").read_text(encoding="utf-8") if f' int: ) # Historical engineering notes may describe earlier release stages. Only # current release-facing documents must avoid stale release wording. - for relative in ("README.md", "THIRD_PARTY.md", "docs/README.md", - "docs/BUILDING.md", "docs/PLATFORMS.md", - "packaging/RELEASE-README.md", "packaging/MACOS-BUILD-README.md"): + for relative in ( + "README.md", + "THIRD_PARTY.md", + "docs/README.md", + "docs/BUILDING.md", + "docs/PLATFORMS.md", + "packaging/RELEASE-README.md", + "packaging/MACOS-BUILD-README.md", + ): path = ROOT / relative match = forbidden.search(path.read_text(encoding="utf-8")) if match: diff --git a/scripts/verify_android_compile.py b/scripts/verify_android_compile.py index 4e4412dc..d538b4c5 100644 --- a/scripts/verify_android_compile.py +++ b/scripts/verify_android_compile.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Verify ROM-free Android objects and the linked platform subset.""" + from __future__ import annotations import argparse @@ -10,71 +11,81 @@ import sys OBJECTS = { - 'DKRAndroidCompileCheck': ('android_platform.cpp', 'android_rt64_window.cpp', 'mobile_ui.cpp'), - 'DKRAndroidPlatformLinkCheck': ('android_file_dialog.cpp', 'runtime_portable.cpp', 'runtime_support.cpp'), + "DKRAndroidCompileCheck": ("android_platform.cpp", "android_rt64_window.cpp", "mobile_ui.cpp"), + "DKRAndroidPlatformLinkCheck": ("android_file_dialog.cpp", "runtime_portable.cpp", "runtime_support.cpp"), } def elf_architecture(description, kind): - for name, expected in (('Class', 'ELF64'), ('Machine', 'AArch64'), ('Type', kind)): - if not re.search(r'^\s*' + name + r':\s+' + expected + r'\b', description, re.MULTILINE): - raise ValueError(f'Expected {name} {expected}') + for name, expected in (("Class", "ELF64"), ("Machine", "AArch64"), ("Type", kind)): + if not re.search(r"^\s*" + name + r":\s+" + expected + r"\b", description, re.MULTILINE): + raise ValueError(f"Expected {name} {expected}") def library_contract(description, shared_cpp=False): - elf_architecture(description, 'DYN') - alignments = [int(line.split()[-1], 16) for line in description.splitlines() - if line.strip().startswith('LOAD ')] + elf_architecture(description, "DYN") + alignments = [int(line.split()[-1], 16) for line in description.splitlines() if line.strip().startswith("LOAD ")] if not alignments or any(value < 16384 for value in alignments): - raise ValueError('Every ELF LOAD segment must have at least 16 KB alignment') - needed = re.findall(r'\(NEEDED\)\s+Shared library: \[([^]]+)\]', description) - if shared_cpp and not {'libc++_shared.so', 'libSDL2.so'} <= set(needed): - raise ValueError('Platform subset must link shared libc++ and SDL2') - note = re.search(r'NT_ANDROID_TYPE_IDENT\s+description data:\s*((?:[0-9a-f]{2} ?)+)', description) - data = bytes.fromhex(note[1]) if note else b'' + raise ValueError("Every ELF LOAD segment must have at least 16 KB alignment") + needed = re.findall(r"\(NEEDED\)\s+Shared library: \[([^]]+)\]", description) + if shared_cpp and not {"libc++_shared.so", "libSDL2.so"} <= set(needed): + raise ValueError("Platform subset must link shared libc++ and SDL2") + note = re.search(r"NT_ANDROID_TYPE_IDENT\s+description data:\s*((?:[0-9a-f]{2} ?)+)", description) + data = bytes.fromhex(note[1]) if note else b"" if len(data) != 132: - raise ValueError('Expected an Android NDK identity note') - api = int.from_bytes(data[:4], 'little') - ndk = data[4:68].split(b'\0')[0].decode('ascii') - build = data[68:].split(b'\0')[0].decode('ascii') - if api < 28 or (ndk, build) != ('r29', '14206865'): - raise ValueError(f'Expected API >= 28 and NDK r29 build 14206865, got {api}, {ndk}, {build}') - return {'architecture': 'AArch64', 'api': api, 'ndk': ndk, 'ndk_build': build, - 'load_alignments': alignments, 'needed': needed} + raise ValueError("Expected an Android NDK identity note") + api = int.from_bytes(data[:4], "little") + ndk = data[4:68].split(b"\0")[0].decode("ascii") + build = data[68:].split(b"\0")[0].decode("ascii") + if api < 28 or (ndk, build) != ("r29", "14206865"): + raise ValueError(f"Expected API >= 28 and NDK r29 build 14206865, got {api}, {ndk}, {build}") + return { + "architecture": "AArch64", + "api": api, + "ndk": ndk, + "ndk_build": build, + "load_alignments": alignments, + "needed": needed, + } def verify(build, readelf): objects = [] for target, sources in OBJECTS.items(): for source in sources: - matches = list((build / f'CMakeFiles/{target}.dir').rglob(source + '.o')) + matches = list((build / f"CMakeFiles/{target}.dir").rglob(source + ".o")) if len(matches) != 1: - raise ValueError(f'{source}: expected exactly one compiled platform object') - description = subprocess.check_output([str(readelf), '-h', str(matches[0])], text=True) - elf_architecture(description, 'REL') + raise ValueError(f"{source}: expected exactly one compiled platform object") + description = subprocess.check_output([str(readelf), "-h", str(matches[0])], text=True) + elf_architecture(description, "REL") objects.append(source) libraries = {} - for name, shared_cpp in (('libDKRAndroidPlatformLinkCheck.so', True), ('sdl2/libSDL2.so', False)): - description = subprocess.check_output([str(readelf), '--wide', '-h', '-l', '-d', '-n', str(build / name)], text=True) + for name, shared_cpp in (("libDKRAndroidPlatformLinkCheck.so", True), ("sdl2/libSDL2.so", False)): + description = subprocess.check_output( + [str(readelf), "--wide", "-h", "-l", "-d", "-n", str(build / name)], text=True + ) libraries[name] = library_contract(description, shared_cpp) - return {'scope': 'Device compilation and platform-subset linking only; no APK or gameplay validation', - 'objects': objects, 'libraries': libraries} + return { + "scope": "Device compilation and platform-subset linking only; no APK or gameplay validation", + "objects": objects, + "libraries": libraries, + } def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('build', type=Path) - parser.add_argument('--readelf', type=Path, required=True, help='NDK llvm-readelf executable') + parser.add_argument("build", type=Path) + parser.add_argument("--readelf", type=Path, required=True, help="NDK llvm-readelf executable") args = parser.parse_args(argv) try: report = verify(args.build, args.readelf) - (args.build / 'device-objects.json').write_text(json.dumps(report, indent=2) + '\n') - print(f'Verified {len(report["objects"])} arm64 objects and two Android libraries with 16 KB LOAD alignment.') + (args.build / "device-objects.json").write_text(json.dumps(report, indent=2) + "\n") + print(f"Verified {len(report['objects'])} arm64 objects and two Android libraries with 16 KB LOAD alignment.") return 0 except (OSError, ValueError, subprocess.SubprocessError) as error: - print(f'Android compile check: {error}', file=sys.stderr) + print(f"Android compile check: {error}", file=sys.stderr) return 1 -if __name__ == '__main__': +if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/verify_ios_compile.py b/scripts/verify_ios_compile.py index ec3dbcaa..eebfc6b2 100644 --- a/scripts/verify_ios_compile.py +++ b/scripts/verify_ios_compile.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Verify the ROM-free compile check produced arm64 iPhoneOS objects.""" + from __future__ import annotations import argparse @@ -9,53 +10,61 @@ import subprocess import sys -SOURCES = ('ios_platform.mm', 'ios_metal_layer.mm', 'ios_file_dialog.mm', - 'mobile_ui.cpp', 'runtime_portable.cpp', 'runtime_support.cpp') +SOURCES = ( + "ios_platform.mm", + "ios_metal_layer.mm", + "ios_file_dialog.mm", + "mobile_ui.cpp", + "runtime_portable.cpp", + "runtime_support.cpp", +) def inspect_object(path): - architecture = subprocess.check_output(['xcrun', 'lipo', '-archs', str(path)], text=True).strip() - if architecture != 'arm64': - raise ValueError(f'{path}: expected only arm64, got {architecture}') - description = subprocess.check_output(['xcrun', 'vtool', '-show-build', str(path)], text=True) - fields = dict(re.findall(r'^\s*(platform|minos|sdk)\s+(\S+)\s*$', description, re.MULTILINE)) - if fields.get('platform') != 'IOS' or fields.get('minos') != '18.0' or fields.get('sdk') in (None, '0.0'): - raise ValueError(f'{path}: expected iPhoneOS device object targeting iOS 18.0, got {fields}') - return {'architecture': architecture, **fields} + architecture = subprocess.check_output(["xcrun", "lipo", "-archs", str(path)], text=True).strip() + if architecture != "arm64": + raise ValueError(f"{path}: expected only arm64, got {architecture}") + description = subprocess.check_output(["xcrun", "vtool", "-show-build", str(path)], text=True) + fields = dict(re.findall(r"^\s*(platform|minos|sdk)\s+(\S+)\s*$", description, re.MULTILINE)) + if fields.get("platform") != "IOS" or fields.get("minos") != "18.0" or fields.get("sdk") in (None, "0.0"): + raise ValueError(f"{path}: expected iPhoneOS device object targeting iOS 18.0, got {fields}") + return {"architecture": architecture, **fields} def verify(build): records = [] for source in SOURCES: - matches = list((build / 'CMakeFiles/DKRIosCompileCheck.dir').rglob(source + '.o')) + matches = list((build / "CMakeFiles/DKRIosCompileCheck.dir").rglob(source + ".o")) if len(matches) != 1: - raise ValueError(f'{source}: expected exactly one compiled platform object, found {len(matches)}') - records.append({'source': source, **inspect_object(matches[0])}) - sdl_objects = list((build / 'sdl2/CMakeFiles/SDL2-static.dir').rglob('SDL_uikitappdelegate.m.o')) + raise ValueError(f"{source}: expected exactly one compiled platform object, found {len(matches)}") + records.append({"source": source, **inspect_object(matches[0])}) + sdl_objects = list((build / "sdl2/CMakeFiles/SDL2-static.dir").rglob("SDL_uikitappdelegate.m.o")) if len(sdl_objects) != 1: - raise ValueError('Expected compiled SDL2 UIKit application delegate') - records.append({'source': 'SDL_uikitappdelegate.m', **inspect_object(sdl_objects[0])}) - archive = build / 'libDKRIosCompileCheck.a' - architecture = subprocess.check_output(['xcrun', 'lipo', '-archs', str(archive)], text=True).strip() - if architecture != 'arm64': - raise ValueError(f'{archive}: expected arm64 validation archive') - return {'scope': 'Device compilation only; no application linking, ROM execution, or IPA validation', - 'objects': records} + raise ValueError("Expected compiled SDL2 UIKit application delegate") + records.append({"source": "SDL_uikitappdelegate.m", **inspect_object(sdl_objects[0])}) + archive = build / "libDKRIosCompileCheck.a" + architecture = subprocess.check_output(["xcrun", "lipo", "-archs", str(archive)], text=True).strip() + if architecture != "arm64": + raise ValueError(f"{archive}: expected arm64 validation archive") + return { + "scope": "Device compilation only; no application linking, ROM execution, or IPA validation", + "objects": records, + } def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('build', type=Path) + parser.add_argument("build", type=Path) args = parser.parse_args(argv) try: report = verify(args.build) - (args.build / 'device-objects.json').write_text(json.dumps(report, indent=2) + '\n') - print(f'Verified {len(report["objects"])} arm64 iPhoneOS device objects; compilation only.') + (args.build / "device-objects.json").write_text(json.dumps(report, indent=2) + "\n") + print(f"Verified {len(report['objects'])} arm64 iPhoneOS device objects; compilation only.") return 0 except (OSError, ValueError, subprocess.SubprocessError) as error: - print(f'iOS compile check: {error}', file=sys.stderr) + print(f"iOS compile check: {error}", file=sys.stderr) return 1 -if __name__ == '__main__': +if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/water_patch_baseline.py b/scripts/water_patch_baseline.py index 78054062..bd675b87 100644 --- a/scripts/water_patch_baseline.py +++ b/scripts/water_patch_baseline.py @@ -1,28 +1,53 @@ """Read composed patch layers back to their baseline, entirely in memory.""" + from pathlib import Path -def read_before(stage,path,patches,text=None): + +def read_before(stage, path, patches, text=None): # An authored composer may supply an in-memory overlay while a build still # owns an earlier staged revision. Never write into that active build tree. - if text is None:text=(Path(stage)/path).read_text() - if path=='src/render/rt64_water_renderer.cpp' and 'WaterRenderer::prepareRays' in text: - text=text.replace(' if(!rayShader) {',' if(!rayShader || !rayLayout) {\n // Do not keep a shader if its companion layout creation failed.\n rayShader.reset();rayLayout.reset();') + if text is None: + text = (Path(stage) / path).read_text() + if path == "src/render/rt64_water_renderer.cpp" and "WaterRenderer::prepareRays" in text: + text = text.replace( + " if(!rayShader) {", + " if(!rayShader || !rayLayout) {\n // Do not keep a shader if its companion layout creation failed.\n rayShader.reset();rayLayout.reset();", + ) for patch in patches: - patch=Path(patch) - if not patch.exists():continue - hunks=[];active=False;old=[];new=[] + patch = Path(patch) + if not patch.exists(): + continue + hunks = [] + active = False + old = [] + new = [] + def finish(): - if active and (old or new):hunks.append((''.join(new),''.join(old))) + if active and (old or new): + hunks.append(("".join(new), "".join(old))) + for line in patch.read_text().splitlines(True): - if line.startswith('diff --git '): - finish();old=[];new=[];active=line.split()[2][2:]==path - elif line.startswith('@@ '):finish();old=[];new=[] - elif line.startswith(('--- ','+++ ')):continue - elif line.startswith(' '):old.append(line[1:]);new.append(line[1:]) - elif line.startswith('-'):old.append(line[1:]) - elif line.startswith('+'):new.append(line[1:]) + if line.startswith("diff --git "): + finish() + old = [] + new = [] + active = line.split()[2][2:] == path + elif line.startswith("@@ "): + finish() + old = [] + new = [] + elif line.startswith(("--- ", "+++ ")): + continue + elif line.startswith(" "): + old.append(line[1:]) + new.append(line[1:]) + elif line.startswith("-"): + old.append(line[1:]) + elif line.startswith("+"): + new.append(line[1:]) finish() - for new,old in reversed(hunks): - if text.count(new)!=1:raise ValueError(f'Cannot reverse {patch.name} in {path}') - text=text.replace(new,old,1) + for new, old in reversed(hunks): + if text.count(new) != 1: + raise ValueError(f"Cannot reverse {patch.name} in {path}") + text = text.replace(new, old, 1) return text diff --git a/scripts/water_profile_policy.py b/scripts/water_profile_policy.py index f11f53b4..9551cdc9 100644 --- a/scripts/water_profile_policy.py +++ b/scripts/water_profile_policy.py @@ -1,15 +1,53 @@ """Checked, opt-in timing hooks and PRIVATE scene qualification; no guest writes in release.""" + import copy import hashlib import struct -from compose_legacy_mod_policy import elf_sections, elf_functions -NAMES = ['waves_update', 'waves_visibility', 'waves_block_hq', 'func_800B92F4', - 'func_800B97A8', 'waves_render', 'func_800BB2F4', 'func_800BDC80', - 'obj_wave_height', 'waves_get_y', 'load_level_for_menu'] +from compose_legacy_mod_policy import elf_functions, elf_sections + +NAMES = [ + "waves_update", + "waves_visibility", + "waves_block_hq", + "func_800B92F4", + "func_800B97A8", + "waves_render", + "func_800BB2F4", + "func_800BDC80", + "obj_wave_height", + "waves_get_y", + "load_level_for_menu", +] HASHES = [ - ['0be5ad4f67d381a40fe42c284896ac308ce2cb0c3b018b38d0f5b0f198763139','27ff5fcb957487f23070e369b93659004bb664076d186c40868f8893da7cc026','660e7c6d4f701024bcd99946a7a53404b80e782417a2c80c7b86da8bdb38bc22','7cec4eebe04598847507c29e0f763b1ef266fd59a460835fa80f7ef1838f8445','9481d1434de804cdc562f814ff9e1f67a8d3b1aa35a5ba803814c30c55721cfa','ca06f0a082d997767983784407eb5cbb25af47b8e92d63673e76890c3f6661e6','f1761b4c091fddd4c1103cc5d678e193b0171e0cbfcd214da9599b47e33ce763','5ce68ee0a11b39ac85a9383e94479259e0d937f1b5f358aee7763e243c0c07a7','9113721a43bb596cdf15e160f197fe01fc885a18aab471f28b6806858c509bf0','3838141634a5c477c2c45058d6fd047f352853a694e8d60fbee0917cdb4d5c71','8d882096ad66310eee2c3c76eeb8352bc0ce58b268ef234e03d3ad2cc90eb027'], - ['4cc0a30e3f6fc7afefce1a171a61815412447f911adc413ade35536127eab84d','40d51f94ec2b588a3be7741ce09d8d6629ec81d8ff00d5160cd2dafb75ff2b10','bb4849d16b18bd89b622f86e030c9f7276d2d090f15e74ac203e09e162890985','8a021c9d576192773311270af588c4ac93c3f0f4d7d4ada1c63cf3e1f5c16427','dacc90a1480a4fa309776545ed6b7567b5d334abf3a383e114893d25dfe2b077','a7e33311d3a6c2ffeb1714a79aa9b4f7ff2b2c880e2818a48c21c25797848fe7','2392ea6922def84e4a29b29d013e1579bbf90be04eefc187f9bdb34c10a400c1','99d6897f469b9dd32ac530bc3933e98326d070ba2f503b4746024a48109ca820','f5f31ea6a7b27d96ac64846b38b825c415a858f2e0bf472cb4c5a10e9088b064','cf044b459b3dccd457d26c1bd309c552f1f10948458de39461f0c98d09f77789','cad1cb56072b5d9a260d56a783fb680bd54203aab268bf8b982b2b5027f01f2e']] + [ + "0be5ad4f67d381a40fe42c284896ac308ce2cb0c3b018b38d0f5b0f198763139", + "27ff5fcb957487f23070e369b93659004bb664076d186c40868f8893da7cc026", + "660e7c6d4f701024bcd99946a7a53404b80e782417a2c80c7b86da8bdb38bc22", + "7cec4eebe04598847507c29e0f763b1ef266fd59a460835fa80f7ef1838f8445", + "9481d1434de804cdc562f814ff9e1f67a8d3b1aa35a5ba803814c30c55721cfa", + "ca06f0a082d997767983784407eb5cbb25af47b8e92d63673e76890c3f6661e6", + "f1761b4c091fddd4c1103cc5d678e193b0171e0cbfcd214da9599b47e33ce763", + "5ce68ee0a11b39ac85a9383e94479259e0d937f1b5f358aee7763e243c0c07a7", + "9113721a43bb596cdf15e160f197fe01fc885a18aab471f28b6806858c509bf0", + "3838141634a5c477c2c45058d6fd047f352853a694e8d60fbee0917cdb4d5c71", + "8d882096ad66310eee2c3c76eeb8352bc0ce58b268ef234e03d3ad2cc90eb027", + ], + [ + "4cc0a30e3f6fc7afefce1a171a61815412447f911adc413ade35536127eab84d", + "40d51f94ec2b588a3be7741ce09d8d6629ec81d8ff00d5160cd2dafb75ff2b10", + "bb4849d16b18bd89b622f86e030c9f7276d2d090f15e74ac203e09e162890985", + "8a021c9d576192773311270af588c4ac93c3f0f4d7d4ada1c63cf3e1f5c16427", + "dacc90a1480a4fa309776545ed6b7567b5d334abf3a383e114893d25dfe2b077", + "a7e33311d3a6c2ffeb1714a79aa9b4f7ff2b2c880e2818a48c21c25797848fe7", + "2392ea6922def84e4a29b29d013e1579bbf90be04eefc187f9bdb34c10a400c1", + "99d6897f469b9dd32ac530bc3933e98326d070ba2f503b4746024a48109ca820", + "f5f31ea6a7b27d96ac64846b38b825c415a858f2e0bf472cb4c5a10e9088b064", + "cf044b459b3dccd457d26c1bd309c552f1f10948458de39461f0c98d09f77789", + "cad1cb56072b5d9a260d56a783fb680bd54203aab268bf8b982b2b5027f01f2e", + ], +] + def compose_water_profile(policy, elf): result = copy.deepcopy(policy) @@ -17,28 +55,52 @@ def compose_water_profile(policy, elf): additions = [] for region, name in enumerate(NAMES): entries = functions.get(name, set()) - if len(entries) != 1: raise ValueError(f'Ambiguous water function: {name}') + if len(entries) != 1: + raise ValueError(f"Ambiguous water function: {name}") base, size = next(iter(entries)) - bodies = [data[base-start:base-start+size] for start,data in sections if start <= base and base+size <= start+len(data)] + bodies = [ + data[base - start : base - start + size] + for start, data in sections + if start <= base and base + size <= start + len(data) + ] if len(bodies) != 1 or hashlib.sha256(bodies[0]).hexdigest() not in [h[region] for h in HASHES]: - raise ValueError(f'Unreviewed water function body: {name}') + raise ValueError(f"Unreviewed water function body: {name}") if region == 10: - sites = [(base, 'extern void dkr_water_private_scene(uint8_t*, recomp_context*); dkr_water_private_scene(rdram, ctx);')] + sites = [ + ( + base, + "extern void dkr_water_private_scene(uint8_t*, recomp_context*); dkr_water_private_scene(rdram, ctx);", + ) + ] else: - returns = [base+i for i in range(0,size,4) if struct.unpack_from('>I',bodies[0],i)[0] == 0x03E00008] - if returns != [base+size-8]: raise ValueError(f'Water return coverage changed: {name}') - sites = [(base, f'extern void dkr_water_profile_begin(uint32_t); dkr_water_profile_begin({region}U);'), - (returns[0], f'extern void dkr_water_profile_end(uint8_t*, uint32_t); dkr_water_profile_end(rdram, {region}U);')] + returns = [base + i for i in range(0, size, 4) if struct.unpack_from(">I", bodies[0], i)[0] == 0x03E00008] + if returns != [base + size - 8]: + raise ValueError(f"Water return coverage changed: {name}") + sites = [ + (base, f"extern void dkr_water_profile_begin(uint32_t); dkr_water_profile_begin({region}U);"), + ( + returns[0], + f"extern void dkr_water_profile_end(uint8_t*, uint32_t); dkr_water_profile_end(rdram, {region}U);", + ), + ] for pc, text in sites: - if any(int(p['vram'],0)==pc for p in policy.get('instructionPatches',[])): - raise ValueError(f'Water instrumentation overlaps an instruction patch: {name}') - existing = [h for h in result.get('functionHooks',[]) if int(h['beforeVram'],0)==pc] - if len(existing)>1: raise ValueError(f'Ambiguous existing hook at {pc:x}') + if any(int(p["vram"], 0) == pc for p in policy.get("instructionPatches", [])): + raise ValueError(f"Water instrumentation overlaps an instruction patch: {name}") + existing = [h for h in result.get("functionHooks", []) if int(h["beforeVram"], 0) == pc] + if len(existing) > 1: + raise ValueError(f"Ambiguous existing hook at {pc:x}") if existing: - if 'dkr_water_' in existing[0]['text']: raise ValueError('Water profile already composed') - existing[0]['text'] = text + ' ' + existing[0]['text'] + if "dkr_water_" in existing[0]["text"]: + raise ValueError("Water profile already composed") + existing[0]["text"] = text + " " + existing[0]["text"] else: - additions.append(dict(function=name,beforeVram=f'0x{pc:08X}',text=text, - reason='Read-only opt-in water timing; verified full MIPS body and sole return. Private scene override is compiled out of release.')) - result.setdefault('functionHooks',[]).extend(additions) + additions.append( + dict( + function=name, + beforeVram=f"0x{pc:08X}", + text=text, + reason="Read-only opt-in water timing; verified full MIPS body and sole return. Private scene override is compiled out of release.", + ) + ) + result.setdefault("functionHooks", []).extend(additions) return result diff --git a/tests/fixtures/pattern-guard/PG001/negative/patches/README.md b/tests/fixtures/pattern-guard/PG001/negative/patches/README.md new file mode 100644 index 00000000..f0a5a878 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG001/negative/patches/README.md @@ -0,0 +1 @@ +Dependency patches only. diff --git a/tests/fixtures/pattern-guard/PG001/negative/patches/manifest.json b/tests/fixtures/pattern-guard/PG001/negative/patches/manifest.json new file mode 100644 index 00000000..a1bbecc4 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG001/negative/patches/manifest.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 1, + "dependencies": [ + { + "name": "RT64", + "repositoryPath": "extern/rt64", + "expectedCommit": "0000000000000000000000000000000000000000", + "patches": [ + { + "path": "patches/rt64/0001-cache-limit.patch", + "sha256": "12d4abba5da0b4f2440519aab67a2e9dfb1e2606d32a2e912a063590b5c2b8ee" + } + ] + } + ] +} diff --git a/tests/fixtures/pattern-guard/PG001/negative/patches/rt64/0001-cache-limit.patch b/tests/fixtures/pattern-guard/PG001/negative/patches/rt64/0001-cache-limit.patch new file mode 100644 index 00000000..864c786f --- /dev/null +++ b/tests/fixtures/pattern-guard/PG001/negative/patches/rt64/0001-cache-limit.patch @@ -0,0 +1,6 @@ +diff --git a/src/render/rt64_texture_cache.cpp b/src/render/rt64_texture_cache.cpp +--- a/src/render/rt64_texture_cache.cpp ++++ b/src/render/rt64_texture_cache.cpp +@@ -1 +1 @@ +-int cache_limit = 1; ++int cache_limit = 2; diff --git a/tests/fixtures/pattern-guard/PG001/negative/runtime-recomp/cmake/Feature.cmake b/tests/fixtures/pattern-guard/PG001/negative/runtime-recomp/cmake/Feature.cmake new file mode 100644 index 00000000..2a845a43 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG001/negative/runtime-recomp/cmake/Feature.cmake @@ -0,0 +1 @@ +target_sources(DKRPortGame PRIVATE src/game/feature.cpp) diff --git a/tests/fixtures/pattern-guard/PG001/positive/patches/feature/CMakeLists.txt b/tests/fixtures/pattern-guard/PG001/positive/patches/feature/CMakeLists.txt new file mode 100644 index 00000000..91ffbfaf --- /dev/null +++ b/tests/fixtures/pattern-guard/PG001/positive/patches/feature/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(native) diff --git a/tests/fixtures/pattern-guard/PG001/positive/patches/feature/ReleaseOverlay.cmake b/tests/fixtures/pattern-guard/PG001/positive/patches/feature/ReleaseOverlay.cmake new file mode 100644 index 00000000..67f64c98 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG001/positive/patches/feature/ReleaseOverlay.cmake @@ -0,0 +1 @@ +message(STATUS "feature overlay") diff --git a/tests/fixtures/pattern-guard/PG002/negative/docs/history.md b/tests/fixtures/pattern-guard/PG002/negative/docs/history.md new file mode 100644 index 00000000..da93da93 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG002/negative/docs/history.md @@ -0,0 +1 @@ +Layers used `CMAKE_PROJECT_INCLUDE` and `cmake_language(DEFER`. diff --git a/tests/fixtures/pattern-guard/PG002/negative/runtime-recomp/cmake/Notes.cmake b/tests/fixtures/pattern-guard/PG002/negative/runtime-recomp/cmake/Notes.cmake new file mode 100644 index 00000000..5890f20a --- /dev/null +++ b/tests/fixtures/pattern-guard/PG002/negative/runtime-recomp/cmake/Notes.cmake @@ -0,0 +1,2 @@ +# CMAKE_PROJECT_INCLUDE is not used here. +set(DKR_NOTES ON) diff --git a/tests/fixtures/pattern-guard/PG002/negative/runtime-recomp/cmake/ProjectTests.cmake b/tests/fixtures/pattern-guard/PG002/negative/runtime-recomp/cmake/ProjectTests.cmake new file mode 100644 index 00000000..48e881db --- /dev/null +++ b/tests/fixtures/pattern-guard/PG002/negative/runtime-recomp/cmake/ProjectTests.cmake @@ -0,0 +1 @@ +cmake_language(DEFER CALL dkr_configure_test_metadata) diff --git a/tests/fixtures/pattern-guard/PG002/positive/.github/workflows/build.yml b/tests/fixtures/pattern-guard/PG002/positive/.github/workflows/build.yml new file mode 100644 index 00000000..c9850622 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG002/positive/.github/workflows/build.yml @@ -0,0 +1,4 @@ +jobs: + build: + steps: + - run: cmake -DCMAKE_PROJECT_INCLUDE=x.cmake . diff --git a/tests/fixtures/pattern-guard/PG002/positive/runtime-recomp/cmake/Overlay.cmake b/tests/fixtures/pattern-guard/PG002/positive/runtime-recomp/cmake/Overlay.cmake new file mode 100644 index 00000000..ec88d1d3 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG002/positive/runtime-recomp/cmake/Overlay.cmake @@ -0,0 +1 @@ +cmake_language(DEFER CALL dkr_apply_overlay) diff --git a/tests/fixtures/pattern-guard/PG002/positive/scripts/configure.sh b/tests/fixtures/pattern-guard/PG002/positive/scripts/configure.sh new file mode 100644 index 00000000..48b07281 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG002/positive/scripts/configure.sh @@ -0,0 +1 @@ +cmake -S runtime-recomp -B build -DCMAKE_PROJECT_INCLUDE=overlay.cmake diff --git a/tests/fixtures/pattern-guard/PG003/negative/runtime-recomp/CMakeLists.txt b/tests/fixtures/pattern-guard/PG003/negative/runtime-recomp/CMakeLists.txt new file mode 100644 index 00000000..e4a0532c --- /dev/null +++ b/tests/fixtures/pattern-guard/PG003/negative/runtime-recomp/CMakeLists.txt @@ -0,0 +1,9 @@ +add_executable(DKRPortGame src/game/main.cpp) +set_target_properties(DKRPortGame PROPERTIES OUTPUT_NAME "DKR-R-Runtime") +# Third-party SOURCES swaps are how staged dependency patches compile. +get_target_property(rt64_sources rt64 SOURCES) +list(FILTER rt64_sources EXCLUDE REGEX "(^|/)rt64_application_window\\.cpp$") +set_property(TARGET rt64 PROPERTY SOURCES "${rt64_sources}") +set_source_files_properties("${CMAKE_BINARY_DIR}/generated/stage/src/render/rt64_texture_cache.cpp" + TARGET_DIRECTORY rt64 PROPERTIES INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}/generated/stage/src/render") +target_sources(DKRPortGame PRIVATE src/game/feature.cpp) diff --git a/tests/fixtures/pattern-guard/PG003/negative/runtime-recomp/cmake/StagedPatch.cmake b/tests/fixtures/pattern-guard/PG003/negative/runtime-recomp/cmake/StagedPatch.cmake new file mode 100644 index 00000000..da8c2269 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG003/negative/runtime-recomp/cmake/StagedPatch.cmake @@ -0,0 +1,5 @@ +function(dkr_stage_patch target source stage_name patch) + get_target_property(sources ${target} SOURCES) + list(FILTER sources EXCLUDE REGEX "${stage_name}") + set_property(TARGET ${target} PROPERTY SOURCES "${sources}") +endfunction() diff --git a/tests/fixtures/pattern-guard/PG003/positive/runtime-recomp/CMakeLists.txt b/tests/fixtures/pattern-guard/PG003/positive/runtime-recomp/CMakeLists.txt new file mode 100644 index 00000000..4fd78a65 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG003/positive/runtime-recomp/CMakeLists.txt @@ -0,0 +1,8 @@ +add_executable(DKRPortGame src/game/main.cpp) +set_target_properties(DKRPortGame PROPERTIES SOURCES "${CMAKE_BINARY_DIR}/generated/main.cpp") +get_target_property(game_sources DKRPortGame SOURCES) +list(TRANSFORM game_sources REPLACE "src/game" "${CMAKE_BINARY_DIR}/generated") +set_property(TARGET DKRPortGame PROPERTY SOURCES "${game_sources}") +set_source_files_properties(src/game/main.cpp PROPERTIES + INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}/generated/feature") +dkr_repair_replace_source(DKRPortGame src/game/main.cpp "${CMAKE_BINARY_DIR}/generated/main.cpp") diff --git a/tests/fixtures/pattern-guard/PG004/negative/scripts/count_sources.py b/tests/fixtures/pattern-guard/PG004/negative/scripts/count_sources.py new file mode 100644 index 00000000..2c3e88dd --- /dev/null +++ b/tests/fixtures/pattern-guard/PG004/negative/scripts/count_sources.py @@ -0,0 +1,5 @@ +"""Report first-party source sizes; writes nothing.""" +from pathlib import Path + +text = Path("runtime-recomp/src/game/runtime_ui.cpp").read_text() +print(len(text.replace("\r\n", "\n"))) diff --git a/tests/fixtures/pattern-guard/PG004/negative/scripts/guest_hooks.py b/tests/fixtures/pattern-guard/PG004/negative/scripts/guest_hooks.py new file mode 100644 index 00000000..03f18e6b --- /dev/null +++ b/tests/fixtures/pattern-guard/PG004/negative/scripts/guest_hooks.py @@ -0,0 +1,5 @@ +"""Apply declared hooks (allow-listed).""" +from pathlib import Path + +text = Path("runtime-recomp/src/game/runtime_ui.cpp").read_text() +Path("out.c").write_text(text.replace("a", "b")) diff --git a/tests/fixtures/pattern-guard/PG004/negative/scripts/render_table.py b/tests/fixtures/pattern-guard/PG004/negative/scripts/render_table.py new file mode 100644 index 00000000..ba71fed6 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG004/negative/scripts/render_table.py @@ -0,0 +1,6 @@ +"""Writes a new table from data; no first-party text involved.""" +from pathlib import Path +import sys + +rows = Path(sys.argv[1]).read_text().splitlines() +Path(sys.argv[2]).write_text("\n".join(row.replace(",", " | ") for row in rows)) diff --git a/tests/fixtures/pattern-guard/PG004/positive/scripts/feature_pipeline.py b/tests/fixtures/pattern-guard/PG004/positive/scripts/feature_pipeline.py new file mode 100644 index 00000000..93d97902 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG004/positive/scripts/feature_pipeline.py @@ -0,0 +1,13 @@ +"""Stage a rewritten copy of a first-party source.""" +from pathlib import Path +import sys + + +def once(text, old, new): + if text.count(old) != 1: + raise SystemExit(f"expected one {old!r}") + return text.replace(old, new) + + +source = Path("runtime-recomp/src/game/runtime_ui.cpp").read_text() +(Path(sys.argv[1]) / "runtime_ui.cpp").write_text(once(source, "int sidebar = 1;", "int sidebar = 2;")) diff --git a/tests/fixtures/pattern-guard/PG005/negative/.gitattributes b/tests/fixtures/pattern-guard/PG005/negative/.gitattributes new file mode 100644 index 00000000..24e115f5 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG005/negative/.gitattributes @@ -0,0 +1 @@ +*.patch -whitespace diff --git a/tests/fixtures/pattern-guard/PG005/negative/docs/adr/0001-retire-source-capsule.md b/tests/fixtures/pattern-guard/PG005/negative/docs/adr/0001-retire-source-capsule.md new file mode 100644 index 00000000..f435f941 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG005/negative/docs/adr/0001-retire-source-capsule.md @@ -0,0 +1 @@ +The capsule is gone. diff --git a/tests/fixtures/pattern-guard/PG005/negative/modding/reference-us-v77/receipt.json b/tests/fixtures/pattern-guard/PG005/negative/modding/reference-us-v77/receipt.json new file mode 100644 index 00000000..131ea68c --- /dev/null +++ b/tests/fixtures/pattern-guard/PG005/negative/modding/reference-us-v77/receipt.json @@ -0,0 +1 @@ +{"sha256": "dcb1e8df777d341f2e96d4ee78cdc4337d32a6ddc86b4b60dbcc46fabfd31efc"} diff --git a/tests/fixtures/pattern-guard/PG005/negative/tools/other/manifest.json b/tests/fixtures/pattern-guard/PG005/negative/tools/other/manifest.json new file mode 100644 index 00000000..142712fc --- /dev/null +++ b/tests/fixtures/pattern-guard/PG005/negative/tools/other/manifest.json @@ -0,0 +1 @@ +{"files": []} diff --git a/tests/fixtures/pattern-guard/PG005/positive/.gitattributes b/tests/fixtures/pattern-guard/PG005/positive/.gitattributes new file mode 100644 index 00000000..f5a21db7 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG005/positive/.gitattributes @@ -0,0 +1 @@ +release/source-capsule/** -text -eol -whitespace diff --git a/tests/fixtures/pattern-guard/PG005/positive/patches/feature/stage-receipt.json b/tests/fixtures/pattern-guard/PG005/positive/patches/feature/stage-receipt.json new file mode 100644 index 00000000..131ea68c --- /dev/null +++ b/tests/fixtures/pattern-guard/PG005/positive/patches/feature/stage-receipt.json @@ -0,0 +1 @@ +{"sha256": "dcb1e8df777d341f2e96d4ee78cdc4337d32a6ddc86b4b60dbcc46fabfd31efc"} diff --git a/tests/fixtures/pattern-guard/PG005/positive/release/source-capsule/notes.txt b/tests/fixtures/pattern-guard/PG005/positive/release/source-capsule/notes.txt new file mode 100644 index 00000000..9163324e --- /dev/null +++ b/tests/fixtures/pattern-guard/PG005/positive/release/source-capsule/notes.txt @@ -0,0 +1 @@ +frozen sources diff --git a/tests/fixtures/pattern-guard/PG005/positive/tools/manifest.json b/tests/fixtures/pattern-guard/PG005/positive/tools/manifest.json new file mode 100644 index 00000000..f6962574 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG005/positive/tools/manifest.json @@ -0,0 +1 @@ +{"roles": {"runtime_ui.cpp": "replace"}} diff --git a/tests/fixtures/pattern-guard/PG006/negative/patches/manifest.json b/tests/fixtures/pattern-guard/PG006/negative/patches/manifest.json new file mode 100644 index 00000000..88efc540 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG006/negative/patches/manifest.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 1, + "dependencies": [ + { + "name": "RT64", + "repositoryPath": "extern/rt64", + "expectedCommit": "0000000000000000000000000000000000000000", + "patches": [ + { + "path": "patches/rt64/0001-new-file.patch", + "sha256": "2063a3c42bf83988edcfd224f0a4657525bff0ed20fc22e3f6665b944ecdf2a6" + } + ] + } + ] +} diff --git a/tests/fixtures/pattern-guard/PG006/negative/patches/rt64/0001-new-file.patch b/tests/fixtures/pattern-guard/PG006/negative/patches/rt64/0001-new-file.patch new file mode 100644 index 00000000..0f9384df --- /dev/null +++ b/tests/fixtures/pattern-guard/PG006/negative/patches/rt64/0001-new-file.patch @@ -0,0 +1,6 @@ +diff --git a/src/render/rt64_extra.cpp b/src/render/rt64_extra.cpp +new file mode 100644 +--- /dev/null ++++ b/src/render/rt64_extra.cpp +@@ -0,0 +1 @@ ++#include "rt64_extra.h" diff --git a/tests/fixtures/pattern-guard/PG006/negative/runtime-recomp/src/game/feature.cpp b/tests/fixtures/pattern-guard/PG006/negative/runtime-recomp/src/game/feature.cpp new file mode 100644 index 00000000..0e73b262 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG006/negative/runtime-recomp/src/game/feature.cpp @@ -0,0 +1 @@ +int feature_enabled = 1; diff --git a/tests/fixtures/pattern-guard/PG006/positive/patches/feature/hooks.toml b/tests/fixtures/pattern-guard/PG006/positive/patches/feature/hooks.toml new file mode 100644 index 00000000..30880315 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG006/positive/patches/feature/hooks.toml @@ -0,0 +1,2 @@ +[hook] +name = "feature" diff --git a/tests/fixtures/pattern-guard/PG006/positive/patches/feature/native/feature.cpp b/tests/fixtures/pattern-guard/PG006/positive/patches/feature/native/feature.cpp new file mode 100644 index 00000000..0e73b262 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG006/positive/patches/feature/native/feature.cpp @@ -0,0 +1 @@ +int feature_enabled = 1; diff --git a/tests/fixtures/pattern-guard/PG006/positive/patches/feature/native/feature.hpp b/tests/fixtures/pattern-guard/PG006/positive/patches/feature/native/feature.hpp new file mode 100644 index 00000000..432e356d --- /dev/null +++ b/tests/fixtures/pattern-guard/PG006/positive/patches/feature/native/feature.hpp @@ -0,0 +1 @@ +extern int feature_enabled; diff --git a/tests/fixtures/pattern-guard/PG007/negative/patches/manifest.json b/tests/fixtures/pattern-guard/PG007/negative/patches/manifest.json new file mode 100644 index 00000000..a1bbecc4 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG007/negative/patches/manifest.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 1, + "dependencies": [ + { + "name": "RT64", + "repositoryPath": "extern/rt64", + "expectedCommit": "0000000000000000000000000000000000000000", + "patches": [ + { + "path": "patches/rt64/0001-cache-limit.patch", + "sha256": "12d4abba5da0b4f2440519aab67a2e9dfb1e2606d32a2e912a063590b5c2b8ee" + } + ] + } + ] +} diff --git a/tests/fixtures/pattern-guard/PG007/negative/patches/rt64/0001-cache-limit.patch b/tests/fixtures/pattern-guard/PG007/negative/patches/rt64/0001-cache-limit.patch new file mode 100644 index 00000000..864c786f --- /dev/null +++ b/tests/fixtures/pattern-guard/PG007/negative/patches/rt64/0001-cache-limit.patch @@ -0,0 +1,6 @@ +diff --git a/src/render/rt64_texture_cache.cpp b/src/render/rt64_texture_cache.cpp +--- a/src/render/rt64_texture_cache.cpp ++++ b/src/render/rt64_texture_cache.cpp +@@ -1 +1 @@ +-int cache_limit = 1; ++int cache_limit = 2; diff --git a/tests/fixtures/pattern-guard/PG007/negative/runtime-recomp/CMakeLists.txt b/tests/fixtures/pattern-guard/PG007/negative/runtime-recomp/CMakeLists.txt new file mode 100644 index 00000000..b5eba5c5 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG007/negative/runtime-recomp/CMakeLists.txt @@ -0,0 +1,2 @@ +dkr_stage_patch(rt64 "${DKR_RT64_SOURCE}" staged + "${DKRPORT_ROOT}/patches/rt64/0001-cache-limit.patch" src/render/rt64_texture_cache.cpp) diff --git a/tests/fixtures/pattern-guard/PG007/negative/runtime-recomp/tests/cmake/build_iteration/CMakeLists.txt b/tests/fixtures/pattern-guard/PG007/negative/runtime-recomp/tests/cmake/build_iteration/CMakeLists.txt new file mode 100644 index 00000000..60876deb --- /dev/null +++ b/tests/fixtures/pattern-guard/PG007/negative/runtime-recomp/tests/cmake/build_iteration/CMakeLists.txt @@ -0,0 +1,2 @@ +dkr_stage_patch(fixture "${CMAKE_CURRENT_SOURCE_DIR}/source" first + "${CMAKE_CURRENT_SOURCE_DIR}/first.patch" value.txt) diff --git a/tests/fixtures/pattern-guard/PG007/positive/patches/manifest.json b/tests/fixtures/pattern-guard/PG007/positive/patches/manifest.json new file mode 100644 index 00000000..836c3fac --- /dev/null +++ b/tests/fixtures/pattern-guard/PG007/positive/patches/manifest.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 1, + "dependencies": [ + { + "name": "RT64", + "repositoryPath": "extern/rt64", + "expectedCommit": "0000000000000000000000000000000000000000", + "patches": [ + { + "path": "patches/rt64/0001-cache-limit.patch", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000" + } + ] + } + ] +} diff --git a/tests/fixtures/pattern-guard/PG007/positive/patches/rt64/0001-cache-limit.patch b/tests/fixtures/pattern-guard/PG007/positive/patches/rt64/0001-cache-limit.patch new file mode 100644 index 00000000..864c786f --- /dev/null +++ b/tests/fixtures/pattern-guard/PG007/positive/patches/rt64/0001-cache-limit.patch @@ -0,0 +1,6 @@ +diff --git a/src/render/rt64_texture_cache.cpp b/src/render/rt64_texture_cache.cpp +--- a/src/render/rt64_texture_cache.cpp ++++ b/src/render/rt64_texture_cache.cpp +@@ -1 +1 @@ +-int cache_limit = 1; ++int cache_limit = 2; diff --git a/tests/fixtures/pattern-guard/PG007/positive/patches/rt64/0002-unlisted.patch b/tests/fixtures/pattern-guard/PG007/positive/patches/rt64/0002-unlisted.patch new file mode 100644 index 00000000..864c786f --- /dev/null +++ b/tests/fixtures/pattern-guard/PG007/positive/patches/rt64/0002-unlisted.patch @@ -0,0 +1,6 @@ +diff --git a/src/render/rt64_texture_cache.cpp b/src/render/rt64_texture_cache.cpp +--- a/src/render/rt64_texture_cache.cpp ++++ b/src/render/rt64_texture_cache.cpp +@@ -1 +1 @@ +-int cache_limit = 1; ++int cache_limit = 2; diff --git a/tests/fixtures/pattern-guard/PG007/positive/runtime-recomp/CMakeLists.txt b/tests/fixtures/pattern-guard/PG007/positive/runtime-recomp/CMakeLists.txt new file mode 100644 index 00000000..5061755c --- /dev/null +++ b/tests/fixtures/pattern-guard/PG007/positive/runtime-recomp/CMakeLists.txt @@ -0,0 +1,2 @@ +dkr_stage_patch(rt64 "${DKR_RT64_SOURCE}" staged + "${DKRPORT_ROOT}/patches/rt64/0003-staged.patch" src/render/rt64_texture_cache.cpp) diff --git a/tests/fixtures/pattern-guard/PG008/negative/patches/manifest.json b/tests/fixtures/pattern-guard/PG008/negative/patches/manifest.json new file mode 100644 index 00000000..a1bbecc4 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG008/negative/patches/manifest.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 1, + "dependencies": [ + { + "name": "RT64", + "repositoryPath": "extern/rt64", + "expectedCommit": "0000000000000000000000000000000000000000", + "patches": [ + { + "path": "patches/rt64/0001-cache-limit.patch", + "sha256": "12d4abba5da0b4f2440519aab67a2e9dfb1e2606d32a2e912a063590b5c2b8ee" + } + ] + } + ] +} diff --git a/tests/fixtures/pattern-guard/PG008/negative/patches/rt64/0001-cache-limit.patch b/tests/fixtures/pattern-guard/PG008/negative/patches/rt64/0001-cache-limit.patch new file mode 100644 index 00000000..864c786f --- /dev/null +++ b/tests/fixtures/pattern-guard/PG008/negative/patches/rt64/0001-cache-limit.patch @@ -0,0 +1,6 @@ +diff --git a/src/render/rt64_texture_cache.cpp b/src/render/rt64_texture_cache.cpp +--- a/src/render/rt64_texture_cache.cpp ++++ b/src/render/rt64_texture_cache.cpp +@@ -1 +1 @@ +-int cache_limit = 1; ++int cache_limit = 2; diff --git a/tests/fixtures/pattern-guard/PG008/positive/patches/manifest.json b/tests/fixtures/pattern-guard/PG008/positive/patches/manifest.json new file mode 100644 index 00000000..a885bfdf --- /dev/null +++ b/tests/fixtures/pattern-guard/PG008/positive/patches/manifest.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 1, + "dependencies": [ + { + "name": "RT64", + "repositoryPath": "extern/rt64", + "expectedCommit": "0000000000000000000000000000000000000000", + "patches": [ + { + "path": "patches/rt64/0001-sidebar.patch", + "sha256": "aa4075ae7e27b7495e269c92ad0d1ecc90a8ed939abb36cf98a98930e3120757" + } + ] + } + ] +} diff --git a/tests/fixtures/pattern-guard/PG008/positive/patches/rt64/0001-sidebar.patch b/tests/fixtures/pattern-guard/PG008/positive/patches/rt64/0001-sidebar.patch new file mode 100644 index 00000000..97e81bf5 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG008/positive/patches/rt64/0001-sidebar.patch @@ -0,0 +1,6 @@ +diff --git a/runtime-recomp/src/game/runtime_ui.cpp b/runtime-recomp/src/game/runtime_ui.cpp +--- a/runtime-recomp/src/game/runtime_ui.cpp ++++ b/runtime-recomp/src/game/runtime_ui.cpp +@@ -1 +1 @@ +-int sidebar = 1; ++int sidebar = 2; diff --git a/tests/fixtures/pattern-guard/PG009/negative/runtime-recomp/src/game/feature.cpp b/tests/fixtures/pattern-guard/PG009/negative/runtime-recomp/src/game/feature.cpp new file mode 100644 index 00000000..4860fff0 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG009/negative/runtime-recomp/src/game/feature.cpp @@ -0,0 +1,4 @@ +#include "generated/selawik_font.h" +#include "../../include/dkr/presentation.h" +#include +int feature_enabled = 1; diff --git a/tests/fixtures/pattern-guard/PG009/positive/runtime-recomp/src/game/feature.cpp b/tests/fixtures/pattern-guard/PG009/positive/runtime-recomp/src/game/feature.cpp new file mode 100644 index 00000000..488b4042 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG009/positive/runtime-recomp/src/game/feature.cpp @@ -0,0 +1,3 @@ +#include "/Users/dev/DKR-R/build/generated/feature.hpp" +#include "../../../build/generated/feature/runtime_ui.hpp" +int feature_enabled = 1; diff --git a/tests/fixtures/pattern-guard/PG010/negative/runtime-recomp/cmake/LegacyMods.cmake b/tests/fixtures/pattern-guard/PG010/negative/runtime-recomp/cmake/LegacyMods.cmake new file mode 100644 index 00000000..5504413e --- /dev/null +++ b/tests/fixtures/pattern-guard/PG010/negative/runtime-recomp/cmake/LegacyMods.cmake @@ -0,0 +1 @@ +set(DKR_LEGACY_PACK_SHA256 dcb1e8df777d341f2e96d4ee78cdc4337d32a6ddc86b4b60dbcc46fabfd31efc) diff --git a/tests/fixtures/pattern-guard/PG010/negative/runtime-recomp/src/game/mods/catalog.cpp b/tests/fixtures/pattern-guard/PG010/negative/runtime-recomp/src/game/mods/catalog.cpp new file mode 100644 index 00000000..5417413f --- /dev/null +++ b/tests/fixtures/pattern-guard/PG010/negative/runtime-recomp/src/game/mods/catalog.cpp @@ -0,0 +1 @@ +constexpr const char* kPackHash = "dcb1e8df777d341f2e96d4ee78cdc4337d32a6ddc86b4b60dbcc46fabfd31efc"; diff --git a/tests/fixtures/pattern-guard/PG010/negative/runtime-recomp/src/game/netplay/legacy_identity.hpp b/tests/fixtures/pattern-guard/PG010/negative/runtime-recomp/src/game/netplay/legacy_identity.hpp new file mode 100644 index 00000000..3baaed77 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG010/negative/runtime-recomp/src/game/netplay/legacy_identity.hpp @@ -0,0 +1,3 @@ +inline void identity(std::string& base) { + base += "dcb1e8df777d341f2e96d4ee78cdc4337d32a6ddc86b4b60dbcc46fabfd31efc"; +} diff --git a/tests/fixtures/pattern-guard/PG010/positive/runtime-recomp/cmake/Identity.cmake b/tests/fixtures/pattern-guard/PG010/positive/runtime-recomp/cmake/Identity.cmake new file mode 100644 index 00000000..4863b1e8 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG010/positive/runtime-recomp/cmake/Identity.cmake @@ -0,0 +1 @@ +string(SHA256 hash "${CMAKE_MATCH_2}\n${ledger}") diff --git a/tests/fixtures/pattern-guard/PG010/positive/runtime-recomp/src/game/netplay/identity.cpp b/tests/fixtures/pattern-guard/PG010/positive/runtime-recomp/src/game/netplay/identity.cpp new file mode 100644 index 00000000..038b4b04 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG010/positive/runtime-recomp/src/game/netplay/identity.cpp @@ -0,0 +1,3 @@ +void identity(std::string& base) { + base += "dcb1e8df777d341f2e96d4ee78cdc4337d32a6ddc86b4b60dbcc46fabfd31efc"; +} diff --git a/tests/fixtures/pattern-guard/PG010/positive/scripts/salt.py b/tests/fixtures/pattern-guard/PG010/positive/scripts/salt.py new file mode 100644 index 00000000..8537656b --- /dev/null +++ b/tests/fixtures/pattern-guard/PG010/positive/scripts/salt.py @@ -0,0 +1 @@ +SALT = "|sdk2:" diff --git a/tests/fixtures/pattern-guard/PG011/negative/runtime-recomp/src/game/generated/selawik_font.h b/tests/fixtures/pattern-guard/PG011/negative/runtime-recomp/src/game/generated/selawik_font.h new file mode 100644 index 00000000..6f70f09b --- /dev/null +++ b/tests/fixtures/pattern-guard/PG011/negative/runtime-recomp/src/game/generated/selawik_font.h @@ -0,0 +1 @@ +#pragma once diff --git a/tests/fixtures/pattern-guard/PG011/negative/runtime-recomp/src/game/symbols.hpp b/tests/fixtures/pattern-guard/PG011/negative/runtime-recomp/src/game/symbols.hpp new file mode 100644 index 00000000..6f70f09b --- /dev/null +++ b/tests/fixtures/pattern-guard/PG011/negative/runtime-recomp/src/game/symbols.hpp @@ -0,0 +1 @@ +#pragma once diff --git a/tests/fixtures/pattern-guard/PG011/negative/runtime-recomp/symbols/polish.toml b/tests/fixtures/pattern-guard/PG011/negative/runtime-recomp/symbols/polish.toml new file mode 100644 index 00000000..33fc131b --- /dev/null +++ b/tests/fixtures/pattern-guard/PG011/negative/runtime-recomp/symbols/polish.toml @@ -0,0 +1,2 @@ +[symbols] +name = "polish" diff --git a/tests/fixtures/pattern-guard/PG011/positive/runtime-recomp/build/generated/feature.hpp b/tests/fixtures/pattern-guard/PG011/positive/runtime-recomp/build/generated/feature.hpp new file mode 100644 index 00000000..6f70f09b --- /dev/null +++ b/tests/fixtures/pattern-guard/PG011/positive/runtime-recomp/build/generated/feature.hpp @@ -0,0 +1 @@ +#pragma once diff --git a/tests/fixtures/pattern-guard/PG011/positive/runtime-recomp/src/game/polish_symbols.hpp b/tests/fixtures/pattern-guard/PG011/positive/runtime-recomp/src/game/polish_symbols.hpp new file mode 100644 index 00000000..6f70f09b --- /dev/null +++ b/tests/fixtures/pattern-guard/PG011/positive/runtime-recomp/src/game/polish_symbols.hpp @@ -0,0 +1 @@ +#pragma once diff --git a/tests/fixtures/pattern-guard/PG012/negative/runtime-recomp/cmake/Ledger.cmake b/tests/fixtures/pattern-guard/PG012/negative/runtime-recomp/cmake/Ledger.cmake new file mode 100644 index 00000000..913b9b38 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG012/negative/runtime-recomp/cmake/Ledger.cmake @@ -0,0 +1,4 @@ +set(protected "runtime-recomp/RecompiledFuncs/") +file(READ "${ledger}" text) +string(REPLACE "\r\n" "\n" text "${text}") +file(WRITE "${ledger}" "${text}") diff --git a/tests/fixtures/pattern-guard/PG012/negative/scripts/generate_probe.py b/tests/fixtures/pattern-guard/PG012/negative/scripts/generate_probe.py new file mode 100644 index 00000000..71955c29 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG012/negative/scripts/generate_probe.py @@ -0,0 +1,9 @@ +"""Generate a NEW probe from recompiled guest code.""" +from pathlib import Path +import re +import sys + +source, output = Path(sys.argv[1]) / "RecompiledFuncs", Path(sys.argv[2]) +for index, path in enumerate(sorted(source.glob("funcs_*.c"))): + text = re.sub(r"\bosRecvMesg\b", "probe_recv_mesg", path.read_text()) + (output / f"guest_{index}.c").write_text(text) diff --git a/tests/fixtures/pattern-guard/PG012/negative/scripts/guest_hooks.py b/tests/fixtures/pattern-guard/PG012/negative/scripts/guest_hooks.py new file mode 100644 index 00000000..e8ae1a38 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG012/negative/scripts/guest_hooks.py @@ -0,0 +1,9 @@ +"""Rewrite recompiled guest code in place.""" +from pathlib import Path +import re +import sys + +funcs = Path(sys.argv[1]) / "RecompiledFuncs" +for path in funcs.glob("funcs_*.c"): + text = path.read_text() + path.write_text(re.sub(r"\bosRecvMesg\b", "dkr_recv_mesg", text)) diff --git a/tests/fixtures/pattern-guard/PG012/positive/runtime-recomp/cmake/HookGuest.cmake b/tests/fixtures/pattern-guard/PG012/positive/runtime-recomp/cmake/HookGuest.cmake new file mode 100644 index 00000000..02e7cacd --- /dev/null +++ b/tests/fixtures/pattern-guard/PG012/positive/runtime-recomp/cmake/HookGuest.cmake @@ -0,0 +1,6 @@ +file(GLOB guest_sources "${DKR_GENERATED}/RecompiledFuncs/funcs_*.c") +foreach(source IN LISTS guest_sources) + file(READ "${source}" text) + string(REPLACE "osRecvMesg" "dkr_recv_mesg" text "${text}") + file(WRITE "${source}" "${text}") +endforeach() diff --git a/tests/fixtures/pattern-guard/PG012/positive/scripts/hook_guest.py b/tests/fixtures/pattern-guard/PG012/positive/scripts/hook_guest.py new file mode 100644 index 00000000..e8ae1a38 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG012/positive/scripts/hook_guest.py @@ -0,0 +1,9 @@ +"""Rewrite recompiled guest code in place.""" +from pathlib import Path +import re +import sys + +funcs = Path(sys.argv[1]) / "RecompiledFuncs" +for path in funcs.glob("funcs_*.c"): + text = path.read_text() + path.write_text(re.sub(r"\bosRecvMesg\b", "dkr_recv_mesg", text)) diff --git a/tests/fixtures/pattern-guard/PG013/negative/patches/README.md b/tests/fixtures/pattern-guard/PG013/negative/patches/README.md new file mode 100644 index 00000000..f0a5a878 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG013/negative/patches/README.md @@ -0,0 +1 @@ +Dependency patches only. diff --git a/tests/fixtures/pattern-guard/PG013/negative/patches/manifest.json b/tests/fixtures/pattern-guard/PG013/negative/patches/manifest.json new file mode 100644 index 00000000..a1bbecc4 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG013/negative/patches/manifest.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 1, + "dependencies": [ + { + "name": "RT64", + "repositoryPath": "extern/rt64", + "expectedCommit": "0000000000000000000000000000000000000000", + "patches": [ + { + "path": "patches/rt64/0001-cache-limit.patch", + "sha256": "12d4abba5da0b4f2440519aab67a2e9dfb1e2606d32a2e912a063590b5c2b8ee" + } + ] + } + ] +} diff --git a/tests/fixtures/pattern-guard/PG013/negative/patches/rt64/0001-cache-limit.patch b/tests/fixtures/pattern-guard/PG013/negative/patches/rt64/0001-cache-limit.patch new file mode 100644 index 00000000..864c786f --- /dev/null +++ b/tests/fixtures/pattern-guard/PG013/negative/patches/rt64/0001-cache-limit.patch @@ -0,0 +1,6 @@ +diff --git a/src/render/rt64_texture_cache.cpp b/src/render/rt64_texture_cache.cpp +--- a/src/render/rt64_texture_cache.cpp ++++ b/src/render/rt64_texture_cache.cpp +@@ -1 +1 @@ +-int cache_limit = 1; ++int cache_limit = 2; diff --git a/tests/fixtures/pattern-guard/PG013/negative/patches/rt64/README.md b/tests/fixtures/pattern-guard/PG013/negative/patches/rt64/README.md new file mode 100644 index 00000000..6cd9b217 --- /dev/null +++ b/tests/fixtures/pattern-guard/PG013/negative/patches/rt64/README.md @@ -0,0 +1 @@ +RT64 patch notes. diff --git a/tests/fixtures/pattern-guard/PG013/positive/patches/android/surface.patch b/tests/fixtures/pattern-guard/PG013/positive/patches/android/surface.patch new file mode 100644 index 00000000..864c786f --- /dev/null +++ b/tests/fixtures/pattern-guard/PG013/positive/patches/android/surface.patch @@ -0,0 +1,6 @@ +diff --git a/src/render/rt64_texture_cache.cpp b/src/render/rt64_texture_cache.cpp +--- a/src/render/rt64_texture_cache.cpp ++++ b/src/render/rt64_texture_cache.cpp +@@ -1 +1 @@ +-int cache_limit = 1; ++int cache_limit = 2; diff --git a/tests/fixtures/pattern-guard/PG013/positive/patches/feature/PLAYTEST.txt b/tests/fixtures/pattern-guard/PG013/positive/patches/feature/PLAYTEST.txt new file mode 100644 index 00000000..344c534a --- /dev/null +++ b/tests/fixtures/pattern-guard/PG013/positive/patches/feature/PLAYTEST.txt @@ -0,0 +1 @@ +Race twice. diff --git a/tests/fixtures/pattern-guard/PG013/positive/patches/feature/README.md b/tests/fixtures/pattern-guard/PG013/positive/patches/feature/README.md new file mode 100644 index 00000000..84f8ff8d --- /dev/null +++ b/tests/fixtures/pattern-guard/PG013/positive/patches/feature/README.md @@ -0,0 +1 @@ +Feature layer notes.